[Auditbeat] Socket dataset: Exclude localhost by default

CHANGELOG.next.asciidoc

@@ -17,6 +17,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Auditd module: Normalized value of `event.category` field from `user-login` to `authentication`. {pull}11432[11432]
 - Auditd module: Unset `auditd.session` and `user.audit.id` fields are removed from audit events. {issue}11431[11431] {pull}11815[11815]
+- Socket dataset: Exclude localhost by default {pull}11993[11993]
 
 *Filebeat*
 - Modify apache/error dataset to follow ECS. {pull}8963[8963]

x-pack/auditbeat/auditbeat.reference.yml

@@ -135,6 +135,10 @@ auditbeat.modules:
   # socket.state.period: 12h
   # user.state.period: 12h
 
+  # Disabled by default. If enabled, the socket dataset will
+  # report sockets to and from localhost.
+  # socket.include_localhost: false
+
   # Enabled by default. Auditbeat will read password fields in
   # /etc/passwd and /etc/shadow and store a hash locally to
   # detect any changes.

x-pack/auditbeat/docs/modules/system.asciidoc

@@ -66,6 +66,9 @@ sample suggested configuration.
     - user
   period: 10s
   state.period: 12h
+
+  socket.include_localhost: false
+
   user.detect_password_changes: true
 ----
 
@@ -75,6 +78,10 @@ datasets - esp. `process` and `socket` - a shorter period is recommended.
 *`state.period`*:: The frequency at which the datasets send full state information.
 This option can be overridden per dataset using `{dataset}.state.period`.
 
+*`socket.include_localhost`*:: If the `socket` dataset is configured and this
+option is set to `true`, Auditbeat will include sockets that have localhost
+as either their source and/or destination IP. Defaults to `false`.
+
 *`user.detect_password_changes`*:: If the `user` dataset is configured and
 this option is set to `true`, Auditbeat will read password information in `/etc/passwd`
 and `/etc/shadow` to detect password changes. A hash will be kept locally in
@@ -100,6 +107,7 @@ so a longer polling interval can be used.
     - package
     - user
   period: 1m
+
   user.detect_password_changes: true
 
 - module: system

x-pack/auditbeat/module/system/_meta/config.yml.tmpl

@@ -23,7 +23,9 @@
   # current state of the system (e.g. all currently
   # running processes, all open sockets).
   state.period: 12h
-{{ if .Reference }}
+
+{{- if .Reference }}
+
   # The state.period can be overridden for any dataset.
   # host.state.period: 12h
   {{ if ne .GOOS "windows" -}}
@@ -33,16 +35,22 @@
   {{ if eq .GOOS "linux" -}}
   # socket.state.period: 12h
   # user.state.period: 12h
-  {{- end }}
-{{ end }}
-  {{ if eq .GOOS "linux" -}}
+  {{- end -}}
+{{- end -}}
+{{- if eq .GOOS "linux" -}}
+
+{{- if .Reference }}
+
+  # Disabled by default. If enabled, the socket dataset will
+  # report sockets to and from localhost.
+  # socket.include_localhost: false
+{{- end }}
+
   # Enabled by default. Auditbeat will read password fields in
   # /etc/passwd and /etc/shadow and store a hash locally to
   # detect any changes.
   user.detect_password_changes: true
-  {{- end }}
 
-  {{ if eq .GOOS "linux" -}}
   # File patterns of the login record files.
 {{- if .Reference }}
   # wtmp: History of successful logins, logouts, and system shutdowns and boots.

x-pack/auditbeat/module/system/_meta/docs.asciidoc

@@ -61,6 +61,9 @@ sample suggested configuration.
     - user
   period: 10s
   state.period: 12h
+
+  socket.include_localhost: false
+
   user.detect_password_changes: true
 ----
 
@@ -70,6 +73,10 @@ datasets - esp. `process` and `socket` - a shorter period is recommended.
 *`state.period`*:: The frequency at which the datasets send full state information.
 This option can be overridden per dataset using `{dataset}.state.period`.
 
+*`socket.include_localhost`*:: If the `socket` dataset is configured and this
+option is set to `true`, Auditbeat will include sockets that have localhost
+as either their source and/or destination IP. Defaults to `false`.
+
 *`user.detect_password_changes`*:: If the `user` dataset is configured and
 this option is set to `true`, Auditbeat will read password information in `/etc/passwd`
 and `/etc/shadow` to detect password changes. A hash will be kept locally in
@@ -95,6 +102,7 @@ so a longer polling interval can be used.
     - package
     - user
   period: 1m
+
   user.detect_password_changes: true
 
 - module: system

x-pack/auditbeat/module/system/socket/_meta/docs.asciidoc

@@ -4,8 +4,20 @@ beta[]
 
 This is the `socket` dataset of the system module.
 
+[float]
+=== Implementation
+
 It is implemented for Linux only.
 
+On Linux, the dataset uses the
+http://man7.org/linux/man-pages/man7/sock_diag.7.html[sock_diag netlink subsystem]
+to periodically receive a list of all sockets from the kernel. The polling frequency
+can be set using the `period` configuration option. A low value (e.g. `1s`) is
+recommended to capture short-lived sockets.
+
+By default, sockets from or to `localhost` will be excluded. This can be controlled using
+the `socket.include_localhost` configuration option.
+
 [float]
 ==== Example dashboard
 

x-pack/auditbeat/module/system/socket/config.go

@@ -12,6 +12,7 @@ import (
 type Config struct {
 	StatePeriod       time.Duration `config:"state.period"`
 	SocketStatePeriod time.Duration `config:"socket.state.period"`
+	IncludeLocalhost  bool          `config:"socket.include_localhost"`
 }
 
 // Validate validates the host metricset config.
@@ -27,5 +28,6 @@ func (c *Config) effectiveStatePeriod() time.Duration {
 }
 
 var defaultConfig = Config{
-	StatePeriod: 1 * time.Hour,
+	StatePeriod:      1 * time.Hour,
+	IncludeLocalhost: false,
 }

x-pack/auditbeat/module/system/socket/socket.go

@@ -220,8 +220,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // Fetch collects the user information. It is invoked periodically.
 func (ms *MetricSet) Fetch(report mb.ReporterV2) {
 	needsStateUpdate := time.Since(ms.lastState) > ms.config.effectiveStatePeriod()
-	if needsStateUpdate || ms.cache.IsEmpty() {
-		ms.log.Debugf("State update needed (needsStateUpdate=%v, cache.IsEmpty()=%v)", needsStateUpdate, ms.cache.IsEmpty())
+	if needsStateUpdate {
+		ms.log.Debug("Sending state")
 		err := ms.reportState(report)
 		if err != nil {
 			ms.log.Error(err)
@@ -391,7 +391,15 @@ func (ms *MetricSet) getSockets() ([]*Socket, error) {
 
 	sockets := make([]*Socket, 0, len(diags))
 	for _, diag := range diags {
-		sockets = append(sockets, newSocket(diag))
+		socket := newSocket(diag)
+
+		if !ms.config.IncludeLocalhost &&
+			(socket.LocalIP.IsLoopback() || socket.RemoteIP.IsLoopback()) {
+
+			continue
+		}
+
+		sockets = append(sockets, socket)
 	}
 
 	return sockets, nil