From 4eed4df925683b94a05e68816a75133f769e7eca Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 6 May 2019 15:18:46 +0200 Subject: [PATCH 1/6] Revert printing template and policy name on export. Ensures behavior on running the `export` cmd does not change compared to last version. --- libbeat/cmd/export/export.go | 15 ++++++++------- libbeat/idxmgmt/client_handler.go | 2 +- libbeat/idxmgmt/ilm/client_handler.go | 7 +++---- .../ilm/client_handler_integration_test.go | 9 +++++---- libbeat/idxmgmt/ilm/ilm.go | 2 +- libbeat/idxmgmt/ilm/ilm_test.go | 6 ++---- libbeat/template/load.go | 7 +++---- libbeat/template/load_test.go | 10 +++++----- metricbeat/tests/system/test_template.py | 4 +--- 9 files changed, 29 insertions(+), 33 deletions(-) diff --git a/libbeat/cmd/export/export.go b/libbeat/cmd/export/export.go index d7acacf97539..218f565c0a28 100644 --- a/libbeat/cmd/export/export.go +++ b/libbeat/cmd/export/export.go @@ -71,10 +71,7 @@ func newFileClient(dir string, ver string) (*fileClient, error) { if err != nil { return nil, err } - err = os.MkdirAll(path, os.ModePerm) - if err != nil { - return nil, err - } + fmt.Println(fmt.Sprintf("Writing to directory %s", path)) return &fileClient{ver: *common.MustNewVersion(ver), dir: path}, nil } @@ -82,7 +79,7 @@ func (c *stdoutClient) GetVersion() common.Version { return c.ver } -func (c *stdoutClient) Write(_ string, body string) error { +func (c *stdoutClient) Write(_ string, _string, body string) error { _, err := c.f.WriteString(body) return err } @@ -91,8 +88,12 @@ func (c *fileClient) GetVersion() common.Version { return c.ver } -func (c *fileClient) Write(name string, body string) error { - f, err := os.Create(filepath.Join(c.dir, fmt.Sprintf("%s.json", name))) +func (c *fileClient) Write(kind string, name string, body string) error { + path := filepath.Join(c.dir, kind) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + return err + } + f, err := os.Create(filepath.Join(path, fmt.Sprintf("%s.json", name))) defer f.Close() if err != nil { return err diff --git a/libbeat/idxmgmt/client_handler.go b/libbeat/idxmgmt/client_handler.go index c4665ce0ecbb..817bea7dd2c5 100644 --- a/libbeat/idxmgmt/client_handler.go +++ b/libbeat/idxmgmt/client_handler.go @@ -45,7 +45,7 @@ type ESClient interface { // prepare a policy and write alias. type FileClient interface { GetVersion() common.Version - Write(name string, body string) error + Write(kind string, name string, body string) error } // NewClientHandler initializes and returns a new instance of ClientHandler diff --git a/libbeat/idxmgmt/ilm/client_handler.go b/libbeat/idxmgmt/ilm/client_handler.go index 1cc8456c21d5..fa35eb990bc5 100644 --- a/libbeat/idxmgmt/ilm/client_handler.go +++ b/libbeat/idxmgmt/ilm/client_handler.go @@ -63,7 +63,7 @@ type FileClientHandler struct { // prepare a policy and write alias. type FileClient interface { GetVersion() common.Version - Write(name string, body string) error + Write(kind string, name string, body string) error } const ( @@ -243,9 +243,8 @@ func (h *FileClientHandler) CheckILMEnabled(mode Mode) (bool, error) { // CreateILMPolicy writes given policy to the configured file. func (h *FileClientHandler) CreateILMPolicy(policy Policy) error { - p := common.MapStr{policy.Name: policy.Body} - str := fmt.Sprintf("%s\n", p.StringToPrint()) - if err := h.client.Write(policy.Name, str); err != nil { + str := fmt.Sprintf("%s\n", policy.Body.StringToPrint()) + if err := h.client.Write("policy", policy.Name, str); err != nil { return fmt.Errorf("error printing policy : %v", err) } return nil diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index 8d328cbca2f1..bd892a3cb7a6 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -260,14 +260,15 @@ func TestFileClientHandler_CreateILMPolicy(t *testing.T) { h.CreateILMPolicy(ilm.Policy{Name: name, Body: body}) assert.Equal(t, name, c.name) + assert.Equal(t, "policy", c.kind) var out common.MapStr json.Unmarshal([]byte(c.body), &out) assert.Equal(t, common.MapStr{name: body}, out) } type mockClient struct { - v common.Version - name, body string + v common.Version + kind, name, body string } func newMockClient(v string) *mockClient { @@ -281,7 +282,7 @@ func (c *mockClient) GetVersion() common.Version { return c.v } -func (c *mockClient) Write(name string, body string) error { - c.name, c.body = name, body +func (c *mockClient) Write(kind string, name string, body string) error { + c.kind, c.name, c.body = kind, name, body return nil } diff --git a/libbeat/idxmgmt/ilm/ilm.go b/libbeat/idxmgmt/ilm/ilm.go index d7e803b5742c..a023386adaef 100644 --- a/libbeat/idxmgmt/ilm/ilm.go +++ b/libbeat/idxmgmt/ilm/ilm.go @@ -59,7 +59,7 @@ type Manager interface { // See: [Policy phases and actions documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/ilm-policy-definition.html). type Policy struct { Name string - Body map[string]interface{} + Body common.MapStr } // Alias describes the alias to be created in Elasticsearch. diff --git a/libbeat/idxmgmt/ilm/ilm_test.go b/libbeat/idxmgmt/ilm/ilm_test.go index 0ca72c866730..5e8f382b8748 100644 --- a/libbeat/idxmgmt/ilm/ilm_test.go +++ b/libbeat/idxmgmt/ilm/ilm_test.go @@ -76,12 +76,10 @@ func TestDefaultSupport_Init(t *testing.T) { t.Run("load external policy", func(t *testing.T) { s, err := DefaultSupport(nil, info, common.MustNewConfigFrom( - map[string]interface{}{ - "policy_file": "testfiles/custom.json", - }, + common.MapStr{"policy_file": "testfiles/custom.json"}, )) require.NoError(t, err) - assert.Equal(t, map[string]interface{}{"hello": "world"}, s.Policy().Body) + assert.Equal(t, common.MapStr{"hello": "world"}, s.Policy().Body) }) } diff --git a/libbeat/template/load.go b/libbeat/template/load.go index d9fe54897a6c..ce87552c3d3b 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -55,7 +55,7 @@ type FileLoader struct { // FileClient defines the minimal interface required for the FileLoader type FileClient interface { GetVersion() common.Version - Write(name string, body string) error + Write(kind string, name string, body string) error } // NewESLoader creates a new template loader for ES @@ -145,9 +145,8 @@ func (l *FileLoader) Load(config TemplateConfig, info beat.Info, fields []byte, return err } - p := common.MapStr{tmpl.name: body} - str := fmt.Sprintf("%s\n", p.StringToPrint()) - if err := l.client.Write(tmpl.name, str); err != nil { + str := fmt.Sprintf("%s\n", body.StringToPrint()) + if err := l.client.Write("template", tmpl.name, str); err != nil { return fmt.Errorf("error printing template: %v", err) } return nil diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go index 22566229ab94..634a1ae92dbb 100644 --- a/libbeat/template/load_test.go +++ b/libbeat/template/load_test.go @@ -63,14 +63,14 @@ func TestFileLoader_Load(t *testing.T) { require.NoError(t, err) body, err := buildBody(tmpl, test.cfg, test.fields) require.NoError(t, err) - assert.Equal(t, common.MapStr{test.name: body}.StringToPrint()+"\n", fc.body) + assert.Equal(t, body.StringToPrint()+"\n", fc.body) }) } } type fileClient struct { - ver common.Version - body string + ver common.Version + kind, name, body string } func newFileClient(ver string) (*fileClient, error) { @@ -88,7 +88,7 @@ func (c *fileClient) GetVersion() common.Version { return c.ver } -func (c *fileClient) Write(name string, body string) error { - c.body = body +func (c *fileClient) Write(kind string, name string, body string) error { + c.kind, c.name, c.body = kind, name, body return nil } diff --git a/metricbeat/tests/system/test_template.py b/metricbeat/tests/system/test_template.py index 335c435119cd..5899796524d6 100644 --- a/metricbeat/tests/system/test_template.py +++ b/metricbeat/tests/system/test_template.py @@ -41,9 +41,7 @@ def test_export_template(self): break t = json.loads(template_content) - keys = [k for k, v in t.iteritems() if k.startswith("metricbeat")] - assert len(keys) == 1 - properties = t[keys[0]]["mappings"]["properties"] + properties = t["mappings"]["properties"] # Check libbeat fields assert properties["@timestamp"] == {"type": "date"} From a78cd2dc8cef515a39bf86d8546f841bb99bbcec Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 6 May 2019 16:01:15 +0200 Subject: [PATCH 2/6] Add changelog note --- CHANGELOG-developer.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 6704596f3171..95e96a7750d4 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -35,5 +35,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only. by `make` and `mage`. Example: `export PYTHON_EXE=python2.7`. {pull}11212[11212] - Prometheus helper for metricbeat contains now `Namespace` field for `prometheus.MetricsMappings` {pull}11424[11424] - Update Jinja2 version to 2.10.1. {pull}11817[11817] -- Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777] +- Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12067[12067] - Update urllib3 version to 1.24.2 {pull}11930[11930] From 0edc3d27a36821f73a0d006e2190de119586c8ea Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 6 May 2019 16:20:56 +0200 Subject: [PATCH 3/6] fix broken test --- libbeat/idxmgmt/ilm/client_handler_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index bd892a3cb7a6..400daa83e970 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -256,14 +256,14 @@ func TestFileClientHandler_CreateILMPolicy(t *testing.T) { c := newMockClient("") h := ilm.NewFileClientHandler(c) name := "test-policy" - body := map[string]interface{}{"foo": "bar"} + body := common.MapStr{"foo": "bar"} h.CreateILMPolicy(ilm.Policy{Name: name, Body: body}) assert.Equal(t, name, c.name) assert.Equal(t, "policy", c.kind) var out common.MapStr json.Unmarshal([]byte(c.body), &out) - assert.Equal(t, common.MapStr{name: body}, out) + assert.Equal(t, body, out) } type mockClient struct { From 12398c64035cf726b743b9cf4dff7257133faa9f Mon Sep 17 00:00:00 2001 From: simitt Date: Mon, 6 May 2019 20:01:18 +0200 Subject: [PATCH 4/6] minor fix --- libbeat/cmd/export/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/cmd/export/export.go b/libbeat/cmd/export/export.go index 218f565c0a28..bf1f0e32eaab 100644 --- a/libbeat/cmd/export/export.go +++ b/libbeat/cmd/export/export.go @@ -79,7 +79,7 @@ func (c *stdoutClient) GetVersion() common.Version { return c.ver } -func (c *stdoutClient) Write(_ string, _string, body string) error { +func (c *stdoutClient) Write(_ string, _ string, body string) error { _, err := c.f.WriteString(body) return err } From 2bae5f5f74c21b1e082b8999b77e20a29efa34f2 Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 8 May 2019 17:28:53 +0200 Subject: [PATCH 5/6] rename kind to component --- libbeat/cmd/export/export.go | 4 ++-- libbeat/idxmgmt/client_handler.go | 2 +- libbeat/idxmgmt/ilm/client_handler.go | 2 +- libbeat/idxmgmt/ilm/client_handler_integration_test.go | 10 +++++----- libbeat/template/load.go | 2 +- libbeat/template/load_test.go | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libbeat/cmd/export/export.go b/libbeat/cmd/export/export.go index bf1f0e32eaab..07ec27edf087 100644 --- a/libbeat/cmd/export/export.go +++ b/libbeat/cmd/export/export.go @@ -88,8 +88,8 @@ func (c *fileClient) GetVersion() common.Version { return c.ver } -func (c *fileClient) Write(kind string, name string, body string) error { - path := filepath.Join(c.dir, kind) +func (c *fileClient) Write(component string, name string, body string) error { + path := filepath.Join(c.dir, component) if err := os.MkdirAll(path, os.ModePerm); err != nil { return err } diff --git a/libbeat/idxmgmt/client_handler.go b/libbeat/idxmgmt/client_handler.go index 817bea7dd2c5..9feba65ebd78 100644 --- a/libbeat/idxmgmt/client_handler.go +++ b/libbeat/idxmgmt/client_handler.go @@ -45,7 +45,7 @@ type ESClient interface { // prepare a policy and write alias. type FileClient interface { GetVersion() common.Version - Write(kind string, name string, body string) error + Write(component string, name string, body string) error } // NewClientHandler initializes and returns a new instance of ClientHandler diff --git a/libbeat/idxmgmt/ilm/client_handler.go b/libbeat/idxmgmt/ilm/client_handler.go index fa35eb990bc5..99daa4aaae33 100644 --- a/libbeat/idxmgmt/ilm/client_handler.go +++ b/libbeat/idxmgmt/ilm/client_handler.go @@ -63,7 +63,7 @@ type FileClientHandler struct { // prepare a policy and write alias. type FileClient interface { GetVersion() common.Version - Write(kind string, name string, body string) error + Write(component string, name string, body string) error } const ( diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index 400daa83e970..960436b4a05b 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -260,15 +260,15 @@ func TestFileClientHandler_CreateILMPolicy(t *testing.T) { h.CreateILMPolicy(ilm.Policy{Name: name, Body: body}) assert.Equal(t, name, c.name) - assert.Equal(t, "policy", c.kind) + assert.Equal(t, "policy", c.component) var out common.MapStr json.Unmarshal([]byte(c.body), &out) assert.Equal(t, body, out) } type mockClient struct { - v common.Version - kind, name, body string + v common.Version + component, name, body string } func newMockClient(v string) *mockClient { @@ -282,7 +282,7 @@ func (c *mockClient) GetVersion() common.Version { return c.v } -func (c *mockClient) Write(kind string, name string, body string) error { - c.kind, c.name, c.body = kind, name, body +func (c *mockClient) Write(component string, name string, body string) error { + c.component, c.name, c.body = component, name, body return nil } diff --git a/libbeat/template/load.go b/libbeat/template/load.go index ce87552c3d3b..2d8fdbf6607e 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -55,7 +55,7 @@ type FileLoader struct { // FileClient defines the minimal interface required for the FileLoader type FileClient interface { GetVersion() common.Version - Write(kind string, name string, body string) error + Write(component string, name string, body string) error } // NewESLoader creates a new template loader for ES diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go index 634a1ae92dbb..07f0ca7ad5ba 100644 --- a/libbeat/template/load_test.go +++ b/libbeat/template/load_test.go @@ -88,7 +88,7 @@ func (c *fileClient) GetVersion() common.Version { return c.ver } -func (c *fileClient) Write(kind string, name string, body string) error { - c.kind, c.name, c.body = kind, name, body +func (c *fileClient) Write(component string, name string, body string) error { + c.kind, c.name, c.body = component, name, body return nil } From 030c269956877b11855b549357b61cf3c81f12e0 Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 8 May 2019 20:55:01 +0200 Subject: [PATCH 6/6] fix broken test --- libbeat/tests/system/test_ilm.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index c016fba71292..2dcc11edd6f4 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -286,9 +286,8 @@ def setUp(self): self.policy_name = self.beat_name + "-9.9.9" self.cmd = "ilm-policy" - def assert_log_contains_policy(self, policy): + def assert_log_contains_policy(self): assert self.log_contains('ILM policy successfully loaded.') - assert self.log_contains(policy) assert self.log_contains('"max_age": "30d"') assert self.log_contains('"max_size": "50gb"') @@ -304,7 +303,7 @@ def test_default(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_policy(self.policy_name) + self.assert_log_contains_policy() self.assert_log_contains_write_alias() def test_load_disabled(self): @@ -316,7 +315,7 @@ def test_load_disabled(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_policy(self.policy_name) + self.assert_log_contains_policy() self.assert_log_contains_write_alias() def test_changed_policy_name(self): @@ -330,5 +329,5 @@ def test_changed_policy_name(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_policy(policy_name) + self.assert_log_contains_policy() self.assert_log_contains_write_alias()