From 75c6c41e373a4c29ba7518c3ac43725acaa95c90 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 19 Jan 2021 13:33:53 +0100 Subject: [PATCH 01/61] Terraform executor as Docker image --- .../data_stream/sns/_dev/deploy/docker/Dockerfile | 6 ++++++ .../sns/_dev/deploy/docker/docker-compose.yml | 6 ++++++ .../aws/data_stream/sns/_dev/deploy/docker/main.tf | 13 +++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile create mode 100644 test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml create mode 100644 test/packages/aws/data_stream/sns/_dev/deploy/docker/main.tf diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile new file mode 100644 index 000000000..16a87baed --- /dev/null +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile @@ -0,0 +1,6 @@ +ARG SERVICE_VERSION=${SERVICE_VERSION:-2.4.20} +FROM httpd:$SERVICE_VERSION +RUN sed -i "/jessie-updates/d" /etc/apt/sources.list +RUN apt-get update && apt-get install -y curl +HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost +COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml b/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 000000000..9387000e7 --- /dev/null +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,6 @@ +version: '2.3' +services: + aws_sns: + build: . + volumes: + - ${TF_DIR}:/workspace diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/main.tf b/test/packages/aws/data_stream/sns/_dev/deploy/docker/main.tf new file mode 100644 index 000000000..4a703a5f2 --- /dev/null +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/main.tf @@ -0,0 +1,13 @@ +provider "aws" {} + +resource "random_string" "id" { + length = 5 + upper = false + lower = true + number = false + special = false +} + +resource "aws_sns_topic" "t" { + name = "ep-sns-topic-${random_string.id.result}" +} \ No newline at end of file From fe54067ad860e2b817eada19831f88aaf265dc79 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 19 Jan 2021 14:54:28 +0100 Subject: [PATCH 02/61] Implement docker-compose for tf executor --- .../sns/_dev/deploy/docker/Dockerfile | 16 ++++++++----- .../sns/_dev/deploy/docker/docker-compose.yml | 8 ++++++- .../data_stream/sns/_dev/deploy/docker/run.sh | 24 +++++++++++++++++++ .../sns/_dev/deploy/docker/{ => tf}/main.tf | 0 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100755 test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh rename test/packages/aws/data_stream/sns/_dev/deploy/docker/{ => tf}/main.tf (100%) diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile index 16a87baed..bb3982085 100644 --- a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile @@ -1,6 +1,10 @@ -ARG SERVICE_VERSION=${SERVICE_VERSION:-2.4.20} -FROM httpd:$SERVICE_VERSION -RUN sed -i "/jessie-updates/d" /etc/apt/sources.list -RUN apt-get update && apt-get install -y curl -HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost -COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf \ No newline at end of file +FROM hashicorp/terraform:light + +ENV TF_IN_AUTOMATION=true + +HEALTHCHECK --timeout=3s CMD sh -c "[ -f /tmp/tf-applied ]" + +ADD run.sh / + +WORKDIR /workspace +ENTRYPOINT sh /run.sh \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml b/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml index 9387000e7..29584142e 100644 --- a/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml @@ -2,5 +2,11 @@ version: '2.3' services: aws_sns: build: . + tty: true + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_PROFILE=${AWS_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} volumes: - - ${TF_DIR}:/workspace + - ${TF_DIR}:/stage diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh b/test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh new file mode 100755 index 000000000..4c0b718ad --- /dev/null +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh @@ -0,0 +1,24 @@ +#!sh + +set -euxo pipefail + +cp -r /stage/*.tf /workspace + +cleanup() { + r=$? + + set -x + terraform destroy -auto-approve + + exit $r +} +trap cleanup EXIT INT TERM + +terraform init +terraform plan +terraform apply -auto-approve && touch /tmp/tf-applied + +echo "Terraform definitions applied." + +set +x +while true; do sleep 1; done # wait for ctrl-c \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/main.tf b/test/packages/aws/data_stream/sns/_dev/deploy/docker/tf/main.tf similarity index 100% rename from test/packages/aws/data_stream/sns/_dev/deploy/docker/main.tf rename to test/packages/aws/data_stream/sns/_dev/deploy/docker/tf/main.tf From 4eb1f8bfd129ca07cc105ebb1b0cd0132dddafc4 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 12:49:43 +0100 Subject: [PATCH 03/61] Enable _dev/deploy for data streams --- go.mod | 1 + go.sum | 2 + internal/testrunner/runners/system/runner.go | 5 +- .../runners/system/servicedeployer/factory.go | 46 ++++++++++++++++--- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index f496b05b8..8740e110d 100644 --- a/go.mod +++ b/go.mod @@ -28,5 +28,6 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 + golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/go.sum b/go.sum index c7e1c9cfa..bedcac688 100644 --- a/go.sum +++ b/go.sum @@ -458,6 +458,8 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 5a1f15e88..0212d49c6 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -258,7 +258,10 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // Setup service. logger.Debug("setting up service...") - serviceDeployer, err := servicedeployer.Factory(r.options.PackageRootPath) + serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{ + PackageRootPath: r.options.PackageRootPath, + DataStreamRootPath: dataStreamPath, + }) if err != nil { return result.withError(errors.Wrap(err, "could not create service runner")) } diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 5527b3a61..315241837 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -5,27 +5,61 @@ package servicedeployer import ( - "errors" "os" "path/filepath" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/logger" ) +const devDir = "_dev" + var ( // ErrNotFound is returned when the appropriate service runner for a package // cannot be found. ErrNotFound = errors.New("unable to find service runner") ) -// Factory chooses the appropriate service runner for the given package, depending -// on service configuration files defined in the package. -func Factory(packageRootPath string) (ServiceDeployer, error) { - packageDevPath := filepath.Join(packageRootPath, "_dev") +// FactoryOptions defines options used to create an instance of a service deployer. +type FactoryOptions struct { + PackageRootPath string + DataStreamRootPath string +} + +// Factory chooses the appropriate service runner for the given data stream, depending +// on service configuration files defined in the package or data stream. +func Factory(options FactoryOptions) (ServiceDeployer, error) { + devPath, err := findDevPath(options) + if err != nil { + logger.Errorf("can't find _dev directory") + return nil, ErrNotFound + } // Is the service defined using a docker compose configuration file? - dockerComposeYMLPath := filepath.Join(packageDevPath, "deploy", "docker", "docker-compose.yml") + dockerComposeYMLPath := filepath.Join(devPath, "deploy", "docker", "docker-compose.yml") if _, err := os.Stat(dockerComposeYMLPath); err == nil { return NewDockerComposeServiceDeployer(dockerComposeYMLPath) } return nil, ErrNotFound } + +func findDevPath(options FactoryOptions) (string, error) { + dataStreamDevPath := filepath.Join(options.DataStreamRootPath, devDir) + _, err := os.Stat(dataStreamDevPath) + if err == nil { + return dataStreamDevPath, nil + } else if !os.IsNotExist(err) { + return "", errors.Wrapf(err, "stat failed (path: %s)", dataStreamDevPath) + } + + packageDevPath := filepath.Join(options.PackageRootPath, devDir) + _, err = os.Stat(packageDevPath) + if err == nil { + return packageDevPath, nil + } else if !os.IsNotExist(err) { + return "", errors.Wrapf(err, "stat failed (path: %s)", packageDevPath) + } + return "", errors.New("_dev directory doesn't exist") +} From 2690b652803ca0026b3f0d580e346f4553b55c16 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 13:07:10 +0100 Subject: [PATCH 04/61] Fix --- .../runners/system/servicedeployer/factory.go | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 315241837..503dfc720 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -5,6 +5,7 @@ package servicedeployer import ( + "fmt" "os" "path/filepath" @@ -13,7 +14,7 @@ import ( "github.com/elastic/elastic-package/internal/logger" ) -const devDir = "_dev" +const devDeployDir = "_dev/deploy" var ( // ErrNotFound is returned when the appropriate service runner for a package @@ -30,36 +31,36 @@ type FactoryOptions struct { // Factory chooses the appropriate service runner for the given data stream, depending // on service configuration files defined in the package or data stream. func Factory(options FactoryOptions) (ServiceDeployer, error) { - devPath, err := findDevPath(options) + devDeployPath, err := findDevDeployPath(options) if err != nil { - logger.Errorf("can't find _dev directory") + logger.Errorf("can't find _dev/deploy directory") return nil, ErrNotFound } + fmt.Println(devDeployPath) // Is the service defined using a docker compose configuration file? - dockerComposeYMLPath := filepath.Join(devPath, "deploy", "docker", "docker-compose.yml") + dockerComposeYMLPath := filepath.Join(devDeployPath, "docker", "docker-compose.yml") if _, err := os.Stat(dockerComposeYMLPath); err == nil { return NewDockerComposeServiceDeployer(dockerComposeYMLPath) } - return nil, ErrNotFound } -func findDevPath(options FactoryOptions) (string, error) { - dataStreamDevPath := filepath.Join(options.DataStreamRootPath, devDir) - _, err := os.Stat(dataStreamDevPath) +func findDevDeployPath(options FactoryOptions) (string, error) { + dataStreamDevDeployPath := filepath.Join(options.DataStreamRootPath, devDeployDir) + _, err := os.Stat(dataStreamDevDeployPath) if err == nil { - return dataStreamDevPath, nil + return dataStreamDevDeployPath, nil } else if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "stat failed (path: %s)", dataStreamDevPath) + return "", errors.Wrapf(err, "stat failed (path: %s)", dataStreamDevDeployPath) } - packageDevPath := filepath.Join(options.PackageRootPath, devDir) - _, err = os.Stat(packageDevPath) + packageDevDeployPath := filepath.Join(options.PackageRootPath, devDeployDir) + _, err = os.Stat(packageDevDeployPath) if err == nil { - return packageDevPath, nil + return packageDevDeployPath, nil } else if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "stat failed (path: %s)", packageDevPath) + return "", errors.Wrapf(err, "stat failed (path: %s)", packageDevDeployPath) } return "", errors.New("_dev directory doesn't exist") } From b324b836c2ccca7494b37b31c17ec679a9f46b30 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 14:02:12 +0100 Subject: [PATCH 05/61] Gohack: github.com/mtojek/package-spec/code/go --- go.mod | 3 +++ go.sum | 10 ++-------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 8740e110d..3c41bc690 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,10 @@ require ( github.com/stretchr/testify v1.6.1 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) + +replace github.com/elastic/package-spec/code/go => github.com/mtojek/package-spec/code/go v0.0.0-20210120110240-1a9cec67766e diff --git a/go.sum b/go.sum index bedcac688..e340c63af 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20210111200056-23a4cfe4f518 h1:zMaYyFfVNHmyjPE/iqjWr/EbfHPVxv+GCXAIEhkKKM4= -github.com/elastic/package-spec/code/go v0.0.0-20210111200056-23a4cfe4f518/go.mod h1:wjN5+18SNfDKwMmrT80XbcD6wpwig3XYYBUctkNmZjI= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -243,6 +241,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mtojek/package-spec/code/go v0.0.0-20210120110240-1a9cec67766e h1:teNPHdo4n1CJ6nZpJr+g32YwSdAAek9kWQh+AqTikYo= +github.com/mtojek/package-spec/code/go v0.0.0-20210120110240-1a9cec67766e/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -314,7 +314,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -362,8 +361,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -422,7 +419,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -513,8 +509,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e h1:Z2uDrs8MyXUWJbwGc4V+nGjV4Ygo+oubBbWSVQw21/I= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 22c92c9ac196e6b2dd27064fb9e5bd7f71181c77 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 15:05:53 +0100 Subject: [PATCH 06/61] Install service deployer as static resource --- internal/install/install.go | 29 +++++++++++++++-- .../static_terraform_deployer_dockerfile.go | 13 ++++++++ .../install/static_terraform_deployer_run.go | 31 +++++++++++++++++++ .../install/static_terraform_deployer_yml.go | 19 ++++++++++++ .../sns/_dev/deploy/docker/Dockerfile | 4 --- 5 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 internal/install/static_terraform_deployer_dockerfile.go create mode 100644 internal/install/static_terraform_deployer_run.go create mode 100644 internal/install/static_terraform_deployer_yml.go diff --git a/internal/install/install.go b/internal/install/install.go index b92ceb985..ef408fba1 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -18,10 +18,12 @@ const ( stackDir = "stack" packagesDir = "development" temporaryDir = "tmp" + deployerDir = "deployer" ) var ( - serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + terraformDeployerDir = filepath.Join(deployerDir, "terraform") ) const versionFilename = "version" @@ -50,7 +52,12 @@ func EnsureInstalled() error { err = writeStackResources(elasticPackagePath) if err != nil { - return errors.Wrap(err, "writing static resources failed") + return errors.Wrap(err, "writing stack resources failed") + } + + err = writeDeployerResources(elasticPackagePath) + if err != nil { + return errors.Wrap(err, "writing deployer resources failed") } if err := createServiceLogsDir(elasticPackagePath); err != nil { @@ -126,7 +133,7 @@ func writeStackResources(elasticPackagePath string) error { packagesPath := filepath.Join(stackPath, packagesDir) err := os.MkdirAll(packagesPath, 0755) if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath) + return errors.Wrapf(err, "creating directory failed (path: %s)", packagesPath) } err = writeStaticResource(err, filepath.Join(stackPath, "kibana.config.yml"), kibanaConfigYml) @@ -139,6 +146,22 @@ func writeStackResources(elasticPackagePath string) error { return nil } +func writeDeployerResources(elasticPackagePath string) error { + terraformDeployer := filepath.Join(elasticPackagePath, terraformDeployerDir) + err := os.MkdirAll(terraformDeployer, 0755) + if err != nil { + return errors.Wrapf(err, "creating directory failed (path: %s)", terraformDeployer) + } + + err = writeStaticResource(err, filepath.Join(terraformDeployer, "terraform-deployer.yml"), terraformDeployerYml) + err = writeStaticResource(err, filepath.Join(terraformDeployer, "Dockerfile"), terraformDeployerDockerfile) + err = writeStaticResource(err, filepath.Join(terraformDeployer, "run.sh"), terraformDeployerRun) + if err != nil { + return errors.Wrap(err, "writing static resource failed") + } + return nil +} + func writeStaticResource(err error, path, content string) error { if err != nil { return err diff --git a/internal/install/static_terraform_deployer_dockerfile.go b/internal/install/static_terraform_deployer_dockerfile.go new file mode 100644 index 000000000..91b320e0e --- /dev/null +++ b/internal/install/static_terraform_deployer_dockerfile.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +const terraformDeployerDockerfile = `FROM hashicorp/terraform:light +ENV TF_IN_AUTOMATION=true +HEALTHCHECK --timeout=3s CMD sh -c "[ -f /tmp/tf-applied ]" +ADD run.sh / +WORKDIR /workspace +ENTRYPOINT sh /run.sh +` diff --git a/internal/install/static_terraform_deployer_run.go b/internal/install/static_terraform_deployer_run.go new file mode 100644 index 000000000..4d2b643bf --- /dev/null +++ b/internal/install/static_terraform_deployer_run.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +const terraformDeployerRun = `#!sh + +set -euxo pipefail + +cp -r /stage/*.tf /workspace + +cleanup() { + r=$? + + set -x + terraform destroy -auto-approve + + exit $r +} +trap cleanup EXIT INT TERM + +terraform init +terraform plan +terraform apply -auto-approve && touch /tmp/tf-applied + +echo "Terraform definitions applied." + +set +x +while true; do sleep 1; done # wait for ctrl-c +` diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go new file mode 100644 index 000000000..a4e740380 --- /dev/null +++ b/internal/install/static_terraform_deployer_yml.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +const terraformDeployerYml = `version: '2.3' +services: + aws_sns: + build: . + tty: true + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_PROFILE=${AWS_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} + volumes: + - ${TF_DIR}:/stage +` diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile index bb3982085..eccb7850e 100644 --- a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile +++ b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile @@ -1,10 +1,6 @@ FROM hashicorp/terraform:light - ENV TF_IN_AUTOMATION=true - HEALTHCHECK --timeout=3s CMD sh -c "[ -f /tmp/tf-applied ]" - ADD run.sh / - WORKDIR /workspace ENTRYPOINT sh /run.sh \ No newline at end of file From 62a6e853069d43b579f21f32c061e74abe4e854f Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 15:06:32 +0100 Subject: [PATCH 07/61] Cleanup --- .../sns/_dev/deploy/docker/Dockerfile | 6 ----- .../sns/_dev/deploy/docker/docker-compose.yml | 12 ---------- .../data_stream/sns/_dev/deploy/docker/run.sh | 24 ------------------- .../sns/_dev/deploy/{docker => }/tf/main.tf | 0 4 files changed, 42 deletions(-) delete mode 100644 test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile delete mode 100644 test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml delete mode 100755 test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh rename test/packages/aws/data_stream/sns/_dev/deploy/{docker => }/tf/main.tf (100%) diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile b/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile deleted file mode 100644 index eccb7850e..000000000 --- a/test/packages/aws/data_stream/sns/_dev/deploy/docker/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM hashicorp/terraform:light -ENV TF_IN_AUTOMATION=true -HEALTHCHECK --timeout=3s CMD sh -c "[ -f /tmp/tf-applied ]" -ADD run.sh / -WORKDIR /workspace -ENTRYPOINT sh /run.sh \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml b/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml deleted file mode 100644 index 29584142e..000000000 --- a/test/packages/aws/data_stream/sns/_dev/deploy/docker/docker-compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '2.3' -services: - aws_sns: - build: . - tty: true - environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_PROFILE=${AWS_PROFILE} - - AWS_REGION=${AWS_REGION:-us-east-1} - volumes: - - ${TF_DIR}:/stage diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh b/test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh deleted file mode 100755 index 4c0b718ad..000000000 --- a/test/packages/aws/data_stream/sns/_dev/deploy/docker/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!sh - -set -euxo pipefail - -cp -r /stage/*.tf /workspace - -cleanup() { - r=$? - - set -x - terraform destroy -auto-approve - - exit $r -} -trap cleanup EXIT INT TERM - -terraform init -terraform plan -terraform apply -auto-approve && touch /tmp/tf-applied - -echo "Terraform definitions applied." - -set +x -while true; do sleep 1; done # wait for ctrl-c \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/docker/tf/main.tf b/test/packages/aws/data_stream/sns/_dev/deploy/tf/main.tf similarity index 100% rename from test/packages/aws/data_stream/sns/_dev/deploy/docker/tf/main.tf rename to test/packages/aws/data_stream/sns/_dev/deploy/tf/main.tf From 7ddd7fe80d1b38dbeca1a52815227e827e78baed Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 15:49:00 +0100 Subject: [PATCH 08/61] Define sample test config for AWS SNS --- .../data_stream/sns/_dev/test/system/test-default-config.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml diff --git a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml new file mode 100644 index 000000000..835dab798 --- /dev/null +++ b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_SECRET_ACCESS_KEY} +data_stream: + vars: ~ From 0f2528412132c7638d0c7f41ecf05e415501d0f2 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 17:48:32 +0100 Subject: [PATCH 09/61] WIP --- internal/install/install.go | 18 ++- .../runners/system/servicedeployer/factory.go | 6 + .../system/servicedeployer/terraform.go | 107 ++++++++++++++++++ .../system/servicedeployer/terraform_env.go | 20 ++++ 4 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 internal/testrunner/runners/system/servicedeployer/terraform.go create mode 100644 internal/testrunner/runners/system/servicedeployer/terraform_env.go diff --git a/internal/install/install.go b/internal/install/install.go index ef408fba1..5682aad73 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -19,6 +19,8 @@ const ( packagesDir = "development" temporaryDir = "tmp" deployerDir = "deployer" + + terraformDeployerYmlFile = "terraform-deployer.yml" ) var ( @@ -96,6 +98,20 @@ func ServiceLogsDir() (string, error) { return filepath.Join(configurationDir, serviceLogsDir), nil } +func ServiceDeployerComposeFile(deployerName string) (string, error) { + configurationDir, err := configurationDir() + if err != nil { + return "", errors.Wrap(err, "locating configuration directory failed") + } + + switch deployerName { + case "terraform": + return filepath.Join(configurationDir, terraformDeployerDir, terraformDeployerYml), nil + default: + return "", errors.New("unsupported service deployer") + } +} + func configurationDir() (string, error) { homeDir, err := os.UserHomeDir() if err != nil { @@ -153,7 +169,7 @@ func writeDeployerResources(elasticPackagePath string) error { return errors.Wrapf(err, "creating directory failed (path: %s)", terraformDeployer) } - err = writeStaticResource(err, filepath.Join(terraformDeployer, "terraform-deployer.yml"), terraformDeployerYml) + err = writeStaticResource(err, filepath.Join(terraformDeployer, terraformDeployerYmlFile), terraformDeployerYml) err = writeStaticResource(err, filepath.Join(terraformDeployer, "Dockerfile"), terraformDeployerDockerfile) err = writeStaticResource(err, filepath.Join(terraformDeployer, "run.sh"), terraformDeployerRun) if err != nil { diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 503dfc720..7bb1ab752 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -43,6 +43,12 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { if _, err := os.Stat(dockerComposeYMLPath); err == nil { return NewDockerComposeServiceDeployer(dockerComposeYMLPath) } + + // Is the service defined using Terraform definition files? + terraformDirPath := filepath.Join(devDeployPath, "tf") + if _, err := os.Stat(terraformDirPath); err == nil { + return NewTerraformServiceDeployer(terraformDirPath) + } return nil, ErrNotFound } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go new file mode 100644 index 000000000..a98d1a7c1 --- /dev/null +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -0,0 +1,107 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +import ( + "bytes" + "fmt" + "os/exec" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/stack" +) + +type TerraformServiceDeployer struct { + definitionsDir string +} + +func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) { + return &TerraformServiceDeployer{ + definitionsDir: definitionsDir, + }, nil +} + +func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { + logger.Debug("setting up service using Terraform service deployer") + terraformDeployerYml, err := install.ServiceDeployerComposeFile("terraform") + if err != nil { + return nil, errors.Wrap(err, "can't locate docker compose file for service deployer") + } + + service := dockerComposeDeployedService{ + ymlPath: terraformDeployerYml, + project: "elastic-package-service", + } + outCtxt := inCtxt + + p, err := compose.NewProject(service.project, service.ymlPath) + if err != nil { + return nil, errors.Wrap(err, "could not create docker compose project for service") + } + + // Clean service logs + err = files.RemoveContent(outCtxt.Logs.Folder.Local) + if err != nil { + return nil, errors.Wrap(err, "removing service logs failed") + } + + // Boot up service + environmentVars, err := buildTerraformEnvironmentVars(inCtxt) + if err != nil { + return nil, errors.Wrap(err, "can't build environment variables") + } + + serviceName := inCtxt.Name + opts := compose.CommandOptions{ + Env: environmentVars, + ExtraArgs: []string{"--build", "-d"}, + } + if err := p.Up(opts); err != nil { + return nil, errors.Wrap(err, "could not boot up service using docker compose") + } + + // Build service container name + serviceContainer := fmt.Sprintf("%s_%s_1", service.project, serviceName) + outCtxt.Hostname = serviceContainer + + // Connect service network with stack network (for the purpose of metrics collection) + stackNetwork := fmt.Sprintf("%s_default", stack.DockerComposeProjectName) + logger.Debugf("attaching service container %s to stack network %s", serviceContainer, stackNetwork) + cmd := exec.Command("docker", "network", "connect", stackNetwork, serviceContainer) + errOutput := new(bytes.Buffer) + cmd.Stderr = errOutput + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "could not attach service container to the stack network (stderr=%q)", errOutput.String()) + } + + logger.Debugf("adding service container %s internal ports to context", serviceContainer) + serviceComposeConfig, err := p.Config(compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, + }) + if err != nil { + return nil, errors.Wrap(err, "could not get Docker Compose configuration for service") + } + + s := serviceComposeConfig.Services[serviceName] + outCtxt.Ports = make([]int, len(s.Ports)) + for idx, port := range s.Ports { + outCtxt.Ports[idx] = port.InternalPort + } + + // Shortcut to first port for convenience + if len(outCtxt.Ports) > 0 { + outCtxt.Port = outCtxt.Ports[0] + } + + service.ctxt = outCtxt + return &service, nil +} + +var _ ServiceDeployer = new(TerraformServiceDeployer) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go new file mode 100644 index 000000000..b7e62eb45 --- /dev/null +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -0,0 +1,20 @@ +package servicedeployer + +import "fmt" + +const ( + awsAccessKeyID = "AWS_ACCESS_KEY_ID" + awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + awsProfile = "AWS_PROFILE" + awsRegion = "AWS_REGION" +) + +func buildTerraformEnvironmentVars(ctxt ServiceContext) ([]string, error) { + var vars []string + vars = append(vars, fmt.Sprintf("%s=%s", serviceLogsDirEnv, ctxt.Logs.Folder.Local)) + + // TODO load vars + // TODO safe cmd prints + + return vars, nil +} From 6f7e4afa2afb0befc91c50f66d4d82a4dacd70c0 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 20 Jan 2021 17:48:52 +0100 Subject: [PATCH 10/61] Fix --- .../runners/system/servicedeployer/terraform_env.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index b7e62eb45..b283f14f5 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package servicedeployer import "fmt" From 6bacf601ffc05056eabc6863e0fc4c7e27eed5b3 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 12:04:39 +0100 Subject: [PATCH 11/61] Build terraform environment --- .../system/servicedeployer/terraform.go | 8 ++---- .../system/servicedeployer/terraform_env.go | 28 ++++++++++++++----- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index a98d1a7c1..fb84389fc 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -53,14 +53,10 @@ func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, } // Boot up service - environmentVars, err := buildTerraformEnvironmentVars(inCtxt) - if err != nil { - return nil, errors.Wrap(err, "can't build environment variables") - } - + tfEnvironment := buildTerraformExecutorEnvironment(inCtxt) serviceName := inCtxt.Name opts := compose.CommandOptions{ - Env: environmentVars, + Env: tfEnvironment, ExtraArgs: []string{"--build", "-d"}, } if err := p.Up(opts); err != nil { diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index b283f14f5..d197adca8 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -4,7 +4,10 @@ package servicedeployer -import "fmt" +import ( + "fmt" + "os" +) const ( awsAccessKeyID = "AWS_ACCESS_KEY_ID" @@ -13,12 +16,23 @@ const ( awsRegion = "AWS_REGION" ) -func buildTerraformEnvironmentVars(ctxt ServiceContext) ([]string, error) { - var vars []string - vars = append(vars, fmt.Sprintf("%s=%s", serviceLogsDirEnv, ctxt.Logs.Folder.Local)) +func buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { + vars := map[string]string{} + vars[serviceLogsDirEnv] = ctxt.Logs.Folder.Local + if os.Getenv(awsAccessKeyID) != "" && os.Getenv(awsSecretAccessKey) != "" { + vars[awsAccessKeyID] = os.Getenv(awsAccessKeyID) + vars[awsSecretAccessKey] = os.Getenv(awsSecretAccessKey) + } else if os.Getenv(awsProfile) != "" { + vars[awsProfile] = os.Getenv(awsProfile) + } - // TODO load vars - // TODO safe cmd prints + if os.Getenv(awsRegion) != "" { + vars[awsRegion] = os.Getenv(awsRegion) + } - return vars, nil + var pairs []string + for k, v := range vars { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + return pairs } From 1916b4fbcbeec81e523f10348b9b79dee71b3ab6 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 12:16:10 +0100 Subject: [PATCH 12/61] Mount TF_DIR --- .../testrunner/runners/system/servicedeployer/terraform.go | 5 ++++- .../runners/system/servicedeployer/terraform_env.go | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index fb84389fc..94c08dd4d 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -18,16 +18,19 @@ import ( "github.com/elastic/elastic-package/internal/stack" ) +// TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. type TerraformServiceDeployer struct { definitionsDir string } +// NewTerraformServiceDeployer creates an instance of TerraformServiceDeployer. func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) { return &TerraformServiceDeployer{ definitionsDir: definitionsDir, }, nil } +// SetUp method boots up the Docker Compose with Terraform executor and mounted .tf definitions. func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { logger.Debug("setting up service using Terraform service deployer") terraformDeployerYml, err := install.ServiceDeployerComposeFile("terraform") @@ -53,7 +56,7 @@ func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, } // Boot up service - tfEnvironment := buildTerraformExecutorEnvironment(inCtxt) + tfEnvironment := t.buildTerraformExecutorEnvironment(inCtxt) serviceName := inCtxt.Name opts := compose.CommandOptions{ Env: tfEnvironment, diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index d197adca8..f81426c9a 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -14,11 +14,15 @@ const ( awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" awsProfile = "AWS_PROFILE" awsRegion = "AWS_REGION" + + tfDir = "TF_DIR" ) -func buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { +func (tsd *TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { vars := map[string]string{} vars[serviceLogsDirEnv] = ctxt.Logs.Folder.Local + vars[tfDir] = tsd.definitionsDir + if os.Getenv(awsAccessKeyID) != "" && os.Getenv(awsSecretAccessKey) != "" { vars[awsAccessKeyID] = os.Getenv(awsAccessKeyID) vars[awsSecretAccessKey] = os.Getenv(awsSecretAccessKey) From 6248a8679d4369936e44bc6a7d4d95319a7a3200 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 12:50:30 +0100 Subject: [PATCH 13/61] Set envs in policies --- .../runners/system/servicedeployer/context.go | 12 +++++++++++- .../runners/system/servicedeployer/terraform.go | 3 +++ .../runners/system/servicedeployer/terraform_env.go | 9 +++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/context.go b/internal/testrunner/runners/system/servicedeployer/context.go index c4e968ae8..1353d02a8 100644 --- a/internal/testrunner/runners/system/servicedeployer/context.go +++ b/internal/testrunner/runners/system/servicedeployer/context.go @@ -38,13 +38,23 @@ type ServiceContext struct { Agent string } } + + // CustomProperties store additional data used to boot up the service, e.g. AWS credentials. + CustomProperties map[string]interface{} } // Aliases method returned aliases to properties of the service context. func (sc *ServiceContext) Aliases() map[string]interface{} { - return map[string]interface{}{ + m := map[string]interface{}{ serviceLogsDirEnv: func() interface{} { return sc.Logs.Folder.Agent }, } + + for k, v := range sc.CustomProperties { + m[k] = func() interface{} { // wrap as function + return v + } + } + return m } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 94c08dd4d..77ec4864a 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -99,6 +99,9 @@ func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, outCtxt.Port = outCtxt.Ports[0] } + // Set custom aliases, which may be used in agent policies. + outCtxt.CustomProperties = buildTerraformAliases() + service.ctxt = outCtxt return &service, nil } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index f81426c9a..80237ddf3 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -40,3 +40,12 @@ func (tsd *TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt Serv } return pairs } + +func buildTerraformAliases() map[string]interface{} { + return map[string]interface{}{ + awsAccessKeyID: os.Getenv(awsAccessKeyID), + awsSecretAccessKey: os.Getenv(awsSecretAccessKey), + awsProfile: os.Getenv(awsProfile), + awsRegion: os.Getenv(awsRegion), + } +} From 3ce33c8daa5f87cf54953242b7b2cee20c90811d Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 12:58:03 +0100 Subject: [PATCH 14/61] Fix: lint --- internal/install/install.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/install/install.go b/internal/install/install.go index 5682aad73..ba771a3b8 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -88,7 +88,7 @@ func StackPackagesDir() (string, error) { return filepath.Join(stackDir, packagesDir), nil } -// ServiceLogsDir method returns the location of the directory to store service logs on the +// ServiceLogsDir function returns the location of the directory to store service logs on the // local filesystem, i.e. the same one where elastic-package is installed. func ServiceLogsDir() (string, error) { configurationDir, err := configurationDir() @@ -98,6 +98,7 @@ func ServiceLogsDir() (string, error) { return filepath.Join(configurationDir, serviceLogsDir), nil } +// ServiceDeployerComposeFile function returns the path to the selected service deployer's Docker Compose definition. func ServiceDeployerComposeFile(deployerName string) (string, error) { configurationDir, err := configurationDir() if err != nil { From 3b5db75629ab785f63116fdd1762b0063ca4ef87 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 13:36:19 +0100 Subject: [PATCH 15/61] Fix: build dir (empty value) --- internal/cleanup/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cleanup/build.go b/internal/cleanup/build.go index fc370db04..1dd5aa7e0 100644 --- a/internal/cleanup/build.go +++ b/internal/cleanup/build.go @@ -35,7 +35,7 @@ func Build() (string, error) { } if !found { - logger.Debugf("Build directory doesn't exist (missing path: %s)", buildDir) + logger.Debug("Build directory doesn't exist") return "", nil } From 958c43b6f8d6d91f418cb4790f55610da4d6e913 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 13:39:34 +0100 Subject: [PATCH 16/61] Fix: yml file --- internal/install/install.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/install/install.go b/internal/install/install.go index ba771a3b8..1fa79931a 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -107,7 +107,7 @@ func ServiceDeployerComposeFile(deployerName string) (string, error) { switch deployerName { case "terraform": - return filepath.Join(configurationDir, terraformDeployerDir, terraformDeployerYml), nil + return filepath.Join(configurationDir, terraformDeployerDir, terraformDeployerYmlFile), nil default: return "", errors.New("unsupported service deployer") } From ebdf53ad637f8b59f0428f38fd88769d92f5deb2 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 13:53:31 +0100 Subject: [PATCH 17/61] Fix: service name --- internal/install/static_terraform_deployer_yml.go | 2 +- internal/testrunner/runners/system/servicedeployer/terraform.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go index a4e740380..0fa6b2b1d 100644 --- a/internal/install/static_terraform_deployer_yml.go +++ b/internal/install/static_terraform_deployer_yml.go @@ -6,7 +6,7 @@ package install const terraformDeployerYml = `version: '2.3' services: - aws_sns: + terraform: build: . tty: true environment: diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 77ec4864a..360b695ab 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -67,7 +67,7 @@ func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, } // Build service container name - serviceContainer := fmt.Sprintf("%s_%s_1", service.project, serviceName) + serviceContainer := fmt.Sprintf("%s_terraform_1", service.project) outCtxt.Hostname = serviceContainer // Connect service network with stack network (for the purpose of metrics collection) From dcfb15ce3e7a881c9b9d4da406237928df5b7acf Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 14:01:57 +0100 Subject: [PATCH 18/61] Fix: brackets --- .../data_stream/sns/_dev/test/system/test-default-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml index 835dab798..17594d6d8 100644 --- a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml @@ -1,5 +1,5 @@ vars: - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_SECRET_ACCESS_KEY} + access_key_id: {{AWS_ACCESS_KEY_ID}} + secret_access_key: {{AWS_SECRET_ACCESS_KEY}} data_stream: vars: ~ From 2b8fac7449275a80aeccf51fb2525db6cae8ce0e Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 14:11:30 +0100 Subject: [PATCH 19/61] Fix --- .../testrunner/runners/system/servicedeployer/terraform.go | 4 ++-- .../runners/system/servicedeployer/terraform_env.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 360b695ab..855517cb3 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -31,7 +31,7 @@ func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeploy } // SetUp method boots up the Docker Compose with Terraform executor and mounted .tf definitions. -func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { +func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { logger.Debug("setting up service using Terraform service deployer") terraformDeployerYml, err := install.ServiceDeployerComposeFile("terraform") if err != nil { @@ -56,7 +56,7 @@ func (t TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, } // Boot up service - tfEnvironment := t.buildTerraformExecutorEnvironment(inCtxt) + tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) serviceName := inCtxt.Name opts := compose.CommandOptions{ Env: tfEnvironment, diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index 80237ddf3..4575112a5 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -18,7 +18,7 @@ const ( tfDir = "TF_DIR" ) -func (tsd *TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { +func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { vars := map[string]string{} vars[serviceLogsDirEnv] = ctxt.Logs.Folder.Local vars[tfDir] = tsd.definitionsDir From 12efcffdaaafe9486c17d8508211faca25a30e2b Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 14:46:28 +0100 Subject: [PATCH 20/61] Fix: reference issue --- internal/testrunner/runners/system/servicedeployer/context.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/context.go b/internal/testrunner/runners/system/servicedeployer/context.go index 1353d02a8..026062d61 100644 --- a/internal/testrunner/runners/system/servicedeployer/context.go +++ b/internal/testrunner/runners/system/servicedeployer/context.go @@ -52,8 +52,9 @@ func (sc *ServiceContext) Aliases() map[string]interface{} { } for k, v := range sc.CustomProperties { + var that = v m[k] = func() interface{} { // wrap as function - return v + return that } } return m From 6ae050c329c5d26e8418aba454a8377157dabc4b Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 14:58:16 +0100 Subject: [PATCH 21/61] Fix: period --- .../data_stream/sns/_dev/test/system/test-default-config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml index 17594d6d8..33caa4fb1 100644 --- a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml @@ -2,4 +2,5 @@ vars: access_key_id: {{AWS_ACCESS_KEY_ID}} secret_access_key: {{AWS_SECRET_ACCESS_KEY}} data_stream: - vars: ~ + vars: + period: 10s From ba9cd9f778d3b77de1664ff21c544a89b9544f6f Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 15:33:44 +0100 Subject: [PATCH 22/61] Set time --- internal/testrunner/runners/system/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 0212d49c6..080931556 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -389,7 +389,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext logger.Debug("checking for expected data in data stream...") passed, err := waitUntilTrue(r.hasNumDocs(dataStream, fieldsValidator, func(n int) bool { return n > 0 - }), 2*time.Minute) + }), 10*time.Minute) if err != nil { return result.withError(err) From b908c9d4d03927a7a46c6c430939dac2f95daa90 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 15:47:55 +0100 Subject: [PATCH 23/61] Do not print request body to prevent leaks --- internal/kibana/client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/kibana/client.go b/internal/kibana/client.go index 32371db13..cd2b700c8 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -72,7 +72,6 @@ func (c *Client) sendRequest(method, resourcePath string, body []byte) (int, []b u := base.ResolveReference(rel) logger.Debugf("%s %s", method, u) - logger.Debugf("%s", body) req, err := http.NewRequest(method, u.String(), reqBody) if err != nil { From c1ded23ab2baae43bb08fa7913d7c06d4ad6da3a Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 21 Jan 2021 15:49:52 +0100 Subject: [PATCH 24/61] Fix: format --- .../data_stream/sns/_dev/test/system/test-default-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml index 33caa4fb1..9018fba99 100644 --- a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml @@ -1,6 +1,6 @@ vars: - access_key_id: {{AWS_ACCESS_KEY_ID}} - secret_access_key: {{AWS_SECRET_ACCESS_KEY}} + access_key_id: '{{AWS_ACCESS_KEY_ID}}' + secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' data_stream: vars: period: 10s From 5e26d8a3bed664c0e6e10e9cddc5141d43370fa4 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 25 Jan 2021 11:49:45 +0100 Subject: [PATCH 25/61] Sample for EC2 metrics --- .../ec2_metrics/_dev/deploy/tf/main.tf | 25 +++++++++++++++++++ .../_dev/test/system/test-default-config.yml | 6 +++++ 2 files changed, 31 insertions(+) create mode 100644 test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf create mode 100644 test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf new file mode 100644 index 000000000..0b80c67a8 --- /dev/null +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -0,0 +1,25 @@ +provider "aws" {} + +resource "random_string" "id" { + length = 5 + upper = false + lower = true + number = false + special = false +} + +resource "aws_instance" "i" { + ami = "${data.aws_ami.latest-amzn.id}" + instance_type = "t1.micro" + tags = { + Name = "elastic-package-test-${random_string.id.result}" + } +} + +data "aws_ami" "latest-amzn" { + most_recent = true + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml new file mode 100644 index 000000000..9018fba99 --- /dev/null +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +vars: + access_key_id: '{{AWS_ACCESS_KEY_ID}}' + secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' +data_stream: + vars: + period: 10s From 0f640983c176affe0cb0092f5d2bbc88d91472bd Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 25 Jan 2021 15:04:38 +0100 Subject: [PATCH 26/61] Fix EC2 system test --- .../data_stream/ec2_metrics/_dev/deploy/tf/main.tf | 13 +++---------- .../_dev/test/system/test-default-config.yml | 5 ++++- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf index 0b80c67a8..dfd6673ee 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -1,23 +1,16 @@ provider "aws" {} -resource "random_string" "id" { - length = 5 - upper = false - lower = true - number = false - special = false -} - resource "aws_instance" "i" { - ami = "${data.aws_ami.latest-amzn.id}" + ami = data.aws_ami.latest-amzn.id instance_type = "t1.micro" tags = { - Name = "elastic-package-test-${random_string.id.result}" + Name = "elastic-package-test" } } data "aws_ami" "latest-amzn" { most_recent = true + owners = [ "amazon" ] # AWS filter { name = "name" values = ["amzn2-ami-hvm-*"] diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index 9018fba99..8500d9872 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -3,4 +3,7 @@ vars: secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' data_stream: vars: - period: 10s + period: 60s + tags_filter: | + - name: Name + - value: "elastic-package-test" \ No newline at end of file From 075a2dbb25c10b2061f0e6c9b3d13ad9582a23d3 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 25 Jan 2021 15:05:04 +0100 Subject: [PATCH 27/61] Disable SNS test --- .../aws/data_stream/sns/_dev/deploy/tf/main.tf | 13 ------------- .../sns/_dev/test/system/test-default-config.yml | 6 ------ 2 files changed, 19 deletions(-) delete mode 100644 test/packages/aws/data_stream/sns/_dev/deploy/tf/main.tf delete mode 100644 test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml diff --git a/test/packages/aws/data_stream/sns/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/sns/_dev/deploy/tf/main.tf deleted file mode 100644 index 4a703a5f2..000000000 --- a/test/packages/aws/data_stream/sns/_dev/deploy/tf/main.tf +++ /dev/null @@ -1,13 +0,0 @@ -provider "aws" {} - -resource "random_string" "id" { - length = 5 - upper = false - lower = true - number = false - special = false -} - -resource "aws_sns_topic" "t" { - name = "ep-sns-topic-${random_string.id.result}" -} \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml deleted file mode 100644 index 9018fba99..000000000 --- a/test/packages/aws/data_stream/sns/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -vars: - access_key_id: '{{AWS_ACCESS_KEY_ID}}' - secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' -data_stream: - vars: - period: 10s From 14174ee9551e4f6973e230e6321769aab99816d5 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 25 Jan 2021 15:20:35 +0100 Subject: [PATCH 28/61] Pass test run ID --- internal/install/static_terraform_deployer_yml.go | 1 + internal/testrunner/runners/system/runner.go | 6 ++++++ .../runners/system/servicedeployer/context.go | 14 +++++++++++++- .../system/servicedeployer/terraform_env.go | 4 +++- .../data_stream/ec2_metrics/_dev/deploy/tf/main.tf | 6 +++++- .../_dev/test/system/test-default-config.yml | 2 +- 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go index 0fa6b2b1d..57c255358 100644 --- a/internal/install/static_terraform_deployer_yml.go +++ b/internal/install/static_terraform_deployer_yml.go @@ -14,6 +14,7 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - AWS_PROFILE=${AWS_PROFILE} - AWS_REGION=${AWS_REGION:-us-east-1} + - TF_VAR_TEST_RUN_ID=${TF_VAR_TEST_RUN_ID:-detached} volumes: - ${TF_DIR}:/stage ` diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 080931556..0fcd27b28 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -7,6 +7,7 @@ package system import ( "encoding/json" "fmt" + "math/rand" "path/filepath" "strings" "time" @@ -158,6 +159,7 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { ctxt.Name = r.options.TestFolder.Package ctxt.Logs.Folder.Local = serviceLogsDir ctxt.Logs.Folder.Agent = serviceLogsAgentDir + ctxt.Test.RunID = createTestRunID() testConfig, err := newConfig(filepath.Join(r.options.TestFolder.Path, cfgFile), ctxt) if err != nil { return result.withError(errors.Wrapf(err, "unable to load system test case file '%s'", cfgFile)) @@ -174,6 +176,10 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { return results, nil } +func createTestRunID() string { + return fmt.Sprintf("%d", rand.Intn(99999-10000)+10000) +} + func (r *runner) hasNumDocs( dataStream string, fieldsValidator *fields.Validator, diff --git a/internal/testrunner/runners/system/servicedeployer/context.go b/internal/testrunner/runners/system/servicedeployer/context.go index 026062d61..890a2c3ff 100644 --- a/internal/testrunner/runners/system/servicedeployer/context.go +++ b/internal/testrunner/runners/system/servicedeployer/context.go @@ -4,7 +4,10 @@ package servicedeployer -const serviceLogsDirEnv = "SERVICE_LOGS_DIR" +const ( + serviceLogsDirEnv = "SERVICE_LOGS_DIR" + testRunIDEnv = "TEST_RUN_ID" +) // ServiceContext encapsulates context that is both available to a ServiceDeployer and // populated by a DeployedService. The fields in ServiceContext may be used in handlebars @@ -39,6 +42,12 @@ type ServiceContext struct { } } + // Test related properties. + Test struct { + // RunID identifies the current test run. + RunID string + } + // CustomProperties store additional data used to boot up the service, e.g. AWS credentials. CustomProperties map[string]interface{} } @@ -49,6 +58,9 @@ func (sc *ServiceContext) Aliases() map[string]interface{} { serviceLogsDirEnv: func() interface{} { return sc.Logs.Folder.Agent }, + testRunIDEnv: func() interface{} { + return sc.Test.RunID + }, } for k, v := range sc.CustomProperties { diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index 4575112a5..dad2d2e46 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -15,12 +15,14 @@ const ( awsProfile = "AWS_PROFILE" awsRegion = "AWS_REGION" - tfDir = "TF_DIR" + tfDir = "TF_DIR" + tfTestRunID = "TF_VAR_TEST_RUN_ID" ) func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { vars := map[string]string{} vars[serviceLogsDirEnv] = ctxt.Logs.Folder.Local + vars[tfTestRunID] = ctxt.Test.RunID vars[tfDir] = tsd.definitionsDir if os.Getenv(awsAccessKeyID) != "" && os.Getenv(awsSecretAccessKey) != "" { diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf index dfd6673ee..da9c1c722 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -1,10 +1,14 @@ +variable "test_run_id" { + default = "detached" +} + provider "aws" {} resource "aws_instance" "i" { ami = data.aws_ami.latest-amzn.id instance_type = "t1.micro" tags = { - Name = "elastic-package-test" + Name = "elastic-package-test-${var.test_run_id}" } } diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index 8500d9872..b7ed711a9 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -6,4 +6,4 @@ data_stream: period: 60s tags_filter: | - name: Name - - value: "elastic-package-test" \ No newline at end of file + - value: "elastic-package-test-{{TEST_RUN_ID}}" \ No newline at end of file From 2d2b18bec1ed3631e82d469c8091eaa62b917357 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 25 Jan 2021 15:35:38 +0100 Subject: [PATCH 29/61] Next fixes --- main.go | 4 ++++ .../aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf | 4 ++-- .../ec2_metrics/_dev/test/system/test-default-config.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index f2895e852..e551273bf 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,9 @@ package main import ( "log" + "math/rand" "os" + "time" "github.com/pkg/errors" @@ -15,6 +17,8 @@ import ( ) func main() { + rand.Seed(time.Now().UnixNano()) + rootCmd := cmd.RootCmd() err := install.EnsureInstalled() diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf index da9c1c722..a57ca8458 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -1,4 +1,4 @@ -variable "test_run_id" { +variable "TEST_RUN_ID" { default = "detached" } @@ -8,7 +8,7 @@ resource "aws_instance" "i" { ami = data.aws_ami.latest-amzn.id instance_type = "t1.micro" tags = { - Name = "elastic-package-test-${var.test_run_id}" + Name = "elastic-package-test-${var.TEST_RUN_ID}" } } diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index b7ed711a9..93ff5a54b 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -6,4 +6,4 @@ data_stream: period: 60s tags_filter: | - name: Name - - value: "elastic-package-test-{{TEST_RUN_ID}}" \ No newline at end of file + value: "elastic-package-test-{{TEST_RUN_ID}}" \ No newline at end of file From 5806a4e0409a92f5cb7d93a8994f9afd8943a104 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 25 Jan 2021 15:49:39 +0100 Subject: [PATCH 30/61] Fix: formatter --- .../ec2_metrics/_dev/test/system/test-default-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index 93ff5a54b..70375c1c2 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -4,6 +4,6 @@ vars: data_stream: vars: period: 60s - tags_filter: | + tags_filter: |- - name: Name - value: "elastic-package-test-{{TEST_RUN_ID}}" \ No newline at end of file + value: "elastic-package-test-{{TEST_RUN_ID}}" From 652755ef95395c408ced7f7f6a4ee6c89562ca2e Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 26 Jan 2021 13:09:54 +0100 Subject: [PATCH 31/61] Try: enable monitoring --- internal/install/static_terraform_deployer_yml.go | 2 +- internal/testrunner/runners/system/servicedeployer/terraform.go | 1 + .../packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go index 57c255358..39b4a019d 100644 --- a/internal/install/static_terraform_deployer_yml.go +++ b/internal/install/static_terraform_deployer_yml.go @@ -10,7 +10,7 @@ services: build: . tty: true environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # TODO Extract to the env-file - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - AWS_PROFILE=${AWS_PROFILE} - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 855517cb3..79df8ecfc 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -89,6 +89,7 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic } s := serviceComposeConfig.Services[serviceName] + // TODO remove ports outCtxt.Ports = make([]int, len(s.Ports)) for idx, port := range s.Ports { outCtxt.Ports[idx] = port.InternalPort diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf index a57ca8458..929994242 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -6,6 +6,7 @@ provider "aws" {} resource "aws_instance" "i" { ami = data.aws_ami.latest-amzn.id + monitoring = true instance_type = "t1.micro" tags = { Name = "elastic-package-test-${var.TEST_RUN_ID}" From 90d0a1ca79796c5ae27c374447c6a42a5acd0c45 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 26 Jan 2021 16:16:02 +0100 Subject: [PATCH 32/61] Update dependency on package-spec --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 256da1979..0e0c24101 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.9.0 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-ucfg v0.8.3 - github.com/elastic/package-spec/code/go v0.0.0-20210125131944-e98a6af2135e + github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-openapi/strfmt v0.19.6 // indirect diff --git a/go.sum b/go.sum index 0a3a5c2a0..c0e4a5dc5 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20210125131944-e98a6af2135e h1:yKE0ydG2iSNrC132l6SYY+79H+hmrV7/IhsCSHeHVFM= -github.com/elastic/package-spec/code/go v0.0.0-20210125131944-e98a6af2135e/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= +github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 h1:QeQvLQvDx1mRAOBzcJ7T7ebF02mJxcK5HYsYTTOrsik= +github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= From be253b7ab00846a80cb8f9a7de9094015fec460c Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 10:58:11 +0100 Subject: [PATCH 33/61] Address PR comments --- .../testrunner/runners/system/servicedeployer/factory.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 503dfc720..813ef64a0 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -5,7 +5,6 @@ package servicedeployer import ( - "fmt" "os" "path/filepath" @@ -36,7 +35,6 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { logger.Errorf("can't find _dev/deploy directory") return nil, ErrNotFound } - fmt.Println(devDeployPath) // Is the service defined using a docker compose configuration file? dockerComposeYMLPath := filepath.Join(devDeployPath, "docker", "docker-compose.yml") @@ -52,7 +50,7 @@ func findDevDeployPath(options FactoryOptions) (string, error) { if err == nil { return dataStreamDevDeployPath, nil } else if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "stat failed (path: %s)", dataStreamDevDeployPath) + return "", errors.Wrapf(err, "stat failed for data stream (path: %s)", dataStreamDevDeployPath) } packageDevDeployPath := filepath.Join(options.PackageRootPath, devDeployDir) @@ -60,7 +58,7 @@ func findDevDeployPath(options FactoryOptions) (string, error) { if err == nil { return packageDevDeployPath, nil } else if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "stat failed (path: %s)", packageDevDeployPath) + return "", errors.Wrapf(err, "stat failed for package (path: %s)", packageDevDeployPath) } return "", errors.New("_dev directory doesn't exist") } From ca8faecf1cc30a751bdd892dc1b4fef9abf2ecee Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 11:20:37 +0100 Subject: [PATCH 34/61] Fix failing test --- .../_dev/test/system/test-default-config.yml | 2 +- .../aws/data_stream/ec2_metrics/fields/ecs.yml | 4 ++++ .../data_stream/ec2_metrics/fields/fields.yml | 16 ++++++++++++++++ test/packages/aws/docs/README.md | 6 ++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index 70375c1c2..41edf7121 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -5,5 +5,5 @@ data_stream: vars: period: 60s tags_filter: |- - - name: Name + - key: Name value: "elastic-package-test-{{TEST_RUN_ID}}" diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml index e49975bc2..432ee5f4d 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml b/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml index 066b1eee4..833aba3ec 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml @@ -78,6 +78,14 @@ format: bytes description: | Bytes read from all instance store volumes available to the instance. + - name: diskio.read.count + type: long + format: bytes + description: The number of disk IO reads + - name: diskio.read.count_per_sec + type: long + format: bytes + description: The number of disk IO reads per second - name: diskio.read.bytes_per_sec type: long description: | @@ -99,6 +107,14 @@ type: long description: | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. + - name: diskio.write.count + type: long + format: bytes + description: The number of disk IO writes + - name: diskio.write.count_per_sec + type: long + format: bytes + description: The number of disk IO writes per second - name: diskio.write.ops type: long description: | diff --git a/test/packages/aws/docs/README.md b/test/packages/aws/docs/README.md index 5a66b4514..a00fe0ef3 100644 --- a/test/packages/aws/docs/README.md +++ b/test/packages/aws/docs/README.md @@ -1311,10 +1311,14 @@ An example event for `ec2` looks as following: | aws.ec2.cpu.total.pct | The percentage of allocated EC2 compute units that are currently in use on the instance. | scaled_float | | aws.ec2.diskio.read.bytes | Bytes read from all instance store volumes available to the instance. | long | | aws.ec2.diskio.read.bytes_per_sec | Bytes read per second from all instance store volumes available to the instance. | long | +| aws.ec2.diskio.read.count | The number of disk IO reads | long | +| aws.ec2.diskio.read.count_per_sec | The number of disk IO reads per second | long | | aws.ec2.diskio.read.ops | Completed read operations from all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.diskio.read.ops_per_sec | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.diskio.write.bytes | Bytes written to all instance store volumes available to the instance. | long | | aws.ec2.diskio.write.bytes_per_sec | Bytes written per second to all instance store volumes available to the instance. | long | +| aws.ec2.diskio.write.count | The number of disk IO writes | long | +| aws.ec2.diskio.write.count_per_sec | The number of disk IO writes per second | long | | aws.ec2.diskio.write.ops | Completed write operations to all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.diskio.write.ops_per_sec | Completed write operations per second to all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.instance.core.count | The number of CPU cores for the instance. | integer | @@ -1357,6 +1361,7 @@ An example event for `ec2` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.cpu.pct | Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. | scaled_float | @@ -1380,6 +1385,7 @@ An example event for `ec2` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### elb From 3c9b61e58d8573fb4ede8efb4ca341f4511253b7 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 11:28:29 +0100 Subject: [PATCH 35/61] Fix: go.sum --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 3e24025db..976f0836b 100644 --- a/go.sum +++ b/go.sum @@ -71,7 +71,6 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From 57ab7ba98db9f9b695c7c4c67d1dc4618686903d Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 11:32:15 +0100 Subject: [PATCH 36/61] Don't set ports --- .../system/servicedeployer/terraform.go | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 79df8ecfc..71f90fe7f 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -57,7 +57,6 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic // Boot up service tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) - serviceName := inCtxt.Name opts := compose.CommandOptions{ Env: tfEnvironment, ExtraArgs: []string{"--build", "-d"}, @@ -80,26 +79,6 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return nil, errors.Wrapf(err, "could not attach service container to the stack network (stderr=%q)", errOutput.String()) } - logger.Debugf("adding service container %s internal ports to context", serviceContainer) - serviceComposeConfig, err := p.Config(compose.CommandOptions{ - Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, - }) - if err != nil { - return nil, errors.Wrap(err, "could not get Docker Compose configuration for service") - } - - s := serviceComposeConfig.Services[serviceName] - // TODO remove ports - outCtxt.Ports = make([]int, len(s.Ports)) - for idx, port := range s.Ports { - outCtxt.Ports[idx] = port.InternalPort - } - - // Shortcut to first port for convenience - if len(outCtxt.Ports) > 0 { - outCtxt.Port = outCtxt.Ports[0] - } - // Set custom aliases, which may be used in agent policies. outCtxt.CustomProperties = buildTerraformAliases() From 4b2cc85060b614c4a7c04ed3c602e0c6da257e10 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 11:47:40 +0100 Subject: [PATCH 37/61] Update package-spec --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0e0c24101..05fa0d69a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.9.0 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-ucfg v0.8.3 - github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 + github.com/elastic/package-spec/code/go v0.0.0-20210126151110-08a493a36954 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-openapi/strfmt v0.19.6 // indirect diff --git a/go.sum b/go.sum index 976f0836b..09e5bfeaf 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 h1:QeQvLQvDx1mRAOBzcJ7T7ebF02mJxcK5HYsYTTOrsik= -github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= +github.com/elastic/package-spec/code/go v0.0.0-20210126151110-08a493a36954 h1:KTExarO1tJf91RNJPoBBeZqjC8IsYamBW0Krhw8dYfg= +github.com/elastic/package-spec/code/go v0.0.0-20210126151110-08a493a36954/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= From 3cc0cfb2ebac80834ec54373130e1f76501ee7c7 Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 12:13:48 +0100 Subject: [PATCH 38/61] Collect logs for integrations --- .ci/Jenkinsfile | 23 ++++++++++++++++++++++- scripts/test-check-packages.sh | 3 +++ scripts/test-stack-command.sh | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 965e0dc9d..4d7aeb454 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { BASE_DIR="src/github.com/elastic/elastic-package" JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" PIPELINE_LOG_LEVEL='INFO' + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' } options { timeout(time: 1, unit: 'HOURS') @@ -49,7 +50,8 @@ pipeline { always { dir("${BASE_DIR}") { archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/test-results/*.xml') - archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/elastic-stack-dump/logs/*.log') + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/elastic-stack-dump/stack/logs/*.log') + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/elastic-stack-dump/check/logs/*.log') junit(allowEmptyResults: false, keepLongStdio: true, testResults: "build/test-results/*.xml") @@ -71,3 +73,22 @@ def cleanup(){ } unstash 'source' } + +def withCloudTestEnv(Closure body) { + def maskedVars = [] + // AWS + def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data + if (!aws.containsKey('access_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") + } + if (!aws.containsKey('secret_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") + } + maskedVars.addAll([ + [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], + [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], + ]) + withEnvMask(vars: maskedVars) { + body() + } +} \ No newline at end of file diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index bba3c31f5..e69edb13d 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -5,6 +5,9 @@ set -euxo pipefail cleanup() { r=$? + # Dump stack logs + elastic-package stack dump -v --output build/elastic-stack-dump/check + # Take down the stack elastic-package stack down -v diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index e38154846..6d2c6d525 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -6,7 +6,7 @@ cleanup() { r=$? # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump + elastic-package stack dump -v --output build/elastic-stack-dump/stack # Take down the stack elastic-package stack down -v From d9f5ec34fb0e93100a9eef72d07a3b0e0b47322a Mon Sep 17 00:00:00 2001 From: mtojek Date: Wed, 27 Jan 2021 12:46:03 +0100 Subject: [PATCH 39/61] Use cloud environment --- .ci/Jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4d7aeb454..5cc10b2e9 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -41,8 +41,10 @@ pipeline { steps { cleanup() withMageEnv(){ - dir("${BASE_DIR}"){ - sh(label: 'Check',script: 'make check') + withCloudTestEnv() { + dir("${BASE_DIR}"){ + sh(label: 'Check',script: 'make check') + } } } } From c990cb7a6e9f0c4f20af53f8cc7639ae0d8f0060 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 14:59:02 +0100 Subject: [PATCH 40/61] Fix: go mod tidy --- go.sum | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/go.sum b/go.sum index 0f369434b..976f0836b 100644 --- a/go.sum +++ b/go.sum @@ -87,19 +87,27 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= +github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 h1:QeQvLQvDx1mRAOBzcJ7T7ebF02mJxcK5HYsYTTOrsik= github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM= github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.0.1 h1:q+IFMfLx200Q3scvt2hN79JsEzy4AmBTp/pqnefH+Bc= github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= +github.com/go-git/go-git/v5 v5.1.0 h1:HxJn9g/E7eYvKW3Fm7Jt4ee8LXfPOm/H1cdDu8vEssk= github.com/go-git/go-git/v5 v5.1.0/go.mod h1:ZKfuPUoY1ZqIG4QG9BDBh3G4gLM5zvPuSJAozQrZuyM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -107,8 +115,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/strfmt v0.19.6 h1:epWc+q5qSgsy7A7+/HYyxLF37vLEYdPSkNB9G8mRqjw= github.com/go-openapi/strfmt v0.19.6/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -136,6 +147,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -145,8 +157,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -158,6 +173,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -168,19 +184,26 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ= github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -188,29 +211,42 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -222,10 +258,12 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -233,9 +271,11 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -245,14 +285,20 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -261,6 +307,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -279,6 +326,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -302,6 +350,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -310,6 +359,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -340,12 +390,14 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -389,11 +441,13 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -439,10 +493,12 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -465,6 +521,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -517,21 +574,26 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 36bf34b5fb8c7b6b7a213cdabdfc4d4b46826463 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:06:02 +0100 Subject: [PATCH 41/61] Update package-spec dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2b30f425d..398243f2b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.9.0 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-ucfg v0.8.3 - github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 + github.com/elastic/package-spec/code/go v0.0.0-20210127201409-dd08da649371 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-openapi/strfmt v0.19.6 // indirect diff --git a/go.sum b/go.sum index 976f0836b..2c8b75c80 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 h1:QeQvLQvDx1mRAOBzcJ7T7ebF02mJxcK5HYsYTTOrsik= -github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= +github.com/elastic/package-spec/code/go v0.0.0-20210127201409-dd08da649371 h1:YjBuT9e9rP5vms2NWJVu6MAccgrSlOUlbykkjfa10aY= +github.com/elastic/package-spec/code/go v0.0.0-20210127201409-dd08da649371/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= From c7ad30193fe51aa77b7548bb67c653302687814e Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:11:35 +0100 Subject: [PATCH 42/61] Support session token --- internal/install/static_terraform_deployer_yml.go | 1 + .../system/servicedeployer/terraform_env.go | 15 +++++++++++++-- .../_dev/test/system/test-default-config.yml | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go index 39b4a019d..2fd63fb08 100644 --- a/internal/install/static_terraform_deployer_yml.go +++ b/internal/install/static_terraform_deployer_yml.go @@ -12,6 +12,7 @@ services: environment: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # TODO Extract to the env-file - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - AWS_PROFILE=${AWS_PROFILE} - AWS_REGION=${AWS_REGION:-us-east-1} - TF_VAR_TEST_RUN_ID=${TF_VAR_TEST_RUN_ID:-detached} diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index dad2d2e46..1ca7a6f9b 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -12,6 +12,7 @@ import ( const ( awsAccessKeyID = "AWS_ACCESS_KEY_ID" awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + awsSessionToken = "AWS_SESSION_TOKEN" awsProfile = "AWS_PROFILE" awsRegion = "AWS_REGION" @@ -25,10 +26,19 @@ func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt Servi vars[tfTestRunID] = ctxt.Test.RunID vars[tfDir] = tsd.definitionsDir - if os.Getenv(awsAccessKeyID) != "" && os.Getenv(awsSecretAccessKey) != "" { + if os.Getenv(awsAccessKeyID) != "" { vars[awsAccessKeyID] = os.Getenv(awsAccessKeyID) + } + + if os.Getenv(awsSecretAccessKey) != "" { vars[awsSecretAccessKey] = os.Getenv(awsSecretAccessKey) - } else if os.Getenv(awsProfile) != "" { + } + + if os.Getenv(awsSessionToken) != "" { + vars[awsSessionToken] = os.Getenv(awsSessionToken) + } + + if os.Getenv(awsProfile) != "" { vars[awsProfile] = os.Getenv(awsProfile) } @@ -47,6 +57,7 @@ func buildTerraformAliases() map[string]interface{} { return map[string]interface{}{ awsAccessKeyID: os.Getenv(awsAccessKeyID), awsSecretAccessKey: os.Getenv(awsSecretAccessKey), + awsSessionToken: os.Getenv(awsSessionToken), awsProfile: os.Getenv(awsProfile), awsRegion: os.Getenv(awsRegion), } diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index 41edf7121..d164a5abd 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -1,6 +1,7 @@ vars: access_key_id: '{{AWS_ACCESS_KEY_ID}}' secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' + session_token: '{{AWS_SESSION_TOKEN}}' data_stream: vars: period: 60s From 77ff5c9189076a8c6bdf8abb88032e80b05345b4 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:21:40 +0100 Subject: [PATCH 43/61] Terraform service deployer --- internal/install/install.go | 12 +++--------- .../runners/system/servicedeployer/terraform.go | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/internal/install/install.go b/internal/install/install.go index 1fa79931a..cc5edf9e8 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -98,19 +98,13 @@ func ServiceLogsDir() (string, error) { return filepath.Join(configurationDir, serviceLogsDir), nil } -// ServiceDeployerComposeFile function returns the path to the selected service deployer's Docker Compose definition. -func ServiceDeployerComposeFile(deployerName string) (string, error) { +// TerraformDeployerComposeFile function returns the path to the Terraform service deployer's definitions. +func TerraformDeployerComposeFile() (string, error) { configurationDir, err := configurationDir() if err != nil { return "", errors.Wrap(err, "locating configuration directory failed") } - - switch deployerName { - case "terraform": - return filepath.Join(configurationDir, terraformDeployerDir, terraformDeployerYmlFile), nil - default: - return "", errors.New("unsupported service deployer") - } + return filepath.Join(configurationDir, terraformDeployerDir, terraformDeployerYmlFile), nil } func configurationDir() (string, error) { diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 71f90fe7f..9f11bae2a 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -33,7 +33,7 @@ func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeploy // SetUp method boots up the Docker Compose with Terraform executor and mounted .tf definitions. func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { logger.Debug("setting up service using Terraform service deployer") - terraformDeployerYml, err := install.ServiceDeployerComposeFile("terraform") + terraformDeployerYml, err := install.TerraformDeployerComposeFile() if err != nil { return nil, errors.Wrap(err, "can't locate docker compose file for service deployer") } From 7ddc1f4f2bd8c75cf9787ed221bcbb9ecf67489a Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:22:27 +0100 Subject: [PATCH 44/61] nit-picks --- .../testrunner/runners/system/servicedeployer/terraform.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 9f11bae2a..8290bbefc 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -32,10 +32,10 @@ func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeploy // SetUp method boots up the Docker Compose with Terraform executor and mounted .tf definitions. func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { - logger.Debug("setting up service using Terraform service deployer") + logger.Debug("setting up service using Terraform deployer") terraformDeployerYml, err := install.TerraformDeployerComposeFile() if err != nil { - return nil, errors.Wrap(err, "can't locate docker compose file for service deployer") + return nil, errors.Wrap(err, "can't locate docker compose file for Terraform deployer") } service := dockerComposeDeployedService{ From 8275c7c13f6c62d68f94d0c18eae3ab7ac359efd Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:27:11 +0100 Subject: [PATCH 45/61] Don't connect to the stack network --- .../runners/system/servicedeployer/terraform.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 8290bbefc..40bc2ed65 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -5,9 +5,7 @@ package servicedeployer import ( - "bytes" "fmt" - "os/exec" "github.com/pkg/errors" @@ -15,7 +13,6 @@ import ( "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/stack" ) // TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. @@ -69,16 +66,6 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic serviceContainer := fmt.Sprintf("%s_terraform_1", service.project) outCtxt.Hostname = serviceContainer - // Connect service network with stack network (for the purpose of metrics collection) - stackNetwork := fmt.Sprintf("%s_default", stack.DockerComposeProjectName) - logger.Debugf("attaching service container %s to stack network %s", serviceContainer, stackNetwork) - cmd := exec.Command("docker", "network", "connect", stackNetwork, serviceContainer) - errOutput := new(bytes.Buffer) - cmd.Stderr = errOutput - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "could not attach service container to the stack network (stderr=%q)", errOutput.String()) - } - // Set custom aliases, which may be used in agent policies. outCtxt.CustomProperties = buildTerraformAliases() From b3c0b59d3e3fec911d44aaf51e94ecfaa53d9ae1 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:35:48 +0100 Subject: [PATCH 46/61] writeTerraformDeployerResources --- internal/install/install.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/install/install.go b/internal/install/install.go index cc5edf9e8..4538e2ec2 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -57,9 +57,9 @@ func EnsureInstalled() error { return errors.Wrap(err, "writing stack resources failed") } - err = writeDeployerResources(elasticPackagePath) + err = writeTerraformDeployerResources(elasticPackagePath) if err != nil { - return errors.Wrap(err, "writing deployer resources failed") + return errors.Wrap(err, "writing Terraform deployer resources failed") } if err := createServiceLogsDir(elasticPackagePath); err != nil { @@ -157,7 +157,7 @@ func writeStackResources(elasticPackagePath string) error { return nil } -func writeDeployerResources(elasticPackagePath string) error { +func writeTerraformDeployerResources(elasticPackagePath string) error { terraformDeployer := filepath.Join(elasticPackagePath, terraformDeployerDir) err := os.MkdirAll(terraformDeployer, 0755) if err != nil { From c044a01e4a19526ef324cabbe1dbc56f51686d0f Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:41:42 +0100 Subject: [PATCH 47/61] Link: env-file --- internal/install/static_terraform_deployer_yml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go index 2fd63fb08..8c06bce4f 100644 --- a/internal/install/static_terraform_deployer_yml.go +++ b/internal/install/static_terraform_deployer_yml.go @@ -10,7 +10,7 @@ services: build: . tty: true environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # TODO Extract to the env-file + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # TODO Extract to the env-file (link: https://github.com/elastic/elastic-package/issues/235) - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - AWS_PROFILE=${AWS_PROFILE} From a128259eb838543decf4f4a97cb1fc2ebb7d97d7 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:45:47 +0100 Subject: [PATCH 48/61] Link GH issue --- .../testrunner/runners/system/servicedeployer/terraform_env.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go index 1ca7a6f9b..edb5d4efd 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform_env.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -10,6 +10,7 @@ import ( ) const ( + // TODO: Replace with an env-file (link: https://github.com/elastic/elastic-package/issues/235) awsAccessKeyID = "AWS_ACCESS_KEY_ID" awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" awsSessionToken = "AWS_SESSION_TOKEN" From 38dd80e4be3f2a83364116e25cc3d2522938c481 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:49:33 +0100 Subject: [PATCH 49/61] Fix comment --- internal/testrunner/runners/system/servicedeployer/terraform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 40bc2ed65..f3be45b69 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -62,7 +62,7 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return nil, errors.Wrap(err, "could not boot up service using docker compose") } - // Build service container name + // Build Terraform executor container name serviceContainer := fmt.Sprintf("%s_terraform_1", service.project) outCtxt.Hostname = serviceContainer From 133bf5d145df6cd3f001e69b1f15c057e80a1efd Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:56:08 +0100 Subject: [PATCH 50/61] Next improvements --- internal/testrunner/runners/system/runner.go | 7 ++++++- .../testrunner/runners/system/servicedeployer/factory.go | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index e793ad116..85624e595 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -26,6 +26,11 @@ import ( "github.com/elastic/elastic-package/internal/testrunner/runners/system/servicedeployer" ) +const ( + testRunMaxID = 99999 + testRunMinID = 10000 +) + func init() { testrunner.RegisterRunner(&runner{}) } @@ -177,7 +182,7 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { } func createTestRunID() string { - return fmt.Sprintf("%d", rand.Intn(99999-10000)+10000) + return fmt.Sprintf("%d", rand.Intn(testRunMaxID-testRunMinID)+testRunMinID) } func (r *runner) hasNumDocs( diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 2f6ad787e..87333f4ce 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -5,6 +5,7 @@ package servicedeployer import ( + "fmt" "os" "path/filepath" @@ -32,7 +33,7 @@ type FactoryOptions struct { func Factory(options FactoryOptions) (ServiceDeployer, error) { devDeployPath, err := findDevDeployPath(options) if err != nil { - logger.Errorf("can't find _dev/deploy directory") + logger.Errorf("can't find \"%s\" directory", devDeployDir) return nil, ErrNotFound } @@ -66,5 +67,5 @@ func findDevDeployPath(options FactoryOptions) (string, error) { } else if !os.IsNotExist(err) { return "", errors.Wrapf(err, "stat failed for package (path: %s)", packageDevDeployPath) } - return "", errors.New("_dev directory doesn't exist") + return "", fmt.Errorf("\"%s\" directory doesn't exist", devDeployDir) } From f14ad49b43489153c9bc5e3b289c675f913b8a0a Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 15:56:37 +0100 Subject: [PATCH 51/61] nit: separator --- internal/testrunner/runners/system/servicedeployer/factory.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 87333f4ce..a4dcbff80 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -48,6 +48,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { if _, err := os.Stat(terraformDirPath); err == nil { return NewTerraformServiceDeployer(terraformDirPath) } + return nil, ErrNotFound } From 8c116fb49498fcb171e11fa7e696b1697214d805 Mon Sep 17 00:00:00 2001 From: mtojek Date: Thu, 28 Jan 2021 16:40:28 +0100 Subject: [PATCH 52/61] Docs --- docs/howto/system_testing.md | 74 +++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 1f8d634db..5dc331614 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -12,19 +12,18 @@ Conceptually, running a system test involves the following steps: 1. Depending on the Elastic Package whose data stream is being tested, deploy an instance of the package's integration service. 1. Create a test policy that configures a single data stream for a single package. 1. Assign the test policy to the enrolled Agent. -1. Wait a reasonable amount of time for the Agent to collect data from the +1. Wait a reasonable amount of time for the Agent to collect data from the integration service and index it into the correct Elasticsearch data stream. 1. Query the first 500 documents based on `@timestamp` for validation. 1. Validate mappings are defined for the fields contained in the indexed documents. 1. Validate that the JSON data types contained `_source` are compatible with - mappings declared for the field. + mappings declared for the field. 1. Delete test artifacts and tear down the instance of the package's integration service. 1. Once all desired data streams have been system tested, tear down the Elastic Stack. ## Limitations At the moment system tests have limitations. The salient ones are: -* They can only test packages whose integration services can be deployed via Docker Compose. Eventually they will be able to test packages that can be deployed via other means, e.g. a Terraform configuration. * There isn't a way to do assert that the indexed data matches data from a file (e.g. golden file testing). ## Defining a system test @@ -39,19 +38,33 @@ Packages have a specific folder structure (only relevant parts shown). manifest.yml ``` -To define a system test we must define configuration at two levels: the package level and each data stream's level. +To define a system test we must define configuration on at least one level: a package or a data stream's one. -### Package-level configuration - -First, we must define the configuration for deploying a package's integration service. As mentioned in the [_Limitations_](#Limitations) section above, only packages whose integration services can be deployed via Docker Compose are supported at the moment. +First, we must define the configuration for deploying a package's integration service. We can define it on either the package-level: ``` / _dev/ deploy/ - docker/ - docker-compose.yml + / + +``` + +or the data-stream's level: + ``` +/ + data_stream/ + / + _dev/ + deploy/ + / + +``` + +`` - a name of the supported service deployer: `docker` or `tf` (terraform). + +### Docker Compose service deployer The `docker-compose.yml` file defines the integration service(s) for the package. If your package has a logs data stream, the log files from your package's integration service must be written to a volume. For example, the `apache` package has the following definition in it's integration service's `docker-compose.yml` file. @@ -66,7 +79,42 @@ services: Here, `SERVICE_LOGS_DIR` is a special keyword. It is something that we will need later. -### Data stream-level configuration +### Terraform service deployer + +The `*.tf` files define the infrastructure using the Terraform syntax. The terraform based service can be handy to boot up +resources on AWS and use them for testing (e.g. spawn EC2 instance and collect its metrics). + +Sample `main.tf` definition: + +``` +variable "TEST_RUN_ID" { + default = "detached" +} + +provider "aws" {} + +resource "aws_instance" "i" { + ami = data.aws_ami.latest-amzn.id + monitoring = true + instance_type = "t1.micro" + tags = { + Name = "elastic-package-test-${var.TEST_RUN_ID}" + } +} + +data "aws_ami" "latest-amzn" { + most_recent = true + owners = [ "amazon" ] # AWS + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } +} +``` + +Notice the use of the `TEST_RUN_ID` variable. It contains a unique ID, which can help differentiate resources created in potential concurrent test runs. + +### Test case definition Next, we must define configuration for each data stream that we want to system test. @@ -97,10 +145,8 @@ The `data_stream.vars` field corresponds to data stream-level variables for the Notice the use of the `{{SERVICE_LOGS_DIR}}` placeholder. This corresponds to the `${SERVICE_LOGS_DIR}` variable we saw in the `docker-compose.yml` file earlier. In the above example, the net effect is as if the `/usr/local/apache2/logs/access.log*` files located inside the Apache integration service container become available at the same path from Elastic Agent's perspective. -When a data stream's manifest declares multiple streams with different inputs -you can use the `input` option to select the stream to test. The first stream -whose input type matches the `input` value will be tested. By default, the first -stream declared in the manifest will be tested. +When a data stream's manifest declares multiple streams with different inputs you can use the `input` option to select the stream to test. The first stream +whose input type matches the `input` value will be tested. By default, the first stream declared in the manifest will be tested. #### Placeholders From ebf857c44f069d1e9541c32018a559a9aa68646b Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 29 Jan 2021 11:40:57 +0100 Subject: [PATCH 53/61] Update docs/howto/system_testing.md Co-authored-by: Shaunak Kashyap --- docs/howto/system_testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 5dc331614..2065b610b 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -40,7 +40,7 @@ Packages have a specific folder structure (only relevant parts shown). To define a system test we must define configuration on at least one level: a package or a data stream's one. -First, we must define the configuration for deploying a package's integration service. We can define it on either the package-level: +First, we must define the configuration for deploying a package's integration service. We can define it on either the package level: ``` / @@ -198,4 +198,4 @@ Finally, when you are done running all system tests, bring down the Elastic Stac ``` elastic-package stack down -``` \ No newline at end of file +``` From 8871d000cdd2ace14b4179c03c28e039ccca3b82 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 29 Jan 2021 11:41:07 +0100 Subject: [PATCH 54/61] Update docs/howto/system_testing.md Co-authored-by: Shaunak Kashyap --- docs/howto/system_testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 2065b610b..9e5e2ed56 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -50,7 +50,7 @@ First, we must define the configuration for deploying a package's integration se ``` -or the data-stream's level: +or the data stream's level: ``` / From b43c3a1b4b4e5b5a9629d4f816e80bee4ca0e6a0 Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 11:54:16 +0100 Subject: [PATCH 55/61] Introductory sentences --- docs/howto/system_testing.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 9e5e2ed56..4b859d487 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -66,7 +66,10 @@ or the data stream's level: ### Docker Compose service deployer -The `docker-compose.yml` file defines the integration service(s) for the package. If your package has a logs data stream, the log files from your package's integration service must be written to a volume. For example, the `apache` package has the following definition in it's integration service's `docker-compose.yml` file. +When using the Docker Compose service deployer, the `` must include a `docker-compose.yml` file. +The `docker-compose.yml` file defines the integration service(s) for the package. If your package has a logs data stream, +the log files from your package's integration service must be written to a volume. For example, the `apache` package has +the following definition in it's integration service's `docker-compose.yml` file. ``` version: '2.3' @@ -81,6 +84,7 @@ Here, `SERVICE_LOGS_DIR` is a special keyword. It is something that we will need ### Terraform service deployer +When using the Terraform service deployer, the `` must include at least one `*.tf` file. The `*.tf` files define the infrastructure using the Terraform syntax. The terraform based service can be handy to boot up resources on AWS and use them for testing (e.g. spawn EC2 instance and collect its metrics). From 2ea691fafb0b2e16229394f4150efd5ebdc97010 Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 11:58:45 +0100 Subject: [PATCH 56/61] Docker compose, terraform --- docs/howto/system_testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 4b859d487..a45e33f2b 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -62,7 +62,7 @@ or the data stream's level: ``` -`` - a name of the supported service deployer: `docker` or `tf` (terraform). +`` - a name of the supported service deployer: `docker` (Docker Compose service deployer) or `tf` (Terraform service deployer). ### Docker Compose service deployer From 5934ad023b6c99959c0f09c00c4ce5a8297aa4b1 Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 12:33:28 +0100 Subject: [PATCH 57/61] More generic --- docs/howto/system_testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index a45e33f2b..5fd4e4cb7 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -86,7 +86,7 @@ Here, `SERVICE_LOGS_DIR` is a special keyword. It is something that we will need When using the Terraform service deployer, the `` must include at least one `*.tf` file. The `*.tf` files define the infrastructure using the Terraform syntax. The terraform based service can be handy to boot up -resources on AWS and use them for testing (e.g. spawn EC2 instance and collect its metrics). +resources using selected cloud provider and use them for testing (e.g. observe and collect metrics). Sample `main.tf` definition: From 4d4011fbc8e15c1ab6f0801ebdc3f59ff6996e01 Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 12:40:55 +0100 Subject: [PATCH 58/61] Dont set hostname --- .../testrunner/runners/system/servicedeployer/terraform.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index f3be45b69..176bf3587 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -5,8 +5,6 @@ package servicedeployer import ( - "fmt" - "github.com/pkg/errors" "github.com/elastic/elastic-package/internal/compose" @@ -62,10 +60,6 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return nil, errors.Wrap(err, "could not boot up service using docker compose") } - // Build Terraform executor container name - serviceContainer := fmt.Sprintf("%s_terraform_1", service.project) - outCtxt.Hostname = serviceContainer - // Set custom aliases, which may be used in agent policies. outCtxt.CustomProperties = buildTerraformAliases() From a97a874786968f150c9fb6309ebf97c71aae5817 Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 17:29:40 +0100 Subject: [PATCH 59/61] Check if there is a single definition of service deployer --- .../runners/system/servicedeployer/factory.go | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index a4dcbff80..a89784ffb 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -6,22 +6,15 @@ package servicedeployer import ( "fmt" + "io/ioutil" "os" "path/filepath" "github.com/pkg/errors" - - "github.com/elastic/elastic-package/internal/logger" ) const devDeployDir = "_dev/deploy" -var ( - // ErrNotFound is returned when the appropriate service runner for a package - // cannot be found. - ErrNotFound = errors.New("unable to find service runner") -) - // FactoryOptions defines options used to create an instance of a service deployer. type FactoryOptions struct { PackageRootPath string @@ -33,23 +26,45 @@ type FactoryOptions struct { func Factory(options FactoryOptions) (ServiceDeployer, error) { devDeployPath, err := findDevDeployPath(options) if err != nil { - logger.Errorf("can't find \"%s\" directory", devDeployDir) - return nil, ErrNotFound + return nil, errors.Wrapf(err, "can't find \"%s\" directory", devDeployDir) + } + + serviceDeployerName, err := findServiceDeployer(devDeployDir) + if err != nil { + return nil, errors.Wrap(err, "can't find any valid service deployer") + } + + switch serviceDeployerName { + case "docker": + dockerComposeYMLPath := filepath.Join(devDeployPath, serviceDeployerName, "docker-compose.yml") + if _, err := os.Stat(dockerComposeYMLPath); err == nil { + return NewDockerComposeServiceDeployer(dockerComposeYMLPath) + } + case "tf": + terraformDirPath := filepath.Join(devDeployPath, serviceDeployerName) + if _, err := os.Stat(terraformDirPath); err == nil { + return NewTerraformServiceDeployer(terraformDirPath) + } + } + return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName) +} + +func findServiceDeployer(devDeployPath string) (string, error) { + fis, err := ioutil.ReadDir(devDeployPath) + if err != nil { + return "", errors.Wrapf(err, "can't read directory (path: %s)", devDeployDir) } - // Is the service defined using a docker compose configuration file? - dockerComposeYMLPath := filepath.Join(devDeployPath, "docker", "docker-compose.yml") - if _, err := os.Stat(dockerComposeYMLPath); err == nil { - return NewDockerComposeServiceDeployer(dockerComposeYMLPath) + if len(fis) != 1 { + return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) } - // Is the service defined using Terraform definition files? - terraformDirPath := filepath.Join(devDeployPath, "tf") - if _, err := os.Stat(terraformDirPath); err == nil { - return NewTerraformServiceDeployer(terraformDirPath) + deployerFileInfo := fis[0] + if !deployerFileInfo.IsDir() { + return "", fmt.Errorf("\"%s\" is expected to be a folder in \"%s\"", deployerFileInfo, devDeployPath) } - return nil, ErrNotFound + return deployerFileInfo.Name(), nil } func findDevDeployPath(options FactoryOptions) (string, error) { From 34097cd1728fa172f2cf7f46536305822a9e14dd Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 17:44:10 +0100 Subject: [PATCH 60/61] Fix --- .../runners/system/servicedeployer/factory.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index a89784ffb..2ca47e1c8 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -55,16 +55,17 @@ func findServiceDeployer(devDeployPath string) (string, error) { return "", errors.Wrapf(err, "can't read directory (path: %s)", devDeployDir) } - if len(fis) != 1 { - return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) + var folders []os.FileInfo + for _, fi := range fis { + if fi.IsDir() { + folders = append(folders, fi) + } } - deployerFileInfo := fis[0] - if !deployerFileInfo.IsDir() { - return "", fmt.Errorf("\"%s\" is expected to be a folder in \"%s\"", deployerFileInfo, devDeployPath) + if len(folders) != 1 { + return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) } - - return deployerFileInfo.Name(), nil + return folders[0].Name(), nil } func findDevDeployPath(options FactoryOptions) (string, error) { From e7e876485db84877314900e856ff3517fff65b2b Mon Sep 17 00:00:00 2001 From: mtojek Date: Fri, 29 Jan 2021 18:05:26 +0100 Subject: [PATCH 61/61] Another fix --- .../runners/system/servicedeployer/factory.go | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 2ca47e1c8..16f781c13 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -29,7 +29,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { return nil, errors.Wrapf(err, "can't find \"%s\" directory", devDeployDir) } - serviceDeployerName, err := findServiceDeployer(devDeployDir) + serviceDeployerName, err := findServiceDeployer(devDeployPath) if err != nil { return nil, errors.Wrap(err, "can't find any valid service deployer") } @@ -49,25 +49,6 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName) } -func findServiceDeployer(devDeployPath string) (string, error) { - fis, err := ioutil.ReadDir(devDeployPath) - if err != nil { - return "", errors.Wrapf(err, "can't read directory (path: %s)", devDeployDir) - } - - var folders []os.FileInfo - for _, fi := range fis { - if fi.IsDir() { - folders = append(folders, fi) - } - } - - if len(folders) != 1 { - return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) - } - return folders[0].Name(), nil -} - func findDevDeployPath(options FactoryOptions) (string, error) { dataStreamDevDeployPath := filepath.Join(options.DataStreamRootPath, devDeployDir) _, err := os.Stat(dataStreamDevDeployPath) @@ -86,3 +67,22 @@ func findDevDeployPath(options FactoryOptions) (string, error) { } return "", fmt.Errorf("\"%s\" directory doesn't exist", devDeployDir) } + +func findServiceDeployer(devDeployPath string) (string, error) { + fis, err := ioutil.ReadDir(devDeployPath) + if err != nil { + return "", errors.Wrapf(err, "can't read directory (path: %s)", devDeployDir) + } + + var folders []os.FileInfo + for _, fi := range fis { + if fi.IsDir() { + folders = append(folders, fi) + } + } + + if len(folders) != 1 { + return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) + } + return folders[0].Name(), nil +}