diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 965e0dc9d..5cc10b2e9 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { BASE_DIR="src/github.com/elastic/elastic-package" JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" PIPELINE_LOG_LEVEL='INFO' + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' } options { timeout(time: 1, unit: 'HOURS') @@ -40,8 +41,10 @@ pipeline { steps { cleanup() withMageEnv(){ - dir("${BASE_DIR}"){ - sh(label: 'Check',script: 'make check') + withCloudTestEnv() { + dir("${BASE_DIR}"){ + sh(label: 'Check',script: 'make check') + } } } } @@ -49,7 +52,8 @@ pipeline { always { dir("${BASE_DIR}") { archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/test-results/*.xml') - archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/elastic-stack-dump/logs/*.log') + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/elastic-stack-dump/stack/logs/*.log') + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/elastic-stack-dump/check/logs/*.log') junit(allowEmptyResults: false, keepLongStdio: true, testResults: "build/test-results/*.xml") @@ -71,3 +75,22 @@ def cleanup(){ } unstash 'source' } + +def withCloudTestEnv(Closure body) { + def maskedVars = [] + // AWS + def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data + if (!aws.containsKey('access_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") + } + if (!aws.containsKey('secret_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") + } + maskedVars.addAll([ + [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], + [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], + ]) + withEnvMask(vars: maskedVars) { + body() + } +} \ No newline at end of file diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 1f8d634db..5fd4e4cb7 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -12,19 +12,18 @@ Conceptually, running a system test involves the following steps: 1. Depending on the Elastic Package whose data stream is being tested, deploy an instance of the package's integration service. 1. Create a test policy that configures a single data stream for a single package. 1. Assign the test policy to the enrolled Agent. -1. Wait a reasonable amount of time for the Agent to collect data from the +1. Wait a reasonable amount of time for the Agent to collect data from the integration service and index it into the correct Elasticsearch data stream. 1. Query the first 500 documents based on `@timestamp` for validation. 1. Validate mappings are defined for the fields contained in the indexed documents. 1. Validate that the JSON data types contained `_source` are compatible with - mappings declared for the field. + mappings declared for the field. 1. Delete test artifacts and tear down the instance of the package's integration service. 1. Once all desired data streams have been system tested, tear down the Elastic Stack. ## Limitations At the moment system tests have limitations. The salient ones are: -* They can only test packages whose integration services can be deployed via Docker Compose. Eventually they will be able to test packages that can be deployed via other means, e.g. a Terraform configuration. * There isn't a way to do assert that the indexed data matches data from a file (e.g. golden file testing). ## Defining a system test @@ -39,21 +38,38 @@ Packages have a specific folder structure (only relevant parts shown). manifest.yml ``` -To define a system test we must define configuration at two levels: the package level and each data stream's level. +To define a system test we must define configuration on at least one level: a package or a data stream's one. -### Package-level configuration - -First, we must define the configuration for deploying a package's integration service. As mentioned in the [_Limitations_](#Limitations) section above, only packages whose integration services can be deployed via Docker Compose are supported at the moment. +First, we must define the configuration for deploying a package's integration service. We can define it on either the package level: ``` / _dev/ deploy/ - docker/ - docker-compose.yml + / + +``` + +or the data stream's level: + ``` +/ + data_stream/ + / + _dev/ + deploy/ + / + +``` + +`` - a name of the supported service deployer: `docker` (Docker Compose service deployer) or `tf` (Terraform service deployer). + +### Docker Compose service deployer -The `docker-compose.yml` file defines the integration service(s) for the package. If your package has a logs data stream, the log files from your package's integration service must be written to a volume. For example, the `apache` package has the following definition in it's integration service's `docker-compose.yml` file. +When using the Docker Compose service deployer, the `` must include a `docker-compose.yml` file. +The `docker-compose.yml` file defines the integration service(s) for the package. If your package has a logs data stream, +the log files from your package's integration service must be written to a volume. For example, the `apache` package has +the following definition in it's integration service's `docker-compose.yml` file. ``` version: '2.3' @@ -66,7 +82,43 @@ services: Here, `SERVICE_LOGS_DIR` is a special keyword. It is something that we will need later. -### Data stream-level configuration +### Terraform service deployer + +When using the Terraform service deployer, the `` must include at least one `*.tf` file. +The `*.tf` files define the infrastructure using the Terraform syntax. The terraform based service can be handy to boot up +resources using selected cloud provider and use them for testing (e.g. observe and collect metrics). + +Sample `main.tf` definition: + +``` +variable "TEST_RUN_ID" { + default = "detached" +} + +provider "aws" {} + +resource "aws_instance" "i" { + ami = data.aws_ami.latest-amzn.id + monitoring = true + instance_type = "t1.micro" + tags = { + Name = "elastic-package-test-${var.TEST_RUN_ID}" + } +} + +data "aws_ami" "latest-amzn" { + most_recent = true + owners = [ "amazon" ] # AWS + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } +} +``` + +Notice the use of the `TEST_RUN_ID` variable. It contains a unique ID, which can help differentiate resources created in potential concurrent test runs. + +### Test case definition Next, we must define configuration for each data stream that we want to system test. @@ -97,10 +149,8 @@ The `data_stream.vars` field corresponds to data stream-level variables for the Notice the use of the `{{SERVICE_LOGS_DIR}}` placeholder. This corresponds to the `${SERVICE_LOGS_DIR}` variable we saw in the `docker-compose.yml` file earlier. In the above example, the net effect is as if the `/usr/local/apache2/logs/access.log*` files located inside the Apache integration service container become available at the same path from Elastic Agent's perspective. -When a data stream's manifest declares multiple streams with different inputs -you can use the `input` option to select the stream to test. The first stream -whose input type matches the `input` value will be tested. By default, the first -stream declared in the manifest will be tested. +When a data stream's manifest declares multiple streams with different inputs you can use the `input` option to select the stream to test. The first stream +whose input type matches the `input` value will be tested. By default, the first stream declared in the manifest will be tested. #### Placeholders @@ -152,4 +202,4 @@ Finally, when you are done running all system tests, bring down the Elastic Stac ``` elastic-package stack down -``` \ No newline at end of file +``` diff --git a/go.mod b/go.mod index 2b30f425d..398243f2b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.9.0 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-ucfg v0.8.3 - github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 + github.com/elastic/package-spec/code/go v0.0.0-20210127201409-dd08da649371 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-openapi/strfmt v0.19.6 // indirect diff --git a/go.sum b/go.sum index c0e4a5dc5..2c8b75c80 100644 --- a/go.sum +++ b/go.sum @@ -71,7 +71,6 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -88,8 +87,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3 h1:QeQvLQvDx1mRAOBzcJ7T7ebF02mJxcK5HYsYTTOrsik= -github.com/elastic/package-spec/code/go v0.0.0-20210126144901-46090e1310d3/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= +github.com/elastic/package-spec/code/go v0.0.0-20210127201409-dd08da649371 h1:YjBuT9e9rP5vms2NWJVu6MAccgrSlOUlbykkjfa10aY= +github.com/elastic/package-spec/code/go v0.0.0-20210127201409-dd08da649371/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -99,7 +98,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -139,7 +137,6 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -155,7 +152,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -191,7 +187,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ= github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -226,18 +221,15 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -282,7 +274,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -293,7 +284,6 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -303,7 +293,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -336,7 +325,6 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -355,14 +343,12 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -395,7 +381,6 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -404,7 +389,6 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -412,7 +396,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -450,7 +433,6 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -458,14 +440,12 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -476,7 +456,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -513,7 +492,6 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -539,7 +517,6 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -612,11 +589,9 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cleanup/build.go b/internal/cleanup/build.go index fc370db04..1dd5aa7e0 100644 --- a/internal/cleanup/build.go +++ b/internal/cleanup/build.go @@ -35,7 +35,7 @@ func Build() (string, error) { } if !found { - logger.Debugf("Build directory doesn't exist (missing path: %s)", buildDir) + logger.Debug("Build directory doesn't exist") return "", nil } diff --git a/internal/install/install.go b/internal/install/install.go index b92ceb985..4538e2ec2 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -18,10 +18,14 @@ const ( stackDir = "stack" packagesDir = "development" temporaryDir = "tmp" + deployerDir = "deployer" + + terraformDeployerYmlFile = "terraform-deployer.yml" ) var ( - serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + terraformDeployerDir = filepath.Join(deployerDir, "terraform") ) const versionFilename = "version" @@ -50,7 +54,12 @@ func EnsureInstalled() error { err = writeStackResources(elasticPackagePath) if err != nil { - return errors.Wrap(err, "writing static resources failed") + return errors.Wrap(err, "writing stack resources failed") + } + + err = writeTerraformDeployerResources(elasticPackagePath) + if err != nil { + return errors.Wrap(err, "writing Terraform deployer resources failed") } if err := createServiceLogsDir(elasticPackagePath); err != nil { @@ -79,7 +88,7 @@ func StackPackagesDir() (string, error) { return filepath.Join(stackDir, packagesDir), nil } -// ServiceLogsDir method returns the location of the directory to store service logs on the +// ServiceLogsDir function returns the location of the directory to store service logs on the // local filesystem, i.e. the same one where elastic-package is installed. func ServiceLogsDir() (string, error) { configurationDir, err := configurationDir() @@ -89,6 +98,15 @@ func ServiceLogsDir() (string, error) { return filepath.Join(configurationDir, serviceLogsDir), nil } +// TerraformDeployerComposeFile function returns the path to the Terraform service deployer's definitions. +func TerraformDeployerComposeFile() (string, error) { + configurationDir, err := configurationDir() + if err != nil { + return "", errors.Wrap(err, "locating configuration directory failed") + } + return filepath.Join(configurationDir, terraformDeployerDir, terraformDeployerYmlFile), nil +} + func configurationDir() (string, error) { homeDir, err := os.UserHomeDir() if err != nil { @@ -126,7 +144,7 @@ func writeStackResources(elasticPackagePath string) error { packagesPath := filepath.Join(stackPath, packagesDir) err := os.MkdirAll(packagesPath, 0755) if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath) + return errors.Wrapf(err, "creating directory failed (path: %s)", packagesPath) } err = writeStaticResource(err, filepath.Join(stackPath, "kibana.config.yml"), kibanaConfigYml) @@ -139,6 +157,22 @@ func writeStackResources(elasticPackagePath string) error { return nil } +func writeTerraformDeployerResources(elasticPackagePath string) error { + terraformDeployer := filepath.Join(elasticPackagePath, terraformDeployerDir) + err := os.MkdirAll(terraformDeployer, 0755) + if err != nil { + return errors.Wrapf(err, "creating directory failed (path: %s)", terraformDeployer) + } + + err = writeStaticResource(err, filepath.Join(terraformDeployer, terraformDeployerYmlFile), terraformDeployerYml) + err = writeStaticResource(err, filepath.Join(terraformDeployer, "Dockerfile"), terraformDeployerDockerfile) + err = writeStaticResource(err, filepath.Join(terraformDeployer, "run.sh"), terraformDeployerRun) + if err != nil { + return errors.Wrap(err, "writing static resource failed") + } + return nil +} + func writeStaticResource(err error, path, content string) error { if err != nil { return err diff --git a/internal/install/static_terraform_deployer_dockerfile.go b/internal/install/static_terraform_deployer_dockerfile.go new file mode 100644 index 000000000..91b320e0e --- /dev/null +++ b/internal/install/static_terraform_deployer_dockerfile.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +const terraformDeployerDockerfile = `FROM hashicorp/terraform:light +ENV TF_IN_AUTOMATION=true +HEALTHCHECK --timeout=3s CMD sh -c "[ -f /tmp/tf-applied ]" +ADD run.sh / +WORKDIR /workspace +ENTRYPOINT sh /run.sh +` diff --git a/internal/install/static_terraform_deployer_run.go b/internal/install/static_terraform_deployer_run.go new file mode 100644 index 000000000..4d2b643bf --- /dev/null +++ b/internal/install/static_terraform_deployer_run.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +const terraformDeployerRun = `#!sh + +set -euxo pipefail + +cp -r /stage/*.tf /workspace + +cleanup() { + r=$? + + set -x + terraform destroy -auto-approve + + exit $r +} +trap cleanup EXIT INT TERM + +terraform init +terraform plan +terraform apply -auto-approve && touch /tmp/tf-applied + +echo "Terraform definitions applied." + +set +x +while true; do sleep 1; done # wait for ctrl-c +` diff --git a/internal/install/static_terraform_deployer_yml.go b/internal/install/static_terraform_deployer_yml.go new file mode 100644 index 000000000..8c06bce4f --- /dev/null +++ b/internal/install/static_terraform_deployer_yml.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +const terraformDeployerYml = `version: '2.3' +services: + terraform: + build: . + tty: true + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} # TODO Extract to the env-file (link: https://github.com/elastic/elastic-package/issues/235) + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} + - AWS_PROFILE=${AWS_PROFILE} + - AWS_REGION=${AWS_REGION:-us-east-1} + - TF_VAR_TEST_RUN_ID=${TF_VAR_TEST_RUN_ID:-detached} + volumes: + - ${TF_DIR}:/stage +` diff --git a/internal/kibana/client.go b/internal/kibana/client.go index 32371db13..cd2b700c8 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -72,7 +72,6 @@ func (c *Client) sendRequest(method, resourcePath string, body []byte) (int, []b u := base.ResolveReference(rel) logger.Debugf("%s %s", method, u) - logger.Debugf("%s", body) req, err := http.NewRequest(method, u.String(), reqBody) if err != nil { diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index a823848bf..85624e595 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -7,6 +7,7 @@ package system import ( "encoding/json" "fmt" + "math/rand" "path/filepath" "strings" "time" @@ -25,6 +26,11 @@ import ( "github.com/elastic/elastic-package/internal/testrunner/runners/system/servicedeployer" ) +const ( + testRunMaxID = 99999 + testRunMinID = 10000 +) + func init() { testrunner.RegisterRunner(&runner{}) } @@ -158,6 +164,7 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { ctxt.Name = r.options.TestFolder.Package ctxt.Logs.Folder.Local = serviceLogsDir ctxt.Logs.Folder.Agent = serviceLogsAgentDir + ctxt.Test.RunID = createTestRunID() testConfig, err := newConfig(filepath.Join(r.options.TestFolder.Path, cfgFile), ctxt) if err != nil { return result.withError(errors.Wrapf(err, "unable to load system test case file '%s'", cfgFile)) @@ -174,6 +181,10 @@ func (r *runner) run() (results []testrunner.TestResult, err error) { return results, nil } +func createTestRunID() string { + return fmt.Sprintf("%d", rand.Intn(testRunMaxID-testRunMinID)+testRunMinID) +} + func (r *runner) hasNumDocs( dataStream string, fieldsValidator *fields.Validator, @@ -396,7 +407,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext logger.Debug("checking for expected data in data stream...") passed, err := waitUntilTrue(r.hasNumDocs(dataStream, fieldsValidator, func(n int) bool { return n > 0 - }), 2*time.Minute) + }), 10*time.Minute) if err != nil { return result.withError(err) diff --git a/internal/testrunner/runners/system/servicedeployer/context.go b/internal/testrunner/runners/system/servicedeployer/context.go index c4e968ae8..890a2c3ff 100644 --- a/internal/testrunner/runners/system/servicedeployer/context.go +++ b/internal/testrunner/runners/system/servicedeployer/context.go @@ -4,7 +4,10 @@ package servicedeployer -const serviceLogsDirEnv = "SERVICE_LOGS_DIR" +const ( + serviceLogsDirEnv = "SERVICE_LOGS_DIR" + testRunIDEnv = "TEST_RUN_ID" +) // ServiceContext encapsulates context that is both available to a ServiceDeployer and // populated by a DeployedService. The fields in ServiceContext may be used in handlebars @@ -38,13 +41,33 @@ type ServiceContext struct { Agent string } } + + // Test related properties. + Test struct { + // RunID identifies the current test run. + RunID string + } + + // CustomProperties store additional data used to boot up the service, e.g. AWS credentials. + CustomProperties map[string]interface{} } // Aliases method returned aliases to properties of the service context. func (sc *ServiceContext) Aliases() map[string]interface{} { - return map[string]interface{}{ + m := map[string]interface{}{ serviceLogsDirEnv: func() interface{} { return sc.Logs.Folder.Agent }, + testRunIDEnv: func() interface{} { + return sc.Test.RunID + }, + } + + for k, v := range sc.CustomProperties { + var that = v + m[k] = func() interface{} { // wrap as function + return that + } } + return m } diff --git a/internal/testrunner/runners/system/servicedeployer/factory.go b/internal/testrunner/runners/system/servicedeployer/factory.go index 813ef64a0..16f781c13 100644 --- a/internal/testrunner/runners/system/servicedeployer/factory.go +++ b/internal/testrunner/runners/system/servicedeployer/factory.go @@ -5,22 +5,16 @@ package servicedeployer import ( + "fmt" + "io/ioutil" "os" "path/filepath" "github.com/pkg/errors" - - "github.com/elastic/elastic-package/internal/logger" ) const devDeployDir = "_dev/deploy" -var ( - // ErrNotFound is returned when the appropriate service runner for a package - // cannot be found. - ErrNotFound = errors.New("unable to find service runner") -) - // FactoryOptions defines options used to create an instance of a service deployer. type FactoryOptions struct { PackageRootPath string @@ -32,16 +26,27 @@ type FactoryOptions struct { func Factory(options FactoryOptions) (ServiceDeployer, error) { devDeployPath, err := findDevDeployPath(options) if err != nil { - logger.Errorf("can't find _dev/deploy directory") - return nil, ErrNotFound + return nil, errors.Wrapf(err, "can't find \"%s\" directory", devDeployDir) } - // Is the service defined using a docker compose configuration file? - dockerComposeYMLPath := filepath.Join(devDeployPath, "docker", "docker-compose.yml") - if _, err := os.Stat(dockerComposeYMLPath); err == nil { - return NewDockerComposeServiceDeployer(dockerComposeYMLPath) + serviceDeployerName, err := findServiceDeployer(devDeployPath) + if err != nil { + return nil, errors.Wrap(err, "can't find any valid service deployer") } - return nil, ErrNotFound + + switch serviceDeployerName { + case "docker": + dockerComposeYMLPath := filepath.Join(devDeployPath, serviceDeployerName, "docker-compose.yml") + if _, err := os.Stat(dockerComposeYMLPath); err == nil { + return NewDockerComposeServiceDeployer(dockerComposeYMLPath) + } + case "tf": + terraformDirPath := filepath.Join(devDeployPath, serviceDeployerName) + if _, err := os.Stat(terraformDirPath); err == nil { + return NewTerraformServiceDeployer(terraformDirPath) + } + } + return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName) } func findDevDeployPath(options FactoryOptions) (string, error) { @@ -60,5 +65,24 @@ func findDevDeployPath(options FactoryOptions) (string, error) { } else if !os.IsNotExist(err) { return "", errors.Wrapf(err, "stat failed for package (path: %s)", packageDevDeployPath) } - return "", errors.New("_dev directory doesn't exist") + return "", fmt.Errorf("\"%s\" directory doesn't exist", devDeployDir) +} + +func findServiceDeployer(devDeployPath string) (string, error) { + fis, err := ioutil.ReadDir(devDeployPath) + if err != nil { + return "", errors.Wrapf(err, "can't read directory (path: %s)", devDeployDir) + } + + var folders []os.FileInfo + for _, fi := range fis { + if fi.IsDir() { + folders = append(folders, fi) + } + } + + if len(folders) != 1 { + return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) + } + return folders[0].Name(), nil } diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go new file mode 100644 index 000000000..176bf3587 --- /dev/null +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -0,0 +1,70 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +import ( + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/install" + "github.com/elastic/elastic-package/internal/logger" +) + +// TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. +type TerraformServiceDeployer struct { + definitionsDir string +} + +// NewTerraformServiceDeployer creates an instance of TerraformServiceDeployer. +func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) { + return &TerraformServiceDeployer{ + definitionsDir: definitionsDir, + }, nil +} + +// SetUp method boots up the Docker Compose with Terraform executor and mounted .tf definitions. +func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { + logger.Debug("setting up service using Terraform deployer") + terraformDeployerYml, err := install.TerraformDeployerComposeFile() + if err != nil { + return nil, errors.Wrap(err, "can't locate docker compose file for Terraform deployer") + } + + service := dockerComposeDeployedService{ + ymlPath: terraformDeployerYml, + project: "elastic-package-service", + } + outCtxt := inCtxt + + p, err := compose.NewProject(service.project, service.ymlPath) + if err != nil { + return nil, errors.Wrap(err, "could not create docker compose project for service") + } + + // Clean service logs + err = files.RemoveContent(outCtxt.Logs.Folder.Local) + if err != nil { + return nil, errors.Wrap(err, "removing service logs failed") + } + + // Boot up service + tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) + opts := compose.CommandOptions{ + Env: tfEnvironment, + ExtraArgs: []string{"--build", "-d"}, + } + if err := p.Up(opts); err != nil { + return nil, errors.Wrap(err, "could not boot up service using docker compose") + } + + // Set custom aliases, which may be used in agent policies. + outCtxt.CustomProperties = buildTerraformAliases() + + service.ctxt = outCtxt + return &service, nil +} + +var _ ServiceDeployer = new(TerraformServiceDeployer) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform_env.go b/internal/testrunner/runners/system/servicedeployer/terraform_env.go new file mode 100644 index 000000000..edb5d4efd --- /dev/null +++ b/internal/testrunner/runners/system/servicedeployer/terraform_env.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +import ( + "fmt" + "os" +) + +const ( + // TODO: Replace with an env-file (link: https://github.com/elastic/elastic-package/issues/235) + awsAccessKeyID = "AWS_ACCESS_KEY_ID" + awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + awsSessionToken = "AWS_SESSION_TOKEN" + awsProfile = "AWS_PROFILE" + awsRegion = "AWS_REGION" + + tfDir = "TF_DIR" + tfTestRunID = "TF_VAR_TEST_RUN_ID" +) + +func (tsd TerraformServiceDeployer) buildTerraformExecutorEnvironment(ctxt ServiceContext) []string { + vars := map[string]string{} + vars[serviceLogsDirEnv] = ctxt.Logs.Folder.Local + vars[tfTestRunID] = ctxt.Test.RunID + vars[tfDir] = tsd.definitionsDir + + if os.Getenv(awsAccessKeyID) != "" { + vars[awsAccessKeyID] = os.Getenv(awsAccessKeyID) + } + + if os.Getenv(awsSecretAccessKey) != "" { + vars[awsSecretAccessKey] = os.Getenv(awsSecretAccessKey) + } + + if os.Getenv(awsSessionToken) != "" { + vars[awsSessionToken] = os.Getenv(awsSessionToken) + } + + if os.Getenv(awsProfile) != "" { + vars[awsProfile] = os.Getenv(awsProfile) + } + + if os.Getenv(awsRegion) != "" { + vars[awsRegion] = os.Getenv(awsRegion) + } + + var pairs []string + for k, v := range vars { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + return pairs +} + +func buildTerraformAliases() map[string]interface{} { + return map[string]interface{}{ + awsAccessKeyID: os.Getenv(awsAccessKeyID), + awsSecretAccessKey: os.Getenv(awsSecretAccessKey), + awsSessionToken: os.Getenv(awsSessionToken), + awsProfile: os.Getenv(awsProfile), + awsRegion: os.Getenv(awsRegion), + } +} diff --git a/main.go b/main.go index f2895e852..e551273bf 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,9 @@ package main import ( "log" + "math/rand" "os" + "time" "github.com/pkg/errors" @@ -15,6 +17,8 @@ import ( ) func main() { + rand.Seed(time.Now().UnixNano()) + rootCmd := cmd.RootCmd() err := install.EnsureInstalled() diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index bba3c31f5..e69edb13d 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -5,6 +5,9 @@ set -euxo pipefail cleanup() { r=$? + # Dump stack logs + elastic-package stack dump -v --output build/elastic-stack-dump/check + # Take down the stack elastic-package stack down -v diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index e38154846..6d2c6d525 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -6,7 +6,7 @@ cleanup() { r=$? # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump + elastic-package stack dump -v --output build/elastic-stack-dump/stack # Take down the stack elastic-package stack down -v diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf new file mode 100644 index 000000000..929994242 --- /dev/null +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf @@ -0,0 +1,23 @@ +variable "TEST_RUN_ID" { + default = "detached" +} + +provider "aws" {} + +resource "aws_instance" "i" { + ami = data.aws_ami.latest-amzn.id + monitoring = true + instance_type = "t1.micro" + tags = { + Name = "elastic-package-test-${var.TEST_RUN_ID}" + } +} + +data "aws_ami" "latest-amzn" { + most_recent = true + owners = [ "amazon" ] # AWS + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml new file mode 100644 index 000000000..d164a5abd --- /dev/null +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -0,0 +1,10 @@ +vars: + access_key_id: '{{AWS_ACCESS_KEY_ID}}' + secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' + session_token: '{{AWS_SESSION_TOKEN}}' +data_stream: + vars: + period: 60s + tags_filter: |- + - key: Name + value: "elastic-package-test-{{TEST_RUN_ID}}" diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml index e49975bc2..432ee5f4d 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml b/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml index 066b1eee4..833aba3ec 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml @@ -78,6 +78,14 @@ format: bytes description: | Bytes read from all instance store volumes available to the instance. + - name: diskio.read.count + type: long + format: bytes + description: The number of disk IO reads + - name: diskio.read.count_per_sec + type: long + format: bytes + description: The number of disk IO reads per second - name: diskio.read.bytes_per_sec type: long description: | @@ -99,6 +107,14 @@ type: long description: | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. + - name: diskio.write.count + type: long + format: bytes + description: The number of disk IO writes + - name: diskio.write.count_per_sec + type: long + format: bytes + description: The number of disk IO writes per second - name: diskio.write.ops type: long description: | diff --git a/test/packages/aws/docs/README.md b/test/packages/aws/docs/README.md index 5a66b4514..a00fe0ef3 100644 --- a/test/packages/aws/docs/README.md +++ b/test/packages/aws/docs/README.md @@ -1311,10 +1311,14 @@ An example event for `ec2` looks as following: | aws.ec2.cpu.total.pct | The percentage of allocated EC2 compute units that are currently in use on the instance. | scaled_float | | aws.ec2.diskio.read.bytes | Bytes read from all instance store volumes available to the instance. | long | | aws.ec2.diskio.read.bytes_per_sec | Bytes read per second from all instance store volumes available to the instance. | long | +| aws.ec2.diskio.read.count | The number of disk IO reads | long | +| aws.ec2.diskio.read.count_per_sec | The number of disk IO reads per second | long | | aws.ec2.diskio.read.ops | Completed read operations from all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.diskio.read.ops_per_sec | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.diskio.write.bytes | Bytes written to all instance store volumes available to the instance. | long | | aws.ec2.diskio.write.bytes_per_sec | Bytes written per second to all instance store volumes available to the instance. | long | +| aws.ec2.diskio.write.count | The number of disk IO writes | long | +| aws.ec2.diskio.write.count_per_sec | The number of disk IO writes per second | long | | aws.ec2.diskio.write.ops | Completed write operations to all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.diskio.write.ops_per_sec | Completed write operations per second to all instance store volumes available to the instance in a specified period of time. | long | | aws.ec2.instance.core.count | The number of CPU cores for the instance. | integer | @@ -1357,6 +1361,7 @@ An example event for `ec2` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.cpu.pct | Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. | scaled_float | @@ -1380,6 +1385,7 @@ An example event for `ec2` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### elb