Add parallel integTest for OpenSearch Plugins

jenkins/opensearch-dashboards/integ-test.jenkinsfile

@@ -28,11 +28,7 @@ pipeline {
     }
     stages {
         stage('verify-parameters') {
-            agent {
-                node {
-                    label AGENT_LABEL
-                }
-            }
+            agent { label AGENT_LABEL }
             steps {
                 script {
                     currentBuild.description = TEST_MANIFEST
@@ -57,8 +53,8 @@ pipeline {
         stage('detect docker image + args') {
             agent {
                 docker {
-                    label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-                    image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028'
+                    label AGENT_LABEL
+                    image 'alpine:3'
                     alwaysPull true
                 }
             }
@@ -90,9 +86,9 @@ pipeline {
 
                     runIntegTestScript(
                         jobName: BUILD_JOB_NAME,
+                        componentName: 'functionalTestDashboards',
                         buildManifest: BUILD_MANIFEST,
                         testManifest: "manifests/${TEST_MANIFEST}",
-                        buildId: BUILD_ID
                     )
                 }
             }
@@ -101,8 +97,7 @@ pipeline {
                     script {
                         uploadTestResults(
                             buildManifestFileName: BUILD_MANIFEST,
-                            jobName: JOB_NAME,
-                            buildNumber: BUILD_ID
+                            jobName: JOB_NAME
                         )
                     }
                     postCleanup()

jenkins/opensearch/integ-test.jenkinsfile

@@ -31,11 +31,7 @@ pipeline {
     }
     stages {
         stage('verify-parameters') {
-            agent {
-                node {
-                    label AGENT_LABEL
-                }
-            }
+            agent { label AGENT_LABEL }
             steps {
                 script {
                     currentBuild.description = TEST_MANIFEST
@@ -60,8 +56,8 @@ pipeline {
         stage('detect docker image + args') {
             agent {
                 docker {
-                    label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-                    image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028'
+                    label AGENT_LABEL
+                    image 'alpine:3'
                     alwaysPull true
                 }
             }
@@ -72,42 +68,73 @@ pipeline {
             }
         }
         stage('integ-test') {
-            agent {
-                docker {
-                    label AGENT_LABEL
-                    image DOCKER_AGENT.image
-                    args DOCKER_AGENT.args
-                    alwaysPull true
-                }
-            }
+            // Required running on agent directly here to trigger docker stages in agent node, not trigger docker within docker container
+            // Can only be run in runner that is at least 50GB per container
+            agent { label AGENT_LABEL }
             steps {
                 script {
-                    def buildManifestObj = downloadBuildManifest(
+
+                    downloadBuildManifest(
                         url: BUILD_MANIFEST_URL,
                         path: BUILD_MANIFEST
                     )
-                    String buildId = buildManifestObj.getArtifactBuildId()
-                    env.BUILD_ID = buildId
-                    echo "BUILD_MANIFEST: ${BUILD_MANIFEST}"
-                    echo "BUILD_ID: ${BUILD_ID}"
 
-                    runIntegTestScript(
-                        jobName: BUILD_JOB_NAME,
-                        buildManifest: BUILD_MANIFEST,
-                        testManifest: "manifests/${TEST_MANIFEST}",
-                        buildId: BUILD_ID
-                    )
+                    // Stash the current working directory files, aka opensearch-build repo
+                    // Unstash later in each triggered stage to run integTest
+                    stash includes: '**', name: 'opensearch-build-repo'
+
+                    def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
+                    def componentList = buildManifestObj.getNames()
+                    echo "componentList: ${componentList}"
+
+                    componentTests = [:]
+
+                    for (component in componentList) {
+                        // Must use local variable due to groovy for loop and closure scope
+                        // Or the stage will be fixed to the last item in return when new stages are triggered here
+                        // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/
+                        def local_component = component
+
+                        echo "Add Component: ${local_component}"
+                        componentTests["Run Integtest ${local_component}"] = {
+                            // Using scripted pipelines to trigger dynamic parallel stages
+                            node(AGENT_LABEL) {
+                                docker.image(DOCKER_AGENT.image).inside(DOCKER_AGENT.args) {
+                                    try {
+                                        stage("Run Integtest ${local_component}") {
+                                            echo "Component Name: ${local_component}"
+                                            unstash 'opensearch-build-repo'
+                                            // Jenkins tend to not clean up workspace at times even though ws clean is called
+                                            // Due to docker is mounting the agent directory so it can communicated with the agent
+                                            // This sometimes causes the workspace to retain last run test-results and ends with build failures
+                                            // https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
+                                            sh("rm -rf test-results")
+                                            runIntegTestScript(
+                                                jobName: BUILD_JOB_NAME,
+                                                componentName: "${local_component}",
+                                                buildManifest: BUILD_MANIFEST,
+                                                testManifest: "manifests/${TEST_MANIFEST}"
+                                            )
+                                        }
+                                    } catch (e) {
+                                        echo "Error running integtest for component ${local_component}"
+                                        throw e
+                                    } finally {
+                                        uploadTestResults(
+                                            buildManifestFileName: BUILD_MANIFEST,
+                                            jobName: JOB_NAME
+                                        )
+                                        postCleanup()
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    parallel componentTests
                 }
             }
             post {
                 always {
-                    script {
-                        uploadTestResults(
-                            buildManifestFileName: BUILD_MANIFEST,
-                            jobName: JOB_NAME,
-                            buildNumber: BUILD_ID
-                        )
-                    }
                     postCleanup()
                 }
             }

src/test_workflow/README.md

@@ -69,6 +69,15 @@ To run OpenSearch Dashboards integration tests.
 opensearch-dashboards=https://ci.opensearch.org/ci/dbc/bundle-build-dashboards/1.2.0/869/linux/x64 
 ```
 
+To run OpenSearch Dashboards integration tests with local artifacts on different distributions
+```bash
+./test.sh integ-test manifests/2.0.0/opensearch-dashboards-2.0.0-test.yml --paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0-rc1/latest/linux/x64/tar opensearch-dashboards=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0-rc1/latest/linux/x64/tar
+./test.sh integ-test manifests/2.0.0/opensearch-dashboards-2.0.0-test.yml --paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0-rc1/latest/linux/x64/rpm opensearch-dashboards=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0-rc1/latest/linux/x64/rpm
+```
+
+:warning: RPM Test requires user to run the `./test.sh` command with sudo permission, as rpm requires root to install and start the service.
+
+
 ### Backwards Compatibility Tests
 
 Runs backward compatibility invoking `run_bwc_test.py` in each component from a distribution manifest.

tests/jenkins/TestOpenSearchBwcTest.groovy

@@ -23,7 +23,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest {
         this.registerLibTester(new DetectTestDockerAgentLibTester())
         this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest))
         this.registerLibTester(new RunBwcTestScriptLibTester(jobName, buildManifest, "manifests/${testManifest}", "${buildId}"))
-        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName, buildId))
+        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName))
         this.registerLibTester(new PublishNotificationLibTester(
                 ':white_check_mark:',
                 'BWC Tests Successful',

tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy

@@ -23,7 +23,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest {
         this.registerLibTester(new DetectTestDockerAgentLibTester())
         this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest))
         this.registerLibTester(new RunBwcTestScriptLibTester(jobName, buildManifest, "manifests/${testManifest}", "${buildId}"))
-        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName, buildId))
+        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName))
         this.registerLibTester(new PublishNotificationLibTester(
                 ':white_check_mark:',
                 'BWC Tests Successful',

tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy

@@ -22,8 +22,8 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
 
         this.registerLibTester(new DetectTestDockerAgentLibTester())
         this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest))
-        this.registerLibTester(new RunIntegTestScriptLibTester(jobName, buildManifest, "manifests/${testManifest}", "${buildId}"))
-        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName, buildId))
+        this.registerLibTester(new RunIntegTestScriptLibTester(jobName, 'functionalTestDashboards', buildManifest, "manifests/${testManifest}"))
+        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName))
         this.registerLibTester(new PublishNotificationLibTester(
                 ':white_check_mark:',
                 'Integration Tests Successful',
@@ -48,6 +48,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
         })
 
         helper.registerAllowedMethod('findFiles', [Map.class], null)
+        helper.registerAllowedMethod('unstash', [String.class], null)
     }
 
     @Test

tests/jenkins/TestOpenSearchIntegTest.groovy

@@ -22,8 +22,8 @@ class TestOpenSearchIntegTest extends BuildPipelineTest {
 
         this.registerLibTester(new DetectTestDockerAgentLibTester())
         this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest))
-        this.registerLibTester(new RunIntegTestScriptLibTester(jobName, buildManifest, "manifests/${testManifest}", "${buildId}"))
-        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName, buildId))
+        this.registerLibTester(new RunIntegTestScriptLibTester(jobName, 'OpenSearch', buildManifest, "manifests/${testManifest}"))
+        this.registerLibTester(new UploadTestResultsLibTester(buildManifest, jobName))
         this.registerLibTester(new PublishNotificationLibTester(
                 ':white_check_mark:',
                 'Integration Tests Successful',
@@ -48,6 +48,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest {
         })
 
         helper.registerAllowedMethod('findFiles', [Map.class], null)
+        helper.registerAllowedMethod('unstash', [String.class], null)
     }
 
     @Test

tests/jenkins/TestRunIntegTestScript.groovy

@@ -17,9 +17,9 @@ class TestRunIntegTestScript extends BuildPipelineTest {
     public void TestRunIntegTestScript() {
         this.registerLibTester(new RunIntegTestScriptLibTester(
             'dummy_job',
+            'OpenSearch',
             'tests/jenkins/data/opensearch-1.3.0-build.yml',
             'tests/jenkins/data/opensearch-1.3.0-test.yml',
-            '717'
             )
         )
 
@@ -30,9 +30,9 @@ class TestRunIntegTestScript extends BuildPipelineTest {
     public void TestRunIntegTestScript_OpenSearch_Dashboards() {
         this.registerLibTester(new RunIntegTestScriptLibTester(
             'dummy_job',
+            'functionalTestDashboards',
             'tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml',
             'tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml',
-            '215'
             )
         )
 

tests/jenkins/TestUploadTestResults.groovy

@@ -7,7 +7,7 @@ class TestUploadTestResults extends BuildPipelineTest {
     @Before
     void setUp() {
 
-        this.registerLibTester(new UploadTestResultsLibTester( 'tests/jenkins/data/opensearch-1.3.0-build.yml', 'dummy_job', 123 ))
+        this.registerLibTester(new UploadTestResultsLibTester( 'tests/jenkins/data/opensearch-1.3.0-build.yml', 'dummy_job'))
 
         super.setUp()
     }

tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt

@@ -72,6 +72,7 @@ CCR Performance tests for 1236 completed})
                uploadTestResults.library({identifier=jenkins@20211123, retriever=null})
                uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
                BuildManifest.asBoolean()
+               uploadTestResults.echo(Build Id: 1236)
                BuildManifest.getArtifactRoot(perf-test, 1236)
                uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64/tar)
                uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)

tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt

@@ -44,6 +44,7 @@
                uploadTestResults.library({identifier=jenkins@20211123, retriever=null})
                uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml})
                BuildManifest.asBoolean()
+               uploadTestResults.echo(Build Id: 215)
                BuildManifest.getArtifactRoot(dummy_job, 215)
                uploadTestResults.echo(Uploading to s3://DUMMY_ARTIFACT_BUCKET_NAME/dummy_job/1.2.0/215/linux/x64/tar)
                uploadTestResults.withAWS({role=opensearch-test, roleAccount=DUMMY_AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)