[HUDI-6902] Containerize the Azure tests

apache · Feb 5, 2024 · a35d70e · a35d70e
1 parent ed6b072
commit a35d70e
Show file tree

Hide file tree

Showing 5 changed files with 135 additions and 54 deletions.
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
@@ -150,7 +150,7 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
+          ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service -Pthrift-gen-source-with-script $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
       - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -335,10 +335,10 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           SCALA_PROFILE: 'scala-2.12'
         run: |
-          mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS
+          mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -Phudi-platform-service -Pthrift-gen-source-with-script
           # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708.
           sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources
-          mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0
+          mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 -Phudi-platform-service -Pthrift-gen-source-with-script
       - name: IT - Bundle Validation - OpenJDK 8
         env:
           FLINK_PROFILE: ${{ matrix.flinkProfile }}

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use a home made image as the base, which includes:
+# utuntu:latest
+# git
+# thrift
+# maven
+# java8
+# Use an official Ubuntu base image
+FROM ubuntu:latest
+
+# Install necessary packages
+RUN apt-get update && \
+    apt-get install -y openjdk-8-jdk maven && \
+    apt-get clean
+
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    automake \
+    bison \
+    flex \
+    libboost-all-dev \
+    libevent-dev \
+    libssl-dev \
+    git \
+    pkg-config
+
+WORKDIR /app
+
+# Clone the Thrift repository from GitHub
+RUN git clone --branch 0.13.0 --depth 1 https://github.com/apache/thrift.git .
+
+RUN export PKG_PROG_PKG_CONFIG=`which pkg-config`
+
+# Build and install Thrift
+RUN ./bootstrap.sh && \
+    ./configure && \
+    make && \
+    make install
+
+# Cleanup unnecessary build dependencies
+RUN apt-get remove -y \
+    build-essential \
+    automake \
+    bison \
+    flex \
+    libboost-all-dev \
+    libevent-dev \
+    libssl-dev && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Set Java 8 as the default version
+RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java 1
+RUN update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+
+CMD ["java", "-version"]
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy git repo into the working directory
+COPY . /app
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
@@ -98,7 +98,7 @@ parameters:
 variables:
   BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18'
   PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
-  MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
+  MVN_OPTS_INSTALL: '-DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
   MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
   JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
   JOB2_MODULES: ${{ join(',',parameters.job2Modules) }}
@@ -201,35 +201,30 @@ stages:
         displayName: UT FT other modules
         timeoutInMinutes: '240'
         steps:
-          - task: Maven@4
-            displayName: maven install
+          - task: Docker@2
+            displayName: "build docker image"
             inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'clean install'
-              options: $(MVN_OPTS_INSTALL)
-              publishJUnitResults: true
-              testResultsFiles: '**/surefire-reports/TEST-*.xml'
-              jdkVersionOption: '1.8'
-          - task: Maven@4
-            displayName: UT other modules
+              containerRegistry: 'onehouse-docker-hub'
+              repository: 'prettydocker/hudi-devops'
+              command: 'build'
+              Dockerfile: '**/Dockerfile'
+              ImageName: $(Build.BuildId)
+          - task: Docker@2
+            displayName: "login to docker"
             inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)
-              publishJUnitResults: true
-              testResultsFiles: '**/surefire-reports/TEST-*.xml'
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - task: Maven@4
-            displayName: FT other modules
+              command: "login"
+              containerRegistry: "onehouse-docker-hub"
+          - task: Docker@2
+            displayName: "UT other modules"
             inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES)
-              publishJUnitResults: true
-              testResultsFiles: '**/surefire-reports/TEST-*.xml'
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - script: |
-              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
-            displayName: Top 100 long-running testcases
+              containerRegistry: 'onehouse-docker-hub'
+              repository: 'prettydocker/hudi-devops'
+              command: 'run'
+              arguments: "-i docker.io/prettydocker/hudi-devops:$(Build.BuildId) /bin/bash -c \"mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source && mvn test  $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)\""
+          - task: Docker@2
+            displayName: "FT other modules"
+            inputs:
+              containerRegistry: 'onehouse-docker-hub'
+              repository: 'prettydocker/hudi-devops'
+              command: 'run'
+              arguments: "-i docker.io/prettydocker/hudi-devops:$(Build.BuildId) /bin/bash -c \"mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source && mvn test  $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES)\""
diff --git a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml
@@ -92,6 +92,34 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>thrift-gen-source-with-script</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <version>1.6.0</version>
+                        <executions>
+                            <execution>
+                                <id>thrift-install-and-generate-source</id>
+                                <phase>generate-sources</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                            </execution>
+                        </executions>
+                        <configuration>
+                            <executable>${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh</executable>
+                            <arguments>
+                                <argument>${thrift.install.env}</argument>
+                            </arguments>
+                            <skip>false</skip>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
         <profile>
             <id>m1-mac</id>
             <properties>
@@ -108,27 +136,6 @@
 
     <build>
         <plugins>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-                <version>1.6.0</version>
-                <executions>
-                    <execution>
-                        <id>thrift-install-and-generate-source</id>
-                        <phase>generate-sources</phase>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                    </execution>
-                </executions>
-                <configuration>
-                    <executable>${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh</executable>
-                    <arguments>
-                        <argument>${thrift.install.env}</argument>
-                    </arguments>
-                    <skip>false</skip>
-                </configuration>
-            </plugin>
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -230,6 +230,7 @@
     <springboot.version>2.7.3</springboot.version>
     <spring.shell.version>2.1.1</spring.shell.version>
     <snappy.version>1.1.8.3</snappy.version>
+    <thrift.executable>/usr/local/bin/thrift</thrift.executable>
   </properties>
 
   <scm>