diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index b14153acb8242..82f5b22a8aad9 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -12,3 +12,8 @@ tests: - added|modified: '**/src/test/java/**/*.java' cpp: - 'pulsar-client-cpp/**' +need_owasp: + - 'pom.xml' + - '**/pom.xml' + - 'src/owasp-dependency-check-false-positives.xml' + - 'src/owasp-dependency-check-suppressions.xml' \ No newline at end of file diff --git a/.github/workflows/ci-owasp-dep-check.yaml b/.github/workflows/ci-owasp-dep-check.yaml deleted file mode 100644 index 27f820fb20a0b..0000000000000 --- a/.github/workflows/ci-owasp-dep-check.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - Misc - OWASP Dependency Check -on: - pull_request: - branches: - - master - push: - branches: - - branch-* - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - - owasp-dep-check: - name: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed pom files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: | - poms: - - 'pom.xml' - - '**/pom.xml' - - 'src/owasp-dependency-check-false-positives.xml' - - 'src/owasp-dependency-check-suppressions.xml' - - '.github/workflows/ci-owasp-dep-check.yaml' - - - name: Cache local Maven repository - if: ${{ steps.changes.outputs.poms == 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 17 - uses: actions/setup-java@v2 - if: ${{ steps.changes.outputs.poms == 'true' }} - with: - distribution: 'temurin' - java-version: 17 - - - name: clean disk - if: ${{ steps.changes.outputs.poms == 'true' }} - run: | - sudo swapoff -a - sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - # Projects dependent on flume, hdfs, hbase, and trino currently excluded from the scan. - - name: run "clean verify" to trigger dependency check - if: ${{ steps.changes.outputs.poms == 'true' }} - run: mvn -q -B -ntp clean verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - - - name: Upload report - uses: actions/upload-artifact@v3 - if: ${{ cancelled() || failure() }} - continue-on-error: true - with: - name: dependency report - path: target/dependency-check-report.html diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 2f2e63ab36751..5532aea0f8406 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -855,6 +855,74 @@ jobs: - name: build package run: mvn -B clean package -DskipTests -T 1C -ntp + owasp-dep-check: + name: OWASP dependency check + runs-on: ubuntu-20.04 + timeout-minutes: 120 + needs: [ 'changed_files_job', 'integration-tests' ] + if: ${{ needs.changed_files_job.outputs.need_owasp == 'true' }} + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Tune Runner VM + uses: ./.github/actions/tune-runner-vm + + - name: Setup ssh access to build runner VM + # ssh access is enabled for builds in own forks + if: ${{ github.repository != 'apache/pulsar' }} + uses: ./.github/actions/ssh-access + with: + limit-access-to-actor: true + + - name: Cache Maven dependencies + uses: actions/cache@v2 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/pulsar + key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2-dependencies-core-modules- + - name: Set up JDK ${{ matrix.jdk || '17' }} + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk || '17' }} + + - name: clean disk + run: | + sudo swapoff -a + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + - name: Install gh-actions-artifact-client.js + uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master + + - name: Restore maven build results from Github artifact cache + run: | + cd $HOME + $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. + - name: run "clean verify" to trigger dependency check + run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + + - name: Upload report + uses: actions/upload-artifact@v3 + if: ${{ cancelled() || failure() }} + continue-on-error: true + with: + name: dependency report + path: target/dependency-check-report.html + + - name: Wait for ssh connection when build fails + # ssh access is enabled for builds in own forks + uses: ./.github/actions/ssh-access + if: ${{ failure() && github.repository != 'apache/pulsar' }} + continue-on-error: true + with: + action: wait # This job is required for pulls to be merged. # It depends on all other jobs in this workflow.