CD.yml

name: Build and Release k-NN
on:
  push:
    tags:
      - v*

jobs:
  Provision-Runners:
    name: Provision-Runners
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v1
        with:
          repository: opendistro-for-elasticsearch/opendistro-build
          ref: master
          path: opendistro-build
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_INFRATEST_EC2_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_INFRATEST_EC2_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: AWS Cli Processing
        run: |
          #!/bin/bash
          # Please add comma at the end of the RUNNERS strings
          # Don't add space after commas
          RUNNERS="odfe-knnlib-build-arm64"
          export GIT_URL_REPO=${GITHUB_REPOSITORY}
          ../opendistro-build/release-tools/scripts/setup_runners.sh run $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} ami-0ed17173ab64255b1

  plugin-build-and-ship-artifacts:
    strategy:
      matrix:
        java: [14]

    name: Build and release plugin artifacts
    runs-on: ubuntu-18.04
    steps:
      - name: Checkout k-NN
        uses: actions/checkout@v2

      - name: Configure AWS
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Setup Java ${{ matrix.java }}
        uses: actions/setup-java@v1
        with:
          java-version: ${{ matrix.java }}

      - name: Build
        run: |
          ./gradlew buildPackages --console=plain -Dbuild.snapshot=false

      - name: Upload plugin artifacts to S3
        shell: bash
        run: |
          zip=`ls build/distributions/*.zip`
          rpm=`ls build/distributions/*.rpm`
          deb=`ls build/distributions/*.deb`

          # Inject the build number before the suffix
          zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip`
          rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm`
          deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb`

          s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/knn/"

          echo "Copying ${zip} to ${s3_prefix}${zip_outfile}"
          aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile}

          echo "Copying ${rpm} to ${s3_prefix}${rpm_outfile}"
          aws s3 cp --quiet $rpm ${s3_prefix}${rpm_outfile}

          echo "Copying ${deb} to ${s3_prefix}${deb_outfile}"
          aws s3 cp --quiet $deb ${s3_prefix}${deb_outfile}

  library-build-and-ship-artifacts-x64:
    name: Build and release JNI library artifacts x64
    runs-on: ubuntu-18.04
    container:
      image: centos:7
    strategy:
      matrix:
        java: [14]
        compiler: [g++]
    steps:
      - name: Checkout k-NN
        uses: actions/checkout@v1
        with:
          submodules: true

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Setup Java ${{ matrix.java }}
        uses: actions/setup-java@v1
        with:
          java-version: ${{ matrix.java }}

      - name: Install dependencies
        run: |
          yum update -y
          yum install -y cmake rpm-build gcc-c++ make epel-release unzip
          yum repolist
          yum install -y dpkg
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          ./aws/install
          aws --version

      - name: Build
        env:
          CXX: ${{ matrix.compiler }}
        run: |
          cd jni
          sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt
          cmake .
          make package

          cd packages
          folder_name=`ls ./*.rpm | sed 's|\(.*\)\..*|\1|'`
          zip_name=$folder_name".zip"
          mkdir $folder_name
          cp ../release/*.so $folder_name
          zip -r $zip_name $folder_name/*

      - name: Upload JNI library x64 artifacts to S3
        shell: bash
        run: |
          zip=`ls jni/packages/*.zip`
          rpm=`ls jni/packages/*.rpm`
          deb=`ls jni/packages/*.deb`

          # Inject the build number before the suffix
          zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip`
          rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm`
          deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb`

          s3_lib_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/opendistro-libs/knnlib/"

          echo "Copying ${zip} to ${s3_lib_prefix}${zip_outfile}"
          aws s3 cp --quiet $zip ${s3_lib_prefix}${zip_outfile}

          echo "Copying ${rpm} to ${s3_lib_prefix}${rpm_outfile}"
          aws s3 cp --quiet $rpm ${s3_lib_prefix}${rpm_outfile}

          echo "Copying ${deb} to ${s3_lib_prefix}${deb_outfile}"
          aws s3 cp --quiet $deb ${s3_lib_prefix}${deb_outfile}

  library-build-and-ship-artifacts-arm64:
    needs: [Provision-Runners]
    runs-on: [self-hosted, Linux, ARM64, odfe-knnlib-build-arm64]
    name: Build and release JNI library artifacts arm64
    strategy:
      matrix:
        java: [14]
        compiler: [g++]
    steps:
      - name: Checkout k-NN
        uses: actions/checkout@v1
        with:
          submodules: true

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      # Arm64 is not supported by setup-java actions as of now
      # Placeholder in case it adds support later
      # Using pre-installed jdk on arm64 self-hosted runners now
      #- name: Setup Java ${{ matrix.java }}
      #  uses: actions/setup-java@v1
      #  with:
      #    java-version: ${{ matrix.java }}

      - name: Install dependencies
        run: |
          sudo yum update -y
          sudo yum install -y cmake rpm-build gcc-c++ make epel-release unzip
          sudo yum repolist
          sudo yum install -y dpkg
          curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install
          aws --version

      - name: Build
        env:
          CXX: ${{ matrix.compiler }}
        run: |

          export JAVA_HOME="/etc/alternatives/jre"
          echo JAVA_HOME $JAVA_HOME

          cd jni
          cmake .
          make package

          cd packages
          folder_name=`ls ./*.rpm | sed 's|\(.*\)\..*|\1|'`
          zip_name=$folder_name".zip"
          mkdir $folder_name
          cp ../release/*.so $folder_name
          zip -r $zip_name $folder_name/*

      - name: Upload JNI library arm64 artifacts to S3
        shell: bash
        run: |
          zip=`ls jni/packages/*.zip`
          rpm=`ls jni/packages/*.rpm`
          deb=`ls jni/packages/*.deb`

          # Inject the build number before the suffix
          zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip`
          rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm`
          deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb`

          s3_lib_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/opendistro-libs/knnlib/"

          echo "Copying ${zip} to ${s3_lib_prefix}${zip_outfile}"
          aws s3 cp --quiet $zip ${s3_lib_prefix}${zip_outfile}

          echo "Copying ${rpm} to ${s3_lib_prefix}${rpm_outfile}"
          aws s3 cp --quiet $rpm ${s3_lib_prefix}${rpm_outfile}

          echo "Copying ${deb} to ${s3_lib_prefix}${deb_outfile}"
          aws s3 cp --quiet $deb ${s3_lib_prefix}${deb_outfile}

  CleanUp-Runners:
    needs: [library-build-and-ship-artifacts-arm64]
    if: always()
    name: CleanUp-Runners
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v1
        with:
          repository: opendistro-for-elasticsearch/opendistro-build
          ref: master
          path: opendistro-build
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_INFRATEST_EC2_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_INFRATEST_EC2_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      - name: AWS Cli Processing
        run: |
          #!/bin/bash
          # Please add comma at the end of the RUNNERS strings
          # Don't add space after commas
          RUNNERS="odfe-knnlib-build-arm64"
          export GIT_URL_REPO=${GITHUB_REPOSITORY}
          ../opendistro-build/release-tools/scripts/setup_runners.sh terminate $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }}