From acd079a4e55e189d1cac8424fa9452857f37baac Mon Sep 17 00:00:00 2001
From: "Masih H. Derkani" <m@derkani.org>
Date: Mon, 11 Sep 2023 16:15:27 +0100
Subject: [PATCH] Add S3 connector integration tests

Add testing of s3 connector to devnet integration tests.

This change will then remove the need to duplicate integration test
setup in motion-s3-connector repo, and would help us catch breaking
changes early since the devnet instance started up in `motion` repo
builds the motion container from `main` branch.
---
 .../aws-cli-s3-upload-test/action.yaml        |  8 +++++
 .../actions/aws-cli-s3-upload-test/upload.sh  | 35 +++++++++++++++++++
 .github/workflows/devnet-test.yml             | 14 ++++++--
 .../test/s3-connector/docker-compose.yaml     | 14 ++++++++
 4 files changed, 69 insertions(+), 2 deletions(-)
 create mode 100644 .github/actions/aws-cli-s3-upload-test/action.yaml
 create mode 100755 .github/actions/aws-cli-s3-upload-test/upload.sh
 create mode 100644 integration/test/s3-connector/docker-compose.yaml

diff --git a/.github/actions/aws-cli-s3-upload-test/action.yaml b/.github/actions/aws-cli-s3-upload-test/action.yaml
new file mode 100644
index 0000000..d97e57f
--- /dev/null
+++ b/.github/actions/aws-cli-s3-upload-test/action.yaml
@@ -0,0 +1,8 @@
+name: AWS CLI S3 Upload Test
+description: Creates a file, uploads it using the AWS CLI S3 API, downloads it, and compares the two files.
+runs:
+  using: "composite"
+  steps:
+    - name: Test
+      run: ./.github/actions/aws-cli-s3-upload-test/upload.sh
+      shell: bash
\ No newline at end of file
diff --git a/.github/actions/aws-cli-s3-upload-test/upload.sh b/.github/actions/aws-cli-s3-upload-test/upload.sh
new file mode 100755
index 0000000..6fff180
--- /dev/null
+++ b/.github/actions/aws-cli-s3-upload-test/upload.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Create a test file to upload
+echo fish > upload.txt
+
+# Create a test bucket, retrying if it fails.
+# We do this to ensure we're not failing too early on a connection error.
+RETRY_NUM=3
+RETRY_EVERY=15
+NUM=$RETRY_NUM
+until aws --endpoint-url http://127.0.0.1:8000 s3 mb s3://test-bucket
+do
+  1>&2 echo Retrying $NUM more times
+  sleep $RETRY_EVERY
+  ((NUM--))
+
+  if [ $NUM -eq 0 ]
+  then
+    1>&2 echo Bucket was not created after $RETRY_NUM tries
+    exit 1
+  fi
+done 
+
+aws --endpoint-url http://127.0.0.1:8000 s3 cp upload.txt s3://test-bucket
+aws --endpoint-url http://127.0.0.1:8000 s3 cp s3://test-bucket/upload.txt download.txt
+cmp upload.txt download.txt
+
+# Compare the downloaded file with the original and echo if successful
+if cmp upload.txt download.txt
+then
+  1>&2 echo "Uploaded and downloaded files are the same"
+else
+  1>&2 echo "Uploaded and downloaded files are different"
+  exit 1
+fi
\ No newline at end of file
diff --git a/.github/workflows/devnet-test.yml b/.github/workflows/devnet-test.yml
index 0428bbe..f9ec234 100644
--- a/.github/workflows/devnet-test.yml
+++ b/.github/workflows/devnet-test.yml
@@ -69,7 +69,17 @@ jobs:
           command: |
             cd integration/test/motionlarity
             docker compose ps --services --filter "status=running" | grep motion
-      - name: Run tests
+      - name: Run Motion integration tests
         env:
           MOTION_INTEGRATION_TEST: 'true'
-        run: go test ./integration/test -v
\ No newline at end of file
+        run: go test ./integration/test -v
+      - name: Start S3 Connector
+        run: |
+          cd integration/test/s3-connector
+          docker compose -f "docker-compose.yaml" up -d --build
+      - name: Run S3 Connector integration tests
+        env:
+          AWS_ACCESS_KEY_ID: 'accessKey1'
+          AWS_SECRET_ACCESS_KEY: 'verySecretKey1'
+          AWS_DEFAULT_REGION: 'location-motion-v1'
+        uses: ./.github/actions/aws-cli-s3-upload-test
\ No newline at end of file
diff --git a/integration/test/s3-connector/docker-compose.yaml b/integration/test/s3-connector/docker-compose.yaml
new file mode 100644
index 0000000..6600e74
--- /dev/null
+++ b/integration/test/s3-connector/docker-compose.yaml
@@ -0,0 +1,14 @@
+services:
+  cloudserver:
+    image: ghcr.io/filecoin-project/motion-cloudserver:0.1.0
+    environment:
+      REMOTE_MANAGEMENT_DISABLE: 1
+      S3DATA: 'multiple'
+      MOTION_HOST: 'motion'
+    ports:
+      - "8000:8000"
+
+networks:
+  default:
+    name: devnet
+    external: true
\ No newline at end of file