Add S3 connector integration tests

Add testing of s3 connector to devnet integration tests.

This change will then remove the need to duplicate integration test
setup in motion-s3-connector repo, and would help us catch breaking
changes early since the devnet instance started up in `motion` repo
builds the motion container from `main` branch.

.github/actions/aws-cli-s3-upload-test/action.yaml

@@ -0,0 +1,8 @@
+name: AWS CLI S3 Upload Test
+description: Creates a file, uploads it using the AWS CLI S3 API, downloads it, and compares the two files.
+runs:
+  using: "composite"
+  steps:
+    - name: Test
+      run: ./.github/actions/aws-cli-s3-upload-test/upload.sh
+      shell: bash

.github/actions/aws-cli-s3-upload-test/upload.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Create a test file to upload
+echo fish > upload.txt
+
+# Create a test bucket, retrying if it fails.
+# We do this to ensure we're not failing too early on a connection error.
+RETRY_NUM=3
+RETRY_EVERY=15
+NUM=$RETRY_NUM
+until aws --endpoint-url http://127.0.0.1:8000 s3 mb s3://test-bucket
+do
+  1>&2 echo Retrying $NUM more times
+  sleep $RETRY_EVERY
+  ((NUM--))
+
+  if [ $NUM -eq 0 ]
+  then
+    1>&2 echo Bucket was not created after $RETRY_NUM tries
+    exit 1
+  fi
+done 
+
+aws --endpoint-url http://127.0.0.1:8000 s3 cp upload.txt s3://test-bucket
+aws --endpoint-url http://127.0.0.1:8000 s3 cp s3://test-bucket/upload.txt download.txt
+cmp upload.txt download.txt
+
+# Compare the downloaded file with the original and echo if successful
+if cmp upload.txt download.txt
+then
+  1>&2 echo "Uploaded and downloaded files are the same"
+else
+  1>&2 echo "Uploaded and downloaded files are different"
+  exit 1
+fi

.github/workflows/devnet-test.yml

@@ -69,7 +69,17 @@ jobs:
           command: |
             cd integration/test/motionlarity
             docker compose ps --services --filter "status=running" | grep motion
-      - name: Run tests
+      - name: Run Motion integration tests
         env:
           MOTION_INTEGRATION_TEST: 'true'
-        run: go test ./integration/test -v
+        run: go test ./integration/test -v
+      - name: Start S3 Connector
+        run: |
+          cd integration/test/s3-connector
+          docker compose -f "docker-compose.yaml" up -d --build
+      - name: Run S3 Connector integration tests
+        env:
+          AWS_ACCESS_KEY_ID: 'accessKey1'
+          AWS_SECRET_ACCESS_KEY: 'verySecretKey1'
+          AWS_DEFAULT_REGION: 'location-motion-v1'
+        uses: ./.github/actions/aws-cli-s3-upload-test

integration/test/s3-connector/docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  cloudserver:
+    image: ghcr.io/filecoin-project/motion-cloudserver:0.1.0
+    environment:
+      REMOTE_MANAGEMENT_DISABLE: 1
+      S3DATA: 'multiple'
+      MOTION_HOST: 'motion'
+    ports:
+      - "8000:8000"
+
+networks:
+  default:
+    name: devnet
+    external: true