Merge pull request #12 from HotoRas/host

Object Storage returns #7 #8
nekoplanet · Jun 27, 2024 · 665dd48 · 665dd48
2 parents 5632537 + 47f3b77
commit 665dd48
Show file tree

Hide file tree

Showing 66 changed files with 1,029 additions and 279 deletions.
diff --git a/.config/example.yml b/.config/example.yml
@@ -129,23 +129,6 @@ redis:
 #  index: ''
 #  scope: local
 
-#   ┌────────────────────────────────────┐
-#───┘ File storage (Drive) configuration └──────────────────────
-
-#s3:
-#  baseUrl: s3.example.com
-#  bucket: example-bucket
-#  prefix: example-prefix
-#  endpoint: s3.example.com
-#  region: us-east-1
-#  useSSL: true
-#  accessKey: example-access-key
-#  secretKey: example-secret-key
-#  options:
-#    setPublicRead: true
-#    forcePathStyle: false
-#    useProxy: false
-
 #   ┌───────────────┐
 #───┘ ID generation └───────────────────────────────────────────
 

diff --git a/.github/workflows/dockle.yml b/.github/workflows/dockle.yml
diff --git a/.github/workflows/reviewer_lottery.yml b/.github/workflows/reviewer_lottery.yml
@@ -0,0 +1,13 @@
+name: "Reviewer lottery"
+on:
+  pull_request_target:
+    types: [] #[opened, ready_for_review, reopened]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: uesteibar/reviewer-lottery@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml
@@ -0,0 +1,112 @@
+name: Storybook
+
+on:
+  push:
+    branches: []
+      #- master
+      #- develop
+  pull_request_target: []
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    env:
+      NODE_OPTIONS: "--max_old_space_size=7168"
+
+    steps:
+    - uses: actions/[email protected]
+      if: github.event_name != 'pull_request_target'
+      with:
+        fetch-depth: 0
+        submodules: true
+    - uses: actions/[email protected]
+      if: github.event_name == 'pull_request_target'
+      with:
+        fetch-depth: 0
+        submodules: true
+        ref: "refs/pull/${{ github.event.number }}/merge"
+    - name: Checkout actual HEAD
+      if: github.event_name == 'pull_request_target'
+      id: rev
+      run: |
+        echo "base=$(git rev-list --parents -n1 HEAD | cut -d" " -f2)" >> $GITHUB_OUTPUT
+        git checkout $(git rev-list --parents -n1 HEAD | cut -d" " -f3)
+    - name: Install pnpm
+      uses: pnpm/action-setup@v2
+      with:
+        version: 8
+        run_install: false
+    - name: Use Node.js 20.x
+      uses: actions/[email protected]
+      with:
+        node-version-file: '.node-version'
+        cache: 'pnpm'
+    - run: corepack enable
+    - run: pnpm i --frozen-lockfile
+    - name: Check pnpm-lock.yaml
+      run: git diff --exit-code pnpm-lock.yaml
+    - name: Build misskey-js
+      run: pnpm --filter misskey-js build
+    - name: Build storybook
+      run: pnpm --filter frontend build-storybook
+    - name: Publish to Chromatic
+      if: github.event_name != 'pull_request_target' && github.ref == 'refs/heads/master'
+      run: pnpm --filter frontend chromatic --exit-once-uploaded -d storybook-static
+      env:
+        CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+    - name: Publish to Chromatic
+      if: github.event_name != 'pull_request_target' && github.ref != 'refs/heads/master'
+      id: chromatic_push
+      run: |
+        DIFF="${{ github.event.before }} HEAD"
+        if [ "$DIFF" = "0000000000000000000000000000000000000000 HEAD" ]; then
+          DIFF="HEAD"
+        fi
+        CHROMATIC_PARAMETER="$(node packages/frontend/.storybook/changes.js $(git diff-tree --no-commit-id --name-only -r $(echo "$DIFF") | xargs))"
+        if [ "$CHROMATIC_PARAMETER" = " --skip" ]; then
+          echo "skip=true" >> $GITHUB_OUTPUT
+        fi
+        if pnpm --filter frontend chromatic -d storybook-static $(echo "$CHROMATIC_PARAMETER"); then
+          echo "success=true" >> $GITHUB_OUTPUT
+        else
+          echo "success=false" >> $GITHUB_OUTPUT
+        fi
+      env:
+        CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+    - name: Publish to Chromatic
+      if: github.event_name == 'pull_request_target'
+      id: chromatic_pull_request
+      run: |
+        DIFF="${{ steps.rev.outputs.base }} HEAD"
+        if [ "$DIFF" = "0000000000000000000000000000000000000000 HEAD" ]; then
+          DIFF="HEAD"
+        fi
+        CHROMATIC_PARAMETER="$(node packages/frontend/.storybook/changes.js $(git diff-tree --no-commit-id --name-only -r $(echo "$DIFF") | xargs))"
+        if [ "$CHROMATIC_PARAMETER" = " --skip" ]; then
+          echo "skip=true" >> $GITHUB_OUTPUT
+        fi
+        BRANCH="${{ github.event.pull_request.head.user.login }}:${{ github.event.pull_request.head.ref }}"
+        if [ "$BRANCH" = "misskey-dev:${{ github.event.pull_request.head.ref }}" ]; then
+          BRANCH="${{ github.event.pull_request.head.ref }}"
+        fi
+        pnpm --filter frontend chromatic --exit-once-uploaded -d storybook-static --branch-name $BRANCH $(echo "$CHROMATIC_PARAMETER")
+      env:
+        CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+    - name: Notify that Chromatic detects changes
+      uses: actions/[email protected]
+      if: github.event_name != 'pull_request_target' && steps.chromatic_push.outputs.success == 'false'
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          github.rest.repos.createCommitComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            commit_sha: context.sha,
+            body: 'Chromatic detects changes. Please [review the changes on Chromatic](https://www.chromatic.com/builds?appId=6428f7d7b962f0b79f97d6e4).'
+          })
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: storybook
+        path: packages/frontend/storybook-static
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 # syntax = docker/dockerfile:1.4
 
-ARG NODE_VERSION=20
+ARG NODE_VERSION=22
 
 # build assets & compile TypeScript
 

diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
@@ -65,21 +65,6 @@ type Source = {
 		index: string;
 		scope?: 'local' | 'global' | string[];
 	};
-	s3?: {
-		baseUrl: string;
-		bucket: string;
-		prefix: string;
-		endpoint: string;
-		region?: string;
-		useSSL: boolean;
-		accessKey: string;
-		secretKey: string;
-		options?: {
-			setPublicRead?: boolean;
-			forcePathStyle?: boolean;
-			useProxy?: boolean;
-		}
-	};
 
 	skebStatus?: {
 		method: string;

diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
@@ -147,7 +147,9 @@ export class DriveService {
 	// thunbnail, webpublic を必要なら生成
 		const alts = await this.generateAlts(path, type, !file.uri);
 
-		if (this.config.s3) {
+		const meta = await this.metaService.fetch();
+
+		if (meta.useObjectStorage) {
 		//#region ObjectStorage params
 			let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);
 
@@ -166,11 +168,11 @@ export class DriveService {
 				ext = '';
 			}
 
-			const baseUrl = this.config.s3.baseUrl
-				?? `${ this.config.s3.useSSL ? 'https' : 'http' }://${ this.config.s3.endpoint }/${ this.config.s3.bucket }`;
+			const baseUrl = meta.objectStorageBaseUrl
+				?? `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`;
 
 			// for original
-			const key = `${this.config.s3.prefix}/${randomUUID()}${ext}`;
+			const key = `${meta.objectStoragePrefix}/${randomUUID()}${ext}`;
 			const url = `${ baseUrl }/${ key }`;
 
 			// for alts
@@ -187,15 +189,15 @@ export class DriveService {
 			];
 
 			if (alts.webpublic) {
-				webpublicKey = `${this.config.s3.prefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
+				webpublicKey = `${meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
 				webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
 
 				this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
 				uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
 			}
 
 			if (alts.thumbnail) {
-				thumbnailKey = `${this.config.s3.prefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
+				thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
 				thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
 
 				this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
@@ -372,8 +374,10 @@ export class DriveService {
 		if (type === 'image/apng') type = 'image/png';
 		if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream';
 
+		const meta = await this.metaService.fetch();
+
 		const params = {
-			Bucket: this.config.s3?.bucket,
+			Bucket: meta.objectStorageBucket,
 			Key: key,
 			Body: stream,
 			ContentType: type,
@@ -386,9 +390,9 @@ export class DriveService {
 			// 許可されているファイル形式でしか拡張子をつけない
 			ext ? correctFilename(filename, ext) : filename,
 		);
-		if (this.config.s3?.options?.setPublicRead) params.ACL = 'public-read';
+		if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
 
-		await this.s3Service.upload(params)
+		await this.s3Service.upload(meta, params)
 			.then(
 				result => {
 					if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
@@ -805,13 +809,14 @@ export class DriveService {
 
 	@bindThis
 	public async deleteObjectStorageFile(key: string) {
+		const meta = await this.metaService.fetch();
 		try {
 			const param = {
-				Bucket: this.config.s3?.bucket,
+				Bucket: meta.objectStorageBucket,
 				Key: key,
 			} as DeleteObjectCommandInput;
 
-			await this.s3Service.delete(param);
+			await this.s3Service.delete(meta, param);
 		} catch (err: any) {
 			if (err.name === 'NoSuchKey') {
 				this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);

diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts
@@ -13,6 +13,7 @@ import { NodeHttpHandler, NodeHttpHandlerOptions } from '@smithy/node-http-handl
 import { bindThis } from '@/decorators.js';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
+import type { Meta } from '@/models/entities/Meta.js';
 import { HttpRequestService } from '@/core/HttpRequestService.js';
 import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3';
 
@@ -27,33 +28,35 @@ export class S3Service {
 	}
 
 	@bindThis
-	public getS3Client(): S3Client {
-		const u = `${this.config.s3?.useSSL ? 'https' : 'http'}://${this.config.s3?.endpoint ?? 'example.net'}`; // dummy url to select http(s) agent
+	public getS3Client(meta: Meta): S3Client {
+		const u = meta.objectStorageEndpoint
+			? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
+			: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
 
-		const agent = this.httpRequestService.getAgentByUrl(new URL(u), !this.config.s3?.options?.useProxy);
+		const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
 		const handlerOption: NodeHttpHandlerOptions = {};
-		if (this.config.s3?.useSSL) {
+		if (meta.objectStorageUseSSL) {
 			handlerOption.httpsAgent = agent as https.Agent;
 		} else {
 			handlerOption.httpAgent = agent as http.Agent;
 		}
 
 		return new S3Client({
-			endpoint: this.config.s3?.endpoint ? u : undefined,
-			credentials: (this.config.s3 && (this.config.s3.accessKey !== null && this.config.s3.secretKey !== null)) ? {
-				accessKeyId: this.config.s3.accessKey,
-				secretAccessKey: this.config.s3.secretKey,
+			endpoint: meta.objectStorageEndpoint ? u : undefined,
+			credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
+				accessKeyId: meta.objectStorageAccessKey,
+				secretAccessKey: meta.objectStorageSecretKey,
 			} : undefined,
-			region: this.config.s3?.region ?? 'us-east-1',
-			tls: this.config.s3?.useSSL,
-			forcePathStyle: this.config.s3?.options?.forcePathStyle ?? false, // AWS with endPoint omitted
+			region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない
+			tls: meta.objectStorageUseSSL,
+			forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
 			requestHandler: new NodeHttpHandler(handlerOption),
 		});
 	}
 
 	@bindThis
-	public async upload(input: PutObjectCommandInput) {
-		const client = this.getS3Client();
+	public async upload(meta: Meta, input: PutObjectCommandInput) {
+		const client = this.getS3Client(meta);
 		return new Upload({
 			client,
 			params: input,
@@ -64,8 +67,8 @@ export class S3Service {
 	}
 
 	@bindThis
-	public delete(input: DeleteObjectCommandInput) {
-		const client = this.getS3Client();
+	public delete(meta: Meta, input: DeleteObjectCommandInput) {
+		const client = this.getS3Client(meta);
 		return client.send(new DeleteObjectCommand(input));
 	}
 }