From 7a0bd04dd379ce41c035f229514c52a8a0c7db6e Mon Sep 17 00:00:00 2001 From: Claas Augner Date: Tue, 21 Jan 2025 13:46:58 +0100 Subject: [PATCH] chore(workflows/build): use static robots.txt --- .github/workflows/prod-build.yml | 3 ++- .github/workflows/stage-build.yml | 3 ++- .github/workflows/test-build.yml | 3 ++- assets/nonprod/robots.txt | 3 +++ assets/prod/robots.txt | 6 ++++++ tool/build-robots-txt.ts | 28 ---------------------------- 6 files changed, 15 insertions(+), 31 deletions(-) create mode 100644 assets/nonprod/robots.txt create mode 100644 assets/prod/robots.txt delete mode 100644 tool/build-robots-txt.ts diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml index 0413b299097e..677b1a148547 100644 --- a/.github/workflows/prod-build.yml +++ b/.github/workflows/prod-build.yml @@ -301,7 +301,8 @@ jobs: yarn build:sw yarn build:client yarn build:ssr - yarn tool:legacy build-robots-txt + + cp assets/prod/robots.txt client/build/robots.txt yarn rari content sync-translated-content yarn rari git-history diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml index 57561d91e1b8..6ec334e75366 100644 --- a/.github/workflows/stage-build.yml +++ b/.github/workflows/stage-build.yml @@ -317,7 +317,8 @@ jobs: yarn build:sw yarn build:client yarn build:ssr - yarn tool:legacy build-robots-txt + + cp assets/nonprod/robots.txt client/build/robots.txt yarn rari content sync-translated-content yarn rari git-history diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 8698df15a2dd..80afbe9caa19 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -249,7 +249,8 @@ jobs: yarn build:sw yarn build:client yarn build:ssr - yarn tool:legacy build-robots-txt + + cp assets/nonprod/robots.txt client/build/robots.txt yarn rari content sync-translated-content yarn rari git-history diff --git a/assets/nonprod/robots.txt b/assets/nonprod/robots.txt new file mode 100644 index 000000000000..4afefd85cadf --- /dev/null +++ b/assets/nonprod/robots.txt @@ -0,0 +1,3 @@ +User-Agent: * + +Disallow: / \ No newline at end of file diff --git a/assets/prod/robots.txt b/assets/prod/robots.txt new file mode 100644 index 000000000000..771cc3ff6ca4 --- /dev/null +++ b/assets/prod/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Sitemap: https://developer.mozilla.org/sitemap.xml + +Disallow: /api/ +Disallow: /*/files/ +Disallow: /media \ No newline at end of file diff --git a/tool/build-robots-txt.ts b/tool/build-robots-txt.ts deleted file mode 100644 index 3bcd6044f2c0..000000000000 --- a/tool/build-robots-txt.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * This script generates a /robots.txt file that depends on - * process.env.BUILD_ALWAYS_ALLOW_ROBOTS. - * - */ -import fs from "node:fs"; - -import { ALWAYS_ALLOW_ROBOTS } from "../libs/env/index.js"; - -const ALLOW_TEXT = ` -User-agent: * -Sitemap: https://developer.mozilla.org/sitemap.xml - -Disallow: /api/ -Disallow: /*/files/ -Disallow: /media -`; - -const DISALLOW_TEXT = ` -User-Agent: * - -Disallow: / -`; - -export async function runBuildRobotsTxt(outfile: string) { - const content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT; - fs.writeFileSync(outfile, `${content.trim()}\n`, "utf-8"); -}