diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml index 0413b299097e..677b1a148547 100644 --- a/.github/workflows/prod-build.yml +++ b/.github/workflows/prod-build.yml @@ -301,7 +301,8 @@ jobs: yarn build:sw yarn build:client yarn build:ssr - yarn tool:legacy build-robots-txt + + cp assets/prod/robots.txt client/build/robots.txt yarn rari content sync-translated-content yarn rari git-history diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml index 57561d91e1b8..6ec334e75366 100644 --- a/.github/workflows/stage-build.yml +++ b/.github/workflows/stage-build.yml @@ -317,7 +317,8 @@ jobs: yarn build:sw yarn build:client yarn build:ssr - yarn tool:legacy build-robots-txt + + cp assets/nonprod/robots.txt client/build/robots.txt yarn rari content sync-translated-content yarn rari git-history diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 2cd1af2f94d8..02e541442f9d 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -256,7 +256,8 @@ jobs: yarn build:sw yarn build:client yarn build:ssr - yarn tool:legacy build-robots-txt + + cp assets/nonprod/robots.txt client/build/robots.txt yarn rari content sync-translated-content yarn rari git-history diff --git a/assets/nonprod/robots.txt b/assets/nonprod/robots.txt new file mode 100644 index 000000000000..d08d2f12524b --- /dev/null +++ b/assets/nonprod/robots.txt @@ -0,0 +1,3 @@ +User-Agent: * + +Disallow: / diff --git a/assets/prod/robots.txt b/assets/prod/robots.txt new file mode 100644 index 000000000000..b1cc531f395a --- /dev/null +++ b/assets/prod/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Sitemap: https://developer.mozilla.org/sitemap.xml + +Disallow: /api/ +Disallow: /*/files/ +Disallow: /media diff --git a/package.json b/package.json index 62e214deec44..8384db628413 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,8 @@ "build:legacy::curriculum": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node build/build-curriculum.ts", "build:legacy::docs": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node build/cli.ts -n", "build:legacy:blog": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node build/build-blog.ts", - "build:legacy:prepare": "yarn build:client && yarn build:ssr && yarn tool:legacy popularities && yarn tool:legacy spas && yarn tool:legacy gather-git-history && yarn tool:legacy build-robots-txt", - "build:prepare": "yarn build:client && yarn build:ssr && yarn tool:legacy build-robots-txt", + "build:legacy:prepare": "yarn build:client && yarn build:ssr && yarn tool:legacy popularities && yarn tool:legacy spas && yarn tool:legacy gather-git-history", + "build:prepare": "yarn build:client && yarn build:ssr", "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node ssr/prepare.ts && webpack --mode=production --config=ssr/webpack.config.js", "build:sw": "cd client/pwa && yarn && yarn build:prod", "build:sw-dev": "cd client/pwa && yarn && yarn build", diff --git a/testing/tests/index.test.ts b/testing/tests/index.test.ts index 7477365dd400..14837bee70d2 100644 --- a/testing/tests/index.test.ts +++ b/testing/tests/index.test.ts @@ -1177,17 +1177,6 @@ test("built search-index.json (en-US)", () => { expect(urlToTitle.get("/en-US/docs/Web/Foo")).toBe(": A test tag"); }); -test("the robots.txt file was created", () => { - const filePath = path.join(buildRoot, "robots.txt"); - const text = fs.readFileSync(filePath, "utf-8"); - // The content of robots file when it's in production mode is - // to ONLY say `Disallow: /api/`. - // When the robots file is for disallowing everything it - // will ONLY say `Disallow: /`. - expect(text).toContain("Disallow: /api/"); - expect(text).not.toContain("Disallow: /\n"); -}); - test("duplicate IDs are de-duplicated", () => { const builtFolder = path.join( buildRoot, diff --git a/tool/build-robots-txt.ts b/tool/build-robots-txt.ts deleted file mode 100644 index 3bcd6044f2c0..000000000000 --- a/tool/build-robots-txt.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * This script generates a /robots.txt file that depends on - * process.env.BUILD_ALWAYS_ALLOW_ROBOTS. - * - */ -import fs from "node:fs"; - -import { ALWAYS_ALLOW_ROBOTS } from "../libs/env/index.js"; - -const ALLOW_TEXT = ` -User-agent: * -Sitemap: https://developer.mozilla.org/sitemap.xml - -Disallow: /api/ -Disallow: /*/files/ -Disallow: /media -`; - -const DISALLOW_TEXT = ` -User-Agent: * - -Disallow: / -`; - -export async function runBuildRobotsTxt(outfile: string) { - const content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT; - fs.writeFileSync(outfile, `${content.trim()}\n`, "utf-8"); -} diff --git a/tool/cli.ts b/tool/cli.ts index e8808ff68470..c1a5e3f43628 100644 --- a/tool/cli.ts +++ b/tool/cli.ts @@ -25,13 +25,11 @@ import { buildDocument, gatherGitHistory, buildSPAs } from "../build/index.js"; import { isValidLocale } from "../libs/locale-utils/index.js"; import type { Doc } from "../libs/types/document.js"; import { - ALWAYS_ALLOW_ROBOTS, BUILD_OUT_ROOT, CONTENT_ROOT, CONTENT_TRANSLATED_ROOT, } from "../libs/env/index.js"; import { runMakePopularitiesFile } from "./popularities.js"; -import { runBuildRobotsTxt } from "./build-robots-txt.js"; import { syncAllTranslatedContent } from "./sync-translated-content.js"; import { macroUsageReport } from "./macro-usage-report.js"; import * as kumascript from "../kumascript/index.js"; @@ -176,15 +174,6 @@ interface PopularitiesActionParameters extends ActionParameters { logger: Logger; } -interface BuildRobotsTxtActionParameters extends ActionParameters { - options: { - outfile: string; - maxUris: number; - refresh: boolean; - }; - logger: Logger; -} - interface MacrosActionParameters extends ActionParameters { args: { cmd: string; @@ -869,28 +858,6 @@ program }) ) - .command( - "build-robots-txt", - "Generate a robots.txt in the build root depending ALWAYS_ALLOW_ROBOTS" - ) - .option("--outfile ", "name of the generated file", { - default: path.join(BUILD_OUT_ROOT, "robots.txt"), - }) - .action( - tryOrExit(async ({ options, logger }: BuildRobotsTxtActionParameters) => { - const { outfile } = options; - await runBuildRobotsTxt(outfile); - logger.info( - chalk.yellow( - `Generated ${path.relative( - ".", - outfile - )} based on ALWAYS_ALLOW_ROBOTS=${ALWAYS_ALLOW_ROBOTS}` - ) - ); - }) - ) - .command("spas", "Build (SSR) all the skeleton apps for single page apps") .action( tryOrExit(async ({ options }) => {