Open source CapCut alternative.
TypeScript
fix(site): let crawlers read auth pages' noindex
David Lebeba4c43af2b
diff --git a/site/src/app/robots.test.ts b/site/src/app/robots.test.tsnew file mode 100644index 00000000..2ee40591--- /dev/null+++ b/site/src/app/robots.test.ts@@ -0,0 +1,33 @@+import { expect, test } from "bun:test";++import robots from "@/app/robots";+import { metadata as signIn } from "@/app/sign-in/page";+import { metadata as signUp } from "@/app/sign-up/page";+import { DONKEYCUT_CANONICAL } from "@/cut/lib/hosts";++// Google must crawl auth pages to read their indexing rules, including email+// login links and callbacks discovered through the editor.+test("auth pages are crawlable and excluded from search", () => {+ const { rules } = robots();+ expect(Array.isArray(rules)).toBe(false);+ if (Array.isArray(rules)) {+ throw new Error("Expected one crawler rule");+ }++ expect(rules.allow).toBe("/");+ for (const path of ["/sign-in", "/sign-up", "/sign-in?method=email&callbackURL=%2Fapp"]) {+ for (const blocked of [rules.disallow].flat()) {+ expect(blocked && path.startsWith(blocked)).toBeFalsy();+ }+ }++ expect(signIn.robots).toEqual({ index: false, follow: true });+ expect(signUp.robots).toEqual({ index: false, follow: true });+});++// Account data keeps its crawl boundary, and discovery stays on the public host.+test("robots keeps app and API exclusions and the canonical sitemap", () => {+ const { rules, sitemap } = robots();+ expect(rules).toMatchObject({ disallow: ["/api/", "/app/", "/unsubscribe"] });+ expect(sitemap).toBe(`${DONKEYCUT_CANONICAL}/sitemap.xml`);+});diff --git a/site/src/app/robots.ts b/site/src/app/robots.tsindex 3236a745..f70f2072 100644--- a/site/src/app/robots.ts+++ b/site/src/app/robots.ts@@ -11,7 +11,8 @@ export default function robots(): MetadataRoute.Robots {rules: {userAgent: "*",allow: "/",- disallow: ["/api/", "/app/", "/sign-in", "/sign-up", "/unsubscribe"],+ // Auth pages stay crawlable so Google can read their noindex metadata.+ disallow: ["/api/", "/app/", "/unsubscribe"],},sitemap: `${DONKEYCUT_CANONICAL}/sitemap.xml`,};diff --git a/site/src/app/sign-in/page.tsx b/site/src/app/sign-in/page.tsxindex 88315efc..a0154701 100644--- a/site/src/app/sign-in/page.tsx+++ b/site/src/app/sign-in/page.tsx@@ -5,6 +5,8 @@ import { AuthPage, type AuthSearchParams } from "@/app/_components/landing/AuthPexport const metadata: Metadata = {title: "Log in | Donkey",description: "Log in to Donkey Cut.",+ // Crawlers can follow the public links while keeping login out of search.+ robots: { index: false, follow: true },};export const instant = true;diff --git a/site/src/app/sign-up/page.tsx b/site/src/app/sign-up/page.tsxindex 6bc84710..976015c6 100644--- a/site/src/app/sign-up/page.tsx+++ b/site/src/app/sign-up/page.tsx@@ -5,6 +5,8 @@ import { AuthPage, type AuthSearchParams } from "@/app/_components/landing/AuthPexport const metadata: Metadata = {title: "Sign up | Donkey",description: "Create a Donkey Cut account.",+ // Crawlers can follow the public links while keeping signup out of search.+ robots: { index: false, follow: true },};export const instant = true;