Skip to content

Open source CapCut alternative.

TypeScript

fix(site): let crawlers read auth pages' noindex

David Lebeba4c43af2b
diff --git a/site/src/app/robots.test.ts b/site/src/app/robots.test.ts
new file mode 100644
index 00000000..2ee40591
--- /dev/null
+++ b/site/src/app/robots.test.ts
@@ -0,0 +1,33 @@
+import { expect, test } from "bun:test";
+
+import robots from "@/app/robots";
+import { metadata as signIn } from "@/app/sign-in/page";
+import { metadata as signUp } from "@/app/sign-up/page";
+import { DONKEYCUT_CANONICAL } from "@/cut/lib/hosts";
+
+// Google must crawl auth pages to read their indexing rules, including email
+// login links and callbacks discovered through the editor.
+test("auth pages are crawlable and excluded from search", () => {
+ const { rules } = robots();
+ expect(Array.isArray(rules)).toBe(false);
+ if (Array.isArray(rules)) {
+ throw new Error("Expected one crawler rule");
+ }
+
+ expect(rules.allow).toBe("/");
+ for (const path of ["/sign-in", "/sign-up", "/sign-in?method=email&callbackURL=%2Fapp"]) {
+ for (const blocked of [rules.disallow].flat()) {
+ expect(blocked && path.startsWith(blocked)).toBeFalsy();
+ }
+ }
+
+ expect(signIn.robots).toEqual({ index: false, follow: true });
+ expect(signUp.robots).toEqual({ index: false, follow: true });
+});
+
+// Account data keeps its crawl boundary, and discovery stays on the public host.
+test("robots keeps app and API exclusions and the canonical sitemap", () => {
+ const { rules, sitemap } = robots();
+ expect(rules).toMatchObject({ disallow: ["/api/", "/app/", "/unsubscribe"] });
+ expect(sitemap).toBe(`${DONKEYCUT_CANONICAL}/sitemap.xml`);
+});
diff --git a/site/src/app/robots.ts b/site/src/app/robots.ts
index 3236a745..f70f2072 100644
--- a/site/src/app/robots.ts
+++ b/site/src/app/robots.ts
@@ -11,7 +11,8 @@ export default function robots(): MetadataRoute.Robots {
rules: {
userAgent: "*",
allow: "/",
- disallow: ["/api/", "/app/", "/sign-in", "/sign-up", "/unsubscribe"],
+ // Auth pages stay crawlable so Google can read their noindex metadata.
+ disallow: ["/api/", "/app/", "/unsubscribe"],
},
sitemap: `${DONKEYCUT_CANONICAL}/sitemap.xml`,
};
diff --git a/site/src/app/sign-in/page.tsx b/site/src/app/sign-in/page.tsx
index 88315efc..a0154701 100644
--- a/site/src/app/sign-in/page.tsx
+++ b/site/src/app/sign-in/page.tsx
@@ -5,6 +5,8 @@ import { AuthPage, type AuthSearchParams } from "@/app/_components/landing/AuthP
export const metadata: Metadata = {
title: "Log in | Donkey",
description: "Log in to Donkey Cut.",
+ // Crawlers can follow the public links while keeping login out of search.
+ robots: { index: false, follow: true },
};
export const instant = true;
diff --git a/site/src/app/sign-up/page.tsx b/site/src/app/sign-up/page.tsx
index 6bc84710..976015c6 100644
--- a/site/src/app/sign-up/page.tsx
+++ b/site/src/app/sign-up/page.tsx
@@ -5,6 +5,8 @@ import { AuthPage, type AuthSearchParams } from "@/app/_components/landing/AuthP
export const metadata: Metadata = {
title: "Sign up | Donkey",
description: "Create a Donkey Cut account.",
+ // Crawlers can follow the public links while keeping signup out of search.
+ robots: { index: false, follow: true },
};
export const instant = true;