Skip to content

Production-ready animations, transitions, backgrounds, and scenes for Remotion

feat: IndexNow, per-section llms-full, slimmer llms.txt

kapishdima87bd69227ada
diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml
new file mode 100644
index 00000000..63fef871
--- /dev/null
+++ b/.github/workflows/indexnow.yml
@@ -0,0 +1,43 @@
+name: IndexNow
+
+# Tells Bing and the other IndexNow engines which pages a push to main
+# changed, once Coolify has deployed it. Run it by hand with "all" to submit
+# every URL in the live sitemap.
+on:
+ push:
+ branches: [main]
+ paths:
+ - "content/**"
+ - "app/(home)/**"
+ workflow_dispatch:
+ inputs:
+ scope:
+ description: "changed (last push) or all (whole sitemap)"
+ default: all
+ type: choice
+ options: [all, changed]
+
+concurrency:
+ group: indexnow
+ cancel-in-progress: false
+
+jobs:
+ submit:
+ runs-on: ubuntu-latest
+ timeout-minutes: 50
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: latest
+ - name: Submit changed URLs
+ if: github.event_name == 'push'
+ run: bun scripts/indexnow.mts --changed ${{ github.event.before }} ${{ github.sha }} --wait-deploy
+ - name: Submit every URL
+ if: github.event_name == 'workflow_dispatch' && inputs.scope == 'all'
+ run: bun scripts/indexnow.mts --all
+ - name: Submit the last commit's URLs
+ if: github.event_name == 'workflow_dispatch' && inputs.scope == 'changed'
+ run: bun scripts/indexnow.mts --changed HEAD~1 HEAD
diff --git a/app/index.md/route.ts b/app/index.md/route.ts
index 8173a38f..78a81ebf 100644
--- a/app/index.md/route.ts
+++ b/app/index.md/route.ts
@@ -1,3 +1,4 @@
+import { COMPONENT_COUNT } from "@/config/site";
import { buildLlmsTxt, MARKDOWN_CONTENT_TYPE } from "@/lib/agent-docs";
import { docsIndexPages } from "@/lib/docs-index";
@@ -7,11 +8,14 @@ export const revalidate = false;
// text/markdown. Agents reach it at `/` through the rewrites in next.config.ts;
// `Vary: Accept, User-Agent` is set there for the URLs that negotiate.
export function GET() {
- return new Response(buildLlmsTxt(docsIndexPages()), {
- headers: {
- "Content-Type": MARKDOWN_CONTENT_TYPE,
- // A copy of the HTML home page for agents; search should index the HTML.
- "X-Robots-Tag": "noindex",
+ return new Response(
+ buildLlmsTxt(docsIndexPages(), { components: COMPONENT_COUNT }),
+ {
+ headers: {
+ "Content-Type": MARKDOWN_CONTENT_TYPE,
+ // A copy of the HTML home page for agents; search should index the HTML.
+ "X-Robots-Tag": "noindex",
+ },
},
- });
+ );
}
diff --git a/app/llms-full.txt/route.ts b/app/llms-full.txt/route.ts
index 646f2dbf..3641f914 100644
--- a/app/llms-full.txt/route.ts
+++ b/app/llms-full.txt/route.ts
@@ -1,19 +1,11 @@
-import { getLLMText } from "@/lib/get-llm-text";
+import { corpusResponse } from "@/lib/llms-corpus";
import { source } from "@/source";
export const revalidate = false;
// llms-full.txt — every doc page concatenated as Markdown, for LLMs that want
-// the entire corpus in one request.
-export async function GET() {
- const pages = source.getPages();
- const docs = await Promise.all(pages.map((page) => getLLMText(page)));
-
- return new Response(docs.join("\n\n---\n\n"), {
- headers: {
- "Content-Type": "text/plain; charset=utf-8",
- // The same text as the docs pages; keep search on the HTML.
- "X-Robots-Tag": "noindex",
- },
- });
+// the entire corpus in one request. It runs to megabytes; llms.txt points
+// agents at the per-section files in app/llms-full/[file] instead.
+export function GET() {
+ return corpusResponse(source.getPages());
}
diff --git a/app/llms-full/[file]/route.ts b/app/llms-full/[file]/route.ts
new file mode 100644
index 00000000..7cac8e0b
--- /dev/null
+++ b/app/llms-full/[file]/route.ts
@@ -0,0 +1,29 @@
+import { markdownNotFound } from "@/lib/agent-docs";
+import { corpusResponse } from "@/lib/llms-corpus";
+import { source } from "@/source";
+
+export const revalidate = false;
+
+/** Top-level docs sections, e.g. "transitions" for /docs/transitions/*. */
+function sections(): string[] {
+ return [...new Set(source.getPages().flatMap((page) => page.slugs[0] ?? []))];
+}
+
+export function generateStaticParams() {
+ return sections().map((section) => ({ file: `${section}.txt` }));
+}
+
+// /llms-full/<section>.txt — one docs section as a single Markdown file, small
+// enough to read in one request, unlike the whole corpus at /llms-full.txt.
+export async function GET(
+ _request: Request,
+ { params }: { params: Promise<{ file: string }> },
+) {
+ const { file } = await params;
+ const section = file.replace(/\.txt$/, "");
+ const pages = source.getPages().filter((page) => page.slugs[0] === section);
+ if (!file.endsWith(".txt") || pages.length === 0) {
+ return markdownNotFound(`/llms-full/${file}`);
+ }
+ return corpusResponse(pages);
+}
diff --git a/app/llms.txt/route.ts b/app/llms.txt/route.ts
index e0003e06..d65a3b74 100644
--- a/app/llms.txt/route.ts
+++ b/app/llms.txt/route.ts
@@ -1,3 +1,4 @@
+import { COMPONENT_COUNT } from "@/config/site";
import { buildLlmsTxt } from "@/lib/agent-docs";
import { docsIndexPages } from "@/lib/docs-index";
@@ -6,7 +7,10 @@ export const revalidate = false;
// llms.txt — curated index of the docs for LLMs (https://llmstxt.org).
// Each entry links to the page's raw Markdown (`*.md`) served by app/llms.md.
export function GET() {
- return new Response(buildLlmsTxt(docsIndexPages()), {
- headers: { "Content-Type": "text/plain; charset=utf-8" },
- });
+ return new Response(
+ buildLlmsTxt(docsIndexPages(), { components: COMPONENT_COUNT }),
+ {
+ headers: { "Content-Type": "text/plain; charset=utf-8" },
+ },
+ );
}
diff --git a/lib/agent-docs.test.ts b/lib/agent-docs.test.ts
index d4e3e69f..1632d14e 100644
--- a/lib/agent-docs.test.ts
+++ b/lib/agent-docs.test.ts
@@ -20,6 +20,46 @@ const PAGES = [
},
];
+describe("buildLlmsTxt with component pages", () => {
+ const text = buildLlmsTxt(
+ [
+ ...PAGES,
+ {
+ slugs: ["transitions"],
+ url: "/docs/transitions",
+ title: "Transitions",
+ description: "Scene transitions",
+ },
+ {
+ slugs: ["transitions", "whip-pan"],
+ url: "/docs/transitions/whip-pan",
+ title: "Whip Pan",
+ component: "whip-pan",
+ },
+ ],
+ { components: 932 },
+ );
+
+ test("states the catalog size once, before the first section", () => {
+ const preamble = text.slice(0, text.indexOf("\n## "));
+ expect(preamble).toContain("Remocn has 932 components across 1 docs sections");
+ });
+
+ test("links a section's whole text instead of each component page", () => {
+ expect(text).not.toContain("whip-pan.md");
+ expect(text).toContain(
+ "- [Transitions](https://remocn.dev/docs/transitions.md): Scene transitions",
+ );
+ expect(text).toContain(
+ "- [Transitions: all 3 pages in one file](https://remocn.dev/llms-full/transitions.txt): 1 component pages",
+ );
+ });
+
+ test("sections without component pages get no corpus link", () => {
+ expect(text).not.toContain("llms-full/getting-started.txt");
+ });
+});
+
describe("buildLlmsTxt", () => {
const text = buildLlmsTxt(PAGES);
const lines = text.split("\n");
diff --git a/lib/agent-docs.ts b/lib/agent-docs.ts
index 2545556e..85043293 100644
--- a/lib/agent-docs.ts
+++ b/lib/agent-docs.ts
@@ -11,8 +11,21 @@ export type IndexPage = {
url: string;
title: string;
description?: string;
+ /** Registry name, when the page documents an installable component. */
+ component?: string;
};
+/** Facts about the catalog, stated once near the top of llms.txt. */
+export type CatalogFacts = {
+ /** Installable registry items, as the site counts them. */
+ components: number;
+};
+
+/** One section of the docs corpus as a single Markdown file. */
+export function sectionCorpusUrl(key: string): string {
+ return `${SITE_URL}/llms-full/${key}.txt`;
+}
+
export const INSTALL_COMMAND = "npx shadcn@latest add @remocn/<name>";
export const MCP_URL = `${SITE_URL}/mcp`;
@@ -51,7 +64,7 @@ const AGENT_TOOLS = [
`- [Component index](${SITE_URL}/llms-components.txt): every installable component with its use / avoid signal, natural length, vibe and dependencies`,
`- [MCP server](${MCP_URL}): remote, read-only, no auth. Search components, read one, plan a video, preview a component`,
`- [OpenAPI spec](${OPENAPI_URL}): the HTTP API (registry items, docs search, GitHub stars video renders)`,
- `- [Full docs corpus](${SITE_URL}/llms-full.txt): every docs page as Markdown in one file`,
+ `- [Full docs corpus](${SITE_URL}/llms-full.txt): every docs page as Markdown in one file. It runs to megabytes, more than a context window holds; prefer one section's file, linked at the end of each section below`,
];
const ABOUT_LINKS = [
@@ -71,17 +84,39 @@ function sectionHeading(key: string): string {
: "Overview";
}
-/** llms.txt (https://llmstxt.org): H1, summary, guidance, then link lists. */
-export function buildLlmsTxt(pages: IndexPage[]): string {
+/**
+ * llms.txt (https://llmstxt.org): H1, summary, guidance, then link lists.
+ * Component pages are not listed one by one: /llms-components.txt has every
+ * one with its use / avoid signal, and each section links its whole text as
+ * one file. Guides, overviews and concept pages are listed.
+ */
+export function buildLlmsTxt(pages: IndexPage[], facts?: CatalogFacts): string {
const lines: string[] = [];
lines.push("# Remocn");
lines.push(
`\n> shadcn registry of production-ready Remotion components — animations, transitions, backgrounds, and full scenes for product demo videos. Install with \`${INSTALL_COMMAND}\`.`,
);
+ // Group pages by their top-level section (first slug segment).
+ const sections = new Map<string, IndexPage[]>();
+ for (const page of pages) {
+ const key = page.slugs[0] ?? "";
+ const bucket = sections.get(key) ?? [];
+ bucket.push(page);
+ sections.set(key, bucket);
+ }
+
+ if (facts) {
+ const componentSections = [...sections.values()].filter((bucket) =>
+ bucket.some((page) => page.component),
+ ).length;
+ lines.push(
+ `\nRemocn has ${facts.components} components across ${componentSections} docs sections, documented on ${pages.length} pages. Everything is MIT-licensed and free; components are copied into the user's project as source files, and Remotion is the only runtime.`,
+ );
+ }
lines.push(
- "\nDocs are also available as raw Markdown: append `.md` to any docs URL, or read [/llms-full.txt](" +
- `${SITE_URL}/llms-full.txt) for the full corpus in a single file.`,
+ "\nDocs are also available as raw Markdown: append `.md` to any docs URL, or read one section as a single file at " +
+ `\`${SITE_URL}/llms-full/<section>.txt\` (linked at the end of each section below).`,
);
lines.push(
`\nBuilding a video? Start at [/llms-components.txt](${SITE_URL}/llms-components.txt) — every` +
@@ -97,21 +132,20 @@ export function buildLlmsTxt(pages: IndexPage[]): string {
lines.push(`\n${WHEN_TO_USE.join("\n")}`);
lines.push(`\n${AGENT_TOOLS.join("\n")}`);
- // Group pages by their top-level section (first slug segment).
- const sections = new Map<string, IndexPage[]>();
- for (const page of pages) {
- const key = page.slugs[0] ?? "";
- const bucket = sections.get(key) ?? [];
- bucket.push(page);
- sections.set(key, bucket);
- }
-
for (const [key, bucket] of sections) {
- lines.push(`\n## ${sectionHeading(key)}`);
+ const heading = sectionHeading(key);
+ lines.push(`\n## ${heading}`);
for (const page of bucket) {
+ if (page.component) continue;
const desc = page.description ? `: ${page.description}` : "";
lines.push(`- [${page.title}](${SITE_URL}${page.url}.md)${desc}`);
}
+ const components = bucket.filter((page) => page.component).length;
+ if (components && key) {
+ lines.push(
+ `- [${heading}: all ${bucket.length} pages in one file](${sectionCorpusUrl(key)}): ${components} component pages with install command, props and usage. Each is also listed in /llms-components.txt`,
+ );
+ }
}
lines.push(`\n${ABOUT_LINKS.join("\n")}`);
diff --git a/lib/docs-index.ts b/lib/docs-index.ts
index a1882fb2..ce2843d0 100644
--- a/lib/docs-index.ts
+++ b/lib/docs-index.ts
@@ -8,5 +8,6 @@ export function docsIndexPages(): IndexPage[] {
url: page.url,
title: page.data.title,
description: page.data.description,
+ component: page.data.component,
}));
}
diff --git a/lib/indexnow.test.ts b/lib/indexnow.test.ts
new file mode 100644
index 00000000..eed560a8
--- /dev/null
+++ b/lib/indexnow.test.ts
@@ -0,0 +1,94 @@
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import {
+ buildIdOf,
+ INDEXNOW_KEY,
+ KEY_LOCATION,
+ MAX_URLS_PER_REQUEST,
+ sitemapUrls,
+ submissionBodies,
+ urlsForChangedFiles,
+} from "./indexnow";
+
+describe("urlsForChangedFiles", () => {
+ test("maps docs, blog, changelog and home page files to their URLs", () => {
+ expect(
+ urlsForChangedFiles([
+ "content/docs/transitions/zoom-blur.mdx",
+ "content/docs/transitions/index.mdx",
+ "content/blog/nextjs-launch-video.mdx",
+ "content/changelog/2026-07-08-transitions-revamp.mdx",
+ "app/(home)/components/landing/sections/hero/prompt-composer.tsx",
+ "app/(home)/about/page.tsx",
+ ]),
+ ).toEqual([
+ "https://remocn.dev/",
+ "https://remocn.dev/about",
+ "https://remocn.dev/blog",
+ "https://remocn.dev/blog/nextjs-launch-video",
+ "https://remocn.dev/changelog",
+ "https://remocn.dev/changelog/video",
+ "https://remocn.dev/docs/transitions",
+ "https://remocn.dev/docs/transitions/zoom-blur",
+ ]);
+ });
+
+ test("ignores files that are not a public page", () => {
+ expect(
+ urlsForChangedFiles([
+ "lib/utils.ts",
+ "registry/remocn/zoom-blur/zoom-blur.tsx",
+ "content/docs/meta.json",
+ "app/(home)/blog/[slug]/page.tsx",
+ ]),
+ ).toEqual([]);
+ });
+});
+
+describe("submission", () => {
+ test("the key file in public/ serves the key", () => {
+ expect(readFileSync(`public/${INDEXNOW_KEY}.txt`, "utf8")).toBe(
+ INDEXNOW_KEY,
+ );
+ expect(KEY_LOCATION).toBe(`https://remocn.dev/${INDEXNOW_KEY}.txt`);
+ });
+
+ test("splits at the per-request limit", () => {
+ const urls = Array.from(
+ { length: MAX_URLS_PER_REQUEST + 1 },
+ (_, i) => `https://remocn.dev/${i}`,
+ );
+ const bodies = submissionBodies(urls);
+ expect(bodies.map((body) => body.urlList.length)).toEqual([
+ MAX_URLS_PER_REQUEST,
+ 1,
+ ]);
+ expect(bodies[0]).toMatchObject({
+ host: "remocn.dev",
+ key: INDEXNOW_KEY,
+ keyLocation: KEY_LOCATION,
+ });
+ });
+
+ test("reads <loc> entries from a sitemap", () => {
+ expect(
+ sitemapUrls(
+ "<urlset><url><loc>https://remocn.dev</loc></url><url><loc>https://remocn.dev/blog</loc></url></urlset>",
+ ),
+ ).toEqual(["https://remocn.dev", "https://remocn.dev/blog"]);
+ });
+});
+
+describe("buildIdOf", () => {
+ test("finds the build id in the escaped RSC payload", () => {
+ expect(
+ buildIdOf(
+ 'self.__next_f.push([1,"0:{\\"P\\":null,\\"b\\":\\"CpSmtBmP7Mx\\",',
+ ),
+ ).toBe("CpSmtBmP7Mx");
+ });
+
+ test("returns null when the page has none", () => {
+ expect(buildIdOf("<html></html>")).toBeNull();
+ });
+});
diff --git a/lib/indexnow.ts b/lib/indexnow.ts
new file mode 100644
index 00000000..8839d447
--- /dev/null
+++ b/lib/indexnow.ts
@@ -0,0 +1,79 @@
+/**
+ * IndexNow (https://www.indexnow.org): tell Bing, Yandex, Naver, Seznam and
+ * the other participating engines which URLs changed, instead of waiting for
+ * a recrawl. Bing's index feeds ChatGPT search and Copilot, so a page edited
+ * here reaches AI answers sooner. The key is public by design: the engines
+ * check it at KEY_LOCATION to confirm the host owns the submission.
+ */
+
+export const SITE_HOST = "remocn.dev";
+const SITE_URL = `https://${SITE_HOST}`;
+
+export const INDEXNOW_KEY = "7a075132af7cd50bca9fa007c1b92d23";
+export const KEY_LOCATION = `${SITE_URL}/${INDEXNOW_KEY}.txt`;
+export const INDEXNOW_ENDPOINT = "https://api.indexnow.org/indexnow";
+
+/** IndexNow takes at most 10,000 URLs per request. */
+export const MAX_URLS_PER_REQUEST = 10_000;
+
+/**
+ * Public URLs whose content a change to these repo files alters. Deleted
+ * pages are included on purpose: IndexNow is also how an engine learns that a
+ * URL now answers 404. Files that map to no page are ignored.
+ */
+export function urlsForChangedFiles(paths: string[]): string[] {
+ const urls = new Set<string>();
+ for (const path of paths) {
+ for (const url of urlsFor(path)) urls.add(`${SITE_URL}${url}`);
+ }
+ return [...urls].sort();
+}
+
+function urlsFor(path: string): string[] {
+ const doc = /^content\/docs\/(.+)\.mdx$/.exec(path);
+ if (doc) {
+ const slug = doc[1].replace(/(^|\/)index$/, "");
+ return [slug ? `/docs/${slug}` : "/docs"];
+ }
+ const post = /^content\/blog\/([^/]+)\.mdx$/.exec(path);
+ if (post) return [`/blog/${post[1]}`, "/blog"];
+ if (/^content\/changelog\/[^/]+\.mdx$/.test(path)) {
+ return ["/changelog", "/changelog/video"];
+ }
+ if (
+ path === "app/(home)/page.tsx" ||
+ path.startsWith("app/(home)/components/landing/")
+ ) {
+ return ["/"];
+ }
+ const page = /^app\/\(home\)\/([a-z-]+)\/page\.tsx$/.exec(path);
+ if (page) return [`/${page[1]}`];
+ return [];
+}
+
+/** The <loc> entries of a sitemap.xml. */
+export function sitemapUrls(xml: string): string[] {
+ return [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) => match[1]);
+}
+
+/** Request bodies for the IndexNow API, split at the per-request limit. */
+export function submissionBodies(urls: string[]) {
+ const bodies = [];
+ for (let i = 0; i < urls.length; i += MAX_URLS_PER_REQUEST) {
+ bodies.push({
+ host: SITE_HOST,
+ key: INDEXNOW_KEY,
+ keyLocation: KEY_LOCATION,
+ urlList: urls.slice(i, i + MAX_URLS_PER_REQUEST),
+ });
+ }
+ return bodies;
+}
+
+/**
+ * The Next build id in a page's RSC payload (`"b":"<id>"`). It changes with
+ * every build, so a new value means the new deploy is live.
+ */
+export function buildIdOf(html: string): string | null {
+ return /\\?"b\\?":\\?"([A-Za-z0-9_-]+)/.exec(html)?.[1] ?? null;
+}
diff --git a/lib/llms-corpus.ts b/lib/llms-corpus.ts
new file mode 100644
index 00000000..0151a4d3
--- /dev/null
+++ b/lib/llms-corpus.ts
@@ -0,0 +1,16 @@
+import { getLLMText } from "@/lib/get-llm-text";
+import type { source } from "@/source";
+
+type DocPage = ReturnType<typeof source.getPages>[number];
+
+/** Docs pages as one Markdown file, separated by horizontal rules. */
+export async function corpusResponse(pages: DocPage[]): Promise<Response> {
+ const docs = await Promise.all(pages.map((page) => getLLMText(page)));
+ return new Response(docs.join("\n\n---\n\n"), {
+ headers: {
+ "Content-Type": "text/plain; charset=utf-8",
+ // The same text as the docs pages; keep search on the HTML.
+ "X-Robots-Tag": "noindex",
+ },
+ });
+}
diff --git a/lib/structured-data.test.ts b/lib/structured-data.test.ts
index b94be7d9..765fedfa 100644
--- a/lib/structured-data.test.ts
+++ b/lib/structured-data.test.ts
@@ -209,3 +209,18 @@ describe("video JSON-LD", () => {
});
});
});
+
+describe("home JSON-LD entity links", () => {
+ test("the Organization names its maintainer and npm package", () => {
+ const org = (homeJsonLd("x")["@graph"] as Node[]).find(
+ (n) => n["@type"] === "Organization",
+ );
+ expect(org?.sameAs).toContain(
+ "https://www.npmjs.com/package/@remocn/render-sdk",
+ );
+ expect(org?.founder).toMatchObject({
+ "@type": "Person",
+ url: "https://github.com/kapishdima",
+ });
+ });
+});
diff --git a/lib/structured-data.ts b/lib/structured-data.ts
index 3b800dad..fb4053f6 100644
--- a/lib/structured-data.ts
+++ b/lib/structured-data.ts
@@ -10,6 +10,13 @@ const SITE_URL = "https://remocn.dev";
const ORG_ID = `${SITE_URL}/#organization`;
+/** The maintainer (/about names them), by their public profiles. */
+const MAINTAINER_GITHUB = "https://github.com/kapishdima";
+const MAINTAINER_X = "https://x.com/kapish_dima";
+
+/** The npm package Remocn publishes (the render SDK). */
+const NPM_URL = "https://www.npmjs.com/package/@remocn/render-sdk";
+
/**
* Product terms people search together with the brand. Ora and other agents
* read the first ones as "what Remocn is", so the most specific term leads,
@@ -41,7 +48,13 @@ export function homeJsonLd(description: string) {
email: SUPPORT_EMAIL,
description:
"Remocn builds open-source video components for Remotion and the tools to make product demo videos with an AI agent.",
- sameAs: [GITHUB_URL, X_URL, THREADS_URL],
+ sameAs: [GITHUB_URL, X_URL, THREADS_URL, NPM_URL],
+ founder: {
+ "@type": "Person",
+ name: "KapishDima",
+ url: MAINTAINER_GITHUB,
+ sameAs: [MAINTAINER_GITHUB, MAINTAINER_X],
+ },
knowsAbout: ["Remotion", "React", "Motion design", "shadcn/ui"],
contactPoint: [
{
diff --git a/public/7a075132af7cd50bca9fa007c1b92d23.txt b/public/7a075132af7cd50bca9fa007c1b92d23.txt
new file mode 100644
index 00000000..1377bcb0
--- /dev/null
+++ b/public/7a075132af7cd50bca9fa007c1b92d23.txt
@@ -0,0 +1 @@
+7a075132af7cd50bca9fa007c1b92d23
\ No newline at end of file
diff --git a/scripts/indexnow.mts b/scripts/indexnow.mts
new file mode 100644
index 00000000..b4fe57d4
--- /dev/null
+++ b/scripts/indexnow.mts
@@ -0,0 +1,109 @@
+/**
+ * Submit changed URLs to IndexNow (see lib/indexnow.ts).
+ *
+ * bun scripts/indexnow.mts --changed <base-sha> <head-sha> URLs the diff touches
+ * bun scripts/indexnow.mts --all every URL in the live sitemap
+ *
+ * Flags:
+ * --wait-deploy first wait (up to 40 min) until the live site serves a new
+ * build, so engines crawl the new content, not the old one
+ * --dry-run print the URLs, submit nothing
+ */
+import { execFileSync } from "node:child_process";
+import {
+ buildIdOf,
+ INDEXNOW_ENDPOINT,
+ INDEXNOW_KEY,
+ KEY_LOCATION,
+ SITE_HOST,
+ sitemapUrls,
+ submissionBodies,
+ urlsForChangedFiles,
+} from "../lib/indexnow";
+
+const SITE_URL = `https://${SITE_HOST}`;
+const POLL_MS = 30_000;
+const WAIT_LIMIT_MS = 40 * 60_000;
+
+const args = process.argv.slice(2);
+const dryRun = args.includes("--dry-run");
+
+async function fetchText(url: string): Promise<string> {
+ const res = await fetch(url, { headers: { "Cache-Control": "no-cache" } });
+ if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);
+ return res.text();
+}
+
+/** Build id of the live site; a page outside the CDN-cached docs is enough. */
+async function liveBuildId(): Promise<string | null> {
+ return buildIdOf(await fetchText(`${SITE_URL}/about?indexnow=${Date.now()}`));
+}
+
+async function waitForDeploy(): Promise<void> {
+ const before = await liveBuildId();
+ console.log(`Live build ${before ?? "unknown"}; waiting for the next one`);
+ const deadline = Date.now() + WAIT_LIMIT_MS;
+ while (Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+ const now = await liveBuildId().catch(() => null);
+ if (now && now !== before) {
+ console.log(`New build ${now} is live`);
+ return;
+ }
+ }
+ console.warn("No new build after 40 minutes; submitting anyway");
+}
+
+function changedFiles(base: string, head: string): string[] {
+ return execFileSync(
+ "git",
+ ["diff", "--name-only", "--no-renames", base, head],
+ { encoding: "utf8" },
+ )
+ .split("\n")
+ .filter(Boolean);
+}
+
+async function main() {
+ let urls: string[];
+ if (args.includes("--all")) {
+ urls = sitemapUrls(await fetchText(`${SITE_URL}/sitemap.xml`));
+ } else {
+ const at = args.indexOf("--changed");
+ const [base, head] = [args[at + 1], args[at + 2]];
+ if (at === -1 || !base || !head) {
+ console.error("Usage: --changed <base> <head> | --all");
+ process.exit(2);
+ }
+ urls = urlsForChangedFiles(changedFiles(base, head));
+ }
+
+ if (urls.length === 0) {
+ console.log("No public URLs changed");
+ return;
+ }
+ console.log(`${urls.length} URL(s):\n${urls.join("\n")}`);
+ if (dryRun) return;
+
+ if (args.includes("--wait-deploy")) await waitForDeploy();
+
+ const key = (await fetchText(KEY_LOCATION)).trim();
+ if (key !== INDEXNOW_KEY) {
+ throw new Error(`${KEY_LOCATION} does not serve the IndexNow key`);
+ }
+
+ for (const body of submissionBodies(urls)) {
+ const res = await fetch(INDEXNOW_ENDPOINT, {
+ method: "POST",
+ headers: { "Content-Type": "application/json; charset=utf-8" },
+ body: JSON.stringify(body),
+ });
+ // 200 OK and 202 Accepted both mean the submission was taken.
+ if (res.status !== 200 && res.status !== 202) {
+ throw new Error(`IndexNow: HTTP ${res.status} ${await res.text()}`);
+ }
+ console.log(`IndexNow: ${res.status} for ${body.urlList.length} URL(s)`);
+ }
+}
+
+await main();