Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(docsite): add dynamic sitemap.xml and robots.txt (#3087)

Joey9ad489c40d7f
diff --git a/apps/docsite/src/__tests__/sitemap.test.ts b/apps/docsite/src/__tests__/sitemap.test.ts
new file mode 100644
index 0000000000..6a5203affe
--- /dev/null
+++ b/apps/docsite/src/__tests__/sitemap.test.ts
@@ -0,0 +1,72 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Sitemap tests for the docsite.
+ *
+ * Validates that the dynamic sitemap covers the static pages plus every
+ * generated dynamic route (components, doc topics, templates, blog posts),
+ * uses absolute URLs, and stays free of duplicates.
+ *
+ * Run: pnpm -F @astryxdesign/docsite test
+ */
+
+import {describe, it, expect} from 'vitest';
+import sitemap from '../app/sitemap';
+import {SITE_URL} from '../lib/siteConfig';
+import {flattenComponentSidebarEntries} from '../components/componentSidebarData';
+import {docTopics} from '../generated/docsRegistry';
+import {packages} from '../generated/packageRegistry';
+import {templates} from '../generated/templateRegistry';
+import {blogPosts} from '../generated/blogRegistry';
+
+const entries = sitemap();
+const urls = entries.map(e => e.url);
+
+describe('docsite sitemap', () => {
+ it('emits only absolute URLs on the configured origin', () => {
+ for (const u of urls) {
+ expect(u.startsWith(`${SITE_URL}/`) || u === `${SITE_URL}/`).toBe(true);
+ }
+ });
+
+ it('has no duplicate URLs', () => {
+ expect(new Set(urls).size).toBe(urls.length);
+ });
+
+ it('includes the primary static pages', () => {
+ for (const path of ['/', '/components', '/docs', '/templates', '/blog']) {
+ expect(urls).toContain(new URL(path, SITE_URL).toString());
+ }
+ });
+
+ it('includes every component detail page', () => {
+ for (const c of flattenComponentSidebarEntries()) {
+ expect(urls).toContain(
+ new URL(`/components/${c.name}`, SITE_URL).toString(),
+ );
+ }
+ });
+
+ it('includes every doc topic and non-theme package page', () => {
+ const topics = [
+ ...docTopics.map(d => d.topic),
+ ...packages
+ .filter(p => !p.name.includes('theme-'))
+ .map(p => p.name.replace('@astryxdesign/', '')),
+ ];
+ for (const topic of topics) {
+ expect(urls).toContain(new URL(`/docs/${topic}`, SITE_URL).toString());
+ }
+ });
+
+ it('includes every template and blog post', () => {
+ for (const t of templates) {
+ expect(urls).toContain(
+ new URL(`/templates/${t.slug}`, SITE_URL).toString(),
+ );
+ }
+ for (const p of blogPosts) {
+ expect(urls).toContain(new URL(`/blog/${p.slug}`, SITE_URL).toString());
+ }
+ });
+});
diff --git a/apps/docsite/src/app/layout.tsx b/apps/docsite/src/app/layout.tsx
index e57ae5c522..f49df14581 100644
--- a/apps/docsite/src/app/layout.tsx
+++ b/apps/docsite/src/app/layout.tsx
@@ -4,6 +4,16 @@ import type {Metadata} from 'next';
import {Analytics} from '@vercel/analytics/react';
import './globals.css';
import {Providers} from './providers';
+// Public origin and identity live in lib/siteConfig so the sitemap and
+// metadata stay in sync. metadataBase resolves relative OG/Twitter image
+// paths to absolute URLs — social scrapers (Facebook, X, LinkedIn, Slack,
+// iMessage) require absolute image URLs or the card image is dropped.
+import {
+ SITE_URL,
+ SITE_NAME,
+ SITE_TITLE,
+ SITE_DESCRIPTION,
+} from '../lib/siteConfig';
// Note: the Astryx theme (src/themes/astryxTheme.ts) is Figtree-first.
// We can't use next/font/google here (it requires SWC, but this app pins
@@ -12,19 +22,6 @@ import {Providers} from './providers';
// loads via the shared Google Fonts <link> in <head> below, which is the
// "Good" path from the theming wiki §Font Declarations.
-// Public origin for the docsite. Set NEXT_PUBLIC_SITE_URL in other
-// environments (e.g. preview deploys); defaults to the production domain so
-// metadataBase resolves relative OG/Twitter image paths to absolute URLs —
-// social scrapers (Facebook, X, LinkedIn, Slack, iMessage) require absolute
-// image URLs or the card image is dropped.
-const SITE_URL =
- process.env.NEXT_PUBLIC_SITE_URL ?? 'https://astryx.atmeta.com';
-
-const SITE_NAME = 'Astryx';
-const SITE_TITLE = 'Astryx Design System';
-const SITE_DESCRIPTION =
- 'An open source design system that is fully customizable and agent ready.';
-
// Default social card image: the launch banner that the announcement blog post
// uses for its cover. Reusing the same branded banner keeps the shared-link
// preview consistent with the launch creative across the site and the post.
diff --git a/apps/docsite/src/app/robots.ts b/apps/docsite/src/app/robots.ts
new file mode 100644
index 0000000000..ce4b54fa49
--- /dev/null
+++ b/apps/docsite/src/app/robots.ts
@@ -0,0 +1,26 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file robots.ts
+ *
+ * Next.js serves this at /robots.txt. It allows full crawling and advertises
+ * the dynamic sitemap so search engines can discover every page the docsite
+ * generates. Uses the same SITE_URL origin as the sitemap and metadata.
+ *
+ * @output MetadataRoute.Robots consumed by Next.js to emit /robots.txt
+ */
+
+import type {MetadataRoute} from 'next';
+import {SITE_URL} from '../lib/siteConfig';
+
+export default function robots(): MetadataRoute.Robots {
+ return {
+ rules: {
+ userAgent: '*',
+ allow: '/',
+ // The playground preview frame is a render target, not indexable content.
+ disallow: '/playground/preview',
+ },
+ sitemap: new URL('/sitemap.xml', SITE_URL).toString(),
+ };
+}
diff --git a/apps/docsite/src/app/sitemap.ts b/apps/docsite/src/app/sitemap.ts
new file mode 100644
index 0000000000..adae524299
--- /dev/null
+++ b/apps/docsite/src/app/sitemap.ts
@@ -0,0 +1,95 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file sitemap.ts
+ *
+ * Dynamic sitemap for the docsite. Next.js serves this at /sitemap.xml and
+ * regenerates it on every build, so the URL set tracks the same generated
+ * registries that drive the routes themselves — add a component, doc topic,
+ * template, or blog post and it appears in the sitemap with no manual edit.
+ *
+ * Mirrors the `generateStaticParams` of each dynamic route so the sitemap and
+ * the actual rendered pages never drift:
+ * - /components/[name] ← flattenComponentSidebarEntries()
+ * - /docs/[topic] ← docTopics + non-theme packages
+ * - /templates/[slug] ← templates
+ * - /blog/[slug] ← blogPosts
+ *
+ * @output MetadataRoute.Sitemap consumed by Next.js to emit /sitemap.xml
+ */
+
+import type {MetadataRoute} from 'next';
+import {SITE_URL} from '../lib/siteConfig';
+import {flattenComponentSidebarEntries} from '../components/componentSidebarData';
+import {docTopics} from '../generated/docsRegistry';
+import {packages} from '../generated/packageRegistry';
+import {templates} from '../generated/templateRegistry';
+import {blogPosts} from '../generated/blogRegistry';
+
+// Theme packages don't get a /docs/[topic] reference page (see the docs route's
+// own filter); keep the sitemap aligned with what actually renders.
+function isThemePackage(name: string): boolean {
+ return name.includes('theme-');
+}
+
+function url(path: string): string {
+ return new URL(path, SITE_URL).toString();
+}
+
+export default function sitemap(): MetadataRoute.Sitemap {
+ const now = new Date();
+
+ // Static, hand-authored pages. `priority` is a relative hint to crawlers;
+ // the home page and primary galleries rank highest.
+ const staticEntries: MetadataRoute.Sitemap = [
+ {url: url('/'), changeFrequency: 'weekly', priority: 1},
+ {url: url('/components'), changeFrequency: 'weekly', priority: 0.9},
+ {url: url('/docs'), changeFrequency: 'weekly', priority: 0.9},
+ {url: url('/templates'), changeFrequency: 'weekly', priority: 0.8},
+ {url: url('/themes'), changeFrequency: 'weekly', priority: 0.8},
+ {url: url('/blog'), changeFrequency: 'weekly', priority: 0.8},
+ {url: url('/changelog'), changeFrequency: 'weekly', priority: 0.6},
+ {url: url('/community'), changeFrequency: 'monthly', priority: 0.5},
+ {url: url('/playground'), changeFrequency: 'monthly', priority: 0.6},
+ ];
+
+ const componentEntries: MetadataRoute.Sitemap =
+ flattenComponentSidebarEntries().map(c => ({
+ url: url(`/components/${c.name}`),
+ changeFrequency: 'weekly',
+ priority: 0.7,
+ }));
+
+ const docTopicEntries: MetadataRoute.Sitemap = [
+ ...docTopics.map(d => d.topic),
+ ...packages
+ .filter(p => !isThemePackage(p.name))
+ .map(p => p.name.replace('@astryxdesign/', '')),
+ ].map(topic => ({
+ url: url(`/docs/${topic}`),
+ changeFrequency: 'weekly' as const,
+ priority: 0.7,
+ }));
+
+ const templateEntries: MetadataRoute.Sitemap = templates.map(t => ({
+ url: url(`/templates/${t.slug}`),
+ changeFrequency: 'monthly',
+ priority: 0.6,
+ }));
+
+ const blogEntries: MetadataRoute.Sitemap = blogPosts.map(p => ({
+ url: url(`/blog/${p.slug}`),
+ // Use the post's own publish date as lastModified when present.
+ lastModified: p.date ? new Date(p.date) : now,
+ changeFrequency: 'monthly',
+ priority: 0.6,
+ }));
+
+ return [
+ ...staticEntries,
+ ...componentEntries,
+ ...docTopicEntries,
+ ...templateEntries,
+ ...blogEntries,
+ ];
+}
diff --git a/apps/docsite/src/lib/siteConfig.ts b/apps/docsite/src/lib/siteConfig.ts
new file mode 100644
index 0000000000..5dadef89b5
--- /dev/null
+++ b/apps/docsite/src/lib/siteConfig.ts
@@ -0,0 +1,21 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file siteConfig.ts
+ *
+ * Single source of truth for the docsite's public origin and identity.
+ * Consumed by the root layout (metadataBase, OG/Twitter) and the sitemap so
+ * absolute URLs stay consistent across metadata and crawler surfaces.
+ *
+ * Override SITE_URL per environment (e.g. preview deploys) via
+ * NEXT_PUBLIC_SITE_URL; it defaults to the production domain so social
+ * scrapers resolve relative OG image paths to absolute URLs.
+ */
+
+export const SITE_URL =
+ process.env.NEXT_PUBLIC_SITE_URL ?? 'https://astryx.atmeta.com';
+
+export const SITE_NAME = 'Astryx';
+export const SITE_TITLE = 'Astryx Design System';
+export const SITE_DESCRIPTION =
+ 'An open source design system that is fully customizable and agent ready.';