An open source design system that's fully customizable and agent ready
TypeScript
feat(docsite): add dynamic sitemap.xml and robots.txt (#3087)
Joey9ad489c40d7f
diff --git a/apps/docsite/src/__tests__/sitemap.test.ts b/apps/docsite/src/__tests__/sitemap.test.tsnew file mode 100644index 0000000000..6a5203affe--- /dev/null+++ b/apps/docsite/src/__tests__/sitemap.test.ts@@ -0,0 +1,72 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Sitemap tests for the docsite.+ *+ * Validates that the dynamic sitemap covers the static pages plus every+ * generated dynamic route (components, doc topics, templates, blog posts),+ * uses absolute URLs, and stays free of duplicates.+ *+ * Run: pnpm -F @astryxdesign/docsite test+ */++import {describe, it, expect} from 'vitest';+import sitemap from '../app/sitemap';+import {SITE_URL} from '../lib/siteConfig';+import {flattenComponentSidebarEntries} from '../components/componentSidebarData';+import {docTopics} from '../generated/docsRegistry';+import {packages} from '../generated/packageRegistry';+import {templates} from '../generated/templateRegistry';+import {blogPosts} from '../generated/blogRegistry';++const entries = sitemap();+const urls = entries.map(e => e.url);++describe('docsite sitemap', () => {+ it('emits only absolute URLs on the configured origin', () => {+ for (const u of urls) {+ expect(u.startsWith(`${SITE_URL}/`) || u === `${SITE_URL}/`).toBe(true);+ }+ });++ it('has no duplicate URLs', () => {+ expect(new Set(urls).size).toBe(urls.length);+ });++ it('includes the primary static pages', () => {+ for (const path of ['/', '/components', '/docs', '/templates', '/blog']) {+ expect(urls).toContain(new URL(path, SITE_URL).toString());+ }+ });++ it('includes every component detail page', () => {+ for (const c of flattenComponentSidebarEntries()) {+ expect(urls).toContain(+ new URL(`/components/${c.name}`, SITE_URL).toString(),+ );+ }+ });++ it('includes every doc topic and non-theme package page', () => {+ const topics = [+ ...docTopics.map(d => d.topic),+ ...packages+ .filter(p => !p.name.includes('theme-'))+ .map(p => p.name.replace('@astryxdesign/', '')),+ ];+ for (const topic of topics) {+ expect(urls).toContain(new URL(`/docs/${topic}`, SITE_URL).toString());+ }+ });++ it('includes every template and blog post', () => {+ for (const t of templates) {+ expect(urls).toContain(+ new URL(`/templates/${t.slug}`, SITE_URL).toString(),+ );+ }+ for (const p of blogPosts) {+ expect(urls).toContain(new URL(`/blog/${p.slug}`, SITE_URL).toString());+ }+ });+});diff --git a/apps/docsite/src/app/layout.tsx b/apps/docsite/src/app/layout.tsxindex e57ae5c522..f49df14581 100644--- a/apps/docsite/src/app/layout.tsx+++ b/apps/docsite/src/app/layout.tsx@@ -4,6 +4,16 @@ import type {Metadata} from 'next';import {Analytics} from '@vercel/analytics/react';import './globals.css';import {Providers} from './providers';+// Public origin and identity live in lib/siteConfig so the sitemap and+// metadata stay in sync. metadataBase resolves relative OG/Twitter image+// paths to absolute URLs — social scrapers (Facebook, X, LinkedIn, Slack,+// iMessage) require absolute image URLs or the card image is dropped.+import {+ SITE_URL,+ SITE_NAME,+ SITE_TITLE,+ SITE_DESCRIPTION,+} from '../lib/siteConfig';// Note: the Astryx theme (src/themes/astryxTheme.ts) is Figtree-first.// We can't use next/font/google here (it requires SWC, but this app pins@@ -12,19 +22,6 @@ import {Providers} from './providers';// loads via the shared Google Fonts <link> in <head> below, which is the// "Good" path from the theming wiki §Font Declarations.-// Public origin for the docsite. Set NEXT_PUBLIC_SITE_URL in other-// environments (e.g. preview deploys); defaults to the production domain so-// metadataBase resolves relative OG/Twitter image paths to absolute URLs —-// social scrapers (Facebook, X, LinkedIn, Slack, iMessage) require absolute-// image URLs or the card image is dropped.-const SITE_URL =- process.env.NEXT_PUBLIC_SITE_URL ?? 'https://astryx.atmeta.com';--const SITE_NAME = 'Astryx';-const SITE_TITLE = 'Astryx Design System';-const SITE_DESCRIPTION =- 'An open source design system that is fully customizable and agent ready.';-// Default social card image: the launch banner that the announcement blog post// uses for its cover. Reusing the same branded banner keeps the shared-link// preview consistent with the launch creative across the site and the post.diff --git a/apps/docsite/src/app/robots.ts b/apps/docsite/src/app/robots.tsnew file mode 100644index 0000000000..ce4b54fa49--- /dev/null+++ b/apps/docsite/src/app/robots.ts@@ -0,0 +1,26 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file robots.ts+ *+ * Next.js serves this at /robots.txt. It allows full crawling and advertises+ * the dynamic sitemap so search engines can discover every page the docsite+ * generates. Uses the same SITE_URL origin as the sitemap and metadata.+ *+ * @output MetadataRoute.Robots consumed by Next.js to emit /robots.txt+ */++import type {MetadataRoute} from 'next';+import {SITE_URL} from '../lib/siteConfig';++export default function robots(): MetadataRoute.Robots {+ return {+ rules: {+ userAgent: '*',+ allow: '/',+ // The playground preview frame is a render target, not indexable content.+ disallow: '/playground/preview',+ },+ sitemap: new URL('/sitemap.xml', SITE_URL).toString(),+ };+}diff --git a/apps/docsite/src/app/sitemap.ts b/apps/docsite/src/app/sitemap.tsnew file mode 100644index 0000000000..adae524299--- /dev/null+++ b/apps/docsite/src/app/sitemap.ts@@ -0,0 +1,95 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file sitemap.ts+ *+ * Dynamic sitemap for the docsite. Next.js serves this at /sitemap.xml and+ * regenerates it on every build, so the URL set tracks the same generated+ * registries that drive the routes themselves — add a component, doc topic,+ * template, or blog post and it appears in the sitemap with no manual edit.+ *+ * Mirrors the `generateStaticParams` of each dynamic route so the sitemap and+ * the actual rendered pages never drift:+ * - /components/[name] ← flattenComponentSidebarEntries()+ * - /docs/[topic] ← docTopics + non-theme packages+ * - /templates/[slug] ← templates+ * - /blog/[slug] ← blogPosts+ *+ * @output MetadataRoute.Sitemap consumed by Next.js to emit /sitemap.xml+ */++import type {MetadataRoute} from 'next';+import {SITE_URL} from '../lib/siteConfig';+import {flattenComponentSidebarEntries} from '../components/componentSidebarData';+import {docTopics} from '../generated/docsRegistry';+import {packages} from '../generated/packageRegistry';+import {templates} from '../generated/templateRegistry';+import {blogPosts} from '../generated/blogRegistry';++// Theme packages don't get a /docs/[topic] reference page (see the docs route's+// own filter); keep the sitemap aligned with what actually renders.+function isThemePackage(name: string): boolean {+ return name.includes('theme-');+}++function url(path: string): string {+ return new URL(path, SITE_URL).toString();+}++export default function sitemap(): MetadataRoute.Sitemap {+ const now = new Date();++ // Static, hand-authored pages. `priority` is a relative hint to crawlers;+ // the home page and primary galleries rank highest.+ const staticEntries: MetadataRoute.Sitemap = [+ {url: url('/'), changeFrequency: 'weekly', priority: 1},+ {url: url('/components'), changeFrequency: 'weekly', priority: 0.9},+ {url: url('/docs'), changeFrequency: 'weekly', priority: 0.9},+ {url: url('/templates'), changeFrequency: 'weekly', priority: 0.8},+ {url: url('/themes'), changeFrequency: 'weekly', priority: 0.8},+ {url: url('/blog'), changeFrequency: 'weekly', priority: 0.8},+ {url: url('/changelog'), changeFrequency: 'weekly', priority: 0.6},+ {url: url('/community'), changeFrequency: 'monthly', priority: 0.5},+ {url: url('/playground'), changeFrequency: 'monthly', priority: 0.6},+ ];++ const componentEntries: MetadataRoute.Sitemap =+ flattenComponentSidebarEntries().map(c => ({+ url: url(`/components/${c.name}`),+ changeFrequency: 'weekly',+ priority: 0.7,+ }));++ const docTopicEntries: MetadataRoute.Sitemap = [+ ...docTopics.map(d => d.topic),+ ...packages+ .filter(p => !isThemePackage(p.name))+ .map(p => p.name.replace('@astryxdesign/', '')),+ ].map(topic => ({+ url: url(`/docs/${topic}`),+ changeFrequency: 'weekly' as const,+ priority: 0.7,+ }));++ const templateEntries: MetadataRoute.Sitemap = templates.map(t => ({+ url: url(`/templates/${t.slug}`),+ changeFrequency: 'monthly',+ priority: 0.6,+ }));++ const blogEntries: MetadataRoute.Sitemap = blogPosts.map(p => ({+ url: url(`/blog/${p.slug}`),+ // Use the post's own publish date as lastModified when present.+ lastModified: p.date ? new Date(p.date) : now,+ changeFrequency: 'monthly',+ priority: 0.6,+ }));++ return [+ ...staticEntries,+ ...componentEntries,+ ...docTopicEntries,+ ...templateEntries,+ ...blogEntries,+ ];+}diff --git a/apps/docsite/src/lib/siteConfig.ts b/apps/docsite/src/lib/siteConfig.tsnew file mode 100644index 0000000000..5dadef89b5--- /dev/null+++ b/apps/docsite/src/lib/siteConfig.ts@@ -0,0 +1,21 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file siteConfig.ts+ *+ * Single source of truth for the docsite's public origin and identity.+ * Consumed by the root layout (metadataBase, OG/Twitter) and the sitemap so+ * absolute URLs stay consistent across metadata and crawler surfaces.+ *+ * Override SITE_URL per environment (e.g. preview deploys) via+ * NEXT_PUBLIC_SITE_URL; it defaults to the production domain so social+ * scrapers resolve relative OG image paths to absolute URLs.+ */++export const SITE_URL =+ process.env.NEXT_PUBLIC_SITE_URL ?? 'https://astryx.atmeta.com';++export const SITE_NAME = 'Astryx';+export const SITE_TITLE = 'Astryx Design System';+export const SITE_DESCRIPTION =+ 'An open source design system that is fully customizable and agent ready.';