Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
feat(ok): LLM-friendly web surfaces for docs and marketing (#4060)
Nick Gomez8f61ba6b88bc
diff --git a/.github/scripts/share-contract-cache-shape.test.mjs b/.github/scripts/share-contract-cache-shape.test.mjsindex 45b220f53..ec2642723 100644--- a/.github/scripts/share-contract-cache-shape.test.mjs+++ b/.github/scripts/share-contract-cache-shape.test.mjs@@ -18,6 +18,13 @@ describe('share contract corpus cache wiring', () => {const turbo = JSON.parse(readFileSync(join(REPO_ROOT, 'turbo.json'), 'utf8'));const docsBuild = turbo.tasks['@inkeep/open-knowledge-docs#build'];- expect(docsBuild.env).toEqual(['VERCEL_GIT_COMMIT_SHA', 'GITHUB_SHA']);+ expect(docsBuild.env).toEqual([+ 'VERCEL_GIT_COMMIT_SHA',+ 'GITHUB_SHA',+ 'VERCEL_ENV',+ 'OK_ALLOW_BLOGLESS_LLMS_TXT',+ 'CI',+ 'VERCEL',+ ]);});});diff --git a/biome.jsonc b/biome.jsoncindex 6aa71d903..57a66e880 100644--- a/biome.jsonc+++ b/biome.jsonc@@ -444,6 +444,9 @@"!packages/native-config/target","!**/.next","!**/.source",+ // Second generated content build, emitted by the docs app's real-source+ // vitest config in the Vite flavour `.source` must never hold.+ "!**/.source-vitest","!**/.turbo","!**/tmp","!**/test-results",diff --git a/docs/.gitignore b/docs/.gitignoreindex 2344cdbab..05fdc4bb7 100644--- a/docs/.gitignore+++ b/docs/.gitignore@@ -1,5 +1,8 @@.next/.source/+# Second content build, emitted by vitest.real-source.config.mts in the Vite+# flavour that `.source/` must never hold.+.source-vitest/.vercel.env.env.localdiff --git a/docs/next.config.ts b/docs/next.config.tsindex 2c68859a0..f8e2f641b 100644--- a/docs/next.config.ts+++ b/docs/next.config.ts@@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url';import { withMicrofrontends } from '@vercel/microfrontends/next/config';import { createMDX } from 'fumadocs-mdx/next';import type { NextConfig } from 'next';+import { markdownRewrites } from './src/lib/markdown-routes';const nextConfig: NextConfig = {reactStrictMode: true,@@ -15,16 +16,7 @@ const nextConfig: NextConfig = {skipTrailingSlashRedirect: true,async rewrites() {return {- beforeFiles: [- {- source: '/docs/:path*.md',- destination: '/llms.mdx/:path*',- },- {- source: '/docs/:path*.mdx',- destination: '/llms.mdx/:path*',- },- ],+ beforeFiles: markdownRewrites(),afterFiles: [{source: '/ingest/static/:path*',diff --git a/docs/package.json b/docs/package.jsonindex 5416e0950..9e99fc72f 100644--- a/docs/package.json+++ b/docs/package.json@@ -9,7 +9,8 @@"build": "next build","prebuild": "pnpm run validate-link","start": "next start --port 3010",- "test": "vitest run src",+ "test": "vitest run src && pnpm run test:real-source",+ "test:real-source": "vitest run --config vitest.real-source.config.mts","typecheck": "next typegen && fumadocs-mdx && tsc --noEmit","validate-link": "tsx scripts/validate-link.ts","generate:og-wordmark": "tsx scripts/generate-og-wordmark.ts",@@ -31,19 +32,28 @@"katex": "^0.16.22","lucide-react": "^0.503.0","marked": "^18.0.6",+ "mdast-util-mdx": "^3.0.0","mermaid": "^11.12.3",- "next": "^16",+ "next": "^16.2.4","next-themes": "^0.4.6","posthog-js": "^1.393.0","radix-ui": "^1.4.3","react": "^19","react-dom": "^19","react-hook-form": "^7.80.0",+ "remark-gfm": "^4.0.1",+ "remark-mdx": "^3.1.1",+ "remark-parse": "^11.0.0",+ "remark-stringify": "^11.0.0","tailwind-merge": "^3.6.0",+ "unified": "^11.0.5",+ "unist-util-visit": "^5.1.0",+ "yaml": "^2.8.2","zod": "^4.3.6"},"devDependencies": {"@tailwindcss/postcss": "^4",+ "@types/mdast": "^4.0.4","@types/node": "^24.0.7","@types/react": "^19","@types/react-dom": "^19",diff --git a/docs/source.config.ts b/docs/source.config.tsindex 4cb96c023..95355c4c1 100644--- a/docs/source.config.ts+++ b/docs/source.config.ts@@ -8,9 +8,6 @@ import { z } from 'zod';export const docs = defineDocs({dir: 'content',docs: {- postprocess: {- includeProcessedMarkdown: true,- },schema: frontmatterSchema.extend({sidebarTitle: z.string().optional(),keywords: z.string().optional(),diff --git a/docs/src/app/app-files.test-helper.ts b/docs/src/app/app-files.test-helper.tsnew file mode 100644index 000000000..2baa4e6be--- /dev/null+++ b/docs/src/app/app-files.test-helper.ts@@ -0,0 +1,46 @@+import { readdirSync, readFileSync } from 'node:fs';+import { fileURLToPath } from 'node:url';++const APP_ROOT = fileURLToPath(new URL('.', import.meta.url));++const ROUTER_EXTENSIONS = ['ts', 'tsx', 'js', 'jsx'] as const;++const spellings = (stem: string): string[] =>+ ROUTER_EXTENSIONS.map((extension) => `${stem}.${extension}`);++export const ROUTE_FILES: readonly string[] = spellings('route');+export const PAGE_FILES: readonly string[] = spellings('page');++const METADATA_ROUTE_STEMS: Readonly<Record<string, string>> = {+ sitemap: 'sitemap.xml',+ robots: 'robots.txt',+ manifest: 'manifest.webmanifest',+ 'opengraph-image': 'opengraph-image',+ 'twitter-image': 'twitter-image',+ icon: 'icon',+ 'apple-icon': 'apple-icon',+};++export const METADATA_ROUTE_FILES: readonly string[] =+ Object.keys(METADATA_ROUTE_STEMS).flatMap(spellings);++export function metadataRouteName(fileName: string, sourcePath: string): string | null {+ const stem = fileName.slice(0, fileName.indexOf('.'));+ const served = METADATA_ROUTE_STEMS[stem];+ if (!served || !METADATA_ROUTE_FILES.includes(fileName)) return null;+ const grouped = sourcePath.split('/').some((segment) => /^[(@].+/.test(segment));+ return served.startsWith('sitemap') || !grouped ? served : `${served}-a1b2c3`;+}++export function filesNamed(name: string | readonly string[], dir = APP_ROOT, base = ''): string[] {+ const names = typeof name === 'string' ? [name] : name;+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {+ const rel = base ? `${base}/${entry.name}` : entry.name;+ if (entry.isDirectory()) return filesNamed(names, `${dir}${entry.name}/`, rel);+ return names.includes(entry.name) ? [rel] : [];+ });+}++export function readAppFile(relativePath: string): string {+ return readFileSync(`${APP_ROOT}${relativePath}`, 'utf8');+}diff --git a/docs/src/app/docs/[...slug]/page.tsx b/docs/src/app/docs/[...slug]/page.tsxindex 944686008..e57216538 100644--- a/docs/src/app/docs/[...slug]/page.tsx+++ b/docs/src/app/docs/[...slug]/page.tsx@@ -4,7 +4,7 @@ import type { Metadata } from 'next';import { notFound } from 'next/navigation';import { PageMarkdownActions } from '@/components/page-markdown-actions';import { ProductUpdatesForm } from '@/components/product-updates-form';-import { metaDescription, SITE_NAME, SITE_URL, TWITTER_HANDLE } from '@/lib/site';+import { absoluteSiteUrl, metaDescription, SITE_NAME, SITE_URL, TWITTER_HANDLE } from '@/lib/site';import { source } from '@/lib/source';import { getMDXComponents } from '@/mdx-components';@@ -67,6 +67,7 @@ export async function generateMetadata(props: PageProps<'/docs/[...slug]'>): Prokeywords,alternates: {canonical: page.url,+ types: { 'text/markdown': absoluteSiteUrl(`${page.url}.md`) },},openGraph: {type: 'article',diff --git a/docs/src/app/docs/markdown-alternate-link.test.ts b/docs/src/app/docs/markdown-alternate-link.test.tsnew file mode 100644index 000000000..207a46363--- /dev/null+++ b/docs/src/app/docs/markdown-alternate-link.test.ts@@ -0,0 +1,39 @@+import { describe, expect, test, vi } from 'vitest';++const overviewPage = {+ url: '/docs/get-started/overview',+ data: { title: 'Overview', description: 'What OpenKnowledge is.' },+};++vi.doMock('@/lib/source', () => ({+ source: { getPage: () => overviewPage, generateParams: () => [] },+}));++const { generateMetadata } = await import('./[...slug]/page.tsx');+const changelog = await import('./changelog/page.tsx');++describe('docs page markdown alternate link', () => {+ test('a docs page advertises its .md twin at an absolute URL', async () => {+ const meta = await generateMetadata({+ params: Promise.resolve({ slug: ['get-started', 'overview'] }),+ } as never);++ expect(meta.alternates?.types).toMatchObject({+ 'text/markdown': 'https://openknowledge.ai/docs/get-started/overview.md',+ });+ });++ test('the advertised twin is the URL the markdown rewrite serves', async () => {+ const meta = await generateMetadata({+ params: Promise.resolve({ slug: ['get-started', 'overview'] }),+ } as never);+ const advertised = String(meta.alternates?.types?.['text/markdown']);++ expect(new URL(advertised).pathname).toBe(`${overviewPage.url}.md`);+ });++ test('a page with no markdown producer advertises no markdown twin', async () => {+ expect(changelog.metadata.alternates?.types).not.toHaveProperty('text/markdown');+ expect(changelog.metadata.alternates?.types).toHaveProperty('application/rss+xml');+ });+});diff --git a/docs/src/app/llms-full.txt/route.test.ts b/docs/src/app/llms-full.txt/route.test.tsnew file mode 100644index 000000000..4a196e882--- /dev/null+++ b/docs/src/app/llms-full.txt/route.test.ts@@ -0,0 +1,35 @@+import { describe, expect, test, vi } from 'vitest';+import { corpusSourcePages, readCorpus } from '@/lib/docs-corpus.test-helper';+import { classifyLlmsFullSize, describeLlmsFullSize } from '@/lib/llms-full-size.test-helper';++const pages = corpusSourcePages(await readCorpus());+vi.doMock('@/lib/source', () => ({ source: { getPages: () => pages } }));++const { GET } = await import('./route.ts');++describe('GET /llms-full.txt', () => {+ test('serves the corpus as cacheable markdown', async () => {+ const res = await GET();+ expect(res.status).toBe(200);+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);+ });++ test('renders no canonical of its own, since it renders no single page', async () => {+ expect((await GET()).headers.get('link')).toBeNull();+ });++ test('stays inside the size an agent can actually fetch', async () => {+ const bytes = Buffer.byteLength(await (await GET()).text(), 'utf8');+ const verdict = classifyLlmsFullSize(bytes);+ if (verdict === 'warn') process.stderr.write(`${describeLlmsFullSize(bytes)}\n`);+ expect(verdict, describeLlmsFullSize(bytes)).not.toBe('fail');+ });++ test('carries every page in the corpus', async () => {+ const body = await (await GET()).text();+ for (const page of pages) {+ expect(body).toContain(`(https://openknowledge.ai${page.url})`);+ }+ });+});diff --git a/docs/src/app/llms-full.txt/route.ts b/docs/src/app/llms-full.txt/route.tsindex 13e048929..957ccfd30 100644--- a/docs/src/app/llms-full.txt/route.ts+++ b/docs/src/app/llms-full.txt/route.ts@@ -1,4 +1,5 @@import { getLLMText } from '@/lib/get-llm-text';+import { markdownResponse } from '@/lib/markdown-response';import { source } from '@/lib/source';export const revalidate = false;@@ -7,5 +8,5 @@ export async function GET() {const pages = source.getPages();const scanned = await Promise.all(pages.map((page) => getLLMText(page)));- return new Response(scanned.join('\n\n'));+ return markdownResponse(scanned.join('\n\n'));}diff --git a/docs/src/app/llms.mdx/[...slug]/route.test.ts b/docs/src/app/llms.mdx/[...slug]/route.test.tsindex 8f8e126ef..2e2e7dc15 100644--- a/docs/src/app/llms.mdx/[...slug]/route.test.ts+++ b/docs/src/app/llms.mdx/[...slug]/route.test.ts@@ -1,11 +1,12 @@import { describe, expect, test, vi } from 'vitest';+import { NO_MARKDOWN_PREFIXES } from '@/lib/markdown-routes';const overviewPage = {url: '/docs/get-started/overview',data: {title: 'Overview',description: 'What OpenKnowledge is.',- getText: async () => 'PROCESSED BODY',+ getText: async () => '<Callout type="info">\n\nEarly days.\n\n</Callout>\n',},};@@ -17,35 +18,80 @@ vi.doMock('@/lib/source', () => ({},}));-vi.doMock('next/navigation', () => ({- notFound: () => {- throw new Error('NEXT_HTTP_ERROR_FALLBACK;404');- },-}));-const { GET, generateStaticParams } = await import('./route.ts');function props(slug: string[]) {return { params: Promise.resolve({ slug }) };}+function get(slug: string[]) {+ return GET(new Request(`https://openknowledge.ai/docs/${slug.join('/')}.md`), {+ ...props(slug),+ });+}+describe('GET /docs/<slug>.md (markdown route handler)', () => {test('serves text/markdown with the page rendered by getLLMText', async () => {- const res = await GET(new Request('https://openknowledge.ai/docs/get-started/overview.md'), {- ...props(['get-started', 'overview']),- });+ const res = await get(['get-started', 'overview']);expect(res.status).toBe(200);expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');const body = await res.text();- expect(body).toContain('# Overview (/docs/get-started/overview)');- expect(body).toContain('PROCESSED BODY');+ expect(body).toContain('# Overview (https://openknowledge.ai/docs/get-started/overview)');+ expect(body).toContain('> Early days.');+ expect(body).not.toContain('<Callout');});- test('calls notFound() for an unknown page', async () => {- await expect(- GET(new Request('https://openknowledge.ai/docs/nope.md'), { ...props(['nope']) }),- ).rejects.toThrow('404');+ test('points at the human page with a canonical Link header', async () => {+ const res = await get(['get-started', 'overview']);+ expect(res.headers.get('link')).toBe(+ '<https://openknowledge.ai/docs/get-started/overview>; rel="canonical"',+ );+ });++ test('is edge-cacheable', async () => {+ const res = await get(['get-started', 'overview']);+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);+ });++ test('answers an unknown page with a hard 404 whose body is markdown', async () => {+ const res = await get(['nope']);+ expect(res.status).toBe(404);+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');++ const body = await res.text();+ expect(body).toContain('# 404 Not Found');+ expect(body).toContain('https://openknowledge.ai/llms.txt');+ });++ test('the 404 body is not mistakable for a real page rendition', async () => {+ const [missing, real] = await Promise.all([+ get(['nope']).then((r) => r.text()),+ get(['get-started', 'overview']).then((r) => r.text()),+ ]);+ expect(missing).not.toBe(real);+ });++ test.each(+ NO_MARKDOWN_PREFIXES,+ )('%s tells a real page apart from a URL no page holds', async (prefix) => {+ const slug = [...prefix.replace('/docs/', '').split('/'), 'some-page'];+ const [subtree, missing] = await Promise.all([+ get(slug).then((r) => r.text()),+ get(['nope']).then((r) => r.text()),+ ]);++ expect(subtree).not.toBe(missing);+ expect(subtree).not.toContain('No OpenKnowledge documentation page exists');+ expect(subtree).toContain(`https://openknowledge.ai${prefix}`);+ });++ test.each(+ NO_MARKDOWN_PREFIXES,+ )('%s still answers with a hard 404 carrying the contract', async (prefix) => {+ const res = await get([...prefix.replace('/docs/', '').split('/'), 'some-page']);+ expect(res.status).toBe(404);+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');});test('generateStaticParams delegates to the loader and yields slug-shaped params', () => {diff --git a/docs/src/app/llms.mdx/[...slug]/route.ts b/docs/src/app/llms.mdx/[...slug]/route.tsindex b12aa707a..8b0390000 100644--- a/docs/src/app/llms.mdx/[...slug]/route.ts+++ b/docs/src/app/llms.mdx/[...slug]/route.ts@@ -1,9 +1,29 @@-import { notFound } from 'next/navigation';import { getLLMText } from '@/lib/get-llm-text';+import { markdownResponse } from '@/lib/markdown-response';+import { unrenditionedSubtree } from '@/lib/markdown-routes';+import { absoluteSiteUrl, SITE_URL } from '@/lib/site';import { source } from '@/lib/source';export const dynamic = 'force-static';+const NOT_FOUND_BODY = `# 404 Not Found++No OpenKnowledge documentation page exists at this URL.++Every published page is listed at <${SITE_URL}/llms.txt>, each with a \`.md\`+rendition of its own.+`;++function noRenditionBody(subtree: string): string {+ return `# 404 Not Found++There is no Markdown rendition under ${subtree}. Those pages are published as+HTML only; read them at <${SITE_URL}${subtree}>.++Every page that does have one is listed at <${SITE_URL}/llms.txt>.+`;+}+interface RouteProps {params: Promise<{ slug: string[] }>;}@@ -11,10 +31,13 @@ interface RouteProps {export async function GET(_request: Request, props: RouteProps) {const { slug } = await props.params;const page = source.getPage(slug);- if (!page) notFound();+ if (!page) {+ const subtree = unrenditionedSubtree(slug);+ return markdownResponse(subtree ? noRenditionBody(subtree) : NOT_FOUND_BODY, { status: 404 });+ }- return new Response(await getLLMText(page), {- headers: { 'Content-Type': 'text/markdown; charset=utf-8' },+ return markdownResponse(await getLLMText(page), {+ canonicalUrl: absoluteSiteUrl(page.url),});}diff --git a/docs/src/app/llms.txt/route.test.ts b/docs/src/app/llms.txt/route.test.tsnew file mode 100644index 000000000..711678e6d--- /dev/null+++ b/docs/src/app/llms.txt/route.test.ts@@ -0,0 +1,113 @@+import { beforeEach, describe, expect, test, vi } from 'vitest';+import { corpusSourcePages, readCorpus } from '@/lib/docs-corpus.test-helper';+import { markdownHandlerForPage } from '@/lib/markdown-routes';+import { MARKETING_MARKDOWN_PAGES } from '@/lib/marketing-markdown-pages';++const pages = corpusSourcePages(await readCorpus());+vi.doMock('@/lib/source', () => ({ source: { getPages: () => pages } }));++const { GET, revalidate } = await import('./route.ts');++const FEED = [+ '<rss version="2.0"><channel>',+ '<item><title><![CDATA[Keeping you and your agents in check with markdownlint]]></title>',+ '<link>https://openknowledge.ai/blog/markdownlint-support</link>',+ '<description><![CDATA[Native markdownlint support in Open Knowledge.]]></description></item>',+ '</channel></rss>',+].join('\n');++beforeEach(() => {+ vi.stubGlobal(+ 'fetch',+ vi.fn(async () => new Response(FEED)),+ );+ return () => vi.unstubAllGlobals();+});++async function body(): Promise<string> {+ return (await GET()).text();+}++function listedUrls(text: string): string[] {+ return [...text.matchAll(/^- \[(?:\\.|[^\]\\])*]\(([^)]+)\)/gm)].map((match) => match[1]);+}++describe('GET /llms.txt', () => {+ test('serves markdown the edge can cache', async () => {+ const res = await GET();+ expect(res.status).toBe(200);+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);+ });++ test('refreshes hourly, since the blog it lists is published elsewhere', () => {+ expect(revalidate).toBe(3600);+ });++ test('lists one entry per docs page, derived from the corpus rather than a count', async () => {+ const docsSection = (await body()).split('## Blog')[0];+ expect(listedUrls(docsSection)).toHaveLength(pages.length);+ expect(pages.length).toBeGreaterThan(50);+ });++ test('links the Markdown rendition of every docs page', async () => {+ const listed = new Set(listedUrls(await body()));+ for (const page of pages) {+ expect(listed).toContain(`https://openknowledge.ai${page.url}.md`);+ }+ });++ test('names each docs page by its own title', async () => {+ const quickstart = pages.find((page) => page.url === '/docs/get-started/quickstart');+ expect(quickstart).toBeDefined();+ expect(await body()).toContain(+ `- [${quickstart?.data.title}](https://openknowledge.ai/docs/get-started/quickstart.md)`,+ );+ });++ test('covers the marketing zone the docs app cannot enumerate', async () => {+ const listed = new Set(listedUrls(await body()));+ for (const page of MARKETING_MARKDOWN_PAGES) {+ expect(listed).toContain(`https://openknowledge.ai${page.path}`);+ }+ });++ test('lists the blog posts the marketing zone reported, under their own titles', async () => {+ expect(await body()).toContain(+ '- [Keeping you and your agents in check with markdownlint]' ++ '(https://openknowledge.ai/blog/markdownlint-support.md): ' ++ 'Native markdownlint support in Open Knowledge.',+ );+ });++ test('keeps its own promise that every page it names negotiates Markdown', async () => {+ const docsPages = [...(await body()).matchAll(/^- \[(?:\\.|[^\]\\])*]\((\S+)\.md\)/gm)]+ .map((match) => match[1].replace('https://openknowledge.ai', ''))+ .filter((path) => path.startsWith('/docs/'));++ expect(+ docsPages.length,+ 'no docs entries found; this check is reading nothing',+ ).toBeGreaterThan(10);+ expect(docsPages.filter((path) => markdownHandlerForPage(path) === null)).toEqual([]);+ });++ test('holds the llms.txt shape: title, summary, then named sections', async () => {+ const text = await body();+ expect(text.startsWith('# OpenKnowledge\n\n> ')).toBe(true);+ expect([...text.matchAll(/^## (.+)$/gm)].map((match) => match[1])).toEqual([+ 'Docs',+ 'Blog',+ 'Product',+ ]);+ });++ test('every link is absolute and resolves to a Markdown rendition', async () => {+ const urls = listedUrls(await body());+ expect(urls.length).toBeGreaterThan(50);+ for (const url of urls) {+ expect(url).toMatch(/^https:\/\/openknowledge\.ai\/[^\s]*\.md$/);+ }+ expect(urls).toEqual([...new Set(urls)]);+ });+});diff --git a/docs/src/app/llms.txt/route.ts b/docs/src/app/llms.txt/route.tsindex a2e621b98..5b7f57ce3 100644--- a/docs/src/app/llms.txt/route.ts+++ b/docs/src/app/llms.txt/route.ts@@ -1,15 +1,17 @@-import { SITE_URL } from '@/lib/site';+import { buildLlmsTxt, type LlmsTxtLink } from '@/lib/llms-txt';+import { markdownResponse } from '@/lib/markdown-response';+import { blogPostLinks } from '@/lib/marketing-blog-index';+import { absoluteSiteUrl } from '@/lib/site';import { source } from '@/lib/source';-export const revalidate = false;+export const revalidate = 3600;export async function GET() {- const pages = source.getPages();- return new Response(- [- '# OpenKnowledge',- '## Docs',- ...pages.map((page) => `- [${page.data.title}](${SITE_URL}${page.url}.md)`),- ].join('\n\n'),- );+ const docs: LlmsTxtLink[] = source.getPages().map((page) => ({+ url: absoluteSiteUrl(`${page.url}.md`),+ name: page.data.title,+ description: page.data.description || undefined,+ }));++ return markdownResponse(buildLlmsTxt({ docs, blogPosts: await blogPostLinks(revalidate) }));}diff --git a/docs/src/app/markdown-route-coverage.test.ts b/docs/src/app/markdown-route-coverage.test.tsnew file mode 100644index 000000000..6672478d2--- /dev/null+++ b/docs/src/app/markdown-route-coverage.test.ts@@ -0,0 +1,148 @@+import { describe, expect, test } from 'vitest';+import { markdownHandlerForPage, PAGE_MARKDOWN_ROUTE } from '@/lib/markdown-routes';+import {+ filesNamed,+ METADATA_ROUTE_FILES,+ metadataRouteName,+ PAGE_FILES,+ ROUTE_FILES,+} from './app-files.test-helper.ts';++const NO_MARKDOWN: Record<string, string> = {+ '/continue': 'hand-off screen for the desktop app; carries no durable content',+ '/d/[encoded]': 'renders a share payload carried in the URL, unlisted and ephemeral by design',+ '/docs': 'redirects to the overview page rather than rendering content of its own',+ '/docs/changelog':+ "built from the CLI's own CHANGELOG.md through a source adapter the Markdown handler cannot read",+ '/docs/changelog/[tag]':+ "built from the CLI's own CHANGELOG.md through a source adapter the Markdown handler cannot read",+};++const ROUTE_GROUP = /^\(.+\)$/;+const DYNAMIC_SEGMENT = /^\[(\.\.\.)?[^[\]./]+\]$/;+const LITERAL_SEGMENT = /^[\w.-]+$/;++function routePathOf(file: string): string {+ const segments = file.split('/').slice(0, -1);+ const parts = segments.filter((segment) => {+ if (ROUTE_GROUP.test(segment)) return false;+ if (DYNAMIC_SEGMENT.test(segment) || LITERAL_SEGMENT.test(segment)) return true;+ throw new Error(`unrecognised app-router segment "${segment}" in ${file}`);+ });+ return `/${parts.join('/')}`;+}++function metadataRoutePathOf(file: string): string {+ const fileName = file.slice(file.lastIndexOf('/') + 1);+ const dir = routePathOf(file);+ const served = metadataRouteName(fileName, file);+ if (!served) throw new Error(`unmapped metadata-route convention "${fileName}" in ${file}`);+ return `${dir === '/' ? '' : dir}/${served}`;+}++function sampleUrlOf(routePath: string): string {+ return routePath.replace(/\[(?:\.\.\.)?([^[\]]+)\]/g, 'sample-$1');+}++function routeMatcher(routePath: string): RegExp {+ const body = routePath+ .split('/')+ .map((segment) => {+ if (/^\[\.\.\..+\]$/.test(segment)) return '.+';+ if (/^\[.+\]$/.test(segment)) return '[^/]+';+ return segment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ })+ .join('/');+ return new RegExp(`^${body}$`);+}++const pageRoutes = filesNamed(PAGE_FILES).map((file) => ({ file, route: routePathOf(file) }));+const handlerRoutes = filesNamed(ROUTE_FILES).map(routePathOf);++const nonPageRoutes = [+ ...filesNamed(ROUTE_FILES).map(routePathOf),+ ...filesNamed(METADATA_ROUTE_FILES).map(metadataRoutePathOf),+].filter((route) => !route.startsWith(`${PAGE_MARKDOWN_ROUTE}/`));++describe('markdown route coverage', () => {+ test('every page route has a markdown producer or a stated exemption', () => {+ const uncovered = pageRoutes+ .filter(({ route }) => !markdownHandlerForPage(sampleUrlOf(route)) && !NO_MARKDOWN[route])+ .map(+ ({ file, route }) =>+ `${route} (src/app/${file}): serve it from src/lib/markdown-routes.ts, or add "${route}" to NO_MARKDOWN in this file with the reason it has none`,+ );++ expect(uncovered, 'page routes with neither a markdown rendition nor an exemption').toEqual([]);+ });++ test('the census reaches the pages it is meant to police', () => {+ expect(pageRoutes.map(({ route }) => route)).toContain('/docs/[...slug]');+ });++ test.each(Object.entries(NO_MARKDOWN))('%s is exempt from markdown: %s', (route, reason) => {+ expect(reason.trim(), `${route} is exempted without saying why`).not.toBe('');+ expect(+ pageRoutes.map((page) => page.route),+ `${route} is exempted but no page renders it; drop the NO_MARKDOWN entry`,+ ).toContain(route);+ expect(+ markdownHandlerForPage(sampleUrlOf(route)),+ `${route} is exempted but does have a markdown rendition; drop the NO_MARKDOWN entry`,+ ).toBeNull();+ });++ test('never negotiates a non-page route into a markdown handler', () => {+ const negotiating = nonPageRoutes+ .filter((route) => markdownHandlerForPage(sampleUrlOf(route)) !== null)+ .map(+ (route) =>+ `${route}: a Markdown-preferring agent gets a 404 instead of this route's own body; add its prefix to NO_MARKDOWN_PREFIXES in src/lib/markdown-routes.ts`,+ );++ expect(negotiating, 'non-page routes that shape alone maps to a slug handler').toEqual([]);+ });++ test('reaches the non-page routes it is meant to police', () => {+ expect(nonPageRoutes).toContain('/docs/changelog/rss.xml');+ });++ test('every markdown producer names a handler this app implements', () => {+ const missing = pageRoutes+ .map(({ route }) => ({ route, handler: markdownHandlerForPage(sampleUrlOf(route)) }))+ .filter(+ ({ handler }) =>+ handler !== null && !handlerRoutes.some((r) => routeMatcher(r).test(handler)),+ )+ .map(({ route, handler }) => `${route} -> ${handler} (no route.ts serves it)`);++ expect(missing, 'markdown producers with no handler').toEqual([]);+ });++ test('enumerates every spelling the router accepts for each convention', () => {+ for (const list of [PAGE_FILES, ROUTE_FILES]) {+ expect(list.map((name) => name.split('.').pop())?.sort()).toEqual(['js', 'jsx', 'ts', 'tsx']);+ }+ expect([...new Set(METADATA_ROUTE_FILES.map((name) => name.split('.')[0]))].sort()).toEqual([+ 'apple-icon',+ 'icon',+ 'manifest',+ 'opengraph-image',+ 'robots',+ 'sitemap',+ 'twitter-image',+ ]);+ expect(METADATA_ROUTE_FILES).toHaveLength(28);+ });++ test.each([+ ['sitemap.ts', 'blog/sitemap.ts', 'sitemap.xml'],+ ['sitemap.ts', '(home)/blog/sitemap.ts', 'sitemap.xml'],+ ['opengraph-image.tsx', 'blog/opengraph-image.tsx', 'opengraph-image'],+ ['opengraph-image.tsx', '(home)/blog/opengraph-image.tsx', 'opengraph-image-a1b2c3'],+ ['opengraph-image.png', 'blog/opengraph-image.png', null],+ ['route.ts', 'blog/route.ts', null],+ ])('serves %s (source %s) at %s', (fileName, sourcePath, expected) => {+ expect(metadataRouteName(fileName, sourcePath)).toBe(expected);+ });+});diff --git a/docs/src/components/agent-icons.tsx b/docs/src/components/agent-icons.tsxindex 81e37520b..261a6475a 100644--- a/docs/src/components/agent-icons.tsx+++ b/docs/src/components/agent-icons.tsx@@ -6,7 +6,7 @@ import { OpenClawIcon } from '@/components/icons/openclaw';import { OpenCodeIcon } from '@/components/icons/opencode';import { PiIcon } from '@/components/icons/pi';-const AGENTS = [+export const AGENTS = [{ name: 'Claude', Icon: ClaudeIcon },{ name: 'Codex', Icon: CodexIcon },{ name: 'Cursor', Icon: CursorIcon },diff --git a/docs/src/components/download-split-button.test.tsx b/docs/src/components/download-split-button.test.tsxnew file mode 100644index 000000000..baad98466--- /dev/null+++ b/docs/src/components/download-split-button.test.tsx@@ -0,0 +1,20 @@+import { renderToStaticMarkup } from 'react-dom/server';+import { describe, expect, test } from 'vitest';+import { DownloadButton } from '@/components/download-button';+import { downloadLabelForOs } from '@/lib/download-targets';++const html = renderToStaticMarkup(<DownloadButton />);++describe('the docs download button before hydration', () => {+ test('names no platform', () => {+ expect(html).toContain(`>${downloadLabelForOs('unknown')}<`);+ for (const os of ['macos', 'windows', 'linux'] as const) {+ expect(html).not.toContain(downloadLabelForOs(os));+ }+ });++ test('links a build the visitor can run without script', () => {+ const hrefs = html.replaceAll('&', '&');+ expect(hrefs).toContain('os=macos&arch=arm64&format=dmg');+ });+});diff --git a/docs/src/components/mcp-install.tsx b/docs/src/components/mcp-install.tsxindex f57080bdb..75791af12 100644--- a/docs/src/components/mcp-install.tsx+++ b/docs/src/components/mcp-install.tsx@@ -1,4 +1,20 @@import type { ReactNode } from 'react';+import type { SerializerApi } from '@/lib/mdx-serializer';++const WEB_APP_GUIDE_HREF = '/docs/get-started/quickstart#ok-install-web-app';++interface McpInstallMarkdownApi extends Pick<SerializerApi, 'url'> {+ link(label: string, url: string): string;+}++export function mcpInstallMarkdown(editor: string, api: McpInstallMarkdownApi): string {+ return [+ `There are two ways to connect ${editor}, depending on how you run OpenKnowledge:`,+ '',+ `- **Desktop app** (macOS, Windows, Linux). The first time you open a project, a consent dialog detects ${editor} and configures it for you. To re-trigger the dialog, choose **File → Set up OpenKnowledge integrations…**.`,+ `- **Web app / terminal** (any platform, including Intel Macs — see the ${api.link('web app guide', api.url(WEB_APP_GUIDE_HREF))}). Run \`ok init\` in your project: it registers the OpenKnowledge MCP server with ${editor} and the other editors it detects. Every \`ok start\` repairs the entry if it has drifted (it never adds one you removed).`,+ ].join('\n');+}export function McpInstall({ editor, children }: { editor: string; children?: ReactNode }) {return (@@ -14,7 +30,7 @@ export function McpInstall({ editor, children }: { editor: string; children?: Re</li><li><strong>Web app / terminal</strong> (any platform, including Intel Macs — see the{' '}- <a href="/docs/get-started/quickstart#ok-install-web-app">web app guide</a>+ <a href={WEB_APP_GUIDE_HREF}>web app guide</a>). Run <code>ok init</code> in your project: it registers the OpenKnowledge MCP serverwith {editor} and the other editors it detects. Every <code>ok start</code> repairs theentry if it has drifted (it never adds one you removed).diff --git a/docs/src/components/overview-blocks.tsx b/docs/src/components/overview-blocks.tsxindex 49926c96d..d6451a090 100644--- a/docs/src/components/overview-blocks.tsx+++ b/docs/src/components/overview-blocks.tsx@@ -1,15 +1,14 @@import { ArrowRight, Bot, FileText, type LucideIcon, NotebookPen } from 'lucide-react';-import type { ReactNode } from 'react';interface Layer {k: string;Icon: LucideIcon;title: string;role: string;- desc: ReactNode;+ desc: string;}-const LAYERS: Layer[] = [+export const LAYERS: Layer[] = [{k: 'LAYER 01',Icon: NotebookPen,@@ -73,7 +72,7 @@ interface Path {desc: string;}-const PATHS: Path[] = [+export const PATHS: Path[] = [{href: '/docs/get-started/quickstart',title: 'Try the Quickstart',diff --git a/docs/src/components/verify-exec.tsx b/docs/src/components/verify-exec.tsxindex 92bbaf4a2..ce346ce87 100644--- a/docs/src/components/verify-exec.tsx+++ b/docs/src/components/verify-exec.tsx@@ -1,9 +1,19 @@import { CopyPrompt } from '@/components/copy-prompt';+const VERIFY_PROMPT = 'List the first 5 documents you come across in this project.';++function verifyExpectation(subject: string): string {+ return `${subject} should call the OpenKnowledge \`exec\` tool and respond with some of your documents.`;+}++export function verifyExecMarkdown(subject: string): string {+ return `> ${VERIFY_PROMPT}\n\n${verifyExpectation(subject)}`;+}+export function VerifyExec({ subject = 'The agent' }: { subject?: string }) {return (<>- <CopyPrompt>List the first 5 documents you come across in this project.</CopyPrompt>+ <CopyPrompt>{VERIFY_PROMPT}</CopyPrompt><p>{subject} should call the OpenKnowledge <code>exec</code> tool and respond with some of yourdocuments.diff --git a/docs/src/lib/accept-negotiation.test.ts b/docs/src/lib/accept-negotiation.test.tsnew file mode 100644index 000000000..7e9b0ceb2--- /dev/null+++ b/docs/src/lib/accept-negotiation.test.ts@@ -0,0 +1,77 @@+import { describe, expect, test } from 'vitest';+import { MARKDOWN_MEDIA_TYPES, prefersMarkdown } from './accept-negotiation.ts';++const BROWSER_ACCEPT =+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8';+const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';+const CODING_AGENT_ACCEPT = 'text/markdown, text/html, */*';++describe('prefersMarkdown serves Markdown only on an explicit, winning ask', () => {+ test.each([+ ['bare text/markdown', 'text/markdown'],+ ['the text/x-markdown spelling', 'text/x-markdown'],+ ['a coding agent listing Markdown first at equal weight', CODING_AGENT_ACCEPT],+ ['Markdown outweighing HTML', 'text/html;q=0.5, text/markdown;q=0.9'],+ ['Markdown above a de-weighted wildcard', 'text/markdown, */*;q=0.1'],+ ['HTML explicitly refused', 'text/html;q=0, text/markdown'],+ ['a wildcard listed first but less specific', '*/*, text/markdown'],+ [+ 'Markdown above a weakened HTML, wildcard present',+ 'text/markdown;q=0.5, text/html;q=0.1, */*',+ ],+ [+ 'a type wildcard not overriding the exact HTML entry',+ 'text/markdown;q=0.3, text/html;q=0.2, text/*;q=0.9',+ ],+ ['parameters and casing the client chose', 'Text/Markdown; charset=utf-8'],+ ])('%s wins Markdown', (_label, accept) => {+ expect(prefersMarkdown(accept)).toBe(true);+ });++ test.each([+ ['Chrome', BROWSER_ACCEPT],+ ['Safari', SAFARI_ACCEPT],+ ['a bare wildcard', '*/*'],+ ['a type wildcard', 'text/*'],+ ['text/plain, which asks for text without markup', 'text/plain'],+ ['text/plain ranked above HTML', 'text/plain, text/html;q=0.5'],+ ['Markdown ranked below HTML', 'text/html, text/markdown;q=0.1'],+ ['Markdown refused outright', 'text/markdown;q=0, text/html'],+ ['Markdown tied with HTML but listed second', 'text/html, text/markdown'],+ ['no Accept at all', null],+ ['an empty Accept', ''],+ ['JSON', 'application/json'],+ ])('%s gets HTML', (_label, accept) => {+ expect(prefersMarkdown(accept)).toBe(false);+ });+});++describe('prefersMarkdown survives malformed input', () => {+ test.each([+ ['a bare token with no slash', 'garbage'],+ ['punctuation', ';;;,,,'],+ ['an unreadable quality on the Markdown entry', 'text/markdown;q=abc'],+ ['a truncated header', 'text/markdown;'],+ ])('%s never throws', (_label, accept) => {+ expect(() => prefersMarkdown(accept)).not.toThrow();+ });++ test('an unreadable quality drops that entry rather than weighting it', () => {+ expect(prefersMarkdown('text/markdown;q=abc')).toBe(false);+ expect(prefersMarkdown('text/markdown;q=abc, text/html')).toBe(false);+ });++ test('stray commas and whitespace do not create phantom entries', () => {+ expect(prefersMarkdown(' , text/markdown , ')).toBe(true);+ });++ test('a quality above 1 is clamped rather than used to outrank', () => {+ expect(prefersMarkdown('text/html;q=1, text/markdown;q=5')).toBe(false);+ });+});++describe('MARKDOWN_MEDIA_TYPES', () => {+ test('does not include text/plain', () => {+ expect(MARKDOWN_MEDIA_TYPES).not.toContain('text/plain');+ });+});diff --git a/docs/src/lib/accept-negotiation.ts b/docs/src/lib/accept-negotiation.tsnew file mode 100644index 000000000..f71ed3a05--- /dev/null+++ b/docs/src/lib/accept-negotiation.ts@@ -0,0 +1,86 @@+import { type NextRequest, NextResponse } from 'next/server';++export const MARKDOWN_MEDIA_TYPES = ['text/markdown', 'text/x-markdown'];++interface AcceptEntry {+ range: string;+ q: number;+ index: number;+}++function parseAccept(header: string): AcceptEntry[] {+ const entries: AcceptEntry[] = [];++ header.split(',').forEach((part, index) => {+ const [rangeText, ...parameters] = part.split(';');+ const range = rangeText.trim().toLowerCase();+ if (!range.includes('/')) return;++ let q = 1;+ for (const parameter of parameters) {+ const separator = parameter.indexOf('=');+ if (separator === -1) continue;+ if (parameter.slice(0, separator).trim().toLowerCase() !== 'q') continue;+ const value = Number.parseFloat(parameter.slice(separator + 1).trim());+ if (Number.isNaN(value)) return;+ q = Math.min(1, Math.max(0, value));+ }++ entries.push({ range, q, index });+ });++ return entries;+}++function specificity(range: string): number {+ if (range === '*/*') return 0;+ if (range.endsWith('/*')) return 1;+ return 2;+}++function outranks(candidate: AcceptEntry, incumbent: AcceptEntry): boolean {+ if (candidate.q !== incumbent.q) return candidate.q > incumbent.q;+ const bySpecificity = specificity(candidate.range) - specificity(incumbent.range);+ if (bySpecificity !== 0) return bySpecificity > 0;+ return candidate.index < incumbent.index;+}++function bestMatch(+ entries: AcceptEntry[],+ matches: (range: string) => boolean,+): AcceptEntry | undefined {+ let best: AcceptEntry | undefined;+ for (const entry of entries) {+ if (!matches(entry.range)) continue;+ if (!best) {+ best = entry;+ continue;+ }+ const bySpecificity = specificity(entry.range) - specificity(best.range);+ if (bySpecificity > 0 || (bySpecificity === 0 && entry.q > best.q)) best = entry;+ }+ return best;+}++const isMarkdownRange = (range: string) => MARKDOWN_MEDIA_TYPES.includes(range);+const coversHtml = (range: string) =>+ range === 'text/html' || range === 'text/*' || range === '*/*';++export function prefersMarkdown(accept: string | null | undefined): boolean {+ if (!accept) return false;++ const entries = parseAccept(accept);+ const markdown = bestMatch(entries, isMarkdownRange);+ if (!markdown || markdown.q === 0) return false;++ const html = bestMatch(entries, coversHtml);+ return !html || outranks(markdown, html);+}++export function markdownRewrite(request: NextRequest, handler: string | null): NextResponse | null {+ if (!handler || !prefersMarkdown(request.headers.get('accept'))) return null;++ const target = new URL(handler, request.nextUrl.origin);+ target.search = request.nextUrl.search;+ return NextResponse.rewrite(target);+}diff --git a/docs/src/lib/docs-corpus.test-helper.ts b/docs/src/lib/docs-corpus.test-helper.tsnew file mode 100644index 000000000..a0cc4f466--- /dev/null+++ b/docs/src/lib/docs-corpus.test-helper.ts@@ -0,0 +1,63 @@+import { readdirSync } from 'node:fs';+import { readFile } from 'node:fs/promises';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { parse } from 'yaml';++const CONTENT_ROOT = fileURLToPath(new URL('../../content/', import.meta.url));++export interface CorpusPage {+ readonly slug: string;+ readonly source: string;+ readonly pageUrl: string;+}++function corpusFiles(dir = CONTENT_ROOT): string[] {+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {+ const full = path.join(dir, entry.name);+ if (entry.isDirectory()) return corpusFiles(full);+ return /\.mdx?$/.test(entry.name) ? [full] : [];+ });+}++export async function readCorpus(): Promise<CorpusPage[]> {+ return Promise.all(+ corpusFiles().map(async (file) => {+ const slug = path+ .relative(CONTENT_ROOT, file)+ .replace(/\.mdx?$/, '')+ .replace(/(^|\/)index$/, '');+ return {+ slug,+ source: await readFile(file, 'utf8'),+ pageUrl: `https://openknowledge.ai/docs/${slug}`.replace(/\/$/, ''),+ };+ }),+ );+}++export interface CorpusSourcePage {+ readonly url: string;+ readonly data: {+ readonly title: string;+ readonly description?: string;+ getText(): Promise<string>;+ };+}++const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---/;++export function corpusSourcePages(pages: readonly CorpusPage[]): CorpusSourcePage[] {+ return pages.map((page) => {+ const block = FRONTMATTER.exec(page.source)?.[1];+ const data = (block ? parse(block) : {}) as { title?: string; description?: string };+ return {+ url: `/docs/${page.slug}`.replace(/\/$/, ''),+ data: {+ title: data.title ?? page.slug,+ description: data.description,+ getText: async () => page.source,+ },+ };+ });+}diff --git a/docs/src/lib/download-targets.test.ts b/docs/src/lib/download-targets.test.tsindex 01c739ff7..68a7a087e 100644--- a/docs/src/lib/download-targets.test.ts+++ b/docs/src/lib/download-targets.test.ts@@ -154,6 +154,8 @@ describe('classifyDownloadOs', () => {expect(classifyDownloadOs('macOS')).toBe('macos');expect(classifyDownloadOs('Windows')).toBe('windows');expect(classifyDownloadOs('Linux')).toBe('linux');+ expect(classifyDownloadOs('Chrome OS')).toBe('linux');+ expect(classifyDownloadOs('Chromium OS')).toBe('linux');});test('mobile falls through to unknown — it cannot run the desktop app', () => {diff --git a/docs/src/lib/download-targets.ts b/docs/src/lib/download-targets.tsindex 533ae0f29..47f69d2b9 100644--- a/docs/src/lib/download-targets.ts+++ b/docs/src/lib/download-targets.ts@@ -149,7 +149,8 @@ export function classifyDownloadOs(input: string | null | undefined): DetectedOslower.includes('linux') ||lower.includes('x11') ||lower.includes('cros') ||- lower.includes('chrome os')+ lower.includes('chrome os') ||+ lower.includes('chromium os')) {return 'linux';}diff --git a/docs/src/lib/get-llm-text.test.ts b/docs/src/lib/get-llm-text.test.tsindex e2bc42ecf..349b4b025 100644--- a/docs/src/lib/get-llm-text.test.ts+++ b/docs/src/lib/get-llm-text.test.ts@@ -3,46 +3,97 @@ import { getLLMText } from './get-llm-text.ts';type Page = Parameters<typeof getLLMText>[0];+const SOURCE = `---+title: Overview+description: What OpenKnowledge is.+---++Read the [quickstart](../get-started/quickstart.mdx).++<Callout type="info" title="Heads up">++It is early.++</Callout>+`;+function fakePage(data: Partial<Page['data']> = {}, url = '/docs/get-started/overview'): Page {return {url,data: {title: 'Overview',description: 'What OpenKnowledge is.',- getText: async (type: 'raw' | 'processed') => `BODY(${type})`,+ getText: async () => SOURCE,...data,},} as unknown as Page;}describe('getLLMText', () => {- test('renders title + URL header, description, then processed body', async () => {+ test('renders title + URL header, description, then the serialized body', async () => {const md = await getLLMText(fakePage());expect(md).toBe(- `# Overview (/docs/get-started/overview)+ `# Overview (https://openknowledge.ai/docs/get-started/overview)What OpenKnowledge is.-BODY(processed)`,+Read the [quickstart](https://openknowledge.ai/docs/get-started/quickstart).++> **Heads up**+>+> It is early.`,);});- test('requests the processed Markdown variant (snippets resolved), not raw', async () => {+ test('heads the document with the absolute human URL, not the site-relative one', async () => {+ const md = await getLLMText(fakePage({}, '/docs/reference/components/callout'));+ expect(md.split('\n')[0]).toBe(+ '# Overview (https://openknowledge.ai/docs/reference/components/callout)',+ );+ });++ test('serializes the raw MDX source, not the compiled Markdown rendition', async () => {let requested: string | undefined;await getLLMText(fakePage({getText: async (type) => {requested = type;- return '';+ return SOURCE;},}),);- expect(requested).toBe('processed');+ expect(requested).toBe('raw');+ });++ test('strips the frontmatter rather than serving it as body text', async () => {+ const md = await getLLMText(fakePage());+ expect(md).not.toContain('---');+ expect(md).not.toContain('title: Overview');+ });++ test('neutralises raw HTML a title names in passing', async () => {+ const md = await getLLMText(fakePage({ title: 'The <details> element' }));+ expect(md.split('\n')[0]).toBe(+ '# The \\<details> element (https://openknowledge.ai/docs/get-started/overview)',+ );+ });++ test('keeps a multi-line title on the heading line', async () => {+ const md = await getLLMText(fakePage({ title: 'Folded\n\ntitle' }));+ expect(md.split('\n')[0]).toBe(+ '# Folded title (https://openknowledge.ai/docs/get-started/overview)',+ );});test('tolerates a missing description without emitting "undefined"', async () => {const md = await getLLMText(fakePage({ description: undefined }));expect(md).not.toContain('undefined');- expect(md).toContain('# Overview (/docs/get-started/overview)');+ expect(md).toContain('# Overview (https://openknowledge.ai/docs/get-started/overview)');+ });++ test('fails loudly on a component with no registry disposition', async () => {+ await expect(+ getLLMText(fakePage({ getText: async () => '<Widget>content</Widget>' })),+ ).rejects.toThrow('No serializer disposition for <Widget>');});});diff --git a/docs/src/lib/get-llm-text.ts b/docs/src/lib/get-llm-text.tsindex 8b96b7312..26348f7bf 100644--- a/docs/src/lib/get-llm-text.ts+++ b/docs/src/lib/get-llm-text.ts@@ -1,12 +1,20 @@import type { InferPageType } from 'fumadocs-core/source';+import { escapeInlineProse, escapeRawHtml, serializeMdx } from '@/lib/mdx-serializer';+import { DOCS_SERIALIZER_REGISTRY } from '@/lib/mdx-serializer-registry';+import { absoluteSiteUrl } from '@/lib/site';import type { source } from '@/lib/source';export async function getLLMText(page: InferPageType<typeof source>): Promise<string> {- const processed = await page.data.getText('processed');+ const canonicalUrl = absoluteSiteUrl(page.url);+ const mdx = await page.data.getText('raw');+ const { body } = serializeMdx(mdx, {+ registry: DOCS_SERIALIZER_REGISTRY,+ pageUrl: canonicalUrl,+ });- return `# ${page.data.title} (${page.url})+ return `# ${escapeInlineProse(page.data.title)} (${canonicalUrl})-${page.data.description || ''}+${escapeRawHtml(page.data.description || '')}-${processed}`;+${body}`;}diff --git a/docs/src/lib/llms-full-size.test-helper.ts b/docs/src/lib/llms-full-size.test-helper.tsnew file mode 100644index 000000000..dcd0b49cf--- /dev/null+++ b/docs/src/lib/llms-full-size.test-helper.ts@@ -0,0 +1,15 @@+export const LLMS_FULL_WARN_BYTES = 2_500_000;+export const LLMS_FULL_MAX_BYTES = 5_000_000;++export type CorpusSizeVerdict = 'ok' | 'warn' | 'fail';++export function classifyLlmsFullSize(bytes: number): CorpusSizeVerdict {+ if (bytes > LLMS_FULL_MAX_BYTES) return 'fail';+ if (bytes > LLMS_FULL_WARN_BYTES) return 'warn';+ return 'ok';+}++export function describeLlmsFullSize(bytes: number): string {+ const mb = (limit: number) => `${(limit / 1_000_000).toFixed(1)} MB`;+ return `/llms-full.txt is ${mb(bytes)} (warn over ${mb(LLMS_FULL_WARN_BYTES)}, fail over ${mb(LLMS_FULL_MAX_BYTES)})`;+}diff --git a/docs/src/lib/llms-full-size.test.ts b/docs/src/lib/llms-full-size.test.tsnew file mode 100644index 000000000..0c63a6af8--- /dev/null+++ b/docs/src/lib/llms-full-size.test.ts@@ -0,0 +1,29 @@+import { describe, expect, test } from 'vitest';+import {+ classifyLlmsFullSize,+ describeLlmsFullSize,+ LLMS_FULL_MAX_BYTES,+ LLMS_FULL_WARN_BYTES,+} from './llms-full-size.test-helper';++describe('llms-full.txt size ceiling', () => {+ test('passes a corpus comfortably inside the budget', () => {+ expect(classifyLlmsFullSize(709_000)).toBe('ok');+ });++ test('warns once the corpus passes the warning threshold', () => {+ expect(classifyLlmsFullSize(LLMS_FULL_WARN_BYTES)).toBe('ok');+ expect(classifyLlmsFullSize(LLMS_FULL_WARN_BYTES + 1)).toBe('warn');+ });++ test('fails once the corpus passes the ceiling', () => {+ expect(classifyLlmsFullSize(LLMS_FULL_MAX_BYTES)).toBe('warn');+ expect(classifyLlmsFullSize(LLMS_FULL_MAX_BYTES + 1)).toBe('fail');+ });++ test('reports the measurement and both thresholds so a failure is actionable', () => {+ expect(describeLlmsFullSize(6_000_000)).toBe(+ '/llms-full.txt is 6.0 MB (warn over 2.5 MB, fail over 5.0 MB)',+ );+ });+});diff --git a/docs/src/lib/llms-txt.test.ts b/docs/src/lib/llms-txt.test.tsnew file mode 100644index 000000000..e1cdcef98--- /dev/null+++ b/docs/src/lib/llms-txt.test.ts@@ -0,0 +1,175 @@+import remarkGfm from 'remark-gfm';+import remarkParse from 'remark-parse';+import { unified } from 'unified';+import { visit } from 'unist-util-visit';+import { describe, expect, test } from 'vitest';+import { buildLlmsTxt, type LlmsTxtLink, marketingLinks } from './llms-txt';+import { MARKETING_MARKDOWN_PAGES } from './marketing-markdown-pages';++const docsLink: LlmsTxtLink = {+ url: 'https://openknowledge.ai/docs/get-started/quickstart.md',+ name: 'Quickstart',+ description: 'Install and open your first file.',+};+const postLink: LlmsTxtLink = {+ url: 'https://openknowledge.ai/blog/markdownlint-support.md',+ name: 'Keeping you and your agents in check with markdownlint',+};++function build(overrides: Partial<Parameters<typeof buildLlmsTxt>[0]> = {}) {+ return buildLlmsTxt({ docs: [docsLink], blogPosts: [postLink], ...overrides });+}++describe('llms.txt index', () => {+ test('opens with a title and a blockquote summary', () => {+ const [title, summary] = build().split('\n\n');+ expect(title).toBe('# OpenKnowledge');+ expect(summary.startsWith('> ')).toBe(true);+ expect(summary.length).toBeGreaterThan('> '.length);+ });++ test('groups entries under Docs, Blog and Product in that order', () => {+ const headings = [...build().matchAll(/^## (.+)$/gm)].map((match) => match[1]);+ expect(headings).toEqual(['Docs', 'Blog', 'Product']);+ });++ test('lists a page an agent can fetch, with its notes', () => {+ expect(build()).toContain(+ '- [Quickstart](https://openknowledge.ai/docs/get-started/quickstart.md): Install and open your first file.',+ );+ });++ test('neutralises raw HTML a description names in passing', () => {+ const body = build({+ docs: [+ {+ url: 'https://openknowledge.ai/docs/reference/components/accordion.md',+ name: 'Accordion',+ description: 'Standalone expand/collapse via native HTML5 <details>/<summary>.',+ },+ ],+ });+ expect(body).toContain('native HTML5 \\<details>/\\<summary>.');+ expect(body).not.toContain(' <details>');+ });++ test('neutralises raw HTML a title names in passing', () => {+ const body = build({+ docs: [+ {+ url: 'https://openknowledge.ai/docs/reference/components/accordion.md',+ name: 'The <details> element',+ },+ ],+ });+ expect(body).toContain('- [The \\<details> element](');+ expect(body).not.toContain('[The <details>');+ });++ test('keeps a bracket in a title inside the link label', () => {+ const body = build({+ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name: 'Arrays [and] slices' }],+ });+ expect(body).toContain('- [Arrays \\[and\\] slices](https://openknowledge.ai/docs/x.md)');+ });++ test.each([+ ['A [link](u) title', 'A link title'],+ ['Bang  in a title', 'Bang img in a title'],+ ])('reduces link syntax in a title to its words: %j', (name, expected) => {+ const bullet = build({ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name }] })+ .split('\n')+ .find((line) => line.startsWith('- [A ') || line.startsWith('- [Bang '));+ const tree = unified()+ .use(remarkParse)+ .use(remarkGfm)+ .parse(bullet ?? '');++ const links: string[] = [];+ const images: string[] = [];+ visit(tree, 'link', (node) => links.push(node.url));+ visit(tree, 'image', (node) => images.push(node.url));++ expect(links).toEqual(['https://openknowledge.ai/docs/x.md']);+ expect(images).toEqual([]);+ expect(bullet).toContain(`[${expected}]`);+ });++ test('leaves a bracket inside a code span alone', () => {+ const body = build({+ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name: 'The `items[]` prop' }],+ });+ const tree = unified().use(remarkParse).use(remarkGfm).parse(body);++ const spans: string[] = [];+ visit(tree, 'inlineCode', (node) => spans.push(node.value));+ expect(spans).toContain('items[]');+ });++ test.each([+ ['a parenthesis', 'https://openknowledge.ai/blog/note-)-end.md'],+ ['a space', 'https://openknowledge.ai/blog/pre post.md'],+ ])('round-trips a URL carrying %s', (_name, url) => {+ const body = build({ blogPosts: [{ url, name: 'Note' }] });+ const tree = unified().use(remarkParse).use(remarkGfm).parse(body);++ const urls: string[] = [];+ visit(tree, 'link', (node) => urls.push(node.url));+ expect(urls).toContain(url);+ });++ test('keeps a multi-line title on the bullet it belongs to', () => {+ const body = build({+ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name: 'Folded\n\ntitle' }],+ });+ expect(body).toContain('- [Folded title](https://openknowledge.ai/docs/x.md)');+ });++ test('omits the notes separator when an entry has no description', () => {+ expect(build()).toContain(+ '- [Keeping you and your agents in check with markdownlint]' ++ '(https://openknowledge.ai/blog/markdownlint-support.md)\n',+ );+ });++ test('every listed URL is absolute and points at a Markdown rendition', () => {+ for (const url of [...build().matchAll(/^- \[(?:\\.|[^\]\\])*]\(([^)]+)\)/gm)].map(+ (m) => m[1],+ )) {+ expect(url).toMatch(/^https:\/\/openknowledge\.ai\//);+ expect(url).toMatch(/\.md$/);+ }+ });++ test('lists no URL twice', () => {+ const urls = [...build().matchAll(/^- \[(?:\\.|[^\]\\])*]\(([^)]+)\)/gm)].map((m) => m[1]);+ expect(urls).toEqual([...new Set(urls)]);+ });++ test('carries every curated marketing rendition', () => {+ const body = build();+ for (const page of MARKETING_MARKDOWN_PAGES) {+ expect(body).toContain(`(https://openknowledge.ai${page.path})`);+ }+ });++ test('lists the blog index ahead of the posts it indexes', () => {+ const body = build();+ expect(body.indexOf('/blog.md)')).toBeLessThan(body.indexOf('/blog/markdownlint-support.md)'));+ });++ test('drops a section with nothing to list rather than emitting it bare', () => {+ const headings = [...build({ docs: [] }).matchAll(/^## (.+)$/gm)].map((match) => match[1]);+ expect(headings).toEqual(['Blog', 'Product']);+ });++ test('keeps the blog section when only the index is available', () => {+ expect(build({ blogPosts: [] })).toContain('## Blog');+ });++ test('resolves marketing paths against the site origin', () => {+ expect(marketingLinks('Product').map((link) => link.url)).toContain(+ 'https://openknowledge.ai/download.md',+ );+ });+});diff --git a/docs/src/lib/llms-txt.ts b/docs/src/lib/llms-txt.tsnew file mode 100644index 000000000..e22aa8968--- /dev/null+++ b/docs/src/lib/llms-txt.ts@@ -0,0 +1,68 @@+import {+ MARKETING_MARKDOWN_PAGES,+ type MarketingMarkdownPage,+} from '@/lib/marketing-markdown-pages';+import { escapeInlineProse, markdownLink } from '@/lib/mdx-serializer';+import { absoluteSiteUrl, SITE_DESCRIPTION, SITE_NAME } from '@/lib/site';++export interface LlmsTxtLink {+ readonly url: string;+ readonly name: string;+ readonly description?: string;+}++const LLMS_TXT_SECTIONS = ['Docs', 'Blog', 'Product'] as const;+export type LlmsTxtSection = (typeof LLMS_TXT_SECTIONS)[number];++const PREAMBLE =+ 'Every link below is the Markdown rendition of a page. Each rendition names ' ++ 'the page a reader would open in its first heading and in its ' ++ '`Link: rel="canonical"` header; that is the URL to cite. Each of those pages ' ++ 'also serves this same Markdown from its own URL to any request sending ' ++ '`Accept: text/markdown`.';++function marketingLink(page: MarketingMarkdownPage): LlmsTxtLink {+ return {+ url: absoluteSiteUrl(page.path),+ name: page.name,+ description: page.description,+ };+}++export function marketingLinks(section: LlmsTxtSection): LlmsTxtLink[] {+ return MARKETING_MARKDOWN_PAGES.filter((page) => page.section === section).map(marketingLink);+}++function inline(prose: string, { asLinkLabel = false } = {}): string {+ return escapeInlineProse(prose, { flattenLinks: asLinkLabel });+}++function bullet({ url, name, description }: LlmsTxtLink): string {+ const link = markdownLink(inline(name, { asLinkLabel: true }), url);+ return description ? `- ${link}: ${inline(description)}` : `- ${link}`;+}++function section(name: LlmsTxtSection, links: readonly LlmsTxtLink[]): string[] {+ return links.length === 0 ? [] : [`## ${name}`, links.map(bullet).join('\n')];+}++export function buildLlmsTxt({+ docs,+ blogPosts,+}: {+ docs: readonly LlmsTxtLink[];+ blogPosts: readonly LlmsTxtLink[];+}): string {+ const sections: Record<LlmsTxtSection, readonly LlmsTxtLink[]> = {+ Docs: docs,+ Blog: [...marketingLinks('Blog'), ...blogPosts],+ Product: marketingLinks('Product'),+ };++ return [+ `# ${SITE_NAME}`,+ `> ${SITE_DESCRIPTION}`,+ PREAMBLE,+ ...LLMS_TXT_SECTIONS.flatMap((name) => section(name, sections[name])),+ ].join('\n\n');+}diff --git a/docs/src/lib/markdown-component-coverage.test.ts b/docs/src/lib/markdown-component-coverage.test.tsnew file mode 100644index 000000000..99056a667--- /dev/null+++ b/docs/src/lib/markdown-component-coverage.test.ts@@ -0,0 +1,68 @@+import { describe, expect, test } from 'vitest';+import { readCorpus } from '@/lib/docs-corpus.test-helper';+import { componentNames } from '@/lib/mdx-serializer';+import { DOCS_SERIALIZER_REGISTRY } from '@/lib/mdx-serializer-registry';+import { getMDXComponents } from '@/mdx-components';++const REGISTRY = 'src/lib/mdx-serializer-registry.ts';++const runtimeMapKeys = Object.keys(getMDXComponents());+const corpus = await readCorpus();++const corpusUsage = new Map<string, string>();+for (const page of corpus) {+ for (const name of componentNames(page.source)) {+ if (!corpusUsage.has(name)) corpusUsage.set(name, `content/${page.slug || 'index'}`);+ }+}++const renderable = new Set([...runtimeMapKeys, ...corpusUsage.keys()]);+const registryKeys = new Set(Object.keys(DOCS_SERIALIZER_REGISTRY));++function originOf(name: string): string {+ const page = corpusUsage.get(name);+ if (runtimeMapKeys.includes(name)) {+ return page ? `component map, used in ${page}` : 'component map';+ }+ return `imported by ${page}`;+}++describe('markdown component coverage', () => {+ test('every component a page can render has a disposition', () => {+ const missing = [...renderable]+ .filter((name) => !registryKeys.has(name))+ .sort()+ .map((name) => `<${name}> (${originOf(name)})`);++ expect(+ missing,+ `components with no serializer disposition. Add each to DOCS_SERIALIZER_REGISTRY in ${REGISTRY} as 'flatten', 'drop', or a serializer that renders what it carries. ` ++ "A component that carries no children in source has nothing for 'flatten' to emit, so " ++ 'it needs a serializer: that includes anything whose content a remark plugin supplies ' ++ '(a snippet include, a generated type table), because the Markdown rendition reads RAW ' ++ "source and never runs those plugins — 'flatten' there ships the page with the section " ++ 'silently absent',+ ).toEqual([]);+ });++ test('every disposition names a component something can render', () => {+ const orphaned = [...registryKeys].filter((name) => !renderable.has(name)).sort();++ expect(+ orphaned,+ `dispositions in ${REGISTRY} for components neither the map nor the corpus has; restore the component or drop the entry`,+ ).toEqual([]);+ });++ test('the component map is read at runtime, spread keys included', () => {+ expect(runtimeMapKeys, 'the component map census is not seeing spread keys').toContain(+ 'Callout',+ );+ expect(runtimeMapKeys.length).toBeGreaterThan(40);+ });++ test('the corpus census reaches the published tree', () => {+ expect(corpus.length).toBeGreaterThan(50);+ expect(corpusUsage.size).toBeGreaterThan(10);+ });+});diff --git a/docs/src/lib/markdown-fidelity.real-source.test.ts b/docs/src/lib/markdown-fidelity.real-source.test.tsnew file mode 100644index 000000000..508ddbeca--- /dev/null+++ b/docs/src/lib/markdown-fidelity.real-source.test.ts@@ -0,0 +1,40 @@+import { describe, expect, test } from 'vitest';+import { getLLMText } from '@/lib/get-llm-text';+import type { FidelityViolation } from '@/lib/markdown-fidelity.test-helper';+import { markdownFidelityViolations } from '@/lib/markdown-fidelity.test-helper';+import { source } from '@/lib/source';++const pages = source.getPages();++describe('every page the real loader hands to the Markdown pipeline', () => {+ test('is a census, so a broken content glob cannot vacuously pass this suite', () => {+ expect(pages.length).toBeGreaterThan(50);+ });++ test('reads as Markdown, with no JSX, indented code, base64 or source-form links', async () => {+ const violations: FidelityViolation[] = [];+ for (const page of pages) {+ violations.push(...markdownFidelityViolations(page.url, await getLLMText(page)));+ }+ expect(violations.map((violation) => violation.message)).toEqual([]);+ });+});++describe('the real loader derives a page from its file', () => {+ test('nests the URL by directory rather than flattening or keeping the extension', () => {+ const page = source.getPage(['reference', 'components', 'callout']);+ expect(page?.url).toBe('/docs/reference/components/callout');+ expect(page?.data.title).toBe('Callout');+ });++ test('reads the title from frontmatter rather than from the filename', () => {+ const page = source.getPage(['get-started', 'quickstart']);+ expect(page?.url).toBe('/docs/get-started/quickstart');+ expect(page?.data.title).toBe('Quickstart');+ });++ test('serves raw source, not the compiled rendition, for the document body', async () => {+ const page = source.getPage(['reference', 'components', 'callout']);+ await expect(page?.data.getText('raw')).resolves.toContain('title: "Callout"');+ });+});diff --git a/docs/src/lib/markdown-fidelity.test-helper.ts b/docs/src/lib/markdown-fidelity.test-helper.tsnew file mode 100644index 000000000..cc1642ead--- /dev/null+++ b/docs/src/lib/markdown-fidelity.test-helper.ts@@ -0,0 +1,221 @@+import type { Root } from 'mdast';+import remarkGfm from 'remark-gfm';+import remarkMdx from 'remark-mdx';+import remarkParse from 'remark-parse';+import { unified } from 'unified';+import { visit } from 'unist-util-visit';++type FidelityViolationKind =+ | 'jsx-element'+ | 'indented-code'+ | 'base64-attribute'+ | 'source-form-link'+ | 'unparseable';++export interface FidelityViolation {+ readonly kind: FidelityViolationKind;+ readonly message: string;+}++const asMarkdown = unified().use(remarkParse).use(remarkGfm);+const asMdx = unified().use(remarkParse).use(remarkMdx).use(remarkGfm);++const ABSOLUTE_URL = /^[a-zA-Z][a-zA-Z0-9+.-]*:|^\/\//;++const SOURCE_EXTENSION = /\.mdx($|[#?])/;++const BASE64_ATTRIBUTE = /([A-Za-z][\w-]*)=(["'])([A-Za-z0-9+/]{64,}={0,2})\2/g;++type Range = readonly [number, number];++export function markdownFidelityViolations(name: string, markdown: string): FidelityViolation[] {+ const violations: FidelityViolation[] = [];+ const report = (+ kind: FidelityViolationKind,+ line: number,+ construct: string,+ fix: string,+ ): void => {+ violations.push({ kind, message: `${name}:${line} — ${construct}. Fix: ${fix}` });+ };++ let tree: Root;+ try {+ tree = asMarkdown.parse(markdown);+ } catch (error) {+ report(+ 'unparseable',+ 1,+ `the document does not parse as Markdown (${(error as Error).message})`,+ 'find the producer that emitted this document and correct what it wrote',+ );+ return violations;+ }++ const literalRegions: Range[] = [];++ visit(tree, (node) => {+ const start = node.position?.start.offset;+ const end = node.position?.end.offset;+ const line = node.position?.start.line ?? 1;++ if (node.type === 'code' || node.type === 'inlineCode') {+ if (start !== undefined && end !== undefined) literalRegions.push([start, end]);+ } else if (node.type === 'html' && node.value.trimStart().startsWith('<!--')) {+ if (start !== undefined && end !== undefined) literalRegions.push([start, end]);+ } else if (node.type === 'link' && start !== undefined && markdown[start] === '<') {+ if (end !== undefined) literalRegions.push([start, end]);+ }++ if (node.type === 'code') {+ const opener = start === undefined ? '' : markdown[start];+ if (opener !== '`' && opener !== '~') {+ report(+ 'indented-code',+ line,+ 'a code block reached the four-space column instead of being fenced, so CommonMark reads the prose around it as code',+ 'fence the block; if a custom serializer wrote it, that output is emitted verbatim and has to carry its own fence',+ );+ }+ }++ if (node.type === 'link' || node.type === 'image' || node.type === 'definition') {+ if (!ABSOLUTE_URL.test(node.url)) {+ report(+ 'source-form-link',+ line,+ `the link target ${JSON.stringify(node.url)} is relative, so it breaks as soon as the Markdown is read away from its own URL`,+ 'absolutise it against the page URL, as the serializer does for every link it rewrites',+ );+ } else if (SOURCE_EXTENSION.test(node.url)) {+ report(+ 'source-form-link',+ line,+ `the link target ${JSON.stringify(node.url)} still points at MDX source rather than at a page`,+ 'drop the source extension so the target is the URL a reader can open',+ );+ }+ }+ });++ const masked = maskRanges(markdown, literalRegions);++ try {+ visit(asMdx.parse(masked), (node) => {+ if (node.type !== 'mdxJsxFlowElement' && node.type !== 'mdxJsxTextElement') return;+ report(+ 'jsx-element',+ node.position?.start.line ?? 1,+ `<${node.name ?? 'fragment'}> survived into the served Markdown as a raw tag`,+ 'give the component a serializer disposition that renders what it contributes, or drop it',+ );+ });+ } catch (error) {+ report(+ 'unparseable',+ 1,+ `the document does not parse as MDX (${(error as Error).message})`,+ 'escape the raw markup the producer interpolated; author prose reaches the document as Markdown, not as HTML',+ );+ }++ for (const match of masked.matchAll(BASE64_ATTRIBUTE)) {+ report(+ 'base64-attribute',+ lineAt(masked, match.index),+ `the attribute ${match[1]} carries ${match[3].length} bytes of base64 rather than content`,+ 'restore what the attribute encodes — a preview fence belongs in the document as a fence',+ );+ }++ return violations;+}++function maskRanges(source: string, ranges: readonly Range[]): string {+ if (ranges.length === 0) return source;+ const chars = source.split('');+ for (const [start, end] of ranges) {+ for (let i = start; i < end && i < chars.length; i++) {+ if (chars[i] !== '\n') chars[i] = ' ';+ }+ }+ return chars.join('');+}++function lineAt(source: string, offset: number): number {+ let line = 1;+ for (let i = 0; i < offset; i++) if (source[i] === '\n') line += 1;+ return line;+}++export interface FidelityFixture {+ readonly name: string;+ readonly markdown: string;+ readonly expect: readonly FidelityViolationKind[];+}++export const FIDELITY_FIXTURES: readonly FidelityFixture[] = [+ {+ name: 'a component that reached the reader as a tag',+ markdown: 'Read this <Callout type="info">before you start</Callout> first.\n',+ expect: ['jsx-element'],+ },+ {+ name: 'a block that reached the four-space column',+ markdown: 'Run the installer:\n\n ok init --here\n',+ expect: ['indented-code'],+ },+ {+ name: 'a preview compiled to base64 and escaped past the JSX check',+ markdown: `A preview: \\<HtmlPreview code="PGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+CgPGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+Cg" />\n`,+ expect: ['base64-attribute'],+ },+ {+ name: 'a link that never left the content tree',+ markdown: 'See the [editor](../features/editor) reference.\n',+ expect: ['source-form-link'],+ },+ {+ name: 'a link still in source form',+ markdown: 'See the [editor](../features/editor.mdx#ask-ai) reference.\n',+ expect: ['source-form-link'],+ },+ {+ name: 'a link pointing at MDX source on another host',+ markdown: 'See [the spec](https://example.com/docs/spec.mdx).\n',+ expect: ['source-form-link'],+ },+ {+ name: 'a block at the four-space column that also holds a component',+ markdown: 'Run this:\n\n <Callout>read the release notes</Callout>\n',+ expect: ['indented-code'],+ },+ {+ name: 'a preview attribute shown inside a fence, as documentation of it',+ markdown:+ '````mdx\n<HtmlPreview code="PGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+CgPGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+Cg" />\n````\n',+ expect: [],+ },+ {+ name: 'a component named inside an inline code span',+ markdown: 'Same props as `<Accordion>` — pick whichever vocabulary fits.\n',+ expect: [],+ },+ {+ name: 'a four-backtick fence wrapping a three-backtick one',+ markdown:+ '````text\n```mdx\n<Mirror src="specs/architecture" />\n```\n````\n\nProse after the fence.\n',+ expect: [],+ },+ {+ name: 'prose nested four list levels deep',+ markdown: '- one\n - two\n - three\n - four, indented past the code column\n',+ expect: [],+ },+ {+ name: 'an autolink and an HTML comment, which an MDX parser rejects outright',+ markdown:+ 'Download it from <https://openknowledge.ai/download>.\n\n<!-- linux-arm64: no published build -->\n',+ expect: [],+ },+];diff --git a/docs/src/lib/markdown-fidelity.test.ts b/docs/src/lib/markdown-fidelity.test.tsnew file mode 100644index 000000000..57238cd04--- /dev/null+++ b/docs/src/lib/markdown-fidelity.test.ts@@ -0,0 +1,67 @@+import { describe, expect, test, vi } from 'vitest';+import { type CorpusSourcePage, corpusSourcePages, readCorpus } from './docs-corpus.test-helper.ts';+import {+ FIDELITY_FIXTURES,+ type FidelityViolation,+ markdownFidelityViolations,+} from './markdown-fidelity.test-helper.ts';++const pages = corpusSourcePages(await readCorpus());+const bySlug = new Map(pages.map((page) => [page.url, page]));++vi.doMock('@/lib/source', () => ({+ source: {+ getPage: (slug: string[]) => bySlug.get(`/docs/${slug.join('/')}`.replace(/\/$/, '')),+ getPages: () => pages,+ generateParams: () => pages.map((page) => ({ slug: slugOf(page) })),+ },+}));++vi.doMock('@/lib/marketing-blog-index', () => ({+ blogPostLinks: async () => [+ { url: 'https://openknowledge.ai/blog/some-post.md', name: 'Some post' },+ ],+}));++const { GET } = await import('@/app/llms.mdx/[...slug]/route.ts');+const { GET: llmsTxtGet } = await import('@/app/llms.txt/route.ts');+const { GET: llmsFullGet } = await import('@/app/llms-full.txt/route.ts');++function slugOf(page: CorpusSourcePage): string[] {+ return page.url.slice('/docs/'.length).split('/').filter(Boolean);+}++async function serve(slug: string[]): Promise<string> {+ const response = await GET(new Request(`https://openknowledge.ai/docs/${slug.join('/')}.md`), {+ params: Promise.resolve({ slug }),+ });+ return response.text();+}++const served = new Map<string, string>();+for (const page of pages) served.set(page.url, await serve(slugOf(page)));+served.set('/docs/<unknown>', await serve(['no', 'such', 'page']));+served.set('/llms.txt', await (await llmsTxtGet()).text());+served.set('/llms-full.txt', await (await llmsFullGet()).text());++describe('the fidelity oracle', () => {+ test.each(FIDELITY_FIXTURES)('$name', ({ name, markdown, expect: expected }) => {+ const kinds = markdownFidelityViolations(name, markdown).map((violation) => violation.kind);+ expect(kinds).toEqual([...expected]);+ });+});++describe('every page the docs app serves as Markdown', () => {+ test('is a census, so a broken content glob cannot vacuously pass this suite', () => {+ expect(served.size).toBeGreaterThan(50);+ for (const [slug, markdown] of served) expect(markdown.length, slug).toBeGreaterThan(0);+ });++ test('reads as Markdown, with no JSX, indented code, base64 or source-form links', () => {+ const violations: FidelityViolation[] = [];+ for (const [slug, markdown] of served) {+ violations.push(...markdownFidelityViolations(slug, markdown));+ }+ expect(violations.map((violation) => violation.message)).toEqual([]);+ });+});diff --git a/docs/src/lib/markdown-response.test.ts b/docs/src/lib/markdown-response.test.tsnew file mode 100644index 000000000..1762341fd--- /dev/null+++ b/docs/src/lib/markdown-response.test.ts@@ -0,0 +1,48 @@+import { describe, expect, test } from 'vitest';+import {+ MARKDOWN_CACHE_CONTROL,+ MARKDOWN_CONTENT_TYPE,+ markdownResponse,+} from './markdown-response.ts';++describe('markdownResponse', () => {+ test('serves the body as markdown with an edge-cacheable directive', async () => {+ const res = markdownResponse('# Hi');++ expect(res.status).toBe(200);+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);+ expect(await res.text()).toBe('# Hi');+ });++ test('points agents at the human page with a canonical Link header', () => {+ const res = markdownResponse('# Hi', {+ canonicalUrl: 'https://openknowledge.ai/docs/get-started/overview',+ });++ expect(res.headers.get('link')).toBe(+ '<https://openknowledge.ai/docs/get-started/overview>; rel="canonical"',+ );+ });++ test('omits the canonical Link when the response renders no single page', () => {+ expect(markdownResponse('# Hi').headers.get('link')).toBeNull();+ });++ test('carries the full header contract on an error status too', () => {+ const res = markdownResponse('# 404', { status: 404 });++ expect(res.status).toBe(404);+ expect(res.headers.get('content-type')).toBe(MARKDOWN_CONTENT_TYPE);+ expect(res.headers.get('cache-control')).toBe(MARKDOWN_CACHE_CONTROL);+ expect(res.headers.get('vary')).toBe('Accept');+ });++ test('varies on Accept, the correct signal this representation can carry', () => {+ expect(markdownResponse('# Hi').headers.get('vary')).toBe('Accept');+ });++ test('charset is explicit — text/markdown has no default one', () => {+ expect(MARKDOWN_CONTENT_TYPE).toContain('charset=utf-8');+ });+});diff --git a/docs/src/lib/markdown-response.ts b/docs/src/lib/markdown-response.tsnew file mode 100644index 000000000..b50169b91--- /dev/null+++ b/docs/src/lib/markdown-response.ts@@ -0,0 +1,25 @@+export const MARKDOWN_CONTENT_TYPE = 'text/markdown; charset=utf-8';++export const MARKDOWN_CACHE_CONTROL =+ 'public, max-age=0, s-maxage=3600, stale-while-revalidate=86400';++const MARKDOWN_VARY = 'Accept';++export interface MarkdownResponseInit {+ canonicalUrl?: string;+ status?: number;+}++function markdownHeaders({ canonicalUrl }: MarkdownResponseInit = {}): Headers {+ const headers = new Headers({+ 'Content-Type': MARKDOWN_CONTENT_TYPE,+ 'Cache-Control': MARKDOWN_CACHE_CONTROL,+ Vary: MARKDOWN_VARY,+ });+ if (canonicalUrl) headers.set('Link', `<${canonicalUrl}>; rel="canonical"`);+ return headers;+}++export function markdownResponse(body: string, init: MarkdownResponseInit = {}): Response {+ return new Response(body, { status: init.status ?? 200, headers: markdownHeaders(init) });+}diff --git a/docs/src/lib/markdown-routes.ts b/docs/src/lib/markdown-routes.tsnew file mode 100644index 000000000..7fd27db8f--- /dev/null+++ b/docs/src/lib/markdown-routes.ts@@ -0,0 +1,47 @@+const DOCS_ROUTE = '/docs';++export const PAGE_MARKDOWN_ROUTE = '/llms.mdx';++const MARKDOWN_TWIN_SUFFIXES = ['.html.md', '.md', '.mdx'];++export const NO_MARKDOWN_PREFIXES = [`${DOCS_ROUTE}/changelog`] as const;++function noMarkdownPrefixFor(docsPath: string): string | null {+ return (+ NO_MARKDOWN_PREFIXES.find(+ (prefix) => docsPath === prefix || docsPath.startsWith(`${prefix}/`),+ ) ?? null+ );+}++export function unrenditionedSubtree(slug: readonly string[]): string | null {+ return noMarkdownPrefixFor(`${DOCS_ROUTE}/${slug.join('/')}`);+}++interface RewriteRule {+ source: string;+ destination: string;+}++export function markdownRewrites(): RewriteRule[] {+ return MARKDOWN_TWIN_SUFFIXES.map((suffix) => ({+ source: `${DOCS_ROUTE}/:path*${suffix}`,+ destination: `${PAGE_MARKDOWN_ROUTE}/:path*`,+ }));+}++function trimTrailingSlashes(pathname: string): string {+ let end = pathname.length;+ while (end > 1 && pathname.charCodeAt(end - 1) === 47) end -= 1;+ return pathname.slice(0, end);+}++export function markdownHandlerForPage(pathname: string): string | null {+ const path = trimTrailingSlashes(pathname);++ if (MARKDOWN_TWIN_SUFFIXES.some((suffix) => path.endsWith(suffix))) return null;+ if (noMarkdownPrefixFor(path)) return null;+ if (!path.startsWith(`${DOCS_ROUTE}/`)) return null;++ return `${PAGE_MARKDOWN_ROUTE}${path.slice(DOCS_ROUTE.length)}`;+}diff --git a/docs/src/lib/marketing-blog-index.test.ts b/docs/src/lib/marketing-blog-index.test.tsnew file mode 100644index 000000000..483f99728--- /dev/null+++ b/docs/src/lib/marketing-blog-index.test.ts@@ -0,0 +1,462 @@+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import {+ attemptPolicy,+ BLOG_FEED_URL,+ blogPostLinks,+ EmptyBlogError,+ fetchBlogPostLinks,+} from './marketing-blog-index';++interface FeedItem {+ title: string;+ link: string;+ description?: string;+}++const cdata = (text: string) => `<![CDATA[${text.replaceAll(']]>', ']]]]><![CDATA[>')}]]>`;++function feed(...items: FeedItem[]): string {+ return [+ '<?xml version="1.0" encoding="UTF-8"?>',+ '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">',+ ' <channel>',+ ' <title>OpenKnowledge Blog</title>',+ ' <link>https://openknowledge.ai/blog</link>',+ ' <description>News, releases, and engineering notes from the OpenKnowledge team.</description>',+ ' <atom:link href="https://openknowledge.ai/blog/rss.xml" rel="self" type="application/rss+xml" />',+ ...items.map((item) =>+ [+ ' <item>',+ ` <title>${cdata(item.title)}</title>`,+ ` <link>${item.link}</link>`,+ ...(item.description === undefined+ ? []+ : [` <description>${cdata(item.description)}</description>`]),+ ' <pubDate>Tue, 21 Jul 2026 00:00:00 GMT</pubDate>',+ ` <guid isPermaLink="true">${item.link}</guid>`,+ ' <category>Release</category>',+ ' </item>',+ ].join('\n'),+ ),+ ' </channel>',+ '</rss>',+ ].join('\n');+}++const MARKDOWNLINT: FeedItem = {+ title: 'Keeping you and your agents in check with markdownlint',+ link: 'https://openknowledge.ai/blog/markdownlint-support',+ description:+ "Native markdownlint support in Open Knowledge: your existing config works as-is, a GUI builds one if you don't have it, and your agents see the same rules you do.",+};++const VISIMER: FeedItem = {+ title: 'Edit Mermaid diagrams visually with Visimer',+ link: 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually',+ description:+ 'Edit Mermaid diagrams visually with Visimer, the open source library that powers the Mermaid editing experience in OpenKnowledge.',+};++const LIVE_FEED = feed(MARKDOWNLINT, VISIMER);++function respondWith(body: string, init: ResponseInit = {}) {+ const fetchMock = vi.fn(async () => new Response(body, init));+ vi.stubGlobal('fetch', fetchMock);+ return fetchMock;+}++beforeEach(() => {+ vi.useFakeTimers();+});++async function settle<T>(pending: Promise<T>): Promise<T> {+ pending.catch(() => undefined);+ await vi.runAllTimersAsync();+ return pending;+}++afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllGlobals();+ vi.unstubAllEnvs();+ vi.restoreAllMocks();+});++describe('blog posts read from the marketing zone feed', () => {+ test('lists each post as the Markdown rendition of its page, under its own title', async () => {+ respondWith(LIVE_FEED);+ expect(await fetchBlogPostLinks(3600)).toEqual([+ {+ url: 'https://openknowledge.ai/blog/markdownlint-support.md',+ name: 'Keeping you and your agents in check with markdownlint',+ description: MARKDOWNLINT.description,+ },+ {+ url: 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',+ name: 'Edit Mermaid diagrams visually with Visimer',+ description: VISIMER.description,+ },+ ]);+ });++ test('reads a title the encoder split across two CDATA sections', async () => {+ respondWith(feed({ title: 'before ]]> after', link: 'https://openknowledge.ai/blog/a-post' }));+ expect((await fetchBlogPostLinks(3600))[0].name).toBe('before ]]> after');+ });++ test('decodes XML entities in a link', async () => {+ respondWith(feed({ title: 'A', link: 'https://openknowledge.ai/blog/a&b' }));+ expect((await fetchBlogPostLinks(3600))[0].url).toBe('https://openknowledge.ai/blog/a&b.md');+ });++ test('carries no description when the feed has none to give', async () => {+ respondWith(feed({ title: 'A', link: 'https://openknowledge.ai/blog/a-post' }));+ expect((await fetchBlogPostLinks(3600))[0]).not.toHaveProperty('description');+ });++ test('warns when the feed ends after its last post but before the closing tag', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const whole = feed(MARKDOWNLINT);+ respondWith(whole.slice(0, whole.lastIndexOf('</item>') + '</item>'.length));+ expect((await fetchBlogPostLinks(3600)).length).toBe(1);+ expect(warn.mock.calls.flat().join('\n')).toMatch(/ends before <\/rss>/);+ });++ test('does not cry truncation over a closing tag written inside a title or a summary', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ respondWith(+ feed(MARKDOWNLINT, {+ ...VISIMER,+ description: 'A summary that mentions </description>, </item> and <item> in passing.',+ }),+ );+ expect((await fetchBlogPostLinks(3600)).length).toBe(2);+ expect(warn).not.toHaveBeenCalled();+ });++ test('reads a closing tag written inside a title or a summary as text', async () => {+ respondWith(+ feed(+ {+ title: 'How </title> and </item> survive CDATA',+ link: 'https://openknowledge.ai/blog/closing-tags',+ description: 'A summary that mentions </description>, </item> and <item> in passing.',+ },+ VISIMER,+ ),+ );+ expect(await fetchBlogPostLinks(3600)).toEqual([+ {+ url: 'https://openknowledge.ai/blog/closing-tags.md',+ name: 'How </title> and </item> survive CDATA',+ description: 'A summary that mentions </description>, </item> and <item> in passing.',+ },+ {+ url: 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',+ name: VISIMER.title,+ description: VISIMER.description,+ },+ ]);+ });++ test('an item left unclosed ends where the next one opens, so its successor is not lost', async () => {+ respondWith(feed(MARKDOWNLINT, VISIMER).replace('</item>', ''));+ expect((await fetchBlogPostLinks(3600)).map((post) => post.url)).toEqual([+ 'https://openknowledge.ai/blog/markdownlint-support.md',+ 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',+ ]);+ });++ test('says so when a complete feed leaves an item unclosed, so a malformation is not silent', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const whole = feed(MARKDOWNLINT, VISIMER);+ const lastClose = whole.lastIndexOf('</item>');+ respondWith(whole.slice(0, lastClose) + whole.slice(lastClose + '</item>'.length));+ await fetchBlogPostLinks(3600);+ expect(warn.mock.calls.flat().join('\n')).toMatch(/leaves an <item> unclosed/);+ });++ test('skips an item that is not a post and keeps the rest, saying which it skipped', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ respondWith(+ feed(+ MARKDOWNLINT,+ { title: 'Meet the team', link: 'https://openknowledge.ai/team' },+ VISIMER,+ ),+ );++ expect((await fetchBlogPostLinks(3600)).map((post) => post.url)).toEqual([+ 'https://openknowledge.ai/blog/markdownlint-support.md',+ 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',+ ]);+ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn.mock.calls[0]?.[0]).toContain('"https://openknowledge.ai/team" is not a post at');+ });++ test('keeps the link out of a log line: one line, bounded, quoted', async () => {+ const link = `https://ok.example/blog/${'x'.repeat(500)}\nforged: line`;+ respondWith(feed({ title: 'A', link }));++ const error = (await fetchBlogPostLinks(3600).catch((thrown: unknown) => thrown)) as Error;+ expect(error.message).not.toContain('\n');+ expect(error.message).toContain(JSON.stringify(link.replace(/\s+/g, ' ').slice(0, 200)));+ expect(error.message).not.toContain('x'.repeat(201));+ });++ test('asks the marketing zone for its own feed, revalidating hourly', async () => {+ const fetchMock = respondWith(LIVE_FEED);+ await fetchBlogPostLinks(3600);+ expect(fetchMock).toHaveBeenCalledWith(+ BLOG_FEED_URL,+ expect.objectContaining({ next: { revalidate: 3600 } }),+ );+ expect(BLOG_FEED_URL).toBe('https://openknowledge.ai/blog/rss.xml');+ });++ test('gives up rather than hanging on an origin that never answers', async () => {+ const fetchMock = respondWith(LIVE_FEED);+ await fetchBlogPostLinks(3600);+ const { signal } = fetchMock.mock.calls[0][1] as RequestInit;+ expect(signal).toBeInstanceOf(AbortSignal);+ });++ test('fails on an error response instead of reporting an empty blog', async () => {+ respondWith('nope', { status: 503 });+ await expect(fetchBlogPostLinks(3600)).rejects.toThrow('responded 503');+ });++ test('reads an empty feed as a content state', async () => {+ respondWith(feed());+ await expect(fetchBlogPostLinks(3600)).rejects.toThrow(EmptyBlogError);+ });++ test.each([+ [+ 'an error page served with a 200',+ '<!doctype html><html><body><h1>Application error</h1></body></html>',+ 'is not an RSS feed',+ ],+ [+ 'a canonical host that drifted between the two apps',+ feed({ title: 'A', link: 'https://ok.example/blog/a-post' }),+ 'canonical host that drifted',+ ],+ [+ 'a link outside the blog',+ feed({ title: 'A', link: 'https://openknowledge.ai/team/someone' }),+ 'is not a post at',+ ],+ [+ 'post URLs that gained a segment',+ feed({ title: 'A', link: 'https://openknowledge.ai/blog/2026/a-post' }),+ 'the post URL shape changed',+ ],+ [+ 'an item with no title',+ feed({ title: '', link: 'https://openknowledge.ai/blog/a-post' }),+ 'no <title> or no <link>',+ ],+ [+ 'a response cut inside the first item',+ feed(MARKDOWNLINT).slice(+ 0,+ feed(MARKDOWNLINT).indexOf('<link>', feed(MARKDOWNLINT).indexOf('<item')),+ ),+ 'ends before </rss>, so it is truncated',+ ],+ [+ 'a response cut in the channel header, before any item',+ feed(MARKDOWNLINT).slice(0, feed(MARKDOWNLINT).indexOf('<item')),+ 'ends before </rss>, so it is truncated',+ ],+ [+ 'a complete feed whose only item never closes',+ feed(MARKDOWNLINT).replace('</item>', ''),+ 'leaves an <item> unclosed, so it is malformed',+ ],+ ])('reads %s as drift, not as an empty blog', async (_name, body, expected) => {+ respondWith(body);+ const error = await settle(fetchBlogPostLinks(3600)).catch((thrown: unknown) => thrown);++ expect(error, 'resolved instead of throwing — these links would have published').toBeInstanceOf(+ Error,+ );+ expect(error).not.toBeInstanceOf(EmptyBlogError);+ expect((error as Error).message).toContain(expected);+ });++ test.each([404, 401])('reports a %s as an Error naming the status, once', async (status) => {+ const fetchMock = vi.fn(async () => new Response('nope', { status }));+ vi.stubGlobal('fetch', fetchMock);++ const error = await settle(fetchBlogPostLinks(3600, true)).catch((thrown: unknown) => thrown);++ expect(error).toBeInstanceOf(Error);+ expect((error as Error).message).toContain(`responded ${status}`);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });++ test('spaces the retries rather than firing them in one burst', async () => {+ const at: number[] = [];+ vi.stubGlobal(+ 'fetch',+ vi.fn(async () => {+ at.push(Date.now());+ throw new TypeError('fetch failed');+ }),+ );++ await settle(fetchBlogPostLinks(3600, true)).catch(() => undefined);++ expect(at).toHaveLength(3);+ expect([at[1] - at[0], at[2] - at[1]]).toEqual([250, 500]);+ });++ test.each([+ [true, { attempts: 3, timeoutMs: 20_000 }],+ [false, { attempts: 1, timeoutMs: 5_000 }],+ ])('building=%s takes %o', (building, expected) => {+ expect(attemptPolicy(building)).toEqual(expected);+ });++ test('retries a dropped connection before believing it', async () => {+ let calls = 0;+ vi.stubGlobal(+ 'fetch',+ vi.fn(async () => {+ calls += 1;+ if (calls < 3) throw new TypeError('fetch failed');+ return new Response(LIVE_FEED);+ }),+ );++ expect(await settle(fetchBlogPostLinks(3600, true))).toHaveLength(2);+ expect(calls).toBe(3);+ });++ test('does not retry a document it has already read and rejected', async () => {+ const fetchMock = respondWith('<!doctype html><title>oops</title>');+ await expect(settle(fetchBlogPostLinks(3600, true))).rejects.toThrow('is not an RSS feed');+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });++ test('a render gives up on the first failure rather than holding itself open', async () => {+ const fetchMock = vi.fn(async () => {+ throw new TypeError('fetch failed');+ });+ vi.stubGlobal('fetch', fetchMock);++ await expect(fetchBlogPostLinks(3600)).rejects.toThrow('fetch failed');+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });+});++describe('failure posture', () => {+ function refuseConnection() {+ const fetchMock = vi.fn(async () => {+ throw new TypeError('fetch failed');+ });+ vi.stubGlobal('fetch', fetchMock);+ return fetchMock;+ }++ function building(env: { CI?: string; VERCEL?: string; VERCEL_ENV?: string } = {}) {+ vi.stubEnv('NEXT_PHASE', 'phase-production-build');+ vi.stubEnv('CI', env.CI ?? '');+ vi.stubEnv('VERCEL', env.VERCEL ?? '');+ vi.stubEnv('VERCEL_ENV', env.VERCEL_ENV ?? '');+ }++ test('propagates a fetch failure so a revalidate keeps the last good index', async () => {+ refuseConnection();+ await expect(settle(blogPostLinks(3600))).rejects.toThrow('fetch failed');+ });++ test('builds without the posts when the marketing origin is unreachable', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ building();+ refuseConnection();++ expect(await settle(blogPostLinks(3600))).toEqual([]);+ expect(warn.mock.calls[0]?.[0]).toContain('could not be read after 3 attempts');+ });++ test('returns the posts normally during a build that can reach the origin', async () => {+ building();+ respondWith(LIVE_FEED);+ expect(await settle(blogPostLinks(3600))).toHaveLength(2);+ });++ test('a CI build off Vercel never reads the feed, since it publishes nothing', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ building({ CI: 'true' });+ const fetchMock = respondWith(LIVE_FEED);++ expect(await settle(blogPostLinks(3600))).toEqual([]);+ expect(fetchMock).not.toHaveBeenCalled();+ expect(warn.mock.calls[0]?.[0]).toContain('CI build off Vercel');+ });++ test('a Vercel build, which also sets CI, still reads the feed', async () => {+ building({ CI: '1', VERCEL: '1', VERCEL_ENV: 'preview' });+ const fetchMock = respondWith(LIVE_FEED);++ expect(await settle(blogPostLinks(3600))).toHaveLength(2);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });++ test('fails the production build rather than deploying a blogless index', async () => {+ building({ CI: '1', VERCEL: '1', VERCEL_ENV: 'production' });+ const fetchMock = refuseConnection();++ await expect(settle(blogPostLinks(3600))).rejects.toThrow('refusing to publish');+ expect(fetchMock).toHaveBeenCalledTimes(3);+ });++ test('the fatal error names the URL, the remedy and the escape hatch', async () => {+ building({ VERCEL_ENV: 'production' });+ refuseConnection();++ const error = (await settle(blogPostLinks(3600)).catch((thrown: unknown) => thrown)) as Error;+ expect(error.message).toContain(BLOG_FEED_URL);+ expect(error.message).toContain('redeploy this commit');+ expect(error.message).toContain('OK_ALLOW_BLOGLESS_LLMS_TXT=1');++ const cause = error.cause as Error;+ expect(cause.message).toContain('after 3 attempts');+ expect((cause.cause as Error).message).toBe('fetch failed');+ });++ test('publishes a production build when the blog is legitimately empty', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ building({ VERCEL_ENV: 'production' });+ respondWith(feed());++ expect(await settle(blogPostLinks(3600))).toEqual([]);+ expect(warn.mock.calls[0]?.[0]).toContain('the blog lists no posts');+ });++ test('lets an operator publish without the posts, fail-closed', async () => {+ building({ VERCEL_ENV: 'production' });+ refuseConnection();++ vi.stubEnv('OK_ALLOW_BLOGLESS_LLMS_TXT', 'true');+ await expect(settle(blogPostLinks(3600))).rejects.toThrow('refusing to publish');++ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.stubEnv('OK_ALLOW_BLOGLESS_LLMS_TXT', '1');+ expect(await settle(blogPostLinks(3600))).toEqual([]);+ });++ test.each([+ 'preview',+ 'development',+ ])('still builds without the posts on a %s deployment', async (env) => {+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ building({ VERCEL_ENV: env });+ refuseConnection();++ expect(await settle(blogPostLinks(3600))).toEqual([]);+ });+});diff --git a/docs/src/lib/marketing-blog-index.ts b/docs/src/lib/marketing-blog-index.tsnew file mode 100644index 000000000..5402dd184--- /dev/null+++ b/docs/src/lib/marketing-blog-index.ts@@ -0,0 +1,246 @@+import { PHASE_PRODUCTION_BUILD } from 'next/constants';+import type { LlmsTxtLink } from '@/lib/llms-txt';+import { SITE_URL } from '@/lib/site';++export const BLOG_FEED_URL = `${SITE_URL}/blog/rss.xml`;++const RENDER_TIMEOUT_MS = 5_000;+const BUILD_TIMEOUT_MS = 20_000;++const BUILD_ATTEMPTS = 3;+const RETRY_BASE_MS = 250;++const CDATA_OPEN = '<![CDATA[';+const CDATA_CLOSE = ']]>';+const CDATA_SECTION = /<!\[CDATA\[([\s\S]*?)\]\]>/g;++const LINK_IN_MESSAGE_MAX = 200;++function decodeXml(value: string): string {+ return value+ .replace(/</g, '<')+ .replace(/>/g, '>')+ .replace(/"/g, '"')+ .replace(/'/g, "'")+ .replace(/&/g, '&');+}++function textOf(value: string): string {+ const sections = [...value.matchAll(CDATA_SECTION)];+ if (sections.length > 0) return sections.map((section) => section[1]).join('');+ return decodeXml(value.trim());+}++function matchOutsideCdata(xml: string, needle: RegExp, from: number): RegExpExecArray | null {+ let at = from;+ for (;;) {+ needle.lastIndex = at;+ const match = needle.exec(xml);+ if (!match) return null;+ const cdata = xml.indexOf(CDATA_OPEN, at);+ if (cdata === -1 || match.index < cdata) return match;+ const end = xml.indexOf(CDATA_CLOSE, cdata + CDATA_OPEN.length);+ if (end === -1) return null;+ at = end + CDATA_CLOSE.length;+ }+}++function elements(xml: string, name: string): { found: string[]; dropped: boolean } {+ const open = new RegExp(`<${name}(?:\\s[^>]*)?>`, 'g');+ const close = new RegExp(`</${name}>`, 'g');+ const found: string[] = [];+ let at = 0;+ for (;;) {+ const opened = matchOutsideCdata(xml, open, at);+ if (!opened) return { found, dropped: false };+ const start = opened.index + opened[0].length;+ const closed = matchOutsideCdata(xml, close, start);+ const reopened = matchOutsideCdata(xml, open, start);+ if (reopened && (!closed || reopened.index < closed.index)) {+ found.push(xml.slice(start, reopened.index));+ at = reopened.index;+ continue;+ }+ if (!closed) return { found, dropped: true };+ found.push(xml.slice(start, closed.index));+ at = closed.index + closed[0].length;+ }+}++function element(item: string, name: string): string | undefined {+ const [inner] = elements(item, name).found;+ return inner === undefined ? undefined : textOf(inner);+}++function quoteForMessage(value: string): string {+ return JSON.stringify(value.replace(/\s+/g, ' ').slice(0, LINK_IN_MESSAGE_MAX));+}++class FeedStatusError extends Error {+ override readonly name = 'FeedStatusError';+ constructor(readonly status: number) {+ super(`${BLOG_FEED_URL} responded ${status}`);+ }+}++export class EmptyBlogError extends Error {+ override readonly name = 'EmptyBlogError';+}++function isTransient(error: unknown): boolean {+ if (error instanceof FeedStatusError) return error.status >= 500;+ return error instanceof DOMException || error instanceof TypeError;+}++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++export function attemptPolicy(building: boolean): { attempts: number; timeoutMs: number } {+ return building+ ? { attempts: BUILD_ATTEMPTS, timeoutMs: BUILD_TIMEOUT_MS }+ : { attempts: 1, timeoutMs: RENDER_TIMEOUT_MS };+}++export async function fetchBlogPostLinks(+ revalidateSeconds: number,+ building = false,+): Promise<LlmsTxtLink[]> {+ const { attempts, timeoutMs } = attemptPolicy(building);++ let lastError: unknown;+ for (let attempt = 0; attempt < attempts; attempt += 1) {+ if (attempt > 0) await sleep(RETRY_BASE_MS * 2 ** (attempt - 1));+ try {+ return await attemptFetch(revalidateSeconds, timeoutMs);+ } catch (error) {+ if (!isTransient(error)) throw error;+ lastError = error;+ }+ }+ throw new Error(+ `${BLOG_FEED_URL} could not be read after ${attempts} attempt${attempts === 1 ? '' : 's'} ` ++ `(${lastError instanceof Error ? lastError.message : String(lastError)})`,+ { cause: lastError },+ );+}++async function attemptFetch(revalidateSeconds: number, timeoutMs: number): Promise<LlmsTxtLink[]> {+ const response = await fetch(BLOG_FEED_URL, {+ signal: AbortSignal.timeout(timeoutMs),+ next: { revalidate: revalidateSeconds },+ });+ if (!response.ok) {+ await response.body?.cancel().catch(() => undefined);+ throw new FeedStatusError(response.status);+ }++ return readPosts(await response.text());+}++function readPost(item: string, postPrefix: string): LlmsTxtLink | string {+ const title = element(item, 'title');+ const link = element(item, 'link');+ if (!title || !link) return 'an <item> with no <title> or no <link>';+ if (!link.startsWith(postPrefix) || link.slice(postPrefix.length).includes('/')) {+ return `${quoteForMessage(link)} is not a post at ${postPrefix}<slug>`;+ }+ const description = element(item, 'description');+ return { url: `${link}.md`, name: title, ...(description ? { description } : {}) };+}++function readPosts(xml: string): LlmsTxtLink[] {+ if (!/<rss[\s>]/.test(xml)) {+ throw new Error(+ `${BLOG_FEED_URL} is not an RSS feed — a different document, or an error page served ` ++ 'with a 200',+ );+ }++ const complete = xml.trimEnd().endsWith('</rss>');+ const { found: items, dropped } = elements(xml, 'item');+ const damage = !complete+ ? 'ends before </rss>, so it is truncated'+ : dropped+ ? 'leaves an <item> unclosed, so it is malformed'+ : null;+ if (items.length === 0) {+ if (damage) {+ throw new Error(`${BLOG_FEED_URL} ${damage} — it is not an empty blog`);+ }+ throw new EmptyBlogError(`${BLOG_FEED_URL} lists no posts`);+ }+ if (damage) {+ console.warn(+ `llms.txt: ${BLOG_FEED_URL} ${damage}, so posts after that point may be missing from ` ++ 'this index',+ );+ }++ const postPrefix = `${SITE_URL}/blog/`;+ const posts: LlmsTxtLink[] = [];+ const skipped: string[] = [];+ for (const item of items) {+ const read = readPost(item, postPrefix);+ if (typeof read === 'string') skipped.push(read);+ else posts.push(read);+ }++ if (posts.length === 0) {+ throw new Error(+ `${BLOG_FEED_URL} lists ${items.length} item${items.length === 1 ? '' : 's'} and not one ` ++ `is a post (${skipped[0]}) — either this is not the blog feed this reads (a different ` ++ 'document, or a canonical host that drifted between the two apps), or the post URL ' ++ 'shape changed',+ );+ }+ for (const reason of skipped) {+ console.warn(`llms.txt: skipping a ${BLOG_FEED_URL} item — ${reason}`);+ }+ return posts;+}++const ALLOW_BLOGLESS = 'OK_ALLOW_BLOGLESS_LLMS_TXT';++function isOffVercelCi(): boolean {+ return Boolean(process.env.CI) && !process.env.VERCEL;+}++export async function blogPostLinks(revalidateSeconds: number): Promise<LlmsTxtLink[]> {+ const building = process.env.NEXT_PHASE === PHASE_PRODUCTION_BUILD;+ if (building && isOffVercelCi()) {+ console.warn(+ `llms.txt: omitting blog posts — a CI build off Vercel publishes nothing, so it does not ` ++ `read ${BLOG_FEED_URL}`,+ );+ return [];+ }++ try {+ return await fetchBlogPostLinks(revalidateSeconds, building);+ } catch (error) {+ if (!building) throw error;++ if (process.env.VERCEL_ENV === 'production' && !(error instanceof EmptyBlogError)) {+ if (process.env[ALLOW_BLOGLESS] !== '1') {+ throw new Error(+ `llms.txt: refusing to publish a production build with no blog posts. ${describe(error)}, ` ++ 'and the index would ship a Blog section with no posts in it for the life of this ' ++ 'deployment. Confirm the marketing zone serves that URL and redeploy this commit; ' ++ `set ${ALLOW_BLOGLESS}=1 to publish without the posts.`,+ { cause: error },+ );+ }+ console.warn(+ `llms.txt: ${ALLOW_BLOGLESS}=1 is set, so this PRODUCTION build publishes an index with ` ++ `no blog posts in it. ${describe(error)}. Unset it once the cause is fixed.`,+ );+ }++ console.warn(`llms.txt: omitting blog posts — ${describe(error)}`, error);+ return [];+ }+}++function describe(error: unknown): string {+ if (error instanceof EmptyBlogError) return 'the blog lists no posts';+ return error instanceof Error ? error.message : String(error);+}diff --git a/docs/src/lib/marketing-markdown-pages.test.ts b/docs/src/lib/marketing-markdown-pages.test.tsnew file mode 100644index 000000000..6cbb0e790--- /dev/null+++ b/docs/src/lib/marketing-markdown-pages.test.ts@@ -0,0 +1,27 @@+import { describe, expect, test } from 'vitest';+import { BRAND_ROUTE } from './brand-assets';+import { MARKETING_MARKDOWN_PAGES } from './marketing-markdown-pages';++describe('curated marketing rendition inventory', () => {+ test('tracks the brand route this app links elsewhere', () => {+ expect(MARKETING_MARKDOWN_PAGES.map((page) => page.path)).toContain(`${BRAND_ROUTE}.md`);+ });++ test('every entry is a site-relative Markdown path', () => {+ for (const page of MARKETING_MARKDOWN_PAGES) {+ expect(page.path).toMatch(/^\/[^\s]*\.md$/);+ }+ });++ test('every entry can be described to an agent', () => {+ for (const page of MARKETING_MARKDOWN_PAGES) {+ expect(page.name.length).toBeGreaterThan(0);+ expect(page.description.length).toBeGreaterThan(0);+ }+ });++ test('lists no path twice', () => {+ const paths = MARKETING_MARKDOWN_PAGES.map((page) => page.path);+ expect(paths).toEqual([...new Set(paths)]);+ });+});diff --git a/docs/src/lib/marketing-markdown-pages.ts b/docs/src/lib/marketing-markdown-pages.tsnew file mode 100644index 000000000..2bd063c47--- /dev/null+++ b/docs/src/lib/marketing-markdown-pages.ts@@ -0,0 +1,39 @@+export interface MarketingMarkdownPage {+ readonly path: string;+ readonly section: 'Blog' | 'Product';+ readonly name: string;+ readonly description: string;+}++export const MARKETING_MARKDOWN_PAGES: readonly MarketingMarkdownPage[] = [+ {+ path: '/index.md',+ section: 'Product',+ name: 'OpenKnowledge',+ description: 'What it is, how to get it, and where everything else lives.',+ },+ {+ path: '/download.md',+ section: 'Product',+ name: 'Download',+ description: 'Direct installer URLs for every macOS, Windows, and Linux build.',+ },+ {+ path: '/team.md',+ section: 'Product',+ name: 'Team',+ description: 'Who builds OpenKnowledge, and the rendition of each profile.',+ },+ {+ path: '/brand.md',+ section: 'Product',+ name: 'Brand',+ description: 'Logo and icon files, and the rules for using them.',+ },+ {+ path: '/blog.md',+ section: 'Blog',+ name: 'Blog index',+ description: 'Every published post with its title, date, and summary.',+ },+];diff --git a/docs/src/lib/mdx-attribute-literal.test.ts b/docs/src/lib/mdx-attribute-literal.test.tsnew file mode 100644index 000000000..7201116bd--- /dev/null+++ b/docs/src/lib/mdx-attribute-literal.test.ts@@ -0,0 +1,84 @@+import { describe, expect, test } from 'vitest';+import { parseAttributeLiteral } from './mdx-attribute-literal.ts';++describe('parseAttributeLiteral', () => {+ test('reads the single-quoted string arrays MDX attributes are written with', () => {+ expect(parseAttributeLiteral("['Desktop app', 'Web app']")).toEqual(['Desktop app', 'Web app']);+ });++ test('reads an object with unquoted keys, quoted keys, and a trailing comma', () => {+ expect(+ parseAttributeLiteral(`{+ "src": { description: 'Image source', required: true, },+ alt: { required: false },+ }`),+ ).toEqual({+ src: { description: 'Image source', required: true },+ alt: { required: false },+ });+ });++ test('reads a multi-line template literal, which is how diagram sources arrive', () => {+ expect(parseAttributeLiteral('`graph LR\n A --> B`')).toBe('graph LR\n A --> B');+ });++ test('applies string escapes rather than passing the backslash through', () => {+ expect(parseAttributeLiteral(String.raw`"a\\b \"c\" é \n"`)).toBe('a\\b "c" é \n');+ });++ test('reads numbers, booleans, null, and undefined', () => {+ expect(parseAttributeLiteral('[1, -2.5, 3e2, true, false, null, undefined]')).toEqual([+ 1,+ -2.5,+ 300,+ true,+ false,+ null,+ undefined,+ ]);+ });++ test('skips comments, which authors leave inside prop objects', () => {+ expect(parseAttributeLiteral('{ /* note */ a: 1, // trailing\n b: 2 }')).toEqual({+ a: 1,+ b: 2,+ });+ });+});++describe('anything that is not a literal', () => {+ test('a free identifier is refused by name', () => {+ expect(() => parseAttributeLiteral('{ items: SUPPORTED_EDITORS }')).toThrow(+ 'identifier "SUPPORTED_EDITORS" is not a literal',+ );+ });++ test('a function call is refused', () => {+ expect(() => parseAttributeLiteral('buildRows()')).toThrow('is not a literal');+ });++ test('a template interpolation is refused instead of being emitted verbatim', () => {+ // biome-ignore lint/suspicious/noTemplateCurlyInString: the placeholder is the input under test+ expect(() => parseAttributeLiteral('`count: ${total}`')).toThrow(+ 'template interpolation is not a literal',+ );+ });++ test('trailing input after a complete literal is refused', () => {+ expect(() => parseAttributeLiteral("['a'] ?? []")).toThrow('unexpected trailing input');+ });++ test('a malformed hex escape is refused, never coerced to NUL', () => {+ expect(() => parseAttributeLiteral("['a\\uZZZZb']")).toThrow('malformed escape');+ expect(() => parseAttributeLiteral("['a\\xZZb']")).toThrow('malformed escape');+ expect(() => parseAttributeLiteral("['\\u{WXYZ}']")).toThrow('malformed escape');+ });++ test('a braced escape beyond the Unicode range is refused with a position', () => {+ expect(() => parseAttributeLiteral("['\\u{110000}']")).toThrow('beyond the Unicode range');+ });++ test('the failure names the line, so a multi-line prop object is locatable', () => {+ expect(() => parseAttributeLiteral('{\n a: 1,\n b: someVar,\n}')).toThrow('at line 3');+ });+});diff --git a/docs/src/lib/mdx-attribute-literal.ts b/docs/src/lib/mdx-attribute-literal.tsnew file mode 100644index 000000000..65f362047--- /dev/null+++ b/docs/src/lib/mdx-attribute-literal.ts@@ -0,0 +1,221 @@+export type AttributeLiteral =+ | string+ | number+ | boolean+ | null+ | undefined+ | AttributeLiteral[]+ | { [key: string]: AttributeLiteral };++class LiteralParser {+ private index = 0;++ constructor(private readonly source: string) {}++ parse(): AttributeLiteral {+ this.skipTrivia();+ const value = this.parseValue();+ this.skipTrivia();+ if (this.index < this.source.length) this.fail('unexpected trailing input');+ return value;+ }++ private parseValue(): AttributeLiteral {+ const char = this.peek();+ if (char === '[') return this.parseArray();+ if (char === '{') return this.parseObject();+ if (char === '"' || char === "'") return this.parseQuoted(char);+ if (char === '`') return this.parseTemplate();+ if (char === '-' || (char >= '0' && char <= '9')) return this.parseNumber();++ const word = /^[A-Za-z_$][\w$]*/.exec(this.source.slice(this.index))?.[0];+ if (word === 'true' || word === 'false' || word === 'null' || word === 'undefined') {+ this.index += word.length;+ return word === 'true' ? true : word === 'false' ? false : word === 'null' ? null : undefined;+ }+ this.fail(word ? `identifier "${word}" is not a literal` : 'expected a literal value');+ }++ private parseArray(): AttributeLiteral[] {+ this.index += 1;+ const items: AttributeLiteral[] = [];+ for (;;) {+ this.skipTrivia();+ if (this.peek() === ']') {+ this.index += 1;+ return items;+ }+ items.push(this.parseValue());+ this.skipTrivia();+ if (this.peek() === ',') this.index += 1;+ else if (this.peek() !== ']') this.fail('expected "," or "]"');+ }+ }++ private parseObject(): { [key: string]: AttributeLiteral } {+ this.index += 1;+ const entries: { [key: string]: AttributeLiteral } = {};+ for (;;) {+ this.skipTrivia();+ if (this.peek() === '}') {+ this.index += 1;+ return entries;+ }+ const key = this.parseKey();+ this.skipTrivia();+ if (this.peek() !== ':') this.fail('expected ":" after an object key');+ this.index += 1;+ this.skipTrivia();+ entries[key] = this.parseValue();+ this.skipTrivia();+ if (this.peek() === ',') this.index += 1;+ else if (this.peek() !== '}') this.fail('expected "," or "}"');+ }+ }++ private parseKey(): string {+ const char = this.peek();+ if (char === '"' || char === "'") return this.parseQuoted(char);+ const identifier = /^[A-Za-z_$][\w$]*/.exec(this.source.slice(this.index))?.[0];+ if (!identifier) this.fail('expected an object key');+ this.index += identifier.length;+ return identifier;+ }++ private parseQuoted(quote: string): string {+ this.index += 1;+ let out = '';+ while (this.index < this.source.length) {+ const char = this.source[this.index];+ if (char === quote) {+ this.index += 1;+ return out;+ }+ if (char === '\\') {+ out += this.readEscape();+ continue;+ }+ if (char === '\n') this.fail('unterminated string');+ out += char;+ this.index += 1;+ }+ this.fail('unterminated string');+ }++ private parseTemplate(): string {+ this.index += 1;+ let out = '';+ while (this.index < this.source.length) {+ const char = this.source[this.index];+ if (char === '`') {+ this.index += 1;+ return out;+ }+ if (char === '\\') {+ out += this.readEscape();+ continue;+ }+ if (char === '$' && this.source[this.index + 1] === '{') {+ this.fail('template interpolation is not a literal');+ }+ out += char;+ this.index += 1;+ }+ this.fail('unterminated template literal');+ }++ private readEscape(): string {+ const escaped = this.source[this.index + 1];+ this.index += 2;+ switch (escaped) {+ case 'n':+ return '\n';+ case 't':+ return '\t';+ case 'r':+ return '\r';+ case 'b':+ return '\b';+ case 'f':+ return '\f';+ case 'v':+ return '\v';+ case '0':+ return '\0';+ case 'u': {+ if (this.source[this.index] === '{') {+ const end = this.source.indexOf('}', this.index);+ if (end === -1) this.fail('unterminated unicode escape');+ const code = this.source.slice(this.index + 1, end);+ this.index = end + 1;+ return this.codePoint(code);+ }+ const code = this.source.slice(this.index, this.index + 4);+ this.index += 4;+ return this.codePoint(code, 4);+ }+ case 'x': {+ const code = this.source.slice(this.index, this.index + 2);+ this.index += 2;+ return this.codePoint(code, 2);+ }+ case '\n':+ return '';+ default:+ return escaped ?? '';+ }+ }++ private codePoint(code: string, width?: number): string {+ if ((width !== undefined && code.length !== width) || !/^[0-9a-fA-F]+$/.test(code)) {+ this.fail('malformed escape sequence');+ }+ const parsed = Number.parseInt(code, 16);+ if (parsed > 0x10_ff_ff) this.fail('escape is beyond the Unicode range');+ return String.fromCodePoint(parsed);+ }++ private parseNumber(): number {+ const matched = /^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?/.exec(this.source.slice(this.index));+ if (!matched) this.fail('expected a number');+ this.index += matched[0].length;+ return Number(matched[0]);+ }++ private peek(): string {+ return this.source[this.index] ?? '';+ }++ private skipTrivia(): void {+ for (;;) {+ while (this.index < this.source.length && /\s/.test(this.source[this.index] ?? '')) {+ this.index += 1;+ }+ if (this.source.startsWith('//', this.index)) {+ const end = this.source.indexOf('\n', this.index);+ this.index = end === -1 ? this.source.length : end;+ continue;+ }+ if (this.source.startsWith('/*', this.index)) {+ const end = this.source.indexOf('*/', this.index);+ if (end === -1) this.fail('unterminated comment');+ this.index = end + 2;+ continue;+ }+ return;+ }+ }++ private fail(reason: string): never {+ const upto = this.source.slice(0, this.index);+ const line = upto.split('\n').length;+ throw new Error(+ `Cannot read MDX attribute expression at line ${line}: ${reason}. ` ++ 'Attribute expressions reachable from served Markdown must be plain literals.',+ );+ }+}++export function parseAttributeLiteral(source: string): AttributeLiteral {+ return new LiteralParser(source).parse();+}diff --git a/docs/src/lib/mdx-serializer-registry.test.ts b/docs/src/lib/mdx-serializer-registry.test.tsnew file mode 100644index 000000000..b1f0106c7--- /dev/null+++ b/docs/src/lib/mdx-serializer-registry.test.ts@@ -0,0 +1,451 @@+import remarkGfm from 'remark-gfm';+import remarkMdx from 'remark-mdx';+import remarkParse from 'remark-parse';+import { unified } from 'unified';+import { visit } from 'unist-util-visit';+import { beforeAll, describe, expect, test } from 'vitest';+import { AGENTS } from '../components/agent-icons.tsx';+import { LAYERS, PATHS } from '../components/overview-blocks.tsx';+import { type CorpusPage, readCorpus } from './docs-corpus.test-helper.ts';+import { DOWNLOAD_TARGETS } from './download-targets.ts';+import { serializeMdx } from './mdx-serializer.ts';+import { DOCS_SERIALIZER_REGISTRY } from './mdx-serializer-registry.ts';++const PAGE_URL = 'https://openknowledge.ai/docs/get-started/quickstart';++function serialize(source: string, pageUrl = PAGE_URL): string {+ return serializeMdx(source, { registry: DOCS_SERIALIZER_REGISTRY, pageUrl }).body;+}++describe('components that carry their content in attributes', () => {+ test('a callout becomes a blockquote led by its title', () => {+ expect(serialize('<Callout type="info" title="Verify Git">\n\nRun it.\n\n</Callout>')).toBe(+ '> **Verify Git**\n>\n> Run it.',+ );+ });++ test('an untitled callout keeps its variant, which is its only severity signal', () => {+ expect(serialize('<Callout type="warning">\n\nThis bites.\n\n</Callout>')).toBe(+ '> **Warning**\n>\n> This bites.',+ );+ });++ test('a TypeTable becomes a table with a row per prop', () => {+ const md = serialize(+ `<TypeTable type={{ "src": { description: 'Image source', type: 'string', required: true } }} />`,+ );+ expect(md).toContain('| Prop | Type | Required | Default | Description |');+ expect(md).toContain('| `src` | `string` | yes | | Image source |');+ });++ test.each([+ ['true', 'yes'],+ ['1', 'yes'],+ ['false', 'no'],+ ['0', 'no'],+ ])('a `required` of %s reads as %s, as the rendered table shows it', (value, expected) => {+ const md = serialize(`<TypeTable type={{ "src": { type: 'string', required: ${value} } }} />`);+ expect(md).toContain(`| \`src\` | \`string\` | ${expected} |`);+ });++ test('a generic type reaches the reader as it was written', () => {+ const md = serialize(`<TypeTable type={{ "items": { type: 'Array<string>' } }} />`);+ const tree = unified().use(remarkParse).use(remarkGfm).parse(md);++ const spans: string[] = [];+ visit(tree, 'inlineCode', (node) => spans.push(node.value));+ expect(spans).toContain('Array<string>');+ });++ test('a TypeTable cell escapes a pipe inside a union type rather than splitting the column', () => {+ const md = serialize(`<TypeTable type={{ "kind": { type: "'a' | 'b'" } }} />`);+ expect(md).toContain("| `'a' \\| 'b'` |");+ });++ test('prose naming an HTML tag survives as words, not as an unclosed tag', () => {+ const md = serialize(`<TypeTable type={{ "title": { description: 'Shown in <summary>' } }} />`);+ expect(md).toContain(String.raw`Shown in \<summary>`);+ });++ test('a Mermaid chart becomes a mermaid fence', () => {+ expect(serialize('<Mermaid chart={`graph LR\nA --> B`} />')).toBe(+ '```mermaid\ngraph LR\nA --> B\n```',+ );+ });++ test('a base64 HtmlPreview attribute is decoded back to its preview fence', () => {+ const encoded = Buffer.from('<p>hi</p>', 'utf8').toString('base64');+ expect(serialize(`<HtmlPreview code="${encoded}" />`)).toBe('```html preview\n<p>hi</p>\n```');+ });++ test('an icon carrying an accessible name contributes that name to the sentence', () => {+ expect(serialize('Click the <Clock role="img" aria-label="Timeline" /> icon.')).toBe(+ 'Click the Timeline icon.',+ );+ });++ test('a decorative icon with no accessible name contributes nothing', () => {+ expect(serialize('Click the <InlineIcon name="Bell" /> bell.')).toBe('Click the bell.');+ });+});++describe('inputs the registry refuses rather than serializes wrong', () => {+ test('a TypeTable whose rows are a free identifier is refused, naming it', () => {+ expect(() => serialize('<TypeTable type={PROP_ROWS} />')).toThrow(+ 'identifier "PROP_ROWS" is not a literal',+ );+ });++ test('a TypeTable without its rows object is refused', () => {+ expect(() => serialize('<TypeTable />')).toThrow(+ '<TypeTable> carries its rows in a `type` object, which is missing.',+ );+ });++ test('a TypeTable whose literal is not an object of descriptors is refused', () => {+ expect(() => serialize("<TypeTable type={['src']} />")).toThrow(+ '<TypeTable type={…}> must be an object of prop descriptors.',+ );+ });++ test('a TypeTable whose type is a quoted string is refused without claiming it is missing', () => {+ expect(() => serialize('<TypeTable type="string" />')).toThrow(+ 'must be a braced object expression',+ );+ });++ test('a TypeTable descriptor that is not an object is refused, naming the prop', () => {+ expect(() => serialize(`<TypeTable type={{ src: 'string' }} />`)).toThrow(+ 'descriptor for `src` must be an object',+ );+ });++ test('a TypeTable field holding an object is refused, naming the field', () => {+ expect(() => serialize(`<TypeTable type={{ src: { type: { kind: 'string' } } }} />`)).toThrow(+ '`src.type` must be a string, number or boolean',+ );+ });++ test('a TypeTable field holding an array is refused too', () => {+ expect(() => serialize(`<TypeTable type={{ src: { default: ['a', 'b'] } }} />`)).toThrow(+ '`src.default` must be a string, number or boolean',+ );+ });++ test.each([+ ['<Image src={hero} alt="Editor" />', 'Image'],+ ['<img src={hero} alt="Editor" />', 'img'],+ ])('refuses %s rather than deleting it from the rendition', (source, name) => {+ expect(() => serialize(source)).toThrow(+ new RegExp(`<${name} src=\\{hero\\}> carries its value in an expression`),+ );+ });++ test('names the absent case differently from the unreadable one', () => {+ expect(() => serialize('<img alt="Editor" />')).toThrow('<img> has no `src`');+ });++ test('a Tabs items expression that is not an array is refused', () => {+ expect(() => serialize('<Tabs items={{ first: 1 }} />')).toThrow(+ '<Tabs items={…}> must be an array of labels.',+ );+ });++ test('an HtmlPreview whose code is not base64 is refused rather than shipped as mojibake', () => {+ expect(() => serialize('<HtmlPreview code="<p>hi</p>" />')).toThrow('is not valid base64');+ });++ test.each([+ ['a`b', 'a`b'],+ ['``', '``'],+ ['`start', '`start'],+ ['end`', 'end`'],+ ])('an inline code span carrying %j round-trips as one span', (value, expected) => {+ const md = serialize(`<code>${value}</code>`);+ const tree = unified().use(remarkParse).use(remarkGfm).parse(md);++ const spans: string[] = [];+ visit(tree, 'inlineCode', (node) => {+ spans.push(node.value);+ });+ expect(spans).toEqual([expected]);+ });++ test.each([+ ['base64url', (value: string) => value.replaceAll('+', '-').replaceAll('/', '_')],+ ['line-wrapped', (value: string) => value.replace(/(.{8})/g, '$1\n')],+ ])('accepts a %s spelling the decoder reads in full', (_name, respell) => {+ const html = '<p>hi & bye</p>';+ const encoded = respell(Buffer.from(html, 'utf8').toString('base64'));++ expect(serialize(`<HtmlPreview code="${encoded}" />`)).toBe(+ `\`\`\`html preview\n${html}\n\`\`\``,+ );+ });++ test('a preview body carrying backtick runs gets a fence longer than the runs', () => {+ const encoded = Buffer.from('<code>```</code>', 'utf8').toString('base64');+ expect(serialize(`<HtmlPreview code="${encoded}" />`)).toBe(+ '````html preview\n<code>```</code>\n````',+ );+ });++ test('a Mermaid without a chart is refused', () => {+ expect(() => serialize('<Mermaid />')).toThrow('<Mermaid> needs a `chart` string.');+ });++ test('an HtmlPreview without its encoded block is refused', () => {+ expect(() => serialize('<HtmlPreview />')).toThrow(+ '<HtmlPreview> carries its block in a `code` attribute, which is missing.',+ );+ });+});++describe('tab groups', () => {+ test('each panel becomes a heading taken from the items list', () => {+ const md = serialize(+ "<Tabs items={['Desktop app', 'Web app']}>\n\n<Tab>\n\nAlpha\n\n</Tab>\n\n<Tab>\n\nBeta\n\n</Tab>\n\n</Tabs>",+ );+ expect(md).toBe('### Desktop app\n\nAlpha\n\n### Web app\n\nBeta');+ });++ test('panel headings sit above the headings inside the panels', () => {+ const md = serialize(+ "<Tabs items={['One']}>\n\n<Tab>\n\n### Inner\n\nBody\n\n</Tab>\n\n</Tabs>",+ );+ expect(md.startsWith('## One\n')).toBe(true);+ });++ test('an explicit panel value wins over its position in the items list', () => {+ const md = serialize(+ '<Tabs items={[\'Ignored\']}>\n\n<Tab value="Chosen">\n\nBody\n\n</Tab>\n\n</Tabs>',+ );+ expect(md).toContain('Chosen');+ expect(md).not.toContain('Ignored');+ });+});++describe('components whose content lives in their own module', () => {+ test('the download button lists every published build with its installer URL', () => {+ const md = serialize('<DownloadButton />');+ for (const target of DOWNLOAD_TARGETS) {+ expect(md).toContain(`[${target.label}](${target.assetUrl})`);+ }+ });++ test('the layer stack lists every layer with its role and description', () => {+ const md = serialize('<LayerStack />');+ for (const layer of LAYERS) {+ expect(md).toContain(layer.title);+ expect(md).toContain(layer.desc);+ }+ });++ test('the start paths render as absolute links', () => {+ const md = serialize('<WhereToStart />', 'https://openknowledge.ai/docs/get-started/overview');+ for (const path of PATHS) {+ expect(md).toContain(`[${path.title}](https://openknowledge.ai${path.href})`);+ }+ });++ test('the agent row lists every agent it advertises', () => {+ const md = serialize('<AgentIcons />');+ for (const agent of AGENTS) expect(md).toContain(agent.name);+ });++ test('the MCP install block names the editor it was given and keeps its guide link absolute', () => {+ const md = serialize('<McpInstall editor="Cursor" />');+ expect(md).toContain('two ways to connect Cursor');+ expect(md).toContain(+ '](https://openknowledge.ai/docs/get-started/quickstart#ok-install-web-app)',+ );+ });++ test('an MCP install block keeps the editor-specific notes nested inside it', () => {+ expect(serialize('<McpInstall editor="Hermes">\n\nRestart Hermes.\n\n</McpInstall>')).toContain(+ 'Restart Hermes.',+ );+ });++ test('the verify block keeps the prompt and names the subject', () => {+ const md = serialize('<VerifyExec subject="Claude Code" />');+ expect(md).toContain('> List the first 5 documents you come across in this project.');+ expect(md).toContain('Claude Code should call the OpenKnowledge `exec` tool');+ });++ test('an editor name that reads as markup is neutralised before it lands in prose', () => {+ const md = serialize('<McpInstall editor="VS_Code <beta>" />');+ expect(md).toContain('two ways to connect VS\\_Code \\<beta>');+ expect(md).not.toContain('VS_Code');+ });++ test('a subject that spans lines still leaves the prompt as one blockquote', () => {+ const md = serialize('<VerifyExec subject="The\nagent" />');+ expect(md).toContain(+ '> List the first 5 documents you come across in this project.\n\nThe agent should call',+ );+ });+});++describe('containers and visual-only demos', () => {+ test('a preview wrapper contributes what its child component would', () => {+ expect(serialize('<ComponentPreview>\n\n<AgentIcons />\n\n</ComponentPreview>')).toContain(+ AGENTS[0].name,+ );+ });++ test('a media demo contributes nothing, because it has nothing to say in text', () => {+ expect(+ serialize('<ComponentPreview>\n\n<ImgPreview src="/a.png" alt="A" />\n\n</ComponentPreview>'),+ ).toBe('');+ });++ test('a card keeps the inline formatting in its description', () => {+ expect(serialize('<Card title="CLI" href="/docs/reference/cli">Run `ok init`.</Card>')).toBe(+ '- [CLI](https://openknowledge.ai/docs/reference/cli): Run `ok init`.',+ );+ });++ test('a card with a block body keeps its blocks instead of folding them onto one line', () => {+ const md = serialize(+ '<Card title="Setup">\n\nFirst do this.\n\n```bash\nok init\n```\n\n</Card>',+ );+ expect(md).toBe('**Setup**\n\nFirst do this.\n\n```bash\nok init\n```');+ });++ test('cards render as one list rather than one list per card', () => {+ const md = serialize(+ '<Cards>\n <Card title="CLI" href="/docs/reference/cli">Command line.</Card>\n <Card title="Config" href="/docs/reference/configuration">Settings.</Card>\n</Cards>',+ );+ expect(md).toBe(+ '- [CLI](https://openknowledge.ai/docs/reference/cli): Command line.\n' ++ '- [Config](https://openknowledge.ai/docs/reference/configuration): Settings.',+ );+ });+});++describe('the docs corpus through the real registry', () => {+ const withMdx = unified().use(remarkParse).use(remarkMdx).use(remarkGfm);+ const asMarkdown = unified().use(remarkParse).use(remarkGfm);++ let pages: CorpusPage[];+ const output = new Map<string, string>();++ beforeAll(async () => {+ pages = await readCorpus();+ for (const page of pages) {+ output.set(+ page.slug,+ serializeMdx(page.source, {+ registry: DOCS_SERIALIZER_REGISTRY,+ pageUrl: page.pageUrl,+ }).body,+ );+ }+ });++ test('every page serializes, so no component is left without a disposition', () => {+ expect(pages.length).toBeGreaterThan(50);+ expect(output.size).toBe(pages.length);+ });++ test('no page leaks a JSX element into the Markdown it serves', () => {+ const leaks: string[] = [];+ for (const [slug, md] of output) {+ visit(withMdx.parse(md), (node) => {+ if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {+ leaks.push(`${slug}: <${node.name}>`);+ }+ });+ }+ expect(leaks).toEqual([]);+ });++ test('no page indents a code block to the four-space column', () => {+ const indented: string[] = [];+ for (const [slug, md] of output) {+ visit(asMarkdown.parse(md), 'code', (node) => {+ const opener = md[node.position?.start.offset ?? 0];+ if (opener !== '`' && opener !== '~') {+ indented.push(`${slug}: line ${node.position?.start.line}`);+ }+ });+ }+ expect(indented).toEqual([]);+ });++ test('no page carries a base64 component attribute', () => {+ for (const [slug, md] of output) {+ expect(md, slug).not.toContain('<HtmlPreview');+ }+ });++ test('the quickstart serves the download builds its CTA stands for', () => {+ const md = output.get('get-started/quickstart') ?? '';+ expect(md).toContain(DOWNLOAD_TARGETS[0].assetUrl);+ expect(md).toContain('### Install the desktop app');+ });++ test('every page embedding the shared install block serves its steps', () => {+ const embedding = pages.filter((page) => page.source.includes('<McpInstall'));+ expect(embedding.length).toBeGreaterThan(5);+ for (const page of embedding) {+ expect(output.get(page.slug), page.slug).toContain('depending on how you run OpenKnowledge');+ }+ });++ test('every page embedding the shared verification block serves its prompt', () => {+ const embedding = pages.filter((page) => page.source.includes('<VerifyExec'));+ expect(embedding.length).toBeGreaterThan(5);+ for (const page of embedding) {+ expect(output.get(page.slug), page.slug).toContain(+ '> List the first 5 documents you come across in this project.',+ );+ }+ });++ test('the overview serves the three layers and the start paths', () => {+ const md = output.get('get-started/overview') ?? '';+ for (const layer of LAYERS) expect(md).toContain(layer.desc);+ for (const path of PATHS) expect(md).toContain(path.desc);+ expect(md).toContain(AGENTS[0].name);+ });++ test('every authored preview fence survives as a fence, not as a base64 attribute', () => {+ const authored = pages.filter((page) => page.source.includes('```html preview'));+ expect(authored.length).toBeGreaterThan(5);+ for (const page of authored) {+ expect(output.get(page.slug), page.slug).toContain('```html preview');+ }+ });++ test('the mermaid page serves its diagram as a mermaid fence', () => {+ expect(output.get('reference/components/mermaid-fence') ?? '').toContain(+ '```mermaid\ngraph LR',+ );+ });++ test('every prop table on a component reference page survives as a table', () => {+ const withTypeTable = pages.filter((page) => page.source.includes('<TypeTable'));+ expect(withTypeTable.length).toBeGreaterThan(10);+ for (const page of withTypeTable) {+ expect(output.get(page.slug), page.slug).toContain('| Prop | Type | Required |');+ }+ });+});++describe('an attribute the author wrote as an expression', () => {+ test.each([+ ['<McpInstall editor={EDITOR} />', 'editor'],+ ['<VerifyExec subject={AGENT} />', 'subject'],+ ['<Callout title={TITLE}>body</Callout>', 'title'],+ ])('%s is refused rather than replaced by a fallback', (source, name) => {+ expect(() => serialize(source)).toThrow(new RegExp(`\`${name}\` is an expression`));+ });++ test('a bare attribute still reads as absent, since it carries no text', () => {+ expect(() =>+ serialize('<Tabs persist items={[{ title: "A", content: "b" }]} />'),+ ).not.toThrow();+ });+});diff --git a/docs/src/lib/mdx-serializer-registry.ts b/docs/src/lib/mdx-serializer-registry.tsnew file mode 100644index 000000000..5a93443db--- /dev/null+++ b/docs/src/lib/mdx-serializer-registry.ts@@ -0,0 +1,340 @@+import { visit } from 'unist-util-visit';+import { AGENTS } from '@/components/agent-icons';+import { mcpInstallMarkdown } from '@/components/mcp-install';+import { LAYERS, PATHS } from '@/components/overview-blocks';+import { verifyExecMarkdown } from '@/components/verify-exec';+import {+ DOWNLOAD_PAGE_HREF,+ DOWNLOAD_TARGETS,+ WEB_APP_HREF,+ WEB_APP_LABEL,+} from '@/lib/download-targets';+import { parseAttributeLiteral } from '@/lib/mdx-attribute-literal';+import {+ type ComponentDisposition,+ escapeInlineProse,+ inlineCodeSpan,+ longestBacktickRun,+ type MdxJsxNode,+ markdownLink,+ requiredLiteral,+ type SerializerApi,+ type SerializerRegistry,+} from '@/lib/mdx-serializer';++function cellSeparators(value: string): string {+ return value.replace(/\|/g, '\\|').replace(/\r?\n/g, ' ').trim();+}++function tableCell(value: string): string {+ return cellSeparators(escapeInlineProse(value));+}++function codeCell(value: string): string {+ return cellSeparators(code(value));+}++const code = inlineCodeSpan;++function titledBlockquote(lead: string | undefined, body: string): string {+ const lines = [...(lead ? [`**${escapeInlineProse(lead)}**`, ''] : []), ...body.split('\n')];+ return lines.map((line) => (line === '' ? '>' : `> ${line}`)).join('\n');+}++const callout: ComponentDisposition = (node, api) => {+ const type = api.attribute(node, 'type');+ const lead =+ api.attribute(node, 'title') ??+ (type ? type.charAt(0).toUpperCase() + type.slice(1) : undefined);+ return titledBlockquote(lead, api.children(node));+};++const accordion: ComponentDisposition = (node, api) => {+ const description = api.attribute(node, 'description');+ const body = [+ ...(description ? [escapeInlineProse(description), ''] : []),+ api.children(node),+ ].join('\n');+ return titledBlockquote(api.attribute(node, 'title'), body);+};++function tabHeadingDepth(tabs: readonly MdxJsxNode[]): number {+ let shallowest = Number.POSITIVE_INFINITY;+ for (const tab of tabs) {+ visit(tab, 'heading', (heading) => {+ shallowest = Math.min(shallowest, heading.depth);+ });+ }+ return Number.isFinite(shallowest) ? Math.max(2, shallowest - 1) : 3;+}++function tabLabels(node: MdxJsxNode): string[] {+ const source = expressionSource(node, 'items');+ if (source === undefined) return [];+ const items = parseAttributeLiteral(source);+ if (!Array.isArray(items)) {+ throw new Error(`<${node.name} items={…}> must be an array of labels.`);+ }+ return items.map(String);+}++function expressionSource(node: MdxJsxNode, name: string): string | undefined {+ const found = node.attributes.find(+ (attribute) => attribute.type === 'mdxJsxAttribute' && attribute.name === name,+ );+ const value = found?.value;+ return typeof value === 'object' && value !== null ? value.value : undefined;+}++function isElement(node: unknown, ...names: string[]): node is MdxJsxNode {+ const typed = node as { type?: string; name?: string };+ return (+ (typed?.type === 'mdxJsxFlowElement' || typed?.type === 'mdxJsxTextElement') &&+ names.includes(typed.name ?? '')+ );+}++function collectItems(+ node: { children?: unknown[] },+ item: string,+ container: string,+): MdxJsxNode[] {+ const found: MdxJsxNode[] = [];+ for (const child of node.children ?? []) {+ if (isElement(child, item)) found.push(child);+ else if (!isElement(child, container) && typeof child === 'object' && child !== null) {+ found.push(...collectItems(child as { children?: unknown[] }, item, container));+ }+ }+ return found;+}++function tabs(...childNames: string[]): ComponentDisposition {+ return (node, api) => {+ const labels = tabLabels(node);+ const panels = childNames.flatMap((name) => collectItems(node, name, node.name ?? ''));+ const depth = '#'.repeat(tabHeadingDepth(panels));+ return panels+ .map((panel, index) => {+ const label =+ api.attribute(panel, 'value') ??+ api.attribute(panel, 'label') ??+ labels[index] ??+ `Tab ${index + 1}`;+ return `${depth} ${escapeInlineProse(label)}\n\n${api.children(panel)}`;+ })+ .join('\n\n');+ };+}++const tabPanel: ComponentDisposition = (node, api) => {+ const label = api.attribute(node, 'value') ?? api.attribute(node, 'label');+ const body = api.children(node);+ return label ? `**${escapeInlineProse(label)}**\n\n${body}` : body;+};++function card(node: MdxJsxNode, api: SerializerApi): string {+ const title = api.attribute(node, 'title') ?? api.attribute(node, 'name') ?? '';+ const href = api.attribute(node, 'href');+ const lead = href+ ? markdownLink(escapeInlineProse(title), api.url(href))+ : `**${escapeInlineProse(title)}**`;+ const body = api.children(node);++ if (/\n[ \t]*\n/.test(body) || /^ {0,3}(```|~~~)/m.test(body)) return `${lead}\n\n${body}`;+ const inline = body.replace(/\s*\n\s*/g, ' ').trim();+ return `- ${lead}${inline ? `: ${inline}` : ''}`;+}++const cards: ComponentDisposition = (node, api) => {+ const rendered = collectItems(node, 'Card', 'Cards').map((child) => card(child, api));+ const everyCardIsALine = rendered.every((entry) => entry.startsWith('- '));+ return rendered.join(everyCardIsALine ? '\n' : '\n\n');+};++function fence(info: string, body: string): string {+ const marker = '`'.repeat(Math.max(2, longestBacktickRun(body)) + 1);+ return `${marker}${info}\n${body}\n${marker}`;+}++const htmlPreview: ComponentDisposition = (node, api) => {+ const encoded = api.attribute(node, 'code');+ if (!encoded)+ throw new Error('<HtmlPreview> carries its block in a `code` attribute, which is missing.');+ const html = Buffer.from(encoded, 'base64').toString('utf8');+ const canonical = (value: string) =>+ value.replace(/\s+/g, '').replaceAll('-', '+').replaceAll('_', '/').replace(/=+$/, '');+ if (canonical(Buffer.from(html, 'utf8').toString('base64')) !== canonical(encoded)) {+ throw new Error('<HtmlPreview> `code` is not valid base64.');+ }+ return fence('html preview', html.replace(/\n$/, ''));+};++const mermaid: ComponentDisposition = (node, api) => {+ const expression = api.expressionAttribute(node, 'chart');+ const chart =+ expression === undefined ? api.attribute(node, 'chart') : parseAttributeLiteral(expression);+ if (typeof chart !== 'string') throw new Error('<Mermaid> needs a `chart` string.');+ return fence('mermaid', chart.replaceAll('\\n', '\n').trim());+};++const typeTable: ComponentDisposition = (node, api) => {+ const source = api.expressionAttribute(node, 'type');+ if (!source) {+ throw new Error(+ api.attribute(node, 'type') === undefined+ ? '<TypeTable> carries its rows in a `type` object, which is missing.'+ : '<TypeTable> `type` must be a braced object expression, not a quoted string.',+ );+ }+ const props = parseAttributeLiteral(source);+ if (typeof props !== 'object' || props === null || Array.isArray(props)) {+ throw new Error('<TypeTable type={…}> must be an object of prop descriptors.');+ }++ const rows = Object.entries(props).map(([name, descriptor]) => {+ if (typeof descriptor !== 'object' || descriptor === null || Array.isArray(descriptor)) {+ throw new Error(`<TypeTable type={…}> descriptor for \`${name}\` must be an object.`);+ }+ const field = (key: string) => {+ const value = descriptor[key];+ if (value === undefined || value === null) return '';+ if (typeof value === 'object') {+ throw new Error(+ `<TypeTable type={…}> \`${name}.${key}\` must be a string, number or boolean.`,+ );+ }+ return String(value);+ };+ return `| ${codeCell(name)} | ${codeCell(field('type'))} | ${+ descriptor.required ? 'yes' : 'no'+ } | ${codeCell(field('default'))} | ${tableCell(field('description'))} |`;+ });++ return [+ '| Prop | Type | Required | Default | Description |',+ '| --- | --- | --- | --- | --- |',+ ...rows,+ ].join('\n');+};++const downloadButton: ComponentDisposition = (_node, api) => {+ const builds = DOWNLOAD_TARGETS.map(+ (target) => `- ${markdownLink(target.label, target.assetUrl)}`,+ );+ return [+ ...builds,+ `- ${markdownLink(WEB_APP_LABEL, api.url(WEB_APP_HREF))}`,+ `- ${markdownLink('All builds and checksums', DOWNLOAD_PAGE_HREF)}`,+ ].join('\n');+};++const layerStack: ComponentDisposition = () =>+ LAYERS.map(+ (layer, index) => `${index + 1}. **${layer.title}** — ${layer.role}. ${layer.desc}`,+ ).join('\n');++const whereToStart: ComponentDisposition = (_node, api) =>+ PATHS.map(+ (path, index) => `${index + 1}. ${markdownLink(path.title, api.url(path.href))} — ${path.desc}`,+ ).join('\n');++const agentIcons: ComponentDisposition = () => AGENTS.map((agent) => agent.name).join(', ');++const iconLabel: ComponentDisposition = (node, api) => {+ const label = api.attribute(node, 'aria-label') ?? api.attribute(node, 'label');+ return label === undefined ? null : escapeInlineProse(label);+};++const image: ComponentDisposition = (node, api) => {+ const src = requiredLiteral(node, api, 'src');+ return markdownLink(escapeInlineProse(api.attribute(node, 'alt') ?? ''), api.url(src), {+ image: true,+ });+};++const anchor: ComponentDisposition = (node, api) => {+ const href = api.attribute(node, 'href');+ const label = api.children(node);+ return href ? markdownLink(label, api.url(href)) : label;+};++function heading(depth: number): ComponentDisposition {+ return (node, api) => `${'#'.repeat(depth)} ${api.children(node)}`;+}++export const DOCS_SERIALIZER_REGISTRY: SerializerRegistry = {+ Accordion: accordion,+ AccordionPreview: accordion,+ AgentIcons: agentIcons,+ Callout: callout,+ CalloutPreview: callout,+ Card: card,+ Cards: cards,+ CopyPrompt: (node, api) => titledBlockquote(undefined, api.children(node)),+ CtaButton: (node, api) => {+ const href = api.attribute(node, 'href');+ const label = escapeInlineProse(api.attribute(node, 'label') ?? href ?? '');+ return href ? markdownLink(label, api.url(href)) : label;+ },+ DownloadButton: downloadButton,+ HtmlPreview: htmlPreview,+ Image: image,+ LayerStack: layerStack,+ McpInstall: (node, api) => {+ const editor = escapeInlineProse(api.attribute(node, 'editor') ?? 'your editor');+ const notes = api.children(node);+ return [+ mcpInstallMarkdown(editor, { url: api.url, link: markdownLink }),+ ...(notes ? ['', notes] : []),+ ].join('\n');+ },+ Mermaid: mermaid,+ Tab: tabPanel,+ TabPreview: tabPanel,+ Tabs: tabs('Tab'),+ TabsPreview: tabs('TabPreview'),+ TypeTable: typeTable,+ VerifyExec: (node, api) =>+ verifyExecMarkdown(escapeInlineProse(api.attribute(node, 'subject') ?? 'The agent')),+ WhereToStart: whereToStart,++ Accordions: 'flatten',+ CalloutContainer: 'flatten',+ CalloutDescription: 'flatten',+ CalloutTitle: 'flatten',+ CodeBlockTab: 'flatten',+ CodeBlockTabs: 'flatten',+ CodeBlockTabsList: 'flatten',+ CodeBlockTabsTrigger: 'flatten',+ ComponentPreview: 'flatten',+ MirrorPreview: 'flatten',+ MirrorSourcePreview: 'flatten',+ Step: 'flatten',+ Steps: 'flatten',++ AudioPreview: 'drop',+ EmbedPreview: 'drop',+ FilePreview: 'drop',+ ImgPreview: 'drop',+ MathPreview: 'drop',+ PdfPreview: 'drop',+ VideoPreview: 'drop',++ Clock: iconLabel,+ Cloud: iconLabel,+ InlineIcon: iconLabel,+ Undo2: iconLabel,++ a: anchor,+ code: (node, api) => code(api.text(node)),+ h1: heading(1),+ h2: heading(2),+ h3: heading(3),+ h4: heading(4),+ h5: heading(5),+ h6: heading(6),+ img: image,+ pre: 'flatten',+ table: 'flatten',+};diff --git a/docs/src/lib/mdx-serializer.test.ts b/docs/src/lib/mdx-serializer.test.tsnew file mode 100644index 000000000..9a680d783--- /dev/null+++ b/docs/src/lib/mdx-serializer.test.ts@@ -0,0 +1,522 @@+import { readdirSync } from 'node:fs';+import { readFile } from 'node:fs/promises';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import remarkGfm from 'remark-gfm';+import remarkParse from 'remark-parse';+import { unified } from 'unified';+import { visit } from 'unist-util-visit';+import { describe, expect, test } from 'vitest';+import {+ componentNames,+ escapeInlineProse,+ escapeRawHtml,+ markdownLink,+ type SerializerRegistry,+ STRINGIFY_OPTIONS,+ serializeMdx,+} from './mdx-serializer.ts';++const PAGE_URL = 'https://openknowledge.ai/docs/get-started/quickstart';++function body(source: string, registry: SerializerRegistry = {}): string {+ return serializeMdx(source, { registry, pageUrl: PAGE_URL }).body;+}++describe('serializeMdx', () => {+ test('serializes a page with frontmatter, a wrapper component, and a relative link', () => {+ const { frontmatter, body: md } = serializeMdx(+ [+ '---',+ 'title: "Quickstart"',+ 'description: Get running in five minutes',+ '---',+ '',+ '<Callout type="tip">',+ ' Read the [editor guide](../features/editor.mdx#ask-ai) first.',+ '</Callout>',+ '',+ '- one',+ '- two',+ '',+ ].join('\n'),+ { registry: { Callout: 'flatten' }, pageUrl: PAGE_URL },+ );++ expect(frontmatter).toEqual({+ title: 'Quickstart',+ description: 'Get running in five minutes',+ });+ expect(md).toBe(+ [+ 'Read the [editor guide](https://openknowledge.ai/docs/features/editor#ask-ai) first.',+ '',+ '- one',+ '- two',+ ].join('\n'),+ );+ });+});++describe('component dispositions', () => {+ test('a component with no disposition fails loudly and names the fix', () => {+ expect(() => body('<Widget>content</Widget>')).toThrow(+ /No serializer disposition for <Widget>.*Add "Widget" to the registry/s,+ );+ });++ test.each([+ 'toString',+ 'constructor',+ 'valueOf',+ ])('a name Object.prototype carries (%s) is refused, not resolved through the prototype', (name) => {+ expect(() => body(`<${name} />`)).toThrow(/No serializer disposition/);+ });++ test('flatten keeps the children and discards the tag', () => {+ expect(body('<Wrapper>\n\nkept text\n\n</Wrapper>', { Wrapper: 'flatten' })).toBe('kept text');+ });++ test('flatten reaches components nested inside other components', () => {+ const md = body('<Outer>\n\n<Inner>deep text</Inner>\n\n</Outer>', {+ Outer: 'flatten',+ Inner: 'flatten',+ });+ expect(md).toBe('deep text');+ });++ test('drop removes the component and everything inside it', () => {+ const md = body('before\n\n<Nav>\n\nnavigation\n\n</Nav>\n\nafter', { Nav: 'drop' });+ expect(md).toBe('before\n\nafter');+ });++ test('a custom serializer emits its markdown verbatim', () => {+ const md = body('<Mermaid chart="graph TD; A-->B" />', {+ Mermaid: (node, api) => `\`\`\`mermaid\n${api.attribute(node, 'chart')}\n\`\`\``,+ });+ expect(md).toBe('```mermaid\ngraph TD; A-->B\n```');+ });++ test('a custom serializer can render the children it was given', () => {+ const md = body('<Callout title="Heads up">\n\nBe careful.\n\n</Callout>', {+ Callout: (node, api) => `> **${api.attribute(node, 'title')}**\n>\n> ${api.children(node)}`,+ });+ expect(md).toBe('> **Heads up**\n>\n> Be careful.');+ });++ test('a custom serializer reads expression attributes as source', () => {+ let seen: string | undefined;+ body("<Tabs items={['CLI', 'UI']}>\n\ncontent\n\n</Tabs>", {+ Tabs: (node, api) => {+ seen = api.expressionAttribute(node, 'items');+ return api.children(node);+ },+ });+ expect(seen).toBe("['CLI', 'UI']");+ });++ test('a custom serializer returning null drops the component', () => {+ expect(body('kept\n\n<Aside>gone</Aside>', { Aside: () => null })).toBe('kept');+ });++ test('a fragment contributes its children without a registry entry', () => {+ expect(body('<>\n\nfragment text\n\n</>')).toBe('fragment text');+ });++ test('imports and expressions carry no prose and are removed', () => {+ const md = body("import { Clock } from 'lucide-react';\n\ntext\n\n{ 2 + 2 }");+ expect(md).toBe('text');+ });+});++describe('frontmatter', () => {+ test('is YAML-parsed, so quoting never leaks into a value', () => {+ const { frontmatter } = serializeMdx('---\ntitle: "Mirror"\n---\n\nbody\n', {+ registry: {},+ pageUrl: PAGE_URL,+ });+ expect(frontmatter.title).toBe('Mirror');+ });++ test('is stripped before the MDX parse, so raw HTML in a description is inert', () => {+ const source = [+ '---',+ 'title: "Accordion"',+ 'description: "Expand/collapse via native HTML5 <details>/<summary>."',+ '---',+ '',+ 'Prose.',+ '',+ ].join('\n');+ const { frontmatter, body: md } = serializeMdx(source, { registry: {}, pageUrl: PAGE_URL });+ expect(frontmatter.description).toContain('<details>/<summary>');+ expect(md).toBe('Prose.');+ });++ test('a source with no frontmatter yields an empty mapping', () => {+ const { frontmatter, body: md } = serializeMdx('Just prose.\n', {+ registry: {},+ pageUrl: PAGE_URL,+ });+ expect(frontmatter).toEqual({});+ expect(md).toBe('Just prose.');+ });++ test('frontmatter that is not a mapping fails loudly', () => {+ expect(() =>+ serializeMdx('---\n- a\n- b\n---\n\nbody\n', { registry: {}, pageUrl: PAGE_URL }),+ ).toThrow(/Frontmatter must be a YAML mapping/);+ });+});++describe('link rewriting', () => {+ test.each([+ [+ 'parent-relative source link',+ '[a](../features/editor.mdx)',+ 'https://openknowledge.ai/docs/features/editor',+ ],+ [+ 'sibling-relative source link',+ '[a](./overview.mdx)',+ 'https://openknowledge.ai/docs/get-started/overview',+ ],+ [+ 'a .md sibling',+ '[a](../reference/core-concepts.md)',+ 'https://openknowledge.ai/docs/reference/core-concepts',+ ],+ [+ 'a fragment on a relative link',+ '[a](../features/editor.mdx#ask-ai)',+ 'https://openknowledge.ai/docs/features/editor#ask-ai',+ ],+ ['a root-relative link', '[a](/download)', 'https://openknowledge.ai/download'],+ [+ 'a bare fragment',+ '[a](#section)',+ 'https://openknowledge.ai/docs/get-started/quickstart#section',+ ],+ ['an image', '', 'https://openknowledge.ai/screenshots/x.png'],+ ])('absolutises %s', (_label, source, expected) => {+ expect(body(source)).toContain(`(${expected})`);+ });++ test.each([+ ['an external URL that happens to end in .md', 'https://github.com/o/r/blob/main/SPEC.md'],+ ['a mailto link', 'mailto:[email protected]'],+ ['a protocol-relative URL', '//cdn.example.com/x.png'],+ ])('leaves %s untouched', (_label, url) => {+ expect(body(`[a](${url})`)).toContain(`(${url})`);+ });++ test('rewrites a link reference definition', () => {+ expect(body('[a][ref]\n\n[ref]: ../features/editor.mdx\n')).toContain(+ 'https://openknowledge.ai/docs/features/editor',+ );+ });++ test('rewrites links inside a component that is flattened away', () => {+ const md = body('<Callout>\n\n[a](../features/editor.mdx)\n\n</Callout>', {+ Callout: 'flatten',+ });+ expect(md).toContain('https://openknowledge.ai/docs/features/editor');+ });+});++describe('pinned stringify options', () => {+ test('are the exact set both apps depend on', () => {+ expect(STRINGIFY_OPTIONS).toEqual({ bullet: '-', rule: '-', listItemIndent: 'one' });+ });++ test('emit hyphen bullets, hyphen rules, and single-space list indent', () => {+ const md = body('* one\n * nested\n\n***\n');+ expect(md).toBe('- one\n - nested\n\n---');+ });++ test('never indent a code block to the four-space column', () => {+ const md = body('```\nplain fence\n```\n');+ expect(md).toBe('```\nplain fence\n```');+ });+});++describe('escapeRawHtml', () => {+ test('turns a tag named in prose into the words it reads as', () => {+ expect(escapeRawHtml('Standalone expand/collapse via native HTML5 <details>/<summary>.')).toBe(+ 'Standalone expand/collapse via native HTML5 \\<details>/\\<summary>.',+ );+ });++ test('leaves a tag named inside a code span alone, where it is already inert', () => {+ const prose = 'Each `<Tab>` child is one panel.';+ expect(escapeRawHtml(prose)).toBe(prose);+ });++ test('keeps the inline formatting the author wrote', () => {+ const prose = 'Prefer **the editor**, or read [the guide](https://openknowledge.ai/docs).';+ expect(escapeRawHtml(prose)).toBe(prose);+ });++ test('gives block-level markup a paragraph of its own', () => {+ expect(escapeRawHtml('<div align="center">\n\nCentred.\n\n</div>')).toBe(+ '\\<div align="center">\n\nCentred.\n\n\\</div>',+ );+ });++ test('leaves prose carrying no markup byte-identical', () => {+ const prose = 'What OpenKnowledge is, and why it exists.';+ expect(escapeRawHtml(prose)).toBe(prose);+ });+});++describe('componentNames', () => {+ test('names every element, component and lowercase HTML alike', () => {+ expect(componentNames('<Callout>text</Callout>\n\n<code>x</code>\n').sort()).toEqual([+ 'Callout',+ 'code',+ ]);+ });++ test('ignores an element inside a fenced code block', () => {+ expect(componentNames('```mdx\n<Callout>example</Callout>\n```\n')).toEqual([]);+ });++ test('ignores an element inside an inline code span', () => {+ expect(componentNames('Write `<Callout>` to add one.\n')).toEqual([]);+ });++ test('ignores markup in frontmatter', () => {+ const source = ['---', 'description: The <details> element', '---', '', 'Body.', ''].join('\n');+ expect(componentNames(source)).toEqual([]);+ });++ test('reaches nested and inline elements, and names each once', () => {+ const source = '<Tabs>\n <Tab>a <InlineIcon /> b</Tab>\n <Tab>c</Tab>\n</Tabs>\n';+ expect(componentNames(source).sort()).toEqual(['InlineIcon', 'Tab', 'Tabs']);+ });++ test('ignores a fragment, which has no name to look up', () => {+ expect(componentNames('<>\n <Card>a</Card>\n</>\n')).toEqual(['Card']);+ });+});++describe('the docs content corpus', () => {+ const CONTENT_ROOT = fileURLToPath(new URL('../../content/', import.meta.url));+ const FLATTEN_EVERY_COMPONENT = new Proxy(+ {},+ {+ get: () => 'flatten',+ getOwnPropertyDescriptor: () => ({+ value: 'flatten',+ configurable: true,+ enumerable: true,+ writable: true,+ }),+ },+ ) as SerializerRegistry;++ function corpusFiles(dir = CONTENT_ROOT): string[] {+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {+ const full = path.join(dir, entry.name);+ if (entry.isDirectory()) return corpusFiles(full);+ return /\.mdx?$/.test(entry.name) ? [full] : [];+ });+ }++ function pageUrlFor(file: string): string {+ const slug = path+ .relative(CONTENT_ROOT, file)+ .replace(/\.mdx?$/, '')+ .replace(/(^|\/)index$/, '');+ return `https://openknowledge.ai/docs/${slug}`.replace(/\/$/, '');+ }++ const files = corpusFiles();++ function hrefsIn(markdown: string): string[] {+ const tree = unified().use(remarkParse).use(remarkGfm).parse(markdown);+ const urls: string[] = [];+ visit(tree, (node) => {+ if (node.type === 'link' || node.type === 'image' || node.type === 'definition') {+ urls.push(node.url);+ }+ });+ return urls;+ }++ test('is non-empty, so a broken glob cannot vacuously pass this suite', () => {+ expect(files.length).toBeGreaterThan(50);+ });++ test('serializes every page with no parse failures and no source-form links', async () => {+ const failures: string[] = [];++ await Promise.all(+ files.map(async (file) => {+ const rel = path.relative(CONTENT_ROOT, file);+ let md: string;+ try {+ md = serializeMdx(await readFile(file, 'utf8'), {+ registry: FLATTEN_EVERY_COMPONENT,+ pageUrl: pageUrlFor(file),+ }).body;+ } catch (error) {+ failures.push(`${rel}: ${(error as Error).message}`);+ return;+ }+ for (const href of hrefsIn(md)) {+ if (href.startsWith('../') || href.startsWith('./')) {+ failures.push(`${rel}: relative href survived (${href})`);+ }+ if (/\.mdx($|[#?])/.test(href)) {+ failures.push(`${rel}: source-form .mdx href survived (${href})`);+ }+ }+ }),+ );++ expect(failures).toEqual([]);+ });+});++describe('a refusal names the page it came from', () => {+ test('prefixes the page URL, so the corpus route does not report 68 candidates', () => {+ expect(() =>+ serializeMdx('<Unknown />', {+ registry: {},+ pageUrl: 'https://openknowledge.ai/docs/x/y',+ }),+ ).toThrow('https://openknowledge.ai/docs/x/y: No serializer disposition for <Unknown>');+ });++ test('keeps the original refusal reachable as the cause', () => {+ const thrown = (() => {+ try {+ serializeMdx('<Unknown />', { registry: {}, pageUrl: 'https://openknowledge.ai/docs/x' });+ } catch (error) {+ return error as Error;+ }+ })();++ expect((thrown?.cause as Error).message).toBe(+ "No serializer disposition for <Unknown>. Add \"Unknown\" to the registry as 'flatten', 'drop', or a custom serializer.",+ );+ });+});++describe('markdownLink', () => {+ const parse = unified().use(remarkParse).use(remarkGfm);++ function linkIn(markdown: string): { url: string; text: string } | undefined {+ let found: { url: string; text: string } | undefined;+ visit(parse.parse(markdown), 'link', (node) => {+ const words: string[] = [];+ visit(node, (child) => {+ if (child.type === 'text' || child.type === 'inlineCode') words.push(child.value);+ });+ found = { url: node.url, text: words.join('') };+ });+ return found;+ }++ test.each([+ ['an unbalanced bracket in the label', 'Unmatched ] here', 'https://x.test/a.md'],+ ['balanced brackets in the label', 'Array[0] access', 'https://x.test/a.md'],+ ['a parenthesis in the destination', 'Note', 'https://x.test/blog/note-)-end.md'],+ ['a space in the destination', 'Note', 'https://x.test/blog/pre post.md'],+ ])('round-trips %s', (_name, label, url) => {+ expect(linkIn(markdownLink(label, url))).toEqual({ url, text: label });+ });++ test.each([+ ['an ordered-list marker', '1. Install the CLI'],+ ['a heading marker', '# Overview'],+ ['a bullet marker', '- Quickstart'],+ ['a blockquote marker', '> Note'],+ ['a thematic break', '---'],+ ])('keeps a label opening with %s', (_name, label) => {+ expect(linkIn(markdownLink(label, 'https://x.test/a.md'))).toEqual({+ url: 'https://x.test/a.md',+ text: label,+ });+ });++ test.each([+ 'Install claude_desktop_config.json',+ '1. Install claude_desktop_config.json',+ 'Use [brackets] here',+ '1. Use [brackets] here',+ 'Set up ~/.codex/config.toml',+ '- Set up ~/.codex/config.toml',+ 'Read the C++ <T> guide',+ '1. Read the C++ <T> guide',+ '# Roadmap',+ ])('serves %s as written through the production composition', (authored) => {+ expect(linkIn(markdownLink(escapeInlineProse(authored), 'https://x.test/a.md'))).toEqual({+ url: 'https://x.test/a.md',+ text: authored,+ });+ });++ test('folds a label that parses to several blocks onto its one line', () => {+ expect(linkIn(markdownLink('para one\n\npara two', 'https://x.test/a.md'))).toEqual({+ url: 'https://x.test/a.md',+ text: 'para one para two',+ });+ });++ test('carries a block-opening label into the image branch too', () => {+ let alt: string | undefined;+ visit(+ parse.parse(markdownLink('1. The editor', 'https://x.test/h.png', { image: true })),+ 'image',+ (node) => {+ alt = node.alt ?? undefined;+ },+ );+ expect(alt).toBe('1. The editor');+ });++ test('leaves a code span in the label byte-identical', () => {+ const rendered = markdownLink('Use ``a`b]c`` today', 'https://x.test/a.md');++ expect(rendered).toBe('[Use ``a`b]c`` today](https://x.test/a.md)');+ expect(rendered).not.toContain('\\]');+ });++ test('flattens a nested link, which CommonMark forbids in a label', () => {+ const rendered = markdownLink('see [the docs](https://x.test/docs)', 'https://x.test/a.md');+ expect(linkIn(rendered)).toEqual({ url: 'https://x.test/a.md', text: 'see the docs' });+ });++ test('renders an image when asked, with the label as alt text', () => {+ let alt: string | undefined;+ let url: string | undefined;+ visit(+ parse.parse(markdownLink('The editor', 'https://x.test/hero.png', { image: true })),+ 'image',+ (n) => {+ alt = n.alt ?? undefined;+ url = n.url;+ },+ );+ expect({ alt, url }).toEqual({ alt: 'The editor', url: 'https://x.test/hero.png' });+ });+});++describe('one escaping rule for author prose placed into authored Markdown', () => {+ const parse = unified().use(remarkParse).use(remarkGfm);++ test.each([+ ['raw HTML a writer names in passing', 'the <summary> element', 'the <summary> element'],+ ['a code span containing a tag', 'Use the `<b>` tag', 'Use the <b> tag'],+ ])('round-trips %s', (_name, prose, expected) => {+ const label = escapeInlineProse(prose);+ const words: string[] = [];+ visit(parse.parse(markdownLink(label, 'https://x.test/a.md')), (node) => {+ if (node.type === 'text' || node.type === 'inlineCode') words.push(node.value);+ });++ expect(words.join('')).toBe(expected);+ });+});diff --git a/docs/src/lib/mdx-serializer.ts b/docs/src/lib/mdx-serializer.tsnew file mode 100644index 000000000..ed123aa66--- /dev/null+++ b/docs/src/lib/mdx-serializer.ts@@ -0,0 +1,301 @@+import type { Html, PhrasingContent, Root, RootContent } from 'mdast';+import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';+import remarkGfm from 'remark-gfm';+import remarkMdx from 'remark-mdx';+import remarkParse from 'remark-parse';+import remarkStringify from 'remark-stringify';+import { unified } from 'unified';+import { visit } from 'unist-util-visit';+import { parse as parseYaml } from 'yaml';++export type ComponentDisposition =+ | 'flatten'+ | 'drop'+ | ((node: MdxJsxNode, api: SerializerApi) => string | null);++export type SerializerRegistry = Readonly<Record<string, ComponentDisposition>>;++export type MdxJsxNode = MdxJsxFlowElement | MdxJsxTextElement;++export interface SerializerApi {+ children(node: MdxJsxNode): string;+ text(node: MdxJsxNode): string;+ attribute(node: MdxJsxNode, name: string): string | undefined;+ expressionAttribute(node: MdxJsxNode, name: string): string | undefined;+ url(href: string): string;+}++export interface SerializeMdxOptions {+ registry: SerializerRegistry;+ pageUrl: string;+}++export interface SerializedMdx {+ frontmatter: Record<string, unknown>;+ body: string;+}++export const STRINGIFY_OPTIONS = {+ bullet: '-',+ rule: '-',+ listItemIndent: 'one',+} as const;++const processor = unified()+ .use(remarkParse)+ .use(remarkMdx)+ .use(remarkGfm)+ .use(remarkStringify, STRINGIFY_OPTIONS);++const proseProcessor = unified()+ .use(remarkParse)+ .use(remarkGfm)+ .use(remarkStringify, STRINGIFY_OPTIONS);++const FRONTMATTER_BLOCK = /^---[ \t]*\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)/;++const ABSOLUTE_URL = /^[a-zA-Z][a-zA-Z0-9+.-]*:|^\/\//;++const SOURCE_EXTENSION = /\.mdx?$/;++const NON_PROSE_TYPES = new Set(['mdxjsEsm', 'mdxFlowExpression', 'mdxTextExpression']);++export function serializeMdx(source: string, options: SerializeMdxOptions): SerializedMdx {+ const { frontmatter, body } = splitFrontmatter(source);+ const base = new URL(options.pageUrl);++ const tree = processor.parse(body);+ absolutiseUrls(tree, base);+ try {+ tree.children = transformChildren(tree, options.registry, base);+ } catch (error) {+ throw new Error(`${options.pageUrl}: ${error instanceof Error ? error.message : error}`, {+ cause: error,+ });+ }++ return { frontmatter, body: processor.stringify(tree).trimEnd() };+}++export function componentNames(source: string): string[] {+ const names = new Set<string>();+ visit(processor.parse(splitFrontmatter(source).body), (node) => {+ if (node.type !== 'mdxJsxFlowElement' && node.type !== 'mdxJsxTextElement') return;+ if (typeof node.name === 'string') names.add(node.name);+ });+ return [...names];+}++export function escapeRawHtml(prose: string, { flattenLinks = false } = {}): string {+ const tree = proseProcessor.parse(prose);+ visit(tree, 'html', (node, index, parent) => {+ if (!parent || index === undefined) return;+ const asText = { type: 'text' as const, value: node.value };+ parent.children[index] =+ parent.type === 'root' || parent.type === 'blockquote' || parent.type === 'listItem'+ ? { type: 'paragraph', children: [asText] }+ : asText;+ });+ if (flattenLinks) flattenLinkNodes(tree);+ return proseProcessor.stringify(tree).trim();+}++function flattenLinkNodes(tree: Root): void {+ visit(tree, (node, index, parent) => {+ if (!parent || index === undefined) return;+ if (node.type === 'link') {+ parent.children.splice(index, 1, ...node.children);+ return index;+ }+ if (node.type === 'image') {+ parent.children[index] = { type: 'text', value: node.alt ?? '' };+ }+ });+}++export function escapeInlineProse(prose: string, options?: { flattenLinks?: boolean }): string {+ return escapeRawHtml(prose.replace(/\s+/g, ' ').trim(), options);+}++export function markdownLink(label: string, url: string, { image = false } = {}): string {+ const children = inlinePhrasing(label);+ const node = image+ ? ({ type: 'image', url, alt: textContent({ children }) } as const)+ : ({ type: 'link', url, children } as const);+ return proseProcessor+ .stringify({ type: 'root', children: [{ type: 'paragraph', children: [node] }] })+ .trim();+}++function inlinePhrasing(markdown: string): PhrasingContent[] {+ const paragraph = singleParagraph(markdown);+ if (paragraph) return paragraph;++ const oneLine = markdown.replace(/\s+/g, ' ').trim();+ const neutralised = oneLine.replace(/^(\d{1,9})([.)])|^[#>*+_=-]+/, (run, digits, delimiter) =>+ digits ? `${digits}\\${delimiter}` : run.replace(/./g, '\\$&'),+ );+ const asInline = singleParagraph(neutralised);+ if (asInline) return asInline;++ return [{ type: 'text', value: textContent(proseProcessor.parse(markdown)) }];+}++function singleParagraph(markdown: string): PhrasingContent[] | null {+ const tree = proseProcessor.parse(markdown);+ flattenLinkNodes(tree);+ const [first, ...rest] = tree.children;+ return first?.type === 'paragraph' && rest.length === 0 ? first.children : null;+}++export function requiredLiteral(node: MdxJsxNode, api: SerializerApi, name: string): string {+ const expression = api.expressionAttribute(node, name);+ if (expression !== undefined) {+ throw new Error(+ `<${node.name} ${name}={${expression}}> carries its value in an expression the Markdown ` ++ 'rendition cannot resolve. Write it as a literal, or give the component a disposition ' ++ 'that renders what it points at.',+ );+ }++ const literal = api.attribute(node, name);+ if (literal) return literal;++ throw new Error(+ `<${node.name}> has no \`${name}\`, so the Markdown rendition has nothing to render.`,+ );+}++export function longestBacktickRun(value: string): number {+ return Math.max(0, ...[...value.matchAll(/`+/g)].map((run) => run[0].length));+}++export function inlineCodeSpan(value: string): string {+ const trimmed = value.trim();+ if (trimmed === '') return '';+ const marker = '`'.repeat(longestBacktickRun(trimmed) + 1);+ const pad = trimmed.startsWith('`') || trimmed.endsWith('`') ? ' ' : '';+ return `${marker}${pad}${trimmed}${pad}${marker}`;+}++function splitFrontmatter(source: string): { frontmatter: Record<string, unknown>; body: string } {+ const match = FRONTMATTER_BLOCK.exec(source);+ if (!match) return { frontmatter: {}, body: source };++ const parsed: unknown = parseYaml(match[1] ?? '') ?? {};+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {+ throw new Error(+ `Frontmatter must be a YAML mapping, got ${Array.isArray(parsed) ? 'a sequence' : typeof parsed}.`,+ );+ }+ return { frontmatter: parsed as Record<string, unknown>, body: source.slice(match[0].length) };+}++function absolutiseUrls(tree: Root, base: URL): void {+ visit(tree, (node) => {+ if (node.type !== 'link' && node.type !== 'image' && node.type !== 'definition') return;+ node.url = absolutise(node.url, base);+ });+}++function absolutise(url: string, base: URL): string {+ if (url === '' || ABSOLUTE_URL.test(url)) return url;+ const suffixAt = url.search(/[#?]/);+ const path = suffixAt === -1 ? url : url.slice(0, suffixAt);+ const suffix = suffixAt === -1 ? '' : url.slice(suffixAt);+ return new URL(path.replace(SOURCE_EXTENSION, '') + suffix, base).href;+}++function transformChildren(+ parent: { children?: unknown[] },+ registry: SerializerRegistry,+ base: URL,+) {+ const out: RootContent[] = [];+ for (const child of parent.children ?? [])+ out.push(...transformNode(child as RootContent, registry, base));+ return out;+}++function transformNode(node: RootContent, registry: SerializerRegistry, base: URL): RootContent[] {+ if (NON_PROSE_TYPES.has(node.type)) return [];++ if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {+ return transformJsx(node, registry, base);+ }++ if ('children' in node) {+ (node as unknown as { children: RootContent[] }).children = transformChildren(+ node,+ registry,+ base,+ );+ }+ return [node];+}++function transformJsx(node: MdxJsxNode, registry: SerializerRegistry, base: URL): RootContent[] {+ const name = node.name;+ if (name === null || name === undefined) return transformChildren(node, registry, base);++ const disposition = Object.hasOwn(registry, name) ? registry[name] : undefined;+ if (disposition === undefined) {+ throw new Error(+ `No serializer disposition for <${name}>. Add "${name}" to the registry as 'flatten', 'drop', or a custom serializer.`,+ );+ }+ if (disposition === 'drop') return [];+ if (disposition === 'flatten') return transformChildren(node, registry, base);++ const rendered = disposition(node, apiFor(registry, base));+ if (rendered === null) return [];+ return [{ type: 'html', value: rendered } satisfies Html];+}++function apiFor(registry: SerializerRegistry, base: URL): SerializerApi {+ return {+ children(node) {+ const children = transformChildren(node, registry, base);+ return processor.stringify({ type: 'root', children }).trimEnd();+ },+ text(node) {+ return textContent(node);+ },+ url(href) {+ return absolutise(href, base);+ },+ attribute(node, name) {+ const found = findAttribute(node, name);+ if (found === undefined) return undefined;+ if (found.value !== null && typeof found.value !== 'string') {+ throw new Error(+ `<${node.name ?? 'component'}> \`${name}\` is an expression, and this serializer reads ` ++ 'it as literal text. Read it with expressionAttribute and parse it, or give the ' ++ 'attribute a quoted value.',+ );+ }+ return typeof found.value === 'string' ? found.value : undefined;+ },+ expressionAttribute(node, name) {+ const value = findAttribute(node, name)?.value;+ return typeof value === 'object' && value !== null ? value.value : undefined;+ },+ };+}++function findAttribute(node: MdxJsxNode, name: string) {+ return node.attributes.find(+ (attribute) => attribute.type === 'mdxJsxAttribute' && attribute.name === name,+ );+}++function textContent(node: unknown): string {+ if (typeof node !== 'object' || node === null) return '';+ const typed = node as { type?: string; value?: unknown; alt?: unknown; children?: unknown[] };+ if (typed.type === 'text' || typed.type === 'inlineCode' || typed.type === 'code') {+ return typeof typed.value === 'string' ? typed.value : '';+ }+ if (typed.type === 'image') return typeof typed.alt === 'string' ? typed.alt : '';+ if (!Array.isArray(typed.children)) return '';+ return typed.children.map(textContent).join('');+}diff --git a/docs/src/lib/site.ts b/docs/src/lib/site.tsindex d41a6dd22..118f3a903 100644--- a/docs/src/lib/site.ts+++ b/docs/src/lib/site.ts@@ -1,6 +1,10 @@import { STABLE_DMG_URL } from './download-links';export const SITE_URL = 'https://openknowledge.ai';++export function absoluteSiteUrl(path: string): string {+ return new URL(path, SITE_URL).href;+}export const SITE_NAME = 'OpenKnowledge';export const TWITTER_HANDLE = '@OpenKnowledge';diff --git a/docs/src/markdown-header-contract.test.ts b/docs/src/markdown-header-contract.test.tsnew file mode 100644index 000000000..76b5018f2--- /dev/null+++ b/docs/src/markdown-header-contract.test.ts@@ -0,0 +1,213 @@+import { autoImplementMethods } from 'next/dist/server/route-modules/app-route/helpers/auto-implement-methods';+import type { AppRouteHandlers } from 'next/dist/server/route-modules/app-route/module';+import { describe, expect, test, vi } from 'vitest';+import nextConfig from '../next.config.ts';+import { filesNamed, ROUTE_FILES, readAppFile } from './app/app-files.test-helper.ts';++const CONTRACT = {+ 'content-type': 'text/markdown; charset=utf-8',+ 'cache-control': 'public, max-age=0, s-maxage=3600, stale-while-revalidate=86400',+ vary: 'Accept',+} as const;++const SHARED_HSTS = 'max-age=63072000; includeSubDomains; preload';++const MARKDOWN_SOURCE_MARKERS = ['markdownResponse', 'markdownRouteHandler', 'text/markdown'];++const markdownRouteFilesOnDisk = () =>+ filesNamed(ROUTE_FILES).filter((file) => {+ const source = readAppFile(file);+ return MARKDOWN_SOURCE_MARKERS.some((marker) => source.includes(marker));+ });++const OVERVIEW_URL = 'https://openknowledge.ai/docs/get-started/overview';++const overviewPage = {+ url: '/docs/get-started/overview',+ data: {+ title: 'Overview',+ description: 'What OpenKnowledge is.',+ getText: async () => 'Early days.\n',+ },+};++vi.doMock('@/lib/source', () => ({+ source: {+ getPage: (slug: string[]) =>+ slug.join('/') === 'get-started/overview' ? overviewPage : undefined,+ getPages: () => [overviewPage],+ generateParams: () => [{ slug: ['get-started', 'overview'] }],+ },+}));++vi.stubGlobal(+ 'fetch',+ async () =>+ new Response(+ '<rss version="2.0"><channel><item><title>markdownlint</title>' ++ '<link>https://openknowledge.ai/blog/markdownlint-support</link></item></channel></rss>',+ ),+);++const perPageRoute = await import('@/app/llms.mdx/[...slug]/route.ts');+const llmsTxtRoute = await import('@/app/llms.txt/route.ts');+const llmsFullRoute = await import('@/app/llms-full.txt/route.ts');++type Handler = (...args: never[]) => Promise<Response> | Response;+type Method = 'GET' | 'HEAD';++interface MarkdownRoute {+ readonly name: string;+ readonly file: string;+ readonly canonicalUrl: string | null;+ readonly status: number;+ readonly module: { readonly GET: Handler };+ readonly call: (handler: Handler, method: Method) => Promise<Response> | Response;+}++const slugCall =+ (url: string, slug: string[]) =>+ (handler: Handler, method: Method): Promise<Response> =>+ (handler as typeof perPageRoute.GET)(new Request(url, { method }), {+ params: Promise.resolve({ slug }),+ });+const bareCall = (handler: Handler) => (handler as () => Promise<Response>)();++const ROUTES: MarkdownRoute[] = [+ {+ name: 'a docs page',+ file: 'llms.mdx/[...slug]/route.ts',+ canonicalUrl: OVERVIEW_URL,+ status: 200,+ module: perPageRoute,+ call: slugCall(`${OVERVIEW_URL}.md`, ['get-started', 'overview']),+ },+ {+ name: 'a docs page nobody holds',+ file: 'llms.mdx/[...slug]/route.ts',+ canonicalUrl: null,+ status: 404,+ module: perPageRoute,+ call: slugCall('https://openknowledge.ai/docs/nope.md', ['nope']),+ },+ {+ name: 'llms.txt',+ file: 'llms.txt/route.ts',+ canonicalUrl: null,+ status: 200,+ module: llmsTxtRoute,+ call: bareCall,+ },+ {+ name: 'llms-full.txt',+ file: 'llms-full.txt/route.ts',+ canonicalUrl: null,+ status: 200,+ module: llmsFullRoute,+ call: bareCall,+ },+];++const serve = (route: MarkdownRoute) => route.call(route.module.GET, 'GET');++const served = new Map(+ await Promise.all(ROUTES.map(async (route) => [route, await serve(route)] as const)),+);++describe('every Markdown response the docs app serves', () => {+ test.each(ROUTES)('$name carries the shared header contract', (route) => {+ const headers = served.get(route)?.headers;+ for (const [header, value] of Object.entries(CONTRACT)) {+ expect(headers?.get(header), `${header} on ${route.name}`).toBe(value);+ }+ });++ test.each(ROUTES)('$name points agents at the page they should cite', (route) => {+ const expected = route.canonicalUrl ? `<${route.canonicalUrl}>; rel="canonical"` : null;+ expect(served.get(route)?.headers.get('link'), `Link on ${route.name}`).toBe(expected);+ });++ test.each(ROUTES)('$name answers with the status it promises', (route) => {+ expect(served.get(route)?.status, route.name).toBe(route.status);+ });++ test('is a census: no route file serves Markdown without being pinned here', () => {+ const unpinned = markdownRouteFilesOnDisk().filter(+ (file) => !ROUTES.some((route) => route.file === file),+ );++ expect(+ unpinned,+ 'route files serving Markdown that this contract never checks; add each to ROUTES',+ ).toEqual([]);+ });++ test('the census sees a handler authored as .tsx', () => {+ expect(filesNamed(ROUTE_FILES).filter((file) => file.endsWith('.tsx')).length).toBeGreaterThan(+ 0,+ );+ });++ test('the census reaches the routes it names', () => {+ const onDisk = new Set(filesNamed(ROUTE_FILES));+ expect(ROUTES.filter((route) => !onDisk.has(route.file)).map((route) => route.file)).toEqual(+ [],+ );+ expect(ROUTES.length).toBeGreaterThan(1);+ });+});++describe('a HEAD request to every Markdown route', () => {+ const HEAD_ROUTES = ROUTES.filter(+ (route, index) => ROUTES.findIndex((other) => other.file === route.file) === index,+ );++ test('covers every Markdown route file on disk, once', () => {+ const files = HEAD_ROUTES.map((route) => route.file);+ expect(new Set(files)).toEqual(new Set(markdownRouteFilesOnDisk()));+ expect(files.length).toBe(new Set(files).size);+ });++ test.each(HEAD_ROUTES)('$name is answered by its GET handler, which reads no method', async ({+ module,+ call,+ }) => {+ expect(module).not.toHaveProperty('HEAD');+ const methods = autoImplementMethods(module as unknown as AppRouteHandlers);+ expect(methods.HEAD).toBe(module.GET);++ const headers = (await call(methods.HEAD as never, 'HEAD')).headers;+ for (const [header, value] of Object.entries(CONTRACT)) {+ expect(headers.get(header), `${header} on HEAD`).toBe(value);+ }+ });+});++describe('the header layer next.config.ts adds on top', () => {+ async function configRules() {+ return (await nextConfig.headers?.()) ?? [];+ }++ test('sets no header the Markdown contract also sets', async () => {+ const collisions = (await configRules()).flatMap((rule) =>+ rule.headers+ .filter((header) => {+ const key = header.key.toLowerCase();+ return key in CONTRACT || key === 'link';+ })+ .map((header) => `${rule.source} sets ${header.key}`),+ );++ expect(collisions, 'config headers colliding with the Markdown contract').toEqual([]);+ });++ test('sends the same Strict-Transport-Security value as the marketing app', async () => {+ const values = (await configRules()).flatMap((rule) =>+ rule.headers+ .filter((header) => header.key.toLowerCase() === 'strict-transport-security')+ .map((header) => header.value),+ );++ expect(values).toEqual([SHARED_HSTS]);+ });+});diff --git a/docs/src/markdown-rewrites.test.ts b/docs/src/markdown-rewrites.test.tsnew file mode 100644index 000000000..2bb6cdbef--- /dev/null+++ b/docs/src/markdown-rewrites.test.ts@@ -0,0 +1,64 @@+import { getPathMatch } from 'next/dist/shared/lib/router/utils/path-match.js';+import { prepareDestination } from 'next/dist/shared/lib/router/utils/prepare-destination.js';+import { describe, expect, test } from 'vitest';+import nextConfig from '../next.config.ts';++async function rewritePhases() {+ const rewrites = await nextConfig.rewrites?.();+ if (!rewrites || Array.isArray(rewrites)) {+ throw new Error('rewrites() must return { beforeFiles, afterFiles }');+ }+ return { beforeFiles: rewrites.beforeFiles ?? [], afterFiles: rewrites.afterFiles ?? [] };+}++async function rewriteOf(pathname: string): Promise<string | null> {+ const rules = (await rewritePhases()).beforeFiles;++ for (const rule of rules) {+ const params = getPathMatch(rule.source, { removeUnnamedParams: true, strict: true })(pathname);+ if (!params) continue;+ return prepareDestination({+ appendParamsToQuery: false,+ destination: rule.destination,+ params,+ query: {},+ }).newUrl;+ }+ return null;+}++describe('docs Markdown rewrites', () => {+ test.each([+ ['/docs/get-started/quickstart.md'],+ ['/docs/get-started/quickstart.html.md'],+ ['/docs/get-started/quickstart.mdx'],+ ])('%s serves the markdown rendition of the same page', async (url) => {+ expect(await rewriteOf(url)).toBe('/llms.mdx/get-started/quickstart');+ });++ test('a single-segment slug resolves too', async () => {+ expect(await rewriteOf('/docs/overview.html.md')).toBe('/llms.mdx/overview');+ });++ test('a page whose own name ends in .html keeps its slug', async () => {+ expect(await rewriteOf('/docs/guides/index.html.md')).toBe('/llms.mdx/guides/index');+ });++ test('the human page is left alone', async () => {+ expect(await rewriteOf('/docs/get-started/quickstart')).toBeNull();+ });++ test('every markdown rule sits in beforeFiles, and none in afterFiles', async () => {+ const { beforeFiles, afterFiles } = await rewritePhases();+ expect(beforeFiles.map((rule) => rule.destination)).toEqual(+ expect.arrayContaining([expect.stringMatching(/^\/llms\.mdx\//)]),+ );+ expect(afterFiles.filter((rule) => rule.destination.startsWith('/llms.mdx'))).toEqual([]);+ });++ test('keeps the PostHog proxy rules in afterFiles, in their existing order', async () => {+ const { afterFiles } = await rewritePhases();+ const sources = afterFiles.map((rule) => rule.source);+ expect(sources).toEqual(['/ingest/static/:path*', '/ingest/array/:path*', '/ingest/:path*']);+ });+});diff --git a/docs/src/proxy.test.ts b/docs/src/proxy.test.tsindex e6381b241..9f6283d81 100644--- a/docs/src/proxy.test.ts+++ b/docs/src/proxy.test.ts@@ -60,3 +60,92 @@ describe('proxy: www -> apex canonicalization', () => {expect(res.status).toBe(200);});});++function negotiate(pathAndQuery: string, accept?: string) {+ const headers: Record<string, string> = { host: APEX };+ if (accept !== undefined) headers.accept = accept;+ const res = proxy(new NextRequest(`https://${APEX}${pathAndQuery}`, { headers }));+ const rewrite = res.headers.get('x-middleware-rewrite');+ return {+ status: res.status,+ location: res.headers.get('location'),+ rewrittenTo: rewrite ? new URL(rewrite).pathname : null,+ };+}++const MARKDOWN = 'text/markdown';+const BROWSER_ACCEPT =+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8';++describe('proxy: Accept negotiation', () => {+ test('a docs page requested as Markdown serves its Markdown rendition', () => {+ expect(negotiate('/docs/get-started/quickstart', MARKDOWN).rewrittenTo).toBe(+ '/llms.mdx/get-started/quickstart',+ );+ });++ test('a one-segment slug resolves too', () => {+ expect(negotiate('/docs/overview', MARKDOWN).rewrittenTo).toBe('/llms.mdx/overview');+ });++ test('a trailing slash still resolves', () => {+ expect(negotiate('/docs/overview/', MARKDOWN).rewrittenTo).toBe('/llms.mdx/overview');+ });++ test('the query string survives the rewrite', () => {+ const res = negotiate('/docs/overview?ref=agent', MARKDOWN);+ expect(res.rewrittenTo).toBe('/llms.mdx/overview');+ });++ test('a browser gets the page untouched', () => {+ const res = negotiate('/docs/get-started/quickstart', BROWSER_ACCEPT);+ expect(res.rewrittenTo).toBeNull();+ expect(res.status).toBe(200);+ });++ test('a request with no Accept gets the page', () => {+ expect(negotiate('/docs/overview').rewrittenTo).toBeNull();+ });++ test('negotiation rewrites rather than redirecting, so the URL stays shareable', () => {+ const res = negotiate('/docs/overview', MARKDOWN);+ expect(res.location).toBeNull();+ expect(res.status).toBe(200);+ });++ test.each([+ ['the docs index, which only redirects', '/docs'],+ ['the changelog, built from a different source adapter', '/docs/changelog'],+ ['a changelog release page', '/docs/changelog/v0.4.0'],+ ['a path outside the docs tree', '/continue'],+ ['a share page', '/d/abc123'],+ ])('%s is never negotiated', (_label, path) => {+ expect(negotiate(path, MARKDOWN).rewrittenTo).toBeNull();+ });++ test.each([+ '/docs/overview.md',+ '/docs/overview.html.md',+ '/docs/overview.mdx',+ ])('%s is left to the rewrite rules rather than negotiated again', (path) => {+ expect(negotiate(path, MARKDOWN).rewrittenTo).toBeNull();+ });++ test('host canonicalization wins over negotiation', () => {+ const res = proxy(+ new NextRequest(`https://${WWW}/docs/overview`, { headers: { host: WWW, accept: MARKDOWN } }),+ );+ expect(res.status).toBe(308);+ expect(res.headers.get('location')).toBe(`https://${APEX}/docs/overview`);+ });++ test('the page a browser falls through to carries no Vary from the proxy', () => {+ const res = proxy(+ new NextRequest(`https://${APEX}/docs/overview`, {+ headers: { host: APEX, accept: BROWSER_ACCEPT },+ }),+ );+ expect(res.headers.get('x-middleware-rewrite')).toBeNull();+ expect(res.headers.get('vary')).toBeNull();+ });+});diff --git a/docs/src/proxy.ts b/docs/src/proxy.tsindex 872cabcc9..11ae20461 100644--- a/docs/src/proxy.ts+++ b/docs/src/proxy.ts@@ -1,4 +1,6 @@import { type NextRequest, NextResponse } from 'next/server';+import { markdownRewrite } from './lib/accept-negotiation';+import { markdownHandlerForPage } from './lib/markdown-routes';const APEX_HOST = 'openknowledge.ai';const WWW_HOST = `www.${APEX_HOST}`;@@ -16,6 +18,10 @@ export function proxy(request: NextRequest): NextResponse {return NextResponse.redirect(url, 308);}+ const handler = markdownHandlerForPage(request.nextUrl.pathname);+ const negotiated = markdownRewrite(request, handler);+ if (negotiated) return negotiated;+return NextResponse.next();}diff --git a/docs/vitest.config.ts b/docs/vitest.config.tsindex f98ab24e5..d12d3f363 100644--- a/docs/vitest.config.ts+++ b/docs/vitest.config.ts@@ -6,6 +6,12 @@ const docsRoot = fileURLToPath(new URL('.', import.meta.url));export default defineConfig({...okVitestBase,+ test: {+ ...okVitestBase.test,+ css: false,+ server: { deps: { inline: [/fumadocs-ui/] } },+ exclude: [...okVitestBase.test.exclude, '**/*.real-source.test.ts'],+ },resolve: {...okVitestBase.resolve,alias: [diff --git a/docs/vitest.real-source.config.mts b/docs/vitest.real-source.config.mtsnew file mode 100644index 000000000..64dd8ea55--- /dev/null+++ b/docs/vitest.real-source.config.mts@@ -0,0 +1,38 @@+import { fileURLToPath } from 'node:url';+import mdx from 'fumadocs-mdx/vite';+import { defineConfig } from 'vitest/config';+import { okVitestBase } from '../test-support/vitest.base';+import * as Config from './source.config.ts';++const docsRoot = fileURLToPath(new URL('.', import.meta.url));++const OUT_DIR = '.source-vitest';++const collectionJsonModule = {+ name: 'docs-collection-json-module',+ enforce: 'post' as const,+ transform(code: string, id: string) {+ if (!/\.json\?collection=/.test(id)) return null;+ if (!code.trimStart().startsWith('{')) return null;+ return { code: `export default ${code};`, map: null };+ },+};++export default defineConfig(async () => ({+ ...okVitestBase,+ plugins: [...okVitestBase.plugins, await mdx(Config, { outDir: OUT_DIR }), collectionJsonModule],+ test: {+ ...okVitestBase.test,+ include: ['src/**/*.real-source.test.ts'],+ css: false,+ server: { deps: { inline: [/fumadocs-ui/] } },+ },+ resolve: {+ ...okVitestBase.resolve,+ alias: [+ { find: /(?:\.\.\/)+\.source(?=$|\/)/, replacement: `${docsRoot}${OUT_DIR}` },+ { find: /^@\/\.source(?=$|\/)/, replacement: `${docsRoot}${OUT_DIR}` },+ { find: /^@\//, replacement: `${docsRoot}src/` },+ ],+ },+}));diff --git a/lint-plugins/no-comments/README.md b/lint-plugins/no-comments/README.mdindex 4c75f6c2c..6081f3e21 100644--- a/lint-plugins/no-comments/README.md+++ b/lint-plugins/no-comments/README.md@@ -237,11 +237,13 @@ the mirror cannot resolve. The infix is not an escape hatch from this policy —file still ships INTERNALLY, still rots, and review holds it to the same bar; the gate skips itonly because the mirrored predicate cannot see it anyway.-`packages/**/{src,tests}/**/*.{ts,tsx}`, `scripts/**/*.mjs`, `.github/scripts/**/*.mjs`,-`docs/**/*.{ts,tsx}`, and root `*.ts`. Excluded: the audit framework's own tree, `*.private.*`,-the fidelity and lume-qa suites, fixtures, `*.d.ts`, generated locales, `knip.config.ts`, and-build output. `scope.mjs` is the one place this is written down; the gate, the sweep, the-codemod, and the hook all read it from there.+Source files across the package, script, docs, lint-plugin and root-config strata, minus an+exclusion set that covers the audit framework's own tree, `*.private.*` files, the fidelity and+lume-qa suites, fixtures, declaration files, generated locales, build output, and specific+single-file carve-outs. `scope.mjs` holds both lists, and it is the one place they are written+down; the gate, the sweep, the codemod, and the hook all read it from there. Read that file+rather than a restatement here: this paragraph has drifted from it twice, because an enumeration+in prose has nothing checking it.This directory ships to the public mirror, which means its own tests may not *spell* a`*.private.*` path: the mirror's content-leak gate rejects a named reference to one in anydiff --git a/lint-plugins/no-comments/scope.mjs b/lint-plugins/no-comments/scope.mjsindex fd9357871..fef4f815b 100644--- a/lint-plugins/no-comments/scope.mjs+++ b/lint-plugins/no-comments/scope.mjs@@ -3,8 +3,11 @@ export const SCOPE_STRATA = [{ id: 'packages-tests', include: ['packages/**/tests/**/*.ts', 'packages/**/tests/**/*.tsx'] },{ id: 'scripts', include: ['scripts/**/*.mjs'] },{ id: 'github-scripts', include: ['.github/scripts/**/*.mjs'] },- { id: 'docs', include: ['docs/**/*.ts', 'docs/**/*.tsx'] },- { id: 'root-configs', include: ['*.ts'] },+ {+ id: 'docs',+ include: ['docs/**/*.ts', 'docs/**/*.tsx', 'docs/**/*.mts', 'docs/**/*.cts'],+ },+ { id: 'root-configs', include: ['*.ts', '*.mts', '*.cts'] },{ id: 'lint-plugins', include: ['lint-plugins/**/*.mjs'] },];@@ -20,6 +23,8 @@ export const SCOPE_EXCLUDE = ['**/*.fixture.tsx','**/*.fixture.mjs','**/*.d.ts',+ '**/*.d.mts',+ '**/*.d.cts','**/fixtures/**','**/__fixtures__/**','knip.config.ts',diff --git a/lint-plugins/no-comments/scope.test.mjs b/lint-plugins/no-comments/scope.test.mjsindex 7add32aff..5ee0bfc24 100644--- a/lint-plugins/no-comments/scope.test.mjs+++ b/lint-plugins/no-comments/scope.test.mjs@@ -24,8 +24,13 @@ describe('glob compilation', () => {['packages/**/src/**/*.ts', 'packages/app/tests/a.ts', false],['**/*.d.ts', 'docs/next-env.d.ts', true],['**/*.d.ts', 'a.d.ts', true],+ ['**/*.d.mts', 'docs/next-env.d.mts', true],+ ['**/*.d.cts', 'docs/lib/types.d.cts', true],+ ['**/*.d.mts', 'docs/vitest.real-source.config.mts', false],['*.ts', 'oxlint.config.ts', true],['*.ts', 'docs/tailwind.config.ts', false],+ ['*.ts', 'vitest.scripts.config.mts', false],+ ['docs/**/*.mts', 'docs/vitest.real-source.config.mts', true],['**/*.private.*', privateTestPath('packages/core/src', 'a'), true],];@@ -42,7 +47,10 @@ describe('scope membership', () => {'scripts/comment-fidelity.mjs','.github/scripts/bridge-public-pr-to-monorepo.mjs','docs/tailwind.config.ts',+ 'docs/vitest.real-source.config.mts',+ 'docs/lib/a.cts','oxlint.config.ts',+ 'vitest.scripts.config.mts',];const outOfScope = ['knip.config.ts',@@ -52,6 +60,9 @@ describe('scope membership', () => {privateTestPath('packages/core/src/markdown', 'a'),'packages/app/src/locales/en/messages.ts','docs/next-env.d.ts',+ 'docs/next-env.d.mts',+ 'docs/lib/types.d.cts',+ 'types.d.mts','lint-plugins/no-comments/__fixtures__/must-fire.fixture.ts','packages/app/src/lib/a.mjs','packages/app/node_modules/x/src/a.ts',@@ -93,7 +104,7 @@ describe('discovery over the real tree', () => {path.includes('/dist/') ||path.includes('.private.') ||path.includes('md-conformance/') ||- path.endsWith('.d.ts'),+ /\.d\.[mc]?ts$/.test(path),);expect(leaks).toEqual([]);});@@ -112,7 +123,12 @@ describe('the scope definition is derived, not restated', () => {});test('globToRegExp refuses syntax it does not implement instead of mis-compiling it', () => {- for (const glob of ['packages/**/{src,tests}/**/*.ts', 'packages/**/*.?s', '!packages/foo/**', 'src/[abc]/*.ts']) {+ for (const glob of [+ 'packages/**/{src,tests}/**/*.ts',+ 'packages/**/*.?s',+ '!packages/foo/**',+ 'src/[abc]/*.ts',+ ]) {expect(() => globToRegExp(glob), glob).toThrow(/unsupported glob syntax/);}});diff --git a/oxlint.config.ts b/oxlint.config.tsindex 589aed256..389e9353c 100644--- a/oxlint.config.ts+++ b/oxlint.config.ts@@ -32,7 +32,7 @@ const NO_COMMENTS_RULE = 'no-comments/no-comments';const NO_COMMENTS_SEVERITY = 'error';export default defineConfig({- ignorePatterns: ['.agents/skills/**', '.codex/skills/**', '/reports/**'],+ ignorePatterns: ['.agents/skills/**', '.codex/skills/**', '/reports/**', '/specs/**'],options: {typeAware: true,},diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 57ca13961..7fd349f66 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -269,6 +269,9 @@ importers:marked:specifier: ^18.0.6version: 18.0.7+ mdast-util-mdx:+ specifier: ^3.0.0+ version: 3.0.0mermaid:specifier: ~11.16.1version: 11.16.1@@ -293,9 +296,30 @@ importers:react-hook-form:specifier: ^7.80.0version: 7.81.0([email protected])+ remark-gfm:+ specifier: ^4.0.1+ version: 4.0.1+ remark-mdx:+ specifier: ^3.1.1+ version: 3.1.1+ remark-parse:+ specifier: ^11.0.0+ version: 11.0.0+ remark-stringify:+ specifier: ^11.0.0+ version: 11.0.0tailwind-merge:specifier: ^3.6.0version: 3.6.0+ unified:+ specifier: ^11.0.5+ version: 11.0.5+ unist-util-visit:+ specifier: ^5.1.0+ version: 5.1.0+ yaml:+ specifier: ^2.8.2+ version: 2.9.0zod:specifier: ^4.3.6version: 4.4.3@@ -303,6 +327,9 @@ importers:'@tailwindcss/postcss':specifier: ^4version: 4.3.2+ '@types/mdast':+ specifier: ^4.0.4+ version: 4.0.4'@types/node':specifier: ^24.7.0version: 24.13.3diff --git a/turbo.json b/turbo.jsonindex c373fc48b..7a3888b24 100644--- a/turbo.json+++ b/turbo.json@@ -42,7 +42,22 @@"@inkeep/open-knowledge-docs#build": {"dependsOn": ["^build", "typecheck"],"outputs": ["dist/**", ".next/**"],- "env": ["VERCEL_GIT_COMMIT_SHA", "GITHUB_SHA"]+ "inputs": [+ "$TURBO_DEFAULT$",+ "!src/**/*.test.ts",+ "!src/**/*.test.tsx",+ "!src/**/*.test-helper.ts",+ "!vitest.config.ts",+ "!vitest.*.config.mts"+ ],+ "env": [+ "VERCEL_GIT_COMMIT_SHA",+ "GITHUB_SHA",+ "VERCEL_ENV",+ "OK_ALLOW_BLOGLESS_LLMS_TXT",+ "CI",+ "VERCEL"+ ]},"test": {"dependsOn": ["^build"],@@ -60,6 +75,26 @@"../test-support/*.ts"]},+ "@inkeep/open-knowledge-docs#test": {+ "dependsOn": ["^build", "build"],+ "cache": true,+ "inputs": [+ "src/**/*.ts",+ "src/**/*.tsx",+ "src/**/*.json",+ "content/**",+ "_snippets/**",+ "next.config.ts",+ "source.config.ts",+ "microfrontends.json",+ "postcss.config.ts",+ "tsconfig.json",+ "vitest.config.ts",+ "vitest.*.config.mts",+ "../../test-support/*.ts",+ "../test-support/*.ts"+ ]+ },"@inkeep/open-knowledge#test": {"dependsOn": ["^build", "build"],"cache": true,