Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

feat(ok): LLM-friendly web surfaces for docs and marketing (#4060)

Nick Gomez8f61ba6b88bc
diff --git a/.github/scripts/share-contract-cache-shape.test.mjs b/.github/scripts/share-contract-cache-shape.test.mjs
index 45b220f53..ec2642723 100644
--- a/.github/scripts/share-contract-cache-shape.test.mjs
+++ b/.github/scripts/share-contract-cache-shape.test.mjs
@@ -18,6 +18,13 @@ describe('share contract corpus cache wiring', () => {
const turbo = JSON.parse(readFileSync(join(REPO_ROOT, 'turbo.json'), 'utf8'));
const docsBuild = turbo.tasks['@inkeep/open-knowledge-docs#build'];
- expect(docsBuild.env).toEqual(['VERCEL_GIT_COMMIT_SHA', 'GITHUB_SHA']);
+ expect(docsBuild.env).toEqual([
+ 'VERCEL_GIT_COMMIT_SHA',
+ 'GITHUB_SHA',
+ 'VERCEL_ENV',
+ 'OK_ALLOW_BLOGLESS_LLMS_TXT',
+ 'CI',
+ 'VERCEL',
+ ]);
});
});
diff --git a/biome.jsonc b/biome.jsonc
index 6aa71d903..57a66e880 100644
--- a/biome.jsonc
+++ b/biome.jsonc
@@ -444,6 +444,9 @@
"!packages/native-config/target",
"!**/.next",
"!**/.source",
+ // Second generated content build, emitted by the docs app's real-source
+ // vitest config in the Vite flavour `.source` must never hold.
+ "!**/.source-vitest",
"!**/.turbo",
"!**/tmp",
"!**/test-results",
diff --git a/docs/.gitignore b/docs/.gitignore
index 2344cdbab..05fdc4bb7 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1,5 +1,8 @@
.next/
.source/
+# Second content build, emitted by vitest.real-source.config.mts in the Vite
+# flavour that `.source/` must never hold.
+.source-vitest/
.vercel
.env
.env.local
diff --git a/docs/next.config.ts b/docs/next.config.ts
index 2c68859a0..f8e2f641b 100644
--- a/docs/next.config.ts
+++ b/docs/next.config.ts
@@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url';
import { withMicrofrontends } from '@vercel/microfrontends/next/config';
import { createMDX } from 'fumadocs-mdx/next';
import type { NextConfig } from 'next';
+import { markdownRewrites } from './src/lib/markdown-routes';
const nextConfig: NextConfig = {
reactStrictMode: true,
@@ -15,16 +16,7 @@ const nextConfig: NextConfig = {
skipTrailingSlashRedirect: true,
async rewrites() {
return {
- beforeFiles: [
- {
- source: '/docs/:path*.md',
- destination: '/llms.mdx/:path*',
- },
- {
- source: '/docs/:path*.mdx',
- destination: '/llms.mdx/:path*',
- },
- ],
+ beforeFiles: markdownRewrites(),
afterFiles: [
{
source: '/ingest/static/:path*',
diff --git a/docs/package.json b/docs/package.json
index 5416e0950..9e99fc72f 100644
--- a/docs/package.json
+++ b/docs/package.json
@@ -9,7 +9,8 @@
"build": "next build",
"prebuild": "pnpm run validate-link",
"start": "next start --port 3010",
- "test": "vitest run src",
+ "test": "vitest run src && pnpm run test:real-source",
+ "test:real-source": "vitest run --config vitest.real-source.config.mts",
"typecheck": "next typegen && fumadocs-mdx && tsc --noEmit",
"validate-link": "tsx scripts/validate-link.ts",
"generate:og-wordmark": "tsx scripts/generate-og-wordmark.ts",
@@ -31,19 +32,28 @@
"katex": "^0.16.22",
"lucide-react": "^0.503.0",
"marked": "^18.0.6",
+ "mdast-util-mdx": "^3.0.0",
"mermaid": "^11.12.3",
- "next": "^16",
+ "next": "^16.2.4",
"next-themes": "^0.4.6",
"posthog-js": "^1.393.0",
"radix-ui": "^1.4.3",
"react": "^19",
"react-dom": "^19",
"react-hook-form": "^7.80.0",
+ "remark-gfm": "^4.0.1",
+ "remark-mdx": "^3.1.1",
+ "remark-parse": "^11.0.0",
+ "remark-stringify": "^11.0.0",
"tailwind-merge": "^3.6.0",
+ "unified": "^11.0.5",
+ "unist-util-visit": "^5.1.0",
+ "yaml": "^2.8.2",
"zod": "^4.3.6"
},
"devDependencies": {
"@tailwindcss/postcss": "^4",
+ "@types/mdast": "^4.0.4",
"@types/node": "^24.0.7",
"@types/react": "^19",
"@types/react-dom": "^19",
diff --git a/docs/source.config.ts b/docs/source.config.ts
index 4cb96c023..95355c4c1 100644
--- a/docs/source.config.ts
+++ b/docs/source.config.ts
@@ -8,9 +8,6 @@ import { z } from 'zod';
export const docs = defineDocs({
dir: 'content',
docs: {
- postprocess: {
- includeProcessedMarkdown: true,
- },
schema: frontmatterSchema.extend({
sidebarTitle: z.string().optional(),
keywords: z.string().optional(),
diff --git a/docs/src/app/app-files.test-helper.ts b/docs/src/app/app-files.test-helper.ts
new file mode 100644
index 000000000..2baa4e6be
--- /dev/null
+++ b/docs/src/app/app-files.test-helper.ts
@@ -0,0 +1,46 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+const APP_ROOT = fileURLToPath(new URL('.', import.meta.url));
+
+const ROUTER_EXTENSIONS = ['ts', 'tsx', 'js', 'jsx'] as const;
+
+const spellings = (stem: string): string[] =>
+ ROUTER_EXTENSIONS.map((extension) => `${stem}.${extension}`);
+
+export const ROUTE_FILES: readonly string[] = spellings('route');
+export const PAGE_FILES: readonly string[] = spellings('page');
+
+const METADATA_ROUTE_STEMS: Readonly<Record<string, string>> = {
+ sitemap: 'sitemap.xml',
+ robots: 'robots.txt',
+ manifest: 'manifest.webmanifest',
+ 'opengraph-image': 'opengraph-image',
+ 'twitter-image': 'twitter-image',
+ icon: 'icon',
+ 'apple-icon': 'apple-icon',
+};
+
+export const METADATA_ROUTE_FILES: readonly string[] =
+ Object.keys(METADATA_ROUTE_STEMS).flatMap(spellings);
+
+export function metadataRouteName(fileName: string, sourcePath: string): string | null {
+ const stem = fileName.slice(0, fileName.indexOf('.'));
+ const served = METADATA_ROUTE_STEMS[stem];
+ if (!served || !METADATA_ROUTE_FILES.includes(fileName)) return null;
+ const grouped = sourcePath.split('/').some((segment) => /^[(@].+/.test(segment));
+ return served.startsWith('sitemap') || !grouped ? served : `${served}-a1b2c3`;
+}
+
+export function filesNamed(name: string | readonly string[], dir = APP_ROOT, base = ''): string[] {
+ const names = typeof name === 'string' ? [name] : name;
+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+ const rel = base ? `${base}/${entry.name}` : entry.name;
+ if (entry.isDirectory()) return filesNamed(names, `${dir}${entry.name}/`, rel);
+ return names.includes(entry.name) ? [rel] : [];
+ });
+}
+
+export function readAppFile(relativePath: string): string {
+ return readFileSync(`${APP_ROOT}${relativePath}`, 'utf8');
+}
diff --git a/docs/src/app/docs/[...slug]/page.tsx b/docs/src/app/docs/[...slug]/page.tsx
index 944686008..e57216538 100644
--- a/docs/src/app/docs/[...slug]/page.tsx
+++ b/docs/src/app/docs/[...slug]/page.tsx
@@ -4,7 +4,7 @@ import type { Metadata } from 'next';
import { notFound } from 'next/navigation';
import { PageMarkdownActions } from '@/components/page-markdown-actions';
import { ProductUpdatesForm } from '@/components/product-updates-form';
-import { metaDescription, SITE_NAME, SITE_URL, TWITTER_HANDLE } from '@/lib/site';
+import { absoluteSiteUrl, metaDescription, SITE_NAME, SITE_URL, TWITTER_HANDLE } from '@/lib/site';
import { source } from '@/lib/source';
import { getMDXComponents } from '@/mdx-components';
@@ -67,6 +67,7 @@ export async function generateMetadata(props: PageProps<'/docs/[...slug]'>): Pro
keywords,
alternates: {
canonical: page.url,
+ types: { 'text/markdown': absoluteSiteUrl(`${page.url}.md`) },
},
openGraph: {
type: 'article',
diff --git a/docs/src/app/docs/markdown-alternate-link.test.ts b/docs/src/app/docs/markdown-alternate-link.test.ts
new file mode 100644
index 000000000..207a46363
--- /dev/null
+++ b/docs/src/app/docs/markdown-alternate-link.test.ts
@@ -0,0 +1,39 @@
+import { describe, expect, test, vi } from 'vitest';
+
+const overviewPage = {
+ url: '/docs/get-started/overview',
+ data: { title: 'Overview', description: 'What OpenKnowledge is.' },
+};
+
+vi.doMock('@/lib/source', () => ({
+ source: { getPage: () => overviewPage, generateParams: () => [] },
+}));
+
+const { generateMetadata } = await import('./[...slug]/page.tsx');
+const changelog = await import('./changelog/page.tsx');
+
+describe('docs page markdown alternate link', () => {
+ test('a docs page advertises its .md twin at an absolute URL', async () => {
+ const meta = await generateMetadata({
+ params: Promise.resolve({ slug: ['get-started', 'overview'] }),
+ } as never);
+
+ expect(meta.alternates?.types).toMatchObject({
+ 'text/markdown': 'https://openknowledge.ai/docs/get-started/overview.md',
+ });
+ });
+
+ test('the advertised twin is the URL the markdown rewrite serves', async () => {
+ const meta = await generateMetadata({
+ params: Promise.resolve({ slug: ['get-started', 'overview'] }),
+ } as never);
+ const advertised = String(meta.alternates?.types?.['text/markdown']);
+
+ expect(new URL(advertised).pathname).toBe(`${overviewPage.url}.md`);
+ });
+
+ test('a page with no markdown producer advertises no markdown twin', async () => {
+ expect(changelog.metadata.alternates?.types).not.toHaveProperty('text/markdown');
+ expect(changelog.metadata.alternates?.types).toHaveProperty('application/rss+xml');
+ });
+});
diff --git a/docs/src/app/llms-full.txt/route.test.ts b/docs/src/app/llms-full.txt/route.test.ts
new file mode 100644
index 000000000..4a196e882
--- /dev/null
+++ b/docs/src/app/llms-full.txt/route.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, test, vi } from 'vitest';
+import { corpusSourcePages, readCorpus } from '@/lib/docs-corpus.test-helper';
+import { classifyLlmsFullSize, describeLlmsFullSize } from '@/lib/llms-full-size.test-helper';
+
+const pages = corpusSourcePages(await readCorpus());
+vi.doMock('@/lib/source', () => ({ source: { getPages: () => pages } }));
+
+const { GET } = await import('./route.ts');
+
+describe('GET /llms-full.txt', () => {
+ test('serves the corpus as cacheable markdown', async () => {
+ const res = await GET();
+ expect(res.status).toBe(200);
+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');
+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);
+ });
+
+ test('renders no canonical of its own, since it renders no single page', async () => {
+ expect((await GET()).headers.get('link')).toBeNull();
+ });
+
+ test('stays inside the size an agent can actually fetch', async () => {
+ const bytes = Buffer.byteLength(await (await GET()).text(), 'utf8');
+ const verdict = classifyLlmsFullSize(bytes);
+ if (verdict === 'warn') process.stderr.write(`${describeLlmsFullSize(bytes)}\n`);
+ expect(verdict, describeLlmsFullSize(bytes)).not.toBe('fail');
+ });
+
+ test('carries every page in the corpus', async () => {
+ const body = await (await GET()).text();
+ for (const page of pages) {
+ expect(body).toContain(`(https://openknowledge.ai${page.url})`);
+ }
+ });
+});
diff --git a/docs/src/app/llms-full.txt/route.ts b/docs/src/app/llms-full.txt/route.ts
index 13e048929..957ccfd30 100644
--- a/docs/src/app/llms-full.txt/route.ts
+++ b/docs/src/app/llms-full.txt/route.ts
@@ -1,4 +1,5 @@
import { getLLMText } from '@/lib/get-llm-text';
+import { markdownResponse } from '@/lib/markdown-response';
import { source } from '@/lib/source';
export const revalidate = false;
@@ -7,5 +8,5 @@ export async function GET() {
const pages = source.getPages();
const scanned = await Promise.all(pages.map((page) => getLLMText(page)));
- return new Response(scanned.join('\n\n'));
+ return markdownResponse(scanned.join('\n\n'));
}
diff --git a/docs/src/app/llms.mdx/[...slug]/route.test.ts b/docs/src/app/llms.mdx/[...slug]/route.test.ts
index 8f8e126ef..2e2e7dc15 100644
--- a/docs/src/app/llms.mdx/[...slug]/route.test.ts
+++ b/docs/src/app/llms.mdx/[...slug]/route.test.ts
@@ -1,11 +1,12 @@
import { describe, expect, test, vi } from 'vitest';
+import { NO_MARKDOWN_PREFIXES } from '@/lib/markdown-routes';
const overviewPage = {
url: '/docs/get-started/overview',
data: {
title: 'Overview',
description: 'What OpenKnowledge is.',
- getText: async () => 'PROCESSED BODY',
+ getText: async () => '<Callout type="info">\n\nEarly days.\n\n</Callout>\n',
},
};
@@ -17,35 +18,80 @@ vi.doMock('@/lib/source', () => ({
},
}));
-vi.doMock('next/navigation', () => ({
- notFound: () => {
- throw new Error('NEXT_HTTP_ERROR_FALLBACK;404');
- },
-}));
-
const { GET, generateStaticParams } = await import('./route.ts');
function props(slug: string[]) {
return { params: Promise.resolve({ slug }) };
}
+function get(slug: string[]) {
+ return GET(new Request(`https://openknowledge.ai/docs/${slug.join('/')}.md`), {
+ ...props(slug),
+ });
+}
+
describe('GET /docs/<slug>.md (markdown route handler)', () => {
test('serves text/markdown with the page rendered by getLLMText', async () => {
- const res = await GET(new Request('https://openknowledge.ai/docs/get-started/overview.md'), {
- ...props(['get-started', 'overview']),
- });
+ const res = await get(['get-started', 'overview']);
expect(res.status).toBe(200);
expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');
const body = await res.text();
- expect(body).toContain('# Overview (/docs/get-started/overview)');
- expect(body).toContain('PROCESSED BODY');
+ expect(body).toContain('# Overview (https://openknowledge.ai/docs/get-started/overview)');
+ expect(body).toContain('> Early days.');
+ expect(body).not.toContain('<Callout');
});
- test('calls notFound() for an unknown page', async () => {
- await expect(
- GET(new Request('https://openknowledge.ai/docs/nope.md'), { ...props(['nope']) }),
- ).rejects.toThrow('404');
+ test('points at the human page with a canonical Link header', async () => {
+ const res = await get(['get-started', 'overview']);
+ expect(res.headers.get('link')).toBe(
+ '<https://openknowledge.ai/docs/get-started/overview>; rel="canonical"',
+ );
+ });
+
+ test('is edge-cacheable', async () => {
+ const res = await get(['get-started', 'overview']);
+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);
+ });
+
+ test('answers an unknown page with a hard 404 whose body is markdown', async () => {
+ const res = await get(['nope']);
+ expect(res.status).toBe(404);
+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');
+
+ const body = await res.text();
+ expect(body).toContain('# 404 Not Found');
+ expect(body).toContain('https://openknowledge.ai/llms.txt');
+ });
+
+ test('the 404 body is not mistakable for a real page rendition', async () => {
+ const [missing, real] = await Promise.all([
+ get(['nope']).then((r) => r.text()),
+ get(['get-started', 'overview']).then((r) => r.text()),
+ ]);
+ expect(missing).not.toBe(real);
+ });
+
+ test.each(
+ NO_MARKDOWN_PREFIXES,
+ )('%s tells a real page apart from a URL no page holds', async (prefix) => {
+ const slug = [...prefix.replace('/docs/', '').split('/'), 'some-page'];
+ const [subtree, missing] = await Promise.all([
+ get(slug).then((r) => r.text()),
+ get(['nope']).then((r) => r.text()),
+ ]);
+
+ expect(subtree).not.toBe(missing);
+ expect(subtree).not.toContain('No OpenKnowledge documentation page exists');
+ expect(subtree).toContain(`https://openknowledge.ai${prefix}`);
+ });
+
+ test.each(
+ NO_MARKDOWN_PREFIXES,
+ )('%s still answers with a hard 404 carrying the contract', async (prefix) => {
+ const res = await get([...prefix.replace('/docs/', '').split('/'), 'some-page']);
+ expect(res.status).toBe(404);
+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');
});
test('generateStaticParams delegates to the loader and yields slug-shaped params', () => {
diff --git a/docs/src/app/llms.mdx/[...slug]/route.ts b/docs/src/app/llms.mdx/[...slug]/route.ts
index b12aa707a..8b0390000 100644
--- a/docs/src/app/llms.mdx/[...slug]/route.ts
+++ b/docs/src/app/llms.mdx/[...slug]/route.ts
@@ -1,9 +1,29 @@
-import { notFound } from 'next/navigation';
import { getLLMText } from '@/lib/get-llm-text';
+import { markdownResponse } from '@/lib/markdown-response';
+import { unrenditionedSubtree } from '@/lib/markdown-routes';
+import { absoluteSiteUrl, SITE_URL } from '@/lib/site';
import { source } from '@/lib/source';
export const dynamic = 'force-static';
+const NOT_FOUND_BODY = `# 404 Not Found
+
+No OpenKnowledge documentation page exists at this URL.
+
+Every published page is listed at <${SITE_URL}/llms.txt>, each with a \`.md\`
+rendition of its own.
+`;
+
+function noRenditionBody(subtree: string): string {
+ return `# 404 Not Found
+
+There is no Markdown rendition under ${subtree}. Those pages are published as
+HTML only; read them at <${SITE_URL}${subtree}>.
+
+Every page that does have one is listed at <${SITE_URL}/llms.txt>.
+`;
+}
+
interface RouteProps {
params: Promise<{ slug: string[] }>;
}
@@ -11,10 +31,13 @@ interface RouteProps {
export async function GET(_request: Request, props: RouteProps) {
const { slug } = await props.params;
const page = source.getPage(slug);
- if (!page) notFound();
+ if (!page) {
+ const subtree = unrenditionedSubtree(slug);
+ return markdownResponse(subtree ? noRenditionBody(subtree) : NOT_FOUND_BODY, { status: 404 });
+ }
- return new Response(await getLLMText(page), {
- headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+ return markdownResponse(await getLLMText(page), {
+ canonicalUrl: absoluteSiteUrl(page.url),
});
}
diff --git a/docs/src/app/llms.txt/route.test.ts b/docs/src/app/llms.txt/route.test.ts
new file mode 100644
index 000000000..711678e6d
--- /dev/null
+++ b/docs/src/app/llms.txt/route.test.ts
@@ -0,0 +1,113 @@
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+import { corpusSourcePages, readCorpus } from '@/lib/docs-corpus.test-helper';
+import { markdownHandlerForPage } from '@/lib/markdown-routes';
+import { MARKETING_MARKDOWN_PAGES } from '@/lib/marketing-markdown-pages';
+
+const pages = corpusSourcePages(await readCorpus());
+vi.doMock('@/lib/source', () => ({ source: { getPages: () => pages } }));
+
+const { GET, revalidate } = await import('./route.ts');
+
+const FEED = [
+ '<rss version="2.0"><channel>',
+ '<item><title><![CDATA[Keeping you and your agents in check with markdownlint]]></title>',
+ '<link>https://openknowledge.ai/blog/markdownlint-support</link>',
+ '<description><![CDATA[Native markdownlint support in Open Knowledge.]]></description></item>',
+ '</channel></rss>',
+].join('\n');
+
+beforeEach(() => {
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async () => new Response(FEED)),
+ );
+ return () => vi.unstubAllGlobals();
+});
+
+async function body(): Promise<string> {
+ return (await GET()).text();
+}
+
+function listedUrls(text: string): string[] {
+ return [...text.matchAll(/^- \[(?:\\.|[^\]\\])*]\(([^)]+)\)/gm)].map((match) => match[1]);
+}
+
+describe('GET /llms.txt', () => {
+ test('serves markdown the edge can cache', async () => {
+ const res = await GET();
+ expect(res.status).toBe(200);
+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');
+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);
+ });
+
+ test('refreshes hourly, since the blog it lists is published elsewhere', () => {
+ expect(revalidate).toBe(3600);
+ });
+
+ test('lists one entry per docs page, derived from the corpus rather than a count', async () => {
+ const docsSection = (await body()).split('## Blog')[0];
+ expect(listedUrls(docsSection)).toHaveLength(pages.length);
+ expect(pages.length).toBeGreaterThan(50);
+ });
+
+ test('links the Markdown rendition of every docs page', async () => {
+ const listed = new Set(listedUrls(await body()));
+ for (const page of pages) {
+ expect(listed).toContain(`https://openknowledge.ai${page.url}.md`);
+ }
+ });
+
+ test('names each docs page by its own title', async () => {
+ const quickstart = pages.find((page) => page.url === '/docs/get-started/quickstart');
+ expect(quickstart).toBeDefined();
+ expect(await body()).toContain(
+ `- [${quickstart?.data.title}](https://openknowledge.ai/docs/get-started/quickstart.md)`,
+ );
+ });
+
+ test('covers the marketing zone the docs app cannot enumerate', async () => {
+ const listed = new Set(listedUrls(await body()));
+ for (const page of MARKETING_MARKDOWN_PAGES) {
+ expect(listed).toContain(`https://openknowledge.ai${page.path}`);
+ }
+ });
+
+ test('lists the blog posts the marketing zone reported, under their own titles', async () => {
+ expect(await body()).toContain(
+ '- [Keeping you and your agents in check with markdownlint]' +
+ '(https://openknowledge.ai/blog/markdownlint-support.md): ' +
+ 'Native markdownlint support in Open Knowledge.',
+ );
+ });
+
+ test('keeps its own promise that every page it names negotiates Markdown', async () => {
+ const docsPages = [...(await body()).matchAll(/^- \[(?:\\.|[^\]\\])*]\((\S+)\.md\)/gm)]
+ .map((match) => match[1].replace('https://openknowledge.ai', ''))
+ .filter((path) => path.startsWith('/docs/'));
+
+ expect(
+ docsPages.length,
+ 'no docs entries found; this check is reading nothing',
+ ).toBeGreaterThan(10);
+ expect(docsPages.filter((path) => markdownHandlerForPage(path) === null)).toEqual([]);
+ });
+
+ test('holds the llms.txt shape: title, summary, then named sections', async () => {
+ const text = await body();
+ expect(text.startsWith('# OpenKnowledge\n\n> ')).toBe(true);
+ expect([...text.matchAll(/^## (.+)$/gm)].map((match) => match[1])).toEqual([
+ 'Docs',
+ 'Blog',
+ 'Product',
+ ]);
+ });
+
+ test('every link is absolute and resolves to a Markdown rendition', async () => {
+ const urls = listedUrls(await body());
+ expect(urls.length).toBeGreaterThan(50);
+ for (const url of urls) {
+ expect(url).toMatch(/^https:\/\/openknowledge\.ai\/[^\s]*\.md$/);
+ }
+ expect(urls).toEqual([...new Set(urls)]);
+ });
+});
diff --git a/docs/src/app/llms.txt/route.ts b/docs/src/app/llms.txt/route.ts
index a2e621b98..5b7f57ce3 100644
--- a/docs/src/app/llms.txt/route.ts
+++ b/docs/src/app/llms.txt/route.ts
@@ -1,15 +1,17 @@
-import { SITE_URL } from '@/lib/site';
+import { buildLlmsTxt, type LlmsTxtLink } from '@/lib/llms-txt';
+import { markdownResponse } from '@/lib/markdown-response';
+import { blogPostLinks } from '@/lib/marketing-blog-index';
+import { absoluteSiteUrl } from '@/lib/site';
import { source } from '@/lib/source';
-export const revalidate = false;
+export const revalidate = 3600;
export async function GET() {
- const pages = source.getPages();
- return new Response(
- [
- '# OpenKnowledge',
- '## Docs',
- ...pages.map((page) => `- [${page.data.title}](${SITE_URL}${page.url}.md)`),
- ].join('\n\n'),
- );
+ const docs: LlmsTxtLink[] = source.getPages().map((page) => ({
+ url: absoluteSiteUrl(`${page.url}.md`),
+ name: page.data.title,
+ description: page.data.description || undefined,
+ }));
+
+ return markdownResponse(buildLlmsTxt({ docs, blogPosts: await blogPostLinks(revalidate) }));
}
diff --git a/docs/src/app/markdown-route-coverage.test.ts b/docs/src/app/markdown-route-coverage.test.ts
new file mode 100644
index 000000000..6672478d2
--- /dev/null
+++ b/docs/src/app/markdown-route-coverage.test.ts
@@ -0,0 +1,148 @@
+import { describe, expect, test } from 'vitest';
+import { markdownHandlerForPage, PAGE_MARKDOWN_ROUTE } from '@/lib/markdown-routes';
+import {
+ filesNamed,
+ METADATA_ROUTE_FILES,
+ metadataRouteName,
+ PAGE_FILES,
+ ROUTE_FILES,
+} from './app-files.test-helper.ts';
+
+const NO_MARKDOWN: Record<string, string> = {
+ '/continue': 'hand-off screen for the desktop app; carries no durable content',
+ '/d/[encoded]': 'renders a share payload carried in the URL, unlisted and ephemeral by design',
+ '/docs': 'redirects to the overview page rather than rendering content of its own',
+ '/docs/changelog':
+ "built from the CLI's own CHANGELOG.md through a source adapter the Markdown handler cannot read",
+ '/docs/changelog/[tag]':
+ "built from the CLI's own CHANGELOG.md through a source adapter the Markdown handler cannot read",
+};
+
+const ROUTE_GROUP = /^\(.+\)$/;
+const DYNAMIC_SEGMENT = /^\[(\.\.\.)?[^[\]./]+\]$/;
+const LITERAL_SEGMENT = /^[\w.-]+$/;
+
+function routePathOf(file: string): string {
+ const segments = file.split('/').slice(0, -1);
+ const parts = segments.filter((segment) => {
+ if (ROUTE_GROUP.test(segment)) return false;
+ if (DYNAMIC_SEGMENT.test(segment) || LITERAL_SEGMENT.test(segment)) return true;
+ throw new Error(`unrecognised app-router segment "${segment}" in ${file}`);
+ });
+ return `/${parts.join('/')}`;
+}
+
+function metadataRoutePathOf(file: string): string {
+ const fileName = file.slice(file.lastIndexOf('/') + 1);
+ const dir = routePathOf(file);
+ const served = metadataRouteName(fileName, file);
+ if (!served) throw new Error(`unmapped metadata-route convention "${fileName}" in ${file}`);
+ return `${dir === '/' ? '' : dir}/${served}`;
+}
+
+function sampleUrlOf(routePath: string): string {
+ return routePath.replace(/\[(?:\.\.\.)?([^[\]]+)\]/g, 'sample-$1');
+}
+
+function routeMatcher(routePath: string): RegExp {
+ const body = routePath
+ .split('/')
+ .map((segment) => {
+ if (/^\[\.\.\..+\]$/.test(segment)) return '.+';
+ if (/^\[.+\]$/.test(segment)) return '[^/]+';
+ return segment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ })
+ .join('/');
+ return new RegExp(`^${body}$`);
+}
+
+const pageRoutes = filesNamed(PAGE_FILES).map((file) => ({ file, route: routePathOf(file) }));
+const handlerRoutes = filesNamed(ROUTE_FILES).map(routePathOf);
+
+const nonPageRoutes = [
+ ...filesNamed(ROUTE_FILES).map(routePathOf),
+ ...filesNamed(METADATA_ROUTE_FILES).map(metadataRoutePathOf),
+].filter((route) => !route.startsWith(`${PAGE_MARKDOWN_ROUTE}/`));
+
+describe('markdown route coverage', () => {
+ test('every page route has a markdown producer or a stated exemption', () => {
+ const uncovered = pageRoutes
+ .filter(({ route }) => !markdownHandlerForPage(sampleUrlOf(route)) && !NO_MARKDOWN[route])
+ .map(
+ ({ file, route }) =>
+ `${route} (src/app/${file}): serve it from src/lib/markdown-routes.ts, or add "${route}" to NO_MARKDOWN in this file with the reason it has none`,
+ );
+
+ expect(uncovered, 'page routes with neither a markdown rendition nor an exemption').toEqual([]);
+ });
+
+ test('the census reaches the pages it is meant to police', () => {
+ expect(pageRoutes.map(({ route }) => route)).toContain('/docs/[...slug]');
+ });
+
+ test.each(Object.entries(NO_MARKDOWN))('%s is exempt from markdown: %s', (route, reason) => {
+ expect(reason.trim(), `${route} is exempted without saying why`).not.toBe('');
+ expect(
+ pageRoutes.map((page) => page.route),
+ `${route} is exempted but no page renders it; drop the NO_MARKDOWN entry`,
+ ).toContain(route);
+ expect(
+ markdownHandlerForPage(sampleUrlOf(route)),
+ `${route} is exempted but does have a markdown rendition; drop the NO_MARKDOWN entry`,
+ ).toBeNull();
+ });
+
+ test('never negotiates a non-page route into a markdown handler', () => {
+ const negotiating = nonPageRoutes
+ .filter((route) => markdownHandlerForPage(sampleUrlOf(route)) !== null)
+ .map(
+ (route) =>
+ `${route}: a Markdown-preferring agent gets a 404 instead of this route's own body; add its prefix to NO_MARKDOWN_PREFIXES in src/lib/markdown-routes.ts`,
+ );
+
+ expect(negotiating, 'non-page routes that shape alone maps to a slug handler').toEqual([]);
+ });
+
+ test('reaches the non-page routes it is meant to police', () => {
+ expect(nonPageRoutes).toContain('/docs/changelog/rss.xml');
+ });
+
+ test('every markdown producer names a handler this app implements', () => {
+ const missing = pageRoutes
+ .map(({ route }) => ({ route, handler: markdownHandlerForPage(sampleUrlOf(route)) }))
+ .filter(
+ ({ handler }) =>
+ handler !== null && !handlerRoutes.some((r) => routeMatcher(r).test(handler)),
+ )
+ .map(({ route, handler }) => `${route} -> ${handler} (no route.ts serves it)`);
+
+ expect(missing, 'markdown producers with no handler').toEqual([]);
+ });
+
+ test('enumerates every spelling the router accepts for each convention', () => {
+ for (const list of [PAGE_FILES, ROUTE_FILES]) {
+ expect(list.map((name) => name.split('.').pop())?.sort()).toEqual(['js', 'jsx', 'ts', 'tsx']);
+ }
+ expect([...new Set(METADATA_ROUTE_FILES.map((name) => name.split('.')[0]))].sort()).toEqual([
+ 'apple-icon',
+ 'icon',
+ 'manifest',
+ 'opengraph-image',
+ 'robots',
+ 'sitemap',
+ 'twitter-image',
+ ]);
+ expect(METADATA_ROUTE_FILES).toHaveLength(28);
+ });
+
+ test.each([
+ ['sitemap.ts', 'blog/sitemap.ts', 'sitemap.xml'],
+ ['sitemap.ts', '(home)/blog/sitemap.ts', 'sitemap.xml'],
+ ['opengraph-image.tsx', 'blog/opengraph-image.tsx', 'opengraph-image'],
+ ['opengraph-image.tsx', '(home)/blog/opengraph-image.tsx', 'opengraph-image-a1b2c3'],
+ ['opengraph-image.png', 'blog/opengraph-image.png', null],
+ ['route.ts', 'blog/route.ts', null],
+ ])('serves %s (source %s) at %s', (fileName, sourcePath, expected) => {
+ expect(metadataRouteName(fileName, sourcePath)).toBe(expected);
+ });
+});
diff --git a/docs/src/components/agent-icons.tsx b/docs/src/components/agent-icons.tsx
index 81e37520b..261a6475a 100644
--- a/docs/src/components/agent-icons.tsx
+++ b/docs/src/components/agent-icons.tsx
@@ -6,7 +6,7 @@ import { OpenClawIcon } from '@/components/icons/openclaw';
import { OpenCodeIcon } from '@/components/icons/opencode';
import { PiIcon } from '@/components/icons/pi';
-const AGENTS = [
+export const AGENTS = [
{ name: 'Claude', Icon: ClaudeIcon },
{ name: 'Codex', Icon: CodexIcon },
{ name: 'Cursor', Icon: CursorIcon },
diff --git a/docs/src/components/download-split-button.test.tsx b/docs/src/components/download-split-button.test.tsx
new file mode 100644
index 000000000..baad98466
--- /dev/null
+++ b/docs/src/components/download-split-button.test.tsx
@@ -0,0 +1,20 @@
+import { renderToStaticMarkup } from 'react-dom/server';
+import { describe, expect, test } from 'vitest';
+import { DownloadButton } from '@/components/download-button';
+import { downloadLabelForOs } from '@/lib/download-targets';
+
+const html = renderToStaticMarkup(<DownloadButton />);
+
+describe('the docs download button before hydration', () => {
+ test('names no platform', () => {
+ expect(html).toContain(`>${downloadLabelForOs('unknown')}<`);
+ for (const os of ['macos', 'windows', 'linux'] as const) {
+ expect(html).not.toContain(downloadLabelForOs(os));
+ }
+ });
+
+ test('links a build the visitor can run without script', () => {
+ const hrefs = html.replaceAll('&amp;', '&');
+ expect(hrefs).toContain('os=macos&arch=arm64&format=dmg');
+ });
+});
diff --git a/docs/src/components/mcp-install.tsx b/docs/src/components/mcp-install.tsx
index f57080bdb..75791af12 100644
--- a/docs/src/components/mcp-install.tsx
+++ b/docs/src/components/mcp-install.tsx
@@ -1,4 +1,20 @@
import type { ReactNode } from 'react';
+import type { SerializerApi } from '@/lib/mdx-serializer';
+
+const WEB_APP_GUIDE_HREF = '/docs/get-started/quickstart#ok-install-web-app';
+
+interface McpInstallMarkdownApi extends Pick<SerializerApi, 'url'> {
+ link(label: string, url: string): string;
+}
+
+export function mcpInstallMarkdown(editor: string, api: McpInstallMarkdownApi): string {
+ return [
+ `There are two ways to connect ${editor}, depending on how you run OpenKnowledge:`,
+ '',
+ `- **Desktop app** (macOS, Windows, Linux). The first time you open a project, a consent dialog detects ${editor} and configures it for you. To re-trigger the dialog, choose **File → Set up OpenKnowledge integrations…**.`,
+ `- **Web app / terminal** (any platform, including Intel Macs — see the ${api.link('web app guide', api.url(WEB_APP_GUIDE_HREF))}). Run \`ok init\` in your project: it registers the OpenKnowledge MCP server with ${editor} and the other editors it detects. Every \`ok start\` repairs the entry if it has drifted (it never adds one you removed).`,
+ ].join('\n');
+}
export function McpInstall({ editor, children }: { editor: string; children?: ReactNode }) {
return (
@@ -14,7 +30,7 @@ export function McpInstall({ editor, children }: { editor: string; children?: Re
</li>
<li>
<strong>Web app / terminal</strong> (any platform, including Intel Macs — see the{' '}
- <a href="/docs/get-started/quickstart#ok-install-web-app">web app guide</a>
+ <a href={WEB_APP_GUIDE_HREF}>web app guide</a>
). Run <code>ok init</code> in your project: it registers the OpenKnowledge MCP server
with {editor} and the other editors it detects. Every <code>ok start</code> repairs the
entry if it has drifted (it never adds one you removed).
diff --git a/docs/src/components/overview-blocks.tsx b/docs/src/components/overview-blocks.tsx
index 49926c96d..d6451a090 100644
--- a/docs/src/components/overview-blocks.tsx
+++ b/docs/src/components/overview-blocks.tsx
@@ -1,15 +1,14 @@
import { ArrowRight, Bot, FileText, type LucideIcon, NotebookPen } from 'lucide-react';
-import type { ReactNode } from 'react';
interface Layer {
k: string;
Icon: LucideIcon;
title: string;
role: string;
- desc: ReactNode;
+ desc: string;
}
-const LAYERS: Layer[] = [
+export const LAYERS: Layer[] = [
{
k: 'LAYER 01',
Icon: NotebookPen,
@@ -73,7 +72,7 @@ interface Path {
desc: string;
}
-const PATHS: Path[] = [
+export const PATHS: Path[] = [
{
href: '/docs/get-started/quickstart',
title: 'Try the Quickstart',
diff --git a/docs/src/components/verify-exec.tsx b/docs/src/components/verify-exec.tsx
index 92bbaf4a2..ce346ce87 100644
--- a/docs/src/components/verify-exec.tsx
+++ b/docs/src/components/verify-exec.tsx
@@ -1,9 +1,19 @@
import { CopyPrompt } from '@/components/copy-prompt';
+const VERIFY_PROMPT = 'List the first 5 documents you come across in this project.';
+
+function verifyExpectation(subject: string): string {
+ return `${subject} should call the OpenKnowledge \`exec\` tool and respond with some of your documents.`;
+}
+
+export function verifyExecMarkdown(subject: string): string {
+ return `> ${VERIFY_PROMPT}\n\n${verifyExpectation(subject)}`;
+}
+
export function VerifyExec({ subject = 'The agent' }: { subject?: string }) {
return (
<>
- <CopyPrompt>List the first 5 documents you come across in this project.</CopyPrompt>
+ <CopyPrompt>{VERIFY_PROMPT}</CopyPrompt>
<p>
{subject} should call the OpenKnowledge <code>exec</code> tool and respond with some of your
documents.
diff --git a/docs/src/lib/accept-negotiation.test.ts b/docs/src/lib/accept-negotiation.test.ts
new file mode 100644
index 000000000..7e9b0ceb2
--- /dev/null
+++ b/docs/src/lib/accept-negotiation.test.ts
@@ -0,0 +1,77 @@
+import { describe, expect, test } from 'vitest';
+import { MARKDOWN_MEDIA_TYPES, prefersMarkdown } from './accept-negotiation.ts';
+
+const BROWSER_ACCEPT =
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8';
+const SAFARI_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
+const CODING_AGENT_ACCEPT = 'text/markdown, text/html, */*';
+
+describe('prefersMarkdown serves Markdown only on an explicit, winning ask', () => {
+ test.each([
+ ['bare text/markdown', 'text/markdown'],
+ ['the text/x-markdown spelling', 'text/x-markdown'],
+ ['a coding agent listing Markdown first at equal weight', CODING_AGENT_ACCEPT],
+ ['Markdown outweighing HTML', 'text/html;q=0.5, text/markdown;q=0.9'],
+ ['Markdown above a de-weighted wildcard', 'text/markdown, */*;q=0.1'],
+ ['HTML explicitly refused', 'text/html;q=0, text/markdown'],
+ ['a wildcard listed first but less specific', '*/*, text/markdown'],
+ [
+ 'Markdown above a weakened HTML, wildcard present',
+ 'text/markdown;q=0.5, text/html;q=0.1, */*',
+ ],
+ [
+ 'a type wildcard not overriding the exact HTML entry',
+ 'text/markdown;q=0.3, text/html;q=0.2, text/*;q=0.9',
+ ],
+ ['parameters and casing the client chose', 'Text/Markdown; charset=utf-8'],
+ ])('%s wins Markdown', (_label, accept) => {
+ expect(prefersMarkdown(accept)).toBe(true);
+ });
+
+ test.each([
+ ['Chrome', BROWSER_ACCEPT],
+ ['Safari', SAFARI_ACCEPT],
+ ['a bare wildcard', '*/*'],
+ ['a type wildcard', 'text/*'],
+ ['text/plain, which asks for text without markup', 'text/plain'],
+ ['text/plain ranked above HTML', 'text/plain, text/html;q=0.5'],
+ ['Markdown ranked below HTML', 'text/html, text/markdown;q=0.1'],
+ ['Markdown refused outright', 'text/markdown;q=0, text/html'],
+ ['Markdown tied with HTML but listed second', 'text/html, text/markdown'],
+ ['no Accept at all', null],
+ ['an empty Accept', ''],
+ ['JSON', 'application/json'],
+ ])('%s gets HTML', (_label, accept) => {
+ expect(prefersMarkdown(accept)).toBe(false);
+ });
+});
+
+describe('prefersMarkdown survives malformed input', () => {
+ test.each([
+ ['a bare token with no slash', 'garbage'],
+ ['punctuation', ';;;,,,'],
+ ['an unreadable quality on the Markdown entry', 'text/markdown;q=abc'],
+ ['a truncated header', 'text/markdown;'],
+ ])('%s never throws', (_label, accept) => {
+ expect(() => prefersMarkdown(accept)).not.toThrow();
+ });
+
+ test('an unreadable quality drops that entry rather than weighting it', () => {
+ expect(prefersMarkdown('text/markdown;q=abc')).toBe(false);
+ expect(prefersMarkdown('text/markdown;q=abc, text/html')).toBe(false);
+ });
+
+ test('stray commas and whitespace do not create phantom entries', () => {
+ expect(prefersMarkdown(' , text/markdown , ')).toBe(true);
+ });
+
+ test('a quality above 1 is clamped rather than used to outrank', () => {
+ expect(prefersMarkdown('text/html;q=1, text/markdown;q=5')).toBe(false);
+ });
+});
+
+describe('MARKDOWN_MEDIA_TYPES', () => {
+ test('does not include text/plain', () => {
+ expect(MARKDOWN_MEDIA_TYPES).not.toContain('text/plain');
+ });
+});
diff --git a/docs/src/lib/accept-negotiation.ts b/docs/src/lib/accept-negotiation.ts
new file mode 100644
index 000000000..f71ed3a05
--- /dev/null
+++ b/docs/src/lib/accept-negotiation.ts
@@ -0,0 +1,86 @@
+import { type NextRequest, NextResponse } from 'next/server';
+
+export const MARKDOWN_MEDIA_TYPES = ['text/markdown', 'text/x-markdown'];
+
+interface AcceptEntry {
+ range: string;
+ q: number;
+ index: number;
+}
+
+function parseAccept(header: string): AcceptEntry[] {
+ const entries: AcceptEntry[] = [];
+
+ header.split(',').forEach((part, index) => {
+ const [rangeText, ...parameters] = part.split(';');
+ const range = rangeText.trim().toLowerCase();
+ if (!range.includes('/')) return;
+
+ let q = 1;
+ for (const parameter of parameters) {
+ const separator = parameter.indexOf('=');
+ if (separator === -1) continue;
+ if (parameter.slice(0, separator).trim().toLowerCase() !== 'q') continue;
+ const value = Number.parseFloat(parameter.slice(separator + 1).trim());
+ if (Number.isNaN(value)) return;
+ q = Math.min(1, Math.max(0, value));
+ }
+
+ entries.push({ range, q, index });
+ });
+
+ return entries;
+}
+
+function specificity(range: string): number {
+ if (range === '*/*') return 0;
+ if (range.endsWith('/*')) return 1;
+ return 2;
+}
+
+function outranks(candidate: AcceptEntry, incumbent: AcceptEntry): boolean {
+ if (candidate.q !== incumbent.q) return candidate.q > incumbent.q;
+ const bySpecificity = specificity(candidate.range) - specificity(incumbent.range);
+ if (bySpecificity !== 0) return bySpecificity > 0;
+ return candidate.index < incumbent.index;
+}
+
+function bestMatch(
+ entries: AcceptEntry[],
+ matches: (range: string) => boolean,
+): AcceptEntry | undefined {
+ let best: AcceptEntry | undefined;
+ for (const entry of entries) {
+ if (!matches(entry.range)) continue;
+ if (!best) {
+ best = entry;
+ continue;
+ }
+ const bySpecificity = specificity(entry.range) - specificity(best.range);
+ if (bySpecificity > 0 || (bySpecificity === 0 && entry.q > best.q)) best = entry;
+ }
+ return best;
+}
+
+const isMarkdownRange = (range: string) => MARKDOWN_MEDIA_TYPES.includes(range);
+const coversHtml = (range: string) =>
+ range === 'text/html' || range === 'text/*' || range === '*/*';
+
+export function prefersMarkdown(accept: string | null | undefined): boolean {
+ if (!accept) return false;
+
+ const entries = parseAccept(accept);
+ const markdown = bestMatch(entries, isMarkdownRange);
+ if (!markdown || markdown.q === 0) return false;
+
+ const html = bestMatch(entries, coversHtml);
+ return !html || outranks(markdown, html);
+}
+
+export function markdownRewrite(request: NextRequest, handler: string | null): NextResponse | null {
+ if (!handler || !prefersMarkdown(request.headers.get('accept'))) return null;
+
+ const target = new URL(handler, request.nextUrl.origin);
+ target.search = request.nextUrl.search;
+ return NextResponse.rewrite(target);
+}
diff --git a/docs/src/lib/docs-corpus.test-helper.ts b/docs/src/lib/docs-corpus.test-helper.ts
new file mode 100644
index 000000000..a0cc4f466
--- /dev/null
+++ b/docs/src/lib/docs-corpus.test-helper.ts
@@ -0,0 +1,63 @@
+import { readdirSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parse } from 'yaml';
+
+const CONTENT_ROOT = fileURLToPath(new URL('../../content/', import.meta.url));
+
+export interface CorpusPage {
+ readonly slug: string;
+ readonly source: string;
+ readonly pageUrl: string;
+}
+
+function corpusFiles(dir = CONTENT_ROOT): string[] {
+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) return corpusFiles(full);
+ return /\.mdx?$/.test(entry.name) ? [full] : [];
+ });
+}
+
+export async function readCorpus(): Promise<CorpusPage[]> {
+ return Promise.all(
+ corpusFiles().map(async (file) => {
+ const slug = path
+ .relative(CONTENT_ROOT, file)
+ .replace(/\.mdx?$/, '')
+ .replace(/(^|\/)index$/, '');
+ return {
+ slug,
+ source: await readFile(file, 'utf8'),
+ pageUrl: `https://openknowledge.ai/docs/${slug}`.replace(/\/$/, ''),
+ };
+ }),
+ );
+}
+
+export interface CorpusSourcePage {
+ readonly url: string;
+ readonly data: {
+ readonly title: string;
+ readonly description?: string;
+ getText(): Promise<string>;
+ };
+}
+
+const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---/;
+
+export function corpusSourcePages(pages: readonly CorpusPage[]): CorpusSourcePage[] {
+ return pages.map((page) => {
+ const block = FRONTMATTER.exec(page.source)?.[1];
+ const data = (block ? parse(block) : {}) as { title?: string; description?: string };
+ return {
+ url: `/docs/${page.slug}`.replace(/\/$/, ''),
+ data: {
+ title: data.title ?? page.slug,
+ description: data.description,
+ getText: async () => page.source,
+ },
+ };
+ });
+}
diff --git a/docs/src/lib/download-targets.test.ts b/docs/src/lib/download-targets.test.ts
index 01c739ff7..68a7a087e 100644
--- a/docs/src/lib/download-targets.test.ts
+++ b/docs/src/lib/download-targets.test.ts
@@ -154,6 +154,8 @@ describe('classifyDownloadOs', () => {
expect(classifyDownloadOs('macOS')).toBe('macos');
expect(classifyDownloadOs('Windows')).toBe('windows');
expect(classifyDownloadOs('Linux')).toBe('linux');
+ expect(classifyDownloadOs('Chrome OS')).toBe('linux');
+ expect(classifyDownloadOs('Chromium OS')).toBe('linux');
});
test('mobile falls through to unknown — it cannot run the desktop app', () => {
diff --git a/docs/src/lib/download-targets.ts b/docs/src/lib/download-targets.ts
index 533ae0f29..47f69d2b9 100644
--- a/docs/src/lib/download-targets.ts
+++ b/docs/src/lib/download-targets.ts
@@ -149,7 +149,8 @@ export function classifyDownloadOs(input: string | null | undefined): DetectedOs
lower.includes('linux') ||
lower.includes('x11') ||
lower.includes('cros') ||
- lower.includes('chrome os')
+ lower.includes('chrome os') ||
+ lower.includes('chromium os')
) {
return 'linux';
}
diff --git a/docs/src/lib/get-llm-text.test.ts b/docs/src/lib/get-llm-text.test.ts
index e2bc42ecf..349b4b025 100644
--- a/docs/src/lib/get-llm-text.test.ts
+++ b/docs/src/lib/get-llm-text.test.ts
@@ -3,46 +3,97 @@ import { getLLMText } from './get-llm-text.ts';
type Page = Parameters<typeof getLLMText>[0];
+const SOURCE = `---
+title: Overview
+description: What OpenKnowledge is.
+---
+
+Read the [quickstart](../get-started/quickstart.mdx).
+
+<Callout type="info" title="Heads up">
+
+It is early.
+
+</Callout>
+`;
+
function fakePage(data: Partial<Page['data']> = {}, url = '/docs/get-started/overview'): Page {
return {
url,
data: {
title: 'Overview',
description: 'What OpenKnowledge is.',
- getText: async (type: 'raw' | 'processed') => `BODY(${type})`,
+ getText: async () => SOURCE,
...data,
},
} as unknown as Page;
}
describe('getLLMText', () => {
- test('renders title + URL header, description, then processed body', async () => {
+ test('renders title + URL header, description, then the serialized body', async () => {
const md = await getLLMText(fakePage());
expect(md).toBe(
- `# Overview (/docs/get-started/overview)
+ `# Overview (https://openknowledge.ai/docs/get-started/overview)
What OpenKnowledge is.
-BODY(processed)`,
+Read the [quickstart](https://openknowledge.ai/docs/get-started/quickstart).
+
+> **Heads up**
+>
+> It is early.`,
);
});
- test('requests the processed Markdown variant (snippets resolved), not raw', async () => {
+ test('heads the document with the absolute human URL, not the site-relative one', async () => {
+ const md = await getLLMText(fakePage({}, '/docs/reference/components/callout'));
+ expect(md.split('\n')[0]).toBe(
+ '# Overview (https://openknowledge.ai/docs/reference/components/callout)',
+ );
+ });
+
+ test('serializes the raw MDX source, not the compiled Markdown rendition', async () => {
let requested: string | undefined;
await getLLMText(
fakePage({
getText: async (type) => {
requested = type;
- return '';
+ return SOURCE;
},
}),
);
- expect(requested).toBe('processed');
+ expect(requested).toBe('raw');
+ });
+
+ test('strips the frontmatter rather than serving it as body text', async () => {
+ const md = await getLLMText(fakePage());
+ expect(md).not.toContain('---');
+ expect(md).not.toContain('title: Overview');
+ });
+
+ test('neutralises raw HTML a title names in passing', async () => {
+ const md = await getLLMText(fakePage({ title: 'The <details> element' }));
+ expect(md.split('\n')[0]).toBe(
+ '# The \\<details> element (https://openknowledge.ai/docs/get-started/overview)',
+ );
+ });
+
+ test('keeps a multi-line title on the heading line', async () => {
+ const md = await getLLMText(fakePage({ title: 'Folded\n\ntitle' }));
+ expect(md.split('\n')[0]).toBe(
+ '# Folded title (https://openknowledge.ai/docs/get-started/overview)',
+ );
});
test('tolerates a missing description without emitting "undefined"', async () => {
const md = await getLLMText(fakePage({ description: undefined }));
expect(md).not.toContain('undefined');
- expect(md).toContain('# Overview (/docs/get-started/overview)');
+ expect(md).toContain('# Overview (https://openknowledge.ai/docs/get-started/overview)');
+ });
+
+ test('fails loudly on a component with no registry disposition', async () => {
+ await expect(
+ getLLMText(fakePage({ getText: async () => '<Widget>content</Widget>' })),
+ ).rejects.toThrow('No serializer disposition for <Widget>');
});
});
diff --git a/docs/src/lib/get-llm-text.ts b/docs/src/lib/get-llm-text.ts
index 8b96b7312..26348f7bf 100644
--- a/docs/src/lib/get-llm-text.ts
+++ b/docs/src/lib/get-llm-text.ts
@@ -1,12 +1,20 @@
import type { InferPageType } from 'fumadocs-core/source';
+import { escapeInlineProse, escapeRawHtml, serializeMdx } from '@/lib/mdx-serializer';
+import { DOCS_SERIALIZER_REGISTRY } from '@/lib/mdx-serializer-registry';
+import { absoluteSiteUrl } from '@/lib/site';
import type { source } from '@/lib/source';
export async function getLLMText(page: InferPageType<typeof source>): Promise<string> {
- const processed = await page.data.getText('processed');
+ const canonicalUrl = absoluteSiteUrl(page.url);
+ const mdx = await page.data.getText('raw');
+ const { body } = serializeMdx(mdx, {
+ registry: DOCS_SERIALIZER_REGISTRY,
+ pageUrl: canonicalUrl,
+ });
- return `# ${page.data.title} (${page.url})
+ return `# ${escapeInlineProse(page.data.title)} (${canonicalUrl})
-${page.data.description || ''}
+${escapeRawHtml(page.data.description || '')}
-${processed}`;
+${body}`;
}
diff --git a/docs/src/lib/llms-full-size.test-helper.ts b/docs/src/lib/llms-full-size.test-helper.ts
new file mode 100644
index 000000000..dcd0b49cf
--- /dev/null
+++ b/docs/src/lib/llms-full-size.test-helper.ts
@@ -0,0 +1,15 @@
+export const LLMS_FULL_WARN_BYTES = 2_500_000;
+export const LLMS_FULL_MAX_BYTES = 5_000_000;
+
+export type CorpusSizeVerdict = 'ok' | 'warn' | 'fail';
+
+export function classifyLlmsFullSize(bytes: number): CorpusSizeVerdict {
+ if (bytes > LLMS_FULL_MAX_BYTES) return 'fail';
+ if (bytes > LLMS_FULL_WARN_BYTES) return 'warn';
+ return 'ok';
+}
+
+export function describeLlmsFullSize(bytes: number): string {
+ const mb = (limit: number) => `${(limit / 1_000_000).toFixed(1)} MB`;
+ return `/llms-full.txt is ${mb(bytes)} (warn over ${mb(LLMS_FULL_WARN_BYTES)}, fail over ${mb(LLMS_FULL_MAX_BYTES)})`;
+}
diff --git a/docs/src/lib/llms-full-size.test.ts b/docs/src/lib/llms-full-size.test.ts
new file mode 100644
index 000000000..0c63a6af8
--- /dev/null
+++ b/docs/src/lib/llms-full-size.test.ts
@@ -0,0 +1,29 @@
+import { describe, expect, test } from 'vitest';
+import {
+ classifyLlmsFullSize,
+ describeLlmsFullSize,
+ LLMS_FULL_MAX_BYTES,
+ LLMS_FULL_WARN_BYTES,
+} from './llms-full-size.test-helper';
+
+describe('llms-full.txt size ceiling', () => {
+ test('passes a corpus comfortably inside the budget', () => {
+ expect(classifyLlmsFullSize(709_000)).toBe('ok');
+ });
+
+ test('warns once the corpus passes the warning threshold', () => {
+ expect(classifyLlmsFullSize(LLMS_FULL_WARN_BYTES)).toBe('ok');
+ expect(classifyLlmsFullSize(LLMS_FULL_WARN_BYTES + 1)).toBe('warn');
+ });
+
+ test('fails once the corpus passes the ceiling', () => {
+ expect(classifyLlmsFullSize(LLMS_FULL_MAX_BYTES)).toBe('warn');
+ expect(classifyLlmsFullSize(LLMS_FULL_MAX_BYTES + 1)).toBe('fail');
+ });
+
+ test('reports the measurement and both thresholds so a failure is actionable', () => {
+ expect(describeLlmsFullSize(6_000_000)).toBe(
+ '/llms-full.txt is 6.0 MB (warn over 2.5 MB, fail over 5.0 MB)',
+ );
+ });
+});
diff --git a/docs/src/lib/llms-txt.test.ts b/docs/src/lib/llms-txt.test.ts
new file mode 100644
index 000000000..e1cdcef98
--- /dev/null
+++ b/docs/src/lib/llms-txt.test.ts
@@ -0,0 +1,175 @@
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+import { describe, expect, test } from 'vitest';
+import { buildLlmsTxt, type LlmsTxtLink, marketingLinks } from './llms-txt';
+import { MARKETING_MARKDOWN_PAGES } from './marketing-markdown-pages';
+
+const docsLink: LlmsTxtLink = {
+ url: 'https://openknowledge.ai/docs/get-started/quickstart.md',
+ name: 'Quickstart',
+ description: 'Install and open your first file.',
+};
+const postLink: LlmsTxtLink = {
+ url: 'https://openknowledge.ai/blog/markdownlint-support.md',
+ name: 'Keeping you and your agents in check with markdownlint',
+};
+
+function build(overrides: Partial<Parameters<typeof buildLlmsTxt>[0]> = {}) {
+ return buildLlmsTxt({ docs: [docsLink], blogPosts: [postLink], ...overrides });
+}
+
+describe('llms.txt index', () => {
+ test('opens with a title and a blockquote summary', () => {
+ const [title, summary] = build().split('\n\n');
+ expect(title).toBe('# OpenKnowledge');
+ expect(summary.startsWith('> ')).toBe(true);
+ expect(summary.length).toBeGreaterThan('> '.length);
+ });
+
+ test('groups entries under Docs, Blog and Product in that order', () => {
+ const headings = [...build().matchAll(/^## (.+)$/gm)].map((match) => match[1]);
+ expect(headings).toEqual(['Docs', 'Blog', 'Product']);
+ });
+
+ test('lists a page an agent can fetch, with its notes', () => {
+ expect(build()).toContain(
+ '- [Quickstart](https://openknowledge.ai/docs/get-started/quickstart.md): Install and open your first file.',
+ );
+ });
+
+ test('neutralises raw HTML a description names in passing', () => {
+ const body = build({
+ docs: [
+ {
+ url: 'https://openknowledge.ai/docs/reference/components/accordion.md',
+ name: 'Accordion',
+ description: 'Standalone expand/collapse via native HTML5 <details>/<summary>.',
+ },
+ ],
+ });
+ expect(body).toContain('native HTML5 \\<details>/\\<summary>.');
+ expect(body).not.toContain(' <details>');
+ });
+
+ test('neutralises raw HTML a title names in passing', () => {
+ const body = build({
+ docs: [
+ {
+ url: 'https://openknowledge.ai/docs/reference/components/accordion.md',
+ name: 'The <details> element',
+ },
+ ],
+ });
+ expect(body).toContain('- [The \\<details> element](');
+ expect(body).not.toContain('[The <details>');
+ });
+
+ test('keeps a bracket in a title inside the link label', () => {
+ const body = build({
+ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name: 'Arrays [and] slices' }],
+ });
+ expect(body).toContain('- [Arrays \\[and\\] slices](https://openknowledge.ai/docs/x.md)');
+ });
+
+ test.each([
+ ['A [link](u) title', 'A link title'],
+ ['Bang ![img](u) in a title', 'Bang img in a title'],
+ ])('reduces link syntax in a title to its words: %j', (name, expected) => {
+ const bullet = build({ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name }] })
+ .split('\n')
+ .find((line) => line.startsWith('- [A ') || line.startsWith('- [Bang '));
+ const tree = unified()
+ .use(remarkParse)
+ .use(remarkGfm)
+ .parse(bullet ?? '');
+
+ const links: string[] = [];
+ const images: string[] = [];
+ visit(tree, 'link', (node) => links.push(node.url));
+ visit(tree, 'image', (node) => images.push(node.url));
+
+ expect(links).toEqual(['https://openknowledge.ai/docs/x.md']);
+ expect(images).toEqual([]);
+ expect(bullet).toContain(`[${expected}]`);
+ });
+
+ test('leaves a bracket inside a code span alone', () => {
+ const body = build({
+ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name: 'The `items[]` prop' }],
+ });
+ const tree = unified().use(remarkParse).use(remarkGfm).parse(body);
+
+ const spans: string[] = [];
+ visit(tree, 'inlineCode', (node) => spans.push(node.value));
+ expect(spans).toContain('items[]');
+ });
+
+ test.each([
+ ['a parenthesis', 'https://openknowledge.ai/blog/note-)-end.md'],
+ ['a space', 'https://openknowledge.ai/blog/pre post.md'],
+ ])('round-trips a URL carrying %s', (_name, url) => {
+ const body = build({ blogPosts: [{ url, name: 'Note' }] });
+ const tree = unified().use(remarkParse).use(remarkGfm).parse(body);
+
+ const urls: string[] = [];
+ visit(tree, 'link', (node) => urls.push(node.url));
+ expect(urls).toContain(url);
+ });
+
+ test('keeps a multi-line title on the bullet it belongs to', () => {
+ const body = build({
+ docs: [{ url: 'https://openknowledge.ai/docs/x.md', name: 'Folded\n\ntitle' }],
+ });
+ expect(body).toContain('- [Folded title](https://openknowledge.ai/docs/x.md)');
+ });
+
+ test('omits the notes separator when an entry has no description', () => {
+ expect(build()).toContain(
+ '- [Keeping you and your agents in check with markdownlint]' +
+ '(https://openknowledge.ai/blog/markdownlint-support.md)\n',
+ );
+ });
+
+ test('every listed URL is absolute and points at a Markdown rendition', () => {
+ for (const url of [...build().matchAll(/^- \[(?:\\.|[^\]\\])*]\(([^)]+)\)/gm)].map(
+ (m) => m[1],
+ )) {
+ expect(url).toMatch(/^https:\/\/openknowledge\.ai\//);
+ expect(url).toMatch(/\.md$/);
+ }
+ });
+
+ test('lists no URL twice', () => {
+ const urls = [...build().matchAll(/^- \[(?:\\.|[^\]\\])*]\(([^)]+)\)/gm)].map((m) => m[1]);
+ expect(urls).toEqual([...new Set(urls)]);
+ });
+
+ test('carries every curated marketing rendition', () => {
+ const body = build();
+ for (const page of MARKETING_MARKDOWN_PAGES) {
+ expect(body).toContain(`(https://openknowledge.ai${page.path})`);
+ }
+ });
+
+ test('lists the blog index ahead of the posts it indexes', () => {
+ const body = build();
+ expect(body.indexOf('/blog.md)')).toBeLessThan(body.indexOf('/blog/markdownlint-support.md)'));
+ });
+
+ test('drops a section with nothing to list rather than emitting it bare', () => {
+ const headings = [...build({ docs: [] }).matchAll(/^## (.+)$/gm)].map((match) => match[1]);
+ expect(headings).toEqual(['Blog', 'Product']);
+ });
+
+ test('keeps the blog section when only the index is available', () => {
+ expect(build({ blogPosts: [] })).toContain('## Blog');
+ });
+
+ test('resolves marketing paths against the site origin', () => {
+ expect(marketingLinks('Product').map((link) => link.url)).toContain(
+ 'https://openknowledge.ai/download.md',
+ );
+ });
+});
diff --git a/docs/src/lib/llms-txt.ts b/docs/src/lib/llms-txt.ts
new file mode 100644
index 000000000..e22aa8968
--- /dev/null
+++ b/docs/src/lib/llms-txt.ts
@@ -0,0 +1,68 @@
+import {
+ MARKETING_MARKDOWN_PAGES,
+ type MarketingMarkdownPage,
+} from '@/lib/marketing-markdown-pages';
+import { escapeInlineProse, markdownLink } from '@/lib/mdx-serializer';
+import { absoluteSiteUrl, SITE_DESCRIPTION, SITE_NAME } from '@/lib/site';
+
+export interface LlmsTxtLink {
+ readonly url: string;
+ readonly name: string;
+ readonly description?: string;
+}
+
+const LLMS_TXT_SECTIONS = ['Docs', 'Blog', 'Product'] as const;
+export type LlmsTxtSection = (typeof LLMS_TXT_SECTIONS)[number];
+
+const PREAMBLE =
+ 'Every link below is the Markdown rendition of a page. Each rendition names ' +
+ 'the page a reader would open in its first heading and in its ' +
+ '`Link: rel="canonical"` header; that is the URL to cite. Each of those pages ' +
+ 'also serves this same Markdown from its own URL to any request sending ' +
+ '`Accept: text/markdown`.';
+
+function marketingLink(page: MarketingMarkdownPage): LlmsTxtLink {
+ return {
+ url: absoluteSiteUrl(page.path),
+ name: page.name,
+ description: page.description,
+ };
+}
+
+export function marketingLinks(section: LlmsTxtSection): LlmsTxtLink[] {
+ return MARKETING_MARKDOWN_PAGES.filter((page) => page.section === section).map(marketingLink);
+}
+
+function inline(prose: string, { asLinkLabel = false } = {}): string {
+ return escapeInlineProse(prose, { flattenLinks: asLinkLabel });
+}
+
+function bullet({ url, name, description }: LlmsTxtLink): string {
+ const link = markdownLink(inline(name, { asLinkLabel: true }), url);
+ return description ? `- ${link}: ${inline(description)}` : `- ${link}`;
+}
+
+function section(name: LlmsTxtSection, links: readonly LlmsTxtLink[]): string[] {
+ return links.length === 0 ? [] : [`## ${name}`, links.map(bullet).join('\n')];
+}
+
+export function buildLlmsTxt({
+ docs,
+ blogPosts,
+}: {
+ docs: readonly LlmsTxtLink[];
+ blogPosts: readonly LlmsTxtLink[];
+}): string {
+ const sections: Record<LlmsTxtSection, readonly LlmsTxtLink[]> = {
+ Docs: docs,
+ Blog: [...marketingLinks('Blog'), ...blogPosts],
+ Product: marketingLinks('Product'),
+ };
+
+ return [
+ `# ${SITE_NAME}`,
+ `> ${SITE_DESCRIPTION}`,
+ PREAMBLE,
+ ...LLMS_TXT_SECTIONS.flatMap((name) => section(name, sections[name])),
+ ].join('\n\n');
+}
diff --git a/docs/src/lib/markdown-component-coverage.test.ts b/docs/src/lib/markdown-component-coverage.test.ts
new file mode 100644
index 000000000..99056a667
--- /dev/null
+++ b/docs/src/lib/markdown-component-coverage.test.ts
@@ -0,0 +1,68 @@
+import { describe, expect, test } from 'vitest';
+import { readCorpus } from '@/lib/docs-corpus.test-helper';
+import { componentNames } from '@/lib/mdx-serializer';
+import { DOCS_SERIALIZER_REGISTRY } from '@/lib/mdx-serializer-registry';
+import { getMDXComponents } from '@/mdx-components';
+
+const REGISTRY = 'src/lib/mdx-serializer-registry.ts';
+
+const runtimeMapKeys = Object.keys(getMDXComponents());
+const corpus = await readCorpus();
+
+const corpusUsage = new Map<string, string>();
+for (const page of corpus) {
+ for (const name of componentNames(page.source)) {
+ if (!corpusUsage.has(name)) corpusUsage.set(name, `content/${page.slug || 'index'}`);
+ }
+}
+
+const renderable = new Set([...runtimeMapKeys, ...corpusUsage.keys()]);
+const registryKeys = new Set(Object.keys(DOCS_SERIALIZER_REGISTRY));
+
+function originOf(name: string): string {
+ const page = corpusUsage.get(name);
+ if (runtimeMapKeys.includes(name)) {
+ return page ? `component map, used in ${page}` : 'component map';
+ }
+ return `imported by ${page}`;
+}
+
+describe('markdown component coverage', () => {
+ test('every component a page can render has a disposition', () => {
+ const missing = [...renderable]
+ .filter((name) => !registryKeys.has(name))
+ .sort()
+ .map((name) => `<${name}> (${originOf(name)})`);
+
+ expect(
+ missing,
+ `components with no serializer disposition. Add each to DOCS_SERIALIZER_REGISTRY in ${REGISTRY} as 'flatten', 'drop', or a serializer that renders what it carries. ` +
+ "A component that carries no children in source has nothing for 'flatten' to emit, so " +
+ 'it needs a serializer: that includes anything whose content a remark plugin supplies ' +
+ '(a snippet include, a generated type table), because the Markdown rendition reads RAW ' +
+ "source and never runs those plugins — 'flatten' there ships the page with the section " +
+ 'silently absent',
+ ).toEqual([]);
+ });
+
+ test('every disposition names a component something can render', () => {
+ const orphaned = [...registryKeys].filter((name) => !renderable.has(name)).sort();
+
+ expect(
+ orphaned,
+ `dispositions in ${REGISTRY} for components neither the map nor the corpus has; restore the component or drop the entry`,
+ ).toEqual([]);
+ });
+
+ test('the component map is read at runtime, spread keys included', () => {
+ expect(runtimeMapKeys, 'the component map census is not seeing spread keys').toContain(
+ 'Callout',
+ );
+ expect(runtimeMapKeys.length).toBeGreaterThan(40);
+ });
+
+ test('the corpus census reaches the published tree', () => {
+ expect(corpus.length).toBeGreaterThan(50);
+ expect(corpusUsage.size).toBeGreaterThan(10);
+ });
+});
diff --git a/docs/src/lib/markdown-fidelity.real-source.test.ts b/docs/src/lib/markdown-fidelity.real-source.test.ts
new file mode 100644
index 000000000..508ddbeca
--- /dev/null
+++ b/docs/src/lib/markdown-fidelity.real-source.test.ts
@@ -0,0 +1,40 @@
+import { describe, expect, test } from 'vitest';
+import { getLLMText } from '@/lib/get-llm-text';
+import type { FidelityViolation } from '@/lib/markdown-fidelity.test-helper';
+import { markdownFidelityViolations } from '@/lib/markdown-fidelity.test-helper';
+import { source } from '@/lib/source';
+
+const pages = source.getPages();
+
+describe('every page the real loader hands to the Markdown pipeline', () => {
+ test('is a census, so a broken content glob cannot vacuously pass this suite', () => {
+ expect(pages.length).toBeGreaterThan(50);
+ });
+
+ test('reads as Markdown, with no JSX, indented code, base64 or source-form links', async () => {
+ const violations: FidelityViolation[] = [];
+ for (const page of pages) {
+ violations.push(...markdownFidelityViolations(page.url, await getLLMText(page)));
+ }
+ expect(violations.map((violation) => violation.message)).toEqual([]);
+ });
+});
+
+describe('the real loader derives a page from its file', () => {
+ test('nests the URL by directory rather than flattening or keeping the extension', () => {
+ const page = source.getPage(['reference', 'components', 'callout']);
+ expect(page?.url).toBe('/docs/reference/components/callout');
+ expect(page?.data.title).toBe('Callout');
+ });
+
+ test('reads the title from frontmatter rather than from the filename', () => {
+ const page = source.getPage(['get-started', 'quickstart']);
+ expect(page?.url).toBe('/docs/get-started/quickstart');
+ expect(page?.data.title).toBe('Quickstart');
+ });
+
+ test('serves raw source, not the compiled rendition, for the document body', async () => {
+ const page = source.getPage(['reference', 'components', 'callout']);
+ await expect(page?.data.getText('raw')).resolves.toContain('title: "Callout"');
+ });
+});
diff --git a/docs/src/lib/markdown-fidelity.test-helper.ts b/docs/src/lib/markdown-fidelity.test-helper.ts
new file mode 100644
index 000000000..cc1642ead
--- /dev/null
+++ b/docs/src/lib/markdown-fidelity.test-helper.ts
@@ -0,0 +1,221 @@
+import type { Root } from 'mdast';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+
+type FidelityViolationKind =
+ | 'jsx-element'
+ | 'indented-code'
+ | 'base64-attribute'
+ | 'source-form-link'
+ | 'unparseable';
+
+export interface FidelityViolation {
+ readonly kind: FidelityViolationKind;
+ readonly message: string;
+}
+
+const asMarkdown = unified().use(remarkParse).use(remarkGfm);
+const asMdx = unified().use(remarkParse).use(remarkMdx).use(remarkGfm);
+
+const ABSOLUTE_URL = /^[a-zA-Z][a-zA-Z0-9+.-]*:|^\/\//;
+
+const SOURCE_EXTENSION = /\.mdx($|[#?])/;
+
+const BASE64_ATTRIBUTE = /([A-Za-z][\w-]*)=(["'])([A-Za-z0-9+/]{64,}={0,2})\2/g;
+
+type Range = readonly [number, number];
+
+export function markdownFidelityViolations(name: string, markdown: string): FidelityViolation[] {
+ const violations: FidelityViolation[] = [];
+ const report = (
+ kind: FidelityViolationKind,
+ line: number,
+ construct: string,
+ fix: string,
+ ): void => {
+ violations.push({ kind, message: `${name}:${line} — ${construct}. Fix: ${fix}` });
+ };
+
+ let tree: Root;
+ try {
+ tree = asMarkdown.parse(markdown);
+ } catch (error) {
+ report(
+ 'unparseable',
+ 1,
+ `the document does not parse as Markdown (${(error as Error).message})`,
+ 'find the producer that emitted this document and correct what it wrote',
+ );
+ return violations;
+ }
+
+ const literalRegions: Range[] = [];
+
+ visit(tree, (node) => {
+ const start = node.position?.start.offset;
+ const end = node.position?.end.offset;
+ const line = node.position?.start.line ?? 1;
+
+ if (node.type === 'code' || node.type === 'inlineCode') {
+ if (start !== undefined && end !== undefined) literalRegions.push([start, end]);
+ } else if (node.type === 'html' && node.value.trimStart().startsWith('<!--')) {
+ if (start !== undefined && end !== undefined) literalRegions.push([start, end]);
+ } else if (node.type === 'link' && start !== undefined && markdown[start] === '<') {
+ if (end !== undefined) literalRegions.push([start, end]);
+ }
+
+ if (node.type === 'code') {
+ const opener = start === undefined ? '' : markdown[start];
+ if (opener !== '`' && opener !== '~') {
+ report(
+ 'indented-code',
+ line,
+ 'a code block reached the four-space column instead of being fenced, so CommonMark reads the prose around it as code',
+ 'fence the block; if a custom serializer wrote it, that output is emitted verbatim and has to carry its own fence',
+ );
+ }
+ }
+
+ if (node.type === 'link' || node.type === 'image' || node.type === 'definition') {
+ if (!ABSOLUTE_URL.test(node.url)) {
+ report(
+ 'source-form-link',
+ line,
+ `the link target ${JSON.stringify(node.url)} is relative, so it breaks as soon as the Markdown is read away from its own URL`,
+ 'absolutise it against the page URL, as the serializer does for every link it rewrites',
+ );
+ } else if (SOURCE_EXTENSION.test(node.url)) {
+ report(
+ 'source-form-link',
+ line,
+ `the link target ${JSON.stringify(node.url)} still points at MDX source rather than at a page`,
+ 'drop the source extension so the target is the URL a reader can open',
+ );
+ }
+ }
+ });
+
+ const masked = maskRanges(markdown, literalRegions);
+
+ try {
+ visit(asMdx.parse(masked), (node) => {
+ if (node.type !== 'mdxJsxFlowElement' && node.type !== 'mdxJsxTextElement') return;
+ report(
+ 'jsx-element',
+ node.position?.start.line ?? 1,
+ `<${node.name ?? 'fragment'}> survived into the served Markdown as a raw tag`,
+ 'give the component a serializer disposition that renders what it contributes, or drop it',
+ );
+ });
+ } catch (error) {
+ report(
+ 'unparseable',
+ 1,
+ `the document does not parse as MDX (${(error as Error).message})`,
+ 'escape the raw markup the producer interpolated; author prose reaches the document as Markdown, not as HTML',
+ );
+ }
+
+ for (const match of masked.matchAll(BASE64_ATTRIBUTE)) {
+ report(
+ 'base64-attribute',
+ lineAt(masked, match.index),
+ `the attribute ${match[1]} carries ${match[3].length} bytes of base64 rather than content`,
+ 'restore what the attribute encodes — a preview fence belongs in the document as a fence',
+ );
+ }
+
+ return violations;
+}
+
+function maskRanges(source: string, ranges: readonly Range[]): string {
+ if (ranges.length === 0) return source;
+ const chars = source.split('');
+ for (const [start, end] of ranges) {
+ for (let i = start; i < end && i < chars.length; i++) {
+ if (chars[i] !== '\n') chars[i] = ' ';
+ }
+ }
+ return chars.join('');
+}
+
+function lineAt(source: string, offset: number): number {
+ let line = 1;
+ for (let i = 0; i < offset; i++) if (source[i] === '\n') line += 1;
+ return line;
+}
+
+export interface FidelityFixture {
+ readonly name: string;
+ readonly markdown: string;
+ readonly expect: readonly FidelityViolationKind[];
+}
+
+export const FIDELITY_FIXTURES: readonly FidelityFixture[] = [
+ {
+ name: 'a component that reached the reader as a tag',
+ markdown: 'Read this <Callout type="info">before you start</Callout> first.\n',
+ expect: ['jsx-element'],
+ },
+ {
+ name: 'a block that reached the four-space column',
+ markdown: 'Run the installer:\n\n ok init --here\n',
+ expect: ['indented-code'],
+ },
+ {
+ name: 'a preview compiled to base64 and escaped past the JSX check',
+ markdown: `A preview: \\<HtmlPreview code="PGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+CgPGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+Cg" />\n`,
+ expect: ['base64-attribute'],
+ },
+ {
+ name: 'a link that never left the content tree',
+ markdown: 'See the [editor](../features/editor) reference.\n',
+ expect: ['source-form-link'],
+ },
+ {
+ name: 'a link still in source form',
+ markdown: 'See the [editor](../features/editor.mdx#ask-ai) reference.\n',
+ expect: ['source-form-link'],
+ },
+ {
+ name: 'a link pointing at MDX source on another host',
+ markdown: 'See [the spec](https://example.com/docs/spec.mdx).\n',
+ expect: ['source-form-link'],
+ },
+ {
+ name: 'a block at the four-space column that also holds a component',
+ markdown: 'Run this:\n\n <Callout>read the release notes</Callout>\n',
+ expect: ['indented-code'],
+ },
+ {
+ name: 'a preview attribute shown inside a fence, as documentation of it',
+ markdown:
+ '````mdx\n<HtmlPreview code="PGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+CgPGRpdiBjbGFzcz0icHJldmlldyI+PHNwYW4+SGVsbG88L3NwYW4+PC9kaXY+Cg" />\n````\n',
+ expect: [],
+ },
+ {
+ name: 'a component named inside an inline code span',
+ markdown: 'Same props as `<Accordion>` — pick whichever vocabulary fits.\n',
+ expect: [],
+ },
+ {
+ name: 'a four-backtick fence wrapping a three-backtick one',
+ markdown:
+ '````text\n```mdx\n<Mirror src="specs/architecture" />\n```\n````\n\nProse after the fence.\n',
+ expect: [],
+ },
+ {
+ name: 'prose nested four list levels deep',
+ markdown: '- one\n - two\n - three\n - four, indented past the code column\n',
+ expect: [],
+ },
+ {
+ name: 'an autolink and an HTML comment, which an MDX parser rejects outright',
+ markdown:
+ 'Download it from <https://openknowledge.ai/download>.\n\n<!-- linux-arm64: no published build -->\n',
+ expect: [],
+ },
+];
diff --git a/docs/src/lib/markdown-fidelity.test.ts b/docs/src/lib/markdown-fidelity.test.ts
new file mode 100644
index 000000000..57238cd04
--- /dev/null
+++ b/docs/src/lib/markdown-fidelity.test.ts
@@ -0,0 +1,67 @@
+import { describe, expect, test, vi } from 'vitest';
+import { type CorpusSourcePage, corpusSourcePages, readCorpus } from './docs-corpus.test-helper.ts';
+import {
+ FIDELITY_FIXTURES,
+ type FidelityViolation,
+ markdownFidelityViolations,
+} from './markdown-fidelity.test-helper.ts';
+
+const pages = corpusSourcePages(await readCorpus());
+const bySlug = new Map(pages.map((page) => [page.url, page]));
+
+vi.doMock('@/lib/source', () => ({
+ source: {
+ getPage: (slug: string[]) => bySlug.get(`/docs/${slug.join('/')}`.replace(/\/$/, '')),
+ getPages: () => pages,
+ generateParams: () => pages.map((page) => ({ slug: slugOf(page) })),
+ },
+}));
+
+vi.doMock('@/lib/marketing-blog-index', () => ({
+ blogPostLinks: async () => [
+ { url: 'https://openknowledge.ai/blog/some-post.md', name: 'Some post' },
+ ],
+}));
+
+const { GET } = await import('@/app/llms.mdx/[...slug]/route.ts');
+const { GET: llmsTxtGet } = await import('@/app/llms.txt/route.ts');
+const { GET: llmsFullGet } = await import('@/app/llms-full.txt/route.ts');
+
+function slugOf(page: CorpusSourcePage): string[] {
+ return page.url.slice('/docs/'.length).split('/').filter(Boolean);
+}
+
+async function serve(slug: string[]): Promise<string> {
+ const response = await GET(new Request(`https://openknowledge.ai/docs/${slug.join('/')}.md`), {
+ params: Promise.resolve({ slug }),
+ });
+ return response.text();
+}
+
+const served = new Map<string, string>();
+for (const page of pages) served.set(page.url, await serve(slugOf(page)));
+served.set('/docs/<unknown>', await serve(['no', 'such', 'page']));
+served.set('/llms.txt', await (await llmsTxtGet()).text());
+served.set('/llms-full.txt', await (await llmsFullGet()).text());
+
+describe('the fidelity oracle', () => {
+ test.each(FIDELITY_FIXTURES)('$name', ({ name, markdown, expect: expected }) => {
+ const kinds = markdownFidelityViolations(name, markdown).map((violation) => violation.kind);
+ expect(kinds).toEqual([...expected]);
+ });
+});
+
+describe('every page the docs app serves as Markdown', () => {
+ test('is a census, so a broken content glob cannot vacuously pass this suite', () => {
+ expect(served.size).toBeGreaterThan(50);
+ for (const [slug, markdown] of served) expect(markdown.length, slug).toBeGreaterThan(0);
+ });
+
+ test('reads as Markdown, with no JSX, indented code, base64 or source-form links', () => {
+ const violations: FidelityViolation[] = [];
+ for (const [slug, markdown] of served) {
+ violations.push(...markdownFidelityViolations(slug, markdown));
+ }
+ expect(violations.map((violation) => violation.message)).toEqual([]);
+ });
+});
diff --git a/docs/src/lib/markdown-response.test.ts b/docs/src/lib/markdown-response.test.ts
new file mode 100644
index 000000000..1762341fd
--- /dev/null
+++ b/docs/src/lib/markdown-response.test.ts
@@ -0,0 +1,48 @@
+import { describe, expect, test } from 'vitest';
+import {
+ MARKDOWN_CACHE_CONTROL,
+ MARKDOWN_CONTENT_TYPE,
+ markdownResponse,
+} from './markdown-response.ts';
+
+describe('markdownResponse', () => {
+ test('serves the body as markdown with an edge-cacheable directive', async () => {
+ const res = markdownResponse('# Hi');
+
+ expect(res.status).toBe(200);
+ expect(res.headers.get('content-type')).toBe('text/markdown; charset=utf-8');
+ expect(res.headers.get('cache-control')).toMatch(/\bs-maxage=\d+/);
+ expect(await res.text()).toBe('# Hi');
+ });
+
+ test('points agents at the human page with a canonical Link header', () => {
+ const res = markdownResponse('# Hi', {
+ canonicalUrl: 'https://openknowledge.ai/docs/get-started/overview',
+ });
+
+ expect(res.headers.get('link')).toBe(
+ '<https://openknowledge.ai/docs/get-started/overview>; rel="canonical"',
+ );
+ });
+
+ test('omits the canonical Link when the response renders no single page', () => {
+ expect(markdownResponse('# Hi').headers.get('link')).toBeNull();
+ });
+
+ test('carries the full header contract on an error status too', () => {
+ const res = markdownResponse('# 404', { status: 404 });
+
+ expect(res.status).toBe(404);
+ expect(res.headers.get('content-type')).toBe(MARKDOWN_CONTENT_TYPE);
+ expect(res.headers.get('cache-control')).toBe(MARKDOWN_CACHE_CONTROL);
+ expect(res.headers.get('vary')).toBe('Accept');
+ });
+
+ test('varies on Accept, the correct signal this representation can carry', () => {
+ expect(markdownResponse('# Hi').headers.get('vary')).toBe('Accept');
+ });
+
+ test('charset is explicit — text/markdown has no default one', () => {
+ expect(MARKDOWN_CONTENT_TYPE).toContain('charset=utf-8');
+ });
+});
diff --git a/docs/src/lib/markdown-response.ts b/docs/src/lib/markdown-response.ts
new file mode 100644
index 000000000..b50169b91
--- /dev/null
+++ b/docs/src/lib/markdown-response.ts
@@ -0,0 +1,25 @@
+export const MARKDOWN_CONTENT_TYPE = 'text/markdown; charset=utf-8';
+
+export const MARKDOWN_CACHE_CONTROL =
+ 'public, max-age=0, s-maxage=3600, stale-while-revalidate=86400';
+
+const MARKDOWN_VARY = 'Accept';
+
+export interface MarkdownResponseInit {
+ canonicalUrl?: string;
+ status?: number;
+}
+
+function markdownHeaders({ canonicalUrl }: MarkdownResponseInit = {}): Headers {
+ const headers = new Headers({
+ 'Content-Type': MARKDOWN_CONTENT_TYPE,
+ 'Cache-Control': MARKDOWN_CACHE_CONTROL,
+ Vary: MARKDOWN_VARY,
+ });
+ if (canonicalUrl) headers.set('Link', `<${canonicalUrl}>; rel="canonical"`);
+ return headers;
+}
+
+export function markdownResponse(body: string, init: MarkdownResponseInit = {}): Response {
+ return new Response(body, { status: init.status ?? 200, headers: markdownHeaders(init) });
+}
diff --git a/docs/src/lib/markdown-routes.ts b/docs/src/lib/markdown-routes.ts
new file mode 100644
index 000000000..7fd27db8f
--- /dev/null
+++ b/docs/src/lib/markdown-routes.ts
@@ -0,0 +1,47 @@
+const DOCS_ROUTE = '/docs';
+
+export const PAGE_MARKDOWN_ROUTE = '/llms.mdx';
+
+const MARKDOWN_TWIN_SUFFIXES = ['.html.md', '.md', '.mdx'];
+
+export const NO_MARKDOWN_PREFIXES = [`${DOCS_ROUTE}/changelog`] as const;
+
+function noMarkdownPrefixFor(docsPath: string): string | null {
+ return (
+ NO_MARKDOWN_PREFIXES.find(
+ (prefix) => docsPath === prefix || docsPath.startsWith(`${prefix}/`),
+ ) ?? null
+ );
+}
+
+export function unrenditionedSubtree(slug: readonly string[]): string | null {
+ return noMarkdownPrefixFor(`${DOCS_ROUTE}/${slug.join('/')}`);
+}
+
+interface RewriteRule {
+ source: string;
+ destination: string;
+}
+
+export function markdownRewrites(): RewriteRule[] {
+ return MARKDOWN_TWIN_SUFFIXES.map((suffix) => ({
+ source: `${DOCS_ROUTE}/:path*${suffix}`,
+ destination: `${PAGE_MARKDOWN_ROUTE}/:path*`,
+ }));
+}
+
+function trimTrailingSlashes(pathname: string): string {
+ let end = pathname.length;
+ while (end > 1 && pathname.charCodeAt(end - 1) === 47) end -= 1;
+ return pathname.slice(0, end);
+}
+
+export function markdownHandlerForPage(pathname: string): string | null {
+ const path = trimTrailingSlashes(pathname);
+
+ if (MARKDOWN_TWIN_SUFFIXES.some((suffix) => path.endsWith(suffix))) return null;
+ if (noMarkdownPrefixFor(path)) return null;
+ if (!path.startsWith(`${DOCS_ROUTE}/`)) return null;
+
+ return `${PAGE_MARKDOWN_ROUTE}${path.slice(DOCS_ROUTE.length)}`;
+}
diff --git a/docs/src/lib/marketing-blog-index.test.ts b/docs/src/lib/marketing-blog-index.test.ts
new file mode 100644
index 000000000..483f99728
--- /dev/null
+++ b/docs/src/lib/marketing-blog-index.test.ts
@@ -0,0 +1,462 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import {
+ attemptPolicy,
+ BLOG_FEED_URL,
+ blogPostLinks,
+ EmptyBlogError,
+ fetchBlogPostLinks,
+} from './marketing-blog-index';
+
+interface FeedItem {
+ title: string;
+ link: string;
+ description?: string;
+}
+
+const cdata = (text: string) => `<![CDATA[${text.replaceAll(']]>', ']]]]><![CDATA[>')}]]>`;
+
+function feed(...items: FeedItem[]): string {
+ return [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ '<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">',
+ ' <channel>',
+ ' <title>OpenKnowledge Blog</title>',
+ ' <link>https://openknowledge.ai/blog</link>',
+ ' <description>News, releases, and engineering notes from the OpenKnowledge team.</description>',
+ ' <atom:link href="https://openknowledge.ai/blog/rss.xml" rel="self" type="application/rss+xml" />',
+ ...items.map((item) =>
+ [
+ ' <item>',
+ ` <title>${cdata(item.title)}</title>`,
+ ` <link>${item.link}</link>`,
+ ...(item.description === undefined
+ ? []
+ : [` <description>${cdata(item.description)}</description>`]),
+ ' <pubDate>Tue, 21 Jul 2026 00:00:00 GMT</pubDate>',
+ ` <guid isPermaLink="true">${item.link}</guid>`,
+ ' <category>Release</category>',
+ ' </item>',
+ ].join('\n'),
+ ),
+ ' </channel>',
+ '</rss>',
+ ].join('\n');
+}
+
+const MARKDOWNLINT: FeedItem = {
+ title: 'Keeping you and your agents in check with markdownlint',
+ link: 'https://openknowledge.ai/blog/markdownlint-support',
+ description:
+ "Native markdownlint support in Open Knowledge: your existing config works as-is, a GUI builds one if you don't have it, and your agents see the same rules you do.",
+};
+
+const VISIMER: FeedItem = {
+ title: 'Edit Mermaid diagrams visually with Visimer',
+ link: 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually',
+ description:
+ 'Edit Mermaid diagrams visually with Visimer, the open source library that powers the Mermaid editing experience in OpenKnowledge.',
+};
+
+const LIVE_FEED = feed(MARKDOWNLINT, VISIMER);
+
+function respondWith(body: string, init: ResponseInit = {}) {
+ const fetchMock = vi.fn(async () => new Response(body, init));
+ vi.stubGlobal('fetch', fetchMock);
+ return fetchMock;
+}
+
+beforeEach(() => {
+ vi.useFakeTimers();
+});
+
+async function settle<T>(pending: Promise<T>): Promise<T> {
+ pending.catch(() => undefined);
+ await vi.runAllTimersAsync();
+ return pending;
+}
+
+afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ vi.unstubAllEnvs();
+ vi.restoreAllMocks();
+});
+
+describe('blog posts read from the marketing zone feed', () => {
+ test('lists each post as the Markdown rendition of its page, under its own title', async () => {
+ respondWith(LIVE_FEED);
+ expect(await fetchBlogPostLinks(3600)).toEqual([
+ {
+ url: 'https://openknowledge.ai/blog/markdownlint-support.md',
+ name: 'Keeping you and your agents in check with markdownlint',
+ description: MARKDOWNLINT.description,
+ },
+ {
+ url: 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',
+ name: 'Edit Mermaid diagrams visually with Visimer',
+ description: VISIMER.description,
+ },
+ ]);
+ });
+
+ test('reads a title the encoder split across two CDATA sections', async () => {
+ respondWith(feed({ title: 'before ]]> after', link: 'https://openknowledge.ai/blog/a-post' }));
+ expect((await fetchBlogPostLinks(3600))[0].name).toBe('before ]]> after');
+ });
+
+ test('decodes XML entities in a link', async () => {
+ respondWith(feed({ title: 'A', link: 'https://openknowledge.ai/blog/a&amp;b' }));
+ expect((await fetchBlogPostLinks(3600))[0].url).toBe('https://openknowledge.ai/blog/a&b.md');
+ });
+
+ test('carries no description when the feed has none to give', async () => {
+ respondWith(feed({ title: 'A', link: 'https://openknowledge.ai/blog/a-post' }));
+ expect((await fetchBlogPostLinks(3600))[0]).not.toHaveProperty('description');
+ });
+
+ test('warns when the feed ends after its last post but before the closing tag', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const whole = feed(MARKDOWNLINT);
+ respondWith(whole.slice(0, whole.lastIndexOf('</item>') + '</item>'.length));
+ expect((await fetchBlogPostLinks(3600)).length).toBe(1);
+ expect(warn.mock.calls.flat().join('\n')).toMatch(/ends before <\/rss>/);
+ });
+
+ test('does not cry truncation over a closing tag written inside a title or a summary', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ respondWith(
+ feed(MARKDOWNLINT, {
+ ...VISIMER,
+ description: 'A summary that mentions </description>, </item> and <item> in passing.',
+ }),
+ );
+ expect((await fetchBlogPostLinks(3600)).length).toBe(2);
+ expect(warn).not.toHaveBeenCalled();
+ });
+
+ test('reads a closing tag written inside a title or a summary as text', async () => {
+ respondWith(
+ feed(
+ {
+ title: 'How </title> and </item> survive CDATA',
+ link: 'https://openknowledge.ai/blog/closing-tags',
+ description: 'A summary that mentions </description>, </item> and <item> in passing.',
+ },
+ VISIMER,
+ ),
+ );
+ expect(await fetchBlogPostLinks(3600)).toEqual([
+ {
+ url: 'https://openknowledge.ai/blog/closing-tags.md',
+ name: 'How </title> and </item> survive CDATA',
+ description: 'A summary that mentions </description>, </item> and <item> in passing.',
+ },
+ {
+ url: 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',
+ name: VISIMER.title,
+ description: VISIMER.description,
+ },
+ ]);
+ });
+
+ test('an item left unclosed ends where the next one opens, so its successor is not lost', async () => {
+ respondWith(feed(MARKDOWNLINT, VISIMER).replace('</item>', ''));
+ expect((await fetchBlogPostLinks(3600)).map((post) => post.url)).toEqual([
+ 'https://openknowledge.ai/blog/markdownlint-support.md',
+ 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',
+ ]);
+ });
+
+ test('says so when a complete feed leaves an item unclosed, so a malformation is not silent', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const whole = feed(MARKDOWNLINT, VISIMER);
+ const lastClose = whole.lastIndexOf('</item>');
+ respondWith(whole.slice(0, lastClose) + whole.slice(lastClose + '</item>'.length));
+ await fetchBlogPostLinks(3600);
+ expect(warn.mock.calls.flat().join('\n')).toMatch(/leaves an <item> unclosed/);
+ });
+
+ test('skips an item that is not a post and keeps the rest, saying which it skipped', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ respondWith(
+ feed(
+ MARKDOWNLINT,
+ { title: 'Meet the team', link: 'https://openknowledge.ai/team' },
+ VISIMER,
+ ),
+ );
+
+ expect((await fetchBlogPostLinks(3600)).map((post) => post.url)).toEqual([
+ 'https://openknowledge.ai/blog/markdownlint-support.md',
+ 'https://openknowledge.ai/blog/edit-mermaid-diagrams-visually.md',
+ ]);
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn.mock.calls[0]?.[0]).toContain('"https://openknowledge.ai/team" is not a post at');
+ });
+
+ test('keeps the link out of a log line: one line, bounded, quoted', async () => {
+ const link = `https://ok.example/blog/${'x'.repeat(500)}\nforged: line`;
+ respondWith(feed({ title: 'A', link }));
+
+ const error = (await fetchBlogPostLinks(3600).catch((thrown: unknown) => thrown)) as Error;
+ expect(error.message).not.toContain('\n');
+ expect(error.message).toContain(JSON.stringify(link.replace(/\s+/g, ' ').slice(0, 200)));
+ expect(error.message).not.toContain('x'.repeat(201));
+ });
+
+ test('asks the marketing zone for its own feed, revalidating hourly', async () => {
+ const fetchMock = respondWith(LIVE_FEED);
+ await fetchBlogPostLinks(3600);
+ expect(fetchMock).toHaveBeenCalledWith(
+ BLOG_FEED_URL,
+ expect.objectContaining({ next: { revalidate: 3600 } }),
+ );
+ expect(BLOG_FEED_URL).toBe('https://openknowledge.ai/blog/rss.xml');
+ });
+
+ test('gives up rather than hanging on an origin that never answers', async () => {
+ const fetchMock = respondWith(LIVE_FEED);
+ await fetchBlogPostLinks(3600);
+ const { signal } = fetchMock.mock.calls[0][1] as RequestInit;
+ expect(signal).toBeInstanceOf(AbortSignal);
+ });
+
+ test('fails on an error response instead of reporting an empty blog', async () => {
+ respondWith('nope', { status: 503 });
+ await expect(fetchBlogPostLinks(3600)).rejects.toThrow('responded 503');
+ });
+
+ test('reads an empty feed as a content state', async () => {
+ respondWith(feed());
+ await expect(fetchBlogPostLinks(3600)).rejects.toThrow(EmptyBlogError);
+ });
+
+ test.each([
+ [
+ 'an error page served with a 200',
+ '<!doctype html><html><body><h1>Application error</h1></body></html>',
+ 'is not an RSS feed',
+ ],
+ [
+ 'a canonical host that drifted between the two apps',
+ feed({ title: 'A', link: 'https://ok.example/blog/a-post' }),
+ 'canonical host that drifted',
+ ],
+ [
+ 'a link outside the blog',
+ feed({ title: 'A', link: 'https://openknowledge.ai/team/someone' }),
+ 'is not a post at',
+ ],
+ [
+ 'post URLs that gained a segment',
+ feed({ title: 'A', link: 'https://openknowledge.ai/blog/2026/a-post' }),
+ 'the post URL shape changed',
+ ],
+ [
+ 'an item with no title',
+ feed({ title: '', link: 'https://openknowledge.ai/blog/a-post' }),
+ 'no <title> or no <link>',
+ ],
+ [
+ 'a response cut inside the first item',
+ feed(MARKDOWNLINT).slice(
+ 0,
+ feed(MARKDOWNLINT).indexOf('<link>', feed(MARKDOWNLINT).indexOf('<item')),
+ ),
+ 'ends before </rss>, so it is truncated',
+ ],
+ [
+ 'a response cut in the channel header, before any item',
+ feed(MARKDOWNLINT).slice(0, feed(MARKDOWNLINT).indexOf('<item')),
+ 'ends before </rss>, so it is truncated',
+ ],
+ [
+ 'a complete feed whose only item never closes',
+ feed(MARKDOWNLINT).replace('</item>', ''),
+ 'leaves an <item> unclosed, so it is malformed',
+ ],
+ ])('reads %s as drift, not as an empty blog', async (_name, body, expected) => {
+ respondWith(body);
+ const error = await settle(fetchBlogPostLinks(3600)).catch((thrown: unknown) => thrown);
+
+ expect(error, 'resolved instead of throwing — these links would have published').toBeInstanceOf(
+ Error,
+ );
+ expect(error).not.toBeInstanceOf(EmptyBlogError);
+ expect((error as Error).message).toContain(expected);
+ });
+
+ test.each([404, 401])('reports a %s as an Error naming the status, once', async (status) => {
+ const fetchMock = vi.fn(async () => new Response('nope', { status }));
+ vi.stubGlobal('fetch', fetchMock);
+
+ const error = await settle(fetchBlogPostLinks(3600, true)).catch((thrown: unknown) => thrown);
+
+ expect(error).toBeInstanceOf(Error);
+ expect((error as Error).message).toContain(`responded ${status}`);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ test('spaces the retries rather than firing them in one burst', async () => {
+ const at: number[] = [];
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async () => {
+ at.push(Date.now());
+ throw new TypeError('fetch failed');
+ }),
+ );
+
+ await settle(fetchBlogPostLinks(3600, true)).catch(() => undefined);
+
+ expect(at).toHaveLength(3);
+ expect([at[1] - at[0], at[2] - at[1]]).toEqual([250, 500]);
+ });
+
+ test.each([
+ [true, { attempts: 3, timeoutMs: 20_000 }],
+ [false, { attempts: 1, timeoutMs: 5_000 }],
+ ])('building=%s takes %o', (building, expected) => {
+ expect(attemptPolicy(building)).toEqual(expected);
+ });
+
+ test('retries a dropped connection before believing it', async () => {
+ let calls = 0;
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async () => {
+ calls += 1;
+ if (calls < 3) throw new TypeError('fetch failed');
+ return new Response(LIVE_FEED);
+ }),
+ );
+
+ expect(await settle(fetchBlogPostLinks(3600, true))).toHaveLength(2);
+ expect(calls).toBe(3);
+ });
+
+ test('does not retry a document it has already read and rejected', async () => {
+ const fetchMock = respondWith('<!doctype html><title>oops</title>');
+ await expect(settle(fetchBlogPostLinks(3600, true))).rejects.toThrow('is not an RSS feed');
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ test('a render gives up on the first failure rather than holding itself open', async () => {
+ const fetchMock = vi.fn(async () => {
+ throw new TypeError('fetch failed');
+ });
+ vi.stubGlobal('fetch', fetchMock);
+
+ await expect(fetchBlogPostLinks(3600)).rejects.toThrow('fetch failed');
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('failure posture', () => {
+ function refuseConnection() {
+ const fetchMock = vi.fn(async () => {
+ throw new TypeError('fetch failed');
+ });
+ vi.stubGlobal('fetch', fetchMock);
+ return fetchMock;
+ }
+
+ function building(env: { CI?: string; VERCEL?: string; VERCEL_ENV?: string } = {}) {
+ vi.stubEnv('NEXT_PHASE', 'phase-production-build');
+ vi.stubEnv('CI', env.CI ?? '');
+ vi.stubEnv('VERCEL', env.VERCEL ?? '');
+ vi.stubEnv('VERCEL_ENV', env.VERCEL_ENV ?? '');
+ }
+
+ test('propagates a fetch failure so a revalidate keeps the last good index', async () => {
+ refuseConnection();
+ await expect(settle(blogPostLinks(3600))).rejects.toThrow('fetch failed');
+ });
+
+ test('builds without the posts when the marketing origin is unreachable', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ building();
+ refuseConnection();
+
+ expect(await settle(blogPostLinks(3600))).toEqual([]);
+ expect(warn.mock.calls[0]?.[0]).toContain('could not be read after 3 attempts');
+ });
+
+ test('returns the posts normally during a build that can reach the origin', async () => {
+ building();
+ respondWith(LIVE_FEED);
+ expect(await settle(blogPostLinks(3600))).toHaveLength(2);
+ });
+
+ test('a CI build off Vercel never reads the feed, since it publishes nothing', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ building({ CI: 'true' });
+ const fetchMock = respondWith(LIVE_FEED);
+
+ expect(await settle(blogPostLinks(3600))).toEqual([]);
+ expect(fetchMock).not.toHaveBeenCalled();
+ expect(warn.mock.calls[0]?.[0]).toContain('CI build off Vercel');
+ });
+
+ test('a Vercel build, which also sets CI, still reads the feed', async () => {
+ building({ CI: '1', VERCEL: '1', VERCEL_ENV: 'preview' });
+ const fetchMock = respondWith(LIVE_FEED);
+
+ expect(await settle(blogPostLinks(3600))).toHaveLength(2);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ test('fails the production build rather than deploying a blogless index', async () => {
+ building({ CI: '1', VERCEL: '1', VERCEL_ENV: 'production' });
+ const fetchMock = refuseConnection();
+
+ await expect(settle(blogPostLinks(3600))).rejects.toThrow('refusing to publish');
+ expect(fetchMock).toHaveBeenCalledTimes(3);
+ });
+
+ test('the fatal error names the URL, the remedy and the escape hatch', async () => {
+ building({ VERCEL_ENV: 'production' });
+ refuseConnection();
+
+ const error = (await settle(blogPostLinks(3600)).catch((thrown: unknown) => thrown)) as Error;
+ expect(error.message).toContain(BLOG_FEED_URL);
+ expect(error.message).toContain('redeploy this commit');
+ expect(error.message).toContain('OK_ALLOW_BLOGLESS_LLMS_TXT=1');
+
+ const cause = error.cause as Error;
+ expect(cause.message).toContain('after 3 attempts');
+ expect((cause.cause as Error).message).toBe('fetch failed');
+ });
+
+ test('publishes a production build when the blog is legitimately empty', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ building({ VERCEL_ENV: 'production' });
+ respondWith(feed());
+
+ expect(await settle(blogPostLinks(3600))).toEqual([]);
+ expect(warn.mock.calls[0]?.[0]).toContain('the blog lists no posts');
+ });
+
+ test('lets an operator publish without the posts, fail-closed', async () => {
+ building({ VERCEL_ENV: 'production' });
+ refuseConnection();
+
+ vi.stubEnv('OK_ALLOW_BLOGLESS_LLMS_TXT', 'true');
+ await expect(settle(blogPostLinks(3600))).rejects.toThrow('refusing to publish');
+
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.stubEnv('OK_ALLOW_BLOGLESS_LLMS_TXT', '1');
+ expect(await settle(blogPostLinks(3600))).toEqual([]);
+ });
+
+ test.each([
+ 'preview',
+ 'development',
+ ])('still builds without the posts on a %s deployment', async (env) => {
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ building({ VERCEL_ENV: env });
+ refuseConnection();
+
+ expect(await settle(blogPostLinks(3600))).toEqual([]);
+ });
+});
diff --git a/docs/src/lib/marketing-blog-index.ts b/docs/src/lib/marketing-blog-index.ts
new file mode 100644
index 000000000..5402dd184
--- /dev/null
+++ b/docs/src/lib/marketing-blog-index.ts
@@ -0,0 +1,246 @@
+import { PHASE_PRODUCTION_BUILD } from 'next/constants';
+import type { LlmsTxtLink } from '@/lib/llms-txt';
+import { SITE_URL } from '@/lib/site';
+
+export const BLOG_FEED_URL = `${SITE_URL}/blog/rss.xml`;
+
+const RENDER_TIMEOUT_MS = 5_000;
+const BUILD_TIMEOUT_MS = 20_000;
+
+const BUILD_ATTEMPTS = 3;
+const RETRY_BASE_MS = 250;
+
+const CDATA_OPEN = '<![CDATA[';
+const CDATA_CLOSE = ']]>';
+const CDATA_SECTION = /<!\[CDATA\[([\s\S]*?)\]\]>/g;
+
+const LINK_IN_MESSAGE_MAX = 200;
+
+function decodeXml(value: string): string {
+ return value
+ .replace(/&lt;/g, '<')
+ .replace(/&gt;/g, '>')
+ .replace(/&quot;/g, '"')
+ .replace(/&apos;/g, "'")
+ .replace(/&amp;/g, '&');
+}
+
+function textOf(value: string): string {
+ const sections = [...value.matchAll(CDATA_SECTION)];
+ if (sections.length > 0) return sections.map((section) => section[1]).join('');
+ return decodeXml(value.trim());
+}
+
+function matchOutsideCdata(xml: string, needle: RegExp, from: number): RegExpExecArray | null {
+ let at = from;
+ for (;;) {
+ needle.lastIndex = at;
+ const match = needle.exec(xml);
+ if (!match) return null;
+ const cdata = xml.indexOf(CDATA_OPEN, at);
+ if (cdata === -1 || match.index < cdata) return match;
+ const end = xml.indexOf(CDATA_CLOSE, cdata + CDATA_OPEN.length);
+ if (end === -1) return null;
+ at = end + CDATA_CLOSE.length;
+ }
+}
+
+function elements(xml: string, name: string): { found: string[]; dropped: boolean } {
+ const open = new RegExp(`<${name}(?:\\s[^>]*)?>`, 'g');
+ const close = new RegExp(`</${name}>`, 'g');
+ const found: string[] = [];
+ let at = 0;
+ for (;;) {
+ const opened = matchOutsideCdata(xml, open, at);
+ if (!opened) return { found, dropped: false };
+ const start = opened.index + opened[0].length;
+ const closed = matchOutsideCdata(xml, close, start);
+ const reopened = matchOutsideCdata(xml, open, start);
+ if (reopened && (!closed || reopened.index < closed.index)) {
+ found.push(xml.slice(start, reopened.index));
+ at = reopened.index;
+ continue;
+ }
+ if (!closed) return { found, dropped: true };
+ found.push(xml.slice(start, closed.index));
+ at = closed.index + closed[0].length;
+ }
+}
+
+function element(item: string, name: string): string | undefined {
+ const [inner] = elements(item, name).found;
+ return inner === undefined ? undefined : textOf(inner);
+}
+
+function quoteForMessage(value: string): string {
+ return JSON.stringify(value.replace(/\s+/g, ' ').slice(0, LINK_IN_MESSAGE_MAX));
+}
+
+class FeedStatusError extends Error {
+ override readonly name = 'FeedStatusError';
+ constructor(readonly status: number) {
+ super(`${BLOG_FEED_URL} responded ${status}`);
+ }
+}
+
+export class EmptyBlogError extends Error {
+ override readonly name = 'EmptyBlogError';
+}
+
+function isTransient(error: unknown): boolean {
+ if (error instanceof FeedStatusError) return error.status >= 500;
+ return error instanceof DOMException || error instanceof TypeError;
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export function attemptPolicy(building: boolean): { attempts: number; timeoutMs: number } {
+ return building
+ ? { attempts: BUILD_ATTEMPTS, timeoutMs: BUILD_TIMEOUT_MS }
+ : { attempts: 1, timeoutMs: RENDER_TIMEOUT_MS };
+}
+
+export async function fetchBlogPostLinks(
+ revalidateSeconds: number,
+ building = false,
+): Promise<LlmsTxtLink[]> {
+ const { attempts, timeoutMs } = attemptPolicy(building);
+
+ let lastError: unknown;
+ for (let attempt = 0; attempt < attempts; attempt += 1) {
+ if (attempt > 0) await sleep(RETRY_BASE_MS * 2 ** (attempt - 1));
+ try {
+ return await attemptFetch(revalidateSeconds, timeoutMs);
+ } catch (error) {
+ if (!isTransient(error)) throw error;
+ lastError = error;
+ }
+ }
+ throw new Error(
+ `${BLOG_FEED_URL} could not be read after ${attempts} attempt${attempts === 1 ? '' : 's'} ` +
+ `(${lastError instanceof Error ? lastError.message : String(lastError)})`,
+ { cause: lastError },
+ );
+}
+
+async function attemptFetch(revalidateSeconds: number, timeoutMs: number): Promise<LlmsTxtLink[]> {
+ const response = await fetch(BLOG_FEED_URL, {
+ signal: AbortSignal.timeout(timeoutMs),
+ next: { revalidate: revalidateSeconds },
+ });
+ if (!response.ok) {
+ await response.body?.cancel().catch(() => undefined);
+ throw new FeedStatusError(response.status);
+ }
+
+ return readPosts(await response.text());
+}
+
+function readPost(item: string, postPrefix: string): LlmsTxtLink | string {
+ const title = element(item, 'title');
+ const link = element(item, 'link');
+ if (!title || !link) return 'an <item> with no <title> or no <link>';
+ if (!link.startsWith(postPrefix) || link.slice(postPrefix.length).includes('/')) {
+ return `${quoteForMessage(link)} is not a post at ${postPrefix}<slug>`;
+ }
+ const description = element(item, 'description');
+ return { url: `${link}.md`, name: title, ...(description ? { description } : {}) };
+}
+
+function readPosts(xml: string): LlmsTxtLink[] {
+ if (!/<rss[\s>]/.test(xml)) {
+ throw new Error(
+ `${BLOG_FEED_URL} is not an RSS feed — a different document, or an error page served ` +
+ 'with a 200',
+ );
+ }
+
+ const complete = xml.trimEnd().endsWith('</rss>');
+ const { found: items, dropped } = elements(xml, 'item');
+ const damage = !complete
+ ? 'ends before </rss>, so it is truncated'
+ : dropped
+ ? 'leaves an <item> unclosed, so it is malformed'
+ : null;
+ if (items.length === 0) {
+ if (damage) {
+ throw new Error(`${BLOG_FEED_URL} ${damage} — it is not an empty blog`);
+ }
+ throw new EmptyBlogError(`${BLOG_FEED_URL} lists no posts`);
+ }
+ if (damage) {
+ console.warn(
+ `llms.txt: ${BLOG_FEED_URL} ${damage}, so posts after that point may be missing from ` +
+ 'this index',
+ );
+ }
+
+ const postPrefix = `${SITE_URL}/blog/`;
+ const posts: LlmsTxtLink[] = [];
+ const skipped: string[] = [];
+ for (const item of items) {
+ const read = readPost(item, postPrefix);
+ if (typeof read === 'string') skipped.push(read);
+ else posts.push(read);
+ }
+
+ if (posts.length === 0) {
+ throw new Error(
+ `${BLOG_FEED_URL} lists ${items.length} item${items.length === 1 ? '' : 's'} and not one ` +
+ `is a post (${skipped[0]}) — either this is not the blog feed this reads (a different ` +
+ 'document, or a canonical host that drifted between the two apps), or the post URL ' +
+ 'shape changed',
+ );
+ }
+ for (const reason of skipped) {
+ console.warn(`llms.txt: skipping a ${BLOG_FEED_URL} item — ${reason}`);
+ }
+ return posts;
+}
+
+const ALLOW_BLOGLESS = 'OK_ALLOW_BLOGLESS_LLMS_TXT';
+
+function isOffVercelCi(): boolean {
+ return Boolean(process.env.CI) && !process.env.VERCEL;
+}
+
+export async function blogPostLinks(revalidateSeconds: number): Promise<LlmsTxtLink[]> {
+ const building = process.env.NEXT_PHASE === PHASE_PRODUCTION_BUILD;
+ if (building && isOffVercelCi()) {
+ console.warn(
+ `llms.txt: omitting blog posts — a CI build off Vercel publishes nothing, so it does not ` +
+ `read ${BLOG_FEED_URL}`,
+ );
+ return [];
+ }
+
+ try {
+ return await fetchBlogPostLinks(revalidateSeconds, building);
+ } catch (error) {
+ if (!building) throw error;
+
+ if (process.env.VERCEL_ENV === 'production' && !(error instanceof EmptyBlogError)) {
+ if (process.env[ALLOW_BLOGLESS] !== '1') {
+ throw new Error(
+ `llms.txt: refusing to publish a production build with no blog posts. ${describe(error)}, ` +
+ 'and the index would ship a Blog section with no posts in it for the life of this ' +
+ 'deployment. Confirm the marketing zone serves that URL and redeploy this commit; ' +
+ `set ${ALLOW_BLOGLESS}=1 to publish without the posts.`,
+ { cause: error },
+ );
+ }
+ console.warn(
+ `llms.txt: ${ALLOW_BLOGLESS}=1 is set, so this PRODUCTION build publishes an index with ` +
+ `no blog posts in it. ${describe(error)}. Unset it once the cause is fixed.`,
+ );
+ }
+
+ console.warn(`llms.txt: omitting blog posts — ${describe(error)}`, error);
+ return [];
+ }
+}
+
+function describe(error: unknown): string {
+ if (error instanceof EmptyBlogError) return 'the blog lists no posts';
+ return error instanceof Error ? error.message : String(error);
+}
diff --git a/docs/src/lib/marketing-markdown-pages.test.ts b/docs/src/lib/marketing-markdown-pages.test.ts
new file mode 100644
index 000000000..6cbb0e790
--- /dev/null
+++ b/docs/src/lib/marketing-markdown-pages.test.ts
@@ -0,0 +1,27 @@
+import { describe, expect, test } from 'vitest';
+import { BRAND_ROUTE } from './brand-assets';
+import { MARKETING_MARKDOWN_PAGES } from './marketing-markdown-pages';
+
+describe('curated marketing rendition inventory', () => {
+ test('tracks the brand route this app links elsewhere', () => {
+ expect(MARKETING_MARKDOWN_PAGES.map((page) => page.path)).toContain(`${BRAND_ROUTE}.md`);
+ });
+
+ test('every entry is a site-relative Markdown path', () => {
+ for (const page of MARKETING_MARKDOWN_PAGES) {
+ expect(page.path).toMatch(/^\/[^\s]*\.md$/);
+ }
+ });
+
+ test('every entry can be described to an agent', () => {
+ for (const page of MARKETING_MARKDOWN_PAGES) {
+ expect(page.name.length).toBeGreaterThan(0);
+ expect(page.description.length).toBeGreaterThan(0);
+ }
+ });
+
+ test('lists no path twice', () => {
+ const paths = MARKETING_MARKDOWN_PAGES.map((page) => page.path);
+ expect(paths).toEqual([...new Set(paths)]);
+ });
+});
diff --git a/docs/src/lib/marketing-markdown-pages.ts b/docs/src/lib/marketing-markdown-pages.ts
new file mode 100644
index 000000000..2bd063c47
--- /dev/null
+++ b/docs/src/lib/marketing-markdown-pages.ts
@@ -0,0 +1,39 @@
+export interface MarketingMarkdownPage {
+ readonly path: string;
+ readonly section: 'Blog' | 'Product';
+ readonly name: string;
+ readonly description: string;
+}
+
+export const MARKETING_MARKDOWN_PAGES: readonly MarketingMarkdownPage[] = [
+ {
+ path: '/index.md',
+ section: 'Product',
+ name: 'OpenKnowledge',
+ description: 'What it is, how to get it, and where everything else lives.',
+ },
+ {
+ path: '/download.md',
+ section: 'Product',
+ name: 'Download',
+ description: 'Direct installer URLs for every macOS, Windows, and Linux build.',
+ },
+ {
+ path: '/team.md',
+ section: 'Product',
+ name: 'Team',
+ description: 'Who builds OpenKnowledge, and the rendition of each profile.',
+ },
+ {
+ path: '/brand.md',
+ section: 'Product',
+ name: 'Brand',
+ description: 'Logo and icon files, and the rules for using them.',
+ },
+ {
+ path: '/blog.md',
+ section: 'Blog',
+ name: 'Blog index',
+ description: 'Every published post with its title, date, and summary.',
+ },
+];
diff --git a/docs/src/lib/mdx-attribute-literal.test.ts b/docs/src/lib/mdx-attribute-literal.test.ts
new file mode 100644
index 000000000..7201116bd
--- /dev/null
+++ b/docs/src/lib/mdx-attribute-literal.test.ts
@@ -0,0 +1,84 @@
+import { describe, expect, test } from 'vitest';
+import { parseAttributeLiteral } from './mdx-attribute-literal.ts';
+
+describe('parseAttributeLiteral', () => {
+ test('reads the single-quoted string arrays MDX attributes are written with', () => {
+ expect(parseAttributeLiteral("['Desktop app', 'Web app']")).toEqual(['Desktop app', 'Web app']);
+ });
+
+ test('reads an object with unquoted keys, quoted keys, and a trailing comma', () => {
+ expect(
+ parseAttributeLiteral(`{
+ "src": { description: 'Image source', required: true, },
+ alt: { required: false },
+ }`),
+ ).toEqual({
+ src: { description: 'Image source', required: true },
+ alt: { required: false },
+ });
+ });
+
+ test('reads a multi-line template literal, which is how diagram sources arrive', () => {
+ expect(parseAttributeLiteral('`graph LR\n A --> B`')).toBe('graph LR\n A --> B');
+ });
+
+ test('applies string escapes rather than passing the backslash through', () => {
+ expect(parseAttributeLiteral(String.raw`"a\\b \"c\" é \n"`)).toBe('a\\b "c" é \n');
+ });
+
+ test('reads numbers, booleans, null, and undefined', () => {
+ expect(parseAttributeLiteral('[1, -2.5, 3e2, true, false, null, undefined]')).toEqual([
+ 1,
+ -2.5,
+ 300,
+ true,
+ false,
+ null,
+ undefined,
+ ]);
+ });
+
+ test('skips comments, which authors leave inside prop objects', () => {
+ expect(parseAttributeLiteral('{ /* note */ a: 1, // trailing\n b: 2 }')).toEqual({
+ a: 1,
+ b: 2,
+ });
+ });
+});
+
+describe('anything that is not a literal', () => {
+ test('a free identifier is refused by name', () => {
+ expect(() => parseAttributeLiteral('{ items: SUPPORTED_EDITORS }')).toThrow(
+ 'identifier "SUPPORTED_EDITORS" is not a literal',
+ );
+ });
+
+ test('a function call is refused', () => {
+ expect(() => parseAttributeLiteral('buildRows()')).toThrow('is not a literal');
+ });
+
+ test('a template interpolation is refused instead of being emitted verbatim', () => {
+ // biome-ignore lint/suspicious/noTemplateCurlyInString: the placeholder is the input under test
+ expect(() => parseAttributeLiteral('`count: ${total}`')).toThrow(
+ 'template interpolation is not a literal',
+ );
+ });
+
+ test('trailing input after a complete literal is refused', () => {
+ expect(() => parseAttributeLiteral("['a'] ?? []")).toThrow('unexpected trailing input');
+ });
+
+ test('a malformed hex escape is refused, never coerced to NUL', () => {
+ expect(() => parseAttributeLiteral("['a\\uZZZZb']")).toThrow('malformed escape');
+ expect(() => parseAttributeLiteral("['a\\xZZb']")).toThrow('malformed escape');
+ expect(() => parseAttributeLiteral("['\\u{WXYZ}']")).toThrow('malformed escape');
+ });
+
+ test('a braced escape beyond the Unicode range is refused with a position', () => {
+ expect(() => parseAttributeLiteral("['\\u{110000}']")).toThrow('beyond the Unicode range');
+ });
+
+ test('the failure names the line, so a multi-line prop object is locatable', () => {
+ expect(() => parseAttributeLiteral('{\n a: 1,\n b: someVar,\n}')).toThrow('at line 3');
+ });
+});
diff --git a/docs/src/lib/mdx-attribute-literal.ts b/docs/src/lib/mdx-attribute-literal.ts
new file mode 100644
index 000000000..65f362047
--- /dev/null
+++ b/docs/src/lib/mdx-attribute-literal.ts
@@ -0,0 +1,221 @@
+export type AttributeLiteral =
+ | string
+ | number
+ | boolean
+ | null
+ | undefined
+ | AttributeLiteral[]
+ | { [key: string]: AttributeLiteral };
+
+class LiteralParser {
+ private index = 0;
+
+ constructor(private readonly source: string) {}
+
+ parse(): AttributeLiteral {
+ this.skipTrivia();
+ const value = this.parseValue();
+ this.skipTrivia();
+ if (this.index < this.source.length) this.fail('unexpected trailing input');
+ return value;
+ }
+
+ private parseValue(): AttributeLiteral {
+ const char = this.peek();
+ if (char === '[') return this.parseArray();
+ if (char === '{') return this.parseObject();
+ if (char === '"' || char === "'") return this.parseQuoted(char);
+ if (char === '`') return this.parseTemplate();
+ if (char === '-' || (char >= '0' && char <= '9')) return this.parseNumber();
+
+ const word = /^[A-Za-z_$][\w$]*/.exec(this.source.slice(this.index))?.[0];
+ if (word === 'true' || word === 'false' || word === 'null' || word === 'undefined') {
+ this.index += word.length;
+ return word === 'true' ? true : word === 'false' ? false : word === 'null' ? null : undefined;
+ }
+ this.fail(word ? `identifier "${word}" is not a literal` : 'expected a literal value');
+ }
+
+ private parseArray(): AttributeLiteral[] {
+ this.index += 1;
+ const items: AttributeLiteral[] = [];
+ for (;;) {
+ this.skipTrivia();
+ if (this.peek() === ']') {
+ this.index += 1;
+ return items;
+ }
+ items.push(this.parseValue());
+ this.skipTrivia();
+ if (this.peek() === ',') this.index += 1;
+ else if (this.peek() !== ']') this.fail('expected "," or "]"');
+ }
+ }
+
+ private parseObject(): { [key: string]: AttributeLiteral } {
+ this.index += 1;
+ const entries: { [key: string]: AttributeLiteral } = {};
+ for (;;) {
+ this.skipTrivia();
+ if (this.peek() === '}') {
+ this.index += 1;
+ return entries;
+ }
+ const key = this.parseKey();
+ this.skipTrivia();
+ if (this.peek() !== ':') this.fail('expected ":" after an object key');
+ this.index += 1;
+ this.skipTrivia();
+ entries[key] = this.parseValue();
+ this.skipTrivia();
+ if (this.peek() === ',') this.index += 1;
+ else if (this.peek() !== '}') this.fail('expected "," or "}"');
+ }
+ }
+
+ private parseKey(): string {
+ const char = this.peek();
+ if (char === '"' || char === "'") return this.parseQuoted(char);
+ const identifier = /^[A-Za-z_$][\w$]*/.exec(this.source.slice(this.index))?.[0];
+ if (!identifier) this.fail('expected an object key');
+ this.index += identifier.length;
+ return identifier;
+ }
+
+ private parseQuoted(quote: string): string {
+ this.index += 1;
+ let out = '';
+ while (this.index < this.source.length) {
+ const char = this.source[this.index];
+ if (char === quote) {
+ this.index += 1;
+ return out;
+ }
+ if (char === '\\') {
+ out += this.readEscape();
+ continue;
+ }
+ if (char === '\n') this.fail('unterminated string');
+ out += char;
+ this.index += 1;
+ }
+ this.fail('unterminated string');
+ }
+
+ private parseTemplate(): string {
+ this.index += 1;
+ let out = '';
+ while (this.index < this.source.length) {
+ const char = this.source[this.index];
+ if (char === '`') {
+ this.index += 1;
+ return out;
+ }
+ if (char === '\\') {
+ out += this.readEscape();
+ continue;
+ }
+ if (char === '$' && this.source[this.index + 1] === '{') {
+ this.fail('template interpolation is not a literal');
+ }
+ out += char;
+ this.index += 1;
+ }
+ this.fail('unterminated template literal');
+ }
+
+ private readEscape(): string {
+ const escaped = this.source[this.index + 1];
+ this.index += 2;
+ switch (escaped) {
+ case 'n':
+ return '\n';
+ case 't':
+ return '\t';
+ case 'r':
+ return '\r';
+ case 'b':
+ return '\b';
+ case 'f':
+ return '\f';
+ case 'v':
+ return '\v';
+ case '0':
+ return '\0';
+ case 'u': {
+ if (this.source[this.index] === '{') {
+ const end = this.source.indexOf('}', this.index);
+ if (end === -1) this.fail('unterminated unicode escape');
+ const code = this.source.slice(this.index + 1, end);
+ this.index = end + 1;
+ return this.codePoint(code);
+ }
+ const code = this.source.slice(this.index, this.index + 4);
+ this.index += 4;
+ return this.codePoint(code, 4);
+ }
+ case 'x': {
+ const code = this.source.slice(this.index, this.index + 2);
+ this.index += 2;
+ return this.codePoint(code, 2);
+ }
+ case '\n':
+ return '';
+ default:
+ return escaped ?? '';
+ }
+ }
+
+ private codePoint(code: string, width?: number): string {
+ if ((width !== undefined && code.length !== width) || !/^[0-9a-fA-F]+$/.test(code)) {
+ this.fail('malformed escape sequence');
+ }
+ const parsed = Number.parseInt(code, 16);
+ if (parsed > 0x10_ff_ff) this.fail('escape is beyond the Unicode range');
+ return String.fromCodePoint(parsed);
+ }
+
+ private parseNumber(): number {
+ const matched = /^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?/.exec(this.source.slice(this.index));
+ if (!matched) this.fail('expected a number');
+ this.index += matched[0].length;
+ return Number(matched[0]);
+ }
+
+ private peek(): string {
+ return this.source[this.index] ?? '';
+ }
+
+ private skipTrivia(): void {
+ for (;;) {
+ while (this.index < this.source.length && /\s/.test(this.source[this.index] ?? '')) {
+ this.index += 1;
+ }
+ if (this.source.startsWith('//', this.index)) {
+ const end = this.source.indexOf('\n', this.index);
+ this.index = end === -1 ? this.source.length : end;
+ continue;
+ }
+ if (this.source.startsWith('/*', this.index)) {
+ const end = this.source.indexOf('*/', this.index);
+ if (end === -1) this.fail('unterminated comment');
+ this.index = end + 2;
+ continue;
+ }
+ return;
+ }
+ }
+
+ private fail(reason: string): never {
+ const upto = this.source.slice(0, this.index);
+ const line = upto.split('\n').length;
+ throw new Error(
+ `Cannot read MDX attribute expression at line ${line}: ${reason}. ` +
+ 'Attribute expressions reachable from served Markdown must be plain literals.',
+ );
+ }
+}
+
+export function parseAttributeLiteral(source: string): AttributeLiteral {
+ return new LiteralParser(source).parse();
+}
diff --git a/docs/src/lib/mdx-serializer-registry.test.ts b/docs/src/lib/mdx-serializer-registry.test.ts
new file mode 100644
index 000000000..b1f0106c7
--- /dev/null
+++ b/docs/src/lib/mdx-serializer-registry.test.ts
@@ -0,0 +1,451 @@
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+import { beforeAll, describe, expect, test } from 'vitest';
+import { AGENTS } from '../components/agent-icons.tsx';
+import { LAYERS, PATHS } from '../components/overview-blocks.tsx';
+import { type CorpusPage, readCorpus } from './docs-corpus.test-helper.ts';
+import { DOWNLOAD_TARGETS } from './download-targets.ts';
+import { serializeMdx } from './mdx-serializer.ts';
+import { DOCS_SERIALIZER_REGISTRY } from './mdx-serializer-registry.ts';
+
+const PAGE_URL = 'https://openknowledge.ai/docs/get-started/quickstart';
+
+function serialize(source: string, pageUrl = PAGE_URL): string {
+ return serializeMdx(source, { registry: DOCS_SERIALIZER_REGISTRY, pageUrl }).body;
+}
+
+describe('components that carry their content in attributes', () => {
+ test('a callout becomes a blockquote led by its title', () => {
+ expect(serialize('<Callout type="info" title="Verify Git">\n\nRun it.\n\n</Callout>')).toBe(
+ '> **Verify Git**\n>\n> Run it.',
+ );
+ });
+
+ test('an untitled callout keeps its variant, which is its only severity signal', () => {
+ expect(serialize('<Callout type="warning">\n\nThis bites.\n\n</Callout>')).toBe(
+ '> **Warning**\n>\n> This bites.',
+ );
+ });
+
+ test('a TypeTable becomes a table with a row per prop', () => {
+ const md = serialize(
+ `<TypeTable type={{ "src": { description: 'Image source', type: 'string', required: true } }} />`,
+ );
+ expect(md).toContain('| Prop | Type | Required | Default | Description |');
+ expect(md).toContain('| `src` | `string` | yes | | Image source |');
+ });
+
+ test.each([
+ ['true', 'yes'],
+ ['1', 'yes'],
+ ['false', 'no'],
+ ['0', 'no'],
+ ])('a `required` of %s reads as %s, as the rendered table shows it', (value, expected) => {
+ const md = serialize(`<TypeTable type={{ "src": { type: 'string', required: ${value} } }} />`);
+ expect(md).toContain(`| \`src\` | \`string\` | ${expected} |`);
+ });
+
+ test('a generic type reaches the reader as it was written', () => {
+ const md = serialize(`<TypeTable type={{ "items": { type: 'Array<string>' } }} />`);
+ const tree = unified().use(remarkParse).use(remarkGfm).parse(md);
+
+ const spans: string[] = [];
+ visit(tree, 'inlineCode', (node) => spans.push(node.value));
+ expect(spans).toContain('Array<string>');
+ });
+
+ test('a TypeTable cell escapes a pipe inside a union type rather than splitting the column', () => {
+ const md = serialize(`<TypeTable type={{ "kind": { type: "'a' | 'b'" } }} />`);
+ expect(md).toContain("| `'a' \\| 'b'` |");
+ });
+
+ test('prose naming an HTML tag survives as words, not as an unclosed tag', () => {
+ const md = serialize(`<TypeTable type={{ "title": { description: 'Shown in <summary>' } }} />`);
+ expect(md).toContain(String.raw`Shown in \<summary>`);
+ });
+
+ test('a Mermaid chart becomes a mermaid fence', () => {
+ expect(serialize('<Mermaid chart={`graph LR\nA --> B`} />')).toBe(
+ '```mermaid\ngraph LR\nA --> B\n```',
+ );
+ });
+
+ test('a base64 HtmlPreview attribute is decoded back to its preview fence', () => {
+ const encoded = Buffer.from('<p>hi</p>', 'utf8').toString('base64');
+ expect(serialize(`<HtmlPreview code="${encoded}" />`)).toBe('```html preview\n<p>hi</p>\n```');
+ });
+
+ test('an icon carrying an accessible name contributes that name to the sentence', () => {
+ expect(serialize('Click the <Clock role="img" aria-label="Timeline" /> icon.')).toBe(
+ 'Click the Timeline icon.',
+ );
+ });
+
+ test('a decorative icon with no accessible name contributes nothing', () => {
+ expect(serialize('Click the <InlineIcon name="Bell" /> bell.')).toBe('Click the bell.');
+ });
+});
+
+describe('inputs the registry refuses rather than serializes wrong', () => {
+ test('a TypeTable whose rows are a free identifier is refused, naming it', () => {
+ expect(() => serialize('<TypeTable type={PROP_ROWS} />')).toThrow(
+ 'identifier "PROP_ROWS" is not a literal',
+ );
+ });
+
+ test('a TypeTable without its rows object is refused', () => {
+ expect(() => serialize('<TypeTable />')).toThrow(
+ '<TypeTable> carries its rows in a `type` object, which is missing.',
+ );
+ });
+
+ test('a TypeTable whose literal is not an object of descriptors is refused', () => {
+ expect(() => serialize("<TypeTable type={['src']} />")).toThrow(
+ '<TypeTable type={…}> must be an object of prop descriptors.',
+ );
+ });
+
+ test('a TypeTable whose type is a quoted string is refused without claiming it is missing', () => {
+ expect(() => serialize('<TypeTable type="string" />')).toThrow(
+ 'must be a braced object expression',
+ );
+ });
+
+ test('a TypeTable descriptor that is not an object is refused, naming the prop', () => {
+ expect(() => serialize(`<TypeTable type={{ src: 'string' }} />`)).toThrow(
+ 'descriptor for `src` must be an object',
+ );
+ });
+
+ test('a TypeTable field holding an object is refused, naming the field', () => {
+ expect(() => serialize(`<TypeTable type={{ src: { type: { kind: 'string' } } }} />`)).toThrow(
+ '`src.type` must be a string, number or boolean',
+ );
+ });
+
+ test('a TypeTable field holding an array is refused too', () => {
+ expect(() => serialize(`<TypeTable type={{ src: { default: ['a', 'b'] } }} />`)).toThrow(
+ '`src.default` must be a string, number or boolean',
+ );
+ });
+
+ test.each([
+ ['<Image src={hero} alt="Editor" />', 'Image'],
+ ['<img src={hero} alt="Editor" />', 'img'],
+ ])('refuses %s rather than deleting it from the rendition', (source, name) => {
+ expect(() => serialize(source)).toThrow(
+ new RegExp(`<${name} src=\\{hero\\}> carries its value in an expression`),
+ );
+ });
+
+ test('names the absent case differently from the unreadable one', () => {
+ expect(() => serialize('<img alt="Editor" />')).toThrow('<img> has no `src`');
+ });
+
+ test('a Tabs items expression that is not an array is refused', () => {
+ expect(() => serialize('<Tabs items={{ first: 1 }} />')).toThrow(
+ '<Tabs items={…}> must be an array of labels.',
+ );
+ });
+
+ test('an HtmlPreview whose code is not base64 is refused rather than shipped as mojibake', () => {
+ expect(() => serialize('<HtmlPreview code="<p>hi</p>" />')).toThrow('is not valid base64');
+ });
+
+ test.each([
+ ['a`b', 'a`b'],
+ ['``', '``'],
+ ['`start', '`start'],
+ ['end`', 'end`'],
+ ])('an inline code span carrying %j round-trips as one span', (value, expected) => {
+ const md = serialize(`<code>${value}</code>`);
+ const tree = unified().use(remarkParse).use(remarkGfm).parse(md);
+
+ const spans: string[] = [];
+ visit(tree, 'inlineCode', (node) => {
+ spans.push(node.value);
+ });
+ expect(spans).toEqual([expected]);
+ });
+
+ test.each([
+ ['base64url', (value: string) => value.replaceAll('+', '-').replaceAll('/', '_')],
+ ['line-wrapped', (value: string) => value.replace(/(.{8})/g, '$1\n')],
+ ])('accepts a %s spelling the decoder reads in full', (_name, respell) => {
+ const html = '<p>hi &amp; bye</p>';
+ const encoded = respell(Buffer.from(html, 'utf8').toString('base64'));
+
+ expect(serialize(`<HtmlPreview code="${encoded}" />`)).toBe(
+ `\`\`\`html preview\n${html}\n\`\`\``,
+ );
+ });
+
+ test('a preview body carrying backtick runs gets a fence longer than the runs', () => {
+ const encoded = Buffer.from('<code>```</code>', 'utf8').toString('base64');
+ expect(serialize(`<HtmlPreview code="${encoded}" />`)).toBe(
+ '````html preview\n<code>```</code>\n````',
+ );
+ });
+
+ test('a Mermaid without a chart is refused', () => {
+ expect(() => serialize('<Mermaid />')).toThrow('<Mermaid> needs a `chart` string.');
+ });
+
+ test('an HtmlPreview without its encoded block is refused', () => {
+ expect(() => serialize('<HtmlPreview />')).toThrow(
+ '<HtmlPreview> carries its block in a `code` attribute, which is missing.',
+ );
+ });
+});
+
+describe('tab groups', () => {
+ test('each panel becomes a heading taken from the items list', () => {
+ const md = serialize(
+ "<Tabs items={['Desktop app', 'Web app']}>\n\n<Tab>\n\nAlpha\n\n</Tab>\n\n<Tab>\n\nBeta\n\n</Tab>\n\n</Tabs>",
+ );
+ expect(md).toBe('### Desktop app\n\nAlpha\n\n### Web app\n\nBeta');
+ });
+
+ test('panel headings sit above the headings inside the panels', () => {
+ const md = serialize(
+ "<Tabs items={['One']}>\n\n<Tab>\n\n### Inner\n\nBody\n\n</Tab>\n\n</Tabs>",
+ );
+ expect(md.startsWith('## One\n')).toBe(true);
+ });
+
+ test('an explicit panel value wins over its position in the items list', () => {
+ const md = serialize(
+ '<Tabs items={[\'Ignored\']}>\n\n<Tab value="Chosen">\n\nBody\n\n</Tab>\n\n</Tabs>',
+ );
+ expect(md).toContain('Chosen');
+ expect(md).not.toContain('Ignored');
+ });
+});
+
+describe('components whose content lives in their own module', () => {
+ test('the download button lists every published build with its installer URL', () => {
+ const md = serialize('<DownloadButton />');
+ for (const target of DOWNLOAD_TARGETS) {
+ expect(md).toContain(`[${target.label}](${target.assetUrl})`);
+ }
+ });
+
+ test('the layer stack lists every layer with its role and description', () => {
+ const md = serialize('<LayerStack />');
+ for (const layer of LAYERS) {
+ expect(md).toContain(layer.title);
+ expect(md).toContain(layer.desc);
+ }
+ });
+
+ test('the start paths render as absolute links', () => {
+ const md = serialize('<WhereToStart />', 'https://openknowledge.ai/docs/get-started/overview');
+ for (const path of PATHS) {
+ expect(md).toContain(`[${path.title}](https://openknowledge.ai${path.href})`);
+ }
+ });
+
+ test('the agent row lists every agent it advertises', () => {
+ const md = serialize('<AgentIcons />');
+ for (const agent of AGENTS) expect(md).toContain(agent.name);
+ });
+
+ test('the MCP install block names the editor it was given and keeps its guide link absolute', () => {
+ const md = serialize('<McpInstall editor="Cursor" />');
+ expect(md).toContain('two ways to connect Cursor');
+ expect(md).toContain(
+ '](https://openknowledge.ai/docs/get-started/quickstart#ok-install-web-app)',
+ );
+ });
+
+ test('an MCP install block keeps the editor-specific notes nested inside it', () => {
+ expect(serialize('<McpInstall editor="Hermes">\n\nRestart Hermes.\n\n</McpInstall>')).toContain(
+ 'Restart Hermes.',
+ );
+ });
+
+ test('the verify block keeps the prompt and names the subject', () => {
+ const md = serialize('<VerifyExec subject="Claude Code" />');
+ expect(md).toContain('> List the first 5 documents you come across in this project.');
+ expect(md).toContain('Claude Code should call the OpenKnowledge `exec` tool');
+ });
+
+ test('an editor name that reads as markup is neutralised before it lands in prose', () => {
+ const md = serialize('<McpInstall editor="VS_Code <beta>" />');
+ expect(md).toContain('two ways to connect VS\\_Code \\<beta>');
+ expect(md).not.toContain('VS_Code');
+ });
+
+ test('a subject that spans lines still leaves the prompt as one blockquote', () => {
+ const md = serialize('<VerifyExec subject="The\nagent" />');
+ expect(md).toContain(
+ '> List the first 5 documents you come across in this project.\n\nThe agent should call',
+ );
+ });
+});
+
+describe('containers and visual-only demos', () => {
+ test('a preview wrapper contributes what its child component would', () => {
+ expect(serialize('<ComponentPreview>\n\n<AgentIcons />\n\n</ComponentPreview>')).toContain(
+ AGENTS[0].name,
+ );
+ });
+
+ test('a media demo contributes nothing, because it has nothing to say in text', () => {
+ expect(
+ serialize('<ComponentPreview>\n\n<ImgPreview src="/a.png" alt="A" />\n\n</ComponentPreview>'),
+ ).toBe('');
+ });
+
+ test('a card keeps the inline formatting in its description', () => {
+ expect(serialize('<Card title="CLI" href="/docs/reference/cli">Run `ok init`.</Card>')).toBe(
+ '- [CLI](https://openknowledge.ai/docs/reference/cli): Run `ok init`.',
+ );
+ });
+
+ test('a card with a block body keeps its blocks instead of folding them onto one line', () => {
+ const md = serialize(
+ '<Card title="Setup">\n\nFirst do this.\n\n```bash\nok init\n```\n\n</Card>',
+ );
+ expect(md).toBe('**Setup**\n\nFirst do this.\n\n```bash\nok init\n```');
+ });
+
+ test('cards render as one list rather than one list per card', () => {
+ const md = serialize(
+ '<Cards>\n <Card title="CLI" href="/docs/reference/cli">Command line.</Card>\n <Card title="Config" href="/docs/reference/configuration">Settings.</Card>\n</Cards>',
+ );
+ expect(md).toBe(
+ '- [CLI](https://openknowledge.ai/docs/reference/cli): Command line.\n' +
+ '- [Config](https://openknowledge.ai/docs/reference/configuration): Settings.',
+ );
+ });
+});
+
+describe('the docs corpus through the real registry', () => {
+ const withMdx = unified().use(remarkParse).use(remarkMdx).use(remarkGfm);
+ const asMarkdown = unified().use(remarkParse).use(remarkGfm);
+
+ let pages: CorpusPage[];
+ const output = new Map<string, string>();
+
+ beforeAll(async () => {
+ pages = await readCorpus();
+ for (const page of pages) {
+ output.set(
+ page.slug,
+ serializeMdx(page.source, {
+ registry: DOCS_SERIALIZER_REGISTRY,
+ pageUrl: page.pageUrl,
+ }).body,
+ );
+ }
+ });
+
+ test('every page serializes, so no component is left without a disposition', () => {
+ expect(pages.length).toBeGreaterThan(50);
+ expect(output.size).toBe(pages.length);
+ });
+
+ test('no page leaks a JSX element into the Markdown it serves', () => {
+ const leaks: string[] = [];
+ for (const [slug, md] of output) {
+ visit(withMdx.parse(md), (node) => {
+ if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
+ leaks.push(`${slug}: <${node.name}>`);
+ }
+ });
+ }
+ expect(leaks).toEqual([]);
+ });
+
+ test('no page indents a code block to the four-space column', () => {
+ const indented: string[] = [];
+ for (const [slug, md] of output) {
+ visit(asMarkdown.parse(md), 'code', (node) => {
+ const opener = md[node.position?.start.offset ?? 0];
+ if (opener !== '`' && opener !== '~') {
+ indented.push(`${slug}: line ${node.position?.start.line}`);
+ }
+ });
+ }
+ expect(indented).toEqual([]);
+ });
+
+ test('no page carries a base64 component attribute', () => {
+ for (const [slug, md] of output) {
+ expect(md, slug).not.toContain('<HtmlPreview');
+ }
+ });
+
+ test('the quickstart serves the download builds its CTA stands for', () => {
+ const md = output.get('get-started/quickstart') ?? '';
+ expect(md).toContain(DOWNLOAD_TARGETS[0].assetUrl);
+ expect(md).toContain('### Install the desktop app');
+ });
+
+ test('every page embedding the shared install block serves its steps', () => {
+ const embedding = pages.filter((page) => page.source.includes('<McpInstall'));
+ expect(embedding.length).toBeGreaterThan(5);
+ for (const page of embedding) {
+ expect(output.get(page.slug), page.slug).toContain('depending on how you run OpenKnowledge');
+ }
+ });
+
+ test('every page embedding the shared verification block serves its prompt', () => {
+ const embedding = pages.filter((page) => page.source.includes('<VerifyExec'));
+ expect(embedding.length).toBeGreaterThan(5);
+ for (const page of embedding) {
+ expect(output.get(page.slug), page.slug).toContain(
+ '> List the first 5 documents you come across in this project.',
+ );
+ }
+ });
+
+ test('the overview serves the three layers and the start paths', () => {
+ const md = output.get('get-started/overview') ?? '';
+ for (const layer of LAYERS) expect(md).toContain(layer.desc);
+ for (const path of PATHS) expect(md).toContain(path.desc);
+ expect(md).toContain(AGENTS[0].name);
+ });
+
+ test('every authored preview fence survives as a fence, not as a base64 attribute', () => {
+ const authored = pages.filter((page) => page.source.includes('```html preview'));
+ expect(authored.length).toBeGreaterThan(5);
+ for (const page of authored) {
+ expect(output.get(page.slug), page.slug).toContain('```html preview');
+ }
+ });
+
+ test('the mermaid page serves its diagram as a mermaid fence', () => {
+ expect(output.get('reference/components/mermaid-fence') ?? '').toContain(
+ '```mermaid\ngraph LR',
+ );
+ });
+
+ test('every prop table on a component reference page survives as a table', () => {
+ const withTypeTable = pages.filter((page) => page.source.includes('<TypeTable'));
+ expect(withTypeTable.length).toBeGreaterThan(10);
+ for (const page of withTypeTable) {
+ expect(output.get(page.slug), page.slug).toContain('| Prop | Type | Required |');
+ }
+ });
+});
+
+describe('an attribute the author wrote as an expression', () => {
+ test.each([
+ ['<McpInstall editor={EDITOR} />', 'editor'],
+ ['<VerifyExec subject={AGENT} />', 'subject'],
+ ['<Callout title={TITLE}>body</Callout>', 'title'],
+ ])('%s is refused rather than replaced by a fallback', (source, name) => {
+ expect(() => serialize(source)).toThrow(new RegExp(`\`${name}\` is an expression`));
+ });
+
+ test('a bare attribute still reads as absent, since it carries no text', () => {
+ expect(() =>
+ serialize('<Tabs persist items={[{ title: "A", content: "b" }]} />'),
+ ).not.toThrow();
+ });
+});
diff --git a/docs/src/lib/mdx-serializer-registry.ts b/docs/src/lib/mdx-serializer-registry.ts
new file mode 100644
index 000000000..5a93443db
--- /dev/null
+++ b/docs/src/lib/mdx-serializer-registry.ts
@@ -0,0 +1,340 @@
+import { visit } from 'unist-util-visit';
+import { AGENTS } from '@/components/agent-icons';
+import { mcpInstallMarkdown } from '@/components/mcp-install';
+import { LAYERS, PATHS } from '@/components/overview-blocks';
+import { verifyExecMarkdown } from '@/components/verify-exec';
+import {
+ DOWNLOAD_PAGE_HREF,
+ DOWNLOAD_TARGETS,
+ WEB_APP_HREF,
+ WEB_APP_LABEL,
+} from '@/lib/download-targets';
+import { parseAttributeLiteral } from '@/lib/mdx-attribute-literal';
+import {
+ type ComponentDisposition,
+ escapeInlineProse,
+ inlineCodeSpan,
+ longestBacktickRun,
+ type MdxJsxNode,
+ markdownLink,
+ requiredLiteral,
+ type SerializerApi,
+ type SerializerRegistry,
+} from '@/lib/mdx-serializer';
+
+function cellSeparators(value: string): string {
+ return value.replace(/\|/g, '\\|').replace(/\r?\n/g, ' ').trim();
+}
+
+function tableCell(value: string): string {
+ return cellSeparators(escapeInlineProse(value));
+}
+
+function codeCell(value: string): string {
+ return cellSeparators(code(value));
+}
+
+const code = inlineCodeSpan;
+
+function titledBlockquote(lead: string | undefined, body: string): string {
+ const lines = [...(lead ? [`**${escapeInlineProse(lead)}**`, ''] : []), ...body.split('\n')];
+ return lines.map((line) => (line === '' ? '>' : `> ${line}`)).join('\n');
+}
+
+const callout: ComponentDisposition = (node, api) => {
+ const type = api.attribute(node, 'type');
+ const lead =
+ api.attribute(node, 'title') ??
+ (type ? type.charAt(0).toUpperCase() + type.slice(1) : undefined);
+ return titledBlockquote(lead, api.children(node));
+};
+
+const accordion: ComponentDisposition = (node, api) => {
+ const description = api.attribute(node, 'description');
+ const body = [
+ ...(description ? [escapeInlineProse(description), ''] : []),
+ api.children(node),
+ ].join('\n');
+ return titledBlockquote(api.attribute(node, 'title'), body);
+};
+
+function tabHeadingDepth(tabs: readonly MdxJsxNode[]): number {
+ let shallowest = Number.POSITIVE_INFINITY;
+ for (const tab of tabs) {
+ visit(tab, 'heading', (heading) => {
+ shallowest = Math.min(shallowest, heading.depth);
+ });
+ }
+ return Number.isFinite(shallowest) ? Math.max(2, shallowest - 1) : 3;
+}
+
+function tabLabels(node: MdxJsxNode): string[] {
+ const source = expressionSource(node, 'items');
+ if (source === undefined) return [];
+ const items = parseAttributeLiteral(source);
+ if (!Array.isArray(items)) {
+ throw new Error(`<${node.name} items={…}> must be an array of labels.`);
+ }
+ return items.map(String);
+}
+
+function expressionSource(node: MdxJsxNode, name: string): string | undefined {
+ const found = node.attributes.find(
+ (attribute) => attribute.type === 'mdxJsxAttribute' && attribute.name === name,
+ );
+ const value = found?.value;
+ return typeof value === 'object' && value !== null ? value.value : undefined;
+}
+
+function isElement(node: unknown, ...names: string[]): node is MdxJsxNode {
+ const typed = node as { type?: string; name?: string };
+ return (
+ (typed?.type === 'mdxJsxFlowElement' || typed?.type === 'mdxJsxTextElement') &&
+ names.includes(typed.name ?? '')
+ );
+}
+
+function collectItems(
+ node: { children?: unknown[] },
+ item: string,
+ container: string,
+): MdxJsxNode[] {
+ const found: MdxJsxNode[] = [];
+ for (const child of node.children ?? []) {
+ if (isElement(child, item)) found.push(child);
+ else if (!isElement(child, container) && typeof child === 'object' && child !== null) {
+ found.push(...collectItems(child as { children?: unknown[] }, item, container));
+ }
+ }
+ return found;
+}
+
+function tabs(...childNames: string[]): ComponentDisposition {
+ return (node, api) => {
+ const labels = tabLabels(node);
+ const panels = childNames.flatMap((name) => collectItems(node, name, node.name ?? ''));
+ const depth = '#'.repeat(tabHeadingDepth(panels));
+ return panels
+ .map((panel, index) => {
+ const label =
+ api.attribute(panel, 'value') ??
+ api.attribute(panel, 'label') ??
+ labels[index] ??
+ `Tab ${index + 1}`;
+ return `${depth} ${escapeInlineProse(label)}\n\n${api.children(panel)}`;
+ })
+ .join('\n\n');
+ };
+}
+
+const tabPanel: ComponentDisposition = (node, api) => {
+ const label = api.attribute(node, 'value') ?? api.attribute(node, 'label');
+ const body = api.children(node);
+ return label ? `**${escapeInlineProse(label)}**\n\n${body}` : body;
+};
+
+function card(node: MdxJsxNode, api: SerializerApi): string {
+ const title = api.attribute(node, 'title') ?? api.attribute(node, 'name') ?? '';
+ const href = api.attribute(node, 'href');
+ const lead = href
+ ? markdownLink(escapeInlineProse(title), api.url(href))
+ : `**${escapeInlineProse(title)}**`;
+ const body = api.children(node);
+
+ if (/\n[ \t]*\n/.test(body) || /^ {0,3}(```|~~~)/m.test(body)) return `${lead}\n\n${body}`;
+ const inline = body.replace(/\s*\n\s*/g, ' ').trim();
+ return `- ${lead}${inline ? `: ${inline}` : ''}`;
+}
+
+const cards: ComponentDisposition = (node, api) => {
+ const rendered = collectItems(node, 'Card', 'Cards').map((child) => card(child, api));
+ const everyCardIsALine = rendered.every((entry) => entry.startsWith('- '));
+ return rendered.join(everyCardIsALine ? '\n' : '\n\n');
+};
+
+function fence(info: string, body: string): string {
+ const marker = '`'.repeat(Math.max(2, longestBacktickRun(body)) + 1);
+ return `${marker}${info}\n${body}\n${marker}`;
+}
+
+const htmlPreview: ComponentDisposition = (node, api) => {
+ const encoded = api.attribute(node, 'code');
+ if (!encoded)
+ throw new Error('<HtmlPreview> carries its block in a `code` attribute, which is missing.');
+ const html = Buffer.from(encoded, 'base64').toString('utf8');
+ const canonical = (value: string) =>
+ value.replace(/\s+/g, '').replaceAll('-', '+').replaceAll('_', '/').replace(/=+$/, '');
+ if (canonical(Buffer.from(html, 'utf8').toString('base64')) !== canonical(encoded)) {
+ throw new Error('<HtmlPreview> `code` is not valid base64.');
+ }
+ return fence('html preview', html.replace(/\n$/, ''));
+};
+
+const mermaid: ComponentDisposition = (node, api) => {
+ const expression = api.expressionAttribute(node, 'chart');
+ const chart =
+ expression === undefined ? api.attribute(node, 'chart') : parseAttributeLiteral(expression);
+ if (typeof chart !== 'string') throw new Error('<Mermaid> needs a `chart` string.');
+ return fence('mermaid', chart.replaceAll('\\n', '\n').trim());
+};
+
+const typeTable: ComponentDisposition = (node, api) => {
+ const source = api.expressionAttribute(node, 'type');
+ if (!source) {
+ throw new Error(
+ api.attribute(node, 'type') === undefined
+ ? '<TypeTable> carries its rows in a `type` object, which is missing.'
+ : '<TypeTable> `type` must be a braced object expression, not a quoted string.',
+ );
+ }
+ const props = parseAttributeLiteral(source);
+ if (typeof props !== 'object' || props === null || Array.isArray(props)) {
+ throw new Error('<TypeTable type={…}> must be an object of prop descriptors.');
+ }
+
+ const rows = Object.entries(props).map(([name, descriptor]) => {
+ if (typeof descriptor !== 'object' || descriptor === null || Array.isArray(descriptor)) {
+ throw new Error(`<TypeTable type={…}> descriptor for \`${name}\` must be an object.`);
+ }
+ const field = (key: string) => {
+ const value = descriptor[key];
+ if (value === undefined || value === null) return '';
+ if (typeof value === 'object') {
+ throw new Error(
+ `<TypeTable type={…}> \`${name}.${key}\` must be a string, number or boolean.`,
+ );
+ }
+ return String(value);
+ };
+ return `| ${codeCell(name)} | ${codeCell(field('type'))} | ${
+ descriptor.required ? 'yes' : 'no'
+ } | ${codeCell(field('default'))} | ${tableCell(field('description'))} |`;
+ });
+
+ return [
+ '| Prop | Type | Required | Default | Description |',
+ '| --- | --- | --- | --- | --- |',
+ ...rows,
+ ].join('\n');
+};
+
+const downloadButton: ComponentDisposition = (_node, api) => {
+ const builds = DOWNLOAD_TARGETS.map(
+ (target) => `- ${markdownLink(target.label, target.assetUrl)}`,
+ );
+ return [
+ ...builds,
+ `- ${markdownLink(WEB_APP_LABEL, api.url(WEB_APP_HREF))}`,
+ `- ${markdownLink('All builds and checksums', DOWNLOAD_PAGE_HREF)}`,
+ ].join('\n');
+};
+
+const layerStack: ComponentDisposition = () =>
+ LAYERS.map(
+ (layer, index) => `${index + 1}. **${layer.title}** — ${layer.role}. ${layer.desc}`,
+ ).join('\n');
+
+const whereToStart: ComponentDisposition = (_node, api) =>
+ PATHS.map(
+ (path, index) => `${index + 1}. ${markdownLink(path.title, api.url(path.href))} — ${path.desc}`,
+ ).join('\n');
+
+const agentIcons: ComponentDisposition = () => AGENTS.map((agent) => agent.name).join(', ');
+
+const iconLabel: ComponentDisposition = (node, api) => {
+ const label = api.attribute(node, 'aria-label') ?? api.attribute(node, 'label');
+ return label === undefined ? null : escapeInlineProse(label);
+};
+
+const image: ComponentDisposition = (node, api) => {
+ const src = requiredLiteral(node, api, 'src');
+ return markdownLink(escapeInlineProse(api.attribute(node, 'alt') ?? ''), api.url(src), {
+ image: true,
+ });
+};
+
+const anchor: ComponentDisposition = (node, api) => {
+ const href = api.attribute(node, 'href');
+ const label = api.children(node);
+ return href ? markdownLink(label, api.url(href)) : label;
+};
+
+function heading(depth: number): ComponentDisposition {
+ return (node, api) => `${'#'.repeat(depth)} ${api.children(node)}`;
+}
+
+export const DOCS_SERIALIZER_REGISTRY: SerializerRegistry = {
+ Accordion: accordion,
+ AccordionPreview: accordion,
+ AgentIcons: agentIcons,
+ Callout: callout,
+ CalloutPreview: callout,
+ Card: card,
+ Cards: cards,
+ CopyPrompt: (node, api) => titledBlockquote(undefined, api.children(node)),
+ CtaButton: (node, api) => {
+ const href = api.attribute(node, 'href');
+ const label = escapeInlineProse(api.attribute(node, 'label') ?? href ?? '');
+ return href ? markdownLink(label, api.url(href)) : label;
+ },
+ DownloadButton: downloadButton,
+ HtmlPreview: htmlPreview,
+ Image: image,
+ LayerStack: layerStack,
+ McpInstall: (node, api) => {
+ const editor = escapeInlineProse(api.attribute(node, 'editor') ?? 'your editor');
+ const notes = api.children(node);
+ return [
+ mcpInstallMarkdown(editor, { url: api.url, link: markdownLink }),
+ ...(notes ? ['', notes] : []),
+ ].join('\n');
+ },
+ Mermaid: mermaid,
+ Tab: tabPanel,
+ TabPreview: tabPanel,
+ Tabs: tabs('Tab'),
+ TabsPreview: tabs('TabPreview'),
+ TypeTable: typeTable,
+ VerifyExec: (node, api) =>
+ verifyExecMarkdown(escapeInlineProse(api.attribute(node, 'subject') ?? 'The agent')),
+ WhereToStart: whereToStart,
+
+ Accordions: 'flatten',
+ CalloutContainer: 'flatten',
+ CalloutDescription: 'flatten',
+ CalloutTitle: 'flatten',
+ CodeBlockTab: 'flatten',
+ CodeBlockTabs: 'flatten',
+ CodeBlockTabsList: 'flatten',
+ CodeBlockTabsTrigger: 'flatten',
+ ComponentPreview: 'flatten',
+ MirrorPreview: 'flatten',
+ MirrorSourcePreview: 'flatten',
+ Step: 'flatten',
+ Steps: 'flatten',
+
+ AudioPreview: 'drop',
+ EmbedPreview: 'drop',
+ FilePreview: 'drop',
+ ImgPreview: 'drop',
+ MathPreview: 'drop',
+ PdfPreview: 'drop',
+ VideoPreview: 'drop',
+
+ Clock: iconLabel,
+ Cloud: iconLabel,
+ InlineIcon: iconLabel,
+ Undo2: iconLabel,
+
+ a: anchor,
+ code: (node, api) => code(api.text(node)),
+ h1: heading(1),
+ h2: heading(2),
+ h3: heading(3),
+ h4: heading(4),
+ h5: heading(5),
+ h6: heading(6),
+ img: image,
+ pre: 'flatten',
+ table: 'flatten',
+};
diff --git a/docs/src/lib/mdx-serializer.test.ts b/docs/src/lib/mdx-serializer.test.ts
new file mode 100644
index 000000000..9a680d783
--- /dev/null
+++ b/docs/src/lib/mdx-serializer.test.ts
@@ -0,0 +1,522 @@
+import { readdirSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+import { describe, expect, test } from 'vitest';
+import {
+ componentNames,
+ escapeInlineProse,
+ escapeRawHtml,
+ markdownLink,
+ type SerializerRegistry,
+ STRINGIFY_OPTIONS,
+ serializeMdx,
+} from './mdx-serializer.ts';
+
+const PAGE_URL = 'https://openknowledge.ai/docs/get-started/quickstart';
+
+function body(source: string, registry: SerializerRegistry = {}): string {
+ return serializeMdx(source, { registry, pageUrl: PAGE_URL }).body;
+}
+
+describe('serializeMdx', () => {
+ test('serializes a page with frontmatter, a wrapper component, and a relative link', () => {
+ const { frontmatter, body: md } = serializeMdx(
+ [
+ '---',
+ 'title: "Quickstart"',
+ 'description: Get running in five minutes',
+ '---',
+ '',
+ '<Callout type="tip">',
+ ' Read the [editor guide](../features/editor.mdx#ask-ai) first.',
+ '</Callout>',
+ '',
+ '- one',
+ '- two',
+ '',
+ ].join('\n'),
+ { registry: { Callout: 'flatten' }, pageUrl: PAGE_URL },
+ );
+
+ expect(frontmatter).toEqual({
+ title: 'Quickstart',
+ description: 'Get running in five minutes',
+ });
+ expect(md).toBe(
+ [
+ 'Read the [editor guide](https://openknowledge.ai/docs/features/editor#ask-ai) first.',
+ '',
+ '- one',
+ '- two',
+ ].join('\n'),
+ );
+ });
+});
+
+describe('component dispositions', () => {
+ test('a component with no disposition fails loudly and names the fix', () => {
+ expect(() => body('<Widget>content</Widget>')).toThrow(
+ /No serializer disposition for <Widget>.*Add "Widget" to the registry/s,
+ );
+ });
+
+ test.each([
+ 'toString',
+ 'constructor',
+ 'valueOf',
+ ])('a name Object.prototype carries (%s) is refused, not resolved through the prototype', (name) => {
+ expect(() => body(`<${name} />`)).toThrow(/No serializer disposition/);
+ });
+
+ test('flatten keeps the children and discards the tag', () => {
+ expect(body('<Wrapper>\n\nkept text\n\n</Wrapper>', { Wrapper: 'flatten' })).toBe('kept text');
+ });
+
+ test('flatten reaches components nested inside other components', () => {
+ const md = body('<Outer>\n\n<Inner>deep text</Inner>\n\n</Outer>', {
+ Outer: 'flatten',
+ Inner: 'flatten',
+ });
+ expect(md).toBe('deep text');
+ });
+
+ test('drop removes the component and everything inside it', () => {
+ const md = body('before\n\n<Nav>\n\nnavigation\n\n</Nav>\n\nafter', { Nav: 'drop' });
+ expect(md).toBe('before\n\nafter');
+ });
+
+ test('a custom serializer emits its markdown verbatim', () => {
+ const md = body('<Mermaid chart="graph TD; A-->B" />', {
+ Mermaid: (node, api) => `\`\`\`mermaid\n${api.attribute(node, 'chart')}\n\`\`\``,
+ });
+ expect(md).toBe('```mermaid\ngraph TD; A-->B\n```');
+ });
+
+ test('a custom serializer can render the children it was given', () => {
+ const md = body('<Callout title="Heads up">\n\nBe careful.\n\n</Callout>', {
+ Callout: (node, api) => `> **${api.attribute(node, 'title')}**\n>\n> ${api.children(node)}`,
+ });
+ expect(md).toBe('> **Heads up**\n>\n> Be careful.');
+ });
+
+ test('a custom serializer reads expression attributes as source', () => {
+ let seen: string | undefined;
+ body("<Tabs items={['CLI', 'UI']}>\n\ncontent\n\n</Tabs>", {
+ Tabs: (node, api) => {
+ seen = api.expressionAttribute(node, 'items');
+ return api.children(node);
+ },
+ });
+ expect(seen).toBe("['CLI', 'UI']");
+ });
+
+ test('a custom serializer returning null drops the component', () => {
+ expect(body('kept\n\n<Aside>gone</Aside>', { Aside: () => null })).toBe('kept');
+ });
+
+ test('a fragment contributes its children without a registry entry', () => {
+ expect(body('<>\n\nfragment text\n\n</>')).toBe('fragment text');
+ });
+
+ test('imports and expressions carry no prose and are removed', () => {
+ const md = body("import { Clock } from 'lucide-react';\n\ntext\n\n{ 2 + 2 }");
+ expect(md).toBe('text');
+ });
+});
+
+describe('frontmatter', () => {
+ test('is YAML-parsed, so quoting never leaks into a value', () => {
+ const { frontmatter } = serializeMdx('---\ntitle: "Mirror"\n---\n\nbody\n', {
+ registry: {},
+ pageUrl: PAGE_URL,
+ });
+ expect(frontmatter.title).toBe('Mirror');
+ });
+
+ test('is stripped before the MDX parse, so raw HTML in a description is inert', () => {
+ const source = [
+ '---',
+ 'title: "Accordion"',
+ 'description: "Expand/collapse via native HTML5 <details>/<summary>."',
+ '---',
+ '',
+ 'Prose.',
+ '',
+ ].join('\n');
+ const { frontmatter, body: md } = serializeMdx(source, { registry: {}, pageUrl: PAGE_URL });
+ expect(frontmatter.description).toContain('<details>/<summary>');
+ expect(md).toBe('Prose.');
+ });
+
+ test('a source with no frontmatter yields an empty mapping', () => {
+ const { frontmatter, body: md } = serializeMdx('Just prose.\n', {
+ registry: {},
+ pageUrl: PAGE_URL,
+ });
+ expect(frontmatter).toEqual({});
+ expect(md).toBe('Just prose.');
+ });
+
+ test('frontmatter that is not a mapping fails loudly', () => {
+ expect(() =>
+ serializeMdx('---\n- a\n- b\n---\n\nbody\n', { registry: {}, pageUrl: PAGE_URL }),
+ ).toThrow(/Frontmatter must be a YAML mapping/);
+ });
+});
+
+describe('link rewriting', () => {
+ test.each([
+ [
+ 'parent-relative source link',
+ '[a](../features/editor.mdx)',
+ 'https://openknowledge.ai/docs/features/editor',
+ ],
+ [
+ 'sibling-relative source link',
+ '[a](./overview.mdx)',
+ 'https://openknowledge.ai/docs/get-started/overview',
+ ],
+ [
+ 'a .md sibling',
+ '[a](../reference/core-concepts.md)',
+ 'https://openknowledge.ai/docs/reference/core-concepts',
+ ],
+ [
+ 'a fragment on a relative link',
+ '[a](../features/editor.mdx#ask-ai)',
+ 'https://openknowledge.ai/docs/features/editor#ask-ai',
+ ],
+ ['a root-relative link', '[a](/download)', 'https://openknowledge.ai/download'],
+ [
+ 'a bare fragment',
+ '[a](#section)',
+ 'https://openknowledge.ai/docs/get-started/quickstart#section',
+ ],
+ ['an image', '![a](/screenshots/x.png)', 'https://openknowledge.ai/screenshots/x.png'],
+ ])('absolutises %s', (_label, source, expected) => {
+ expect(body(source)).toContain(`(${expected})`);
+ });
+
+ test.each([
+ ['an external URL that happens to end in .md', 'https://github.com/o/r/blob/main/SPEC.md'],
+ ['a mailto link', 'mailto:[email protected]'],
+ ['a protocol-relative URL', '//cdn.example.com/x.png'],
+ ])('leaves %s untouched', (_label, url) => {
+ expect(body(`[a](${url})`)).toContain(`(${url})`);
+ });
+
+ test('rewrites a link reference definition', () => {
+ expect(body('[a][ref]\n\n[ref]: ../features/editor.mdx\n')).toContain(
+ 'https://openknowledge.ai/docs/features/editor',
+ );
+ });
+
+ test('rewrites links inside a component that is flattened away', () => {
+ const md = body('<Callout>\n\n[a](../features/editor.mdx)\n\n</Callout>', {
+ Callout: 'flatten',
+ });
+ expect(md).toContain('https://openknowledge.ai/docs/features/editor');
+ });
+});
+
+describe('pinned stringify options', () => {
+ test('are the exact set both apps depend on', () => {
+ expect(STRINGIFY_OPTIONS).toEqual({ bullet: '-', rule: '-', listItemIndent: 'one' });
+ });
+
+ test('emit hyphen bullets, hyphen rules, and single-space list indent', () => {
+ const md = body('* one\n * nested\n\n***\n');
+ expect(md).toBe('- one\n - nested\n\n---');
+ });
+
+ test('never indent a code block to the four-space column', () => {
+ const md = body('```\nplain fence\n```\n');
+ expect(md).toBe('```\nplain fence\n```');
+ });
+});
+
+describe('escapeRawHtml', () => {
+ test('turns a tag named in prose into the words it reads as', () => {
+ expect(escapeRawHtml('Standalone expand/collapse via native HTML5 <details>/<summary>.')).toBe(
+ 'Standalone expand/collapse via native HTML5 \\<details>/\\<summary>.',
+ );
+ });
+
+ test('leaves a tag named inside a code span alone, where it is already inert', () => {
+ const prose = 'Each `<Tab>` child is one panel.';
+ expect(escapeRawHtml(prose)).toBe(prose);
+ });
+
+ test('keeps the inline formatting the author wrote', () => {
+ const prose = 'Prefer **the editor**, or read [the guide](https://openknowledge.ai/docs).';
+ expect(escapeRawHtml(prose)).toBe(prose);
+ });
+
+ test('gives block-level markup a paragraph of its own', () => {
+ expect(escapeRawHtml('<div align="center">\n\nCentred.\n\n</div>')).toBe(
+ '\\<div align="center">\n\nCentred.\n\n\\</div>',
+ );
+ });
+
+ test('leaves prose carrying no markup byte-identical', () => {
+ const prose = 'What OpenKnowledge is, and why it exists.';
+ expect(escapeRawHtml(prose)).toBe(prose);
+ });
+});
+
+describe('componentNames', () => {
+ test('names every element, component and lowercase HTML alike', () => {
+ expect(componentNames('<Callout>text</Callout>\n\n<code>x</code>\n').sort()).toEqual([
+ 'Callout',
+ 'code',
+ ]);
+ });
+
+ test('ignores an element inside a fenced code block', () => {
+ expect(componentNames('```mdx\n<Callout>example</Callout>\n```\n')).toEqual([]);
+ });
+
+ test('ignores an element inside an inline code span', () => {
+ expect(componentNames('Write `<Callout>` to add one.\n')).toEqual([]);
+ });
+
+ test('ignores markup in frontmatter', () => {
+ const source = ['---', 'description: The <details> element', '---', '', 'Body.', ''].join('\n');
+ expect(componentNames(source)).toEqual([]);
+ });
+
+ test('reaches nested and inline elements, and names each once', () => {
+ const source = '<Tabs>\n <Tab>a <InlineIcon /> b</Tab>\n <Tab>c</Tab>\n</Tabs>\n';
+ expect(componentNames(source).sort()).toEqual(['InlineIcon', 'Tab', 'Tabs']);
+ });
+
+ test('ignores a fragment, which has no name to look up', () => {
+ expect(componentNames('<>\n <Card>a</Card>\n</>\n')).toEqual(['Card']);
+ });
+});
+
+describe('the docs content corpus', () => {
+ const CONTENT_ROOT = fileURLToPath(new URL('../../content/', import.meta.url));
+ const FLATTEN_EVERY_COMPONENT = new Proxy(
+ {},
+ {
+ get: () => 'flatten',
+ getOwnPropertyDescriptor: () => ({
+ value: 'flatten',
+ configurable: true,
+ enumerable: true,
+ writable: true,
+ }),
+ },
+ ) as SerializerRegistry;
+
+ function corpusFiles(dir = CONTENT_ROOT): string[] {
+ return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) return corpusFiles(full);
+ return /\.mdx?$/.test(entry.name) ? [full] : [];
+ });
+ }
+
+ function pageUrlFor(file: string): string {
+ const slug = path
+ .relative(CONTENT_ROOT, file)
+ .replace(/\.mdx?$/, '')
+ .replace(/(^|\/)index$/, '');
+ return `https://openknowledge.ai/docs/${slug}`.replace(/\/$/, '');
+ }
+
+ const files = corpusFiles();
+
+ function hrefsIn(markdown: string): string[] {
+ const tree = unified().use(remarkParse).use(remarkGfm).parse(markdown);
+ const urls: string[] = [];
+ visit(tree, (node) => {
+ if (node.type === 'link' || node.type === 'image' || node.type === 'definition') {
+ urls.push(node.url);
+ }
+ });
+ return urls;
+ }
+
+ test('is non-empty, so a broken glob cannot vacuously pass this suite', () => {
+ expect(files.length).toBeGreaterThan(50);
+ });
+
+ test('serializes every page with no parse failures and no source-form links', async () => {
+ const failures: string[] = [];
+
+ await Promise.all(
+ files.map(async (file) => {
+ const rel = path.relative(CONTENT_ROOT, file);
+ let md: string;
+ try {
+ md = serializeMdx(await readFile(file, 'utf8'), {
+ registry: FLATTEN_EVERY_COMPONENT,
+ pageUrl: pageUrlFor(file),
+ }).body;
+ } catch (error) {
+ failures.push(`${rel}: ${(error as Error).message}`);
+ return;
+ }
+ for (const href of hrefsIn(md)) {
+ if (href.startsWith('../') || href.startsWith('./')) {
+ failures.push(`${rel}: relative href survived (${href})`);
+ }
+ if (/\.mdx($|[#?])/.test(href)) {
+ failures.push(`${rel}: source-form .mdx href survived (${href})`);
+ }
+ }
+ }),
+ );
+
+ expect(failures).toEqual([]);
+ });
+});
+
+describe('a refusal names the page it came from', () => {
+ test('prefixes the page URL, so the corpus route does not report 68 candidates', () => {
+ expect(() =>
+ serializeMdx('<Unknown />', {
+ registry: {},
+ pageUrl: 'https://openknowledge.ai/docs/x/y',
+ }),
+ ).toThrow('https://openknowledge.ai/docs/x/y: No serializer disposition for <Unknown>');
+ });
+
+ test('keeps the original refusal reachable as the cause', () => {
+ const thrown = (() => {
+ try {
+ serializeMdx('<Unknown />', { registry: {}, pageUrl: 'https://openknowledge.ai/docs/x' });
+ } catch (error) {
+ return error as Error;
+ }
+ })();
+
+ expect((thrown?.cause as Error).message).toBe(
+ "No serializer disposition for <Unknown>. Add \"Unknown\" to the registry as 'flatten', 'drop', or a custom serializer.",
+ );
+ });
+});
+
+describe('markdownLink', () => {
+ const parse = unified().use(remarkParse).use(remarkGfm);
+
+ function linkIn(markdown: string): { url: string; text: string } | undefined {
+ let found: { url: string; text: string } | undefined;
+ visit(parse.parse(markdown), 'link', (node) => {
+ const words: string[] = [];
+ visit(node, (child) => {
+ if (child.type === 'text' || child.type === 'inlineCode') words.push(child.value);
+ });
+ found = { url: node.url, text: words.join('') };
+ });
+ return found;
+ }
+
+ test.each([
+ ['an unbalanced bracket in the label', 'Unmatched ] here', 'https://x.test/a.md'],
+ ['balanced brackets in the label', 'Array[0] access', 'https://x.test/a.md'],
+ ['a parenthesis in the destination', 'Note', 'https://x.test/blog/note-)-end.md'],
+ ['a space in the destination', 'Note', 'https://x.test/blog/pre post.md'],
+ ])('round-trips %s', (_name, label, url) => {
+ expect(linkIn(markdownLink(label, url))).toEqual({ url, text: label });
+ });
+
+ test.each([
+ ['an ordered-list marker', '1. Install the CLI'],
+ ['a heading marker', '# Overview'],
+ ['a bullet marker', '- Quickstart'],
+ ['a blockquote marker', '> Note'],
+ ['a thematic break', '---'],
+ ])('keeps a label opening with %s', (_name, label) => {
+ expect(linkIn(markdownLink(label, 'https://x.test/a.md'))).toEqual({
+ url: 'https://x.test/a.md',
+ text: label,
+ });
+ });
+
+ test.each([
+ 'Install claude_desktop_config.json',
+ '1. Install claude_desktop_config.json',
+ 'Use [brackets] here',
+ '1. Use [brackets] here',
+ 'Set up ~/.codex/config.toml',
+ '- Set up ~/.codex/config.toml',
+ 'Read the C++ <T> guide',
+ '1. Read the C++ <T> guide',
+ '# Roadmap',
+ ])('serves %s as written through the production composition', (authored) => {
+ expect(linkIn(markdownLink(escapeInlineProse(authored), 'https://x.test/a.md'))).toEqual({
+ url: 'https://x.test/a.md',
+ text: authored,
+ });
+ });
+
+ test('folds a label that parses to several blocks onto its one line', () => {
+ expect(linkIn(markdownLink('para one\n\npara two', 'https://x.test/a.md'))).toEqual({
+ url: 'https://x.test/a.md',
+ text: 'para one para two',
+ });
+ });
+
+ test('carries a block-opening label into the image branch too', () => {
+ let alt: string | undefined;
+ visit(
+ parse.parse(markdownLink('1. The editor', 'https://x.test/h.png', { image: true })),
+ 'image',
+ (node) => {
+ alt = node.alt ?? undefined;
+ },
+ );
+ expect(alt).toBe('1. The editor');
+ });
+
+ test('leaves a code span in the label byte-identical', () => {
+ const rendered = markdownLink('Use ``a`b]c`` today', 'https://x.test/a.md');
+
+ expect(rendered).toBe('[Use ``a`b]c`` today](https://x.test/a.md)');
+ expect(rendered).not.toContain('\\]');
+ });
+
+ test('flattens a nested link, which CommonMark forbids in a label', () => {
+ const rendered = markdownLink('see [the docs](https://x.test/docs)', 'https://x.test/a.md');
+ expect(linkIn(rendered)).toEqual({ url: 'https://x.test/a.md', text: 'see the docs' });
+ });
+
+ test('renders an image when asked, with the label as alt text', () => {
+ let alt: string | undefined;
+ let url: string | undefined;
+ visit(
+ parse.parse(markdownLink('The editor', 'https://x.test/hero.png', { image: true })),
+ 'image',
+ (n) => {
+ alt = n.alt ?? undefined;
+ url = n.url;
+ },
+ );
+ expect({ alt, url }).toEqual({ alt: 'The editor', url: 'https://x.test/hero.png' });
+ });
+});
+
+describe('one escaping rule for author prose placed into authored Markdown', () => {
+ const parse = unified().use(remarkParse).use(remarkGfm);
+
+ test.each([
+ ['raw HTML a writer names in passing', 'the <summary> element', 'the <summary> element'],
+ ['a code span containing a tag', 'Use the `<b>` tag', 'Use the <b> tag'],
+ ])('round-trips %s', (_name, prose, expected) => {
+ const label = escapeInlineProse(prose);
+ const words: string[] = [];
+ visit(parse.parse(markdownLink(label, 'https://x.test/a.md')), (node) => {
+ if (node.type === 'text' || node.type === 'inlineCode') words.push(node.value);
+ });
+
+ expect(words.join('')).toBe(expected);
+ });
+});
diff --git a/docs/src/lib/mdx-serializer.ts b/docs/src/lib/mdx-serializer.ts
new file mode 100644
index 000000000..ed123aa66
--- /dev/null
+++ b/docs/src/lib/mdx-serializer.ts
@@ -0,0 +1,301 @@
+import type { Html, PhrasingContent, Root, RootContent } from 'mdast';
+import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import remarkParse from 'remark-parse';
+import remarkStringify from 'remark-stringify';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+import { parse as parseYaml } from 'yaml';
+
+export type ComponentDisposition =
+ | 'flatten'
+ | 'drop'
+ | ((node: MdxJsxNode, api: SerializerApi) => string | null);
+
+export type SerializerRegistry = Readonly<Record<string, ComponentDisposition>>;
+
+export type MdxJsxNode = MdxJsxFlowElement | MdxJsxTextElement;
+
+export interface SerializerApi {
+ children(node: MdxJsxNode): string;
+ text(node: MdxJsxNode): string;
+ attribute(node: MdxJsxNode, name: string): string | undefined;
+ expressionAttribute(node: MdxJsxNode, name: string): string | undefined;
+ url(href: string): string;
+}
+
+export interface SerializeMdxOptions {
+ registry: SerializerRegistry;
+ pageUrl: string;
+}
+
+export interface SerializedMdx {
+ frontmatter: Record<string, unknown>;
+ body: string;
+}
+
+export const STRINGIFY_OPTIONS = {
+ bullet: '-',
+ rule: '-',
+ listItemIndent: 'one',
+} as const;
+
+const processor = unified()
+ .use(remarkParse)
+ .use(remarkMdx)
+ .use(remarkGfm)
+ .use(remarkStringify, STRINGIFY_OPTIONS);
+
+const proseProcessor = unified()
+ .use(remarkParse)
+ .use(remarkGfm)
+ .use(remarkStringify, STRINGIFY_OPTIONS);
+
+const FRONTMATTER_BLOCK = /^---[ \t]*\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)/;
+
+const ABSOLUTE_URL = /^[a-zA-Z][a-zA-Z0-9+.-]*:|^\/\//;
+
+const SOURCE_EXTENSION = /\.mdx?$/;
+
+const NON_PROSE_TYPES = new Set(['mdxjsEsm', 'mdxFlowExpression', 'mdxTextExpression']);
+
+export function serializeMdx(source: string, options: SerializeMdxOptions): SerializedMdx {
+ const { frontmatter, body } = splitFrontmatter(source);
+ const base = new URL(options.pageUrl);
+
+ const tree = processor.parse(body);
+ absolutiseUrls(tree, base);
+ try {
+ tree.children = transformChildren(tree, options.registry, base);
+ } catch (error) {
+ throw new Error(`${options.pageUrl}: ${error instanceof Error ? error.message : error}`, {
+ cause: error,
+ });
+ }
+
+ return { frontmatter, body: processor.stringify(tree).trimEnd() };
+}
+
+export function componentNames(source: string): string[] {
+ const names = new Set<string>();
+ visit(processor.parse(splitFrontmatter(source).body), (node) => {
+ if (node.type !== 'mdxJsxFlowElement' && node.type !== 'mdxJsxTextElement') return;
+ if (typeof node.name === 'string') names.add(node.name);
+ });
+ return [...names];
+}
+
+export function escapeRawHtml(prose: string, { flattenLinks = false } = {}): string {
+ const tree = proseProcessor.parse(prose);
+ visit(tree, 'html', (node, index, parent) => {
+ if (!parent || index === undefined) return;
+ const asText = { type: 'text' as const, value: node.value };
+ parent.children[index] =
+ parent.type === 'root' || parent.type === 'blockquote' || parent.type === 'listItem'
+ ? { type: 'paragraph', children: [asText] }
+ : asText;
+ });
+ if (flattenLinks) flattenLinkNodes(tree);
+ return proseProcessor.stringify(tree).trim();
+}
+
+function flattenLinkNodes(tree: Root): void {
+ visit(tree, (node, index, parent) => {
+ if (!parent || index === undefined) return;
+ if (node.type === 'link') {
+ parent.children.splice(index, 1, ...node.children);
+ return index;
+ }
+ if (node.type === 'image') {
+ parent.children[index] = { type: 'text', value: node.alt ?? '' };
+ }
+ });
+}
+
+export function escapeInlineProse(prose: string, options?: { flattenLinks?: boolean }): string {
+ return escapeRawHtml(prose.replace(/\s+/g, ' ').trim(), options);
+}
+
+export function markdownLink(label: string, url: string, { image = false } = {}): string {
+ const children = inlinePhrasing(label);
+ const node = image
+ ? ({ type: 'image', url, alt: textContent({ children }) } as const)
+ : ({ type: 'link', url, children } as const);
+ return proseProcessor
+ .stringify({ type: 'root', children: [{ type: 'paragraph', children: [node] }] })
+ .trim();
+}
+
+function inlinePhrasing(markdown: string): PhrasingContent[] {
+ const paragraph = singleParagraph(markdown);
+ if (paragraph) return paragraph;
+
+ const oneLine = markdown.replace(/\s+/g, ' ').trim();
+ const neutralised = oneLine.replace(/^(\d{1,9})([.)])|^[#>*+_=-]+/, (run, digits, delimiter) =>
+ digits ? `${digits}\\${delimiter}` : run.replace(/./g, '\\$&'),
+ );
+ const asInline = singleParagraph(neutralised);
+ if (asInline) return asInline;
+
+ return [{ type: 'text', value: textContent(proseProcessor.parse(markdown)) }];
+}
+
+function singleParagraph(markdown: string): PhrasingContent[] | null {
+ const tree = proseProcessor.parse(markdown);
+ flattenLinkNodes(tree);
+ const [first, ...rest] = tree.children;
+ return first?.type === 'paragraph' && rest.length === 0 ? first.children : null;
+}
+
+export function requiredLiteral(node: MdxJsxNode, api: SerializerApi, name: string): string {
+ const expression = api.expressionAttribute(node, name);
+ if (expression !== undefined) {
+ throw new Error(
+ `<${node.name} ${name}={${expression}}> carries its value in an expression the Markdown ` +
+ 'rendition cannot resolve. Write it as a literal, or give the component a disposition ' +
+ 'that renders what it points at.',
+ );
+ }
+
+ const literal = api.attribute(node, name);
+ if (literal) return literal;
+
+ throw new Error(
+ `<${node.name}> has no \`${name}\`, so the Markdown rendition has nothing to render.`,
+ );
+}
+
+export function longestBacktickRun(value: string): number {
+ return Math.max(0, ...[...value.matchAll(/`+/g)].map((run) => run[0].length));
+}
+
+export function inlineCodeSpan(value: string): string {
+ const trimmed = value.trim();
+ if (trimmed === '') return '';
+ const marker = '`'.repeat(longestBacktickRun(trimmed) + 1);
+ const pad = trimmed.startsWith('`') || trimmed.endsWith('`') ? ' ' : '';
+ return `${marker}${pad}${trimmed}${pad}${marker}`;
+}
+
+function splitFrontmatter(source: string): { frontmatter: Record<string, unknown>; body: string } {
+ const match = FRONTMATTER_BLOCK.exec(source);
+ if (!match) return { frontmatter: {}, body: source };
+
+ const parsed: unknown = parseYaml(match[1] ?? '') ?? {};
+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+ throw new Error(
+ `Frontmatter must be a YAML mapping, got ${Array.isArray(parsed) ? 'a sequence' : typeof parsed}.`,
+ );
+ }
+ return { frontmatter: parsed as Record<string, unknown>, body: source.slice(match[0].length) };
+}
+
+function absolutiseUrls(tree: Root, base: URL): void {
+ visit(tree, (node) => {
+ if (node.type !== 'link' && node.type !== 'image' && node.type !== 'definition') return;
+ node.url = absolutise(node.url, base);
+ });
+}
+
+function absolutise(url: string, base: URL): string {
+ if (url === '' || ABSOLUTE_URL.test(url)) return url;
+ const suffixAt = url.search(/[#?]/);
+ const path = suffixAt === -1 ? url : url.slice(0, suffixAt);
+ const suffix = suffixAt === -1 ? '' : url.slice(suffixAt);
+ return new URL(path.replace(SOURCE_EXTENSION, '') + suffix, base).href;
+}
+
+function transformChildren(
+ parent: { children?: unknown[] },
+ registry: SerializerRegistry,
+ base: URL,
+) {
+ const out: RootContent[] = [];
+ for (const child of parent.children ?? [])
+ out.push(...transformNode(child as RootContent, registry, base));
+ return out;
+}
+
+function transformNode(node: RootContent, registry: SerializerRegistry, base: URL): RootContent[] {
+ if (NON_PROSE_TYPES.has(node.type)) return [];
+
+ if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
+ return transformJsx(node, registry, base);
+ }
+
+ if ('children' in node) {
+ (node as unknown as { children: RootContent[] }).children = transformChildren(
+ node,
+ registry,
+ base,
+ );
+ }
+ return [node];
+}
+
+function transformJsx(node: MdxJsxNode, registry: SerializerRegistry, base: URL): RootContent[] {
+ const name = node.name;
+ if (name === null || name === undefined) return transformChildren(node, registry, base);
+
+ const disposition = Object.hasOwn(registry, name) ? registry[name] : undefined;
+ if (disposition === undefined) {
+ throw new Error(
+ `No serializer disposition for <${name}>. Add "${name}" to the registry as 'flatten', 'drop', or a custom serializer.`,
+ );
+ }
+ if (disposition === 'drop') return [];
+ if (disposition === 'flatten') return transformChildren(node, registry, base);
+
+ const rendered = disposition(node, apiFor(registry, base));
+ if (rendered === null) return [];
+ return [{ type: 'html', value: rendered } satisfies Html];
+}
+
+function apiFor(registry: SerializerRegistry, base: URL): SerializerApi {
+ return {
+ children(node) {
+ const children = transformChildren(node, registry, base);
+ return processor.stringify({ type: 'root', children }).trimEnd();
+ },
+ text(node) {
+ return textContent(node);
+ },
+ url(href) {
+ return absolutise(href, base);
+ },
+ attribute(node, name) {
+ const found = findAttribute(node, name);
+ if (found === undefined) return undefined;
+ if (found.value !== null && typeof found.value !== 'string') {
+ throw new Error(
+ `<${node.name ?? 'component'}> \`${name}\` is an expression, and this serializer reads ` +
+ 'it as literal text. Read it with expressionAttribute and parse it, or give the ' +
+ 'attribute a quoted value.',
+ );
+ }
+ return typeof found.value === 'string' ? found.value : undefined;
+ },
+ expressionAttribute(node, name) {
+ const value = findAttribute(node, name)?.value;
+ return typeof value === 'object' && value !== null ? value.value : undefined;
+ },
+ };
+}
+
+function findAttribute(node: MdxJsxNode, name: string) {
+ return node.attributes.find(
+ (attribute) => attribute.type === 'mdxJsxAttribute' && attribute.name === name,
+ );
+}
+
+function textContent(node: unknown): string {
+ if (typeof node !== 'object' || node === null) return '';
+ const typed = node as { type?: string; value?: unknown; alt?: unknown; children?: unknown[] };
+ if (typed.type === 'text' || typed.type === 'inlineCode' || typed.type === 'code') {
+ return typeof typed.value === 'string' ? typed.value : '';
+ }
+ if (typed.type === 'image') return typeof typed.alt === 'string' ? typed.alt : '';
+ if (!Array.isArray(typed.children)) return '';
+ return typed.children.map(textContent).join('');
+}
diff --git a/docs/src/lib/site.ts b/docs/src/lib/site.ts
index d41a6dd22..118f3a903 100644
--- a/docs/src/lib/site.ts
+++ b/docs/src/lib/site.ts
@@ -1,6 +1,10 @@
import { STABLE_DMG_URL } from './download-links';
export const SITE_URL = 'https://openknowledge.ai';
+
+export function absoluteSiteUrl(path: string): string {
+ return new URL(path, SITE_URL).href;
+}
export const SITE_NAME = 'OpenKnowledge';
export const TWITTER_HANDLE = '@OpenKnowledge';
diff --git a/docs/src/markdown-header-contract.test.ts b/docs/src/markdown-header-contract.test.ts
new file mode 100644
index 000000000..76b5018f2
--- /dev/null
+++ b/docs/src/markdown-header-contract.test.ts
@@ -0,0 +1,213 @@
+import { autoImplementMethods } from 'next/dist/server/route-modules/app-route/helpers/auto-implement-methods';
+import type { AppRouteHandlers } from 'next/dist/server/route-modules/app-route/module';
+import { describe, expect, test, vi } from 'vitest';
+import nextConfig from '../next.config.ts';
+import { filesNamed, ROUTE_FILES, readAppFile } from './app/app-files.test-helper.ts';
+
+const CONTRACT = {
+ 'content-type': 'text/markdown; charset=utf-8',
+ 'cache-control': 'public, max-age=0, s-maxage=3600, stale-while-revalidate=86400',
+ vary: 'Accept',
+} as const;
+
+const SHARED_HSTS = 'max-age=63072000; includeSubDomains; preload';
+
+const MARKDOWN_SOURCE_MARKERS = ['markdownResponse', 'markdownRouteHandler', 'text/markdown'];
+
+const markdownRouteFilesOnDisk = () =>
+ filesNamed(ROUTE_FILES).filter((file) => {
+ const source = readAppFile(file);
+ return MARKDOWN_SOURCE_MARKERS.some((marker) => source.includes(marker));
+ });
+
+const OVERVIEW_URL = 'https://openknowledge.ai/docs/get-started/overview';
+
+const overviewPage = {
+ url: '/docs/get-started/overview',
+ data: {
+ title: 'Overview',
+ description: 'What OpenKnowledge is.',
+ getText: async () => 'Early days.\n',
+ },
+};
+
+vi.doMock('@/lib/source', () => ({
+ source: {
+ getPage: (slug: string[]) =>
+ slug.join('/') === 'get-started/overview' ? overviewPage : undefined,
+ getPages: () => [overviewPage],
+ generateParams: () => [{ slug: ['get-started', 'overview'] }],
+ },
+}));
+
+vi.stubGlobal(
+ 'fetch',
+ async () =>
+ new Response(
+ '<rss version="2.0"><channel><item><title>markdownlint</title>' +
+ '<link>https://openknowledge.ai/blog/markdownlint-support</link></item></channel></rss>',
+ ),
+);
+
+const perPageRoute = await import('@/app/llms.mdx/[...slug]/route.ts');
+const llmsTxtRoute = await import('@/app/llms.txt/route.ts');
+const llmsFullRoute = await import('@/app/llms-full.txt/route.ts');
+
+type Handler = (...args: never[]) => Promise<Response> | Response;
+type Method = 'GET' | 'HEAD';
+
+interface MarkdownRoute {
+ readonly name: string;
+ readonly file: string;
+ readonly canonicalUrl: string | null;
+ readonly status: number;
+ readonly module: { readonly GET: Handler };
+ readonly call: (handler: Handler, method: Method) => Promise<Response> | Response;
+}
+
+const slugCall =
+ (url: string, slug: string[]) =>
+ (handler: Handler, method: Method): Promise<Response> =>
+ (handler as typeof perPageRoute.GET)(new Request(url, { method }), {
+ params: Promise.resolve({ slug }),
+ });
+const bareCall = (handler: Handler) => (handler as () => Promise<Response>)();
+
+const ROUTES: MarkdownRoute[] = [
+ {
+ name: 'a docs page',
+ file: 'llms.mdx/[...slug]/route.ts',
+ canonicalUrl: OVERVIEW_URL,
+ status: 200,
+ module: perPageRoute,
+ call: slugCall(`${OVERVIEW_URL}.md`, ['get-started', 'overview']),
+ },
+ {
+ name: 'a docs page nobody holds',
+ file: 'llms.mdx/[...slug]/route.ts',
+ canonicalUrl: null,
+ status: 404,
+ module: perPageRoute,
+ call: slugCall('https://openknowledge.ai/docs/nope.md', ['nope']),
+ },
+ {
+ name: 'llms.txt',
+ file: 'llms.txt/route.ts',
+ canonicalUrl: null,
+ status: 200,
+ module: llmsTxtRoute,
+ call: bareCall,
+ },
+ {
+ name: 'llms-full.txt',
+ file: 'llms-full.txt/route.ts',
+ canonicalUrl: null,
+ status: 200,
+ module: llmsFullRoute,
+ call: bareCall,
+ },
+];
+
+const serve = (route: MarkdownRoute) => route.call(route.module.GET, 'GET');
+
+const served = new Map(
+ await Promise.all(ROUTES.map(async (route) => [route, await serve(route)] as const)),
+);
+
+describe('every Markdown response the docs app serves', () => {
+ test.each(ROUTES)('$name carries the shared header contract', (route) => {
+ const headers = served.get(route)?.headers;
+ for (const [header, value] of Object.entries(CONTRACT)) {
+ expect(headers?.get(header), `${header} on ${route.name}`).toBe(value);
+ }
+ });
+
+ test.each(ROUTES)('$name points agents at the page they should cite', (route) => {
+ const expected = route.canonicalUrl ? `<${route.canonicalUrl}>; rel="canonical"` : null;
+ expect(served.get(route)?.headers.get('link'), `Link on ${route.name}`).toBe(expected);
+ });
+
+ test.each(ROUTES)('$name answers with the status it promises', (route) => {
+ expect(served.get(route)?.status, route.name).toBe(route.status);
+ });
+
+ test('is a census: no route file serves Markdown without being pinned here', () => {
+ const unpinned = markdownRouteFilesOnDisk().filter(
+ (file) => !ROUTES.some((route) => route.file === file),
+ );
+
+ expect(
+ unpinned,
+ 'route files serving Markdown that this contract never checks; add each to ROUTES',
+ ).toEqual([]);
+ });
+
+ test('the census sees a handler authored as .tsx', () => {
+ expect(filesNamed(ROUTE_FILES).filter((file) => file.endsWith('.tsx')).length).toBeGreaterThan(
+ 0,
+ );
+ });
+
+ test('the census reaches the routes it names', () => {
+ const onDisk = new Set(filesNamed(ROUTE_FILES));
+ expect(ROUTES.filter((route) => !onDisk.has(route.file)).map((route) => route.file)).toEqual(
+ [],
+ );
+ expect(ROUTES.length).toBeGreaterThan(1);
+ });
+});
+
+describe('a HEAD request to every Markdown route', () => {
+ const HEAD_ROUTES = ROUTES.filter(
+ (route, index) => ROUTES.findIndex((other) => other.file === route.file) === index,
+ );
+
+ test('covers every Markdown route file on disk, once', () => {
+ const files = HEAD_ROUTES.map((route) => route.file);
+ expect(new Set(files)).toEqual(new Set(markdownRouteFilesOnDisk()));
+ expect(files.length).toBe(new Set(files).size);
+ });
+
+ test.each(HEAD_ROUTES)('$name is answered by its GET handler, which reads no method', async ({
+ module,
+ call,
+ }) => {
+ expect(module).not.toHaveProperty('HEAD');
+ const methods = autoImplementMethods(module as unknown as AppRouteHandlers);
+ expect(methods.HEAD).toBe(module.GET);
+
+ const headers = (await call(methods.HEAD as never, 'HEAD')).headers;
+ for (const [header, value] of Object.entries(CONTRACT)) {
+ expect(headers.get(header), `${header} on HEAD`).toBe(value);
+ }
+ });
+});
+
+describe('the header layer next.config.ts adds on top', () => {
+ async function configRules() {
+ return (await nextConfig.headers?.()) ?? [];
+ }
+
+ test('sets no header the Markdown contract also sets', async () => {
+ const collisions = (await configRules()).flatMap((rule) =>
+ rule.headers
+ .filter((header) => {
+ const key = header.key.toLowerCase();
+ return key in CONTRACT || key === 'link';
+ })
+ .map((header) => `${rule.source} sets ${header.key}`),
+ );
+
+ expect(collisions, 'config headers colliding with the Markdown contract').toEqual([]);
+ });
+
+ test('sends the same Strict-Transport-Security value as the marketing app', async () => {
+ const values = (await configRules()).flatMap((rule) =>
+ rule.headers
+ .filter((header) => header.key.toLowerCase() === 'strict-transport-security')
+ .map((header) => header.value),
+ );
+
+ expect(values).toEqual([SHARED_HSTS]);
+ });
+});
diff --git a/docs/src/markdown-rewrites.test.ts b/docs/src/markdown-rewrites.test.ts
new file mode 100644
index 000000000..2bb6cdbef
--- /dev/null
+++ b/docs/src/markdown-rewrites.test.ts
@@ -0,0 +1,64 @@
+import { getPathMatch } from 'next/dist/shared/lib/router/utils/path-match.js';
+import { prepareDestination } from 'next/dist/shared/lib/router/utils/prepare-destination.js';
+import { describe, expect, test } from 'vitest';
+import nextConfig from '../next.config.ts';
+
+async function rewritePhases() {
+ const rewrites = await nextConfig.rewrites?.();
+ if (!rewrites || Array.isArray(rewrites)) {
+ throw new Error('rewrites() must return { beforeFiles, afterFiles }');
+ }
+ return { beforeFiles: rewrites.beforeFiles ?? [], afterFiles: rewrites.afterFiles ?? [] };
+}
+
+async function rewriteOf(pathname: string): Promise<string | null> {
+ const rules = (await rewritePhases()).beforeFiles;
+
+ for (const rule of rules) {
+ const params = getPathMatch(rule.source, { removeUnnamedParams: true, strict: true })(pathname);
+ if (!params) continue;
+ return prepareDestination({
+ appendParamsToQuery: false,
+ destination: rule.destination,
+ params,
+ query: {},
+ }).newUrl;
+ }
+ return null;
+}
+
+describe('docs Markdown rewrites', () => {
+ test.each([
+ ['/docs/get-started/quickstart.md'],
+ ['/docs/get-started/quickstart.html.md'],
+ ['/docs/get-started/quickstart.mdx'],
+ ])('%s serves the markdown rendition of the same page', async (url) => {
+ expect(await rewriteOf(url)).toBe('/llms.mdx/get-started/quickstart');
+ });
+
+ test('a single-segment slug resolves too', async () => {
+ expect(await rewriteOf('/docs/overview.html.md')).toBe('/llms.mdx/overview');
+ });
+
+ test('a page whose own name ends in .html keeps its slug', async () => {
+ expect(await rewriteOf('/docs/guides/index.html.md')).toBe('/llms.mdx/guides/index');
+ });
+
+ test('the human page is left alone', async () => {
+ expect(await rewriteOf('/docs/get-started/quickstart')).toBeNull();
+ });
+
+ test('every markdown rule sits in beforeFiles, and none in afterFiles', async () => {
+ const { beforeFiles, afterFiles } = await rewritePhases();
+ expect(beforeFiles.map((rule) => rule.destination)).toEqual(
+ expect.arrayContaining([expect.stringMatching(/^\/llms\.mdx\//)]),
+ );
+ expect(afterFiles.filter((rule) => rule.destination.startsWith('/llms.mdx'))).toEqual([]);
+ });
+
+ test('keeps the PostHog proxy rules in afterFiles, in their existing order', async () => {
+ const { afterFiles } = await rewritePhases();
+ const sources = afterFiles.map((rule) => rule.source);
+ expect(sources).toEqual(['/ingest/static/:path*', '/ingest/array/:path*', '/ingest/:path*']);
+ });
+});
diff --git a/docs/src/proxy.test.ts b/docs/src/proxy.test.ts
index e6381b241..9f6283d81 100644
--- a/docs/src/proxy.test.ts
+++ b/docs/src/proxy.test.ts
@@ -60,3 +60,92 @@ describe('proxy: www -> apex canonicalization', () => {
expect(res.status).toBe(200);
});
});
+
+function negotiate(pathAndQuery: string, accept?: string) {
+ const headers: Record<string, string> = { host: APEX };
+ if (accept !== undefined) headers.accept = accept;
+ const res = proxy(new NextRequest(`https://${APEX}${pathAndQuery}`, { headers }));
+ const rewrite = res.headers.get('x-middleware-rewrite');
+ return {
+ status: res.status,
+ location: res.headers.get('location'),
+ rewrittenTo: rewrite ? new URL(rewrite).pathname : null,
+ };
+}
+
+const MARKDOWN = 'text/markdown';
+const BROWSER_ACCEPT =
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8';
+
+describe('proxy: Accept negotiation', () => {
+ test('a docs page requested as Markdown serves its Markdown rendition', () => {
+ expect(negotiate('/docs/get-started/quickstart', MARKDOWN).rewrittenTo).toBe(
+ '/llms.mdx/get-started/quickstart',
+ );
+ });
+
+ test('a one-segment slug resolves too', () => {
+ expect(negotiate('/docs/overview', MARKDOWN).rewrittenTo).toBe('/llms.mdx/overview');
+ });
+
+ test('a trailing slash still resolves', () => {
+ expect(negotiate('/docs/overview/', MARKDOWN).rewrittenTo).toBe('/llms.mdx/overview');
+ });
+
+ test('the query string survives the rewrite', () => {
+ const res = negotiate('/docs/overview?ref=agent', MARKDOWN);
+ expect(res.rewrittenTo).toBe('/llms.mdx/overview');
+ });
+
+ test('a browser gets the page untouched', () => {
+ const res = negotiate('/docs/get-started/quickstart', BROWSER_ACCEPT);
+ expect(res.rewrittenTo).toBeNull();
+ expect(res.status).toBe(200);
+ });
+
+ test('a request with no Accept gets the page', () => {
+ expect(negotiate('/docs/overview').rewrittenTo).toBeNull();
+ });
+
+ test('negotiation rewrites rather than redirecting, so the URL stays shareable', () => {
+ const res = negotiate('/docs/overview', MARKDOWN);
+ expect(res.location).toBeNull();
+ expect(res.status).toBe(200);
+ });
+
+ test.each([
+ ['the docs index, which only redirects', '/docs'],
+ ['the changelog, built from a different source adapter', '/docs/changelog'],
+ ['a changelog release page', '/docs/changelog/v0.4.0'],
+ ['a path outside the docs tree', '/continue'],
+ ['a share page', '/d/abc123'],
+ ])('%s is never negotiated', (_label, path) => {
+ expect(negotiate(path, MARKDOWN).rewrittenTo).toBeNull();
+ });
+
+ test.each([
+ '/docs/overview.md',
+ '/docs/overview.html.md',
+ '/docs/overview.mdx',
+ ])('%s is left to the rewrite rules rather than negotiated again', (path) => {
+ expect(negotiate(path, MARKDOWN).rewrittenTo).toBeNull();
+ });
+
+ test('host canonicalization wins over negotiation', () => {
+ const res = proxy(
+ new NextRequest(`https://${WWW}/docs/overview`, { headers: { host: WWW, accept: MARKDOWN } }),
+ );
+ expect(res.status).toBe(308);
+ expect(res.headers.get('location')).toBe(`https://${APEX}/docs/overview`);
+ });
+
+ test('the page a browser falls through to carries no Vary from the proxy', () => {
+ const res = proxy(
+ new NextRequest(`https://${APEX}/docs/overview`, {
+ headers: { host: APEX, accept: BROWSER_ACCEPT },
+ }),
+ );
+ expect(res.headers.get('x-middleware-rewrite')).toBeNull();
+ expect(res.headers.get('vary')).toBeNull();
+ });
+});
diff --git a/docs/src/proxy.ts b/docs/src/proxy.ts
index 872cabcc9..11ae20461 100644
--- a/docs/src/proxy.ts
+++ b/docs/src/proxy.ts
@@ -1,4 +1,6 @@
import { type NextRequest, NextResponse } from 'next/server';
+import { markdownRewrite } from './lib/accept-negotiation';
+import { markdownHandlerForPage } from './lib/markdown-routes';
const APEX_HOST = 'openknowledge.ai';
const WWW_HOST = `www.${APEX_HOST}`;
@@ -16,6 +18,10 @@ export function proxy(request: NextRequest): NextResponse {
return NextResponse.redirect(url, 308);
}
+ const handler = markdownHandlerForPage(request.nextUrl.pathname);
+ const negotiated = markdownRewrite(request, handler);
+ if (negotiated) return negotiated;
+
return NextResponse.next();
}
diff --git a/docs/vitest.config.ts b/docs/vitest.config.ts
index f98ab24e5..d12d3f363 100644
--- a/docs/vitest.config.ts
+++ b/docs/vitest.config.ts
@@ -6,6 +6,12 @@ const docsRoot = fileURLToPath(new URL('.', import.meta.url));
export default defineConfig({
...okVitestBase,
+ test: {
+ ...okVitestBase.test,
+ css: false,
+ server: { deps: { inline: [/fumadocs-ui/] } },
+ exclude: [...okVitestBase.test.exclude, '**/*.real-source.test.ts'],
+ },
resolve: {
...okVitestBase.resolve,
alias: [
diff --git a/docs/vitest.real-source.config.mts b/docs/vitest.real-source.config.mts
new file mode 100644
index 000000000..64dd8ea55
--- /dev/null
+++ b/docs/vitest.real-source.config.mts
@@ -0,0 +1,38 @@
+import { fileURLToPath } from 'node:url';
+import mdx from 'fumadocs-mdx/vite';
+import { defineConfig } from 'vitest/config';
+import { okVitestBase } from '../test-support/vitest.base';
+import * as Config from './source.config.ts';
+
+const docsRoot = fileURLToPath(new URL('.', import.meta.url));
+
+const OUT_DIR = '.source-vitest';
+
+const collectionJsonModule = {
+ name: 'docs-collection-json-module',
+ enforce: 'post' as const,
+ transform(code: string, id: string) {
+ if (!/\.json\?collection=/.test(id)) return null;
+ if (!code.trimStart().startsWith('{')) return null;
+ return { code: `export default ${code};`, map: null };
+ },
+};
+
+export default defineConfig(async () => ({
+ ...okVitestBase,
+ plugins: [...okVitestBase.plugins, await mdx(Config, { outDir: OUT_DIR }), collectionJsonModule],
+ test: {
+ ...okVitestBase.test,
+ include: ['src/**/*.real-source.test.ts'],
+ css: false,
+ server: { deps: { inline: [/fumadocs-ui/] } },
+ },
+ resolve: {
+ ...okVitestBase.resolve,
+ alias: [
+ { find: /(?:\.\.\/)+\.source(?=$|\/)/, replacement: `${docsRoot}${OUT_DIR}` },
+ { find: /^@\/\.source(?=$|\/)/, replacement: `${docsRoot}${OUT_DIR}` },
+ { find: /^@\//, replacement: `${docsRoot}src/` },
+ ],
+ },
+}));
diff --git a/lint-plugins/no-comments/README.md b/lint-plugins/no-comments/README.md
index 4c75f6c2c..6081f3e21 100644
--- a/lint-plugins/no-comments/README.md
+++ b/lint-plugins/no-comments/README.md
@@ -237,11 +237,13 @@ the mirror cannot resolve. The infix is not an escape hatch from this policy —
file still ships INTERNALLY, still rots, and review holds it to the same bar; the gate skips it
only because the mirrored predicate cannot see it anyway.
-`packages/**/{src,tests}/**/*.{ts,tsx}`, `scripts/**/*.mjs`, `.github/scripts/**/*.mjs`,
-`docs/**/*.{ts,tsx}`, and root `*.ts`. Excluded: the audit framework's own tree, `*.private.*`,
-the fidelity and lume-qa suites, fixtures, `*.d.ts`, generated locales, `knip.config.ts`, and
-build output. `scope.mjs` is the one place this is written down; the gate, the sweep, the
-codemod, and the hook all read it from there.
+Source files across the package, script, docs, lint-plugin and root-config strata, minus an
+exclusion set that covers the audit framework's own tree, `*.private.*` files, the fidelity and
+lume-qa suites, fixtures, declaration files, generated locales, build output, and specific
+single-file carve-outs. `scope.mjs` holds both lists, and it is the one place they are written
+down; the gate, the sweep, the codemod, and the hook all read it from there. Read that file
+rather than a restatement here: this paragraph has drifted from it twice, because an enumeration
+in prose has nothing checking it.
This directory ships to the public mirror, which means its own tests may not *spell* a
`*.private.*` path: the mirror's content-leak gate rejects a named reference to one in any
diff --git a/lint-plugins/no-comments/scope.mjs b/lint-plugins/no-comments/scope.mjs
index fd9357871..fef4f815b 100644
--- a/lint-plugins/no-comments/scope.mjs
+++ b/lint-plugins/no-comments/scope.mjs
@@ -3,8 +3,11 @@ export const SCOPE_STRATA = [
{ id: 'packages-tests', include: ['packages/**/tests/**/*.ts', 'packages/**/tests/**/*.tsx'] },
{ id: 'scripts', include: ['scripts/**/*.mjs'] },
{ id: 'github-scripts', include: ['.github/scripts/**/*.mjs'] },
- { id: 'docs', include: ['docs/**/*.ts', 'docs/**/*.tsx'] },
- { id: 'root-configs', include: ['*.ts'] },
+ {
+ id: 'docs',
+ include: ['docs/**/*.ts', 'docs/**/*.tsx', 'docs/**/*.mts', 'docs/**/*.cts'],
+ },
+ { id: 'root-configs', include: ['*.ts', '*.mts', '*.cts'] },
{ id: 'lint-plugins', include: ['lint-plugins/**/*.mjs'] },
];
@@ -20,6 +23,8 @@ export const SCOPE_EXCLUDE = [
'**/*.fixture.tsx',
'**/*.fixture.mjs',
'**/*.d.ts',
+ '**/*.d.mts',
+ '**/*.d.cts',
'**/fixtures/**',
'**/__fixtures__/**',
'knip.config.ts',
diff --git a/lint-plugins/no-comments/scope.test.mjs b/lint-plugins/no-comments/scope.test.mjs
index 7add32aff..5ee0bfc24 100644
--- a/lint-plugins/no-comments/scope.test.mjs
+++ b/lint-plugins/no-comments/scope.test.mjs
@@ -24,8 +24,13 @@ describe('glob compilation', () => {
['packages/**/src/**/*.ts', 'packages/app/tests/a.ts', false],
['**/*.d.ts', 'docs/next-env.d.ts', true],
['**/*.d.ts', 'a.d.ts', true],
+ ['**/*.d.mts', 'docs/next-env.d.mts', true],
+ ['**/*.d.cts', 'docs/lib/types.d.cts', true],
+ ['**/*.d.mts', 'docs/vitest.real-source.config.mts', false],
['*.ts', 'oxlint.config.ts', true],
['*.ts', 'docs/tailwind.config.ts', false],
+ ['*.ts', 'vitest.scripts.config.mts', false],
+ ['docs/**/*.mts', 'docs/vitest.real-source.config.mts', true],
['**/*.private.*', privateTestPath('packages/core/src', 'a'), true],
];
@@ -42,7 +47,10 @@ describe('scope membership', () => {
'scripts/comment-fidelity.mjs',
'.github/scripts/bridge-public-pr-to-monorepo.mjs',
'docs/tailwind.config.ts',
+ 'docs/vitest.real-source.config.mts',
+ 'docs/lib/a.cts',
'oxlint.config.ts',
+ 'vitest.scripts.config.mts',
];
const outOfScope = [
'knip.config.ts',
@@ -52,6 +60,9 @@ describe('scope membership', () => {
privateTestPath('packages/core/src/markdown', 'a'),
'packages/app/src/locales/en/messages.ts',
'docs/next-env.d.ts',
+ 'docs/next-env.d.mts',
+ 'docs/lib/types.d.cts',
+ 'types.d.mts',
'lint-plugins/no-comments/__fixtures__/must-fire.fixture.ts',
'packages/app/src/lib/a.mjs',
'packages/app/node_modules/x/src/a.ts',
@@ -93,7 +104,7 @@ describe('discovery over the real tree', () => {
path.includes('/dist/') ||
path.includes('.private.') ||
path.includes('md-conformance/') ||
- path.endsWith('.d.ts'),
+ /\.d\.[mc]?ts$/.test(path),
);
expect(leaks).toEqual([]);
});
@@ -112,7 +123,12 @@ describe('the scope definition is derived, not restated', () => {
});
test('globToRegExp refuses syntax it does not implement instead of mis-compiling it', () => {
- for (const glob of ['packages/**/{src,tests}/**/*.ts', 'packages/**/*.?s', '!packages/foo/**', 'src/[abc]/*.ts']) {
+ for (const glob of [
+ 'packages/**/{src,tests}/**/*.ts',
+ 'packages/**/*.?s',
+ '!packages/foo/**',
+ 'src/[abc]/*.ts',
+ ]) {
expect(() => globToRegExp(glob), glob).toThrow(/unsupported glob syntax/);
}
});
diff --git a/oxlint.config.ts b/oxlint.config.ts
index 589aed256..389e9353c 100644
--- a/oxlint.config.ts
+++ b/oxlint.config.ts
@@ -32,7 +32,7 @@ const NO_COMMENTS_RULE = 'no-comments/no-comments';
const NO_COMMENTS_SEVERITY = 'error';
export default defineConfig({
- ignorePatterns: ['.agents/skills/**', '.codex/skills/**', '/reports/**'],
+ ignorePatterns: ['.agents/skills/**', '.codex/skills/**', '/reports/**', '/specs/**'],
options: {
typeAware: true,
},
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 57ca13961..7fd349f66 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -269,6 +269,9 @@ importers:
marked:
specifier: ^18.0.6
version: 18.0.7
+ mdast-util-mdx:
+ specifier: ^3.0.0
+ version: 3.0.0
mermaid:
specifier: ~11.16.1
version: 11.16.1
@@ -293,9 +296,30 @@ importers:
react-hook-form:
specifier: ^7.80.0
version: 7.81.0([email protected])
+ remark-gfm:
+ specifier: ^4.0.1
+ version: 4.0.1
+ remark-mdx:
+ specifier: ^3.1.1
+ version: 3.1.1
+ remark-parse:
+ specifier: ^11.0.0
+ version: 11.0.0
+ remark-stringify:
+ specifier: ^11.0.0
+ version: 11.0.0
tailwind-merge:
specifier: ^3.6.0
version: 3.6.0
+ unified:
+ specifier: ^11.0.5
+ version: 11.0.5
+ unist-util-visit:
+ specifier: ^5.1.0
+ version: 5.1.0
+ yaml:
+ specifier: ^2.8.2
+ version: 2.9.0
zod:
specifier: ^4.3.6
version: 4.4.3
@@ -303,6 +327,9 @@ importers:
'@tailwindcss/postcss':
specifier: ^4
version: 4.3.2
+ '@types/mdast':
+ specifier: ^4.0.4
+ version: 4.0.4
'@types/node':
specifier: ^24.7.0
version: 24.13.3
diff --git a/turbo.json b/turbo.json
index c373fc48b..7a3888b24 100644
--- a/turbo.json
+++ b/turbo.json
@@ -42,7 +42,22 @@
"@inkeep/open-knowledge-docs#build": {
"dependsOn": ["^build", "typecheck"],
"outputs": ["dist/**", ".next/**"],
- "env": ["VERCEL_GIT_COMMIT_SHA", "GITHUB_SHA"]
+ "inputs": [
+ "$TURBO_DEFAULT$",
+ "!src/**/*.test.ts",
+ "!src/**/*.test.tsx",
+ "!src/**/*.test-helper.ts",
+ "!vitest.config.ts",
+ "!vitest.*.config.mts"
+ ],
+ "env": [
+ "VERCEL_GIT_COMMIT_SHA",
+ "GITHUB_SHA",
+ "VERCEL_ENV",
+ "OK_ALLOW_BLOGLESS_LLMS_TXT",
+ "CI",
+ "VERCEL"
+ ]
},
"test": {
"dependsOn": ["^build"],
@@ -60,6 +75,26 @@
"../test-support/*.ts"
]
},
+ "@inkeep/open-knowledge-docs#test": {
+ "dependsOn": ["^build", "build"],
+ "cache": true,
+ "inputs": [
+ "src/**/*.ts",
+ "src/**/*.tsx",
+ "src/**/*.json",
+ "content/**",
+ "_snippets/**",
+ "next.config.ts",
+ "source.config.ts",
+ "microfrontends.json",
+ "postcss.config.ts",
+ "tsconfig.json",
+ "vitest.config.ts",
+ "vitest.*.config.mts",
+ "../../test-support/*.ts",
+ "../test-support/*.ts"
+ ]
+ },
"@inkeep/open-knowledge#test": {
"dependsOn": ["^build", "build"],
"cache": true,