Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

Read docs through one internal compiler (no output change) (#6484)

Joeybcf01fe01895
diff --git a/.changeset/docs-compiled-topics.md b/.changeset/docs-compiled-topics.md
new file mode 100644
index 0000000000..db6fce0407
--- /dev/null
+++ b/.changeset/docs-compiled-topics.md
@@ -0,0 +1,7 @@
+---
+'@astryxdesign/cli': patch
+---
+
+[chore] Docs reads go through one internal compiler. `astryx docs` (and `docs()`), `astryx doctor` and `astryx search` read compiled topic nodes instead of each loading, merging, translating and linking doc files on its own. Output is unchanged.
+
+@josephfarina
diff --git a/eslint.config.js b/eslint.config.js
index 5101f96696..e80e61f82c 100644
--- a/eslint.config.js
+++ b/eslint.config.js
@@ -80,6 +80,58 @@ const tseslintRecommended = /** @type {import('eslint').Linter.Config[]} */ (
tseslint.configs.recommended
);
+const ZOD_SEALED = {
+ name: 'zod',
+ message:
+ 'zod is sealed behind the authoring/ parsers. Validate at the load boundary (parseDoc/parseConfig/parseIntegration) and pass typed data inward.',
+};
+
+// Doc merging, section keys and token-reference linking belong to the doc
+// compiler (packages/cli/foundation/doc-compiler). The import scan in its
+// tests enforces the same list and also catches a dynamic import().
+/** @param {string[]} [allowed] compiler functions the file may call */
+const docCompilerOnly = (allowed = []) =>
+ [
+ {
+ group: ['**/foundation/discovery/docs-discovery.mjs'],
+ importNames: ['mergeTopic'],
+ },
+ {
+ group: ['**/foundation/discovery/docs-section-key.mjs'],
+ importNames: ['withSectionKeys'],
+ },
+ {
+ group: ['**/foundation/doc-compiler/compile.mjs'],
+ importNames: [
+ 'lowerReferenceTopic',
+ 'linkReferenceTopic',
+ 'linkReferenceSection',
+ ].filter(name => !allowed.includes(name)),
+ },
+ ]
+ .filter(pattern => pattern.importNames.length > 0)
+ .map(pattern => ({
+ ...pattern,
+ message:
+ 'Doc merging, section keys and token-reference linking belong to the doc compiler. Read compiled nodes through api/docs/_adapter.mjs and foundation/doc-compiler/lenses.mjs.',
+ }));
+
+/** api/'s import rules, allowing the compiler functions one docs file calls. */
+const apiImportRules = (/** @type {string[]} */ allowed = []) => [
+ 'error',
+ {
+ patterns: [
+ {
+ group: ['**/clients/**'],
+ message:
+ 'api/ is the behavior source of truth and must not import clients/ (the CLI presentation layer). Return data in the { type, data } envelope and let the command handler render it.',
+ },
+ ...docCompilerOnly(allowed),
+ ],
+ paths: [ZOD_SEALED],
+ },
+];
+
export default defineConfig(
js.configs.recommended,
tseslintRecommended,
@@ -595,46 +647,38 @@ export default defineConfig(
},
// api/ is the behavior source of truth — it must never reach up into the
// CLI presentation layer (that's what keeps `astryx --json` and the imported
- // function returning identical data).
+ // function returning identical data). It reads docs only as compiled nodes.
{
files: ['packages/cli/api/**/*.mjs'],
+ rules: {'no-restricted-imports': apiImportRules()},
+ },
+ // The docs adapter and leaves are the only files that drive the doc
+ // compiler. These overlap the api/ block on purpose: each repeats its rules
+ // and allows only the compiler functions that file calls.
+ {
+ files: ['packages/cli/api/docs/_adapter.mjs'],
rules: {
- 'no-restricted-imports': [
- 'error',
- {
- patterns: [
- {
- group: ['**/clients/**'],
- message:
- 'api/ is the behavior source of truth and must not import clients/ (the CLI presentation layer). Return data in the { type, data } envelope and let the command handler render it.',
- },
- ],
- paths: [
- {
- name: 'zod',
- message:
- 'zod is sealed behind the authoring/ parsers. Validate at the load boundary (parseDoc/parseConfig/parseIntegration) and pass typed data inward.',
- },
- ],
- },
- ],
+ 'no-restricted-imports': apiImportRules([
+ 'lowerReferenceTopic',
+ 'linkReferenceTopic',
+ ]),
},
},
+ {
+ files: ['packages/cli/api/docs/detail/detail.mjs'],
+ rules: {'no-restricted-imports': apiImportRules(['linkReferenceTopic'])},
+ },
+ {
+ files: ['packages/cli/api/docs/detail/section/section.mjs'],
+ rules: {'no-restricted-imports': apiImportRules(['linkReferenceSection'])},
+ },
// Everything above the contracts consumes already-parsed, typed data.
{
files: ['packages/cli/clients/**/*.mjs'],
rules: {
'no-restricted-imports': [
'error',
- {
- paths: [
- {
- name: 'zod',
- message:
- 'zod is sealed behind the authoring/ parsers. Validate at the load boundary (parseDoc/parseConfig/parseIntegration) and pass typed data inward.',
- },
- ],
- },
+ {patterns: docCompilerOnly(), paths: [ZOD_SEALED]},
],
},
},
diff --git a/packages/cli/api/docs/_adapter.mjs b/packages/cli/api/docs/_adapter.mjs
index 6194bd629a..3dcf11ab79 100644
--- a/packages/cli/api/docs/_adapter.mjs
+++ b/packages/cli/api/docs/_adapter.mjs
@@ -7,29 +7,24 @@
* packages/cli/assets/docs/{topic}.doc.mjs plus every topic the configured
* integrations contribute — and, when a --dense/--zh overlay is requested,
* the sibling {topic}.doc.dense.mjs / {topic}.doc.zh.mjs.
- * @output Catalog access, overlay- and extension-merged reference-doc data,
- * and a combined resolve step ({catalog, docsData}) that the detail and
- * section leaves share.
- * @position Sits beside docs.mjs (api/docs/). Owns everything ≥2 leaves need so
- * no leaf re-implements resolution, overlay merging, or unknown-topic
- * handling. Discovery itself lives in foundation/discovery/docs-discovery,
- * which api/search and the agent-docs block read through the same catalog.
+ * @output Catalog access, the compiler input for a topic, and the compiled
+ * node for it: lowered (overlaid, extensions merged, keys stamped) or linked
+ * (token references resolved too), memoized per catalog.
+ * @position Sits beside docs.mjs (api/docs/). Loads authored files and hands
+ * them to foundation/doc-compiler, so no leaf, doctor check or search loads,
+ * merges, or resolves docs on its own. Discovery itself lives in
+ * foundation/discovery/docs-discovery, which the catalog comes from.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import {pathToFileURL} from 'node:url';
import {Project} from '../../foundation/config/project.mjs';
+import {DocsCatalog} from '../../foundation/discovery/docs-discovery.mjs';
import {
- DocsCatalog,
- mergeTopic,
- problemsInTopic,
- withSourceTitle,
-} from '../../foundation/discovery/docs-discovery.mjs';
-import {
- sectionKeyProblems,
- withSectionKeys,
-} from '../../foundation/discovery/docs-section-key.mjs';
+ linkReferenceTopic,
+ lowerReferenceTopic,
+} from '../../foundation/doc-compiler/compile.mjs';
import {AstryxError} from '../error.mjs';
import {ERROR_CODES} from '../../foundation/response/error-codes.mjs';
import {parseDoc} from '../../authoring/doctypes/parse.mjs';
@@ -56,108 +51,6 @@ export async function loadDocsCatalog(cwd = process.cwd()) {
}
}
-/**
- * @param {string} docPath
- * @param {{lang?: string|null}} [opts]
- * @returns {Promise<import('./docs.type.mjs').DocsDetailResponse['data']>}
- */
-export async function loadReferenceDocs(docPath, {lang} = {}) {
- const mod = await import(pathToFileURL(docPath).href);
- const parsed = parseDoc(mod.docs ?? mod.default, path.basename(docPath));
- if (!('sections' in parsed)) {
- throw new Error(`${path.basename(docPath)} is not a reference document.`);
- }
- const problems = problemsInTopic(parsed);
- if (problems.length > 0) {
- throw new Error(
- `${path.basename(docPath)} is invalid: ${problems.join('; ')}`,
- );
- }
- const docs = parsed;
- if (!lang || lang === 'en') return docs;
-
- const translationPath = overlayPath(docPath, lang);
- if (!fs.existsSync(translationPath)) return docs;
-
- const translationMod = await import(pathToFileURL(translationPath).href);
- const translation = translationMod.docsZh || translationMod.docsDense;
- if (!translation) return docs;
-
- // Overlays are keyed to a base section by title (`section`), not by array
- // position. Position-keying silently grafted each overlay title onto whatever
- // base section happened to share its index, so an overlay that omitted or
- // reordered a section corrupted every section after it — `docs tokens --dense`
- // printed the colour table under a "Spacing" heading (#2182). An overlay may
- // now cover any subset of sections, in any order; sections it does not name
- // keep their base content.
- /** @type {Map<string, any>} */
- const bySection = new Map();
- for (const ts of translation.sections ?? []) {
- if (ts?.section != null) bySection.set(ts.section, ts);
- }
-
- return {
- ...docs,
- description: translation.description || docs.description,
- sections: docs.sections.map(
- (
- /** @type {import('@astryxdesign/cli/authoring').ReferenceSection} */ section,
- ) => {
- const ts = bySection.get(section.title);
- if (!ts) return section;
- const localized = {
- ...section,
- title: ts.title || section.title,
- content: section.content.map(
- (
- /** @type {import('@astryxdesign/cli/authoring').ReferenceContentBlock} */ block,
- /** @type {number} */ bi,
- ) => {
- const tb = ts.content?.[bi];
- if (!tb) return block;
- if (tb.type === 'prose' && block.type === 'prose')
- return {...block, text: tb.text};
- if (tb.type === 'list' && block.type === 'list')
- return {...block, items: tb.items};
- return block;
- },
- ),
- };
- return withSourceTitle(localized, section.title);
- },
- ),
- };
-}
-
-/**
- * Load one catalog entry: its own doc, plus any extension an integration
- * merged onto it, in configuration order.
- *
- * A localization overlay applies to each file before the extensions are
- * merged, so an extension written in the base language stays readable under
- * `--dense`/`--zh` (it replaces its own sections and leaves the rest
- * translated) rather than being dropped.
- *
- * @param {import('../../foundation/discovery/docs-discovery.mjs').DocsTopicEntry} entry
- * @param {{lang?: string|null}} [opts]
- * @returns {Promise<import('./docs.type.mjs').DocsDetailResponse['data']>}
- */
-export async function loadTopicDoc(entry, {lang} = {}) {
- let doc = await loadReferenceDocs(entry.path, {lang});
- for (const extension of entry.extensions) {
- doc = mergeTopic(doc, await loadReferenceDocs(extension.path, {lang}));
- // Merging matches on keys, so this holds unless merge itself regresses.
- const problems = sectionKeyProblems(doc.sections);
- if (problems.length > 0) {
- throw new Error(
- `${path.basename(extension.path)}, extending ${entry.name}, leaves two sections with one key: ${problems.join('; ')}`,
- );
- }
- }
- // Derived keys are stamped only now, so they never take part in merging.
- return withSectionKeys(doc);
-}
-
/** The localized overlays a docs read can apply. */
export const OVERLAY_LANGUAGES = ['zh', 'dense'];
@@ -186,11 +79,148 @@ export function overlayLanguages(entry) {
);
}
+/**
+ * The overlay a read applies: none for the authored language.
+ * @param {string | null | undefined} lang
+ * @returns {string | null}
+ */
+function overlayLanguage(lang) {
+ return lang && lang !== 'en' ? lang : null;
+}
+
+/**
+ * Load one authored file and the overlay for `lang`. A failure is recorded on
+ * the result, not thrown, so the compiler reports it in reading order.
+ * @param {string} docPath
+ * @param {string | null} lang
+ * @returns {Promise<import('../../foundation/doc-compiler/compile.mjs').AuthoredFile>}
+ */
+async function loadAuthoredFile(docPath, lang) {
+ const file = path.basename(docPath);
+ let doc;
+ try {
+ const mod = await import(pathToFileURL(docPath).href);
+ doc = parseDoc(mod.docs ?? mod.default, file);
+ } catch (error) {
+ return {file, error};
+ }
+ if (!lang) return {file, doc};
+ const translationPath = overlayPath(docPath, lang);
+ if (!fs.existsSync(translationPath)) return {file, doc};
+ try {
+ const translationMod = await import(pathToFileURL(translationPath).href);
+ return {
+ file,
+ doc,
+ overlay: translationMod.docsZh || translationMod.docsDense || null,
+ };
+ } catch (overlayError) {
+ return {file, doc, overlayError};
+ }
+}
+
+/**
+ * Everything the compiler needs for one topic, read from disk.
+ * @param {import('../../foundation/discovery/docs-discovery.mjs').DocsTopicEntry} entry
+ * @param {string | null} lang
+ * @returns {Promise<import('../../foundation/doc-compiler/compile.mjs').ReferenceTopicInput>}
+ */
+async function loadCompilerInput(entry, lang) {
+ const extensions = [];
+ for (const extension of entry.extensions) {
+ extensions.push({
+ ...(await loadAuthoredFile(extension.path, lang)),
+ provider: extension.package,
+ });
+ }
+ return {
+ id: entry.name,
+ provider: entry.package,
+ replaces: entry.replaces ?? null,
+ lang,
+ base: await loadAuthoredFile(entry.path, lang),
+ extensions,
+ };
+}
+
+/** @type {WeakMap<DocsCatalog, Map<string, Promise<import('../../foundation/doc-compiler/compile.mjs').CompiledReferenceNode>>>} */
+const loweredByCatalog = new WeakMap();
+
+/**
+ * One topic, lowered for `lang`: overlaid, extensions merged, keys stamped.
+ * Memoized per catalog, so a read that references a topic twice loads it once.
+ * Every read of the catalog shares the memoized node, so it is frozen; the
+ * lenses hand readers copies.
+ * @param {DocsCatalog} catalog
+ * @param {import('../../foundation/discovery/docs-discovery.mjs').DocsTopicEntry} entry
+ * @param {string | null} [lang]
+ * @returns {Promise<import('../../foundation/doc-compiler/compile.mjs').CompiledReferenceNode>}
+ */
+export function lowerTopic(catalog, entry, lang = null) {
+ const overlay = overlayLanguage(lang);
+ let cache = loweredByCatalog.get(catalog);
+ if (!cache) {
+ cache = new Map();
+ loweredByCatalog.set(catalog, cache);
+ }
+ const key = `${entry.name.toLowerCase()}\u0000${overlay ?? ''}`;
+ let lowered = cache.get(key);
+ if (!lowered) {
+ lowered = loadCompilerInput(entry, overlay).then(input =>
+ deepFreeze(lowerReferenceTopic(input)),
+ );
+ cache.set(key, lowered);
+ }
+ return lowered;
+}
+
+/**
+ * Freeze a value and everything in it.
+ * @template T
+ * @param {T} value
+ * @returns {T}
+ */
+function deepFreeze(value) {
+ if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) {
+ Object.freeze(value);
+ for (const child of Object.values(value)) deepFreeze(child);
+ }
+ return value;
+}
+
+/**
+ * How a token reference finds its target: the topic it names in `catalog`,
+ * lowered for the same language.
+ * @param {DocsCatalog} catalog
+ * @param {string | null} lang
+ * @returns {(topic: string) => Promise<import('../../foundation/doc-compiler/compile.mjs').CompiledReferenceNode | null>}
+ */
+export function referenceTargets(catalog, lang) {
+ return async topic => {
+ const target = catalog.resolve(topic);
+ return target ? lowerTopic(catalog, target, lang) : null;
+ };
+}
+
+/**
+ * One topic, compiled for `lang`: lowered, then every token reference linked.
+ * @param {DocsCatalog} catalog
+ * @param {import('../../foundation/discovery/docs-discovery.mjs').DocsTopicEntry} entry
+ * @param {string | null} [lang]
+ * @returns {Promise<import('../../foundation/doc-compiler/compile.mjs').CompiledReferenceNode>}
+ */
+export async function compileTopic(catalog, entry, lang = null) {
+ return linkReferenceTopic(
+ await lowerTopic(catalog, entry, lang),
+ referenceTargets(catalog, lang),
+ );
+}
+
/**
* Resolve `topic` against the project's catalog (throwing `ERR_UNKNOWN_TOPIC`
- * when unmatched), and load it with any --dense/--zh overlay and any
- * integration extension applied. Shared by the detail and section leaves so
- * topic normalization and unknown-topic handling live in exactly one place.
+ * when unmatched) and lower it with any --dense/--zh overlay and any
+ * integration extension applied. Shared by the leaves so topic normalization
+ * and unknown-topic handling live in exactly one place.
*
* @param {string} topic
* @param {object} [options]
@@ -200,7 +230,7 @@ export function overlayLanguages(entry) {
* @param {string} [options.cwd]
* @returns {Promise<{
* catalog: DocsCatalog,
- * docsData: import('./docs.type.mjs').DocsDetailResponse['data'],
+ * node: import('../../foundation/doc-compiler/compile.mjs').CompiledReferenceNode,
* lang: string | null,
* }>}
*/
@@ -221,6 +251,6 @@ export async function resolveTopicDocs(topic, options = {}) {
);
}
- const docsData = await loadTopicDoc(entry, {lang: effectiveLang});
- return {catalog, docsData, lang: effectiveLang};
+ const node = await lowerTopic(catalog, entry, effectiveLang);
+ return {catalog, node, lang: effectiveLang};
}
diff --git a/packages/cli/api/docs/compiled-topics.test.mjs b/packages/cli/api/docs/compiled-topics.test.mjs
new file mode 100644
index 0000000000..6d7053863a
--- /dev/null
+++ b/packages/cli/api/docs/compiled-topics.test.mjs
@@ -0,0 +1,78 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Every shipped topic compiles to a plain-JSON node that answers every
+ * docs read exactly as the live one does, and no read can change another read
+ * of the same catalog.
+ */
+
+import {describe, expect, it} from 'vitest';
+import {parseCompiledReferenceNode} from '../../foundation/doc-compiler/ir.mjs';
+import {detailView, indexView} from '../../foundation/doc-compiler/lenses.mjs';
+import {
+ compileTopic,
+ loadDocsCatalog,
+ lowerTopic,
+ overlayLanguages,
+} from './_adapter.mjs';
+
+const SLOW = 60_000;
+
+describe('every shipped topic compiles to plain JSON', () => {
+ it(
+ 'survives a JSON round trip with identical responses in every language',
+ async () => {
+ const catalog = await loadDocsCatalog();
+ let compiled = 0;
+ for (const entry of catalog.entries()) {
+ for (const lang of [null, ...overlayLanguages(entry)]) {
+ const lowered = await lowerTopic(catalog, entry, lang);
+ expect(parseCompiledReferenceNode(lowered)).toBe(lowered);
+ const node = await compileTopic(catalog, entry, lang);
+ expect(parseCompiledReferenceNode(node)).toBe(node);
+ const copy = parseCompiledReferenceNode(
+ JSON.parse(JSON.stringify(node)),
+ );
+ expect(JSON.stringify(detailView(copy))).toBe(
+ JSON.stringify(detailView(node)),
+ );
+ expect(JSON.stringify(indexView(copy))).toBe(
+ JSON.stringify(indexView(node)),
+ );
+ compiled += 1;
+ }
+ }
+ expect(compiled).toBeGreaterThan(catalog.entries().length);
+ },
+ SLOW,
+ );
+});
+
+describe('reads that share a catalog', () => {
+ it(
+ 'never let one read change another',
+ async () => {
+ const catalog = await loadDocsCatalog();
+ const tokens = catalog.resolve('tokens');
+ const first = detailView(await compileTopic(catalog, tokens));
+ for (const section of first.sections) {
+ section.title = 'EDITED';
+ for (const block of section.content) {
+ if (typeof block.text === 'string') block.text = 'EDITED';
+ if (Array.isArray(block.rows)) block.rows.push(['EDITED']);
+ }
+ }
+ const again = detailView(await compileTopic(catalog, tokens));
+ const index = indexView(await lowerTopic(catalog, tokens));
+ const spacing = detailView(
+ await compileTopic(catalog, catalog.resolve('spacing')),
+ );
+ for (const read of [again, index, spacing]) {
+ expect(JSON.stringify(read)).not.toContain('EDITED');
+ }
+ const lowered = await lowerTopic(catalog, tokens);
+ expect(Object.isFrozen(lowered.doc.sections[0].content)).toBe(true);
+ },
+ SLOW,
+ );
+});
diff --git a/packages/cli/api/docs/detail/detail.mjs b/packages/cli/api/docs/detail/detail.mjs
index d51474d10f..e8c160a12b 100644
--- a/packages/cli/api/docs/detail/detail.mjs
+++ b/packages/cli/api/docs/detail/detail.mjs
@@ -3,84 +3,17 @@
/**
* @file docs.detail leaf — load one topic's full reference doc.
*
- * @input A topic name plus optional {lang, zh, dense}. Resolves and loads the
- * topic via the shared adapter, then inlines any token-ref blocks.
+ * @input A topic name plus optional {lang, zh, dense, cwd}. Resolves the topic
+ * via the shared adapter and compiles it with its token references linked.
* @output { type: 'docs.detail', data: ReferenceDoc } — the full doc with
- * token-refs resolved, matching `xds --json docs <topic>`.
- * @position Leaf under api/docs. Owns resolveTokenRefs (used only here); shares
- * discovery/loading/topic-resolution with the section leaf via _adapter.mjs.
+ * token-refs inlined, matching `astryx --json docs <topic>`.
+ * @position Leaf under api/docs. Reads the compiled node through the detail
+ * lens; resolution, overlays and extensions happen in the compiler.
*/
-import {loadTopicDoc, resolveTopicDocs} from '../_adapter.mjs';
-import {
- sectionKey,
- sourceTitle,
- withSourceTitle,
-} from '../../../foundation/discovery/docs-section-key.mjs';
-
-/**
- * Resolve token-ref blocks by inlining the referenced section's table.
- * This allows section docs to reference token tables without duplicating data.
- *
- * The reference is resolved through the catalog, so a topic may point at one
- * an integration contributed (or replaced) rather than only at a built-in. It
- * names the section by key or authored title, so it resolves in every language.
- * @param {import('../docs.type.mjs').DocsDetailResponse['data']} docsData
- * @param {import('../../../foundation/discovery/docs-discovery.mjs').DocsCatalog} catalog
- * @param {{lang?: string | null}} [options]
- * @returns {Promise<import('../docs.type.mjs').DocsDetailResponse['data']>}
- */
-export async function resolveTokenRefs(docsData, catalog, {lang = null} = {}) {
- const resolved = {...docsData, sections: [...docsData.sections]};
- for (let si = 0; si < resolved.sections.length; si++) {
- const section = resolved.sections[si];
- /** @type {import('@astryxdesign/cli/authoring').ReferenceContentBlock[]} */
- const newContent = [];
- for (const block of section.content) {
- if (block.type === 'token-ref') {
- const refEntry = catalog.resolve(block.topic);
- if (!refEntry) {
- newContent.push({type: 'prose', text: `[token-ref: unknown topic "${block.topic}"]`});
- continue;
- }
- const refDocs = await loadTopicDoc(refEntry, {lang});
- const wanted = block.section.toLowerCase();
- const refSection = refDocs.sections.find(
- (/** @type {import('@astryxdesign/cli/authoring').ReferenceSection} */ s) =>
- sectionKey(s) === block.section ||
- sourceTitle(s).toLowerCase() === wanted,
- );
- if (!refSection) {
- newContent.push({type: 'prose', text: `[token-ref: section "${block.section}" not found in "${block.topic}"]`});
- continue;
- }
- // Inline the referenced section's content blocks (tables, prose, etc.)
- // and carry over the previewType
- for (const refBlock of refSection.content) {
- newContent.push(refBlock);
- }
- // If the referenced section has a previewType, attach it to our section
- if (refSection.previewType && !section.previewType) {
- resolved.sections[si] = withSourceTitle(
- {...section, previewType: refSection.previewType, content: newContent},
- sourceTitle(section),
- );
- }
- } else {
- newContent.push(block);
- }
- }
- if (resolved.sections[si] === section) {
- resolved.sections[si] = withSourceTitle(
- {...section, content: newContent},
- sourceTitle(section),
- );
- } else {
- resolved.sections[si].content = newContent;
- }
- }
- return resolved;
-}
+import {linkReferenceTopic} from '../../../foundation/doc-compiler/compile.mjs';
+import {detailView} from '../../../foundation/doc-compiler/lenses.mjs';
+import {referenceTargets, resolveTopicDocs} from '../_adapter.mjs';
/**
* @param {string} topic
@@ -92,7 +25,10 @@ export async function resolveTokenRefs(docsData, catalog, {lang = null} = {}) {
* @returns {Promise<import('../docs.type.mjs').DocsDetailResponse>}
*/
export async function detail(topic, options = {}) {
- const {catalog, docsData, lang} = await resolveTopicDocs(topic, options);
- const resolved = await resolveTokenRefs(docsData, catalog, {lang});
- return {type: 'docs.detail', data: resolved};
+ const {catalog, node, lang} = await resolveTopicDocs(topic, options);
+ const linked = await linkReferenceTopic(
+ node,
+ referenceTargets(catalog, lang),
+ );
+ return {type: 'docs.detail', data: detailView(linked)};
}
diff --git a/packages/cli/api/docs/detail/section/section.mjs b/packages/cli/api/docs/detail/section/section.mjs
index 0c103ce2f5..40ce3d04a4 100644
--- a/packages/cli/api/docs/detail/section/section.mjs
+++ b/packages/cli/api/docs/detail/section/section.mjs
@@ -4,14 +4,15 @@
* @file docs.detail.section leaf — load a single section of a topic.
*
* @input A topic name, a section query, and optional {lang, zh, dense}. Resolves
- * and loads the topic via the shared adapter, then finds the section by its
- * stable key, then by exact title, then by a title that contains the query.
+ * the topic via the shared adapter, finds the section in its lowered compiled
+ * node by its stable key, then by exact title, then by a title that contains
+ * the query, and links only that section.
* @output { type: 'docs.detail.section', data: ReferenceSection } with any
* token-ref blocks inlined — matching `astryx --json docs <topic> <section>`.
* Throws ERR_UNKNOWN_SECTION when nothing matches, or when the query matches
* more than one section (the candidates come back as suggestions).
- * @position Leaf nested under api/docs/detail. Shares discovery/loading/
- * topic-resolution with the detail leaf via _adapter.mjs.
+ * @position Leaf nested under api/docs/detail. Shares topic resolution with the
+ * detail leaf via _adapter.mjs and reads through the compiler's lenses.
*/
import {AstryxError} from '../../../error.mjs';
@@ -20,8 +21,12 @@ import {
findDocSection,
sectionKey,
} from '../../../../foundation/discovery/docs-section-key.mjs';
-import {resolveTopicDocs} from '../../_adapter.mjs';
-import {resolveTokenRefs} from '../detail.mjs';
+import {linkReferenceSection} from '../../../../foundation/doc-compiler/compile.mjs';
+import {
+ readerSections,
+ sectionView,
+} from '../../../../foundation/doc-compiler/lenses.mjs';
+import {referenceTargets, resolveTopicDocs} from '../../_adapter.mjs';
/**
* @param {string} topic
@@ -44,31 +49,30 @@ export async function section(topic, sectionName, options = {}) {
ERROR_CODES.ERR_UNKNOWN_SECTION,
);
}
- const {catalog, docsData, lang} = await resolveTopicDocs(topic, options);
- const {section: match, candidates} = findDocSection(
- docsData.sections,
- sectionName,
- );
+ const {catalog, node, lang} = await resolveTopicDocs(topic, options);
+ const sections = readerSections(node);
+ const {section: match, candidates} = findDocSection(sections, sectionName);
if (!match) {
const ambiguous = candidates.length > 1;
throw new AstryxError(
ambiguous
? `Section "${sectionName}" matches ${candidates.length} sections in "${topic}". Read one by its key.`
: `Section "${sectionName}" not found in "${topic}"`,
- (ambiguous ? candidates : docsData.sections).map(s => ({
+ (ambiguous ? candidates : sections).map(s => ({
name: sectionKey(s),
reason: s.title,
})),
ERROR_CODES.ERR_UNKNOWN_SECTION,
);
}
+
// A section read on its own inlines its token refs, as the whole topic does;
- // otherwise a section that is only a token-ref prints blank.
- const {sections} = await resolveTokenRefs(
- {...docsData, sections: [match]},
- catalog,
- {lang},
+ // otherwise a section that is only a token-ref prints blank. Only this
+ // section is linked, so a broken reference elsewhere cannot fail the read.
+ const linked = await linkReferenceSection(
+ match,
+ referenceTargets(catalog, lang),
);
- return {type: 'docs.detail.section', data: sections[0]};
+ return {type: 'docs.detail.section', data: sectionView(node, linked)};
}
diff --git a/packages/cli/api/docs/detail/section/section.test.mjs b/packages/cli/api/docs/detail/section/section.test.mjs
index 5ca38b9123..afecd3ace4 100644
--- a/packages/cli/api/docs/detail/section/section.test.mjs
+++ b/packages/cli/api/docs/detail/section/section.test.mjs
@@ -10,7 +10,7 @@
import {describe, it, expect} from 'vitest';
import {section} from './section.mjs';
import {AstryxError} from '../../../error.mjs';
-import {loadDocsCatalog, loadTopicDoc} from '../../_adapter.mjs';
+import {loadDocsCatalog, lowerTopic} from '../../_adapter.mjs';
const SLOW = 30_000;
@@ -42,7 +42,8 @@ describe('docs.detail.section leaf', () => {
}, SLOW);
it('reads a section by its stable key', async () => {
- const doc = await loadTopicDoc((await loadDocsCatalog()).resolve('theme'));
+ const catalog = await loadDocsCatalog();
+ const {doc} = await lowerTopic(catalog, catalog.resolve('theme'));
const target = doc.sections[doc.sections.length - 1];
const res = await section('theme', target.id);
expect(res.data.title).toBe(target.title);
@@ -63,7 +64,7 @@ describe('docs.detail.section leaf', () => {
const catalog = await loadDocsCatalog();
let checked = 0;
for (const entry of catalog.entries()) {
- const doc = await loadTopicDoc(entry);
+ const {doc} = await lowerTopic(catalog, entry);
for (const own of doc.sections) {
if (!own.content.some(block => block.type === 'token-ref')) continue;
const res = await section(entry.name, own.id, lang ? {lang} : {});
diff --git a/packages/cli/api/docs/index/index.mjs b/packages/cli/api/docs/index/index.mjs
index 0fb9305c03..c56b5181a3 100644
--- a/packages/cli/api/docs/index/index.mjs
+++ b/packages/cli/api/docs/index/index.mjs
@@ -3,8 +3,9 @@
/**
* @file docs.index leaf — the section index of one topic.
*
- * @input A topic name plus optional {lang, zh, dense, cwd}. Resolves and loads
- * the topic via the shared adapter.
+ * @input A topic name plus optional {lang, zh, dense, cwd}. Resolves the topic
+ * via the shared adapter and reads its lowered compiled node; nothing is
+ * linked, since the index never inlines a token reference.
* @output { type: 'docs.index', data: DocsIndex } — the topic's name, title and
* description and, for each section, the key it is read by, its title, and a
* one-line summary. Matches `astryx --json docs <topic> --index`.
@@ -13,7 +14,7 @@
* read, is the detail leaf.
*/
-import {buildDocsIndexData} from '../../../foundation/discovery/docs-section-key.mjs';
+import {indexView} from '../../../foundation/doc-compiler/lenses.mjs';
import {resolveTopicDocs} from '../_adapter.mjs';
/**
@@ -26,6 +27,6 @@ import {resolveTopicDocs} from '../_adapter.mjs';
* @returns {Promise<import('../docs.type.mjs').DocsIndexResponse>}
*/
export async function index(topic, options = {}) {
- const {docsData} = await resolveTopicDocs(topic, options);
- return {type: 'docs.index', data: buildDocsIndexData(docsData)};
+ const {node} = await resolveTopicDocs(topic, options);
+ return {type: 'docs.index', data: indexView(node)};
}
diff --git a/packages/cli/api/doctor/doctor.mjs b/packages/cli/api/doctor/doctor.mjs
index e8f6045354..6b61ca4993 100644
--- a/packages/cli/api/doctor/doctor.mjs
+++ b/packages/cli/api/doctor/doctor.mjs
@@ -34,8 +34,8 @@ import {
docsIndexBytes,
oversizedDocSections,
} from '../../foundation/discovery/docs-output-budget.mjs';
-import {loadTopicDoc, overlayLanguages} from '../docs/_adapter.mjs';
-import {resolveTokenRefs} from '../docs/detail/detail.mjs';
+import {compileTopic, overlayLanguages} from '../docs/_adapter.mjs';
+import {detailView} from '../../foundation/doc-compiler/lenses.mjs';
import {semverCompare, isValidSemver, satisfiesRange} from '../../foundation/env/semver.mjs';
/**
@@ -798,11 +798,7 @@ export async function checkDocsProgressiveDisclosure(ctx) {
for (const lang of [null, ...overlayLanguages(entry)]) {
const where = lang ? `${entry.name} [${lang}]` : entry.name;
try {
- const doc = await resolveTokenRefs(
- await loadTopicDoc(entry, {lang}),
- catalog,
- {lang},
- );
+ const doc = detailView(await compileTopic(catalog, entry, lang));
if (lang == null) topics += 1;
const indexBytes = docsIndexBytes(buildDocsIndexData(doc));
if (indexBytes > DOC_OUTPUT_BUDGET_BYTES) {
diff --git a/packages/cli/api/search/search.mjs b/packages/cli/api/search/search.mjs
index 1a52650804..20fb501371 100644
--- a/packages/cli/api/search/search.mjs
+++ b/packages/cli/api/search/search.mjs
@@ -66,7 +66,7 @@ import {
import {loadIntegrationsSafely} from '../component/_adapter.mjs';
import {levenshteinDistance} from '../../foundation/text/string-utils.mjs';
import {discoverTemplates, extractComponents} from '../template/template.mjs';
-import {loadDocsCatalog, loadTopicDoc} from '../docs/_adapter.mjs';
+import {loadDocsCatalog, lowerTopic} from '../docs/_adapter.mjs';
import {AstryxError} from '../error.mjs';
import {ERROR_CODES} from '../../foundation/response/error-codes.mjs';
import {setResultCoverage} from './coverage.mjs';
@@ -732,16 +732,16 @@ async function gatherHooks(coreDir) {
async function gatherDocs(cwd) {
/** @type {Candidate[]} */
const candidates = [];
- let entries;
+ let catalog;
try {
- entries = (await loadDocsCatalog(cwd)).entries();
+ catalog = await loadDocsCatalog(cwd);
} catch {
return candidates;
}
- for (const entry of entries) {
+ for (const entry of catalog.entries()) {
let doc = null;
try {
- doc = await loadTopicDoc(entry);
+ doc = (await lowerTopic(catalog, entry)).doc;
} catch {
// A topic that cannot be loaded is reported by the commands that own
// integration issues; search just cannot index it.
diff --git a/packages/cli/foundation/doc-compiler/compile.mjs b/packages/cli/foundation/doc-compiler/compile.mjs
new file mode 100644
index 0000000000..1de2d28d8d
--- /dev/null
+++ b/packages/cli/foundation/doc-compiler/compile.mjs
@@ -0,0 +1,262 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Doc compiler — authored reference topics in, compiled nodes out.
+ *
+ * @input A {@link ReferenceTopicInput}: one topic's own file and the files of
+ * the extensions merged onto it, each already through the authored-doc parser
+ * and each carrying the overlay for the language being read. Discovery
+ * assembles it, and records a file that failed to load instead of throwing,
+ * so problems surface here in the order a reader meets them.
+ * @output A compiled reference node: plain JSON carrying a schema version, the
+ * topic after overlay, extension merge and key stamping, and the authored
+ * title of every section. Linking then resolves each token reference against
+ * its target. Nothing in a node is a function, a symbol, or a file path: an
+ * authored value JSON cannot hold (a function, a Date, `undefined`) takes its
+ * JSON form, the one `--json` output has always shown.
+ * @position The one step between authored docs and every docs reader. The docs
+ * API, doctor and search read compiled nodes, and ./lenses.mjs turns them into
+ * response shapes. Internal to the CLI: the public way in is `docs()`.
+ */
+
+import {mergeTopic, problemsInTopic} from '../discovery/docs-discovery.mjs';
+import {
+ sectionKey,
+ sectionKeyProblems,
+ sourceTitle,
+ withSectionKeys,
+ withSourceTitle,
+} from '../discovery/docs-section-key.mjs';
+
+/** Bumped whenever the shape of a compiled node changes. */
+export const COMPILED_DOC_SCHEMA_VERSION = 1;
+
+/**
+ * One authored file, as discovery loaded it.
+ * @typedef {object} AuthoredFile
+ * @property {string} file the file's name, for messages only
+ * @property {any} [doc] the parsed doc, when it loaded
+ * @property {unknown} [error] why it did not load or parse
+ * @property {any} [overlay] the language overlay's export, when one applies
+ * @property {unknown} [overlayError] why the overlay did not load
+ */
+
+/**
+ * @typedef {object} ReferenceTopicInput
+ * @property {string} id the topic's name in the catalog
+ * @property {string} provider the package that owns the topic
+ * @property {string | null} replaces the topic it took the place of
+ * @property {string | null} lang the overlay language, or null for authored text
+ * @property {AuthoredFile} base
+ * @property {Array<AuthoredFile & {provider: string}>} extensions in merge order
+ */
+
+/**
+ * A token reference after linking: the target section's content, or why it
+ * has none.
+ * @typedef {{status: 'resolved', topic: string, section: string, previewType?: string, content: any[]}
+ * | {status: 'unknown-topic'}
+ * | {status: 'unknown-section'}} TokenRefResolution
+ */
+
+/**
+ * @typedef {object} CompiledReferenceNode
+ * @property {number} schemaVersion
+ * @property {'reference'} kind
+ * @property {'lowered' | 'linked'} stage `linked` once every token reference
+ * carries its resolution; a lowered node carries none
+ * @property {string} id the topic's name in the catalog
+ * @property {string | null} lang
+ * @property {{provider: string, replaces: string | null, extensions: string[]}} provenance
+ * @property {Record<string, string>} sourceTitles section key -> authored title
+ * @property {any} doc the topic: authored fields in authored order, every
+ * section keyed; a linked node's token references carry `resolved`
+ */
+
+/**
+ * Lower one topic: overlay each file, merge the extensions in order, and stamp
+ * every section with its key.
+ * @param {ReferenceTopicInput} input
+ * @returns {CompiledReferenceNode}
+ */
+export function lowerReferenceTopic(input) {
+ let doc = readAuthoredFile(input.base);
+ for (const extension of input.extensions) {
+ doc = mergeTopic(doc, readAuthoredFile(extension));
+ // Merging matches on keys, so this holds unless merge itself regresses.
+ const problems = sectionKeyProblems(doc.sections);
+ if (problems.length > 0) {
+ throw new Error(
+ `${extension.file}, extending ${input.id}, leaves two sections with one key: ${problems.join('; ')}`,
+ );
+ }
+ }
+ // Derived keys are stamped only now, so they never take part in merging.
+ const keyed = withSectionKeys(doc);
+ /** @type {Record<string, string>} */
+ const sourceTitles = {};
+ for (const section of keyed.sections) {
+ sourceTitles[section.id] = sourceTitle(section);
+ }
+ return {
+ schemaVersion: COMPILED_DOC_SCHEMA_VERSION,
+ kind: 'reference',
+ stage: 'lowered',
+ id: input.id,
+ lang: input.lang,
+ provenance: {
+ provider: input.provider,
+ replaces: input.replaces,
+ extensions: input.extensions.map(extension => extension.provider),
+ },
+ sourceTitles,
+ // The authored title travels in sourceTitles; JSON drops the symbol.
+ doc: asJson(keyed, input.id),
+ };
+}
+
+/**
+ * Link every section of a lowered node.
+ * @param {CompiledReferenceNode} node
+ * @param {(topic: string) => Promise<CompiledReferenceNode | null>} lowerTarget
+ * the lowered node a reference names, or null when no topic has that name
+ * @returns {Promise<CompiledReferenceNode>}
+ */
+export async function linkReferenceTopic(node, lowerTarget) {
+ const sections = [];
+ for (const section of node.doc.sections) {
+ sections.push(await linkReferenceSection(section, lowerTarget));
+ }
+ return {...node, stage: 'linked', doc: {...node.doc, sections}};
+}
+
+/**
+ * Resolve the token references in one section. A section with none comes back
+ * as it went in.
+ * @template {{content: any[]}} S
+ * @param {S} section
+ * @param {(topic: string) => Promise<CompiledReferenceNode | null>} lowerTarget
+ * @returns {Promise<S>}
+ */
+export async function linkReferenceSection(section, lowerTarget) {
+ if (!section.content.some(block => block?.type === 'token-ref')) {
+ return section;
+ }
+ const content = [];
+ for (const block of section.content) {
+ content.push(
+ block?.type === 'token-ref'
+ ? {...block, resolved: await resolveTokenRef(block, lowerTarget)}
+ : block,
+ );
+ }
+ return {...section, content};
+}
+
+/**
+ * The section a token reference names — by key, or by authored title in any
+ * letter case — in the target topic lowered for the same language.
+ * @param {any} block
+ * @param {(topic: string) => Promise<CompiledReferenceNode | null>} lowerTarget
+ * @returns {Promise<TokenRefResolution>}
+ */
+async function resolveTokenRef(block, lowerTarget) {
+ const target = await lowerTarget(block.topic);
+ if (!target) return {status: 'unknown-topic'};
+ const wanted = block.section.toLowerCase();
+ const found = target.doc.sections.find(
+ (/** @type {any} */ section) =>
+ sectionKey(section) === block.section ||
+ (target.sourceTitles[section.id] ?? section.title).toLowerCase() ===
+ wanted,
+ );
+ if (!found) return {status: 'unknown-section'};
+ return {
+ status: 'resolved',
+ topic: target.id,
+ section: found.id,
+ ...(found.previewType ? {previewType: found.previewType} : {}),
+ content: found.content,
+ };
+}
+
+/**
+ * A value as JSON carries it: functions, symbols and `undefined` dropped,
+ * Dates as ISO strings.
+ * @param {any} value
+ * @param {string} topic for the message when the value cannot be serialized
+ * @returns {any}
+ */
+function asJson(value, topic) {
+ try {
+ return JSON.parse(JSON.stringify(value));
+ } catch (err) {
+ throw new Error(
+ `${topic} cannot be compiled: ${err instanceof Error ? err.message : String(err)}`,
+ {cause: err},
+ );
+ }
+}
+
+/**
+ * One file's doc, checked and in the reading language. Problems throw in the
+ * order a reader meets them: the file itself, then its overlay.
+ * @param {AuthoredFile} file
+ * @returns {any}
+ */
+function readAuthoredFile(file) {
+ if ('error' in file) throw file.error;
+ const parsed = file.doc;
+ if (!('sections' in parsed)) {
+ throw new Error(`${file.file} is not a reference document.`);
+ }
+ const problems = problemsInTopic(parsed);
+ if (problems.length > 0) {
+ throw new Error(`${file.file} is invalid: ${problems.join('; ')}`);
+ }
+ if ('overlayError' in file) throw file.overlayError;
+ return file.overlay ? applyOverlay(parsed, file.overlay) : parsed;
+}
+
+/**
+ * Lay a translation over a doc. Overlays are keyed to a base section by title
+ * (`section`), not by array position: position keying grafted each overlay
+ * title onto whatever base section shared its index, so an overlay that omitted
+ * or reordered a section corrupted every section after it (#2182). An overlay
+ * may cover any subset of sections, in any order; the rest keep their base
+ * content.
+ * @param {any} docs
+ * @param {any} translation
+ * @returns {any}
+ */
+function applyOverlay(docs, translation) {
+ /** @type {Map<string, any>} */
+ const bySection = new Map();
+ for (const ts of translation.sections ?? []) {
+ if (ts?.section != null) bySection.set(ts.section, ts);
+ }
+ return {
+ ...docs,
+ description: translation.description || docs.description,
+ sections: docs.sections.map((/** @type {any} */ section) => {
+ const ts = bySection.get(section.title);
+ if (!ts) return section;
+ const localized = {
+ ...section,
+ title: ts.title || section.title,
+ content: section.content.map(
+ (/** @type {any} */ block, /** @type {number} */ bi) => {
+ const tb = ts.content?.[bi];
+ if (!tb) return block;
+ if (tb.type === 'prose' && block.type === 'prose')
+ return {...block, text: tb.text};
+ if (tb.type === 'list' && block.type === 'list')
+ return {...block, items: tb.items};
+ return block;
+ },
+ ),
+ };
+ return withSourceTitle(localized, section.title);
+ }),
+ };
+}
diff --git a/packages/cli/foundation/doc-compiler/doc-compiler.test.mjs b/packages/cli/foundation/doc-compiler/doc-compiler.test.mjs
new file mode 100644
index 0000000000..fa8b8bb447
--- /dev/null
+++ b/packages/cli/foundation/doc-compiler/doc-compiler.test.mjs
@@ -0,0 +1,687 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Tests for the doc compiler: lowering, linking, the compiled-node
+ * contract, the lenses, and the rule that readers go through the compiler.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import jscodeshift from 'jscodeshift';
+import {describe, expect, it} from 'vitest';
+import {parseDoc} from '../../authoring/doctypes/parse.mjs';
+import {
+ COMPILED_DOC_SCHEMA_VERSION,
+ linkReferenceTopic,
+ lowerReferenceTopic,
+} from './compile.mjs';
+import {parseCompiledReferenceNode} from './ir.mjs';
+import {detailView, indexView, readerSections, sectionView} from './lenses.mjs';
+
+const CLI_ROOT = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '../..',
+);
+
+/**
+ * @param {string} name
+ * @param {any[]} sections
+ * @param {object} [fields]
+ */
+function authored(name, sections, fields = {}) {
+ return parseDoc(
+ {
+ name,
+ title: `${name} title`,
+ description: `About ${name}.`,
+ ...fields,
+ sections,
+ },
+ `${name}.doc.mjs`,
+ );
+}
+
+/** @param {any} doc @param {object} [more] */
+const file = (doc, more = {}) => ({file: `${doc.name}.doc.mjs`, doc, ...more});
+
+/**
+ * @param {any} base
+ * @param {any[]} [extensions]
+ * @param {string | null} [lang]
+ */
+const input = (base, extensions = [], lang = null) => ({
+ id: base.doc?.name ?? 'demo',
+ provider: '@acme/base',
+ replaces: null,
+ lang,
+ base,
+ extensions,
+});
+
+const prose = (/** @type {string} */ text) => ({type: 'prose', text});
+
+const demo = () =>
+ authored('demo', [
+ {title: 'Quick Start', content: [prose('Start here.')]},
+ {title: 'Props', content: [prose('Every prop.')]},
+ ]);
+
+/** @param {string} name @param {string} text */
+const extension = (name, text) => ({
+ ...file(
+ authored(name, [{title: 'Quick Start', content: [prose(text)]}], {
+ extends: 'demo',
+ }),
+ ),
+ provider: `@acme/${name}`,
+});
+
+describe('lowerReferenceTopic', () => {
+ it('lays the overlay over authored titles and keys every section', () => {
+ const overlay = {
+ description: '演示。',
+ sections: [
+ {
+ section: 'Quick Start',
+ title: '快速开始',
+ content: [prose('从这里开始。')],
+ },
+ ],
+ };
+ const node = lowerReferenceTopic(input(file(demo(), {overlay}), [], 'zh'));
+ expect(node.schemaVersion).toBe(COMPILED_DOC_SCHEMA_VERSION);
+ expect(node.stage).toBe('lowered');
+ expect(node.doc.description).toBe('演示。');
+ expect(
+ node.doc.sections.map((/** @type {any} */ s) => [s.id, s.title]),
+ ).toEqual([
+ ['quick-start', '快速开始'],
+ ['props', 'Props'],
+ ]);
+ expect(node.sourceTitles).toEqual({
+ 'quick-start': 'Quick Start',
+ props: 'Props',
+ });
+ expect(parseCompiledReferenceNode(node)).toBe(node);
+ });
+
+ it('merges extensions by key in order and names every provider', () => {
+ const acme = authored(
+ 'demo-acme',
+ [
+ {title: 'Quick Start', content: [prose('Acme start.')]},
+ {title: 'Acme Palettes', content: [prose('Palettes.')]},
+ ],
+ {extends: 'demo'},
+ );
+ const node = lowerReferenceTopic(
+ input(file(demo()), [{...file(acme), provider: '@acme/ext'}]),
+ );
+ expect(node.doc.sections.map((/** @type {any} */ s) => s.id)).toEqual([
+ 'quick-start',
+ 'props',
+ 'acme-palettes',
+ ]);
+ expect(node.doc.sections[0].content).toEqual([prose('Acme start.')]);
+ expect(node.provenance).toEqual({
+ provider: '@acme/base',
+ replaces: null,
+ extensions: ['@acme/ext'],
+ });
+ });
+
+ it('applies extensions in configuration order, so the last one wins', () => {
+ const node = lowerReferenceTopic(
+ input(file(demo()), [
+ extension('one', 'First.'),
+ extension('two', 'Second.'),
+ ]),
+ );
+ expect(node.doc.sections[0].content).toEqual([prose('Second.')]);
+ expect(node.provenance.extensions).toEqual(['@acme/one', '@acme/two']);
+ });
+
+ it('reports problems in the order a reader meets them', () => {
+ const broken = new Error('extension failed to load');
+ const invalidBase = file(
+ authored('demo', [{title: 'Only', content: [prose('x')]}]),
+ );
+ invalidBase.doc = {...invalidBase.doc, title: ''};
+ expect(() =>
+ lowerReferenceTopic(
+ input(invalidBase, [
+ {file: 'ext.doc.mjs', error: broken, provider: '@acme/ext'},
+ ]),
+ ),
+ ).toThrow(/demo\.doc\.mjs is invalid/);
+
+ const overlayError = new Error('overlay failed to load');
+ expect(() =>
+ lowerReferenceTopic(input(file(demo(), {overlayError}), [], 'zh')),
+ ).toThrow(overlayError);
+
+ expect(() =>
+ lowerReferenceTopic(
+ input(file(demo()), [
+ {file: 'ext.doc.mjs', error: broken, provider: '@acme/ext'},
+ ]),
+ ),
+ ).toThrow(broken);
+ });
+
+ it('carries authored values the way JSON does', () => {
+ const doc = {
+ ...demo(),
+ when: new Date(0),
+ render: () => 1,
+ note: undefined,
+ };
+ doc.sections = [
+ {...doc.sections[0], previewType: undefined},
+ doc.sections[1],
+ ];
+ const node = lowerReferenceTopic(input(file(doc)));
+ expect(node.doc.when).toBe('1970-01-01T00:00:00.000Z');
+ expect('render' in node.doc).toBe(false);
+ expect('note' in node.doc).toBe(false);
+ expect('previewType' in node.doc.sections[0]).toBe(false);
+ expect(parseCompiledReferenceNode(node)).toBe(node);
+ });
+});
+
+describe('linking and lenses', () => {
+ const tokens = () =>
+ lowerReferenceTopic(
+ input(
+ file(
+ authored('tokens', [
+ {
+ title: 'Spacing',
+ previewType: 'spacing-bar',
+ content: [
+ {type: 'table', headers: ['Token'], rows: [['--space-1']]},
+ ],
+ },
+ ]),
+ {overlay: {sections: [{section: 'Spacing', title: '间距'}]}},
+ ),
+ [],
+ 'zh',
+ ),
+ );
+ const guide = () =>
+ lowerReferenceTopic(
+ input(
+ file(
+ authored('guide', [
+ {
+ title: 'By Title',
+ content: [
+ {type: 'token-ref', topic: 'tokens', section: 'SPACING'},
+ ],
+ },
+ {
+ title: 'By Key',
+ content: [
+ {type: 'token-ref', topic: 'tokens', section: 'spacing'},
+ ],
+ },
+ {
+ title: 'No Topic',
+ content: [{type: 'token-ref', topic: 'nope', section: 'x'}],
+ },
+ {
+ title: 'No Section',
+ content: [{type: 'token-ref', topic: 'tokens', section: 'nope'}],
+ },
+ ]),
+ ),
+ [],
+ 'zh',
+ ),
+ );
+ const targets = async (/** @type {string} */ name) =>
+ name === 'tokens' ? tokens() : null;
+
+ it('resolves a reference by key or authored title in the reading language', async () => {
+ const linked = await linkReferenceTopic(guide(), targets);
+ expect(linked.stage).toBe('linked');
+ expect(parseCompiledReferenceNode(linked)).toBe(linked);
+ const [byTitle, byKey, noTopic, noSection] = detailView(linked).sections;
+ for (const section of [byTitle, byKey]) {
+ expect(section.content).toEqual([
+ {type: 'table', headers: ['Token'], rows: [['--space-1']]},
+ ]);
+ expect(section.previewType).toBe('spacing-bar');
+ }
+ expect(noTopic.content).toEqual([
+ prose('[token-ref: unknown topic "nope"]'),
+ ]);
+ expect(noSection.content).toEqual([
+ prose('[token-ref: section "nope" not found in "tokens"]'),
+ ]);
+ });
+
+ it('takes the preview type of the last reference that has one', async () => {
+ const palette = lowerReferenceTopic(
+ input(
+ file(
+ authored('palette', [
+ {
+ title: 'Spacing',
+ previewType: 'spacing-bar',
+ content: [{type: 'table', headers: ['S'], rows: [['1']]}],
+ },
+ {title: 'Plain', content: [prose('No preview.')]},
+ {
+ title: 'Radius',
+ previewType: 'radius-box',
+ content: [{type: 'table', headers: ['R'], rows: [['2']]}],
+ },
+ ]),
+ ),
+ ),
+ );
+ const both = lowerReferenceTopic(
+ input(
+ file(
+ authored('guide', [
+ {
+ title: 'Both',
+ content: ['spacing', 'plain', 'radius'].map(section => ({
+ type: 'token-ref',
+ topic: 'palette',
+ section,
+ })),
+ },
+ ]),
+ ),
+ ),
+ );
+ const [section] = detailView(
+ await linkReferenceTopic(both, async () => palette),
+ ).sections;
+ expect(section.previewType).toBe('radius-box');
+ expect(Object.keys(section)).toEqual([
+ 'title',
+ 'content',
+ 'id',
+ 'previewType',
+ ]);
+ expect(section.content.map((/** @type {any} */ b) => b.type)).toEqual([
+ 'table',
+ 'prose',
+ 'table',
+ ]);
+ });
+
+ it('gives every inlined copy of a referenced section its own blocks', async () => {
+ const shared = tokens();
+ const sameTarget = async (/** @type {string} */ name) =>
+ name === 'tokens' ? shared : null;
+ const twice = lowerReferenceTopic(
+ input(
+ file(
+ authored('guide', [
+ {
+ title: 'First',
+ content: [
+ {type: 'token-ref', topic: 'tokens', section: 'spacing'},
+ ],
+ },
+ {
+ title: 'Second',
+ content: [
+ {type: 'token-ref', topic: 'tokens', section: 'spacing'},
+ ],
+ },
+ ]),
+ ),
+ [],
+ 'zh',
+ ),
+ );
+ const [first, second] = detailView(
+ await linkReferenceTopic(twice, sameTarget),
+ ).sections;
+ expect(first.content).toEqual(second.content);
+ expect(first.content[0]).not.toBe(second.content[0]);
+ expect(first.content[0]).not.toBe(shared.doc.sections[0].content[0]);
+ });
+
+ it('hands out views that share nothing with the node', async () => {
+ const lowered = lowerReferenceTopic(
+ input(file({...demo(), extra: {tags: ['a']}})),
+ );
+ // Authored sections hold only scalars besides content; a node read back
+ // from JSON may hold more, and the lens copies that too.
+ lowered.doc.sections[0] = {
+ ...lowered.doc.sections[0],
+ meta: {level: 1},
+ content: [{type: 'table', headers: ['A'], rows: [['1']]}],
+ };
+ const node = await linkReferenceTopic(lowered, async () => null);
+ const view = detailView(node);
+ expect(JSON.stringify(view)).toBe(JSON.stringify(node.doc));
+ expect(view.extra).not.toBe(node.doc.extra);
+ expect(view.sections[0].meta).not.toBe(node.doc.sections[0].meta);
+ expect(view.sections[0].content[0]).not.toBe(
+ node.doc.sections[0].content[0],
+ );
+ expect(view.sections[0].content[0].rows).not.toBe(
+ node.doc.sections[0].content[0].rows,
+ );
+ });
+
+ it('reads the index and section lookups without linking', () => {
+ const node = guide();
+ expect(indexView(node).sections.map(s => s.id)).toEqual([
+ 'by-title',
+ 'by-key',
+ 'no-topic',
+ 'no-section',
+ ]);
+ expect(readerSections(node).map(s => s.title)).toEqual([
+ 'By Title',
+ 'By Key',
+ 'No Topic',
+ 'No Section',
+ ]);
+ expect(() => sectionView(node, node.doc.sections[0])).toThrow(
+ /before it was linked/,
+ );
+ expect(() => detailView(node)).toThrow(/must be linked/);
+ });
+});
+
+describe('parseCompiledReferenceNode', () => {
+ const node = () => lowerReferenceTopic(input(file(demo())));
+ const linked = async () =>
+ linkReferenceTopic(
+ lowerReferenceTopic(
+ input(
+ file(
+ authored('guide', [
+ {
+ title: 'Ref',
+ content: [{type: 'token-ref', topic: 'nope', section: 'x'}],
+ },
+ ]),
+ ),
+ ),
+ ),
+ async () => null,
+ );
+
+ it('returns a valid node as given', () => {
+ const value = JSON.parse(JSON.stringify(node()));
+ expect(parseCompiledReferenceNode(value)).toBe(value);
+ });
+
+ it('names an unsupported schema version', () => {
+ expect(() =>
+ parseCompiledReferenceNode({...node(), schemaVersion: 99}),
+ ).toThrow(/schema version 99 is not supported/);
+ });
+
+ it('rejects a node that lost an authored title or duplicated a key', () => {
+ expect(() =>
+ parseCompiledReferenceNode({...node(), sourceTitles: {}}),
+ ).toThrow(/no authored title for section "quick-start"/);
+ const twice = node();
+ twice.doc.sections[1] = {...twice.doc.sections[1], id: 'quick-start'};
+ expect(() => parseCompiledReferenceNode(twice)).toThrow(
+ /two sections have the key/,
+ );
+ });
+
+ it.each([
+ [
+ 'a function in a block',
+ (/** @type {any} */ n) => {
+ n.doc.sections[0].content[0].text = () => 1;
+ },
+ /a function is not JSON/,
+ ],
+ [
+ 'a Date',
+ (/** @type {any} */ n) => {
+ n.doc.when = new Date(0);
+ },
+ /a Date is not JSON/,
+ ],
+ [
+ 'an undefined value',
+ (/** @type {any} */ n) => {
+ n.doc.note = undefined;
+ },
+ /undefined is not JSON/,
+ ],
+ [
+ 'a symbol key',
+ (/** @type {any} */ n) => {
+ n.doc[Symbol('x')] = 1;
+ },
+ /symbol keys/,
+ ],
+ [
+ 'a cycle',
+ (/** @type {any} */ n) => {
+ n.doc.self = n.doc;
+ },
+ /refers back to itself/,
+ ],
+ [
+ 'a block that is a number',
+ (/** @type {any} */ n) => {
+ n.doc.sections[0].content.push(42);
+ },
+ /expected a block with a type/,
+ ],
+ [
+ 'a null block',
+ (/** @type {any} */ n) => {
+ n.doc.sections[0].content.push(null);
+ },
+ /expected a block with a type/,
+ ],
+ [
+ 'a block with no type',
+ (/** @type {any} */ n) => {
+ n.doc.sections[0].content.push({text: 'x'});
+ },
+ /expected a block with a type/,
+ ],
+ [
+ 'a path in the provenance',
+ (/** @type {any} */ n) => {
+ n.provenance.provider = '/home/me/acme';
+ },
+ /naming packages, not paths/,
+ ],
+ [
+ 'a resolution on a lowered node',
+ (/** @type {any} */ n) => {
+ n.doc.sections[0].content.push({
+ type: 'token-ref',
+ topic: 't',
+ section: 's',
+ resolved: {status: 'unknown-topic'},
+ });
+ },
+ /a lowered node carries no resolution/,
+ ],
+ ])('rejects %s', (_, mutate, message) => {
+ const value = node();
+ mutate(value);
+ expect(() => parseCompiledReferenceNode(value)).toThrow(message);
+ });
+
+ it('rejects a linked node with an unresolved or malformed reference', async () => {
+ const unresolved = await linked();
+ delete unresolved.doc.sections[0].content[0].resolved;
+ expect(() => parseCompiledReferenceNode(unresolved)).toThrow(
+ /a linked node resolves every reference/,
+ );
+ const malformed = await linked();
+ malformed.doc.sections[0].content[0] = {
+ ...malformed.doc.sections[0].content[0],
+ resolved: {status: 'maybe'},
+ };
+ expect(() => parseCompiledReferenceNode(malformed)).toThrow(
+ /token reference to "nope"/,
+ );
+ });
+});
+
+describe('readers go through the compiler', () => {
+ /** Compiler functions, by module. ESLint enforces the same list. */
+ const RESTRICTED = {
+ 'foundation/discovery/docs-discovery.mjs': ['mergeTopic'],
+ 'foundation/discovery/docs-section-key.mjs': ['withSectionKeys'],
+ 'foundation/doc-compiler/compile.mjs': [
+ 'lowerReferenceTopic',
+ 'linkReferenceTopic',
+ 'linkReferenceSection',
+ ],
+ };
+ /** The docs adapter and leaves drive the compiler; nothing else does. */
+ const ALLOWED = {
+ 'api/docs/_adapter.mjs': ['lowerReferenceTopic', 'linkReferenceTopic'],
+ 'api/docs/detail/detail.mjs': ['linkReferenceTopic'],
+ 'api/docs/detail/section/section.mjs': ['linkReferenceSection'],
+ };
+ const j = jscodeshift.withParser('babel');
+
+ /**
+ * Every import of a compiler function that `rel` may not make: named under
+ * any alias, namespace, re-export, and dynamic import().
+ * @param {string} source
+ * @param {string} rel path under packages/cli
+ * @returns {string[]}
+ */
+ function forbiddenImports(source, rel) {
+ /** @type {string[]} */
+ const allowed = ALLOWED[/** @type {keyof typeof ALLOWED} */ (rel)] ?? [];
+ /** @type {string[]} */
+ const found = [];
+ /** @param {unknown} specifier @returns {string[]} */
+ const bannedFrom = specifier => {
+ if (typeof specifier !== 'string' || !specifier.startsWith('.')) {
+ return [];
+ }
+ const target = path.posix.normalize(
+ path.posix.join(path.posix.dirname(rel), specifier),
+ );
+ const names =
+ RESTRICTED[/** @type {keyof typeof RESTRICTED} */ (target)] ?? [];
+ return names.filter(name => !allowed.includes(name));
+ };
+ /** @param {any} node */
+ const checkStatic = node => {
+ const banned = bannedFrom(node.source?.value);
+ if (banned.length === 0) return;
+ if (node.type === 'ExportAllDeclaration') {
+ found.push(`${rel}: export * from ${node.source.value}`);
+ return;
+ }
+ for (const spec of node.specifiers ?? []) {
+ const name =
+ spec.type === 'ImportNamespaceSpecifier'
+ ? '*'
+ : spec.type === 'ImportDefaultSpecifier'
+ ? 'default'
+ : (spec.imported ?? spec.local)?.name;
+ if (name === '*' || name === 'default' || banned.includes(name)) {
+ found.push(`${rel}: ${name} from ${node.source.value}`);
+ }
+ }
+ };
+ /** @param {any} node */
+ const checkDynamic = node => {
+ const specifier = node.source?.value ?? node.arguments?.[0]?.value;
+ if (bannedFrom(specifier).length > 0) {
+ found.push(`${rel}: import(${specifier})`);
+ }
+ };
+ const root = j(source);
+ root.find(j.ImportDeclaration).forEach(p => checkStatic(p.node));
+ root
+ .find(j.ExportNamedDeclaration)
+ .forEach(p => p.node.source && checkStatic(p.node));
+ root.find(j.ExportAllDeclaration).forEach(p => checkStatic(p.node));
+ root.find(j.ImportExpression).forEach(p => checkDynamic(p.node));
+ root
+ .find(j.CallExpression, {callee: {type: 'Import'}})
+ .forEach(p => checkDynamic(p.node));
+ return found;
+ }
+
+ /** @param {string} dir @returns {string[]} */
+ const sources = dir =>
+ fs.readdirSync(dir, {withFileTypes: true}).flatMap(entry => {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ return entry.name === 'node_modules' ? [] : sources(full);
+ }
+ return entry.name.endsWith('.mjs') && !entry.name.endsWith('.test.mjs')
+ ? [full]
+ : [];
+ });
+
+ it('keeps compiler functions out of api/ and clients/', () => {
+ const offenders = ['api', 'clients']
+ .flatMap(dir => sources(path.join(CLI_ROOT, dir)))
+ .flatMap(full =>
+ forbiddenImports(
+ fs.readFileSync(full, 'utf8'),
+ path.relative(CLI_ROOT, full).split(path.sep).join('/'),
+ ),
+ );
+ expect(offenders).toEqual([]);
+ });
+
+ it('catches every way around the rule', () => {
+ const doctor = 'api/doctor/doctor.mjs';
+ const discovery = '../../foundation/discovery/docs-discovery.mjs';
+ expect(
+ forbiddenImports(
+ `import {mergeTopic as merge} from '${discovery}'; merge({}, {});`,
+ doctor,
+ ),
+ ).toHaveLength(1);
+ expect(
+ forbiddenImports(
+ `import * as dd from '${discovery}'; dd['mergeTopic']({}, {});`,
+ doctor,
+ ),
+ ).toHaveLength(1);
+ expect(
+ forbiddenImports(
+ `async function f() { const dd = await import('${discovery}'); dd.mergeTopic.call(null, {}, {}); }`,
+ doctor,
+ ),
+ ).toHaveLength(1);
+ expect(
+ forbiddenImports(
+ `export {withSectionKeys} from '../../foundation/discovery/docs-section-key.mjs';`,
+ doctor,
+ ),
+ ).toHaveLength(1);
+ expect(
+ forbiddenImports(`import {DocsCatalog} from '${discovery}';`, doctor),
+ ).toEqual([]);
+ const leaf = 'api/docs/detail/detail.mjs';
+ const compiler = '../../../foundation/doc-compiler/compile.mjs';
+ expect(
+ forbiddenImports(`import {linkReferenceTopic} from '${compiler}';`, leaf),
+ ).toEqual([]);
+ expect(
+ forbiddenImports(
+ `import {lowerReferenceTopic} from '${compiler}';`,
+ leaf,
+ ),
+ ).toHaveLength(1);
+ });
+});
diff --git a/packages/cli/foundation/doc-compiler/ir.mjs b/packages/cli/foundation/doc-compiler/ir.mjs
new file mode 100644
index 0000000000..448ecfae4c
--- /dev/null
+++ b/packages/cli/foundation/doc-compiler/ir.mjs
@@ -0,0 +1,287 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file The compiled-node contract and its sealed parser.
+ *
+ * @input Any value that claims to be a compiled reference node — typically one
+ * read back from JSON.
+ * @output The same value once it validates; a thrown Error naming the problems
+ * otherwise. An unsupported schema version fails with its own message before
+ * anything else is checked.
+ * @position The load boundary for compiled nodes that did not come straight
+ * from ./compile.mjs in this process. The value is returned as given, not
+ * rebuilt, so key order (which response JSON follows) survives. A node is
+ * plain JSON throughout, so nothing it holds can surprise a reader.
+ */
+
+import {SECTION_KEY_RE} from '../discovery/docs-section-key.mjs';
+import {COMPILED_DOC_SCHEMA_VERSION} from './compile.mjs';
+
+const NODE_FIELDS = new Set([
+ 'schemaVersion',
+ 'kind',
+ 'stage',
+ 'id',
+ 'lang',
+ 'provenance',
+ 'sourceTitles',
+ 'doc',
+]);
+const RESOLVED_FIELDS = new Set([
+ 'status',
+ 'topic',
+ 'section',
+ 'previewType',
+ 'content',
+]);
+
+/** How many problems one message lists before it stops. */
+const MAX_PROBLEMS = 10;
+
+/** @param {unknown} value @returns {value is Record<string, any>} */
+const isRecord = value =>
+ value != null && typeof value === 'object' && !Array.isArray(value);
+
+/** @param {unknown} value @returns {value is string} */
+const isText = value => typeof value === 'string' && value !== '';
+
+/**
+ * A package name, never a location: provenance must not leak a path.
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+const isPackageName = value =>
+ isText(value) &&
+ !value.startsWith('/') &&
+ !value.startsWith('.') &&
+ !value.startsWith('\\') &&
+ !value.startsWith('file:') &&
+ !/^[A-Za-z]:[\\/]/.test(value);
+
+/**
+ * Validate a compiled reference node.
+ * @param {unknown} value
+ * @returns {import('./compile.mjs').CompiledReferenceNode}
+ */
+export function parseCompiledReferenceNode(value) {
+ const node = /** @type {any} */ (value);
+ if (node?.schemaVersion !== COMPILED_DOC_SCHEMA_VERSION) {
+ throw new Error(
+ `Compiled doc schema version ${JSON.stringify(node?.schemaVersion)} is not supported; this CLI reads version ${COMPILED_DOC_SCHEMA_VERSION}. Compile the docs again with this CLI.`,
+ );
+ }
+ const problems = jsonProblems(node, 'node');
+ if (problems.length === 0) problems.push(...structureProblems(node));
+ if (problems.length > 0) {
+ throw new Error(
+ `Invalid compiled doc node: ${problems.slice(0, MAX_PROBLEMS).join('; ')}`,
+ );
+ }
+ return node;
+}
+
+/**
+ * Where a value stops being plain JSON: anything but null, booleans, finite
+ * numbers, strings, arrays and plain objects; a symbol key; or a cycle.
+ * @param {unknown} value
+ * @param {string} at
+ * @param {Set<object>} [ancestors]
+ * @param {string[]} [out]
+ * @returns {string[]}
+ */
+function jsonProblems(value, at, ancestors = new Set(), out = []) {
+ if (out.length >= MAX_PROBLEMS) return out;
+ if (
+ value === null ||
+ typeof value === 'string' ||
+ typeof value === 'boolean'
+ ) {
+ return out;
+ }
+ if (typeof value === 'number') {
+ if (!Number.isFinite(value))
+ out.push(`${at}: ${value} is not a JSON number`);
+ return out;
+ }
+ if (value === undefined) {
+ out.push(`${at}: undefined is not JSON`);
+ return out;
+ }
+ if (typeof value !== 'object') {
+ out.push(`${at}: a ${typeof value} is not JSON`);
+ return out;
+ }
+ if (ancestors.has(value)) {
+ out.push(`${at}: refers back to itself`);
+ return out;
+ }
+ const proto = Object.getPrototypeOf(value);
+ if (!Array.isArray(value) && proto !== Object.prototype && proto !== null) {
+ out.push(
+ `${at}: a ${proto?.constructor?.name ?? 'non-plain object'} is not JSON`,
+ );
+ return out;
+ }
+ if (Object.getOwnPropertySymbols(value).length > 0) {
+ out.push(`${at}: has symbol keys`);
+ }
+ ancestors.add(value);
+ if (Array.isArray(value)) {
+ value.forEach((item, index) =>
+ jsonProblems(item, `${at}[${index}]`, ancestors, out),
+ );
+ } else {
+ for (const [key, item] of Object.entries(value)) {
+ jsonProblems(item, `${at}.${key}`, ancestors, out);
+ }
+ }
+ ancestors.delete(value);
+ return out;
+}
+
+/**
+ * The node's own shape, once it is known to be JSON.
+ * @param {Record<string, any>} node
+ * @returns {string[]}
+ */
+function structureProblems(node) {
+ /** @type {string[]} */
+ const problems = [];
+ const unknown = Object.keys(node).filter(key => !NODE_FIELDS.has(key));
+ if (unknown.length > 0)
+ problems.push(`unknown fields: ${unknown.join(', ')}`);
+ if (node.kind !== 'reference') problems.push('kind: expected "reference"');
+ if (node.stage !== 'lowered' && node.stage !== 'linked') {
+ problems.push('stage: expected "lowered" or "linked"');
+ }
+ if (!isText(node.id)) problems.push('id: expected a topic name');
+ if (node.lang !== null && !isText(node.lang)) {
+ problems.push('lang: expected a language or null');
+ }
+ const provenance = node.provenance;
+ if (
+ !isRecord(provenance) ||
+ !isPackageName(provenance.provider) ||
+ (provenance.replaces !== null && !isText(provenance.replaces)) ||
+ !Array.isArray(provenance.extensions) ||
+ !provenance.extensions.every(isPackageName)
+ ) {
+ problems.push(
+ 'provenance: expected {provider, replaces, extensions} naming packages, not paths',
+ );
+ }
+ const titles = isRecord(node.sourceTitles) ? node.sourceTitles : null;
+ if (!titles || !Object.values(titles).every(isText)) {
+ problems.push('sourceTitles: expected section key -> authored title');
+ }
+ const doc = node.doc;
+ if (
+ !isRecord(doc) ||
+ !isText(doc.name) ||
+ !isText(doc.title) ||
+ typeof doc.description !== 'string' ||
+ !Array.isArray(doc.sections) ||
+ doc.sections.length === 0
+ ) {
+ problems.push('doc: expected {name, title, description, sections}');
+ } else {
+ problems.push(...sectionProblems(doc.sections, titles, node.stage));
+ }
+ return problems;
+}
+
+/**
+ * @param {any[]} sections
+ * @param {Record<string, any> | null} titles
+ * @param {unknown} stage
+ * @returns {string[]}
+ */
+function sectionProblems(sections, titles, stage) {
+ /** @type {string[]} */
+ const problems = [];
+ const seen = new Set();
+ sections.forEach((section, index) => {
+ const at = `doc.sections[${index}]`;
+ if (
+ !isRecord(section) ||
+ typeof section.id !== 'string' ||
+ !SECTION_KEY_RE.test(section.id)
+ ) {
+ problems.push(`${at}.id: expected a section key`);
+ return;
+ }
+ if (seen.has(section.id)) {
+ problems.push(`${at}.id: two sections have the key "${section.id}"`);
+ }
+ seen.add(section.id);
+ if (!isText(section.title)) problems.push(`${at}.title: expected a title`);
+ if (titles && !Object.hasOwn(titles, section.id)) {
+ problems.push(
+ `sourceTitles: no authored title for section "${section.id}"`,
+ );
+ }
+ if (!Array.isArray(section.content)) {
+ problems.push(`${at}.content: expected an array of blocks`);
+ return;
+ }
+ section.content.forEach((/** @type {unknown} */ block, blockIndex) => {
+ const where = `${at}.content[${blockIndex}]`;
+ if (!isRecord(block) || !isText(block.type)) {
+ problems.push(`${where}: expected a block with a type`);
+ return;
+ }
+ if (block.type !== 'token-ref') return;
+ const ref = `${where}: token reference to "${block.topic}"`;
+ if (stage === 'lowered') {
+ if ('resolved' in block) {
+ problems.push(`${ref}: a lowered node carries no resolution`);
+ }
+ return;
+ }
+ if (!('resolved' in block)) {
+ problems.push(`${ref}: a linked node resolves every reference`);
+ return;
+ }
+ const problem = resolutionProblem(block.resolved);
+ if (problem) problems.push(`${ref}: ${problem}`);
+ });
+ });
+ return problems;
+}
+
+/**
+ * @param {unknown} resolved
+ * @returns {string | null}
+ */
+function resolutionProblem(resolved) {
+ if (!isRecord(resolved)) return 'expected a resolution';
+ switch (resolved.status) {
+ case 'unknown-topic':
+ case 'unknown-section':
+ return Object.keys(resolved).length === 1 ? null : 'unexpected fields';
+ case 'resolved': {
+ if (!isText(resolved.topic)) return 'topic: expected a topic name';
+ if (
+ typeof resolved.section !== 'string' ||
+ !SECTION_KEY_RE.test(resolved.section)
+ ) {
+ return 'section: expected a section key';
+ }
+ if ('previewType' in resolved && !isText(resolved.previewType)) {
+ return 'previewType: expected a preview type';
+ }
+ if (
+ !Array.isArray(resolved.content) ||
+ !resolved.content.every(block => isRecord(block) && isText(block.type))
+ ) {
+ return 'content: expected blocks with a type';
+ }
+ const extra = Object.keys(resolved).filter(
+ key => !RESOLVED_FIELDS.has(key),
+ );
+ return extra.length > 0 ? `unexpected fields: ${extra.join(', ')}` : null;
+ }
+ default:
+ return `status: expected resolved, unknown-topic or unknown-section, got ${JSON.stringify(resolved.status)}`;
+ }
+}
diff --git a/packages/cli/foundation/doc-compiler/lenses.mjs b/packages/cli/foundation/doc-compiler/lenses.mjs
new file mode 100644
index 0000000000..ced72373f5
--- /dev/null
+++ b/packages/cli/foundation/doc-compiler/lenses.mjs
@@ -0,0 +1,127 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Lenses — the docs API's response shapes, read off compiled nodes.
+ *
+ * @input A compiled reference node from ./compile.mjs: lowered for the index
+ * and for section lookup, linked for anything that inlines token references.
+ * @output The `docs.detail` topic, the `docs.index` section index, one
+ * `docs.detail.section` section, and the sections as readers look them up.
+ * Every view is a fresh copy, so a reader may edit what it gets back without
+ * touching the node, which other reads of the same catalog share.
+ * @position Between the compiler and api/docs. A lens only projects: it never
+ * loads, merges, overlays, keys, or resolves a reference itself.
+ */
+
+import {
+ buildDocsIndexData,
+ withSourceTitle,
+} from '../discovery/docs-section-key.mjs';
+
+/**
+ * @param {import('./compile.mjs').CompiledReferenceNode} node
+ * @param {any} section
+ * @returns {string}
+ */
+function authoredTitle(node, section) {
+ return node.sourceTitles[section.id] ?? section.title;
+}
+
+/**
+ * The node's sections as readers look them up: each one knows its authored
+ * title, so a query in the authoring language finds a translated section.
+ * For lookup only; a section a reader gets back comes from
+ * {@link sectionView}.
+ * @param {import('./compile.mjs').CompiledReferenceNode} node
+ * @returns {any[]}
+ */
+export function readerSections(node) {
+ return node.doc.sections.map((/** @type {any} */ section) =>
+ withSourceTitle({...section}, authoredTitle(node, section)),
+ );
+}
+
+/**
+ * `docs.detail`: the whole topic, with every token reference inlined.
+ * @param {import('./compile.mjs').CompiledReferenceNode} node a linked node
+ * @returns {any}
+ */
+export function detailView(node) {
+ if (node.stage !== 'linked') {
+ throw new Error(
+ `"${node.id}" must be linked before its whole doc is read.`,
+ );
+ }
+ // Assigning `sections` keeps it where the authored doc put it.
+ const view = structuredClone({...node.doc, sections: []});
+ view.sections = node.doc.sections.map((/** @type {any} */ section) =>
+ sectionView(node, section),
+ );
+ return view;
+}
+
+/**
+ * `docs.index`: what the topic is, and each section's key, title and summary.
+ * @param {import('./compile.mjs').CompiledReferenceNode} node
+ * @returns {import('../../api/docs/docs.type.mjs').DocsIndex}
+ */
+export function indexView(node) {
+ return buildDocsIndexData(node.doc);
+}
+
+/**
+ * `docs.detail.section`: one section with its token references inlined. A
+ * referenced section's content takes the reference's place; the section takes
+ * the preview type of the last reference that has one, unless it has its own.
+ * @param {import('./compile.mjs').CompiledReferenceNode} node
+ * @param {any} section a linked section of `node`
+ * @returns {any}
+ */
+export function sectionView(node, section) {
+ /** @type {any[]} */
+ const content = [];
+ /** @type {string | null} */
+ let previewType = null;
+ for (const block of section.content) {
+ if (block?.type !== 'token-ref') {
+ content.push(structuredClone(block));
+ continue;
+ }
+ const target = block.resolved;
+ if (target == null) {
+ throw new Error(
+ `The token reference to "${block.topic}" in "${node.id}" was read before it was linked.`,
+ );
+ }
+ if (target.status === 'unknown-topic') {
+ content.push({
+ type: 'prose',
+ text: `[token-ref: unknown topic "${block.topic}"]`,
+ });
+ continue;
+ }
+ if (target.status === 'unknown-section') {
+ content.push({
+ type: 'prose',
+ text: `[token-ref: section "${block.section}" not found in "${block.topic}"]`,
+ });
+ continue;
+ }
+ // A copy per reference: two references to one section share nothing.
+ for (const refBlock of target.content) {
+ content.push(structuredClone(refBlock));
+ }
+ if (target.previewType && !section.previewType) {
+ previewType = target.previewType;
+ }
+ }
+ // Assigning `content` keeps it where the section put it; a carried preview
+ // type lands after the section's own keys, as it always has.
+ const view = structuredClone(
+ previewType == null
+ ? {...section, content: []}
+ : {...section, previewType, content: []},
+ );
+ view.content = content;
+ return withSourceTitle(view, authoredTitle(node, section));
+}