Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

chore(docs): drop the verify scripts and their dependencies

Hampuscf662fa01622
diff --git a/fluxer_docs/package.json b/fluxer_docs/package.json
index 035befa15..c993b2943 100644
--- a/fluxer_docs/package.json
+++ b/fluxer_docs/package.json
@@ -7,12 +7,7 @@
"build": "astro build",
"dev": "astro dev",
"start": "tsx scripts/StartServer.ts",
- "typecheck": "astro check --minimumFailingSeverity warning",
- "verify": "pnpm verify:sidebar && pnpm verify:coverage && pnpm verify:schemas && pnpm verify:style && pnpm verify:commands",
- "verify:coverage": "tsx scripts/VerifyDocsCoverage.ts",
- "verify:schemas": "tsx scripts/VerifyDocsSchemas.ts",
- "verify:sidebar": "tsx scripts/VerifyDocsSidebar.ts",
- "verify:style": "tsx scripts/VerifyDocsStyle.ts"
+ "typecheck": "astro check --minimumFailingSeverity warning"
},
"dependencies": {
"@astrojs/markdown-remark": "catalog:",
@@ -29,7 +24,6 @@
"devDependencies": {
"@astrojs/check": "catalog:",
"@types/hast": "catalog:",
- "@types/mdast": "catalog:",
"@types/node": "catalog:",
"@types/send": "catalog:",
"mdast-util-to-hast": "catalog:",
diff --git a/fluxer_docs/scripts/DocsRouteHeaders.ts b/fluxer_docs/scripts/DocsRouteHeaders.ts
deleted file mode 100644
index 7ad1ecbe3..000000000
--- a/fluxer_docs/scripts/DocsRouteHeaders.ts
+++ /dev/null
@@ -1,120 +0,0 @@
-import assert from 'node:assert/strict';
-import {parseFrontmatter} from '@astrojs/markdown-remark';
-import {createProcessor} from '@mdx-js/mdx';
-import type {Nodes, Root} from 'mdast';
-import type {MdxJsxAttribute, MdxJsxFlowElement, MdxJsxTextElement} from 'mdast-util-mdx-jsx';
-import remarkGfm from 'remark-gfm';
-import {HTTP_METHODS, type MarkdownPage} from './DocsSource.ts';
-
-export interface DocsRouteHeader {
- readonly method: string;
- readonly path: string;
- readonly file: string;
- readonly bot: boolean;
- readonly unauthenticated: boolean;
- readonly line: number;
- readonly endLine: number;
-}
-
-const STRING_ATTRIBUTES = new Set(['method', 'path', 'oauth2']);
-const BOOLEAN_ATTRIBUTES = new Set(['bot', 'unauthenticated', 'auditReason', 'mfa']);
-const markdownProcessor = createProcessor({format: 'md', remarkPlugins: [remarkGfm]});
-const mdxProcessor = createProcessor({format: 'mdx', remarkPlugins: [remarkGfm]});
-
-function readAttributeValue(attribute: MdxJsxAttribute, location: string): string | boolean {
- const value = attribute.value;
- if (value === null || value === undefined) return true;
- if (typeof value === 'string') return value;
- const body = value.data?.estree?.body;
- if (body?.length === 1 && body[0].type === 'ExpressionStatement') {
- const expression = body[0].expression;
- if (
- expression.type === 'Literal' &&
- (typeof expression.value === 'string' || typeof expression.value === 'boolean')
- ) {
- return expression.value;
- }
- if (
- expression.type === 'TemplateLiteral' &&
- expression.expressions.length === 0 &&
- expression.quasis.length === 1 &&
- typeof expression.quasis[0].value.cooked === 'string'
- ) {
- return expression.quasis[0].value.cooked;
- }
- }
- throw new Error(`${location}: RouteHeader ${attribute.name} requires a literal string or boolean`);
-}
-
-function readHeader(node: MdxJsxFlowElement | MdxJsxTextElement, file: string): DocsRouteHeader {
- assert(node.position !== undefined, 'Parsed RouteHeader has no source position');
- const line = node.position.start.line;
- const endLine = node.position.end.line;
- assert(Number.isInteger(line) && line > 0 && Number.isInteger(endLine) && endLine >= line);
- const location = `${file}:${line}`;
- const attributes = new Map<string, string | boolean>();
- for (const attribute of node.attributes) {
- if (attribute.type === 'mdxJsxExpressionAttribute') {
- throw new Error(`${location}: RouteHeader spread attributes are unsupported`);
- }
- const name = attribute.name;
- if (attributes.has(name)) {
- throw new Error(`${location}: duplicate RouteHeader attribute ${name}`);
- }
- const isString = STRING_ATTRIBUTES.has(name);
- if (!isString && !BOOLEAN_ATTRIBUTES.has(name)) {
- throw new Error(`${location}: unknown RouteHeader attribute ${name}`);
- }
- const value = readAttributeValue(attribute, location);
- if (isString ? typeof value !== 'string' : typeof value !== 'boolean') {
- throw new Error(`${location}: RouteHeader ${name} must be ${isString ? 'a string' : 'a boolean'}`);
- }
- attributes.set(name, value);
- }
- const method = attributes.get('method');
- const routePath = attributes.get('path');
- if (typeof method !== 'string' || typeof routePath !== 'string') {
- throw new Error(`${location}: RouteHeader requires method and path attributes`);
- }
- if (!HTTP_METHODS.has(method)) {
- throw new Error(`${location}: unsupported RouteHeader method ${method}`);
- }
- if (!routePath.startsWith('/')) {
- throw new Error(`${location}: RouteHeader path must be absolute`);
- }
- return {
- method,
- path: routePath,
- file,
- bot: attributes.get('bot') === true,
- unauthenticated: attributes.get('unauthenticated') === true,
- line,
- endLine,
- };
-}
-
-export function readRouteHeaders(page: MarkdownPage): Array<DocsRouteHeader> {
- let tree: Root;
- try {
- const {content} = parseFrontmatter(page.source, {frontmatter: 'empty-with-spaces'});
- const processor = page.file.endsWith('.mdx') ? mdxProcessor : markdownProcessor;
- tree = processor.parse({value: content, path: page.file});
- } catch (error) {
- throw new Error(`${page.relativePath}: ${String(error)}`, {cause: error});
- }
- const headers: Array<DocsRouteHeader> = [];
- const pending: Array<Nodes> = [tree];
- while (pending.length > 0) {
- const node = pending.pop();
- assert(node !== undefined);
- if ((node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') && node.name === 'RouteHeader') {
- headers.push(readHeader(node, page.relativePath));
- }
- if ('children' in node) {
- for (let index = node.children.length - 1; index >= 0; index -= 1) {
- pending.push(node.children[index]);
- }
- }
- }
- return headers;
-}
diff --git a/fluxer_docs/scripts/DocsSource.ts b/fluxer_docs/scripts/DocsSource.ts
deleted file mode 100644
index 862547ffc..000000000
--- a/fluxer_docs/scripts/DocsSource.ts
+++ /dev/null
@@ -1,119 +0,0 @@
-import {readdir, readFile} from 'node:fs/promises';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-import {parseFrontmatter} from '@astrojs/markdown-remark';
-
-export const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url));
-export const HTTP_METHODS: ReadonlySet<string> = new Set(['GET', 'HEAD', 'POST', 'PATCH', 'PUT', 'DELETE', 'OPTIONS']);
-
-export interface MarkdownPage {
- readonly file: string;
- readonly relativePath: string;
- readonly source: string;
- readonly lines: ReadonlyArray<string>;
-}
-
-async function collectMarkdownFiles(directory: string): Promise<Array<string>> {
- const files: Array<string> = [];
- for (const entry of await readdir(directory, {withFileTypes: true})) {
- if (entry.name === 'node_modules') {
- continue;
- }
- const resolved = path.join(directory, entry.name);
- if (entry.isDirectory()) {
- files.push(...(await collectMarkdownFiles(resolved)));
- } else if (entry.isFile() && /\.mdx?$/u.test(entry.name)) {
- files.push(resolved);
- }
- }
- return files.sort();
-}
-
-function isDraftSource(source: string): boolean {
- return parseFrontmatter(source).frontmatter.draft === true;
-}
-
-async function partitionMarkdownFiles(directory: string, drafts: boolean): Promise<Array<string>> {
- const files: Array<string> = [];
- for (const file of await collectMarkdownFiles(directory)) {
- if (isDraftSource(await readFile(file, 'utf8')) === drafts) {
- files.push(file);
- }
- }
- return files;
-}
-
-export async function listMarkdownFiles(directory: string): Promise<Array<string>> {
- return partitionMarkdownFiles(directory, false);
-}
-
-export async function listDraftMarkdownFiles(directory: string): Promise<Array<string>> {
- return partitionMarkdownFiles(directory, true);
-}
-
-export async function readMarkdownPages(
- directory: string,
- options: {includeDrafts?: boolean} = {},
-): Promise<Array<MarkdownPage>> {
- const pages: Array<MarkdownPage> = [];
- const files = options.includeDrafts ? await collectMarkdownFiles(directory) : await listMarkdownFiles(directory);
- for (const file of files) {
- const source = await readFile(file, 'utf8');
- pages.push({
- file,
- relativePath: path.relative(directory, file).split(path.sep).join('/'),
- source,
- lines: source.split('\n'),
- });
- }
- return pages;
-}
-
-export function splitTableRow(line: string): Array<string> {
- const row = line.trim();
- const cells: Array<string> = [];
- let start = row.startsWith('|') ? 1 : 0;
- for (let index = start; index < row.length; index += 1) {
- if (row[index] === '\\' && (row[index + 1] === '\\' || row[index + 1] === '|')) {
- index += 1;
- continue;
- }
- if (row[index] === '|') {
- cells.push(row.slice(start, index).trim());
- start = index + 1;
- }
- }
- if (start < row.length) {
- cells.push(row.slice(start).trim());
- }
- return cells;
-}
-
-export function routeShape(method: string, routePath: string): string {
- return `${method} ${routePath.split('?')[0].replace(/\{[^}]*\}/gu, '{}')}`;
-}
-
-export function slugifyHeading(heading: string): string {
- return heading
- .replace(/`/gu, '')
- .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1')
- .replace(/<[^>]*>/gu, '')
- .toLowerCase()
- .replace(/[^a-z0-9\s-]/gu, '')
- .trim()
- .replace(/\s+/gu, '-');
-}
-
-export function stripVersion(routePath: string): string {
- if (routePath.startsWith('/v1/')) {
- return routePath.slice(3);
- }
- return routePath;
-}
-
-export function pageSlug(relativePath: string): string {
- return relativePath
- .replace(/\.(mdx|md)$/u, '')
- .replace(/\/index$/u, '')
- .replace(/^index$/u, '');
-}
diff --git a/fluxer_docs/scripts/DocsTableWidth.ts b/fluxer_docs/scripts/DocsTableWidth.ts
deleted file mode 100644
index a0d5d2928..000000000
--- a/fluxer_docs/scripts/DocsTableWidth.ts
+++ /dev/null
@@ -1,226 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-import {
- columnWidthPercents,
- maxContentPx,
- minContentPx,
- type Piece,
- PX,
- TABLE_BUDGET_PX,
- TABLE_FLOOR_CAP_SHARE,
- TABLE_MAX_CELL_CHARS,
- TABLE_MAX_COLUMNS,
- TABLE_MAX_IDENT_CHARS,
- TABLE_WIDE_TIER_PX,
-} from '@/table/DocsTableMetrics.ts';
-import {splitTableRow} from './DocsSource.ts';
-
-export {
- columnWidthPercents,
- PX,
- TABLE_BUDGET_PX,
- TABLE_FLOOR_CAP_SHARE,
- TABLE_MAX_CELL_CHARS,
- TABLE_MAX_COLUMNS,
- TABLE_MAX_IDENT_CHARS,
- TABLE_WIDE_TIER_PX,
-};
-
-const CANONICAL_SHAPES = new Set([
- 'field|type|description',
- 'status|body|condition',
- 'value|name|description',
- 'value|description',
-]);
-
-const PARALLEL_SENTINELS = new Set(['unset', 'empty', 'none', 'required', 'n/a', '-', 'same target', 'no default']);
-
-export interface DocsTable {
- readonly line: number;
- readonly endLine: number;
- readonly columns: number;
- readonly rows: number;
- readonly header: ReadonlyArray<string>;
- readonly canonical: boolean;
- readonly minDemandPx: number;
- readonly columnMinPx: ReadonlyArray<number>;
- readonly columnMaxPx: ReadonlyArray<number>;
- readonly worstCellChars: number;
- readonly worstCellColumn: number;
- readonly worstCellText: string;
- readonly col1IdentChars: number;
- readonly col1IdentText: string;
- readonly parallel: boolean;
- readonly nonParallelReason: string;
-}
-
-function isDelimiter(line: string, columns: number): boolean {
- const cells = splitTableRow(line);
- return cells.length === columns && cells.every((cell) => /^:?-+:?$/u.test(cell));
-}
-
-function stripInline(text: string): string {
- return text
- .replace(/!\[([^\]]*)\]\([^)]*\)/gu, '$1')
- .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1')
- .replace(/<sup>([^<]*)<\/sup>/gu, '$1')
- .replace(/<br\s*\/?>/giu, ' ')
- .replace(/<[^>]+>/gu, '')
- .replace(/\*\*([^*]*)\*\*/gu, '$1')
- .replace(/\*([^*]*)\*/gu, '$1')
- .replace(/&nbsp;/gu, ' ')
- .replace(/&lt;/gu, '<')
- .replace(/&gt;/gu, '>')
- .replace(/&amp;/gu, '&')
- .replace(/\\([!-/:-@[-`{-~])/gu, '$1');
-}
-
-function pieces(cell: string): Array<Piece> {
- const out: Array<Piece> = [];
- const pattern = /`([^`]*)`/gu;
- let last = 0;
- let match: RegExpExecArray | null = pattern.exec(cell);
- while (match != null) {
- if (match.index > last) {
- out.push({text: stripInline(cell.slice(last, match.index)), code: false});
- }
- out.push({
- text: match[1].replace(/\\([\\|])/gu, (escaped, character: string) => (character === '|' ? character : escaped)),
- code: true,
- });
- last = pattern.lastIndex;
- match = pattern.exec(cell);
- }
- if (last < cell.length) {
- out.push({text: stripInline(cell.slice(last)), code: false});
- }
- return out.filter((piece) => piece.text.length > 0);
-}
-
-function renderedText(parts: ReadonlyArray<Piece>): string {
- return parts.map((piece) => piece.text).join('');
-}
-
-function cellKind(cell: string): string {
- const stripped = cell.replace(/<sup>[^<]*<\/sup>/gu, '');
- const parts = pieces(stripped);
- if (parts.length === 0) {
- return 'empty';
- }
- if (parts.every((piece) => piece.code || /^[\s,]+$/u.test(piece.text))) {
- return 'code';
- }
- if (parts.every((piece) => !piece.code)) {
- return /[.!?]\s|[.!?]$/u.test(stripped) ? 'sentence' : 'phrase';
- }
- return 'mixed';
-}
-
-export function extractTables(source: string): Array<DocsTable> {
- const lines = source.split('\n');
- const tables: Array<DocsTable> = [];
- let fenced = false;
- let index = 0;
- while (index < lines.length) {
- if (/^\s*(?:```|~~~)/u.test(lines[index])) {
- fenced = !fenced;
- index += 1;
- continue;
- }
- if (fenced) {
- index += 1;
- continue;
- }
- if (!lines[index].trim().startsWith('|') || index + 1 >= lines.length) {
- index += 1;
- continue;
- }
- const header = splitTableRow(lines[index]);
- if (header.length === 0 || !isDelimiter(lines[index + 1], header.length)) {
- index += 1;
- continue;
- }
- const start = index;
- index += 2;
- const body: Array<Array<string>> = [];
- while (index < lines.length && lines[index].trim().startsWith('|')) {
- body.push(splitTableRow(lines[index]));
- index += 1;
- }
- const columns = header.length;
- const columnMinPx = new Array<number>(columns).fill(0);
- const columnMaxPx = new Array<number>(columns).fill(0);
- let worstCellChars = 0;
- let worstCellColumn = 0;
- let worstCellText = '';
- let col1IdentChars = 0;
- let col1IdentText = '';
- for (const [rowIndex, row] of [header, ...body].entries()) {
- const isHeader = rowIndex === 0;
- for (let column = 0; column < columns; column += 1) {
- const parts = pieces(isHeader ? (row[column] ?? '').toUpperCase() : (row[column] ?? ''));
- const minimum = minContentPx(parts);
- const maximum = maxContentPx(parts, isHeader);
- if (maximum > columnMaxPx[column]) {
- columnMaxPx[column] = maximum;
- }
- if (minimum > columnMinPx[column]) {
- columnMinPx[column] = minimum;
- }
- const text = renderedText(parts);
- if (!isHeader && text.length > worstCellChars) {
- worstCellChars = text.length;
- worstCellColumn = column;
- worstCellText = text;
- }
- if (column === 0 && !isHeader) {
- for (const run of text.split(/(?<=\s)|(?<=-)(?!\d)/u)) {
- const token = run.trim();
- if (token.length > col1IdentChars) {
- col1IdentChars = token.length;
- col1IdentText = token;
- }
- }
- }
- }
- }
- const headerShape = header.map((cell) => cell.trim().toLowerCase()).join('|');
- let parallel = true;
- let nonParallelReason = '';
- for (let column = 0; column < columns && parallel; column += 1) {
- if (headerShape === 'status|body|condition' && column === 1) {
- continue;
- }
- const shape = body
- .map((row) => (row[column] ?? '').trim())
- .filter((cell) => cell.length > 0 && !PARALLEL_SENTINELS.has(cell.toLowerCase()))
- .map(cellKind);
- const kinds = new Set(shape);
- kinds.delete('empty');
- const prose = kinds.has('sentence') || kinds.has('phrase') || kinds.has('mixed');
- if (kinds.has('code') && prose && shape.length > 2) {
- parallel = false;
- nonParallelReason = `column ${(column + 1).toString()} mixes code-only cells with prose cells`;
- }
- }
- tables.push({
- line: start + 1,
- endLine: index,
- columns,
- rows: body.length,
- header,
- canonical: CANONICAL_SHAPES.has(headerShape),
- minDemandPx: Math.round(columnMinPx.reduce((a, b) => a + b, 0) + columns * PX.cellPad),
- columnMinPx: columnMinPx.map((value) => Math.round(value)),
- columnMaxPx: columnMaxPx.map((value) => Math.round(value)),
- worstCellChars,
- worstCellColumn,
- worstCellText,
- col1IdentChars,
- col1IdentText,
- parallel,
- nonParallelReason,
- });
- }
- return tables;
-}
diff --git a/fluxer_docs/scripts/VerifyDocsCoverage.ts b/fluxer_docs/scripts/VerifyDocsCoverage.ts
deleted file mode 100644
index 7b1081ab6..000000000
--- a/fluxer_docs/scripts/VerifyDocsCoverage.ts
+++ /dev/null
@@ -1,2482 +0,0 @@
-import {stripVersion as stripVersionPrefix} from './DocsSource.ts';
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-import {spawnSync} from 'node:child_process';
-import {createHash} from 'node:crypto';
-import type {Dirent} from 'node:fs';
-import {mkdir, mkdtemp, readdir, readFile, rm, writeFile} from 'node:fs/promises';
-import {tmpdir} from 'node:os';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-import {parseArgs} from 'node:util';
-import {extractRoutesFromControllers} from '@fluxer/openapi/src/extractors/RouteExtractor';
-import type {OpenAPIDocument} from '@fluxer/openapi/src/OpenAPITypes';
-import {installerChecksumLine} from '@/installer/InstallerDigest.ts';
-import {type DocsRouteHeader, readRouteHeaders} from './DocsRouteHeaders.ts';
-import {DOCS_ROOT, HTTP_METHODS, type MarkdownPage, readMarkdownPages, routeShape} from './DocsSource.ts';
-
-const {values: options} = parseArgs({options: {'source-only': {type: 'boolean', default: false}}});
-
-const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url));
-const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json');
-const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json');
-const MEDIA_PROXY_SERVER_DIR = path.join(REPO_ROOT, 'fluxer_media_proxy/src/server');
-
-const BLUESKY_OAUTH_CONTROLLER = 'fluxer_api/src/api/bluesky/BlueskyOAuthController.ts';
-const DOWNLOAD_CONTROLLER = 'fluxer_api/src/api/download/DownloadController.ts';
-
-const MAIN_SPEC_EXEMPT = new Map<string, {file: string; anchor: string; reason: string}>([
- [
- 'GET /oauth2/authorize',
- {
- file: 'fluxer_api/src/api/oauth/OAuth2Controller.ts',
- anchor: "'/oauth2/authorize',",
- reason: 'the generator excludes the path outright, at OpenAPIGeneratorCatalog excluded.paths',
- },
- ],
- [
- 'GET /openapi.json',
- {
- file: 'fluxer_api/src/api/openapi/OpenAPIController.ts',
- anchor: "app.get('/openapi.json'",
- reason: 'the handler serves the spec file itself and has no OpenAPI({...}) block',
- },
- ],
- [
- 'GET /connections/bluesky/client-metadata.json',
- {
- file: BLUESKY_OAUTH_CONTROLLER,
- anchor: "'/connections/bluesky/client-metadata.json'",
- reason: 'an AT Protocol client document with no OpenAPI({...}) block',
- },
- ],
- [
- 'GET /connections/bluesky/jwks.json',
- {
- file: BLUESKY_OAUTH_CONTROLLER,
- anchor: "'/connections/bluesky/jwks.json'",
- reason: 'an AT Protocol client document with no OpenAPI({...}) block',
- },
- ],
-
- [
- 'GET /dl/desktop/{}/{}/{}/latest/{}.sha256',
- {
- file: DOWNLOAD_CONTROLLER,
- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\\\.sha256}`,',
- reason: 'the path constrains :format by regex and has no OpenAPI path template',
- },
- ],
- [
- 'GET /dl/desktop/{}/{}/{}/{}/{}.sha256',
- {
- file: DOWNLOAD_CONTROLLER,
- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\\\.sha256}`,',
- reason: 'the path constrains :format by regex and has no OpenAPI path template',
- },
- ],
- [
- 'GET /dl/desktop/{}/{}/{}/latest/{}.zsync',
- {
- file: DOWNLOAD_CONTROLLER,
- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\\\.zsync}`,',
- reason: 'the path constrains :format by regex and has no OpenAPI path template',
- },
- ],
- [
- 'GET /dl/desktop/{}/{}/{}/{}/{}.zsync',
- {
- file: DOWNLOAD_CONTROLLER,
- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\\\.zsync}`,',
- reason: 'the path constrains :format by regex and has no OpenAPI path template',
- },
- ],
- [
- 'GET /dl/{}',
- {
- file: DOWNLOAD_CONTROLLER,
- anchor: '`${DOWNLOAD_PREFIX}/*`,',
- reason: 'a Hono wildcard, which is not an OpenAPI path template',
- },
- ],
-]);
-
-const DELIBERATELY_UNDOCUMENTED = new Map([
- ['GET /users/@me/mobile-devices', 'mobile notifications, backported separately'],
- ['DELETE /users/@me/mobile-devices/{}', 'mobile notifications, backported separately'],
- ['GET /users/@me/push/subscriptions', 'push API, backported separately'],
- ['POST /users/@me/push/subscribe', 'push API, backported separately'],
- ['POST /users/@me/push/rotate', 'push API, backported separately'],
- ['DELETE /users/@me/push/subscriptions/{}', 'push API, backported separately'],
-]);
-
-interface OutOfBandRoute {
- readonly reason: string;
- readonly documentedIn: {readonly file: string; readonly anchor: string} | null;
-}
-const OUT_OF_BAND_CREDENTIAL = new Map<string, OutOfBandRoute>([
- [
- 'POST /internal/rpc',
- {
- reason:
- 'x-fluxer-rpc-auth must timing-safe-equal Config.internal.gatewayRpcAuthToken, and an empty token rejects every caller. The caller is fluxer_gateway',
- documentedIn: null,
- },
- ],
- [
- 'POST /webhooks/livekit',
- {
- reason: 'a LiveKit-signed callback whose shipped target is the internal address http://api:8080/webhooks/livekit',
- documentedIn: {file: 'operator/configuration.mdx', anchor: 'http://api:8080/webhooks/livekit'},
- },
- ],
- [
- 'POST /webhooks/sweego',
- {
- reason:
- 'a Standard Webhooks delivery signed with FLUXER_EMAIL_WEBHOOK_SECRET, which answers 404 Email not enabled while email is off',
- documentedIn: {file: 'operator/configuration.mdx', anchor: 'POST /webhooks/sweego'},
- },
- ],
- [
- 'POST /webhooks/app-store',
- {
- reason:
- 'an App Store Server Notification whose signedPayload must verify against the pinned Apple root before it is queued. The route is hosted-only',
- documentedIn: {file: 'http-api/in-app-purchases.mdx', anchor: 'POST /webhooks/app-store'},
- },
- ],
- [
- 'POST /webhooks/google-play',
- {
- reason:
- 'a Pub/Sub push whose Google-signed OIDC token must match the configured audience and service account before it is queued. The route is hosted-only',
- documentedIn: {file: 'http-api/in-app-purchases.mdx', anchor: 'POST /webhooks/google-play'},
- },
- ],
- [
- 'GET /connections/bluesky/callback',
- {
- reason:
- 'the atproto redirect URI, reached with a provider-issued authorisation code. A client MUST NOT call it directly',
- documentedIn: {file: 'http-api/connections.mdx', anchor: '/connections/bluesky/callback'},
- },
- ],
-]);
-
-const HEALTH_AND_METRICS_PATHS = new Set(['/_health', '/_health/ready', '/_health/drain', '/_healthz', '/_metrics']);
-
-const LEGACY_REPORT_ROUTES = new Set(['POST /reports/message', 'POST /reports/user', 'POST /reports/guild']);
-
-interface ExemptionRule {
- readonly name: string;
- readonly justification: string;
- readonly mayCoverNothing?: boolean;
-
- readonly anchors: ReadonlyArray<{readonly file: string; readonly anchor: string}>;
- readonly covers: (shape: string, routePath: string) => boolean;
-}
-const EXEMPTION_RULES: ReadonlyArray<ExemptionRule> = [
- {
- name: 'test harness',
- justification:
- 'ControllerRegistry registers TestHarnessController only when config.dev.testModeEnabled or NODE_ENV is development, and ensureHarnessAccess rechecks that per request. No production instance serves a /test/ route',
- anchors: [
- {
- file: 'fluxer_api/src/api/app/ControllerRegistry.ts',
- anchor: "if (config.dev.testModeEnabled || config.nodeEnv === 'development') {",
- },
- {file: 'fluxer_api/src/api/test/TestHarnessController.ts', anchor: 'function ensureHarnessAccess'},
- ],
- covers: (_shape, routePath) => routePath.startsWith('/test/'),
- },
- {
- name: 'deprecated desktop download redirect',
- justification:
- 'every /dl route is an undocumented deprecated redirect onto pkgs.fluxer.com, kept only for desktop clients already in the field. Nothing current calls one, so documenting them would advertise a path new callers must not use',
- anchors: [
- {
- file: 'fluxer_api/src/api/download/DownloadController.ts',
- anchor: 'function redirectToPackageOrigin',
- },
- ],
- covers: (_shape, routePath) => routePath === '/dl' || routePath.startsWith('/dl/'),
- },
- {
- name: 'backported separately',
- justification:
- 'part of a system this backport does not ship. Each shape is listed with its system, and a page that documents one fails the run below',
- anchors: [],
- covers: (shape) => DELIBERATELY_UNDOCUMENTED.has(shape),
- },
- {
- name: 'process health and metrics',
- justification:
- 'loopback gated or reachable only inside the compose network. operator/reverse-proxy.mdx states which probes answer through a proxy once for every service, rather than per route',
- anchors: [{file: 'fluxer_docs/src/content/docs/operator/reverse-proxy.mdx', anchor: '/_metrics'}],
- covers: (_shape, routePath) => HEALTH_AND_METRICS_PATHS.has(routePath),
- },
- {
- name: 'out-of-band credential',
- justification:
- 'no ordinary client holds the credential. Each entry states its guard, and five are covered in prose',
- anchors: [{file: 'fluxer_api/src/api/app/ControllerRegistry.ts', anchor: 'InternalRpcController(routes);'}],
- covers: (shape) => OUT_OF_BAND_CREDENTIAL.has(shape),
- },
- {
- name: 'legacy report routes',
- justification:
- 'still served for clients released before report flows, which new callers must use instead. The generator drops them at OpenAPIGeneratorCatalog excluded.paths',
- anchors: [{file: 'packages/openapi/src/generator/OpenAPIGeneratorCatalog.ts', anchor: "'/reports/message'"}],
- covers: (shape) => LEGACY_REPORT_ROUTES.has(shape),
- },
-];
-
-const MEDIA_PROXY_ROUTES = new Map([
- ['GET /_health', '.route("/_health", get(routes::ops::health))'],
- ['HEAD /_health', '.route("/_health", get(routes::ops::health))'],
- ['GET /_metrics', '.route("/_metrics", get(routes::ops::metrics_handler))'],
- ['POST /_metadata', '.route("/_metadata", post(routes::internal::metadata_handler))'],
- ['POST /_sniff', '.route("/_sniff", post(routes::internal::sniff_handler))'],
- ['POST /_thumbnail', '.route("/_thumbnail", post(routes::internal::thumbnail_handler))'],
- ['POST /_frames', '.route("/_frames", post(routes::internal::frames_handler))'],
- [
- 'PUT /v1/relay/{}',
- '"/v1/relay/{*key}",\n put(routes::relay::relay_put).options(routes::relay::relay_options),',
- ],
- [
- 'OPTIONS /v1/relay/{}',
- '"/v1/relay/{*key}",\n put(routes::relay::relay_put).options(routes::relay::relay_options),',
- ],
- ['GET /{}', 'if app.cfg.mode == DeploymentMode::Static {'],
- ['HEAD /{}', 'if app.cfg.mode == DeploymentMode::Static {'],
-]);
-
-const MEDIA_PROXY_ASSET_PREFIXES = new Map([
- ['external', 'path.strip_prefix("/external/")'],
- ['attachments', 'path.starts_with("/attachments/")'],
- ['themes', 'path.starts_with("/themes/")'],
- ['entrance-sounds', '!= "entrance-sounds"'],
- ['emojis', 'AssetKind::Emoji => "emojis"'],
- ['stickers', 'AssetKind::Sticker => "stickers"'],
- ['avatars', '"avatars" => AssetKind::Avatar'],
- ['icons', '"icons" => AssetKind::GuildIcon'],
- ['branding', '"branding" => AssetKind::GuildIcon'],
- ['banners', '"banners" => AssetKind::Banner'],
- ['splashes', '"splashes" => AssetKind::Splash'],
- ['embed-splashes', '"embed-splashes" => AssetKind::EmbedSplash'],
- ['guilds', 'fn parse_guild_member_asset_path'],
-]);
-
-interface SpecOperation {
- readonly method: string;
- readonly path: string;
- readonly tags: ReadonlyArray<string>;
- readonly security: ReadonlyArray<Record<string, Array<string>>> | null;
-}
-
-interface EffectiveSpecSecurity {
- readonly schemes: ReadonlySet<string>;
- readonly allowsAnonymous: boolean;
-}
-
-function indexDocumentedRoutes(routes: ReadonlyArray<DocsRouteHeader>): Map<string, DocsRouteHeader> {
- const index = new Map<string, DocsRouteHeader>();
- for (const route of routes) {
- const key = routeShape(route.method, stripVersionPrefix(route.path));
- const existing = index.get(key);
- if (existing !== undefined) {
- throw new Error(
- `Duplicate documented route ${key}: ${existing.file}:${existing.line} and ${route.file}:${route.line}`,
- );
- }
- index.set(key, route);
- }
- return index;
-}
-
-interface AliasRoute {
- readonly shape: string;
- readonly file: string;
-}
-
-const ALIAS_TABLE_HEADER = '| Method | Deprecated path | Successor |';
-const ALIAS_ROW_PATTERN =
- /^\|\s*(GET|HEAD|POST|PATCH|PUT|DELETE|OPTIONS)\s*\|\s*`(\/[^`]+)`\s*\|\s*\[[^\]]+\]\([^)]+\)\s*\|\s*$/u;
-
-function aliasDocumentedRoutes(pages: ReadonlyArray<MarkdownPage>): Array<AliasRoute> {
- const aliases: Array<AliasRoute> = [];
- for (const {relativePath: file, lines} of pages) {
- let inTable = false;
- for (const line of lines) {
- if (line.trim() === ALIAS_TABLE_HEADER) {
- inTable = true;
- continue;
- }
- if (inTable && line.trim().startsWith('| ---')) {
- continue;
- }
- const row = inTable ? ALIAS_ROW_PATTERN.exec(line.trim()) : null;
- if (row == null) {
- inTable = false;
- continue;
- }
- aliases.push({
- shape: routeShape(row[1], stripVersionPrefix(row[2])),
- file,
- });
- }
- }
- return aliases;
-}
-
-async function specOperations(specPath: string): Promise<Array<SpecOperation>> {
- const spec: unknown = JSON.parse(await readFile(specPath, 'utf8'));
- if (typeof spec !== 'object' || spec == null || !('paths' in spec)) {
- throw new Error(`Spec has no paths: ${specPath}`);
- }
- const document = spec as OpenAPIDocument;
- const operations: Array<SpecOperation> = [];
- for (const [routePath, item] of Object.entries(document.paths)) {
- for (const [method, operation] of Object.entries(item)) {
- const upper = method.toUpperCase();
- if (!HTTP_METHODS.has(upper)) {
- continue;
- }
- operations.push({
- method: upper,
- path: stripVersionPrefix(routePath),
- tags: operation.tags ?? [],
- security: operation.security === undefined ? (document.security ?? null) : operation.security,
- });
- }
- }
- return operations;
-}
-
-function section(title: string, entries: ReadonlyArray<string>): number {
- if (entries.length === 0) {
- console.log(` ${title}: none`);
- return 0;
- }
- console.log(` ${title}: ${entries.length.toString()}`);
- for (const entry of entries) {
- console.log(` ${entry}`);
- }
- return entries.length;
-}
-
-async function registrationSources(directory: string): Promise<Array<string>> {
- const entries = await readdir(directory, {withFileTypes: true});
- const files: Array<string> = [];
- for (const entry of entries) {
- const resolved = path.join(directory, entry.name);
- if (entry.isDirectory()) {
- if (entry.name === 'node_modules' || entry.name === 'tests') {
- continue;
- }
- files.push(...(await registrationSources(resolved)));
- continue;
- }
- if (entry.name.endsWith('.ts') && !entry.name.endsWith('.test.ts')) {
- files.push(resolved);
- }
- }
- return files;
-}
-const controllerRoutes = extractRoutesFromControllers(
- await registrationSources(path.join(REPO_ROOT, 'fluxer_api/src')),
-);
-
-function constraintTail(constraint: string): string {
- const body = constraint.slice(1, -1);
- const tail = body.match(/(?:\\[^A-Za-z0-9]|[A-Za-z0-9_])+$/u);
- if (tail?.index == null || !/[+*?})\]]$/u.test(body.slice(0, tail.index))) {
- return `<constraint ${body}>`;
- }
- return tail[0].replace(/\\(.)/gu, '$1');
-}
-const astRoute = (route: {method: string; path: string}): string => {
- const templated = route.path
- .replace(
- /:([A-Za-z_][A-Za-z0-9_]*)(\{(?:[^{}]|\{[^{}]*\})*\})?/gu,
- (_match: string, name: string, constraint: string | undefined) =>
- `{${name}}${constraint == null ? '' : constraintTail(constraint)}`,
- )
- .replace(/\*/gu, '{wildcard}');
- return routeShape(route.method.toUpperCase(), stripVersionPrefix(templated));
-};
-
-const pages = await readMarkdownPages(DOCS_ROOT);
-const documented = pages.flatMap(readRouteHeaders);
-const aliasDocumented = aliasDocumentedRoutes(pages);
-const aliasShapes = new Map<string, AliasRoute>();
-for (const alias of aliasDocumented) {
- const existing = aliasShapes.get(alias.shape);
- if (existing !== undefined) {
- throw new Error(`Duplicate deprecated alias ${alias.shape}: ${existing.file} and ${alias.file}`);
- }
- aliasShapes.set(alias.shape, alias);
-}
-const mediaProxySource = await (async () => {
- const sources: Array<string> = [];
- const walk = async (dir: string): Promise<void> => {
- for (const entry of await readdir(dir, {withFileTypes: true})) {
- const full = path.join(dir, entry.name);
- if (entry.isDirectory()) {
- await walk(full);
- } else if (entry.name.endsWith('.rs')) {
- sources.push(await readFile(full, 'utf8'));
- }
- }
- };
- await walk(MEDIA_PROXY_SERVER_DIR);
- return sources.join('\n');
-})();
-
-let failures = 0;
-
-console.log('source anchors');
-const staleAnchors: Array<string> = [];
-for (const [route, anchor] of [...MEDIA_PROXY_ROUTES, ...MEDIA_PROXY_ASSET_PREFIXES]) {
- if (!mediaProxySource.includes(anchor)) {
- staleAnchors.push(`${route}: fluxer_media_proxy/src/server no longer contains ${anchor}`);
- }
-}
-const exemptSources = new Map<string, string>();
-const sourceOf = async (file: string): Promise<string> => {
- let source = exemptSources.get(file);
- if (source == null) {
- source = await readFile(path.join(REPO_ROOT, file), 'utf8');
- exemptSources.set(file, source);
- }
- return source;
-};
-for (const [route, {file, anchor}] of MAIN_SPEC_EXEMPT) {
- if (!(await sourceOf(file)).includes(anchor)) {
- staleAnchors.push(`${route}: ${file} no longer contains ${anchor}`);
- }
-}
-for (const rule of EXEMPTION_RULES) {
- for (const {file, anchor} of rule.anchors) {
- if (!(await sourceOf(file)).includes(anchor)) {
- staleAnchors.push(`exemption rule "${rule.name}": ${file} no longer contains ${anchor}`);
- }
- }
-}
-for (const [route, {documentedIn}] of OUT_OF_BAND_CREDENTIAL) {
- if (documentedIn == null) {
- continue;
- }
- const page = await readFile(path.join(DOCS_ROOT, documentedIn.file), 'utf8');
- if (!page.includes(documentedIn.anchor)) {
- staleAnchors.push(`${route}: ${documentedIn.file} no longer contains ${documentedIn.anchor}`);
- }
-}
-failures += section('stale anchors (the code moved, update this script)', staleAnchors);
-
-const mediaProxyRouterSource = await readFile(path.join(MEDIA_PROXY_SERVER_DIR, 'runtime.rs'), 'utf8');
-const mediaProxyRegisteredPaths = [...mediaProxyRouterSource.matchAll(/\.route\(\s*"([^"]+)"/gu)].map((match) =>
- match[1].replace(/\{[^}]*\}/gu, '{}'),
-);
-const mediaProxyListedPaths = new Set([...MEDIA_PROXY_ROUTES.keys()].map((shape) => shape.split(' ')[1]));
-const unlistedMediaProxyPaths = [...new Set(mediaProxyRegisteredPaths)]
- .filter((routePath) => !mediaProxyListedPaths.has(routePath))
- .map((routePath) => `${routePath}: build_router registers it and MEDIA_PROXY_ROUTES does not name it`)
- .sort();
-if (mediaProxyRegisteredPaths.length === 0) {
- unlistedMediaProxyPaths.push(
- 'fluxer_media_proxy/src/server/runtime.rs registers no route, so this check has gone blind',
- );
-}
-failures += section('registered by fluxer_media_proxy but absent from this script', unlistedMediaProxyPaths);
-console.log(
- ` routes registered in fluxer_media_proxy/src/server/runtime.rs: ${mediaProxyRegisteredPaths.length.toString()}`,
-);
-
-const mediaProxyDocumented = documented.filter((route) => route.file.startsWith('media-proxy/'));
-const adminDocumented = documented.filter(
- (route) => !route.file.startsWith('media-proxy/') && stripVersionPrefix(route.path).startsWith('/admin'),
-);
-const mainDocumented = documented.filter(
- (route) => !route.file.startsWith('media-proxy/') && !stripVersionPrefix(route.path).startsWith('/admin'),
-);
-
-const main = await specOperations(MAIN_SPEC);
-const admin = await specOperations(ADMIN_SPEC);
-const mainShapes = new Set(main.map((operation) => routeShape(operation.method, operation.path)));
-const adminShapes = new Set(admin.map((operation) => routeShape(operation.method, operation.path)));
-
-const documentedMain = indexDocumentedRoutes(mainDocumented);
-const documentedAdmin = indexDocumentedRoutes(adminDocumented);
-
-const registered = new Map<string, string>();
-for (const route of controllerRoutes) {
- const shape = astRoute(route);
- if (!registered.has(shape)) {
- registered.set(shape, `${path.relative(REPO_ROOT, route.controllerFile)}:${route.lineNumber.toString()}`);
- }
-}
-
-console.log('main API and admin API (from route registration)');
-{
- const ruleCounts = new Map<string, number>();
- for (const rule of EXEMPTION_RULES) {
- ruleCounts.set(rule.name, 0);
- }
- const undocumented: Array<string> = [];
- for (const [shape, site] of [...registered].sort()) {
- if (documentedMain.has(shape) || documentedAdmin.has(shape) || aliasShapes.has(shape)) {
- continue;
- }
- const rule = EXEMPTION_RULES.find((candidate) => candidate.covers(shape, shape.split(' ')[1]));
- if (rule == null) {
- undocumented.push(`${shape} registered at ${site}`);
- continue;
- }
- ruleCounts.set(rule.name, (ruleCounts.get(rule.name) ?? 0) + 1);
- }
-
- const documentedNotRegistered = [...documentedMain, ...documentedAdmin]
- .filter(([shape]) => !registered.has(shape))
- .map(([shape, route]) => `${shape} (${route.file}) is documented but fluxer_api registers no such route`)
- .sort();
- const deadRules = EXEMPTION_RULES.filter(
- (rule) => rule.mayCoverNothing !== true && (ruleCounts.get(rule.name) ?? 0) === 0,
- ).map((rule) => `the "${rule.name}" rule covers no registered route, so it is either stale or too narrow to matter`);
- const staleEntries = [
- ...[...DELIBERATELY_UNDOCUMENTED.keys()].map((shape) => [shape, 'DELIBERATELY_UNDOCUMENTED'] as const),
- ...[...OUT_OF_BAND_CREDENTIAL.keys()].map((shape) => [shape, 'OUT_OF_BAND_CREDENTIAL'] as const),
- ]
- .filter(([shape]) => !registered.has(shape))
- .map(([shape, list]) => `${list} names ${shape}, which fluxer_api no longer registers`)
- .sort();
-
- const exemptTotal = [...ruleCounts.values()].reduce((a, b) => a + b, 0);
- console.log(` registered routes: ${registered.size.toString()}`);
- console.log(
- ` documented with a RouteHeader: ${[...registered.keys()].filter((shape) => documentedMain.has(shape) || documentedAdmin.has(shape)).length.toString()}`,
- );
- console.log(
- ` documented as a deprecated alias: ${[...registered.keys()].filter((shape) => aliasShapes.has(shape)).length.toString()}`,
- );
- console.log(` exempt by rule: ${exemptTotal.toString()}`);
- for (const rule of EXEMPTION_RULES) {
- console.log(` ${rule.name}: ${(ruleCounts.get(rule.name) ?? 0).toString()}. ${rule.justification}`);
- }
- console.log(' fluxer_admin is a Rust crate and its 83 HTML panel routes are NOT checked here. It serves');
- console.log(' zero API operations, so the admin API above is the /admin routes fluxer_api registers.');
- console.log(' fluxer_media_proxy and fluxer_app_proxy are Rust too. The media proxy is checked against');
- console.log(' the hand-maintained list below, and the app proxy is not checked at all.');
- failures += section('registered but neither documented nor covered by an exemption rule', undocumented);
- failures += section('documented but not registered', documentedNotRegistered);
- failures += section('exemption rules that cover nothing', deadRules);
- failures += section('exemption entries naming a route that is gone', staleEntries);
-}
-
-console.log('main API (against the generated spec)');
-failures += section(
- 'documented but absent from the live spec',
- [...documentedMain.entries()]
- .filter(([shape]) => !mainShapes.has(shape) && !MAIN_SPEC_EXEMPT.has(shape))
- .map(([, route]) => `${route.method} ${route.path} (${route.file})`)
- .sort(),
-);
-failures += section(
- 'present in the live spec but undocumented',
- main
- .filter((operation) => {
- const shape = routeShape(operation.method, operation.path);
- if (documentedMain.has(shape)) {
- return false;
- }
- if (aliasShapes.has(shape)) {
- return false;
- }
- return !DELIBERATELY_UNDOCUMENTED.has(shape);
- })
- .map((operation) => `${operation.method} ${operation.path} [${operation.tags.join(', ')}]`)
- .sort(),
-);
-failures += section(
- 'documented as a deprecated alias but not registered',
- [...aliasShapes.values()]
- .filter((alias) => !registered.has(alias.shape))
- .map((alias) => `${alias.shape} (${alias.file}) names a route fluxer_api does not register`)
- .sort(),
-);
-failures += section(
- 'documented as a deprecated alias and with its own RouteHeader',
- [...aliasShapes.keys()]
- .filter((shape) => documentedMain.has(shape))
- .map((shape) => `${shape} is an alias row and a RouteHeader, which double counts it`)
- .sort(),
-);
-console.log(` documented in deprecated alias tables: ${aliasShapes.size.toString()}`);
-const wronglyDocumented = [...DELIBERATELY_UNDOCUMENTED.entries()]
- .filter(([shape]) => documentedMain.has(shape))
- .map(([shape, reason]) => `${shape} must not be documented: ${reason}`);
-failures += section('documented despite belonging to an unshipped system', wronglyDocumented);
-console.log(
- ` deliberately undocumented and correctly absent: ${(DELIBERATELY_UNDOCUMENTED.size - wronglyDocumented.length).toString()}/${DELIBERATELY_UNDOCUMENTED.size.toString()}`,
-);
-console.log(
- ` exempt non-spec routes documented: ${[...MAIN_SPEC_EXEMPT.keys()].filter((shape) => documentedMain.has(shape)).length.toString()}/${MAIN_SPEC_EXEMPT.size.toString()}`,
-);
-for (const [shape, {reason}] of MAIN_SPEC_EXEMPT) {
- console.log(` ${shape}: ${reason}`);
-}
-
-console.log('media proxy');
-const mediaProxyProblems: Array<string> = [];
-for (const route of mediaProxyDocumented) {
- const shape = routeShape(route.method, route.path);
- if (MEDIA_PROXY_ROUTES.has(shape)) {
- continue;
- }
- const firstSegment = route.path.replace(/^\//u, '').split('/')[0].split('.')[0];
- if (MEDIA_PROXY_ASSET_PREFIXES.has(firstSegment)) {
- if (route.method !== 'GET' && route.method !== 'HEAD') {
- mediaProxyProblems.push(`${route.method} ${route.path} (${route.file}) asset paths serve GET and HEAD only`);
- }
- continue;
- }
- mediaProxyProblems.push(`${route.method} ${route.path} (${route.file}) is not a route the media proxy serves`);
-}
-failures += section('documented but not served by fluxer_media_proxy', mediaProxyProblems);
-
-console.log('gateway dispatch events');
-{
- const gatewayConstants = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/constants/Gateway.ts'), 'utf8');
- const unionMatch = gatewayConstants.match(/export type GatewayDispatchEvent\s*=([\s\S]*?);/u);
- const apiEvents = new Set<string>();
- if (unionMatch != null) {
- for (const entry of unionMatch[1].matchAll(/'([A-Z][A-Z0-9_]*)'/gu)) {
- apiEvents.add(entry[1]);
- }
- }
- const gatewayDirectory = path.join(REPO_ROOT, 'fluxer_gateway/src');
- const erlangSources: Array<string> = [];
- const collect = async (directory: string): Promise<void> => {
- for (const entry of await readdir(directory, {withFileTypes: true})) {
- const resolved = path.join(directory, entry.name);
- if (entry.isDirectory()) {
- await collect(resolved);
- continue;
- }
- if (entry.name.endsWith('.erl') || entry.name.endsWith('.hrl')) {
- erlangSources.push(await readFile(resolved, 'utf8'));
- }
- }
- };
- await collect(gatewayDirectory);
- const erlang = erlangSources.join('\n');
-
- const eventsPage = [
- await readFile(path.join(DOCS_ROOT, 'gateway/events.md'), 'utf8'),
- await readFile(path.join(DOCS_ROOT, 'gateway/threads.md'), 'utf8'),
- ].join('\n');
- const documentedEvents = new Set<string>();
- for (const entry of eventsPage.matchAll(/^###\s+(?:<span[^>]*><\/span>)?([A-Z][A-Z0-9_]{3,})\s*$/gmu)) {
- documentedEvents.add(entry[1]);
- }
-
- const isReal = (event: string): boolean => {
- if (apiEvents.has(event)) {
- return true;
- }
- if (erlang.includes(event)) {
- return true;
- }
- return new RegExp(`\\b${event.toLowerCase()}\\b`, 'u').test(erlang);
- };
-
- const fabricated = [...documentedEvents].filter((event) => !isReal(event)).sort();
- const undocumented = [...apiEvents].filter((event) => !documentedEvents.has(event)).sort();
- console.log(` events in the GatewayDispatchEvent union: ${apiEvents.size.toString()}`);
- console.log(` events documented in gateway/events.md and gateway/threads.md: ${documentedEvents.size.toString()}`);
- console.log(' a documented event counts as real if it is in the union, or appears in fluxer_gateway');
- console.log(' as an uppercase binary or a lowercase atom');
- failures += section('documented but not emitted by any service', fabricated);
- failures += section('in the dispatch union but undocumented', undocumented);
-}
-
-console.log('gateway opcodes and close codes');
-{
- const erl = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/utils/constants.erl'), 'utf8');
- const realOpcodes = new Set<number>();
- for (const entry of erl.matchAll(/gateway_opcode\((\d+)\) -> [a-z_]+/gu)) {
- realOpcodes.add(Number.parseInt(entry[1], 10));
- }
- const realCloseCodes = new Set<number>();
- for (const entry of erl.matchAll(/close_code_to_num\([a-z_]+\) -> (\d+)/gu)) {
- realCloseCodes.add(Number.parseInt(entry[1], 10));
- }
- const page = await readFile(path.join(DOCS_ROOT, 'gateway/opcodes-and-close-codes.md'), 'utf8');
- const docOpcodes = new Set<number>();
- const docCloseCodes = new Set<number>();
- const strippedPage = page.replace(/<sup>.*?<\/sup>/gu, '');
- for (const row of strippedPage.matchAll(/^\|\s*(\d{1,4})\s*\|/gmu)) {
- const value = Number.parseInt(row[1], 10);
- if (value >= 4000) {
- docCloseCodes.add(value);
- continue;
- }
- if (value <= 99) {
- docOpcodes.add(value);
- }
- }
- console.log(` opcodes: ${realOpcodes.size.toString()} real, ${docOpcodes.size.toString()} documented`);
- console.log(` close codes: ${realCloseCodes.size.toString()} real, ${docCloseCodes.size.toString()} documented`);
- failures += section(
- 'opcodes documented but absent from constants.erl',
- [...docOpcodes]
- .filter((code) => !realOpcodes.has(code))
- .sort((a, b) => a - b)
- .map(String),
- );
- failures += section(
- 'opcodes in constants.erl but undocumented',
- [...realOpcodes]
- .filter((code) => !docOpcodes.has(code))
- .sort((a, b) => a - b)
- .map(String),
- );
- failures += section(
- 'close codes documented but absent from constants.erl',
- [...docCloseCodes]
- .filter((code) => !realCloseCodes.has(code))
- .sort((a, b) => a - b)
- .map(String),
- );
- failures += section(
- 'close codes in constants.erl but undocumented',
- [...realCloseCodes]
- .filter((code) => !docCloseCodes.has(code))
- .sort((a, b) => a - b)
- .map(String),
- );
-}
-
-console.log('enum names and error codes');
-{
- const liveTokens = new Set<string>();
- const scanDirectory = async (directory: string): Promise<void> => {
- let entries: Array<Dirent>;
- try {
- entries = await readdir(directory, {withFileTypes: true});
- } catch {
- return;
- }
- for (const entry of entries) {
- if (entry.name === 'node_modules' || entry.name === 'target' || entry.name === 'dist') {
- continue;
- }
- const resolved = path.join(directory, entry.name);
- if (entry.isDirectory()) {
- await scanDirectory(resolved);
- continue;
- }
- if (!/\.(ts|rs|erl|hrl|json)$/u.test(entry.name)) {
- continue;
- }
- const source = await readFile(resolved, 'utf8');
- for (const token of source.matchAll(/[A-Z][A-Z0-9_]+/gu)) {
- liveTokens.add(token[0]);
- }
- }
- };
- for (const sub of [
- 'packages',
- 'fluxer_api/src',
- 'fluxer_admin/src',
- 'fluxer_gateway/src',
- 'fluxer_media_proxy/src',
- ]) {
- await scanDirectory(path.join(REPO_ROOT, sub));
- }
-
- const enumRows: Array<{file: string; line: number; name: string}> = [];
- const codeRows: Array<{file: string; line: number; code: string}> = [];
- for (const {relativePath: relative, lines} of pages) {
- let inEnumTable = false;
- for (let i = 0; i < lines.length; i += 1) {
- const line = lines[i];
- if (/^\|\s*Value\s*\|\s*Name\s*\|/u.test(line)) {
- inEnumTable = true;
- continue;
- }
- if (inEnumTable && !line.startsWith('|')) {
- inEnumTable = false;
- }
- if (inEnumTable && !line.startsWith('| ---')) {
- const row = line.match(/^\|\s*-?\d+\s*\|\s*([A-Z][A-Z0-9_]*)/u);
- if (row != null) {
- enumRows.push({file: relative, line: i + 1, name: row[1]});
- }
- }
- for (const cited of line.matchAll(/\b[1-5]\d\d\s+`([A-Z][A-Z0-9_]{2,})`/gu)) {
- codeRows.push({file: relative, line: i + 1, code: cited[1]});
- }
- }
- }
-
- const fabricatedEnums = enumRows.filter((row) => !liveTokens.has(row.name));
- const fabricatedCodes = codeRows.filter((row) => !liveTokens.has(row.code));
- console.log(` enum rows checked: ${enumRows.length.toString()}`);
- console.log(` error codes cited with a status: ${new Set(codeRows.map((r) => r.code)).size.toString()}`);
- console.log(' names only, not value bindings: the same name is reused across enums, so a global');
- console.log(' value comparison misattributes. ReportType.GUILD is 2 while ChannelContext.GUILD is 0');
- failures += section(
- 'enum names absent from live source',
- fabricatedEnums.map((row) => `${row.file}:${row.line.toString()} ${row.name}`).sort(),
- );
- failures += section(
- 'error codes absent from live source',
- [...new Set(fabricatedCodes.map((row) => `${row.file}:${row.line.toString()} ${row.code}`))].sort(),
- );
-}
-
-console.log('registry codes with a producer');
-{
- const CODE_REGISTRIES = ['packages/constants/src/ApiErrorCodes.ts', 'packages/constants/src/ValidationErrorCodes.ts'];
- const NON_PRODUCERS = [...CODE_REGISTRIES, 'packages/errors/src/i18n/ErrorCodeMappings.ts'];
- const registryKeys = new Map<string, string>();
- for (const relative of CODE_REGISTRIES) {
- const source = await readFile(path.join(REPO_ROOT, relative), 'utf8');
- for (const entry of source.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {
- registryKeys.set(entry[1], relative);
- }
- }
-
- const producerTokens = new Set<string>();
- const scanProducers = async (directory: string): Promise<void> => {
- let entries: Array<Dirent>;
- try {
- entries = await readdir(directory, {withFileTypes: true});
- } catch {
- return;
- }
- for (const entry of entries) {
- if (entry.name === 'node_modules' || entry.name === 'target' || entry.name === 'dist' || entry.name === 'tests') {
- continue;
- }
- const resolved = path.join(directory, entry.name);
- if (entry.isDirectory()) {
- await scanProducers(resolved);
- continue;
- }
- if (!/\.(ts|rs|erl|hrl)$/u.test(entry.name) || entry.name.endsWith('.test.ts')) {
- continue;
- }
- if (NON_PRODUCERS.includes(path.relative(REPO_ROOT, resolved))) {
- continue;
- }
- const source = await readFile(resolved, 'utf8');
- for (const token of source.matchAll(/\b[A-Z][A-Z0-9_]+\b/gu)) {
- producerTokens.add(token[0]);
- }
- }
- };
- for (const sub of [
- 'packages',
- 'fluxer_api/src',
- 'fluxer_admin/src',
- 'fluxer_gateway/src',
- 'fluxer_media_proxy/src',
- 'fluxer_unfurl/src',
- ]) {
- await scanProducers(path.join(REPO_ROOT, sub));
- }
-
- const withoutProducer = [...registryKeys]
- .filter(([code]) => !producerTokens.has(code))
- .map(([code, relative]) => `${relative} ${code}`)
- .sort();
- console.log(` registry codes checked: ${registryKeys.size.toString()}`);
- console.log(' a producer is any live server file outside the two registries, ErrorCodeMappings.ts and the');
- console.log(' generated specs. Test files do not count, the api test harness controller does');
- failures += section('registry codes with no producer', withoutProducer);
-}
-
-console.log('admin audit actions');
-{
- const registrySource = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/admin/AdminAuditActions.ts'), 'utf8');
- const registryBlock = registrySource.match(/export const AdminAuditReadActions = \{([\s\S]*?)\} as const;/u);
- const readActions = new Set([...(registryBlock?.[1] ?? '').matchAll(/:\s*'([^']+)'/gu)].map((match) => match[1]));
-
- const recorded = new Map<string, string>();
- const scanAuditSources = async (directory: string, everyFile: boolean): Promise<void> => {
- for (const entry of await readdir(directory, {withFileTypes: true})) {
- if (entry.name === 'node_modules' || entry.name === 'tests') continue;
- const resolved = path.join(directory, entry.name);
- if (entry.isDirectory()) {
- await scanAuditSources(resolved, everyFile);
- continue;
- }
- if (!entry.name.endsWith('.ts') || entry.name.endsWith('.test.ts')) continue;
- const source = await readFile(resolved, 'utf8');
- if (!everyFile && !source.includes('createAuditLog(')) continue;
- for (const assignment of source.matchAll(/\baction\s*[:=]\s*([^,;\n]*(?:\n\s*[?:][^,;\n]*)*)/gu)) {
- const expression = assignment[1];
- const value = expression.includes('?') ? expression.slice(expression.indexOf('?') + 1) : expression;
- for (const literal of value.matchAll(/'([^']+)'/gu)) {
- if (!recorded.has(literal[1])) recorded.set(literal[1], path.relative(REPO_ROOT, resolved));
- }
- }
- }
- };
- await scanAuditSources(path.join(REPO_ROOT, 'fluxer_api/src/api/admin'), true);
- await scanAuditSources(path.join(REPO_ROOT, 'fluxer_api/src/api/worker/tasks'), true);
- await scanAuditSources(path.join(REPO_ROOT, 'fluxer_api/src'), false);
- for (const action of readActions) {
- if (!recorded.has(action)) recorded.set(action, 'fluxer_api/src/api/admin/AdminAuditActions.ts');
- }
-
- const indexPage = await readFile(path.join(DOCS_ROOT, 'admin-api/index.mdx'), 'utf8');
- const actionTable = (heading: string): Set<string> => {
- const start = indexPage.indexOf(`\n${heading}\n`);
- if (start === -1) return new Set();
- const rest = indexPage.slice(start + heading.length + 2);
- const end = rest.search(/\n#{1,4} /u);
- const values = new Set<string>();
- for (const line of (end === -1 ? rest : rest.slice(0, end)).split('\n')) {
- const cell = line
- .match(/^\|\s*([^|]+?)\s*\|/u)?.[1]
- ?.replace(/<sup>.*?<\/sup>/gu, '')
- .trim();
- if (cell == null || cell === 'Value' || /^-+$/u.test(cell)) continue;
- values.add(cell);
- }
- return values;
- };
- const documentedRead = actionTable('#### Read actions');
- const documentedWrite = actionTable('#### Write actions');
- console.log(
- ` actions recorded by fluxer_api: ${recorded.size.toString()}, of which read: ${readActions.size.toString()}`,
- );
- console.log(
- ` actions documented in admin-api/index.mdx: read ${documentedRead.size.toString()}, write ${documentedWrite.size.toString()}`,
- );
- const emptyParses = [
- ...(readActions.size === 0 ? ['the read action registry in AdminAuditActions.ts parsed empty'] : []),
- ...(recorded.size <= readActions.size ? ['no write action was found in fluxer_api source'] : []),
- ...(documentedRead.size === 0 ? ['the Read actions table parsed empty'] : []),
- ...(documentedWrite.size === 0 ? ['the Write actions table parsed empty'] : []),
- ];
- failures += section('audit action parses that came back empty (update this script)', emptyParses);
- failures += section(
- 'recorded but undocumented',
- [...recorded]
- .filter(([action]) => !documentedRead.has(action) && !documentedWrite.has(action))
- .map(([action, file]) => `${action} (${file})`)
- .sort(),
- );
- failures += section(
- 'documented but never recorded',
- [...documentedRead, ...documentedWrite].filter((action) => !recorded.has(action)).sort(),
- );
- failures += section(
- 'documented under the wrong access',
- [
- ...[...documentedRead].filter((action) => !readActions.has(action)).map((action) => `${action}: listed as read`),
- ...[...documentedWrite].filter((action) => readActions.has(action)).map((action) => `${action}: listed as write`),
- ].sort(),
- );
-}
-
-console.log('permission bits');
-{
- const constants = await readFile(path.join(REPO_ROOT, 'packages/constants/src/ChannelConstants.ts'), 'utf8');
- const block = constants.match(/export const Permissions = \{([\s\S]*?)\} as const;/u);
- const livePermissions = new Map<string, number>();
- const threadConstants = await readFile(
- path.join(REPO_ROOT, 'packages/constants/src/ThreadPermissionUtils.ts'),
- 'utf8',
- );
- const threadBlock = threadConstants.match(/export const ThreadPermissionFlags = \{([\s\S]*?)\} as const;/u);
- for (const source of [block, threadBlock]) {
- if (source == null) {
- continue;
- }
- for (const entry of source[1].matchAll(/([A-Z][A-Z0-9_]*):\s*1n\s*<<\s*(\d+)n/gu)) {
- livePermissions.set(entry[1], Number.parseInt(entry[2], 10));
- }
- }
- if (threadBlock == null) {
- failures += section('ThreadPermissionFlags block not found in ThreadPermissionUtils.ts', ['parser drift']);
- }
- const page = (await readFile(path.join(DOCS_ROOT, 'http-api/permissions.mdx'), 'utf8'))
- .replace(/&lt;/gu, '<')
- .replace(/&gt;/gu, '>');
- const documentedPermissions = new Map<string, number>();
- for (const row of page.matchAll(/^\|\s*1\s*<<\s*(\d+)\s*\|\s*([A-Z][A-Z0-9_]*)/gmu)) {
- documentedPermissions.set(row[2], Number.parseInt(row[1], 10));
- }
- console.log(` live permissions: ${livePermissions.size.toString()}`);
- console.log(` documented permissions: ${documentedPermissions.size.toString()}`);
- const bitMismatches: Array<string> = [];
- for (const [name, bit] of documentedPermissions) {
- const liveBit = livePermissions.get(name);
- if (liveBit != null && liveBit !== bit) {
- bitMismatches.push(`${name}: documented 1 << ${bit.toString()}, live 1 << ${liveBit.toString()}`);
- }
- }
- failures += section('permission bit mismatches', bitMismatches);
- failures += section(
- 'documented but not a live permission',
- [...documentedPermissions.keys()].filter((name) => !livePermissions.has(name)).sort(),
- );
- failures += section(
- 'live permission but undocumented',
- [...livePermissions.keys()].filter((name) => !documentedPermissions.has(name)).sort(),
- );
-}
-
-console.log('rate limit buckets, limits and windows');
-{
- const UNIT_MS = new Map([
- ['second', 1000],
- ['seconds', 1000],
- ['minute', 60_000],
- ['minutes', 60_000],
- ['hour', 3_600_000],
- ['hours', 3_600_000],
- ['day', 86_400_000],
- ['days', 86_400_000],
- ]);
- const toMs = (text: string): number | null => {
- const parsed = text.trim().match(/^(\d+)?\s*([a-z]+)$/u);
- if (parsed == null) {
- return null;
- }
- const unit = UNIT_MS.get(parsed[2]);
- if (unit == null) {
- return null;
- }
- return Number.parseInt(parsed[1] ?? '1', 10) * unit;
- };
-
- const configDirectory = path.join(REPO_ROOT, 'fluxer_api/src/api/rate_limit_configs');
- const liveBuckets = new Map<string, {limit: number; windowMs: number | null}>();
- for (const entry of await readdir(configDirectory)) {
- if (!entry.endsWith('.ts')) {
- continue;
- }
- const source = await readFile(path.join(configDirectory, entry), 'utf8');
- for (const config of source.matchAll(
- /bucket:\s*'([^']+)',\s*config:\s*\{limit:\s*(\d+),\s*windowMs:\s*ms\('([^']+)'\)/gu,
- )) {
- liveBuckets.set(config[1], {limit: Number.parseInt(config[2], 10), windowMs: toMs(config[3])});
- }
- }
-
- const claimPattern =
- /([\d,]+) requests? per ([a-z0-9 ]+?)(?:,| for [^.]*?,) on the (?:shared )?`([a-z0-9_:@{}]+)` bucket/gu;
- const problems: Array<string> = [];
- let claims = 0;
- for (const {relativePath: relative, lines} of pages) {
- for (let i = 0; i < lines.length; i += 1) {
- for (const claim of lines[i].matchAll(claimPattern)) {
- claims += 1;
- const bucket = claim[3];
- const limit = Number.parseInt(claim[1].replace(/,/gu, ''), 10);
- const windowMs = toMs(claim[2]);
- const live = liveBuckets.get(bucket);
- if (live == null) {
- problems.push(`${relative}:${(i + 1).toString()} bucket ${bucket} is not in any rate limit config`);
- continue;
- }
- if (live.limit !== limit) {
- problems.push(
- `${relative}:${(i + 1).toString()} ${bucket} documented limit ${limit.toString()}, live ${live.limit.toString()}`,
- );
- }
- if (windowMs != null && live.windowMs != null && live.windowMs !== windowMs) {
- problems.push(
- `${relative}:${(i + 1).toString()} ${bucket} documented window ${windowMs.toString()}ms, live ${live.windowMs.toString()}ms`,
- );
- }
- }
- }
- }
- console.log(` buckets defined with a limit and window: ${liveBuckets.size.toString()}`);
- console.log(` documented rate limit claims parsed: ${claims.toString()}`);
- failures += section('rate limit disagreements', problems);
-}
-
-console.log('instance limit keys');
-{
- const defaults = await readFile(path.join(REPO_ROOT, 'packages/limits/src/LimitDefaults.ts'), 'utf8');
- const liveKeys = new Set<string>();
- for (const entry of defaults.matchAll(/^\s+([a-z][a-z0-9_]*):/gmu)) {
- liveKeys.add(entry[1]);
- }
- const instancePage = await readFile(path.join(DOCS_ROOT, 'http-api/instance.mdx'), 'utf8');
- const documentedKeys = new Set<string>();
- for (const row of instancePage.matchAll(/^\|\s*([^|]+?)\s*\|/gmu)) {
- const name = row[1]
- .replace(/<sup>.*?<\/sup>/gu, '')
- .replace(/`/gu, '')
- .trim();
- if (/^[a-z][a-z0-9_]*$/u.test(name)) {
- documentedKeys.add(name);
- }
- }
- console.log(` live limit keys: ${liveKeys.size.toString()}`);
- console.log(
- ` documented in the instance limit-keys table: ${[...liveKeys].filter((k) => documentedKeys.has(k)).length.toString()}`,
- );
- failures += section(
- 'live limit keys missing from the instance table',
- [...liveKeys].filter((key) => !documentedKeys.has(key)).sort(),
- );
-}
-
-console.log('media proxy image constants');
-{
- const constants = await readFile(path.join(REPO_ROOT, 'fluxer_media_proxy/src/constants.rs'), 'utf8');
- const ladderBlock = constants.match(/pub const IMAGE_SIZES: &\[u32\] = &\[([\s\S]*?)\];/u);
- const ladder = ladderBlock == null ? [] : [...ladderBlock[1].matchAll(/\d+/gu)].map((m) => Number.parseInt(m[0], 10));
- const defaultSize = constants.match(/pub const DEFAULT_IMAGE_SIZE: u32 = (\d+);/u);
- const dimsBlock = constants.match(/pub fn dims_for\(kind: AssetKind\) -> Dims \{([\s\S]*?)\n\}/u);
- const dims: Array<[string, number, number]> = [];
- if (dimsBlock != null) {
- for (const entry of dimsBlock[1].matchAll(
- /((?:AssetKind::\w+\s*\|?\s*)+)=> Dims \{\s*min:\s*(\d+),\s*max:\s*(\d+),?\s*\}/gu,
- )) {
- dims.push([entry[1].trim(), Number.parseInt(entry[2], 10), Number.parseInt(entry[3], 10)]);
- }
- }
-
- const LADDER_PAGE = 'media-proxy/transformations.md';
-
- const ladderPage = (await readFile(path.join(DOCS_ROOT, LADDER_PAGE), 'utf8')).replace(/(?<=\d),(?=\d)/gu, '');
- const problems: Array<string> = [];
-
- const documentedLadder = ladderPage.match(
- /`size` selects from a fixed ladder: ([\d,\s]+(?:and\s+\d+)?)\. A requested value snaps up/u,
- );
- if (documentedLadder == null) {
- problems.push(`${LADDER_PAGE} no longer states the size ladder in the expected sentence`);
- } else {
- const listed = [...documentedLadder[1].matchAll(/\d+/gu)].map((m) => Number.parseInt(m[0], 10));
- if (listed.join(',') !== ladder.join(',')) {
- problems.push(`size ladder differs. documented [${listed.join(', ')}] vs IMAGE_SIZES [${ladder.join(', ')}]`);
- }
- }
-
- if (dims.length === 0) {
- problems.push('no asset class clamps parsed from dims_for in fluxer_media_proxy/src/constants.rs');
- }
-
- if (defaultSize != null && !ladderPage.includes(`resolves to ${defaultSize[1]} before clamping`)) {
- problems.push(`${LADDER_PAGE} does not state that an absent size resolves to ${defaultSize[1]} before clamping`);
- }
-
- for (const [kinds, min, max] of dims) {
- if (!ladderPage.includes(`${min.toString()} through ${max.toString()}`)) {
- problems.push(`${LADDER_PAGE} does not state the ${kinds} clamp of ${min.toString()} to ${max.toString()}`);
- }
- }
-
- console.log(` size ladder rungs in constants.rs: ${ladder.length.toString()}`);
- console.log(` asset classes with a clamp: ${dims.length.toString()}`);
- failures += section('media proxy constant disagreements', problems);
-}
-
-console.log('self-hosting guide against deploy/self-hosting');
-{
- const envExample = await readFile(path.join(REPO_ROOT, 'deploy/self-hosting/.env.example'), 'utf8');
- const guide = await readFile(path.join(DOCS_ROOT, 'operator/get-started.mdx'), 'utf8');
-
- const placeholders = new Set<string>();
- const declared = new Set<string>();
- const assigned = new Set<string>();
- for (const line of envExample.split('\n')) {
- const commented = line.startsWith('#');
- const assignment = (commented ? line.slice(1) : line).match(/^([A-Z][A-Z0-9_]*)=(.*)$/u);
- if (assignment == null) {
- continue;
- }
- declared.add(assignment[1]);
- if (commented) {
- continue;
- }
- assigned.add(assignment[1]);
- if (assignment[2].includes('CHANGE_ME')) {
- placeholders.add(assignment[1]);
- }
- }
-
- const INSTALLER_ROOT = fileURLToPath(new URL('../src/installer/', import.meta.url));
- const installers: ReadonlyArray<readonly [string, string]> = [
- ['install.sh', await readFile(path.join(INSTALLER_ROOT, 'install.sh'), 'utf8')],
- ['install.ps1', await readFile(path.join(INSTALLER_ROOT, 'install.ps1'), 'utf8')],
- ];
- const shellSource = installers[0][1];
- const powershellSource = installers[1][1];
-
- const problems: Array<string> = [];
-
- const INSTALLER_SECRET_FLOOR = 14;
- const INSTALLER_NON_SECRET_FLOOR = 7;
- const COMPOSE_REQUIRED_FLOOR = 17;
-
- const blockBetween = (source: string, opener: string, closer: string, label: string): string | null => {
- const from = source.indexOf(opener);
- if (from === -1) {
- problems.push(`${label} is gone. Nothing in the script opens with \`${opener.trim()}\``);
- return null;
- }
- const rest = source.slice(from + opener.length);
- const to = rest.indexOf(closer);
- if (to === -1) {
- problems.push(`${label} opens but never closes at \`${closer.trim()}\``);
- return null;
- }
- return rest.slice(0, to);
- };
-
- const shellRows = (fn: string, delimiter: string, label: string): Array<string> => {
- const body = blockBetween(shellSource, `${fn}() {\n\tcat <<'${delimiter}'\n`, `\n${delimiter}\n`, label);
- if (body == null) {
- return [];
- }
- return body.split('\n').filter((row) => row.trim().length > 0);
- };
-
- const powershellRows = (variable: string, label: string): Array<string> => {
- const body = blockBetween(powershellSource, `$${variable} = @(\n`, '\n)\n', label);
- if (body == null) {
- return [];
- }
- return body.split('\n').filter((row) => row.trim().length > 0);
- };
-
- const parseShellKeys = (fn: string, label: string): Map<string, string> => {
- const keys = new Map<string, string>();
- for (const row of shellRows(fn, 'KEYS', label)) {
- const parsed = row.match(/^([A-Z][A-Z0-9_]*) ([a-z0-9_]+)(?: .*)?$/u);
- if (parsed == null) {
- problems.push(`${label} has the row \`${row}\`, which is not "NAME kind"`);
- continue;
- }
- keys.set(parsed[1], parsed[2]);
- }
- return keys;
- };
-
- const parsePowerShellKeys = (variable: string, label: string): Map<string, string> => {
- const keys = new Map<string, string>();
- for (const row of powershellRows(variable, label)) {
- const parsed = row.match(/^\s*@\{Name = '([A-Z][A-Z0-9_]*)'; Kind = '([a-z0-9_]+)'/u);
- if (parsed == null) {
- problems.push(`${label} has the row \`${row.trim()}\`, which is not an @{Name; Kind} entry`);
- continue;
- }
- keys.set(parsed[1], parsed[2]);
- }
- return keys;
- };
-
- const shellSecrets = parseShellKeys('fluxer_secret_keys', 'the install.sh secret list');
- const shellNonSecrets = parseShellKeys('fluxer_non_secret_keys', 'the install.sh non-secret list');
- const powershellSecrets = parsePowerShellKeys('FluxerSecretKeys', 'the install.ps1 secret list');
- const powershellNonSecrets = parsePowerShellKeys('FluxerNonSecretKeys', 'the install.ps1 non-secret list');
-
- const shellWrites = new Set<string>([...shellSecrets.keys(), ...shellNonSecrets.keys()]);
- const powershellWrites = new Set<string>([...powershellSecrets.keys(), ...powershellNonSecrets.keys()]);
-
- for (const [name, secrets, nonSecrets] of [
- ['install.sh', shellSecrets, shellNonSecrets],
- ['install.ps1', powershellSecrets, powershellNonSecrets],
- ] as const) {
- if (secrets.size < INSTALLER_SECRET_FLOOR) {
- problems.push(
- `${name} declares ${secrets.size.toString()} secret keys, floor is ${INSTALLER_SECRET_FLOOR.toString()}`,
- );
- }
- if (nonSecrets.size < INSTALLER_NON_SECRET_FLOOR) {
- problems.push(
- `${name} declares ${nonSecrets.size.toString()} non-secret keys, floor is ${INSTALLER_NON_SECRET_FLOOR.toString()}`,
- );
- }
- }
-
- for (const name of placeholders) {
- for (const [script, secrets] of [
- ['install.sh', shellSecrets],
- ['install.ps1', powershellSecrets],
- ] as const) {
- if (!secrets.has(name)) {
- problems.push(`${name} ships as CHANGE_ME but ${script} never sets it`);
- }
- }
- }
-
- for (const name of new Set<string>([...shellWrites, ...powershellWrites])) {
- if (!declared.has(name)) {
- problems.push(`the installer writes ${name}, which is not declared in .env.example`);
- }
- }
-
- const compareKeyLists = (label: string, shell: Map<string, string>, powershell: Map<string, string>): void => {
- for (const [name, kind] of shell) {
- const other = powershell.get(name);
- if (other == null) {
- problems.push(`install.sh writes the ${label} ${name} and install.ps1 does not`);
- continue;
- }
- if (other !== kind) {
- problems.push(`${name} is ${kind} in install.sh and ${other} in install.ps1`);
- }
- }
- for (const name of powershell.keys()) {
- if (!shell.has(name)) {
- problems.push(`install.ps1 writes the ${label} ${name} and install.sh does not`);
- }
- }
- };
- compareKeyLists('secret', shellSecrets, powershellSecrets);
- compareKeyLists('non-secret value', shellNonSecrets, powershellNonSecrets);
-
- const shellUpgrade = parseShellKeys('fluxer_upgrade_secret_keys', 'the install.sh upgrade secret list');
- const powershellUpgrade = parsePowerShellKeys('FluxerUpgradeSecretKeys', 'the install.ps1 upgrade secret list');
- compareKeyLists('upgrade secret', shellUpgrade, powershellUpgrade);
-
- for (const [script, upgrade, secrets] of [
- ['install.sh', shellUpgrade, shellSecrets],
- ['install.ps1', powershellUpgrade, powershellSecrets],
- ] as const) {
- if (upgrade.size === 0) {
- problems.push(`${script} declares no upgrade secret keys, so an upgrade mints nothing`);
- }
- for (const [name, kind] of upgrade) {
- const asInstalled = secrets.get(name);
- if (asInstalled == null) {
- problems.push(`${script} mints ${name} on an upgrade and never writes it on an install`);
- continue;
- }
- if (asInstalled !== kind) {
- problems.push(`${name} is ${kind} on an upgrade and ${asInstalled} on an install in ${script}`);
- }
- }
- }
-
- const expectedSecretKind = (name: string): string => {
- if (name === 'FLUXER_VAPID_PUBLIC_KEY') {
- return 'vapid_public';
- }
- if (name === 'FLUXER_VAPID_PRIVATE_KEY') {
- return 'vapid_private';
- }
- if (name.endsWith('_BASE64')) {
- return 'base64';
- }
- return 'hex';
- };
- for (const [script, secrets] of [
- ['install.sh', shellSecrets],
- ['install.ps1', powershellSecrets],
- ] as const) {
- for (const [name, kind] of secrets) {
- const wanted = expectedSecretKind(name);
- if (kind !== wanted) {
- problems.push(`${script} generates ${name} as ${kind}, and the value has to be ${wanted}`);
- }
- }
- }
-
- const shippedAssets = (await readdir(path.join(REPO_ROOT, 'deploy/self-hosting')))
- .filter((entry) => entry !== '.gitignore')
- .sort();
- const shellStackFiles = shellRows('fluxer_stack_files', 'FILES', 'the install.sh download list');
- const powershellStackFiles: Array<string> = [];
- for (const row of powershellRows('FluxerStackFiles', 'the install.ps1 download list')) {
- const parsed = row.match(/^\s*'([^']+)'\s*$/u);
- if (parsed == null) {
- problems.push(`the install.ps1 download list has \`${row.trim()}\`, which is not a quoted file name`);
- continue;
- }
- powershellStackFiles.push(parsed[1]);
- }
- for (const [script, list] of [
- ['install.sh', shellStackFiles],
- ['install.ps1', powershellStackFiles],
- ] as const) {
- const sorted = [...list].sort();
- if (sorted.join(', ') !== shippedAssets.join(', ')) {
- problems.push(
- `${script} downloads [${sorted.join(', ')}] and deploy/self-hosting ships [${shippedAssets.join(', ')}]`,
- );
- }
- }
-
- const COMPOSE_DEFAULT_NAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yml', 'docker-compose.yaml'];
- const shellComposeNames = shellRows('fluxer_compose_names', 'NAMES', 'the install.sh compose name list');
- const powershellComposeNames: Array<string> = [];
- for (const row of powershellRows('FluxerComposeNames', 'the install.ps1 compose name list')) {
- const parsed = row.match(/^\s*'([^']+)'\s*$/u);
- if (parsed == null) {
- problems.push(`the install.ps1 compose name list has \`${row.trim()}\`, which is not a quoted file name`);
- continue;
- }
- powershellComposeNames.push(parsed[1]);
- }
- for (const [script, list] of [
- ['install.sh', shellComposeNames],
- ['install.ps1', powershellComposeNames],
- ] as const) {
- if (list.join(', ') !== COMPOSE_DEFAULT_NAMES.join(', ')) {
- problems.push(
- `${script} resolves compose files as [${list.join(', ')}] and Compose resolves them as [${COMPOSE_DEFAULT_NAMES.join(', ')}]`,
- );
- }
- }
- if (!shellStackFiles.includes('docker-compose.yml') || !COMPOSE_DEFAULT_NAMES.includes('docker-compose.yml')) {
- problems.push(
- 'the installers no longer download docker-compose.yml, so the name they map onto an instance is unclear',
- );
- }
-
- const PIPE_TO_SHELL =
- /(?:curl|wget|iwr|Invoke-WebRequest)[^\n|]*\|\s*(?:sudo\s+)?(?:sh|bash|zsh|iex|Invoke-Expression)\b/iu;
- for (const [name, source] of installers) {
- if (PIPE_TO_SHELL.test(source)) {
- problems.push(`${name} pipes a download into a shell`);
- }
- }
- for (const {relativePath, source} of pages) {
- if (PIPE_TO_SHELL.test(source)) {
- problems.push(`${relativePath} pipes a download into a shell`);
- }
- }
-
- for (const [name, source] of installers) {
- if (source.includes('—')) {
- problems.push(`${name} contains an em dash`);
- }
- if (source.includes('\r')) {
- problems.push(`${name} contains a CR byte, so the served digest would disagree with sha256sum on the file`);
- }
- if (!source.endsWith('\n') || source.endsWith('\n\n')) {
- problems.push(`${name} does not end with exactly one newline`);
- }
- }
-
- for (const [name, source] of installers) {
- const published = installerChecksumLine(name, source);
- const expected = `${createHash('sha256').update(Buffer.from(source, 'utf8')).digest('hex')} ${name}\n`;
- if (published !== expected) {
- problems.push(
- `the published checksum for ${name} is \`${published.trim()}\` and sha256 of the file is \`${expected.trim()}\``,
- );
- }
- }
-
- const INSTALLER_ENDPOINTS = ['install.sh.ts', 'install.sh.sha256.ts', 'install.ps1.ts', 'install.ps1.sha256.ts'];
- const digestBearing: Array<[string, string]> = [];
- for (const endpoint of INSTALLER_ENDPOINTS) {
- const source = await readFile(fileURLToPath(new URL(`../src/pages/${endpoint}`, import.meta.url)), 'utf8');
- if (!source.includes("from '@/installer/Installer'")) {
- problems.push(`src/pages/${endpoint} no longer serves the script through src/installer/Installer`);
- }
- digestBearing.push([`src/pages/${endpoint}`, source]);
- }
- digestBearing.push([
- 'src/components/InstallerChecksum.astro',
- await readFile(fileURLToPath(new URL('../src/components/InstallerChecksum.astro', import.meta.url)), 'utf8'),
- ]);
- for (const {relativePath, source} of pages) {
- if (source.includes('install.sh') || source.includes('install.ps1')) {
- digestBearing.push([relativePath, source]);
- }
- }
- for (const [where, text] of digestBearing) {
- if (/\b[0-9a-f]{64}\b/u.test(text)) {
- problems.push(
- `${where} contains a literal 64-character hex digest, which goes stale the next time a script changes`,
- );
- }
- }
-
- if (options['source-only']) {
- console.log(' installer execution skipped (--source-only)');
- } else {
- problems.push(...(await verifyInstallerExecution(INSTALLER_ROOT)));
- }
-
- const covered = new Set<string>(
- [...placeholders].filter((name) => shellSecrets.has(name) && powershellSecrets.has(name)),
- );
-
- const compose = await readFile(path.join(REPO_ROOT, 'deploy/self-hosting/docker-compose.yml'), 'utf8');
- const composeLines = compose.split('\n');
-
- const composeRequired = new Set<string>();
- for (const required of compose.matchAll(/\$\{([A-Z][A-Z0-9_]*):\?/gu)) {
- composeRequired.add(required[1]);
- }
- if (composeRequired.size < COMPOSE_REQUIRED_FLOOR) {
- problems.push(
- `docker-compose.yml parsed to ${composeRequired.size.toString()} :?-required variables, floor is ${COMPOSE_REQUIRED_FLOOR.toString()}`,
- );
- }
- for (const required of compose.matchAll(/\$\{([A-Z][A-Z0-9_]*):\?([^}]*)\}/gu)) {
- if (required[2].trim().length === 0) {
- problems.push(
- `docker-compose.yml requires ${required[1]} with an empty :? message, so Compose stops on a bare variable name`,
- );
- }
- }
- const vapidEmailDerivation = 'FLUXER_VAPID_EMAIL: ${FLUXER_VAPID_EMAIL:-admin@${FLUXER_DOMAIN}}';
- if (!compose.includes(vapidEmailDerivation)) {
- problems.push('docker-compose.yml no longer derives FLUXER_VAPID_EMAIL from FLUXER_DOMAIN');
- } else if (assigned.has('FLUXER_VAPID_EMAIL')) {
- problems.push(
- '.env.example assigns FLUXER_VAPID_EMAIL, so a copied .env defeats the admin@FLUXER_DOMAIN derivation',
- );
- }
- let composeRequiredCovered = 0;
- for (const name of composeRequired) {
- const missing: Array<string> = [];
- if (!shellWrites.has(name)) {
- missing.push('install.sh');
- }
- if (!powershellWrites.has(name)) {
- missing.push('install.ps1');
- }
- if (missing.length === 0) {
- composeRequiredCovered += 1;
- continue;
- }
- problems.push(`docker-compose.yml requires ${name} with :? and ${missing.join(' and ')} never writes it`);
- }
- const publishers = new Set<string>();
- const publishedPorts = new Set<string>();
- let currentService: string | null = null;
- for (let i = 0; i < composeLines.length; i += 1) {
- const service = composeLines[i].match(/^ {2}([a-z][a-z0-9_-]*):\s*$/u);
- if (service != null) {
- currentService = service[1];
- continue;
- }
- if (!/^ {4}ports:\s*$/u.test(composeLines[i]) || currentService == null) {
- continue;
- }
- publishers.add(currentService);
- for (let j = i + 1; j < composeLines.length && /^\s*-\s/u.test(composeLines[j]); j += 1) {
- const mapping = composeLines[j].match(/(\d+)\}?(\/udp)?"\s*$/u);
- if (mapping != null) {
- publishedPorts.add(`${mapping[1]}${mapping[2] ?? ''}`);
- }
- }
- }
- if (!guide.includes('The edge and LiveKit are the only services that publish ports')) {
- problems.push('the guide no longer states which services publish ports');
- } else if (publishers.size !== 2 || !publishers.has('edge') || !publishers.has('livekit')) {
- problems.push(
- `the guide says only the edge and LiveKit publish ports, but the compose file publishes from [${[...publishers].sort().join(', ')}]`,
- );
- }
- for (const port of publishedPorts) {
- const bare = port.replace('/udp', '');
- if (!guide.includes(bare)) {
- problems.push(`the compose file publishes ${port} but the guide never mentions ${bare}`);
- }
- }
-
- const livekitConfig = blockBetween(
- compose,
- ' LIVEKIT_CONFIG: |\n',
- '\n ports:\n',
- "the livekit service's LIVEKIT_CONFIG",
- );
- if (livekitConfig != null) {
- const coupled: ReadonlyArray<readonly [string, string]> = [
- ['tcp_port', '${FLUXER_LIVEKIT_TCP_PORT:-7881}'],
- ['udp_port', '${FLUXER_LIVEKIT_UDP_PORT:-7882}'],
- ['api_key', '${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}'],
- ];
- for (const [key, expression] of coupled) {
- if (!livekitConfig.includes(`${key}: ${expression}`)) {
- problems.push(`LIVEKIT_CONFIG sets ${key} to something other than ${expression}, so it no longer follows .env`);
- }
- }
- for (const mapping of [
- '${FLUXER_LIVEKIT_TCP_PORT:-7881}:${FLUXER_LIVEKIT_TCP_PORT:-7881}',
- '${FLUXER_LIVEKIT_UDP_PORT:-7882}:${FLUXER_LIVEKIT_UDP_PORT:-7882}/udp',
- ]) {
- if (!compose.includes(`"${mapping}"`)) {
- problems.push(
- `the livekit service no longer publishes ${mapping}, so the port it advertises in ICE candidates is not the one the host forwards`,
- );
- }
- }
- }
-
- const configPage = await readFile(path.join(DOCS_ROOT, 'operator/configuration.mdx'), 'utf8');
- const configLines = configPage.split('\n');
- const envValues = new Map<string, string>();
- for (const line of envExample.split('\n')) {
- const assignment = line.match(/^([A-Z][A-Z0-9_]*)=(.*)$/u);
- if (assignment != null) {
- envValues.set(assignment[1], assignment[2]);
- }
- }
-
- const ENV_VALUE_FLOOR = 16;
- const ENV_VALUE_UNSET = 'unset';
- let inEnvValueTable = false;
- let envEntry: string | null = null;
- let envRowsChecked = 0;
- const checkEnvValue = (name: string, documented: string): void => {
- envRowsChecked += 1;
- const actual = envValues.get(name);
- if (documented === ENV_VALUE_UNSET) {
- if (actual != null) {
- problems.push(`configuration.mdx says ${name} is unset but .env.example assigns \`${actual}\``);
- }
- return;
- }
- if (actual == null) {
- problems.push(`configuration.mdx documents ${name} under "Value in .env.example" but it is not in that file`);
- return;
- }
- if (actual !== documented) {
- problems.push(`configuration.mdx says ${name} is \`${documented}\` but .env.example has \`${actual}\``);
- }
- };
- for (const line of configLines) {
- const entryHeading = line.match(/^#### `([A-Z][A-Z0-9_]*)`(?:<sup>[^<]*<\/sup>)?\s*$/u);
- if (entryHeading != null) {
- envEntry = entryHeading[1];
- inEnvValueTable = false;
- continue;
- }
- if (/^#{1,6}\s/u.test(line)) {
- envEntry = null;
- }
- if (envEntry != null) {
- const marker = line.match(/^`\.env\.example`\s+(?:`([^`]*)`|(unset)\b)/u);
- if (marker != null) {
- checkEnvValue(envEntry, marker[1] ?? ENV_VALUE_UNSET);
- envEntry = null;
- continue;
- }
- }
- if (/^\|\s*Variable\s*\|/u.test(line)) {
- inEnvValueTable = line.includes('Value in `.env.example`');
- continue;
- }
- if (!line.startsWith('|')) {
- inEnvValueTable = false;
- continue;
- }
- if (!inEnvValueTable) {
- continue;
- }
- const row = line.match(/^\|\s*([A-Z][A-Z0-9_]*)(?:<sup>[^<]*<\/sup>)?\s*\|\s*(?:`([^`]*)`|(unset))\s*\|/u);
- if (row == null) {
- continue;
- }
- checkEnvValue(row[1], row[2] ?? ENV_VALUE_UNSET);
- }
- console.log(` configuration.mdx .env.example values checked: ${envRowsChecked.toString()}`);
- if (envRowsChecked < ENV_VALUE_FLOOR) {
- problems.push(
- `configuration.mdx .env.example values checked fell to ${envRowsChecked.toString()}, floor is ${ENV_VALUE_FLOOR.toString()}`,
- );
- }
-
- const composeServices = new Set<string>();
- {
- let inServices = false;
- for (const line of composeLines) {
- if (/^services:\s*$/u.test(line)) {
- inServices = true;
- continue;
- }
- if (/^[a-z]/u.test(line)) {
- inServices = false;
- }
- const service = line.match(/^ {2}([a-z][a-z0-9_-]*):\s*$/u);
- if (inServices && service != null) {
- composeServices.add(service[1]);
- }
- }
- }
- const operatorPages = ['operator/get-started.mdx', 'operator/configuration.mdx', 'operator/upgrading.mdx'];
- const namedServices = new Set<string>();
- for (const page of operatorPages) {
- const text = await readFile(path.join(DOCS_ROOT, page), 'utf8');
- for (const backticked of text.matchAll(/`([a-z][a-z0-9-]{2,})`/gu)) {
- namedServices.add(backticked[1]);
- }
- }
- const knownNonServices = new Set(['docker-compose.yml', 'chat.example.com', 'admin.example.com', 'localhost']);
- for (const candidate of namedServices) {
- if (!candidate.endsWith('-shard') && !candidate.startsWith('fluxer-') && !composeServices.has(candidate)) {
- continue;
- }
- if (knownNonServices.has(candidate)) {
- continue;
- }
- const isShard = candidate.endsWith('-shard');
- const isImage = candidate.startsWith('fluxer-');
- if (!isShard && !isImage) {
- continue;
- }
- if (isShard && !composeServices.has(candidate)) {
- problems.push(`the operator docs name the service \`${candidate}\`, which is not in docker-compose.yml`);
- }
- if (isImage && !compose.includes(candidate)) {
- problems.push(`the operator docs name the image \`${candidate}\`, which docker-compose.yml never references`);
- }
- }
- console.log(` compose services: ${composeServices.size.toString()}`);
-
- const files = ['docker-compose.yml', 'Caddyfile', '.env.example'];
- for (const file of files) {
- if (!guide.includes(file)) {
- problems.push(`the guide never mentions ${file}, which the stack ships`);
- }
- }
-
- console.log(` .env.example placeholders: ${placeholders.size.toString()}`);
- console.log(` set by the guide: ${[...placeholders].filter((n) => covered.has(n)).length.toString()}`);
- console.log(` installer secret keys (sh): ${shellSecrets.size.toString()}`);
- console.log(` installer secret keys (ps1): ${powershellSecrets.size.toString()}`);
- console.log(` installer non-secret keys: ${shellNonSecrets.size.toString()}`);
- console.log(` compose required variables covered: ${composeRequiredCovered.toString()}`);
- console.log(` services publishing ports: ${[...publishers].sort().join(', ')}`);
- console.log(` published ports: ${[...publishedPorts].sort().join(', ')}`);
- failures += section('self-hosting guide disagreements', problems);
-}
-
-console.log('Kubernetes page against deploy/helm/fluxer');
-{
- const chartValues = await readFile(path.join(REPO_ROOT, 'deploy/helm/fluxer/values.yaml'), 'utf8');
- const page = await readFile(path.join(DOCS_ROOT, 'operator/kubernetes.mdx'), 'utf8');
- const problems: Array<string> = [];
-
- const chartKeys: Array<string> = [];
- let inSecrets = false;
- let inValues = false;
- for (const line of chartValues.split('\n')) {
- if (/^\S/u.test(line)) {
- inSecrets = line.startsWith('secrets:');
- inValues = false;
- continue;
- }
- if (inSecrets && /^ {2}\S/u.test(line)) {
- inValues = line.startsWith(' values:');
- continue;
- }
- const key = inValues ? line.match(/^ {4}([A-Z][A-Z0-9_]*):/u) : null;
- if (key != null) {
- chartKeys.push(key[1]);
- }
- }
- if (chartKeys.length === 0) {
- problems.push('found no keys under secrets.values in deploy/helm/fluxer/values.yaml');
- }
-
- const tableKeys = new Set<string>();
- const header = '| Key | Chart default |';
- const lines = page.split('\n');
- const start = lines.indexOf(header);
- if (start === -1) {
- problems.push(`operator/kubernetes.mdx has no table headed \`${header}\``);
- } else {
- for (const row of lines.slice(start + 2)) {
- const cell = row.match(/^\| `([A-Z][A-Z0-9_]*)` \|/u);
- if (cell == null) {
- break;
- }
- tableKeys.add(cell[1]);
- }
- }
- for (const key of chartKeys) {
- if (!tableKeys.has(key)) {
- problems.push(`secrets.values.${key} is missing from the secret table on operator/kubernetes.mdx`);
- }
- }
- for (const key of tableKeys) {
- if (!chartKeys.includes(key)) {
- problems.push(`operator/kubernetes.mdx lists ${key}, which secrets.values does not declare`);
- }
- }
- console.log(` chart secret keys: ${chartKeys.length.toString()}`);
- console.log(` documented in the secret table: ${tableKeys.size.toString()}`);
- failures += section('Kubernetes page disagreements', problems);
-}
-
-console.log('unthrottled routes and global bucket claims');
-{
- const unthrottled = controllerRoutes
- .filter((route) => route.rateLimitConfig == null)
- .map((route) => ({
- method: route.method.toUpperCase(),
- route: route.path,
- admin: route.path.startsWith('/admin'),
- }));
-
- const publicShapes = new Set(main.map((operation) => routeShape(operation.method, operation.path)));
- const publicUnthrottled = unthrottled
- .filter((entry) => !entry.admin)
- .map((entry) => routeShape(entry.method, stripVersionPrefix(entry.route.replace(/:([a-zA-Z_]+)/gu, '{$1}'))))
- .filter((shape) => publicShapes.has(shape));
- const adminUnthrottled = unthrottled.filter((entry) => entry.admin);
-
- const page = await readFile(path.join(DOCS_ROOT, 'topics/rate-limits.md'), 'utf8');
- const problems: Array<string> = [];
- const uniquePublic = [...new Set(publicUnthrottled)].sort();
-
- const unexpected = uniquePublic;
-
- if (unexpected.length > 0) {
- problems.push(
- `rate-limits.md says every HTTP API operation declares a bucket, but ${unexpected.length.toString()} declare none: ${unexpected.join(', ')}`,
- );
- }
- if (!page.includes('Every HTTP API and Admin API operation declares a bucket')) {
- problems.push('rate-limits.md no longer states that every operation declares a bucket');
- }
- if (adminUnthrottled.length > 0) {
- const named = adminUnthrottled.map((entry) => `${entry.method} ${entry.route}`).sort();
- problems.push(
- `rate-limits.md says every Admin API operation declares a bucket, but ${named.length.toString()} declare none: ${named.join(', ')}`,
- );
- }
- for (const stale of ['declares no bucket', 'Two [Admin API]']) {
- if (page.includes(stale)) {
- problems.push(`rate-limits.md still says \`${stale}\`, and every operation now declares one`);
- }
- }
-
- const middleware = await readFile(
- path.join(REPO_ROOT, 'fluxer_api/src/api/middleware/RateLimitMiddleware.ts'),
- 'utf8',
- );
- const high = middleware.match(/HIGH_GLOBAL_RATE_LIMIT\) !== 0n\) \{\s*return (\d+);/u);
- if (high != null && !page.includes(`${Number.parseInt(high[1], 10).toLocaleString('en-US')} requests per second`)) {
- problems.push(`rate-limits.md does not state the HIGH_GLOBAL_RATE_LIMIT allowance of ${high[1]}`);
- }
-
- console.log(` public operations with no rate limit middleware: ${uniquePublic.length.toString()}`);
- console.log(` admin operations with no rate limit middleware: ${adminUnthrottled.length.toString()}`);
- failures += section('rate limit prose disagreements', problems);
-}
-
-console.log('error registry');
-{
- const errorsPage = await readFile(path.join(DOCS_ROOT, 'http-api/errors.md'), 'utf8');
- const documentedCodes = new Set<string>();
- let documentedEntries = 0;
- for (const row of errorsPage.matchAll(/^\|\s*`?([A-Z][A-Z0-9_]{2,})`?\s*\|/gmu)) {
- documentedCodes.add(row[1]);
- documentedEntries += 1;
- }
- for (const entry of errorsPage.matchAll(/^#{3,4} `([A-Z][A-Z0-9_]{2,})`\s*$/gmu)) {
- documentedCodes.add(entry[1]);
- documentedEntries += 1;
- }
- const registrySource = await readFile(path.join(REPO_ROOT, 'packages/constants/src/ApiErrorCodes.ts'), 'utf8');
- const registryCodes = new Set<string>();
- for (const entry of registrySource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {
- registryCodes.add(entry[1]);
- }
- const validationSource = await readFile(
- path.join(REPO_ROOT, 'packages/constants/src/ValidationErrorCodes.ts'),
- 'utf8',
- );
- const validationCodes = new Set<string>();
- for (const entry of validationSource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {
- validationCodes.add(entry[1]);
- }
- const problems: Array<string> = [];
- for (const code of registryCodes) {
- if (!documentedCodes.has(code)) {
- problems.push(`APIErrorCodes.${code} is not listed in the errors page registry`);
- }
- }
-
- const documentedRegistryCodes = [...registryCodes].filter((c) => documentedCodes.has(c)).length;
- const UNDOCUMENTED_VALIDATION_CODES = new Set(['EMAIL_DOMAIN_CANNOT_RECEIVE_MAIL']);
- const expectedEntries = registryCodes.size + validationCodes.size - UNDOCUMENTED_VALIDATION_CODES.size;
- if (documentedEntries < expectedEntries) {
- problems.push(
- `errors.md code entries parsed fell to ${documentedEntries.toString()}, expected ${expectedEntries.toString()}, the ${registryCodes.size.toString()} API codes plus the ${validationCodes.size.toString()} validation codes less the ${UNDOCUMENTED_VALIDATION_CODES.size.toString()} named as undocumented. Either a code lost its entry, or the registry parser stopped matching one of the \`| CODE |\` table row and \`### \`CODE\`\` or \`#### \`CODE\`\` heading shapes it reads`,
- );
- }
- for (const code of documentedCodes) {
- if (!registryCodes.has(code) && !validationCodes.has(code)) {
- problems.push(`errors.md documents ${code}, which is in neither code registry`);
- }
- }
- if (documentedRegistryCodes < registryCodes.size) {
- problems.push(
- `errors.md documents ${documentedRegistryCodes.toString()} of the ${registryCodes.size.toString()} registry codes, floor is every one of them`,
- );
- }
-
- console.log(` registry codes: ${registryCodes.size.toString()}, documented: ${documentedRegistryCodes.toString()}`);
- failures += section('error registry disagreements', problems);
-}
-
-console.log('gateway numeric limits');
-{
- const erlConstants = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/utils/constants.erl'), 'utf8');
- const sharding = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/gateway_sharding.erl'), 'utf8');
- const abuse = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/session_abuse_protection.erl'), 'utf8');
- const heartbeat = await readFile(
- path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/gateway_handler_heartbeat.erl'),
- 'utf8',
- );
- const limitsPage = await readFile(path.join(DOCS_ROOT, 'gateway/limits-and-rate-limits.md'), 'utf8');
- const overviewPage = await readFile(path.join(DOCS_ROOT, 'gateway/overview.md'), 'utf8');
- const corpus = `${limitsPage}\n${overviewPage}`;
-
- const readFn = (source: string, name: string): number | null => {
- const found = source.match(new RegExp(`${name}\\(\\) -> (\\d+)`, 'u'));
- return found == null ? null : Number.parseInt(found[1], 10);
- };
- const readDefine = (source: string, name: string): number | null => {
- const found = source.match(new RegExp(`-define\\(${name}, (\\d+)\\)`, 'u'));
- return found == null ? null : Number.parseInt(found[1], 10);
- };
- const grouped = (value: number): string => value.toLocaleString('en-US');
-
- const problems: Array<string> = [];
- const interval = readFn(erlConstants, 'heartbeat_interval');
-
- const expectations: Array<[string, number | null, (value: string) => string]> = [
- ['heartbeat interval', interval, (v) => `heartbeat interval of ${v} ms`],
- ['resume timeout', readFn(erlConstants, 'resume_timeout'), (v) => `resumable for ${v} ms`],
- ['max payload size', readFn(erlConstants, 'max_payload_size'), (v) => v],
- ['max shard count', readDefine(sharding, 'MAX_SHARD_COUNT'), (v) => `through ${v}`],
- ['max guilds per shard', readDefine(sharding, 'MAX_GUILDS_PER_SHARD'), (v) => `at most ${v} guilds`],
- ['max sessions per user', readDefine(abuse, 'MAX_SESSIONS_PER_USER'), (v) => `at most ${v} live sessions`],
- ];
- if (interval != null && heartbeat.includes('div 3')) {
- expectations.push(['heartbeat check cadence, interval div 3', Math.floor(interval / 3), (v) => `every ${v} ms`]);
- }
- if (interval != null && heartbeat.includes('* 10) >= (Interval * 9)')) {
- expectations.push([
- 'heartbeat ping threshold, interval times 9 over 10',
- (interval * 9) / 10,
- (v) => `When ${v} ms have elapsed`,
- ]);
- }
- let compared = 0;
- for (const [label, value, phrase] of expectations) {
- if (value == null) {
- problems.push(`could not read ${label} from the Gateway source`);
- continue;
- }
- compared += 1;
- if (!corpus.includes(phrase(grouped(value))) && !corpus.includes(phrase(String(value)))) {
- problems.push(`the Gateway docs never state the ${label} of ${grouped(value)} in the expected phrasing`);
- }
- }
- console.log(` gateway constants compared: ${compared.toString()}`);
- failures += section('gateway limit disagreements', problems);
-}
-
-console.log('snowflake layout');
-{
- const core = await readFile(path.join(REPO_ROOT, 'packages/constants/src/Core.ts'), 'utf8');
- const snowflakeSource = await readFile(path.join(REPO_ROOT, 'packages/snowflake/src/Snowflake.ts'), 'utf8');
- const page = await readFile(path.join(DOCS_ROOT, 'snowflakes.md'), 'utf8');
- const problems: Array<string> = [];
-
- const epochMatch = core.match(/export const FLUXER_EPOCH = (\d+);/u);
- const workerBits = snowflakeSource.match(/WORKER_ID_BITS = (\d+)n/u);
- const sequenceBits = snowflakeSource.match(/SEQUENCE_BITS = (\d+)n/u);
-
- if (epochMatch == null || workerBits == null || sequenceBits == null) {
- problems.push('could not read FLUXER_EPOCH, WORKER_ID_BITS or SEQUENCE_BITS from source');
- } else {
- const epoch = Number.parseInt(epochMatch[1], 10);
- const worker = Number.parseInt(workerBits[1], 10);
- const sequence = Number.parseInt(sequenceBits[1], 10);
- const shift = worker + sequence;
- if (!page.includes(String(epoch))) {
- problems.push(`snowflakes.md does not state the epoch ${epoch.toString()}`);
- }
- const epochIso = new Date(epoch).toISOString();
- if (!page.includes(epochIso.slice(0, 10))) {
- problems.push(`snowflakes.md does not state the epoch date ${epochIso}`);
- }
- if (!page.includes(`right by ${shift.toString()} bits`)) {
- problems.push(`snowflakes.md does not state the timestamp shift of ${shift.toString()} bits`);
- }
- if (!page.includes(`through \`${((1 << worker) - 1).toString()}\``)) {
- problems.push(`snowflakes.md does not state the maximum worker id of ${((1 << worker) - 1).toString()}`);
- }
- const timestampBits = 63 - shift;
- const lastMs = epoch + 2 ** timestampBits - 1;
- const lastIso = new Date(lastMs).toISOString();
- if (!page.includes(lastIso.slice(0, 19))) {
- problems.push(`snowflakes.md does not state the last representable instant ${lastIso}`);
- }
- console.log(
- ` epoch ${epoch.toString()} (${epochIso.slice(0, 10)}), shift ${shift.toString()}, worker ${worker.toString()} bits, sequence ${sequence.toString()} bits`,
- );
- console.log(` last representable instant: ${lastIso}`);
- }
- failures += section('snowflake layout disagreements', problems);
-}
-
-console.log('attachment upload geometry');
-{
- const limits = await readFile(path.join(REPO_ROOT, 'packages/constants/src/LimitConstants.ts'), 'utf8');
- const problems: Array<string> = [];
- const readConst = (name: string): number | null => {
- const found = limits.match(new RegExp(`${name} = ([^;]+);`, 'u'));
- if (found == null) {
- return null;
- }
- const expression = found[1].replace(/_/gu, '').trim();
- if (!/^[\d*+\s]+$/u.test(expression)) {
- return null;
- }
- return expression
- .split('+')
- .map((term) => term.split('*').reduce((a, b) => a * Number.parseInt(b.trim(), 10), 1))
- .reduce((a, b) => a + b, 0);
- };
-
- const constants: Array<[string, string, string]> = [
- ['ATTACHMENT_UPLOAD_CHUNK_THRESHOLD', 'singlepart threshold', 'topics/uploads.md'],
- ['ATTACHMENT_MAX_SIZE_NON_PREMIUM', 'non-premium attachment ceiling', 'http-api/messages.mdx'],
- ['ATTACHMENT_MAX_SIZE_PREMIUM', 'premium attachment ceiling', 'http-api/messages.mdx'],
- ['ATTACHMENT_MAX_SIZE_BOT', 'bot attachment ceiling', 'http-api/messages.mdx'],
- ];
- let compared = 0;
- for (const [name, label, file] of constants) {
- const page = pages.find((entry) => entry.relativePath === file);
- if (!page) throw new Error(`Missing upload documentation: ${file}`);
- const value = readConst(name);
- if (value == null) {
- problems.push(`could not read ${name} from LimitConstants.ts`);
- continue;
- }
- compared += 1;
- const bare = new RegExp(`(?<![\\d,.])${value.toString()}(?![\\d,.])`, 'u');
- const grouped = new RegExp(`(?<![\\d,.])${value.toLocaleString('en-US')}(?![\\d,.])`, 'u');
- if (!bare.test(page.source) && !grouped.test(page.source)) {
- problems.push(`${file} does not state the ${label} of ${value.toString()}`);
- }
- }
- console.log(` upload constants compared: ${compared.toString()}`);
- failures += section('upload geometry disagreements', problems);
-}
-
-console.log('captcha gated operations');
-{
- const page = await readFile(path.join(DOCS_ROOT, 'topics/captcha.md'), 'utf8');
- const documented = new Set<string>();
- const documentedLabels = new Map<string, string>();
- for (const row of page.matchAll(/^\|\s*(GET|POST|PUT|PATCH|DELETE)\s*\|\s*(\/v1\/\S+?)\s*\|/gmu)) {
- const normalised = routeShape(row[1], stripVersionPrefix(row[2]));
- documented.add(normalised);
- documentedLabels.set(normalised, `${row[1]} ${row[2]}`);
- }
-
- const gated = new Set<string>();
- for (const route of controllerRoutes) {
- if (route.middlewares.some((name) => /Captcha|GroupDm\w*Protection/u.test(name))) {
- gated.add(astRoute(route));
- }
- }
-
- const problems: Array<string> = [];
- for (const route of gated) {
- if (!documented.has(route)) {
- problems.push(`${route} is captcha gated in source but not listed on the captcha page`);
- }
- }
- for (const route of documented) {
- if (!gated.has(route)) {
- problems.push(
- `the captcha page lists ${documentedLabels.get(route) ?? route}, which is not captcha gated in source`,
- );
- }
- }
- console.log(` captcha gated routes in source: ${gated.size.toString()}, documented: ${documented.size.toString()}`);
- failures += section('captcha gating disagreements', problems);
-}
-
-console.log('bot capability flag (from the middleware chain)');
-{
- const BOT_EXEMPT = new Map([
- ['GET /applications/@me', {docs: true, why: 'reads the Authorization header directly in the handler'}],
- ['GET /gateway/bot', {docs: true, why: 'requires a bot-shaped token, parsed in the handler'}],
- ['POST /guilds', {docs: false, why: 'the service rejects bots with BOTS_CANNOT_CREATE_GUILDS'}],
- ['POST /guilds/{}/delete', {docs: false, why: 'the caller must own the guild and a bot never can'}],
- ]);
- const problems: Array<string> = [];
- let compared = 0;
- for (const route of controllerRoutes) {
- const key = astRoute(route);
- if (key.startsWith('GET /admin') || key.includes(' /admin/')) {
- continue;
- }
- const documented = documentedMain.get(key);
- if (documented == null) {
- continue;
- }
- const anyLogin = route.hasLoginRequired;
- const sourceAcceptsBot = (anyLogin || route.hasBotOnly) && !route.hasDefaultUserOnly;
- const exemption = BOT_EXEMPT.get(key);
- if (exemption != null) {
- if (documented.bot !== exemption.docs) {
- problems.push(
- `${key} is an exempted route expected to document bot=${String(exemption.docs)} (${exemption.why}), but the page says ${String(documented.bot)}`,
- );
- }
- continue;
- }
- compared += 1;
- if (documented.bot !== sourceAcceptsBot) {
- problems.push(
- `${key} documents bot=${String(documented.bot)} but the middleware chain gives ${String(sourceAcceptsBot)}`,
- );
- }
- }
- console.log(` routes compared against the middleware chain: ${compared.toString()}`);
- console.log(` exempted, each verified by hand: ${BOT_EXEMPT.size.toString()}`);
- failures += section('bot flag disagreements', problems);
-}
-
-console.log('unauthenticated capability flag (from the middleware chain)');
-{
- const WEBHOOK_TOKEN_CREDENTIAL = 'the webhook ID and token in the path are the complete credential';
- const OAUTH2_CLIENT_CREDENTIAL =
- 'the client authenticates with HTTP Basic or the client_id and client_secret form fields';
- const UNAUTHENTICATED_EXEMPT = new Map([
- ['GET /applications/@me', {docs: false, why: 'reads the Authorization header directly in the handler'}],
- ['GET /gateway/bot', {docs: false, why: 'requires a bot-shaped token, parsed in the handler'}],
- [
- 'POST /auth/handoff/complete',
- {docs: false, why: 'the approving session token comes from the Authorization header or the body token field'},
- ],
- ['POST /oauth2/token', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}],
- ['POST /oauth2/introspect', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}],
- ['POST /oauth2/token/revoke', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}],
- ['GET /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['PATCH /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['DELETE /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['POST /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['GET /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['PATCH /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['DELETE /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['POST /webhooks/{}/{}/github', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['POST /webhooks/{}/{}/slack', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['POST /webhooks/{}/{}/instatus', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ['POST /webhooks/{}/{}/libredesk', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],
- ]);
- const problems: Array<string> = [];
- let compared = 0;
- for (const route of controllerRoutes) {
- const key = astRoute(route);
- if (key.startsWith('GET /admin') || key.includes(' /admin/')) {
- continue;
- }
- const documented = documentedMain.get(key);
- if (documented == null) {
- continue;
- }
- const anyLogin = route.hasLoginRequired;
- const sourceIsOpen = !anyLogin && !route.middlewares.some((name) => /OAuth2Scope/u.test(name));
- const exemption = UNAUTHENTICATED_EXEMPT.get(key);
- if (exemption != null) {
- if (documented.unauthenticated !== exemption.docs) {
- problems.push(
- `${key} is an exempted route expected to document unauthenticated=${String(exemption.docs)} (${exemption.why}), but the page says ${String(documented.unauthenticated)}`,
- );
- }
- continue;
- }
- compared += 1;
- if (documented.unauthenticated !== sourceIsOpen) {
- problems.push(
- `${key} documents unauthenticated=${String(documented.unauthenticated)} but the middleware chain gives ${String(sourceIsOpen)}`,
- );
- }
- }
- console.log(` routes compared against the middleware chain: ${compared.toString()}`);
- console.log(` exempted, each verified by hand: ${UNAUTHENTICATED_EXEMPT.size.toString()}`);
- failures += section('unauthenticated flag disagreements', problems);
-}
-
-console.log('spec security field against the middleware chain');
-{
- const MANUAL_CREDENTIAL = new Set(['GET /applications/@me']);
- const BOT_SCHEME_EXEMPT = new Map<string, string>([
- [
- 'POST /guilds',
- 'LoginRequired admits a bot token, but GuildOperationsService rejects every bot with 400 BOTS_CANNOT_CREATE_GUILDS, so the spec must not advertise botToken',
- ],
- ]);
- const specSecurity = new Map<string, EffectiveSpecSecurity>();
- for (const operation of main) {
- const security = operation.security ?? [];
- const schemes = new Set<string>();
- for (const entry of security) {
- for (const scheme of Object.keys(entry)) {
- schemes.add(scheme);
- }
- }
- specSecurity.set(routeShape(operation.method, operation.path), {
- schemes,
- allowsAnonymous: security.length === 0 || security.some((entry) => Object.keys(entry).length === 0),
- });
- }
- const specBugs: Array<string> = [];
- let compared = 0;
- for (const route of controllerRoutes) {
- const key = astRoute(route);
- const declaredSecurity = specSecurity.get(key);
- if (declaredSecurity == null || MANUAL_CREDENTIAL.has(key)) {
- continue;
- }
- const declaredSchemes = declaredSecurity.schemes;
- compared += 1;
- const anyLogin = route.hasLoginRequired;
- const acceptsBot = (anyLogin || route.hasBotOnly) && !route.hasDefaultUserOnly;
- const requiresAuthentication =
- anyLogin ||
- route.hasDefaultUserOnly ||
- route.hasBotOnly ||
- route.oauth2BearerTokenRequired ||
- route.middlewares.includes('requireOAuth2Scope');
- if (requiresAuthentication && declaredSecurity.allowsAnonymous) {
- specBugs.push(`${key} allows unauthenticated requests, but the middleware chain requires authentication`);
- }
- const exemption = BOT_SCHEME_EXEMPT.get(key);
- if (exemption != null) {
- if (declaredSchemes.has('botToken')) {
- specBugs.push(`${key} declares botToken, but is exempted because ${exemption}`);
- }
- continue;
- }
- if (declaredSchemes.has('botToken') && !acceptsBot) {
- specBugs.push(`${key} declares botToken, but DefaultUserOnly rejects bots`);
- }
- if (!declaredSchemes.has('botToken') && acceptsBot && declaredSchemes.size > 0) {
- specBugs.push(`${key} omits botToken, but the middleware chain admits a bot token`);
- }
- if (route.hasBotOnly && [...declaredSchemes].some((scheme) => scheme !== 'botToken')) {
- specBugs.push(`${key} declares [${[...declaredSchemes].join(', ')}], but BotOnly admits only a bot token`);
- }
- if (
- declaredSchemes.size > 0 &&
- !anyLogin &&
- !route.hasBotOnly &&
- !route.middlewares.some((name) => /Admin|OAuth2Scope/iu.test(name))
- ) {
- specBugs.push(`${key} declares [${[...declaredSchemes].join(', ')}], but the chain has no login policy`);
- }
- }
- console.log(` routes compared: ${compared.toString()}`);
- console.log(' GET /applications/@me is excluded because it reads the Authorization header in the');
- console.log(' handler, so it has no middleware to compare against.');
- console.log(` exempted, each verified by hand: ${BOT_SCHEME_EXEMPT.size.toString()}`);
- for (const [key, why] of BOT_SCHEME_EXEMPT) {
- console.log(` ${key}: ${why}`);
- }
- failures += section('spec security disagreements with the middleware chain', specBugs);
-}
-
-console.log('admin API (against the generated spec)');
-const adminTargetOnly = [...documentedAdmin.entries()]
- .filter(([shape]) => !adminShapes.has(shape))
- .map(([, route]) => `${route.method} ${route.path} (${route.file})`)
- .sort();
-const adminLiveOnly = admin
- .filter((operation) => {
- const shape = routeShape(operation.method, operation.path);
- return !documentedAdmin.has(shape);
- })
- .map((operation) => `${operation.method} ${operation.path}`)
- .sort();
-console.log(` live admin operations: ${admin.length.toString()}`);
-console.log(` documented admin operations: ${adminDocumented.length.toString()}`);
-console.log(
- ` documented in the RESTful target shape, awaiting the live admin refactor: ${adminTargetOnly.length.toString()}`,
-);
-console.log(` live admin operations with no documented RESTful equivalent: ${adminLiveOnly.length.toString()}`);
-failures += section('target-shape routes awaiting the live admin refactor', adminTargetOnly);
-failures += section('live admin routes with no documented equivalent', adminLiveOnly);
-
-console.log('');
-console.log(
- `documented routes: ${documented.length.toString()} (main ${mainDocumented.length.toString()}, admin ${adminDocumented.length.toString()}, media proxy ${mediaProxyDocumented.length.toString()})`,
-);
-if (failures > 0) {
- console.error(`FAIL: ${failures.toString()} coverage problems`);
- process.exit(1);
-}
-console.log('OK: every registered fluxer_api route is documented or covered by an exemption rule, and the');
-console.log('documented routes match the live main API, media proxy, and admin target shape');
-
-async function verifyInstallerExecution(installerRoot: string): Promise<Array<string>> {
- const problems: Array<string> = [];
- const shellParse = spawnSync('sh', ['-n', path.join(installerRoot, 'install.sh')], {encoding: 'utf8'});
- if (shellParse.error != null) {
- problems.push(`sh -n could not run against install.sh: ${shellParse.error.message}`);
- } else if (shellParse.status !== 0) {
- problems.push(`sh -n rejects install.sh: ${shellParse.stderr.trim()}`);
- }
-
- const DOCKER_STUB = [
- '#!/bin/sh',
- 'case "$1 $2" in',
- " '--version ') echo 'Docker version 27.1.1, build stub' ;;",
- " 'compose version') if [ \"$3\" = '--short' ]; then echo '2.30.3'; else echo 'v2.30.3'; fi ;;",
- " 'compose config') echo 'ghcr.io/fluxerapp/fluxer-api:v1' ;;",
- 'esac',
- 'exit 0',
- '',
- ].join('\n');
- const CURL_STUB = [
- '#!/bin/sh',
- "out=''",
- "prev=''",
- 'for arg in "$@"; do',
- ' if [ "$prev" = \'-o\' ]; then out=$arg; fi',
- ' prev=$arg',
- 'done',
- '[ -z "$out" ] || printf \'name: fluxer\\nservices:\\n api:\\n image: stub\\n\' > "$out"',
- '',
- ].join('\n');
-
- const sandbox = await mkdtemp(path.join(tmpdir(), 'fluxer-installer-'));
- try {
- const stubBin = path.join(sandbox, 'bin');
- await mkdir(stubBin, {recursive: true});
- await writeFile(path.join(stubBin, 'docker'), DOCKER_STUB, {mode: 0o755});
- await writeFile(path.join(stubBin, 'curl'), CURL_STUB, {mode: 0o755});
- await writeFile(path.join(stubBin, 'openssl'), '#!/bin/sh\nexit 0\n', {mode: 0o755});
-
- const instance = path.join(sandbox, 'instance');
- await mkdir(instance, {recursive: true});
- await writeFile(path.join(instance, '.env'), 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n');
- await writeFile(path.join(instance, 'docker-compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');
-
- const planned = (label: string, args: ReadonlyArray<string>, cwd?: string): string | null => {
- const run = spawnSync('sh', [path.join(installerRoot, 'install.sh'), ...args], {
- cwd,
- encoding: 'utf8',
- env: {
- ...process.env,
- PATH: `${stubBin}${path.delimiter}${process.env.PATH ?? ''}`,
- FLUXER_INSTALLER_REFRESHED: '1',
- ...(cwd == null ? {} : {PWD: cwd}),
- },
- });
- if (run.error != null) {
- problems.push(`install.sh ${label} could not run: ${run.error.message}`);
- return null;
- }
- if (run.status !== 0) {
- problems.push(`install.sh ${label} exited ${String(run.status)}: ${run.stderr.trim()}`);
- return null;
- }
- return run.stdout;
- };
-
- const plannedRef = (label: string, args: ReadonlyArray<string>): string | null => {
- const stdout = planned(label, args);
- if (stdout == null) {
- return null;
- }
- const line = stdout.match(/^ {2}ref\s+(\S+)$/mu);
- if (line == null) {
- problems.push(`install.sh ${label} printed no ref line`);
- return null;
- }
- return line[1];
- };
-
- const INSTALL_ARGS = [
- '--dry-run',
- '--non-interactive',
- '--allow-root',
- '--domain',
- 'x.example',
- '--email',
- '--dir',
- path.join(sandbox, 'target'),
- ];
- const REF_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string]> = [
- ['on the default image tag', INSTALL_ARGS, 'main'],
- ['under --image-tag latest', [...INSTALL_ARGS, '--image-tag', 'latest'], 'main'],
- ['under --image-tag 2026.813.205040', [...INSTALL_ARGS, '--image-tag', '2026.813.205040'], '2026.813.205040'],
- [
- 'under --ref feature/x --image-tag 2026.813.205040',
- [...INSTALL_ARGS, '--ref', 'feature/x', '--image-tag', '2026.813.205040'],
- 'feature/x',
- ],
- [
- 'under --update against a pinned .env',
- ['--update', '--dry-run', '--allow-root', '--dir', instance],
- '2026.813.205040',
- ],
- ];
- for (const [label, args, expected] of REF_CASES) {
- const resolved = plannedRef(label, args);
- if (resolved != null && resolved !== expected) {
- problems.push(`install.sh ${label} plans ref ${resolved}, and the image tag it pairs with wants ${expected}`);
- }
- }
-
- const withoutEmail = INSTALL_ARGS.filter(
- (arg, index) => arg !== '--email' && INSTALL_ARGS[index - 1] !== '--email',
- );
- const withoutEmailPlan = planned('without --email', withoutEmail);
- if (withoutEmailPlan != null && !/^ {2}email\s+admin@x\.example, derived by compose$/mu.test(withoutEmailPlan)) {
- problems.push('install.sh without --email does not plan the admin@FLUXER_DOMAIN contact that compose derives');
- }
-
- const composeYmlInstance = path.join(sandbox, 'compose-yml-instance');
- await mkdir(composeYmlInstance, {recursive: true});
- await writeFile(
- path.join(composeYmlInstance, '.env'),
- 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n',
- );
- await writeFile(path.join(composeYmlInstance, 'compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');
- const COMPOSE_NAME_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string | undefined]> = [
- [
- 'under --update against a compose.yml instance',
- ['--update', '--dry-run', '--allow-root', '--dir', composeYmlInstance],
- undefined,
- ],
- [
- 'under --update standing in a compose.yml instance',
- ['--update', '--dry-run', '--allow-root'],
- composeYmlInstance,
- ],
- ];
- for (const [label, args, cwd] of COMPOSE_NAME_CASES) {
- const stdout = planned(label, args, cwd);
- if (stdout == null) {
- continue;
- }
- if (!/^ {4}compose\.yml is unchanged$/mu.test(stdout)) {
- problems.push(`install.sh ${label} does not plan the refreshed stack file onto compose.yml`);
- }
- if (/^ {4}docker-compose\.yml is new$/mu.test(stdout)) {
- problems.push(`install.sh ${label} plans a docker-compose.yml that Compose would never load there`);
- }
- }
- } finally {
- await rm(sandbox, {recursive: true, force: true});
- }
- return problems;
-}
diff --git a/fluxer_docs/scripts/VerifyDocsSchemas.test.ts b/fluxer_docs/scripts/VerifyDocsSchemas.test.ts
deleted file mode 100644
index b4a26ba3f..000000000
--- a/fluxer_docs/scripts/VerifyDocsSchemas.test.ts
+++ /dev/null
@@ -1,666 +0,0 @@
-import assert from 'node:assert/strict';
-import {test} from 'node:test';
-import type {OpenAPIDocument} from '@fluxer/openapi/src/OpenAPITypes';
-import type {MarkdownPage} from './DocsSource.ts';
-import {type SchemaVerification, verifyPages} from './VerifyDocsSchemas.ts';
-
-const SNOWFLAKE = {$ref: '#/components/schemas/SnowflakeType'};
-const WIDGET = {$ref: '#/components/schemas/Widget'};
-
-const spec = {
- openapi: '3.1.0',
- info: {title: 'fixture', version: '1'},
- paths: {
- '/widgets/{widget_id}': {
- patch: {
- operationId: 'update_widget',
- tags: [],
- requestBody: {content: {'application/json': {schema: {$ref: '#/components/schemas/WidgetUpdate'}}}},
- responses: {'200': {description: 'ok', content: {'application/json': {schema: WIDGET}}}},
- },
- },
- '/widgets': {
- get: {
- operationId: 'list_widgets',
- tags: [],
- parameters: [
- {name: 'limit', in: 'query', required: false, schema: {type: 'integer'}},
- {name: 'owner_id', in: 'query', required: false, schema: SNOWFLAKE},
- ],
- responses: {
- '200': {
- description: 'ok',
- content: {
- 'application/json': {
- schema: {
- type: 'object',
- properties: {widgets: {type: 'array', items: WIDGET}, total: {type: 'integer'}},
- required: ['widgets', 'total'],
- additionalProperties: false,
- },
- },
- },
- },
- },
- },
- post: {
- operationId: 'create_widget',
- tags: [],
- requestBody: {
- content: {
- 'application/json': {
- schema: {
- oneOf: [
- {
- type: 'object',
- properties: {kind: {type: 'string', const: 'alpha'}, alpha: {type: 'string'}},
- required: ['kind', 'alpha'],
- },
- {
- type: 'object',
- properties: {kind: {type: 'string', const: 'beta'}, beta: {type: 'integer'}},
- required: ['kind', 'beta'],
- },
- ],
- },
- },
- },
- },
- responses: {'200': {description: 'ok', content: {'application/json': {schema: WIDGET}}}},
- },
- },
- '/widgets/positions': {
- put: {
- operationId: 'move_widgets',
- tags: [],
- requestBody: {
- content: {
- 'application/json': {schema: {type: 'array', items: {$ref: '#/components/schemas/WidgetPosition'}}},
- },
- },
- responses: {'204': {description: 'empty'}},
- },
- },
- '/owners': {
- get: {
- operationId: 'list_owners',
- tags: [],
- responses: {
- '200': {
- description: 'ok',
- content: {
- 'application/json': {
- schema: {
- type: 'object',
- additionalProperties: {anyOf: [{$ref: '#/components/schemas/Owner'}, {type: 'null'}]},
- },
- },
- },
- },
- },
- },
- },
- '/gadgets': {
- post: {
- operationId: 'create_gadget',
- tags: [],
- requestBody: {content: {'application/json': {schema: {$ref: '#/components/schemas/GadgetCreate'}}}},
- responses: {'204': {description: 'empty'}},
- },
- },
- },
- components: {
- schemas: {
- SnowflakeType: {
- anyOf: [{type: 'string'}, {type: 'integer'}],
- format: 'snowflake',
- },
- Widget: {
- type: 'object',
- properties: {
- id: SNOWFLAKE,
- name: {type: 'string'},
- value: {anyOf: [{type: 'string'}, {type: 'boolean'}]},
- part: {anyOf: [{$ref: '#/components/schemas/Part'}, {type: 'null'}]},
- labels: {type: 'object', additionalProperties: {$ref: '#/components/schemas/Label'}},
- shape: {anyOf: [{$ref: '#/components/schemas/Circle'}, {$ref: '#/components/schemas/Square'}]},
- legacy: {type: 'boolean', description: 'Deprecated. Always false.'},
- },
- required: ['id', 'name', 'value', 'part'],
- additionalProperties: false,
- },
- Part: {
- type: 'object',
- properties: {part_id: SNOWFLAKE, weight: {type: 'number'}, unused: {type: 'string'}},
- required: ['part_id', 'weight'],
- additionalProperties: false,
- },
- Label: {
- type: 'object',
- properties: {text: {type: 'string'}, color: {type: 'string'}},
- required: ['text'],
- additionalProperties: false,
- },
- Owner: {
- type: 'object',
- properties: {owner_id: SNOWFLAKE, name: {type: 'string'}},
- required: ['owner_id', 'name'],
- additionalProperties: false,
- },
- Circle: {
- type: 'object',
- properties: {kind: {type: 'string'}, radius: {type: 'number'}},
- required: ['kind', 'radius'],
- additionalProperties: false,
- },
- Square: {
- type: 'object',
- properties: {kind: {type: 'string'}, side: {type: 'number'}},
- required: ['kind', 'side'],
- additionalProperties: false,
- },
- WidgetPosition: {
- type: 'object',
- properties: {id: SNOWFLAKE, position: {type: 'integer'}},
- required: ['id'],
- additionalProperties: false,
- },
- WidgetUpdate: {
- type: 'object',
- properties: {name: {type: 'string'}, owner_id: SNOWFLAKE, size: {type: 'integer'}},
- required: ['name'],
- additionalProperties: false,
- },
- GadgetCreate: {
- type: 'object',
- properties: {label: {type: 'string'}, color: {type: 'string'}},
- required: ['label'],
- additionalProperties: false,
- },
- },
- securitySchemes: {},
- },
-} as unknown as OpenAPIDocument;
-
-const emptySpec = {
- openapi: '3.1.0',
- info: {title: 'empty', version: '1'},
- paths: {},
- components: {schemas: {}, securitySchemes: {}},
-} as OpenAPIDocument;
-
-const PAGE = `---
-title: Widgets
----
-
-import RouteHeader from '@/components/RouteHeader.astro';
-
-## Widget object
-
-### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| id | snowflake | The ID of the widget |
-| name | string | The name |
-| value | boolean | The value |
-| part | ?[part](#part-object) object | The part |
-| labels? | map[string, [label](#label-object) object] | The labels by key |
-| shape? | [circle](#circle-object) object \\| [square](#square-object) object | The shape |
-| legacy | boolean | Always false |
-| shown<sup>1</sup> | boolean | Whether the widget is shown |
-
-<sup>1</sup> Emitted on every response although the response schema does not declare it
-
-### Part object
-
-#### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| part_id | snowflake | The ID of the part |
-| weight | float | The weight |
-
-The response schema also declares \`unused\`, which a part never has.
-
-### Label object
-
-#### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| text | string | The text |
-| color? | string | The colour |
-
-### Circle object
-
-#### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| kind | string | \`circle\` |
-| radius | float | The radius |
-
-### Square object
-
-#### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| kind | string | \`square\` |
-| side | float | The side length |
-
-## Widget position object
-
-### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| id | snowflake | The ID of the widget |
-| position? | integer | The new position |
-
-## Owner object
-
-### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| owner_id | snowflake | The ID of the owner |
-| name | string | The name |
-
-## Owners by ID object
-
-Each property name is the ID of one owner.
-
-### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| &#123;owner_id&#125; | ?[owner](#owner-object) object | The owner, or null |
-
-## Gadget creation object
-
-### Structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| label | string | The label |
-| color? | string | The colour |
-
-## Update widget
-
-<RouteHeader method="PATCH" path="/v1/widgets/{widget_id}" />
-
-### JSON body
-
-| Field | Type | Description |
-| --- | --- | --- |
-| name | string | The new name |
-| owner_id? | snowflake | The new owner |
-| size? | integer | The new size |
-
-### Response
-
-| Status | Body | Condition |
-| --- | --- | --- |
-| 200 | [widget](#widget-object) object | The widget was updated |
-
-## List widgets
-
-<RouteHeader method="GET" path="/v1/widgets" />
-
-### Query parameters
-
-| Field | Type | Description |
-| --- | --- | --- |
-| limit? | integer | The page size |
-| owner_id? | snowflake | Restrict to one owner |
-
-### Response body
-
-| Field | Type | Description |
-| --- | --- | --- |
-| widgets | array[[widget](#widget-object) object] | The widgets |
-| total | integer | Every matching widget |
-
-### Response
-
-| Status | Body | Condition |
-| --- | --- | --- |
-| 200 | response body | The page was returned |
-
-## Create widget
-
-<RouteHeader method="POST" path="/v1/widgets" />
-
-### JSON body
-
-#### Alpha structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| kind | string | \`alpha\` |
-| alpha | string | The alpha value |
-
-#### Beta structure
-
-| Field | Type | Description |
-| --- | --- | --- |
-| kind | string | \`beta\` |
-| beta | integer | The beta value |
-
-### Response
-
-| Status | Body | Condition |
-| --- | --- | --- |
-| 200 | [widget](#widget-object) object | The widget was created |
-
-## Move widgets
-
-<RouteHeader method="PUT" path="/v1/widgets/positions" />
-
-### JSON body
-
-The body is an array of [widget position objects](#widget-position-object).
-
-### Response
-
-| Status | Body | Condition |
-| --- | --- | --- |
-| 204 | empty | The widgets were moved |
-
-## List owners
-
-<RouteHeader method="GET" path="/v1/owners" />
-
-### Response
-
-| Status | Body | Condition |
-| --- | --- | --- |
-| 200 | [owners by ID](#owners-by-id-object) object | The owners were returned |
-
-## Create gadget
-
-<RouteHeader method="POST" path="/v1/gadgets" />
-
-### JSON body
-
-The body is a [gadget creation object](#gadget-creation-object).
-
-### Response
-
-| Status | Body | Condition |
-| --- | --- | --- |
-| 204 | empty | The gadget was created |
-`;
-
-function page(source: string): MarkdownPage {
- return {
- file: '/fixture/http-api/widgets.mdx',
- relativePath: 'http-api/widgets.mdx',
- source,
- lines: source.split('\n'),
- };
-}
-
-function verify(source: string): SchemaVerification {
- return verifyPages([page(source)], spec, emptySpec);
-}
-
-function mutate(from: string, to: string): string {
- assert.ok(PAGE.includes(from), `fixture text not found: ${from}`);
- return PAGE.replace(from, to);
-}
-
-function findings(result: SchemaVerification): Array<string> {
- return result.mismatches.map((mismatch) => `${mismatch.kind} ${mismatch.operation} ${mismatch.detail}`);
-}
-
-test('the clean fixture passes and every check runs', () => {
- const result = verify(PAGE);
- assert.deepEqual(findings(result), []);
- assert.deepEqual(result.optionalityAdvisories, []);
- assert.equal(result.counters.bodyTables, 2);
- assert.equal(result.counters.bodiesByReference, 2);
- assert.equal(result.counters.unionBodies, 1);
- assert.equal(result.counters.queryTables, 1);
- assert.equal(result.counters.responseTables, 1);
- assert.equal(result.counters.responseObjects, 3);
- assert.ok(result.counters.nestedObjects >= 2);
- assert.equal(result.counters.unlinkedObjects, 0);
- assert.ok(result.counters.typesCompared > 10);
- assert.equal(result.counters.optionalityCompared, 11);
-});
-
-test('a union body field missing from every variant table fails', () => {
- const result = verify(mutate('| beta | integer | The beta value |\n', ''));
- assert.deepEqual(findings(result), ['body-missing POST /widgets beta']);
-});
-
-test('a field documented in one variant table but missing from another fails', () => {
- const result = verify(mutate('| kind | string | `beta` |\n', ''));
- assert.deepEqual(findings(result), ['body-missing POST /widgets kind (variant 2)']);
-});
-
-test('a body documented by reference is compared with the object section', () => {
- const result = verify(mutate('| label | string | The label |', '| title | string | The label |'));
- assert.deepEqual(findings(result), ['body-extra POST /gadgets title', 'body-missing POST /gadgets label']);
-});
-
-test('a nested object linked from a field type is compared with its schema', () => {
- const result = verify(mutate('| weight | float | The weight |', '| mass | float | The weight |'));
- assert.deepEqual(findings(result), [
- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).part (http-api/widgets#part-object).weight',
- 'object-extra PATCH /widgets/{} (http-api/widgets#part-object).mass',
- ]);
-});
-
-test('a query row is not found in another route or table of the page', () => {
- const result = verify(mutate('| owner_id? | snowflake | Restrict to one owner |\n', ''));
- assert.deepEqual(findings(result), ['query-missing GET /widgets owner_id']);
-});
-
-test('a snowflake is typed string, so a snowflake documented as integer fails', () => {
- const result = verify(mutate('| id | snowflake | The ID of the widget |', '| id | integer | The ID of the widget |'));
- assert.deepEqual(findings(result), [
- 'type-mismatch PATCH /widgets/{} (http-api/widgets#widget-object).id: documented integer, schema string',
- ]);
-});
-
-test('a type outside every anyOf branch fails', () => {
- const result = verify(mutate('| value | boolean | The value |', '| value | integer | The value |'));
- assert.deepEqual(findings(result), [
- 'type-mismatch PATCH /widgets/{} (http-api/widgets#widget-object).value: documented integer, schema boolean | string',
- ]);
-});
-
-test('a route-local response body table is type checked', () => {
- const result = verify(
- mutate('| total | integer | Every matching widget |', '| total | string | Every matching widget |'),
- );
- assert.deepEqual(findings(result), ['type-mismatch GET /widgets total: documented string, schema integer']);
-});
-
-test('a response body field the schema does not have fails', () => {
- const result = verify(
- mutate(
- '| total | integer | Every matching widget |\n',
- '| total | integer | Every matching widget |\n| count | integer | Stale |\n',
- ),
- );
- assert.deepEqual(findings(result), ['response-extra GET /widgets count']);
-});
-
-test('every table under JSON body is checked, not only the first', () => {
- const result = verify(
- mutate(
- '| size? | integer | The new size |\n',
- '| size? | integer | The new size |\n\n#### Extra structure\n\n| Field | Type | Description |\n| --- | --- | --- |\n| colour? | string | Stale |\n',
- ),
- );
- assert.deepEqual(findings(result), ['body-extra PATCH /widgets/{} colour']);
-});
-
-test('an optionality disagreement is fatal', () => {
- const result = verify(mutate('| name | string | The new name |', '| name? | string | The new name |'));
- assert.deepEqual(findings(result), [
- 'optionality PATCH /widgets/{} name: documented optional, schema marks it required',
- ]);
- assert.equal(result.optionalityAdvisories.length, 1);
-});
-
-test('an object member the schema does not declare needs the footnote', () => {
- const result = verify(
- mutate('<sup>1</sup> Emitted on every response although the response schema does not declare it\n', ''),
- );
- assert.deepEqual(findings(result), ['object-extra PATCH /widgets/{} (http-api/widgets#widget-object).shown']);
-});
-
-test('a schema member an object never has needs the declared note', () => {
- const result = verify(mutate('The response schema also declares `unused`, which a part never has.\n', ''));
- assert.deepEqual(findings(result), [
- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).part (http-api/widgets#part-object).unused',
- ]);
-});
-
-test('a member missing from a linked response object fails', () => {
- const result = verify(mutate('| name | string | The name |\n', ''));
- assert.deepEqual(findings(result), ['object-missing PATCH /widgets/{} (http-api/widgets#widget-object).name']);
-});
-
-test('a route whose schema has a JSON body needs a JSON body section', () => {
- const result = verify(
- mutate('### JSON body\n\nThe body is a [gadget creation object](#gadget-creation-object).\n\n', ''),
- );
- assert.deepEqual(findings(result), ['body-missing POST /gadgets no JSON body section documents the request body']);
-});
-
-test('a route whose schema has query parameters needs a query parameter table', () => {
- const result = verify(
- mutate(
- '### Query parameters\n\n| Field | Type | Description |\n| --- | --- | --- |\n| limit? | integer | The page size |\n| owner_id? | snowflake | Restrict to one owner |\n\n',
- '',
- ),
- );
- assert.deepEqual(findings(result), ['query-missing GET /widgets limit', 'query-missing GET /widgets owner_id']);
-});
-
-test('an object linked as the value of a map is compared with the value schema', () => {
- const result = verify(mutate('| text | string | The text |', '| title | string | The text |'));
- assert.deepEqual(findings(result), [
- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).labels (http-api/widgets#label-object).text',
- 'object-extra PATCH /widgets/{} (http-api/widgets#label-object).title',
- ]);
-});
-
-test('a body that is an array of a linked object is compared with the item schema', () => {
- const result = verify(mutate('| position? | integer | The new position |', '| rank? | integer | The new position |'));
- assert.deepEqual(findings(result), [
- 'body-extra PUT /widgets/positions rank',
- 'body-missing PUT /widgets/positions position',
- ]);
-});
-
-test('an object documented as a keyed map is compared through its value link', () => {
- const result = verify(mutate('| name | string | The name |\n\n## Owners by ID object', '\n## Owners by ID object'));
- assert.deepEqual(findings(result), ['object-missing GET /owners (http-api/widgets#owners-by-id-object).name']);
-});
-
-test('an object linked where the schema declares no members fails', () => {
- const result = verify(
- mutate(
- '| total | integer | Every matching widget |',
- '| total | [part](#part-object) object | Every matching widget |',
- ),
- );
- assert.deepEqual(findings(result), [
- 'type-mismatch GET /widgets total: documented object, schema integer',
- 'object-unresolved GET /widgets total: the schema declares no members for (http-api/widgets#part-object)',
- ]);
-});
-
-test('a query table on a route with no query parameters fails unless the footnote says so', () => {
- const table =
- '### Query parameters\n\n| Field | Type | Description |\n| --- | --- | --- |\n| token<sup>1</sup> | string | The token |\n\n';
- const anchor =
- '### JSON body\n\n| Field | Type | Description |\n| --- | --- | --- |\n| name | string | The new name |';
- const undocumented = verify(mutate(anchor, `${table}${anchor}`));
- assert.deepEqual(findings(undocumented), ['query-extra PATCH /widgets/{} token']);
- const note = '<sup>1</sup> Required, although the request schema does not declare it\n\n';
- const footnoted = verify(mutate(anchor, `${table}${note}${anchor}`));
- assert.deepEqual(findings(footnoted), []);
-});
-
-test('a body or response table on a route whose schema has neither fails', () => {
- const result = verify(
- mutate(
- '| 204 | empty | The widgets were moved |',
- '| 204 | empty | The widgets were moved |\n\n### Response body\n\n| Field | Type | Description |\n| --- | --- | --- |\n| moved | integer | Stale |',
- ),
- );
- assert.deepEqual(findings(result), ['response-extra PUT /widgets/positions moved']);
-});
-
-test('each object linked for a union documents the members of its own variant', () => {
- const result = verify(mutate('| kind | string | `square` |\n', ''));
- assert.deepEqual(findings(result), [
- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).shape (http-api/widgets#circle-object, http-api/widgets#square-object).kind (variant 2)',
- ]);
-});
-
-test('a member repeated by a second object linked for a union is type checked', () => {
- const result = verify(mutate('| kind | string | `square` |', '| kind | integer | `square` |'));
- assert.deepEqual(findings(result), [
- 'type-mismatch PATCH /widgets/{} (http-api/widgets#widget-object).shape (http-api/widgets#circle-object, http-api/widgets#square-object).kind: documented integer, schema string',
- ]);
-});
-
-test('a member with a declared shape and no linked object section is counted', () => {
- const result = verify(mutate('| part | ?[part](#part-object) object | The part |', '| part | ?object | The part |'));
- assert.deepEqual(findings(result), []);
- assert.equal(result.counters.unlinkedObjects, 1);
-});
-
-test('a declared note naming a field the object schema does not have fails', () => {
- const result = verify(
- mutate(
- 'The response schema also declares `unused`, which a part never has.',
- 'The response schema also declares `unused` and `retired`, which a part never has.',
- ),
- );
- assert.deepEqual(findings(result), [
- 'object-stale-declared PATCH /widgets/{} (http-api/widgets#part-object).retired',
- ]);
-});
-
-test('a declared note naming a field the response schema does not have fails', () => {
- const result = verify(
- mutate(
- '| total | integer | Every matching widget |\n',
- '| total | integer | Every matching widget |\n\nThe response schema also declares `cursor`.\n',
- ),
- );
- assert.deepEqual(findings(result), ['response-stale-declared GET /widgets cursor']);
-});
-
-test('a not-declared footnote on a member the object schema declares fails', () => {
- const result = verify(mutate('| name | string | The name |', '| name<sup>1</sup> | string | The name |'));
- assert.deepEqual(findings(result), [
- 'object-stale-undeclared PATCH /widgets/{} (http-api/widgets#widget-object).name',
- ]);
-});
-
-test('a not-declared footnote on a declared body field or query parameter fails', () => {
- const note = '<sup>1</sup> Required, although the request schema does not declare it\n\n';
- const body = verify(
- mutate('| size? | integer | The new size |', `| size?<sup>1</sup> | integer | The new size |\n\n${note.trimEnd()}`),
- );
- assert.deepEqual(findings(body), ['body-stale-undeclared PATCH /widgets/{} size']);
- const query = verify(
- mutate(
- '| limit? | integer | The page size |\n| owner_id? | snowflake | Restrict to one owner |',
- `| limit?<sup>1</sup> | integer | The page size |\n| owner_id? | snowflake | Restrict to one owner |\n\n${note.trimEnd()}`,
- ),
- );
- assert.deepEqual(findings(query), ['query-stale-undeclared GET /widgets limit']);
-});
diff --git a/fluxer_docs/scripts/VerifyDocsSchemas.ts b/fluxer_docs/scripts/VerifyDocsSchemas.ts
deleted file mode 100644
index a0618274d..000000000
--- a/fluxer_docs/scripts/VerifyDocsSchemas.ts
+++ /dev/null
@@ -1,1307 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-import {readFile, writeFile} from 'node:fs/promises';
-import path from 'node:path';
-import {fileURLToPath, pathToFileURL} from 'node:url';
-import type {
- OpenAPIOperation as Operation,
- OpenAPISchema as SchemaNode,
- OpenAPIDocument as Spec,
-} from '@fluxer/openapi/src/OpenAPITypes';
-import {readRouteHeaders} from './DocsRouteHeaders.ts';
-import {
- DOCS_ROOT,
- HTTP_METHODS,
- type MarkdownPage,
- pageSlug,
- readMarkdownPages,
- routeShape,
- slugifyHeading,
- splitTableRow,
- stripVersion,
-} from './DocsSource.ts';
-
-const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url));
-const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json');
-const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json');
-interface Mismatch {
- readonly page: string;
- readonly operation: string;
- readonly kind: string;
- readonly detail: string;
-}
-
-interface SchemaCounters {
- bodyTables: number;
- bodiesByReference: number;
- unionBodies: number;
- queryTables: number;
- responseTables: number;
- responseObjects: number;
- responseFieldsFound: number;
- nestedObjects: number;
- unlinkedObjects: number;
- typesCompared: number;
- optionalityCompared: number;
-}
-
-export interface SchemaVerification {
- readonly mismatches: ReadonlyArray<Mismatch>;
- readonly optionalityAdvisories: ReadonlyArray<string>;
- readonly counters: SchemaCounters;
-}
-
-interface DocField {
- readonly type: string | null;
- readonly typed: boolean;
- readonly optional: boolean;
- readonly anchors: ReadonlyArray<string>;
- readonly notes: ReadonlyArray<string>;
- readonly undeclared: boolean;
-}
-
-interface Heading {
- readonly level: number;
- readonly text: string;
- readonly line: number;
- readonly anchors: Array<string>;
- isObject: boolean;
-}
-
-interface Table {
- readonly line: number;
- readonly anchors: ReadonlyArray<string>;
- readonly stack: ReadonlyArray<Heading>;
- readonly header: ReadonlyArray<string>;
- readonly rows: ReadonlyArray<ReadonlyArray<string>>;
-}
-
-interface ProseLine {
- readonly line: number;
- readonly stack: ReadonlyArray<Heading>;
- readonly text: string;
-}
-
-interface ParsedPage {
- readonly page: MarkdownPage;
- readonly slug: string;
- readonly headings: ReadonlyArray<Heading>;
- readonly tables: ReadonlyArray<Table>;
- readonly prose: ReadonlyArray<ProseLine>;
-}
-
-interface ObjectUsage {
- readonly page: string;
- readonly operation: string;
- readonly anchors: string;
- readonly own: ReadonlyMap<string, DocField>;
- readonly properties: Set<string>;
- open: boolean;
-}
-
-interface Inheritance {
- readonly anchor: string;
- readonly except: ReadonlySet<string>;
-}
-
-interface ObjectBlock {
- readonly own: Map<string, DocField>;
- readonly tables: Array<ReadonlyMap<string, DocField>>;
- readonly inherits: Array<Inheritance>;
- readonly links: Array<Inheritance>;
- readonly undeclared: Set<string>;
- readonly declaredOnly: Set<string>;
-}
-
-function resolveSchemaPointer(spec: Spec, reference: string): SchemaNode | boolean {
- if (!reference.startsWith('#')) throw new Error(`Unsupported schema reference: ${reference}`);
- const pointer = decodeURIComponent(reference.slice(1));
- const prefix = '/components/schemas/';
- if (!pointer.startsWith(prefix)) throw new Error(`Unsupported schema reference: ${reference}`);
- let target: unknown = spec.components.schemas;
- for (const token of pointer.slice(prefix.length).split('/')) {
- if (/~(?:[^01]|$)/u.test(token)) throw new Error(`Invalid schema reference escape: ${reference}`);
- const key = token.replace(/~1/gu, '/').replace(/~0/gu, '~');
- if (
- target === null ||
- typeof target !== 'object' ||
- (Array.isArray(target) && !/^(0|[1-9][0-9]*)$/u.test(key)) ||
- !Object.hasOwn(target, key)
- ) {
- throw new Error(`Missing schema reference: ${reference}`);
- }
- target = (target as Record<string, unknown>)[key];
- }
- if (typeof target === 'boolean') return target;
- if (target === null || typeof target !== 'object' || Array.isArray(target)) {
- throw new Error(`Reference does not identify a schema: ${reference}`);
- }
- return target as SchemaNode;
-}
-
-function resolveRef(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): SchemaNode | undefined {
- if (node == null || node === false) {
- return undefined;
- }
- if (node === true) {
- return {};
- }
- if (depth > 64) {
- throw new Error('OpenAPI reference chain exceeds the supported depth');
- }
- if (node.$ref != null) {
- const target = resolveSchemaPointer(spec, node.$ref);
- const resolved = resolveRef(spec, target, depth + 1);
- const {$ref, ...siblings} = node;
- if (resolved == null || Object.keys(siblings).length === 0) {
- return resolved;
- }
- if (Object.keys(resolved).length === 0) {
- return siblings;
- }
- return {...resolved, allOf: [...(resolved.allOf ?? []), siblings]};
- }
- return node;
-}
-
-function asSchema(node: unknown): SchemaNode | boolean | undefined {
- if (typeof node === 'boolean') return node;
- if (node == null || typeof node !== 'object' || Array.isArray(node)) return undefined;
- return node as SchemaNode;
-}
-
-function isSchemaNode(node: SchemaNode | boolean): node is SchemaNode {
- return typeof node !== 'boolean';
-}
-
-function branchesOf(resolved: SchemaNode): Array<SchemaNode | boolean> {
- return [...(resolved.allOf ?? []), ...(resolved.oneOf ?? []), ...(resolved.anyOf ?? [])];
-}
-
-function isUnion(resolved: SchemaNode | undefined): boolean {
- return resolved != null && ((resolved.oneOf ?? []).length > 0 || (resolved.anyOf ?? []).length > 0);
-}
-
-function collectRequired(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): Set<string> {
- const out = new Set<string>();
- const resolved = resolveRef(spec, node, depth);
- if (resolved == null) {
- return out;
- }
- for (const name of resolved.required ?? []) {
- out.add(name);
- }
- for (const branch of resolved.allOf ?? []) {
- for (const name of collectRequired(spec, branch, depth + 1)) {
- out.add(name);
- }
- }
- for (const union of [resolved.oneOf ?? [], resolved.anyOf ?? []]) {
- const branchRequirements = union.map((branch) => collectRequired(spec, branch, depth + 1));
- for (const name of branchRequirements[0] ?? []) {
- if (branchRequirements.every((required) => required.has(name))) {
- out.add(name);
- }
- }
- }
- return out;
-}
-
-function collectTypes(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): Set<string> {
- const resolved = resolveRef(spec, node, depth);
- const out = new Set<string>();
- if (resolved == null) {
- return out;
- }
- if (resolved.format === 'snowflake') {
- out.add('string');
- return out;
- }
- for (const type of Array.isArray(resolved.type) ? resolved.type : [resolved.type]) {
- if (type != null && type !== 'null') {
- out.add(type);
- }
- }
- for (const branch of branchesOf(resolved)) {
- for (const type of collectTypes(spec, branch, depth + 1)) {
- out.add(type);
- }
- }
- return out;
-}
-
-function isDeprecatedProperty(property: unknown): boolean {
- if (property == null || typeof property !== 'object') {
- return false;
- }
- const node = property as {deprecated?: unknown; description?: unknown};
- if (node.deprecated === true) {
- return true;
- }
- return typeof node.description === 'string' && node.description.trimStart().toLowerCase().startsWith('deprecated');
-}
-
-function collectPropertySchemas(
- spec: Spec,
- node: SchemaNode | boolean | undefined,
- depth = 0,
- out = new Map<string, Array<SchemaNode | boolean>>(),
- includeDeprecated = false,
-): Map<string, Array<SchemaNode | boolean>> {
- const resolved = resolveRef(spec, node, depth);
- if (resolved == null) {
- return out;
- }
- for (const [key, property] of Object.entries(resolved.properties ?? {})) {
- if (!includeDeprecated && isDeprecatedProperty(property)) {
- continue;
- }
- const schema = asSchema(property);
- if (schema === undefined) {
- continue;
- }
- const list = out.get(key) ?? [];
- list.push(schema);
- out.set(key, list);
- }
- for (const branch of branchesOf(resolved)) {
- collectPropertySchemas(spec, branch, depth + 1, out, includeDeprecated);
- }
- return out;
-}
-
-function objectTarget(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): SchemaNode | undefined {
- if (depth > 32) {
- return undefined;
- }
- const resolved = resolveRef(spec, node, depth);
- if (resolved == null) {
- return undefined;
- }
- if (resolved.properties != null) {
- return resolved;
- }
- const items = asSchema(resolved.items);
- if (items !== undefined && collectTypes(spec, resolved).has('array')) {
- return objectTarget(spec, items, depth + 1);
- }
- const values = asSchema(resolved.additionalProperties);
- if (values !== undefined && isSchemaNode(values)) {
- return objectTarget(spec, values, depth + 1);
- }
- const branches = branchesOf(resolved).filter((branch) => collectTypes(spec, branch).size > 0);
- if (branches.length === 1) {
- return objectTarget(spec, branches[0], depth + 1);
- }
- if (collectPropertySchemas(spec, resolved).size > 0) {
- return resolved;
- }
- const targets = new Map<string, SchemaNode>();
- for (const branch of branches) {
- const target = objectTarget(spec, branch, depth + 1);
- if (target != null) targets.set(JSON.stringify(target), target);
- }
- const [first, ...rest] = targets.values();
- return rest.length === 0 ? first : {anyOf: [first, ...rest]};
-}
-
-function acceptsUndeclaredProperties(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): boolean {
- const resolved = resolveRef(spec, node, depth);
- if (resolved == null) return false;
- const additional = resolved.additionalProperties;
- if (resolved.properties != null && additional != null && additional !== false) return true;
- return branchesOf(resolved).some((branch) => acceptsUndeclaredProperties(spec, branch, depth + 1));
-}
-
-const DOC_TYPE_TO_JSON = new Map([
- ['snowflake', 'string'],
- ['string', 'string'],
- ['integer', 'integer'],
- ['boolean', 'boolean'],
- ['iso8601 timestamp', 'string'],
- ['base64 string', 'string'],
- ['float', 'number'],
- ['number', 'number'],
-]);
-
-function normaliseDocType(cell: string): string | null {
- const text = cell
- .replace(/<sup>.*?<\/sup>/gu, '')
- .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1')
- .replace(/`/gu, '')
- .trim()
- .replace(/^\?/u, '')
- .toLowerCase();
- if (text.startsWith('array')) {
- return 'array';
- }
- if (text.endsWith(' object') || text.includes('object')) {
- return 'object';
- }
- return DOC_TYPE_TO_JSON.get(text) ?? null;
-}
-
-function typeAgrees(docType: string, specTypes: ReadonlySet<string>): boolean {
- return specTypes.has(docType) || (docType === 'integer' && specTypes.has('number'));
-}
-
-function fieldNameText(cell: string): string {
- return cell
- .replace(/<sup>.*?<\/sup>/gu, '')
- .replace(/\*\*/gu, '')
- .replace(/`/gu, '')
- .replace(/\\/gu, '')
- .trim();
-}
-
-function cleanFieldName(cell: string): string | null {
- const name = fieldNameText(cell).replace(/\?$/u, '');
- if (name.length === 0) {
- return null;
- }
- if (!/^[a-z_][a-z0-9_.]*$/iu.test(name)) {
- return null;
- }
- return name;
-}
-
-function operationIndex(spec: Spec): Map<string, Operation> {
- const index = new Map<string, Operation>();
- for (const [routePath, item] of Object.entries(spec.paths)) {
- for (const [method, operation] of Object.entries(item)) {
- const upper = method.toUpperCase();
- if (!HTTP_METHODS.has(upper)) continue;
- const key = routeShape(upper, stripVersion(routePath));
- if (index.has(key)) throw new Error(`Duplicate OpenAPI operation: ${key}`);
- index.set(key, operation);
- }
- }
- return index;
-}
-
-function documentReferences(page: string, line: string): Set<string> {
- const references = new Set<string>();
- for (const link of line.matchAll(/\]\(([^)\s]*)#([a-z0-9-]+)\)/gu)) {
- if (/^[a-z][a-z0-9+.-]*:/iu.test(link[1])) {
- continue;
- }
- const target =
- link[1].length === 0
- ? page
- : link[1].startsWith('/')
- ? link[1].slice(1)
- : path.posix.join(path.posix.dirname(page), link[1]);
- const slug = target
- .replace(/\.(mdx|md)$/u, '')
- .replace(/\/$/u, '')
- .replace(/\/index$/u, '');
- references.add(`${slug}#${link[2]}`);
- }
- return references;
-}
-
-function parsePage(page: MarkdownPage): ParsedPage {
- const slug = pageSlug(page.relativePath);
- const headings: Array<Heading> = [];
- const tables: Array<Table> = [];
- const prose: Array<ProseLine> = [];
- const stack: Array<Heading> = [];
- const pendingAnchors: Array<string> = [];
- const tableAnchors: Array<string> = [];
- let fence: string | null = null;
- let table: {line: number; anchors: Array<string>; stack: ReadonlyArray<Heading>; rows: Array<Array<string>>} | null =
- null;
- const closeTable = () => {
- if (table == null) return;
- const [header, , ...rows] = table.rows;
- tables.push({
- line: table.line,
- anchors: table.anchors,
- stack: table.stack,
- header: (header ?? []).map(fieldNameText),
- rows,
- });
- table = null;
- };
- for (let i = 0; i < page.lines.length; i += 1) {
- const line = page.lines[i];
- const fenceMatch = line.match(/^\s*(```|~~~)/u);
- if (fence != null) {
- if (fenceMatch != null && fenceMatch[1] === fence) fence = null;
- continue;
- }
- if (fenceMatch != null) {
- closeTable();
- fence = fenceMatch[1];
- continue;
- }
- if (line.startsWith('|')) {
- if (table == null) table = {line: i, anchors: tableAnchors.splice(0), stack: [...stack], rows: []};
- table.rows.push(splitTableRow(line));
- continue;
- }
- closeTable();
- for (const explicit of line.matchAll(/<a\s+id=["']([^"']+)["']/gu)) {
- const nextContent = page.lines.slice(i + 1).find((nextLine) => nextLine.trim().length > 0);
- if (nextContent != null && /^#{2,6}\s/u.test(nextContent)) {
- pendingAnchors.push(explicit[1]);
- } else if (nextContent?.startsWith('|')) {
- tableAnchors.push(explicit[1]);
- } else if (stack.length > 0) {
- stack[stack.length - 1].anchors.push(explicit[1]);
- }
- }
- const heading = line.match(/^(#{2,6})\s+(.+?)\s*$/u);
- if (heading != null) {
- const level = heading[1].length;
- const entry: Heading = {
- level,
- text: heading[2],
- line: i,
- anchors: [...new Set([slugifyHeading(heading[2]), ...pendingAnchors])],
- isObject: /\bobject\b/iu.test(heading[2]),
- };
- pendingAnchors.length = 0;
- while (stack.length > 0 && stack[stack.length - 1].level >= level) stack.pop();
- stack.push(entry);
- headings.push(entry);
- continue;
- }
- if (line.trim().length > 0) {
- prose.push({line: i, stack: [...stack], text: line});
- }
- }
- closeTable();
- return {page, slug, headings, tables, prose};
-}
-
-function tableFields(page: ParsedPage, table: Table): Map<string, DocField> {
- const out = new Map<string, DocField>();
- const nameColumn = table.header.indexOf('Field');
- if (nameColumn === -1) {
- return out;
- }
- const typeColumn = table.header.indexOf('Type');
- for (const cells of table.rows) {
- const nameCell = cells[nameColumn];
- if (nameCell === undefined) continue;
- const name = cleanFieldName(nameCell);
- if (name == null) continue;
- const typeCell = typeColumn === -1 ? undefined : cells[typeColumn];
- out.set(name, {
- type: typeCell === undefined ? null : normaliseDocType(typeCell),
- typed: typeCell !== undefined,
- optional: fieldNameText(nameCell).endsWith('?'),
- anchors: typeCell === undefined ? [] : [...documentReferences(page.slug, typeCell)],
- notes: [...nameCell.matchAll(/<sup>(\d+)<\/sup>/gu)].map((match) => match[1]),
- undeclared: false,
- });
- }
- return out;
-}
-
-function keyedValueAnchors(page: ParsedPage, table: Table): Array<string> {
- const nameColumn = table.header.indexOf('Field');
- const typeColumn = table.header.indexOf('Type');
- if (nameColumn === -1 || typeColumn === -1) {
- return [];
- }
- return table.rows
- .filter((cells) => /^(?:&#123;|\{)/u.test(fieldNameText(cells[nameColumn] ?? '')))
- .flatMap((cells) => [...documentReferences(page.slug, cells[typeColumn] ?? '')]);
-}
-
-const BODY_HEADING = /^JSON body$|\bresponse body$/iu;
-
-const RESPONSE_BODY_HEADING = /\bresponse body$/iu;
-
-const STRUCTURE_HEADING = /(?:^|\s)structure$|^(?:additional|value) fields?$/iu;
-
-const INHERITANCE = /\b(?:extends|every (?:field|member) of)\b/iu;
-
-function inheritedAnchors(slug: string, text: string, everyLink = false): Array<Inheritance> {
- const out: Array<Inheritance> = [];
- for (const sentence of text.split(/(?<=\.)\s+(?=[A-Z])/u)) {
- if (!everyLink && !INHERITANCE.test(sentence)) continue;
- const exceptClause = sentence.match(/\b(?:except|without)\b(.*)$/u)?.[1] ?? '';
- const except = new Set([...exceptClause.matchAll(/`([a-z_][a-z0-9_]*)`/giu)].map((match) => match[1]));
- for (const anchor of documentReferences(slug, sentence)) {
- out.push({anchor, except});
- }
- }
- return out;
-}
-
-const DECLARED_ONLY = /\bschema also declares\b/iu;
-
-function declaredOnlySentences(text: string): Array<string> {
- return text.split(/(?<=\.)\s+(?=[A-Z])/u).filter((sentence) => DECLARED_ONLY.test(sentence));
-}
-
-function declaredOnlyNames(text: string): Array<string> {
- return declaredOnlySentences(text).flatMap((sentence) =>
- [...sentence.matchAll(/`([a-z_][a-z0-9_]*)`/giu)].map((match) => match[1]),
- );
-}
-
-const UNDECLARED_NOTE = /^<sup>(\d+)<\/sup>.*\bschema does not declare\b/u;
-
-function markUndeclared(fields: ReadonlyMap<string, DocField>, notes: ReadonlySet<string>): Map<string, DocField> {
- const out = new Map<string, DocField>();
- for (const [name, field] of fields) {
- out.set(name, field.notes.some((note) => notes.has(note)) ? {...field, undeclared: true} : field);
- }
- return out;
-}
-
-function mergeGroups(groups: ReadonlyArray<ReadonlyMap<string, DocField>>): Map<string, DocField> {
- const out = new Map<string, DocField>();
- for (const group of groups) {
- for (const [field, docField] of group) {
- if (!out.has(field)) out.set(field, docField);
- }
- }
- return out;
-}
-
-function innermostObject(stack: ReadonlyArray<Heading>): Heading | undefined {
- for (let index = stack.length - 1; index >= 0; index -= 1) {
- if (stack[index].isObject) return stack[index];
- }
- return undefined;
-}
-
-class Verifier {
- readonly mismatches: Array<Mismatch> = [];
- readonly optionalityAdvisories: Array<string> = [];
- readonly counters: SchemaCounters = {
- bodyTables: 0,
- bodiesByReference: 0,
- unionBodies: 0,
- queryTables: 0,
- responseTables: 0,
- responseObjects: 0,
- responseFieldsFound: 0,
- nestedObjects: 0,
- unlinkedObjects: 0,
- typesCompared: 0,
- optionalityCompared: 0,
- };
- private readonly objects = new Map<string, ObjectBlock>();
- private readonly compared = new Set<string>();
- private readonly unlinked = new Set<string>();
- private readonly usages = new Map<string, ObjectUsage>();
- private readonly routeSections = new Map<ParsedPage, ReadonlySet<Heading>>();
- private readonly routeBodies = new Map<string, ReadonlyArray<ReadonlyMap<string, DocField>>>();
- private readonly identities = new Map<string, number>();
- private readonly parsed: ReadonlyArray<ParsedPage>;
-
- constructor(
- pages: ReadonlyArray<MarkdownPage>,
- private readonly mainSpec: Spec,
- private readonly adminSpec: Spec,
- ) {
- this.parsed = pages.map(parsePage);
- const typeTargets = new Set<string>();
- for (const page of this.parsed) {
- for (const table of page.tables) {
- for (const field of tableFields(page, table).values()) {
- for (const anchor of field.anchors) typeTargets.add(anchor);
- }
- if (table.header[0] !== 'Status') continue;
- for (const row of table.rows) {
- for (const anchor of documentReferences(page.slug, row[1] ?? '')) typeTargets.add(anchor);
- }
- }
- for (const line of page.prose) {
- if (!line.stack.some((heading) => heading.level === 3 && BODY_HEADING.test(heading.text.trim()))) continue;
- for (const anchor of documentReferences(page.slug, line.text)) typeTargets.add(anchor);
- }
- }
- for (const page of this.parsed) {
- const routeLines = readRouteHeaders(page.page).map((header) => header.line);
- const sections = page.headings.filter((heading) => heading.level === 2);
- const routeSections = new Set(
- sections.filter((heading, index) => {
- const end = sections[index + 1]?.line ?? page.page.lines.length;
- return routeLines.some((line) => line > heading.line && line <= end);
- }),
- );
- this.routeSections.set(page, routeSections);
- for (const heading of page.headings) {
- if (routeSections.has(heading)) {
- heading.isObject = false;
- continue;
- }
- if (heading.anchors.some((anchor) => typeTargets.has(`${page.slug}#${anchor}`))) heading.isObject = true;
- }
- }
- for (const page of this.parsed) {
- this.indexObjects(page);
- for (const section of this.routeSections.get(page) ?? []) {
- const groups = page.tables
- .filter(
- (table) =>
- table.stack.includes(section) &&
- table.stack.some((heading) => heading.level === 3 && heading.text.trim() === 'JSON body') &&
- (innermostObject(table.stack)?.level ?? 0) <= 2,
- )
- .map((table) => tableFields(page, table))
- .filter((fields) => fields.size > 0);
- for (const anchor of section.anchors) this.routeBodies.set(`${page.slug}#${anchor}`, groups);
- }
- }
- }
-
- private schemaIdentity(node: SchemaNode): number {
- const shape = JSON.stringify(node);
- const known = this.identities.get(shape);
- if (known !== undefined) return known;
- const identity = this.identities.size;
- this.identities.set(shape, identity);
- return identity;
- }
-
- private indexObjects(page: ParsedPage): void {
- const blocks = new Map<Heading, ObjectBlock>();
- for (const heading of page.headings) {
- if (!heading.isObject) continue;
- const block: ObjectBlock = {
- own: new Map(),
- tables: [],
- inherits: [],
- links: [],
- undeclared: new Set(),
- declaredOnly: new Set(),
- };
- blocks.set(heading, block);
- for (const anchor of heading.anchors) {
- this.objects.set(`${page.slug}#${anchor}`, block);
- }
- }
- for (const table of page.tables) {
- if (table.anchors.length > 0) {
- const fields = tableFields(page, table);
- const block: ObjectBlock = {
- own: new Map(fields),
- tables: [fields],
- inherits: [],
- links: [],
- undeclared: new Set(),
- declaredOnly: new Set(),
- };
- for (const anchor of table.anchors) this.objects.set(`${page.slug}#${anchor}`, block);
- }
- const owner = innermostObject(table.stack);
- if (owner == null) continue;
- const inner = table.stack[table.stack.length - 1];
- if (inner !== owner && !STRUCTURE_HEADING.test(inner.text)) continue;
- const block = blocks.get(owner);
- for (const anchor of keyedValueAnchors(page, table)) block?.links.push({anchor, except: new Set()});
- const fields = tableFields(page, table);
- if (fields.size === 0) continue;
- block?.tables.push(fields);
- for (const [name, field] of fields) {
- if (!block?.own.has(name)) block?.own.set(name, field);
- }
- }
- for (const line of page.prose) {
- const owner = innermostObject(line.stack);
- const block = owner == null ? undefined : blocks.get(owner);
- if (block == null) continue;
- block.inherits.push(...inheritedAnchors(page.slug, line.text));
- const note = line.text.match(UNDECLARED_NOTE);
- if (note != null) block.undeclared.add(note[1]);
- for (const name of declaredOnlyNames(line.text)) block.declaredOnly.add(name);
- for (const anchor of documentReferences(page.slug, line.text)) {
- block.links.push({anchor, except: new Set()});
- }
- }
- for (const block of blocks.values()) {
- if (block.undeclared.size === 0) continue;
- for (const [name, field] of markUndeclared(block.own, block.undeclared)) block.own.set(name, field);
- block.tables.splice(
- 0,
- block.tables.length,
- ...block.tables.map((table) => markUndeclared(table, block.undeclared)),
- );
- }
- }
-
- private parents(block: ObjectBlock): Array<Inheritance> {
- const parents = block.own.size > 0 ? block.inherits : block.links;
- return parents.filter((parent) => this.isObjectAnchor(parent.anchor));
- }
-
- private isObjectAnchor(anchor: string): boolean {
- return this.objects.has(anchor);
- }
-
- private objectFields(anchors: Iterable<string>, seen = new Set<string>()): Map<string, DocField> {
- const out = new Map<string, DocField>();
- for (const anchor of anchors) {
- if (seen.has(anchor)) continue;
- seen.add(anchor);
- const block = this.objects.get(anchor);
- if (block == null) continue;
- for (const [name, field] of block.own) {
- if (!out.has(name)) out.set(name, field);
- }
- for (const parent of this.parents(block)) {
- for (const [name, field] of this.objectFields([parent.anchor], seen)) {
- if (!out.has(name) && !parent.except.has(name)) out.set(name, field);
- }
- }
- }
- return out;
- }
-
- private coverageFields(anchors: Iterable<string>, extra: Iterable<string> = []): Set<string> {
- const out = new Set<string>([...this.objectFields(anchors).keys(), ...extra]);
- for (const anchor of anchors) {
- for (const name of this.objects.get(anchor)?.declaredOnly ?? []) out.add(name);
- }
- return out;
- }
-
- private ownFields(anchors: Iterable<string>): Map<string, DocField> {
- const out = new Map<string, DocField>();
- for (const anchor of anchors) {
- const block = this.objects.get(anchor);
- if (block == null) continue;
- const own = block.own.size > 0 ? block.own : this.objectFields([anchor]);
- for (const [name, field] of own) {
- if (!out.has(name)) out.set(name, field);
- }
- }
- return out;
- }
-
- private push(page: string, operation: string, kind: string, detail: string): void {
- this.mismatches.push({page, operation, kind, detail});
- }
-
- private compareFields(
- spec: Spec,
- context: {page: string; operation: string; kind: string; label: string},
- documented: Iterable<readonly [string, DocField]>,
- coverage: ReadonlySet<string>,
- schema: SchemaNode | boolean | undefined,
- usages?: ReadonlyArray<ObjectUsage>,
- ): number {
- const properties = collectPropertySchemas(spec, schema);
- const known = collectPropertySchemas(spec, schema, 0, new Map(), true);
- const prefix = context.label.length > 0 ? `${context.label}.` : '';
- const open = acceptsUndeclaredProperties(spec, schema);
- for (const usage of usages ?? []) {
- for (const field of known.keys()) usage.properties.add(field);
- usage.open ||= open;
- }
- for (const [field, docField] of documented) {
- const schemas = known.get(field);
- if (schemas == null) {
- if (usages == null && !open && !docField.undeclared) {
- this.push(context.page, context.operation, `${context.kind}-extra`, `${prefix}${field}`);
- }
- continue;
- }
- if (usages == null && docField.undeclared) {
- this.push(context.page, context.operation, `${context.kind}-stale-undeclared`, `${prefix}${field}`);
- }
- const specTypes = new Set<string>();
- for (const node of schemas) {
- for (const type of collectTypes(spec, node)) specTypes.add(type);
- }
- if (docField.type != null && specTypes.size > 0) {
- this.counters.typesCompared += 1;
- if (!typeAgrees(docField.type, specTypes)) {
- this.push(
- context.page,
- context.operation,
- 'type-mismatch',
- `${prefix}${field}: documented ${docField.type}, schema ${[...specTypes].sort().join(' | ')}`,
- );
- }
- }
- const anchors = docField.anchors.filter((anchor) => this.isObjectAnchor(anchor));
- const target: SchemaNode | boolean = schemas.length === 1 ? schemas[0] : {anyOf: schemas.filter(isSchemaNode)};
- const nested = objectTarget(spec, target);
- if (anchors.length === 0 && nested != null) {
- this.unlinked.add(`${context.page} ${prefix}${field}`);
- }
- if (anchors.length > 0) {
- if (nested != null) {
- this.compareObject(spec, context.page, context.operation, anchors, target, `${prefix}${field}`);
- } else if (!docField.undeclared && this.ownFields(anchors).size > 0) {
- this.push(
- context.page,
- context.operation,
- 'object-unresolved',
- `${prefix}${field}: the schema declares no members for (${anchors.join(', ')})`,
- );
- }
- }
- }
- let found = 0;
- for (const field of properties.keys()) {
- if (coverage.has(field)) {
- found += 1;
- continue;
- }
- this.push(context.page, context.operation, `${context.kind}-missing`, `${prefix}${field}`);
- }
- return found;
- }
-
- private compareObject(
- spec: Spec,
- page: string,
- operation: string,
- anchors: ReadonlyArray<string>,
- node: SchemaNode | boolean | undefined,
- label: string,
- extraCoverage: Iterable<string> = [],
- ): number {
- const target = objectTarget(spec, node);
- if (target == null) {
- return 0;
- }
- const specName = spec === this.adminSpec ? 'admin' : 'main';
- const key = `${specName}|${[...anchors].sort().join(',')}|${this.schemaIdentity(target).toString()}`;
- if (this.compared.has(key)) {
- return 0;
- }
- this.compared.add(key);
- this.counters.nestedObjects += 1;
- const usages = anchors.map((anchor) => {
- let usage = this.usages.get(anchor);
- if (usage == null) {
- usage = {page, operation, anchors: anchor, own: this.ownFields([anchor]), properties: new Set(), open: false};
- this.usages.set(anchor, usage);
- }
- return usage;
- });
- const objectLabel = `${label.length > 0 ? `${label} ` : ''}(${anchors.join(', ')})`;
- this.compareObjectVariants(spec, page, operation, anchors, target, objectLabel);
- return this.compareFields(
- spec,
- {page, operation, kind: 'object', label: objectLabel},
- anchors.flatMap((anchor) => [...this.ownFields([anchor])]),
- this.coverageFields(anchors, extraCoverage),
- target,
- usages,
- );
- }
-
- private compareObjectVariants(
- spec: Spec,
- page: string,
- operation: string,
- anchors: ReadonlyArray<string>,
- target: SchemaNode,
- label: string,
- ): void {
- const branches = [...(target.oneOf ?? []), ...(target.anyOf ?? [])];
- if (anchors.length < 2 || branches.length < 2) return;
- const branchProperties = branches.map((branch) => new Set(collectPropertySchemas(spec, branch).keys()));
- const documentedFor = branches.map(() => new Set<string>());
- const documented = new Set<string>();
- const matchedBranches = new Set<number>();
- for (const anchor of anchors) {
- const fields = [...this.ownFields([anchor]).keys()].filter((field) =>
- branchProperties.some((properties) => properties.has(field)),
- );
- const candidates = branchProperties
- .map((properties, index) => (fields.every((field) => properties.has(field)) ? index : -1))
- .filter((index) => index !== -1);
- if (candidates.length === 0) return;
- const coverage = this.coverageFields([anchor]);
- for (const field of coverage) documented.add(field);
- for (const index of candidates) {
- for (const field of coverage) documentedFor[index].add(field);
- if (candidates.length < branches.length) matchedBranches.add(index);
- }
- }
- for (const index of matchedBranches) {
- for (const field of branchProperties[index]) {
- if (documentedFor[index].has(field) || !documented.has(field)) continue;
- this.push(page, operation, 'object-missing', `${label}.${field} (variant ${(index + 1).toString()})`);
- }
- }
- }
-
- private objectTables(anchors: Iterable<string>, seen = new Set<string>()): Array<ReadonlyMap<string, DocField>> {
- const out: Array<ReadonlyMap<string, DocField>> = [];
- for (const anchor of anchors) {
- if (seen.has(anchor)) continue;
- seen.add(anchor);
- const block = this.objects.get(anchor);
- if (block == null) continue;
- out.push(...block.tables);
- for (const parent of this.parents(block)) {
- for (const table of this.objectTables([parent.anchor], seen)) {
- out.push(new Map([...table].filter(([name]) => !parent.except.has(name))));
- }
- }
- }
- return out;
- }
-
- private compareBodyBranches(
- spec: Spec,
- page: string,
- operation: string,
- schema: SchemaNode | boolean,
- groups: ReadonlyArray<ReadonlyMap<string, DocField>>,
- ): void {
- const resolved = resolveRef(spec, schema);
- const allBranches = isUnion(resolved) ? [...(resolved?.oneOf ?? []), ...(resolved?.anyOf ?? [])] : [schema];
- const branches = allBranches;
- if (branches.length > 1) this.counters.unionBodies += 1;
- const branchProperties = branches.map((branch) => new Set(collectPropertySchemas(spec, branch).keys()));
- const branchRequired = branches.map((branch) => collectRequired(spec, branch));
- const allProperties = new Set(branchProperties.flatMap((properties) => [...properties]));
- const commonRequired = collectRequired(spec, schema);
- const documentedFor = branches.map(() => new Set<string>());
- const matchedBranches = new Set<number>();
- for (const group of groups) {
- const fields = [...group.keys()].filter((field) => allProperties.has(field));
- const candidates = branchProperties
- .map((properties, index) => (fields.every((field) => properties.has(field)) ? index : -1))
- .filter((index) => index !== -1);
- const specific = candidates.length > 0 && candidates.length < branches.length;
- for (const index of candidates) {
- for (const field of fields) documentedFor[index].add(field);
- if (specific) matchedBranches.add(index);
- }
- for (const [field, docField] of group) {
- if (!allProperties.has(field) || !docField.typed) continue;
- const specRequired =
- candidates.length > 0
- ? candidates.every((index) => branchRequired[index].has(field))
- : commonRequired.has(field);
- this.counters.optionalityCompared += 1;
- if (specRequired === !docField.optional) continue;
- const detail = specRequired
- ? `${field}: documented optional, schema marks it required`
- : `${field}: documented required, schema marks it optional`;
- this.optionalityAdvisories.push(`${page} ${operation} ${detail}`);
- this.push(page, operation, 'optionality', detail);
- }
- }
- const documented = new Set(groups.flatMap((group) => [...group.keys()]));
- for (const index of matchedBranches) {
- for (const field of branchProperties[index]) {
- if (documentedFor[index].has(field) || !documented.has(field)) continue;
- this.push(page, operation, 'body-missing', `${field} (variant ${(index + 1).toString()})`);
- }
- }
- }
-
- private reportObjectExtras(): void {
- for (const usage of this.usages.values()) {
- for (const [field, docField] of usage.own) {
- if (docField.undeclared && usage.properties.has(field)) {
- this.push(usage.page, usage.operation, 'object-stale-undeclared', `(${usage.anchors}).${field}`);
- }
- }
- if (usage.open) continue;
- for (const [field, docField] of usage.own) {
- if (usage.properties.has(field) || docField.undeclared) continue;
- this.push(usage.page, usage.operation, 'object-extra', `(${usage.anchors}).${field}`);
- }
- for (const field of this.objects.get(usage.anchors)?.declaredOnly ?? []) {
- if (!usage.properties.has(field)) {
- this.push(usage.page, usage.operation, 'object-stale-declared', `(${usage.anchors}).${field}`);
- }
- }
- }
- }
-
- run(): SchemaVerification {
- const mainIndex = operationIndex(this.mainSpec);
- const adminIndex = operationIndex(this.adminSpec);
- for (const page of this.parsed) {
- if (page.page.relativePath.startsWith('media-proxy/')) continue;
- const isAdmin = page.page.relativePath.startsWith('admin-api/');
- this.verifyPage(page, isAdmin ? this.adminSpec : this.mainSpec, isAdmin ? adminIndex : mainIndex);
- }
- this.reportObjectExtras();
- this.counters.unlinkedObjects = this.unlinked.size;
- return {mismatches: this.mismatches, optionalityAdvisories: this.optionalityAdvisories, counters: this.counters};
- }
-
- private verifyPage(page: ParsedPage, spec: Spec, index: ReadonlyMap<string, Operation>): void {
- const relative = page.page.relativePath;
- const routeHeaders = readRouteHeaders(page.page);
- const sections = page.headings.filter((heading) => heading.level === 2);
- for (let s = 0; s < sections.length; s += 1) {
- const section = sections[s];
- const end = sections[s + 1]?.line ?? page.page.lines.length;
- const header = routeHeaders.find((candidate) => candidate.line > section.line && candidate.line <= end);
- if (header == null) continue;
- if (header.endLine > end) {
- throw new Error(`${relative}:${header.line}: RouteHeader crosses a section boundary`);
- }
- const key = routeShape(header.method, stripVersion(header.path));
- const operation = index.get(key);
- if (operation == null) continue;
- this.verifyRoute(page, spec, section, end, key, operation);
- }
- }
-
- private verifyRoute(
- page: ParsedPage,
- spec: Spec,
- section: Heading,
- end: number,
- key: string,
- operation: Operation,
- ): void {
- const relative = page.page.relativePath;
- const inSection = (stack: ReadonlyArray<Heading>) => stack.includes(section);
- const subsection = (stack: ReadonlyArray<Heading>) => {
- const text = stack.find((heading) => heading.level === 3 && stack.includes(section))?.text.trim() ?? null;
- return text != null && RESPONSE_BODY_HEADING.test(text) ? 'Response body' : text;
- };
- const sectionTables = page.tables.filter((table) => inSection(table.stack));
- const sectionProse = page.prose.filter((line) => inSection(line.stack));
-
- const fieldsUnder = (name: string) => {
- const groups: Array<ReadonlyMap<string, DocField>> = [];
- const undeclared = new Set<string>();
- for (const line of sectionProse) {
- const note = subsection(line.stack) === name ? line.text.match(UNDECLARED_NOTE) : null;
- if (note != null) undeclared.add(note[1]);
- }
- for (const table of sectionTables) {
- if (subsection(table.stack) !== name) continue;
- if ((innermostObject(table.stack)?.level ?? 0) > 2) continue;
- const fields = markUndeclared(tableFields(page, table), undeclared);
- if (fields.size > 0) groups.push(fields);
- }
- const tables = groups.length;
- if (tables > 0) {
- for (const line of sectionProse) {
- if (subsection(line.stack) !== name) continue;
- for (const parent of inheritedAnchors(page.slug, line.text)) {
- for (const table of this.objectTables([parent.anchor])) {
- groups.push(new Map([...table].filter(([field]) => !parent.except.has(field))));
- }
- }
- }
- }
- return {fields: mergeGroups(groups), groups, tables};
- };
- const referencesUnder = (name: string) => {
- const out: Array<Inheritance> = [];
- for (const line of sectionProse) {
- if (subsection(line.stack) !== name) continue;
- for (const parent of inheritedAnchors(page.slug, line.text, true)) {
- if (this.isObjectAnchor(parent.anchor)) out.push(parent);
- }
- }
- return out;
- };
-
- const jsonSchema = objectTarget(spec, operation.requestBody?.content?.['application/json']?.schema);
- const body = fieldsUnder('JSON body');
- const bodyReferences = referencesUnder('JSON body');
- const hasBodySection = page.headings.some(
- (heading) =>
- heading.level === 3 && heading.text.trim() === 'JSON body' && heading.line > section.line && heading.line < end,
- );
- if (jsonSchema == null) {
- for (const [field, docField] of body.fields) {
- if (!docField.undeclared) this.push(relative, key, 'body-extra', field);
- }
- }
- if (jsonSchema != null && !hasBodySection && collectPropertySchemas(spec, jsonSchema).size > 0) {
- this.push(relative, key, 'body-missing', 'no JSON body section documents the request body');
- }
- if (jsonSchema != null && hasBodySection && collectPropertySchemas(spec, jsonSchema).size > 0) {
- let documented = body.fields;
- let groups = body.groups;
- if (body.tables > 0) {
- this.counters.bodyTables += 1;
- } else if (bodyReferences.length > 0) {
- this.counters.bodiesByReference += 1;
- groups = bodyReferences.flatMap((parent) =>
- this.objectTables([parent.anchor]).map(
- (table) => new Map([...table].filter(([field]) => !parent.except.has(field))),
- ),
- );
- documented = mergeGroups(groups);
- } else {
- const sameAs = sectionProse
- .filter((line) => subsection(line.stack) === 'JSON body')
- .flatMap((line) => [...documentReferences(page.slug, line.text)])
- .flatMap((anchor) => this.routeBodies.get(anchor) ?? []);
- if (sameAs.length > 0) {
- this.counters.bodiesByReference += 1;
- groups = sameAs;
- documented = mergeGroups(groups);
- }
- }
- this.compareFields(
- spec,
- {page: relative, operation: key, kind: 'body', label: ''},
- documented,
- new Set(documented.keys()),
- jsonSchema,
- );
- this.compareBodyBranches(spec, relative, key, jsonSchema, groups);
- }
-
- const query = fieldsUnder('Query parameters');
- const actualQuery = new Set((operation.parameters ?? []).filter((p) => p.in === 'query').map((p) => p.name));
- if (query.tables > 0) this.counters.queryTables += 1;
- for (const [field, docField] of query.fields) {
- if (!actualQuery.has(field) && !docField.undeclared) this.push(relative, key, 'query-extra', field);
- if (actualQuery.has(field) && docField.undeclared) this.push(relative, key, 'query-stale-undeclared', field);
- }
- for (const field of actualQuery) {
- if (!query.fields.has(field)) this.push(relative, key, 'query-missing', field);
- }
-
- const success = Object.entries(operation.responses ?? {}).find(([status]) => status.startsWith('2'));
- const responseSchema = success?.[1].content?.['application/json']?.schema;
- const responseTarget = objectTarget(spec, responseSchema);
- const responseBody = fieldsUnder('Response body');
- if (responseTarget == null) {
- for (const [field, docField] of responseBody.fields) {
- if (!docField.undeclared) this.push(relative, key, 'response-extra', field);
- }
- return;
- }
- const responseDeclared = {anchors: new Set<string>(), names: new Set<string>()};
- for (const line of sectionProse) {
- const name = subsection(line.stack);
- if (name !== 'Response' && name !== 'Response body') continue;
- for (const sentence of declaredOnlySentences(line.text)) {
- for (const reference of documentReferences(page.slug, sentence)) {
- if (this.isObjectAnchor(reference)) responseDeclared.anchors.add(reference);
- }
- }
- for (const field of declaredOnlyNames(line.text)) responseDeclared.names.add(field);
- }
- const responseKnown = collectPropertySchemas(spec, responseSchema, 0, new Map(), true);
- if (!acceptsUndeclaredProperties(spec, responseSchema)) {
- for (const field of responseDeclared.names) {
- if (!responseKnown.has(field)) this.push(relative, key, 'response-stale-declared', field);
- }
- }
- if (responseBody.tables > 0) {
- this.counters.responseTables += 1;
- this.counters.responseFieldsFound += this.compareFields(
- spec,
- {page: relative, operation: key, kind: 'response', label: ''},
- responseBody.fields,
- new Set([...responseBody.fields.keys(), ...responseDeclared.names]),
- responseTarget,
- );
- return;
- }
- const statusAnchors = new Set<string>(responseDeclared.anchors);
- for (const parent of referencesUnder('Response body')) statusAnchors.add(parent.anchor);
- for (const table of sectionTables) {
- if (subsection(table.stack) !== 'Response' || table.header[0] !== 'Status') continue;
- for (const row of table.rows) {
- if (!row[0]?.trim().startsWith('2')) continue;
- for (const reference of documentReferences(page.slug, row[1] ?? '')) {
- if (this.isObjectAnchor(reference)) statusAnchors.add(reference);
- }
- }
- }
- if (statusAnchors.size > 0) {
- this.counters.responseObjects += 1;
- this.counters.responseFieldsFound += this.compareObject(
- spec,
- relative,
- key,
- [...statusAnchors],
- responseSchema,
- '',
- responseDeclared.names,
- );
- return;
- }
- for (const field of collectPropertySchemas(spec, responseTarget).keys()) {
- if (!responseDeclared.names.has(field)) this.push(relative, key, 'response-missing', field);
- }
- }
-}
-
-export function verifyPages(pages: ReadonlyArray<MarkdownPage>, mainSpec: Spec, adminSpec: Spec): SchemaVerification {
- return new Verifier(pages, mainSpec, adminSpec).run();
-}
-
-const KIND_LABELS = new Map([
- ['extra', 'documented but not in the schema'],
- ['missing', 'in the schema but undocumented'],
- ['declared', 'named by a "schema also declares" sentence but not in the schema'],
- ['undeclared', 'footnoted as not declared by the schema but in the schema'],
-]);
-
-async function main(): Promise<void> {
- const mainSpec: Spec = JSON.parse(await readFile(MAIN_SPEC, 'utf8'));
- const adminSpec: Spec = JSON.parse(await readFile(ADMIN_SPEC, 'utf8'));
- const pages = await readMarkdownPages(DOCS_ROOT);
- const {mismatches, optionalityAdvisories, counters} = verifyPages(pages, mainSpec, adminSpec);
-
- if (process.env.FLUXER_DOCS_SCHEMA_JSON != null) {
- const payload: Record<string, Array<{operation: string; kind: string; field: string}>> = {};
- for (const m of mismatches) {
- const list = payload[m.page] ?? [];
- list.push({operation: m.operation, kind: m.kind, field: m.detail});
- payload[m.page] = list;
- }
- await writeFile(process.env.FLUXER_DOCS_SCHEMA_JSON, JSON.stringify(payload, null, 1));
- console.log(`wrote ${process.env.FLUXER_DOCS_SCHEMA_JSON}`);
- }
-
- const byKind = new Map<string, number>();
- for (const m of mismatches) {
- byKind.set(m.kind, (byKind.get(m.kind) ?? 0) + 1);
- }
-
- console.log(`request body tables checked: ${counters.bodyTables.toString()}`);
- console.log(`request bodies documented by reference to an object or route: ${counters.bodiesByReference.toString()}`);
- console.log(`union request bodies checked: ${counters.unionBodies.toString()}`);
- console.log(`query parameter tables checked: ${counters.queryTables.toString()}`);
- console.log(`success responses checked against a response body table: ${counters.responseTables.toString()}`);
- console.log(`success responses checked against a linked object section: ${counters.responseObjects.toString()}`);
- console.log(`response fields found documented: ${counters.responseFieldsFound.toString()}`);
- console.log(`linked object sections compared with their schema: ${counters.nestedObjects.toString()}`);
- console.log(
- `fields whose schema has members but whose row links no object section: ${counters.unlinkedObjects.toString()}`,
- );
- console.log(`request and response field types compared: ${counters.typesCompared.toString()}`);
- console.log(`request body optionality compared: ${counters.optionalityCompared.toString()}`);
- console.log(`optionality advisories: ${optionalityAdvisories.length.toString()}`);
- for (const entry of optionalityAdvisories) {
- console.log(` ${entry}`);
- }
- for (const [kind, count] of [...byKind.entries()].sort()) {
- console.log(` ${kind}: ${count.toString()}`);
- }
- if (mismatches.length > 0) {
- console.log('');
- for (const m of mismatches.slice(0, 200)) {
- let label = KIND_LABELS.get(m.kind.split('-').at(-1) ?? '') ?? m.kind;
- if (m.kind === 'type-mismatch') label = 'type disagreement';
- if (m.kind === 'optionality') label = 'optionality disagreement';
- console.log(`${m.page} ${m.operation} ${label}: ${m.detail}`);
- }
- if (mismatches.length > 200) {
- console.log(`... and ${(mismatches.length - 200).toString()} more`);
- }
- console.error(`FAIL: ${mismatches.length.toString()} field mismatches`);
- process.exit(1);
- }
- console.log('OK: no field mismatches found in the checked tables and checked-in OpenAPI schemas');
-}
-
-if (process.argv[1] != null && import.meta.url === pathToFileURL(process.argv[1]).href) {
- await main();
-}
diff --git a/fluxer_docs/scripts/VerifyDocsSidebar.ts b/fluxer_docs/scripts/VerifyDocsSidebar.ts
deleted file mode 100644
index 2bc9434fb..000000000
--- a/fluxer_docs/scripts/VerifyDocsSidebar.ts
+++ /dev/null
@@ -1,179 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-import {readFile} from 'node:fs/promises';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-import {DOCS_ROOT, listDraftMarkdownFiles, listMarkdownFiles, pageSlug, slugifyHeading} from './DocsSource.ts';
-
-const ASTRO_CONFIG = fileURLToPath(new URL('../astro.config.ts', import.meta.url));
-
-const SIDEBAR_START = 'sidebar: [';
-const QUOTED_ENTRY = /'([^']+)'/gu;
-const LINK_ENTRY = /link:\s*'([^']+)'/gu;
-
-function sidebarBlock(source: string): string {
- const start = source.indexOf(SIDEBAR_START);
- if (start === -1) {
- throw new Error('sidebar block not found in astro.config.ts');
- }
- let depth = 0;
- for (let cursor = start + SIDEBAR_START.length - 1; cursor < source.length; cursor += 1) {
- const character = source[cursor];
- if (character === '[') {
- depth += 1;
- }
- if (character === ']') {
- depth -= 1;
- if (depth === 0) {
- return source.slice(start, cursor + 1);
- }
- }
- }
- throw new Error('sidebar block is unbalanced');
-}
-
-const config = await readFile(ASTRO_CONFIG, 'utf8');
-const block = sidebarBlock(config);
-
-const labels = new Set<string>();
-for (const match of block.matchAll(/label:\s*'([^']+)'/gu)) {
- labels.add(match[1]);
-}
-
-const linkSlugs = new Set<string>();
-for (const match of block.matchAll(LINK_ENTRY)) {
- linkSlugs.add(match[1].replace(/^\//u, '').replace(/\/$/u, ''));
-}
-
-const referenced = new Set<string>(linkSlugs);
-for (const match of block.matchAll(QUOTED_ENTRY)) {
- const value = match[1];
- if (labels.has(value)) {
- continue;
- }
- if (value.startsWith('/')) {
- continue;
- }
- if (!value.includes('/') && !value.match(/^[a-z0-9-]+$/u)) {
- continue;
- }
- referenced.add(value);
-}
-
-const files = await listMarkdownFiles(DOCS_ROOT);
-const actual = new Set(files.map((file) => pageSlug(path.relative(DOCS_ROOT, file))));
-
-const missingPages = [...referenced].filter((slug) => !actual.has(slug)).sort();
-const orphanPages = [...actual].filter((slug) => !referenced.has(slug)).sort();
-
-console.log(`sidebar entries: ${referenced.size.toString()}`);
-console.log(`content pages: ${actual.size.toString()}`);
-console.log(`draft pages withheld from the build: ${(await listDraftMarkdownFiles(DOCS_ROOT)).length.toString()}`);
-
-let failures = 0;
-if (missingPages.length > 0) {
- console.log('');
- console.log(`sidebar references a page that does not exist: ${missingPages.length.toString()}`);
- for (const slug of missingPages) {
- console.log(` ${slug}`);
- }
- failures += missingPages.length;
-}
-if (orphanPages.length > 0) {
- console.log('');
- console.log(`page exists but is not in the sidebar: ${orphanPages.length.toString()}`);
- for (const slug of orphanPages) {
- console.log(` ${slug === '' ? '(index)' : slug}`);
- }
- failures += orphanPages.length;
-}
-
-const INTERNAL_LINK = /\]\((\/[^)\s#]*)(#[^)\s]*)?\)/gu;
-const SAME_PAGE_LINK = /\]\((#[^)\s]*)\)/gu;
-const HEADING = /^#{2,6}\s+(.+?)\s*$/u;
-const EXPLICIT_ANCHOR = /<[a-z]+\s+id=["']([^"']+)["']/gu;
-const dangling: Array<string> = [];
-const anchorable = new Set(actual);
-
-const anchorsByPage = new Map<string, Set<string>>();
-const ambiguousByPage = new Map<string, Set<string>>();
-for (const file of files) {
- const slug = pageSlug(path.relative(DOCS_ROOT, file));
- const source = await readFile(file, 'utf8');
- const anchors = new Set<string>();
- const counts = new Map<string, number>();
- for (const line of source.split('\n')) {
- const heading = line.match(HEADING);
- if (heading != null) {
- const generated = slugifyHeading(heading[1]);
- anchors.add(generated);
- counts.set(generated, (counts.get(generated) ?? 0) + 1);
- }
- }
- for (const explicit of source.matchAll(EXPLICIT_ANCHOR)) {
- anchors.add(explicit[1]);
- }
- anchorsByPage.set(slug, anchors);
- const ambiguous = new Set<string>();
- for (const [generated, count] of counts) {
- if (count > 1 && !source.includes(`id="${generated}"`)) {
- ambiguous.add(generated);
- }
- }
- ambiguousByPage.set(slug, ambiguous);
-}
-
-for (const file of files) {
- const relative = path.relative(DOCS_ROOT, file);
- const selfSlug = pageSlug(path.relative(DOCS_ROOT, file));
- const source = await readFile(file, 'utf8');
- for (const match of source.matchAll(INTERNAL_LINK)) {
- const target = match[1].replace(/^\//u, '').replace(/\/$/u, '');
- if (target.length === 0) {
- if (match[2] != null) {
- const fragment = match[2].slice(1);
- const rootAnchors = anchorsByPage.get('');
- if (rootAnchors != null && fragment.length > 0 && !rootAnchors.has(fragment)) {
- dangling.push(`${relative} -> /#${fragment} (anchor)`);
- }
- }
- continue;
- }
- if (!anchorable.has(target)) {
- dangling.push(`${relative} -> /${target}/`);
- continue;
- }
- if (match[2] != null) {
- const fragment = match[2].slice(1);
- const anchors = anchorsByPage.get(target);
- if (fragment.length > 0 && anchors != null && !anchors.has(fragment)) {
- dangling.push(`${relative} -> /${target}/#${fragment} (anchor)`);
- } else if (fragment.length > 0 && (ambiguousByPage.get(target)?.has(fragment) ?? false)) {
- dangling.push(`${relative} -> /${target}/#${fragment} (that heading repeats on the target page)`);
- }
- }
- }
- for (const match of source.matchAll(SAME_PAGE_LINK)) {
- const fragment = match[1].slice(1);
- const anchors = anchorsByPage.get(selfSlug);
- if (fragment.length > 0 && anchors != null && !anchors.has(fragment)) {
- dangling.push(`${relative} -> #${fragment} (same-page anchor)`);
- } else if (fragment.length > 0 && (ambiguousByPage.get(selfSlug)?.has(fragment) ?? false)) {
- dangling.push(`${relative} -> #${fragment} (that heading repeats on this page)`);
- }
- }
-}
-if (dangling.length > 0) {
- console.log('');
- console.log(`links to a page that does not exist: ${dangling.length.toString()}`);
- for (const entry of [...new Set(dangling)].sort()) {
- console.log(` ${entry}`);
- }
- failures += dangling.length;
-}
-
-if (failures > 0) {
- console.error(`FAIL: ${failures.toString()} structure problems`);
- process.exit(1);
-}
-console.log('OK: sidebar, content and internal links are in sync');
diff --git a/fluxer_docs/scripts/VerifyDocsStyle.ts b/fluxer_docs/scripts/VerifyDocsStyle.ts
deleted file mode 100644
index 1de2b58ef..000000000
--- a/fluxer_docs/scripts/VerifyDocsStyle.ts
+++ /dev/null
@@ -1,545 +0,0 @@
-// SPDX-License-Identifier: AGPL-3.0-or-later
-
-import {readdir, readFile} from 'node:fs/promises';
-import path from 'node:path';
-import {fileURLToPath} from 'node:url';
-import {DOCS_ROOT, readMarkdownPages} from './DocsSource.ts';
-import {
- columnWidthPercents,
- extractTables,
- PX,
- TABLE_BUDGET_PX,
- TABLE_FLOOR_CAP_SHARE,
- TABLE_MAX_CELL_CHARS,
- TABLE_MAX_COLUMNS,
- TABLE_MAX_IDENT_CHARS,
- TABLE_WIDE_TIER_PX,
-} from './DocsTableWidth.ts';
-
-const STYLES_ROOT = fileURLToPath(new URL('../src/styles/', import.meta.url));
-const STARLIGHT_STYLES = fileURLToPath(new URL('../node_modules/@astrojs/starlight/dist/style/', import.meta.url));
-
-const FORBIDDEN_MARKETING_WORDS = [
- 'seamless',
- 'effortless',
- 'blazing',
- 'cutting-edge',
- 'game-changing',
- 'powerful and',
- 'just simply',
- 'simply use',
- 'simply call',
- 'simply add',
- 'simply set',
-];
-
-const FORBIDDEN_TOPICS = [
- {pattern: /push subscription/iu, reason: 'push API does not exist in the live era'},
- {pattern: /\/push\/events/u, reason: 'push events API does not exist in the live era'},
- {pattern: /voice[- ]public[- ]key/iu, reason: 'voice connection API does not exist in the live era'},
- {pattern: /match-public-key/u, reason: 'voice connection API does not exist in the live era'},
-];
-
-const CANONICAL_HEADINGS = new Map([
- ['### JSON params', '### JSON body'],
- ['### Query string parameters', '### Query parameters'],
- ['### Response Body', '### Response body'],
- ['### JSON Body', '### JSON body'],
- ['### Query Parameters', '### Query parameters'],
-]);
-
-interface Finding {
- readonly file: string;
- readonly line: number;
- readonly rule: string;
- readonly detail: string;
-}
-
-function frontmatterOf(source: string): string | null {
- if (!source.startsWith('---\n')) {
- return null;
- }
- const end = source.indexOf('\n---', 4);
- if (end === -1) {
- return null;
- }
- return source.slice(4, end);
-}
-
-const pages = await readMarkdownPages(DOCS_ROOT, {includeDrafts: true});
-const findings: Array<Finding> = [];
-
-for (const {file, relativePath: relative, source, lines} of pages) {
- const frontmatter = frontmatterOf(source);
-
- if (frontmatter == null) {
- findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing frontmatter block'});
- } else {
- if (!/^# SPDX-License-Identifier: AGPL-3\.0-or-later$/mu.test(frontmatter)) {
- findings.push({file: relative, line: 2, rule: 'spdx', detail: 'missing SPDX comment in frontmatter'});
- }
- if (!/^title:\s*\S/mu.test(frontmatter)) {
- findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing title'});
- }
- if (!/^description:\s*\S/mu.test(frontmatter)) {
- findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing description'});
- }
- }
-
- const usesRouteHeader = source.includes('<RouteHeader');
- const importsRouteHeader = source.includes("import RouteHeader from '@/components/RouteHeader.astro'");
- if (usesRouteHeader && !importsRouteHeader) {
- findings.push({file: relative, line: 1, rule: 'import', detail: 'uses RouteHeader without importing it'});
- }
- if (usesRouteHeader && !file.endsWith('.mdx')) {
- findings.push({file: relative, line: 1, rule: 'extension', detail: 'uses RouteHeader but is not .mdx'});
- }
-
- let fenced = false;
- for (let index = 0; index < lines.length; index += 1) {
- const line = lines[index];
- const number = index + 1;
- const insideFence = fenced;
- if (line.startsWith('```')) {
- fenced = !fenced;
- }
- if (insideFence) {
- continue;
- }
- if (line.includes('—')) {
- findings.push({file: relative, line: number, rule: 'em-dash', detail: line.trim().slice(0, 100)});
- }
- for (const word of FORBIDDEN_MARKETING_WORDS) {
- if (line.toLowerCase().includes(word)) {
- findings.push({file: relative, line: number, rule: 'tone', detail: `contains "${word.trim()}"`});
- }
- }
- const canonical = CANONICAL_HEADINGS.get(line.trim());
- if (canonical != null) {
- findings.push({
- file: relative,
- line: number,
- rule: 'heading',
- detail: `use "${canonical}" for consistency with the rest of the corpus`,
- });
- }
- for (const topic of FORBIDDEN_TOPICS) {
- if (topic.pattern.test(line)) {
- findings.push({file: relative, line: number, rule: 'unshipped', detail: topic.reason});
- }
- }
- }
-}
-
-for (const {relativePath: relative, lines} of pages) {
- const sectionStarts: Array<number> = [];
- for (let index = 0; index < lines.length; index += 1) {
- if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) {
- sectionStarts.push(index);
- }
- }
- sectionStarts.push(lines.length);
- for (let s = 0; s + 1 < sectionStarts.length; s += 1) {
- let responseBody = -1;
- let response = -1;
- for (let index = sectionStarts[s]; index < sectionStarts[s + 1]; index += 1) {
- const heading = lines[index].trim();
- if (heading === '### Response body' && responseBody === -1) {
- responseBody = index;
- }
- if (heading === '### Response' && response === -1) {
- response = index;
- }
- }
- if (responseBody !== -1 && response !== -1 && responseBody > response) {
- findings.push({
- file: relative,
- line: response + 1,
- rule: 'order',
- detail: '"### Response body" must precede "### Response"',
- });
- }
- }
-}
-
-for (const {relativePath: relative, lines} of pages) {
- let block = new Map<string, number>();
- let sinceFootnote = 0;
- for (let index = 0; index < lines.length; index += 1) {
- const definition = lines[index].match(/^<sup>(\d+)<\/sup>\s/u);
- if (definition == null) {
- if (lines[index].trim().length > 0) {
- sinceFootnote += 1;
- }
- if (sinceFootnote > 1) {
- block = new Map<string, number>();
- }
- continue;
- }
- sinceFootnote = 0;
- const previous = block.get(definition[1]);
- if (previous != null) {
- findings.push({
- file: relative,
- line: index + 1,
- rule: 'footnote',
- detail: `footnote ${definition[1]} is defined twice in the same block, first at line ${previous.toString()}`,
- });
- continue;
- }
- block.set(definition[1], index + 1);
- }
-}
-
-const NOTATION_EXAMPLE = 'A superscript marker such as <sup>1</sup> refers to the numbered footnote';
-
-for (const {relativePath: relative, source, lines} of pages) {
- const explainsNotation = source.includes(NOTATION_EXAMPLE);
- const bounds: Array<number> = [0];
- for (let index = 0; index < lines.length; index += 1) {
- if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) {
- bounds.push(index);
- }
- }
- bounds.push(lines.length);
- for (let b = 0; b + 1 < bounds.length; b += 1) {
- const definitions = new Map<string, number>();
- const references = new Set<string>();
- for (let index = bounds[b]; index < bounds[b + 1]; index += 1) {
- const definition = lines[index].match(/^<sup>(\d+)<\/sup>\s/u);
- if (definition != null) {
- if (!definitions.has(definition[1])) {
- definitions.set(definition[1], index + 1);
- }
- continue;
- }
- for (const marker of lines[index].matchAll(/<sup>(\d+)<\/sup>/gu)) {
- references.add(marker[1]);
- }
- }
- for (const [number, line] of definitions) {
- if (!references.has(number)) {
- findings.push({
- file: relative,
- line,
- rule: 'footnote',
- detail: `footnote ${number} is defined but never referenced in its section`,
- });
- }
- }
- for (const number of references) {
- if (definitions.has(number)) {
- continue;
- }
- if (explainsNotation) {
- continue;
- }
- findings.push({
- file: relative,
- line: bounds[b] + 1,
- rule: 'footnote',
- detail: `footnote ${number} is referenced but never defined in its section`,
- });
- }
- }
-}
-
-type TableRule =
- | 'table-columns'
- | 'table-fit'
- | 'table-cell'
- | 'table-identifier'
- | 'table-rows'
- | 'table-parallel'
- | 'table-wide-tight';
-
-const TABLE_RULES: ReadonlyArray<TableRule> = [
- 'table-columns',
- 'table-fit',
- 'table-cell',
- 'table-identifier',
- 'table-rows',
- 'table-parallel',
- 'table-wide-tight',
-];
-
-const TABLE_COUNT_FLOOR = 1800;
-
-const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRule, number>>>>([
- ['admin-api/applications.mdx', {'table-identifier': 1}],
- ['admin-api/discovery.mdx', {'table-identifier': 1}],
- ['admin-api/gateway.mdx', {'table-fit': 1, 'table-identifier': 3}],
- ['admin-api/guilds.mdx', {'table-identifier': 3}],
- ['admin-api/index.mdx', {'table-identifier': 3}],
- ['admin-api/instance.mdx', {'table-identifier': 8}],
- ['admin-api/messages.mdx', {'table-identifier': 1}],
- ['admin-api/reports.mdx', {'table-fit': 1, 'table-identifier': 3}],
- ['admin-api/users.mdx', {'table-identifier': 4}],
- ['admin-api/voice.mdx', {'table-identifier': 3}],
- ['gateway/event-filtering.md', {'table-cell': 2}],
- ['gateway/events.md', {'table-identifier': 1}],
- ['gateway/opcodes-and-close-codes.md', {'table-cell': 1}],
- ['gateway/overview.md', {'table-cell': 1}],
- ['http-api/authentication.mdx', {'table-identifier': 1}],
- ['http-api/billing.mdx', {'table-identifier': 4}],
- ['http-api/calls.mdx', {'table-fit': 1, 'table-cell': 3}],
- ['http-api/channels.mdx', {'table-cell': 6, 'table-identifier': 2}],
- ['http-api/connections.mdx', {'table-cell': 2}],
- ['http-api/discovery.mdx', {'table-cell': 2}],
- ['http-api/donations.mdx', {'table-cell': 1}],
- ['http-api/entrance-sounds.mdx', {'table-cell': 3, 'table-parallel': 1}],
- ['http-api/forums.mdx', {'table-identifier': 2}],
- ['http-api/gifs.mdx', {'table-cell': 5}],
- ['http-api/gifts.mdx', {'table-cell': 1}],
- ['http-api/guild-audit-logs.mdx', {'table-identifier': 3}],
- ['http-api/guild-channels.mdx', {'table-cell': 2, 'table-identifier': 1}],
- ['http-api/guild-emojis.mdx', {'table-cell': 4}],
- ['http-api/guild-members.mdx', {'table-identifier': 2}],
- ['http-api/guild-moderation.mdx', {'table-cell': 1}],
- ['http-api/guild-stickers.mdx', {'table-cell': 3}],
- ['http-api/guilds.mdx', {'table-fit': 1, 'table-identifier': 5}],
- ['http-api/instance.mdx', {'table-identifier': 8}],
- ['http-api/invites.mdx', {'table-cell': 6}],
- ['http-api/messages.mdx', {'table-fit': 2, 'table-cell': 20, 'table-identifier': 2}],
- ['http-api/permissions.mdx', {'table-cell': 8}],
- ['http-api/premium.mdx', {'table-identifier': 5}],
- ['http-api/reports.mdx', {'table-identifier': 1}],
- ['http-api/search.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1}],
- ['http-api/streams.mdx', {'table-cell': 5}],
- ['http-api/unfurl.mdx', {'table-cell': 2, 'table-parallel': 1}],
- ['http-api/users.mdx', {'table-fit': 1, 'table-identifier': 3}],
- ['http-api/users/content.mdx', {'table-cell': 2}],
- ['http-api/users/current-user.mdx', {'table-identifier': 1}],
- ['http-api/users/data-harvest.mdx', {'table-cell': 2}],
- ['http-api/users/email-and-password.mdx', {'table-identifier': 1}],
- ['http-api/users/mfa.mdx', {'table-cell': 3, 'table-parallel': 1}],
- ['http-api/users/private-channels.mdx', {'table-cell': 3}],
- ['http-api/users/relationships.mdx', {'table-fit': 2, 'table-cell': 3, 'table-identifier': 2}],
- ['http-api/users/settings-protobuf.md', {'table-fit': 2, 'table-identifier': 7, 'table-parallel': 1}],
- ['http-api/users/settings.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1, 'table-parallel': 1}],
- ['http-api/webhooks.mdx', {'table-identifier': 1, 'table-parallel': 1}],
- ['media-proxy/responses-and-limits.md', {'table-cell': 2}],
- ['media-proxy/routes.mdx', {'table-cell': 1}],
- ['operator/kubernetes.mdx', {'table-identifier': 1}],
- ['voice/index.md', {'table-parallel': 1}],
-]);
-
-const tableFindingsByPage = new Map<string, Map<TableRule, Array<Finding>>>();
-let tablesMeasured = 0;
-const overWideTier: Array<Finding> = [];
-
-for (const {relativePath: relative, source} of pages) {
- for (const table of extractTables(source)) {
- tablesMeasured += 1;
- const raise = (rule: TableRule, detail: string): void => {
- const perRule = tableFindingsByPage.get(relative) ?? new Map<TableRule, Array<Finding>>();
- const list = perRule.get(rule) ?? [];
- list.push({file: relative, line: table.line, rule, detail});
- perRule.set(rule, list);
- tableFindingsByPage.set(relative, perRule);
- };
- const shape = table.header.join(' | ');
- if (table.columns > TABLE_MAX_COLUMNS) {
- raise(
- 'table-columns',
- `[${shape}] has ${table.columns.toString()} columns, at most ${TABLE_MAX_COLUMNS.toString()} are allowed`,
- );
- }
- if (table.minDemandPx > TABLE_BUDGET_PX) {
- raise(
- 'table-fit',
- `[${shape}] demands ${table.minDemandPx.toString()}px against the ${TABLE_BUDGET_PX.toString()}px budget, columns ${table.columnMinPx.join('/')}px`,
- );
- }
- if (table.minDemandPx > TABLE_WIDE_TIER_PX) {
- overWideTier.push({
- file: relative,
- line: table.line,
- rule: 'table-wide-tier',
- detail: `[${shape}] demands ${table.minDemandPx.toString()}px, over the ${TABLE_WIDE_TIER_PX.toString()}px width where the wide CSS tier switches to content sizing, so this table overflows its column instead of wrapping`,
- });
- }
- if (table.worstCellChars > TABLE_MAX_CELL_CHARS) {
- raise(
- 'table-cell',
- `[${shape}] column ${(table.worstCellColumn + 1).toString()} holds ${table.worstCellChars.toString()} rendered characters, over ${TABLE_MAX_CELL_CHARS.toString()}: "${table.worstCellText.slice(0, 60)}..."`,
- );
- }
- if (table.col1IdentChars > TABLE_MAX_IDENT_CHARS) {
- raise(
- 'table-identifier',
- `[${shape}] first column holds "${table.col1IdentText}", ${table.col1IdentChars.toString()} characters, over ${TABLE_MAX_IDENT_CHARS.toString()}`,
- );
- }
- if (table.rows < 2 && !table.canonical) {
- raise(
- 'table-rows',
- `[${shape}] has ${table.rows.toString()} body row(s) and is not a canonical schema shape, so it is a sentence`,
- );
- }
- if (!table.parallel) {
- raise('table-parallel', `[${shape}] ${table.nonParallelReason}`);
- }
- const shares = columnWidthPercents(
- table.columnMinPx.map((minPx, index) => ({minPx, maxPx: table.columnMaxPx[index]})),
- TABLE_WIDE_TIER_PX,
- );
- const floorCap = TABLE_WIDE_TIER_PX * TABLE_FLOOR_CAP_SHARE;
- for (const [index, share] of shares.entries()) {
- const allotted = (share / 100) * TABLE_WIDE_TIER_PX;
- const unbreakable = table.columnMinPx[index] + PX.cellPad;
- if (unbreakable <= floorCap && allotted + PX.proseFallback < unbreakable) {
- raise(
- 'table-wide-tight',
- `[${shape}] column ${(index + 1).toString()} is allotted ${Math.round(allotted).toString()}px but its longest unbreakable word needs ${Math.round(unbreakable).toString()}px, so that word breaks mid-word even at the widest tier`,
- );
- }
- }
- }
-}
-
-if (tablesMeasured < TABLE_COUNT_FLOOR) {
- findings.push({
- file: 'scripts/DocsTableWidth.ts',
- line: 1,
- rule: 'table-extractor',
- detail: `only ${tablesMeasured.toString()} tables were measured, floor is ${TABLE_COUNT_FLOOR.toString()}. The table rules have gone blind`,
- });
-}
-
-findings.push(...overWideTier);
-
-const acceptedByRule = new Map<TableRule, number>();
-for (const [relative, perRule] of tableFindingsByPage) {
- const allowance = ACCEPTED_TABLE_FINDINGS.get(relative) ?? {};
- for (const [rule, list] of perRule) {
- const allowed = allowance[rule] ?? 0;
- acceptedByRule.set(rule, (acceptedByRule.get(rule) ?? 0) + Math.min(allowed, list.length));
- for (const finding of list.slice(allowed)) {
- findings.push(finding);
- }
- }
-}
-for (const [relative, allowance] of ACCEPTED_TABLE_FINDINGS) {
- const perRule = tableFindingsByPage.get(relative);
- for (const rule of TABLE_RULES) {
- const allowed = allowance[rule] ?? 0;
- const actual = perRule?.get(rule)?.length ?? 0;
- if (allowed > actual) {
- findings.push({
- file: relative,
- line: 1,
- rule: 'table-ledger',
- detail: `the ledger allows ${allowed.toString()} ${rule} findings but the page now produces ${actual.toString()}. Lower the entry in VerifyDocsStyle.ts`,
- });
- }
- }
-}
-
-interface CssBlock {
- readonly selector: string;
- readonly body: string;
- readonly line: number;
-}
-
-function cssBlocks(source: string): Array<CssBlock> {
- const blocks: Array<CssBlock> = [];
- const pattern = /([^{}]+)\{([^{}]*)\}/gu;
- let match: RegExpExecArray | null = pattern.exec(source);
- while (match != null) {
- blocks.push({
- selector: match[1]
- .replace(/\/\*[\s\S]*?\*\//gu, ' ')
- .replace(/\s+/gu, ' ')
- .trim(),
- body: match[2],
- line: source.slice(0, match.index).split('\n').length,
- });
- match = pattern.exec(source);
- }
- return blocks;
-}
-
-const SIDEWAYS_SCROLL = /(?:^|;)\s*overflow(?:-x)?\s*:\s*(?:auto|scroll)\s*(?:;|$)/mu;
-const SCROLL_FORBIDDEN_SELECTOR =
- /\btable\b|\bthead\b|\btbody\b|\btr\b|\.sl-markdown-content\s*(?:,|$)|table-wrapper|overflow-wrapper/u;
-
-for (const entry of await readdir(STYLES_ROOT, {withFileTypes: true})) {
- if (!entry.name.endsWith('.css')) {
- continue;
- }
- const source = await readFile(path.join(STYLES_ROOT, entry.name), 'utf8');
- for (const block of cssBlocks(source)) {
- if (!SCROLL_FORBIDDEN_SELECTOR.test(block.selector) || !SIDEWAYS_SCROLL.test(block.body)) {
- continue;
- }
- findings.push({
- file: `src/styles/${entry.name}`,
- line: block.line,
- rule: 'table-scroll',
- detail: `"${block.selector}" makes a table or the markdown container a sideways scroller. No table may ever scroll horizontally`,
- });
- }
-}
-
-{
- const starlightTableScrollers: Array<string> = [];
- for (const entry of await readdir(STARLIGHT_STYLES, {withFileTypes: true})) {
- if (!entry.name.endsWith('.css')) {
- continue;
- }
- const source = await readFile(path.join(STARLIGHT_STYLES, entry.name), 'utf8');
- for (const block of cssBlocks(source)) {
- if (block.selector.includes('table') && SIDEWAYS_SCROLL.test(block.body)) {
- starlightTableScrollers.push(`${entry.name}:${block.line.toString()} ${block.selector}`);
- }
- }
- }
- const content = await readFile(path.join(STYLES_ROOT, 'content.css'), 'utf8');
- const neutralised = cssBlocks(content).some(
- (block) =>
- /\.sl-markdown-content\s+table\b/u.test(block.selector) &&
- /(?:^|;)\s*display\s*:\s*table\s*(?:;|$)/mu.test(block.body) &&
- /(?:^|;)\s*overflow\s*:\s*visible\s*(?:;|$)/mu.test(block.body),
- );
- if (starlightTableScrollers.length > 0 && !neutralised) {
- findings.push({
- file: 'src/styles/content.css',
- line: 1,
- rule: 'table-scroll',
- detail: `Starlight still ships a scrolling table (${starlightTableScrollers.join(', ')}) and content.css no longer overrides it with "display: table" and "overflow: visible"`,
- });
- }
-}
-
-const byRule = new Map<string, number>();
-for (const finding of findings) {
- byRule.set(finding.rule, (byRule.get(finding.rule) ?? 0) + 1);
-}
-
-console.log(`pages checked: ${pages.length.toString()}`);
-console.log(`tables measured: ${tablesMeasured.toString()}`);
-
-for (const rule of TABLE_RULES) {
- console.log(
- ` ${rule}: ${(byRule.get(rule) ?? 0).toString()} new, ${(acceptedByRule.get(rule) ?? 0).toString()} accepted`,
- );
-}
-for (const [rule, count] of [...byRule.entries()].sort()) {
- if ((TABLE_RULES as ReadonlyArray<string>).includes(rule)) {
- continue;
- }
- console.log(` ${rule}: ${count.toString()}`);
-}
-if (findings.length > 0) {
- console.log('');
- for (const finding of findings.slice(0, 200)) {
- console.log(`${finding.file}:${finding.line.toString()} [${finding.rule}] ${finding.detail}`);
- }
- if (findings.length > 200) {
- console.log(`... and ${(findings.length - 200).toString()} more`);
- }
- console.error(`FAIL: ${findings.length.toString()} style problems`);
- process.exit(1);
-}
-console.log('OK: house style clean');
diff --git a/fluxer_docs/src/table/DocsTableMetrics.ts b/fluxer_docs/src/table/DocsTableMetrics.ts
index e258d47a3..acd5dc039 100644
--- a/fluxer_docs/src/table/DocsTableMetrics.ts
+++ b/fluxer_docs/src/table/DocsTableMetrics.ts
@@ -27,9 +27,6 @@ export const TABLE_FLOOR_CAP_SHARE = 1 / 3;
export const TABLE_MIDDLE_TIER_PX = 768;
export const TABLE_WIDE_TIER_PX = 960;
-export const TABLE_MAX_COLUMNS = 3;
-export const TABLE_MAX_CELL_CHARS = 120;
-export const TABLE_MAX_IDENT_CHARS = 24;
const TABLE_FIXED_REFERENCE_PX = TABLE_BUDGET_PX;