A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
chore(docs): drop the verify scripts and their dependencies
Hampuscf662fa01622
diff --git a/fluxer_docs/package.json b/fluxer_docs/package.jsonindex 035befa15..c993b2943 100644--- a/fluxer_docs/package.json+++ b/fluxer_docs/package.json@@ -7,12 +7,7 @@"build": "astro build","dev": "astro dev","start": "tsx scripts/StartServer.ts",- "typecheck": "astro check --minimumFailingSeverity warning",- "verify": "pnpm verify:sidebar && pnpm verify:coverage && pnpm verify:schemas && pnpm verify:style && pnpm verify:commands",- "verify:coverage": "tsx scripts/VerifyDocsCoverage.ts",- "verify:schemas": "tsx scripts/VerifyDocsSchemas.ts",- "verify:sidebar": "tsx scripts/VerifyDocsSidebar.ts",- "verify:style": "tsx scripts/VerifyDocsStyle.ts"+ "typecheck": "astro check --minimumFailingSeverity warning"},"dependencies": {"@astrojs/markdown-remark": "catalog:",@@ -29,7 +24,6 @@"devDependencies": {"@astrojs/check": "catalog:","@types/hast": "catalog:",- "@types/mdast": "catalog:","@types/node": "catalog:","@types/send": "catalog:","mdast-util-to-hast": "catalog:",diff --git a/fluxer_docs/scripts/DocsRouteHeaders.ts b/fluxer_docs/scripts/DocsRouteHeaders.tsdeleted file mode 100644index 7ad1ecbe3..000000000--- a/fluxer_docs/scripts/DocsRouteHeaders.ts+++ /dev/null@@ -1,120 +0,0 @@-import assert from 'node:assert/strict';-import {parseFrontmatter} from '@astrojs/markdown-remark';-import {createProcessor} from '@mdx-js/mdx';-import type {Nodes, Root} from 'mdast';-import type {MdxJsxAttribute, MdxJsxFlowElement, MdxJsxTextElement} from 'mdast-util-mdx-jsx';-import remarkGfm from 'remark-gfm';-import {HTTP_METHODS, type MarkdownPage} from './DocsSource.ts';--export interface DocsRouteHeader {- readonly method: string;- readonly path: string;- readonly file: string;- readonly bot: boolean;- readonly unauthenticated: boolean;- readonly line: number;- readonly endLine: number;-}--const STRING_ATTRIBUTES = new Set(['method', 'path', 'oauth2']);-const BOOLEAN_ATTRIBUTES = new Set(['bot', 'unauthenticated', 'auditReason', 'mfa']);-const markdownProcessor = createProcessor({format: 'md', remarkPlugins: [remarkGfm]});-const mdxProcessor = createProcessor({format: 'mdx', remarkPlugins: [remarkGfm]});--function readAttributeValue(attribute: MdxJsxAttribute, location: string): string | boolean {- const value = attribute.value;- if (value === null || value === undefined) return true;- if (typeof value === 'string') return value;- const body = value.data?.estree?.body;- if (body?.length === 1 && body[0].type === 'ExpressionStatement') {- const expression = body[0].expression;- if (- expression.type === 'Literal' &&- (typeof expression.value === 'string' || typeof expression.value === 'boolean')- ) {- return expression.value;- }- if (- expression.type === 'TemplateLiteral' &&- expression.expressions.length === 0 &&- expression.quasis.length === 1 &&- typeof expression.quasis[0].value.cooked === 'string'- ) {- return expression.quasis[0].value.cooked;- }- }- throw new Error(`${location}: RouteHeader ${attribute.name} requires a literal string or boolean`);-}--function readHeader(node: MdxJsxFlowElement | MdxJsxTextElement, file: string): DocsRouteHeader {- assert(node.position !== undefined, 'Parsed RouteHeader has no source position');- const line = node.position.start.line;- const endLine = node.position.end.line;- assert(Number.isInteger(line) && line > 0 && Number.isInteger(endLine) && endLine >= line);- const location = `${file}:${line}`;- const attributes = new Map<string, string | boolean>();- for (const attribute of node.attributes) {- if (attribute.type === 'mdxJsxExpressionAttribute') {- throw new Error(`${location}: RouteHeader spread attributes are unsupported`);- }- const name = attribute.name;- if (attributes.has(name)) {- throw new Error(`${location}: duplicate RouteHeader attribute ${name}`);- }- const isString = STRING_ATTRIBUTES.has(name);- if (!isString && !BOOLEAN_ATTRIBUTES.has(name)) {- throw new Error(`${location}: unknown RouteHeader attribute ${name}`);- }- const value = readAttributeValue(attribute, location);- if (isString ? typeof value !== 'string' : typeof value !== 'boolean') {- throw new Error(`${location}: RouteHeader ${name} must be ${isString ? 'a string' : 'a boolean'}`);- }- attributes.set(name, value);- }- const method = attributes.get('method');- const routePath = attributes.get('path');- if (typeof method !== 'string' || typeof routePath !== 'string') {- throw new Error(`${location}: RouteHeader requires method and path attributes`);- }- if (!HTTP_METHODS.has(method)) {- throw new Error(`${location}: unsupported RouteHeader method ${method}`);- }- if (!routePath.startsWith('/')) {- throw new Error(`${location}: RouteHeader path must be absolute`);- }- return {- method,- path: routePath,- file,- bot: attributes.get('bot') === true,- unauthenticated: attributes.get('unauthenticated') === true,- line,- endLine,- };-}--export function readRouteHeaders(page: MarkdownPage): Array<DocsRouteHeader> {- let tree: Root;- try {- const {content} = parseFrontmatter(page.source, {frontmatter: 'empty-with-spaces'});- const processor = page.file.endsWith('.mdx') ? mdxProcessor : markdownProcessor;- tree = processor.parse({value: content, path: page.file});- } catch (error) {- throw new Error(`${page.relativePath}: ${String(error)}`, {cause: error});- }- const headers: Array<DocsRouteHeader> = [];- const pending: Array<Nodes> = [tree];- while (pending.length > 0) {- const node = pending.pop();- assert(node !== undefined);- if ((node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') && node.name === 'RouteHeader') {- headers.push(readHeader(node, page.relativePath));- }- if ('children' in node) {- for (let index = node.children.length - 1; index >= 0; index -= 1) {- pending.push(node.children[index]);- }- }- }- return headers;-}diff --git a/fluxer_docs/scripts/DocsSource.ts b/fluxer_docs/scripts/DocsSource.tsdeleted file mode 100644index 862547ffc..000000000--- a/fluxer_docs/scripts/DocsSource.ts+++ /dev/null@@ -1,119 +0,0 @@-import {readdir, readFile} from 'node:fs/promises';-import path from 'node:path';-import {fileURLToPath} from 'node:url';-import {parseFrontmatter} from '@astrojs/markdown-remark';--export const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url));-export const HTTP_METHODS: ReadonlySet<string> = new Set(['GET', 'HEAD', 'POST', 'PATCH', 'PUT', 'DELETE', 'OPTIONS']);--export interface MarkdownPage {- readonly file: string;- readonly relativePath: string;- readonly source: string;- readonly lines: ReadonlyArray<string>;-}--async function collectMarkdownFiles(directory: string): Promise<Array<string>> {- const files: Array<string> = [];- for (const entry of await readdir(directory, {withFileTypes: true})) {- if (entry.name === 'node_modules') {- continue;- }- const resolved = path.join(directory, entry.name);- if (entry.isDirectory()) {- files.push(...(await collectMarkdownFiles(resolved)));- } else if (entry.isFile() && /\.mdx?$/u.test(entry.name)) {- files.push(resolved);- }- }- return files.sort();-}--function isDraftSource(source: string): boolean {- return parseFrontmatter(source).frontmatter.draft === true;-}--async function partitionMarkdownFiles(directory: string, drafts: boolean): Promise<Array<string>> {- const files: Array<string> = [];- for (const file of await collectMarkdownFiles(directory)) {- if (isDraftSource(await readFile(file, 'utf8')) === drafts) {- files.push(file);- }- }- return files;-}--export async function listMarkdownFiles(directory: string): Promise<Array<string>> {- return partitionMarkdownFiles(directory, false);-}--export async function listDraftMarkdownFiles(directory: string): Promise<Array<string>> {- return partitionMarkdownFiles(directory, true);-}--export async function readMarkdownPages(- directory: string,- options: {includeDrafts?: boolean} = {},-): Promise<Array<MarkdownPage>> {- const pages: Array<MarkdownPage> = [];- const files = options.includeDrafts ? await collectMarkdownFiles(directory) : await listMarkdownFiles(directory);- for (const file of files) {- const source = await readFile(file, 'utf8');- pages.push({- file,- relativePath: path.relative(directory, file).split(path.sep).join('/'),- source,- lines: source.split('\n'),- });- }- return pages;-}--export function splitTableRow(line: string): Array<string> {- const row = line.trim();- const cells: Array<string> = [];- let start = row.startsWith('|') ? 1 : 0;- for (let index = start; index < row.length; index += 1) {- if (row[index] === '\\' && (row[index + 1] === '\\' || row[index + 1] === '|')) {- index += 1;- continue;- }- if (row[index] === '|') {- cells.push(row.slice(start, index).trim());- start = index + 1;- }- }- if (start < row.length) {- cells.push(row.slice(start).trim());- }- return cells;-}--export function routeShape(method: string, routePath: string): string {- return `${method} ${routePath.split('?')[0].replace(/\{[^}]*\}/gu, '{}')}`;-}--export function slugifyHeading(heading: string): string {- return heading- .replace(/`/gu, '')- .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1')- .replace(/<[^>]*>/gu, '')- .toLowerCase()- .replace(/[^a-z0-9\s-]/gu, '')- .trim()- .replace(/\s+/gu, '-');-}--export function stripVersion(routePath: string): string {- if (routePath.startsWith('/v1/')) {- return routePath.slice(3);- }- return routePath;-}--export function pageSlug(relativePath: string): string {- return relativePath- .replace(/\.(mdx|md)$/u, '')- .replace(/\/index$/u, '')- .replace(/^index$/u, '');-}diff --git a/fluxer_docs/scripts/DocsTableWidth.ts b/fluxer_docs/scripts/DocsTableWidth.tsdeleted file mode 100644index a0d5d2928..000000000--- a/fluxer_docs/scripts/DocsTableWidth.ts+++ /dev/null@@ -1,226 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import {- columnWidthPercents,- maxContentPx,- minContentPx,- type Piece,- PX,- TABLE_BUDGET_PX,- TABLE_FLOOR_CAP_SHARE,- TABLE_MAX_CELL_CHARS,- TABLE_MAX_COLUMNS,- TABLE_MAX_IDENT_CHARS,- TABLE_WIDE_TIER_PX,-} from '@/table/DocsTableMetrics.ts';-import {splitTableRow} from './DocsSource.ts';--export {- columnWidthPercents,- PX,- TABLE_BUDGET_PX,- TABLE_FLOOR_CAP_SHARE,- TABLE_MAX_CELL_CHARS,- TABLE_MAX_COLUMNS,- TABLE_MAX_IDENT_CHARS,- TABLE_WIDE_TIER_PX,-};--const CANONICAL_SHAPES = new Set([- 'field|type|description',- 'status|body|condition',- 'value|name|description',- 'value|description',-]);--const PARALLEL_SENTINELS = new Set(['unset', 'empty', 'none', 'required', 'n/a', '-', 'same target', 'no default']);--export interface DocsTable {- readonly line: number;- readonly endLine: number;- readonly columns: number;- readonly rows: number;- readonly header: ReadonlyArray<string>;- readonly canonical: boolean;- readonly minDemandPx: number;- readonly columnMinPx: ReadonlyArray<number>;- readonly columnMaxPx: ReadonlyArray<number>;- readonly worstCellChars: number;- readonly worstCellColumn: number;- readonly worstCellText: string;- readonly col1IdentChars: number;- readonly col1IdentText: string;- readonly parallel: boolean;- readonly nonParallelReason: string;-}--function isDelimiter(line: string, columns: number): boolean {- const cells = splitTableRow(line);- return cells.length === columns && cells.every((cell) => /^:?-+:?$/u.test(cell));-}--function stripInline(text: string): string {- return text- .replace(/!\[([^\]]*)\]\([^)]*\)/gu, '$1')- .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1')- .replace(/<sup>([^<]*)<\/sup>/gu, '$1')- .replace(/<br\s*\/?>/giu, ' ')- .replace(/<[^>]+>/gu, '')- .replace(/\*\*([^*]*)\*\*/gu, '$1')- .replace(/\*([^*]*)\*/gu, '$1')- .replace(/ /gu, ' ')- .replace(/</gu, '<')- .replace(/>/gu, '>')- .replace(/&/gu, '&')- .replace(/\\([!-/:-@[-`{-~])/gu, '$1');-}--function pieces(cell: string): Array<Piece> {- const out: Array<Piece> = [];- const pattern = /`([^`]*)`/gu;- let last = 0;- let match: RegExpExecArray | null = pattern.exec(cell);- while (match != null) {- if (match.index > last) {- out.push({text: stripInline(cell.slice(last, match.index)), code: false});- }- out.push({- text: match[1].replace(/\\([\\|])/gu, (escaped, character: string) => (character === '|' ? character : escaped)),- code: true,- });- last = pattern.lastIndex;- match = pattern.exec(cell);- }- if (last < cell.length) {- out.push({text: stripInline(cell.slice(last)), code: false});- }- return out.filter((piece) => piece.text.length > 0);-}--function renderedText(parts: ReadonlyArray<Piece>): string {- return parts.map((piece) => piece.text).join('');-}--function cellKind(cell: string): string {- const stripped = cell.replace(/<sup>[^<]*<\/sup>/gu, '');- const parts = pieces(stripped);- if (parts.length === 0) {- return 'empty';- }- if (parts.every((piece) => piece.code || /^[\s,]+$/u.test(piece.text))) {- return 'code';- }- if (parts.every((piece) => !piece.code)) {- return /[.!?]\s|[.!?]$/u.test(stripped) ? 'sentence' : 'phrase';- }- return 'mixed';-}--export function extractTables(source: string): Array<DocsTable> {- const lines = source.split('\n');- const tables: Array<DocsTable> = [];- let fenced = false;- let index = 0;- while (index < lines.length) {- if (/^\s*(?:```|~~~)/u.test(lines[index])) {- fenced = !fenced;- index += 1;- continue;- }- if (fenced) {- index += 1;- continue;- }- if (!lines[index].trim().startsWith('|') || index + 1 >= lines.length) {- index += 1;- continue;- }- const header = splitTableRow(lines[index]);- if (header.length === 0 || !isDelimiter(lines[index + 1], header.length)) {- index += 1;- continue;- }- const start = index;- index += 2;- const body: Array<Array<string>> = [];- while (index < lines.length && lines[index].trim().startsWith('|')) {- body.push(splitTableRow(lines[index]));- index += 1;- }- const columns = header.length;- const columnMinPx = new Array<number>(columns).fill(0);- const columnMaxPx = new Array<number>(columns).fill(0);- let worstCellChars = 0;- let worstCellColumn = 0;- let worstCellText = '';- let col1IdentChars = 0;- let col1IdentText = '';- for (const [rowIndex, row] of [header, ...body].entries()) {- const isHeader = rowIndex === 0;- for (let column = 0; column < columns; column += 1) {- const parts = pieces(isHeader ? (row[column] ?? '').toUpperCase() : (row[column] ?? ''));- const minimum = minContentPx(parts);- const maximum = maxContentPx(parts, isHeader);- if (maximum > columnMaxPx[column]) {- columnMaxPx[column] = maximum;- }- if (minimum > columnMinPx[column]) {- columnMinPx[column] = minimum;- }- const text = renderedText(parts);- if (!isHeader && text.length > worstCellChars) {- worstCellChars = text.length;- worstCellColumn = column;- worstCellText = text;- }- if (column === 0 && !isHeader) {- for (const run of text.split(/(?<=\s)|(?<=-)(?!\d)/u)) {- const token = run.trim();- if (token.length > col1IdentChars) {- col1IdentChars = token.length;- col1IdentText = token;- }- }- }- }- }- const headerShape = header.map((cell) => cell.trim().toLowerCase()).join('|');- let parallel = true;- let nonParallelReason = '';- for (let column = 0; column < columns && parallel; column += 1) {- if (headerShape === 'status|body|condition' && column === 1) {- continue;- }- const shape = body- .map((row) => (row[column] ?? '').trim())- .filter((cell) => cell.length > 0 && !PARALLEL_SENTINELS.has(cell.toLowerCase()))- .map(cellKind);- const kinds = new Set(shape);- kinds.delete('empty');- const prose = kinds.has('sentence') || kinds.has('phrase') || kinds.has('mixed');- if (kinds.has('code') && prose && shape.length > 2) {- parallel = false;- nonParallelReason = `column ${(column + 1).toString()} mixes code-only cells with prose cells`;- }- }- tables.push({- line: start + 1,- endLine: index,- columns,- rows: body.length,- header,- canonical: CANONICAL_SHAPES.has(headerShape),- minDemandPx: Math.round(columnMinPx.reduce((a, b) => a + b, 0) + columns * PX.cellPad),- columnMinPx: columnMinPx.map((value) => Math.round(value)),- columnMaxPx: columnMaxPx.map((value) => Math.round(value)),- worstCellChars,- worstCellColumn,- worstCellText,- col1IdentChars,- col1IdentText,- parallel,- nonParallelReason,- });- }- return tables;-}diff --git a/fluxer_docs/scripts/VerifyDocsCoverage.ts b/fluxer_docs/scripts/VerifyDocsCoverage.tsdeleted file mode 100644index 7b1081ab6..000000000--- a/fluxer_docs/scripts/VerifyDocsCoverage.ts+++ /dev/null@@ -1,2482 +0,0 @@-import {stripVersion as stripVersionPrefix} from './DocsSource.ts';-// SPDX-License-Identifier: AGPL-3.0-or-later--import {spawnSync} from 'node:child_process';-import {createHash} from 'node:crypto';-import type {Dirent} from 'node:fs';-import {mkdir, mkdtemp, readdir, readFile, rm, writeFile} from 'node:fs/promises';-import {tmpdir} from 'node:os';-import path from 'node:path';-import {fileURLToPath} from 'node:url';-import {parseArgs} from 'node:util';-import {extractRoutesFromControllers} from '@fluxer/openapi/src/extractors/RouteExtractor';-import type {OpenAPIDocument} from '@fluxer/openapi/src/OpenAPITypes';-import {installerChecksumLine} from '@/installer/InstallerDigest.ts';-import {type DocsRouteHeader, readRouteHeaders} from './DocsRouteHeaders.ts';-import {DOCS_ROOT, HTTP_METHODS, type MarkdownPage, readMarkdownPages, routeShape} from './DocsSource.ts';--const {values: options} = parseArgs({options: {'source-only': {type: 'boolean', default: false}}});--const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url));-const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json');-const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json');-const MEDIA_PROXY_SERVER_DIR = path.join(REPO_ROOT, 'fluxer_media_proxy/src/server');--const BLUESKY_OAUTH_CONTROLLER = 'fluxer_api/src/api/bluesky/BlueskyOAuthController.ts';-const DOWNLOAD_CONTROLLER = 'fluxer_api/src/api/download/DownloadController.ts';--const MAIN_SPEC_EXEMPT = new Map<string, {file: string; anchor: string; reason: string}>([- [- 'GET /oauth2/authorize',- {- file: 'fluxer_api/src/api/oauth/OAuth2Controller.ts',- anchor: "'/oauth2/authorize',",- reason: 'the generator excludes the path outright, at OpenAPIGeneratorCatalog excluded.paths',- },- ],- [- 'GET /openapi.json',- {- file: 'fluxer_api/src/api/openapi/OpenAPIController.ts',- anchor: "app.get('/openapi.json'",- reason: 'the handler serves the spec file itself and has no OpenAPI({...}) block',- },- ],- [- 'GET /connections/bluesky/client-metadata.json',- {- file: BLUESKY_OAUTH_CONTROLLER,- anchor: "'/connections/bluesky/client-metadata.json'",- reason: 'an AT Protocol client document with no OpenAPI({...}) block',- },- ],- [- 'GET /connections/bluesky/jwks.json',- {- file: BLUESKY_OAUTH_CONTROLLER,- anchor: "'/connections/bluesky/jwks.json'",- reason: 'an AT Protocol client document with no OpenAPI({...}) block',- },- ],-- [- 'GET /dl/desktop/{}/{}/{}/latest/{}.sha256',- {- file: DOWNLOAD_CONTROLLER,- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\\\.sha256}`,',- reason: 'the path constrains :format by regex and has no OpenAPI path template',- },- ],- [- 'GET /dl/desktop/{}/{}/{}/{}/{}.sha256',- {- file: DOWNLOAD_CONTROLLER,- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\\\.sha256}`,',- reason: 'the path constrains :format by regex and has no OpenAPI path template',- },- ],- [- 'GET /dl/desktop/{}/{}/{}/latest/{}.zsync',- {- file: DOWNLOAD_CONTROLLER,- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\\\.zsync}`,',- reason: 'the path constrains :format by regex and has no OpenAPI path template',- },- ],- [- 'GET /dl/desktop/{}/{}/{}/{}/{}.zsync',- {- file: DOWNLOAD_CONTROLLER,- anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\\\.zsync}`,',- reason: 'the path constrains :format by regex and has no OpenAPI path template',- },- ],- [- 'GET /dl/{}',- {- file: DOWNLOAD_CONTROLLER,- anchor: '`${DOWNLOAD_PREFIX}/*`,',- reason: 'a Hono wildcard, which is not an OpenAPI path template',- },- ],-]);--const DELIBERATELY_UNDOCUMENTED = new Map([- ['GET /users/@me/mobile-devices', 'mobile notifications, backported separately'],- ['DELETE /users/@me/mobile-devices/{}', 'mobile notifications, backported separately'],- ['GET /users/@me/push/subscriptions', 'push API, backported separately'],- ['POST /users/@me/push/subscribe', 'push API, backported separately'],- ['POST /users/@me/push/rotate', 'push API, backported separately'],- ['DELETE /users/@me/push/subscriptions/{}', 'push API, backported separately'],-]);--interface OutOfBandRoute {- readonly reason: string;- readonly documentedIn: {readonly file: string; readonly anchor: string} | null;-}-const OUT_OF_BAND_CREDENTIAL = new Map<string, OutOfBandRoute>([- [- 'POST /internal/rpc',- {- reason:- 'x-fluxer-rpc-auth must timing-safe-equal Config.internal.gatewayRpcAuthToken, and an empty token rejects every caller. The caller is fluxer_gateway',- documentedIn: null,- },- ],- [- 'POST /webhooks/livekit',- {- reason: 'a LiveKit-signed callback whose shipped target is the internal address http://api:8080/webhooks/livekit',- documentedIn: {file: 'operator/configuration.mdx', anchor: 'http://api:8080/webhooks/livekit'},- },- ],- [- 'POST /webhooks/sweego',- {- reason:- 'a Standard Webhooks delivery signed with FLUXER_EMAIL_WEBHOOK_SECRET, which answers 404 Email not enabled while email is off',- documentedIn: {file: 'operator/configuration.mdx', anchor: 'POST /webhooks/sweego'},- },- ],- [- 'POST /webhooks/app-store',- {- reason:- 'an App Store Server Notification whose signedPayload must verify against the pinned Apple root before it is queued. The route is hosted-only',- documentedIn: {file: 'http-api/in-app-purchases.mdx', anchor: 'POST /webhooks/app-store'},- },- ],- [- 'POST /webhooks/google-play',- {- reason:- 'a Pub/Sub push whose Google-signed OIDC token must match the configured audience and service account before it is queued. The route is hosted-only',- documentedIn: {file: 'http-api/in-app-purchases.mdx', anchor: 'POST /webhooks/google-play'},- },- ],- [- 'GET /connections/bluesky/callback',- {- reason:- 'the atproto redirect URI, reached with a provider-issued authorisation code. A client MUST NOT call it directly',- documentedIn: {file: 'http-api/connections.mdx', anchor: '/connections/bluesky/callback'},- },- ],-]);--const HEALTH_AND_METRICS_PATHS = new Set(['/_health', '/_health/ready', '/_health/drain', '/_healthz', '/_metrics']);--const LEGACY_REPORT_ROUTES = new Set(['POST /reports/message', 'POST /reports/user', 'POST /reports/guild']);--interface ExemptionRule {- readonly name: string;- readonly justification: string;- readonly mayCoverNothing?: boolean;-- readonly anchors: ReadonlyArray<{readonly file: string; readonly anchor: string}>;- readonly covers: (shape: string, routePath: string) => boolean;-}-const EXEMPTION_RULES: ReadonlyArray<ExemptionRule> = [- {- name: 'test harness',- justification:- 'ControllerRegistry registers TestHarnessController only when config.dev.testModeEnabled or NODE_ENV is development, and ensureHarnessAccess rechecks that per request. No production instance serves a /test/ route',- anchors: [- {- file: 'fluxer_api/src/api/app/ControllerRegistry.ts',- anchor: "if (config.dev.testModeEnabled || config.nodeEnv === 'development') {",- },- {file: 'fluxer_api/src/api/test/TestHarnessController.ts', anchor: 'function ensureHarnessAccess'},- ],- covers: (_shape, routePath) => routePath.startsWith('/test/'),- },- {- name: 'deprecated desktop download redirect',- justification:- 'every /dl route is an undocumented deprecated redirect onto pkgs.fluxer.com, kept only for desktop clients already in the field. Nothing current calls one, so documenting them would advertise a path new callers must not use',- anchors: [- {- file: 'fluxer_api/src/api/download/DownloadController.ts',- anchor: 'function redirectToPackageOrigin',- },- ],- covers: (_shape, routePath) => routePath === '/dl' || routePath.startsWith('/dl/'),- },- {- name: 'backported separately',- justification:- 'part of a system this backport does not ship. Each shape is listed with its system, and a page that documents one fails the run below',- anchors: [],- covers: (shape) => DELIBERATELY_UNDOCUMENTED.has(shape),- },- {- name: 'process health and metrics',- justification:- 'loopback gated or reachable only inside the compose network. operator/reverse-proxy.mdx states which probes answer through a proxy once for every service, rather than per route',- anchors: [{file: 'fluxer_docs/src/content/docs/operator/reverse-proxy.mdx', anchor: '/_metrics'}],- covers: (_shape, routePath) => HEALTH_AND_METRICS_PATHS.has(routePath),- },- {- name: 'out-of-band credential',- justification:- 'no ordinary client holds the credential. Each entry states its guard, and five are covered in prose',- anchors: [{file: 'fluxer_api/src/api/app/ControllerRegistry.ts', anchor: 'InternalRpcController(routes);'}],- covers: (shape) => OUT_OF_BAND_CREDENTIAL.has(shape),- },- {- name: 'legacy report routes',- justification:- 'still served for clients released before report flows, which new callers must use instead. The generator drops them at OpenAPIGeneratorCatalog excluded.paths',- anchors: [{file: 'packages/openapi/src/generator/OpenAPIGeneratorCatalog.ts', anchor: "'/reports/message'"}],- covers: (shape) => LEGACY_REPORT_ROUTES.has(shape),- },-];--const MEDIA_PROXY_ROUTES = new Map([- ['GET /_health', '.route("/_health", get(routes::ops::health))'],- ['HEAD /_health', '.route("/_health", get(routes::ops::health))'],- ['GET /_metrics', '.route("/_metrics", get(routes::ops::metrics_handler))'],- ['POST /_metadata', '.route("/_metadata", post(routes::internal::metadata_handler))'],- ['POST /_sniff', '.route("/_sniff", post(routes::internal::sniff_handler))'],- ['POST /_thumbnail', '.route("/_thumbnail", post(routes::internal::thumbnail_handler))'],- ['POST /_frames', '.route("/_frames", post(routes::internal::frames_handler))'],- [- 'PUT /v1/relay/{}',- '"/v1/relay/{*key}",\n put(routes::relay::relay_put).options(routes::relay::relay_options),',- ],- [- 'OPTIONS /v1/relay/{}',- '"/v1/relay/{*key}",\n put(routes::relay::relay_put).options(routes::relay::relay_options),',- ],- ['GET /{}', 'if app.cfg.mode == DeploymentMode::Static {'],- ['HEAD /{}', 'if app.cfg.mode == DeploymentMode::Static {'],-]);--const MEDIA_PROXY_ASSET_PREFIXES = new Map([- ['external', 'path.strip_prefix("/external/")'],- ['attachments', 'path.starts_with("/attachments/")'],- ['themes', 'path.starts_with("/themes/")'],- ['entrance-sounds', '!= "entrance-sounds"'],- ['emojis', 'AssetKind::Emoji => "emojis"'],- ['stickers', 'AssetKind::Sticker => "stickers"'],- ['avatars', '"avatars" => AssetKind::Avatar'],- ['icons', '"icons" => AssetKind::GuildIcon'],- ['branding', '"branding" => AssetKind::GuildIcon'],- ['banners', '"banners" => AssetKind::Banner'],- ['splashes', '"splashes" => AssetKind::Splash'],- ['embed-splashes', '"embed-splashes" => AssetKind::EmbedSplash'],- ['guilds', 'fn parse_guild_member_asset_path'],-]);--interface SpecOperation {- readonly method: string;- readonly path: string;- readonly tags: ReadonlyArray<string>;- readonly security: ReadonlyArray<Record<string, Array<string>>> | null;-}--interface EffectiveSpecSecurity {- readonly schemes: ReadonlySet<string>;- readonly allowsAnonymous: boolean;-}--function indexDocumentedRoutes(routes: ReadonlyArray<DocsRouteHeader>): Map<string, DocsRouteHeader> {- const index = new Map<string, DocsRouteHeader>();- for (const route of routes) {- const key = routeShape(route.method, stripVersionPrefix(route.path));- const existing = index.get(key);- if (existing !== undefined) {- throw new Error(- `Duplicate documented route ${key}: ${existing.file}:${existing.line} and ${route.file}:${route.line}`,- );- }- index.set(key, route);- }- return index;-}--interface AliasRoute {- readonly shape: string;- readonly file: string;-}--const ALIAS_TABLE_HEADER = '| Method | Deprecated path | Successor |';-const ALIAS_ROW_PATTERN =- /^\|\s*(GET|HEAD|POST|PATCH|PUT|DELETE|OPTIONS)\s*\|\s*`(\/[^`]+)`\s*\|\s*\[[^\]]+\]\([^)]+\)\s*\|\s*$/u;--function aliasDocumentedRoutes(pages: ReadonlyArray<MarkdownPage>): Array<AliasRoute> {- const aliases: Array<AliasRoute> = [];- for (const {relativePath: file, lines} of pages) {- let inTable = false;- for (const line of lines) {- if (line.trim() === ALIAS_TABLE_HEADER) {- inTable = true;- continue;- }- if (inTable && line.trim().startsWith('| ---')) {- continue;- }- const row = inTable ? ALIAS_ROW_PATTERN.exec(line.trim()) : null;- if (row == null) {- inTable = false;- continue;- }- aliases.push({- shape: routeShape(row[1], stripVersionPrefix(row[2])),- file,- });- }- }- return aliases;-}--async function specOperations(specPath: string): Promise<Array<SpecOperation>> {- const spec: unknown = JSON.parse(await readFile(specPath, 'utf8'));- if (typeof spec !== 'object' || spec == null || !('paths' in spec)) {- throw new Error(`Spec has no paths: ${specPath}`);- }- const document = spec as OpenAPIDocument;- const operations: Array<SpecOperation> = [];- for (const [routePath, item] of Object.entries(document.paths)) {- for (const [method, operation] of Object.entries(item)) {- const upper = method.toUpperCase();- if (!HTTP_METHODS.has(upper)) {- continue;- }- operations.push({- method: upper,- path: stripVersionPrefix(routePath),- tags: operation.tags ?? [],- security: operation.security === undefined ? (document.security ?? null) : operation.security,- });- }- }- return operations;-}--function section(title: string, entries: ReadonlyArray<string>): number {- if (entries.length === 0) {- console.log(` ${title}: none`);- return 0;- }- console.log(` ${title}: ${entries.length.toString()}`);- for (const entry of entries) {- console.log(` ${entry}`);- }- return entries.length;-}--async function registrationSources(directory: string): Promise<Array<string>> {- const entries = await readdir(directory, {withFileTypes: true});- const files: Array<string> = [];- for (const entry of entries) {- const resolved = path.join(directory, entry.name);- if (entry.isDirectory()) {- if (entry.name === 'node_modules' || entry.name === 'tests') {- continue;- }- files.push(...(await registrationSources(resolved)));- continue;- }- if (entry.name.endsWith('.ts') && !entry.name.endsWith('.test.ts')) {- files.push(resolved);- }- }- return files;-}-const controllerRoutes = extractRoutesFromControllers(- await registrationSources(path.join(REPO_ROOT, 'fluxer_api/src')),-);--function constraintTail(constraint: string): string {- const body = constraint.slice(1, -1);- const tail = body.match(/(?:\\[^A-Za-z0-9]|[A-Za-z0-9_])+$/u);- if (tail?.index == null || !/[+*?})\]]$/u.test(body.slice(0, tail.index))) {- return `<constraint ${body}>`;- }- return tail[0].replace(/\\(.)/gu, '$1');-}-const astRoute = (route: {method: string; path: string}): string => {- const templated = route.path- .replace(- /:([A-Za-z_][A-Za-z0-9_]*)(\{(?:[^{}]|\{[^{}]*\})*\})?/gu,- (_match: string, name: string, constraint: string | undefined) =>- `{${name}}${constraint == null ? '' : constraintTail(constraint)}`,- )- .replace(/\*/gu, '{wildcard}');- return routeShape(route.method.toUpperCase(), stripVersionPrefix(templated));-};--const pages = await readMarkdownPages(DOCS_ROOT);-const documented = pages.flatMap(readRouteHeaders);-const aliasDocumented = aliasDocumentedRoutes(pages);-const aliasShapes = new Map<string, AliasRoute>();-for (const alias of aliasDocumented) {- const existing = aliasShapes.get(alias.shape);- if (existing !== undefined) {- throw new Error(`Duplicate deprecated alias ${alias.shape}: ${existing.file} and ${alias.file}`);- }- aliasShapes.set(alias.shape, alias);-}-const mediaProxySource = await (async () => {- const sources: Array<string> = [];- const walk = async (dir: string): Promise<void> => {- for (const entry of await readdir(dir, {withFileTypes: true})) {- const full = path.join(dir, entry.name);- if (entry.isDirectory()) {- await walk(full);- } else if (entry.name.endsWith('.rs')) {- sources.push(await readFile(full, 'utf8'));- }- }- };- await walk(MEDIA_PROXY_SERVER_DIR);- return sources.join('\n');-})();--let failures = 0;--console.log('source anchors');-const staleAnchors: Array<string> = [];-for (const [route, anchor] of [...MEDIA_PROXY_ROUTES, ...MEDIA_PROXY_ASSET_PREFIXES]) {- if (!mediaProxySource.includes(anchor)) {- staleAnchors.push(`${route}: fluxer_media_proxy/src/server no longer contains ${anchor}`);- }-}-const exemptSources = new Map<string, string>();-const sourceOf = async (file: string): Promise<string> => {- let source = exemptSources.get(file);- if (source == null) {- source = await readFile(path.join(REPO_ROOT, file), 'utf8');- exemptSources.set(file, source);- }- return source;-};-for (const [route, {file, anchor}] of MAIN_SPEC_EXEMPT) {- if (!(await sourceOf(file)).includes(anchor)) {- staleAnchors.push(`${route}: ${file} no longer contains ${anchor}`);- }-}-for (const rule of EXEMPTION_RULES) {- for (const {file, anchor} of rule.anchors) {- if (!(await sourceOf(file)).includes(anchor)) {- staleAnchors.push(`exemption rule "${rule.name}": ${file} no longer contains ${anchor}`);- }- }-}-for (const [route, {documentedIn}] of OUT_OF_BAND_CREDENTIAL) {- if (documentedIn == null) {- continue;- }- const page = await readFile(path.join(DOCS_ROOT, documentedIn.file), 'utf8');- if (!page.includes(documentedIn.anchor)) {- staleAnchors.push(`${route}: ${documentedIn.file} no longer contains ${documentedIn.anchor}`);- }-}-failures += section('stale anchors (the code moved, update this script)', staleAnchors);--const mediaProxyRouterSource = await readFile(path.join(MEDIA_PROXY_SERVER_DIR, 'runtime.rs'), 'utf8');-const mediaProxyRegisteredPaths = [...mediaProxyRouterSource.matchAll(/\.route\(\s*"([^"]+)"/gu)].map((match) =>- match[1].replace(/\{[^}]*\}/gu, '{}'),-);-const mediaProxyListedPaths = new Set([...MEDIA_PROXY_ROUTES.keys()].map((shape) => shape.split(' ')[1]));-const unlistedMediaProxyPaths = [...new Set(mediaProxyRegisteredPaths)]- .filter((routePath) => !mediaProxyListedPaths.has(routePath))- .map((routePath) => `${routePath}: build_router registers it and MEDIA_PROXY_ROUTES does not name it`)- .sort();-if (mediaProxyRegisteredPaths.length === 0) {- unlistedMediaProxyPaths.push(- 'fluxer_media_proxy/src/server/runtime.rs registers no route, so this check has gone blind',- );-}-failures += section('registered by fluxer_media_proxy but absent from this script', unlistedMediaProxyPaths);-console.log(- ` routes registered in fluxer_media_proxy/src/server/runtime.rs: ${mediaProxyRegisteredPaths.length.toString()}`,-);--const mediaProxyDocumented = documented.filter((route) => route.file.startsWith('media-proxy/'));-const adminDocumented = documented.filter(- (route) => !route.file.startsWith('media-proxy/') && stripVersionPrefix(route.path).startsWith('/admin'),-);-const mainDocumented = documented.filter(- (route) => !route.file.startsWith('media-proxy/') && !stripVersionPrefix(route.path).startsWith('/admin'),-);--const main = await specOperations(MAIN_SPEC);-const admin = await specOperations(ADMIN_SPEC);-const mainShapes = new Set(main.map((operation) => routeShape(operation.method, operation.path)));-const adminShapes = new Set(admin.map((operation) => routeShape(operation.method, operation.path)));--const documentedMain = indexDocumentedRoutes(mainDocumented);-const documentedAdmin = indexDocumentedRoutes(adminDocumented);--const registered = new Map<string, string>();-for (const route of controllerRoutes) {- const shape = astRoute(route);- if (!registered.has(shape)) {- registered.set(shape, `${path.relative(REPO_ROOT, route.controllerFile)}:${route.lineNumber.toString()}`);- }-}--console.log('main API and admin API (from route registration)');-{- const ruleCounts = new Map<string, number>();- for (const rule of EXEMPTION_RULES) {- ruleCounts.set(rule.name, 0);- }- const undocumented: Array<string> = [];- for (const [shape, site] of [...registered].sort()) {- if (documentedMain.has(shape) || documentedAdmin.has(shape) || aliasShapes.has(shape)) {- continue;- }- const rule = EXEMPTION_RULES.find((candidate) => candidate.covers(shape, shape.split(' ')[1]));- if (rule == null) {- undocumented.push(`${shape} registered at ${site}`);- continue;- }- ruleCounts.set(rule.name, (ruleCounts.get(rule.name) ?? 0) + 1);- }-- const documentedNotRegistered = [...documentedMain, ...documentedAdmin]- .filter(([shape]) => !registered.has(shape))- .map(([shape, route]) => `${shape} (${route.file}) is documented but fluxer_api registers no such route`)- .sort();- const deadRules = EXEMPTION_RULES.filter(- (rule) => rule.mayCoverNothing !== true && (ruleCounts.get(rule.name) ?? 0) === 0,- ).map((rule) => `the "${rule.name}" rule covers no registered route, so it is either stale or too narrow to matter`);- const staleEntries = [- ...[...DELIBERATELY_UNDOCUMENTED.keys()].map((shape) => [shape, 'DELIBERATELY_UNDOCUMENTED'] as const),- ...[...OUT_OF_BAND_CREDENTIAL.keys()].map((shape) => [shape, 'OUT_OF_BAND_CREDENTIAL'] as const),- ]- .filter(([shape]) => !registered.has(shape))- .map(([shape, list]) => `${list} names ${shape}, which fluxer_api no longer registers`)- .sort();-- const exemptTotal = [...ruleCounts.values()].reduce((a, b) => a + b, 0);- console.log(` registered routes: ${registered.size.toString()}`);- console.log(- ` documented with a RouteHeader: ${[...registered.keys()].filter((shape) => documentedMain.has(shape) || documentedAdmin.has(shape)).length.toString()}`,- );- console.log(- ` documented as a deprecated alias: ${[...registered.keys()].filter((shape) => aliasShapes.has(shape)).length.toString()}`,- );- console.log(` exempt by rule: ${exemptTotal.toString()}`);- for (const rule of EXEMPTION_RULES) {- console.log(` ${rule.name}: ${(ruleCounts.get(rule.name) ?? 0).toString()}. ${rule.justification}`);- }- console.log(' fluxer_admin is a Rust crate and its 83 HTML panel routes are NOT checked here. It serves');- console.log(' zero API operations, so the admin API above is the /admin routes fluxer_api registers.');- console.log(' fluxer_media_proxy and fluxer_app_proxy are Rust too. The media proxy is checked against');- console.log(' the hand-maintained list below, and the app proxy is not checked at all.');- failures += section('registered but neither documented nor covered by an exemption rule', undocumented);- failures += section('documented but not registered', documentedNotRegistered);- failures += section('exemption rules that cover nothing', deadRules);- failures += section('exemption entries naming a route that is gone', staleEntries);-}--console.log('main API (against the generated spec)');-failures += section(- 'documented but absent from the live spec',- [...documentedMain.entries()]- .filter(([shape]) => !mainShapes.has(shape) && !MAIN_SPEC_EXEMPT.has(shape))- .map(([, route]) => `${route.method} ${route.path} (${route.file})`)- .sort(),-);-failures += section(- 'present in the live spec but undocumented',- main- .filter((operation) => {- const shape = routeShape(operation.method, operation.path);- if (documentedMain.has(shape)) {- return false;- }- if (aliasShapes.has(shape)) {- return false;- }- return !DELIBERATELY_UNDOCUMENTED.has(shape);- })- .map((operation) => `${operation.method} ${operation.path} [${operation.tags.join(', ')}]`)- .sort(),-);-failures += section(- 'documented as a deprecated alias but not registered',- [...aliasShapes.values()]- .filter((alias) => !registered.has(alias.shape))- .map((alias) => `${alias.shape} (${alias.file}) names a route fluxer_api does not register`)- .sort(),-);-failures += section(- 'documented as a deprecated alias and with its own RouteHeader',- [...aliasShapes.keys()]- .filter((shape) => documentedMain.has(shape))- .map((shape) => `${shape} is an alias row and a RouteHeader, which double counts it`)- .sort(),-);-console.log(` documented in deprecated alias tables: ${aliasShapes.size.toString()}`);-const wronglyDocumented = [...DELIBERATELY_UNDOCUMENTED.entries()]- .filter(([shape]) => documentedMain.has(shape))- .map(([shape, reason]) => `${shape} must not be documented: ${reason}`);-failures += section('documented despite belonging to an unshipped system', wronglyDocumented);-console.log(- ` deliberately undocumented and correctly absent: ${(DELIBERATELY_UNDOCUMENTED.size - wronglyDocumented.length).toString()}/${DELIBERATELY_UNDOCUMENTED.size.toString()}`,-);-console.log(- ` exempt non-spec routes documented: ${[...MAIN_SPEC_EXEMPT.keys()].filter((shape) => documentedMain.has(shape)).length.toString()}/${MAIN_SPEC_EXEMPT.size.toString()}`,-);-for (const [shape, {reason}] of MAIN_SPEC_EXEMPT) {- console.log(` ${shape}: ${reason}`);-}--console.log('media proxy');-const mediaProxyProblems: Array<string> = [];-for (const route of mediaProxyDocumented) {- const shape = routeShape(route.method, route.path);- if (MEDIA_PROXY_ROUTES.has(shape)) {- continue;- }- const firstSegment = route.path.replace(/^\//u, '').split('/')[0].split('.')[0];- if (MEDIA_PROXY_ASSET_PREFIXES.has(firstSegment)) {- if (route.method !== 'GET' && route.method !== 'HEAD') {- mediaProxyProblems.push(`${route.method} ${route.path} (${route.file}) asset paths serve GET and HEAD only`);- }- continue;- }- mediaProxyProblems.push(`${route.method} ${route.path} (${route.file}) is not a route the media proxy serves`);-}-failures += section('documented but not served by fluxer_media_proxy', mediaProxyProblems);--console.log('gateway dispatch events');-{- const gatewayConstants = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/constants/Gateway.ts'), 'utf8');- const unionMatch = gatewayConstants.match(/export type GatewayDispatchEvent\s*=([\s\S]*?);/u);- const apiEvents = new Set<string>();- if (unionMatch != null) {- for (const entry of unionMatch[1].matchAll(/'([A-Z][A-Z0-9_]*)'/gu)) {- apiEvents.add(entry[1]);- }- }- const gatewayDirectory = path.join(REPO_ROOT, 'fluxer_gateway/src');- const erlangSources: Array<string> = [];- const collect = async (directory: string): Promise<void> => {- for (const entry of await readdir(directory, {withFileTypes: true})) {- const resolved = path.join(directory, entry.name);- if (entry.isDirectory()) {- await collect(resolved);- continue;- }- if (entry.name.endsWith('.erl') || entry.name.endsWith('.hrl')) {- erlangSources.push(await readFile(resolved, 'utf8'));- }- }- };- await collect(gatewayDirectory);- const erlang = erlangSources.join('\n');-- const eventsPage = [- await readFile(path.join(DOCS_ROOT, 'gateway/events.md'), 'utf8'),- await readFile(path.join(DOCS_ROOT, 'gateway/threads.md'), 'utf8'),- ].join('\n');- const documentedEvents = new Set<string>();- for (const entry of eventsPage.matchAll(/^###\s+(?:<span[^>]*><\/span>)?([A-Z][A-Z0-9_]{3,})\s*$/gmu)) {- documentedEvents.add(entry[1]);- }-- const isReal = (event: string): boolean => {- if (apiEvents.has(event)) {- return true;- }- if (erlang.includes(event)) {- return true;- }- return new RegExp(`\\b${event.toLowerCase()}\\b`, 'u').test(erlang);- };-- const fabricated = [...documentedEvents].filter((event) => !isReal(event)).sort();- const undocumented = [...apiEvents].filter((event) => !documentedEvents.has(event)).sort();- console.log(` events in the GatewayDispatchEvent union: ${apiEvents.size.toString()}`);- console.log(` events documented in gateway/events.md and gateway/threads.md: ${documentedEvents.size.toString()}`);- console.log(' a documented event counts as real if it is in the union, or appears in fluxer_gateway');- console.log(' as an uppercase binary or a lowercase atom');- failures += section('documented but not emitted by any service', fabricated);- failures += section('in the dispatch union but undocumented', undocumented);-}--console.log('gateway opcodes and close codes');-{- const erl = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/utils/constants.erl'), 'utf8');- const realOpcodes = new Set<number>();- for (const entry of erl.matchAll(/gateway_opcode\((\d+)\) -> [a-z_]+/gu)) {- realOpcodes.add(Number.parseInt(entry[1], 10));- }- const realCloseCodes = new Set<number>();- for (const entry of erl.matchAll(/close_code_to_num\([a-z_]+\) -> (\d+)/gu)) {- realCloseCodes.add(Number.parseInt(entry[1], 10));- }- const page = await readFile(path.join(DOCS_ROOT, 'gateway/opcodes-and-close-codes.md'), 'utf8');- const docOpcodes = new Set<number>();- const docCloseCodes = new Set<number>();- const strippedPage = page.replace(/<sup>.*?<\/sup>/gu, '');- for (const row of strippedPage.matchAll(/^\|\s*(\d{1,4})\s*\|/gmu)) {- const value = Number.parseInt(row[1], 10);- if (value >= 4000) {- docCloseCodes.add(value);- continue;- }- if (value <= 99) {- docOpcodes.add(value);- }- }- console.log(` opcodes: ${realOpcodes.size.toString()} real, ${docOpcodes.size.toString()} documented`);- console.log(` close codes: ${realCloseCodes.size.toString()} real, ${docCloseCodes.size.toString()} documented`);- failures += section(- 'opcodes documented but absent from constants.erl',- [...docOpcodes]- .filter((code) => !realOpcodes.has(code))- .sort((a, b) => a - b)- .map(String),- );- failures += section(- 'opcodes in constants.erl but undocumented',- [...realOpcodes]- .filter((code) => !docOpcodes.has(code))- .sort((a, b) => a - b)- .map(String),- );- failures += section(- 'close codes documented but absent from constants.erl',- [...docCloseCodes]- .filter((code) => !realCloseCodes.has(code))- .sort((a, b) => a - b)- .map(String),- );- failures += section(- 'close codes in constants.erl but undocumented',- [...realCloseCodes]- .filter((code) => !docCloseCodes.has(code))- .sort((a, b) => a - b)- .map(String),- );-}--console.log('enum names and error codes');-{- const liveTokens = new Set<string>();- const scanDirectory = async (directory: string): Promise<void> => {- let entries: Array<Dirent>;- try {- entries = await readdir(directory, {withFileTypes: true});- } catch {- return;- }- for (const entry of entries) {- if (entry.name === 'node_modules' || entry.name === 'target' || entry.name === 'dist') {- continue;- }- const resolved = path.join(directory, entry.name);- if (entry.isDirectory()) {- await scanDirectory(resolved);- continue;- }- if (!/\.(ts|rs|erl|hrl|json)$/u.test(entry.name)) {- continue;- }- const source = await readFile(resolved, 'utf8');- for (const token of source.matchAll(/[A-Z][A-Z0-9_]+/gu)) {- liveTokens.add(token[0]);- }- }- };- for (const sub of [- 'packages',- 'fluxer_api/src',- 'fluxer_admin/src',- 'fluxer_gateway/src',- 'fluxer_media_proxy/src',- ]) {- await scanDirectory(path.join(REPO_ROOT, sub));- }-- const enumRows: Array<{file: string; line: number; name: string}> = [];- const codeRows: Array<{file: string; line: number; code: string}> = [];- for (const {relativePath: relative, lines} of pages) {- let inEnumTable = false;- for (let i = 0; i < lines.length; i += 1) {- const line = lines[i];- if (/^\|\s*Value\s*\|\s*Name\s*\|/u.test(line)) {- inEnumTable = true;- continue;- }- if (inEnumTable && !line.startsWith('|')) {- inEnumTable = false;- }- if (inEnumTable && !line.startsWith('| ---')) {- const row = line.match(/^\|\s*-?\d+\s*\|\s*([A-Z][A-Z0-9_]*)/u);- if (row != null) {- enumRows.push({file: relative, line: i + 1, name: row[1]});- }- }- for (const cited of line.matchAll(/\b[1-5]\d\d\s+`([A-Z][A-Z0-9_]{2,})`/gu)) {- codeRows.push({file: relative, line: i + 1, code: cited[1]});- }- }- }-- const fabricatedEnums = enumRows.filter((row) => !liveTokens.has(row.name));- const fabricatedCodes = codeRows.filter((row) => !liveTokens.has(row.code));- console.log(` enum rows checked: ${enumRows.length.toString()}`);- console.log(` error codes cited with a status: ${new Set(codeRows.map((r) => r.code)).size.toString()}`);- console.log(' names only, not value bindings: the same name is reused across enums, so a global');- console.log(' value comparison misattributes. ReportType.GUILD is 2 while ChannelContext.GUILD is 0');- failures += section(- 'enum names absent from live source',- fabricatedEnums.map((row) => `${row.file}:${row.line.toString()} ${row.name}`).sort(),- );- failures += section(- 'error codes absent from live source',- [...new Set(fabricatedCodes.map((row) => `${row.file}:${row.line.toString()} ${row.code}`))].sort(),- );-}--console.log('registry codes with a producer');-{- const CODE_REGISTRIES = ['packages/constants/src/ApiErrorCodes.ts', 'packages/constants/src/ValidationErrorCodes.ts'];- const NON_PRODUCERS = [...CODE_REGISTRIES, 'packages/errors/src/i18n/ErrorCodeMappings.ts'];- const registryKeys = new Map<string, string>();- for (const relative of CODE_REGISTRIES) {- const source = await readFile(path.join(REPO_ROOT, relative), 'utf8');- for (const entry of source.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {- registryKeys.set(entry[1], relative);- }- }-- const producerTokens = new Set<string>();- const scanProducers = async (directory: string): Promise<void> => {- let entries: Array<Dirent>;- try {- entries = await readdir(directory, {withFileTypes: true});- } catch {- return;- }- for (const entry of entries) {- if (entry.name === 'node_modules' || entry.name === 'target' || entry.name === 'dist' || entry.name === 'tests') {- continue;- }- const resolved = path.join(directory, entry.name);- if (entry.isDirectory()) {- await scanProducers(resolved);- continue;- }- if (!/\.(ts|rs|erl|hrl)$/u.test(entry.name) || entry.name.endsWith('.test.ts')) {- continue;- }- if (NON_PRODUCERS.includes(path.relative(REPO_ROOT, resolved))) {- continue;- }- const source = await readFile(resolved, 'utf8');- for (const token of source.matchAll(/\b[A-Z][A-Z0-9_]+\b/gu)) {- producerTokens.add(token[0]);- }- }- };- for (const sub of [- 'packages',- 'fluxer_api/src',- 'fluxer_admin/src',- 'fluxer_gateway/src',- 'fluxer_media_proxy/src',- 'fluxer_unfurl/src',- ]) {- await scanProducers(path.join(REPO_ROOT, sub));- }-- const withoutProducer = [...registryKeys]- .filter(([code]) => !producerTokens.has(code))- .map(([code, relative]) => `${relative} ${code}`)- .sort();- console.log(` registry codes checked: ${registryKeys.size.toString()}`);- console.log(' a producer is any live server file outside the two registries, ErrorCodeMappings.ts and the');- console.log(' generated specs. Test files do not count, the api test harness controller does');- failures += section('registry codes with no producer', withoutProducer);-}--console.log('admin audit actions');-{- const registrySource = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/admin/AdminAuditActions.ts'), 'utf8');- const registryBlock = registrySource.match(/export const AdminAuditReadActions = \{([\s\S]*?)\} as const;/u);- const readActions = new Set([...(registryBlock?.[1] ?? '').matchAll(/:\s*'([^']+)'/gu)].map((match) => match[1]));-- const recorded = new Map<string, string>();- const scanAuditSources = async (directory: string, everyFile: boolean): Promise<void> => {- for (const entry of await readdir(directory, {withFileTypes: true})) {- if (entry.name === 'node_modules' || entry.name === 'tests') continue;- const resolved = path.join(directory, entry.name);- if (entry.isDirectory()) {- await scanAuditSources(resolved, everyFile);- continue;- }- if (!entry.name.endsWith('.ts') || entry.name.endsWith('.test.ts')) continue;- const source = await readFile(resolved, 'utf8');- if (!everyFile && !source.includes('createAuditLog(')) continue;- for (const assignment of source.matchAll(/\baction\s*[:=]\s*([^,;\n]*(?:\n\s*[?:][^,;\n]*)*)/gu)) {- const expression = assignment[1];- const value = expression.includes('?') ? expression.slice(expression.indexOf('?') + 1) : expression;- for (const literal of value.matchAll(/'([^']+)'/gu)) {- if (!recorded.has(literal[1])) recorded.set(literal[1], path.relative(REPO_ROOT, resolved));- }- }- }- };- await scanAuditSources(path.join(REPO_ROOT, 'fluxer_api/src/api/admin'), true);- await scanAuditSources(path.join(REPO_ROOT, 'fluxer_api/src/api/worker/tasks'), true);- await scanAuditSources(path.join(REPO_ROOT, 'fluxer_api/src'), false);- for (const action of readActions) {- if (!recorded.has(action)) recorded.set(action, 'fluxer_api/src/api/admin/AdminAuditActions.ts');- }-- const indexPage = await readFile(path.join(DOCS_ROOT, 'admin-api/index.mdx'), 'utf8');- const actionTable = (heading: string): Set<string> => {- const start = indexPage.indexOf(`\n${heading}\n`);- if (start === -1) return new Set();- const rest = indexPage.slice(start + heading.length + 2);- const end = rest.search(/\n#{1,4} /u);- const values = new Set<string>();- for (const line of (end === -1 ? rest : rest.slice(0, end)).split('\n')) {- const cell = line- .match(/^\|\s*([^|]+?)\s*\|/u)?.[1]- ?.replace(/<sup>.*?<\/sup>/gu, '')- .trim();- if (cell == null || cell === 'Value' || /^-+$/u.test(cell)) continue;- values.add(cell);- }- return values;- };- const documentedRead = actionTable('#### Read actions');- const documentedWrite = actionTable('#### Write actions');- console.log(- ` actions recorded by fluxer_api: ${recorded.size.toString()}, of which read: ${readActions.size.toString()}`,- );- console.log(- ` actions documented in admin-api/index.mdx: read ${documentedRead.size.toString()}, write ${documentedWrite.size.toString()}`,- );- const emptyParses = [- ...(readActions.size === 0 ? ['the read action registry in AdminAuditActions.ts parsed empty'] : []),- ...(recorded.size <= readActions.size ? ['no write action was found in fluxer_api source'] : []),- ...(documentedRead.size === 0 ? ['the Read actions table parsed empty'] : []),- ...(documentedWrite.size === 0 ? ['the Write actions table parsed empty'] : []),- ];- failures += section('audit action parses that came back empty (update this script)', emptyParses);- failures += section(- 'recorded but undocumented',- [...recorded]- .filter(([action]) => !documentedRead.has(action) && !documentedWrite.has(action))- .map(([action, file]) => `${action} (${file})`)- .sort(),- );- failures += section(- 'documented but never recorded',- [...documentedRead, ...documentedWrite].filter((action) => !recorded.has(action)).sort(),- );- failures += section(- 'documented under the wrong access',- [- ...[...documentedRead].filter((action) => !readActions.has(action)).map((action) => `${action}: listed as read`),- ...[...documentedWrite].filter((action) => readActions.has(action)).map((action) => `${action}: listed as write`),- ].sort(),- );-}--console.log('permission bits');-{- const constants = await readFile(path.join(REPO_ROOT, 'packages/constants/src/ChannelConstants.ts'), 'utf8');- const block = constants.match(/export const Permissions = \{([\s\S]*?)\} as const;/u);- const livePermissions = new Map<string, number>();- const threadConstants = await readFile(- path.join(REPO_ROOT, 'packages/constants/src/ThreadPermissionUtils.ts'),- 'utf8',- );- const threadBlock = threadConstants.match(/export const ThreadPermissionFlags = \{([\s\S]*?)\} as const;/u);- for (const source of [block, threadBlock]) {- if (source == null) {- continue;- }- for (const entry of source[1].matchAll(/([A-Z][A-Z0-9_]*):\s*1n\s*<<\s*(\d+)n/gu)) {- livePermissions.set(entry[1], Number.parseInt(entry[2], 10));- }- }- if (threadBlock == null) {- failures += section('ThreadPermissionFlags block not found in ThreadPermissionUtils.ts', ['parser drift']);- }- const page = (await readFile(path.join(DOCS_ROOT, 'http-api/permissions.mdx'), 'utf8'))- .replace(/</gu, '<')- .replace(/>/gu, '>');- const documentedPermissions = new Map<string, number>();- for (const row of page.matchAll(/^\|\s*1\s*<<\s*(\d+)\s*\|\s*([A-Z][A-Z0-9_]*)/gmu)) {- documentedPermissions.set(row[2], Number.parseInt(row[1], 10));- }- console.log(` live permissions: ${livePermissions.size.toString()}`);- console.log(` documented permissions: ${documentedPermissions.size.toString()}`);- const bitMismatches: Array<string> = [];- for (const [name, bit] of documentedPermissions) {- const liveBit = livePermissions.get(name);- if (liveBit != null && liveBit !== bit) {- bitMismatches.push(`${name}: documented 1 << ${bit.toString()}, live 1 << ${liveBit.toString()}`);- }- }- failures += section('permission bit mismatches', bitMismatches);- failures += section(- 'documented but not a live permission',- [...documentedPermissions.keys()].filter((name) => !livePermissions.has(name)).sort(),- );- failures += section(- 'live permission but undocumented',- [...livePermissions.keys()].filter((name) => !documentedPermissions.has(name)).sort(),- );-}--console.log('rate limit buckets, limits and windows');-{- const UNIT_MS = new Map([- ['second', 1000],- ['seconds', 1000],- ['minute', 60_000],- ['minutes', 60_000],- ['hour', 3_600_000],- ['hours', 3_600_000],- ['day', 86_400_000],- ['days', 86_400_000],- ]);- const toMs = (text: string): number | null => {- const parsed = text.trim().match(/^(\d+)?\s*([a-z]+)$/u);- if (parsed == null) {- return null;- }- const unit = UNIT_MS.get(parsed[2]);- if (unit == null) {- return null;- }- return Number.parseInt(parsed[1] ?? '1', 10) * unit;- };-- const configDirectory = path.join(REPO_ROOT, 'fluxer_api/src/api/rate_limit_configs');- const liveBuckets = new Map<string, {limit: number; windowMs: number | null}>();- for (const entry of await readdir(configDirectory)) {- if (!entry.endsWith('.ts')) {- continue;- }- const source = await readFile(path.join(configDirectory, entry), 'utf8');- for (const config of source.matchAll(- /bucket:\s*'([^']+)',\s*config:\s*\{limit:\s*(\d+),\s*windowMs:\s*ms\('([^']+)'\)/gu,- )) {- liveBuckets.set(config[1], {limit: Number.parseInt(config[2], 10), windowMs: toMs(config[3])});- }- }-- const claimPattern =- /([\d,]+) requests? per ([a-z0-9 ]+?)(?:,| for [^.]*?,) on the (?:shared )?`([a-z0-9_:@{}]+)` bucket/gu;- const problems: Array<string> = [];- let claims = 0;- for (const {relativePath: relative, lines} of pages) {- for (let i = 0; i < lines.length; i += 1) {- for (const claim of lines[i].matchAll(claimPattern)) {- claims += 1;- const bucket = claim[3];- const limit = Number.parseInt(claim[1].replace(/,/gu, ''), 10);- const windowMs = toMs(claim[2]);- const live = liveBuckets.get(bucket);- if (live == null) {- problems.push(`${relative}:${(i + 1).toString()} bucket ${bucket} is not in any rate limit config`);- continue;- }- if (live.limit !== limit) {- problems.push(- `${relative}:${(i + 1).toString()} ${bucket} documented limit ${limit.toString()}, live ${live.limit.toString()}`,- );- }- if (windowMs != null && live.windowMs != null && live.windowMs !== windowMs) {- problems.push(- `${relative}:${(i + 1).toString()} ${bucket} documented window ${windowMs.toString()}ms, live ${live.windowMs.toString()}ms`,- );- }- }- }- }- console.log(` buckets defined with a limit and window: ${liveBuckets.size.toString()}`);- console.log(` documented rate limit claims parsed: ${claims.toString()}`);- failures += section('rate limit disagreements', problems);-}--console.log('instance limit keys');-{- const defaults = await readFile(path.join(REPO_ROOT, 'packages/limits/src/LimitDefaults.ts'), 'utf8');- const liveKeys = new Set<string>();- for (const entry of defaults.matchAll(/^\s+([a-z][a-z0-9_]*):/gmu)) {- liveKeys.add(entry[1]);- }- const instancePage = await readFile(path.join(DOCS_ROOT, 'http-api/instance.mdx'), 'utf8');- const documentedKeys = new Set<string>();- for (const row of instancePage.matchAll(/^\|\s*([^|]+?)\s*\|/gmu)) {- const name = row[1]- .replace(/<sup>.*?<\/sup>/gu, '')- .replace(/`/gu, '')- .trim();- if (/^[a-z][a-z0-9_]*$/u.test(name)) {- documentedKeys.add(name);- }- }- console.log(` live limit keys: ${liveKeys.size.toString()}`);- console.log(- ` documented in the instance limit-keys table: ${[...liveKeys].filter((k) => documentedKeys.has(k)).length.toString()}`,- );- failures += section(- 'live limit keys missing from the instance table',- [...liveKeys].filter((key) => !documentedKeys.has(key)).sort(),- );-}--console.log('media proxy image constants');-{- const constants = await readFile(path.join(REPO_ROOT, 'fluxer_media_proxy/src/constants.rs'), 'utf8');- const ladderBlock = constants.match(/pub const IMAGE_SIZES: &\[u32\] = &\[([\s\S]*?)\];/u);- const ladder = ladderBlock == null ? [] : [...ladderBlock[1].matchAll(/\d+/gu)].map((m) => Number.parseInt(m[0], 10));- const defaultSize = constants.match(/pub const DEFAULT_IMAGE_SIZE: u32 = (\d+);/u);- const dimsBlock = constants.match(/pub fn dims_for\(kind: AssetKind\) -> Dims \{([\s\S]*?)\n\}/u);- const dims: Array<[string, number, number]> = [];- if (dimsBlock != null) {- for (const entry of dimsBlock[1].matchAll(- /((?:AssetKind::\w+\s*\|?\s*)+)=> Dims \{\s*min:\s*(\d+),\s*max:\s*(\d+),?\s*\}/gu,- )) {- dims.push([entry[1].trim(), Number.parseInt(entry[2], 10), Number.parseInt(entry[3], 10)]);- }- }-- const LADDER_PAGE = 'media-proxy/transformations.md';-- const ladderPage = (await readFile(path.join(DOCS_ROOT, LADDER_PAGE), 'utf8')).replace(/(?<=\d),(?=\d)/gu, '');- const problems: Array<string> = [];-- const documentedLadder = ladderPage.match(- /`size` selects from a fixed ladder: ([\d,\s]+(?:and\s+\d+)?)\. A requested value snaps up/u,- );- if (documentedLadder == null) {- problems.push(`${LADDER_PAGE} no longer states the size ladder in the expected sentence`);- } else {- const listed = [...documentedLadder[1].matchAll(/\d+/gu)].map((m) => Number.parseInt(m[0], 10));- if (listed.join(',') !== ladder.join(',')) {- problems.push(`size ladder differs. documented [${listed.join(', ')}] vs IMAGE_SIZES [${ladder.join(', ')}]`);- }- }-- if (dims.length === 0) {- problems.push('no asset class clamps parsed from dims_for in fluxer_media_proxy/src/constants.rs');- }-- if (defaultSize != null && !ladderPage.includes(`resolves to ${defaultSize[1]} before clamping`)) {- problems.push(`${LADDER_PAGE} does not state that an absent size resolves to ${defaultSize[1]} before clamping`);- }-- for (const [kinds, min, max] of dims) {- if (!ladderPage.includes(`${min.toString()} through ${max.toString()}`)) {- problems.push(`${LADDER_PAGE} does not state the ${kinds} clamp of ${min.toString()} to ${max.toString()}`);- }- }-- console.log(` size ladder rungs in constants.rs: ${ladder.length.toString()}`);- console.log(` asset classes with a clamp: ${dims.length.toString()}`);- failures += section('media proxy constant disagreements', problems);-}--console.log('self-hosting guide against deploy/self-hosting');-{- const envExample = await readFile(path.join(REPO_ROOT, 'deploy/self-hosting/.env.example'), 'utf8');- const guide = await readFile(path.join(DOCS_ROOT, 'operator/get-started.mdx'), 'utf8');-- const placeholders = new Set<string>();- const declared = new Set<string>();- const assigned = new Set<string>();- for (const line of envExample.split('\n')) {- const commented = line.startsWith('#');- const assignment = (commented ? line.slice(1) : line).match(/^([A-Z][A-Z0-9_]*)=(.*)$/u);- if (assignment == null) {- continue;- }- declared.add(assignment[1]);- if (commented) {- continue;- }- assigned.add(assignment[1]);- if (assignment[2].includes('CHANGE_ME')) {- placeholders.add(assignment[1]);- }- }-- const INSTALLER_ROOT = fileURLToPath(new URL('../src/installer/', import.meta.url));- const installers: ReadonlyArray<readonly [string, string]> = [- ['install.sh', await readFile(path.join(INSTALLER_ROOT, 'install.sh'), 'utf8')],- ['install.ps1', await readFile(path.join(INSTALLER_ROOT, 'install.ps1'), 'utf8')],- ];- const shellSource = installers[0][1];- const powershellSource = installers[1][1];-- const problems: Array<string> = [];-- const INSTALLER_SECRET_FLOOR = 14;- const INSTALLER_NON_SECRET_FLOOR = 7;- const COMPOSE_REQUIRED_FLOOR = 17;-- const blockBetween = (source: string, opener: string, closer: string, label: string): string | null => {- const from = source.indexOf(opener);- if (from === -1) {- problems.push(`${label} is gone. Nothing in the script opens with \`${opener.trim()}\``);- return null;- }- const rest = source.slice(from + opener.length);- const to = rest.indexOf(closer);- if (to === -1) {- problems.push(`${label} opens but never closes at \`${closer.trim()}\``);- return null;- }- return rest.slice(0, to);- };-- const shellRows = (fn: string, delimiter: string, label: string): Array<string> => {- const body = blockBetween(shellSource, `${fn}() {\n\tcat <<'${delimiter}'\n`, `\n${delimiter}\n`, label);- if (body == null) {- return [];- }- return body.split('\n').filter((row) => row.trim().length > 0);- };-- const powershellRows = (variable: string, label: string): Array<string> => {- const body = blockBetween(powershellSource, `$${variable} = @(\n`, '\n)\n', label);- if (body == null) {- return [];- }- return body.split('\n').filter((row) => row.trim().length > 0);- };-- const parseShellKeys = (fn: string, label: string): Map<string, string> => {- const keys = new Map<string, string>();- for (const row of shellRows(fn, 'KEYS', label)) {- const parsed = row.match(/^([A-Z][A-Z0-9_]*) ([a-z0-9_]+)(?: .*)?$/u);- if (parsed == null) {- problems.push(`${label} has the row \`${row}\`, which is not "NAME kind"`);- continue;- }- keys.set(parsed[1], parsed[2]);- }- return keys;- };-- const parsePowerShellKeys = (variable: string, label: string): Map<string, string> => {- const keys = new Map<string, string>();- for (const row of powershellRows(variable, label)) {- const parsed = row.match(/^\s*@\{Name = '([A-Z][A-Z0-9_]*)'; Kind = '([a-z0-9_]+)'/u);- if (parsed == null) {- problems.push(`${label} has the row \`${row.trim()}\`, which is not an @{Name; Kind} entry`);- continue;- }- keys.set(parsed[1], parsed[2]);- }- return keys;- };-- const shellSecrets = parseShellKeys('fluxer_secret_keys', 'the install.sh secret list');- const shellNonSecrets = parseShellKeys('fluxer_non_secret_keys', 'the install.sh non-secret list');- const powershellSecrets = parsePowerShellKeys('FluxerSecretKeys', 'the install.ps1 secret list');- const powershellNonSecrets = parsePowerShellKeys('FluxerNonSecretKeys', 'the install.ps1 non-secret list');-- const shellWrites = new Set<string>([...shellSecrets.keys(), ...shellNonSecrets.keys()]);- const powershellWrites = new Set<string>([...powershellSecrets.keys(), ...powershellNonSecrets.keys()]);-- for (const [name, secrets, nonSecrets] of [- ['install.sh', shellSecrets, shellNonSecrets],- ['install.ps1', powershellSecrets, powershellNonSecrets],- ] as const) {- if (secrets.size < INSTALLER_SECRET_FLOOR) {- problems.push(- `${name} declares ${secrets.size.toString()} secret keys, floor is ${INSTALLER_SECRET_FLOOR.toString()}`,- );- }- if (nonSecrets.size < INSTALLER_NON_SECRET_FLOOR) {- problems.push(- `${name} declares ${nonSecrets.size.toString()} non-secret keys, floor is ${INSTALLER_NON_SECRET_FLOOR.toString()}`,- );- }- }-- for (const name of placeholders) {- for (const [script, secrets] of [- ['install.sh', shellSecrets],- ['install.ps1', powershellSecrets],- ] as const) {- if (!secrets.has(name)) {- problems.push(`${name} ships as CHANGE_ME but ${script} never sets it`);- }- }- }-- for (const name of new Set<string>([...shellWrites, ...powershellWrites])) {- if (!declared.has(name)) {- problems.push(`the installer writes ${name}, which is not declared in .env.example`);- }- }-- const compareKeyLists = (label: string, shell: Map<string, string>, powershell: Map<string, string>): void => {- for (const [name, kind] of shell) {- const other = powershell.get(name);- if (other == null) {- problems.push(`install.sh writes the ${label} ${name} and install.ps1 does not`);- continue;- }- if (other !== kind) {- problems.push(`${name} is ${kind} in install.sh and ${other} in install.ps1`);- }- }- for (const name of powershell.keys()) {- if (!shell.has(name)) {- problems.push(`install.ps1 writes the ${label} ${name} and install.sh does not`);- }- }- };- compareKeyLists('secret', shellSecrets, powershellSecrets);- compareKeyLists('non-secret value', shellNonSecrets, powershellNonSecrets);-- const shellUpgrade = parseShellKeys('fluxer_upgrade_secret_keys', 'the install.sh upgrade secret list');- const powershellUpgrade = parsePowerShellKeys('FluxerUpgradeSecretKeys', 'the install.ps1 upgrade secret list');- compareKeyLists('upgrade secret', shellUpgrade, powershellUpgrade);-- for (const [script, upgrade, secrets] of [- ['install.sh', shellUpgrade, shellSecrets],- ['install.ps1', powershellUpgrade, powershellSecrets],- ] as const) {- if (upgrade.size === 0) {- problems.push(`${script} declares no upgrade secret keys, so an upgrade mints nothing`);- }- for (const [name, kind] of upgrade) {- const asInstalled = secrets.get(name);- if (asInstalled == null) {- problems.push(`${script} mints ${name} on an upgrade and never writes it on an install`);- continue;- }- if (asInstalled !== kind) {- problems.push(`${name} is ${kind} on an upgrade and ${asInstalled} on an install in ${script}`);- }- }- }-- const expectedSecretKind = (name: string): string => {- if (name === 'FLUXER_VAPID_PUBLIC_KEY') {- return 'vapid_public';- }- if (name === 'FLUXER_VAPID_PRIVATE_KEY') {- return 'vapid_private';- }- if (name.endsWith('_BASE64')) {- return 'base64';- }- return 'hex';- };- for (const [script, secrets] of [- ['install.sh', shellSecrets],- ['install.ps1', powershellSecrets],- ] as const) {- for (const [name, kind] of secrets) {- const wanted = expectedSecretKind(name);- if (kind !== wanted) {- problems.push(`${script} generates ${name} as ${kind}, and the value has to be ${wanted}`);- }- }- }-- const shippedAssets = (await readdir(path.join(REPO_ROOT, 'deploy/self-hosting')))- .filter((entry) => entry !== '.gitignore')- .sort();- const shellStackFiles = shellRows('fluxer_stack_files', 'FILES', 'the install.sh download list');- const powershellStackFiles: Array<string> = [];- for (const row of powershellRows('FluxerStackFiles', 'the install.ps1 download list')) {- const parsed = row.match(/^\s*'([^']+)'\s*$/u);- if (parsed == null) {- problems.push(`the install.ps1 download list has \`${row.trim()}\`, which is not a quoted file name`);- continue;- }- powershellStackFiles.push(parsed[1]);- }- for (const [script, list] of [- ['install.sh', shellStackFiles],- ['install.ps1', powershellStackFiles],- ] as const) {- const sorted = [...list].sort();- if (sorted.join(', ') !== shippedAssets.join(', ')) {- problems.push(- `${script} downloads [${sorted.join(', ')}] and deploy/self-hosting ships [${shippedAssets.join(', ')}]`,- );- }- }-- const COMPOSE_DEFAULT_NAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yml', 'docker-compose.yaml'];- const shellComposeNames = shellRows('fluxer_compose_names', 'NAMES', 'the install.sh compose name list');- const powershellComposeNames: Array<string> = [];- for (const row of powershellRows('FluxerComposeNames', 'the install.ps1 compose name list')) {- const parsed = row.match(/^\s*'([^']+)'\s*$/u);- if (parsed == null) {- problems.push(`the install.ps1 compose name list has \`${row.trim()}\`, which is not a quoted file name`);- continue;- }- powershellComposeNames.push(parsed[1]);- }- for (const [script, list] of [- ['install.sh', shellComposeNames],- ['install.ps1', powershellComposeNames],- ] as const) {- if (list.join(', ') !== COMPOSE_DEFAULT_NAMES.join(', ')) {- problems.push(- `${script} resolves compose files as [${list.join(', ')}] and Compose resolves them as [${COMPOSE_DEFAULT_NAMES.join(', ')}]`,- );- }- }- if (!shellStackFiles.includes('docker-compose.yml') || !COMPOSE_DEFAULT_NAMES.includes('docker-compose.yml')) {- problems.push(- 'the installers no longer download docker-compose.yml, so the name they map onto an instance is unclear',- );- }-- const PIPE_TO_SHELL =- /(?:curl|wget|iwr|Invoke-WebRequest)[^\n|]*\|\s*(?:sudo\s+)?(?:sh|bash|zsh|iex|Invoke-Expression)\b/iu;- for (const [name, source] of installers) {- if (PIPE_TO_SHELL.test(source)) {- problems.push(`${name} pipes a download into a shell`);- }- }- for (const {relativePath, source} of pages) {- if (PIPE_TO_SHELL.test(source)) {- problems.push(`${relativePath} pipes a download into a shell`);- }- }-- for (const [name, source] of installers) {- if (source.includes('—')) {- problems.push(`${name} contains an em dash`);- }- if (source.includes('\r')) {- problems.push(`${name} contains a CR byte, so the served digest would disagree with sha256sum on the file`);- }- if (!source.endsWith('\n') || source.endsWith('\n\n')) {- problems.push(`${name} does not end with exactly one newline`);- }- }-- for (const [name, source] of installers) {- const published = installerChecksumLine(name, source);- const expected = `${createHash('sha256').update(Buffer.from(source, 'utf8')).digest('hex')} ${name}\n`;- if (published !== expected) {- problems.push(- `the published checksum for ${name} is \`${published.trim()}\` and sha256 of the file is \`${expected.trim()}\``,- );- }- }-- const INSTALLER_ENDPOINTS = ['install.sh.ts', 'install.sh.sha256.ts', 'install.ps1.ts', 'install.ps1.sha256.ts'];- const digestBearing: Array<[string, string]> = [];- for (const endpoint of INSTALLER_ENDPOINTS) {- const source = await readFile(fileURLToPath(new URL(`../src/pages/${endpoint}`, import.meta.url)), 'utf8');- if (!source.includes("from '@/installer/Installer'")) {- problems.push(`src/pages/${endpoint} no longer serves the script through src/installer/Installer`);- }- digestBearing.push([`src/pages/${endpoint}`, source]);- }- digestBearing.push([- 'src/components/InstallerChecksum.astro',- await readFile(fileURLToPath(new URL('../src/components/InstallerChecksum.astro', import.meta.url)), 'utf8'),- ]);- for (const {relativePath, source} of pages) {- if (source.includes('install.sh') || source.includes('install.ps1')) {- digestBearing.push([relativePath, source]);- }- }- for (const [where, text] of digestBearing) {- if (/\b[0-9a-f]{64}\b/u.test(text)) {- problems.push(- `${where} contains a literal 64-character hex digest, which goes stale the next time a script changes`,- );- }- }-- if (options['source-only']) {- console.log(' installer execution skipped (--source-only)');- } else {- problems.push(...(await verifyInstallerExecution(INSTALLER_ROOT)));- }-- const covered = new Set<string>(- [...placeholders].filter((name) => shellSecrets.has(name) && powershellSecrets.has(name)),- );-- const compose = await readFile(path.join(REPO_ROOT, 'deploy/self-hosting/docker-compose.yml'), 'utf8');- const composeLines = compose.split('\n');-- const composeRequired = new Set<string>();- for (const required of compose.matchAll(/\$\{([A-Z][A-Z0-9_]*):\?/gu)) {- composeRequired.add(required[1]);- }- if (composeRequired.size < COMPOSE_REQUIRED_FLOOR) {- problems.push(- `docker-compose.yml parsed to ${composeRequired.size.toString()} :?-required variables, floor is ${COMPOSE_REQUIRED_FLOOR.toString()}`,- );- }- for (const required of compose.matchAll(/\$\{([A-Z][A-Z0-9_]*):\?([^}]*)\}/gu)) {- if (required[2].trim().length === 0) {- problems.push(- `docker-compose.yml requires ${required[1]} with an empty :? message, so Compose stops on a bare variable name`,- );- }- }- const vapidEmailDerivation = 'FLUXER_VAPID_EMAIL: ${FLUXER_VAPID_EMAIL:-admin@${FLUXER_DOMAIN}}';- if (!compose.includes(vapidEmailDerivation)) {- problems.push('docker-compose.yml no longer derives FLUXER_VAPID_EMAIL from FLUXER_DOMAIN');- } else if (assigned.has('FLUXER_VAPID_EMAIL')) {- problems.push(- '.env.example assigns FLUXER_VAPID_EMAIL, so a copied .env defeats the admin@FLUXER_DOMAIN derivation',- );- }- let composeRequiredCovered = 0;- for (const name of composeRequired) {- const missing: Array<string> = [];- if (!shellWrites.has(name)) {- missing.push('install.sh');- }- if (!powershellWrites.has(name)) {- missing.push('install.ps1');- }- if (missing.length === 0) {- composeRequiredCovered += 1;- continue;- }- problems.push(`docker-compose.yml requires ${name} with :? and ${missing.join(' and ')} never writes it`);- }- const publishers = new Set<string>();- const publishedPorts = new Set<string>();- let currentService: string | null = null;- for (let i = 0; i < composeLines.length; i += 1) {- const service = composeLines[i].match(/^ {2}([a-z][a-z0-9_-]*):\s*$/u);- if (service != null) {- currentService = service[1];- continue;- }- if (!/^ {4}ports:\s*$/u.test(composeLines[i]) || currentService == null) {- continue;- }- publishers.add(currentService);- for (let j = i + 1; j < composeLines.length && /^\s*-\s/u.test(composeLines[j]); j += 1) {- const mapping = composeLines[j].match(/(\d+)\}?(\/udp)?"\s*$/u);- if (mapping != null) {- publishedPorts.add(`${mapping[1]}${mapping[2] ?? ''}`);- }- }- }- if (!guide.includes('The edge and LiveKit are the only services that publish ports')) {- problems.push('the guide no longer states which services publish ports');- } else if (publishers.size !== 2 || !publishers.has('edge') || !publishers.has('livekit')) {- problems.push(- `the guide says only the edge and LiveKit publish ports, but the compose file publishes from [${[...publishers].sort().join(', ')}]`,- );- }- for (const port of publishedPorts) {- const bare = port.replace('/udp', '');- if (!guide.includes(bare)) {- problems.push(`the compose file publishes ${port} but the guide never mentions ${bare}`);- }- }-- const livekitConfig = blockBetween(- compose,- ' LIVEKIT_CONFIG: |\n',- '\n ports:\n',- "the livekit service's LIVEKIT_CONFIG",- );- if (livekitConfig != null) {- const coupled: ReadonlyArray<readonly [string, string]> = [- ['tcp_port', '${FLUXER_LIVEKIT_TCP_PORT:-7881}'],- ['udp_port', '${FLUXER_LIVEKIT_UDP_PORT:-7882}'],- ['api_key', '${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}'],- ];- for (const [key, expression] of coupled) {- if (!livekitConfig.includes(`${key}: ${expression}`)) {- problems.push(`LIVEKIT_CONFIG sets ${key} to something other than ${expression}, so it no longer follows .env`);- }- }- for (const mapping of [- '${FLUXER_LIVEKIT_TCP_PORT:-7881}:${FLUXER_LIVEKIT_TCP_PORT:-7881}',- '${FLUXER_LIVEKIT_UDP_PORT:-7882}:${FLUXER_LIVEKIT_UDP_PORT:-7882}/udp',- ]) {- if (!compose.includes(`"${mapping}"`)) {- problems.push(- `the livekit service no longer publishes ${mapping}, so the port it advertises in ICE candidates is not the one the host forwards`,- );- }- }- }-- const configPage = await readFile(path.join(DOCS_ROOT, 'operator/configuration.mdx'), 'utf8');- const configLines = configPage.split('\n');- const envValues = new Map<string, string>();- for (const line of envExample.split('\n')) {- const assignment = line.match(/^([A-Z][A-Z0-9_]*)=(.*)$/u);- if (assignment != null) {- envValues.set(assignment[1], assignment[2]);- }- }-- const ENV_VALUE_FLOOR = 16;- const ENV_VALUE_UNSET = 'unset';- let inEnvValueTable = false;- let envEntry: string | null = null;- let envRowsChecked = 0;- const checkEnvValue = (name: string, documented: string): void => {- envRowsChecked += 1;- const actual = envValues.get(name);- if (documented === ENV_VALUE_UNSET) {- if (actual != null) {- problems.push(`configuration.mdx says ${name} is unset but .env.example assigns \`${actual}\``);- }- return;- }- if (actual == null) {- problems.push(`configuration.mdx documents ${name} under "Value in .env.example" but it is not in that file`);- return;- }- if (actual !== documented) {- problems.push(`configuration.mdx says ${name} is \`${documented}\` but .env.example has \`${actual}\``);- }- };- for (const line of configLines) {- const entryHeading = line.match(/^#### `([A-Z][A-Z0-9_]*)`(?:<sup>[^<]*<\/sup>)?\s*$/u);- if (entryHeading != null) {- envEntry = entryHeading[1];- inEnvValueTable = false;- continue;- }- if (/^#{1,6}\s/u.test(line)) {- envEntry = null;- }- if (envEntry != null) {- const marker = line.match(/^`\.env\.example`\s+(?:`([^`]*)`|(unset)\b)/u);- if (marker != null) {- checkEnvValue(envEntry, marker[1] ?? ENV_VALUE_UNSET);- envEntry = null;- continue;- }- }- if (/^\|\s*Variable\s*\|/u.test(line)) {- inEnvValueTable = line.includes('Value in `.env.example`');- continue;- }- if (!line.startsWith('|')) {- inEnvValueTable = false;- continue;- }- if (!inEnvValueTable) {- continue;- }- const row = line.match(/^\|\s*([A-Z][A-Z0-9_]*)(?:<sup>[^<]*<\/sup>)?\s*\|\s*(?:`([^`]*)`|(unset))\s*\|/u);- if (row == null) {- continue;- }- checkEnvValue(row[1], row[2] ?? ENV_VALUE_UNSET);- }- console.log(` configuration.mdx .env.example values checked: ${envRowsChecked.toString()}`);- if (envRowsChecked < ENV_VALUE_FLOOR) {- problems.push(- `configuration.mdx .env.example values checked fell to ${envRowsChecked.toString()}, floor is ${ENV_VALUE_FLOOR.toString()}`,- );- }-- const composeServices = new Set<string>();- {- let inServices = false;- for (const line of composeLines) {- if (/^services:\s*$/u.test(line)) {- inServices = true;- continue;- }- if (/^[a-z]/u.test(line)) {- inServices = false;- }- const service = line.match(/^ {2}([a-z][a-z0-9_-]*):\s*$/u);- if (inServices && service != null) {- composeServices.add(service[1]);- }- }- }- const operatorPages = ['operator/get-started.mdx', 'operator/configuration.mdx', 'operator/upgrading.mdx'];- const namedServices = new Set<string>();- for (const page of operatorPages) {- const text = await readFile(path.join(DOCS_ROOT, page), 'utf8');- for (const backticked of text.matchAll(/`([a-z][a-z0-9-]{2,})`/gu)) {- namedServices.add(backticked[1]);- }- }- const knownNonServices = new Set(['docker-compose.yml', 'chat.example.com', 'admin.example.com', 'localhost']);- for (const candidate of namedServices) {- if (!candidate.endsWith('-shard') && !candidate.startsWith('fluxer-') && !composeServices.has(candidate)) {- continue;- }- if (knownNonServices.has(candidate)) {- continue;- }- const isShard = candidate.endsWith('-shard');- const isImage = candidate.startsWith('fluxer-');- if (!isShard && !isImage) {- continue;- }- if (isShard && !composeServices.has(candidate)) {- problems.push(`the operator docs name the service \`${candidate}\`, which is not in docker-compose.yml`);- }- if (isImage && !compose.includes(candidate)) {- problems.push(`the operator docs name the image \`${candidate}\`, which docker-compose.yml never references`);- }- }- console.log(` compose services: ${composeServices.size.toString()}`);-- const files = ['docker-compose.yml', 'Caddyfile', '.env.example'];- for (const file of files) {- if (!guide.includes(file)) {- problems.push(`the guide never mentions ${file}, which the stack ships`);- }- }-- console.log(` .env.example placeholders: ${placeholders.size.toString()}`);- console.log(` set by the guide: ${[...placeholders].filter((n) => covered.has(n)).length.toString()}`);- console.log(` installer secret keys (sh): ${shellSecrets.size.toString()}`);- console.log(` installer secret keys (ps1): ${powershellSecrets.size.toString()}`);- console.log(` installer non-secret keys: ${shellNonSecrets.size.toString()}`);- console.log(` compose required variables covered: ${composeRequiredCovered.toString()}`);- console.log(` services publishing ports: ${[...publishers].sort().join(', ')}`);- console.log(` published ports: ${[...publishedPorts].sort().join(', ')}`);- failures += section('self-hosting guide disagreements', problems);-}--console.log('Kubernetes page against deploy/helm/fluxer');-{- const chartValues = await readFile(path.join(REPO_ROOT, 'deploy/helm/fluxer/values.yaml'), 'utf8');- const page = await readFile(path.join(DOCS_ROOT, 'operator/kubernetes.mdx'), 'utf8');- const problems: Array<string> = [];-- const chartKeys: Array<string> = [];- let inSecrets = false;- let inValues = false;- for (const line of chartValues.split('\n')) {- if (/^\S/u.test(line)) {- inSecrets = line.startsWith('secrets:');- inValues = false;- continue;- }- if (inSecrets && /^ {2}\S/u.test(line)) {- inValues = line.startsWith(' values:');- continue;- }- const key = inValues ? line.match(/^ {4}([A-Z][A-Z0-9_]*):/u) : null;- if (key != null) {- chartKeys.push(key[1]);- }- }- if (chartKeys.length === 0) {- problems.push('found no keys under secrets.values in deploy/helm/fluxer/values.yaml');- }-- const tableKeys = new Set<string>();- const header = '| Key | Chart default |';- const lines = page.split('\n');- const start = lines.indexOf(header);- if (start === -1) {- problems.push(`operator/kubernetes.mdx has no table headed \`${header}\``);- } else {- for (const row of lines.slice(start + 2)) {- const cell = row.match(/^\| `([A-Z][A-Z0-9_]*)` \|/u);- if (cell == null) {- break;- }- tableKeys.add(cell[1]);- }- }- for (const key of chartKeys) {- if (!tableKeys.has(key)) {- problems.push(`secrets.values.${key} is missing from the secret table on operator/kubernetes.mdx`);- }- }- for (const key of tableKeys) {- if (!chartKeys.includes(key)) {- problems.push(`operator/kubernetes.mdx lists ${key}, which secrets.values does not declare`);- }- }- console.log(` chart secret keys: ${chartKeys.length.toString()}`);- console.log(` documented in the secret table: ${tableKeys.size.toString()}`);- failures += section('Kubernetes page disagreements', problems);-}--console.log('unthrottled routes and global bucket claims');-{- const unthrottled = controllerRoutes- .filter((route) => route.rateLimitConfig == null)- .map((route) => ({- method: route.method.toUpperCase(),- route: route.path,- admin: route.path.startsWith('/admin'),- }));-- const publicShapes = new Set(main.map((operation) => routeShape(operation.method, operation.path)));- const publicUnthrottled = unthrottled- .filter((entry) => !entry.admin)- .map((entry) => routeShape(entry.method, stripVersionPrefix(entry.route.replace(/:([a-zA-Z_]+)/gu, '{$1}'))))- .filter((shape) => publicShapes.has(shape));- const adminUnthrottled = unthrottled.filter((entry) => entry.admin);-- const page = await readFile(path.join(DOCS_ROOT, 'topics/rate-limits.md'), 'utf8');- const problems: Array<string> = [];- const uniquePublic = [...new Set(publicUnthrottled)].sort();-- const unexpected = uniquePublic;-- if (unexpected.length > 0) {- problems.push(- `rate-limits.md says every HTTP API operation declares a bucket, but ${unexpected.length.toString()} declare none: ${unexpected.join(', ')}`,- );- }- if (!page.includes('Every HTTP API and Admin API operation declares a bucket')) {- problems.push('rate-limits.md no longer states that every operation declares a bucket');- }- if (adminUnthrottled.length > 0) {- const named = adminUnthrottled.map((entry) => `${entry.method} ${entry.route}`).sort();- problems.push(- `rate-limits.md says every Admin API operation declares a bucket, but ${named.length.toString()} declare none: ${named.join(', ')}`,- );- }- for (const stale of ['declares no bucket', 'Two [Admin API]']) {- if (page.includes(stale)) {- problems.push(`rate-limits.md still says \`${stale}\`, and every operation now declares one`);- }- }-- const middleware = await readFile(- path.join(REPO_ROOT, 'fluxer_api/src/api/middleware/RateLimitMiddleware.ts'),- 'utf8',- );- const high = middleware.match(/HIGH_GLOBAL_RATE_LIMIT\) !== 0n\) \{\s*return (\d+);/u);- if (high != null && !page.includes(`${Number.parseInt(high[1], 10).toLocaleString('en-US')} requests per second`)) {- problems.push(`rate-limits.md does not state the HIGH_GLOBAL_RATE_LIMIT allowance of ${high[1]}`);- }-- console.log(` public operations with no rate limit middleware: ${uniquePublic.length.toString()}`);- console.log(` admin operations with no rate limit middleware: ${adminUnthrottled.length.toString()}`);- failures += section('rate limit prose disagreements', problems);-}--console.log('error registry');-{- const errorsPage = await readFile(path.join(DOCS_ROOT, 'http-api/errors.md'), 'utf8');- const documentedCodes = new Set<string>();- let documentedEntries = 0;- for (const row of errorsPage.matchAll(/^\|\s*`?([A-Z][A-Z0-9_]{2,})`?\s*\|/gmu)) {- documentedCodes.add(row[1]);- documentedEntries += 1;- }- for (const entry of errorsPage.matchAll(/^#{3,4} `([A-Z][A-Z0-9_]{2,})`\s*$/gmu)) {- documentedCodes.add(entry[1]);- documentedEntries += 1;- }- const registrySource = await readFile(path.join(REPO_ROOT, 'packages/constants/src/ApiErrorCodes.ts'), 'utf8');- const registryCodes = new Set<string>();- for (const entry of registrySource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {- registryCodes.add(entry[1]);- }- const validationSource = await readFile(- path.join(REPO_ROOT, 'packages/constants/src/ValidationErrorCodes.ts'),- 'utf8',- );- const validationCodes = new Set<string>();- for (const entry of validationSource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {- validationCodes.add(entry[1]);- }- const problems: Array<string> = [];- for (const code of registryCodes) {- if (!documentedCodes.has(code)) {- problems.push(`APIErrorCodes.${code} is not listed in the errors page registry`);- }- }-- const documentedRegistryCodes = [...registryCodes].filter((c) => documentedCodes.has(c)).length;- const UNDOCUMENTED_VALIDATION_CODES = new Set(['EMAIL_DOMAIN_CANNOT_RECEIVE_MAIL']);- const expectedEntries = registryCodes.size + validationCodes.size - UNDOCUMENTED_VALIDATION_CODES.size;- if (documentedEntries < expectedEntries) {- problems.push(- `errors.md code entries parsed fell to ${documentedEntries.toString()}, expected ${expectedEntries.toString()}, the ${registryCodes.size.toString()} API codes plus the ${validationCodes.size.toString()} validation codes less the ${UNDOCUMENTED_VALIDATION_CODES.size.toString()} named as undocumented. Either a code lost its entry, or the registry parser stopped matching one of the \`| CODE |\` table row and \`### \`CODE\`\` or \`#### \`CODE\`\` heading shapes it reads`,- );- }- for (const code of documentedCodes) {- if (!registryCodes.has(code) && !validationCodes.has(code)) {- problems.push(`errors.md documents ${code}, which is in neither code registry`);- }- }- if (documentedRegistryCodes < registryCodes.size) {- problems.push(- `errors.md documents ${documentedRegistryCodes.toString()} of the ${registryCodes.size.toString()} registry codes, floor is every one of them`,- );- }-- console.log(` registry codes: ${registryCodes.size.toString()}, documented: ${documentedRegistryCodes.toString()}`);- failures += section('error registry disagreements', problems);-}--console.log('gateway numeric limits');-{- const erlConstants = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/utils/constants.erl'), 'utf8');- const sharding = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/gateway_sharding.erl'), 'utf8');- const abuse = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/session_abuse_protection.erl'), 'utf8');- const heartbeat = await readFile(- path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/gateway_handler_heartbeat.erl'),- 'utf8',- );- const limitsPage = await readFile(path.join(DOCS_ROOT, 'gateway/limits-and-rate-limits.md'), 'utf8');- const overviewPage = await readFile(path.join(DOCS_ROOT, 'gateway/overview.md'), 'utf8');- const corpus = `${limitsPage}\n${overviewPage}`;-- const readFn = (source: string, name: string): number | null => {- const found = source.match(new RegExp(`${name}\\(\\) -> (\\d+)`, 'u'));- return found == null ? null : Number.parseInt(found[1], 10);- };- const readDefine = (source: string, name: string): number | null => {- const found = source.match(new RegExp(`-define\\(${name}, (\\d+)\\)`, 'u'));- return found == null ? null : Number.parseInt(found[1], 10);- };- const grouped = (value: number): string => value.toLocaleString('en-US');-- const problems: Array<string> = [];- const interval = readFn(erlConstants, 'heartbeat_interval');-- const expectations: Array<[string, number | null, (value: string) => string]> = [- ['heartbeat interval', interval, (v) => `heartbeat interval of ${v} ms`],- ['resume timeout', readFn(erlConstants, 'resume_timeout'), (v) => `resumable for ${v} ms`],- ['max payload size', readFn(erlConstants, 'max_payload_size'), (v) => v],- ['max shard count', readDefine(sharding, 'MAX_SHARD_COUNT'), (v) => `through ${v}`],- ['max guilds per shard', readDefine(sharding, 'MAX_GUILDS_PER_SHARD'), (v) => `at most ${v} guilds`],- ['max sessions per user', readDefine(abuse, 'MAX_SESSIONS_PER_USER'), (v) => `at most ${v} live sessions`],- ];- if (interval != null && heartbeat.includes('div 3')) {- expectations.push(['heartbeat check cadence, interval div 3', Math.floor(interval / 3), (v) => `every ${v} ms`]);- }- if (interval != null && heartbeat.includes('* 10) >= (Interval * 9)')) {- expectations.push([- 'heartbeat ping threshold, interval times 9 over 10',- (interval * 9) / 10,- (v) => `When ${v} ms have elapsed`,- ]);- }- let compared = 0;- for (const [label, value, phrase] of expectations) {- if (value == null) {- problems.push(`could not read ${label} from the Gateway source`);- continue;- }- compared += 1;- if (!corpus.includes(phrase(grouped(value))) && !corpus.includes(phrase(String(value)))) {- problems.push(`the Gateway docs never state the ${label} of ${grouped(value)} in the expected phrasing`);- }- }- console.log(` gateway constants compared: ${compared.toString()}`);- failures += section('gateway limit disagreements', problems);-}--console.log('snowflake layout');-{- const core = await readFile(path.join(REPO_ROOT, 'packages/constants/src/Core.ts'), 'utf8');- const snowflakeSource = await readFile(path.join(REPO_ROOT, 'packages/snowflake/src/Snowflake.ts'), 'utf8');- const page = await readFile(path.join(DOCS_ROOT, 'snowflakes.md'), 'utf8');- const problems: Array<string> = [];-- const epochMatch = core.match(/export const FLUXER_EPOCH = (\d+);/u);- const workerBits = snowflakeSource.match(/WORKER_ID_BITS = (\d+)n/u);- const sequenceBits = snowflakeSource.match(/SEQUENCE_BITS = (\d+)n/u);-- if (epochMatch == null || workerBits == null || sequenceBits == null) {- problems.push('could not read FLUXER_EPOCH, WORKER_ID_BITS or SEQUENCE_BITS from source');- } else {- const epoch = Number.parseInt(epochMatch[1], 10);- const worker = Number.parseInt(workerBits[1], 10);- const sequence = Number.parseInt(sequenceBits[1], 10);- const shift = worker + sequence;- if (!page.includes(String(epoch))) {- problems.push(`snowflakes.md does not state the epoch ${epoch.toString()}`);- }- const epochIso = new Date(epoch).toISOString();- if (!page.includes(epochIso.slice(0, 10))) {- problems.push(`snowflakes.md does not state the epoch date ${epochIso}`);- }- if (!page.includes(`right by ${shift.toString()} bits`)) {- problems.push(`snowflakes.md does not state the timestamp shift of ${shift.toString()} bits`);- }- if (!page.includes(`through \`${((1 << worker) - 1).toString()}\``)) {- problems.push(`snowflakes.md does not state the maximum worker id of ${((1 << worker) - 1).toString()}`);- }- const timestampBits = 63 - shift;- const lastMs = epoch + 2 ** timestampBits - 1;- const lastIso = new Date(lastMs).toISOString();- if (!page.includes(lastIso.slice(0, 19))) {- problems.push(`snowflakes.md does not state the last representable instant ${lastIso}`);- }- console.log(- ` epoch ${epoch.toString()} (${epochIso.slice(0, 10)}), shift ${shift.toString()}, worker ${worker.toString()} bits, sequence ${sequence.toString()} bits`,- );- console.log(` last representable instant: ${lastIso}`);- }- failures += section('snowflake layout disagreements', problems);-}--console.log('attachment upload geometry');-{- const limits = await readFile(path.join(REPO_ROOT, 'packages/constants/src/LimitConstants.ts'), 'utf8');- const problems: Array<string> = [];- const readConst = (name: string): number | null => {- const found = limits.match(new RegExp(`${name} = ([^;]+);`, 'u'));- if (found == null) {- return null;- }- const expression = found[1].replace(/_/gu, '').trim();- if (!/^[\d*+\s]+$/u.test(expression)) {- return null;- }- return expression- .split('+')- .map((term) => term.split('*').reduce((a, b) => a * Number.parseInt(b.trim(), 10), 1))- .reduce((a, b) => a + b, 0);- };-- const constants: Array<[string, string, string]> = [- ['ATTACHMENT_UPLOAD_CHUNK_THRESHOLD', 'singlepart threshold', 'topics/uploads.md'],- ['ATTACHMENT_MAX_SIZE_NON_PREMIUM', 'non-premium attachment ceiling', 'http-api/messages.mdx'],- ['ATTACHMENT_MAX_SIZE_PREMIUM', 'premium attachment ceiling', 'http-api/messages.mdx'],- ['ATTACHMENT_MAX_SIZE_BOT', 'bot attachment ceiling', 'http-api/messages.mdx'],- ];- let compared = 0;- for (const [name, label, file] of constants) {- const page = pages.find((entry) => entry.relativePath === file);- if (!page) throw new Error(`Missing upload documentation: ${file}`);- const value = readConst(name);- if (value == null) {- problems.push(`could not read ${name} from LimitConstants.ts`);- continue;- }- compared += 1;- const bare = new RegExp(`(?<![\\d,.])${value.toString()}(?![\\d,.])`, 'u');- const grouped = new RegExp(`(?<![\\d,.])${value.toLocaleString('en-US')}(?![\\d,.])`, 'u');- if (!bare.test(page.source) && !grouped.test(page.source)) {- problems.push(`${file} does not state the ${label} of ${value.toString()}`);- }- }- console.log(` upload constants compared: ${compared.toString()}`);- failures += section('upload geometry disagreements', problems);-}--console.log('captcha gated operations');-{- const page = await readFile(path.join(DOCS_ROOT, 'topics/captcha.md'), 'utf8');- const documented = new Set<string>();- const documentedLabels = new Map<string, string>();- for (const row of page.matchAll(/^\|\s*(GET|POST|PUT|PATCH|DELETE)\s*\|\s*(\/v1\/\S+?)\s*\|/gmu)) {- const normalised = routeShape(row[1], stripVersionPrefix(row[2]));- documented.add(normalised);- documentedLabels.set(normalised, `${row[1]} ${row[2]}`);- }-- const gated = new Set<string>();- for (const route of controllerRoutes) {- if (route.middlewares.some((name) => /Captcha|GroupDm\w*Protection/u.test(name))) {- gated.add(astRoute(route));- }- }-- const problems: Array<string> = [];- for (const route of gated) {- if (!documented.has(route)) {- problems.push(`${route} is captcha gated in source but not listed on the captcha page`);- }- }- for (const route of documented) {- if (!gated.has(route)) {- problems.push(- `the captcha page lists ${documentedLabels.get(route) ?? route}, which is not captcha gated in source`,- );- }- }- console.log(` captcha gated routes in source: ${gated.size.toString()}, documented: ${documented.size.toString()}`);- failures += section('captcha gating disagreements', problems);-}--console.log('bot capability flag (from the middleware chain)');-{- const BOT_EXEMPT = new Map([- ['GET /applications/@me', {docs: true, why: 'reads the Authorization header directly in the handler'}],- ['GET /gateway/bot', {docs: true, why: 'requires a bot-shaped token, parsed in the handler'}],- ['POST /guilds', {docs: false, why: 'the service rejects bots with BOTS_CANNOT_CREATE_GUILDS'}],- ['POST /guilds/{}/delete', {docs: false, why: 'the caller must own the guild and a bot never can'}],- ]);- const problems: Array<string> = [];- let compared = 0;- for (const route of controllerRoutes) {- const key = astRoute(route);- if (key.startsWith('GET /admin') || key.includes(' /admin/')) {- continue;- }- const documented = documentedMain.get(key);- if (documented == null) {- continue;- }- const anyLogin = route.hasLoginRequired;- const sourceAcceptsBot = (anyLogin || route.hasBotOnly) && !route.hasDefaultUserOnly;- const exemption = BOT_EXEMPT.get(key);- if (exemption != null) {- if (documented.bot !== exemption.docs) {- problems.push(- `${key} is an exempted route expected to document bot=${String(exemption.docs)} (${exemption.why}), but the page says ${String(documented.bot)}`,- );- }- continue;- }- compared += 1;- if (documented.bot !== sourceAcceptsBot) {- problems.push(- `${key} documents bot=${String(documented.bot)} but the middleware chain gives ${String(sourceAcceptsBot)}`,- );- }- }- console.log(` routes compared against the middleware chain: ${compared.toString()}`);- console.log(` exempted, each verified by hand: ${BOT_EXEMPT.size.toString()}`);- failures += section('bot flag disagreements', problems);-}--console.log('unauthenticated capability flag (from the middleware chain)');-{- const WEBHOOK_TOKEN_CREDENTIAL = 'the webhook ID and token in the path are the complete credential';- const OAUTH2_CLIENT_CREDENTIAL =- 'the client authenticates with HTTP Basic or the client_id and client_secret form fields';- const UNAUTHENTICATED_EXEMPT = new Map([- ['GET /applications/@me', {docs: false, why: 'reads the Authorization header directly in the handler'}],- ['GET /gateway/bot', {docs: false, why: 'requires a bot-shaped token, parsed in the handler'}],- [- 'POST /auth/handoff/complete',- {docs: false, why: 'the approving session token comes from the Authorization header or the body token field'},- ],- ['POST /oauth2/token', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}],- ['POST /oauth2/introspect', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}],- ['POST /oauth2/token/revoke', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}],- ['GET /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['PATCH /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['DELETE /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['POST /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['GET /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['PATCH /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['DELETE /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['POST /webhooks/{}/{}/github', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['POST /webhooks/{}/{}/slack', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['POST /webhooks/{}/{}/instatus', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ['POST /webhooks/{}/{}/libredesk', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}],- ]);- const problems: Array<string> = [];- let compared = 0;- for (const route of controllerRoutes) {- const key = astRoute(route);- if (key.startsWith('GET /admin') || key.includes(' /admin/')) {- continue;- }- const documented = documentedMain.get(key);- if (documented == null) {- continue;- }- const anyLogin = route.hasLoginRequired;- const sourceIsOpen = !anyLogin && !route.middlewares.some((name) => /OAuth2Scope/u.test(name));- const exemption = UNAUTHENTICATED_EXEMPT.get(key);- if (exemption != null) {- if (documented.unauthenticated !== exemption.docs) {- problems.push(- `${key} is an exempted route expected to document unauthenticated=${String(exemption.docs)} (${exemption.why}), but the page says ${String(documented.unauthenticated)}`,- );- }- continue;- }- compared += 1;- if (documented.unauthenticated !== sourceIsOpen) {- problems.push(- `${key} documents unauthenticated=${String(documented.unauthenticated)} but the middleware chain gives ${String(sourceIsOpen)}`,- );- }- }- console.log(` routes compared against the middleware chain: ${compared.toString()}`);- console.log(` exempted, each verified by hand: ${UNAUTHENTICATED_EXEMPT.size.toString()}`);- failures += section('unauthenticated flag disagreements', problems);-}--console.log('spec security field against the middleware chain');-{- const MANUAL_CREDENTIAL = new Set(['GET /applications/@me']);- const BOT_SCHEME_EXEMPT = new Map<string, string>([- [- 'POST /guilds',- 'LoginRequired admits a bot token, but GuildOperationsService rejects every bot with 400 BOTS_CANNOT_CREATE_GUILDS, so the spec must not advertise botToken',- ],- ]);- const specSecurity = new Map<string, EffectiveSpecSecurity>();- for (const operation of main) {- const security = operation.security ?? [];- const schemes = new Set<string>();- for (const entry of security) {- for (const scheme of Object.keys(entry)) {- schemes.add(scheme);- }- }- specSecurity.set(routeShape(operation.method, operation.path), {- schemes,- allowsAnonymous: security.length === 0 || security.some((entry) => Object.keys(entry).length === 0),- });- }- const specBugs: Array<string> = [];- let compared = 0;- for (const route of controllerRoutes) {- const key = astRoute(route);- const declaredSecurity = specSecurity.get(key);- if (declaredSecurity == null || MANUAL_CREDENTIAL.has(key)) {- continue;- }- const declaredSchemes = declaredSecurity.schemes;- compared += 1;- const anyLogin = route.hasLoginRequired;- const acceptsBot = (anyLogin || route.hasBotOnly) && !route.hasDefaultUserOnly;- const requiresAuthentication =- anyLogin ||- route.hasDefaultUserOnly ||- route.hasBotOnly ||- route.oauth2BearerTokenRequired ||- route.middlewares.includes('requireOAuth2Scope');- if (requiresAuthentication && declaredSecurity.allowsAnonymous) {- specBugs.push(`${key} allows unauthenticated requests, but the middleware chain requires authentication`);- }- const exemption = BOT_SCHEME_EXEMPT.get(key);- if (exemption != null) {- if (declaredSchemes.has('botToken')) {- specBugs.push(`${key} declares botToken, but is exempted because ${exemption}`);- }- continue;- }- if (declaredSchemes.has('botToken') && !acceptsBot) {- specBugs.push(`${key} declares botToken, but DefaultUserOnly rejects bots`);- }- if (!declaredSchemes.has('botToken') && acceptsBot && declaredSchemes.size > 0) {- specBugs.push(`${key} omits botToken, but the middleware chain admits a bot token`);- }- if (route.hasBotOnly && [...declaredSchemes].some((scheme) => scheme !== 'botToken')) {- specBugs.push(`${key} declares [${[...declaredSchemes].join(', ')}], but BotOnly admits only a bot token`);- }- if (- declaredSchemes.size > 0 &&- !anyLogin &&- !route.hasBotOnly &&- !route.middlewares.some((name) => /Admin|OAuth2Scope/iu.test(name))- ) {- specBugs.push(`${key} declares [${[...declaredSchemes].join(', ')}], but the chain has no login policy`);- }- }- console.log(` routes compared: ${compared.toString()}`);- console.log(' GET /applications/@me is excluded because it reads the Authorization header in the');- console.log(' handler, so it has no middleware to compare against.');- console.log(` exempted, each verified by hand: ${BOT_SCHEME_EXEMPT.size.toString()}`);- for (const [key, why] of BOT_SCHEME_EXEMPT) {- console.log(` ${key}: ${why}`);- }- failures += section('spec security disagreements with the middleware chain', specBugs);-}--console.log('admin API (against the generated spec)');-const adminTargetOnly = [...documentedAdmin.entries()]- .filter(([shape]) => !adminShapes.has(shape))- .map(([, route]) => `${route.method} ${route.path} (${route.file})`)- .sort();-const adminLiveOnly = admin- .filter((operation) => {- const shape = routeShape(operation.method, operation.path);- return !documentedAdmin.has(shape);- })- .map((operation) => `${operation.method} ${operation.path}`)- .sort();-console.log(` live admin operations: ${admin.length.toString()}`);-console.log(` documented admin operations: ${adminDocumented.length.toString()}`);-console.log(- ` documented in the RESTful target shape, awaiting the live admin refactor: ${adminTargetOnly.length.toString()}`,-);-console.log(` live admin operations with no documented RESTful equivalent: ${adminLiveOnly.length.toString()}`);-failures += section('target-shape routes awaiting the live admin refactor', adminTargetOnly);-failures += section('live admin routes with no documented equivalent', adminLiveOnly);--console.log('');-console.log(- `documented routes: ${documented.length.toString()} (main ${mainDocumented.length.toString()}, admin ${adminDocumented.length.toString()}, media proxy ${mediaProxyDocumented.length.toString()})`,-);-if (failures > 0) {- console.error(`FAIL: ${failures.toString()} coverage problems`);- process.exit(1);-}-console.log('OK: every registered fluxer_api route is documented or covered by an exemption rule, and the');-console.log('documented routes match the live main API, media proxy, and admin target shape');--async function verifyInstallerExecution(installerRoot: string): Promise<Array<string>> {- const problems: Array<string> = [];- const shellParse = spawnSync('sh', ['-n', path.join(installerRoot, 'install.sh')], {encoding: 'utf8'});- if (shellParse.error != null) {- problems.push(`sh -n could not run against install.sh: ${shellParse.error.message}`);- } else if (shellParse.status !== 0) {- problems.push(`sh -n rejects install.sh: ${shellParse.stderr.trim()}`);- }-- const DOCKER_STUB = [- '#!/bin/sh',- 'case "$1 $2" in',- " '--version ') echo 'Docker version 27.1.1, build stub' ;;",- " 'compose version') if [ \"$3\" = '--short' ]; then echo '2.30.3'; else echo 'v2.30.3'; fi ;;",- " 'compose config') echo 'ghcr.io/fluxerapp/fluxer-api:v1' ;;",- 'esac',- 'exit 0',- '',- ].join('\n');- const CURL_STUB = [- '#!/bin/sh',- "out=''",- "prev=''",- 'for arg in "$@"; do',- ' if [ "$prev" = \'-o\' ]; then out=$arg; fi',- ' prev=$arg',- 'done',- '[ -z "$out" ] || printf \'name: fluxer\\nservices:\\n api:\\n image: stub\\n\' > "$out"',- '',- ].join('\n');-- const sandbox = await mkdtemp(path.join(tmpdir(), 'fluxer-installer-'));- try {- const stubBin = path.join(sandbox, 'bin');- await mkdir(stubBin, {recursive: true});- await writeFile(path.join(stubBin, 'docker'), DOCKER_STUB, {mode: 0o755});- await writeFile(path.join(stubBin, 'curl'), CURL_STUB, {mode: 0o755});- await writeFile(path.join(stubBin, 'openssl'), '#!/bin/sh\nexit 0\n', {mode: 0o755});-- const instance = path.join(sandbox, 'instance');- await mkdir(instance, {recursive: true});- await writeFile(path.join(instance, '.env'), 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n');- await writeFile(path.join(instance, 'docker-compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');-- const planned = (label: string, args: ReadonlyArray<string>, cwd?: string): string | null => {- const run = spawnSync('sh', [path.join(installerRoot, 'install.sh'), ...args], {- cwd,- encoding: 'utf8',- env: {- ...process.env,- PATH: `${stubBin}${path.delimiter}${process.env.PATH ?? ''}`,- FLUXER_INSTALLER_REFRESHED: '1',- ...(cwd == null ? {} : {PWD: cwd}),- },- });- if (run.error != null) {- problems.push(`install.sh ${label} could not run: ${run.error.message}`);- return null;- }- if (run.status !== 0) {- problems.push(`install.sh ${label} exited ${String(run.status)}: ${run.stderr.trim()}`);- return null;- }- return run.stdout;- };-- const plannedRef = (label: string, args: ReadonlyArray<string>): string | null => {- const stdout = planned(label, args);- if (stdout == null) {- return null;- }- const line = stdout.match(/^ {2}ref\s+(\S+)$/mu);- if (line == null) {- problems.push(`install.sh ${label} printed no ref line`);- return null;- }- return line[1];- };-- const INSTALL_ARGS = [- '--dry-run',- '--non-interactive',- '--allow-root',- '--domain',- 'x.example',- '--email',- '[email protected]',- '--dir',- path.join(sandbox, 'target'),- ];- const REF_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string]> = [- ['on the default image tag', INSTALL_ARGS, 'main'],- ['under --image-tag latest', [...INSTALL_ARGS, '--image-tag', 'latest'], 'main'],- ['under --image-tag 2026.813.205040', [...INSTALL_ARGS, '--image-tag', '2026.813.205040'], '2026.813.205040'],- [- 'under --ref feature/x --image-tag 2026.813.205040',- [...INSTALL_ARGS, '--ref', 'feature/x', '--image-tag', '2026.813.205040'],- 'feature/x',- ],- [- 'under --update against a pinned .env',- ['--update', '--dry-run', '--allow-root', '--dir', instance],- '2026.813.205040',- ],- ];- for (const [label, args, expected] of REF_CASES) {- const resolved = plannedRef(label, args);- if (resolved != null && resolved !== expected) {- problems.push(`install.sh ${label} plans ref ${resolved}, and the image tag it pairs with wants ${expected}`);- }- }-- const withoutEmail = INSTALL_ARGS.filter(- (arg, index) => arg !== '--email' && INSTALL_ARGS[index - 1] !== '--email',- );- const withoutEmailPlan = planned('without --email', withoutEmail);- if (withoutEmailPlan != null && !/^ {2}email\s+admin@x\.example, derived by compose$/mu.test(withoutEmailPlan)) {- problems.push('install.sh without --email does not plan the admin@FLUXER_DOMAIN contact that compose derives');- }-- const composeYmlInstance = path.join(sandbox, 'compose-yml-instance');- await mkdir(composeYmlInstance, {recursive: true});- await writeFile(- path.join(composeYmlInstance, '.env'),- 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n',- );- await writeFile(path.join(composeYmlInstance, 'compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');- const COMPOSE_NAME_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string | undefined]> = [- [- 'under --update against a compose.yml instance',- ['--update', '--dry-run', '--allow-root', '--dir', composeYmlInstance],- undefined,- ],- [- 'under --update standing in a compose.yml instance',- ['--update', '--dry-run', '--allow-root'],- composeYmlInstance,- ],- ];- for (const [label, args, cwd] of COMPOSE_NAME_CASES) {- const stdout = planned(label, args, cwd);- if (stdout == null) {- continue;- }- if (!/^ {4}compose\.yml is unchanged$/mu.test(stdout)) {- problems.push(`install.sh ${label} does not plan the refreshed stack file onto compose.yml`);- }- if (/^ {4}docker-compose\.yml is new$/mu.test(stdout)) {- problems.push(`install.sh ${label} plans a docker-compose.yml that Compose would never load there`);- }- }- } finally {- await rm(sandbox, {recursive: true, force: true});- }- return problems;-}diff --git a/fluxer_docs/scripts/VerifyDocsSchemas.test.ts b/fluxer_docs/scripts/VerifyDocsSchemas.test.tsdeleted file mode 100644index b4a26ba3f..000000000--- a/fluxer_docs/scripts/VerifyDocsSchemas.test.ts+++ /dev/null@@ -1,666 +0,0 @@-import assert from 'node:assert/strict';-import {test} from 'node:test';-import type {OpenAPIDocument} from '@fluxer/openapi/src/OpenAPITypes';-import type {MarkdownPage} from './DocsSource.ts';-import {type SchemaVerification, verifyPages} from './VerifyDocsSchemas.ts';--const SNOWFLAKE = {$ref: '#/components/schemas/SnowflakeType'};-const WIDGET = {$ref: '#/components/schemas/Widget'};--const spec = {- openapi: '3.1.0',- info: {title: 'fixture', version: '1'},- paths: {- '/widgets/{widget_id}': {- patch: {- operationId: 'update_widget',- tags: [],- requestBody: {content: {'application/json': {schema: {$ref: '#/components/schemas/WidgetUpdate'}}}},- responses: {'200': {description: 'ok', content: {'application/json': {schema: WIDGET}}}},- },- },- '/widgets': {- get: {- operationId: 'list_widgets',- tags: [],- parameters: [- {name: 'limit', in: 'query', required: false, schema: {type: 'integer'}},- {name: 'owner_id', in: 'query', required: false, schema: SNOWFLAKE},- ],- responses: {- '200': {- description: 'ok',- content: {- 'application/json': {- schema: {- type: 'object',- properties: {widgets: {type: 'array', items: WIDGET}, total: {type: 'integer'}},- required: ['widgets', 'total'],- additionalProperties: false,- },- },- },- },- },- },- post: {- operationId: 'create_widget',- tags: [],- requestBody: {- content: {- 'application/json': {- schema: {- oneOf: [- {- type: 'object',- properties: {kind: {type: 'string', const: 'alpha'}, alpha: {type: 'string'}},- required: ['kind', 'alpha'],- },- {- type: 'object',- properties: {kind: {type: 'string', const: 'beta'}, beta: {type: 'integer'}},- required: ['kind', 'beta'],- },- ],- },- },- },- },- responses: {'200': {description: 'ok', content: {'application/json': {schema: WIDGET}}}},- },- },- '/widgets/positions': {- put: {- operationId: 'move_widgets',- tags: [],- requestBody: {- content: {- 'application/json': {schema: {type: 'array', items: {$ref: '#/components/schemas/WidgetPosition'}}},- },- },- responses: {'204': {description: 'empty'}},- },- },- '/owners': {- get: {- operationId: 'list_owners',- tags: [],- responses: {- '200': {- description: 'ok',- content: {- 'application/json': {- schema: {- type: 'object',- additionalProperties: {anyOf: [{$ref: '#/components/schemas/Owner'}, {type: 'null'}]},- },- },- },- },- },- },- },- '/gadgets': {- post: {- operationId: 'create_gadget',- tags: [],- requestBody: {content: {'application/json': {schema: {$ref: '#/components/schemas/GadgetCreate'}}}},- responses: {'204': {description: 'empty'}},- },- },- },- components: {- schemas: {- SnowflakeType: {- anyOf: [{type: 'string'}, {type: 'integer'}],- format: 'snowflake',- },- Widget: {- type: 'object',- properties: {- id: SNOWFLAKE,- name: {type: 'string'},- value: {anyOf: [{type: 'string'}, {type: 'boolean'}]},- part: {anyOf: [{$ref: '#/components/schemas/Part'}, {type: 'null'}]},- labels: {type: 'object', additionalProperties: {$ref: '#/components/schemas/Label'}},- shape: {anyOf: [{$ref: '#/components/schemas/Circle'}, {$ref: '#/components/schemas/Square'}]},- legacy: {type: 'boolean', description: 'Deprecated. Always false.'},- },- required: ['id', 'name', 'value', 'part'],- additionalProperties: false,- },- Part: {- type: 'object',- properties: {part_id: SNOWFLAKE, weight: {type: 'number'}, unused: {type: 'string'}},- required: ['part_id', 'weight'],- additionalProperties: false,- },- Label: {- type: 'object',- properties: {text: {type: 'string'}, color: {type: 'string'}},- required: ['text'],- additionalProperties: false,- },- Owner: {- type: 'object',- properties: {owner_id: SNOWFLAKE, name: {type: 'string'}},- required: ['owner_id', 'name'],- additionalProperties: false,- },- Circle: {- type: 'object',- properties: {kind: {type: 'string'}, radius: {type: 'number'}},- required: ['kind', 'radius'],- additionalProperties: false,- },- Square: {- type: 'object',- properties: {kind: {type: 'string'}, side: {type: 'number'}},- required: ['kind', 'side'],- additionalProperties: false,- },- WidgetPosition: {- type: 'object',- properties: {id: SNOWFLAKE, position: {type: 'integer'}},- required: ['id'],- additionalProperties: false,- },- WidgetUpdate: {- type: 'object',- properties: {name: {type: 'string'}, owner_id: SNOWFLAKE, size: {type: 'integer'}},- required: ['name'],- additionalProperties: false,- },- GadgetCreate: {- type: 'object',- properties: {label: {type: 'string'}, color: {type: 'string'}},- required: ['label'],- additionalProperties: false,- },- },- securitySchemes: {},- },-} as unknown as OpenAPIDocument;--const emptySpec = {- openapi: '3.1.0',- info: {title: 'empty', version: '1'},- paths: {},- components: {schemas: {}, securitySchemes: {}},-} as OpenAPIDocument;--const PAGE = `----title: Widgets------import RouteHeader from '@/components/RouteHeader.astro';--## Widget object--### Structure--| Field | Type | Description |-| --- | --- | --- |-| id | snowflake | The ID of the widget |-| name | string | The name |-| value | boolean | The value |-| part | ?[part](#part-object) object | The part |-| labels? | map[string, [label](#label-object) object] | The labels by key |-| shape? | [circle](#circle-object) object \\| [square](#square-object) object | The shape |-| legacy | boolean | Always false |-| shown<sup>1</sup> | boolean | Whether the widget is shown |--<sup>1</sup> Emitted on every response although the response schema does not declare it--### Part object--#### Structure--| Field | Type | Description |-| --- | --- | --- |-| part_id | snowflake | The ID of the part |-| weight | float | The weight |--The response schema also declares \`unused\`, which a part never has.--### Label object--#### Structure--| Field | Type | Description |-| --- | --- | --- |-| text | string | The text |-| color? | string | The colour |--### Circle object--#### Structure--| Field | Type | Description |-| --- | --- | --- |-| kind | string | \`circle\` |-| radius | float | The radius |--### Square object--#### Structure--| Field | Type | Description |-| --- | --- | --- |-| kind | string | \`square\` |-| side | float | The side length |--## Widget position object--### Structure--| Field | Type | Description |-| --- | --- | --- |-| id | snowflake | The ID of the widget |-| position? | integer | The new position |--## Owner object--### Structure--| Field | Type | Description |-| --- | --- | --- |-| owner_id | snowflake | The ID of the owner |-| name | string | The name |--## Owners by ID object--Each property name is the ID of one owner.--### Structure--| Field | Type | Description |-| --- | --- | --- |-| {owner_id} | ?[owner](#owner-object) object | The owner, or null |--## Gadget creation object--### Structure--| Field | Type | Description |-| --- | --- | --- |-| label | string | The label |-| color? | string | The colour |--## Update widget--<RouteHeader method="PATCH" path="/v1/widgets/{widget_id}" />--### JSON body--| Field | Type | Description |-| --- | --- | --- |-| name | string | The new name |-| owner_id? | snowflake | The new owner |-| size? | integer | The new size |--### Response--| Status | Body | Condition |-| --- | --- | --- |-| 200 | [widget](#widget-object) object | The widget was updated |--## List widgets--<RouteHeader method="GET" path="/v1/widgets" />--### Query parameters--| Field | Type | Description |-| --- | --- | --- |-| limit? | integer | The page size |-| owner_id? | snowflake | Restrict to one owner |--### Response body--| Field | Type | Description |-| --- | --- | --- |-| widgets | array[[widget](#widget-object) object] | The widgets |-| total | integer | Every matching widget |--### Response--| Status | Body | Condition |-| --- | --- | --- |-| 200 | response body | The page was returned |--## Create widget--<RouteHeader method="POST" path="/v1/widgets" />--### JSON body--#### Alpha structure--| Field | Type | Description |-| --- | --- | --- |-| kind | string | \`alpha\` |-| alpha | string | The alpha value |--#### Beta structure--| Field | Type | Description |-| --- | --- | --- |-| kind | string | \`beta\` |-| beta | integer | The beta value |--### Response--| Status | Body | Condition |-| --- | --- | --- |-| 200 | [widget](#widget-object) object | The widget was created |--## Move widgets--<RouteHeader method="PUT" path="/v1/widgets/positions" />--### JSON body--The body is an array of [widget position objects](#widget-position-object).--### Response--| Status | Body | Condition |-| --- | --- | --- |-| 204 | empty | The widgets were moved |--## List owners--<RouteHeader method="GET" path="/v1/owners" />--### Response--| Status | Body | Condition |-| --- | --- | --- |-| 200 | [owners by ID](#owners-by-id-object) object | The owners were returned |--## Create gadget--<RouteHeader method="POST" path="/v1/gadgets" />--### JSON body--The body is a [gadget creation object](#gadget-creation-object).--### Response--| Status | Body | Condition |-| --- | --- | --- |-| 204 | empty | The gadget was created |-`;--function page(source: string): MarkdownPage {- return {- file: '/fixture/http-api/widgets.mdx',- relativePath: 'http-api/widgets.mdx',- source,- lines: source.split('\n'),- };-}--function verify(source: string): SchemaVerification {- return verifyPages([page(source)], spec, emptySpec);-}--function mutate(from: string, to: string): string {- assert.ok(PAGE.includes(from), `fixture text not found: ${from}`);- return PAGE.replace(from, to);-}--function findings(result: SchemaVerification): Array<string> {- return result.mismatches.map((mismatch) => `${mismatch.kind} ${mismatch.operation} ${mismatch.detail}`);-}--test('the clean fixture passes and every check runs', () => {- const result = verify(PAGE);- assert.deepEqual(findings(result), []);- assert.deepEqual(result.optionalityAdvisories, []);- assert.equal(result.counters.bodyTables, 2);- assert.equal(result.counters.bodiesByReference, 2);- assert.equal(result.counters.unionBodies, 1);- assert.equal(result.counters.queryTables, 1);- assert.equal(result.counters.responseTables, 1);- assert.equal(result.counters.responseObjects, 3);- assert.ok(result.counters.nestedObjects >= 2);- assert.equal(result.counters.unlinkedObjects, 0);- assert.ok(result.counters.typesCompared > 10);- assert.equal(result.counters.optionalityCompared, 11);-});--test('a union body field missing from every variant table fails', () => {- const result = verify(mutate('| beta | integer | The beta value |\n', ''));- assert.deepEqual(findings(result), ['body-missing POST /widgets beta']);-});--test('a field documented in one variant table but missing from another fails', () => {- const result = verify(mutate('| kind | string | `beta` |\n', ''));- assert.deepEqual(findings(result), ['body-missing POST /widgets kind (variant 2)']);-});--test('a body documented by reference is compared with the object section', () => {- const result = verify(mutate('| label | string | The label |', '| title | string | The label |'));- assert.deepEqual(findings(result), ['body-extra POST /gadgets title', 'body-missing POST /gadgets label']);-});--test('a nested object linked from a field type is compared with its schema', () => {- const result = verify(mutate('| weight | float | The weight |', '| mass | float | The weight |'));- assert.deepEqual(findings(result), [- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).part (http-api/widgets#part-object).weight',- 'object-extra PATCH /widgets/{} (http-api/widgets#part-object).mass',- ]);-});--test('a query row is not found in another route or table of the page', () => {- const result = verify(mutate('| owner_id? | snowflake | Restrict to one owner |\n', ''));- assert.deepEqual(findings(result), ['query-missing GET /widgets owner_id']);-});--test('a snowflake is typed string, so a snowflake documented as integer fails', () => {- const result = verify(mutate('| id | snowflake | The ID of the widget |', '| id | integer | The ID of the widget |'));- assert.deepEqual(findings(result), [- 'type-mismatch PATCH /widgets/{} (http-api/widgets#widget-object).id: documented integer, schema string',- ]);-});--test('a type outside every anyOf branch fails', () => {- const result = verify(mutate('| value | boolean | The value |', '| value | integer | The value |'));- assert.deepEqual(findings(result), [- 'type-mismatch PATCH /widgets/{} (http-api/widgets#widget-object).value: documented integer, schema boolean | string',- ]);-});--test('a route-local response body table is type checked', () => {- const result = verify(- mutate('| total | integer | Every matching widget |', '| total | string | Every matching widget |'),- );- assert.deepEqual(findings(result), ['type-mismatch GET /widgets total: documented string, schema integer']);-});--test('a response body field the schema does not have fails', () => {- const result = verify(- mutate(- '| total | integer | Every matching widget |\n',- '| total | integer | Every matching widget |\n| count | integer | Stale |\n',- ),- );- assert.deepEqual(findings(result), ['response-extra GET /widgets count']);-});--test('every table under JSON body is checked, not only the first', () => {- const result = verify(- mutate(- '| size? | integer | The new size |\n',- '| size? | integer | The new size |\n\n#### Extra structure\n\n| Field | Type | Description |\n| --- | --- | --- |\n| colour? | string | Stale |\n',- ),- );- assert.deepEqual(findings(result), ['body-extra PATCH /widgets/{} colour']);-});--test('an optionality disagreement is fatal', () => {- const result = verify(mutate('| name | string | The new name |', '| name? | string | The new name |'));- assert.deepEqual(findings(result), [- 'optionality PATCH /widgets/{} name: documented optional, schema marks it required',- ]);- assert.equal(result.optionalityAdvisories.length, 1);-});--test('an object member the schema does not declare needs the footnote', () => {- const result = verify(- mutate('<sup>1</sup> Emitted on every response although the response schema does not declare it\n', ''),- );- assert.deepEqual(findings(result), ['object-extra PATCH /widgets/{} (http-api/widgets#widget-object).shown']);-});--test('a schema member an object never has needs the declared note', () => {- const result = verify(mutate('The response schema also declares `unused`, which a part never has.\n', ''));- assert.deepEqual(findings(result), [- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).part (http-api/widgets#part-object).unused',- ]);-});--test('a member missing from a linked response object fails', () => {- const result = verify(mutate('| name | string | The name |\n', ''));- assert.deepEqual(findings(result), ['object-missing PATCH /widgets/{} (http-api/widgets#widget-object).name']);-});--test('a route whose schema has a JSON body needs a JSON body section', () => {- const result = verify(- mutate('### JSON body\n\nThe body is a [gadget creation object](#gadget-creation-object).\n\n', ''),- );- assert.deepEqual(findings(result), ['body-missing POST /gadgets no JSON body section documents the request body']);-});--test('a route whose schema has query parameters needs a query parameter table', () => {- const result = verify(- mutate(- '### Query parameters\n\n| Field | Type | Description |\n| --- | --- | --- |\n| limit? | integer | The page size |\n| owner_id? | snowflake | Restrict to one owner |\n\n',- '',- ),- );- assert.deepEqual(findings(result), ['query-missing GET /widgets limit', 'query-missing GET /widgets owner_id']);-});--test('an object linked as the value of a map is compared with the value schema', () => {- const result = verify(mutate('| text | string | The text |', '| title | string | The text |'));- assert.deepEqual(findings(result), [- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).labels (http-api/widgets#label-object).text',- 'object-extra PATCH /widgets/{} (http-api/widgets#label-object).title',- ]);-});--test('a body that is an array of a linked object is compared with the item schema', () => {- const result = verify(mutate('| position? | integer | The new position |', '| rank? | integer | The new position |'));- assert.deepEqual(findings(result), [- 'body-extra PUT /widgets/positions rank',- 'body-missing PUT /widgets/positions position',- ]);-});--test('an object documented as a keyed map is compared through its value link', () => {- const result = verify(mutate('| name | string | The name |\n\n## Owners by ID object', '\n## Owners by ID object'));- assert.deepEqual(findings(result), ['object-missing GET /owners (http-api/widgets#owners-by-id-object).name']);-});--test('an object linked where the schema declares no members fails', () => {- const result = verify(- mutate(- '| total | integer | Every matching widget |',- '| total | [part](#part-object) object | Every matching widget |',- ),- );- assert.deepEqual(findings(result), [- 'type-mismatch GET /widgets total: documented object, schema integer',- 'object-unresolved GET /widgets total: the schema declares no members for (http-api/widgets#part-object)',- ]);-});--test('a query table on a route with no query parameters fails unless the footnote says so', () => {- const table =- '### Query parameters\n\n| Field | Type | Description |\n| --- | --- | --- |\n| token<sup>1</sup> | string | The token |\n\n';- const anchor =- '### JSON body\n\n| Field | Type | Description |\n| --- | --- | --- |\n| name | string | The new name |';- const undocumented = verify(mutate(anchor, `${table}${anchor}`));- assert.deepEqual(findings(undocumented), ['query-extra PATCH /widgets/{} token']);- const note = '<sup>1</sup> Required, although the request schema does not declare it\n\n';- const footnoted = verify(mutate(anchor, `${table}${note}${anchor}`));- assert.deepEqual(findings(footnoted), []);-});--test('a body or response table on a route whose schema has neither fails', () => {- const result = verify(- mutate(- '| 204 | empty | The widgets were moved |',- '| 204 | empty | The widgets were moved |\n\n### Response body\n\n| Field | Type | Description |\n| --- | --- | --- |\n| moved | integer | Stale |',- ),- );- assert.deepEqual(findings(result), ['response-extra PUT /widgets/positions moved']);-});--test('each object linked for a union documents the members of its own variant', () => {- const result = verify(mutate('| kind | string | `square` |\n', ''));- assert.deepEqual(findings(result), [- 'object-missing PATCH /widgets/{} (http-api/widgets#widget-object).shape (http-api/widgets#circle-object, http-api/widgets#square-object).kind (variant 2)',- ]);-});--test('a member repeated by a second object linked for a union is type checked', () => {- const result = verify(mutate('| kind | string | `square` |', '| kind | integer | `square` |'));- assert.deepEqual(findings(result), [- 'type-mismatch PATCH /widgets/{} (http-api/widgets#widget-object).shape (http-api/widgets#circle-object, http-api/widgets#square-object).kind: documented integer, schema string',- ]);-});--test('a member with a declared shape and no linked object section is counted', () => {- const result = verify(mutate('| part | ?[part](#part-object) object | The part |', '| part | ?object | The part |'));- assert.deepEqual(findings(result), []);- assert.equal(result.counters.unlinkedObjects, 1);-});--test('a declared note naming a field the object schema does not have fails', () => {- const result = verify(- mutate(- 'The response schema also declares `unused`, which a part never has.',- 'The response schema also declares `unused` and `retired`, which a part never has.',- ),- );- assert.deepEqual(findings(result), [- 'object-stale-declared PATCH /widgets/{} (http-api/widgets#part-object).retired',- ]);-});--test('a declared note naming a field the response schema does not have fails', () => {- const result = verify(- mutate(- '| total | integer | Every matching widget |\n',- '| total | integer | Every matching widget |\n\nThe response schema also declares `cursor`.\n',- ),- );- assert.deepEqual(findings(result), ['response-stale-declared GET /widgets cursor']);-});--test('a not-declared footnote on a member the object schema declares fails', () => {- const result = verify(mutate('| name | string | The name |', '| name<sup>1</sup> | string | The name |'));- assert.deepEqual(findings(result), [- 'object-stale-undeclared PATCH /widgets/{} (http-api/widgets#widget-object).name',- ]);-});--test('a not-declared footnote on a declared body field or query parameter fails', () => {- const note = '<sup>1</sup> Required, although the request schema does not declare it\n\n';- const body = verify(- mutate('| size? | integer | The new size |', `| size?<sup>1</sup> | integer | The new size |\n\n${note.trimEnd()}`),- );- assert.deepEqual(findings(body), ['body-stale-undeclared PATCH /widgets/{} size']);- const query = verify(- mutate(- '| limit? | integer | The page size |\n| owner_id? | snowflake | Restrict to one owner |',- `| limit?<sup>1</sup> | integer | The page size |\n| owner_id? | snowflake | Restrict to one owner |\n\n${note.trimEnd()}`,- ),- );- assert.deepEqual(findings(query), ['query-stale-undeclared GET /widgets limit']);-});diff --git a/fluxer_docs/scripts/VerifyDocsSchemas.ts b/fluxer_docs/scripts/VerifyDocsSchemas.tsdeleted file mode 100644index a0618274d..000000000--- a/fluxer_docs/scripts/VerifyDocsSchemas.ts+++ /dev/null@@ -1,1307 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import {readFile, writeFile} from 'node:fs/promises';-import path from 'node:path';-import {fileURLToPath, pathToFileURL} from 'node:url';-import type {- OpenAPIOperation as Operation,- OpenAPISchema as SchemaNode,- OpenAPIDocument as Spec,-} from '@fluxer/openapi/src/OpenAPITypes';-import {readRouteHeaders} from './DocsRouteHeaders.ts';-import {- DOCS_ROOT,- HTTP_METHODS,- type MarkdownPage,- pageSlug,- readMarkdownPages,- routeShape,- slugifyHeading,- splitTableRow,- stripVersion,-} from './DocsSource.ts';--const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url));-const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json');-const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json');-interface Mismatch {- readonly page: string;- readonly operation: string;- readonly kind: string;- readonly detail: string;-}--interface SchemaCounters {- bodyTables: number;- bodiesByReference: number;- unionBodies: number;- queryTables: number;- responseTables: number;- responseObjects: number;- responseFieldsFound: number;- nestedObjects: number;- unlinkedObjects: number;- typesCompared: number;- optionalityCompared: number;-}--export interface SchemaVerification {- readonly mismatches: ReadonlyArray<Mismatch>;- readonly optionalityAdvisories: ReadonlyArray<string>;- readonly counters: SchemaCounters;-}--interface DocField {- readonly type: string | null;- readonly typed: boolean;- readonly optional: boolean;- readonly anchors: ReadonlyArray<string>;- readonly notes: ReadonlyArray<string>;- readonly undeclared: boolean;-}--interface Heading {- readonly level: number;- readonly text: string;- readonly line: number;- readonly anchors: Array<string>;- isObject: boolean;-}--interface Table {- readonly line: number;- readonly anchors: ReadonlyArray<string>;- readonly stack: ReadonlyArray<Heading>;- readonly header: ReadonlyArray<string>;- readonly rows: ReadonlyArray<ReadonlyArray<string>>;-}--interface ProseLine {- readonly line: number;- readonly stack: ReadonlyArray<Heading>;- readonly text: string;-}--interface ParsedPage {- readonly page: MarkdownPage;- readonly slug: string;- readonly headings: ReadonlyArray<Heading>;- readonly tables: ReadonlyArray<Table>;- readonly prose: ReadonlyArray<ProseLine>;-}--interface ObjectUsage {- readonly page: string;- readonly operation: string;- readonly anchors: string;- readonly own: ReadonlyMap<string, DocField>;- readonly properties: Set<string>;- open: boolean;-}--interface Inheritance {- readonly anchor: string;- readonly except: ReadonlySet<string>;-}--interface ObjectBlock {- readonly own: Map<string, DocField>;- readonly tables: Array<ReadonlyMap<string, DocField>>;- readonly inherits: Array<Inheritance>;- readonly links: Array<Inheritance>;- readonly undeclared: Set<string>;- readonly declaredOnly: Set<string>;-}--function resolveSchemaPointer(spec: Spec, reference: string): SchemaNode | boolean {- if (!reference.startsWith('#')) throw new Error(`Unsupported schema reference: ${reference}`);- const pointer = decodeURIComponent(reference.slice(1));- const prefix = '/components/schemas/';- if (!pointer.startsWith(prefix)) throw new Error(`Unsupported schema reference: ${reference}`);- let target: unknown = spec.components.schemas;- for (const token of pointer.slice(prefix.length).split('/')) {- if (/~(?:[^01]|$)/u.test(token)) throw new Error(`Invalid schema reference escape: ${reference}`);- const key = token.replace(/~1/gu, '/').replace(/~0/gu, '~');- if (- target === null ||- typeof target !== 'object' ||- (Array.isArray(target) && !/^(0|[1-9][0-9]*)$/u.test(key)) ||- !Object.hasOwn(target, key)- ) {- throw new Error(`Missing schema reference: ${reference}`);- }- target = (target as Record<string, unknown>)[key];- }- if (typeof target === 'boolean') return target;- if (target === null || typeof target !== 'object' || Array.isArray(target)) {- throw new Error(`Reference does not identify a schema: ${reference}`);- }- return target as SchemaNode;-}--function resolveRef(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): SchemaNode | undefined {- if (node == null || node === false) {- return undefined;- }- if (node === true) {- return {};- }- if (depth > 64) {- throw new Error('OpenAPI reference chain exceeds the supported depth');- }- if (node.$ref != null) {- const target = resolveSchemaPointer(spec, node.$ref);- const resolved = resolveRef(spec, target, depth + 1);- const {$ref, ...siblings} = node;- if (resolved == null || Object.keys(siblings).length === 0) {- return resolved;- }- if (Object.keys(resolved).length === 0) {- return siblings;- }- return {...resolved, allOf: [...(resolved.allOf ?? []), siblings]};- }- return node;-}--function asSchema(node: unknown): SchemaNode | boolean | undefined {- if (typeof node === 'boolean') return node;- if (node == null || typeof node !== 'object' || Array.isArray(node)) return undefined;- return node as SchemaNode;-}--function isSchemaNode(node: SchemaNode | boolean): node is SchemaNode {- return typeof node !== 'boolean';-}--function branchesOf(resolved: SchemaNode): Array<SchemaNode | boolean> {- return [...(resolved.allOf ?? []), ...(resolved.oneOf ?? []), ...(resolved.anyOf ?? [])];-}--function isUnion(resolved: SchemaNode | undefined): boolean {- return resolved != null && ((resolved.oneOf ?? []).length > 0 || (resolved.anyOf ?? []).length > 0);-}--function collectRequired(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): Set<string> {- const out = new Set<string>();- const resolved = resolveRef(spec, node, depth);- if (resolved == null) {- return out;- }- for (const name of resolved.required ?? []) {- out.add(name);- }- for (const branch of resolved.allOf ?? []) {- for (const name of collectRequired(spec, branch, depth + 1)) {- out.add(name);- }- }- for (const union of [resolved.oneOf ?? [], resolved.anyOf ?? []]) {- const branchRequirements = union.map((branch) => collectRequired(spec, branch, depth + 1));- for (const name of branchRequirements[0] ?? []) {- if (branchRequirements.every((required) => required.has(name))) {- out.add(name);- }- }- }- return out;-}--function collectTypes(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): Set<string> {- const resolved = resolveRef(spec, node, depth);- const out = new Set<string>();- if (resolved == null) {- return out;- }- if (resolved.format === 'snowflake') {- out.add('string');- return out;- }- for (const type of Array.isArray(resolved.type) ? resolved.type : [resolved.type]) {- if (type != null && type !== 'null') {- out.add(type);- }- }- for (const branch of branchesOf(resolved)) {- for (const type of collectTypes(spec, branch, depth + 1)) {- out.add(type);- }- }- return out;-}--function isDeprecatedProperty(property: unknown): boolean {- if (property == null || typeof property !== 'object') {- return false;- }- const node = property as {deprecated?: unknown; description?: unknown};- if (node.deprecated === true) {- return true;- }- return typeof node.description === 'string' && node.description.trimStart().toLowerCase().startsWith('deprecated');-}--function collectPropertySchemas(- spec: Spec,- node: SchemaNode | boolean | undefined,- depth = 0,- out = new Map<string, Array<SchemaNode | boolean>>(),- includeDeprecated = false,-): Map<string, Array<SchemaNode | boolean>> {- const resolved = resolveRef(spec, node, depth);- if (resolved == null) {- return out;- }- for (const [key, property] of Object.entries(resolved.properties ?? {})) {- if (!includeDeprecated && isDeprecatedProperty(property)) {- continue;- }- const schema = asSchema(property);- if (schema === undefined) {- continue;- }- const list = out.get(key) ?? [];- list.push(schema);- out.set(key, list);- }- for (const branch of branchesOf(resolved)) {- collectPropertySchemas(spec, branch, depth + 1, out, includeDeprecated);- }- return out;-}--function objectTarget(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): SchemaNode | undefined {- if (depth > 32) {- return undefined;- }- const resolved = resolveRef(spec, node, depth);- if (resolved == null) {- return undefined;- }- if (resolved.properties != null) {- return resolved;- }- const items = asSchema(resolved.items);- if (items !== undefined && collectTypes(spec, resolved).has('array')) {- return objectTarget(spec, items, depth + 1);- }- const values = asSchema(resolved.additionalProperties);- if (values !== undefined && isSchemaNode(values)) {- return objectTarget(spec, values, depth + 1);- }- const branches = branchesOf(resolved).filter((branch) => collectTypes(spec, branch).size > 0);- if (branches.length === 1) {- return objectTarget(spec, branches[0], depth + 1);- }- if (collectPropertySchemas(spec, resolved).size > 0) {- return resolved;- }- const targets = new Map<string, SchemaNode>();- for (const branch of branches) {- const target = objectTarget(spec, branch, depth + 1);- if (target != null) targets.set(JSON.stringify(target), target);- }- const [first, ...rest] = targets.values();- return rest.length === 0 ? first : {anyOf: [first, ...rest]};-}--function acceptsUndeclaredProperties(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): boolean {- const resolved = resolveRef(spec, node, depth);- if (resolved == null) return false;- const additional = resolved.additionalProperties;- if (resolved.properties != null && additional != null && additional !== false) return true;- return branchesOf(resolved).some((branch) => acceptsUndeclaredProperties(spec, branch, depth + 1));-}--const DOC_TYPE_TO_JSON = new Map([- ['snowflake', 'string'],- ['string', 'string'],- ['integer', 'integer'],- ['boolean', 'boolean'],- ['iso8601 timestamp', 'string'],- ['base64 string', 'string'],- ['float', 'number'],- ['number', 'number'],-]);--function normaliseDocType(cell: string): string | null {- const text = cell- .replace(/<sup>.*?<\/sup>/gu, '')- .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1')- .replace(/`/gu, '')- .trim()- .replace(/^\?/u, '')- .toLowerCase();- if (text.startsWith('array')) {- return 'array';- }- if (text.endsWith(' object') || text.includes('object')) {- return 'object';- }- return DOC_TYPE_TO_JSON.get(text) ?? null;-}--function typeAgrees(docType: string, specTypes: ReadonlySet<string>): boolean {- return specTypes.has(docType) || (docType === 'integer' && specTypes.has('number'));-}--function fieldNameText(cell: string): string {- return cell- .replace(/<sup>.*?<\/sup>/gu, '')- .replace(/\*\*/gu, '')- .replace(/`/gu, '')- .replace(/\\/gu, '')- .trim();-}--function cleanFieldName(cell: string): string | null {- const name = fieldNameText(cell).replace(/\?$/u, '');- if (name.length === 0) {- return null;- }- if (!/^[a-z_][a-z0-9_.]*$/iu.test(name)) {- return null;- }- return name;-}--function operationIndex(spec: Spec): Map<string, Operation> {- const index = new Map<string, Operation>();- for (const [routePath, item] of Object.entries(spec.paths)) {- for (const [method, operation] of Object.entries(item)) {- const upper = method.toUpperCase();- if (!HTTP_METHODS.has(upper)) continue;- const key = routeShape(upper, stripVersion(routePath));- if (index.has(key)) throw new Error(`Duplicate OpenAPI operation: ${key}`);- index.set(key, operation);- }- }- return index;-}--function documentReferences(page: string, line: string): Set<string> {- const references = new Set<string>();- for (const link of line.matchAll(/\]\(([^)\s]*)#([a-z0-9-]+)\)/gu)) {- if (/^[a-z][a-z0-9+.-]*:/iu.test(link[1])) {- continue;- }- const target =- link[1].length === 0- ? page- : link[1].startsWith('/')- ? link[1].slice(1)- : path.posix.join(path.posix.dirname(page), link[1]);- const slug = target- .replace(/\.(mdx|md)$/u, '')- .replace(/\/$/u, '')- .replace(/\/index$/u, '');- references.add(`${slug}#${link[2]}`);- }- return references;-}--function parsePage(page: MarkdownPage): ParsedPage {- const slug = pageSlug(page.relativePath);- const headings: Array<Heading> = [];- const tables: Array<Table> = [];- const prose: Array<ProseLine> = [];- const stack: Array<Heading> = [];- const pendingAnchors: Array<string> = [];- const tableAnchors: Array<string> = [];- let fence: string | null = null;- let table: {line: number; anchors: Array<string>; stack: ReadonlyArray<Heading>; rows: Array<Array<string>>} | null =- null;- const closeTable = () => {- if (table == null) return;- const [header, , ...rows] = table.rows;- tables.push({- line: table.line,- anchors: table.anchors,- stack: table.stack,- header: (header ?? []).map(fieldNameText),- rows,- });- table = null;- };- for (let i = 0; i < page.lines.length; i += 1) {- const line = page.lines[i];- const fenceMatch = line.match(/^\s*(```|~~~)/u);- if (fence != null) {- if (fenceMatch != null && fenceMatch[1] === fence) fence = null;- continue;- }- if (fenceMatch != null) {- closeTable();- fence = fenceMatch[1];- continue;- }- if (line.startsWith('|')) {- if (table == null) table = {line: i, anchors: tableAnchors.splice(0), stack: [...stack], rows: []};- table.rows.push(splitTableRow(line));- continue;- }- closeTable();- for (const explicit of line.matchAll(/<a\s+id=["']([^"']+)["']/gu)) {- const nextContent = page.lines.slice(i + 1).find((nextLine) => nextLine.trim().length > 0);- if (nextContent != null && /^#{2,6}\s/u.test(nextContent)) {- pendingAnchors.push(explicit[1]);- } else if (nextContent?.startsWith('|')) {- tableAnchors.push(explicit[1]);- } else if (stack.length > 0) {- stack[stack.length - 1].anchors.push(explicit[1]);- }- }- const heading = line.match(/^(#{2,6})\s+(.+?)\s*$/u);- if (heading != null) {- const level = heading[1].length;- const entry: Heading = {- level,- text: heading[2],- line: i,- anchors: [...new Set([slugifyHeading(heading[2]), ...pendingAnchors])],- isObject: /\bobject\b/iu.test(heading[2]),- };- pendingAnchors.length = 0;- while (stack.length > 0 && stack[stack.length - 1].level >= level) stack.pop();- stack.push(entry);- headings.push(entry);- continue;- }- if (line.trim().length > 0) {- prose.push({line: i, stack: [...stack], text: line});- }- }- closeTable();- return {page, slug, headings, tables, prose};-}--function tableFields(page: ParsedPage, table: Table): Map<string, DocField> {- const out = new Map<string, DocField>();- const nameColumn = table.header.indexOf('Field');- if (nameColumn === -1) {- return out;- }- const typeColumn = table.header.indexOf('Type');- for (const cells of table.rows) {- const nameCell = cells[nameColumn];- if (nameCell === undefined) continue;- const name = cleanFieldName(nameCell);- if (name == null) continue;- const typeCell = typeColumn === -1 ? undefined : cells[typeColumn];- out.set(name, {- type: typeCell === undefined ? null : normaliseDocType(typeCell),- typed: typeCell !== undefined,- optional: fieldNameText(nameCell).endsWith('?'),- anchors: typeCell === undefined ? [] : [...documentReferences(page.slug, typeCell)],- notes: [...nameCell.matchAll(/<sup>(\d+)<\/sup>/gu)].map((match) => match[1]),- undeclared: false,- });- }- return out;-}--function keyedValueAnchors(page: ParsedPage, table: Table): Array<string> {- const nameColumn = table.header.indexOf('Field');- const typeColumn = table.header.indexOf('Type');- if (nameColumn === -1 || typeColumn === -1) {- return [];- }- return table.rows- .filter((cells) => /^(?:{|\{)/u.test(fieldNameText(cells[nameColumn] ?? '')))- .flatMap((cells) => [...documentReferences(page.slug, cells[typeColumn] ?? '')]);-}--const BODY_HEADING = /^JSON body$|\bresponse body$/iu;--const RESPONSE_BODY_HEADING = /\bresponse body$/iu;--const STRUCTURE_HEADING = /(?:^|\s)structure$|^(?:additional|value) fields?$/iu;--const INHERITANCE = /\b(?:extends|every (?:field|member) of)\b/iu;--function inheritedAnchors(slug: string, text: string, everyLink = false): Array<Inheritance> {- const out: Array<Inheritance> = [];- for (const sentence of text.split(/(?<=\.)\s+(?=[A-Z])/u)) {- if (!everyLink && !INHERITANCE.test(sentence)) continue;- const exceptClause = sentence.match(/\b(?:except|without)\b(.*)$/u)?.[1] ?? '';- const except = new Set([...exceptClause.matchAll(/`([a-z_][a-z0-9_]*)`/giu)].map((match) => match[1]));- for (const anchor of documentReferences(slug, sentence)) {- out.push({anchor, except});- }- }- return out;-}--const DECLARED_ONLY = /\bschema also declares\b/iu;--function declaredOnlySentences(text: string): Array<string> {- return text.split(/(?<=\.)\s+(?=[A-Z])/u).filter((sentence) => DECLARED_ONLY.test(sentence));-}--function declaredOnlyNames(text: string): Array<string> {- return declaredOnlySentences(text).flatMap((sentence) =>- [...sentence.matchAll(/`([a-z_][a-z0-9_]*)`/giu)].map((match) => match[1]),- );-}--const UNDECLARED_NOTE = /^<sup>(\d+)<\/sup>.*\bschema does not declare\b/u;--function markUndeclared(fields: ReadonlyMap<string, DocField>, notes: ReadonlySet<string>): Map<string, DocField> {- const out = new Map<string, DocField>();- for (const [name, field] of fields) {- out.set(name, field.notes.some((note) => notes.has(note)) ? {...field, undeclared: true} : field);- }- return out;-}--function mergeGroups(groups: ReadonlyArray<ReadonlyMap<string, DocField>>): Map<string, DocField> {- const out = new Map<string, DocField>();- for (const group of groups) {- for (const [field, docField] of group) {- if (!out.has(field)) out.set(field, docField);- }- }- return out;-}--function innermostObject(stack: ReadonlyArray<Heading>): Heading | undefined {- for (let index = stack.length - 1; index >= 0; index -= 1) {- if (stack[index].isObject) return stack[index];- }- return undefined;-}--class Verifier {- readonly mismatches: Array<Mismatch> = [];- readonly optionalityAdvisories: Array<string> = [];- readonly counters: SchemaCounters = {- bodyTables: 0,- bodiesByReference: 0,- unionBodies: 0,- queryTables: 0,- responseTables: 0,- responseObjects: 0,- responseFieldsFound: 0,- nestedObjects: 0,- unlinkedObjects: 0,- typesCompared: 0,- optionalityCompared: 0,- };- private readonly objects = new Map<string, ObjectBlock>();- private readonly compared = new Set<string>();- private readonly unlinked = new Set<string>();- private readonly usages = new Map<string, ObjectUsage>();- private readonly routeSections = new Map<ParsedPage, ReadonlySet<Heading>>();- private readonly routeBodies = new Map<string, ReadonlyArray<ReadonlyMap<string, DocField>>>();- private readonly identities = new Map<string, number>();- private readonly parsed: ReadonlyArray<ParsedPage>;-- constructor(- pages: ReadonlyArray<MarkdownPage>,- private readonly mainSpec: Spec,- private readonly adminSpec: Spec,- ) {- this.parsed = pages.map(parsePage);- const typeTargets = new Set<string>();- for (const page of this.parsed) {- for (const table of page.tables) {- for (const field of tableFields(page, table).values()) {- for (const anchor of field.anchors) typeTargets.add(anchor);- }- if (table.header[0] !== 'Status') continue;- for (const row of table.rows) {- for (const anchor of documentReferences(page.slug, row[1] ?? '')) typeTargets.add(anchor);- }- }- for (const line of page.prose) {- if (!line.stack.some((heading) => heading.level === 3 && BODY_HEADING.test(heading.text.trim()))) continue;- for (const anchor of documentReferences(page.slug, line.text)) typeTargets.add(anchor);- }- }- for (const page of this.parsed) {- const routeLines = readRouteHeaders(page.page).map((header) => header.line);- const sections = page.headings.filter((heading) => heading.level === 2);- const routeSections = new Set(- sections.filter((heading, index) => {- const end = sections[index + 1]?.line ?? page.page.lines.length;- return routeLines.some((line) => line > heading.line && line <= end);- }),- );- this.routeSections.set(page, routeSections);- for (const heading of page.headings) {- if (routeSections.has(heading)) {- heading.isObject = false;- continue;- }- if (heading.anchors.some((anchor) => typeTargets.has(`${page.slug}#${anchor}`))) heading.isObject = true;- }- }- for (const page of this.parsed) {- this.indexObjects(page);- for (const section of this.routeSections.get(page) ?? []) {- const groups = page.tables- .filter(- (table) =>- table.stack.includes(section) &&- table.stack.some((heading) => heading.level === 3 && heading.text.trim() === 'JSON body') &&- (innermostObject(table.stack)?.level ?? 0) <= 2,- )- .map((table) => tableFields(page, table))- .filter((fields) => fields.size > 0);- for (const anchor of section.anchors) this.routeBodies.set(`${page.slug}#${anchor}`, groups);- }- }- }-- private schemaIdentity(node: SchemaNode): number {- const shape = JSON.stringify(node);- const known = this.identities.get(shape);- if (known !== undefined) return known;- const identity = this.identities.size;- this.identities.set(shape, identity);- return identity;- }-- private indexObjects(page: ParsedPage): void {- const blocks = new Map<Heading, ObjectBlock>();- for (const heading of page.headings) {- if (!heading.isObject) continue;- const block: ObjectBlock = {- own: new Map(),- tables: [],- inherits: [],- links: [],- undeclared: new Set(),- declaredOnly: new Set(),- };- blocks.set(heading, block);- for (const anchor of heading.anchors) {- this.objects.set(`${page.slug}#${anchor}`, block);- }- }- for (const table of page.tables) {- if (table.anchors.length > 0) {- const fields = tableFields(page, table);- const block: ObjectBlock = {- own: new Map(fields),- tables: [fields],- inherits: [],- links: [],- undeclared: new Set(),- declaredOnly: new Set(),- };- for (const anchor of table.anchors) this.objects.set(`${page.slug}#${anchor}`, block);- }- const owner = innermostObject(table.stack);- if (owner == null) continue;- const inner = table.stack[table.stack.length - 1];- if (inner !== owner && !STRUCTURE_HEADING.test(inner.text)) continue;- const block = blocks.get(owner);- for (const anchor of keyedValueAnchors(page, table)) block?.links.push({anchor, except: new Set()});- const fields = tableFields(page, table);- if (fields.size === 0) continue;- block?.tables.push(fields);- for (const [name, field] of fields) {- if (!block?.own.has(name)) block?.own.set(name, field);- }- }- for (const line of page.prose) {- const owner = innermostObject(line.stack);- const block = owner == null ? undefined : blocks.get(owner);- if (block == null) continue;- block.inherits.push(...inheritedAnchors(page.slug, line.text));- const note = line.text.match(UNDECLARED_NOTE);- if (note != null) block.undeclared.add(note[1]);- for (const name of declaredOnlyNames(line.text)) block.declaredOnly.add(name);- for (const anchor of documentReferences(page.slug, line.text)) {- block.links.push({anchor, except: new Set()});- }- }- for (const block of blocks.values()) {- if (block.undeclared.size === 0) continue;- for (const [name, field] of markUndeclared(block.own, block.undeclared)) block.own.set(name, field);- block.tables.splice(- 0,- block.tables.length,- ...block.tables.map((table) => markUndeclared(table, block.undeclared)),- );- }- }-- private parents(block: ObjectBlock): Array<Inheritance> {- const parents = block.own.size > 0 ? block.inherits : block.links;- return parents.filter((parent) => this.isObjectAnchor(parent.anchor));- }-- private isObjectAnchor(anchor: string): boolean {- return this.objects.has(anchor);- }-- private objectFields(anchors: Iterable<string>, seen = new Set<string>()): Map<string, DocField> {- const out = new Map<string, DocField>();- for (const anchor of anchors) {- if (seen.has(anchor)) continue;- seen.add(anchor);- const block = this.objects.get(anchor);- if (block == null) continue;- for (const [name, field] of block.own) {- if (!out.has(name)) out.set(name, field);- }- for (const parent of this.parents(block)) {- for (const [name, field] of this.objectFields([parent.anchor], seen)) {- if (!out.has(name) && !parent.except.has(name)) out.set(name, field);- }- }- }- return out;- }-- private coverageFields(anchors: Iterable<string>, extra: Iterable<string> = []): Set<string> {- const out = new Set<string>([...this.objectFields(anchors).keys(), ...extra]);- for (const anchor of anchors) {- for (const name of this.objects.get(anchor)?.declaredOnly ?? []) out.add(name);- }- return out;- }-- private ownFields(anchors: Iterable<string>): Map<string, DocField> {- const out = new Map<string, DocField>();- for (const anchor of anchors) {- const block = this.objects.get(anchor);- if (block == null) continue;- const own = block.own.size > 0 ? block.own : this.objectFields([anchor]);- for (const [name, field] of own) {- if (!out.has(name)) out.set(name, field);- }- }- return out;- }-- private push(page: string, operation: string, kind: string, detail: string): void {- this.mismatches.push({page, operation, kind, detail});- }-- private compareFields(- spec: Spec,- context: {page: string; operation: string; kind: string; label: string},- documented: Iterable<readonly [string, DocField]>,- coverage: ReadonlySet<string>,- schema: SchemaNode | boolean | undefined,- usages?: ReadonlyArray<ObjectUsage>,- ): number {- const properties = collectPropertySchemas(spec, schema);- const known = collectPropertySchemas(spec, schema, 0, new Map(), true);- const prefix = context.label.length > 0 ? `${context.label}.` : '';- const open = acceptsUndeclaredProperties(spec, schema);- for (const usage of usages ?? []) {- for (const field of known.keys()) usage.properties.add(field);- usage.open ||= open;- }- for (const [field, docField] of documented) {- const schemas = known.get(field);- if (schemas == null) {- if (usages == null && !open && !docField.undeclared) {- this.push(context.page, context.operation, `${context.kind}-extra`, `${prefix}${field}`);- }- continue;- }- if (usages == null && docField.undeclared) {- this.push(context.page, context.operation, `${context.kind}-stale-undeclared`, `${prefix}${field}`);- }- const specTypes = new Set<string>();- for (const node of schemas) {- for (const type of collectTypes(spec, node)) specTypes.add(type);- }- if (docField.type != null && specTypes.size > 0) {- this.counters.typesCompared += 1;- if (!typeAgrees(docField.type, specTypes)) {- this.push(- context.page,- context.operation,- 'type-mismatch',- `${prefix}${field}: documented ${docField.type}, schema ${[...specTypes].sort().join(' | ')}`,- );- }- }- const anchors = docField.anchors.filter((anchor) => this.isObjectAnchor(anchor));- const target: SchemaNode | boolean = schemas.length === 1 ? schemas[0] : {anyOf: schemas.filter(isSchemaNode)};- const nested = objectTarget(spec, target);- if (anchors.length === 0 && nested != null) {- this.unlinked.add(`${context.page} ${prefix}${field}`);- }- if (anchors.length > 0) {- if (nested != null) {- this.compareObject(spec, context.page, context.operation, anchors, target, `${prefix}${field}`);- } else if (!docField.undeclared && this.ownFields(anchors).size > 0) {- this.push(- context.page,- context.operation,- 'object-unresolved',- `${prefix}${field}: the schema declares no members for (${anchors.join(', ')})`,- );- }- }- }- let found = 0;- for (const field of properties.keys()) {- if (coverage.has(field)) {- found += 1;- continue;- }- this.push(context.page, context.operation, `${context.kind}-missing`, `${prefix}${field}`);- }- return found;- }-- private compareObject(- spec: Spec,- page: string,- operation: string,- anchors: ReadonlyArray<string>,- node: SchemaNode | boolean | undefined,- label: string,- extraCoverage: Iterable<string> = [],- ): number {- const target = objectTarget(spec, node);- if (target == null) {- return 0;- }- const specName = spec === this.adminSpec ? 'admin' : 'main';- const key = `${specName}|${[...anchors].sort().join(',')}|${this.schemaIdentity(target).toString()}`;- if (this.compared.has(key)) {- return 0;- }- this.compared.add(key);- this.counters.nestedObjects += 1;- const usages = anchors.map((anchor) => {- let usage = this.usages.get(anchor);- if (usage == null) {- usage = {page, operation, anchors: anchor, own: this.ownFields([anchor]), properties: new Set(), open: false};- this.usages.set(anchor, usage);- }- return usage;- });- const objectLabel = `${label.length > 0 ? `${label} ` : ''}(${anchors.join(', ')})`;- this.compareObjectVariants(spec, page, operation, anchors, target, objectLabel);- return this.compareFields(- spec,- {page, operation, kind: 'object', label: objectLabel},- anchors.flatMap((anchor) => [...this.ownFields([anchor])]),- this.coverageFields(anchors, extraCoverage),- target,- usages,- );- }-- private compareObjectVariants(- spec: Spec,- page: string,- operation: string,- anchors: ReadonlyArray<string>,- target: SchemaNode,- label: string,- ): void {- const branches = [...(target.oneOf ?? []), ...(target.anyOf ?? [])];- if (anchors.length < 2 || branches.length < 2) return;- const branchProperties = branches.map((branch) => new Set(collectPropertySchemas(spec, branch).keys()));- const documentedFor = branches.map(() => new Set<string>());- const documented = new Set<string>();- const matchedBranches = new Set<number>();- for (const anchor of anchors) {- const fields = [...this.ownFields([anchor]).keys()].filter((field) =>- branchProperties.some((properties) => properties.has(field)),- );- const candidates = branchProperties- .map((properties, index) => (fields.every((field) => properties.has(field)) ? index : -1))- .filter((index) => index !== -1);- if (candidates.length === 0) return;- const coverage = this.coverageFields([anchor]);- for (const field of coverage) documented.add(field);- for (const index of candidates) {- for (const field of coverage) documentedFor[index].add(field);- if (candidates.length < branches.length) matchedBranches.add(index);- }- }- for (const index of matchedBranches) {- for (const field of branchProperties[index]) {- if (documentedFor[index].has(field) || !documented.has(field)) continue;- this.push(page, operation, 'object-missing', `${label}.${field} (variant ${(index + 1).toString()})`);- }- }- }-- private objectTables(anchors: Iterable<string>, seen = new Set<string>()): Array<ReadonlyMap<string, DocField>> {- const out: Array<ReadonlyMap<string, DocField>> = [];- for (const anchor of anchors) {- if (seen.has(anchor)) continue;- seen.add(anchor);- const block = this.objects.get(anchor);- if (block == null) continue;- out.push(...block.tables);- for (const parent of this.parents(block)) {- for (const table of this.objectTables([parent.anchor], seen)) {- out.push(new Map([...table].filter(([name]) => !parent.except.has(name))));- }- }- }- return out;- }-- private compareBodyBranches(- spec: Spec,- page: string,- operation: string,- schema: SchemaNode | boolean,- groups: ReadonlyArray<ReadonlyMap<string, DocField>>,- ): void {- const resolved = resolveRef(spec, schema);- const allBranches = isUnion(resolved) ? [...(resolved?.oneOf ?? []), ...(resolved?.anyOf ?? [])] : [schema];- const branches = allBranches;- if (branches.length > 1) this.counters.unionBodies += 1;- const branchProperties = branches.map((branch) => new Set(collectPropertySchemas(spec, branch).keys()));- const branchRequired = branches.map((branch) => collectRequired(spec, branch));- const allProperties = new Set(branchProperties.flatMap((properties) => [...properties]));- const commonRequired = collectRequired(spec, schema);- const documentedFor = branches.map(() => new Set<string>());- const matchedBranches = new Set<number>();- for (const group of groups) {- const fields = [...group.keys()].filter((field) => allProperties.has(field));- const candidates = branchProperties- .map((properties, index) => (fields.every((field) => properties.has(field)) ? index : -1))- .filter((index) => index !== -1);- const specific = candidates.length > 0 && candidates.length < branches.length;- for (const index of candidates) {- for (const field of fields) documentedFor[index].add(field);- if (specific) matchedBranches.add(index);- }- for (const [field, docField] of group) {- if (!allProperties.has(field) || !docField.typed) continue;- const specRequired =- candidates.length > 0- ? candidates.every((index) => branchRequired[index].has(field))- : commonRequired.has(field);- this.counters.optionalityCompared += 1;- if (specRequired === !docField.optional) continue;- const detail = specRequired- ? `${field}: documented optional, schema marks it required`- : `${field}: documented required, schema marks it optional`;- this.optionalityAdvisories.push(`${page} ${operation} ${detail}`);- this.push(page, operation, 'optionality', detail);- }- }- const documented = new Set(groups.flatMap((group) => [...group.keys()]));- for (const index of matchedBranches) {- for (const field of branchProperties[index]) {- if (documentedFor[index].has(field) || !documented.has(field)) continue;- this.push(page, operation, 'body-missing', `${field} (variant ${(index + 1).toString()})`);- }- }- }-- private reportObjectExtras(): void {- for (const usage of this.usages.values()) {- for (const [field, docField] of usage.own) {- if (docField.undeclared && usage.properties.has(field)) {- this.push(usage.page, usage.operation, 'object-stale-undeclared', `(${usage.anchors}).${field}`);- }- }- if (usage.open) continue;- for (const [field, docField] of usage.own) {- if (usage.properties.has(field) || docField.undeclared) continue;- this.push(usage.page, usage.operation, 'object-extra', `(${usage.anchors}).${field}`);- }- for (const field of this.objects.get(usage.anchors)?.declaredOnly ?? []) {- if (!usage.properties.has(field)) {- this.push(usage.page, usage.operation, 'object-stale-declared', `(${usage.anchors}).${field}`);- }- }- }- }-- run(): SchemaVerification {- const mainIndex = operationIndex(this.mainSpec);- const adminIndex = operationIndex(this.adminSpec);- for (const page of this.parsed) {- if (page.page.relativePath.startsWith('media-proxy/')) continue;- const isAdmin = page.page.relativePath.startsWith('admin-api/');- this.verifyPage(page, isAdmin ? this.adminSpec : this.mainSpec, isAdmin ? adminIndex : mainIndex);- }- this.reportObjectExtras();- this.counters.unlinkedObjects = this.unlinked.size;- return {mismatches: this.mismatches, optionalityAdvisories: this.optionalityAdvisories, counters: this.counters};- }-- private verifyPage(page: ParsedPage, spec: Spec, index: ReadonlyMap<string, Operation>): void {- const relative = page.page.relativePath;- const routeHeaders = readRouteHeaders(page.page);- const sections = page.headings.filter((heading) => heading.level === 2);- for (let s = 0; s < sections.length; s += 1) {- const section = sections[s];- const end = sections[s + 1]?.line ?? page.page.lines.length;- const header = routeHeaders.find((candidate) => candidate.line > section.line && candidate.line <= end);- if (header == null) continue;- if (header.endLine > end) {- throw new Error(`${relative}:${header.line}: RouteHeader crosses a section boundary`);- }- const key = routeShape(header.method, stripVersion(header.path));- const operation = index.get(key);- if (operation == null) continue;- this.verifyRoute(page, spec, section, end, key, operation);- }- }-- private verifyRoute(- page: ParsedPage,- spec: Spec,- section: Heading,- end: number,- key: string,- operation: Operation,- ): void {- const relative = page.page.relativePath;- const inSection = (stack: ReadonlyArray<Heading>) => stack.includes(section);- const subsection = (stack: ReadonlyArray<Heading>) => {- const text = stack.find((heading) => heading.level === 3 && stack.includes(section))?.text.trim() ?? null;- return text != null && RESPONSE_BODY_HEADING.test(text) ? 'Response body' : text;- };- const sectionTables = page.tables.filter((table) => inSection(table.stack));- const sectionProse = page.prose.filter((line) => inSection(line.stack));-- const fieldsUnder = (name: string) => {- const groups: Array<ReadonlyMap<string, DocField>> = [];- const undeclared = new Set<string>();- for (const line of sectionProse) {- const note = subsection(line.stack) === name ? line.text.match(UNDECLARED_NOTE) : null;- if (note != null) undeclared.add(note[1]);- }- for (const table of sectionTables) {- if (subsection(table.stack) !== name) continue;- if ((innermostObject(table.stack)?.level ?? 0) > 2) continue;- const fields = markUndeclared(tableFields(page, table), undeclared);- if (fields.size > 0) groups.push(fields);- }- const tables = groups.length;- if (tables > 0) {- for (const line of sectionProse) {- if (subsection(line.stack) !== name) continue;- for (const parent of inheritedAnchors(page.slug, line.text)) {- for (const table of this.objectTables([parent.anchor])) {- groups.push(new Map([...table].filter(([field]) => !parent.except.has(field))));- }- }- }- }- return {fields: mergeGroups(groups), groups, tables};- };- const referencesUnder = (name: string) => {- const out: Array<Inheritance> = [];- for (const line of sectionProse) {- if (subsection(line.stack) !== name) continue;- for (const parent of inheritedAnchors(page.slug, line.text, true)) {- if (this.isObjectAnchor(parent.anchor)) out.push(parent);- }- }- return out;- };-- const jsonSchema = objectTarget(spec, operation.requestBody?.content?.['application/json']?.schema);- const body = fieldsUnder('JSON body');- const bodyReferences = referencesUnder('JSON body');- const hasBodySection = page.headings.some(- (heading) =>- heading.level === 3 && heading.text.trim() === 'JSON body' && heading.line > section.line && heading.line < end,- );- if (jsonSchema == null) {- for (const [field, docField] of body.fields) {- if (!docField.undeclared) this.push(relative, key, 'body-extra', field);- }- }- if (jsonSchema != null && !hasBodySection && collectPropertySchemas(spec, jsonSchema).size > 0) {- this.push(relative, key, 'body-missing', 'no JSON body section documents the request body');- }- if (jsonSchema != null && hasBodySection && collectPropertySchemas(spec, jsonSchema).size > 0) {- let documented = body.fields;- let groups = body.groups;- if (body.tables > 0) {- this.counters.bodyTables += 1;- } else if (bodyReferences.length > 0) {- this.counters.bodiesByReference += 1;- groups = bodyReferences.flatMap((parent) =>- this.objectTables([parent.anchor]).map(- (table) => new Map([...table].filter(([field]) => !parent.except.has(field))),- ),- );- documented = mergeGroups(groups);- } else {- const sameAs = sectionProse- .filter((line) => subsection(line.stack) === 'JSON body')- .flatMap((line) => [...documentReferences(page.slug, line.text)])- .flatMap((anchor) => this.routeBodies.get(anchor) ?? []);- if (sameAs.length > 0) {- this.counters.bodiesByReference += 1;- groups = sameAs;- documented = mergeGroups(groups);- }- }- this.compareFields(- spec,- {page: relative, operation: key, kind: 'body', label: ''},- documented,- new Set(documented.keys()),- jsonSchema,- );- this.compareBodyBranches(spec, relative, key, jsonSchema, groups);- }-- const query = fieldsUnder('Query parameters');- const actualQuery = new Set((operation.parameters ?? []).filter((p) => p.in === 'query').map((p) => p.name));- if (query.tables > 0) this.counters.queryTables += 1;- for (const [field, docField] of query.fields) {- if (!actualQuery.has(field) && !docField.undeclared) this.push(relative, key, 'query-extra', field);- if (actualQuery.has(field) && docField.undeclared) this.push(relative, key, 'query-stale-undeclared', field);- }- for (const field of actualQuery) {- if (!query.fields.has(field)) this.push(relative, key, 'query-missing', field);- }-- const success = Object.entries(operation.responses ?? {}).find(([status]) => status.startsWith('2'));- const responseSchema = success?.[1].content?.['application/json']?.schema;- const responseTarget = objectTarget(spec, responseSchema);- const responseBody = fieldsUnder('Response body');- if (responseTarget == null) {- for (const [field, docField] of responseBody.fields) {- if (!docField.undeclared) this.push(relative, key, 'response-extra', field);- }- return;- }- const responseDeclared = {anchors: new Set<string>(), names: new Set<string>()};- for (const line of sectionProse) {- const name = subsection(line.stack);- if (name !== 'Response' && name !== 'Response body') continue;- for (const sentence of declaredOnlySentences(line.text)) {- for (const reference of documentReferences(page.slug, sentence)) {- if (this.isObjectAnchor(reference)) responseDeclared.anchors.add(reference);- }- }- for (const field of declaredOnlyNames(line.text)) responseDeclared.names.add(field);- }- const responseKnown = collectPropertySchemas(spec, responseSchema, 0, new Map(), true);- if (!acceptsUndeclaredProperties(spec, responseSchema)) {- for (const field of responseDeclared.names) {- if (!responseKnown.has(field)) this.push(relative, key, 'response-stale-declared', field);- }- }- if (responseBody.tables > 0) {- this.counters.responseTables += 1;- this.counters.responseFieldsFound += this.compareFields(- spec,- {page: relative, operation: key, kind: 'response', label: ''},- responseBody.fields,- new Set([...responseBody.fields.keys(), ...responseDeclared.names]),- responseTarget,- );- return;- }- const statusAnchors = new Set<string>(responseDeclared.anchors);- for (const parent of referencesUnder('Response body')) statusAnchors.add(parent.anchor);- for (const table of sectionTables) {- if (subsection(table.stack) !== 'Response' || table.header[0] !== 'Status') continue;- for (const row of table.rows) {- if (!row[0]?.trim().startsWith('2')) continue;- for (const reference of documentReferences(page.slug, row[1] ?? '')) {- if (this.isObjectAnchor(reference)) statusAnchors.add(reference);- }- }- }- if (statusAnchors.size > 0) {- this.counters.responseObjects += 1;- this.counters.responseFieldsFound += this.compareObject(- spec,- relative,- key,- [...statusAnchors],- responseSchema,- '',- responseDeclared.names,- );- return;- }- for (const field of collectPropertySchemas(spec, responseTarget).keys()) {- if (!responseDeclared.names.has(field)) this.push(relative, key, 'response-missing', field);- }- }-}--export function verifyPages(pages: ReadonlyArray<MarkdownPage>, mainSpec: Spec, adminSpec: Spec): SchemaVerification {- return new Verifier(pages, mainSpec, adminSpec).run();-}--const KIND_LABELS = new Map([- ['extra', 'documented but not in the schema'],- ['missing', 'in the schema but undocumented'],- ['declared', 'named by a "schema also declares" sentence but not in the schema'],- ['undeclared', 'footnoted as not declared by the schema but in the schema'],-]);--async function main(): Promise<void> {- const mainSpec: Spec = JSON.parse(await readFile(MAIN_SPEC, 'utf8'));- const adminSpec: Spec = JSON.parse(await readFile(ADMIN_SPEC, 'utf8'));- const pages = await readMarkdownPages(DOCS_ROOT);- const {mismatches, optionalityAdvisories, counters} = verifyPages(pages, mainSpec, adminSpec);-- if (process.env.FLUXER_DOCS_SCHEMA_JSON != null) {- const payload: Record<string, Array<{operation: string; kind: string; field: string}>> = {};- for (const m of mismatches) {- const list = payload[m.page] ?? [];- list.push({operation: m.operation, kind: m.kind, field: m.detail});- payload[m.page] = list;- }- await writeFile(process.env.FLUXER_DOCS_SCHEMA_JSON, JSON.stringify(payload, null, 1));- console.log(`wrote ${process.env.FLUXER_DOCS_SCHEMA_JSON}`);- }-- const byKind = new Map<string, number>();- for (const m of mismatches) {- byKind.set(m.kind, (byKind.get(m.kind) ?? 0) + 1);- }-- console.log(`request body tables checked: ${counters.bodyTables.toString()}`);- console.log(`request bodies documented by reference to an object or route: ${counters.bodiesByReference.toString()}`);- console.log(`union request bodies checked: ${counters.unionBodies.toString()}`);- console.log(`query parameter tables checked: ${counters.queryTables.toString()}`);- console.log(`success responses checked against a response body table: ${counters.responseTables.toString()}`);- console.log(`success responses checked against a linked object section: ${counters.responseObjects.toString()}`);- console.log(`response fields found documented: ${counters.responseFieldsFound.toString()}`);- console.log(`linked object sections compared with their schema: ${counters.nestedObjects.toString()}`);- console.log(- `fields whose schema has members but whose row links no object section: ${counters.unlinkedObjects.toString()}`,- );- console.log(`request and response field types compared: ${counters.typesCompared.toString()}`);- console.log(`request body optionality compared: ${counters.optionalityCompared.toString()}`);- console.log(`optionality advisories: ${optionalityAdvisories.length.toString()}`);- for (const entry of optionalityAdvisories) {- console.log(` ${entry}`);- }- for (const [kind, count] of [...byKind.entries()].sort()) {- console.log(` ${kind}: ${count.toString()}`);- }- if (mismatches.length > 0) {- console.log('');- for (const m of mismatches.slice(0, 200)) {- let label = KIND_LABELS.get(m.kind.split('-').at(-1) ?? '') ?? m.kind;- if (m.kind === 'type-mismatch') label = 'type disagreement';- if (m.kind === 'optionality') label = 'optionality disagreement';- console.log(`${m.page} ${m.operation} ${label}: ${m.detail}`);- }- if (mismatches.length > 200) {- console.log(`... and ${(mismatches.length - 200).toString()} more`);- }- console.error(`FAIL: ${mismatches.length.toString()} field mismatches`);- process.exit(1);- }- console.log('OK: no field mismatches found in the checked tables and checked-in OpenAPI schemas');-}--if (process.argv[1] != null && import.meta.url === pathToFileURL(process.argv[1]).href) {- await main();-}diff --git a/fluxer_docs/scripts/VerifyDocsSidebar.ts b/fluxer_docs/scripts/VerifyDocsSidebar.tsdeleted file mode 100644index 2bc9434fb..000000000--- a/fluxer_docs/scripts/VerifyDocsSidebar.ts+++ /dev/null@@ -1,179 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import {readFile} from 'node:fs/promises';-import path from 'node:path';-import {fileURLToPath} from 'node:url';-import {DOCS_ROOT, listDraftMarkdownFiles, listMarkdownFiles, pageSlug, slugifyHeading} from './DocsSource.ts';--const ASTRO_CONFIG = fileURLToPath(new URL('../astro.config.ts', import.meta.url));--const SIDEBAR_START = 'sidebar: [';-const QUOTED_ENTRY = /'([^']+)'/gu;-const LINK_ENTRY = /link:\s*'([^']+)'/gu;--function sidebarBlock(source: string): string {- const start = source.indexOf(SIDEBAR_START);- if (start === -1) {- throw new Error('sidebar block not found in astro.config.ts');- }- let depth = 0;- for (let cursor = start + SIDEBAR_START.length - 1; cursor < source.length; cursor += 1) {- const character = source[cursor];- if (character === '[') {- depth += 1;- }- if (character === ']') {- depth -= 1;- if (depth === 0) {- return source.slice(start, cursor + 1);- }- }- }- throw new Error('sidebar block is unbalanced');-}--const config = await readFile(ASTRO_CONFIG, 'utf8');-const block = sidebarBlock(config);--const labels = new Set<string>();-for (const match of block.matchAll(/label:\s*'([^']+)'/gu)) {- labels.add(match[1]);-}--const linkSlugs = new Set<string>();-for (const match of block.matchAll(LINK_ENTRY)) {- linkSlugs.add(match[1].replace(/^\//u, '').replace(/\/$/u, ''));-}--const referenced = new Set<string>(linkSlugs);-for (const match of block.matchAll(QUOTED_ENTRY)) {- const value = match[1];- if (labels.has(value)) {- continue;- }- if (value.startsWith('/')) {- continue;- }- if (!value.includes('/') && !value.match(/^[a-z0-9-]+$/u)) {- continue;- }- referenced.add(value);-}--const files = await listMarkdownFiles(DOCS_ROOT);-const actual = new Set(files.map((file) => pageSlug(path.relative(DOCS_ROOT, file))));--const missingPages = [...referenced].filter((slug) => !actual.has(slug)).sort();-const orphanPages = [...actual].filter((slug) => !referenced.has(slug)).sort();--console.log(`sidebar entries: ${referenced.size.toString()}`);-console.log(`content pages: ${actual.size.toString()}`);-console.log(`draft pages withheld from the build: ${(await listDraftMarkdownFiles(DOCS_ROOT)).length.toString()}`);--let failures = 0;-if (missingPages.length > 0) {- console.log('');- console.log(`sidebar references a page that does not exist: ${missingPages.length.toString()}`);- for (const slug of missingPages) {- console.log(` ${slug}`);- }- failures += missingPages.length;-}-if (orphanPages.length > 0) {- console.log('');- console.log(`page exists but is not in the sidebar: ${orphanPages.length.toString()}`);- for (const slug of orphanPages) {- console.log(` ${slug === '' ? '(index)' : slug}`);- }- failures += orphanPages.length;-}--const INTERNAL_LINK = /\]\((\/[^)\s#]*)(#[^)\s]*)?\)/gu;-const SAME_PAGE_LINK = /\]\((#[^)\s]*)\)/gu;-const HEADING = /^#{2,6}\s+(.+?)\s*$/u;-const EXPLICIT_ANCHOR = /<[a-z]+\s+id=["']([^"']+)["']/gu;-const dangling: Array<string> = [];-const anchorable = new Set(actual);--const anchorsByPage = new Map<string, Set<string>>();-const ambiguousByPage = new Map<string, Set<string>>();-for (const file of files) {- const slug = pageSlug(path.relative(DOCS_ROOT, file));- const source = await readFile(file, 'utf8');- const anchors = new Set<string>();- const counts = new Map<string, number>();- for (const line of source.split('\n')) {- const heading = line.match(HEADING);- if (heading != null) {- const generated = slugifyHeading(heading[1]);- anchors.add(generated);- counts.set(generated, (counts.get(generated) ?? 0) + 1);- }- }- for (const explicit of source.matchAll(EXPLICIT_ANCHOR)) {- anchors.add(explicit[1]);- }- anchorsByPage.set(slug, anchors);- const ambiguous = new Set<string>();- for (const [generated, count] of counts) {- if (count > 1 && !source.includes(`id="${generated}"`)) {- ambiguous.add(generated);- }- }- ambiguousByPage.set(slug, ambiguous);-}--for (const file of files) {- const relative = path.relative(DOCS_ROOT, file);- const selfSlug = pageSlug(path.relative(DOCS_ROOT, file));- const source = await readFile(file, 'utf8');- for (const match of source.matchAll(INTERNAL_LINK)) {- const target = match[1].replace(/^\//u, '').replace(/\/$/u, '');- if (target.length === 0) {- if (match[2] != null) {- const fragment = match[2].slice(1);- const rootAnchors = anchorsByPage.get('');- if (rootAnchors != null && fragment.length > 0 && !rootAnchors.has(fragment)) {- dangling.push(`${relative} -> /#${fragment} (anchor)`);- }- }- continue;- }- if (!anchorable.has(target)) {- dangling.push(`${relative} -> /${target}/`);- continue;- }- if (match[2] != null) {- const fragment = match[2].slice(1);- const anchors = anchorsByPage.get(target);- if (fragment.length > 0 && anchors != null && !anchors.has(fragment)) {- dangling.push(`${relative} -> /${target}/#${fragment} (anchor)`);- } else if (fragment.length > 0 && (ambiguousByPage.get(target)?.has(fragment) ?? false)) {- dangling.push(`${relative} -> /${target}/#${fragment} (that heading repeats on the target page)`);- }- }- }- for (const match of source.matchAll(SAME_PAGE_LINK)) {- const fragment = match[1].slice(1);- const anchors = anchorsByPage.get(selfSlug);- if (fragment.length > 0 && anchors != null && !anchors.has(fragment)) {- dangling.push(`${relative} -> #${fragment} (same-page anchor)`);- } else if (fragment.length > 0 && (ambiguousByPage.get(selfSlug)?.has(fragment) ?? false)) {- dangling.push(`${relative} -> #${fragment} (that heading repeats on this page)`);- }- }-}-if (dangling.length > 0) {- console.log('');- console.log(`links to a page that does not exist: ${dangling.length.toString()}`);- for (const entry of [...new Set(dangling)].sort()) {- console.log(` ${entry}`);- }- failures += dangling.length;-}--if (failures > 0) {- console.error(`FAIL: ${failures.toString()} structure problems`);- process.exit(1);-}-console.log('OK: sidebar, content and internal links are in sync');diff --git a/fluxer_docs/scripts/VerifyDocsStyle.ts b/fluxer_docs/scripts/VerifyDocsStyle.tsdeleted file mode 100644index 1de2b58ef..000000000--- a/fluxer_docs/scripts/VerifyDocsStyle.ts+++ /dev/null@@ -1,545 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import {readdir, readFile} from 'node:fs/promises';-import path from 'node:path';-import {fileURLToPath} from 'node:url';-import {DOCS_ROOT, readMarkdownPages} from './DocsSource.ts';-import {- columnWidthPercents,- extractTables,- PX,- TABLE_BUDGET_PX,- TABLE_FLOOR_CAP_SHARE,- TABLE_MAX_CELL_CHARS,- TABLE_MAX_COLUMNS,- TABLE_MAX_IDENT_CHARS,- TABLE_WIDE_TIER_PX,-} from './DocsTableWidth.ts';--const STYLES_ROOT = fileURLToPath(new URL('../src/styles/', import.meta.url));-const STARLIGHT_STYLES = fileURLToPath(new URL('../node_modules/@astrojs/starlight/dist/style/', import.meta.url));--const FORBIDDEN_MARKETING_WORDS = [- 'seamless',- 'effortless',- 'blazing',- 'cutting-edge',- 'game-changing',- 'powerful and',- 'just simply',- 'simply use',- 'simply call',- 'simply add',- 'simply set',-];--const FORBIDDEN_TOPICS = [- {pattern: /push subscription/iu, reason: 'push API does not exist in the live era'},- {pattern: /\/push\/events/u, reason: 'push events API does not exist in the live era'},- {pattern: /voice[- ]public[- ]key/iu, reason: 'voice connection API does not exist in the live era'},- {pattern: /match-public-key/u, reason: 'voice connection API does not exist in the live era'},-];--const CANONICAL_HEADINGS = new Map([- ['### JSON params', '### JSON body'],- ['### Query string parameters', '### Query parameters'],- ['### Response Body', '### Response body'],- ['### JSON Body', '### JSON body'],- ['### Query Parameters', '### Query parameters'],-]);--interface Finding {- readonly file: string;- readonly line: number;- readonly rule: string;- readonly detail: string;-}--function frontmatterOf(source: string): string | null {- if (!source.startsWith('---\n')) {- return null;- }- const end = source.indexOf('\n---', 4);- if (end === -1) {- return null;- }- return source.slice(4, end);-}--const pages = await readMarkdownPages(DOCS_ROOT, {includeDrafts: true});-const findings: Array<Finding> = [];--for (const {file, relativePath: relative, source, lines} of pages) {- const frontmatter = frontmatterOf(source);-- if (frontmatter == null) {- findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing frontmatter block'});- } else {- if (!/^# SPDX-License-Identifier: AGPL-3\.0-or-later$/mu.test(frontmatter)) {- findings.push({file: relative, line: 2, rule: 'spdx', detail: 'missing SPDX comment in frontmatter'});- }- if (!/^title:\s*\S/mu.test(frontmatter)) {- findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing title'});- }- if (!/^description:\s*\S/mu.test(frontmatter)) {- findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing description'});- }- }-- const usesRouteHeader = source.includes('<RouteHeader');- const importsRouteHeader = source.includes("import RouteHeader from '@/components/RouteHeader.astro'");- if (usesRouteHeader && !importsRouteHeader) {- findings.push({file: relative, line: 1, rule: 'import', detail: 'uses RouteHeader without importing it'});- }- if (usesRouteHeader && !file.endsWith('.mdx')) {- findings.push({file: relative, line: 1, rule: 'extension', detail: 'uses RouteHeader but is not .mdx'});- }-- let fenced = false;- for (let index = 0; index < lines.length; index += 1) {- const line = lines[index];- const number = index + 1;- const insideFence = fenced;- if (line.startsWith('```')) {- fenced = !fenced;- }- if (insideFence) {- continue;- }- if (line.includes('—')) {- findings.push({file: relative, line: number, rule: 'em-dash', detail: line.trim().slice(0, 100)});- }- for (const word of FORBIDDEN_MARKETING_WORDS) {- if (line.toLowerCase().includes(word)) {- findings.push({file: relative, line: number, rule: 'tone', detail: `contains "${word.trim()}"`});- }- }- const canonical = CANONICAL_HEADINGS.get(line.trim());- if (canonical != null) {- findings.push({- file: relative,- line: number,- rule: 'heading',- detail: `use "${canonical}" for consistency with the rest of the corpus`,- });- }- for (const topic of FORBIDDEN_TOPICS) {- if (topic.pattern.test(line)) {- findings.push({file: relative, line: number, rule: 'unshipped', detail: topic.reason});- }- }- }-}--for (const {relativePath: relative, lines} of pages) {- const sectionStarts: Array<number> = [];- for (let index = 0; index < lines.length; index += 1) {- if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) {- sectionStarts.push(index);- }- }- sectionStarts.push(lines.length);- for (let s = 0; s + 1 < sectionStarts.length; s += 1) {- let responseBody = -1;- let response = -1;- for (let index = sectionStarts[s]; index < sectionStarts[s + 1]; index += 1) {- const heading = lines[index].trim();- if (heading === '### Response body' && responseBody === -1) {- responseBody = index;- }- if (heading === '### Response' && response === -1) {- response = index;- }- }- if (responseBody !== -1 && response !== -1 && responseBody > response) {- findings.push({- file: relative,- line: response + 1,- rule: 'order',- detail: '"### Response body" must precede "### Response"',- });- }- }-}--for (const {relativePath: relative, lines} of pages) {- let block = new Map<string, number>();- let sinceFootnote = 0;- for (let index = 0; index < lines.length; index += 1) {- const definition = lines[index].match(/^<sup>(\d+)<\/sup>\s/u);- if (definition == null) {- if (lines[index].trim().length > 0) {- sinceFootnote += 1;- }- if (sinceFootnote > 1) {- block = new Map<string, number>();- }- continue;- }- sinceFootnote = 0;- const previous = block.get(definition[1]);- if (previous != null) {- findings.push({- file: relative,- line: index + 1,- rule: 'footnote',- detail: `footnote ${definition[1]} is defined twice in the same block, first at line ${previous.toString()}`,- });- continue;- }- block.set(definition[1], index + 1);- }-}--const NOTATION_EXAMPLE = 'A superscript marker such as <sup>1</sup> refers to the numbered footnote';--for (const {relativePath: relative, source, lines} of pages) {- const explainsNotation = source.includes(NOTATION_EXAMPLE);- const bounds: Array<number> = [0];- for (let index = 0; index < lines.length; index += 1) {- if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) {- bounds.push(index);- }- }- bounds.push(lines.length);- for (let b = 0; b + 1 < bounds.length; b += 1) {- const definitions = new Map<string, number>();- const references = new Set<string>();- for (let index = bounds[b]; index < bounds[b + 1]; index += 1) {- const definition = lines[index].match(/^<sup>(\d+)<\/sup>\s/u);- if (definition != null) {- if (!definitions.has(definition[1])) {- definitions.set(definition[1], index + 1);- }- continue;- }- for (const marker of lines[index].matchAll(/<sup>(\d+)<\/sup>/gu)) {- references.add(marker[1]);- }- }- for (const [number, line] of definitions) {- if (!references.has(number)) {- findings.push({- file: relative,- line,- rule: 'footnote',- detail: `footnote ${number} is defined but never referenced in its section`,- });- }- }- for (const number of references) {- if (definitions.has(number)) {- continue;- }- if (explainsNotation) {- continue;- }- findings.push({- file: relative,- line: bounds[b] + 1,- rule: 'footnote',- detail: `footnote ${number} is referenced but never defined in its section`,- });- }- }-}--type TableRule =- | 'table-columns'- | 'table-fit'- | 'table-cell'- | 'table-identifier'- | 'table-rows'- | 'table-parallel'- | 'table-wide-tight';--const TABLE_RULES: ReadonlyArray<TableRule> = [- 'table-columns',- 'table-fit',- 'table-cell',- 'table-identifier',- 'table-rows',- 'table-parallel',- 'table-wide-tight',-];--const TABLE_COUNT_FLOOR = 1800;--const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRule, number>>>>([- ['admin-api/applications.mdx', {'table-identifier': 1}],- ['admin-api/discovery.mdx', {'table-identifier': 1}],- ['admin-api/gateway.mdx', {'table-fit': 1, 'table-identifier': 3}],- ['admin-api/guilds.mdx', {'table-identifier': 3}],- ['admin-api/index.mdx', {'table-identifier': 3}],- ['admin-api/instance.mdx', {'table-identifier': 8}],- ['admin-api/messages.mdx', {'table-identifier': 1}],- ['admin-api/reports.mdx', {'table-fit': 1, 'table-identifier': 3}],- ['admin-api/users.mdx', {'table-identifier': 4}],- ['admin-api/voice.mdx', {'table-identifier': 3}],- ['gateway/event-filtering.md', {'table-cell': 2}],- ['gateway/events.md', {'table-identifier': 1}],- ['gateway/opcodes-and-close-codes.md', {'table-cell': 1}],- ['gateway/overview.md', {'table-cell': 1}],- ['http-api/authentication.mdx', {'table-identifier': 1}],- ['http-api/billing.mdx', {'table-identifier': 4}],- ['http-api/calls.mdx', {'table-fit': 1, 'table-cell': 3}],- ['http-api/channels.mdx', {'table-cell': 6, 'table-identifier': 2}],- ['http-api/connections.mdx', {'table-cell': 2}],- ['http-api/discovery.mdx', {'table-cell': 2}],- ['http-api/donations.mdx', {'table-cell': 1}],- ['http-api/entrance-sounds.mdx', {'table-cell': 3, 'table-parallel': 1}],- ['http-api/forums.mdx', {'table-identifier': 2}],- ['http-api/gifs.mdx', {'table-cell': 5}],- ['http-api/gifts.mdx', {'table-cell': 1}],- ['http-api/guild-audit-logs.mdx', {'table-identifier': 3}],- ['http-api/guild-channels.mdx', {'table-cell': 2, 'table-identifier': 1}],- ['http-api/guild-emojis.mdx', {'table-cell': 4}],- ['http-api/guild-members.mdx', {'table-identifier': 2}],- ['http-api/guild-moderation.mdx', {'table-cell': 1}],- ['http-api/guild-stickers.mdx', {'table-cell': 3}],- ['http-api/guilds.mdx', {'table-fit': 1, 'table-identifier': 5}],- ['http-api/instance.mdx', {'table-identifier': 8}],- ['http-api/invites.mdx', {'table-cell': 6}],- ['http-api/messages.mdx', {'table-fit': 2, 'table-cell': 20, 'table-identifier': 2}],- ['http-api/permissions.mdx', {'table-cell': 8}],- ['http-api/premium.mdx', {'table-identifier': 5}],- ['http-api/reports.mdx', {'table-identifier': 1}],- ['http-api/search.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1}],- ['http-api/streams.mdx', {'table-cell': 5}],- ['http-api/unfurl.mdx', {'table-cell': 2, 'table-parallel': 1}],- ['http-api/users.mdx', {'table-fit': 1, 'table-identifier': 3}],- ['http-api/users/content.mdx', {'table-cell': 2}],- ['http-api/users/current-user.mdx', {'table-identifier': 1}],- ['http-api/users/data-harvest.mdx', {'table-cell': 2}],- ['http-api/users/email-and-password.mdx', {'table-identifier': 1}],- ['http-api/users/mfa.mdx', {'table-cell': 3, 'table-parallel': 1}],- ['http-api/users/private-channels.mdx', {'table-cell': 3}],- ['http-api/users/relationships.mdx', {'table-fit': 2, 'table-cell': 3, 'table-identifier': 2}],- ['http-api/users/settings-protobuf.md', {'table-fit': 2, 'table-identifier': 7, 'table-parallel': 1}],- ['http-api/users/settings.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1, 'table-parallel': 1}],- ['http-api/webhooks.mdx', {'table-identifier': 1, 'table-parallel': 1}],- ['media-proxy/responses-and-limits.md', {'table-cell': 2}],- ['media-proxy/routes.mdx', {'table-cell': 1}],- ['operator/kubernetes.mdx', {'table-identifier': 1}],- ['voice/index.md', {'table-parallel': 1}],-]);--const tableFindingsByPage = new Map<string, Map<TableRule, Array<Finding>>>();-let tablesMeasured = 0;-const overWideTier: Array<Finding> = [];--for (const {relativePath: relative, source} of pages) {- for (const table of extractTables(source)) {- tablesMeasured += 1;- const raise = (rule: TableRule, detail: string): void => {- const perRule = tableFindingsByPage.get(relative) ?? new Map<TableRule, Array<Finding>>();- const list = perRule.get(rule) ?? [];- list.push({file: relative, line: table.line, rule, detail});- perRule.set(rule, list);- tableFindingsByPage.set(relative, perRule);- };- const shape = table.header.join(' | ');- if (table.columns > TABLE_MAX_COLUMNS) {- raise(- 'table-columns',- `[${shape}] has ${table.columns.toString()} columns, at most ${TABLE_MAX_COLUMNS.toString()} are allowed`,- );- }- if (table.minDemandPx > TABLE_BUDGET_PX) {- raise(- 'table-fit',- `[${shape}] demands ${table.minDemandPx.toString()}px against the ${TABLE_BUDGET_PX.toString()}px budget, columns ${table.columnMinPx.join('/')}px`,- );- }- if (table.minDemandPx > TABLE_WIDE_TIER_PX) {- overWideTier.push({- file: relative,- line: table.line,- rule: 'table-wide-tier',- detail: `[${shape}] demands ${table.minDemandPx.toString()}px, over the ${TABLE_WIDE_TIER_PX.toString()}px width where the wide CSS tier switches to content sizing, so this table overflows its column instead of wrapping`,- });- }- if (table.worstCellChars > TABLE_MAX_CELL_CHARS) {- raise(- 'table-cell',- `[${shape}] column ${(table.worstCellColumn + 1).toString()} holds ${table.worstCellChars.toString()} rendered characters, over ${TABLE_MAX_CELL_CHARS.toString()}: "${table.worstCellText.slice(0, 60)}..."`,- );- }- if (table.col1IdentChars > TABLE_MAX_IDENT_CHARS) {- raise(- 'table-identifier',- `[${shape}] first column holds "${table.col1IdentText}", ${table.col1IdentChars.toString()} characters, over ${TABLE_MAX_IDENT_CHARS.toString()}`,- );- }- if (table.rows < 2 && !table.canonical) {- raise(- 'table-rows',- `[${shape}] has ${table.rows.toString()} body row(s) and is not a canonical schema shape, so it is a sentence`,- );- }- if (!table.parallel) {- raise('table-parallel', `[${shape}] ${table.nonParallelReason}`);- }- const shares = columnWidthPercents(- table.columnMinPx.map((minPx, index) => ({minPx, maxPx: table.columnMaxPx[index]})),- TABLE_WIDE_TIER_PX,- );- const floorCap = TABLE_WIDE_TIER_PX * TABLE_FLOOR_CAP_SHARE;- for (const [index, share] of shares.entries()) {- const allotted = (share / 100) * TABLE_WIDE_TIER_PX;- const unbreakable = table.columnMinPx[index] + PX.cellPad;- if (unbreakable <= floorCap && allotted + PX.proseFallback < unbreakable) {- raise(- 'table-wide-tight',- `[${shape}] column ${(index + 1).toString()} is allotted ${Math.round(allotted).toString()}px but its longest unbreakable word needs ${Math.round(unbreakable).toString()}px, so that word breaks mid-word even at the widest tier`,- );- }- }- }-}--if (tablesMeasured < TABLE_COUNT_FLOOR) {- findings.push({- file: 'scripts/DocsTableWidth.ts',- line: 1,- rule: 'table-extractor',- detail: `only ${tablesMeasured.toString()} tables were measured, floor is ${TABLE_COUNT_FLOOR.toString()}. The table rules have gone blind`,- });-}--findings.push(...overWideTier);--const acceptedByRule = new Map<TableRule, number>();-for (const [relative, perRule] of tableFindingsByPage) {- const allowance = ACCEPTED_TABLE_FINDINGS.get(relative) ?? {};- for (const [rule, list] of perRule) {- const allowed = allowance[rule] ?? 0;- acceptedByRule.set(rule, (acceptedByRule.get(rule) ?? 0) + Math.min(allowed, list.length));- for (const finding of list.slice(allowed)) {- findings.push(finding);- }- }-}-for (const [relative, allowance] of ACCEPTED_TABLE_FINDINGS) {- const perRule = tableFindingsByPage.get(relative);- for (const rule of TABLE_RULES) {- const allowed = allowance[rule] ?? 0;- const actual = perRule?.get(rule)?.length ?? 0;- if (allowed > actual) {- findings.push({- file: relative,- line: 1,- rule: 'table-ledger',- detail: `the ledger allows ${allowed.toString()} ${rule} findings but the page now produces ${actual.toString()}. Lower the entry in VerifyDocsStyle.ts`,- });- }- }-}--interface CssBlock {- readonly selector: string;- readonly body: string;- readonly line: number;-}--function cssBlocks(source: string): Array<CssBlock> {- const blocks: Array<CssBlock> = [];- const pattern = /([^{}]+)\{([^{}]*)\}/gu;- let match: RegExpExecArray | null = pattern.exec(source);- while (match != null) {- blocks.push({- selector: match[1]- .replace(/\/\*[\s\S]*?\*\//gu, ' ')- .replace(/\s+/gu, ' ')- .trim(),- body: match[2],- line: source.slice(0, match.index).split('\n').length,- });- match = pattern.exec(source);- }- return blocks;-}--const SIDEWAYS_SCROLL = /(?:^|;)\s*overflow(?:-x)?\s*:\s*(?:auto|scroll)\s*(?:;|$)/mu;-const SCROLL_FORBIDDEN_SELECTOR =- /\btable\b|\bthead\b|\btbody\b|\btr\b|\.sl-markdown-content\s*(?:,|$)|table-wrapper|overflow-wrapper/u;--for (const entry of await readdir(STYLES_ROOT, {withFileTypes: true})) {- if (!entry.name.endsWith('.css')) {- continue;- }- const source = await readFile(path.join(STYLES_ROOT, entry.name), 'utf8');- for (const block of cssBlocks(source)) {- if (!SCROLL_FORBIDDEN_SELECTOR.test(block.selector) || !SIDEWAYS_SCROLL.test(block.body)) {- continue;- }- findings.push({- file: `src/styles/${entry.name}`,- line: block.line,- rule: 'table-scroll',- detail: `"${block.selector}" makes a table or the markdown container a sideways scroller. No table may ever scroll horizontally`,- });- }-}--{- const starlightTableScrollers: Array<string> = [];- for (const entry of await readdir(STARLIGHT_STYLES, {withFileTypes: true})) {- if (!entry.name.endsWith('.css')) {- continue;- }- const source = await readFile(path.join(STARLIGHT_STYLES, entry.name), 'utf8');- for (const block of cssBlocks(source)) {- if (block.selector.includes('table') && SIDEWAYS_SCROLL.test(block.body)) {- starlightTableScrollers.push(`${entry.name}:${block.line.toString()} ${block.selector}`);- }- }- }- const content = await readFile(path.join(STYLES_ROOT, 'content.css'), 'utf8');- const neutralised = cssBlocks(content).some(- (block) =>- /\.sl-markdown-content\s+table\b/u.test(block.selector) &&- /(?:^|;)\s*display\s*:\s*table\s*(?:;|$)/mu.test(block.body) &&- /(?:^|;)\s*overflow\s*:\s*visible\s*(?:;|$)/mu.test(block.body),- );- if (starlightTableScrollers.length > 0 && !neutralised) {- findings.push({- file: 'src/styles/content.css',- line: 1,- rule: 'table-scroll',- detail: `Starlight still ships a scrolling table (${starlightTableScrollers.join(', ')}) and content.css no longer overrides it with "display: table" and "overflow: visible"`,- });- }-}--const byRule = new Map<string, number>();-for (const finding of findings) {- byRule.set(finding.rule, (byRule.get(finding.rule) ?? 0) + 1);-}--console.log(`pages checked: ${pages.length.toString()}`);-console.log(`tables measured: ${tablesMeasured.toString()}`);--for (const rule of TABLE_RULES) {- console.log(- ` ${rule}: ${(byRule.get(rule) ?? 0).toString()} new, ${(acceptedByRule.get(rule) ?? 0).toString()} accepted`,- );-}-for (const [rule, count] of [...byRule.entries()].sort()) {- if ((TABLE_RULES as ReadonlyArray<string>).includes(rule)) {- continue;- }- console.log(` ${rule}: ${count.toString()}`);-}-if (findings.length > 0) {- console.log('');- for (const finding of findings.slice(0, 200)) {- console.log(`${finding.file}:${finding.line.toString()} [${finding.rule}] ${finding.detail}`);- }- if (findings.length > 200) {- console.log(`... and ${(findings.length - 200).toString()} more`);- }- console.error(`FAIL: ${findings.length.toString()} style problems`);- process.exit(1);-}-console.log('OK: house style clean');diff --git a/fluxer_docs/src/table/DocsTableMetrics.ts b/fluxer_docs/src/table/DocsTableMetrics.tsindex e258d47a3..acd5dc039 100644--- a/fluxer_docs/src/table/DocsTableMetrics.ts+++ b/fluxer_docs/src/table/DocsTableMetrics.ts@@ -27,9 +27,6 @@ export const TABLE_FLOOR_CAP_SHARE = 1 / 3;export const TABLE_MIDDLE_TIER_PX = 768;export const TABLE_WIDE_TIER_PX = 960;-export const TABLE_MAX_COLUMNS = 3;-export const TABLE_MAX_CELL_CHARS = 120;-export const TABLE_MAX_IDENT_CHARS = 24;const TABLE_FIXED_REFERENCE_PX = TABLE_BUDGET_PX;