Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Add expectKnownBug and a guard for known reds and CI skips (#5119)

Nick Gomezabb5fa922577
diff --git a/docs/vitest.real-source.config.mts b/docs/vitest.real-source.config.mts
index 64dd8ea55..fd53b03c6 100644
--- a/docs/vitest.real-source.config.mts
+++ b/docs/vitest.real-source.config.mts
@@ -1,5 +1,6 @@
import { fileURLToPath } from 'node:url';
import mdx from 'fumadocs-mdx/vite';
+import type { Plugin } from 'vite';
import { defineConfig } from 'vitest/config';
import { okVitestBase } from '../test-support/vitest.base';
import * as Config from './source.config.ts';
@@ -18,9 +19,25 @@ const collectionJsonModule = {
},
};
+// UPSTREAM(fuma-nama/fumadocs#3469): fumadocs-mdx 14.0.x's config hook returns the whole config merged with its additions, so Vite merges every array twice; drop this wrapper once the dependency carries that fix
+function withConfigAdditionsOnly(plugin: Plugin): Plugin {
+ const addConfig = plugin.config;
+ if (typeof addConfig !== 'function') return plugin;
+ return {
+ ...plugin,
+ config(_config, env) {
+ return addConfig.call(this, {}, env);
+ },
+ };
+}
+
export default defineConfig(async () => ({
...okVitestBase,
- plugins: [...okVitestBase.plugins, await mdx(Config, { outDir: OUT_DIR }), collectionJsonModule],
+ plugins: [
+ ...okVitestBase.plugins,
+ withConfigAdditionsOnly(await mdx(Config, { outDir: OUT_DIR })),
+ collectionJsonModule,
+ ],
test: {
...okVitestBase.test,
include: ['src/**/*.real-source.test.ts'],
diff --git a/package.json b/package.json
index 7511c93ff..d4602b408 100644
--- a/package.json
+++ b/package.json
@@ -24,6 +24,7 @@
"typecheck": "turbo run typecheck",
"test": "turbo run test",
"test:uncached": "vitest run --config vitest.uncached.config.ts",
+ "known-reds": "node scripts/known-reds.mjs",
"test:vitest-selftest": "vitest run",
"smoke:share-v2": "node --import tsx --conditions=development packages/server/share-v2-smoke.ts",
"lint": "pnpm run lint:biome && pnpm run lint:oxlint",
diff --git a/packages/app/tests/lint-plugins/ok-rules-vocabulary.test.ts b/packages/app/tests/lint-plugins/ok-rules-vocabulary.test.ts
index 6265ca6d5..3a4542260 100644
--- a/packages/app/tests/lint-plugins/ok-rules-vocabulary.test.ts
+++ b/packages/app/tests/lint-plugins/ok-rules-vocabulary.test.ts
@@ -216,8 +216,11 @@ describe('ok-rules corpus vocabulary', () => {
).toEqual([]);
});
- test('on a full checkout the exempt directories are held to the same standard', () => {
- if (!existsSync(join(REPO_ROOT, MIRROR_ABSENT_PACKAGE_ROOT))) return;
+ test('on a full checkout the exempt directories are held to the same standard', (ctx) => {
+ ctx.skip(
+ !existsSync(join(REPO_ROOT, MIRROR_ABSENT_PACKAGE_ROOT)),
+ 'the public mirror omits the exempt package root',
+ );
const counts = filesPerDir();
expect(ABSENT_ON_MIRROR.filter((dir) => (counts.get(dir) ?? 0) === 0)).toEqual([]);
});
diff --git a/packages/app/tests/perf/scenarios/sweep-convention-cap-graduation-real-cycle.test.ts b/packages/app/tests/perf/scenarios/sweep-convention-cap-graduation-real-cycle.test.ts
index 5c16b0af7..86754db26 100644
--- a/packages/app/tests/perf/scenarios/sweep-convention-cap-graduation-real-cycle.test.ts
+++ b/packages/app/tests/perf/scenarios/sweep-convention-cap-graduation-real-cycle.test.ts
@@ -25,14 +25,12 @@ async function isDevServerReachable(target: string): Promise<boolean> {
describe('convention-cap-graduation sweep — real-cycle integration', () => {
test.skipIf(!INTEGRATION_GATE)(
'one real localhost cycle produces a well-formed CycleOutcome',
- async () => {
+ async (ctx) => {
const reachable = await isDevServerReachable(DEFAULT_TARGET);
- if (!reachable) {
- console.warn(
- `[sweep-real-cycle] OK_SWEEP_INTEGRATION=1 but ${DEFAULT_TARGET} is not reachable — start the dev server with: cd packages/app && bun run dev`,
- );
- return;
- }
+ ctx.skip(
+ !reachable,
+ `OK_SWEEP_INTEGRATION=1 but ${DEFAULT_TARGET} is not reachable — start the dev server with: cd packages/app && pnpm dev`,
+ );
const browser = await chromium.launch({
headless: true,
diff --git a/packages/desktop/src/main/check-target-exists.test.ts b/packages/desktop/src/main/check-target-exists.test.ts
index 550390f9c..0c9a9c264 100644
--- a/packages/desktop/src/main/check-target-exists.test.ts
+++ b/packages/desktop/src/main/check-target-exists.test.ts
@@ -220,8 +220,11 @@ describe('checkTargetExists', () => {
).toEqual('missing');
});
- test('handles unreadable directory (EACCES) as unreadable, not missing', () => {
- if (process.platform === 'win32' || process.getuid?.() === 0) return;
+ test('handles unreadable directory (EACCES) as unreadable, not missing', (ctx) => {
+ ctx.skip(
+ process.platform === 'win32' || process.getuid?.() === 0,
+ 'a mode-000 directory stays readable on Windows and to root, so EACCES cannot be provoked',
+ );
const project = makeProject();
try {
mkdirSync(join(project, 'locked'), { recursive: true });
@@ -430,8 +433,8 @@ describe('checkProjectDirExists', () => {
}
});
- test('returns unreadable when a present folder has an unreadable parent (EACCES)', () => {
- if (isRoot) return;
+ test('returns unreadable when a present folder has an unreadable parent (EACCES)', (ctx) => {
+ ctx.skip(isRoot, 'root reads a mode-000 directory, so EACCES cannot be provoked');
const parent = makeTmp();
const child = join(parent, 'proj');
mkdirSync(child);
diff --git a/packages/desktop/tests/integration/packaged-helper-runs-as-node.test.ts b/packages/desktop/tests/integration/packaged-helper-runs-as-node.test.ts
index 0d54fba91..77031b4e7 100644
--- a/packages/desktop/tests/integration/packaged-helper-runs-as-node.test.ts
+++ b/packages/desktop/tests/integration/packaged-helper-runs-as-node.test.ts
@@ -40,21 +40,13 @@ const packagedHelperBinary = haveDarwin ? findPackagedHelperBinary() : null;
const havePackagedBuild = packagedHelperBinary !== null;
describe('packaged helper binary runs under ELECTRON_RUN_AS_NODE=1', () => {
- test('test environment gate (packaged build present)', () => {
- if (!haveDarwin) {
- console.log(
- `[packaged-helper-runs-as-node] platform=${process.platform} — darwin-only test, skipping`,
- );
- return;
- }
- if (!havePackagedBuild) {
- console.log(
- `[packaged-helper-runs-as-node] no packaged helper binary found under ` +
- `${distDesktopDir}/mac-<arch>/OpenKnowledge.app/... — run ` +
- `\`bunx electron-builder --dir --publish never\` (or \`okdesk\`) to enable this test`,
- );
- return;
- }
+ test('test environment gate (packaged build present)', (ctx) => {
+ ctx.skip(!haveDarwin, `darwin-only test; platform=${process.platform}`);
+ ctx.skip(
+ !havePackagedBuild,
+ `no packaged helper binary found under ${distDesktopDir}/mac-<arch>/OpenKnowledge.app/... — run ` +
+ `\`pnpm exec electron-builder --dir --publish never\` (or \`okdesk\`) to enable this test`,
+ );
expect(existsSync(packagedHelperBinary as string)).toBe(true);
});
diff --git a/packages/desktop/tests/main/menu.test.ts b/packages/desktop/tests/main/menu.test.ts
index 251b1b95f..38dd82196 100644
--- a/packages/desktop/tests/main/menu.test.ts
+++ b/packages/desktop/tests/main/menu.test.ts
@@ -466,8 +466,8 @@ describe('buildMenuTemplate', () => {
expect(roles).toContain('minimize');
});
- test('Close tab click dispatches deps.onCloseActiveTabOrWindow on macOS', () => {
- if (process.platform !== 'darwin') return;
+ test('Close tab click dispatches deps.onCloseActiveTabOrWindow on macOS', (ctx) => {
+ ctx.skip(process.platform !== 'darwin', 'macOS-only menu behaviour');
const onCloseActiveTabOrWindow = vi.fn(() => {});
const template = buildMenuTemplate(makeDeps({ onCloseActiveTabOrWindow }));
const closeTab = findByLabel(template, 'Close tab');
diff --git a/packages/desktop/tests/unit/electron-builder-cli-native-deps.test.ts b/packages/desktop/tests/unit/electron-builder-cli-native-deps.test.ts
index ad385c88f..9cc72b826 100644
--- a/packages/desktop/tests/unit/electron-builder-cli-native-deps.test.ts
+++ b/packages/desktop/tests/unit/electron-builder-cli-native-deps.test.ts
@@ -206,19 +206,16 @@ describe('@inkeep/open-knowledge-native-config ships its napi loader + platform
expect(existsSync(resolve(nativeConfigDir, 'package.json'))).toBe(true);
});
- test('the napi-built loader + a platform binary exist after a build', () => {
+ test('the napi-built loader + a platform binary exist after a build', (ctx) => {
const loader = resolve(nativeConfigDir, 'index.js');
const nodeBinaries = existsSync(nativeConfigDir)
? readdirSync(nativeConfigDir).filter((f) => f.endsWith('.node'))
: [];
- if (!existsSync(loader) || nodeBinaries.length === 0) {
- console.warn(
- `[electron-builder-cli-native-deps] SKIP: ${NATIVE_CONFIG} not built ` +
- `(no index.js / *.node in ${nativeConfigDir}). Run \`pnpm run build\` first; ` +
- 'the gate builds it upstream of this tier.',
- );
- return;
- }
+ ctx.skip(
+ !existsSync(loader) || nodeBinaries.length === 0,
+ `${NATIVE_CONFIG} not built (no index.js / *.node in ${nativeConfigDir}). ` +
+ 'Run `pnpm run build` first; the gate builds it upstream of this tier.',
+ );
expect(nodeBinaries.length).toBeGreaterThan(0);
if (process.platform === 'darwin' && process.arch === 'arm64') {
expect(nodeBinaries).toContain('native-config.darwin-arm64.node');
@@ -246,20 +243,17 @@ describe('@inkeep/open-knowledge-native-config ships bundled in cli/dist/native'
}
});
- test('the bundled loader + platform binary exist in cli/dist/native after a build', () => {
+ test('the bundled loader + platform binary exist in cli/dist/native after a build', (ctx) => {
const nativeBundle = resolve(cliDist, 'native');
const loader = resolve(nativeBundle, 'index.js');
const pkgJson = resolve(nativeBundle, 'package.json');
const nodeBinaries = existsSync(nativeBundle)
? readdirSync(nativeBundle).filter((f) => f.endsWith('.node'))
: [];
- if (!existsSync(loader) || nodeBinaries.length === 0) {
- console.warn(
- '[electron-builder-cli-native-deps] SKIP: cli/dist/native not built ' +
- `(no index.js / *.node in ${nativeBundle}). Run \`pnpm run build\` first.`,
- );
- return;
- }
+ ctx.skip(
+ !existsSync(loader) || nodeBinaries.length === 0,
+ `cli/dist/native not built (no index.js / *.node in ${nativeBundle}). Run \`pnpm run build\` first.`,
+ );
expect(existsSync(pkgJson)).toBe(true);
expect(nodeBinaries.length).toBeGreaterThan(0);
});
diff --git a/packages/desktop/tests/unit/linux-package-native-guards.test.ts b/packages/desktop/tests/unit/linux-package-native-guards.test.ts
index 05d8926ef..eb736b5ee 100644
--- a/packages/desktop/tests/unit/linux-package-native-guards.test.ts
+++ b/packages/desktop/tests/unit/linux-package-native-guards.test.ts
@@ -89,13 +89,11 @@ describe('Linux package native-version guard helpers', () => {
});
describe('Linux package native guard lane visibility', () => {
- test('reports whether OK_LINUX_PACKAGE_DIR enabled the assertive lane', () => {
- if (packageDir === null) {
- console.log(
- '[linux-package-native-guards] OK_LINUX_PACKAGE_DIR is unset — packaged-artifact assertions skipped',
- );
- return;
- }
+ test('reports whether OK_LINUX_PACKAGE_DIR enabled the assertive lane', (ctx) => {
+ ctx.skip(
+ packageDir === null,
+ 'OK_LINUX_PACKAGE_DIR is unset, so the packaged-artifact assertions are skipped',
+ );
console.log(`[linux-package-native-guards] checking ${packageDir}`);
expect(packageDirInput).not.toBe('');
});
diff --git a/packages/desktop/tests/unit/preload-single-entry.test.ts b/packages/desktop/tests/unit/preload-single-entry.test.ts
index 410318d16..eafa285c6 100644
--- a/packages/desktop/tests/unit/preload-single-entry.test.ts
+++ b/packages/desktop/tests/unit/preload-single-entry.test.ts
@@ -24,11 +24,9 @@ describe('preload bundle self-containment', () => {
expect(Object.keys(await preloadInput())).toEqual(['index']);
});
- test('the built preload requires only specifiers the sandbox can resolve', () => {
+ test('the built preload requires only specifiers the sandbox can resolve', (ctx) => {
const built = resolve(desktopRoot, 'out', 'preload', 'index.js');
- if (!existsSync(built)) {
- return;
- }
+ ctx.skip(!existsSync(built), 'the desktop preload is not built (out/preload/index.js)');
const requires = [...readFileSync(built, 'utf-8').matchAll(/require\(["']([^"']+)["']\)/g)].map(
(match) => match[1],
);
diff --git a/packages/desktop/tests/unit/verify-native-config-driver.test.mjs b/packages/desktop/tests/unit/verify-native-config-driver.test.mjs
index 4e0c571cc..14ae7fe06 100644
--- a/packages/desktop/tests/unit/verify-native-config-driver.test.mjs
+++ b/packages/desktop/tests/unit/verify-native-config-driver.test.mjs
@@ -174,14 +174,11 @@ describe('runDriver (full orchestration)', () => {
});
describe('real bundle (end-to-end, requires a built CLI)', () => {
- test('loads + round-trips the actual packages/cli/dist/native bundle', async () => {
- if (!existsSync(resolve(cliDist, 'native', 'index.js'))) {
- console.warn(
- '[verify-native-config-driver] SKIP: packages/cli/dist/native not built. ' +
- 'Run `bun run build` (the gate builds it upstream of this tier).',
- );
- return;
- }
+ test('loads + round-trips the actual packages/cli/dist/native bundle', async (ctx) => {
+ ctx.skip(
+ !existsSync(resolve(cliDist, 'native', 'index.js')),
+ 'packages/cli/dist/native not built. Run `pnpm run build` (the gate builds it upstream of this tier).',
+ );
const messages = [];
const code = await runDriver(['node', 'script', cliDist], {
writeStream: (s) => messages.push(s),
diff --git a/packages/server/src/acp/browser-mcp.test.ts b/packages/server/src/acp/browser-mcp.test.ts
index 0f428d4d7..70cc6a55a 100644
--- a/packages/server/src/acp/browser-mcp.test.ts
+++ b/packages/server/src/acp/browser-mcp.test.ts
@@ -759,9 +759,9 @@ describe.skipIf(process.platform === 'win32')('the browser server launch, run fo
const probeArgs = (args: readonly string[]) =>
args.map((arg) => (arg === AGENT_BROWSER_MCP_PACKAGE ? `${PROBE_PACKAGE}@1.0.0` : arg));
- test("never runs the project's own npm packages or reads its .npmrc", () => {
+ test("never runs the project's own npm packages or reads its .npmrc", (ctx) => {
const npx = resolveOnPath('npx', process.env.PATH);
- if (npx === null) return;
+ ctx.skip(npx === null, 'npx is not on PATH');
const root = tmp();
const project = hostileProject(root);
const prefix = isolatedPrefix(root);
diff --git a/packages/server/src/git-preflight.test.ts b/packages/server/src/git-preflight.test.ts
index 1905ef557..28c4fe23b 100644
--- a/packages/server/src/git-preflight.test.ts
+++ b/packages/server/src/git-preflight.test.ts
@@ -399,10 +399,11 @@ describe('detectGit (integration)', () => {
expect(['PATH', 'fallback']).toContain(detected.source);
});
- test('Stage 2 fallback fires when PATH has no git and a fallback candidate exists', () => {
- if (process.platform === 'win32') {
- return;
- }
+ test('Stage 2 fallback fires when PATH has no git and a fallback candidate exists', (ctx) => {
+ ctx.skip(
+ process.platform === 'win32',
+ 'the PATH override and the fallback candidates are POSIX',
+ );
const originalPath = process.env.PATH;
process.env.PATH = '/nonexistent-stage2-fallback-test-xyz';
try {
diff --git a/packages/server/src/saved-themes-write.test.ts b/packages/server/src/saved-themes-write.test.ts
index 81c27a309..b2906df31 100644
--- a/packages/server/src/saved-themes-write.test.ts
+++ b/packages/server/src/saved-themes-write.test.ts
@@ -412,14 +412,17 @@ describe('updateSavedTheme', () => {
expect(readFileSync(path, 'utf-8')).toBe(original);
});
- test('updates only the exact lowercase stem when a mixed-case pair exists', async () => {
+ test('updates only the exact lowercase stem when a mixed-case pair exists', async (ctx) => {
const dir = savedThemesDir(home);
mkdirSync(dir, { recursive: true });
const uppercasePath = join(dir, 'Ocean.yaml');
const lowercasePath = join(dir, 'ocean.yaml');
const uppercase = base16ToYaml(scheme('Uppercase stem'));
writeFileSync(uppercasePath, uppercase);
- if (existsSync(lowercasePath)) return;
+ ctx.skip(
+ existsSync(lowercasePath),
+ 'a case-insensitive filesystem holds Ocean.yaml and ocean.yaml as one file',
+ );
writeFileSync(lowercasePath, base16ToYaml(scheme('Lowercase stem')));
expect(
@@ -530,14 +533,17 @@ describe('deleteSavedTheme', () => {
expect(readFileSync(path, 'utf-8')).toBe(original);
});
- test('deletes only the exact lowercase stem when a mixed-case pair exists', async () => {
+ test('deletes only the exact lowercase stem when a mixed-case pair exists', async (ctx) => {
const dir = savedThemesDir(home);
mkdirSync(dir, { recursive: true });
const uppercasePath = join(dir, 'Ocean.yaml');
const lowercasePath = join(dir, 'ocean.yaml');
const uppercase = base16ToYaml(scheme('Uppercase stem'));
writeFileSync(uppercasePath, uppercase);
- if (existsSync(lowercasePath)) return;
+ ctx.skip(
+ existsSync(lowercasePath),
+ 'a case-insensitive filesystem holds Ocean.yaml and ocean.yaml as one file',
+ );
writeFileSync(lowercasePath, base16ToYaml(scheme('Lowercase stem')));
expect(await deleteSavedTheme({ id: 'saved-ocean', homedirOverride: home })).toEqual({
diff --git a/packages/server/src/spawn-detached.test.ts b/packages/server/src/spawn-detached.test.ts
index 2ee42412a..800225fb1 100644
--- a/packages/server/src/spawn-detached.test.ts
+++ b/packages/server/src/spawn-detached.test.ts
@@ -28,8 +28,8 @@ describe('spawnDetached — error classification', () => {
expect(outcome).toEqual({ ok: false, reason: 'not-installed' });
});
- test('EACCES (no exec permission on POSIX) → { ok: false, reason: "not-installed" }', async () => {
- if (process.platform === 'win32') return;
+ test('EACCES (no exec permission on POSIX) → { ok: false, reason: "not-installed" }', async (ctx) => {
+ ctx.skip(process.platform === 'win32', 'exec permission bits are POSIX-only');
const dir = await mkdtemp(join(tmpdir(), 'ok-spawn-noexec-'));
try {
const script = join(dir, 'noexec.sh');
diff --git a/scripts/check-known-reds.uncached.test.mjs b/scripts/check-known-reds.uncached.test.mjs
new file mode 100644
index 000000000..3e1b3867d
--- /dev/null
+++ b/scripts/check-known-reds.uncached.test.mjs
@@ -0,0 +1,827 @@
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterAll, describe, expect, test } from 'vitest';
+import { okVitestBase } from '../test-support/vitest.base.ts';
+import { gitCleanEnv } from './git-clean-env.mjs';
+import {
+ ALLOWLIST_PATH,
+ CONVENTION_DOC,
+ EXPIRY_WARNING_DAYS,
+ expiringSoon,
+ KNOWN_BUG_TAG,
+ listTestFiles,
+ loadAllowlist,
+ MAX_HORIZON_DAYS,
+ MIN_SIGNATURE_LITERAL,
+ OK_ROOT,
+ OWNERS,
+ QUARANTINE_TAG,
+ render,
+ scanSource,
+ scanTree,
+ todayUtc,
+ validate,
+} from './known-reds.mjs';
+
+const SCRIPT = fileURLToPath(new URL('./known-reds.mjs', import.meta.url));
+const ISSUE = 'https://github.com/inkeep/agents-private/issues/1056';
+const TODAY = '2026-10-01';
+const NO_ALLOWLIST = { ciSkips: [] };
+
+const vitestPin = ({
+ tags = "['known-bug']",
+ meta = `{ issue: '${ISSUE}', owner: 'get-main-green', until: '2026-10-15' }`,
+ declare = 'test',
+ body = 'await expectKnownBug(/row still visible/, async () => { expect(1).toBe(2); });',
+} = {}) => `
+import { expect, test } from 'vitest';
+import { expectKnownBug } from '../../test-support/known-bug.vitest.test-helper';
+${declare}('deleted file leaves the sidebar', { tags: ${tags}, meta: ${meta} }, async () => {
+ ${body}
+});
+`;
+
+const playwrightPin = (configure) => `
+import { expect, test } from '@playwright/test';
+import { expectKnownBug } from '../../../../test-support/known-bug.test-helper';
+test.describe('sidebar', () => {
+ ${configure}
+ test('hide removes the row', {
+ tag: '@known-bug',
+ annotation: [
+ { type: 'issue', description: '${ISSUE}' },
+ { type: 'owner', description: 'get-main-green' },
+ { type: 'until', description: '2026-10-15' },
+ ],
+ }, async ({ page }) => {
+ await expectKnownBug(/unexpected value "visible"/, async () => {
+ await expect(page.getByRole('treeitem')).toBeHidden();
+ });
+ });
+});
+`;
+
+const rules = (scan) => scan.problems.map((problem) => problem.rule);
+const ciSkipCount = (path, source) =>
+ scanSource(path, source).gates.filter((gate) => gate.ci === 'skips-on-ci').length;
+
+describe('known-reds scanner: pins and quarantines', () => {
+ test('reads a Vitest pin with its fields and signature', () => {
+ const scan = scanSource('packages/app/src/a.test.ts', vitestPin());
+ expect(scan.problems).toEqual([]);
+ expect(scan.pins).toEqual([
+ {
+ path: 'packages/app/src/a.test.ts',
+ line: 4,
+ runner: 'vitest',
+ title: 'deleted file leaves the sidebar',
+ issue: ISSUE,
+ owner: 'get-main-green',
+ until: '2026-10-15',
+ signature: '/row still visible/',
+ },
+ ]);
+ });
+
+ test('a test without the known-bug tag is not a pin', () => {
+ const scan = scanSource('packages/app/src/a.test.ts', vitestPin({ tags: "['slow']" }));
+ expect(scan.pins).toEqual([]);
+ expect(scan.problems).toEqual([]);
+ });
+
+ test('accepts a single-string tag', () => {
+ expect(
+ scanSource('packages/app/src/a.test.ts', vitestPin({ tags: "'known-bug'" })).pins,
+ ).toHaveLength(1);
+ });
+
+ test('refuses a tag list the listing cannot read', () => {
+ expect(
+ rules(scanSource('packages/app/src/a.test.ts', vitestPin({ tags: 'PIN_TAGS' }))),
+ ).toEqual(['tags-not-literal']);
+ });
+
+ test('refuses meta fields the listing cannot read', () => {
+ const meta = `{ issue: ISSUE_URL, owner: 'get-main-green', until: '2026-10-15' }`;
+ expect(rules(scanSource('packages/app/src/a.test.ts', vitestPin({ meta })))).toEqual([
+ 'fields-not-literal',
+ ]);
+ });
+
+ test('a pin must assert through expectKnownBug', () => {
+ const scan = scanSource(
+ 'packages/app/src/a.test.ts',
+ vitestPin({ body: 'expect(1).toBe(2);' }),
+ );
+ expect(rules(scan)).toEqual(['pin-without-helper']);
+ });
+
+ test('a pin signature must be a regular-expression literal', () => {
+ const body = "await expectKnownBug('row still visible', () => { expect(1).toBe(2); });";
+ expect(rules(scanSource('packages/app/src/a.test.ts', vitestPin({ body })))).toEqual([
+ 'signature-not-literal',
+ ]);
+ });
+
+ test.each(['/./', '/.*/', '/[a-z]+/', '/a.b.c/', '/.{10,20}/', '/\\w{2,10}/'])(
+ 'refuses the signature %s, which names no wrong outcome',
+ (signature) => {
+ const body = `await expectKnownBug(${signature}, () => { expect(1).toBe(2); });`;
+ expect(rules(scanSource('packages/app/src/a.test.ts', vitestPin({ body })))).toEqual([
+ 'signature-too-broad',
+ ]);
+ },
+ );
+
+ test('accepts a signature with a literal run of the minimum length', () => {
+ expect(MIN_SIGNATURE_LITERAL).toBe(4);
+ for (const signature of ['/ENOENT/', '/\\d+ items/', '/Received: "visible"/i', '/\\{ab\\}/']) {
+ const body = `await expectKnownBug(${signature}, () => { expect(1).toBe(2); });`;
+ expect(rules(scanSource('packages/app/src/a.test.ts', vitestPin({ body })))).toEqual([]);
+ }
+ });
+
+ test('a test tagged both known-bug and quarantine is refused', () => {
+ const scan = scanSource(
+ 'packages/app/src/a.test.ts',
+ vitestPin({ tags: "['known-bug', 'quarantine']" }),
+ );
+ expect(rules(scan)).toEqual(['pin-and-quarantine']);
+ });
+
+ test('reads a single-object Playwright annotation, and refuses a non-literal one', () => {
+ const single = playwrightPin('test.describe.configure({ retries: 0 });').replace(
+ /annotation: \[[\s\S]*?\],/,
+ `annotation: { type: 'issue', description: '${ISSUE}' },`,
+ );
+ const singleScan = scanSource('packages/app/tests/stress/a.e2e.ts', single);
+ expect(singleScan.problems).toEqual([]);
+ expect(singleScan.pins[0]).toMatchObject({ issue: ISSUE });
+ expect(singleScan.pins[0].owner).toBeUndefined();
+ const nonLiteral = playwrightPin('test.describe.configure({ retries: 0 });').replace(
+ `{ type: 'issue', description: '${ISSUE}' }`,
+ "{ type: 'issue', description: ISSUE_URL }",
+ );
+ expect(rules(scanSource('packages/app/tests/stress/a.e2e.ts', nonLiteral))).toEqual([
+ 'fields-not-literal',
+ ]);
+ });
+
+ test('a skipped pin is refused, because it checks nothing', () => {
+ expect(
+ rules(scanSource('packages/app/src/a.test.ts', vitestPin({ declare: 'test.skip' }))),
+ ).toEqual(['pin-not-run']);
+ });
+
+ test('a known-bug tag on a describe is refused', () => {
+ const source = `
+import { describe, test } from 'vitest';
+describe('sidebar', { tags: ['known-bug'] }, () => {
+ test('x', () => {});
+});
+`;
+ expect(rules(scanSource('packages/app/src/a.test.ts', source))).toEqual(['tag-on-describe']);
+ });
+
+ test('a Playwright pin runs with retries: 0 in its describe', () => {
+ const ok = scanSource(
+ 'packages/app/tests/stress/a.e2e.ts',
+ playwrightPin('test.describe.configure({ retries: 0 });'),
+ );
+ expect(ok.problems).toEqual([]);
+ expect(ok.pins).toHaveLength(1);
+ expect(ok.pins[0]).toMatchObject({
+ runner: 'playwright',
+ issue: ISSUE,
+ owner: 'get-main-green',
+ until: '2026-10-15',
+ });
+ expect(rules(scanSource('packages/app/tests/stress/a.e2e.ts', playwrightPin('')))).toEqual([
+ 'pin-retries',
+ ]);
+ expect(
+ rules(
+ scanSource(
+ 'packages/app/tests/stress/a.e2e.ts',
+ playwrightPin('test.describe.configure({ retries: 2 });'),
+ ),
+ ),
+ ).toEqual(['pin-retries']);
+ });
+
+ test('the nearest retries setting decides, not an outer one', () => {
+ const source = `test.describe.configure({ retries: 0 });\n${playwrightPin('test.describe.configure({ retries: 1 });')}`;
+ expect(rules(scanSource('packages/app/tests/stress/a.e2e.ts', source))).toEqual([
+ 'pin-retries',
+ ]);
+ const outerOnly = `test.describe.configure({ retries: 0 });\n${playwrightPin('')}`;
+ expect(rules(scanSource('packages/app/tests/stress/a.e2e.ts', outerOnly))).toEqual([]);
+ });
+
+ test('a quarantine is declared skipped', () => {
+ const quarantine = (declare) => `
+import { test } from 'vitest';
+${declare}('flaky under load', { tags: ['quarantine'], meta: { issue: '${ISSUE}', owner: 'get-main-green', until: '2026-10-15' } }, () => {});
+`;
+ const skipped = scanSource('packages/app/src/a.test.ts', quarantine('test.skip'));
+ expect(skipped.problems).toEqual([]);
+ expect(skipped.quarantines).toHaveLength(1);
+ expect(skipped.notRun).toEqual([]);
+ expect(rules(scanSource('packages/app/src/a.test.ts', quarantine('test')))).toEqual([
+ 'quarantine-runs',
+ ]);
+ });
+
+ test('uses the runner-appropriate tag spelling', () => {
+ expect(KNOWN_BUG_TAG).toBe('known-bug');
+ expect(QUARANTINE_TAG).toBe('quarantine');
+ const playwrightWithVitestSpelling = playwrightPin(
+ 'test.describe.configure({ retries: 0 });',
+ ).replace("tag: '@known-bug'", "tag: 'known-bug'");
+ expect(
+ scanSource('packages/app/tests/stress/a.e2e.ts', playwrightWithVitestSpelling).pins,
+ ).toEqual([]);
+ });
+});
+
+describe('known-reds scanner: bare expected-failure markers', () => {
+ test.each([
+ ["test.fails('x', () => { expect(1).toBe(2); });", 'packages/app/src/a.test.ts'],
+ ["it.fails('x', () => { expect(1).toBe(2); });", 'packages/app/src/a.test.ts'],
+ ["test('x', { fails: true }, () => { expect(1).toBe(2); });", 'packages/app/src/a.test.ts'],
+ ["test('x', async () => { test.fail(); });", 'packages/app/tests/stress/a.e2e.ts'],
+ ["test.fail('x', async () => {});", 'packages/app/tests/stress/a.e2e.ts'],
+ ])('refuses %s', (source, path) => {
+ expect(rules(scanSource(path, source))).toEqual(['bare-expected-failure']);
+ });
+
+ test.each([
+ "import assert from 'node:assert'; test('x', () => { assert.fail('unreachable'); });",
+ "test('x', () => { deferred.fail(new Error('x')); });",
+ "test('x', () => { expect(['test.only(', 'test.fail(']).toHaveLength(2); });",
+ "test('x', () => { report({ fails: 3 }); });",
+ ])('does not mistake %s for a marker', (source) => {
+ expect(scanSource('packages/app/src/a.test.ts', source).problems).toEqual([]);
+ });
+});
+
+describe('known-reds scanner: CI-keyed skips', () => {
+ test.each([
+ "test.skip(IS_CI, 'x');",
+ "test.skip(isCI, 'x');",
+ "test.skip(CI, 'x');",
+ "test.skip(process.env.CI === 'true', 'x');",
+ 'test.skip(process.env.GITHUB_ACTIONS, "x");',
+ 'test.skip(process.env.GITHUB_RUN_ID, "x");',
+ 'test.skip(process.env.GITHUB_WORKFLOW, "x");',
+ ['test.skip(', ' IS_CI,', " 'disabled on CI',", ');'].join('\n'),
+ ])('finds the Playwright skip %s', (source) => {
+ expect(ciSkipCount('packages/app/tests/stress/a.e2e.ts', source)).toBe(1);
+ });
+
+ test.each([
+ "test.skip(!SMOKE_ENABLED, 'x');",
+ "test.skip(!DARWIN, 'x');",
+ 'test.skip(!BUILD_EXISTS, `x`);',
+ "test.skip('cold-start deferred', () => {});",
+ 'test.skip();',
+ "test.skip(decisionMade, 'x');",
+ "test.skip(CIRCLE_FLAG, 'x');",
+ 'test.skip(true, "x");',
+ "test.skip(config.CI, 'x');",
+ ])('does not count %s as CI-keyed', (source) => {
+ expect(ciSkipCount('packages/app/tests/stress/a.e2e.ts', source)).toBe(0);
+ });
+
+ test('counts each site', () => {
+ expect(
+ ciSkipCount(
+ 'packages/app/tests/stress/a.e2e.ts',
+ "test.skip(IS_CI, 'a');\ntest.skip(process.env.CI, 'b');",
+ ),
+ ).toBe(2);
+ });
+
+ test.each([
+ [
+ "import { describe as _d } from 'vitest';\nconst describe = process.env.CI ? _d.skip : _d;",
+ 'file',
+ ],
+ ["(process.env.CI ? describe.skip : describe)('x', () => {});", 'describe'],
+ ["describe.skipIf(process.env.CI)('x', () => {});", 'describe'],
+ ["test.skipIf(process.env.CI ? a : b)('x', () => {});", 'test'],
+ ["test('x', (ctx) => { ctx.skip(process.env.CI === 'true', 'no keyring on CI'); });", 'test'],
+ ["test('x', ({ skip }) => { skip(process.env.GITHUB_ACTIONS !== undefined); });", 'test'],
+ ["const ON_CI = Boolean(process.env.CI);\ntest.skipIf(ON_CI)('x', () => {});", 'test'],
+ ["test('x', { skip: process.env.CI === 'true' }, () => {});", 'test'],
+ ["test('x', (ctx) => { if (process.env.CI) { ctx.skip(); } });", 'test'],
+ ["test.runIf(process.env.CI && FEATURE_FLAG)('x', () => {});", 'test'],
+ ["test.skipIf(!process.env.CI || FLAG)('x', () => {});", 'test'],
+ ["test.skipIf(process.env.CI || FLAG)('x', () => {});", 'test'],
+ ["test.skipIf(process.env.CI && FLAG)('x', () => {});", 'test'],
+ ["const RUN = process.env.CI && FEATURE_FLAG;\ntest.runIf(RUN)('x', () => {});", 'test'],
+ ["const SKIP = !process.env.CI || FLAG;\ntest.skipIf(SKIP)('x', () => {});", 'test'],
+ [
+ "const TIMEOUT = process.env.CI ? 60_000 : 5_000;\ntest.skipIf(TIMEOUT > 10_000)('x', () => {});",
+ 'test',
+ ],
+ ])('finds the Vitest skip in %s', (source, scope) => {
+ const gates = scanSource('packages/server/src/a.test.ts', source).gates;
+ expect(gates.filter((gate) => gate.ci === 'skips-on-ci').map((gate) => gate.scope)).toEqual([
+ scope,
+ ]);
+ });
+
+ test.each([
+ "test.runIf(process.env.CI)('x', () => {});",
+ "test.skipIf(!process.env.CI)('x', () => {});",
+ "test.skipIf(process.env.CI !== 'true')('x', () => {});",
+ "describe.skipIf(process.env.CI === undefined)('x', () => {});",
+ "test.runIf(process.env.CI || FLAG)('x', () => {});",
+ "test.skipIf(!process.env.CI && FLAG)('x', () => {});",
+ ])('lists %s as running only on CI, not as a CI skip', (source) => {
+ const gates = scanSource('packages/server/src/a.test.ts', source).gates;
+ expect(gates.map((gate) => gate.ci)).toEqual(['runs-only-on-ci']);
+ });
+
+ test.each([
+ "test.skipIf(process.env.CIRCLE)('x', () => {});",
+ "test.skipIf({ CI: true }.CI_FLAG)('x', () => {});",
+ "test.skipIf(process.platform === 'win32')('x', () => {});",
+ ])('lists %s as an environment gate', (source) => {
+ const gates = scanSource('packages/server/src/a.test.ts', source).gates;
+ expect(gates.map((gate) => gate.ci)).toEqual([null]);
+ });
+});
+
+describe('known-reds scanner: early returns and not-run tests', () => {
+ test.each([
+ "test('x', () => { if (process.platform === 'win32') return; expect(1).toBe(1); });",
+ "const IS_WIN = process.platform === 'win32';\ntest('x', () => { if (IS_WIN) { console.log('skip'); return; } expect(1).toBe(1); });",
+ "import { existsSync } from 'node:fs';\ntest('x', () => { if (!existsSync(built)) return; expect(1).toBe(1); });",
+ "import os from 'node:os';\ntest('x', () => { if (os.platform() !== 'darwin') return; expect(1).toBe(1); });",
+ "test('x', () => { if (process.env.OK_AUDIT === '1') { report(); return; } expect(1).toBe(1); });",
+ "describe('x', () => { const isWin = process.platform === 'win32'; test('y', () => { if (isWin) return; expect(1).toBe(1); }); });",
+ ])('flags the environment early return in %s', (source) => {
+ expect(scanSource('packages/server/src/a.test.ts', source).earlyReturns).toHaveLength(1);
+ });
+
+ test.each([
+ "test('x', () => { const button = find(); if (!button) return; expect(button).toBeTruthy(); });",
+ "test('x', () => { expect(a).toBe(1); if (process.platform === 'win32') return; expect(b).toBe(2); });",
+ "test('x', async () => { await vi.waitFor(() => expect(a).toBe(1)); if (process.platform === 'win32') return; });",
+ "test('x', () => { const f = () => { if (process.platform === 'win32') return; }; f(); expect(1).toBe(1); });",
+ "test('x', () => { if (process.platform === 'win32') return promise; expect(1).toBe(1); });",
+ "test('x', (ctx) => { if (process.platform === 'win32') { ctx.skip(); return; } expect(1).toBe(1); });",
+ "describe('x', () => { if (process.platform === 'win32') return; test('y', () => { expect(1).toBe(1); }); });",
+ ])('leaves %s alone', (source) => {
+ expect(scanSource('packages/server/src/a.test.ts', source).earlyReturns).toEqual([]);
+ });
+
+ test('flags a Playwright early return too', () => {
+ const source = "test('x', async () => { if (process.platform === 'win32') return; });";
+ expect(scanSource('packages/app/tests/stress/a.e2e.ts', source).earlyReturns).toHaveLength(1);
+ });
+
+ test('reads the test context from test.for, and none from test.each', () => {
+ const forSource =
+ "test.for(cases)('x $name', (row, ctx) => { ctx.skip(process.platform === 'win32', 'posix'); expect(row).toBeTruthy(); });";
+ expect(
+ scanSource('packages/server/src/a.test.ts', forSource).gates.map((gate) => gate.form),
+ ).toEqual(['context-skip']);
+ const eachSource =
+ "test.each(cases)('x $name', (row) => { row.skip(process.platform === 'win32'); expect(row).toBeTruthy(); });";
+ expect(scanSource('packages/server/src/a.test.ts', eachSource).gates).toEqual([]);
+ });
+
+ test('lists unconditional skips, fixme and todo as not run', () => {
+ const vitest = scanSource(
+ 'packages/server/src/a.test.ts',
+ "test.skip('a', () => {});\ntest.todo('b');\ntest('c', { skip: true }, () => {});\ntest('d', (ctx) => { ctx.skip('not yet'); });",
+ );
+ expect(vitest.notRun.map((row) => row.form)).toEqual([
+ 'skip',
+ 'todo',
+ 'skip-option',
+ 'context-skip',
+ ]);
+ const playwright = scanSource(
+ 'packages/app/tests/stress/a.e2e.ts',
+ "test('e', async () => { test.skip(true, 'PR #1010 broke it'); });\ntest.fixme('f', async () => {});",
+ );
+ expect(playwright.notRun.map((row) => [row.form, row.scope])).toEqual([
+ ['skip', 'test'],
+ ['fixme', 'test'],
+ ]);
+ });
+
+ test('takes a literal-true skip’s condition from the if around it', () => {
+ const source =
+ "async function settle() { if (workArea.width < width) { test.skip(true, 'too small'); } }";
+ const scan = scanSource('packages/desktop/tests/smoke/a.e2e.ts', source);
+ expect(scan.notRun).toEqual([]);
+ expect(scan.gates.map((gate) => [gate.scope, gate.condition, gate.reason])).toEqual([
+ ['helper', 'workArea.width < width', 'too small'],
+ ]);
+ });
+
+ test('ignores runner-like calls on other objects', () => {
+ const scan = scanSource(
+ 'packages/server/src/a.test.ts',
+ "test('x', () => { harness.test.skip(process.env.CI); runner.skipIf(process.env.CI); });",
+ );
+ expect(scan.gates).toEqual([]);
+ expect(scan.problems).toEqual([]);
+ });
+});
+
+describe('known-reds validation', () => {
+ const pinReport = (fields) => ({
+ problems: [],
+ pins: [
+ {
+ path: 'a.test.ts',
+ line: 1,
+ runner: 'vitest',
+ title: 't',
+ issue: ISSUE,
+ owner: 'get-main-green',
+ until: '2026-10-15',
+ ...fields,
+ },
+ ],
+ quarantines: [],
+ earlyReturns: [],
+ ciSkips: [],
+ });
+ const messages = (report, allowlist = NO_ALLOWLIST) =>
+ validate(report, { today: TODAY, allowlist }).map((violation) => violation.message);
+
+ test('accepts a complete pin', () => {
+ expect(messages(pinReport({}))).toEqual([]);
+ });
+
+ test.each([
+ [{ owner: undefined }, 'known-bug pin has no owner'],
+ [
+ { owner: 'someone-else' },
+ `known-bug pin has owner "someone-else", which is not one of ${OWNERS.join(', ')}`,
+ ],
+ [{ issue: undefined }, 'known-bug pin has no issue link'],
+ [
+ { issue: 'inkeep/agents-private#1056' },
+ 'known-bug pin has issue "inkeep/agents-private#1056", which is not a GitHub or Linear issue URL',
+ ],
+ [{ until: undefined }, 'known-bug pin has no until date'],
+ [{ until: '15/10/2026' }, 'known-bug pin has until "15/10/2026", which is not YYYY-MM-DD'],
+ [{ until: '2026-02-30' }, 'known-bug pin has until "2026-02-30", which is not a date'],
+ [{ until: '2026-09-30' }, 'known-bug pin expired on 2026-09-30'],
+ [
+ { until: '2026-12-31' },
+ `known-bug pin has until 2026-12-31, beyond the ${MAX_HORIZON_DAYS}-day horizon (2026-12-30)`,
+ ],
+ ])('refuses %j', (fields, message) => {
+ expect(messages(pinReport(fields))).toEqual([message]);
+ });
+
+ test(`lists what expires within ${EXPIRY_WARNING_DAYS} days, without failing it`, () => {
+ const report = {
+ ...pinReport({}),
+ pins: [
+ { path: 'a.test.ts', line: 1, until: '2026-10-08', owner: 'get-main-green' },
+ { path: 'b.test.ts', line: 2, until: '2026-10-15', owner: 'get-main-green' },
+ { path: 'c.test.ts', line: 3, until: '2026-10-16', owner: 'get-main-green' },
+ ],
+ };
+ const allowlist = {
+ ciSkips: [{ path: 'd.test.ts', condition: 'CI', owner: 'known-reds', until: '2026-10-02' }],
+ };
+ expect(expiringSoon(report, { today: TODAY, allowlist })).toEqual([
+ { kind: 'pin', path: 'a.test.ts', line: 1, owner: 'get-main-green', until: '2026-10-08' },
+ { kind: 'pin', path: 'b.test.ts', line: 2, owner: 'get-main-green', until: '2026-10-15' },
+ {
+ kind: 'ci-skip-entry',
+ path: 'd.test.ts',
+ line: null,
+ owner: 'known-reds',
+ until: '2026-10-02',
+ },
+ ]);
+ });
+
+ test('the listing names each expiring kind', () => {
+ const report = {
+ files: 3,
+ problems: [],
+ pins: [{ path: 'a.test.ts', line: 1, until: '2026-10-08', owner: 'get-main-green' }],
+ quarantines: [{ path: 'q.test.ts', line: 4, until: '2026-10-09', owner: 'get-main-green' }],
+ earlyReturns: [],
+ ciSkips: [],
+ notRun: [],
+ envGates: [],
+ };
+ const allowlist = {
+ ciSkips: [{ path: 'd.test.ts', condition: 'CI', owner: 'known-reds', until: '2026-10-02' }],
+ };
+ const upcoming = expiringSoon(report, { today: TODAY, allowlist });
+ expect(render(report, [], upcoming, TODAY, { all: false })).toContain(
+ [
+ `Expiring within ${EXPIRY_WARNING_DAYS} days (3)`,
+ ' pin a.test.ts:1 owner get-main-green, until 2026-10-08',
+ ' quarantine q.test.ts:4 owner get-main-green, until 2026-10-09',
+ ' allowlisted CI skip d.test.ts owner known-reds, until 2026-10-02',
+ ].join('\n'),
+ );
+ });
+
+ test('the date boundaries are inclusive', () => {
+ expect(messages(pinReport({ until: TODAY }))).toEqual([]);
+ expect(messages(pinReport({ until: '2026-12-30' }))).toEqual([]);
+ });
+
+ test('accepts a Linear issue', () => {
+ expect(
+ messages(pinReport({ issue: 'https://linear.app/inkeep/issue/PRD-8603/symlink-reads' })),
+ ).toEqual([]);
+ });
+
+ test('holds a quarantine to the same fields', () => {
+ const report = {
+ ...pinReport({}),
+ pins: [],
+ quarantines: [
+ { path: 'a.test.ts', line: 1, title: 't', issue: ISSUE, owner: 'get-main-green' },
+ ],
+ };
+ expect(messages(report)).toEqual(['quarantine has no until date']);
+ });
+
+ test('refuses an environment early return', () => {
+ const report = {
+ ...pinReport({}),
+ pins: [],
+ earlyReturns: [{ path: 'a.test.ts', line: 3, condition: "process.platform === 'win32'" }],
+ };
+ expect(
+ validate(report, { today: TODAY, allowlist: NO_ALLOWLIST }).map(
+ (violation) => violation.rule,
+ ),
+ ).toEqual(['early-return']);
+ });
+
+ describe('the CI-skip allowlist', () => {
+ const skip = {
+ path: 'packages/server/src/boot.test.ts',
+ line: 3,
+ form: 'alias',
+ scope: 'file',
+ condition: 'process.env.CI',
+ };
+ const entry = {
+ path: skip.path,
+ condition: skip.condition,
+ owner: 'known-reds',
+ until: '2026-11-01',
+ reason: 'skips the whole file on CI',
+ };
+ const report = { ...pinReport({}), pins: [], ciSkips: [skip] };
+ const ruleNames = (allowlist) =>
+ validate(report, { today: TODAY, allowlist }).map((violation) => violation.rule);
+
+ test('refuses a CI skip that is not listed', () => {
+ expect(ruleNames(NO_ALLOWLIST)).toEqual(['ci-skip']);
+ });
+
+ test('accepts a listed, owned, dated CI skip', () => {
+ expect(ruleNames({ ciSkips: [entry] })).toEqual([]);
+ });
+
+ test('permits exactly one site per entry', () => {
+ const twice = { ...report, ciSkips: [skip, { ...skip, line: 40 }] };
+ expect(
+ validate(twice, { today: TODAY, allowlist: { ciSkips: [entry] } }).map(
+ (violation) => `${violation.line} ${violation.rule}`,
+ ),
+ ).toEqual(['40 ci-skip']);
+ });
+
+ test('matches on the condition as well as the path', () => {
+ expect(
+ ruleNames({ ciSkips: [{ ...entry, condition: "process.env.CI === 'true'" }] }),
+ ).toEqual(['ci-skip', 'allowlist-stale']);
+ });
+
+ test('refuses a stale, expired, unowned or unexplained entry', () => {
+ expect(
+ validate({ ...report, ciSkips: [] }, { today: TODAY, allowlist: { ciSkips: [entry] } }).map(
+ (violation) => violation.rule,
+ ),
+ ).toEqual(['allowlist-stale']);
+ expect(ruleNames({ ciSkips: [{ ...entry, until: '2026-09-01' }] })).toEqual([
+ 'allowlist-fields',
+ ]);
+ expect(ruleNames({ ciSkips: [{ ...entry, owner: 'nobody' }] })).toEqual(['allowlist-fields']);
+ expect(ruleNames({ ciSkips: [{ ...entry, reason: '' }] })).toEqual(['allowlist-fields']);
+ });
+ });
+});
+
+describe('known-reds through its real invocation', () => {
+ const root = mkdtempSync(join(tmpdir(), 'ok-known-reds-'));
+ afterAll(() => rmSync(root, { recursive: true, force: true }));
+ const write = (path, source) => {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), source);
+ };
+ write(
+ 'packages/server/src/planted-boot.test.ts',
+ "import { describe as _d } from 'vitest';\nconst describe = process.env.CI ? _d.skip : _d;\ndescribe('boot', () => {});\n",
+ );
+ write(
+ 'packages/server/src/spawn.test.ts',
+ "test('x', () => { if (process.platform === 'win32') return; expect(1).toBe(1); });\n",
+ );
+ write('packages/app/tests/stress/hide.e2e.ts', playwrightPin(''));
+ write(
+ 'packages/app/src/helper.test-helper.ts',
+ "test.skip(process.env.CI, 'not a test file');\n",
+ );
+ write('reports/spike/probe.test.ts', "test.skip(process.env.CI, 'evidence');\n");
+ write('specs/spike/probe.e2e.ts', "test.skip(process.env.CI, 'evidence');\n");
+ write('node_modules/pkg/dep.test.js', "test.skip(process.env.CI, 'vendored');\n");
+ const init = spawnSync('git', ['init', '-q'], {
+ cwd: root,
+ env: gitCleanEnv(),
+ windowsHide: true,
+ encoding: 'utf8',
+ });
+
+ test('scans exactly the test files a runner could run', () => {
+ expect(init.status).toBe(0);
+ expect(listTestFiles(root)).toEqual([
+ 'packages/app/tests/stress/hide.e2e.ts',
+ 'packages/server/src/planted-boot.test.ts',
+ 'packages/server/src/spawn.test.ts',
+ ]);
+ });
+
+ test('finds every planted violation', () => {
+ const report = scanTree(root);
+ const violations = validate(report, { today: TODAY, allowlist: NO_ALLOWLIST });
+ expect(
+ violations.map((violation) => `${violation.path}:${violation.line} ${violation.rule}`).sort(),
+ ).toEqual([
+ 'packages/app/tests/stress/hide.e2e.ts:6 pin-retries',
+ 'packages/server/src/planted-boot.test.ts:2 ci-skip',
+ 'packages/server/src/spawn.test.ts:1 early-return',
+ ]);
+ });
+
+ test('the command exits 1 on violations, and lists them in its JSON feed', () => {
+ const run = spawnSync(process.execPath, [SCRIPT, '--json', '--root', root], {
+ encoding: 'utf8',
+ env: gitCleanEnv(),
+ windowsHide: true,
+ });
+ expect(run.status).toBe(1);
+ const feed = JSON.parse(run.stdout);
+ expect(feed.schemaVersion).toBe(1);
+ expect(feed.files).toBe(3);
+ expect(feed.violations.map((violation) => violation.rule)).toEqual(
+ expect.arrayContaining(['pin-retries', 'ci-skip', 'early-return']),
+ );
+ });
+
+ test('the command warns about a pin that expires soon, in the feed and the listing', () => {
+ const soon = new Date(Date.parse(`${todayUtc()}T00:00:00Z`) + 7 * 86_400_000)
+ .toISOString()
+ .slice(0, 10);
+ write(
+ 'packages/server/src/expiring.test.ts',
+ vitestPin({ meta: `{ issue: '${ISSUE}', owner: 'get-main-green', until: '${soon}' }` }),
+ );
+ try {
+ const json = spawnSync(process.execPath, [SCRIPT, '--json', '--root', root], {
+ encoding: 'utf8',
+ env: gitCleanEnv(),
+ windowsHide: true,
+ });
+ expect(JSON.parse(json.stdout).expiringSoon).toEqual(
+ expect.arrayContaining([
+ {
+ kind: 'pin',
+ path: 'packages/server/src/expiring.test.ts',
+ line: 4,
+ owner: 'get-main-green',
+ until: soon,
+ },
+ ]),
+ );
+ const listing = spawnSync(process.execPath, [SCRIPT, '--root', root], {
+ encoding: 'utf8',
+ env: gitCleanEnv(),
+ windowsHide: true,
+ });
+ expect(listing.stdout).toMatch(
+ new RegExp(`Expiring within ${EXPIRY_WARNING_DAYS} days \\(\\d+\\)`),
+ );
+ expect(listing.stdout).toContain(
+ `pin packages/server/src/expiring.test.ts:4 owner get-main-green, until ${soon}`,
+ );
+ } finally {
+ rmSync(join(root, 'packages/server/src/expiring.test.ts'), { force: true });
+ }
+ });
+
+ test('the listing names the planted sites and points at the convention', () => {
+ const run = spawnSync(process.execPath, [SCRIPT, '--root', root], {
+ encoding: 'utf8',
+ env: gitCleanEnv(),
+ windowsHide: true,
+ });
+ expect(run.status).toBe(1);
+ expect(run.stdout).toContain('Known reds in open-knowledge on ');
+ expect(run.stdout).toContain(': 3 test files scanned.');
+ expect(run.stdout).toContain(
+ 'packages/server/src/planted-boot.test.ts:2 file process.env.CI',
+ );
+ expect(run.stdout).toContain(`The convention: ${CONVENTION_DOC}.`);
+ });
+
+ test('the command exits 2, not 0, when it cannot scan', () => {
+ const notARepo = mkdtempSync(join(tmpdir(), 'ok-known-reds-bare-'));
+ try {
+ const run = spawnSync(process.execPath, [SCRIPT, '--root', notARepo], {
+ encoding: 'utf8',
+ env: { ...gitCleanEnv(), GIT_CEILING_DIRECTORIES: tmpdir() },
+ windowsHide: true,
+ });
+ expect(run.status).toBe(2);
+ expect(run.stderr).toMatch(/^known-reds: could not run: /);
+ } finally {
+ rmSync(notARepo, { recursive: true, force: true });
+ }
+ });
+});
+
+describe('known-reds on this tree', () => {
+ const files = listTestFiles(OK_ROOT);
+
+ test('scans every runner’s test files and nothing under reports/ or specs/', () => {
+ expect(files).toEqual(
+ expect.arrayContaining([
+ 'scripts/check-known-reds.uncached.test.mjs',
+ 'test-support/known-bug.test.ts',
+ 'packages/app/tests/stress/file-tree-create.e2e.ts',
+ 'packages/desktop/tests/smoke/mcp-wiring.e2e.ts',
+ ]),
+ );
+ expect(
+ files.filter((path) => path.startsWith('reports/') || path.startsWith('specs/')),
+ ).toEqual([]);
+ });
+
+ test('conforms: pins, quarantines, CI skips and early returns follow the convention', () => {
+ const violations = validate(scanTree(OK_ROOT), {
+ today: todayUtc(),
+ allowlist: loadAllowlist(),
+ });
+ expect(
+ violations.map(
+ (violation) =>
+ `${violation.path}:${violation.line} [${violation.rule}] ${violation.message}`,
+ ),
+ ).toEqual([]);
+ });
+
+ test('the allowlist is the file the command reads', () => {
+ expect(JSON.parse(readFileSync(ALLOWLIST_PATH, 'utf8'))).toEqual(loadAllowlist());
+ });
+
+ test('every Vitest tier declares the known-bug and quarantine tags', () => {
+ expect(okVitestBase.test.tags.map((tag) => tag.name)).toEqual([KNOWN_BUG_TAG, QUARANTINE_TAG]);
+ });
+
+ test('the convention doc and the guard cite each other', () => {
+ const doc = readFileSync(join(OK_ROOT, CONVENTION_DOC), 'utf8');
+ for (const cited of [
+ 'scripts/check-known-reds.uncached.test.mjs',
+ 'scripts/known-reds.mjs',
+ 'scripts/known-reds-allowlist.json',
+ 'pnpm known-reds',
+ `no more than ${MAX_HORIZON_DAYS} days away`,
+ `at least ${MIN_SIGNATURE_LITERAL} literal characters in a row`,
+ `Expiring within ${EXPIRY_WARNING_DAYS} days`,
+ ]) {
+ expect(doc, `${CONVENTION_DOC} must mention ${cited}`).toContain(cited);
+ }
+ });
+
+ test('pnpm known-reds runs this command', () => {
+ const manifest = JSON.parse(readFileSync(join(OK_ROOT, 'package.json'), 'utf8'));
+ expect(manifest.scripts['known-reds']).toBe('node scripts/known-reds.mjs');
+ });
+});
diff --git a/scripts/known-reds-allowlist.json b/scripts/known-reds-allowlist.json
new file mode 100644
index 000000000..3269aa5d7
--- /dev/null
+++ b/scripts/known-reds-allowlist.json
@@ -0,0 +1,144 @@
+{
+ "ciSkips": [
+ {
+ "path": "packages/app/tests/integration/conflict-aware-write-surfaces.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips the suites declared through `describeBoot` on CI. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/cli/src/utils/open-browser.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/cli/tests/integration/detached-spawn-lifetime.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_describe.skip` there. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/cli/tests/integration/mcp-host-config-race.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips the suites declared through `describeCrossProcess` on CI. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/cli/tests/integration/multi-project-locks.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips this describe on CI. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/boot.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp-local-dispatch-parity.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/logger.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tool-logging.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/audit.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/checkpoint.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/config.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/history.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/links.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/lint.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/preview-url.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/restore-version.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/mcp/tools/summary-passthrough.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/persistence-ytext-truth.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_vitestDescribe.skip` there. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ },
+ {
+ "path": "packages/server/src/serialize-doc-vs-git-show-byte-equality.test.ts",
+ "condition": "process.env.CI",
+ "owner": "known-reds",
+ "until": "2026-12-15",
+ "reason": "Skips every describe in the file on CI: `describe` is rebound to `_bunDescribe.skip` there, a guard kept from the Bun migration. The known-reds cleanup runs it on CI, pins any deterministic red and quarantines any flake."
+ }
+ ]
+}
diff --git a/scripts/known-reds.mjs b/scripts/known-reds.mjs
new file mode 100644
index 000000000..af3500913
--- /dev/null
+++ b/scripts/known-reds.mjs
@@ -0,0 +1,1282 @@
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { gitCleanEnv } from './git-clean-env.mjs';
+
+const ts = createRequire(import.meta.url)('typescript');
+
+export const OK_ROOT = fileURLToPath(new URL('..', import.meta.url));
+export const ALLOWLIST_PATH = fileURLToPath(
+ new URL('./known-reds-allowlist.json', import.meta.url),
+);
+export const SCHEMA_VERSION = 1;
+export const KNOWN_BUG_TAG = 'known-bug';
+export const QUARANTINE_TAG = 'quarantine';
+export const OWNERS = Object.freeze(['get-main-green', 'known-reds']);
+export const MAX_HORIZON_DAYS = 90;
+export const EXCLUDED_PREFIXES = Object.freeze(['reports/', 'specs/']);
+export const CONVENTION_DOC = 'test-support/KNOWN-REDS.md';
+export const MIN_SIGNATURE_LITERAL = 4;
+export const EXPIRY_WARNING_DAYS = 14;
+
+const TEST_FILE = /\.test\.[cm]?[jt]sx?$|\.e2e\.tsx?$/;
+const PLAYWRIGHT_FILE = /\.e2e\.tsx?$/;
+const ISSUE_URL =
+ /^https:\/\/(?:github\.com\/inkeep\/[\w.-]+\/issues\/\d+|linear\.app\/inkeep\/issue\/[A-Z]+-\d+(?:\/[\w-]*)?)$/;
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
+const DAY_MS = 86_400_000;
+const RUNNER_KINDS = new Map([
+ ['test', 'test'],
+ ['it', 'test'],
+ ['describe', 'describe'],
+ ['suite', 'describe'],
+]);
+const HOOKS = new Set(['beforeEach', 'afterEach', 'beforeAll', 'afterAll']);
+const NON_DECLARING = new Set([
+ 'configure',
+ 'use',
+ 'step',
+ 'extend',
+ 'info',
+ 'setTimeout',
+ 'slow',
+ 'expect',
+]);
+const NOT_RUN_MODIFIERS = new Set(['skip', 'fixme', 'todo']);
+const BARE_EXPECTED_FAILURE = new Set(['fails', 'fail']);
+const ANNOTATION_MODIFIERS = new Set(['skip', 'fixme', 'fail', 'slow']);
+const CI_ENV_NAMES = new Set([
+ 'CI',
+ 'GITHUB_ACTIONS',
+ 'GITHUB_RUN_ID',
+ 'GITHUB_WORKFLOW',
+ 'RUNNER_OS',
+]);
+const CI_IDENTIFIERS = new Set(['CI', 'IS_CI', 'isCI', 'isCi', 'ON_CI', 'onCI', 'IN_CI', 'inCI']);
+const FIELDS = ['issue', 'owner', 'until'];
+
+function scriptKind(path) {
+ if (path.endsWith('.tsx')) return ts.ScriptKind.TSX;
+ if (/\.[cm]?ts$/.test(path)) return ts.ScriptKind.TS;
+ if (path.endsWith('.jsx')) return ts.ScriptKind.JSX;
+ return ts.ScriptKind.JS;
+}
+
+function unwrap(node) {
+ let current = node;
+ while (
+ current &&
+ (ts.isParenthesizedExpression(current) ||
+ ts.isAsExpression(current) ||
+ ts.isNonNullExpression(current) ||
+ ts.isSatisfiesExpression(current) ||
+ ts.isTypeAssertionExpression(current))
+ ) {
+ current = current.expression;
+ }
+ return current;
+}
+
+function isFunction(node) {
+ return Boolean(node) && (ts.isArrowFunction(node) || ts.isFunctionExpression(node));
+}
+
+function isTitle(node) {
+ return (
+ Boolean(node) &&
+ (ts.isStringLiteral(node) ||
+ ts.isNoSubstitutionTemplateLiteral(node) ||
+ ts.isTemplateExpression(node))
+ );
+}
+
+function stringValue(node) {
+ const value = unwrap(node);
+ return value && (ts.isStringLiteral(value) || ts.isNoSubstitutionTemplateLiteral(value))
+ ? value.text
+ : null;
+}
+
+function oneLine(node, sourceFile) {
+ return node.getText(sourceFile).replace(/\s+/g, ' ').trim();
+}
+
+function propertyName(property) {
+ const { name } = property;
+ if (name && (ts.isIdentifier(name) || ts.isStringLiteral(name))) return name.text;
+ return null;
+}
+
+function objectProperty(object, name) {
+ return object.properties.find(
+ (property) =>
+ (ts.isPropertyAssignment(property) || ts.isShorthandPropertyAssignment(property)) &&
+ propertyName(property) === name,
+ );
+}
+
+function chainOf(node) {
+ const members = [];
+ let current = unwrap(node);
+ while (current && ts.isPropertyAccessExpression(current)) {
+ members.unshift(current.name.text);
+ current = unwrap(current.expression);
+ }
+ return { root: current, members };
+}
+
+function runnerRef(node, kinds) {
+ const { root, members } = chainOf(node);
+ if (!root || !ts.isIdentifier(root) || !kinds.has(root.text)) return null;
+ return {
+ root: root.text,
+ members,
+ kind: members.includes('describe') ? 'describe' : kinds.get(root.text),
+ };
+}
+
+function isSkipVariant(ref) {
+ return ref.members.includes('skip') || ref.members.includes('fixme');
+}
+
+function skipTernary(node, kinds) {
+ const value = unwrap(node);
+ if (!value || !ts.isConditionalExpression(value)) return null;
+ const whenTrue = runnerRef(value.whenTrue, kinds);
+ const whenFalse = runnerRef(value.whenFalse, kinds);
+ if (!whenTrue || !whenFalse || isSkipVariant(whenTrue) === isSkipVariant(whenFalse)) return null;
+ return {
+ condition: value.condition,
+ skipWhen: isSkipVariant(whenTrue) ? 'condition' : 'negation',
+ ref: isSkipVariant(whenTrue) ? whenFalse : whenTrue,
+ };
+}
+
+function runnerBindings(sourceFile) {
+ const kinds = new Map(RUNNER_KINDS);
+ for (const statement of sourceFile.statements) {
+ if (!ts.isImportDeclaration(statement)) continue;
+ const named = statement.importClause?.namedBindings;
+ if (!named || !ts.isNamedImports(named)) continue;
+ for (const element of named.elements) {
+ const kind = RUNNER_KINDS.get((element.propertyName ?? element.name).text);
+ if (kind) kinds.set(element.name.text, kind);
+ }
+ }
+ const visit = (node) => {
+ if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) {
+ const ternary = skipTernary(node.initializer, kinds);
+ if (ternary) kinds.set(node.name.text, ternary.ref.kind);
+ }
+ ts.forEachChild(node, visit);
+ };
+ visit(sourceFile);
+ return kinds;
+}
+
+function runnerCall(call, kinds) {
+ const callee = unwrap(call.expression);
+ const ternary = skipTernary(callee, kinds);
+ if (ternary) {
+ return {
+ ref: ternary.ref,
+ gate: { form: 'ternary', condition: ternary.condition, skipWhen: ternary.skipWhen },
+ };
+ }
+ if (ts.isCallExpression(callee)) {
+ const inner = runnerRef(callee.expression, kinds);
+ if (!inner) return null;
+ const last = inner.members.at(-1);
+ const ref = { ...inner, members: inner.members.slice(0, -1) };
+ if (last === 'skipIf' || last === 'runIf') {
+ return {
+ ref,
+ gate: {
+ form: last,
+ condition: callee.arguments[0] ?? null,
+ skipWhen: last === 'skipIf' ? 'condition' : 'negation',
+ },
+ };
+ }
+ return last === 'each' || last === 'for' ? { ref, gate: null, via: last } : null;
+ }
+ const ref = runnerRef(callee, kinds);
+ return ref ? { ref, gate: null } : null;
+}
+
+function declarationKind(ref) {
+ if (
+ HOOKS.has(ref.root) ||
+ ref.members.some((member) => HOOKS.has(member) || NON_DECLARING.has(member))
+ ) {
+ return 'other';
+ }
+ return ref.kind;
+}
+
+function declarationParts(call) {
+ const args = call.arguments.map(unwrap);
+ const fn = [...args].reverse().find(isFunction) ?? null;
+ const options = args.slice(1).find((arg) => ts.isObjectLiteralExpression(arg)) ?? null;
+ return { title: isTitle(args[0]) ? args[0] : null, fn, options };
+}
+
+function rootIdentifier(node) {
+ let current = unwrap(node);
+ while (current && (ts.isPropertyAccessExpression(current) || ts.isCallExpression(current))) {
+ current = unwrap(current.expression);
+ }
+ return current && ts.isIdentifier(current) ? current.text : null;
+}
+
+function isAssertionCall(node) {
+ if (!ts.isCallExpression(node)) return false;
+ const root = rootIdentifier(node.expression);
+ return root !== null && /^(expect|assert)/.test(root);
+}
+
+function isExpressionIdentifier(node) {
+ const { parent } = node;
+ if (!parent) return true;
+ if (ts.isPropertyAccessExpression(parent) && parent.name === node) return false;
+ if ((ts.isPropertyAssignment(parent) || ts.isBindingElement(parent)) && parent.name === node)
+ return false;
+ if (ts.isVariableDeclaration(parent) && parent.name === node) return false;
+ if (ts.isPropertyAssignment(parent) && parent.propertyName === node) return false;
+ return true;
+}
+
+function isEnvObject(node) {
+ const value = unwrap(node);
+ if (ts.isIdentifier(value)) return value.text === 'env';
+ return ts.isPropertyAccessExpression(value) && value.name.text === 'env';
+}
+
+function ciAtom(node, aliases) {
+ if (ts.isIdentifier(node)) {
+ if (!isExpressionIdentifier(node)) return null;
+ if (aliases.has(node.text)) return aliases.get(node.text);
+ return CI_IDENTIFIERS.has(node.text) ? 'positive' : null;
+ }
+ if (
+ ts.isPropertyAccessExpression(node) &&
+ CI_ENV_NAMES.has(node.name.text) &&
+ isEnvObject(node.expression)
+ ) {
+ return 'positive';
+ }
+ if (ts.isElementAccessExpression(node) && isEnvObject(node.expression)) {
+ const key = stringValue(node.argumentExpression);
+ return key !== null && CI_ENV_NAMES.has(key) ? 'positive' : null;
+ }
+ return null;
+}
+
+function containsCi(node, aliases) {
+ let found = false;
+ const visit = (current) => {
+ if (found) return;
+ if (ciAtom(current, aliases)) {
+ found = true;
+ return;
+ }
+ if (isFunction(current) || ts.isFunctionDeclaration(current)) return;
+ ts.forEachChild(current, visit);
+ };
+ visit(node);
+ return found;
+}
+
+function literalTruth(node) {
+ const value = unwrap(node);
+ if (value.kind === ts.SyntaxKind.TrueKeyword) return true;
+ if (value.kind === ts.SyntaxKind.FalseKeyword || value.kind === ts.SyntaxKind.NullKeyword)
+ return false;
+ if (ts.isIdentifier(value) && value.text === 'undefined') return false;
+ if (ts.isStringLiteral(value) || ts.isNoSubstitutionTemplateLiteral(value)) {
+ return !['', '0', 'false'].includes(value.text);
+ }
+ if (ts.isNumericLiteral(value)) return Number(value.text) !== 0;
+ return null;
+}
+
+const EQUALITY = new Set([ts.SyntaxKind.EqualsEqualsEqualsToken, ts.SyntaxKind.EqualsEqualsToken]);
+const INEQUALITY = new Set([
+ ts.SyntaxKind.ExclamationEqualsEqualsToken,
+ ts.SyntaxKind.ExclamationEqualsToken,
+]);
+
+function not(value) {
+ return value === null ? null : !value;
+}
+
+function ciValue(node, aliases, onCi) {
+ const value = unwrap(node);
+ if (!value || !containsCi(value, aliases)) return null;
+ const atom = ciAtom(value, aliases);
+ if (atom === 'positive') return onCi;
+ if (atom === 'negative') return !onCi;
+ if (atom === 'unknown') return null;
+ if (ts.isPrefixUnaryExpression(value) && value.operator === ts.SyntaxKind.ExclamationToken) {
+ return not(ciValue(value.operand, aliases, onCi));
+ }
+ if (ts.isBinaryExpression(value)) {
+ const operator = value.operatorToken.kind;
+ if (operator === ts.SyntaxKind.AmpersandAmpersandToken) {
+ const left = ciValue(value.left, aliases, onCi);
+ const right = ciValue(value.right, aliases, onCi);
+ if (left === false || right === false) return false;
+ return left === true && right === true ? true : null;
+ }
+ if (operator === ts.SyntaxKind.BarBarToken) {
+ const left = ciValue(value.left, aliases, onCi);
+ const right = ciValue(value.right, aliases, onCi);
+ if (left === true || right === true) return true;
+ return left === false && right === false ? false : null;
+ }
+ if (EQUALITY.has(operator) || INEQUALITY.has(operator)) {
+ const ciOnLeft = containsCi(value.left, aliases);
+ if (ciOnLeft === containsCi(value.right, aliases)) return null;
+ const side = ciValue(ciOnLeft ? value.left : value.right, aliases, onCi);
+ const truth = literalTruth(ciOnLeft ? value.right : value.left);
+ if (side === null || truth === null) return null;
+ return EQUALITY.has(operator) ? side === truth : side !== truth;
+ }
+ }
+ if (
+ ts.isCallExpression(value) &&
+ ts.isIdentifier(value.expression) &&
+ value.expression.text === 'Boolean' &&
+ value.arguments.length === 1
+ ) {
+ return ciValue(value.arguments[0], aliases, onCi);
+ }
+ return null;
+}
+
+function declarationsAtAnyDepth(sourceFile, kinds) {
+ const declarations = [];
+ const collect = (node) => {
+ if (
+ ts.isVariableDeclaration(node) &&
+ ts.isIdentifier(node.name) &&
+ node.initializer &&
+ !skipTernary(node.initializer, kinds)
+ ) {
+ declarations.push(node);
+ }
+ ts.forEachChild(node, collect);
+ };
+ collect(sourceFile);
+ return declarations;
+}
+
+function ciAliases(sourceFile, kinds) {
+ const declarations = declarationsAtAnyDepth(sourceFile, kinds);
+ const aliases = new Map();
+ for (let pass = 0; pass < 3; pass += 1) {
+ for (const declaration of declarations) {
+ if (!containsCi(declaration.initializer, aliases)) continue;
+ const onCi = ciValue(declaration.initializer, aliases, true);
+ const offCi = ciValue(declaration.initializer, aliases, false);
+ const polarity =
+ onCi === true && offCi === false
+ ? 'positive'
+ : onCi === false && offCi === true
+ ? 'negative'
+ : 'unknown';
+ aliases.set(declaration.name.text, polarity);
+ }
+ }
+ return aliases;
+}
+
+const PROCESS_ENVIRONMENT = new Set([
+ 'platform',
+ 'arch',
+ 'env',
+ 'getuid',
+ 'getgid',
+ 'versions',
+ 'release',
+]);
+const OS_MODULES = new Set(['os', 'node:os']);
+const FS_PRESENCE = new Set(['existsSync']);
+
+function environmentImports(sourceFile) {
+ const names = new Set();
+ const namespaces = new Set();
+ for (const statement of sourceFile.statements) {
+ if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier))
+ continue;
+ if (!OS_MODULES.has(statement.moduleSpecifier.text)) continue;
+ const clause = statement.importClause;
+ if (clause?.name) namespaces.add(clause.name.text);
+ const named = clause?.namedBindings;
+ if (named && ts.isNamespaceImport(named)) namespaces.add(named.name.text);
+ if (named && ts.isNamedImports(named))
+ for (const element of named.elements) names.add(element.name.text);
+ }
+ return { names, namespaces };
+}
+
+function isEnvironmentAtom(node, environment) {
+ if (ciAtom(node, environment.ciAliases)) return true;
+ if (ts.isPropertyAccessExpression(node)) {
+ const target = unwrap(node.expression);
+ if (
+ ts.isIdentifier(target) &&
+ target.text === 'process' &&
+ PROCESS_ENVIRONMENT.has(node.name.text)
+ )
+ return true;
+ if (ts.isIdentifier(target) && environment.imports.namespaces.has(target.text)) return true;
+ }
+ if (ts.isIdentifier(node) && isExpressionIdentifier(node)) {
+ return environment.imports.names.has(node.text) || environment.aliases.has(node.text);
+ }
+ if (ts.isCallExpression(node)) {
+ const callee = unwrap(node.expression);
+ const name = ts.isIdentifier(callee)
+ ? callee.text
+ : ts.isPropertyAccessExpression(callee)
+ ? callee.name.text
+ : null;
+ return name !== null && FS_PRESENCE.has(name);
+ }
+ return false;
+}
+
+function dependsOnEnvironment(node, environment) {
+ let found = false;
+ const visit = (current) => {
+ if (found) return;
+ if (isEnvironmentAtom(current, environment)) {
+ found = true;
+ return;
+ }
+ if (isFunction(current) || ts.isFunctionDeclaration(current)) return;
+ ts.forEachChild(current, visit);
+ };
+ visit(node);
+ return found;
+}
+
+function environmentBindings(sourceFile, ciAliasMap, kinds) {
+ const environment = {
+ ciAliases: ciAliasMap,
+ imports: environmentImports(sourceFile),
+ aliases: new Set(),
+ };
+ const declarations = declarationsAtAnyDepth(sourceFile, kinds);
+ for (let pass = 0; pass < 3; pass += 1) {
+ for (const declaration of declarations) {
+ if (dependsOnEnvironment(declaration.initializer, environment))
+ environment.aliases.add(declaration.name.text);
+ }
+ }
+ return environment;
+}
+
+function ciEffect(condition, skipWhen, aliases) {
+ if (!containsCi(condition, aliases)) return null;
+ const skipWhenTrue = ciValue(condition, aliases, true);
+ const skipsOnCi = skipWhen === 'condition' ? skipWhenTrue : not(skipWhenTrue);
+ return skipsOnCi === false ? 'runs-only-on-ci' : 'skips-on-ci';
+}
+
+function enclosingCondition(node, boundary) {
+ let child = node;
+ let current = node.parent;
+ while (
+ current &&
+ current !== boundary &&
+ !isFunction(current) &&
+ !ts.isFunctionDeclaration(current)
+ ) {
+ if (ts.isIfStatement(current) && child !== current.expression) {
+ return {
+ condition: current.expression,
+ skipWhen: child === current.thenStatement ? 'condition' : 'negation',
+ };
+ }
+ child = current;
+ current = current.parent;
+ }
+ return null;
+}
+
+function containsBefore(scope, position, predicate) {
+ let found = false;
+ const visit = (node) => {
+ if (found || node.getStart() >= position) return;
+ if (predicate(node)) {
+ found = true;
+ return;
+ }
+ ts.forEachChild(node, visit);
+ };
+ visit(scope);
+ return found;
+}
+
+function readTags(options, key) {
+ const property = options && objectProperty(options, key);
+ if (!property) return { tags: [], literal: true };
+ if (!ts.isPropertyAssignment(property)) return { tags: [], literal: false };
+ const value = unwrap(property.initializer);
+ const single = stringValue(value);
+ if (single !== null) return { tags: [single], literal: true };
+ if (!ts.isArrayLiteralExpression(value)) return { tags: [], literal: false };
+ const tags = value.elements.map(stringValue);
+ return tags.includes(null)
+ ? { tags: tags.filter((tag) => tag !== null), literal: false }
+ : { tags, literal: true };
+}
+
+function readVitestFields(options) {
+ const property = options && objectProperty(options, 'meta');
+ if (!property) return { fields: {}, literal: true };
+ const value = ts.isPropertyAssignment(property) ? unwrap(property.initializer) : null;
+ if (!value || !ts.isObjectLiteralExpression(value)) return { fields: {}, literal: false };
+ const fields = {};
+ let literal = true;
+ for (const name of FIELDS) {
+ const field = objectProperty(value, name);
+ if (!field) continue;
+ const text = ts.isPropertyAssignment(field) ? stringValue(field.initializer) : null;
+ if (text === null) literal = false;
+ else fields[name] = text;
+ }
+ return { fields, literal };
+}
+
+function readPlaywrightFields(options) {
+ const property = options && objectProperty(options, 'annotation');
+ if (!property) return { fields: {}, literal: true };
+ const value = ts.isPropertyAssignment(property) ? unwrap(property.initializer) : null;
+ const entries =
+ value && ts.isArrayLiteralExpression(value) ? value.elements.map(unwrap) : [value];
+ const fields = {};
+ let literal = true;
+ for (const entry of entries) {
+ if (!entry || !ts.isObjectLiteralExpression(entry)) {
+ literal = false;
+ continue;
+ }
+ const typeProperty = objectProperty(entry, 'type');
+ const descriptionProperty = objectProperty(entry, 'description');
+ const type =
+ typeProperty && ts.isPropertyAssignment(typeProperty)
+ ? stringValue(typeProperty.initializer)
+ : null;
+ if (type === null) {
+ literal = false;
+ continue;
+ }
+ if (!FIELDS.includes(type)) continue;
+ const description =
+ descriptionProperty && ts.isPropertyAssignment(descriptionProperty)
+ ? stringValue(descriptionProperty.initializer)
+ : null;
+ if (description === null) literal = false;
+ else fields[type] = description;
+ }
+ return { fields, literal };
+}
+
+function configuredRetries(statements, kinds) {
+ for (const statement of statements) {
+ if (!ts.isExpressionStatement(statement)) continue;
+ const call = unwrap(statement.expression);
+ if (!ts.isCallExpression(call)) continue;
+ const ref = runnerRef(call.expression, kinds);
+ if (ref?.members.at(-1) !== 'configure') continue;
+ const options = call.arguments[0] ? unwrap(call.arguments[0]) : null;
+ if (!options || !ts.isObjectLiteralExpression(options)) continue;
+ const retries = objectProperty(options, 'retries');
+ if (!retries) continue;
+ const value = ts.isPropertyAssignment(retries) ? unwrap(retries.initializer) : null;
+ return value && ts.isNumericLiteral(value) ? Number(value.text) : Number.NaN;
+ }
+ return undefined;
+}
+
+function effectiveRetries(call, kinds) {
+ let current = call.parent;
+ while (current) {
+ if (
+ isFunction(current) &&
+ current.parent &&
+ ts.isCallExpression(current.parent) &&
+ ts.isBlock(current.body)
+ ) {
+ const owner = runnerCall(current.parent, kinds);
+ if (owner && declarationKind(owner.ref) === 'describe') {
+ const retries = configuredRetries(current.body.statements, kinds);
+ if (retries !== undefined) return retries;
+ }
+ }
+ if (ts.isSourceFile(current)) return configuredRetries(current.statements, kinds);
+ current = current.parent;
+ }
+ return undefined;
+}
+
+function longestLiteralRun(regexLiteral) {
+ const pattern = regexLiteral.slice(1, regexLiteral.lastIndexOf('/'));
+ const cut = String.fromCharCode(0);
+ const literal = pattern
+ .replace(/\[(?:\\.|[^\]\\])*\]/g, cut)
+ .replace(/\\[pPu]\{[^}]*\}|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\c[A-Za-z]/g, cut)
+ .replace(/\\[dDwWsSbB]/g, cut)
+ .replace(/\\./g, 'x')
+ .replace(/\{\d*,?\d*\}/g, cut)
+ .replace(/[.^$*+?()|{}]/g, cut);
+ return Math.max(0, ...literal.split(cut).map((part) => part.length));
+}
+
+function findSignature(fn) {
+ let found = null;
+ const visit = (node) => {
+ if (found) return;
+ if (ts.isCallExpression(node)) {
+ const callee = unwrap(node.expression);
+ const name = ts.isIdentifier(callee)
+ ? callee.text
+ : ts.isPropertyAccessExpression(callee)
+ ? callee.name.text
+ : null;
+ if (name === 'expectKnownBug') {
+ const signature = node.arguments[0] ? unwrap(node.arguments[0]) : null;
+ found = {
+ node,
+ signature: signature && ts.isRegularExpressionLiteral(signature) ? signature.text : null,
+ };
+ return;
+ }
+ }
+ ts.forEachChild(node, visit);
+ };
+ visit(fn);
+ return found;
+}
+
+function scopeOf(node, kinds) {
+ let current = node.parent;
+ while (current) {
+ if (
+ isFunction(current) ||
+ ts.isFunctionDeclaration(current) ||
+ ts.isMethodDeclaration(current)
+ ) {
+ const owner =
+ current.parent && ts.isCallExpression(current.parent)
+ ? runnerCall(current.parent, kinds)
+ : null;
+ const kind = owner ? declarationKind(owner.ref) : null;
+ return kind === 'test' || kind === 'describe' ? kind : 'helper';
+ }
+ current = current.parent;
+ }
+ return 'file';
+}
+
+export function scanSource(path, source) {
+ const sourceFile = ts.createSourceFile(
+ path,
+ source,
+ ts.ScriptTarget.Latest,
+ true,
+ scriptKind(path),
+ );
+ const runner = PLAYWRIGHT_FILE.test(path) ? 'playwright' : 'vitest';
+ const kinds = runnerBindings(sourceFile);
+ const aliases = ciAliases(sourceFile, kinds);
+ const environment = environmentBindings(sourceFile, aliases, kinds);
+ const scan = {
+ path,
+ runner,
+ pins: [],
+ quarantines: [],
+ gates: [],
+ notRun: [],
+ earlyReturns: [],
+ problems: [],
+ };
+ const lineOf = (node) =>
+ sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + 1;
+ const problem = (node, rule, message) =>
+ scan.problems.push({ path, line: lineOf(node), rule, message });
+ const gate = (node, form, scope, condition, skipWhen, reason = '') => {
+ scan.gates.push({
+ path,
+ line: lineOf(node),
+ form,
+ scope,
+ condition: condition ? oneLine(condition, sourceFile) : '',
+ skipWhen,
+ ci: condition ? ciEffect(condition, skipWhen, aliases) : null,
+ reason,
+ });
+ };
+ const notRun = (node, form, scope, title, reason = '') =>
+ scan.notRun.push({ path, line: lineOf(node), form, scope, title, reason });
+
+ const inspectTestBody = (fn, via) => {
+ const context = via === 'each' ? undefined : fn.parameters[via === 'for' ? 1 : 0]?.name;
+ const contextNames = new Set();
+ const skipNames = new Set();
+ if (context && ts.isIdentifier(context)) contextNames.add(context.text);
+ if (context && ts.isObjectBindingPattern(context)) {
+ for (const element of context.elements) {
+ const key = (element.propertyName ?? element.name).getText(sourceFile);
+ if (key === 'skip' && ts.isIdentifier(element.name)) skipNames.add(element.name.text);
+ }
+ }
+ const isContextSkip = (node) => {
+ if (!ts.isCallExpression(node)) return false;
+ const callee = unwrap(node.expression);
+ if (ts.isIdentifier(callee)) return skipNames.has(callee.text);
+ return (
+ ts.isPropertyAccessExpression(callee) &&
+ callee.name.text === 'skip' &&
+ ts.isIdentifier(unwrap(callee.expression)) &&
+ contextNames.has(unwrap(callee.expression).text)
+ );
+ };
+ const isAnySkip = (node) => {
+ if (isContextSkip(node)) return true;
+ if (!ts.isCallExpression(node)) return false;
+ const ref = runnerRef(node.expression, kinds);
+ return Boolean(ref) && ref.members.at(-1) === 'skip';
+ };
+ const visit = (node) => {
+ if (node !== fn && (isFunction(node) || ts.isFunctionDeclaration(node))) return;
+ if (isContextSkip(node)) {
+ const [first, second] = node.arguments;
+ if (first && !isTitle(unwrap(first))) {
+ gate(
+ node,
+ 'context-skip',
+ 'test',
+ first,
+ 'condition',
+ second ? (stringValue(second) ?? '') : '',
+ );
+ } else {
+ const around = enclosingCondition(node, fn);
+ const reason = first ? (stringValue(first) ?? '') : '';
+ if (around) gate(node, 'context-skip', 'test', around.condition, around.skipWhen, reason);
+ else notRun(node, 'context-skip', 'test', '', reason);
+ }
+ }
+ if (ts.isReturnStatement(node) && !node.expression) {
+ const around = enclosingCondition(node, fn);
+ if (around && dependsOnEnvironment(around.condition, environment)) {
+ const guarded = containsBefore(
+ fn.body,
+ node.getStart(),
+ (candidate) => isAssertionCall(candidate) || isAnySkip(candidate),
+ );
+ if (!guarded) {
+ scan.earlyReturns.push({
+ path,
+ line: lineOf(node),
+ condition: oneLine(around.condition, sourceFile),
+ });
+ }
+ }
+ }
+ ts.forEachChild(node, visit);
+ };
+ ts.forEachChild(fn, visit);
+ };
+
+ const inspectDeclaration = (call, info, kind) => {
+ const parts = declarationParts(call);
+ const modifiers = new Set(info.ref.members);
+ const title = parts.title ? (stringValue(parts.title) ?? oneLine(parts.title, sourceFile)) : '';
+ const tagKey = runner === 'playwright' ? 'tag' : 'tags';
+ const { tags, literal } = readTags(parts.options, tagKey);
+ if (!literal) {
+ problem(
+ call,
+ 'tags-not-literal',
+ `write \`${tagKey}\` as inline string literals so the known-reds listing can see it`,
+ );
+ }
+ const pinTag = runner === 'playwright' ? `@${KNOWN_BUG_TAG}` : KNOWN_BUG_TAG;
+ const quarantineTag = runner === 'playwright' ? `@${QUARANTINE_TAG}` : QUARANTINE_TAG;
+ const isPin = tags.includes(pinTag);
+ const isQuarantine = tags.includes(quarantineTag);
+ const skipOption = parts.options && objectProperty(parts.options, 'skip');
+ const skipValue =
+ skipOption && ts.isPropertyAssignment(skipOption)
+ ? unwrap(skipOption.initializer)
+ : skipOption
+ ? skipOption.name
+ : null;
+ const skipTruth = skipValue ? literalTruth(skipValue) : false;
+ const todoOption = parts.options && objectProperty(parts.options, 'todo');
+ const failsOption = parts.options && objectProperty(parts.options, 'fails');
+ if (failsOption) {
+ problem(
+ call,
+ 'bare-expected-failure',
+ 'an expected-failure marker accepts any failure; pin the bug with expectKnownBug instead',
+ );
+ }
+ const declaredNotRun =
+ [...modifiers].find((modifier) => NOT_RUN_MODIFIERS.has(modifier)) ?? null;
+ const skipped = Boolean(declaredNotRun) || skipTruth === true || Boolean(todoOption);
+ if (skipValue && skipTruth === null) gate(call, 'skip-option', kind, skipValue, 'condition');
+ if ((isPin || isQuarantine) && kind === 'describe') {
+ problem(
+ call,
+ 'tag-on-describe',
+ `tag each test ${isPin ? pinTag : quarantineTag}, not the describe, so every pin carries its own issue, owner and date`,
+ );
+ } else if (isPin || isQuarantine) {
+ const { fields, literal: fieldsLiteral } =
+ runner === 'playwright'
+ ? readPlaywrightFields(parts.options)
+ : readVitestFields(parts.options);
+ if (!fieldsLiteral) {
+ problem(
+ call,
+ 'fields-not-literal',
+ `write the issue, owner and until ${runner === 'playwright' ? 'annotations' : 'meta'} as inline string literals`,
+ );
+ }
+ const record = { path, line: lineOf(call), runner, title, ...fields };
+ if (isPin && isQuarantine)
+ problem(
+ call,
+ 'pin-and-quarantine',
+ 'a test is either a pinned known bug or a quarantined flake, not both',
+ );
+ if (isPin) {
+ const found = parts.fn ? findSignature(parts.fn) : null;
+ record.signature = found?.signature ?? null;
+ if (!found) {
+ problem(
+ call,
+ 'pin-without-helper',
+ 'a known-bug pin must wrap its correct assertion in expectKnownBug(/signature/, fn)',
+ );
+ } else if (found.signature === null) {
+ problem(
+ found.node,
+ 'signature-not-literal',
+ 'pass expectKnownBug a regular-expression literal naming the observed wrong outcome',
+ );
+ } else if (longestLiteralRun(found.signature) < MIN_SIGNATURE_LITERAL) {
+ problem(
+ found.node,
+ 'signature-too-broad',
+ `a pin's signature must name the observed wrong outcome with at least ${MIN_SIGNATURE_LITERAL} literal characters in a row; a pattern like /./ absorbs any failure`,
+ );
+ }
+ if (skipped)
+ problem(
+ call,
+ 'pin-not-run',
+ 'a known-bug pin must run; a pin that is skipped checks nothing',
+ );
+ if (runner === 'playwright') {
+ const retries = effectiveRetries(call, kinds);
+ if (retries !== 0) {
+ problem(
+ call,
+ 'pin-retries',
+ 'run a Playwright pin with `test.describe.configure({ retries: 0 })` in its describe, so a retry cannot turn a changed failure into a flaky pass',
+ );
+ }
+ }
+ scan.pins.push(record);
+ } else {
+ if (!skipped)
+ problem(call, 'quarantine-runs', 'a quarantined flake is declared skipped, with `.skip`');
+ scan.quarantines.push(record);
+ }
+ } else if (skipped) {
+ const form = declaredNotRun ?? (todoOption ? 'todo-option' : 'skip-option');
+ notRun(call, form, kind, title);
+ }
+ if (info.gate) {
+ gate(call, info.gate.form, kind, info.gate.condition, info.gate.skipWhen);
+ }
+ if (kind === 'test' && parts.fn) inspectTestBody(parts.fn, info.via);
+ };
+
+ const inspectAnnotation = (call, modifier) => {
+ const [first, second] = call.arguments.map(unwrap);
+ const scope = scopeOf(call, kinds);
+ const reason = second ? (stringValue(second) ?? '') : '';
+ if (modifier === 'fail') {
+ problem(
+ call,
+ 'bare-expected-failure',
+ 'test.fail() accepts any failure in the test body; pin the bug with expectKnownBug instead',
+ );
+ return;
+ }
+ if (modifier === 'slow') return;
+ if (first && isFunction(first)) {
+ const body = ts.isBlock(first.body) ? null : first.body;
+ gate(call, `${modifier}-callback`, scope, body ?? first, 'condition', reason);
+ return;
+ }
+ const truth = first ? literalTruth(first) : true;
+ if (truth === false) return;
+ if (truth === null) {
+ gate(call, modifier, scope, first, 'condition', reason);
+ return;
+ }
+ const around = enclosingCondition(call, null);
+ if (around) gate(call, modifier, scope, around.condition, around.skipWhen, reason);
+ else notRun(call, modifier, scope, '', reason);
+ };
+
+ const visit = (node) => {
+ if (ts.isVariableDeclaration(node) && node.initializer) {
+ const ternary = skipTernary(node.initializer, kinds);
+ if (ternary) {
+ const topLevel =
+ ts.isVariableStatement(node.parent?.parent) && ts.isSourceFile(node.parent.parent.parent);
+ gate(
+ node,
+ 'alias',
+ topLevel ? 'file' : scopeOf(node, kinds),
+ ternary.condition,
+ ternary.skipWhen,
+ );
+ }
+ }
+ if (ts.isCallExpression(node)) {
+ const info = runnerCall(node, kinds);
+ if (info) {
+ const kind = declarationKind(info.ref);
+ const bare = info.ref.members.find((member) => BARE_EXPECTED_FAILURE.has(member));
+ const parts = kind === 'test' || kind === 'describe' ? declarationParts(node) : null;
+ const annotation =
+ kind === 'test' &&
+ parts &&
+ !parts.title &&
+ ANNOTATION_MODIFIERS.has(info.ref.members.at(-1) ?? '') &&
+ !info.gate;
+ if (annotation) {
+ inspectAnnotation(node, info.ref.members.at(-1));
+ } else if (parts && (parts.fn || info.ref.members.includes('todo'))) {
+ if (bare) {
+ problem(
+ node,
+ 'bare-expected-failure',
+ `\`.${bare}\` accepts any failure; pin the bug with expectKnownBug instead`,
+ );
+ }
+ inspectDeclaration(node, info, kind);
+ }
+ }
+ }
+ ts.forEachChild(node, visit);
+ };
+ visit(sourceFile);
+ return scan;
+}
+
+export function listTestFiles(root) {
+ const listed = spawnSync(
+ 'git',
+ ['ls-files', '-z', '--cached', '--others', '--exclude-standard'],
+ {
+ cwd: root,
+ encoding: 'utf8',
+ env: gitCleanEnv(),
+ maxBuffer: 1 << 28,
+ windowsHide: true,
+ },
+ );
+ if (listed.error || listed.status !== 0) {
+ throw new Error(
+ `known-reds: git ls-files failed in ${root} (${listed.error?.message ?? listed.stderr.trim()}), so the scan cannot show that it read every test file.`,
+ );
+ }
+ return listed.stdout
+ .split('\0')
+ .filter(
+ (path) =>
+ path !== '' &&
+ TEST_FILE.test(path) &&
+ !path.includes('node_modules/') &&
+ !EXCLUDED_PREFIXES.some((prefix) => path.startsWith(prefix)) &&
+ existsSync(join(root, path)),
+ )
+ .sort();
+}
+
+export function scanTree(root) {
+ if (typeof ts.createSourceFile !== 'function') {
+ throw new Error(
+ `known-reds: the resolved typescript ${ts.version} has no createSourceFile, so the scan cannot run.`,
+ );
+ }
+ const files = listTestFiles(root);
+ const report = {
+ schemaVersion: SCHEMA_VERSION,
+ files: files.length,
+ pins: [],
+ quarantines: [],
+ ciSkips: [],
+ envGates: [],
+ notRun: [],
+ earlyReturns: [],
+ problems: [],
+ };
+ for (const path of files) {
+ const scan = scanSource(path, readFileSync(join(root, path), 'utf8'));
+ report.pins.push(...scan.pins);
+ report.quarantines.push(...scan.quarantines);
+ report.notRun.push(...scan.notRun);
+ report.earlyReturns.push(...scan.earlyReturns);
+ report.problems.push(...scan.problems);
+ for (const entry of scan.gates)
+ (entry.ci === 'skips-on-ci' ? report.ciSkips : report.envGates).push(entry);
+ }
+ return report;
+}
+
+export function todayUtc(now = new Date()) {
+ return now.toISOString().slice(0, 10);
+}
+
+function addDays(day, days) {
+ return new Date(Date.parse(`${day}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
+}
+
+function dateProblem(until, today) {
+ if (!until) return 'has no until date';
+ if (!ISO_DATE.test(until)) return `has until "${until}", which is not YYYY-MM-DD`;
+ const time = Date.parse(`${until}T00:00:00Z`);
+ if (Number.isNaN(time) || new Date(time).toISOString().slice(0, 10) !== until)
+ return `has until "${until}", which is not a date`;
+ if (until < today) return `expired on ${until}`;
+ const horizon = addDays(today, MAX_HORIZON_DAYS);
+ if (until > horizon)
+ return `has until ${until}, beyond the ${MAX_HORIZON_DAYS}-day horizon (${horizon})`;
+ return null;
+}
+
+function trackedProblems(entry, today, { needsIssue }) {
+ const problems = [];
+ if (needsIssue && !entry.issue) problems.push('has no issue link');
+ if (entry.issue && !ISSUE_URL.test(entry.issue))
+ problems.push(`has issue "${entry.issue}", which is not a GitHub or Linear issue URL`);
+ if (!entry.owner) problems.push('has no owner');
+ else if (!OWNERS.includes(entry.owner))
+ problems.push(`has owner "${entry.owner}", which is not one of ${OWNERS.join(', ')}`);
+ const date = dateProblem(entry.until, today);
+ if (date) problems.push(date);
+ return problems;
+}
+
+export function loadAllowlist(path = ALLOWLIST_PATH) {
+ const parsed = JSON.parse(readFileSync(path, 'utf8'));
+ if (!parsed || !Array.isArray(parsed.ciSkips)) {
+ throw new Error(`known-reds: ${path} must hold { "ciSkips": [...] }.`);
+ }
+ return parsed;
+}
+
+export function validate(report, { today, allowlist }) {
+ const violations = [...report.problems];
+ for (const pin of report.pins) {
+ for (const message of trackedProblems(pin, today, { needsIssue: true })) {
+ violations.push({
+ path: pin.path,
+ line: pin.line,
+ rule: 'pin-fields',
+ message: `known-bug pin ${message}`,
+ });
+ }
+ }
+ for (const quarantine of report.quarantines) {
+ for (const message of trackedProblems(quarantine, today, { needsIssue: true })) {
+ violations.push({
+ path: quarantine.path,
+ line: quarantine.line,
+ rule: 'quarantine-fields',
+ message: `quarantine ${message}`,
+ });
+ }
+ }
+ for (const early of report.earlyReturns) {
+ violations.push({
+ path: early.path,
+ line: early.line,
+ rule: 'early-return',
+ message: `returns early when ${early.condition}, so the test reports a pass without running; skip it instead (ctx.skip(condition, note) in Vitest, test.skip(condition, reason) in Playwright)`,
+ });
+ }
+ const entries = allowlist.ciSkips;
+ const matched = new Set();
+ for (const skip of report.ciSkips) {
+ const index = entries.findIndex(
+ (entry, at) =>
+ !matched.has(at) && entry.path === skip.path && entry.condition === skip.condition,
+ );
+ if (index === -1) {
+ violations.push({
+ path: skip.path,
+ line: skip.line,
+ rule: 'ci-skip',
+ message: `skips on CI (${skip.condition}), so CI reports it green without running it; run it on CI, or key the skip on the missing capability rather than on CI`,
+ });
+ } else {
+ matched.add(index);
+ }
+ }
+ entries.forEach((entry, index) => {
+ const where = { path: `scripts/known-reds-allowlist.json`, line: index + 1 };
+ if (!matched.has(index)) {
+ violations.push({
+ ...where,
+ rule: 'allowlist-stale',
+ message: `entry ${entry.path} (${entry.condition}) matches no CI-keyed skip; remove it`,
+ });
+ }
+ if (!entry.reason)
+ violations.push({
+ ...where,
+ rule: 'allowlist-fields',
+ message: `entry ${entry.path} has no reason`,
+ });
+ for (const message of trackedProblems(entry, today, { needsIssue: false })) {
+ violations.push({
+ ...where,
+ rule: 'allowlist-fields',
+ message: `entry ${entry.path} ${message}`,
+ });
+ }
+ });
+ return violations;
+}
+
+export function expiringSoon(report, { today, allowlist }) {
+ const limit = addDays(today, EXPIRY_WARNING_DAYS);
+ const due = (entry) =>
+ typeof entry.until === 'string' &&
+ ISO_DATE.test(entry.until) &&
+ entry.until >= today &&
+ entry.until <= limit;
+ const row = (kind) => (entry) => ({
+ kind,
+ path: entry.path,
+ line: entry.line ?? null,
+ owner: entry.owner ?? null,
+ until: entry.until,
+ });
+ return [
+ ...report.pins.filter(due).map(row('pin')),
+ ...report.quarantines.filter(due).map(row('quarantine')),
+ ...allowlist.ciSkips.filter(due).map(row('ci-skip-entry')),
+ ];
+}
+
+const EXPIRING_KIND_LABELS = {
+ pin: 'pin',
+ quarantine: 'quarantine',
+ 'ci-skip-entry': 'allowlisted CI skip',
+};
+
+export function render(report, violations, upcoming, today, { all }) {
+ const lines = [
+ `Known reds in open-knowledge on ${today}: ${report.files} test files scanned.`,
+ '',
+ ];
+ const section = (heading, rows, format, show = true) => {
+ lines.push(`${heading} (${rows.length})`);
+ if (show) for (const row of rows) lines.push(` ${format(row)}`);
+ lines.push('');
+ };
+ section(
+ 'Pinned known bugs',
+ report.pins,
+ (pin) =>
+ `${pin.path}:${pin.line} ${pin.title} ${pin.issue ?? '?'} owner ${pin.owner ?? '?'}, until ${pin.until ?? '?'} ${pin.signature ?? ''}`,
+ );
+ section(
+ 'Quarantined flakes',
+ report.quarantines,
+ (row) =>
+ `${row.path}:${row.line} ${row.title} ${row.issue ?? '?'} owner ${row.owner ?? '?'}, until ${row.until ?? '?'}`,
+ );
+ section(
+ 'Skipped on CI',
+ report.ciSkips,
+ (row) => `${row.path}:${row.line} ${row.scope} ${row.condition}`,
+ );
+ section(
+ 'Not run: skip, fixme and todo',
+ report.notRun,
+ (row) => `${row.path}:${row.line} ${row.form} ${row.title || row.reason}`,
+ all,
+ );
+ section(
+ 'Environment gates',
+ report.envGates,
+ (row) =>
+ `${row.path}:${row.line} ${row.form} ${row.condition}${row.ci ? ' (runs only on CI)' : ''}`,
+ all,
+ );
+ section(
+ `Expiring within ${EXPIRY_WARNING_DAYS} days`,
+ upcoming,
+ (row) =>
+ `${EXPIRING_KIND_LABELS[row.kind] ?? row.kind} ${row.path}${row.line === null ? '' : `:${row.line}`} owner ${row.owner ?? '?'}, until ${row.until}`,
+ );
+ section(
+ 'Violations',
+ violations,
+ (row) => `${row.path}:${row.line} [${row.rule}] ${row.message}`,
+ );
+ if (!all)
+ lines.push(
+ 'Pass --all to list every not-run test and environment gate, or --json for the full feed.',
+ );
+ lines.push(`The convention: ${CONVENTION_DOC}.`);
+ return `${lines.join('\n')}\n`;
+}
+
+function main(argv) {
+ try {
+ const rootAt = argv.indexOf('--root');
+ const root = rootAt === -1 ? OK_ROOT : resolve(argv[rootAt + 1] ?? '');
+ const today = todayUtc();
+ const report = scanTree(root);
+ const allowlist = loadAllowlist();
+ const violations = validate(report, { today, allowlist });
+ const upcoming = expiringSoon(report, { today, allowlist });
+ if (argv.includes('--json')) {
+ process.stdout.write(
+ `${JSON.stringify({ ...report, today, expiringSoon: upcoming, violations }, null, 2)}\n`,
+ );
+ } else {
+ process.stdout.write(
+ render(report, violations, upcoming, today, { all: argv.includes('--all') }),
+ );
+ }
+ process.exitCode = violations.length > 0 ? 1 : 0;
+ } catch (error) {
+ process.stderr.write(
+ `known-reds: could not run: ${error instanceof Error ? error.message : String(error)}\n`,
+ );
+ process.exitCode = 2;
+ }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url))
+ main(process.argv.slice(2));
diff --git a/test-support/KNOWN-REDS.md b/test-support/KNOWN-REDS.md
new file mode 100644
index 000000000..c263867fb
--- /dev/null
+++ b/test-support/KNOWN-REDS.md
@@ -0,0 +1,86 @@
+# Known reds, quarantines and skips
+
+How Open Knowledge marks a test that is expected to fail or that deliberately does not run.
+
+- **The listing command:** `pnpm known-reds` lists every such test.
+- **The checks:** `pnpm known-reds` exits 1 when a test breaks a rule below, and the guard test `scripts/check-known-reds.uncached.test.mjs` runs the same checks in the uncached test tier.
+- **The scanner and its data:** `scripts/known-reds.mjs` holds the scanner and the owner list, and `scripts/known-reds-allowlist.json` holds the CI-keyed skips still waiting to be removed.
+
+## A known bug: pin it
+
+A test that fails because the product is wrong is still a working test. Write the assertion the product should satisfy, and wrap it in `expectKnownBug`. Its signature names the wrong outcome you observed:
+
+```ts
+import { expectKnownBug } from '../../../test-support/known-bug.vitest.test-helper';
+
+test(
+ 'hiding a file removes its sidebar row',
+ {
+ tags: ['known-bug'],
+ meta: {
+ issue: 'https://github.com/inkeep/agents-private/issues/2816',
+ owner: 'get-main-green',
+ until: '2026-12-01',
+ },
+ },
+ async () => {
+ await expectKnownBug(/row still visible/, async () => {
+ expect(await sidebarRowVisible('draft.md')).toBe(false);
+ });
+ },
+);
+```
+
+**How a pin behaves:**
+- While the bug is present, the pin passes.
+- Once the bug is fixed, it fails with "known bug appears fixed".
+- If anything else goes wrong, it fails with the original error.
+
+**The signature** names the wrong outcome itself: the received value, or the Playwright call-log line that shows it. An assertion's name or a timeout alone is not enough, because a broken selector would match it too. The guard refuses a signature without at least 4 literal characters in a row, such as `/./`.
+
+**The three fields:**
+- `issue` is a GitHub or Linear issue URL.
+- `owner` is one of the owners listed in `scripts/known-reds.mjs`.
+- `until` is a date no more than 90 days away.
+
+When the date passes, the guard fails. Either fix the bug, or move the date and record why on the issue. The listing shows every pin, quarantine and allowlist entry under "Expiring within 14 days" before that happens.
+
+**Never use** `test.fails`, `{ fails: true }` or Playwright's `test.fail()`. Each accepts any failure, including one that has nothing to do with the bug.
+
+**Playwright pins** differ in four ways:
+- they import `test-support/known-bug.test-helper`;
+- they use the tag `'@known-bug'`;
+- they carry the three fields as `annotation` entries, `{ type: 'issue' | 'owner' | 'until', description }`;
+- they sit in a describe that calls `test.describe.configure({ retries: 0 })`, so that a retry cannot turn a changed failure into a flaky pass.
+
+**When the bug is fixed,** keep the assertion, remove `expectKnownBug` and the tag, and close the issue.
+
+## A limitation accepted by design
+
+This is not a known red. Assert today's behaviour in an ordinary test whose title says what it is.
+
+## A flake: quarantine it
+
+Declare the test skipped, tag it `quarantine` (`@quarantine` in Playwright), and give it the same `issue`, `owner` and `until` fields. A flake is never pinned, because a pin has to fail on every run.
+
+## A test that cannot run here
+
+Skip it with the runner's own conditional skip, and say why:
+- `test.skipIf(condition)` or `ctx.skip(condition, note)` in Vitest;
+- `test.skip(condition, reason)` in Playwright.
+
+Never `return` early from a test body on an environment condition, such as the platform, an environment variable or a missing build: the runner reports that as a pass.
+
+A test in the uncached tier (`*.uncached.test.*`) may not skip at all. That tier fails any run in which a test did not execute, and prints the rule for such tests (`test-support/uncached-tier-floor.ts`).
+
+A skip keyed on CI (`process.env.CI`, `IS_CI` and the like) hides the test from the run that gates merges. It fails the guard unless `scripts/known-reds-allowlist.json` lists it with an owner, a date and a reason. Prefer to key the skip on the missing capability instead.
+
+## A test not written yet
+
+Use `test.todo('title')`.
+
+## The listing
+
+- `pnpm known-reds` prints the pins, the quarantines and the CI-keyed skips.
+- `--all` adds every test that does not run and every environment gate.
+- `--json` prints the full feed (`schemaVersion` 1).
diff --git a/test-support/known-bug.test-helper.ts b/test-support/known-bug.test-helper.ts
new file mode 100644
index 000000000..0d524322d
--- /dev/null
+++ b/test-support/known-bug.test-helper.ts
@@ -0,0 +1,28 @@
+const ANSI_ESCAPE = new RegExp(`${String.fromCharCode(27)}\\[[0-?]*[ -/]*[@-~]`, 'g');
+
+function failureMessage(failure: unknown): string {
+ const message = failure instanceof Error ? failure.message : String(failure);
+ return message.replace(ANSI_ESCAPE, '');
+}
+
+export async function expectKnownBug(
+ signature: RegExp,
+ assertCorrectBehaviour: () => unknown,
+): Promise<string> {
+ let failure: unknown;
+ let failed = false;
+ try {
+ await assertCorrectBehaviour();
+ } catch (error) {
+ failed = true;
+ failure = error;
+ }
+ if (!failed) {
+ throw new Error(
+ `known bug appears fixed: the correct assertion passed, so ${signature} no longer describes a failure. Remove the pin: keep the assertion, drop expectKnownBug and the known-bug tag, and close the issue.`,
+ );
+ }
+ const message = failureMessage(failure);
+ if (message.search(signature) === -1) throw failure;
+ return message;
+}
diff --git a/test-support/known-bug.test.ts b/test-support/known-bug.test.ts
new file mode 100644
index 000000000..93bbb8f12
--- /dev/null
+++ b/test-support/known-bug.test.ts
@@ -0,0 +1,107 @@
+import { describe, expect, test } from 'vitest';
+import { expectKnownBug } from './known-bug.test-helper';
+import { expectKnownBug as expectKnownBugInVitest } from './known-bug.vitest.test-helper';
+
+const ESC = String.fromCharCode(27);
+
+describe('expectKnownBug', () => {
+ test('passes and returns the message when the correct assertion fails with the signature', async () => {
+ const message = await expectKnownBug(/row still visible/, () => {
+ throw new Error('sidebar row still visible after hide');
+ });
+ expect(message).toBe('sidebar row still visible after hide');
+ });
+
+ test('accepts an asynchronous assertion that rejects with the signature', async () => {
+ await expect(
+ expectKnownBug(/ordinal 1/, async () => {
+ await Promise.resolve();
+ throw new Error('expected ordinal 2, received ordinal 1');
+ }),
+ ).resolves.toBe('expected ordinal 2, received ordinal 1');
+ });
+
+ test('fails with "appears fixed" when the correct assertion passes', async () => {
+ await expect(expectKnownBug(/row still visible/, () => undefined)).rejects.toThrow(
+ /^known bug appears fixed: .*\/row still visible\/.*Remove the pin/,
+ );
+ });
+
+ test('rethrows an unrelated failure unchanged', async () => {
+ const unrelated = new TypeError('cannot read properties of undefined');
+ await expect(
+ expectKnownBug(/row still visible/, () => {
+ throw unrelated;
+ }),
+ ).rejects.toBe(unrelated);
+ });
+
+ test('matches against the message with ANSI colour codes removed', async () => {
+ const coloured = `Expected: ${ESC}[32m"hidden"${ESC}[39m\nReceived: ${ESC}[31m"visible"${ESC}[39m`;
+ await expect(
+ expectKnownBug(/Received: "visible"/, () => {
+ throw new Error(coloured);
+ }),
+ ).resolves.toBe('Expected: "hidden"\nReceived: "visible"');
+ });
+
+ test('does not match a signature that only appears inside the colour codes', async () => {
+ const coloured = `Received: ${ESC}[31m"visible"${ESC}[39m`;
+ const failure = new Error(coloured);
+ await expect(
+ expectKnownBug(/\[31m/, () => {
+ throw failure;
+ }),
+ ).rejects.toBe(failure);
+ });
+
+ test('matches a thrown non-Error value by its string form', async () => {
+ await expect(
+ expectKnownBug(/ENOENT/, () => {
+ throw 'ENOENT: no such file';
+ }),
+ ).resolves.toBe('ENOENT: no such file');
+ });
+
+ test('ignores a global signature’s lastIndex', async () => {
+ const signature = /visible/g;
+ signature.lastIndex = 99;
+ await expect(
+ expectKnownBug(signature, () => {
+ throw new Error('row visible');
+ }),
+ ).resolves.toBe('row visible');
+ });
+});
+
+describe('expectKnownBug for Vitest', () => {
+ test('counts as one assertion even when the correct assertion never reaches expect', async () => {
+ const before = expect.getState().assertionCalls;
+ await expectKnownBugInVitest(/ENOENT/, () => {
+ throw new Error('ENOENT: no such file');
+ });
+ const made = expect.getState().assertionCalls - before;
+ expect(made).toBe(1);
+ });
+
+ test('passes on a failing Vitest assertion that names the wrong value', async () => {
+ await expectKnownBugInVitest(/expected 1 to be 2/, () => {
+ expect(1).toBe(2);
+ });
+ });
+
+ test('fails with "appears fixed" when the bug is gone', async () => {
+ await expect(expectKnownBugInVitest(/expected 1 to be 2/, () => undefined)).rejects.toThrow(
+ /^known bug appears fixed/,
+ );
+ });
+
+ test('rethrows an unrelated failure unchanged', async () => {
+ const unrelated = new RangeError('out of range');
+ await expect(
+ expectKnownBugInVitest(/expected 1 to be 2/, () => {
+ throw unrelated;
+ }),
+ ).rejects.toBe(unrelated);
+ });
+});
diff --git a/test-support/known-bug.vitest.test-helper.ts b/test-support/known-bug.vitest.test-helper.ts
new file mode 100644
index 000000000..465c67956
--- /dev/null
+++ b/test-support/known-bug.vitest.test-helper.ts
@@ -0,0 +1,11 @@
+import { expect } from 'vitest';
+import { expectKnownBug as catchKnownBug } from './known-bug.test-helper';
+
+export async function expectKnownBug(
+ signature: RegExp,
+ assertCorrectBehaviour: () => unknown,
+): Promise<string> {
+ const message = await catchKnownBug(signature, assertCorrectBehaviour);
+ expect(message).toMatch(signature);
+ return message;
+}
diff --git a/test-support/vitest.base.ts b/test-support/vitest.base.ts
index 3c5979d77..bf2d6f825 100644
--- a/test-support/vitest.base.ts
+++ b/test-support/vitest.base.ts
@@ -40,6 +40,18 @@ export const okVitestBase = {
testTimeout: 30_000,
hookTimeout: 30_000,
env: { DO_NOT_TRACK: '1' },
+ tags: [
+ {
+ name: 'known-bug',
+ description:
+ 'Asserts the correct behaviour of a tracked bug through expectKnownBug; listed by pnpm known-reds.',
+ },
+ {
+ name: 'quarantine',
+ description:
+ 'A flaky test declared skipped with an issue, an owner and an expiry; listed by pnpm known-reds.',
+ },
+ ],
setupFiles: [bunGlobalShimPath, noNetConnectPath],
include: ['**/*.test.ts?(x)'],
exclude: [