Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Repair audit scope CLI child lifetimes (#5632)

Nick Gomez6f5ad4555907
diff --git a/packages/cli/src/commands/audit-scope-composition.test.ts b/packages/cli/src/commands/audit-scope-composition.test.ts
index 1f1332479..06980e85c 100644
--- a/packages/cli/src/commands/audit-scope-composition.test.ts
+++ b/packages/cli/src/commands/audit-scope-composition.test.ts
@@ -1,4 +1,5 @@
-import { spawn } from 'node:child_process';
+import { type ChildProcessByStdio, spawn } from 'node:child_process';
+import { EventEmitter } from 'node:events';
import {
existsSync,
mkdirSync,
@@ -12,6 +13,7 @@ import {
import { createServer } from 'node:http';
import { hostname, tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
+import { PassThrough, type Readable } from 'node:stream';
import { fileURLToPath } from 'node:url';
import {
ProblemDetailsSchema,
@@ -19,7 +21,8 @@ import {
ValidationAuditResponseSchema,
} from '@inkeep/open-knowledge-core';
import { ConfigSchema, readServerLock } from '@inkeep/open-knowledge-server';
-import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import { CallToolResultSchema } from '@modelcontextprotocol/sdk/types.js';
+import { afterAll, afterEach, test as baseTest, beforeAll, describe, expect, vi } from 'vitest';
import type { BootedServer } from '../../../server/src/boot.ts';
import { bootCompositionRig } from '../../../server/src/composition-rig.test-helper.ts';
import {
@@ -27,9 +30,12 @@ import {
resolveAuditScope,
} from '../../../server/src/lint/audit-scope.ts';
import { connectMcpTestClient } from '../../../server/src/mcp/client.test-helper.ts';
+import { watchCliOwner } from './cli-owner.test-helper.ts';
const CLI_PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
-const CLI_ENTRY = join(CLI_PACKAGE_ROOT, 'src/cli.ts');
+const CLI_ENTRY = join(CLI_PACKAGE_ROOT, 'dist/cli.mjs');
+const CLI_OWNER_PRELOAD = new URL('./cli-owner-preload.test-helper.mjs', import.meta.url).href;
+const CLI_COMPLETION_RESERVE_MS = 1_000;
const CONFIG = 'contentRules:\n markdownlint:\n enabled: true\nvalidation:\n links: warning\n';
const TABBED = '# Guide\n\nA\ttab.\n\nSee [[scope-cli-missing-target]].\n';
let root: string;
@@ -43,31 +49,62 @@ function seed(directory: string, path: string, body: string): void {
}
interface CliResult {
- code: number | null;
+ code: number;
stdout: string;
stderr: string;
}
-async function cli(cwd: string, args: string[]): Promise<CliResult> {
- const child = spawn(
- process.execPath,
- [
- '--import',
- 'tsx',
- '--conditions=development',
- CLI_ENTRY,
- '--cwd',
- cwd,
- '--log-level',
- 'silent',
- ...args,
- ],
- {
- cwd: CLI_PACKAGE_ROOT,
- env: { ...process.env, NO_COLOR: '1', OK_BUNDLE_PROXY: '0' },
- stdio: ['ignore', 'pipe', 'pipe'],
- },
- );
+interface CliChild {
+ stdout: Readable;
+ stderr: Readable;
+ on(event: 'error', listener: (error: Error) => void): unknown;
+ on(
+ event: 'close',
+ listener: (code: number | null, signal: NodeJS.Signals | null) => void,
+ ): unknown;
+ kill(signal: NodeJS.Signals): boolean;
+}
+
+type Cli = (cwd: string, args: string[], child?: CliChild) => Promise<CliResult>;
+
+const test = baseTest.extend<{ cli: Cli }>({
+ cli: async ({ task }, use) => {
+ const deadline = Date.now() + task.timeout - CLI_COMPLETION_RESERVE_MS;
+ await withCli(deadline, use);
+ },
+});
+
+async function withCli(deadline: number, use: (cli: Cli) => Promise<void>): Promise<void> {
+ const active = new Set<CliChild>();
+ try {
+ await use((cwd, args, child = spawnCli(cwd, args)) => {
+ active.add(child);
+ child.on('close', () => active.delete(child));
+ return runCli(deadline, cwd, args, child);
+ });
+ } finally {
+ for (const child of active) child.kill('SIGKILL');
+ }
+}
+
+function spawnCli(cwd: string, args: string[]) {
+ return spawnCliProcess(CLI_ENTRY, ['--cwd', cwd, '--log-level', 'silent', ...args]);
+}
+
+function spawnCliProcess(entry: string, args: string[]) {
+ return spawn(process.execPath, ['--import', CLI_OWNER_PRELOAD, entry, ...args], {
+ cwd: CLI_PACKAGE_ROOT,
+ env: { ...process.env, NO_COLOR: '1', OK_BUNDLE_PROXY: '0' },
+ stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+ }) as ChildProcessByStdio<null, Readable, Readable>;
+}
+
+async function runCli(
+ deadline: number,
+ cwd: string,
+ args: string[],
+ child: CliChild,
+): Promise<CliResult> {
let stdout = '';
let stderr = '';
child.stdout.setEncoding('utf8').on('data', (chunk: string) => {
@@ -76,14 +113,27 @@ async function cli(cwd: string, args: string[]): Promise<CliResult> {
child.stderr.setEncoding('utf8').on('data', (chunk: string) => {
stderr += chunk;
});
+ const command = args.join(' ');
+ const failure = (message: string, options?: ErrorOptions) =>
+ new Error(`${message}\ncwd: ${cwd}\nstdout: ${stdout}\nstderr: ${stderr}`, options);
return await new Promise((resolveResult, reject) => {
- const timer = setTimeout(() => child.kill('SIGKILL'), 25_000);
+ const timer = setTimeout(
+ () => {
+ reject(failure(`CLI child liveness bound expired: ${command}`));
+ child.kill('SIGKILL');
+ },
+ Math.max(0, deadline - Date.now()),
+ );
child.on('error', (error) => {
clearTimeout(timer);
- reject(error);
+ reject(failure(`CLI child ${command} errored: ${error.message}`, { cause: error }));
});
- child.on('close', (code) => {
+ child.on('close', (code, signal) => {
clearTimeout(timer);
+ if (code === null) {
+ reject(failure(`CLI child ${command} terminated by ${signal ?? 'unknown signal'}`));
+ return;
+ }
resolveResult({ code, stdout, stderr });
});
});
@@ -103,7 +153,12 @@ async function http(name: string, path: string): Promise<ValidationAuditResponse
return ValidationAuditResponseSchema.parse(await response.json());
}
-beforeAll(async () => {
+beforeAll(() => {
+ if (!existsSync(CLI_ENTRY)) {
+ throw new Error(
+ 'Build the CLI first: pnpm exec turbo run build --filter=@inkeep/open-knowledge',
+ );
+ }
root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-cli-scope-')));
headless = realpathSync(mkdtempSync(join(tmpdir(), 'ok-headless-scope-')));
for (const directory of [root, headless]) {
@@ -120,22 +175,163 @@ beforeAll(async () => {
mkdirSync(join(directory, 'empty'));
symlinkSync(join(directory, 'loop'), join(directory, 'loop'));
}
- server = await bootCompositionRig(root, { config: ConfigSchema.parse({}) });
- seed(root, '.ok/config.yml', CONFIG);
- await server.ready;
- client = await connectMcpTestClient(`http://127.0.0.1:${server.port}/mcp`);
- expect(readServerLock(server.lockDir)?.port).toBe(server.port);
-}, 60_000);
-
-afterAll(async () => {
- await client?.close();
- await server?.destroy();
+});
+
+afterAll(() => {
if (root) rmSync(root, { recursive: true, force: true });
if (headless) rmSync(headless, { recursive: true, force: true });
});
+class RecordingCliChild extends EventEmitter implements CliChild {
+ readonly stdout = new PassThrough();
+ readonly stderr = new PassThrough();
+ readonly signals: NodeJS.Signals[] = [];
+ closeOnSignal = true;
+
+ kill(signal: NodeJS.Signals): boolean {
+ this.signals.push(signal);
+ if (this.closeOnSignal) queueMicrotask(() => this.emit('close', null, signal));
+ return true;
+ }
+}
+
+describe('CLI child supervision', () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ test('lets a progressing CLI child finish within its test budget', async ({ cli, task }) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const child = new RecordingCliChild();
+ const completion = cli(root, ['audit', 'empty', '--json'], child).then(
+ (result) => ({ result }),
+ (error: unknown) => ({ error }),
+ );
+ try {
+ await vi.advanceTimersByTimeAsync(task.timeout * 0.9);
+ expect(child.signals).toEqual([]);
+ child.emit('close', 0, null);
+ expect(await completion).toEqual({ result: expect.objectContaining({ code: 0 }) });
+ } finally {
+ child.emit('close', 0, null);
+ await completion;
+ vi.useRealTimers();
+ }
+ });
+
+ test('reports the active CLI step before the enclosing test expires', async ({ cli, task }) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ let enclosingExpired = false;
+ setTimeout(() => {
+ enclosingExpired = true;
+ }, task.timeout);
+ const first = new RecordingCliChild();
+ const firstCompletion = cli(root, ['audit', 'empty', '--json'], first);
+ await vi.advanceTimersByTimeAsync(task.timeout / 2);
+ first.emit('close', 0, null);
+ await firstCompletion;
+ const child = new RecordingCliChild();
+ child.closeOnSignal = false;
+ let outcome: { result: CliResult } | { error: unknown } | undefined;
+ const completion = cli(root, ['audit', 'ignored', '--json'], child).then(
+ (result) => {
+ outcome = { result };
+ },
+ (error: unknown) => {
+ outcome = { error };
+ },
+ );
+ try {
+ await vi.advanceTimersToNextTimerAsync();
+ expect(enclosingExpired).toBe(false);
+ expect(outcome).toEqual({
+ error: expect.objectContaining({
+ message: expect.stringMatching(/CLI child liveness.*audit ignored/),
+ }),
+ });
+ expect(child.signals).toEqual(['SIGKILL']);
+ } finally {
+ child.emit('close', 0, null);
+ await completion;
+ vi.useRealTimers();
+ }
+ });
+
+ test.for([true, false])('stops a CLI after owner loss from connected=%s', (connected) => {
+ const owner = Object.assign(new EventEmitter(), { connected, channel: { unref() {} } });
+ const stops: string[] = [];
+ watchCliOwner(owner, () => stops.push('owner-lost'));
+ if (connected) {
+ expect(stops).toEqual([]);
+ owner.connected = false;
+ owner.emit('disconnect');
+ }
+ expect(stops).toEqual(['owner-lost']);
+ });
+
+ test('stops the CLI lifetime fixture after its owner exits', async ({ cli, task }) => {
+ const fixture = fileURLToPath(new URL('./cli-owner-exit.test-helper.mjs', import.meta.url));
+ const child = spawnCliProcess(fixture, [
+ String(Date.now() + task.timeout - CLI_COMPLETION_RESERVE_MS),
+ ]);
+ const result = await cli(root, ['owner-exit'], child);
+ expect(result.code, result.stderr).toBe(0);
+ expect(JSON.parse(result.stdout)).toEqual({ code: 1 });
+ expect(result.stderr).toContain('CLI child owner disconnected:');
+ });
+
+ test('cleans up a CLI child when its test scope ends', async ({ task }) => {
+ const child = new RecordingCliChild();
+ let completion: Promise<unknown> | undefined;
+ try {
+ await withCli(Date.now() + task.timeout, async (cli) => {
+ completion = cli(root, ['audit', 'empty', '--json'], child).catch(() => undefined);
+ });
+ expect(child.signals).toEqual(['SIGKILL']);
+ } finally {
+ child.emit('close', 0, null);
+ await completion;
+ }
+ });
+
+ test('reports a CLI child terminated by a signal', async ({ cli }) => {
+ const child = new RecordingCliChild();
+ const completion = cli(root, ['audit', 'empty', '--json'], child);
+ const observed = expect(completion).rejects.toThrow(/audit empty.*SIGABRT/);
+ child.emit('close', null, 'SIGABRT');
+ await observed;
+ });
+
+ test('reports a CLI child that fails to start', async ({ cli }) => {
+ const child = new RecordingCliChild();
+ const startFailure = new Error('spawn node ENOENT');
+ const completion = cli(root, ['audit', 'empty', '--json'], child);
+ const observed = expect(completion).rejects.toMatchObject({
+ message: expect.stringMatching(/audit empty.*spawn node ENOENT/),
+ cause: startFailure,
+ });
+ child.emit('error', startFailure);
+ await observed;
+ });
+});
+
describe('CLI scope parity across real process, HTTP and MCP boundaries', () => {
- test('headless lint exits with a not-found problem when content.dir is missing', async () => {
+ beforeAll(async () => {
+ server = await bootCompositionRig(root, { config: ConfigSchema.parse({}) });
+ seed(root, '.ok/config.yml', CONFIG);
+ await server.ready;
+ client = await connectMcpTestClient(`http://127.0.0.1:${server.port}/mcp`);
+ expect(readServerLock(server.lockDir)?.port).toBe(server.port);
+ }, 60_000);
+
+ afterAll(async () => {
+ await client?.close();
+ await server?.destroy();
+ });
+
+ test('headless lint exits with a not-found problem when content.dir is missing', async ({
+ cli,
+ }) => {
const project = realpathSync(mkdtempSync(join(tmpdir(), 'ok-missing-lint-content-')));
try {
seed(project, '.ok/config.yml', 'content:\n dir: missing-content\n');
@@ -173,7 +369,9 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
}
}, 60_000);
- test('audit exits with a not-found problem for a missing root through the HTTP boundary', async () => {
+ test('audit exits with a not-found problem for a missing root through the HTTP boundary', async ({
+ cli,
+ }) => {
const project = realpathSync(mkdtempSync(join(tmpdir(), 'ok-missing-audit-content-')));
const contentDir = join(project, 'missing-content');
const seenPaths: string[] = [];
@@ -218,9 +416,10 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
}
}, 60_000);
- test.each(['lint', 'audit'])(
+ test.for(['lint', 'audit'])(
'%s explicit and extensionless paths select the same validation plane',
- async (name) => {
+ { timeout: 120_000 },
+ async (name, { cli }) => {
await expect
.poll(
async () => {
@@ -247,7 +446,9 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
planes.push(plane);
const remote = await http(name, `guides/${path}`);
expect(plane).toEqual(remote);
- const mcp = await client.callTool({ name, arguments: { path: `guides/${path}` } });
+ const mcp = CallToolResultSchema.parse(
+ await client.callTool({ name, arguments: { path: `guides/${path}` } }),
+ );
expect(mcp.isError).toBeUndefined();
expect(
ValidationAuditResponseSchema.parse({ warnings: [], ...mcp.structuredContent }),
@@ -256,12 +457,12 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
expect(planes[0]).toEqual(planes[1]);
}
},
- 120_000,
);
- test.each(['lint', 'audit'])(
+ test.for(['lint', 'audit'])(
'%s preserves not-found problems and teaches the original path',
- async (name) => {
+ { timeout: 120_000 },
+ async (name, { cli }) => {
for (const path of ['unknown', 'guides/mdx-only.md', 'parent/child']) {
const result = await cli(root, [
name,
@@ -277,16 +478,16 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
});
}
const text = await cli(root, [name, 'unknown']);
- expect(text.code).toBe(1);
+ expect(text.code, text.stderr).toBe(1);
expect(text.stderr).toContain(auditScopeNotFoundTitle('unknown'));
expect(readFileSync(join(root, 'guides/mdx-only.mdx'), 'utf8')).toBe(TABBED);
},
- 120_000,
);
- test.each(['overlong segment', 'symlink loop'])(
+ test.for(['overlong segment', 'symlink loop'])(
'%s produces a teaching ProblemDetails envelope from live and headless CLI processes',
- async (kind) => {
+ { timeout: 120_000 },
+ async (kind, { cli }) => {
const path = kind === 'overlong segment' ? 'x'.repeat(300) : 'loop';
for (const [cwd, name] of [
[root, 'lint'],
@@ -304,12 +505,12 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
expect(result.stderr).not.toContain(' at ');
}
},
- 120_000,
);
- test.each(['lint', 'audit'])(
+ test.for(['lint', 'audit'])(
'%s distinguishes ignored and empty scopes from nonempty clean checks',
- async (name) => {
+ { timeout: 150_000 },
+ async (name, { cli }) => {
for (const path of ['empty', 'ignored']) {
const result = await cli(root, [name, path, '--json']);
expect(result.code, result.stderr).toBe(0);
@@ -329,12 +530,12 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
expect(clean.code, clean.stderr).toBe(0);
expect(clean.stdout).toContain('No problems in 1 file');
},
- 150_000,
);
- test.each(['lint', 'audit'])(
+ test.for(['lint', 'audit'])(
'%s checks explicitly requested ignored files across CLI, HTTP and MCP',
- async (name) => {
+ { timeout: 60_000 },
+ async (name, { cli }) => {
const path = 'ignored/guide.md';
const result = await cli(root, [name, path, '--json']);
expect(result.code, result.stderr).toBe(1);
@@ -346,16 +547,17 @@ describe('CLI scope parity across real process, HTTP and MCP boundaries', () =>
);
expect(plane.warnings).toEqual([]);
expect(plane).toEqual(await http(name, path));
- const mcp = await client.callTool({ name, arguments: { path } });
+ const mcp = CallToolResultSchema.parse(await client.callTool({ name, arguments: { path } }));
expect(mcp.isError).toBeUndefined();
expect(
ValidationAuditResponseSchema.parse({ warnings: [], ...mcp.structuredContent }),
).toEqual(plane);
},
- 60_000,
);
- test('headless lint retains hidden-file admission and resolves without a server', async () => {
+ test('headless lint retains hidden-file admission and resolves without a server', async ({
+ cli,
+ }) => {
const hidden = await cli(headless, ['lint', '.hidden', '--json', '--errors-only']);
expect(hidden.code, hidden.stderr).toBe(0);
expect(lintPlane(hidden.stdout)).toMatchObject({
diff --git a/packages/cli/src/commands/cli-owner-exit.test-helper.mjs b/packages/cli/src/commands/cli-owner-exit.test-helper.mjs
new file mode 100644
index 000000000..6aa84debe
--- /dev/null
+++ b/packages/cli/src/commands/cli-owner-exit.test-helper.mjs
@@ -0,0 +1,20 @@
+import { spawn } from 'node:child_process';
+
+const preload = new URL('./cli-owner-preload.test-helper.mjs', import.meta.url).href;
+const fixture = `
+ const { writeSync } = require('node:fs');
+ process.on('exit', (code) => writeSync(1, JSON.stringify({ code })));
+ process.channel.ref();
+ process.send('ready');
+`;
+setTimeout(
+ () => {
+ process.stderr.write('CLI owner-exit fixture liveness bound expired\n');
+ process.exit(2);
+ },
+ Math.max(0, Number(process.argv[2]) - Date.now()),
+);
+const child = spawn(process.execPath, ['--import', preload, '--eval', fixture], {
+ stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+});
+child.once('message', () => process.exit(0));
diff --git a/packages/cli/src/commands/cli-owner-preload.test-helper.mjs b/packages/cli/src/commands/cli-owner-preload.test-helper.mjs
new file mode 100644
index 000000000..fb707a97c
--- /dev/null
+++ b/packages/cli/src/commands/cli-owner-preload.test-helper.mjs
@@ -0,0 +1,6 @@
+import { watchCliOwner } from './cli-owner.test-helper.ts';
+
+watchCliOwner(process, () => {
+ process.stderr.write(`CLI child owner disconnected: ${process.argv.slice(2).join(' ')}\n`);
+ process.exit(1);
+});
diff --git a/packages/cli/src/commands/cli-owner.test-helper.ts b/packages/cli/src/commands/cli-owner.test-helper.ts
new file mode 100644
index 000000000..597949472
--- /dev/null
+++ b/packages/cli/src/commands/cli-owner.test-helper.ts
@@ -0,0 +1,14 @@
+interface CliOwner {
+ connected?: boolean;
+ channel?: { unref(): void };
+ once(event: 'disconnect', listener: () => void): unknown;
+}
+
+export function watchCliOwner(owner: CliOwner, stop: () => void): void {
+ if (!owner.connected) {
+ stop();
+ return;
+ }
+ owner.once('disconnect', stop);
+ owner.channel?.unref();
+}