Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Record worker server owner loss diagnostics (#7569)

Nick Gomez73fb00d93e48
diff --git a/packages/app/tests/foundation/worker-server-lifetime-diagnostics.node.test.ts b/packages/app/tests/foundation/worker-server-lifetime-diagnostics.node.test.ts
new file mode 100644
index 000000000..6d161e1fe
--- /dev/null
+++ b/packages/app/tests/foundation/worker-server-lifetime-diagnostics.node.test.ts
@@ -0,0 +1,287 @@
+import { type ChildProcess, spawn, spawnSync } from 'node:child_process';
+import {
+ closeSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ openSync,
+ readFileSync,
+ rmSync,
+ writeSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setTimeout as wait } from 'node:timers/promises';
+import { expect, onTestFinished, test } from 'vitest';
+import { holdServerLogTail, SERVER_LOG_TAIL_LINES } from '../stress/_helpers/fixtures.ts';
+import { productionShellLockPath } from '../stress/_helpers/production-shell-readiness.ts';
+import {
+ PRODUCTION_SHELL_ENV,
+ type ProductionShellLaunch,
+} from '../stress/_helpers/production-shell-server.ts';
+import { APP_PACKAGE_ROOT } from '../stress/_helpers/seed-key.ts';
+import {
+ closeServerLog,
+ createViteStartupRequest,
+ getFreePort,
+ openServerLog,
+} from '../stress/_helpers/server-process.ts';
+import { runTeardownPhases } from '../stress/_helpers/teardown-fs.ts';
+import type { WorkerServerLaunch } from '../stress/_helpers/worker-server-command.ts';
+
+const TEST_TIMEOUT_MS = 30_000;
+const REAP_MS = 5_000;
+const POLL_MS = 25;
+const CLI_SOURCE_ENTRY = join(APP_PACKAGE_ROOT, '..', 'cli', 'src', 'cli.ts');
+const LIFETIME = join(
+ APP_PACKAGE_ROOT,
+ 'tests',
+ 'stress',
+ '_helpers',
+ 'worker-server-lifetime.cjs',
+);
+const OWNER_LOSS_RECORD =
+ /\[worker-server-lifetime\] owner-loss trigger=(\S+) ownerPid=(\d+) ownerAlive=(true|false|null) serverPid=(\d+|null) elapsedMs=(\d+) readiness=(\S+)\n/;
+
+type Exit = { code: number | null; signal: NodeJS.Signals | null };
+type Lock = Record<string, unknown>;
+
+interface ProductionShellLifetime {
+ readonly child: ChildProcess;
+ readonly boundLock: Lock;
+ readonly port: number;
+ readonly messages: readonly unknown[];
+ readonly serverLog: () => string;
+ readonly serverLogPath: string;
+ readonly setupFailureReport: () => string;
+ readonly exit: () => Exit | undefined;
+ readonly until: <T>(what: string, attempt: () => T | undefined) => Promise<T>;
+}
+
+function openLogReader(path: string): { read: () => string; close: () => void } {
+ const fd = openSync(path, 'r');
+ const chunks: Buffer[] = [];
+ return {
+ read: () => {
+ chunks.push(readFileSync(fd));
+ return Buffer.concat(chunks).toString('utf8');
+ },
+ close: () => closeSync(fd),
+ };
+}
+
+function readLock(lockPath: string): Lock | undefined {
+ let text: string;
+ try {
+ text = readFileSync(lockPath, 'utf8');
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
+ throw error;
+ }
+ let value: unknown;
+ try {
+ value = JSON.parse(text);
+ } catch (error) {
+ if (error instanceof SyntaxError) return undefined;
+ throw error;
+ }
+ return typeof value === 'object' && value !== null && !Array.isArray(value)
+ ? (value as Lock)
+ : undefined;
+}
+
+function sourceShellLaunch(host: string, port: number): ProductionShellLaunch {
+ return {
+ mode: 'production',
+ command: process.execPath,
+ args: [
+ CLI_SOURCE_ENTRY,
+ 'start',
+ '--port',
+ String(port),
+ '--bind',
+ host,
+ '--no-open-browser',
+ '--mode',
+ 'browser',
+ '--react-shell-dist-dir',
+ join(APP_PACKAGE_ROOT, 'dist'),
+ '--idle-shutdown',
+ 'off',
+ ],
+ };
+}
+
+async function acquireProductionShellLifetime(): Promise<ProductionShellLifetime> {
+ const deadlineAt = Date.now() + TEST_TIMEOUT_MS - REAP_MS;
+ const teardown: Array<() => void | Promise<void>> = [];
+ onTestFinished(() => runTeardownPhases(...teardown.reverse()));
+ const root = mkdtempSync(join(tmpdir(), 'ok-lifetime-diagnostics-'));
+ teardown.push(() => rmSync(root, { recursive: true, force: true }));
+ const request = createViteStartupRequest('127.0.0.1');
+ teardown.push(() => request.dispose());
+ const serverLog = openServerLog('lifetime-diagnostics');
+ teardown.push(() => closeServerLog(serverLog));
+ const log = openLogReader(serverLog.path);
+ teardown.push(log.close);
+ const reportTail = holdServerLogTail(serverLog);
+ teardown.push(reportTail.close);
+ const contentDir = join(root, 'content');
+ const viteCacheDir = join(root, 'vite-cache');
+ mkdirSync(contentDir);
+ mkdirSync(viteCacheDir);
+ const messages: unknown[] = [];
+ let stderr = '';
+ let exit: Exit | undefined;
+ const context = () =>
+ `--- worker server lifetime stderr ---\n${stderr}\n--- server log ---\n${log.read()}`;
+ const until = async <T>(what: string, attempt: () => T | undefined): Promise<T> => {
+ const startedAt = Date.now();
+ while (true) {
+ const value = attempt();
+ if (value !== undefined) return value;
+ const remainingMs = deadlineAt - Date.now();
+ if (remainingMs <= 0) {
+ throw new Error(
+ `${what} within the ${Date.now() - startedAt}ms left of this test's budget\n${context()}`,
+ );
+ }
+ await wait(Math.min(POLL_MS, remainingMs));
+ }
+ };
+
+ const initialized = spawnSync(
+ process.execPath,
+ [CLI_SOURCE_ENTRY, 'init', '--no-mcp', '--no-skills', '--content-dir', '.', '--json'],
+ { cwd: contentDir, encoding: 'utf8', timeout: deadlineAt - Date.now() },
+ );
+ if (initialized.status !== 0) {
+ throw new Error(
+ `ok init from source failed (status=${initialized.status}, signal=${initialized.signal}, error=${initialized.error?.message ?? 'none'})\n${initialized.stderr}`,
+ );
+ }
+ const port = await getFreePort(request.host);
+ const launch: WorkerServerLaunch = {
+ request: { ...request, candidatePort: port },
+ workerServerEnv: { [PRODUCTION_SHELL_ENV]: '1' },
+ contentDir,
+ viteCacheDir,
+ serverLog: { path: serverLog.path, fd: 1 },
+ serving: { kind: 'production-shell', launch: sourceShellLaunch(request.host, port) },
+ command: null,
+ reapMs: REAP_MS,
+ };
+ const child = spawn(process.execPath, [LIFETIME, JSON.stringify(launch)], {
+ cwd: APP_PACKAGE_ROOT,
+ stdio: ['ignore', serverLog.fd, 'pipe', 'ipc'],
+ });
+ const exited = new Promise<Exit>((resolve) =>
+ child.once('exit', (code, signal) => resolve({ code, signal })),
+ );
+ const stderrStream = child.stderr;
+ if (stderrStream === null) throw new Error('the worker server lifetime has no stderr pipe');
+ const stderrClosed = new Promise<void>((resolve) => stderrStream.once('close', resolve));
+ const closed = Promise.all([exited, stderrClosed]).then(([status]) => {
+ exit = status;
+ });
+ teardown.push(async () => {
+ if (child.connected) child.disconnect();
+ await closed;
+ if (stderr.length > 0) process.stderr.write(stderr);
+ });
+ stderrStream.on('data', (chunk: Buffer) => {
+ stderr += chunk.toString('utf8');
+ });
+ child.on('message', (message: unknown) => messages.push(message));
+
+ const lockPath = productionShellLockPath(contentDir);
+ const boundLock = await until('the production-shell server did not bind its lock', () => {
+ const lock = readLock(lockPath);
+ return lock !== undefined && typeof lock.port === 'number' && lock.port > 0 ? lock : undefined;
+ });
+ return {
+ child,
+ boundLock,
+ port,
+ messages,
+ serverLog: log.read,
+ serverLogPath: serverLog.path,
+ setupFailureReport: reportTail.report,
+ exit: () => exit,
+ until,
+ };
+}
+
+test(
+ 'records the owner-loss trigger before stopping a production-shell server',
+ async () => {
+ const lifetime = await acquireProductionShellLifetime();
+ expect(lifetime.boundLock.pid).toEqual(expect.any(Number));
+ lifetime.child.disconnect();
+ const record = await lifetime.until(
+ 'the lifetime did not record the loss of its owner',
+ () => OWNER_LOSS_RECORD.exec(lifetime.serverLog()) ?? undefined,
+ );
+ expect(record.slice(1)).toEqual([
+ 'ipc-disconnect',
+ String(process.pid),
+ 'true',
+ String(lifetime.boundLock.pid),
+ expect.any(String),
+ 'lock-bound',
+ ]);
+ },
+ TEST_TIMEOUT_MS,
+);
+
+test(
+ 'leaves no owner-loss record when its owner requests the release of a production-shell server',
+ async () => {
+ const lifetime = await acquireProductionShellLifetime();
+ await new Promise<void>((resolve, reject) =>
+ lifetime.child.send({ type: 'release' }, (error) =>
+ error === null ? resolve() : reject(error),
+ ),
+ );
+ const exit = await lifetime.until(
+ 'the lifetime did not close after its release',
+ lifetime.exit,
+ );
+ expect(lifetime.messages).toContainEqual({ type: 'released', error: null });
+ expect(exit).toEqual({ code: 0, signal: null });
+ const log = lifetime.serverLog();
+ expect(log).toContain(`http://127.0.0.1:${lifetime.port}`);
+ expect(log).not.toContain('[worker-server-lifetime] owner-loss ');
+ },
+ TEST_TIMEOUT_MS,
+);
+
+test(
+ 'keeps the owner-loss record in the setup failure report after the lifetime removes the server log',
+ async () => {
+ const lifetime = await acquireProductionShellLifetime();
+ lifetime.child.disconnect();
+ await lifetime.until('the lifetime did not close after losing its owner', lifetime.exit);
+ expect(existsSync(lifetime.serverLogPath)).toBe(false);
+ expect(OWNER_LOSS_RECORD.exec(lifetime.setupFailureReport())?.[1]).toBe('ipc-disconnect');
+ },
+ TEST_TIMEOUT_MS,
+);
+
+test('keeps the owner-loss record in the setup failure report when later server output fills the tail', () => {
+ const serverLog = openServerLog('lifetime-diagnostics-tail');
+ const reportTail = holdServerLogTail(serverLog);
+ onTestFinished(() =>
+ runTeardownPhases(
+ reportTail.close,
+ () => closeServerLog(serverLog),
+ () => rmSync(serverLog.path, { force: true }),
+ ),
+ );
+ const record =
+ '[worker-server-lifetime] owner-loss trigger=signal-SIGTERM ownerPid=2 ownerAlive=true serverPid=3 elapsedMs=4 readiness=lock-draining\n';
+ writeSync(serverLog.fd, `${record}${'server shutdown output\n'.repeat(SERVER_LOG_TAIL_LINES)}`);
+ const report = reportTail.report();
+ expect(report).toContain('server shutdown output');
+ expect(report).toContain(record);
+});
diff --git a/packages/app/tests/foundation/worker-server-lifetime-diagnostics.reads.ts b/packages/app/tests/foundation/worker-server-lifetime-diagnostics.reads.ts
new file mode 100644
index 000000000..7e67b98d0
--- /dev/null
+++ b/packages/app/tests/foundation/worker-server-lifetime-diagnostics.reads.ts
@@ -0,0 +1,10 @@
+export const ROUND_READS = {
+ test: './worker-server-lifetime-diagnostics.node.test.ts',
+ reads:
+ "the lifetime owner and the CLI source entry the test starts, with everything they import, then what those processes read at run time: the server package manifest the server source reads for its version, and the worker fixtures module the test process reads when it imports the setup failure report's log tail",
+ roots: [
+ 'packages/app/tests/stress/_helpers/worker-server-lifetime.cjs',
+ 'packages/cli/src/cli.ts',
+ ],
+ include: ['packages/app/tests/stress/_helpers/fixtures.ts', 'packages/server/package.json'],
+};
diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.ts
index 9af2a2370..e72a07754 100644
--- a/packages/app/tests/stress/_helpers/fixtures.ts
+++ b/packages/app/tests/stress/_helpers/fixtures.ts
@@ -1,4 +1,4 @@
-import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import { closeSync, mkdirSync, mkdtempSync, openSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { ProblemType } from '@inkeep/open-knowledge-core';
@@ -27,7 +27,6 @@ import {
prepareViteCacheDir,
requireBoundMs,
type ServerLog,
- tailServerLog,
type ViteStartupRequest,
waitForBoundViteEndpoint,
waitForHttpReady,
@@ -483,6 +482,31 @@ async function waitForSeededPagesSettled(baseURL: string, seededNames: string[])
);
}
+export const SERVER_LOG_TAIL_LINES = 40;
+const WORKER_SERVER_LIFETIME_RECORD = '[worker-server-lifetime] ';
+
+export interface ServerLogTail {
+ readonly report: () => string;
+ readonly close: () => void;
+}
+
+export function holdServerLogTail(serverLog: ServerLog): ServerLogTail {
+ const fd = openSync(serverLog.path, 'r');
+ const chunks: Buffer[] = [];
+ return {
+ report: () => {
+ chunks.push(readFileSync(fd));
+ const lines = Buffer.concat(chunks).toString('utf-8').split('\n');
+ const tail = `--- dev server log tail (${serverLog.path}) ---\n${lines.slice(-SERVER_LOG_TAIL_LINES).join('\n')}`;
+ const records = lines.filter((line) => line.startsWith(WORKER_SERVER_LIFETIME_RECORD));
+ return records.length === 0
+ ? tail
+ : `--- worker server lifetime records ---\n${records.join('\n')}\n${tail}`;
+ },
+ close: () => closeSync(fd),
+ };
+}
+
export function seedRequiredFixtureFiles(contentDir: string): void {
writeFileSync(join(contentDir, 'test-doc.md'), '', 'utf-8');
mkdirSync(join(contentDir, 'sidebar-folder'), { recursive: true });
@@ -552,7 +576,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
WORKER_SERVER_BUDGET_RESERVES.setupOverhead,
);
const releaseSetupResources: Array<() => void | Promise<void>> = [];
- let openedServerLog: ServerLog | undefined;
+ let serverLogTail: ServerLogTail | undefined;
let started: StartedWorkerServer | undefined;
let ownedProc: WorkerServerProcess | undefined;
let endpoint: BoundViteEndpoint;
@@ -573,8 +597,10 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
if (serving.kind === 'production-shell') initProductionShellContent(contentDir);
const serverLog = openServerLog(`w${workerInfo.workerIndex}`);
- openedServerLog = serverLog;
releaseSetupResources.push(() => closeServerLog(serverLog));
+ const heldTail = holdServerLogTail(serverLog);
+ serverLogTail = heldTail;
+ releaseSetupResources.push(heldTail.close);
const proc = spawnWorkerDevServer(
request,
@@ -617,6 +643,10 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
refuseStarvedBudgetSlot(setupOverhead, Date.now() - fixtureStartedAt, residue);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
+ const tail =
+ serverLogTail === undefined
+ ? '--- no dev server log was open when this failure ended the setup ---'
+ : serverLogTail.report();
const borrowed = budgetPhaseOverrunMessage(
setupOverhead,
'this failure reaps the dev server and removes the dirs the setup had created',
@@ -627,10 +657,6 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
} catch (cleanupErr) {
drainFailure = `--- cleanup after this failure did not complete: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)} ---`;
}
- const tail =
- openedServerLog === undefined
- ? '--- no dev server log was open when this failure ended the setup ---'
- : `--- dev server log tail (${openedServerLog.path}) ---\n${tailServerLog(openedServerLog)}`;
try {
if (ownedProc !== undefined) await completeWorkerServerRelease(ownedProc);
} catch (cleanupErr) {
@@ -646,6 +672,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
);
}
+ serverLogTail?.close();
const { contentDir, viteCacheDir, serverLog, proc, pending, serving } = started;
await use({
diff --git a/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs b/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs
index 8228a4a9a..49753eb55 100644
--- a/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs
+++ b/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs
@@ -1,3 +1,5 @@
+const { readFileSync } = require('node:fs');
+const { productionShellLockPath } = require('./production-shell-readiness.ts');
const { removeAllDuringTeardown, runTeardownPhases } = require('./teardown-fs.ts');
const { killGracefully } = require('./server-process.ts');
const { spawnWorkerServerCommand } = require('./worker-server-command.ts');
@@ -8,6 +10,43 @@ let releasing;
let finishing;
let ownerLost = !process.connected;
let terminal;
+const lifetimeStartedAt = Date.now();
+const ownerPid = process.ppid;
+let commandSpawnedAt;
+let ownerLossRecorded = false;
+
+function ownerAlive() {
+ const pid = ownerPid;
+ if (!Number.isSafeInteger(pid) || pid <= 1) return null;
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (error) {
+ return error?.code === 'EPERM';
+ }
+}
+
+function readinessState() {
+ try {
+ const lock = JSON.parse(readFileSync(productionShellLockPath(launch.contentDir), 'utf8'));
+ if (lock.draining === true) return 'lock-draining';
+ if (typeof lock.port === 'number' && lock.port > 0) return 'lock-bound';
+ return 'lock-pending';
+ } catch (error) {
+ return error?.code === 'ENOENT' ? 'lock-missing' : 'lock-unreadable';
+ }
+}
+
+function recordOwnerLoss(trigger) {
+ if (ownerLossRecorded || finishing !== undefined) return;
+ ownerLossRecorded = true;
+ const elapsedMs = Date.now() - (commandSpawnedAt ?? lifetimeStartedAt);
+ const serverPid = proc?.pid ?? null;
+ const line = `[worker-server-lifetime] owner-loss trigger=${trigger} ownerPid=${ownerPid} ownerAlive=${String(ownerAlive())} serverPid=${String(serverPid)} elapsedMs=${elapsedMs} readiness=${readinessState()}\n`;
+ try {
+ process.stdout.write(line);
+ } catch {}
+}
function send(message) {
if (process.connected) process.send(message, () => {});
@@ -75,6 +114,7 @@ function finish() {
process.once('disconnect', () => {
ownerLost = true;
+ recordOwnerLoss('ipc-disconnect');
void finish();
});
process.on('message', (message) => {
@@ -88,14 +128,18 @@ process.on('message', (message) => {
for (const signal of ['SIGTERM', 'SIGINT']) {
process.once(signal, () => {
ownerLost = true;
+ recordOwnerLoss(`signal-${signal}`);
void finish();
});
}
-if (ownerLost) void finish();
-else {
+if (ownerLost) {
+ recordOwnerLoss('owner-not-connected-at-start');
+ void finish();
+} else {
try {
proc = spawnWorkerServerCommand(launch);
+ commandSpawnedAt = Date.now();
proc.once('error', (error) => {
terminal = commandStatus();
send({ type: 'command-error', status: terminal, error: commandError(error) });