Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Fix desktop watchdog worker teardown ordering (#7815)

Nick Gomezda51b1124afc
diff --git a/.changeset/watchdog-worker-teardown.md b/.changeset/watchdog-worker-teardown.md
new file mode 100644
index 000000000..4225ac1c5
--- /dev/null
+++ b/.changeset/watchdog-worker-teardown.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge-desktop': patch
+---
+
+Wait for the desktop watchdog worker to finish inspector teardown before starting the next session.
diff --git a/packages/desktop/src/main/main-thread-watchdog.test.ts b/packages/desktop/src/main/main-thread-watchdog.test.ts
index 48e3800bc..9f1c4e8a6 100644
--- a/packages/desktop/src/main/main-thread-watchdog.test.ts
+++ b/packages/desktop/src/main/main-thread-watchdog.test.ts
@@ -10,7 +10,9 @@ import {
classifyPreviousLiveness,
classifyPreviousStall,
computeWatchdogTick,
+ createLifecycleProbeForTests,
createMainThreadWatchdog,
+ LIFECYCLE_PROBE_SLOTS,
livenessLogFields,
type MainThreadWatchdog,
type MainThreadWatchdogHandle,
@@ -31,8 +33,8 @@ import {
const tmpDirs: string[] = [];
const handles: MainThreadWatchdogHandle[] = [];
-afterEach(() => {
- for (const handle of handles.splice(0)) handle.stop();
+afterEach(async () => {
+ await Promise.all(handles.splice(0).map((handle) => handle.stop()));
for (const dir of tmpDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
@@ -92,6 +94,17 @@ async function pollFor<T>(read: () => T | null, timeoutMs = 10_000): Promise<T |
return read();
}
+async function nextLifecycleMark(
+ probe: Int32Array,
+ slot: number,
+ after: number,
+ timeoutMs = 10_000,
+): Promise<number> {
+ const waited = Atomics.waitAsync(probe, slot, after, timeoutMs);
+ if (waited.async) await waited.value;
+ return Atomics.load(probe, slot);
+}
+
function capturedStall(watchdog: MainThreadWatchdog, after = ''): StallSnapshot | null {
const read = watchdog.readPreviousStall();
if (read.kind !== 'record' || read.value.outcome !== 'captured') return null;
@@ -752,6 +765,49 @@ describe('stallLogFields', () => {
});
describe('the worker witness', () => {
+ test('a stop that arrives while the stall session is connected waits for its disconnect, and the next start waits for the exit', async () => {
+ const dir = makeDir();
+ const lifecycleProbeForTests = createLifecycleProbeForTests();
+ const probe = new Int32Array(lifecycleProbeForTests);
+ const markOf = (slot: keyof typeof LIFECYCLE_PROBE_SLOTS): number =>
+ Atomics.load(probe, LIFECYCLE_PROBE_SLOTS[slot]);
+ const watchdog = createMainThreadWatchdog({
+ path: join(dir, 'watchdog.json'),
+ stallPath: join(dir, 'stall.json'),
+ logger: { warn: () => {} },
+ tickMs: 20,
+ lifecycleProbeForTests,
+ });
+ const handle = watchdog.start('boot-stop');
+ handles.push(handle);
+ const firstSpawn = await nextLifecycleMark(probe, LIFECYCLE_PROBE_SLOTS.workerSpawned, 0);
+ expect(firstSpawn).toBeGreaterThan(0);
+
+ expect(Atomics.wait(probe, LIFECYCLE_PROBE_SLOTS.sessionConnected, 0, 10_000)).not.toBe(
+ 'timed-out',
+ );
+ const connected = markOf('sessionConnected');
+ expect(connected).toBeGreaterThan(firstSpawn);
+ expect(markOf('sessionDisconnected')).toBe(0);
+
+ const stopping = handle.stop();
+ const next = watchdog.start('boot-after-stop');
+ handles.push(next);
+ await stopping;
+
+ const stopPosted = markOf('stopPosted');
+ const disconnected = markOf('sessionDisconnected');
+ const stopCompleted = markOf('stopCompleted');
+ const exited = markOf('workerExited');
+ expect(stopPosted).toBeGreaterThan(connected);
+ expect(disconnected).toBeGreaterThan(stopPosted);
+ expect(stopCompleted).toBeGreaterThan(disconnected);
+ expect(exited).toBeGreaterThan(stopCompleted);
+ expect(
+ await nextLifecycleMark(probe, LIFECYCLE_PROBE_SLOTS.workerSpawned, firstSpawn),
+ ).toBeGreaterThan(exited);
+ });
+
test('it records the running session and the pings it heard from main', async () => {
const path = join(makeDir(), 'nested', 'watchdog.json');
const warnings: Record<string, unknown>[] = [];
diff --git a/packages/desktop/src/main/main-thread-watchdog.ts b/packages/desktop/src/main/main-thread-watchdog.ts
index 22183e171..253b46010 100644
--- a/packages/desktop/src/main/main-thread-watchdog.ts
+++ b/packages/desktop/src/main/main-thread-watchdog.ts
@@ -113,7 +113,7 @@ export interface StallLogFields {
}
export interface MainThreadWatchdogHandle {
- stop(): void;
+ stop(): void | Promise<void>;
}
export interface MainThreadWatchdog {
@@ -183,17 +183,49 @@ export function boundStallFrames(
return { frames, framesTruncated: all.length > maxFrames };
}
-/* STOP: computeWatchdogTick and boundStallFrames are serialized into WORKER_SOURCE through
- toString(), so they must stay closure-free and import-free. WORKER_SOURCE is an eval string
- rather than a file-backed worker entry because electron-builder.yml unpacks node-pty's worker
- on the recorded ground that worker_threads cannot load JS from inside app.asar; a file-backed
- entry has to prove it loads from a packaged build before it replaces this. */
+export const LIFECYCLE_PROBE_SLOTS = {
+ sequence: 0,
+ workerSpawned: 1,
+ sessionConnected: 2,
+ stopPosted: 3,
+ sessionDisconnected: 4,
+ stopCompleted: 5,
+ workerExited: 6,
+} as const;
+
+type LifecycleProbeMark = Exclude<keyof typeof LIFECYCLE_PROBE_SLOTS, 'sequence'>;
+
+export function createLifecycleProbeForTests(): SharedArrayBuffer {
+ return new SharedArrayBuffer(
+ Int32Array.BYTES_PER_ELEMENT * Object.keys(LIFECYCLE_PROBE_SLOTS).length,
+ );
+}
+
+function markLifecycleProbe(probe: Int32Array, sequenceSlot: number, slot: number): void {
+ Atomics.store(probe, slot, Atomics.add(probe, sequenceSlot, 1) + 1);
+ Atomics.notify(probe, slot);
+}
+
+/* STOP: computeWatchdogTick, boundStallFrames and markLifecycleProbe are serialized into
+ WORKER_SOURCE through toString(), so they must stay closure-free and import-free. WORKER_SOURCE
+ is an eval string rather than a file-backed worker entry because electron-builder.yml unpacks
+ node-pty's worker on the recorded ground that worker_threads cannot load JS from inside
+ app.asar; a file-backed entry has to prove it loads from a packaged build before it replaces
+ this. */
const WORKER_SOURCE = `
const { mkdirSync, renameSync, unlinkSync, writeFileSync } = require('node:fs');
const { dirname } = require('node:path');
const { parentPort, workerData } = require('node:worker_threads');
-const { path, stallPath, bootId, tickMs } = workerData;
+const { path, stallPath, bootId, tickMs, lifecycleProbe } = workerData;
+const probe = lifecycleProbe ? new Int32Array(lifecycleProbe) : null;
+const markProbe = ${markLifecycleProbe.toString()};
+const markLifecycle = (slot) => {
+ if (probe !== null) markProbe(probe, ${LIFECYCLE_PROBE_SLOTS.sequence}, slot);
+};
+const holdSessionUntilStopPosted = () => {
+ if (probe !== null) Atomics.wait(probe, ${LIFECYCLE_PROBE_SLOTS.stopPosted}, 0);
+};
const stallThresholdMs = tickMs * ${STALL_THRESHOLD_TICKS};
const computeTick = ${computeWatchdogTick.toString()};
const boundFrames = ${boundStallFrames.toString()};
@@ -204,8 +236,69 @@ const state = {
};
let reportedFailure = false;
let stallEpisodeCaptured = false;
+let activeSession = null;
+let sessionDisconnecting = false;
+let disconnectWaiters = [];
+let stopRequested = false;
+let tickTimer = null;
+
+function disconnectSession(done) {
+ if (sessionDisconnecting) {
+ disconnectWaiters.push(done);
+ return;
+ }
+ const current = activeSession;
+ if (current === null) {
+ done();
+ return;
+ }
+ activeSession = null;
+ sessionDisconnecting = true;
+ disconnectWaiters = [done];
+ let finished = false;
+ const finish = () => {
+ if (finished) return;
+ finished = true;
+ sessionDisconnecting = false;
+ markLifecycle(${LIFECYCLE_PROBE_SLOTS.sessionDisconnected});
+ const waiters = disconnectWaiters;
+ disconnectWaiters = [];
+ for (const waiter of waiters) waiter();
+ };
+ try {
+ current.post('Debugger.disable', () => {
+ try {
+ current.disconnect();
+ } catch {}
+ finish();
+ });
+ } catch {
+ try {
+ current.disconnect();
+ } catch {}
+ finish();
+ }
+}
-parentPort.on('message', () => {
+function finishStop() {
+ if (!stopRequested || activeSession !== null || sessionDisconnecting) return;
+ markLifecycle(${LIFECYCLE_PROBE_SLOTS.stopCompleted});
+ parentPort.close();
+}
+
+function stopWorker() {
+ if (stopRequested) return;
+ stopRequested = true;
+ if (tickTimer !== null) clearInterval(tickTimer);
+ disconnectSession(finishStop);
+}
+
+parentPort.on('message', (message) => {
+ if (message === 'stop') {
+ stopWorker();
+ return;
+ }
+ if (stopRequested) return;
state.lastMainAt = performance.now();
state.mainTicksObserved += 1;
stallEpisodeCaptured = false;
@@ -227,6 +320,7 @@ function writeJsonAtomically(target, value) {
}
function writeWitness() {
+ if (stopRequested) return;
const record = computeTick(
state,
performance.now(),
@@ -248,6 +342,7 @@ function writeWitness() {
}
function requestStallSnapshot(record) {
+ if (sessionDisconnecting || activeSession !== null) return;
const request = {
schemaVersion: 1,
bootId,
@@ -266,14 +361,12 @@ function requestStallSnapshot(record) {
settled = true;
const snapshot = Object.assign({}, request, fields);
const writeError = writeJsonAtomically(stallPath, snapshot);
+ if (stopRequested) {
+ disconnectSession(finishStop);
+ return;
+ }
parentPort.postMessage({ type: 'stall-snapshot', snapshot, writeError });
- if (session === null) return;
- try {
- session.post('Debugger.disable');
- } catch {}
- try {
- session.disconnect();
- } catch {}
+ disconnectSession(finishStop);
};
const fail = (err) => {
settle({ outcome: 'failed', error: describeError(err) });
@@ -281,7 +374,10 @@ function requestStallSnapshot(record) {
try {
const inspector = require('node:inspector');
session = new inspector.Session();
+ activeSession = session;
session.connectToMainThread();
+ markLifecycle(${LIFECYCLE_PROBE_SLOTS.sessionConnected});
+ holdSessionUntilStopPosted();
const urlByScriptId = new Map();
session.on('Debugger.scriptParsed', (message) => {
const params = message && message.params;
@@ -329,7 +425,7 @@ function requestStallSnapshot(record) {
}
writeWitness();
-setInterval(writeWitness, tickMs);
+tickTimer = setInterval(writeWitness, tickMs);
function errorCode(err) {
return err && typeof err.code === 'string' ? err.code : 'unknown';
@@ -619,8 +715,16 @@ export function createMainThreadWatchdog(opts: {
stallPath: string;
logger: WatchdogLogger;
tickMs?: number;
+ lifecycleProbeForTests?: SharedArrayBuffer;
}): MainThreadWatchdog {
const tickMs = opts.tickMs ?? WATCHDOG_TICK_MS;
+ let lastStop = Promise.resolve();
+ const lifecycleProbe =
+ opts.lifecycleProbeForTests === undefined ? null : new Int32Array(opts.lifecycleProbeForTests);
+ const markLifecycle = (mark: LifecycleProbeMark): void => {
+ if (lifecycleProbe === null) return;
+ markLifecycleProbe(lifecycleProbe, LIFECYCLE_PROBE_SLOTS.sequence, LIFECYCLE_PROBE_SLOTS[mark]);
+ };
return {
readPrevious(): WatchdogRead {
@@ -645,75 +749,99 @@ export function createMainThreadWatchdog(opts: {
},
start(bootId: string): MainThreadWatchdogHandle {
- let worker: Worker;
- try {
- worker = new Worker(WORKER_SOURCE, {
- eval: true,
- workerData: { path: opts.path, stallPath: opts.stallPath, bootId, tickMs },
+ let stopped = false;
+ let stopPromise: Promise<void> | null = null;
+ const workerPromise = lastStop.then(() => {
+ let worker: Worker;
+ try {
+ worker = new Worker(WORKER_SOURCE, {
+ eval: true,
+ workerData: {
+ path: opts.path,
+ stallPath: opts.stallPath,
+ bootId,
+ tickMs,
+ lifecycleProbe: opts.lifecycleProbeForTests,
+ },
+ });
+ } catch (err) {
+ opts.logger.warn(
+ { event: 'main-thread-watchdog.spawn-failed', ...watchdogFailureFields(err) },
+ 'could not start the main-thread watchdog — a hang will not be told apart from a death',
+ );
+ return null;
+ }
+ markLifecycle('workerSpawned');
+ worker.unref();
+ const ping = setInterval(() => {
+ worker.postMessage(1);
+ }, tickMs);
+ ping.unref();
+
+ let warnedWorkerFailed = false;
+ const warnWorkerFailed = (cause: Error | number): void => {
+ if (stopped || warnedWorkerFailed) return;
+ warnedWorkerFailed = true;
+ opts.logger.warn(
+ { event: 'main-thread-watchdog.worker-failed', ...watchdogFailureFields(cause) },
+ 'the main-thread watchdog stopped witnessing this session',
+ );
+ };
+ worker.on('error', (err) => {
+ warnWorkerFailed(err);
+ });
+ const exited = new Promise<void>((resolve) => {
+ worker.on('exit', (code) => {
+ clearInterval(ping);
+ markLifecycle('workerExited');
+ if (code !== 0) warnWorkerFailed(code);
+ resolve();
+ });
});
- } catch (err) {
- opts.logger.warn(
- { event: 'main-thread-watchdog.spawn-failed', ...watchdogFailureFields(err) },
- 'could not start the main-thread watchdog — a hang will not be told apart from a death',
- );
- return { stop() {} };
- }
- worker.unref();
- const ping = setInterval(() => {
- worker.postMessage(1);
- }, tickMs);
- ping.unref();
- let stopped = false;
- let warnedWorkerFailed = false;
- const warnWorkerFailed = (cause: Error | number): void => {
- if (stopped || warnedWorkerFailed) return;
- warnedWorkerFailed = true;
- opts.logger.warn(
- { event: 'main-thread-watchdog.worker-failed', ...watchdogFailureFields(cause) },
- 'the main-thread watchdog stopped witnessing this session',
- );
- };
- worker.on('error', (err) => {
- warnWorkerFailed(err);
- });
- worker.on('exit', (code) => {
- clearInterval(ping);
- if (code !== 0) warnWorkerFailed(code);
- });
+ let warnedWriteFailed = false;
+ worker.on('message', (message: unknown) => {
+ if (typeof message !== 'object' || message === null) return;
+ const { type, code, snapshot, writeError } = message as {
+ type?: unknown;
+ code?: unknown;
+ snapshot?: unknown;
+ writeError?: unknown;
+ };
+ if (type === 'stall-snapshot') {
+ logStallSnapshot(opts.logger, snapshot, writeError);
+ return;
+ }
+ if (type !== 'write-failed' || warnedWriteFailed) return;
+ warnedWriteFailed = true;
+ opts.logger.warn(
+ {
+ event: 'main-thread-watchdog.write-failed',
+ code: typeof code === 'string' ? code : 'unknown',
+ },
+ 'the main-thread watchdog could not write its witness file',
+ );
+ });
- let warnedWriteFailed = false;
- worker.on('message', (message: unknown) => {
- if (typeof message !== 'object' || message === null) return;
- const { type, code, snapshot, writeError } = message as {
- type?: unknown;
- code?: unknown;
- snapshot?: unknown;
- writeError?: unknown;
- };
- if (type === 'stall-snapshot') {
- logStallSnapshot(opts.logger, snapshot, writeError);
- return;
- }
- if (type !== 'write-failed' || warnedWriteFailed) return;
- warnedWriteFailed = true;
- opts.logger.warn(
- {
- event: 'main-thread-watchdog.write-failed',
- code: typeof code === 'string' ? code : 'unknown',
+ return {
+ stop(): Promise<void> {
+ clearInterval(ping);
+ worker.postMessage('stop');
+ markLifecycle('stopPosted');
+ return exited;
},
- 'the main-thread watchdog could not write its witness file',
- );
+ };
});
- return {
- stop() {
- if (stopped) return;
- stopped = true;
- clearInterval(ping);
- void worker.terminate();
- },
+ const stop = (): Promise<void> => {
+ if (stopPromise !== null) return stopPromise;
+ stopped = true;
+ stopPromise = workerPromise.then((state) => state?.stop()).then(() => {});
+ lastStop = stopPromise;
+ return stopPromise;
};
+
+ return { stop };
},
};
}