Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Release watcher handles acquired during server shutdown (#7530)

Nick Gomez5052ed6b2c41
diff --git a/.changeset/watcher-retirement-adoption.md b/.changeset/watcher-retirement-adoption.md
new file mode 100644
index 000000000..d1bd5fad6
--- /dev/null
+++ b/.changeset/watcher-retirement-adoption.md
@@ -0,0 +1,6 @@
+---
+"@inkeep/open-knowledge": patch
+"@inkeep/open-knowledge-server": patch
+---
+
+Release watcher subscriptions that finish initializing after server shutdown begins.
diff --git a/packages/server/src/server-factory.init-join-deadline.test-helper.ts b/packages/server/src/server-factory.init-join-deadline.test-helper.ts
new file mode 100644
index 000000000..911679b4a
--- /dev/null
+++ b/packages/server/src/server-factory.init-join-deadline.test-helper.ts
@@ -0,0 +1,306 @@
+import assert from 'node:assert/strict';
+import { createHook } from 'node:async_hooks';
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, realpathSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import type * as parcel from '@parcel/watcher';
+import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts';
+import type { NativeWatcherBackend } from './parcel-subscription.ts';
+import type { ServerInstance } from './server-factory.ts';
+
+interface Operation {
+ id: number;
+ kind: 'subscribe' | 'unsubscribe';
+ directory: string;
+ completed: boolean;
+}
+
+interface NativeRecord {
+ directory: string;
+ subscription: parcel.AsyncSubscription;
+ releaseCalls: number;
+ released: boolean;
+}
+
+export type WatcherTarget = 'content' | 'head';
+
+export const RESULT_PREFIX = 'FACTORY_NATIVE_RESULT ';
+export const READY_LINE = 'FACTORY_NATIVE_READY';
+
+async function factoryProbe(root: string, expireInit: boolean, target: WatcherTarget) {
+ const runRoot = dirname(root);
+ const project = join(root, 'project');
+ const gitDirectory = join(project, '.git');
+ const home = join(root, 'home');
+ mkdirSync(project);
+ mkdirSync(home);
+ const environmentKeys = new Set([
+ 'HOME',
+ 'USERPROFILE',
+ 'OK_FILE_WATCHER_BACKEND',
+ 'GIT_CONFIG_GLOBAL',
+ 'GIT_CONFIG_NOSYSTEM',
+ ...Object.keys(process.env).filter((key) => key.startsWith('GIT_')),
+ ]);
+ const environment = new Map([...environmentKeys].map((key) => [key, process.env[key]]));
+ for (const key of environmentKeys) delete process.env[key];
+ process.env.HOME = home;
+ process.env.USERPROFILE = home;
+ process.env.GIT_CONFIG_GLOBAL = '/dev/null';
+ process.env.GIT_CONFIG_NOSYSTEM = '1';
+ try {
+ const { createServer } = await import('./server-factory.ts');
+ const { getLogger } = await import('./logger.ts');
+ const parcel = await import('@parcel/watcher');
+ execFileSync('git', ['init', '--quiet', '--initial-branch=main', '--template='], {
+ cwd: project,
+ env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1' },
+ });
+ configureTestGitRepository(project);
+ mkdirSync(join(project, '.ok'));
+ writeFileSync(join(project, '.ok', 'config.yml'), '');
+ writeFileSync(join(project, 'start.md'), '# Start\n');
+
+ const operations: Operation[] = [];
+ const nativeRecords: NativeRecord[] = [];
+ const events: Array<Record<string, unknown>> = [];
+ let callingNative: { kind: Operation['kind']; directory: string } | undefined;
+ const hook = createHook({
+ init(id, type) {
+ if (type !== 'undefined' || callingNative === undefined) return;
+ operations.push({ id, ...callingNative, completed: false });
+ events.push({ event: 'native-queued', id, ...callingNative });
+ },
+ before(id) {
+ const operation = operations.find((entry) => entry.id === id);
+ if (operation === undefined) return;
+ operation.completed = true;
+ events.push({ event: 'native-completion-entered', id });
+ },
+ }).enable();
+ const pending = () => operations.filter((entry) => !entry.completed).map((entry) => entry.id);
+ const timers = new Set<ReturnType<typeof setTimeout>>();
+ const realSetTimeout = globalThis.setTimeout;
+ const realSetInterval = globalThis.setInterval;
+ let teardownStarted = false;
+ let initExpired = false;
+ globalThis.setTimeout = Object.assign(
+ <T extends unknown[]>(callback: (...args: T) => void, delay?: number, ...args: T) => {
+ const handle = realSetTimeout(callback, delay, ...args);
+ timers.add(handle);
+ const stack = new Error('deadline site').stack ?? '';
+ if (
+ expireInit &&
+ teardownStarted &&
+ !initExpired &&
+ stack.includes('/server-factory.ts:') &&
+ !stack.includes('joinPrelude')
+ ) {
+ initExpired = true;
+ clearTimeout(handle);
+ events.push({ event: 'deadline-selected', delay, stack, pending: pending() });
+ queueMicrotask(() => {
+ events.push({ event: 'deadline-fired', pending: pending() });
+ callback(...args);
+ });
+ }
+ return handle;
+ },
+ realSetTimeout,
+ );
+ globalThis.setInterval = Object.assign(
+ <T extends unknown[]>(callback: (...args: T) => void, delay?: number, ...args: T) => {
+ const handle = realSetInterval(callback, delay, ...args);
+ timers.add(handle);
+ return handle;
+ },
+ realSetInterval,
+ );
+ const logger = getLogger('server');
+ const warn = logger.warn;
+ logger.warn = (fields, message) => {
+ if (message === '[server] init did not complete before the pre-flush deadline') {
+ events.push({ event: 'initialization-drain-timeout', pending: pending() });
+ }
+ return warn.call(logger, fields, message);
+ };
+
+ let server: ServerInstance | undefined;
+ const destruction = Promise.withResolvers<void>();
+ let destroyPromise: Promise<void> | undefined;
+ let pendingAtStart: number[] = [];
+ const backend: NativeWatcherBackend = {
+ async subscribe(directory, callback, options) {
+ callingNative = { kind: 'subscribe', directory };
+ const acquisition = parcel.subscribe(directory, callback, options);
+ void acquisition.catch((error: unknown) => {
+ events.push({ event: 'native-acquisition-failed', directory, error: String(error) });
+ });
+ callingNative = undefined;
+ const targetDirectory = target === 'head' ? gitDirectory : project;
+ if (expireInit && directory === targetDirectory && destroyPromise === undefined) {
+ const activeServer = server;
+ if (activeServer === undefined)
+ throw new Error('Factory was not returned before subscription');
+ pendingAtStart = pending();
+ teardownStarted = true;
+ events.push({ event: 'destroy-entered', directory, pending: pendingAtStart });
+ destroyPromise = activeServer.destroy();
+ void destroyPromise.then(destruction.resolve, destruction.reject);
+ }
+ const subscription = await acquisition;
+ const record: NativeRecord = { directory, subscription, releaseCalls: 0, released: false };
+ nativeRecords.push(record);
+ events.push({ event: 'subscription-returned', directory });
+ return {
+ async unsubscribe() {
+ record.releaseCalls++;
+ callingNative = { kind: 'unsubscribe', directory };
+ const release = subscription.unsubscribe();
+ callingNative = undefined;
+ await release;
+ record.released = true;
+ },
+ };
+ },
+ };
+
+ console.log(READY_LINE);
+ try {
+ server = createServer({
+ contentDir: project,
+ projectDir: project,
+ configHomedirOverride: home,
+ authorityRegistryPath: join(root, 'authority.sqlite'),
+ watcherBackend: backend,
+ freshnessIntervalMs: null,
+ gitEnabled: false,
+ quiet: true,
+ });
+ const readiness = server.ready.then(
+ () => ({ status: 'fulfilled' }),
+ (error: unknown) => ({ status: 'rejected', error: String(error) }),
+ );
+ if (expireInit) {
+ await destruction.promise;
+ events.push({ event: 'destroy-settled', pending: pending() });
+ } else {
+ await readiness;
+ await server.destroy();
+ }
+ const ready = await readiness;
+ await server.destroy();
+ const result = {
+ mode: expireInit ? 'expired-init' : 'control',
+ target,
+ pendingAtStart,
+ ready,
+ operations,
+ subscriptions: nativeRecords.map(({ directory, releaseCalls, released }) => ({
+ directory,
+ releaseCalls,
+ released,
+ })),
+ events,
+ };
+ writeFileSync(
+ join(runRoot, `factory-${result.mode}-${target}.json`),
+ JSON.stringify(result, null, 2),
+ );
+ console.log(RESULT_PREFIX + JSON.stringify(result));
+ assertFactoryResult(result, project, expireInit, target);
+ return result;
+ } finally {
+ globalThis.setTimeout = realSetTimeout;
+ globalThis.setInterval = realSetInterval;
+ logger.warn = warn;
+ for (const handle of timers) {
+ clearTimeout(handle);
+ clearInterval(handle);
+ }
+ for (const record of nativeRecords) {
+ if (!record.released) {
+ await record.subscription.unsubscribe();
+ record.released = true;
+ }
+ }
+ if (server !== undefined) {
+ await server.syncEngine?.destroy();
+ await server.destroy();
+ }
+ hook.disable();
+ }
+ } finally {
+ for (const [key, value] of environment) {
+ if (value === undefined) delete process.env[key];
+ else process.env[key] = value;
+ }
+ }
+}
+
+export type FactoryProbeResult = Awaited<ReturnType<typeof factoryProbe>>;
+
+export function assertFactoryResult(
+ result: FactoryProbeResult,
+ project: string,
+ expireInit: boolean,
+ target: WatcherTarget,
+): void {
+ const { operations, pendingAtStart, subscriptions, events, ready } = result;
+ const gitDirectory = join(project, '.git');
+ assert.equal(result.mode, expireInit ? 'expired-init' : 'control');
+ assert.equal(result.target, target);
+ assert.equal(
+ operations.some((entry) => entry.kind === 'subscribe'),
+ true,
+ );
+ if (expireInit) {
+ assert.ok(pendingAtStart.length > 0);
+ const entered = events.find((entry) => entry.event === 'destroy-entered');
+ assert.ok(entered, 'shutdown begins from the targeted real subscription');
+ assert.equal(entered.directory, target === 'head' ? gitDirectory : project);
+ assert.ok(Array.isArray(entered.pending));
+ for (const id of pendingAtStart) assert.ok(entered.pending.includes(id));
+ assert.equal(
+ subscriptions.some(
+ (entry) => entry.directory === (target === 'head' ? gitDirectory : project),
+ ),
+ true,
+ );
+ const deadline = events.find((entry) => entry.event === 'initialization-drain-timeout');
+ assert.ok(deadline);
+ assert.ok(Array.isArray(deadline.pending));
+ for (const id of pendingAtStart) assert.ok(deadline.pending.includes(id));
+ } else {
+ assert.deepEqual(
+ subscriptions.map((entry) => entry.directory).sort(),
+ [project, gitDirectory].sort(),
+ 'the completed startup control acquires both real Parcel subscriptions',
+ );
+ assert.equal(ready.status, 'fulfilled');
+ }
+ assert.deepEqual(
+ subscriptions.filter((entry) => !entry.released).map((entry) => entry.directory),
+ [],
+ 'every real native subscription is released by factory shutdown',
+ );
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {
+ const [root, name] = process.argv.slice(2);
+ assert.ok(root);
+ switch (name) {
+ case 'factory control releases real native subscriptions after initialization':
+ await factoryProbe(root, false, 'content');
+ break;
+ case 'factory releases a native acquisition crossing the initialization deadline':
+ await factoryProbe(root, true, 'content');
+ break;
+ case 'factory releases a HEAD acquisition crossing the initialization deadline':
+ await factoryProbe(root, true, 'head');
+ break;
+ default:
+ throw new Error('Unknown factory probe case');
+ }
+}
diff --git a/packages/server/src/server-factory.init-join-deadline.test.ts b/packages/server/src/server-factory.init-join-deadline.test.ts
new file mode 100644
index 000000000..80bf88129
--- /dev/null
+++ b/packages/server/src/server-factory.init-join-deadline.test.ts
@@ -0,0 +1,124 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { beforeEach, expect, type TestContext, test } from 'vitest';
+import { spawnUnderOwnerLifeline } from '../../../test-support/owner-lifeline.ts';
+import {
+ assertFactoryResult,
+ type FactoryProbeResult,
+ READY_LINE,
+ RESULT_PREFIX,
+ type WatcherTarget,
+} from './server-factory.init-join-deadline.test-helper.ts';
+
+const retainedEvidenceRoot = process.env.OK_FACTORY_NATIVE_EVIDENCE_DIR;
+const evidenceRoot = retainedEvidenceRoot ?? tmpdir();
+const HARNESS = fileURLToPath(
+ new URL('./server-factory.init-join-deadline.test-helper.ts', import.meta.url),
+);
+
+async function prepareFactoryProbe(ctx: TestContext) {
+ mkdirSync(evidenceRoot, { recursive: true });
+ const runRoot = realpathSync(mkdtempSync(join(evidenceRoot, 'run-')));
+ const root = realpathSync(mkdtempSync(join(runRoot, 'fixture-')));
+ const env = { ...process.env, OK_FILE_WATCHER_BACKEND: 'chokidar' };
+ delete env.NODE_OPTIONS;
+ delete env.OK_VITEST_RUN_EVIDENCE_OWNER;
+ delete env.GATE_BASE_SHA;
+ delete env.CLAUDE_ENV_FILE;
+ const { child, commandResult } = spawnUnderOwnerLifeline(
+ process.execPath,
+ ['--import', 'tsx', '--conditions=development', HARNESS, root, ctx.task.name],
+ { cwd: resolve(import.meta.dirname, '..'), env },
+ );
+ const ready = Promise.withResolvers<void>();
+ let spawnError: Error | undefined;
+ child.once('error', (error) => {
+ spawnError = error;
+ });
+ const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((done) => {
+ child.once('close', (code, signal) => done({ code, signal }));
+ });
+ let stdout = '';
+ let stderr = '';
+ child.stdout.setEncoding('utf8').on('data', (chunk: string) => {
+ stdout += chunk;
+ if (stdout.split('\n').includes(READY_LINE)) ready.resolve();
+ });
+ child.stderr.setEncoding('utf8').on('data', (chunk: string) => {
+ stderr += chunk;
+ });
+ ctx.onTestFinished(async () => {
+ child.stdin.end();
+ const completion = await closed;
+ const command = await commandResult;
+ writeFileSync(join(runRoot, 'child.stdout.log'), stdout);
+ writeFileSync(join(runRoot, 'child.stderr.log'), stderr);
+ writeFileSync(
+ join(runRoot, 'completion.json'),
+ JSON.stringify({ completion, command, error: spawnError?.message }),
+ );
+ rmSync(root, { recursive: true, force: true });
+ if (retainedEvidenceRoot === undefined) rmSync(runRoot, { recursive: true, force: true });
+ });
+ void closed.then(() =>
+ ready.reject(new Error(`Child closed before readiness\n${stdout}\n${stderr}`)),
+ );
+ await ready.promise;
+ return async (expireInit: boolean, target: WatcherTarget) => {
+ const completion = await closed;
+ const command = await commandResult;
+ const diagnostics =
+ JSON.stringify({ completion, command, error: spawnError?.message }) +
+ '\n' +
+ stdout +
+ '\n' +
+ stderr;
+ const line = stdout.split('\n').find((entry) => entry.startsWith(RESULT_PREFIX));
+ expect(line, diagnostics).toBeDefined();
+ if (line === undefined) throw new Error(diagnostics);
+ const result = JSON.parse(line.slice(RESULT_PREFIX.length)) as FactoryProbeResult;
+ console.log(RESULT_PREFIX + JSON.stringify(result));
+ assertFactoryResult(result, join(root, 'project'), expireInit, target);
+ expect(spawnError, diagnostics).toBeUndefined();
+ expect(command, diagnostics).toEqual({ code: 0, signal: null });
+ };
+}
+
+const preparedProbes = new WeakMap<TestContext, Awaited<ReturnType<typeof prepareFactoryProbe>>>();
+
+beforeEach(async (ctx) => {
+ if (process.platform === 'win32') return;
+ preparedProbes.set(ctx, await prepareFactoryProbe(ctx));
+});
+
+async function runFactoryProbe(ctx: TestContext, expireInit: boolean, target: WatcherTarget) {
+ const probe = preparedProbes.get(ctx);
+ if (probe === undefined) throw new Error('Owned factory probe was not prepared');
+ await probe(expireInit, target);
+}
+
+test('factory control releases real native subscriptions after initialization', async (ctx) => {
+ ctx.skip(
+ process.platform === 'win32',
+ 'requires POSIX process-group ownership for the real fixture Git descendants; Node v24.21.0 rejects Windows process-group kills, and this file has no Windows CI lane',
+ );
+ await runFactoryProbe(ctx, false, 'content');
+});
+
+test('factory releases a native acquisition crossing the initialization deadline', async (ctx) => {
+ ctx.skip(
+ process.platform === 'win32',
+ 'requires POSIX process-group ownership for the real fixture Git descendants; Node v24.21.0 rejects Windows process-group kills, and this file has no Windows CI lane',
+ );
+ await runFactoryProbe(ctx, true, 'content');
+});
+
+test('factory releases a HEAD acquisition crossing the initialization deadline', async (ctx) => {
+ ctx.skip(
+ process.platform === 'win32',
+ 'requires POSIX process-group ownership for the real fixture Git descendants; Node v24.21.0 rejects Windows process-group kills, and this file has no Windows CI lane',
+ );
+ await runFactoryProbe(ctx, true, 'head');
+});
diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.ts
index eaa5c07a8..fc73e2099 100644
--- a/packages/server/src/server-factory.test.ts
+++ b/packages/server/src/server-factory.test.ts
@@ -16,6 +16,7 @@ import {
import { mkdtemp, readFile, realpath, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
import {
BridgeMergeContentLossError,
DEFAULT_LINTER_CONFIG,
@@ -60,6 +61,7 @@ import {
writeManagedRenameJournal,
} from './managed-rename-journal.ts';
import { getMetrics, resetMetrics } from './metrics.ts';
+import type { NativeWatcherBackend } from './parcel-subscription.ts';
import { ensureProjectGit } from './project-git.ts';
import { MAX_LCS_CELLS } from './reconciliation.ts';
import { saveRemovedDocsJournal } from './removed-docs-journal.ts';
@@ -71,6 +73,7 @@ import {
type ShadowHandle,
shadowGit,
} from './shadow-repo.ts';
+import { SyncEngine } from './sync-engine.ts';
import { TagIndex } from './tag-index.ts';
import { contentHash } from './version-hash.ts';
import { createWatcherDouble } from './watcher-double.test-helper.ts';
@@ -509,17 +512,83 @@ async function executePendingStore(server: ServerInstance, docName: string): Pro
describe('createServer().destroy() — graceful shutdown flush', () => {
let tmpDir: string;
let logCapture: ReturnType<typeof captureAllLoggers>;
+ let finishControlledInit: (() => Promise<void>) | undefined;
beforeEach(async () => {
+ finishControlledInit = undefined;
tmpDir = await mkdtemp(join(tmpdir(), 'ok-destroy-test-'));
logCapture = captureAllLoggers();
});
afterEach(async () => {
+ await finishControlledInit?.();
loggerFactory.reset();
await rm(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
+ test('late watcher release failure is a shutdown phase error and retains authority', async () => {
+ const contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ const authorityRegistryPath = join(tmpDir, 'authority.sqlite');
+ const releaseFailure = new Error('late watcher release failed');
+ let server: ServerInstance | undefined;
+ let destroyPromise: Promise<void> | undefined;
+ const backend: NativeWatcherBackend = {
+ async subscribe() {
+ if (destroyPromise === undefined) {
+ if (server === undefined) throw new Error('server was not returned before subscription');
+ destroyPromise = server.destroy();
+ }
+ return {
+ async unsubscribe() {
+ throw releaseFailure;
+ },
+ };
+ },
+ };
+
+ server = createServer({
+ contentDir,
+ projectDir: tmpDir,
+ authorityRegistryPath,
+ watcherBackend: backend,
+ gitEnabled: false,
+ quiet: true,
+ });
+
+ try {
+ await server.ready;
+ await destroyPromise;
+
+ expect(server.degraded).not.toContain('file-watcher');
+ expect(server.degraded).not.toContain('head-watcher');
+ const shutdownLogs = logCapture.getCalls('warn', 'shutdown flushed');
+ expect(shutdownLogs).toHaveLength(1);
+ expect(shutdownLogs[0]?.payload.phaseErrors).toContainEqual({
+ phase: 'watcher-unsubscribe',
+ error: releaseFailure.message,
+ });
+ expect(logCapture.getCalls('error', 'shutdown phase-1 watcher unsubscribe failed')).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ payload: expect.objectContaining({ watched: 'content' }) }),
+ ]),
+ );
+ expect(logCapture.getCalls('error', 'disk bridge watcher failed to start')).toHaveLength(0);
+ expect(logCapture.getCalls('error', 'HEAD watcher failed to start')).toHaveLength(0);
+
+ const registry = new DatabaseSync(authorityRegistryPath, { readOnly: true });
+ try {
+ expect(registry.prepare('SELECT count(*) AS count FROM server_authority').get()).toEqual({
+ count: 1,
+ });
+ } finally {
+ registry.close();
+ }
+ } finally {
+ await server.destroy();
+ }
+ });
+
test('flushes L1 markdown writes before destroy() resolves + emits shutdown log', async () => {
const server = createServer({
contentDir: tmpDir,
@@ -1454,16 +1523,70 @@ describe('createServer().destroy() — graceful shutdown flush', () => {
await server.destroy();
});
- test('destroy() during async init — before ready resolves', async () => {
+ test('destroy() during async init — before ready resolves', async (ctx) => {
+ const startCompleted = Promise.withResolvers<SyncEngine>();
+ const releaseStart = Promise.withResolvers<void>();
+ const realStart = SyncEngine.prototype.start;
+ let startReturned = false;
+ const start = vi.spyOn(SyncEngine.prototype, 'start').mockImplementation(async function () {
+ await realStart.call(this);
+ startCompleted.resolve(this);
+ await releaseStart.promise;
+ startReturned = true;
+ });
+ let restored = false;
+ const releaseAndRestore = () => {
+ releaseStart.resolve();
+ if (!restored) {
+ restored = true;
+ start.mockRestore();
+ }
+ };
+ finishControlledInit = async () => releaseAndRestore();
+ ctx.signal.addEventListener('abort', releaseAndRestore, { once: true });
+ ctx.onTestFinished(() => {
+ ctx.signal.removeEventListener('abort', releaseAndRestore);
+ releaseAndRestore();
+ });
const server = createServer({
contentDir: tmpDir,
projectDir: tmpDir,
quiet: true,
});
- await server.destroy();
-
- const shutdownLogs = logCapture.getCalls('info', 'shutdown flushed');
- expect(shutdownLogs).toHaveLength(1);
+ finishControlledInit = async () => {
+ releaseAndRestore();
+ await server.destroy();
+ };
+ let readySettled = false;
+ const readiness = server.ready.then(
+ () => {
+ readySettled = true;
+ },
+ (error: unknown) => {
+ readySettled = true;
+ throw error;
+ },
+ );
+ try {
+ const engine = await Promise.race([
+ startCompleted.promise,
+ readiness.then(() => {
+ throw new Error('Initialization finished without reaching the held startup');
+ }),
+ ]);
+ ctx.signal.throwIfAborted();
+ expect(engine === server.syncEngine).toBe(true);
+ expect(startReturned).toBe(false);
+ expect(readySettled).toBe(false);
+ const destroying = server.destroy();
+ releaseStart.resolve();
+ await Promise.all([readiness, destroying]);
+
+ const shutdownLogs = logCapture.getCalls('info', 'shutdown flushed');
+ expect(shutdownLogs).toHaveLength(1);
+ } finally {
+ releaseAndRestore();
+ }
});
test('destroy() with zero documents loaded (short-circuit path)', async () => {
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 98bcbaad1..bd9843b25 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -4502,6 +4502,42 @@ export function createServer(options: ServerOptions): ServerInstance {
}
let syncEngine: SyncEngine | null = null;
let inflightDestroy: Promise<void> | null = null;
+ let retirementStarted = false;
+ let watcherReleaseState: {
+ phaseErrors: Array<{ phase: string; error: string }>;
+ authorityGateReached: boolean;
+ } | null = null;
+
+ function recordWatcherReleaseFailure(watched: 'head' | 'content', err: unknown): void {
+ const state = watcherReleaseState;
+ if (state === null || state.authorityGateReached) {
+ log.error({ err, watched }, '[server] late watcher release failed after shutdown');
+ return;
+ }
+ state.phaseErrors.push({
+ phase: 'watcher-unsubscribe',
+ error: err instanceof Error ? err.message : String(err),
+ });
+ log.error({ err, watched }, '[server] shutdown phase-1 watcher unsubscribe failed');
+ }
+
+ async function adoptWatcher<T extends { unsubscribe(): Promise<void> }>(
+ start: () => Promise<T>,
+ publish: (handle: T) => void,
+ watched: 'head' | 'content',
+ ): Promise<void> {
+ if (retirementStarted) return;
+ const handle = await start();
+ if (retirementStarted) {
+ try {
+ await handle.unsubscribe();
+ } catch (err) {
+ recordWatcherReleaseFailure(watched, err);
+ }
+ return;
+ }
+ publish(handle);
+ }
const rescueDocToShadowBuffer = (
docName: string,
@@ -4748,10 +4784,13 @@ export function createServer(options: ServerOptions): ServerInstance {
async function destroy(): Promise<void> {
if (inflightDestroy) return inflightDestroy;
+ retirementStarted = true;
inflightDestroy = (async () => {
const t0 = Date.now();
const phaseErrors: Array<{ phase: string; error: string }> = [];
+ const releaseState = { phaseErrors, authorityGateReached: false };
+ watcherReleaseState = releaseState;
const graceShareMs = Math.min(destroyTimeoutMs, DEFAULT_SIGTERM_GRACE_MS / 5);
const preludeDeadline = Date.now() + graceShareMs;
const preludeRemaining = () => Math.max(1, preludeDeadline - Date.now());
@@ -4907,11 +4946,7 @@ export function createServer(options: ServerOptions): ServerInstance {
try {
await release.unsubscribe();
} catch (err) {
- phaseErrors.push({
- phase: 'watcher-unsubscribe',
- error: err instanceof Error ? err.message : String(err),
- });
- log.error({ err, watched }, '[server] shutdown phase-1 watcher unsubscribe failed');
+ recordWatcherReleaseFailure(watched, err);
}
};
@@ -5115,6 +5150,7 @@ export function createServer(options: ServerOptions): ServerInstance {
log.error({ err: reason }, '[server] shutdown local-op subprocess drain failed');
}
const durationMs = Date.now() - t0;
+ releaseState.authorityGateReached = true;
if (phaseErrors.length === 0) {
authorityRetired = true;
try {
@@ -5152,6 +5188,7 @@ export function createServer(options: ServerOptions): ServerInstance {
error: err instanceof Error ? err.message : String(err),
});
}
+ if (watcherReleaseState === releaseState) watcherReleaseState = null;
}
if (authResult?.status === 'rejected') throw authResult.reason;
})();
@@ -5671,18 +5708,25 @@ export function createServer(options: ServerOptions): ServerInstance {
}
}, IN_PLACE_RESCAN_DEBOUNCE_MS);
};
- watcher = await withSpan('ok.boot.seed-walk', undefined, async () =>
- startWatcher(canonicalContentDir, onDiskEvent, contentFilter, {
- onRawBatch,
- onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),
- onRecoveryPassCompleted: (pass, whole) => settleVanishedDocuments(pass, whole),
- nativeBackend: options.watcherBackend,
- clock: options.watcherClock,
- freshnessIntervalMs:
- options.freshnessIntervalMs === undefined
- ? FRESHNESS_BASE_INTERVAL_MS
- : options.freshnessIntervalMs,
- }),
+ await adoptWatcher(
+ () =>
+ withSpan('ok.boot.seed-walk', undefined, async () =>
+ startWatcher(canonicalContentDir, onDiskEvent, contentFilter, {
+ onRawBatch,
+ onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),
+ onRecoveryPassCompleted: (pass, whole) => settleVanishedDocuments(pass, whole),
+ nativeBackend: options.watcherBackend,
+ clock: options.watcherClock,
+ freshnessIntervalMs:
+ options.freshnessIntervalMs === undefined
+ ? FRESHNESS_BASE_INTERVAL_MS
+ : options.freshnessIntervalMs,
+ }),
+ ),
+ (handle) => {
+ watcher = handle;
+ },
+ 'content',
);
void resyncRecordedSkillCopies(projectDir, contentDir)
.then((n) => {
@@ -5780,277 +5824,284 @@ export function createServer(options: ServerOptions): ServerInstance {
let releaseHeadBatchAdmission = () => {};
try {
- headWatcher = await startHeadWatcher(
- projectDir,
- async ({ trigger }) => {
- log.info({ trigger }, `[batch] begin trigger=${trigger}`);
- incrementBatch();
- hocuspocus.flushPendingStores();
- await persistence.flushPendingGitCommit();
-
- holdBatch('head-watcher');
- releaseHeadBatchAdmission();
- releaseHeadBatchAdmission = durabilityState.holdWriteAdmission();
-
- if (shadowRef.current) {
- const currentBranch = getActiveBranch();
- const newBranch = readProjectHeadState(projectDir).branch ?? currentBranch;
- const docs: ParkableDoc[] = [];
- let absentBaseParkSkips = 0;
- for (const [docName, document] of hocuspocus.documents) {
- if (isReservedForUserTree(docName)) continue;
- let markdown: string | null = null;
- document.transact(() => {
- markdown = serializeDoc(docName);
- }, PARK_SNAPSHOT_ORIGIN);
- if (markdown === null) continue;
- const diskSnapshot = getReconciledBase(docName);
- if (diskSnapshot === undefined) {
- absentBaseParkSkips++;
- continue;
- }
- docs.push({ docName, markdown, diskSnapshot });
- }
- if (absentBaseParkSkips > 0) {
- log.warn(
- { count: absentBaseParkSkips, branch: currentBranch },
- `[history] skipped parking ${absentBaseParkSkips} doc(s) with no reconciled base; their content was not parked to the shadow history`,
- );
- }
- if (docs.length > 0) {
- try {
- const sha = await parkBranch(
- shadowRef.current,
- currentBranch,
- SERVICE_WRITER.id,
- docs,
- newBranch,
- );
- if (sha) {
- incrementPark();
- log.info(
- { count: docs.length, branch: currentBranch, sha: sha.slice(0, 8) },
- `[history] parked ${docs.length} docs on ${currentBranch} → ${sha.slice(0, 8)}`,
+ await adoptWatcher(
+ () =>
+ startHeadWatcher(
+ projectDir,
+ async ({ trigger }) => {
+ log.info({ trigger }, `[batch] begin trigger=${trigger}`);
+ incrementBatch();
+ hocuspocus.flushPendingStores();
+ await persistence.flushPendingGitCommit();
+
+ holdBatch('head-watcher');
+ releaseHeadBatchAdmission();
+ releaseHeadBatchAdmission = durabilityState.holdWriteAdmission();
+
+ if (shadowRef.current) {
+ const currentBranch = getActiveBranch();
+ const newBranch = readProjectHeadState(projectDir).branch ?? currentBranch;
+ const docs: ParkableDoc[] = [];
+ let absentBaseParkSkips = 0;
+ for (const [docName, document] of hocuspocus.documents) {
+ if (isReservedForUserTree(docName)) continue;
+ let markdown: string | null = null;
+ document.transact(() => {
+ markdown = serializeDoc(docName);
+ }, PARK_SNAPSHOT_ORIGIN);
+ if (markdown === null) continue;
+ const diskSnapshot = getReconciledBase(docName);
+ if (diskSnapshot === undefined) {
+ absentBaseParkSkips++;
+ continue;
+ }
+ docs.push({ docName, markdown, diskSnapshot });
+ }
+ if (absentBaseParkSkips > 0) {
+ log.warn(
+ { count: absentBaseParkSkips, branch: currentBranch },
+ `[history] skipped parking ${absentBaseParkSkips} doc(s) with no reconciled base; their content was not parked to the shadow history`,
);
}
- } catch (e) {
- log.error({ err: e }, '[shadow] park failed');
+ if (docs.length > 0) {
+ try {
+ const sha = await parkBranch(
+ shadowRef.current,
+ currentBranch,
+ SERVICE_WRITER.id,
+ docs,
+ newBranch,
+ );
+ if (sha) {
+ incrementPark();
+ log.info(
+ { count: docs.length, branch: currentBranch, sha: sha.slice(0, 8) },
+ `[history] parked ${docs.length} docs on ${currentBranch} → ${sha.slice(0, 8)}`,
+ );
+ }
+ } catch (e) {
+ log.error({ err: e }, '[shadow] park failed');
+ }
+ }
}
- }
- }
- },
- async (info) => {
- const bufferedCount = eventBuffer.length;
- const newBranch = info.newBranch ?? 'main';
-
- log.info(
- {
- kind: info.batchKind,
- headMoved: info.headMoved,
- docs: bufferedCount,
- timeout: !!info.timeout,
},
- `[batch] end kind=${info.batchKind} headMoved=${info.headMoved} docs=${bufferedCount}${info.timeout ? ' timeout' : ''}`,
- );
+ async (info) => {
+ const bufferedCount = eventBuffer.length;
+ const newBranch = info.newBranch ?? 'main';
- if (info.batchKind === 'within-branch') {
- try {
- try {
- await drainEventBuffer();
- } finally {
- disposeBufferedRemovalScopes();
- }
+ log.info(
+ {
+ kind: info.batchKind,
+ headMoved: info.headMoved,
+ docs: bufferedCount,
+ timeout: !!info.timeout,
+ },
+ `[batch] end kind=${info.batchKind} headMoved=${info.headMoved} docs=${bufferedCount}${info.timeout ? ' timeout' : ''}`,
+ );
- if (info.headMoved && info.newHead) {
- const changes = resolveUpstreamChanges(
- projectDir,
- resolve(contentDir),
- info.oldHead,
- info.newHead,
- );
- if (changes.size > 0) {
- dropPendingDocs(changes.keys());
- for (const [docName, author] of changes) {
- recordContributor(
- docName,
- gitAuthorWriterId(author.email),
- author.name,
- author.email,
- formatReconcileSubject(docName),
+ if (info.batchKind === 'within-branch') {
+ try {
+ try {
+ await drainEventBuffer();
+ } finally {
+ disposeBufferedRemovalScopes();
+ }
+
+ if (info.headMoved && info.newHead) {
+ const changes = resolveUpstreamChanges(
+ projectDir,
+ resolve(contentDir),
+ info.oldHead,
+ info.newHead,
);
+ if (changes.size > 0) {
+ dropPendingDocs(changes.keys());
+ for (const [docName, author] of changes) {
+ recordContributor(
+ docName,
+ gitAuthorWriterId(author.email),
+ author.name,
+ author.email,
+ formatReconcileSubject(docName),
+ );
+ }
+ await persistence.flushContributors();
+ }
+ }
+ } finally {
+ try {
+ if (releaseBatchHold('head-watcher')) await finishBatch();
+ } finally {
+ releaseHeadBatchAdmission();
}
- await persistence.flushContributors();
}
- }
- } finally {
- try {
- if (releaseBatchHold('head-watcher')) await finishBatch();
- } finally {
- releaseHeadBatchAdmission();
- }
- }
- try {
- await conflicts.pruneMergeNativeAgainstGit();
- } catch (err) {
- log.warn({ err }, '[head-watcher] conflict prune against git failed');
- }
- } else {
- incrementBranchSwitch();
- supersedeBranchSwitchFollowUps();
- let batchReleased = false;
- let switchPassFellShort = false;
- let branchTransition: DerivedDocumentIndexBranchTransition | undefined;
- try {
- switchReconciledBaseScope(newBranch);
- conflicts.setBranch(newBranch);
- hydrateStaleExternalWrites();
- branchTransition = await derivedDocumentIndex.beginBranchSwitch(newBranch);
+ try {
+ await conflicts.pruneMergeNativeAgainstGit();
+ } catch (err) {
+ log.warn({ err }, '[head-watcher] conflict prune against git failed');
+ }
+ } else {
+ incrementBranchSwitch();
+ supersedeBranchSwitchFollowUps();
+ let batchReleased = false;
+ let switchPassFellShort = false;
+ let branchTransition: DerivedDocumentIndexBranchTransition | undefined;
+ try {
+ switchReconciledBaseScope(newBranch);
+ conflicts.setBranch(newBranch);
+ hydrateStaleExternalWrites();
+ branchTransition = await derivedDocumentIndex.beginBranchSwitch(newBranch);
- contentFilter.rebuildDirCount();
+ contentFilter.rebuildDirCount();
- try {
- let reseedSkipCount = 0;
- basenameIndex.clear();
- await seedBasenameIndex({
- contentDir,
- contentFilter,
- basenameIndex,
- onSkip: (reason, code, path) => {
- reseedSkipCount++;
- log.warn(
- { reason, code, path, branch: newBranch },
- `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,
+ try {
+ let reseedSkipCount = 0;
+ basenameIndex.clear();
+ await seedBasenameIndex({
+ contentDir,
+ contentFilter,
+ basenameIndex,
+ onSkip: (reason, code, path) => {
+ reseedSkipCount++;
+ log.warn(
+ { reason, code, path, branch: newBranch },
+ `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,
+ );
+ },
+ });
+ if (reseedSkipCount > 0) {
+ log.warn(
+ { count: reseedSkipCount, branch: newBranch },
+ `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,
+ );
+ if (!degraded.includes('basename-index-partial')) {
+ degraded.push('basename-index-partial');
+ }
+ }
+ } catch (err) {
+ log.error(
+ { err, branch: newBranch },
+ '[basename-index] branch-switch reseed failed',
);
- },
- });
- if (reseedSkipCount > 0) {
- log.warn(
- { count: reseedSkipCount, branch: newBranch },
- `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,
- );
- if (!degraded.includes('basename-index-partial')) {
- degraded.push('basename-index-partial');
}
- }
- } catch (err) {
- log.error(
- { err, branch: newBranch },
- '[basename-index] branch-switch reseed failed',
- );
- }
- try {
- await derivedDocumentIndex.settleBranchFromDisk(branchTransition);
- } catch (err) {
- derivedDocumentIndex.abortBranchSwitch(branchTransition);
- log.error(
- { err, branch: newBranch },
- '[derived-index] branch-switch rebuild failed; relationship views may be stale',
- );
- }
+ try {
+ await derivedDocumentIndex.settleBranchFromDisk(branchTransition);
+ } catch (err) {
+ derivedDocumentIndex.abortBranchSwitch(branchTransition);
+ log.error(
+ { err, branch: newBranch },
+ '[derived-index] branch-switch rebuild failed; relationship views may be stale',
+ );
+ }
- try {
- await drainEventBuffer();
- const switchPass = await watcher?.recheck('full');
- if (switchPass && (switchPass.status !== 'completed' || switchPass.partial)) {
- switchPassFellShort = true;
- incrementBranchSwitchRecheckShortfalls();
- log.warn(
- {
- branch: newBranch,
- status: switchPass.status,
- partial: switchPass.partial === true,
- },
- `[branch-switch] the re-check after switching to ${newBranch} did not observe the whole tree; another one runs once the switch ends`,
+ try {
+ await drainEventBuffer();
+ const switchPass = await watcher?.recheck('full');
+ if (switchPass && (switchPass.status !== 'completed' || switchPass.partial)) {
+ switchPassFellShort = true;
+ incrementBranchSwitchRecheckShortfalls();
+ log.warn(
+ {
+ branch: newBranch,
+ status: switchPass.status,
+ partial: switchPass.partial === true,
+ },
+ `[branch-switch] the re-check after switching to ${newBranch} did not observe the whole tree; another one runs once the switch ends`,
+ );
+ }
+ await drainEventBuffer();
+ } finally {
+ disposeBufferedRemovalScopes();
+ }
+ tombstoneResidentsResolvingIntoPrivateState(newBranch);
+ log.info(
+ { branch: newBranch, docCount: hocuspocus.documents.size },
+ `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,
);
- }
- await drainEventBuffer();
- } finally {
- disposeBufferedRemovalScopes();
- }
- tombstoneResidentsResolvingIntoPrivateState(newBranch);
- log.info(
- { branch: newBranch, docCount: hocuspocus.documents.size },
- `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,
- );
- if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {
- try {
- const sg = shadowGit(shadowRef.current);
- const refs = (
- await sg.raw(
- 'for-each-ref',
- `refs/wip/${info.oldBranch}/`,
- '--format=%(refname)',
- )
- ).trim();
- if (refs) {
- await shadowOpGateFor(shadowRef.current).withMutator(async () => {
- for (const ref of refs.split('\n')) {
- if (ref) {
- await sg.raw('update-ref', '-d', ref);
- }
+ if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {
+ try {
+ const sg = shadowGit(shadowRef.current);
+ const refs = (
+ await sg.raw(
+ 'for-each-ref',
+ `refs/wip/${info.oldBranch}/`,
+ '--format=%(refname)',
+ )
+ ).trim();
+ if (refs) {
+ await shadowOpGateFor(shadowRef.current).withMutator(async () => {
+ for (const ref of refs.split('\n')) {
+ if (ref) {
+ await sg.raw('update-ref', '-d', ref);
+ }
+ }
+ });
+ log.info(
+ { context: info.oldBranch },
+ `[branch-switch] cleaned up detached context ${info.oldBranch}`,
+ );
}
- });
- log.info(
- { context: info.oldBranch },
- `[branch-switch] cleaned up detached context ${info.oldBranch}`,
- );
+ } catch (e) {
+ log.error({ err: e }, '[branch-switch] detached cleanup failed');
+ }
+ }
+
+ batchReleased = true;
+ if (releaseBatchHold('head-watcher')) await finishBatch();
+ cc1Broadcaster?.emitBranchSwitched(newBranch);
+ if (switchPassFellShort) followUpBranchSwitchRecheck(newBranch);
+ } finally {
+ try {
+ derivedDocumentIndex.abortBranchSwitch(branchTransition);
+ if (!batchReleased && releaseBatchHold('head-watcher')) await finishBatch();
+ } finally {
+ releaseHeadBatchAdmission();
}
- } catch (e) {
- log.error({ err: e }, '[branch-switch] detached cleanup failed');
}
}
- batchReleased = true;
- if (releaseBatchHold('head-watcher')) await finishBatch();
- cc1Broadcaster?.emitBranchSwitched(newBranch);
- if (switchPassFellShort) followUpBranchSwitchRecheck(newBranch);
- } finally {
- try {
- derivedDocumentIndex.abortBranchSwitch(branchTransition);
- if (!batchReleased && releaseBatchHold('head-watcher')) await finishBatch();
- } finally {
- releaseHeadBatchAdmission();
+ if (info.headMoved && info.newHead && shadowRef.current && bufferedCount > 0) {
+ const contentRootForShadow = contentRoot ?? '.';
+ try {
+ const sha = await commitUpstreamImport(
+ shadowRef.current,
+ contentRootForShadow,
+ info.oldHead,
+ info.newHead,
+ newBranch,
+ );
+ incrementUpstreamImport();
+ log.info(
+ {
+ oldHead: info.oldHead?.slice(0, 8) ?? 'null',
+ newHead: info.newHead.slice(0, 8),
+ sha: sha.slice(0, 8),
+ },
+ `[history] upstream-import from ${info.oldHead?.slice(0, 8) ?? 'null'}..${info.newHead.slice(0, 8)} → ${sha.slice(0, 8)}`,
+ );
+ } catch (e) {
+ log.error({ err: e }, '[shadow] upstream-import failed');
+ }
}
- }
- }
-
- if (info.headMoved && info.newHead && shadowRef.current && bufferedCount > 0) {
- const contentRootForShadow = contentRoot ?? '.';
- try {
- const sha = await commitUpstreamImport(
- shadowRef.current,
- contentRootForShadow,
- info.oldHead,
- info.newHead,
- newBranch,
- );
- incrementUpstreamImport();
- log.info(
- {
- oldHead: info.oldHead?.slice(0, 8) ?? 'null',
- newHead: info.newHead.slice(0, 8),
- sha: sha.slice(0, 8),
- },
- `[history] upstream-import from ${info.oldHead?.slice(0, 8) ?? 'null'}..${info.newHead.slice(0, 8)} → ${sha.slice(0, 8)}`,
- );
- } catch (e) {
- log.error({ err: e }, '[shadow] upstream-import failed');
- }
- }
- },
- {
- nativeBackend: options.watcherBackend,
- clock: options.watcherClock,
- onGaveUp: () => {
- if (!degraded.includes('head-watcher')) degraded.push('head-watcher');
- },
- onBack: () => {
- const at = degraded.indexOf('head-watcher');
- if (at !== -1) degraded.splice(at, 1);
- },
+ },
+ {
+ nativeBackend: options.watcherBackend,
+ clock: options.watcherClock,
+ onGaveUp: () => {
+ if (!degraded.includes('head-watcher')) degraded.push('head-watcher');
+ },
+ onBack: () => {
+ const at = degraded.indexOf('head-watcher');
+ if (at !== -1) degraded.splice(at, 1);
+ },
+ },
+ ),
+ (handle) => {
+ headWatcher = handle;
},
+ 'head',
);
} catch (err) {
log.error({ err }, '[server] HEAD watcher failed to start');