Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Release watcher handles acquired during server shutdown (#7530)
Nick Gomez5052ed6b2c41
diff --git a/.changeset/watcher-retirement-adoption.md b/.changeset/watcher-retirement-adoption.mdnew file mode 100644index 000000000..d1bd5fad6--- /dev/null+++ b/.changeset/watcher-retirement-adoption.md@@ -0,0 +1,6 @@+---+"@inkeep/open-knowledge": patch+"@inkeep/open-knowledge-server": patch+---++Release watcher subscriptions that finish initializing after server shutdown begins.diff --git a/packages/server/src/server-factory.init-join-deadline.test-helper.ts b/packages/server/src/server-factory.init-join-deadline.test-helper.tsnew file mode 100644index 000000000..911679b4a--- /dev/null+++ b/packages/server/src/server-factory.init-join-deadline.test-helper.ts@@ -0,0 +1,306 @@+import assert from 'node:assert/strict';+import { createHook } from 'node:async_hooks';+import { execFileSync } from 'node:child_process';+import { mkdirSync, realpathSync, writeFileSync } from 'node:fs';+import { dirname, join } from 'node:path';+import { pathToFileURL } from 'node:url';+import type * as parcel from '@parcel/watcher';+import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts';+import type { NativeWatcherBackend } from './parcel-subscription.ts';+import type { ServerInstance } from './server-factory.ts';++interface Operation {+ id: number;+ kind: 'subscribe' | 'unsubscribe';+ directory: string;+ completed: boolean;+}++interface NativeRecord {+ directory: string;+ subscription: parcel.AsyncSubscription;+ releaseCalls: number;+ released: boolean;+}++export type WatcherTarget = 'content' | 'head';++export const RESULT_PREFIX = 'FACTORY_NATIVE_RESULT ';+export const READY_LINE = 'FACTORY_NATIVE_READY';++async function factoryProbe(root: string, expireInit: boolean, target: WatcherTarget) {+ const runRoot = dirname(root);+ const project = join(root, 'project');+ const gitDirectory = join(project, '.git');+ const home = join(root, 'home');+ mkdirSync(project);+ mkdirSync(home);+ const environmentKeys = new Set([+ 'HOME',+ 'USERPROFILE',+ 'OK_FILE_WATCHER_BACKEND',+ 'GIT_CONFIG_GLOBAL',+ 'GIT_CONFIG_NOSYSTEM',+ ...Object.keys(process.env).filter((key) => key.startsWith('GIT_')),+ ]);+ const environment = new Map([...environmentKeys].map((key) => [key, process.env[key]]));+ for (const key of environmentKeys) delete process.env[key];+ process.env.HOME = home;+ process.env.USERPROFILE = home;+ process.env.GIT_CONFIG_GLOBAL = '/dev/null';+ process.env.GIT_CONFIG_NOSYSTEM = '1';+ try {+ const { createServer } = await import('./server-factory.ts');+ const { getLogger } = await import('./logger.ts');+ const parcel = await import('@parcel/watcher');+ execFileSync('git', ['init', '--quiet', '--initial-branch=main', '--template='], {+ cwd: project,+ env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1' },+ });+ configureTestGitRepository(project);+ mkdirSync(join(project, '.ok'));+ writeFileSync(join(project, '.ok', 'config.yml'), '');+ writeFileSync(join(project, 'start.md'), '# Start\n');++ const operations: Operation[] = [];+ const nativeRecords: NativeRecord[] = [];+ const events: Array<Record<string, unknown>> = [];+ let callingNative: { kind: Operation['kind']; directory: string } | undefined;+ const hook = createHook({+ init(id, type) {+ if (type !== 'undefined' || callingNative === undefined) return;+ operations.push({ id, ...callingNative, completed: false });+ events.push({ event: 'native-queued', id, ...callingNative });+ },+ before(id) {+ const operation = operations.find((entry) => entry.id === id);+ if (operation === undefined) return;+ operation.completed = true;+ events.push({ event: 'native-completion-entered', id });+ },+ }).enable();+ const pending = () => operations.filter((entry) => !entry.completed).map((entry) => entry.id);+ const timers = new Set<ReturnType<typeof setTimeout>>();+ const realSetTimeout = globalThis.setTimeout;+ const realSetInterval = globalThis.setInterval;+ let teardownStarted = false;+ let initExpired = false;+ globalThis.setTimeout = Object.assign(+ <T extends unknown[]>(callback: (...args: T) => void, delay?: number, ...args: T) => {+ const handle = realSetTimeout(callback, delay, ...args);+ timers.add(handle);+ const stack = new Error('deadline site').stack ?? '';+ if (+ expireInit &&+ teardownStarted &&+ !initExpired &&+ stack.includes('/server-factory.ts:') &&+ !stack.includes('joinPrelude')+ ) {+ initExpired = true;+ clearTimeout(handle);+ events.push({ event: 'deadline-selected', delay, stack, pending: pending() });+ queueMicrotask(() => {+ events.push({ event: 'deadline-fired', pending: pending() });+ callback(...args);+ });+ }+ return handle;+ },+ realSetTimeout,+ );+ globalThis.setInterval = Object.assign(+ <T extends unknown[]>(callback: (...args: T) => void, delay?: number, ...args: T) => {+ const handle = realSetInterval(callback, delay, ...args);+ timers.add(handle);+ return handle;+ },+ realSetInterval,+ );+ const logger = getLogger('server');+ const warn = logger.warn;+ logger.warn = (fields, message) => {+ if (message === '[server] init did not complete before the pre-flush deadline') {+ events.push({ event: 'initialization-drain-timeout', pending: pending() });+ }+ return warn.call(logger, fields, message);+ };++ let server: ServerInstance | undefined;+ const destruction = Promise.withResolvers<void>();+ let destroyPromise: Promise<void> | undefined;+ let pendingAtStart: number[] = [];+ const backend: NativeWatcherBackend = {+ async subscribe(directory, callback, options) {+ callingNative = { kind: 'subscribe', directory };+ const acquisition = parcel.subscribe(directory, callback, options);+ void acquisition.catch((error: unknown) => {+ events.push({ event: 'native-acquisition-failed', directory, error: String(error) });+ });+ callingNative = undefined;+ const targetDirectory = target === 'head' ? gitDirectory : project;+ if (expireInit && directory === targetDirectory && destroyPromise === undefined) {+ const activeServer = server;+ if (activeServer === undefined)+ throw new Error('Factory was not returned before subscription');+ pendingAtStart = pending();+ teardownStarted = true;+ events.push({ event: 'destroy-entered', directory, pending: pendingAtStart });+ destroyPromise = activeServer.destroy();+ void destroyPromise.then(destruction.resolve, destruction.reject);+ }+ const subscription = await acquisition;+ const record: NativeRecord = { directory, subscription, releaseCalls: 0, released: false };+ nativeRecords.push(record);+ events.push({ event: 'subscription-returned', directory });+ return {+ async unsubscribe() {+ record.releaseCalls++;+ callingNative = { kind: 'unsubscribe', directory };+ const release = subscription.unsubscribe();+ callingNative = undefined;+ await release;+ record.released = true;+ },+ };+ },+ };++ console.log(READY_LINE);+ try {+ server = createServer({+ contentDir: project,+ projectDir: project,+ configHomedirOverride: home,+ authorityRegistryPath: join(root, 'authority.sqlite'),+ watcherBackend: backend,+ freshnessIntervalMs: null,+ gitEnabled: false,+ quiet: true,+ });+ const readiness = server.ready.then(+ () => ({ status: 'fulfilled' }),+ (error: unknown) => ({ status: 'rejected', error: String(error) }),+ );+ if (expireInit) {+ await destruction.promise;+ events.push({ event: 'destroy-settled', pending: pending() });+ } else {+ await readiness;+ await server.destroy();+ }+ const ready = await readiness;+ await server.destroy();+ const result = {+ mode: expireInit ? 'expired-init' : 'control',+ target,+ pendingAtStart,+ ready,+ operations,+ subscriptions: nativeRecords.map(({ directory, releaseCalls, released }) => ({+ directory,+ releaseCalls,+ released,+ })),+ events,+ };+ writeFileSync(+ join(runRoot, `factory-${result.mode}-${target}.json`),+ JSON.stringify(result, null, 2),+ );+ console.log(RESULT_PREFIX + JSON.stringify(result));+ assertFactoryResult(result, project, expireInit, target);+ return result;+ } finally {+ globalThis.setTimeout = realSetTimeout;+ globalThis.setInterval = realSetInterval;+ logger.warn = warn;+ for (const handle of timers) {+ clearTimeout(handle);+ clearInterval(handle);+ }+ for (const record of nativeRecords) {+ if (!record.released) {+ await record.subscription.unsubscribe();+ record.released = true;+ }+ }+ if (server !== undefined) {+ await server.syncEngine?.destroy();+ await server.destroy();+ }+ hook.disable();+ }+ } finally {+ for (const [key, value] of environment) {+ if (value === undefined) delete process.env[key];+ else process.env[key] = value;+ }+ }+}++export type FactoryProbeResult = Awaited<ReturnType<typeof factoryProbe>>;++export function assertFactoryResult(+ result: FactoryProbeResult,+ project: string,+ expireInit: boolean,+ target: WatcherTarget,+): void {+ const { operations, pendingAtStart, subscriptions, events, ready } = result;+ const gitDirectory = join(project, '.git');+ assert.equal(result.mode, expireInit ? 'expired-init' : 'control');+ assert.equal(result.target, target);+ assert.equal(+ operations.some((entry) => entry.kind === 'subscribe'),+ true,+ );+ if (expireInit) {+ assert.ok(pendingAtStart.length > 0);+ const entered = events.find((entry) => entry.event === 'destroy-entered');+ assert.ok(entered, 'shutdown begins from the targeted real subscription');+ assert.equal(entered.directory, target === 'head' ? gitDirectory : project);+ assert.ok(Array.isArray(entered.pending));+ for (const id of pendingAtStart) assert.ok(entered.pending.includes(id));+ assert.equal(+ subscriptions.some(+ (entry) => entry.directory === (target === 'head' ? gitDirectory : project),+ ),+ true,+ );+ const deadline = events.find((entry) => entry.event === 'initialization-drain-timeout');+ assert.ok(deadline);+ assert.ok(Array.isArray(deadline.pending));+ for (const id of pendingAtStart) assert.ok(deadline.pending.includes(id));+ } else {+ assert.deepEqual(+ subscriptions.map((entry) => entry.directory).sort(),+ [project, gitDirectory].sort(),+ 'the completed startup control acquires both real Parcel subscriptions',+ );+ assert.equal(ready.status, 'fulfilled');+ }+ assert.deepEqual(+ subscriptions.filter((entry) => !entry.released).map((entry) => entry.directory),+ [],+ 'every real native subscription is released by factory shutdown',+ );+}++if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {+ const [root, name] = process.argv.slice(2);+ assert.ok(root);+ switch (name) {+ case 'factory control releases real native subscriptions after initialization':+ await factoryProbe(root, false, 'content');+ break;+ case 'factory releases a native acquisition crossing the initialization deadline':+ await factoryProbe(root, true, 'content');+ break;+ case 'factory releases a HEAD acquisition crossing the initialization deadline':+ await factoryProbe(root, true, 'head');+ break;+ default:+ throw new Error('Unknown factory probe case');+ }+}diff --git a/packages/server/src/server-factory.init-join-deadline.test.ts b/packages/server/src/server-factory.init-join-deadline.test.tsnew file mode 100644index 000000000..80bf88129--- /dev/null+++ b/packages/server/src/server-factory.init-join-deadline.test.ts@@ -0,0 +1,124 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import { beforeEach, expect, type TestContext, test } from 'vitest';+import { spawnUnderOwnerLifeline } from '../../../test-support/owner-lifeline.ts';+import {+ assertFactoryResult,+ type FactoryProbeResult,+ READY_LINE,+ RESULT_PREFIX,+ type WatcherTarget,+} from './server-factory.init-join-deadline.test-helper.ts';++const retainedEvidenceRoot = process.env.OK_FACTORY_NATIVE_EVIDENCE_DIR;+const evidenceRoot = retainedEvidenceRoot ?? tmpdir();+const HARNESS = fileURLToPath(+ new URL('./server-factory.init-join-deadline.test-helper.ts', import.meta.url),+);++async function prepareFactoryProbe(ctx: TestContext) {+ mkdirSync(evidenceRoot, { recursive: true });+ const runRoot = realpathSync(mkdtempSync(join(evidenceRoot, 'run-')));+ const root = realpathSync(mkdtempSync(join(runRoot, 'fixture-')));+ const env = { ...process.env, OK_FILE_WATCHER_BACKEND: 'chokidar' };+ delete env.NODE_OPTIONS;+ delete env.OK_VITEST_RUN_EVIDENCE_OWNER;+ delete env.GATE_BASE_SHA;+ delete env.CLAUDE_ENV_FILE;+ const { child, commandResult } = spawnUnderOwnerLifeline(+ process.execPath,+ ['--import', 'tsx', '--conditions=development', HARNESS, root, ctx.task.name],+ { cwd: resolve(import.meta.dirname, '..'), env },+ );+ const ready = Promise.withResolvers<void>();+ let spawnError: Error | undefined;+ child.once('error', (error) => {+ spawnError = error;+ });+ const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((done) => {+ child.once('close', (code, signal) => done({ code, signal }));+ });+ let stdout = '';+ let stderr = '';+ child.stdout.setEncoding('utf8').on('data', (chunk: string) => {+ stdout += chunk;+ if (stdout.split('\n').includes(READY_LINE)) ready.resolve();+ });+ child.stderr.setEncoding('utf8').on('data', (chunk: string) => {+ stderr += chunk;+ });+ ctx.onTestFinished(async () => {+ child.stdin.end();+ const completion = await closed;+ const command = await commandResult;+ writeFileSync(join(runRoot, 'child.stdout.log'), stdout);+ writeFileSync(join(runRoot, 'child.stderr.log'), stderr);+ writeFileSync(+ join(runRoot, 'completion.json'),+ JSON.stringify({ completion, command, error: spawnError?.message }),+ );+ rmSync(root, { recursive: true, force: true });+ if (retainedEvidenceRoot === undefined) rmSync(runRoot, { recursive: true, force: true });+ });+ void closed.then(() =>+ ready.reject(new Error(`Child closed before readiness\n${stdout}\n${stderr}`)),+ );+ await ready.promise;+ return async (expireInit: boolean, target: WatcherTarget) => {+ const completion = await closed;+ const command = await commandResult;+ const diagnostics =+ JSON.stringify({ completion, command, error: spawnError?.message }) ++ '\n' ++ stdout ++ '\n' ++ stderr;+ const line = stdout.split('\n').find((entry) => entry.startsWith(RESULT_PREFIX));+ expect(line, diagnostics).toBeDefined();+ if (line === undefined) throw new Error(diagnostics);+ const result = JSON.parse(line.slice(RESULT_PREFIX.length)) as FactoryProbeResult;+ console.log(RESULT_PREFIX + JSON.stringify(result));+ assertFactoryResult(result, join(root, 'project'), expireInit, target);+ expect(spawnError, diagnostics).toBeUndefined();+ expect(command, diagnostics).toEqual({ code: 0, signal: null });+ };+}++const preparedProbes = new WeakMap<TestContext, Awaited<ReturnType<typeof prepareFactoryProbe>>>();++beforeEach(async (ctx) => {+ if (process.platform === 'win32') return;+ preparedProbes.set(ctx, await prepareFactoryProbe(ctx));+});++async function runFactoryProbe(ctx: TestContext, expireInit: boolean, target: WatcherTarget) {+ const probe = preparedProbes.get(ctx);+ if (probe === undefined) throw new Error('Owned factory probe was not prepared');+ await probe(expireInit, target);+}++test('factory control releases real native subscriptions after initialization', async (ctx) => {+ ctx.skip(+ process.platform === 'win32',+ 'requires POSIX process-group ownership for the real fixture Git descendants; Node v24.21.0 rejects Windows process-group kills, and this file has no Windows CI lane',+ );+ await runFactoryProbe(ctx, false, 'content');+});++test('factory releases a native acquisition crossing the initialization deadline', async (ctx) => {+ ctx.skip(+ process.platform === 'win32',+ 'requires POSIX process-group ownership for the real fixture Git descendants; Node v24.21.0 rejects Windows process-group kills, and this file has no Windows CI lane',+ );+ await runFactoryProbe(ctx, true, 'content');+});++test('factory releases a HEAD acquisition crossing the initialization deadline', async (ctx) => {+ ctx.skip(+ process.platform === 'win32',+ 'requires POSIX process-group ownership for the real fixture Git descendants; Node v24.21.0 rejects Windows process-group kills, and this file has no Windows CI lane',+ );+ await runFactoryProbe(ctx, true, 'head');+});diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.tsindex eaa5c07a8..fc73e2099 100644--- a/packages/server/src/server-factory.test.ts+++ b/packages/server/src/server-factory.test.ts@@ -16,6 +16,7 @@ import {import { mkdtemp, readFile, realpath, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join, resolve } from 'node:path';+import { DatabaseSync } from 'node:sqlite';import {BridgeMergeContentLossError,DEFAULT_LINTER_CONFIG,@@ -60,6 +61,7 @@ import {writeManagedRenameJournal,} from './managed-rename-journal.ts';import { getMetrics, resetMetrics } from './metrics.ts';+import type { NativeWatcherBackend } from './parcel-subscription.ts';import { ensureProjectGit } from './project-git.ts';import { MAX_LCS_CELLS } from './reconciliation.ts';import { saveRemovedDocsJournal } from './removed-docs-journal.ts';@@ -71,6 +73,7 @@ import {type ShadowHandle,shadowGit,} from './shadow-repo.ts';+import { SyncEngine } from './sync-engine.ts';import { TagIndex } from './tag-index.ts';import { contentHash } from './version-hash.ts';import { createWatcherDouble } from './watcher-double.test-helper.ts';@@ -509,17 +512,83 @@ async function executePendingStore(server: ServerInstance, docName: string): Prodescribe('createServer().destroy() — graceful shutdown flush', () => {let tmpDir: string;let logCapture: ReturnType<typeof captureAllLoggers>;+ let finishControlledInit: (() => Promise<void>) | undefined;beforeEach(async () => {+ finishControlledInit = undefined;tmpDir = await mkdtemp(join(tmpdir(), 'ok-destroy-test-'));logCapture = captureAllLoggers();});afterEach(async () => {+ await finishControlledInit?.();loggerFactory.reset();await rm(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });});+ test('late watcher release failure is a shutdown phase error and retains authority', async () => {+ const contentDir = join(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ const authorityRegistryPath = join(tmpDir, 'authority.sqlite');+ const releaseFailure = new Error('late watcher release failed');+ let server: ServerInstance | undefined;+ let destroyPromise: Promise<void> | undefined;+ const backend: NativeWatcherBackend = {+ async subscribe() {+ if (destroyPromise === undefined) {+ if (server === undefined) throw new Error('server was not returned before subscription');+ destroyPromise = server.destroy();+ }+ return {+ async unsubscribe() {+ throw releaseFailure;+ },+ };+ },+ };++ server = createServer({+ contentDir,+ projectDir: tmpDir,+ authorityRegistryPath,+ watcherBackend: backend,+ gitEnabled: false,+ quiet: true,+ });++ try {+ await server.ready;+ await destroyPromise;++ expect(server.degraded).not.toContain('file-watcher');+ expect(server.degraded).not.toContain('head-watcher');+ const shutdownLogs = logCapture.getCalls('warn', 'shutdown flushed');+ expect(shutdownLogs).toHaveLength(1);+ expect(shutdownLogs[0]?.payload.phaseErrors).toContainEqual({+ phase: 'watcher-unsubscribe',+ error: releaseFailure.message,+ });+ expect(logCapture.getCalls('error', 'shutdown phase-1 watcher unsubscribe failed')).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ payload: expect.objectContaining({ watched: 'content' }) }),+ ]),+ );+ expect(logCapture.getCalls('error', 'disk bridge watcher failed to start')).toHaveLength(0);+ expect(logCapture.getCalls('error', 'HEAD watcher failed to start')).toHaveLength(0);++ const registry = new DatabaseSync(authorityRegistryPath, { readOnly: true });+ try {+ expect(registry.prepare('SELECT count(*) AS count FROM server_authority').get()).toEqual({+ count: 1,+ });+ } finally {+ registry.close();+ }+ } finally {+ await server.destroy();+ }+ });+test('flushes L1 markdown writes before destroy() resolves + emits shutdown log', async () => {const server = createServer({contentDir: tmpDir,@@ -1454,16 +1523,70 @@ describe('createServer().destroy() — graceful shutdown flush', () => {await server.destroy();});- test('destroy() during async init — before ready resolves', async () => {+ test('destroy() during async init — before ready resolves', async (ctx) => {+ const startCompleted = Promise.withResolvers<SyncEngine>();+ const releaseStart = Promise.withResolvers<void>();+ const realStart = SyncEngine.prototype.start;+ let startReturned = false;+ const start = vi.spyOn(SyncEngine.prototype, 'start').mockImplementation(async function () {+ await realStart.call(this);+ startCompleted.resolve(this);+ await releaseStart.promise;+ startReturned = true;+ });+ let restored = false;+ const releaseAndRestore = () => {+ releaseStart.resolve();+ if (!restored) {+ restored = true;+ start.mockRestore();+ }+ };+ finishControlledInit = async () => releaseAndRestore();+ ctx.signal.addEventListener('abort', releaseAndRestore, { once: true });+ ctx.onTestFinished(() => {+ ctx.signal.removeEventListener('abort', releaseAndRestore);+ releaseAndRestore();+ });const server = createServer({contentDir: tmpDir,projectDir: tmpDir,quiet: true,});- await server.destroy();-- const shutdownLogs = logCapture.getCalls('info', 'shutdown flushed');- expect(shutdownLogs).toHaveLength(1);+ finishControlledInit = async () => {+ releaseAndRestore();+ await server.destroy();+ };+ let readySettled = false;+ const readiness = server.ready.then(+ () => {+ readySettled = true;+ },+ (error: unknown) => {+ readySettled = true;+ throw error;+ },+ );+ try {+ const engine = await Promise.race([+ startCompleted.promise,+ readiness.then(() => {+ throw new Error('Initialization finished without reaching the held startup');+ }),+ ]);+ ctx.signal.throwIfAborted();+ expect(engine === server.syncEngine).toBe(true);+ expect(startReturned).toBe(false);+ expect(readySettled).toBe(false);+ const destroying = server.destroy();+ releaseStart.resolve();+ await Promise.all([readiness, destroying]);++ const shutdownLogs = logCapture.getCalls('info', 'shutdown flushed');+ expect(shutdownLogs).toHaveLength(1);+ } finally {+ releaseAndRestore();+ }});test('destroy() with zero documents loaded (short-circuit path)', async () => {diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex 98bcbaad1..bd9843b25 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -4502,6 +4502,42 @@ export function createServer(options: ServerOptions): ServerInstance {}let syncEngine: SyncEngine | null = null;let inflightDestroy: Promise<void> | null = null;+ let retirementStarted = false;+ let watcherReleaseState: {+ phaseErrors: Array<{ phase: string; error: string }>;+ authorityGateReached: boolean;+ } | null = null;++ function recordWatcherReleaseFailure(watched: 'head' | 'content', err: unknown): void {+ const state = watcherReleaseState;+ if (state === null || state.authorityGateReached) {+ log.error({ err, watched }, '[server] late watcher release failed after shutdown');+ return;+ }+ state.phaseErrors.push({+ phase: 'watcher-unsubscribe',+ error: err instanceof Error ? err.message : String(err),+ });+ log.error({ err, watched }, '[server] shutdown phase-1 watcher unsubscribe failed');+ }++ async function adoptWatcher<T extends { unsubscribe(): Promise<void> }>(+ start: () => Promise<T>,+ publish: (handle: T) => void,+ watched: 'head' | 'content',+ ): Promise<void> {+ if (retirementStarted) return;+ const handle = await start();+ if (retirementStarted) {+ try {+ await handle.unsubscribe();+ } catch (err) {+ recordWatcherReleaseFailure(watched, err);+ }+ return;+ }+ publish(handle);+ }const rescueDocToShadowBuffer = (docName: string,@@ -4748,10 +4784,13 @@ export function createServer(options: ServerOptions): ServerInstance {async function destroy(): Promise<void> {if (inflightDestroy) return inflightDestroy;+ retirementStarted = true;inflightDestroy = (async () => {const t0 = Date.now();const phaseErrors: Array<{ phase: string; error: string }> = [];+ const releaseState = { phaseErrors, authorityGateReached: false };+ watcherReleaseState = releaseState;const graceShareMs = Math.min(destroyTimeoutMs, DEFAULT_SIGTERM_GRACE_MS / 5);const preludeDeadline = Date.now() + graceShareMs;const preludeRemaining = () => Math.max(1, preludeDeadline - Date.now());@@ -4907,11 +4946,7 @@ export function createServer(options: ServerOptions): ServerInstance {try {await release.unsubscribe();} catch (err) {- phaseErrors.push({- phase: 'watcher-unsubscribe',- error: err instanceof Error ? err.message : String(err),- });- log.error({ err, watched }, '[server] shutdown phase-1 watcher unsubscribe failed');+ recordWatcherReleaseFailure(watched, err);}};@@ -5115,6 +5150,7 @@ export function createServer(options: ServerOptions): ServerInstance {log.error({ err: reason }, '[server] shutdown local-op subprocess drain failed');}const durationMs = Date.now() - t0;+ releaseState.authorityGateReached = true;if (phaseErrors.length === 0) {authorityRetired = true;try {@@ -5152,6 +5188,7 @@ export function createServer(options: ServerOptions): ServerInstance {error: err instanceof Error ? err.message : String(err),});}+ if (watcherReleaseState === releaseState) watcherReleaseState = null;}if (authResult?.status === 'rejected') throw authResult.reason;})();@@ -5671,18 +5708,25 @@ export function createServer(options: ServerOptions): ServerInstance {}}, IN_PLACE_RESCAN_DEBOUNCE_MS);};- watcher = await withSpan('ok.boot.seed-walk', undefined, async () =>- startWatcher(canonicalContentDir, onDiskEvent, contentFilter, {- onRawBatch,- onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),- onRecoveryPassCompleted: (pass, whole) => settleVanishedDocuments(pass, whole),- nativeBackend: options.watcherBackend,- clock: options.watcherClock,- freshnessIntervalMs:- options.freshnessIntervalMs === undefined- ? FRESHNESS_BASE_INTERVAL_MS- : options.freshnessIntervalMs,- }),+ await adoptWatcher(+ () =>+ withSpan('ok.boot.seed-walk', undefined, async () =>+ startWatcher(canonicalContentDir, onDiskEvent, contentFilter, {+ onRawBatch,+ onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),+ onRecoveryPassCompleted: (pass, whole) => settleVanishedDocuments(pass, whole),+ nativeBackend: options.watcherBackend,+ clock: options.watcherClock,+ freshnessIntervalMs:+ options.freshnessIntervalMs === undefined+ ? FRESHNESS_BASE_INTERVAL_MS+ : options.freshnessIntervalMs,+ }),+ ),+ (handle) => {+ watcher = handle;+ },+ 'content',);void resyncRecordedSkillCopies(projectDir, contentDir).then((n) => {@@ -5780,277 +5824,284 @@ export function createServer(options: ServerOptions): ServerInstance {let releaseHeadBatchAdmission = () => {};try {- headWatcher = await startHeadWatcher(- projectDir,- async ({ trigger }) => {- log.info({ trigger }, `[batch] begin trigger=${trigger}`);- incrementBatch();- hocuspocus.flushPendingStores();- await persistence.flushPendingGitCommit();-- holdBatch('head-watcher');- releaseHeadBatchAdmission();- releaseHeadBatchAdmission = durabilityState.holdWriteAdmission();-- if (shadowRef.current) {- const currentBranch = getActiveBranch();- const newBranch = readProjectHeadState(projectDir).branch ?? currentBranch;- const docs: ParkableDoc[] = [];- let absentBaseParkSkips = 0;- for (const [docName, document] of hocuspocus.documents) {- if (isReservedForUserTree(docName)) continue;- let markdown: string | null = null;- document.transact(() => {- markdown = serializeDoc(docName);- }, PARK_SNAPSHOT_ORIGIN);- if (markdown === null) continue;- const diskSnapshot = getReconciledBase(docName);- if (diskSnapshot === undefined) {- absentBaseParkSkips++;- continue;- }- docs.push({ docName, markdown, diskSnapshot });- }- if (absentBaseParkSkips > 0) {- log.warn(- { count: absentBaseParkSkips, branch: currentBranch },- `[history] skipped parking ${absentBaseParkSkips} doc(s) with no reconciled base; their content was not parked to the shadow history`,- );- }- if (docs.length > 0) {- try {- const sha = await parkBranch(- shadowRef.current,- currentBranch,- SERVICE_WRITER.id,- docs,- newBranch,- );- if (sha) {- incrementPark();- log.info(- { count: docs.length, branch: currentBranch, sha: sha.slice(0, 8) },- `[history] parked ${docs.length} docs on ${currentBranch} → ${sha.slice(0, 8)}`,+ await adoptWatcher(+ () =>+ startHeadWatcher(+ projectDir,+ async ({ trigger }) => {+ log.info({ trigger }, `[batch] begin trigger=${trigger}`);+ incrementBatch();+ hocuspocus.flushPendingStores();+ await persistence.flushPendingGitCommit();++ holdBatch('head-watcher');+ releaseHeadBatchAdmission();+ releaseHeadBatchAdmission = durabilityState.holdWriteAdmission();++ if (shadowRef.current) {+ const currentBranch = getActiveBranch();+ const newBranch = readProjectHeadState(projectDir).branch ?? currentBranch;+ const docs: ParkableDoc[] = [];+ let absentBaseParkSkips = 0;+ for (const [docName, document] of hocuspocus.documents) {+ if (isReservedForUserTree(docName)) continue;+ let markdown: string | null = null;+ document.transact(() => {+ markdown = serializeDoc(docName);+ }, PARK_SNAPSHOT_ORIGIN);+ if (markdown === null) continue;+ const diskSnapshot = getReconciledBase(docName);+ if (diskSnapshot === undefined) {+ absentBaseParkSkips++;+ continue;+ }+ docs.push({ docName, markdown, diskSnapshot });+ }+ if (absentBaseParkSkips > 0) {+ log.warn(+ { count: absentBaseParkSkips, branch: currentBranch },+ `[history] skipped parking ${absentBaseParkSkips} doc(s) with no reconciled base; their content was not parked to the shadow history`,);}- } catch (e) {- log.error({ err: e }, '[shadow] park failed');+ if (docs.length > 0) {+ try {+ const sha = await parkBranch(+ shadowRef.current,+ currentBranch,+ SERVICE_WRITER.id,+ docs,+ newBranch,+ );+ if (sha) {+ incrementPark();+ log.info(+ { count: docs.length, branch: currentBranch, sha: sha.slice(0, 8) },+ `[history] parked ${docs.length} docs on ${currentBranch} → ${sha.slice(0, 8)}`,+ );+ }+ } catch (e) {+ log.error({ err: e }, '[shadow] park failed');+ }+ }}- }- }- },- async (info) => {- const bufferedCount = eventBuffer.length;- const newBranch = info.newBranch ?? 'main';-- log.info(- {- kind: info.batchKind,- headMoved: info.headMoved,- docs: bufferedCount,- timeout: !!info.timeout,},- `[batch] end kind=${info.batchKind} headMoved=${info.headMoved} docs=${bufferedCount}${info.timeout ? ' timeout' : ''}`,- );+ async (info) => {+ const bufferedCount = eventBuffer.length;+ const newBranch = info.newBranch ?? 'main';- if (info.batchKind === 'within-branch') {- try {- try {- await drainEventBuffer();- } finally {- disposeBufferedRemovalScopes();- }+ log.info(+ {+ kind: info.batchKind,+ headMoved: info.headMoved,+ docs: bufferedCount,+ timeout: !!info.timeout,+ },+ `[batch] end kind=${info.batchKind} headMoved=${info.headMoved} docs=${bufferedCount}${info.timeout ? ' timeout' : ''}`,+ );- if (info.headMoved && info.newHead) {- const changes = resolveUpstreamChanges(- projectDir,- resolve(contentDir),- info.oldHead,- info.newHead,- );- if (changes.size > 0) {- dropPendingDocs(changes.keys());- for (const [docName, author] of changes) {- recordContributor(- docName,- gitAuthorWriterId(author.email),- author.name,- author.email,- formatReconcileSubject(docName),+ if (info.batchKind === 'within-branch') {+ try {+ try {+ await drainEventBuffer();+ } finally {+ disposeBufferedRemovalScopes();+ }++ if (info.headMoved && info.newHead) {+ const changes = resolveUpstreamChanges(+ projectDir,+ resolve(contentDir),+ info.oldHead,+ info.newHead,);+ if (changes.size > 0) {+ dropPendingDocs(changes.keys());+ for (const [docName, author] of changes) {+ recordContributor(+ docName,+ gitAuthorWriterId(author.email),+ author.name,+ author.email,+ formatReconcileSubject(docName),+ );+ }+ await persistence.flushContributors();+ }+ }+ } finally {+ try {+ if (releaseBatchHold('head-watcher')) await finishBatch();+ } finally {+ releaseHeadBatchAdmission();}- await persistence.flushContributors();}- }- } finally {- try {- if (releaseBatchHold('head-watcher')) await finishBatch();- } finally {- releaseHeadBatchAdmission();- }- }- try {- await conflicts.pruneMergeNativeAgainstGit();- } catch (err) {- log.warn({ err }, '[head-watcher] conflict prune against git failed');- }- } else {- incrementBranchSwitch();- supersedeBranchSwitchFollowUps();- let batchReleased = false;- let switchPassFellShort = false;- let branchTransition: DerivedDocumentIndexBranchTransition | undefined;- try {- switchReconciledBaseScope(newBranch);- conflicts.setBranch(newBranch);- hydrateStaleExternalWrites();- branchTransition = await derivedDocumentIndex.beginBranchSwitch(newBranch);+ try {+ await conflicts.pruneMergeNativeAgainstGit();+ } catch (err) {+ log.warn({ err }, '[head-watcher] conflict prune against git failed');+ }+ } else {+ incrementBranchSwitch();+ supersedeBranchSwitchFollowUps();+ let batchReleased = false;+ let switchPassFellShort = false;+ let branchTransition: DerivedDocumentIndexBranchTransition | undefined;+ try {+ switchReconciledBaseScope(newBranch);+ conflicts.setBranch(newBranch);+ hydrateStaleExternalWrites();+ branchTransition = await derivedDocumentIndex.beginBranchSwitch(newBranch);- contentFilter.rebuildDirCount();+ contentFilter.rebuildDirCount();- try {- let reseedSkipCount = 0;- basenameIndex.clear();- await seedBasenameIndex({- contentDir,- contentFilter,- basenameIndex,- onSkip: (reason, code, path) => {- reseedSkipCount++;- log.warn(- { reason, code, path, branch: newBranch },- `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,+ try {+ let reseedSkipCount = 0;+ basenameIndex.clear();+ await seedBasenameIndex({+ contentDir,+ contentFilter,+ basenameIndex,+ onSkip: (reason, code, path) => {+ reseedSkipCount++;+ log.warn(+ { reason, code, path, branch: newBranch },+ `[basename-index] skipped entry during branch-switch reseed (${reason}${code ? ` ${code}` : ''})`,+ );+ },+ });+ if (reseedSkipCount > 0) {+ log.warn(+ { count: reseedSkipCount, branch: newBranch },+ `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,+ );+ if (!degraded.includes('basename-index-partial')) {+ degraded.push('basename-index-partial');+ }+ }+ } catch (err) {+ log.error(+ { err, branch: newBranch },+ '[basename-index] branch-switch reseed failed',);- },- });- if (reseedSkipCount > 0) {- log.warn(- { count: reseedSkipCount, branch: newBranch },- `[basename-index] branch-switch reseed completed with ${reseedSkipCount} skipped entries — embeds under inaccessible subtrees will not resolve on this branch`,- );- if (!degraded.includes('basename-index-partial')) {- degraded.push('basename-index-partial');}- }- } catch (err) {- log.error(- { err, branch: newBranch },- '[basename-index] branch-switch reseed failed',- );- }- try {- await derivedDocumentIndex.settleBranchFromDisk(branchTransition);- } catch (err) {- derivedDocumentIndex.abortBranchSwitch(branchTransition);- log.error(- { err, branch: newBranch },- '[derived-index] branch-switch rebuild failed; relationship views may be stale',- );- }+ try {+ await derivedDocumentIndex.settleBranchFromDisk(branchTransition);+ } catch (err) {+ derivedDocumentIndex.abortBranchSwitch(branchTransition);+ log.error(+ { err, branch: newBranch },+ '[derived-index] branch-switch rebuild failed; relationship views may be stale',+ );+ }- try {- await drainEventBuffer();- const switchPass = await watcher?.recheck('full');- if (switchPass && (switchPass.status !== 'completed' || switchPass.partial)) {- switchPassFellShort = true;- incrementBranchSwitchRecheckShortfalls();- log.warn(- {- branch: newBranch,- status: switchPass.status,- partial: switchPass.partial === true,- },- `[branch-switch] the re-check after switching to ${newBranch} did not observe the whole tree; another one runs once the switch ends`,+ try {+ await drainEventBuffer();+ const switchPass = await watcher?.recheck('full');+ if (switchPass && (switchPass.status !== 'completed' || switchPass.partial)) {+ switchPassFellShort = true;+ incrementBranchSwitchRecheckShortfalls();+ log.warn(+ {+ branch: newBranch,+ status: switchPass.status,+ partial: switchPass.partial === true,+ },+ `[branch-switch] the re-check after switching to ${newBranch} did not observe the whole tree; another one runs once the switch ends`,+ );+ }+ await drainEventBuffer();+ } finally {+ disposeBufferedRemovalScopes();+ }+ tombstoneResidentsResolvingIntoPrivateState(newBranch);+ log.info(+ { branch: newBranch, docCount: hocuspocus.documents.size },+ `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,);- }- await drainEventBuffer();- } finally {- disposeBufferedRemovalScopes();- }- tombstoneResidentsResolvingIntoPrivateState(newBranch);- log.info(- { branch: newBranch, docCount: hocuspocus.documents.size },- `[branch-switch] loaded branch ${newBranch} (${hocuspocus.documents.size} docs)`,- );- if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {- try {- const sg = shadowGit(shadowRef.current);- const refs = (- await sg.raw(- 'for-each-ref',- `refs/wip/${info.oldBranch}/`,- '--format=%(refname)',- )- ).trim();- if (refs) {- await shadowOpGateFor(shadowRef.current).withMutator(async () => {- for (const ref of refs.split('\n')) {- if (ref) {- await sg.raw('update-ref', '-d', ref);- }+ if (info.oldBranch?.startsWith('detached-') && shadowRef.current) {+ try {+ const sg = shadowGit(shadowRef.current);+ const refs = (+ await sg.raw(+ 'for-each-ref',+ `refs/wip/${info.oldBranch}/`,+ '--format=%(refname)',+ )+ ).trim();+ if (refs) {+ await shadowOpGateFor(shadowRef.current).withMutator(async () => {+ for (const ref of refs.split('\n')) {+ if (ref) {+ await sg.raw('update-ref', '-d', ref);+ }+ }+ });+ log.info(+ { context: info.oldBranch },+ `[branch-switch] cleaned up detached context ${info.oldBranch}`,+ );}- });- log.info(- { context: info.oldBranch },- `[branch-switch] cleaned up detached context ${info.oldBranch}`,- );+ } catch (e) {+ log.error({ err: e }, '[branch-switch] detached cleanup failed');+ }+ }++ batchReleased = true;+ if (releaseBatchHold('head-watcher')) await finishBatch();+ cc1Broadcaster?.emitBranchSwitched(newBranch);+ if (switchPassFellShort) followUpBranchSwitchRecheck(newBranch);+ } finally {+ try {+ derivedDocumentIndex.abortBranchSwitch(branchTransition);+ if (!batchReleased && releaseBatchHold('head-watcher')) await finishBatch();+ } finally {+ releaseHeadBatchAdmission();}- } catch (e) {- log.error({ err: e }, '[branch-switch] detached cleanup failed');}}- batchReleased = true;- if (releaseBatchHold('head-watcher')) await finishBatch();- cc1Broadcaster?.emitBranchSwitched(newBranch);- if (switchPassFellShort) followUpBranchSwitchRecheck(newBranch);- } finally {- try {- derivedDocumentIndex.abortBranchSwitch(branchTransition);- if (!batchReleased && releaseBatchHold('head-watcher')) await finishBatch();- } finally {- releaseHeadBatchAdmission();+ if (info.headMoved && info.newHead && shadowRef.current && bufferedCount > 0) {+ const contentRootForShadow = contentRoot ?? '.';+ try {+ const sha = await commitUpstreamImport(+ shadowRef.current,+ contentRootForShadow,+ info.oldHead,+ info.newHead,+ newBranch,+ );+ incrementUpstreamImport();+ log.info(+ {+ oldHead: info.oldHead?.slice(0, 8) ?? 'null',+ newHead: info.newHead.slice(0, 8),+ sha: sha.slice(0, 8),+ },+ `[history] upstream-import from ${info.oldHead?.slice(0, 8) ?? 'null'}..${info.newHead.slice(0, 8)} → ${sha.slice(0, 8)}`,+ );+ } catch (e) {+ log.error({ err: e }, '[shadow] upstream-import failed');+ }}- }- }-- if (info.headMoved && info.newHead && shadowRef.current && bufferedCount > 0) {- const contentRootForShadow = contentRoot ?? '.';- try {- const sha = await commitUpstreamImport(- shadowRef.current,- contentRootForShadow,- info.oldHead,- info.newHead,- newBranch,- );- incrementUpstreamImport();- log.info(- {- oldHead: info.oldHead?.slice(0, 8) ?? 'null',- newHead: info.newHead.slice(0, 8),- sha: sha.slice(0, 8),- },- `[history] upstream-import from ${info.oldHead?.slice(0, 8) ?? 'null'}..${info.newHead.slice(0, 8)} → ${sha.slice(0, 8)}`,- );- } catch (e) {- log.error({ err: e }, '[shadow] upstream-import failed');- }- }- },- {- nativeBackend: options.watcherBackend,- clock: options.watcherClock,- onGaveUp: () => {- if (!degraded.includes('head-watcher')) degraded.push('head-watcher');- },- onBack: () => {- const at = degraded.indexOf('head-watcher');- if (at !== -1) degraded.splice(at, 1);- },+ },+ {+ nativeBackend: options.watcherBackend,+ clock: options.watcherClock,+ onGaveUp: () => {+ if (!degraded.includes('head-watcher')) degraded.push('head-watcher');+ },+ onBack: () => {+ const at = degraded.indexOf('head-watcher');+ if (at !== -1) degraded.splice(at, 1);+ },+ },+ ),+ (handle) => {+ headWatcher = handle;},+ 'head',);} catch (err) {log.error({ err }, '[server] HEAD watcher failed to start');