Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Keep server shutdown alive while documents retire (#5531)
Nick Gomezaeb3eecce87d
diff --git a/.changeset/progress-aware-shutdown.md b/.changeset/progress-aware-shutdown.mdnew file mode 100644index 000000000..01adf0c18--- /dev/null+++ b/.changeset/progress-aware-shutdown.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Avoid false shutdown timeouts while documents continue unloading.diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex 05a7c9bd0..40f219a76 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -54,7 +54,12 @@ import { createMcpHttpHandler } from './mcp-http.ts';import { mountMcpAndApi, type ReadinessState } from './mcp-mount.ts';import { MissingOkConfigError } from './missing-ok-config-error.ts';import { createProjectRuntime, type ProjectRuntime } from './project-runtime.ts';-import { createServer, type ServerInstance, type ServerOptions } from './server-factory.ts';+import {+ createServer,+ observeShutdownDocumentRetirements,+ type ServerInstance,+ type ServerOptions,+} from './server-factory.ts';import { installServerMemoryGauge, installServerRuntimeGauges } from './server-memory-telemetry.ts';import {genuineInPlaceNames,@@ -643,19 +648,45 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {);let destroyed = false;- const withDestroyTimeout = async (name: string, work: () => Promise<void>): Promise<void> => {+ const withDestroyTimeout = async (+ name: string,+ work: () => Promise<void>,+ retirementSource?: ServerInstance['hocuspocus'],+ ): Promise<void> => {let timer: ReturnType<typeof setTimeout> | undefined;+ let stopObserving: (() => void) | undefined;+ let finished = false;+ const timeout = Promise.withResolvers<never>();+ const expire = () => {+ if (finished) return;+ finished = true;+ timeout.reject(+ new Error(+ retirementSource+ ? `${name} timed out: no document retired for ${DESTROY_STEP_TIMEOUT_MS}ms`+ : `${name} timed out after ${DESTROY_STEP_TIMEOUT_MS}ms`,+ ),+ );+ };+ const arm = () => {+ timer = setTimeout(expire, DESTROY_STEP_TIMEOUT_MS);+ timer.unref?.();+ };try {- await Promise.race([- work(),- new Promise<never>((_, reject) => {- timer = setTimeout(() => {- reject(new Error(`${name} timed out after ${DESTROY_STEP_TIMEOUT_MS}ms`));- }, DESTROY_STEP_TIMEOUT_MS);- timer.unref?.();- }),- ]);+ if (retirementSource) {+ stopObserving = observeShutdownDocumentRetirements(retirementSource, () => {+ if (finished) return;+ if (timer !== undefined) clearTimeout(timer);+ arm();+ });+ arm();+ }+ const workPromise = work();+ if (!retirementSource) arm();+ await Promise.race([workPromise, timeout.promise]);} finally {+ finished = true;+ stopObserving?.();if (timer !== undefined) clearTimeout(timer);}};@@ -665,9 +696,13 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {log.info({ reason, pid: process.pid, lockDir }, `[server] shutdown initiated (${reason})`);readinessState = 'draining';const errors: unknown[] = [];- const runStep = async (name: string, work: () => Promise<void>): Promise<void> => {+ const runStep = async (+ name: string,+ work: () => Promise<void>,+ retirementSource?: ServerInstance['hocuspocus'],+ ): Promise<void> => {try {- await withDestroyTimeout(name, work);+ await withDestroyTimeout(name, work, retirementSource);} catch (err) {errors.push(err);log.warn({ err, step: name }, 'bootServer destroy step failed');@@ -734,7 +769,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {);}),);- await runStep('destroyHocuspocus', () => destroyHocuspocus());+ await runStep('destroyHocuspocus', () => destroyHocuspocus(), hocuspocus);await runStep('shutdownTelemetry', () => shutdownTelemetry());await runStep('teardownToleranceTelemetry', () => teardownToleranceTelemetryWriter());await runStep('flushLogFileSinks', () => loggerFactory.flushAllFileSinks());diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex b697b4597..3e89680d6 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -380,6 +380,25 @@ export interface ServerInstance {readonly acpPermissions: AcpPermissionStore;}+export function observeShutdownDocumentRetirements(+ hocuspocus: Hocuspocus,+ onDocumentRetired: () => void,+): () => void {+ let active = true;+ const observer: Extension = {+ async afterUnloadDocument() {+ if (active) onDocumentRetired();+ },+ };+ hocuspocus.configuration.extensions.unshift(observer);+ return () => {+ active = false;+ const extensions = hocuspocus.configuration.extensions;+ const index = extensions.indexOf(observer);+ if (index !== -1) extensions.splice(index, 1);+ };+}+export const SHADOW_FANOUT_WARMUP_MS = 3000;const PARK_SNAPSHOT_ORIGIN = (() => {@@ -2992,117 +3011,138 @@ export function createServer(options: ServerOptions): ServerInstance {async function flushAllStoresAndWait(timeoutMs: number): Promise<void> {if (hocuspocus.documents.size === 0) return;- let resolved = false;- const allDone = new Promise<void>((resolve) => {- hocuspocus.configuration.extensions.push({- async afterUnloadDocument({ instance }) {- if (!resolved && instance.getDocumentsCount() === 0) {- resolved = true;- resolve();- }- },- });+ let settled = false;+ let flushDeadline = Date.now() + timeoutMs;+ const raceFloorMs = 250;+ let timeoutState: { id: ReturnType<typeof setTimeout>; expire: () => void } | undefined;+ const allDone = Promise.withResolvers<void>();+ const stopObserving = observeShutdownDocumentRetirements(hocuspocus, () => {+ if (settled) return;+ flushDeadline = Date.now() + timeoutMs;+ if (timeoutState) {+ clearTimeout(timeoutState.id);+ timeoutState.id = setTimeout(+ timeoutState.expire,+ Math.max(flushDeadline - Date.now(), raceFloorMs),+ );+ }});+ const allDoneExtension: Extension = {+ async afterUnloadDocument({ instance }) {+ if (!settled && instance.getDocumentsCount() === 0) {+ settled = true;+ allDone.resolve();+ }+ },+ };+ hocuspocus.configuration.extensions.push(allDoneExtension);- const pendingDocNames = Array.from(hocuspocus.documents.keys());+ try {+ const pendingDocNames = Array.from(hocuspocus.documents.keys());- hocuspocus.closeConnections();- hocuspocus.flushPendingStores();+ hocuspocus.closeConnections();+ hocuspocus.flushPendingStores();- for (const doc of hocuspocus.documents.values()) {- if (doc.getConnectionsCount() === 0) {- void hocuspocus.unloadDocument(doc).catch((err: unknown) => {- console.warn(- JSON.stringify({- event: 'ok-shutdown-unload-document-failed',- docName: doc.name,- reason: err instanceof Error ? err.message : String(err),- }),- );- });+ for (const doc of hocuspocus.documents.values()) {+ if (doc.getConnectionsCount() === 0) {+ void hocuspocus.unloadDocument(doc).catch((err: unknown) => {+ console.warn(+ JSON.stringify({+ event: 'ok-shutdown-unload-document-failed',+ docName: doc.name,+ reason: err instanceof Error ? err.message : String(err),+ }),+ );+ });+ }}- }- const flushDeadline = Date.now() + timeoutMs;- const raceFloorMs = 250;- while (Date.now() < flushDeadline - raceFloorMs) {- const unsettled = Array.from(hocuspocus.documents.values()).some(- (doc) => doc.getConnectionsCount() === 0 && !defaultShouldUnloadDocument(doc),- );- if (!unsettled) break;- await new Promise((r) => setTimeout(r, 25));- }-- const perDocUnloadFloorMs = 50;- for (const docName of durabilityState.getRefusedStoreDocNames()) {- if (isReservedForUserTree(docName)) continue;- const doc = hocuspocus.documents.get(docName);- if (!doc) continue;- if (!defaultShouldUnloadDocument(doc)) continue;- if (flushDeadline - Date.now() < perDocUnloadFloorMs) {- log.warn(- { docName },- `[rescue] refused-store pass out of budget before ${docName}; leaving it and any remaining refused docs to the flush timeout path`,+ while (Date.now() < flushDeadline - raceFloorMs) {+ const unsettled = Array.from(hocuspocus.documents.values()).some(+ (doc) => doc.getConnectionsCount() === 0 && !defaultShouldUnloadDocument(doc),);- break;+ if (!unsettled) break;+ await new Promise((r) => setTimeout(r, 25));}- if (rescueDocToShadowBuffer(docName, 'refused-store-shutdown') !== 'rescued') {- log.warn(- { docName },- `[rescue] refused-store doc rescue failed; leaving ${docName} to the flush timeout path`,- );- continue;- }- let unloadSettled = false;- let unloadFailed = false;- const unload = forceUnloadDocument(doc)- .then(() => {- unloadSettled = true;- })- .catch((err: unknown) => {- unloadFailed = true;- unloadSettled = true;++ const perDocUnloadFloorMs = 50;+ for (const docName of durabilityState.getRefusedStoreDocNames()) {+ if (isReservedForUserTree(docName)) continue;+ const doc = hocuspocus.documents.get(docName);+ if (!doc) continue;+ if (!defaultShouldUnloadDocument(doc)) continue;+ if (flushDeadline - Date.now() < perDocUnloadFloorMs) {log.warn(- { docName, err },- `[rescue] refused-store doc unload failed at shutdown: ${docName}`,+ { docName },+ `[rescue] refused-store pass out of budget before ${docName}; leaving it and any remaining refused docs to the flush timeout path`,);- });- const unloadDeadline = Math.max(flushDeadline - Date.now(), perDocUnloadFloorMs);- await new Promise<void>((resolve) => {- const timer = setTimeout(resolve, unloadDeadline);- void unload.then(() => {- clearTimeout(timer);- resolve();- });- });- if (!unloadSettled) {- log.warn(- { docName, unloadDeadline },- `[rescue] refused-store doc unload did not finish before the flush deadline; leaving ${docName} to the flush timeout path`,- );- void unload.then(() => {- if (!unloadFailed) {- log.info(- { docName },- `[rescue] refused-store doc unload completed after the flush deadline: ${docName}`,+ break;+ }+ if (rescueDocToShadowBuffer(docName, 'refused-store-shutdown') !== 'rescued') {+ log.warn(+ { docName },+ `[rescue] refused-store doc rescue failed; leaving ${docName} to the flush timeout path`,+ );+ continue;+ }+ let unloadSettled = false;+ let unloadFailed = false;+ const unload = forceUnloadDocument(doc)+ .then(() => {+ unloadSettled = true;+ })+ .catch((err: unknown) => {+ unloadFailed = true;+ unloadSettled = true;+ log.warn(+ { docName, err },+ `[rescue] refused-store doc unload failed at shutdown: ${docName}`,);+ });+ let unloadDeadline = 0;+ do {+ const unloadSlice = Math.max(flushDeadline - Date.now(), perDocUnloadFloorMs);+ unloadDeadline += unloadSlice;+ let waitTimer: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ unload,+ new Promise<void>((resolve) => {+ waitTimer = setTimeout(resolve, unloadSlice);+ }),+ ]);+ } finally {+ if (waitTimer !== undefined) clearTimeout(waitTimer);}- });- continue;- }- if (!unloadFailed) {- log.info(- { docName },- `[rescue] refused-store doc rescued and unloaded at shutdown: ${docName}`,- );+ } while (!unloadSettled && Date.now() < flushDeadline);+ if (!unloadSettled) {+ log.warn(+ { docName, unloadDeadline },+ `[rescue] refused-store doc unload did not finish before the flush deadline; leaving ${docName} to the flush timeout path`,+ );+ void unload.then(() => {+ if (!unloadFailed) {+ log.info(+ { docName },+ `[rescue] refused-store doc unload completed after the flush deadline: ${docName}`,+ );+ }+ });+ continue;+ }+ if (!unloadFailed) {+ log.info(+ { docName },+ `[rescue] refused-store doc rescued and unloaded at shutdown: ${docName}`,+ );+ }}- }- let timeoutId: ReturnType<typeof setTimeout> | undefined;- const timeout = new Promise<void>((_, reject) => {- timeoutId = setTimeout(- () => {- resolved = true;+ const timeout = new Promise<never>((_, reject) => {+ const expire = () => {+ if (settled) return;+ settled = true;+ stopObserving();const stillLoaded = Array.from(hocuspocus.documents.keys());const rescued: string[] = [];@@ -3137,18 +3177,24 @@ export function createServer(options: ServerOptions): ServerInstance {reject(new Error(- `flushAllStoresAndWait timeout after ${timeoutMs}ms — ${stillLoaded.length}/${pendingDocNames.length} docs did not unload: [${stillLoaded.join(', ')}]${rescueSummary}`,+ `flushAllStoresAndWait timeout: no document retired for ${timeoutMs}ms — ${stillLoaded.length}/${pendingDocNames.length} docs did not unload: [${stillLoaded.join(', ')}]${rescueSummary}`,),);- },- Math.max(flushDeadline - Date.now(), raceFloorMs),- );- });+ };+ timeoutState = {+ id: setTimeout(expire, Math.max(flushDeadline - Date.now(), raceFloorMs)),+ expire,+ };+ });- try {- await Promise.race([allDone, timeout]);+ await Promise.race([allDone.promise, timeout]);} finally {- if (timeoutId !== undefined) clearTimeout(timeoutId);+ settled = true;+ stopObserving();+ const extensions = hocuspocus.configuration.extensions;+ const index = extensions.indexOf(allDoneExtension);+ if (index !== -1) extensions.splice(index, 1);+ if (timeoutState) clearTimeout(timeoutState.id);}}diff --git a/packages/server/src/shutdown-progress-refused.test.ts b/packages/server/src/shutdown-progress-refused.test.tsnew file mode 100644index 000000000..d6e1eb8a1--- /dev/null+++ b/packages/server/src/shutdown-progress-refused.test.ts@@ -0,0 +1,331 @@+import { ChildProcess } from 'node:child_process';+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { mkdir, mkdtemp, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { expect, test, vi } from 'vitest';+import * as Y from 'yjs';+import { getLogger } from './logger.ts';+import { ensureProjectGit } from './project-git.ts';+import { createServer, type ServerInstance } from './server-factory.ts';+import { initShadowRepo } from './shadow-repo.ts';++const homeState = vi.hoisted(() => ({ path: undefined as string | undefined }));++vi.mock('node:os', async (importOriginal) => {+ const actual = await importOriginal<typeof import('node:os')>();+ return { ...actual, homedir: () => homeState.path ?? actual.homedir() };+});++const normalName = 'normal-shutdown-doc';+const refusedName = 'refused-shutdown-doc';+const innerBudget = 10_000;+const unsavedMarker = 'Unflushed content held through shutdown.';++async function executePendingStore(server: ServerInstance, docName: string): Promise<void> {+ const debounceId = `onStoreDocument-${docName}`;+ expect(server.hocuspocus.debouncer.isDebounced(debounceId)).toBe(true);+ const pending = server.hocuspocus.debouncer.executeNow(debounceId);+ expect(pending).toBeDefined();+ await pending;+}++async function withRefusedStoreShutdown(+ run: (fixture: {+ server: ServerInstance;+ gates: Map<string, PromiseWithResolvers<void>>;+ entered: Map<string, PromiseWithResolvers<void>>;+ unloaded: Map<string, PromiseWithResolvers<void>>;+ observations: Array<{ event: string; loaded: string[] }>;+ rescuePath: string;+ flushBegan: Promise<void>;+ forcedRefusedUnloadBegan: Promise<void>;+ blockedSignalRequests: Array<{+ pid: number | undefined;+ signal: string | number | undefined;+ }>;+ }) => Promise<void>,+): Promise<void> {+ const blockedSignalRequests: Array<{+ pid: number | undefined;+ signal: string | number | undefined;+ }> = [];+ vi.spyOn(process, 'kill').mockImplementation((pid, signal) => {+ blockedSignalRequests.push({ pid, signal });+ throw new Error('Test signal seam refuses delivery');+ });+ vi.spyOn(ChildProcess.prototype, 'kill').mockImplementation(function (+ this: ChildProcess,+ signal,+ ) {+ blockedSignalRequests.push({ pid: this.pid, signal });+ return false;+ });+ const homeDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-refused-home-'));+ const projectDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-refused-'));+ homeState.path = homeDir;+ const contentDir = join(projectDir, 'content');+ const refusedPath = join(contentDir, `${refusedName}.md`);+ const gates = new Map(+ [normalName, refusedName].map((name) => [name, Promise.withResolvers<void>()]),+ );+ const entered = new Map(+ [normalName, refusedName].map((name) => [name, Promise.withResolvers<void>()]),+ );+ const unloaded = new Map(+ [normalName, refusedName].map((name) => [name, Promise.withResolvers<void>()]),+ );+ const observations: Array<{ event: string; loaded: string[] }> = [];+ let server: ServerInstance | undefined;+ try {+ await mkdir(contentDir, { recursive: true });+ await ensureProjectGit(projectDir);+ const shadowHandle = await initShadowRepo(projectDir);+ const refusedInitial = '# Refused shutdown doc\n\nPersisted paragraph.\n';+ writeFileSync(refusedPath, refusedInitial, 'utf-8');+ writeFileSync(join(contentDir, `${normalName}.md`), '# Normal shutdown doc\n', 'utf-8');+ writeFileSync(+ join(projectDir, '.okignore'),+ `content/${normalName}.md\ncontent/${refusedName}.md\n`,+ 'utf-8',+ );+ server = createServer({+ contentDir,+ projectDir,+ quiet: true,+ shadowRepo: shadowHandle,+ });+ await server.ready;+ const hp = server.hocuspocus;+ const refusedConnection = await hp.openDirectConnection(refusedName);+ const refusedDoc = hp.documents.get(refusedName);+ expect(refusedDoc).toBeDefined();+ if (!refusedDoc) throw new Error('Refused document was not loaded');+ expect(server.durabilityState.getReconciledBase(refusedName)).toBe(refusedInitial);+ rmSync(refusedPath);+ mkdirSync(refusedPath);+ server.durabilityState.deleteReconciledBase(refusedName);+ await refusedConnection.transact((doc) => {+ const paragraph = new Y.XmlElement('paragraph');+ paragraph.insert(0, [new Y.XmlText(unsavedMarker)]);+ doc.getXmlFragment('default').insert(0, [paragraph]);+ });+ await executePendingStore(server, refusedName);+ expect(server.durabilityState.isStoreRefused(refusedName)).toBe(true);+ const rescuePath = join(shadowHandle.gitDir, 'rescue', `${refusedName}.md`);+ expect(existsSync(rescuePath)).toBe(true);+ expect(readFileSync(rescuePath, 'utf-8')).toContain(unsavedMarker);++ await hp.openDirectConnection(normalName);+ const normalDoc = hp.documents.get(normalName);+ expect(normalDoc).toBeDefined();+ if (!normalDoc) throw new Error('Normal document was not loaded');+ hp.configuration.extensions.push({+ beforeUnloadDocument({ documentName }) {+ entered.get(documentName)?.resolve();+ return gates.get(documentName)?.promise ?? Promise.resolve();+ },+ afterUnloadDocument({ documentName }) {+ if (unloaded.has(documentName)) {+ observations.push({+ event: documentName,+ loaded: [normalName, refusedName].filter((name) => hp.documents.has(name)),+ });+ unloaded.get(documentName)?.resolve();+ }+ return Promise.resolve();+ },+ });+ refusedDoc.removeDirectConnection();+ normalDoc.removeDirectConnection();+ expect(hp.documents.has(refusedName)).toBe(true);+ expect(hp.documents.has(normalName)).toBe(true);++ const flushBegan = Promise.withResolvers<void>();+ const forcedRefusedUnloadBegan = Promise.withResolvers<void>();+ let flushObserved = false;+ const flushPendingStores = hp.flushPendingStores.bind(hp);+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {+ const result = flushPendingStores();+ flushObserved = true;+ flushBegan.resolve();+ return result;+ });+ const shouldUnloadDocument = hp.shouldUnloadDocument.bind(hp);+ vi.spyOn(hp, 'shouldUnloadDocument').mockImplementation((document) => {+ const result = shouldUnloadDocument(document);+ if (flushObserved && document.name === refusedName && result) {+ forcedRefusedUnloadBegan.resolve();+ }+ return result;+ });+ await run({+ server,+ gates,+ entered,+ unloaded,+ observations,+ rescuePath,+ flushBegan: flushBegan.promise,+ forcedRefusedUnloadBegan: forcedRefusedUnloadBegan.promise,+ blockedSignalRequests,+ });+ } finally {+ for (const gate of gates.values()) gate.resolve();+ vi.useRealTimers();+ await server?.destroy();+ vi.restoreAllMocks();+ homeState.path = undefined;+ await rm(projectDir, { recursive: true, force: true });+ await rm(homeDir, { recursive: true, force: true });+ }+}++test('factory shutdown keeps a refused-store unload pending while other documents retire', async () => {+ await withRefusedStoreShutdown(+ async ({+ server,+ gates,+ entered,+ unloaded,+ observations,+ rescuePath,+ flushBegan,+ forcedRefusedUnloadBegan,+ blockedSignalRequests,+ }) => {+ const hp = server.hocuspocus;+ const warnings = vi.spyOn(getLogger('server'), 'warn');+ const errors = vi.spyOn(getLogger('server'), 'error');+ const refusedDeadlineWarnings = () =>+ warnings.mock.calls.filter(([fields, message]) => {+ if (!fields || typeof fields !== 'object') return false;+ return (+ (fields as { docName?: unknown }).docName === refusedName &&+ typeof message === 'string' &&+ message.includes('did not finish before the flush deadline')+ );+ });+ const flushTimeoutReports = () =>+ errors.mock.calls.filter(([fields]) => {+ if (!fields || typeof fields !== 'object') return false;+ const report = fields as { err?: unknown };+ return (+ report.err instanceof Error &&+ report.err.message.startsWith('flushAllStoresAndWait timeout')+ );+ });++ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ const destroyed = server.destroy().then(+ () => ({ status: 'fulfilled' as const }),+ (error: unknown) => ({ status: 'rejected' as const, error }),+ );+ await flushBegan;+ await forcedRefusedUnloadBegan;+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ expect(server.durabilityState.isStoreRefused(refusedName)).toBe(true);+ expect(hp.documents.has(refusedName)).toBe(true);+ await vi.advanceTimersByTimeAsync(innerBudget * 0.4);+ gates.get(normalName)?.resolve();+ await unloaded.get(normalName)?.promise;+ expect(observations).toEqual([{ event: normalName, loaded: [refusedName] }]);+ await vi.advanceTimersByTimeAsync(innerBudget * 0.8);+ expect(blockedSignalRequests).toEqual([]);+ expect(hp.documents.has(refusedName)).toBe(true);+ expect(refusedDeadlineWarnings()).toEqual([]);+ expect(flushTimeoutReports()).toEqual([]);+ gates.get(refusedName)?.resolve();+ await unloaded.get(refusedName)?.promise;+ vi.useRealTimers();+ const result = await destroyed;+ expect(result).toEqual({ status: 'fulfilled' });+ expect(observations.map((entry) => entry.loaded.length)).toEqual([1, 0]);+ expect(refusedDeadlineWarnings()).toEqual([]);+ expect(flushTimeoutReports()).toEqual([]);+ expect(readFileSync(rescuePath, 'utf-8')).toContain(unsavedMarker);+ expect(blockedSignalRequests).toEqual([]);+ },+ );+});++test('factory shutdown bounds a refused-store unload that stalls after other documents retire', async () => {+ await withRefusedStoreShutdown(+ async ({+ server,+ gates,+ entered,+ unloaded,+ observations,+ rescuePath,+ flushBegan,+ forcedRefusedUnloadBegan,+ blockedSignalRequests,+ }) => {+ const hp = server.hocuspocus;+ const serverLogger = getLogger('server');+ const error = serverLogger.error.bind(serverLogger);+ const isFlushTimeoutReport = (fields: unknown) => {+ if (!fields || typeof fields !== 'object') return false;+ const report = fields as { err?: unknown };+ return (+ report.err instanceof Error &&+ report.err.message.startsWith('flushAllStoresAndWait timeout')+ );+ };+ const warnings = vi.spyOn(serverLogger, 'warn');+ const errors = vi.spyOn(serverLogger, 'error').mockImplementation((fields, message) => {+ const result = error(fields, message);+ if (isFlushTimeoutReport(fields)) vi.useRealTimers();+ return result;+ });+ const refusedDeadlineWarnings = () =>+ warnings.mock.calls.filter(([fields, message]) => {+ if (!fields || typeof fields !== 'object') return false;+ return (+ (fields as { docName?: unknown }).docName === refusedName &&+ typeof message === 'string' &&+ message.includes('did not finish before the flush deadline')+ );+ });+ const flushTimeoutReports = () =>+ errors.mock.calls.filter(([fields]) => isFlushTimeoutReport(fields));++ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ const destroyed = server.destroy().then(+ () => ({ status: 'fulfilled' as const }),+ (error: unknown) => ({ status: 'rejected' as const, error }),+ );+ await flushBegan;+ await forcedRefusedUnloadBegan;+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ expect(server.durabilityState.isStoreRefused(refusedName)).toBe(true);+ expect(hp.documents.has(refusedName)).toBe(true);+ const normalRetiresAfter = innerBudget * 0.4;+ await vi.advanceTimersByTimeAsync(normalRetiresAfter);+ gates.get(normalName)?.resolve();+ await unloaded.get(normalName)?.promise;+ expect(observations).toEqual([{ event: normalName, loaded: [refusedName] }]);+ await vi.advanceTimersByTimeAsync(innerBudget * 0.8);+ expect(refusedDeadlineWarnings()).toEqual([]);+ expect(flushTimeoutReports()).toEqual([]);+ expect(hp.documents.has(refusedName)).toBe(true);+ await vi.advanceTimersByTimeAsync(innerBudget);+ expect(refusedDeadlineWarnings()).toHaveLength(1);+ const deadlineFields = refusedDeadlineWarnings()[0]?.[0] as+ | { unloadDeadline?: number }+ | undefined;+ expect(deadlineFields?.unloadDeadline).toBe(normalRetiresAfter + innerBudget);+ expect(flushTimeoutReports()).toHaveLength(1);+ expect(hp.documents.has(refusedName)).toBe(true);+ expect(blockedSignalRequests).toEqual([]);+ gates.get(refusedName)?.resolve();+ await unloaded.get(refusedName)?.promise;+ vi.useRealTimers();+ const result = await destroyed;+ expect(result).toEqual({ status: 'fulfilled' });+ expect(readFileSync(rescuePath, 'utf-8')).toContain(unsavedMarker);+ expect(blockedSignalRequests).toEqual([]);+ },+ );+});diff --git a/packages/server/src/shutdown-progress.test.ts b/packages/server/src/shutdown-progress.test.tsnew file mode 100644index 000000000..733344dff--- /dev/null+++ b/packages/server/src/shutdown-progress.test.ts@@ -0,0 +1,536 @@+import { ChildProcess } from 'node:child_process';+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { expect, test, vi } from 'vitest';+import { bootServer } from './boot.ts';+import { ConfigSchema } from './config/schema.ts';+import { getLogger } from './logger.ts';+import { ensureProjectGit } from './project-git.ts';++const homeState = vi.hoisted(() => ({ path: undefined as string | undefined }));++vi.mock('node:os', async (importOriginal) => {+ const actual = await importOriginal<typeof import('node:os')>();+ return { ...actual, homedir: () => homeState.path ?? actual.homedir() };+});++const names = ['first', 'second', 'third'];+const outerBudget = Number(process.env.OK_DESTROY_STEP_TIMEOUT_MS) || 5000;+const innerBudget = 10_000;++function captureShutdownReports() {+ const firstReport = Promise.withResolvers<'outer' | 'inner'>();+ const bootLogger = getLogger('boot');+ const serverLogger = getLogger('server');+ const warn = bootLogger.warn.bind(bootLogger);+ const error = serverLogger.error.bind(serverLogger);+ const isOuterTimeout = (fields: unknown) => {+ if (!fields || typeof fields !== 'object') return false;+ const report = fields as { step?: unknown; err?: unknown };+ return (+ report.step === 'destroyHocuspocus' &&+ report.err instanceof Error &&+ report.err.message === `destroyHocuspocus timed out: no document retired for ${outerBudget}ms`+ );+ };+ const isInnerTimeout = (fields: unknown, message: unknown) => {+ if (!fields || typeof fields !== 'object') return false;+ const report = fields as { err?: unknown };+ return (+ message === '[server] shutdown phase-3 flush failed' &&+ report.err instanceof Error &&+ report.err.message.startsWith(+ `flushAllStoresAndWait timeout: no document retired for ${innerBudget}ms`,+ )+ );+ };+ const bootWarnings = vi.spyOn(bootLogger, 'warn').mockImplementation((fields, message) => {+ const result = warn(fields, message);+ if (isOuterTimeout(fields)) {+ vi.useRealTimers();+ firstReport.resolve('outer');+ }+ return result;+ });+ const factoryErrors = vi.spyOn(serverLogger, 'error').mockImplementation((fields, message) => {+ const result = error(fields, message);+ if (isInnerTimeout(fields, message)) {+ vi.useRealTimers();+ firstReport.resolve('inner');+ }+ return result;+ });+ return {+ firstReport: firstReport.promise,+ outer: () => bootWarnings.mock.calls.filter(([fields]) => isOuterTimeout(fields)),+ inner: () =>+ factoryErrors.mock.calls.filter(([fields, message]) => isInnerTimeout(fields, message)),+ };+}++async function withLoadedDocuments(+ run: (fixture: {+ booted: Awaited<ReturnType<typeof bootServer>>;+ gates: Map<string, PromiseWithResolvers<void>>;+ entered: Map<string, PromiseWithResolvers<void>>;+ unloaded: Map<string, PromiseWithResolvers<void>>;+ observations: Array<{ event: string; loaded: string[] }>;+ retirements: string[];+ acknowledgeBootRejection: () => void;+ blockedSignalRequests: Array<{+ pid: number | undefined;+ signal: string | number | undefined;+ }>;+ }) => Promise<void>,+): Promise<void> {+ const blockedSignalRequests: Array<{+ pid: number | undefined;+ signal: string | number | undefined;+ }> = [];+ vi.spyOn(process, 'kill').mockImplementation((pid, signal) => {+ blockedSignalRequests.push({ pid, signal });+ throw new Error('Test signal seam refuses delivery');+ });+ vi.spyOn(ChildProcess.prototype, 'kill').mockImplementation(function (+ this: ChildProcess,+ signal,+ ) {+ blockedSignalRequests.push({ pid: this.pid, signal });+ return false;+ });+ const homeDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-home-'));+ const projectDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-progress-'));+ homeState.path = homeDir;+ let booted: Awaited<ReturnType<typeof bootServer>> | undefined;+ let startedBootDestroy: Promise<void> | undefined;+ let bootRejectionAcknowledged = false;+ const gates = new Map(names.map((name) => [name, Promise.withResolvers<void>()]));+ const entered = new Map(names.map((name) => [name, Promise.withResolvers<void>()]));+ const unloaded = new Map(names.map((name) => [name, Promise.withResolvers<void>()]));+ const observations: Array<{ event: string; loaded: string[] }> = [];+ const retirements: string[] = [];+ let bodyError: unknown;+ let bodyFailed = false;+ const cleanupErrors: unknown[] = [];+ try {+ await ensureProjectGit(projectDir);+ await mkdir(join(projectDir, '.ok'), { recursive: true });+ await writeFile(join(projectDir, '.ok', 'config.yml'), '');+ for (const name of names) await writeFile(join(projectDir, `${name}.md`), `# ${name}\n`);+ booted = await bootServer({+ host: '127.0.0.1',+ config: ConfigSchema.parse({}),+ contentDir: projectDir,+ configHomedirOverride: homeDir,+ port: 0,+ quiet: true,+ gitEnabled: false,+ idleShutdownMs: null,+ ephemeral: true,+ });+ await booted.ready;+ const destroyBoot = booted.destroy.bind(booted);+ booted.destroy = (reason) => {+ const inFlight = destroyBoot(reason);+ startedBootDestroy ??= inFlight;+ return inFlight;+ };+ const hp = booted.serverInstance.hocuspocus;+ for (const name of names) {+ const connection = await hp.openDirectConnection(name);+ await connection.disconnect();+ }+ expect(names.every((name) => hp.documents.has(name))).toBe(true);+ hp.configuration.extensions.push({+ beforeUnloadDocument({ documentName }) {+ entered.get(documentName)?.resolve();+ return gates.get(documentName)?.promise ?? Promise.resolve();+ },+ afterUnloadDocument({ documentName }) {+ retirements.push(documentName);+ if (unloaded.has(documentName)) {+ observations.push({+ event: documentName,+ loaded: names.filter((name) => hp.documents.has(name)),+ });+ unloaded.get(documentName)?.resolve();+ }+ return Promise.resolve();+ },+ });+ await run({+ booted,+ gates,+ entered,+ unloaded,+ observations,+ retirements,+ acknowledgeBootRejection: () => {+ bootRejectionAcknowledged = true;+ },+ blockedSignalRequests,+ });+ } catch (error) {+ bodyError = error;+ bodyFailed = true;+ } finally {+ for (const gate of gates.values()) gate.resolve();+ try {+ await booted?.serverInstance.destroy();+ } catch (error) {+ cleanupErrors.push(error);+ }+ try {+ vi.useRealTimers();+ } catch (error) {+ cleanupErrors.push(error);+ }+ try {+ if (startedBootDestroy) await startedBootDestroy;+ else await booted?.destroy();+ } catch (error) {+ if (!bootRejectionAcknowledged) cleanupErrors.push(error);+ }+ try {+ vi.restoreAllMocks();+ } catch (error) {+ cleanupErrors.push(error);+ }+ homeState.path = undefined;+ try {+ await rm(projectDir, { recursive: true, force: true });+ } catch (error) {+ cleanupErrors.push(error);+ }+ try {+ await rm(homeDir, { recursive: true, force: true });+ } catch (error) {+ cleanupErrors.push(error);+ }+ }+ if (bodyFailed) {+ if (cleanupErrors.length > 0) {+ throw new AggregateError(+ [bodyError, ...cleanupErrors],+ 'Shutdown fixture and cleanup failed',+ );+ }+ throw bodyError;+ }+ if (cleanupErrors.length > 0) {+ throw cleanupErrors.length === 1+ ? cleanupErrors[0]+ : new AggregateError(cleanupErrors, 'Shutdown fixture cleanup failed');+ }+}++test('destroy accepts real document unload progress across the outer deadline', async () => {+ await withLoadedDocuments(+ async ({ booted, gates, entered, unloaded, observations, blockedSignalRequests }) => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ const destroyed = booted.destroy().then(+ () => ({ status: 'fulfilled' as const }),+ (error: unknown) => ({ status: 'rejected' as const, error }),+ );+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ for (const name of names) {+ await vi.advanceTimersByTimeAsync(outerBudget * 0.4);+ gates.get(name)?.resolve();+ await unloaded.get(name)?.promise;+ }+ vi.useRealTimers();+ const result = await destroyed;+ await booted.serverInstance.destroy();+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);+ expect(blockedSignalRequests).toEqual([]);+ expect(result).toEqual({ status: 'fulfilled' });+ },+ );+});++test('destroy reports a held real document unload without waiting for its release', async () => {+ await withLoadedDocuments(+ async ({+ booted,+ gates,+ entered,+ unloaded,+ observations,+ retirements,+ acknowledgeBootRejection,+ blockedSignalRequests,+ }) => {+ const hp = booted.serverInstance.hocuspocus;+ const flushBegan = Promise.withResolvers<void>();+ const flushPendingStores = hp.flushPendingStores.bind(hp);+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {+ const result = flushPendingStores();+ flushBegan.resolve();+ return result;+ });+ const reports = captureShutdownReports();+ const stallWindow = Math.min(outerBudget, innerBudget);+ const outerFirst = outerBudget < innerBudget;+ expect(outerBudget).not.toBe(innerBudget);+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ const destroyed = booted.destroy().then(+ () => ({ status: 'fulfilled' as const }),+ (error: unknown) => ({ status: 'rejected' as const, error }),+ );+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ await flushBegan.promise;+ expect([...hp.documents.keys()].sort()).toEqual([...names].sort());+ const retirementsAtFlush = retirements.length;+ await vi.advanceTimersByTimeAsync(stallWindow);+ if (outerFirst) {+ expect(reports.outer()).toHaveLength(1);+ expect(reports.inner()).toEqual([]);+ } else {+ expect(reports.inner()).toHaveLength(1);+ expect(reports.outer()).toEqual([]);+ }+ expect(await reports.firstReport).toBe(outerFirst ? 'outer' : 'inner');+ const result = await destroyed;+ const loadedBeforeRelease = names.filter((name) => hp.documents.has(name));+ expect(loadedBeforeRelease).toEqual(names);+ expect(observations).toEqual([]);+ expect(retirements.slice(retirementsAtFlush)).toEqual([]);+ if (outerFirst) {+ expect(reports.outer()).toHaveLength(1);+ expect(reports.inner()).toEqual([]);+ expect(result).toEqual({ status: 'rejected', error: expect.any(AggregateError) });+ expect(result).toMatchObject({+ error: {+ errors: [+ expect.objectContaining({+ message: `destroyHocuspocus timed out: no document retired for ${outerBudget}ms`,+ }),+ ],+ },+ });+ acknowledgeBootRejection();+ } else {+ expect(reports.outer()).toEqual([]);+ expect(reports.inner()).toHaveLength(1);+ expect(reports.inner()[0]?.[0]).toMatchObject({+ err: expect.objectContaining({+ message: expect.stringContaining('3/3 docs did not unload: [first, second, third]'),+ }),+ });+ expect(reports.inner()[0]?.[0]).toMatchObject({+ err: expect.objectContaining({+ message: expect.stringContaining('rescued [first, second, third]'),+ }),+ });+ expect(result).toEqual({ status: 'fulfilled' });+ }+ for (const name of names) {+ gates.get(name)?.resolve();+ await unloaded.get(name)?.promise;+ }+ vi.useRealTimers();+ await booted.serverInstance.destroy();+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);+ expect(blockedSignalRequests).toEqual([]);+ },+ );+});++test('destroy bounds a stall after a real document unload', async () => {+ await withLoadedDocuments(+ async ({+ booted,+ gates,+ entered,+ unloaded,+ observations,+ retirements,+ acknowledgeBootRejection,+ blockedSignalRequests,+ }) => {+ const hp = booted.serverInstance.hocuspocus;+ const flushBegan = Promise.withResolvers<void>();+ const flushPendingStores = hp.flushPendingStores.bind(hp);+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {+ const result = flushPendingStores();+ flushBegan.resolve();+ return result;+ });+ const reports = captureShutdownReports();+ const stallWindow = Math.min(outerBudget, innerBudget);+ const outerFirst = outerBudget < innerBudget;+ expect(outerBudget).not.toBe(innerBudget);+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ let settled = false;+ const destroyed = booted.destroy().then(+ () => {+ settled = true;+ return { status: 'fulfilled' as const };+ },+ (error: unknown) => {+ settled = true;+ return { status: 'rejected' as const, error };+ },+ );+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ await flushBegan.promise;+ expect([...hp.documents.keys()].sort()).toEqual([...names].sort());+ const retirementsAtFlush = retirements.length;+ await vi.advanceTimersByTimeAsync(stallWindow * 0.4);+ gates.get('first')?.resolve();+ await unloaded.get('first')?.promise;+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2]);+ expect(retirements.slice(retirementsAtFlush)).toEqual(['first']);+ await vi.advanceTimersByTimeAsync(stallWindow * 0.8);+ expect(settled).toBe(false);+ expect(reports.outer()).toEqual([]);+ expect(reports.inner()).toEqual([]);+ expect(blockedSignalRequests).toEqual([]);+ expect(names.filter((name) => hp.documents.has(name))).toEqual(['second', 'third']);+ expect(retirements.slice(retirementsAtFlush)).toEqual(['first']);+ await vi.advanceTimersByTimeAsync(stallWindow * 0.2);+ if (outerFirst) {+ expect(reports.outer()).toHaveLength(1);+ expect(reports.inner()).toEqual([]);+ } else {+ expect(reports.inner()).toHaveLength(1);+ expect(reports.outer()).toEqual([]);+ }+ expect(await reports.firstReport).toBe(outerFirst ? 'outer' : 'inner');+ const result = await destroyed;+ if (outerFirst) {+ expect(reports.outer()).toHaveLength(1);+ expect(reports.inner()).toEqual([]);+ expect(result).toEqual({ status: 'rejected', error: expect.any(AggregateError) });+ expect(result).toMatchObject({+ error: {+ errors: [+ expect.objectContaining({+ message: `destroyHocuspocus timed out: no document retired for ${outerBudget}ms`,+ }),+ ],+ },+ });+ acknowledgeBootRejection();+ } else {+ expect(reports.outer()).toEqual([]);+ expect(reports.inner()).toHaveLength(1);+ expect(reports.inner()[0]?.[0]).toMatchObject({+ err: expect.objectContaining({+ message: expect.stringContaining('2/3 docs did not unload: [second, third]'),+ }),+ });+ expect(reports.inner()[0]?.[0]).toMatchObject({+ err: expect.objectContaining({+ message: expect.stringContaining('rescued [second, third]'),+ }),+ });+ expect(result).toEqual({ status: 'fulfilled' });+ }+ for (const name of ['second', 'third']) {+ gates.get(name)?.resolve();+ await unloaded.get(name)?.promise;+ }+ vi.useRealTimers();+ await booted.serverInstance.destroy();+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);+ expect(blockedSignalRequests).toEqual([]);+ },+ );+});++test('factory shutdown does not report continuing real unloads as a flush timeout', async () => {+ await withLoadedDocuments(+ async ({ booted, gates, entered, unloaded, observations, blockedSignalRequests }) => {+ const hp = booted.serverInstance.hocuspocus;+ const flushBegan = Promise.withResolvers<void>();+ const flushPendingStores = hp.flushPendingStores.bind(hp);+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {+ const result = flushPendingStores();+ flushBegan.resolve();+ return result;+ });+ const errors = vi.spyOn(getLogger('server'), 'error');+ const flushTimeoutReports = () =>+ errors.mock.calls.filter(([fields]) => {+ if (!fields || typeof fields !== 'object') return false;+ const report = fields as { err?: unknown };+ return (+ report.err instanceof Error &&+ report.err.message.startsWith('flushAllStoresAndWait timeout')+ );+ });+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ const destroyed = booted.serverInstance.destroy().then(+ () => ({ status: 'fulfilled' as const }),+ (error: unknown) => ({ status: 'rejected' as const, error }),+ );+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ await flushBegan.promise;+ for (const name of names) {+ await vi.advanceTimersByTimeAsync(innerBudget * 0.4);+ gates.get(name)?.resolve();+ await unloaded.get(name)?.promise;+ }+ vi.useRealTimers();+ const result = await destroyed;+ await booted.serverInstance.destroy();+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);+ expect(blockedSignalRequests).toEqual([]);+ expect(result).toEqual({ status: 'fulfilled' });+ expect(flushTimeoutReports()).toEqual([]);+ },+ );+});++test('factory shutdown bounds a stall after a real document unload', async () => {+ await withLoadedDocuments(+ async ({ booted, gates, entered, unloaded, observations, blockedSignalRequests }) => {+ const hp = booted.serverInstance.hocuspocus;+ const flushBegan = Promise.withResolvers<void>();+ const flushPendingStores = hp.flushPendingStores.bind(hp);+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {+ const result = flushPendingStores();+ flushBegan.resolve();+ return result;+ });+ const errors = vi.spyOn(getLogger('server'), 'error');+ const flushTimeoutReports = () =>+ errors.mock.calls.filter(([fields]) => {+ if (!fields || typeof fields !== 'object') return false;+ const report = fields as { err?: unknown };+ return (+ report.err instanceof Error &&+ report.err.message.startsWith('flushAllStoresAndWait timeout')+ );+ });+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ const destroyed = booted.serverInstance.destroy().then(+ () => ({ status: 'fulfilled' as const }),+ (error: unknown) => ({ status: 'rejected' as const, error }),+ );+ await Promise.all([...entered.values()].map((entry) => entry.promise));+ await flushBegan.promise;+ await vi.advanceTimersByTimeAsync(innerBudget * 0.4);+ gates.get('first')?.resolve();+ await unloaded.get('first')?.promise;+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2]);+ await vi.advanceTimersByTimeAsync(innerBudget * 0.8);+ expect(flushTimeoutReports()).toEqual([]);+ expect(blockedSignalRequests).toEqual([]);+ expect(names.filter((name) => hp.documents.has(name))).toEqual(['second', 'third']);+ await vi.advanceTimersByTimeAsync(innerBudget * 0.2);+ expect(flushTimeoutReports()).toHaveLength(1);+ for (const name of ['second', 'third']) {+ gates.get(name)?.resolve();+ await unloaded.get(name)?.promise;+ }+ vi.useRealTimers();+ const result = await destroyed;+ expect(result).toEqual({ status: 'fulfilled' });+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);+ expect(blockedSignalRequests).toEqual([]);+ },+ );+});