Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Keep server shutdown alive while documents retire (#5531)

Nick Gomezaeb3eecce87d
diff --git a/.changeset/progress-aware-shutdown.md b/.changeset/progress-aware-shutdown.md
new file mode 100644
index 000000000..01adf0c18
--- /dev/null
+++ b/.changeset/progress-aware-shutdown.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Avoid false shutdown timeouts while documents continue unloading.
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index 05a7c9bd0..40f219a76 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -54,7 +54,12 @@ import { createMcpHttpHandler } from './mcp-http.ts';
import { mountMcpAndApi, type ReadinessState } from './mcp-mount.ts';
import { MissingOkConfigError } from './missing-ok-config-error.ts';
import { createProjectRuntime, type ProjectRuntime } from './project-runtime.ts';
-import { createServer, type ServerInstance, type ServerOptions } from './server-factory.ts';
+import {
+ createServer,
+ observeShutdownDocumentRetirements,
+ type ServerInstance,
+ type ServerOptions,
+} from './server-factory.ts';
import { installServerMemoryGauge, installServerRuntimeGauges } from './server-memory-telemetry.ts';
import {
genuineInPlaceNames,
@@ -643,19 +648,45 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
);
let destroyed = false;
- const withDestroyTimeout = async (name: string, work: () => Promise<void>): Promise<void> => {
+ const withDestroyTimeout = async (
+ name: string,
+ work: () => Promise<void>,
+ retirementSource?: ServerInstance['hocuspocus'],
+ ): Promise<void> => {
let timer: ReturnType<typeof setTimeout> | undefined;
+ let stopObserving: (() => void) | undefined;
+ let finished = false;
+ const timeout = Promise.withResolvers<never>();
+ const expire = () => {
+ if (finished) return;
+ finished = true;
+ timeout.reject(
+ new Error(
+ retirementSource
+ ? `${name} timed out: no document retired for ${DESTROY_STEP_TIMEOUT_MS}ms`
+ : `${name} timed out after ${DESTROY_STEP_TIMEOUT_MS}ms`,
+ ),
+ );
+ };
+ const arm = () => {
+ timer = setTimeout(expire, DESTROY_STEP_TIMEOUT_MS);
+ timer.unref?.();
+ };
try {
- await Promise.race([
- work(),
- new Promise<never>((_, reject) => {
- timer = setTimeout(() => {
- reject(new Error(`${name} timed out after ${DESTROY_STEP_TIMEOUT_MS}ms`));
- }, DESTROY_STEP_TIMEOUT_MS);
- timer.unref?.();
- }),
- ]);
+ if (retirementSource) {
+ stopObserving = observeShutdownDocumentRetirements(retirementSource, () => {
+ if (finished) return;
+ if (timer !== undefined) clearTimeout(timer);
+ arm();
+ });
+ arm();
+ }
+ const workPromise = work();
+ if (!retirementSource) arm();
+ await Promise.race([workPromise, timeout.promise]);
} finally {
+ finished = true;
+ stopObserving?.();
if (timer !== undefined) clearTimeout(timer);
}
};
@@ -665,9 +696,13 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
log.info({ reason, pid: process.pid, lockDir }, `[server] shutdown initiated (${reason})`);
readinessState = 'draining';
const errors: unknown[] = [];
- const runStep = async (name: string, work: () => Promise<void>): Promise<void> => {
+ const runStep = async (
+ name: string,
+ work: () => Promise<void>,
+ retirementSource?: ServerInstance['hocuspocus'],
+ ): Promise<void> => {
try {
- await withDestroyTimeout(name, work);
+ await withDestroyTimeout(name, work, retirementSource);
} catch (err) {
errors.push(err);
log.warn({ err, step: name }, 'bootServer destroy step failed');
@@ -734,7 +769,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
);
}),
);
- await runStep('destroyHocuspocus', () => destroyHocuspocus());
+ await runStep('destroyHocuspocus', () => destroyHocuspocus(), hocuspocus);
await runStep('shutdownTelemetry', () => shutdownTelemetry());
await runStep('teardownToleranceTelemetry', () => teardownToleranceTelemetryWriter());
await runStep('flushLogFileSinks', () => loggerFactory.flushAllFileSinks());
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index b697b4597..3e89680d6 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -380,6 +380,25 @@ export interface ServerInstance {
readonly acpPermissions: AcpPermissionStore;
}
+export function observeShutdownDocumentRetirements(
+ hocuspocus: Hocuspocus,
+ onDocumentRetired: () => void,
+): () => void {
+ let active = true;
+ const observer: Extension = {
+ async afterUnloadDocument() {
+ if (active) onDocumentRetired();
+ },
+ };
+ hocuspocus.configuration.extensions.unshift(observer);
+ return () => {
+ active = false;
+ const extensions = hocuspocus.configuration.extensions;
+ const index = extensions.indexOf(observer);
+ if (index !== -1) extensions.splice(index, 1);
+ };
+}
+
export const SHADOW_FANOUT_WARMUP_MS = 3000;
const PARK_SNAPSHOT_ORIGIN = (() => {
@@ -2992,117 +3011,138 @@ export function createServer(options: ServerOptions): ServerInstance {
async function flushAllStoresAndWait(timeoutMs: number): Promise<void> {
if (hocuspocus.documents.size === 0) return;
- let resolved = false;
- const allDone = new Promise<void>((resolve) => {
- hocuspocus.configuration.extensions.push({
- async afterUnloadDocument({ instance }) {
- if (!resolved && instance.getDocumentsCount() === 0) {
- resolved = true;
- resolve();
- }
- },
- });
+ let settled = false;
+ let flushDeadline = Date.now() + timeoutMs;
+ const raceFloorMs = 250;
+ let timeoutState: { id: ReturnType<typeof setTimeout>; expire: () => void } | undefined;
+ const allDone = Promise.withResolvers<void>();
+ const stopObserving = observeShutdownDocumentRetirements(hocuspocus, () => {
+ if (settled) return;
+ flushDeadline = Date.now() + timeoutMs;
+ if (timeoutState) {
+ clearTimeout(timeoutState.id);
+ timeoutState.id = setTimeout(
+ timeoutState.expire,
+ Math.max(flushDeadline - Date.now(), raceFloorMs),
+ );
+ }
});
+ const allDoneExtension: Extension = {
+ async afterUnloadDocument({ instance }) {
+ if (!settled && instance.getDocumentsCount() === 0) {
+ settled = true;
+ allDone.resolve();
+ }
+ },
+ };
+ hocuspocus.configuration.extensions.push(allDoneExtension);
- const pendingDocNames = Array.from(hocuspocus.documents.keys());
+ try {
+ const pendingDocNames = Array.from(hocuspocus.documents.keys());
- hocuspocus.closeConnections();
- hocuspocus.flushPendingStores();
+ hocuspocus.closeConnections();
+ hocuspocus.flushPendingStores();
- for (const doc of hocuspocus.documents.values()) {
- if (doc.getConnectionsCount() === 0) {
- void hocuspocus.unloadDocument(doc).catch((err: unknown) => {
- console.warn(
- JSON.stringify({
- event: 'ok-shutdown-unload-document-failed',
- docName: doc.name,
- reason: err instanceof Error ? err.message : String(err),
- }),
- );
- });
+ for (const doc of hocuspocus.documents.values()) {
+ if (doc.getConnectionsCount() === 0) {
+ void hocuspocus.unloadDocument(doc).catch((err: unknown) => {
+ console.warn(
+ JSON.stringify({
+ event: 'ok-shutdown-unload-document-failed',
+ docName: doc.name,
+ reason: err instanceof Error ? err.message : String(err),
+ }),
+ );
+ });
+ }
}
- }
- const flushDeadline = Date.now() + timeoutMs;
- const raceFloorMs = 250;
- while (Date.now() < flushDeadline - raceFloorMs) {
- const unsettled = Array.from(hocuspocus.documents.values()).some(
- (doc) => doc.getConnectionsCount() === 0 && !defaultShouldUnloadDocument(doc),
- );
- if (!unsettled) break;
- await new Promise((r) => setTimeout(r, 25));
- }
-
- const perDocUnloadFloorMs = 50;
- for (const docName of durabilityState.getRefusedStoreDocNames()) {
- if (isReservedForUserTree(docName)) continue;
- const doc = hocuspocus.documents.get(docName);
- if (!doc) continue;
- if (!defaultShouldUnloadDocument(doc)) continue;
- if (flushDeadline - Date.now() < perDocUnloadFloorMs) {
- log.warn(
- { docName },
- `[rescue] refused-store pass out of budget before ${docName}; leaving it and any remaining refused docs to the flush timeout path`,
+ while (Date.now() < flushDeadline - raceFloorMs) {
+ const unsettled = Array.from(hocuspocus.documents.values()).some(
+ (doc) => doc.getConnectionsCount() === 0 && !defaultShouldUnloadDocument(doc),
);
- break;
+ if (!unsettled) break;
+ await new Promise((r) => setTimeout(r, 25));
}
- if (rescueDocToShadowBuffer(docName, 'refused-store-shutdown') !== 'rescued') {
- log.warn(
- { docName },
- `[rescue] refused-store doc rescue failed; leaving ${docName} to the flush timeout path`,
- );
- continue;
- }
- let unloadSettled = false;
- let unloadFailed = false;
- const unload = forceUnloadDocument(doc)
- .then(() => {
- unloadSettled = true;
- })
- .catch((err: unknown) => {
- unloadFailed = true;
- unloadSettled = true;
+
+ const perDocUnloadFloorMs = 50;
+ for (const docName of durabilityState.getRefusedStoreDocNames()) {
+ if (isReservedForUserTree(docName)) continue;
+ const doc = hocuspocus.documents.get(docName);
+ if (!doc) continue;
+ if (!defaultShouldUnloadDocument(doc)) continue;
+ if (flushDeadline - Date.now() < perDocUnloadFloorMs) {
log.warn(
- { docName, err },
- `[rescue] refused-store doc unload failed at shutdown: ${docName}`,
+ { docName },
+ `[rescue] refused-store pass out of budget before ${docName}; leaving it and any remaining refused docs to the flush timeout path`,
);
- });
- const unloadDeadline = Math.max(flushDeadline - Date.now(), perDocUnloadFloorMs);
- await new Promise<void>((resolve) => {
- const timer = setTimeout(resolve, unloadDeadline);
- void unload.then(() => {
- clearTimeout(timer);
- resolve();
- });
- });
- if (!unloadSettled) {
- log.warn(
- { docName, unloadDeadline },
- `[rescue] refused-store doc unload did not finish before the flush deadline; leaving ${docName} to the flush timeout path`,
- );
- void unload.then(() => {
- if (!unloadFailed) {
- log.info(
- { docName },
- `[rescue] refused-store doc unload completed after the flush deadline: ${docName}`,
+ break;
+ }
+ if (rescueDocToShadowBuffer(docName, 'refused-store-shutdown') !== 'rescued') {
+ log.warn(
+ { docName },
+ `[rescue] refused-store doc rescue failed; leaving ${docName} to the flush timeout path`,
+ );
+ continue;
+ }
+ let unloadSettled = false;
+ let unloadFailed = false;
+ const unload = forceUnloadDocument(doc)
+ .then(() => {
+ unloadSettled = true;
+ })
+ .catch((err: unknown) => {
+ unloadFailed = true;
+ unloadSettled = true;
+ log.warn(
+ { docName, err },
+ `[rescue] refused-store doc unload failed at shutdown: ${docName}`,
);
+ });
+ let unloadDeadline = 0;
+ do {
+ const unloadSlice = Math.max(flushDeadline - Date.now(), perDocUnloadFloorMs);
+ unloadDeadline += unloadSlice;
+ let waitTimer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ unload,
+ new Promise<void>((resolve) => {
+ waitTimer = setTimeout(resolve, unloadSlice);
+ }),
+ ]);
+ } finally {
+ if (waitTimer !== undefined) clearTimeout(waitTimer);
}
- });
- continue;
- }
- if (!unloadFailed) {
- log.info(
- { docName },
- `[rescue] refused-store doc rescued and unloaded at shutdown: ${docName}`,
- );
+ } while (!unloadSettled && Date.now() < flushDeadline);
+ if (!unloadSettled) {
+ log.warn(
+ { docName, unloadDeadline },
+ `[rescue] refused-store doc unload did not finish before the flush deadline; leaving ${docName} to the flush timeout path`,
+ );
+ void unload.then(() => {
+ if (!unloadFailed) {
+ log.info(
+ { docName },
+ `[rescue] refused-store doc unload completed after the flush deadline: ${docName}`,
+ );
+ }
+ });
+ continue;
+ }
+ if (!unloadFailed) {
+ log.info(
+ { docName },
+ `[rescue] refused-store doc rescued and unloaded at shutdown: ${docName}`,
+ );
+ }
}
- }
- let timeoutId: ReturnType<typeof setTimeout> | undefined;
- const timeout = new Promise<void>((_, reject) => {
- timeoutId = setTimeout(
- () => {
- resolved = true;
+ const timeout = new Promise<never>((_, reject) => {
+ const expire = () => {
+ if (settled) return;
+ settled = true;
+ stopObserving();
const stillLoaded = Array.from(hocuspocus.documents.keys());
const rescued: string[] = [];
@@ -3137,18 +3177,24 @@ export function createServer(options: ServerOptions): ServerInstance {
reject(
new Error(
- `flushAllStoresAndWait timeout after ${timeoutMs}ms — ${stillLoaded.length}/${pendingDocNames.length} docs did not unload: [${stillLoaded.join(', ')}]${rescueSummary}`,
+ `flushAllStoresAndWait timeout: no document retired for ${timeoutMs}ms — ${stillLoaded.length}/${pendingDocNames.length} docs did not unload: [${stillLoaded.join(', ')}]${rescueSummary}`,
),
);
- },
- Math.max(flushDeadline - Date.now(), raceFloorMs),
- );
- });
+ };
+ timeoutState = {
+ id: setTimeout(expire, Math.max(flushDeadline - Date.now(), raceFloorMs)),
+ expire,
+ };
+ });
- try {
- await Promise.race([allDone, timeout]);
+ await Promise.race([allDone.promise, timeout]);
} finally {
- if (timeoutId !== undefined) clearTimeout(timeoutId);
+ settled = true;
+ stopObserving();
+ const extensions = hocuspocus.configuration.extensions;
+ const index = extensions.indexOf(allDoneExtension);
+ if (index !== -1) extensions.splice(index, 1);
+ if (timeoutState) clearTimeout(timeoutState.id);
}
}
diff --git a/packages/server/src/shutdown-progress-refused.test.ts b/packages/server/src/shutdown-progress-refused.test.ts
new file mode 100644
index 000000000..d6e1eb8a1
--- /dev/null
+++ b/packages/server/src/shutdown-progress-refused.test.ts
@@ -0,0 +1,331 @@
+import { ChildProcess } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdir, mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { expect, test, vi } from 'vitest';
+import * as Y from 'yjs';
+import { getLogger } from './logger.ts';
+import { ensureProjectGit } from './project-git.ts';
+import { createServer, type ServerInstance } from './server-factory.ts';
+import { initShadowRepo } from './shadow-repo.ts';
+
+const homeState = vi.hoisted(() => ({ path: undefined as string | undefined }));
+
+vi.mock('node:os', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('node:os')>();
+ return { ...actual, homedir: () => homeState.path ?? actual.homedir() };
+});
+
+const normalName = 'normal-shutdown-doc';
+const refusedName = 'refused-shutdown-doc';
+const innerBudget = 10_000;
+const unsavedMarker = 'Unflushed content held through shutdown.';
+
+async function executePendingStore(server: ServerInstance, docName: string): Promise<void> {
+ const debounceId = `onStoreDocument-${docName}`;
+ expect(server.hocuspocus.debouncer.isDebounced(debounceId)).toBe(true);
+ const pending = server.hocuspocus.debouncer.executeNow(debounceId);
+ expect(pending).toBeDefined();
+ await pending;
+}
+
+async function withRefusedStoreShutdown(
+ run: (fixture: {
+ server: ServerInstance;
+ gates: Map<string, PromiseWithResolvers<void>>;
+ entered: Map<string, PromiseWithResolvers<void>>;
+ unloaded: Map<string, PromiseWithResolvers<void>>;
+ observations: Array<{ event: string; loaded: string[] }>;
+ rescuePath: string;
+ flushBegan: Promise<void>;
+ forcedRefusedUnloadBegan: Promise<void>;
+ blockedSignalRequests: Array<{
+ pid: number | undefined;
+ signal: string | number | undefined;
+ }>;
+ }) => Promise<void>,
+): Promise<void> {
+ const blockedSignalRequests: Array<{
+ pid: number | undefined;
+ signal: string | number | undefined;
+ }> = [];
+ vi.spyOn(process, 'kill').mockImplementation((pid, signal) => {
+ blockedSignalRequests.push({ pid, signal });
+ throw new Error('Test signal seam refuses delivery');
+ });
+ vi.spyOn(ChildProcess.prototype, 'kill').mockImplementation(function (
+ this: ChildProcess,
+ signal,
+ ) {
+ blockedSignalRequests.push({ pid: this.pid, signal });
+ return false;
+ });
+ const homeDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-refused-home-'));
+ const projectDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-refused-'));
+ homeState.path = homeDir;
+ const contentDir = join(projectDir, 'content');
+ const refusedPath = join(contentDir, `${refusedName}.md`);
+ const gates = new Map(
+ [normalName, refusedName].map((name) => [name, Promise.withResolvers<void>()]),
+ );
+ const entered = new Map(
+ [normalName, refusedName].map((name) => [name, Promise.withResolvers<void>()]),
+ );
+ const unloaded = new Map(
+ [normalName, refusedName].map((name) => [name, Promise.withResolvers<void>()]),
+ );
+ const observations: Array<{ event: string; loaded: string[] }> = [];
+ let server: ServerInstance | undefined;
+ try {
+ await mkdir(contentDir, { recursive: true });
+ await ensureProjectGit(projectDir);
+ const shadowHandle = await initShadowRepo(projectDir);
+ const refusedInitial = '# Refused shutdown doc\n\nPersisted paragraph.\n';
+ writeFileSync(refusedPath, refusedInitial, 'utf-8');
+ writeFileSync(join(contentDir, `${normalName}.md`), '# Normal shutdown doc\n', 'utf-8');
+ writeFileSync(
+ join(projectDir, '.okignore'),
+ `content/${normalName}.md\ncontent/${refusedName}.md\n`,
+ 'utf-8',
+ );
+ server = createServer({
+ contentDir,
+ projectDir,
+ quiet: true,
+ shadowRepo: shadowHandle,
+ });
+ await server.ready;
+ const hp = server.hocuspocus;
+ const refusedConnection = await hp.openDirectConnection(refusedName);
+ const refusedDoc = hp.documents.get(refusedName);
+ expect(refusedDoc).toBeDefined();
+ if (!refusedDoc) throw new Error('Refused document was not loaded');
+ expect(server.durabilityState.getReconciledBase(refusedName)).toBe(refusedInitial);
+ rmSync(refusedPath);
+ mkdirSync(refusedPath);
+ server.durabilityState.deleteReconciledBase(refusedName);
+ await refusedConnection.transact((doc) => {
+ const paragraph = new Y.XmlElement('paragraph');
+ paragraph.insert(0, [new Y.XmlText(unsavedMarker)]);
+ doc.getXmlFragment('default').insert(0, [paragraph]);
+ });
+ await executePendingStore(server, refusedName);
+ expect(server.durabilityState.isStoreRefused(refusedName)).toBe(true);
+ const rescuePath = join(shadowHandle.gitDir, 'rescue', `${refusedName}.md`);
+ expect(existsSync(rescuePath)).toBe(true);
+ expect(readFileSync(rescuePath, 'utf-8')).toContain(unsavedMarker);
+
+ await hp.openDirectConnection(normalName);
+ const normalDoc = hp.documents.get(normalName);
+ expect(normalDoc).toBeDefined();
+ if (!normalDoc) throw new Error('Normal document was not loaded');
+ hp.configuration.extensions.push({
+ beforeUnloadDocument({ documentName }) {
+ entered.get(documentName)?.resolve();
+ return gates.get(documentName)?.promise ?? Promise.resolve();
+ },
+ afterUnloadDocument({ documentName }) {
+ if (unloaded.has(documentName)) {
+ observations.push({
+ event: documentName,
+ loaded: [normalName, refusedName].filter((name) => hp.documents.has(name)),
+ });
+ unloaded.get(documentName)?.resolve();
+ }
+ return Promise.resolve();
+ },
+ });
+ refusedDoc.removeDirectConnection();
+ normalDoc.removeDirectConnection();
+ expect(hp.documents.has(refusedName)).toBe(true);
+ expect(hp.documents.has(normalName)).toBe(true);
+
+ const flushBegan = Promise.withResolvers<void>();
+ const forcedRefusedUnloadBegan = Promise.withResolvers<void>();
+ let flushObserved = false;
+ const flushPendingStores = hp.flushPendingStores.bind(hp);
+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {
+ const result = flushPendingStores();
+ flushObserved = true;
+ flushBegan.resolve();
+ return result;
+ });
+ const shouldUnloadDocument = hp.shouldUnloadDocument.bind(hp);
+ vi.spyOn(hp, 'shouldUnloadDocument').mockImplementation((document) => {
+ const result = shouldUnloadDocument(document);
+ if (flushObserved && document.name === refusedName && result) {
+ forcedRefusedUnloadBegan.resolve();
+ }
+ return result;
+ });
+ await run({
+ server,
+ gates,
+ entered,
+ unloaded,
+ observations,
+ rescuePath,
+ flushBegan: flushBegan.promise,
+ forcedRefusedUnloadBegan: forcedRefusedUnloadBegan.promise,
+ blockedSignalRequests,
+ });
+ } finally {
+ for (const gate of gates.values()) gate.resolve();
+ vi.useRealTimers();
+ await server?.destroy();
+ vi.restoreAllMocks();
+ homeState.path = undefined;
+ await rm(projectDir, { recursive: true, force: true });
+ await rm(homeDir, { recursive: true, force: true });
+ }
+}
+
+test('factory shutdown keeps a refused-store unload pending while other documents retire', async () => {
+ await withRefusedStoreShutdown(
+ async ({
+ server,
+ gates,
+ entered,
+ unloaded,
+ observations,
+ rescuePath,
+ flushBegan,
+ forcedRefusedUnloadBegan,
+ blockedSignalRequests,
+ }) => {
+ const hp = server.hocuspocus;
+ const warnings = vi.spyOn(getLogger('server'), 'warn');
+ const errors = vi.spyOn(getLogger('server'), 'error');
+ const refusedDeadlineWarnings = () =>
+ warnings.mock.calls.filter(([fields, message]) => {
+ if (!fields || typeof fields !== 'object') return false;
+ return (
+ (fields as { docName?: unknown }).docName === refusedName &&
+ typeof message === 'string' &&
+ message.includes('did not finish before the flush deadline')
+ );
+ });
+ const flushTimeoutReports = () =>
+ errors.mock.calls.filter(([fields]) => {
+ if (!fields || typeof fields !== 'object') return false;
+ const report = fields as { err?: unknown };
+ return (
+ report.err instanceof Error &&
+ report.err.message.startsWith('flushAllStoresAndWait timeout')
+ );
+ });
+
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const destroyed = server.destroy().then(
+ () => ({ status: 'fulfilled' as const }),
+ (error: unknown) => ({ status: 'rejected' as const, error }),
+ );
+ await flushBegan;
+ await forcedRefusedUnloadBegan;
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ expect(server.durabilityState.isStoreRefused(refusedName)).toBe(true);
+ expect(hp.documents.has(refusedName)).toBe(true);
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.4);
+ gates.get(normalName)?.resolve();
+ await unloaded.get(normalName)?.promise;
+ expect(observations).toEqual([{ event: normalName, loaded: [refusedName] }]);
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.8);
+ expect(blockedSignalRequests).toEqual([]);
+ expect(hp.documents.has(refusedName)).toBe(true);
+ expect(refusedDeadlineWarnings()).toEqual([]);
+ expect(flushTimeoutReports()).toEqual([]);
+ gates.get(refusedName)?.resolve();
+ await unloaded.get(refusedName)?.promise;
+ vi.useRealTimers();
+ const result = await destroyed;
+ expect(result).toEqual({ status: 'fulfilled' });
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([1, 0]);
+ expect(refusedDeadlineWarnings()).toEqual([]);
+ expect(flushTimeoutReports()).toEqual([]);
+ expect(readFileSync(rescuePath, 'utf-8')).toContain(unsavedMarker);
+ expect(blockedSignalRequests).toEqual([]);
+ },
+ );
+});
+
+test('factory shutdown bounds a refused-store unload that stalls after other documents retire', async () => {
+ await withRefusedStoreShutdown(
+ async ({
+ server,
+ gates,
+ entered,
+ unloaded,
+ observations,
+ rescuePath,
+ flushBegan,
+ forcedRefusedUnloadBegan,
+ blockedSignalRequests,
+ }) => {
+ const hp = server.hocuspocus;
+ const serverLogger = getLogger('server');
+ const error = serverLogger.error.bind(serverLogger);
+ const isFlushTimeoutReport = (fields: unknown) => {
+ if (!fields || typeof fields !== 'object') return false;
+ const report = fields as { err?: unknown };
+ return (
+ report.err instanceof Error &&
+ report.err.message.startsWith('flushAllStoresAndWait timeout')
+ );
+ };
+ const warnings = vi.spyOn(serverLogger, 'warn');
+ const errors = vi.spyOn(serverLogger, 'error').mockImplementation((fields, message) => {
+ const result = error(fields, message);
+ if (isFlushTimeoutReport(fields)) vi.useRealTimers();
+ return result;
+ });
+ const refusedDeadlineWarnings = () =>
+ warnings.mock.calls.filter(([fields, message]) => {
+ if (!fields || typeof fields !== 'object') return false;
+ return (
+ (fields as { docName?: unknown }).docName === refusedName &&
+ typeof message === 'string' &&
+ message.includes('did not finish before the flush deadline')
+ );
+ });
+ const flushTimeoutReports = () =>
+ errors.mock.calls.filter(([fields]) => isFlushTimeoutReport(fields));
+
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const destroyed = server.destroy().then(
+ () => ({ status: 'fulfilled' as const }),
+ (error: unknown) => ({ status: 'rejected' as const, error }),
+ );
+ await flushBegan;
+ await forcedRefusedUnloadBegan;
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ expect(server.durabilityState.isStoreRefused(refusedName)).toBe(true);
+ expect(hp.documents.has(refusedName)).toBe(true);
+ const normalRetiresAfter = innerBudget * 0.4;
+ await vi.advanceTimersByTimeAsync(normalRetiresAfter);
+ gates.get(normalName)?.resolve();
+ await unloaded.get(normalName)?.promise;
+ expect(observations).toEqual([{ event: normalName, loaded: [refusedName] }]);
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.8);
+ expect(refusedDeadlineWarnings()).toEqual([]);
+ expect(flushTimeoutReports()).toEqual([]);
+ expect(hp.documents.has(refusedName)).toBe(true);
+ await vi.advanceTimersByTimeAsync(innerBudget);
+ expect(refusedDeadlineWarnings()).toHaveLength(1);
+ const deadlineFields = refusedDeadlineWarnings()[0]?.[0] as
+ | { unloadDeadline?: number }
+ | undefined;
+ expect(deadlineFields?.unloadDeadline).toBe(normalRetiresAfter + innerBudget);
+ expect(flushTimeoutReports()).toHaveLength(1);
+ expect(hp.documents.has(refusedName)).toBe(true);
+ expect(blockedSignalRequests).toEqual([]);
+ gates.get(refusedName)?.resolve();
+ await unloaded.get(refusedName)?.promise;
+ vi.useRealTimers();
+ const result = await destroyed;
+ expect(result).toEqual({ status: 'fulfilled' });
+ expect(readFileSync(rescuePath, 'utf-8')).toContain(unsavedMarker);
+ expect(blockedSignalRequests).toEqual([]);
+ },
+ );
+});
diff --git a/packages/server/src/shutdown-progress.test.ts b/packages/server/src/shutdown-progress.test.ts
new file mode 100644
index 000000000..733344dff
--- /dev/null
+++ b/packages/server/src/shutdown-progress.test.ts
@@ -0,0 +1,536 @@
+import { ChildProcess } from 'node:child_process';
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { expect, test, vi } from 'vitest';
+import { bootServer } from './boot.ts';
+import { ConfigSchema } from './config/schema.ts';
+import { getLogger } from './logger.ts';
+import { ensureProjectGit } from './project-git.ts';
+
+const homeState = vi.hoisted(() => ({ path: undefined as string | undefined }));
+
+vi.mock('node:os', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('node:os')>();
+ return { ...actual, homedir: () => homeState.path ?? actual.homedir() };
+});
+
+const names = ['first', 'second', 'third'];
+const outerBudget = Number(process.env.OK_DESTROY_STEP_TIMEOUT_MS) || 5000;
+const innerBudget = 10_000;
+
+function captureShutdownReports() {
+ const firstReport = Promise.withResolvers<'outer' | 'inner'>();
+ const bootLogger = getLogger('boot');
+ const serverLogger = getLogger('server');
+ const warn = bootLogger.warn.bind(bootLogger);
+ const error = serverLogger.error.bind(serverLogger);
+ const isOuterTimeout = (fields: unknown) => {
+ if (!fields || typeof fields !== 'object') return false;
+ const report = fields as { step?: unknown; err?: unknown };
+ return (
+ report.step === 'destroyHocuspocus' &&
+ report.err instanceof Error &&
+ report.err.message === `destroyHocuspocus timed out: no document retired for ${outerBudget}ms`
+ );
+ };
+ const isInnerTimeout = (fields: unknown, message: unknown) => {
+ if (!fields || typeof fields !== 'object') return false;
+ const report = fields as { err?: unknown };
+ return (
+ message === '[server] shutdown phase-3 flush failed' &&
+ report.err instanceof Error &&
+ report.err.message.startsWith(
+ `flushAllStoresAndWait timeout: no document retired for ${innerBudget}ms`,
+ )
+ );
+ };
+ const bootWarnings = vi.spyOn(bootLogger, 'warn').mockImplementation((fields, message) => {
+ const result = warn(fields, message);
+ if (isOuterTimeout(fields)) {
+ vi.useRealTimers();
+ firstReport.resolve('outer');
+ }
+ return result;
+ });
+ const factoryErrors = vi.spyOn(serverLogger, 'error').mockImplementation((fields, message) => {
+ const result = error(fields, message);
+ if (isInnerTimeout(fields, message)) {
+ vi.useRealTimers();
+ firstReport.resolve('inner');
+ }
+ return result;
+ });
+ return {
+ firstReport: firstReport.promise,
+ outer: () => bootWarnings.mock.calls.filter(([fields]) => isOuterTimeout(fields)),
+ inner: () =>
+ factoryErrors.mock.calls.filter(([fields, message]) => isInnerTimeout(fields, message)),
+ };
+}
+
+async function withLoadedDocuments(
+ run: (fixture: {
+ booted: Awaited<ReturnType<typeof bootServer>>;
+ gates: Map<string, PromiseWithResolvers<void>>;
+ entered: Map<string, PromiseWithResolvers<void>>;
+ unloaded: Map<string, PromiseWithResolvers<void>>;
+ observations: Array<{ event: string; loaded: string[] }>;
+ retirements: string[];
+ acknowledgeBootRejection: () => void;
+ blockedSignalRequests: Array<{
+ pid: number | undefined;
+ signal: string | number | undefined;
+ }>;
+ }) => Promise<void>,
+): Promise<void> {
+ const blockedSignalRequests: Array<{
+ pid: number | undefined;
+ signal: string | number | undefined;
+ }> = [];
+ vi.spyOn(process, 'kill').mockImplementation((pid, signal) => {
+ blockedSignalRequests.push({ pid, signal });
+ throw new Error('Test signal seam refuses delivery');
+ });
+ vi.spyOn(ChildProcess.prototype, 'kill').mockImplementation(function (
+ this: ChildProcess,
+ signal,
+ ) {
+ blockedSignalRequests.push({ pid: this.pid, signal });
+ return false;
+ });
+ const homeDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-home-'));
+ const projectDir = await mkdtemp(join(tmpdir(), 'ok-shutdown-progress-'));
+ homeState.path = homeDir;
+ let booted: Awaited<ReturnType<typeof bootServer>> | undefined;
+ let startedBootDestroy: Promise<void> | undefined;
+ let bootRejectionAcknowledged = false;
+ const gates = new Map(names.map((name) => [name, Promise.withResolvers<void>()]));
+ const entered = new Map(names.map((name) => [name, Promise.withResolvers<void>()]));
+ const unloaded = new Map(names.map((name) => [name, Promise.withResolvers<void>()]));
+ const observations: Array<{ event: string; loaded: string[] }> = [];
+ const retirements: string[] = [];
+ let bodyError: unknown;
+ let bodyFailed = false;
+ const cleanupErrors: unknown[] = [];
+ try {
+ await ensureProjectGit(projectDir);
+ await mkdir(join(projectDir, '.ok'), { recursive: true });
+ await writeFile(join(projectDir, '.ok', 'config.yml'), '');
+ for (const name of names) await writeFile(join(projectDir, `${name}.md`), `# ${name}\n`);
+ booted = await bootServer({
+ host: '127.0.0.1',
+ config: ConfigSchema.parse({}),
+ contentDir: projectDir,
+ configHomedirOverride: homeDir,
+ port: 0,
+ quiet: true,
+ gitEnabled: false,
+ idleShutdownMs: null,
+ ephemeral: true,
+ });
+ await booted.ready;
+ const destroyBoot = booted.destroy.bind(booted);
+ booted.destroy = (reason) => {
+ const inFlight = destroyBoot(reason);
+ startedBootDestroy ??= inFlight;
+ return inFlight;
+ };
+ const hp = booted.serverInstance.hocuspocus;
+ for (const name of names) {
+ const connection = await hp.openDirectConnection(name);
+ await connection.disconnect();
+ }
+ expect(names.every((name) => hp.documents.has(name))).toBe(true);
+ hp.configuration.extensions.push({
+ beforeUnloadDocument({ documentName }) {
+ entered.get(documentName)?.resolve();
+ return gates.get(documentName)?.promise ?? Promise.resolve();
+ },
+ afterUnloadDocument({ documentName }) {
+ retirements.push(documentName);
+ if (unloaded.has(documentName)) {
+ observations.push({
+ event: documentName,
+ loaded: names.filter((name) => hp.documents.has(name)),
+ });
+ unloaded.get(documentName)?.resolve();
+ }
+ return Promise.resolve();
+ },
+ });
+ await run({
+ booted,
+ gates,
+ entered,
+ unloaded,
+ observations,
+ retirements,
+ acknowledgeBootRejection: () => {
+ bootRejectionAcknowledged = true;
+ },
+ blockedSignalRequests,
+ });
+ } catch (error) {
+ bodyError = error;
+ bodyFailed = true;
+ } finally {
+ for (const gate of gates.values()) gate.resolve();
+ try {
+ await booted?.serverInstance.destroy();
+ } catch (error) {
+ cleanupErrors.push(error);
+ }
+ try {
+ vi.useRealTimers();
+ } catch (error) {
+ cleanupErrors.push(error);
+ }
+ try {
+ if (startedBootDestroy) await startedBootDestroy;
+ else await booted?.destroy();
+ } catch (error) {
+ if (!bootRejectionAcknowledged) cleanupErrors.push(error);
+ }
+ try {
+ vi.restoreAllMocks();
+ } catch (error) {
+ cleanupErrors.push(error);
+ }
+ homeState.path = undefined;
+ try {
+ await rm(projectDir, { recursive: true, force: true });
+ } catch (error) {
+ cleanupErrors.push(error);
+ }
+ try {
+ await rm(homeDir, { recursive: true, force: true });
+ } catch (error) {
+ cleanupErrors.push(error);
+ }
+ }
+ if (bodyFailed) {
+ if (cleanupErrors.length > 0) {
+ throw new AggregateError(
+ [bodyError, ...cleanupErrors],
+ 'Shutdown fixture and cleanup failed',
+ );
+ }
+ throw bodyError;
+ }
+ if (cleanupErrors.length > 0) {
+ throw cleanupErrors.length === 1
+ ? cleanupErrors[0]
+ : new AggregateError(cleanupErrors, 'Shutdown fixture cleanup failed');
+ }
+}
+
+test('destroy accepts real document unload progress across the outer deadline', async () => {
+ await withLoadedDocuments(
+ async ({ booted, gates, entered, unloaded, observations, blockedSignalRequests }) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const destroyed = booted.destroy().then(
+ () => ({ status: 'fulfilled' as const }),
+ (error: unknown) => ({ status: 'rejected' as const, error }),
+ );
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ for (const name of names) {
+ await vi.advanceTimersByTimeAsync(outerBudget * 0.4);
+ gates.get(name)?.resolve();
+ await unloaded.get(name)?.promise;
+ }
+ vi.useRealTimers();
+ const result = await destroyed;
+ await booted.serverInstance.destroy();
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);
+ expect(blockedSignalRequests).toEqual([]);
+ expect(result).toEqual({ status: 'fulfilled' });
+ },
+ );
+});
+
+test('destroy reports a held real document unload without waiting for its release', async () => {
+ await withLoadedDocuments(
+ async ({
+ booted,
+ gates,
+ entered,
+ unloaded,
+ observations,
+ retirements,
+ acknowledgeBootRejection,
+ blockedSignalRequests,
+ }) => {
+ const hp = booted.serverInstance.hocuspocus;
+ const flushBegan = Promise.withResolvers<void>();
+ const flushPendingStores = hp.flushPendingStores.bind(hp);
+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {
+ const result = flushPendingStores();
+ flushBegan.resolve();
+ return result;
+ });
+ const reports = captureShutdownReports();
+ const stallWindow = Math.min(outerBudget, innerBudget);
+ const outerFirst = outerBudget < innerBudget;
+ expect(outerBudget).not.toBe(innerBudget);
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const destroyed = booted.destroy().then(
+ () => ({ status: 'fulfilled' as const }),
+ (error: unknown) => ({ status: 'rejected' as const, error }),
+ );
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ await flushBegan.promise;
+ expect([...hp.documents.keys()].sort()).toEqual([...names].sort());
+ const retirementsAtFlush = retirements.length;
+ await vi.advanceTimersByTimeAsync(stallWindow);
+ if (outerFirst) {
+ expect(reports.outer()).toHaveLength(1);
+ expect(reports.inner()).toEqual([]);
+ } else {
+ expect(reports.inner()).toHaveLength(1);
+ expect(reports.outer()).toEqual([]);
+ }
+ expect(await reports.firstReport).toBe(outerFirst ? 'outer' : 'inner');
+ const result = await destroyed;
+ const loadedBeforeRelease = names.filter((name) => hp.documents.has(name));
+ expect(loadedBeforeRelease).toEqual(names);
+ expect(observations).toEqual([]);
+ expect(retirements.slice(retirementsAtFlush)).toEqual([]);
+ if (outerFirst) {
+ expect(reports.outer()).toHaveLength(1);
+ expect(reports.inner()).toEqual([]);
+ expect(result).toEqual({ status: 'rejected', error: expect.any(AggregateError) });
+ expect(result).toMatchObject({
+ error: {
+ errors: [
+ expect.objectContaining({
+ message: `destroyHocuspocus timed out: no document retired for ${outerBudget}ms`,
+ }),
+ ],
+ },
+ });
+ acknowledgeBootRejection();
+ } else {
+ expect(reports.outer()).toEqual([]);
+ expect(reports.inner()).toHaveLength(1);
+ expect(reports.inner()[0]?.[0]).toMatchObject({
+ err: expect.objectContaining({
+ message: expect.stringContaining('3/3 docs did not unload: [first, second, third]'),
+ }),
+ });
+ expect(reports.inner()[0]?.[0]).toMatchObject({
+ err: expect.objectContaining({
+ message: expect.stringContaining('rescued [first, second, third]'),
+ }),
+ });
+ expect(result).toEqual({ status: 'fulfilled' });
+ }
+ for (const name of names) {
+ gates.get(name)?.resolve();
+ await unloaded.get(name)?.promise;
+ }
+ vi.useRealTimers();
+ await booted.serverInstance.destroy();
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);
+ expect(blockedSignalRequests).toEqual([]);
+ },
+ );
+});
+
+test('destroy bounds a stall after a real document unload', async () => {
+ await withLoadedDocuments(
+ async ({
+ booted,
+ gates,
+ entered,
+ unloaded,
+ observations,
+ retirements,
+ acknowledgeBootRejection,
+ blockedSignalRequests,
+ }) => {
+ const hp = booted.serverInstance.hocuspocus;
+ const flushBegan = Promise.withResolvers<void>();
+ const flushPendingStores = hp.flushPendingStores.bind(hp);
+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {
+ const result = flushPendingStores();
+ flushBegan.resolve();
+ return result;
+ });
+ const reports = captureShutdownReports();
+ const stallWindow = Math.min(outerBudget, innerBudget);
+ const outerFirst = outerBudget < innerBudget;
+ expect(outerBudget).not.toBe(innerBudget);
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ let settled = false;
+ const destroyed = booted.destroy().then(
+ () => {
+ settled = true;
+ return { status: 'fulfilled' as const };
+ },
+ (error: unknown) => {
+ settled = true;
+ return { status: 'rejected' as const, error };
+ },
+ );
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ await flushBegan.promise;
+ expect([...hp.documents.keys()].sort()).toEqual([...names].sort());
+ const retirementsAtFlush = retirements.length;
+ await vi.advanceTimersByTimeAsync(stallWindow * 0.4);
+ gates.get('first')?.resolve();
+ await unloaded.get('first')?.promise;
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2]);
+ expect(retirements.slice(retirementsAtFlush)).toEqual(['first']);
+ await vi.advanceTimersByTimeAsync(stallWindow * 0.8);
+ expect(settled).toBe(false);
+ expect(reports.outer()).toEqual([]);
+ expect(reports.inner()).toEqual([]);
+ expect(blockedSignalRequests).toEqual([]);
+ expect(names.filter((name) => hp.documents.has(name))).toEqual(['second', 'third']);
+ expect(retirements.slice(retirementsAtFlush)).toEqual(['first']);
+ await vi.advanceTimersByTimeAsync(stallWindow * 0.2);
+ if (outerFirst) {
+ expect(reports.outer()).toHaveLength(1);
+ expect(reports.inner()).toEqual([]);
+ } else {
+ expect(reports.inner()).toHaveLength(1);
+ expect(reports.outer()).toEqual([]);
+ }
+ expect(await reports.firstReport).toBe(outerFirst ? 'outer' : 'inner');
+ const result = await destroyed;
+ if (outerFirst) {
+ expect(reports.outer()).toHaveLength(1);
+ expect(reports.inner()).toEqual([]);
+ expect(result).toEqual({ status: 'rejected', error: expect.any(AggregateError) });
+ expect(result).toMatchObject({
+ error: {
+ errors: [
+ expect.objectContaining({
+ message: `destroyHocuspocus timed out: no document retired for ${outerBudget}ms`,
+ }),
+ ],
+ },
+ });
+ acknowledgeBootRejection();
+ } else {
+ expect(reports.outer()).toEqual([]);
+ expect(reports.inner()).toHaveLength(1);
+ expect(reports.inner()[0]?.[0]).toMatchObject({
+ err: expect.objectContaining({
+ message: expect.stringContaining('2/3 docs did not unload: [second, third]'),
+ }),
+ });
+ expect(reports.inner()[0]?.[0]).toMatchObject({
+ err: expect.objectContaining({
+ message: expect.stringContaining('rescued [second, third]'),
+ }),
+ });
+ expect(result).toEqual({ status: 'fulfilled' });
+ }
+ for (const name of ['second', 'third']) {
+ gates.get(name)?.resolve();
+ await unloaded.get(name)?.promise;
+ }
+ vi.useRealTimers();
+ await booted.serverInstance.destroy();
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);
+ expect(blockedSignalRequests).toEqual([]);
+ },
+ );
+});
+
+test('factory shutdown does not report continuing real unloads as a flush timeout', async () => {
+ await withLoadedDocuments(
+ async ({ booted, gates, entered, unloaded, observations, blockedSignalRequests }) => {
+ const hp = booted.serverInstance.hocuspocus;
+ const flushBegan = Promise.withResolvers<void>();
+ const flushPendingStores = hp.flushPendingStores.bind(hp);
+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {
+ const result = flushPendingStores();
+ flushBegan.resolve();
+ return result;
+ });
+ const errors = vi.spyOn(getLogger('server'), 'error');
+ const flushTimeoutReports = () =>
+ errors.mock.calls.filter(([fields]) => {
+ if (!fields || typeof fields !== 'object') return false;
+ const report = fields as { err?: unknown };
+ return (
+ report.err instanceof Error &&
+ report.err.message.startsWith('flushAllStoresAndWait timeout')
+ );
+ });
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const destroyed = booted.serverInstance.destroy().then(
+ () => ({ status: 'fulfilled' as const }),
+ (error: unknown) => ({ status: 'rejected' as const, error }),
+ );
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ await flushBegan.promise;
+ for (const name of names) {
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.4);
+ gates.get(name)?.resolve();
+ await unloaded.get(name)?.promise;
+ }
+ vi.useRealTimers();
+ const result = await destroyed;
+ await booted.serverInstance.destroy();
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);
+ expect(blockedSignalRequests).toEqual([]);
+ expect(result).toEqual({ status: 'fulfilled' });
+ expect(flushTimeoutReports()).toEqual([]);
+ },
+ );
+});
+
+test('factory shutdown bounds a stall after a real document unload', async () => {
+ await withLoadedDocuments(
+ async ({ booted, gates, entered, unloaded, observations, blockedSignalRequests }) => {
+ const hp = booted.serverInstance.hocuspocus;
+ const flushBegan = Promise.withResolvers<void>();
+ const flushPendingStores = hp.flushPendingStores.bind(hp);
+ vi.spyOn(hp, 'flushPendingStores').mockImplementation(() => {
+ const result = flushPendingStores();
+ flushBegan.resolve();
+ return result;
+ });
+ const errors = vi.spyOn(getLogger('server'), 'error');
+ const flushTimeoutReports = () =>
+ errors.mock.calls.filter(([fields]) => {
+ if (!fields || typeof fields !== 'object') return false;
+ const report = fields as { err?: unknown };
+ return (
+ report.err instanceof Error &&
+ report.err.message.startsWith('flushAllStoresAndWait timeout')
+ );
+ });
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const destroyed = booted.serverInstance.destroy().then(
+ () => ({ status: 'fulfilled' as const }),
+ (error: unknown) => ({ status: 'rejected' as const, error }),
+ );
+ await Promise.all([...entered.values()].map((entry) => entry.promise));
+ await flushBegan.promise;
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.4);
+ gates.get('first')?.resolve();
+ await unloaded.get('first')?.promise;
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2]);
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.8);
+ expect(flushTimeoutReports()).toEqual([]);
+ expect(blockedSignalRequests).toEqual([]);
+ expect(names.filter((name) => hp.documents.has(name))).toEqual(['second', 'third']);
+ await vi.advanceTimersByTimeAsync(innerBudget * 0.2);
+ expect(flushTimeoutReports()).toHaveLength(1);
+ for (const name of ['second', 'third']) {
+ gates.get(name)?.resolve();
+ await unloaded.get(name)?.promise;
+ }
+ vi.useRealTimers();
+ const result = await destroyed;
+ expect(result).toEqual({ status: 'fulfilled' });
+ expect(observations.map((entry) => entry.loaded.length)).toEqual([2, 1, 0]);
+ expect(blockedSignalRequests).toEqual([]);
+ },
+ );
+});