Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Fix derived index shutdown ordering (#7317)
Nick Gomezb08a29c5b12e
diff --git a/.changeset/derived-index-stop-drain.md b/.changeset/derived-index-stop-drain.mdnew file mode 100644index 000000000..bd3bb1811--- /dev/null+++ b/.changeset/derived-index-stop-drain.md@@ -0,0 +1,6 @@+---+"@inkeep/open-knowledge": patch+"@inkeep/open-knowledge-server": patch+---++Server shutdown now waits, up to a time limit, for backlink and tag cache writes to finish, so removing a project folder right after stopping its server no longer races those writes.diff --git a/packages/server/src/server-factory.test.ts b/packages/server/src/server-factory.test.tsindex 7be396ba5..eaa5c07a8 100644--- a/packages/server/src/server-factory.test.ts+++ b/packages/server/src/server-factory.test.ts@@ -31,7 +31,7 @@ import {} from '@inkeep/open-knowledge-core/server';import { parseCheckpoint } from '@inkeep/open-knowledge-core/shadow-repo-layout';import simpleGit from 'simple-git';-import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import { stringify as stringifyYaml } from 'yaml';import * as Y from 'yjs';import { runningAsRoot } from '../../../test-support/capabilities.test-helper.ts';@@ -1485,35 +1485,46 @@ describe('createServer().destroy() — graceful shutdown flush', () => {expect(shutdownLogs[0].payload.documentCount).toBe(5);});- test('destroy does not await the derived-index cache drain', async () => {+ test('destroy bounds a derived-index close that never settles and records the phase error', async () => {const beginStartup = vi.spyOn(DerivedDocumentIndex.prototype, 'beginStartup');const server = createServer({contentDir: tmpDir,projectDir: tmpDir,quiet: true,- destroyTimeoutMs: 500,+ destroyTimeoutMs: 200,});await server.ready;const coordinator = beginStartup.mock.instances[0] as DerivedDocumentIndex;beginStartup.mockRestore();- const close = vi- .spyOn(DerivedDocumentIndex.prototype, 'close')- .mockImplementation(() => new Promise<void>(() => {}));- let timeout: ReturnType<typeof setTimeout> | undefined;+ let releaseTagClose!: () => void;+ const tagCloseReleased = new Promise<void>((resolve) => {+ releaseTagClose = resolve;+ });+ let tagCloseEntered = false;+ const tagCloseImplementation = TagIndex.prototype.close;+ const tagClose = vi.spyOn(TagIndex.prototype, 'close').mockImplementation(async function (+ this: TagIndex,+ ) {+ tagCloseEntered = true;+ await tagCloseImplementation.call(this);+ await tagCloseReleased;+ });try {- const result = await Promise.race([- server.destroy().then(() => 'destroyed' as const),- new Promise<'timed-out'>((resolveTimeout) => {- timeout = setTimeout(() => resolveTimeout('timed-out'), 2_000);- }),- ]);+ await server.destroy();- expect(result).toBe('destroyed');- expect(close).toHaveBeenCalledTimes(1);+ expect(tagCloseEntered).toBe(true);+ const closePhaseErrors = logCapture+ .getCalls('warn', 'shutdown flushed')+ .flatMap(+ (entry) => (entry.payload.phaseErrors as Array<{ phase: string; error: string }>) ?? [],+ )+ .filter((e) => e.phase === 'derived-index-close');+ expect(closePhaseErrors).toHaveLength(1);+ expect(logCapture.getCalls('info', 'shutdown flushed')).toHaveLength(0);} finally {- if (timeout) clearTimeout(timeout);- close.mockRestore();+ releaseTagClose();+ tagClose.mockRestore();await coordinator.close();}});@@ -4194,12 +4205,6 @@ describe('createServer() — generated index wiring', () => {let server: ServerInstance | null;let shadowHandle: Awaited<ReturnType<typeof initShadowRepo>>;let localHttp: import('node:http').Server | null = null;- let indexDrains: Promise<void>[];- let holdIndexClose: Promise<void> | undefined;- let releaseIndexClose: (() => void) | undefined;- let restoreIndexClose: () => void;- let onDestroyComplete: (() => void) | undefined;- let onBeforeRemove: (() => void) | undefined;const indexPath = () => join(contentDir, 'index.md');const readIndex = () => readFileSync(indexPath(), 'utf-8');@@ -4340,28 +4345,8 @@ describe('createServer() — generated index wiring', () => {expect(res.status, body).toBe(200);}- beforeAll(() => {- const close = DerivedDocumentIndex.prototype.close;- const closeIndex = vi- .spyOn(DerivedDocumentIndex.prototype, 'close')- .mockImplementation(function (this: DerivedDocumentIndex) {- const closing = close.call(this);- const draining = holdIndexClose- ? Promise.all([closing, holdIndexClose]).then(() => undefined)- : closing;- indexDrains.push(draining);- return draining;- });- restoreIndexClose = () => closeIndex.mockRestore();- });-- afterAll(() => restoreIndexClose());-beforeEach(async () => {projectDir = await mkdtemp(join(tmpdir(), 'ok-index-wiring-'));- indexDrains = [];- holdIndexClose = undefined;- releaseIndexClose = undefined;contentDir = join(projectDir, 'content');mkdirSync(contentDir, { recursive: true });mkdirSync(join(projectDir, '.ok'), { recursive: true });@@ -4382,10 +4367,6 @@ describe('createServer() — generated index wiring', () => {}try {await server?.destroy();- onDestroyComplete?.();- if (releaseIndexClose) setImmediate(releaseIndexClose);- await Promise.all(indexDrains);- onBeforeRemove?.();} finally {loggerFactory.reset();await rm(projectDir, { recursive: true, force: true });@@ -4394,33 +4375,6 @@ describe('createServer() — generated index wiring', () => {afterEach(cleanupFixture);- test('fixture cleanup waits for the derived-index close promise', async () => {- holdIndexClose = new Promise<void>((resolve) => {- releaseIndexClose = resolve;- });- try {- await bootServer();- let drainsSettled = false;- let drainsRecordedAtRemoval = 0;- let drainsSettledAtRemoval = false;- onDestroyComplete = () => {- void Promise.all(indexDrains).then(() => {- drainsSettled = true;- });- };- onBeforeRemove = () => {- drainsRecordedAtRemoval = indexDrains.length;- drainsSettledAtRemoval = drainsSettled;- };- await cleanupFixture();- expect(drainsRecordedAtRemoval).toBeGreaterThan(0);- expect(drainsSettledAtRemoval).toBe(true);- } finally {- onDestroyComplete = undefined;- onBeforeRemove = undefined;- }- });-test('the setting reaches the server through the real config file', async () => {await bootServer();writeDoc('note.md', 'A note', 'note', 'Something to index.');diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex de3021dac..0e1811548 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -4752,7 +4752,8 @@ export function createServer(options: ServerOptions): ServerInstance {inflightDestroy = (async () => {const t0 = Date.now();const phaseErrors: Array<{ phase: string; error: string }> = [];- const preludeDeadline = Date.now() + Math.min(destroyTimeoutMs, DEFAULT_SIGTERM_GRACE_MS / 5);+ const graceShareMs = Math.min(destroyTimeoutMs, DEFAULT_SIGTERM_GRACE_MS / 5);+ const preludeDeadline = Date.now() + graceShareMs;const preludeRemaining = () => Math.max(1, preludeDeadline - Date.now());const joinPrelude = async <T>(operation: Promise<T>, phase: string): Promise<T> => {let timer: ReturnType<typeof setTimeout> | undefined;@@ -4994,7 +4995,29 @@ export function createServer(options: ServerOptions): ServerInstance {log.error({ err }, '[server] shutdown phase-3 flush failed');}- void derivedDocumentIndex.close();+ let derivedIndexCloseTimeoutId: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ derivedDocumentIndex.close(),+ new Promise<never>((_, reject) => {+ derivedIndexCloseTimeoutId = setTimeout(+ () =>+ reject(+ new Error(`derived index close did not settle within ${graceShareMs}ms`),+ ),+ graceShareMs,+ );+ }),+ ]);+ } catch (err) {+ phaseErrors.push({+ phase: 'derived-index-close',+ error: err instanceof Error ? err.message : String(err),+ });+ log.error({ err }, '[server] shutdown derived-index close failed');+ } finally {+ if (derivedIndexCloseTimeoutId !== undefined) clearTimeout(derivedIndexCloseTimeoutId);+ }let l2TimeoutId: ReturnType<typeof setTimeout> | undefined;try {