Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-9738 Re-check the tree on a timer, resubscribe, and re-read on focus (#7133)

mike-inkeepf8500ff0ef4e
diff --git a/.changeset/find-missed-outside-changes.md b/.changeset/find-missed-outside-changes.md
new file mode 100644
index 000000000..8649b3d52
--- /dev/null
+++ b/.changeset/find-missed-outside-changes.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Outside edits the file watcher misses now appear within seconds, or about two minutes on very large projects: OpenKnowledge re-checks the project on a timer, watches again after a watcher failure, and re-reads open documents when the window regains focus or reconnects. Large projects use less memory.
diff --git a/packages/app/src/editor/EditorLifecycleFlush.tsx b/packages/app/src/editor/EditorLifecycleFlush.tsx
index 619fb1c63..f5a23b0a1 100644
--- a/packages/app/src/editor/EditorLifecycleFlush.tsx
+++ b/packages/app/src/editor/EditorLifecycleFlush.tsx
@@ -2,6 +2,7 @@ import { useEffect } from 'react';
import { useConfigContext } from '@/lib/config-context';
import { getPool, useDocumentContext } from './DocumentContext';
import { installEditorLifecycleFlush } from './install-editor-lifecycle-flush';
+import { refreshDocumentsFromDisk } from './refresh-documents-from-disk';
export function EditorLifecycleFlush(): null {
const { collabUrl } = useDocumentContext();
@@ -15,6 +16,9 @@ export function EditorLifecycleFlush(): null {
return installEditorLifecycleFlush({
onHide: () => pool.flushOnHide(),
onVisible: () => pool.resyncOnVisible(),
+ onResume: () => {
+ void refreshDocumentsFromDisk(pool.openDocNames());
+ },
});
}, [collabUrl, enabled]);
diff --git a/packages/app/src/editor/install-editor-lifecycle-flush.test.ts b/packages/app/src/editor/install-editor-lifecycle-flush.test.ts
index df1d50703..5b41b7207 100644
--- a/packages/app/src/editor/install-editor-lifecycle-flush.test.ts
+++ b/packages/app/src/editor/install-editor-lifecycle-flush.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from 'vitest';
-import { installEditorLifecycleFlush } from './install-editor-lifecycle-flush';
+import { installEditorLifecycleFlush, RESUME_THROTTLE_MS } from './install-editor-lifecycle-flush';
type Listener = (event: Event) => void;
@@ -129,4 +129,75 @@ describe('installEditorLifecycleFlush', () => {
expect(() => uninstall()).not.toThrow();
expect(onHide).not.toHaveBeenCalled();
});
+
+ it('asks for a re-read from disk when the window regains focus', () => {
+ const onResume = vi.fn();
+ const win = makeFakeTarget();
+ const doc = makeFakeDoc('visible');
+ let clock = 10_000;
+ installEditorLifecycleFlush({
+ onHide: vi.fn(),
+ onVisible: vi.fn(),
+ onResume,
+ now: () => clock,
+ win,
+ doc,
+ });
+
+ win.emit('focus');
+ expect(onResume).toHaveBeenCalledTimes(1);
+
+ clock += RESUME_THROTTLE_MS;
+ win.emit('focus');
+ expect(onResume).toHaveBeenCalledTimes(2);
+ });
+
+ it('asks once when the tab becomes visible and the window takes focus together', () => {
+ const onResume = vi.fn();
+ const onVisible = vi.fn();
+ const win = makeFakeTarget();
+ const doc = makeFakeDoc('visible');
+ installEditorLifecycleFlush({
+ onHide: vi.fn(),
+ onVisible,
+ onResume,
+ now: () => 10_000,
+ win,
+ doc,
+ });
+
+ doc.emit('visibilitychange');
+ win.emit('focus');
+
+ expect(onVisible).toHaveBeenCalledTimes(1);
+ expect(onResume).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not ask while the page is hidden, and stops asking once uninstalled', () => {
+ const onResume = vi.fn();
+ const win = makeFakeTarget();
+ const doc = makeFakeDoc('hidden');
+ let clock = 10_000;
+ const uninstall = installEditorLifecycleFlush({
+ onHide: vi.fn(),
+ onVisible: vi.fn(),
+ onResume,
+ now: () => clock,
+ win,
+ doc,
+ });
+
+ win.emit('focus');
+ expect(onResume).not.toHaveBeenCalled();
+
+ doc.setVisibility('visible');
+ win.emit('focus');
+ expect(onResume).toHaveBeenCalledTimes(1);
+
+ uninstall();
+ clock += RESUME_THROTTLE_MS;
+ expect(win.has('focus')).toBe(false);
+ win.emit('focus');
+ expect(onResume).toHaveBeenCalledTimes(1);
+ });
});
diff --git a/packages/app/src/editor/install-editor-lifecycle-flush.ts b/packages/app/src/editor/install-editor-lifecycle-flush.ts
index d58ac028b..d9464f224 100644
--- a/packages/app/src/editor/install-editor-lifecycle-flush.ts
+++ b/packages/app/src/editor/install-editor-lifecycle-flush.ts
@@ -1,6 +1,10 @@
+export const RESUME_THROTTLE_MS = 1_000;
+
export interface EditorLifecycleFlushDeps {
onHide: () => void;
onVisible: () => void;
+ onResume?: () => void;
+ now?: () => number;
win?: Pick<Window, 'addEventListener' | 'removeEventListener'>;
doc?: Pick<Document, 'addEventListener' | 'removeEventListener'> & {
readonly visibilityState: DocumentVisibilityState;
@@ -13,6 +17,15 @@ export function installEditorLifecycleFlush(deps: EditorLifecycleFlushDeps): ()
if (win === undefined || doc === undefined) return () => {};
let hidden = false;
+ let lastResumeAt = Number.NEGATIVE_INFINITY;
+ const now = deps.now ?? Date.now;
+ const fireResume = (): void => {
+ if (deps.onResume === undefined) return;
+ const at = now();
+ if (at - lastResumeAt < RESUME_THROTTLE_MS) return;
+ lastResumeAt = at;
+ deps.onResume();
+ };
const fireHide = (): void => {
if (hidden) return;
hidden = true;
@@ -25,14 +38,20 @@ export function installEditorLifecycleFlush(deps: EditorLifecycleFlushDeps): ()
else if (doc.visibilityState === 'visible') {
hidden = false;
deps.onVisible();
+ fireResume();
}
};
+ const onFocus = (): void => {
+ if (doc.visibilityState === 'visible') fireResume();
+ };
win.addEventListener('pagehide', onPageHide as (event: Event) => void);
+ win.addEventListener('focus', onFocus);
doc.addEventListener('visibilitychange', onVisibility);
return () => {
win.removeEventListener('pagehide', onPageHide as (event: Event) => void);
+ win.removeEventListener('focus', onFocus);
doc.removeEventListener('visibilitychange', onVisibility);
};
}
diff --git a/packages/app/src/editor/provider-pool.ts b/packages/app/src/editor/provider-pool.ts
index 29364940e..e86964c41 100644
--- a/packages/app/src/editor/provider-pool.ts
+++ b/packages/app/src/editor/provider-pool.ts
@@ -1973,6 +1973,14 @@ export class ProviderPool {
return false;
}
+ openDocNames(): string[] {
+ const docNames: string[] = [];
+ for (const [docName, entry] of this._entries) {
+ if (entry.kind === 'active') docNames.push(docName);
+ }
+ return docNames;
+ }
+
docNamesToOpenForReplay(): string[] {
const docNames: string[] = [];
for (const [docName, buffered] of this.bufferedUpdates) {
diff --git a/packages/app/src/editor/refresh-documents-from-disk.test.ts b/packages/app/src/editor/refresh-documents-from-disk.test.ts
new file mode 100644
index 000000000..05814ad84
--- /dev/null
+++ b/packages/app/src/editor/refresh-documents-from-disk.test.ts
@@ -0,0 +1,66 @@
+import { DOCUMENT_REFRESH_MAX_DOCS } from '@inkeep/open-knowledge-core/schemas/api';
+import { describe, expect, it, vi } from 'vitest';
+import { refreshDocumentsFromDisk } from './refresh-documents-from-disk';
+
+function requestedBatches(fetchImpl: ReturnType<typeof vi.fn<typeof fetch>>): string[][] {
+ return fetchImpl.mock.calls.map(([, init]) => {
+ const body: { docNames?: unknown } = JSON.parse(String(init?.body));
+ return Array.isArray(body.docNames) ? body.docNames.map(String) : [];
+ });
+}
+
+describe('refreshDocumentsFromDisk', () => {
+ it('asks the server to re-read every open document, in batches the route accepts', async () => {
+ const fetchImpl = vi.fn<typeof fetch>(async () => new Response('{"checked":0,"found":0}'));
+ const docNames = Array.from({ length: DOCUMENT_REFRESH_MAX_DOCS + 6 }, (_, i) => `doc-${i}`);
+
+ await refreshDocumentsFromDisk(docNames, fetchImpl);
+
+ const batches = requestedBatches(fetchImpl);
+ expect(batches.map((batch) => batch.length)).toEqual([DOCUMENT_REFRESH_MAX_DOCS, 6]);
+ expect(batches.flat()).toEqual(docNames);
+ expect(fetchImpl.mock.calls[0]?.[0]).toBe('/api/documents/refresh');
+ });
+
+ it('sends nothing when no document is open', async () => {
+ const fetchImpl = vi.fn<typeof fetch>(async () => new Response('{}'));
+ await refreshDocumentsFromDisk([], fetchImpl);
+ expect(fetchImpl).not.toHaveBeenCalled();
+ });
+
+ it('keeps going after a failed request', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const fetchImpl = vi
+ .fn<typeof fetch>()
+ .mockRejectedValueOnce(new Error('offline'))
+ .mockResolvedValue(new Response('{}'));
+ const docNames = Array.from({ length: DOCUMENT_REFRESH_MAX_DOCS + 1 }, (_, i) => `doc-${i}`);
+
+ await refreshDocumentsFromDisk(docNames, fetchImpl);
+
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ expect(warn).toHaveBeenCalledTimes(1);
+ warn.mockRestore();
+ });
+
+ it('logs a refused re-read with its status and still sends the next batch', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const fetchImpl = vi
+ .fn<typeof fetch>()
+ .mockResolvedValueOnce(new Response('{}', { status: 500 }))
+ .mockResolvedValue(new Response('{"checked":1,"found":0}'));
+ const docNames = Array.from({ length: DOCUMENT_REFRESH_MAX_DOCS + 1 }, (_, i) => `doc-${i}`);
+
+ await refreshDocumentsFromDisk(docNames, fetchImpl);
+
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ expect(warn.mock.calls.map(([line]) => JSON.parse(String(line)))).toEqual([
+ {
+ event: 'ok-document-refresh-request-failed',
+ docCount: DOCUMENT_REFRESH_MAX_DOCS,
+ status: 500,
+ },
+ ]);
+ warn.mockRestore();
+ });
+});
diff --git a/packages/app/src/editor/refresh-documents-from-disk.ts b/packages/app/src/editor/refresh-documents-from-disk.ts
new file mode 100644
index 000000000..d8b17fb82
--- /dev/null
+++ b/packages/app/src/editor/refresh-documents-from-disk.ts
@@ -0,0 +1,27 @@
+import { DOCUMENT_REFRESH_MAX_DOCS } from '@inkeep/open-knowledge-core/schemas/api';
+
+function warnRefreshFailed(details: Record<string, unknown>): void {
+ console.warn(JSON.stringify({ event: 'ok-document-refresh-request-failed', ...details }));
+}
+
+export async function refreshDocumentsFromDisk(
+ docNames: readonly string[],
+ fetchImpl: typeof fetch = fetch,
+): Promise<void> {
+ for (let start = 0; start < docNames.length; start += DOCUMENT_REFRESH_MAX_DOCS) {
+ const batch = docNames.slice(start, start + DOCUMENT_REFRESH_MAX_DOCS);
+ try {
+ const response = await fetchImpl('/api/documents/refresh', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docNames: batch }),
+ });
+ if (!response.ok) warnRefreshFailed({ docCount: batch.length, status: response.status });
+ } catch (err) {
+ warnRefreshFailed({
+ docCount: batch.length,
+ message: String(err instanceof Error ? err.message : err),
+ });
+ }
+ }
+}
diff --git a/packages/app/tests/integration/api-error-envelope/document-refresh.test.ts b/packages/app/tests/integration/api-error-envelope/document-refresh.test.ts
new file mode 100644
index 000000000..b2d400be8
--- /dev/null
+++ b/packages/app/tests/integration/api-error-envelope/document-refresh.test.ts
@@ -0,0 +1,56 @@
+import { DocumentRefreshSuccessSchema, ProblemDetailsSchema } from '@inkeep/open-knowledge-core';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import { HARNESS_BOOT_TIMEOUT_MS } from '../harness-boot-timeout';
+import { createTestServer, type TestServer } from '../test-harness';
+
+let server: TestServer;
+
+beforeAll(async () => {
+ server = await createTestServer();
+}, HARNESS_BOOT_TIMEOUT_MS);
+
+afterAll(async () => {
+ await server.cleanup();
+});
+
+function refresh(body: unknown, method = 'POST'): Promise<Response> {
+ return fetch(`http://127.0.0.1:${server.port}/api/documents/refresh`, {
+ method,
+ headers: { 'Content-Type': 'application/json' },
+ ...(method === 'POST' ? { body: JSON.stringify(body) } : {}),
+ });
+}
+
+describe('document-refresh envelope (RFC 9457)', () => {
+ test('happy path emits a flat counts body with application/json', async () => {
+ const res = await refresh({ docNames: ['test-doc'] });
+ expect(res.status).toBe(200);
+ expect(res.headers.get('content-type')).toBe('application/json');
+
+ const body = await res.json();
+ expect(DocumentRefreshSuccessSchema.safeParse(body).success).toBe(true);
+ expect((body as Record<string, unknown>).ok).toBeUndefined();
+ expect(body).toMatchObject({ checked: 1 });
+ });
+
+ test('an empty list emits problem+json invalid-request', async () => {
+ const res = await refresh({ docNames: [] });
+ expect(res.status).toBe(400);
+ expect(res.headers.get('content-type')).toBe('application/problem+json');
+
+ const parsed = ProblemDetailsSchema.safeParse(await res.json());
+ expect(parsed.success).toBe(true);
+ if (parsed.success) expect(parsed.data.type).toBe('urn:ok:error:invalid-request');
+ });
+
+ test('method-not-allowed on GET emits problem+json with Allow: POST', async () => {
+ const res = await refresh(undefined, 'GET');
+ expect(res.status).toBe(405);
+ expect(res.headers.get('allow')).toBe('POST');
+ expect(res.headers.get('content-type')).toBe('application/problem+json');
+
+ const parsed = ProblemDetailsSchema.safeParse(await res.json());
+ expect(parsed.success).toBe(true);
+ if (parsed.success) expect(parsed.data.type).toBe('urn:ok:error:method-not-allowed');
+ });
+});
diff --git a/packages/app/tests/integration/attribution-sweep-coverage.test.ts b/packages/app/tests/integration/attribution-sweep-coverage.test.ts
index eb80fc256..0425804a4 100644
--- a/packages/app/tests/integration/attribution-sweep-coverage.test.ts
+++ b/packages/app/tests/integration/attribution-sweep-coverage.test.ts
@@ -46,6 +46,7 @@ const REQUIRED_HANDLERS = [
const EXEMPT_HANDLERS = new Set([
'handleDocumentRead',
'handleDocumentList',
+ 'handleDocumentRefresh',
'handleAcpCatalog',
'handleAcpModels',
'handleEmbedDetect',
diff --git a/packages/app/tests/integration/conflict-gate-coverage.test.ts b/packages/app/tests/integration/conflict-gate-coverage.test.ts
index 21832b489..5af29c162 100644
--- a/packages/app/tests/integration/conflict-gate-coverage.test.ts
+++ b/packages/app/tests/integration/conflict-gate-coverage.test.ts
@@ -51,6 +51,7 @@ const REQUIRED_HANDLERS = [
const EXEMPT_HANDLERS = new Set([
'handleDocumentRead',
'handleDocumentList',
+ 'handleDocumentRefresh',
'handleAcpCatalog',
'handleAcpModels',
'handleAsset',
diff --git a/packages/app/tests/integration/freshness-recheck.test.ts b/packages/app/tests/integration/freshness-recheck.test.ts
new file mode 100644
index 000000000..e95925dec
--- /dev/null
+++ b/packages/app/tests/integration/freshness-recheck.test.ts
@@ -0,0 +1,196 @@
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import { getMetrics } from '../../../server/src/metrics';
+import { createManualClock, type ManualClock } from '../../../server/src/watcher-clock.test-helper';
+import { createWatcherDouble } from '../../../server/src/watcher-double.test-helper';
+import {
+ agentWriteMd,
+ createTestClient,
+ createTestServer,
+ pollUntil,
+ type TestClient,
+ type TestServer,
+} from './test-harness';
+
+const SEED = '# Doc\n\noriginal\n';
+
+let server: TestServer | undefined;
+const clients: TestClient[] = [];
+
+afterEach(async () => {
+ for (const client of clients.splice(0)) await client.cleanup();
+ await server?.cleanup();
+ server = undefined;
+});
+
+interface Rig {
+ server: TestServer;
+ double: ReturnType<typeof createWatcherDouble>;
+ clock: ManualClock;
+ docName: string;
+ file: string;
+ client: TestClient;
+}
+
+async function openSeededDocument(options: {
+ freshnessIntervalMs: number | null;
+ debounce?: number;
+ maxDebounce?: number;
+}): Promise<Rig> {
+ const double = createWatcherDouble();
+ const clock = createManualClock(Date.now());
+ server = await createTestServer({
+ watcherBackend: double.backend,
+ watcherClock: clock,
+ freshnessIntervalMs: options.freshnessIntervalMs,
+ debounce: options.debounce ?? 50,
+ maxDebounce: options.maxDebounce ?? 200,
+ });
+ const docName = `freshness-${randomUUID()}`;
+ const file = join(server.contentDir, `${docName}.md`);
+ await agentWriteMd(server.port, SEED, { docName, position: 'replace' });
+ await pollUntil(
+ () => existsSync(file) && readFileSync(file, 'utf-8') === SEED,
+ 5_000,
+ 25,
+ 'the seed write to reach disk',
+ );
+ const client = await createTestClient(server.port, docName);
+ clients.push(client);
+ await pollUntil(() => client.ytext.toString() === SEED, 5_000, 25, 'the client to load the seed');
+ return { server, double, clock, docName, file, client };
+}
+
+async function refresh(port: number, docNames: string[]): Promise<Response> {
+ return fetch(`http://127.0.0.1:${port}/api/documents/refresh`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docNames }),
+ });
+}
+
+describe('freshness through the server', () => {
+ test('with native events silenced, the timed re-check applies an outside write to an open document byte for byte', async () => {
+ const rig = await openSeededDocument({ freshnessIntervalMs: 5_000 });
+ const outside = '# Doc\n\nchanged outside, never delivered\n';
+ writeFileSync(rig.file, outside);
+ const findsBefore = getMetrics().freshnessFinds;
+
+ rig.clock.advance(5_000);
+ await pollUntil(
+ () => rig.client.ytext.toString() === outside,
+ 5_000,
+ 25,
+ 'the timed re-check to apply the outside bytes',
+ );
+ expect(readFileSync(rig.file, 'utf-8')).toBe(outside);
+ expect(getMetrics().freshnessFinds - findsBefore).toBeGreaterThanOrEqual(1);
+ });
+
+ test('after a watcher error the server subscribes again, applies the write made in the gap, and takes later events', async () => {
+ const rig = await openSeededDocument({ freshnessIntervalMs: null });
+ const first = rig.double.subscriptionFor(rig.server.contentDir);
+ first.fail(new Error('the native watcher died'));
+ const gap = '# Doc\n\nwritten while the watcher was dead\n';
+ writeFileSync(rig.file, gap);
+
+ rig.clock.advance(0);
+ await pollUntil(
+ () => rig.client.ytext.toString() === gap,
+ 5_000,
+ 25,
+ 'the resubscribe re-check to apply the gap write',
+ );
+ expect(rig.double.subscriptionCount(rig.server.contentDir)).toBe(2);
+
+ const later = '# Doc\n\ndelivered by the new subscription\n';
+ writeFileSync(rig.file, later);
+ rig.double.subscriptionFor(rig.server.contentDir).deliver([{ type: 'update', path: rig.file }]);
+ await pollUntil(
+ () => rig.client.ytext.toString() === later,
+ 5_000,
+ 25,
+ 'the new subscription to deliver the later write',
+ );
+ });
+
+ test('a focus re-read merges an outside change made while the window was unfocused with unsaved typing', async () => {
+ const rig = await openSeededDocument({
+ freshnessIntervalMs: null,
+ debounce: 60_000,
+ maxDebounce: 120_000,
+ });
+ rig.client.ytext.insert(rig.client.ytext.length, '\nlive typing\n');
+ await pollUntil(
+ () =>
+ rig.server.instance.hocuspocus.documents
+ .get(rig.docName)
+ ?.getText('source')
+ .toString()
+ .includes('live typing') ?? false,
+ 5_000,
+ 25,
+ 'the typing to reach the server',
+ );
+ writeFileSync(rig.file, '# Changed outside\n\noriginal\n');
+
+ const response = await refresh(rig.server.port, [rig.docName]);
+ expect(response.status).toBe(200);
+ expect(await response.json()).toMatchObject({ checked: 1, found: 1 });
+ await pollUntil(
+ () => rig.client.ytext.toString().includes('# Changed outside'),
+ 5_000,
+ 25,
+ 'the focus re-read to merge the outside change',
+ );
+ expect(rig.client.ytext.toString()).toContain('live typing');
+ expect(rig.client.ytext.toString()).not.toContain('<<<<<<<');
+ });
+
+ test('a client that reconnects to an open document gets the outside change made while it was away', async () => {
+ const rig = await openSeededDocument({ freshnessIntervalMs: null });
+ const away = await createTestClient(rig.server.port, rig.docName);
+ clients.push(away);
+ await pollUntil(() => away.ytext.toString() === SEED, 5_000, 25, 'the second client to load');
+
+ let disconnected = false;
+ away.provider.on('disconnect', () => {
+ disconnected = true;
+ });
+ away.provider.disconnect();
+ await pollUntil(() => disconnected, 5_000, 25, 'the second client to disconnect');
+
+ const changed = '# Doc\n\nchanged while the client was away\n';
+ writeFileSync(rig.file, changed);
+ rig.clock.advance(1_000);
+ let resynced = false;
+ away.provider.on('synced', () => {
+ resynced = true;
+ });
+ await away.provider.connect();
+ await pollUntil(() => resynced, 10_000, 25, 'the second client to sync again');
+
+ await pollUntil(
+ () => away.ytext.toString() === changed,
+ 5_000,
+ 25,
+ 'the reconnect re-read to apply the outside change',
+ );
+ expect(rig.client.ytext.toString()).toBe(changed);
+ });
+
+ test('the refresh route skips the system document and names outside the content tree', async () => {
+ const rig = await openSeededDocument({ freshnessIntervalMs: null });
+ const response = await refresh(rig.server.port, ['__system__', '../outside', 'not-there']);
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual({ checked: 1, found: 0 });
+ });
+
+ test('the refresh route refuses an empty list', async () => {
+ const rig = await openSeededDocument({ freshnessIntervalMs: null });
+ const response = await refresh(rig.server.port, []);
+ expect(response.status).toBe(400);
+ });
+});
diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.ts
index 6707743b4..d68bbbcb1 100644
--- a/packages/app/tests/integration/test-harness.ts
+++ b/packages/app/tests/integration/test-harness.ts
@@ -110,6 +110,7 @@ export interface CreateTestServerOptions {
seedSkillHostRoot?: boolean;
watcherBackend?: ServerOptions['watcherBackend'];
watcherClock?: ServerOptions['watcherClock'];
+ freshnessIntervalMs?: ServerOptions['freshnessIntervalMs'];
mergeHistoryWriter?: ServerOptions['mergeHistoryWriter'];
}
@@ -189,6 +190,7 @@ export async function createTestServer(options: CreateTestServerOptions = {}): P
agentSessionOptions: options.agentSessionOptions,
watcherBackend: options.watcherBackend,
watcherClock: options.watcherClock,
+ freshnessIntervalMs: options.freshnessIntervalMs ?? null,
mergeHistoryWriter: options.mergeHistoryWriter,
...(ephemeral ? { ephemeral: true, singleDocRelPath: options.singleDocRelPath } : {}),
skipStateManifestCheck: true,
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 63b0fba52..dd5cfe5f9 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1332,6 +1332,7 @@ export {
DismissDocNoticeSuccessSchema,
type DispatchPayload,
DispatchPayloadSchema,
+ DOCUMENT_REFRESH_MAX_DOCS,
DocNoticesSuccessSchema,
type DocNoticeWire,
DocNoticeWireSchema,
@@ -1341,6 +1342,10 @@ export {
DocumentListSuccessSchema,
type DocumentReadSuccess,
DocumentReadSuccessSchema,
+ type DocumentRefreshRequest,
+ DocumentRefreshRequestSchema,
+ type DocumentRefreshSuccess,
+ DocumentRefreshSuccessSchema,
type DuplicatePathRequest,
DuplicatePathRequestSchema,
type DuplicatePathSuccess,
diff --git a/packages/core/src/schemas/api/document-read.ts b/packages/core/src/schemas/api/document-read.ts
index cb218bc1c..c26ff005d 100644
--- a/packages/core/src/schemas/api/document-read.ts
+++ b/packages/core/src/schemas/api/document-read.ts
@@ -228,3 +228,20 @@ export const LinkGraphSuccessSchema = z
})
.loose() satisfies StandardSchemaV1;
export type LinkGraphSuccess = z.infer<typeof LinkGraphSuccessSchema>;
+
+export const DOCUMENT_REFRESH_MAX_DOCS = 64;
+
+export const DocumentRefreshRequestSchema = z
+ .object({
+ docNames: z.array(z.string().min(1).max(1024)).min(1).max(DOCUMENT_REFRESH_MAX_DOCS),
+ })
+ .loose() satisfies StandardSchemaV1;
+export type DocumentRefreshRequest = z.infer<typeof DocumentRefreshRequestSchema>;
+
+export const DocumentRefreshSuccessSchema = z
+ .object({
+ checked: z.number().int().nonnegative(),
+ found: z.number().int().nonnegative(),
+ })
+ .loose() satisfies StandardSchemaV1;
+export type DocumentRefreshSuccess = z.infer<typeof DocumentRefreshSuccessSchema>;
diff --git a/packages/server/src/api-agent-write-batch-persistence-failure.test.ts b/packages/server/src/api-agent-write-batch-persistence-failure.test.ts
index 02c5ce555..d00005939 100644
--- a/packages/server/src/api-agent-write-batch-persistence-failure.test.ts
+++ b/packages/server/src/api-agent-write-batch-persistence-failure.test.ts
@@ -73,6 +73,7 @@ afterEach(async () => {
function createExtension() {
return createApiExtension({
hocuspocus,
+ refreshDocumentsFromDisk: async () => ({ checked: 0, found: 0 }),
durabilityState,
sessionManager,
contentDir,
diff --git a/packages/server/src/api-extension.test-helper.ts b/packages/server/src/api-extension.test-helper.ts
index 725909503..2a86226e3 100644
--- a/packages/server/src/api-extension.test-helper.ts
+++ b/packages/server/src/api-extension.test-helper.ts
@@ -252,6 +252,7 @@ export function createApiExtension(
| 'conflicts'
| 'getProjectConfigEpoch'
| 'getLinkAdvisoryPolicy'
+ | 'refreshDocumentsFromDisk'
> &
LegacyIndexOptions & {
conflicts?: ApiExtensionOptions['conflicts'];
@@ -260,6 +261,7 @@ export function createApiExtension(
signalChannel?: LegacySignalChannel;
getProjectConfigEpoch?: () => number;
getLinkAdvisoryPolicy?: () => LinkAdvisoryPolicy;
+ refreshDocumentsFromDisk?: ApiExtensionOptions['refreshDocumentsFromDisk'];
},
): ReturnType<typeof createApiExtensionBase> {
const {
@@ -268,6 +270,7 @@ export function createApiExtension(
derivedDocumentIndex,
durabilityState = new DocumentDurabilityState(),
getProjectConfigEpoch = () => 0,
+ refreshDocumentsFromDisk = async () => ({ checked: 0, found: 0 }),
getLinkAdvisoryPolicy = () => ({
links: DEFAULT_LINKS_VALIDATION,
suppressLogLinkAdvisories: DEFAULT_SUPPRESS_LOG_LINK_ADVISORIES,
@@ -277,6 +280,7 @@ export function createApiExtension(
const pendingLegacyWrites = new Set<Promise<unknown>>();
const extension = createApiExtensionBase({
...apiOptions,
+ refreshDocumentsFromDisk,
getProjectConfigEpoch,
getLinkAdvisoryPolicy,
conflicts:
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index 9d01d89e3..6aa0d3ae4 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -234,6 +234,7 @@ import {
OK_STORE_REFUSED,
type StoreFailure,
} from './document-durability-state.ts';
+import type { RefreshDocumentsFromDisk } from './document-refresh.ts';
import {
buildReconcileWarning,
reconcileDiskBeforeAgentWrite,
@@ -272,6 +273,7 @@ import { createAssetRoutes } from './http/asset-routes.ts';
import { createCommentRoutes } from './http/comment-routes.ts';
import { createConfigSystemRoutes } from './http/config-system-routes.ts';
import { createDocNoticeRoutes } from './http/doc-notice-routes.ts';
+import { createDocumentRefreshRoutes } from './http/document-refresh-routes.ts';
import { createDocumentRoutes } from './http/document-routes.ts';
import {
type ErrorExtensions,
@@ -1334,6 +1336,7 @@ interface ApiExtensionBaseOptions {
resolveSyncCredentialConfig?: () => Promise<string[]>;
conflicts: ConflictAuthority;
setBatchInProgress?: (value: boolean) => void;
+ refreshDocumentsFromDisk: RefreshDocumentsFromDisk;
localOpCliArgs?: string[];
authStreamHeartbeatMs?: number;
projectDir?: string;
@@ -5476,6 +5479,9 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {
agentIntegrations,
});
const handoffInstallRoutes = createHandoffInstallRoutes({ checkLocalOpSecurity });
+ const documentRefreshRoutes = createDocumentRefreshRoutes({
+ refreshDocumentsFromDisk: options.refreshDocumentsFromDisk,
+ });
const skillsInstallRoutes = createSkillsInstallRoutes({
resolveSkillsRoot,
validateSkillName,
@@ -5613,6 +5619,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {
linkGraphRoutes,
metricsRoutes,
documentRoutes,
+ documentRefreshRoutes,
configSystemRoutes,
lintRoutes,
lintWriteRoutes,
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index cef298c29..39a77051d 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -156,6 +156,7 @@ export interface BootServerOptions
| 'acpRegistryFetchImpl'
| 'watcherBackend'
| 'watcherClock'
+ | 'freshnessIntervalMs'
> {
config: Config;
skipAutoInit?: boolean;
@@ -417,6 +418,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
configHomedirOverride: opts.configHomedirOverride,
watcherBackend: opts.watcherBackend,
watcherClock: opts.watcherClock,
+ freshnessIntervalMs: opts.freshnessIntervalMs,
});
const {
diff --git a/packages/server/src/document-refresh.test.ts b/packages/server/src/document-refresh.test.ts
new file mode 100644
index 000000000..48a61d1a9
--- /dev/null
+++ b/packages/server/src/document-refresh.test.ts
@@ -0,0 +1,51 @@
+import { CONFIG_DOC_NAMES } from '@inkeep/open-knowledge-core';
+import { describe, expect, test, vi } from 'vitest';
+import { SYSTEM_DOC_NAME } from './cc1-broadcast.ts';
+import {
+ createReconnectRefreshExtension,
+ type RefreshDocumentsFromDisk,
+} from './document-refresh.ts';
+import { createManualClock } from './watcher-clock.test-helper.ts';
+
+type HookPayload = { documentName: string };
+
+function extensionWith(refresh: RefreshDocumentsFromDisk) {
+ const clock = createManualClock(1_000_000);
+ const extension = createReconnectRefreshExtension({ clock, refresh });
+ const hooks = extension as unknown as {
+ afterLoadDocument(payload: HookPayload): Promise<void>;
+ connected(payload: HookPayload): Promise<void>;
+ };
+ return { clock, hooks };
+}
+
+describe('re-read on reconnect', () => {
+ test('a connection more than a second after the last read re-reads that document', async () => {
+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => ({ checked: 1, found: 0 }));
+ const { clock, hooks } = extensionWith(refresh);
+
+ await hooks.afterLoadDocument({ documentName: 'notes' });
+ await hooks.connected({ documentName: 'notes' });
+ expect(refresh).not.toHaveBeenCalled();
+
+ clock.advance(1_000);
+ await hooks.connected({ documentName: 'notes' });
+ expect(refresh).toHaveBeenCalledWith(['notes'], 'reconnect');
+
+ await hooks.connected({ documentName: 'notes' });
+ expect(refresh).toHaveBeenCalledTimes(1);
+ });
+
+ test('the system document and config documents are never re-read', async () => {
+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => ({ checked: 0, found: 0 }));
+ const { clock, hooks } = extensionWith(refresh);
+ const reserved = [SYSTEM_DOC_NAME, ...CONFIG_DOC_NAMES];
+
+ for (const documentName of reserved) await hooks.afterLoadDocument({ documentName });
+ clock.advance(5_000);
+ for (const documentName of reserved) await hooks.connected({ documentName });
+
+ expect(reserved.length).toBeGreaterThan(1);
+ expect(refresh).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/server/src/document-refresh.ts b/packages/server/src/document-refresh.ts
new file mode 100644
index 000000000..c377a5ae4
--- /dev/null
+++ b/packages/server/src/document-refresh.ts
@@ -0,0 +1,49 @@
+import type { Extension } from '@hocuspocus/server';
+import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts';
+import { getLogger } from './logger.ts';
+import type { WatcherClock } from './watcher-clock.ts';
+
+const log = getLogger('document-refresh');
+
+const RECONNECT_REFRESH_COALESCE_MS = 1_000;
+
+const isRefreshable = (documentName: string): boolean =>
+ !isSystemDoc(documentName) && !isConfigDoc(documentName);
+
+export type DocumentRefreshTrigger = 'focus' | 'reconnect';
+
+export interface DocumentRefreshResult {
+ checked: number;
+ found: number;
+}
+
+export type RefreshDocumentsFromDisk = (
+ docNames: readonly string[],
+ trigger: DocumentRefreshTrigger,
+) => Promise<DocumentRefreshResult>;
+
+export function createReconnectRefreshExtension(deps: {
+ clock: WatcherClock;
+ refresh: RefreshDocumentsFromDisk;
+}): Extension {
+ const lastReadAt = new Map<string, number>();
+ return {
+ async afterLoadDocument({ documentName }) {
+ if (!isRefreshable(documentName)) return;
+ lastReadAt.set(documentName, deps.clock.now());
+ },
+ async connected({ documentName }) {
+ if (!isRefreshable(documentName)) return;
+ const now = deps.clock.now();
+ const last = lastReadAt.get(documentName);
+ if (last !== undefined && now - last < RECONNECT_REFRESH_COALESCE_MS) return;
+ lastReadAt.set(documentName, now);
+ void deps.refresh([documentName], 'reconnect').catch((err: unknown) => {
+ log.warn({ err, docName: documentName }, 'the re-read on reconnect failed');
+ });
+ },
+ async afterUnloadDocument({ documentName }) {
+ lastReadAt.delete(documentName);
+ },
+ };
+}
diff --git a/packages/server/src/file-watcher-freshness.test.ts b/packages/server/src/file-watcher-freshness.test.ts
new file mode 100644
index 000000000..be1ca6747
--- /dev/null
+++ b/packages/server/src/file-watcher-freshness.test.ts
@@ -0,0 +1,636 @@
+import { execFileSync } from 'node:child_process';
+import {
+ chmodSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ statSync,
+ symlinkSync,
+ utimesSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { createContentFilter } from './content-filter.ts';
+import {
+ contentHash,
+ type DiskEvent,
+ lastKnownHash,
+ removeLastKnownHash,
+ type StartWatcherOptions,
+ startWatcher,
+ type WatcherHandle,
+} from './file-watcher.ts';
+import { scanInPlaceSkillDirs, skillRootPathsFor } from './in-place-skills.ts';
+import { getMetrics } from './metrics.ts';
+import { createManualClock, type ManualClock } from './watcher-clock.test-helper.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+import {
+ FRESHNESS_HASH_BUDGET,
+ FRESHNESS_MAX_INTERVAL_MS,
+ nextFreshnessDelayMs,
+} from './watcher-freshness.ts';
+
+const SETTLED_TIMESTAMPS_MS = 10_000;
+
+function readCanBeRevoked(): boolean {
+ if (process.platform === 'win32') return false;
+ const dir = mkdtempSync(join(tmpdir(), 'ok-revoke-probe-'));
+ const probe = join(dir, 'probe');
+ try {
+ writeFileSync(probe, 'probe');
+ chmodSync(probe, 0o000);
+ try {
+ readFileSync(probe);
+ return false;
+ } catch (err) {
+ if ((err as NodeJS.ErrnoException).code === 'EACCES') return true;
+ throw err;
+ }
+ } finally {
+ chmodSync(probe, 0o600);
+ rmSync(dir, { recursive: true, force: true });
+ }
+}
+
+const CAN_REVOKE_READ = readCanBeRevoked();
+
+const walkGate = vi.hoisted(() => ({
+ dir: null as string | null,
+ afterRead: null as string | null,
+ entered: false,
+ release: null as (() => void) | null,
+ directoryReads: new Map<string, number>(),
+}));
+
+vi.mock('node:fs/promises', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('node:fs/promises')>();
+ const readdir = actual.readdir as (...args: unknown[]) => Promise<unknown>;
+ const readFile = actual.readFile as (...args: unknown[]) => Promise<unknown>;
+ const hold = async (): Promise<void> => {
+ walkGate.entered = true;
+ await new Promise<void>((resolve) => {
+ walkGate.release = resolve;
+ });
+ };
+ return {
+ ...actual,
+ readdir: async (...args: unknown[]) => {
+ const dir = String(args[0]);
+ walkGate.directoryReads.set(dir, (walkGate.directoryReads.get(dir) ?? 0) + 1);
+ if (walkGate.dir !== null && dir === walkGate.dir) {
+ walkGate.dir = null;
+ await hold();
+ }
+ return readdir(...args);
+ },
+ readFile: async (...args: unknown[]) => {
+ const content = await readFile(...args);
+ if (walkGate.afterRead !== null && String(args[0]) === walkGate.afterRead) {
+ walkGate.afterRead = null;
+ await hold();
+ }
+ return content;
+ },
+ };
+});
+
+async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
+ const start = performance.now();
+ while (!condition()) {
+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+function eventsFor(events: readonly DiskEvent[], kind: DiskEvent['kind'], docName: string) {
+ return events.filter(
+ (event) => event.kind === kind && 'docName' in event && event.docName === docName,
+ );
+}
+
+describe('timed freshness re-check', () => {
+ let tmpDir: string;
+ let contentDir: string;
+ let handle: WatcherHandle | undefined;
+ let clock: ManualClock;
+ let events: DiskEvent[];
+
+ beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-freshness-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ contentDir = realpathSync(contentDir);
+ lastKnownHash.clear();
+ clock = createManualClock(Date.now());
+ events = [];
+ });
+
+ afterEach(async () => {
+ walkGate.release?.();
+ walkGate.dir = null;
+ walkGate.afterRead = null;
+ walkGate.entered = false;
+ walkGate.release = null;
+ walkGate.directoryReads.clear();
+ await handle?.unsubscribe();
+ handle = undefined;
+ vi.useRealTimers();
+ for (const locked of revoked.splice(0)) chmodSync(locked, 0o755);
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ const revoked: string[] = [];
+ function revokeRead(path: string): void {
+ chmodSync(path, 0o000);
+ revoked.push(path);
+ }
+
+ async function flush(): Promise<void> {
+ for (let round = 0; round < 5; round++) {
+ await new Promise((resolve) => setImmediate(resolve));
+ }
+ }
+
+ let double: ReturnType<typeof createWatcherDouble>;
+
+ async function watch(
+ freshnessIntervalMs?: number | null,
+ extra: Pick<StartWatcherOptions, 'onRecoveryComplete'> = {},
+ ): Promise<WatcherHandle> {
+ double = createWatcherDouble();
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ {
+ nativeBackend: double.backend,
+ clock,
+ ...(freshnessIntervalMs === undefined ? {} : { freshnessIntervalMs }),
+ ...extra,
+ },
+ );
+ return handle;
+ }
+
+ function moveModificationTimeLater(path: string): void {
+ const { atime, mtime } = statSync(path);
+ utimesSync(path, atime, new Date(mtime.getTime() + 2_000));
+ }
+
+ function keepModificationTime(path: string, write: () => void): void {
+ const { atime, mtime } = statSync(path);
+ write();
+ utimesSync(path, atime, mtime);
+ }
+
+ async function runTimedPass(advanceMs: number): Promise<void> {
+ const before = getMetrics().freshnessPasses;
+ clock.advance(advanceMs);
+ await until(() => getMetrics().freshnessPasses > before, 'the timed pass to finish');
+ }
+
+ function settleTimestamps(): void {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime(Date.now() + SETTLED_TIMESTAMPS_MS);
+ }
+
+ function freezeTimestampsNow(): void {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ }
+
+ test('a write the native watcher never delivered is applied by the next timed pass', async () => {
+ await watch(5_000);
+ writeFileSync(join(contentDir, 'silent.md'), '# Silent\n');
+ const findsBefore = getMetrics().freshnessFinds;
+
+ clock.advance(4_999);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(eventsFor(events, 'create', 'silent')).toHaveLength(0);
+
+ await runTimedPass(1);
+ expect(eventsFor(events, 'create', 'silent')).toHaveLength(1);
+ expect(handle?.getFileIndex().has('silent')).toBe(true);
+ expect(getMetrics().freshnessFinds - findsBefore).toBe(1);
+ });
+
+ test('a watcher started without an interval option runs no timed pass', async () => {
+ await watch();
+ writeFileSync(join(contentDir, 'library-default.md'), '# Library default\n');
+ const passesBefore = getMetrics().freshnessPasses;
+ clock.advance(FRESHNESS_MAX_INTERVAL_MS * 2);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(getMetrics().freshnessPasses).toBe(passesBefore);
+ expect(eventsFor(events, 'create', 'library-default')).toHaveLength(0);
+ });
+
+ test('a null interval turns the timed pass off, and an explicit re-check still finds the write', async () => {
+ await watch(null);
+ writeFileSync(join(contentDir, 'off.md'), '# Off\n');
+ const passesBefore = getMetrics().freshnessPasses;
+ clock.advance(FRESHNESS_MAX_INTERVAL_MS * 2);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(getMetrics().freshnessPasses).toBe(passesBefore);
+ expect(eventsFor(events, 'create', 'off')).toHaveLength(0);
+
+ const result = await (handle as WatcherHandle).recheck();
+ expect(result.status).toBe('completed');
+ expect(eventsFor(events, 'create', 'off')).toHaveLength(1);
+ });
+
+ test('a pass reads only the files whose size, time or identity changed', async () => {
+ writeFileSync(join(contentDir, 'one.md'), '# One\n');
+ writeFileSync(join(contentDir, 'two.md'), '# Two\n');
+ writeFileSync(join(contentDir, 'three.md'), '# Three\n');
+ settleTimestamps();
+ await watch(5_000);
+
+ const hashedBefore = getMetrics().freshnessFilesHashed;
+ await runTimedPass(5_000);
+ expect(getMetrics().freshnessFilesHashed - hashedBefore).toBe(0);
+ expect(events).toHaveLength(0);
+
+ writeFileSync(join(contentDir, 'two.md'), '# Two, edited outside\n');
+ const hashedBeforeEdit = getMetrics().freshnessFilesHashed;
+ await runTimedPass(5_000);
+ expect(getMetrics().freshnessFilesHashed - hashedBeforeEdit).toBe(1);
+ expect(eventsFor(events, 'update', 'two')).toHaveLength(1);
+ expect(events).toHaveLength(1);
+
+ const hashedAfterIntake = getMetrics().freshnessFilesHashed;
+ await runTimedPass(5_000);
+ expect(getMetrics().freshnessFilesHashed - hashedAfterIntake).toBe(0);
+ });
+
+ test('a file changed within the timestamp granularity is read again until its times settle', async () => {
+ freezeTimestampsNow();
+ writeFileSync(join(contentDir, 'fresh.md'), '# Fresh\n');
+ await watch(5_000);
+
+ const hashedBefore = getMetrics().freshnessFilesHashed;
+ await runTimedPass(5_000);
+ await runTimedPass(5_000);
+ expect(getMetrics().freshnessFilesHashed - hashedBefore).toBe(2);
+
+ settleTimestamps();
+ await runTimedPass(5_000);
+ const hashedAfterSettle = getMetrics().freshnessFilesHashed;
+ await runTimedPass(5_000);
+ expect(getMetrics().freshnessFilesHashed - hashedAfterSettle).toBe(0);
+ expect(events).toHaveLength(0);
+ });
+
+ test('changes beyond one pass budget are carried to the next pass', async () => {
+ settleTimestamps();
+ await watch(5_000);
+ const total = FRESHNESS_HASH_BUDGET.files + 40;
+ for (let index = 0; index < total; index++) {
+ writeFileSync(join(contentDir, `bulk-${String(index).padStart(4, '0')}.md`), `# ${index}\n`);
+ }
+
+ const carriedBefore = getMetrics().freshnessCarryOver;
+ await runTimedPass(5_000);
+ expect(events.filter((event) => event.kind === 'create')).toHaveLength(
+ FRESHNESS_HASH_BUDGET.files,
+ );
+ expect(getMetrics().freshnessCarryOver - carriedBefore).toBe(40);
+
+ await runTimedPass(5_000);
+ expect(events.filter((event) => event.kind === 'create')).toHaveLength(total);
+ });
+
+ test('an edit that keeps the file size and moves the modification time is found by the timed pass', async () => {
+ const same = join(contentDir, 'same.md');
+ writeFileSync(same, '# AAAA\n');
+ settleTimestamps();
+ await watch(5_000);
+
+ writeFileSync(same, '# BBBB\n');
+ moveModificationTimeLater(same);
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'update', 'same')).toHaveLength(1);
+ });
+
+ test('an edit that keeps the file size and the modification time is still found by the timed pass', async () => {
+ const same = join(contentDir, 'same.md');
+ writeFileSync(same, '# AAAA\n');
+ await watch(5_000);
+
+ keepModificationTime(same, () => writeFileSync(same, '# BBBB\n'));
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'update', 'same')).toHaveLength(1);
+ });
+
+ test.runIf(CAN_REVOKE_READ).each([
+ ['a later modification time', true],
+ ['an unchanged modification time', false],
+ ] as const)(
+ 'a file that cannot be read, with %s, does not stop the pass from applying a change beside it',
+ async (_label, settled) => {
+ const locked = join(contentDir, 'locked.md');
+ writeFileSync(locked, '# Locked\n');
+ writeFileSync(join(contentDir, 'open.md'), '# Open\n');
+ if (settled) settleTimestamps();
+ const reconciled = vi.fn(async () => {});
+ await watch(5_000, { onRecoveryComplete: reconciled });
+ const unreadableBefore = getMetrics().watcherUnreadableEntries;
+
+ if (settled) {
+ revokeRead(locked);
+ moveModificationTimeLater(locked);
+ } else {
+ keepModificationTime(locked, () => revokeRead(locked));
+ }
+ writeFileSync(join(contentDir, 'open.md'), '# Open, edited outside\n');
+ reconciled.mockClear();
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'update', 'open')).toHaveLength(1);
+ expect(eventsFor(events, 'delete', 'locked')).toHaveLength(0);
+ expect(handle?.getFileIndex().has('locked')).toBe(true);
+ expect(reconciled).not.toHaveBeenCalled();
+
+ writeFileSync(join(contentDir, 'open.md'), '# Open, edited again\n');
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'update', 'open')).toHaveLength(2);
+ expect(getMetrics().watcherUnreadableEntries - unreadableBefore).toBe(1);
+
+ chmodSync(locked, 0o644);
+ moveModificationTimeLater(locked);
+ await runTimedPass(5_000);
+ expect(reconciled).toHaveBeenCalledTimes(1);
+ },
+ );
+
+ test.runIf(CAN_REVOKE_READ)(
+ 'a folder that cannot be listed keeps its documents while the rest of the tree is checked',
+ async () => {
+ mkdirSync(join(contentDir, 'private'));
+ writeFileSync(join(contentDir, 'private', 'inside.md'), '# Inside\n');
+ writeFileSync(join(contentDir, 'open.md'), '# Open\n');
+ settleTimestamps();
+ await watch(5_000);
+
+ revokeRead(join(contentDir, 'private'));
+ writeFileSync(join(contentDir, 'open.md'), '# Open, edited outside\n');
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'update', 'open')).toHaveLength(1);
+ expect(events.filter((event) => event.kind === 'delete')).toHaveLength(0);
+ expect(events.filter((event) => event.kind === 'folder-delete')).toHaveLength(0);
+ expect(handle?.getFileIndex().has('private/inside')).toBe(true);
+ },
+ );
+
+ test('an event the native watcher delivers while a timed pass walks the tree is applied before the pass ends', async () => {
+ mkdirSync(join(contentDir, 'slow'));
+ writeFileSync(join(contentDir, 'slow', 'a.md'), '# A\n');
+ settleTimestamps();
+ const double = createWatcherDouble();
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { nativeBackend: double.backend, clock, freshnessIntervalMs: 5_000 },
+ );
+ const passesBefore = getMetrics().freshnessPasses;
+ walkGate.dir = join(contentDir, 'slow');
+ clock.advance(5_000);
+ await until(() => walkGate.entered, 'the timed pass to reach the slow folder');
+
+ const delivered = join(contentDir, 'delivered.md');
+ writeFileSync(delivered, '# Delivered\n');
+ double.subscriptionFor(contentDir).deliver([{ type: 'create', path: delivered }]);
+ await until(
+ () => eventsFor(events, 'create', 'delivered').length === 1,
+ 'the delivered event while the pass is still walking',
+ );
+ expect(getMetrics().freshnessPasses).toBe(passesBefore);
+
+ walkGate.release?.();
+ await until(() => getMetrics().freshnessPasses > passesBefore, 'the timed pass to finish');
+ expect(eventsFor(events, 'create', 'delivered')).toHaveLength(1);
+ });
+
+ test('a skill folder added outside the app between passes is admitted by the next timed pass', async () => {
+ const filter = createContentFilter({
+ projectDir: contentDir,
+ contentDir,
+ inPlaceSkillDirs: scanInPlaceSkillDirs(contentDir),
+ rescanInPlaceSkillDirs: () => scanInPlaceSkillDirs(contentDir),
+ skillRootPaths: skillRootPathsFor(contentDir),
+ });
+ double = createWatcherDouble();
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ filter,
+ { nativeBackend: double.backend, clock, freshnessIntervalMs: 5_000 },
+ );
+ const bundle = join(contentDir, '.claude', 'skills', 'added-later');
+ mkdirSync(bundle, { recursive: true });
+ writeFileSync(
+ join(bundle, 'SKILL.md'),
+ '---\nname: added-later\ndescription: fixture\n---\n\n# Added later\n',
+ );
+ expect(filter.isExcluded('.claude/skills/added-later/SKILL.md')).toBe(true);
+
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'create', '.claude/skills/added-later/SKILL')).toHaveLength(1);
+ expect(handle.getFileIndex().has('.claude/skills/added-later/SKILL')).toBe(true);
+ });
+
+ test('a change intake applies while a timed pass walks is kept, and the pass walks the tree once', async () => {
+ const raced = join(contentDir, 'raced.md');
+ writeFileSync(raced, '# Raced v1\n');
+ settleTimestamps();
+ await watch(5_000);
+ writeFileSync(raced, '# Raced v2, edited outside\n');
+ moveModificationTimeLater(raced);
+ walkGate.afterRead = raced;
+ walkGate.directoryReads.clear();
+ const passesBefore = getMetrics().freshnessPasses;
+ clock.advance(5_000);
+ await until(() => walkGate.entered, 'the timed pass to read the edited file');
+
+ const latest = '# Raced v3, edited again while the pass walks\n';
+ writeFileSync(raced, latest);
+ double.subscriptionFor(contentDir).deliver([{ type: 'update', path: raced }]);
+ await until(() => lastKnownHash.get(raced) === contentHash(latest), 'intake to apply v3');
+ walkGate.release?.();
+ await until(() => getMetrics().freshnessPasses > passesBefore, 'the timed pass to finish');
+
+ expect(walkGate.directoryReads.get(contentDir)).toBe(1);
+ expect(lastKnownHash.get(raced)).toBe(contentHash(latest));
+ expect(eventsFor(events, 'update', 'raced')).toHaveLength(1);
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'update', 'raced')).toHaveLength(1);
+ });
+
+ test('a Markdown link pointed at another file outside the app is followed by the timed pass', async () => {
+ writeFileSync(join(contentDir, 'first.md'), '# First\n');
+ writeFileSync(join(contentDir, 'second.md'), '# Second\n');
+ const pointer = join(contentDir, 'pointer.md');
+ symlinkSync(join(contentDir, 'first.md'), pointer);
+ settleTimestamps();
+ await watch(5_000);
+ expect(handle?.getAliasMap().get('pointer')).toBe('first');
+
+ rmSync(pointer);
+ symlinkSync(join(contentDir, 'second.md'), pointer);
+ await runTimedPass(5_000);
+ expect(handle?.getAliasMap().get('pointer')).toBe('second');
+ });
+
+ test('a re-read while the content folder is missing reports no document deleted', async () => {
+ const kept = join(contentDir, 'kept.md');
+ writeFileSync(kept, '# Kept\n');
+ await watch(null);
+
+ rmSync(contentDir, { recursive: true, force: true });
+ const result = await (handle as WatcherHandle).recheckPaths([kept]);
+ expect(result).toEqual({ checked: 0, found: 0 });
+ expect(events.filter((event) => event.kind === 'delete')).toHaveLength(0);
+ expect(handle?.getFileIndex().has('kept')).toBe(true);
+ });
+
+ test('a re-read under an intact content folder reports a file deleted outside the app', async () => {
+ const gone = join(contentDir, 'gone.md');
+ writeFileSync(gone, '# Gone\n');
+ await watch(null);
+
+ rmSync(gone);
+ const result = await (handle as WatcherHandle).recheckPaths([gone]);
+ expect(result).toEqual({ checked: 1, found: 1 });
+ expect(eventsFor(events, 'delete', 'gone')).toHaveLength(1);
+ });
+
+ test('stopping the watcher while a timed pass walks the tree waits for that pass, which counts nothing afterwards', async () => {
+ mkdirSync(join(contentDir, 'slow'));
+ writeFileSync(join(contentDir, 'slow', 'a.md'), '# A\n');
+ settleTimestamps();
+ const watching = await watch(5_000);
+ const passesBefore = getMetrics().freshnessPasses;
+ const incompleteBefore = getMetrics().freshnessPassesIncomplete;
+ walkGate.dir = join(contentDir, 'slow');
+ clock.advance(5_000);
+ await until(() => walkGate.entered, 'the timed pass to reach the slow folder');
+
+ let stopped = false;
+ handle = undefined;
+ const stopping = watching.unsubscribe().then(() => {
+ stopped = true;
+ });
+ await flush();
+ expect(stopped).toBe(false);
+ walkGate.release?.();
+ await stopping;
+
+ expect(getMetrics().freshnessPasses).toBe(passesBefore);
+ expect(getMetrics().freshnessPassesIncomplete).toBe(incompleteBefore);
+ expect(lastKnownHash.size).toBe(0);
+ });
+
+ test('a full re-check requested while a timed recovery is queued runs that recovery in full', async () => {
+ settleTimestamps();
+ const double = createWatcherDouble();
+ let releaseQueue: () => void = () => {};
+ const queueHeld = new Promise<void>((resolve) => {
+ releaseQueue = resolve;
+ });
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ if (event.kind === 'create' && event.docName === 'blocker') await queueHeld;
+ },
+ undefined,
+ { nativeBackend: double.backend, clock, freshnessIntervalMs: 5_000 },
+ );
+ const blocker = join(contentDir, 'blocker.md');
+ writeFileSync(blocker, '# Blocker\n');
+ double.subscriptionFor(contentDir).deliver([{ type: 'create', path: blocker }]);
+ await until(() => eventsFor(events, 'create', 'blocker').length === 1, 'the queue to be held');
+
+ const total = FRESHNESS_HASH_BUDGET.files + 40;
+ for (let index = 0; index < total; index++) {
+ writeFileSync(join(contentDir, `bulk-${String(index).padStart(4, '0')}.md`), `# ${index}\n`);
+ }
+ const hashedBefore = getMetrics().freshnessFilesHashed;
+ clock.advance(5_000);
+ await until(() => getMetrics().freshnessFilesHashed > hashedBefore, 'the timed probe');
+ await flush();
+
+ const full = (handle as WatcherHandle).recheck();
+ releaseQueue();
+ expect((await full).status).toBe('completed');
+ const bulkCreates = events.filter(
+ (event) => event.kind === 'create' && event.docName.startsWith('bulk-'),
+ );
+ expect(bulkCreates).toHaveLength(total);
+ });
+
+ test('an indexed document with no stored hash is read, not reported deleted, when the budget runs out', async () => {
+ writeFileSync(join(contentDir, 'zz-kept.md'), '# Kept\n');
+ settleTimestamps();
+ await watch(5_000);
+ removeLastKnownHash(join(contentDir, 'zz-kept.md'));
+ for (let index = 0; index < FRESHNESS_HASH_BUDGET.files * 2 + 100; index++) {
+ writeFileSync(join(contentDir, `bulk-${String(index).padStart(4, '0')}.md`), `# ${index}\n`);
+ }
+
+ await runTimedPass(5_000);
+ expect(eventsFor(events, 'delete', 'zz-kept')).toHaveLength(0);
+ expect(handle?.getFileIndex().has('zz-kept')).toBe(true);
+ });
+
+ test.runIf(process.platform !== 'win32')(
+ 'a re-read of a FIFO or a link to a device returns without reading it',
+ async () => {
+ await watch(null);
+ const fifo = join(contentDir, 'pipe.md');
+ execFileSync('mkfifo', [fifo]);
+ const device = join(contentDir, 'zero.md');
+ symlinkSync('/dev/zero', device);
+
+ const result = await (handle as WatcherHandle).recheckPaths([fifo, device]);
+ expect(result).toEqual({ checked: 2, found: 0 });
+ expect(events).toHaveLength(0);
+ },
+ );
+
+ test('a content root that is gone leaves the known documents in place instead of deleting them', async () => {
+ writeFileSync(join(contentDir, 'kept.md'), '# Kept\n');
+ await watch(null);
+ expect(handle?.getFileIndex().has('kept')).toBe(true);
+
+ rmSync(contentDir, { recursive: true, force: true });
+ const result = await (handle as WatcherHandle).recheck();
+ expect(result.status).toBe('incomplete');
+ expect(events.filter((event) => event.kind === 'delete')).toHaveLength(0);
+ expect(handle?.getFileIndex().has('kept')).toBe(true);
+ });
+});
+
+describe('freshness pass spacing', () => {
+ test('a quick pass keeps the base interval', () => {
+ expect(nextFreshnessDelayMs(0, 5_000)).toBe(5_000);
+ expect(nextFreshnessDelayMs(200, 5_000)).toBe(5_000);
+ });
+
+ test('a slow pass spaces the next one so passes take about a twentieth of the time', () => {
+ expect(nextFreshnessDelayMs(1_000, 5_000)).toBe(20_000);
+ });
+
+ test('the spacing never exceeds the maximum unless the base interval does', () => {
+ expect(nextFreshnessDelayMs(30_000, 5_000)).toBe(FRESHNESS_MAX_INTERVAL_MS);
+ expect(nextFreshnessDelayMs(30_000, 90_000)).toBe(90_000);
+ });
+});
diff --git a/packages/server/src/file-watcher-index-retention.test.ts b/packages/server/src/file-watcher-index-retention.test.ts
new file mode 100644
index 000000000..03b8b99f5
--- /dev/null
+++ b/packages/server/src/file-watcher-index-retention.test.ts
@@ -0,0 +1,62 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setFlagsFromString } from 'node:v8';
+import { runInNewContext } from 'node:vm';
+import { afterEach, beforeEach, expect, test } from 'vitest';
+import { lastKnownHash, startWatcher, type WatcherHandle } from './file-watcher.ts';
+import { createManualClock } from './watcher-clock.test-helper.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+
+setFlagsFromString('--expose_gc');
+const collectGarbage = runInNewContext('gc') as () => void;
+
+const DOCUMENT_COUNT = 300;
+const BODY_BYTES = 64 * 1024;
+
+let tmpDir: string;
+let contentDir: string;
+let handle: WatcherHandle | undefined;
+
+beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-retention-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ contentDir = realpathSync(contentDir);
+ lastKnownHash.clear();
+});
+
+afterEach(async () => {
+ await handle?.unsubscribe();
+ handle = undefined;
+ rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function settledHeapBytes(): number {
+ collectGarbage();
+ collectGarbage();
+ return process.memoryUsage().heapUsed;
+}
+
+test('the file index keeps titles and descriptions without keeping each document text alive', async () => {
+ for (let index = 0; index < DOCUMENT_COUNT; index++) {
+ const name = `doc-${String(index).padStart(4, '0')}`;
+ writeFileSync(
+ join(contentDir, `${name}.md`),
+ `---\ntitle: Document title ${index}\ndescription: A description of document ${index}\n---\n\n# Heading of document ${index}\n\n${'x'.repeat(BODY_BYTES)}\n`,
+ );
+ }
+
+ const before = settledHeapBytes();
+ handle = await startWatcher(contentDir, async () => {}, undefined, {
+ nativeBackend: createWatcherDouble().backend,
+ clock: createManualClock(Date.now()),
+ freshnessIntervalMs: null,
+ });
+ const retained = settledHeapBytes() - before;
+
+ const entry = handle.getFileIndex().get('doc-0007');
+ expect(entry?.title).toBe('Document title 7');
+ expect(entry?.description).toBe('A description of document 7');
+ expect(retained).toBeLessThan((DOCUMENT_COUNT * BODY_BYTES) / 4);
+});
diff --git a/packages/server/src/file-watcher-settle.test.ts b/packages/server/src/file-watcher-settle.test.ts
index 4dc69318d..96724730c 100644
--- a/packages/server/src/file-watcher-settle.test.ts
+++ b/packages/server/src/file-watcher-settle.test.ts
@@ -183,7 +183,7 @@ test('settle rejects when the watcher stops before its fence arrives, and leaves
expect(fencesOnDisk()).toEqual([]);
});
-test('a watcher error that ends the subscription fails a pending settle and every later one, and leaves no fence behind', async () => {
+test('a watcher error that ends the subscription fails a pending settle and later ones until it is subscribed again, and leaves no fence behind', async () => {
const { watcher } = await watchContent('parcel');
const pending = observed(watcher.settle());
expect(fencesOnDisk()).toHaveLength(1);
@@ -191,7 +191,7 @@ test('a watcher error that ends the subscription fails a pending settle and ever
const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;
await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));
- const refusal = `the file watcher failed before it saw its settle fence: ${failure}`;
+ const refusal = `the file watcher is subscribing again after an error and cannot settle until it is back: ${failure}`;
expect(pending()).toEqual({ rejected: refusal });
expect(fencesOnDisk()).toEqual([]);
const later = observed(watcher.settle());
diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts
index 5b1c8c74b..23ea050cc 100644
--- a/packages/server/src/file-watcher.ts
+++ b/packages/server/src/file-watcher.ts
@@ -13,7 +13,7 @@ import {
} from 'node:fs';
import { lstat, readdir, readFile, realpath, stat } from 'node:fs/promises';
import { basename, dirname, extname, isAbsolute, join, relative, sep } from 'node:path';
-import { LINKABLE_ASSET_EXTENSIONS } from '@inkeep/open-knowledge-core';
+import { DOCUMENT_OPEN_BYTE_LIMIT, LINKABLE_ASSET_EXTENSIONS } from '@inkeep/open-knowledge-core';
import { filesystemIdentity, statMtime } from '@inkeep/open-knowledge-core/server';
import { isConfigDoc, isReservedForUserTree, isSystemDoc } from './cc1-broadcast.ts';
import { type ContentFilter, WATCHER_STRUCTURAL_IGNORE_DIRS } from './content-filter.ts';
@@ -38,6 +38,15 @@ import {
} from './fs-traced.ts';
import { errnoCode } from './http/handler-utils.ts';
import { getLogger } from './logger.ts';
+import {
+ addFreshnessCarryOver,
+ addFreshnessFilesHashed,
+ addFreshnessFinds,
+ incrementFreshnessPasses,
+ incrementFreshnessPassesIncomplete,
+ incrementWatcherErrors,
+ incrementWatcherUnreadableEntries,
+} from './metrics.ts';
import {
extractPageDescription,
extractPageIcon,
@@ -48,6 +57,7 @@ import { classifyParcelNotification } from './parcel-recovery.ts';
import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';
import { toPosix } from './path-utils.ts';
import { containsConflictMarkers } from './reconciliation.ts';
+import { isRacyStat, sampleWallClockNs, statSignature } from './stat-signature.ts';
import { getMeter, withSpan } from './telemetry.ts';
import { contentHash } from './version-hash.ts';
import {
@@ -56,6 +66,15 @@ import {
type WatcherTimer,
watcherClock,
} from './watcher-clock.ts';
+import {
+ createResubscriber,
+ FRESHNESS_HASH_BUDGET,
+ FRESHNESS_MAX_INTERVAL_MS,
+ nextFreshnessDelayMs,
+ RESUBSCRIBE_SLOW_RETRY_MS,
+ type RecoveryMode,
+ resolveFreshnessInterval,
+} from './watcher-freshness.ts';
export type { NativeWatcherBackend } from './parcel-subscription.ts';
export { contentHash } from './version-hash.ts';
@@ -237,14 +256,21 @@ function derivePageMeta(
description: string | undefined;
type: string | undefined;
} {
+ const icon = extractPageIcon(content);
+ const description = extractPageDescription(content);
+ const type = extractPageType(content);
return {
- title: extractPageTitle(content, docName),
- icon: extractPageIcon(content),
- description: extractPageDescription(content),
- type: extractPageType(content),
+ title: copyDetachedFromSource(extractPageTitle(content, docName)),
+ icon: icon === undefined ? undefined : copyDetachedFromSource(icon),
+ description: description === undefined ? undefined : copyDetachedFromSource(description),
+ type: type === undefined ? undefined : copyDetachedFromSource(type),
};
}
+function copyDetachedFromSource(value: string): string {
+ return Buffer.from(value, 'utf16le').toString('utf16le');
+}
+
function markdownIndexView(
inner: ReadonlyMap<string, FileIndexEntry>,
): ReadonlyMap<string, FileIndexEntry> {
@@ -266,6 +292,7 @@ export interface RecheckResult {
interface RecoveryOutcome {
completed: boolean;
candidates: number;
+ partial?: boolean;
}
interface DispatchCounter {
@@ -275,6 +302,7 @@ interface DispatchCounter {
export interface WatcherHandle {
unsubscribe: () => Promise<void>;
recheck: () => Promise<RecheckResult>;
+ recheckPaths: (paths: readonly string[]) => Promise<{ checked: number; found: number }>;
getFileIndex: () => ReadonlyMap<string, FileIndexEntry>;
getAllFilesIndex: () => AllFileEntries;
getFileIndexGeneration: () => number;
@@ -648,16 +676,129 @@ function extractDocExtension(path: string): string | null {
export const lastKnownHash = new Map<string, string>();
+const lastKnownStat = new Map<string, string>();
+
+let intakeEpoch = 0;
+let openObservationWindows = 0;
+const intakeEpochByPath = new Map<string, number>();
+
+function noteIntakeMutation(path: string): void {
+ intakeEpoch++;
+ if (openObservationWindows > 0) intakeEpochByPath.set(path, intakeEpoch);
+}
+
+function openObservationWindow(): number {
+ openObservationWindows++;
+ return intakeEpoch;
+}
+
+function closeObservationWindow(): void {
+ openObservationWindows--;
+ if (openObservationWindows === 0) intakeEpochByPath.clear();
+}
+
+function changedSince(epoch: number, contentDir: string): ((path: string) => boolean) | undefined {
+ const changed = new Set<string>();
+ for (const [path, at] of intakeEpochByPath) if (at > epoch) changed.add(path);
+ if (changed.size === 0) return undefined;
+ return (path) => {
+ let current = path;
+ while (true) {
+ if (changed.has(current)) return true;
+ if (current === contentDir) return false;
+ const parent = dirname(current);
+ if (parent === current) return false;
+ current = parent;
+ }
+ };
+}
+
export function updateLastKnownHash(filePath: string, hash: string): void {
+ noteIntakeMutation(filePath);
lastKnownHash.set(filePath, hash);
+ lastKnownStat.delete(filePath);
}
export function removeLastKnownHash(filePath: string): string | undefined {
+ noteIntakeMutation(filePath);
const hash = lastKnownHash.get(filePath);
lastKnownHash.delete(filePath);
+ lastKnownStat.delete(filePath);
return hash;
}
+function recordObservedHash(filePath: string, hash: string, signature: string | undefined): void {
+ lastKnownHash.set(filePath, hash);
+ if (signature === undefined) lastKnownStat.delete(filePath);
+ else lastKnownStat.set(filePath, signature);
+}
+
+interface FreshnessReuse {
+ readonly previousByPath: ReadonlyMap<string, FileIndexEntry>;
+ readonly carriedOver: ReadonlySet<string>;
+ readonly deferred: Set<string>;
+ carriedSlots: number;
+ freeSlots: number;
+ bytesLeft: number;
+ hashed: number;
+}
+
+type MarkdownReadDecision =
+ | { kind: 'read' }
+ | { kind: 'reuse'; entry: FileIndexEntry; hash: string }
+ | { kind: 'defer'; entry: FileIndexEntry | undefined; hash: string | undefined };
+
+function beginFreshnessReuse(
+ mode: RecoveryMode,
+ fileIndex: ReadonlyMap<string, FileIndexEntry>,
+ carryOver: Set<string>,
+): FreshnessReuse | undefined {
+ const carriedOver = new Set(carryOver);
+ carryOver.clear();
+ if (mode === 'full') return undefined;
+ const previousByPath = new Map<string, FileIndexEntry>();
+ for (const entry of fileIndex.values()) {
+ if (entry.kind === 'markdown') previousByPath.set(entry.canonicalPath, entry);
+ }
+ const carriedSlots = Math.min(carriedOver.size, FRESHNESS_HASH_BUDGET.files);
+ return {
+ previousByPath,
+ carriedOver,
+ deferred: carryOver,
+ carriedSlots,
+ freeSlots: FRESHNESS_HASH_BUDGET.files - carriedSlots,
+ bytesLeft: FRESHNESS_HASH_BUDGET.bytes,
+ hashed: 0,
+ };
+}
+
+function decideMarkdownRead(
+ reuse: FreshnessReuse | undefined,
+ path: string,
+ stats: BigIntStats,
+ signature: string,
+): MarkdownReadDecision {
+ if (reuse === undefined) return { kind: 'read' };
+ const previous = reuse.previousByPath.get(path);
+ const hash = lastKnownHash.get(path);
+ if (previous !== undefined && hash !== undefined && lastKnownStat.get(path) === signature) {
+ return { kind: 'reuse', entry: previous, hash };
+ }
+ const size = Number(stats.size);
+ const carried = reuse.carriedOver.has(path);
+ const slotFree = carried ? reuse.carriedSlots > 0 : reuse.freeSlots > 0;
+ const indexedWithoutHash = previous !== undefined && hash === undefined;
+ if (indexedWithoutHash || (slotFree && (reuse.hashed === 0 || size <= reuse.bytesLeft))) {
+ if (carried) reuse.carriedSlots--;
+ else reuse.freeSlots--;
+ reuse.bytesLeft -= size;
+ reuse.hashed++;
+ return { kind: 'read' };
+ }
+ reuse.deferred.add(path);
+ return { kind: 'defer', entry: previous, hash };
+}
+
interface RawFileEvent {
type: 'create' | 'update' | 'delete';
path: string;
@@ -1070,11 +1211,72 @@ interface DiskObservation {
aliasPaths: Map<string, string>;
folderAliasIndex: Map<string, string>;
hashes: Map<string, string>;
+ unreadable: Map<string, string>;
+ signatures: Map<string, string>;
+ sampledAtNs: bigint;
+ reuse: FreshnessReuse | undefined;
extensions: Array<{ docName: string; ext: string }>;
preferredExtensions: Map<string, string>;
structuralIgnoreDirs: Set<string>;
paths: Set<string>;
complete: boolean;
+ stopped?: () => boolean;
+}
+
+const reportedUnreadable = new Map<string, string>();
+
+function markUnreadable(
+ observation: DiskObservation,
+ path: string,
+ code: string | undefined,
+): void {
+ observation.unreadable.set(path, code ?? 'unknown');
+}
+
+function reportUnreadableEntries(observation: DiskObservation): void {
+ for (const [path, code] of observation.unreadable) {
+ if (reportedUnreadable.has(path)) continue;
+ reportedUnreadable.set(path, code);
+ incrementWatcherUnreadableEntries();
+ log.warn(
+ { path, code },
+ `Cannot read ${path} (${code}); it keeps its last known state until it can be read`,
+ );
+ }
+ for (const path of reportedUnreadable.keys()) {
+ if (!observation.unreadable.has(path)) reportedUnreadable.delete(path);
+ }
+}
+
+function isUnderUnreadable(observation: DiskObservation, path: string): boolean {
+ for (const unreadable of observation.unreadable.keys()) {
+ if (path === unreadable || path.startsWith(`${unreadable}${sep}`)) return true;
+ }
+ return false;
+}
+
+function observeMarkdownSignature(
+ observation: DiskObservation,
+ path: string,
+ stats: BigIntStats,
+ signature: string,
+): void {
+ if (!isRacyStat(stats, observation.sampledAtNs)) observation.signatures.set(path, signature);
+}
+
+function reusedMarkdownEntry(
+ decision: Exclude<MarkdownReadDecision, { kind: 'read' }>,
+ observation: DiskObservation,
+ path: string,
+ identity: string,
+ aliases: string[],
+ signature: string,
+): FileIndexEntry | undefined {
+ if (decision.entry === undefined || decision.hash === undefined) return undefined;
+ observation.hashes.set(path, decision.hash);
+ observation.paths.add(path);
+ if (decision.kind === 'reuse') observation.signatures.set(path, signature);
+ return { ...decision.entry, canonicalPath: path, inode: identity, aliases };
}
type GeneralIdentity = ReturnType<typeof filesystemIdentity>;
@@ -1841,6 +2043,10 @@ async function scanDisk(
): Promise<void> {
const visited = visitedInodes ?? new Set<string>();
const generalVisited = generalVisitedInodes ?? new Set<string>();
+ if (observation.stopped?.()) {
+ observation.complete = false;
+ return;
+ }
try {
const entries = await readdir(dir, { withFileTypes: true });
for (const entry of entries) {
@@ -1850,10 +2056,7 @@ async function scanDisk(
lst = await lstat(fullPath, { bigint: true });
} catch (e) {
const code = errnoCode(e);
- if (code !== 'ENOENT') {
- observation.complete = false;
- log.warn({ path: fullPath, err: e }, `Failed to lstat ${fullPath}, skipping`);
- }
+ if (code !== 'ENOENT') markUnreadable(observation, fullPath, code);
continue;
}
const lstIdentity = filesystemIdentity(lst);
@@ -1867,8 +2070,7 @@ async function scanDisk(
if (code === 'ENOENT' || code === 'ELOOP') {
log.warn({ path: fullPath, code }, `Broken/cyclic symlink at ${fullPath}, skipping`);
} else {
- observation.complete = false;
- log.warn({ path: fullPath, err: e }, `Failed to resolve symlink ${fullPath}`);
+ markUnreadable(observation, fullPath, code);
}
continue;
}
@@ -1972,10 +2174,28 @@ async function scanDisk(
observation.aliasPaths.set(aliasDocName, fullPath);
observation.paths.add(fullPath);
+ const signature = statSignature(canonStat);
+ const decision = decideMarkdownRead(observation.reuse, canonical, canonStat, signature);
+ if (decision.kind !== 'read') {
+ const reused = reusedMarkdownEntry(
+ decision,
+ observation,
+ canonical,
+ canonIdentity,
+ [aliasDocName],
+ signature,
+ );
+ if (reused === undefined) continue;
+ const ext = extractDocExtension(canonical);
+ if (ext && !recordObservedExtension(observation, canonicalDocName, ext)) continue;
+ observation.fileIndex.set(canonicalDocName, reused);
+ continue;
+ }
try {
const content = await readFile(canonical, 'utf-8');
const hash = contentHash(content);
observation.hashes.set(canonical, hash);
+ observeMarkdownSignature(observation, canonical, canonStat, signature);
observation.paths.add(canonical);
const ext = extractDocExtension(canonical);
if (ext && !recordObservedExtension(observation, canonicalDocName, ext)) continue;
@@ -1990,18 +2210,16 @@ async function scanDisk(
});
} catch (err) {
const code = errnoCode(err);
- if (code !== 'ENOENT') {
- observation.complete = false;
- log.warn({ path: canonical, err }, `Failed to seed hash for ${canonical}`);
- }
+ if (code !== 'ENOENT') markUnreadable(observation, canonical, code);
}
}
} catch (e) {
const code = errnoCode(e);
if (code !== 'ENOENT' && code !== 'ENOTDIR') {
- observation.complete = false;
+ markUnreadable(observation, fullPath, code);
+ } else {
+ log.warn({ path: canonical, err: e }, `Failed to stat symlink target ${canonical}`);
}
- log.warn({ path: canonical, err: e }, `Failed to stat symlink target ${canonical}`);
}
} else if (lst.isDirectory()) {
const relPath = contentRelativePath(contentDir, fullPath);
@@ -2031,9 +2249,28 @@ async function scanDisk(
const relPath = toPosix(relative(contentDir, fullPath));
if (contentFilter.isExcluded(relPath)) continue;
}
+ const signature = statSignature(lst);
+ const decision = decideMarkdownRead(observation.reuse, fullPath, lst, signature);
+ if (decision.kind !== 'read') {
+ const reused = reusedMarkdownEntry(
+ decision,
+ observation,
+ fullPath,
+ lstIdentity,
+ [],
+ signature,
+ );
+ if (reused === undefined) continue;
+ const docName = pathToDocName(fullPath, contentDir);
+ const ext = extractDocExtension(fullPath);
+ if (ext && !recordObservedExtension(observation, docName, ext)) continue;
+ observation.fileIndex.set(docName, reused);
+ continue;
+ }
try {
const content = await readFile(fullPath, 'utf-8');
observation.hashes.set(fullPath, contentHash(content));
+ observeMarkdownSignature(observation, fullPath, lst, signature);
observation.paths.add(fullPath);
const docName = pathToDocName(fullPath, contentDir);
@@ -2050,16 +2287,7 @@ async function scanDisk(
});
} catch (err) {
const code = errnoCode(err);
- if (code === 'EACCES') {
- observation.complete = false;
- log.warn(
- { path: fullPath, code },
- `Permission denied reading ${fullPath}, file excluded from index`,
- );
- } else if (code !== 'ENOENT') {
- observation.complete = false;
- log.warn({ path: fullPath, err }, `Failed to seed hash for ${fullPath}`);
- }
+ if (code !== 'ENOENT') markUnreadable(observation, fullPath, code);
}
} else if (lst.isFile()) {
if (contentFilter) {
@@ -2088,9 +2316,11 @@ async function scanDisk(
}
} catch (err) {
const code = errnoCode(err);
- if (code !== 'ENOENT' && code !== 'ENOTDIR') {
+ if (dir === contentDir) {
observation.complete = false;
- log.warn({ dir, err }, `Failed to read directory ${dir}`);
+ log.warn({ dir, err }, `Failed to read the content directory ${dir}`);
+ } else if (code !== 'ENOENT' && code !== 'ENOTDIR') {
+ markUnreadable(observation, dir, code);
}
}
}
@@ -2098,6 +2328,8 @@ async function scanDisk(
async function observeDisk(
contentDir: string,
contentFilter: ContentFilter | undefined,
+ reuse?: FreshnessReuse,
+ stopped?: () => boolean,
): Promise<DiskObservation> {
const observation: DiskObservation = {
fileIndex: new Map(),
@@ -2107,17 +2339,27 @@ async function observeDisk(
aliasPaths: new Map(),
folderAliasIndex: new Map(),
hashes: new Map(),
+ unreadable: new Map(),
+ signatures: new Map(),
+ sampledAtNs: sampleWallClockNs(),
+ reuse,
extensions: [],
preferredExtensions: new Map(),
structuralIgnoreDirs: new Set(),
paths: new Set(),
complete: true,
+ stopped,
};
const generalScan: GeneralScanAccumulator = {
facts: new Map(),
metadataEffects: new Map(),
};
await scanDisk(contentDir, contentDir, contentFilter, observation, generalScan);
+ if (observation.stopped?.()) {
+ observation.complete = false;
+ return observation;
+ }
+ reportUnreadableEntries(observation);
materializeScanGeneralIndex(observation, contentDir, generalScan);
for (const [name, entry] of new Map(observation.generalFileIndex)) {
const state = knownGeneralState(entry);
@@ -2202,7 +2444,9 @@ function publishDiskObservation(
);
}
}
- for (const [path, hash] of observation.hashes) lastKnownHash.set(path, hash);
+ for (const [path, hash] of observation.hashes) {
+ recordObservedHash(path, hash, observation.signatures.get(path));
+ }
for (const [docName, entry] of inventory.fileIndex) {
if (getDocExtension(docName).toLowerCase() !== extname(entry.canonicalPath).toLowerCase()) {
continue;
@@ -2507,6 +2751,7 @@ async function handleRawEventsInternal(
generalFileIndex?: GeneralFileIndex,
aliasPaths?: Map<string, string>,
): Promise<void> {
+ for (const event of rawEvents) noteIntakeMutation(event.path);
const declaredBeforeBatch: ReadonlySet<string> = new Set(removalTracker.keys());
const safeEvents = rawEvents.filter((e) => {
if (!eventEscapesContentDir(e.path, contentDir)) return true;
@@ -3058,6 +3303,43 @@ type RecoveredAliasCandidate =
targetIdentity: GeneralIdentity;
};
+function protectedGeneralNames(
+ observed: DiskObservation,
+ contentDir: string,
+ previousGeneralFacts: ReadonlyMap<string, GeneralPathFact>,
+ observedGeneralFacts: ReadonlyMap<string, GeneralPathFact>,
+ changedDuringWalk: ((path: string) => boolean) | undefined,
+): Set<string> {
+ const names = new Set<string>();
+ if (observed.unreadable.size === 0 && changedDuringWalk === undefined) return names;
+ const keepKnownAt = (name: string): boolean => {
+ const path = join(contentDir, name);
+ return isUnderUnreadable(observed, path) || changedDuringWalk?.(path) === true;
+ };
+ const identities = new Set<GeneralIdentity>();
+ for (const [name, fact] of previousGeneralFacts) {
+ if (fact.kind !== 'unresolved' && keepKnownAt(name)) identities.add(fact.identity);
+ }
+ for (const [name, fact] of previousGeneralFacts) {
+ if (
+ keepKnownAt(name) ||
+ (fact.kind !== 'unresolved' && identities.has(fact.identity)) ||
+ (fact.kind === 'symlink' && keepKnownAt(fact.targetPath))
+ ) {
+ names.add(name);
+ }
+ }
+ for (const name of observedGeneralFacts.keys()) {
+ if (keepKnownAt(name)) names.add(name);
+ }
+ return names;
+}
+
+interface HandedObservation {
+ readonly observed: DiskObservation;
+ readonly sinceEpoch: number;
+}
+
async function recoverDiskChanges(
contentDir: string,
contentFilter: ContentFilter | undefined,
@@ -3073,6 +3355,8 @@ async function recoverDiskChanges(
getFileIndexGeneration: () => number,
counter: DispatchCounter,
onRawBatch?: (absPaths: readonly string[]) => void,
+ reuse?: FreshnessReuse,
+ handed?: HandedObservation,
): Promise<RecoveryOutcome> {
const countingOnDiskEvent = async (event: DiskEvent): Promise<void> => {
counter.dispatched++;
@@ -3095,18 +3379,35 @@ async function recoverDiskChanges(
const previousAliasPaths = new Map(aliasPaths);
const previousFolderAliases = new Map(folderAliasIndex);
- contentFilter?.refreshInPlaceSkillDirs();
- const observed = await observeDisk(contentDir, contentFilter);
+ let observed: DiskObservation;
+ if (handed) {
+ observed = handed.observed;
+ } else {
+ contentFilter?.refreshInPlaceSkillDirs();
+ observed = await observeDisk(contentDir, contentFilter, reuse);
+ }
if (!observed.complete) return { completed: false, candidates: 0 };
+ const changedDuringWalk = handed ? changedSince(handed.sinceEpoch, contentDir) : undefined;
+ const intakeOwns = (path: string): boolean => changedDuringWalk?.(path) === true;
const markdownPublicMutationFor = (name: string, entry: FileIndexEntry): boolean => {
const names = publicMutations.markdownNames;
+ if (names.size === 0) return false;
if (names.has(name)) return true;
const canonicalName = pathToDocName(entry.canonicalPath, contentDir);
return names.has(canonicalName) || entry.aliases.some((alias) => names.has(alias));
};
const previousGeneralFacts = generalPathFacts(previousGeneralFiles);
const observedGeneralFacts = generalPathFacts(observed.generalFileIndex);
+ const unreadableAt = (path: string): boolean =>
+ (observed.unreadable.size > 0 && isUnderUnreadable(observed, path)) || intakeOwns(path);
+ const protectedGeneral = protectedGeneralNames(
+ observed,
+ contentDir,
+ previousGeneralFacts,
+ observedGeneralFacts,
+ changedDuringWalk,
+ );
const rawEvents: RecoveredFileEvent[] = [];
for (const [docName, previous] of previousFiles) {
@@ -3116,6 +3417,10 @@ async function recoverDiskChanges(
(current && markdownPublicMutationFor(docName, current))
)
continue;
+ if (current === undefined && unreadableAt(previous.canonicalPath)) continue;
+ if (intakeOwns(previous.canonicalPath) || (current && intakeOwns(current.canonicalPath))) {
+ continue;
+ }
if (current?.canonicalPath !== previous.canonicalPath) {
const removal: KnownDeletion = {
type: 'delete',
@@ -3138,7 +3443,12 @@ async function recoverDiskChanges(
}
}
for (const [name, previous] of previousGeneralFacts) {
- if (previous.kind === 'symlink' || publicMutations.generalEffects.has(name)) continue;
+ if (
+ previous.kind === 'symlink' ||
+ publicMutations.generalEffects.has(name) ||
+ protectedGeneral.has(name)
+ )
+ continue;
const current = observedGeneralFacts.get(name);
if (current === undefined) {
rawEvents.push({ type: 'delete', path: join(contentDir, name), entryKind: 'file' });
@@ -3157,7 +3467,9 @@ async function recoverDiskChanges(
const previous = previousFiles.get(docName);
if (
markdownPublicMutationFor(docName, current) ||
- (previous && markdownPublicMutationFor(docName, previous))
+ (previous && markdownPublicMutationFor(docName, previous)) ||
+ intakeOwns(current.canonicalPath) ||
+ (previous && intakeOwns(previous.canonicalPath))
)
continue;
if (current.canonicalPath !== previous?.canonicalPath) {
@@ -3165,7 +3477,12 @@ async function recoverDiskChanges(
}
}
for (const [name, current] of observedGeneralFacts) {
- if (current.kind !== 'regular' || publicMutations.generalEffects.has(name)) continue;
+ if (
+ current.kind !== 'regular' ||
+ publicMutations.generalEffects.has(name) ||
+ protectedGeneral.has(name)
+ )
+ continue;
const previous = previousGeneralFacts.get(name);
if (previous === undefined) {
rawEvents.push({ type: 'create', path: join(contentDir, name) });
@@ -3175,17 +3492,25 @@ async function recoverDiskChanges(
}
for (const [relativePath, previous] of folderIndex) {
if (!observed.folderIndex.has(relativePath)) {
+ if (unreadableAt(join(contentDir, relativePath))) continue;
rawEvents.push({ type: 'delete', path: join(contentDir, relativePath), entryKind: 'folder' });
- } else if (observed.folderIndex.get(relativePath)?.canonicalPath !== previous.canonicalPath) {
+ } else if (
+ observed.folderIndex.get(relativePath)?.canonicalPath !== previous.canonicalPath &&
+ !intakeOwns(join(contentDir, relativePath))
+ ) {
rawEvents.push({ type: 'update', path: join(contentDir, relativePath) });
}
}
for (const [relativePath, current] of observed.folderIndex) {
- if (!folderIndex.has(relativePath))
+ if (
+ !folderIndex.has(relativePath) &&
+ !intakeOwns(join(contentDir, relativePath)) &&
+ !intakeOwns(current.canonicalPath)
+ )
rawEvents.push({ type: 'create', path: current.canonicalPath });
}
for (const [alias, path] of aliasPaths) {
- if (!observed.aliasPaths.has(alias))
+ if (!observed.aliasPaths.has(alias) && !unreadableAt(path))
rawEvents.push({ type: 'delete', path, entryKind: 'file' });
}
@@ -3240,21 +3565,26 @@ async function recoverDiskChanges(
);
}
+ const publishedDuringPass = (path: string): boolean =>
+ publicMutations.markdownNames.size > 0 &&
+ publicMutations.markdownNames.has(pathToDocName(path, contentDir));
for (const [path, prior] of previousHashes) {
if (
- isWithinContentDir(path, contentDir) &&
- !publicMutations.markdownNames.has(pathToDocName(path, contentDir)) &&
!observed.hashes.has(path) &&
- lastKnownHash.get(path) === prior
+ lastKnownHash.get(path) === prior &&
+ !unreadableAt(path) &&
+ isWithinContentDir(path, contentDir) &&
+ !publishedDuringPass(path)
) {
- lastKnownHash.delete(path);
+ removeLastKnownHash(path);
}
}
for (const [path, hash] of observed.hashes) {
- if (publicMutations.markdownNames.has(pathToDocName(path, contentDir))) continue;
+ if (publishedDuringPass(path) || intakeOwns(path)) continue;
const current = lastKnownHash.get(path);
- if (current === undefined || current === previousHashes.get(path))
- lastKnownHash.set(path, hash);
+ if (current === undefined || current === previousHashes.get(path) || current === hash) {
+ recordObservedHash(path, hash, observed.signatures.get(path));
+ }
}
for (const [name, prior] of previousFiles) {
@@ -3262,7 +3592,8 @@ async function recoverDiskChanges(
if (
!markdownPublicMutationFor(name, prior) &&
!observed.fileIndex.has(name) &&
- current === prior
+ current === prior &&
+ !unreadableAt(prior.canonicalPath)
) {
fileIndex.delete(name);
forgetDocExtension(name);
@@ -3271,10 +3602,17 @@ async function recoverDiskChanges(
const inventory = selectObservedInventory(observed, contentDir, contentFilter);
for (const [name, entry] of observed.fileIndex) {
if (markdownPublicMutationFor(name, entry)) continue;
+ const live = fileIndex.get(name);
+ if (intakeOwns(entry.canonicalPath) || (live && intakeOwns(live.canonicalPath))) continue;
if (inventory.fileIndex.has(name)) fileIndex.set(name, entry);
else if (fileIndex.get(name)?.canonicalPath === entry.canonicalPath) fileIndex.delete(name);
}
const selectedGeneralFacts = inventory.generalFacts;
+ for (const name of protectedGeneral) {
+ const fact = previousGeneralFacts.get(name);
+ if (fact) selectedGeneralFacts.set(name, fact);
+ else selectedGeneralFacts.delete(name);
+ }
for (const [path, effect] of publicMutations.generalEffects) {
if (effect) selectedGeneralFacts.set(path, effect);
else selectedGeneralFacts.delete(path);
@@ -3288,10 +3626,15 @@ async function recoverDiskChanges(
);
for (const [path, prior] of previousFolders) {
- if (!observed.folderIndex.has(path) && folderIndex.get(path) === prior)
+ if (
+ !observed.folderIndex.has(path) &&
+ folderIndex.get(path) === prior &&
+ !unreadableAt(join(contentDir, path))
+ )
folderIndex.delete(path);
}
for (const [path, entry] of inventory.folderIndex) {
+ if (intakeOwns(join(contentDir, path))) continue;
const current = folderIndex.get(path);
if (
current === undefined ||
@@ -3304,28 +3647,38 @@ async function recoverDiskChanges(
}
}
for (const [alias, prior] of previousAliases) {
+ const aliasPath = previousAliasPaths.get(alias);
+ if (aliasPath !== undefined && unreadableAt(aliasPath)) continue;
if (!observed.aliasMap.has(alias) && aliasMap.get(alias) === prior) aliasMap.delete(alias);
}
for (const [alias, canonical] of observed.aliasMap) {
+ const aliasPath = observed.aliasPaths.get(alias) ?? previousAliasPaths.get(alias);
+ if (aliasPath !== undefined && intakeOwns(aliasPath)) continue;
const current = aliasMap.get(alias);
if (current === undefined || current === previousAliases.get(alias))
aliasMap.set(alias, canonical);
}
for (const [alias, prior] of previousAliasPaths) {
- if (!observed.aliasPaths.has(alias) && aliasPaths.get(alias) === prior)
+ if (!observed.aliasPaths.has(alias) && aliasPaths.get(alias) === prior && !unreadableAt(prior))
aliasPaths.delete(alias);
}
for (const [alias, path] of observed.aliasPaths) {
+ if (intakeOwns(path)) continue;
const current = aliasPaths.get(alias);
if (current === undefined || current === previousAliasPaths.get(alias))
aliasPaths.set(alias, path);
}
for (const [alias, prior] of previousFolderAliases) {
- if (!observed.folderAliasIndex.has(alias) && folderAliasIndex.get(alias) === prior) {
+ if (
+ !observed.folderAliasIndex.has(alias) &&
+ folderAliasIndex.get(alias) === prior &&
+ !unreadableAt(join(contentDir, alias))
+ ) {
folderAliasIndex.delete(alias);
}
}
for (const [alias, canonical] of inventory.folderAliasIndex) {
+ if (intakeOwns(join(contentDir, alias))) continue;
const current = folderAliasIndex.get(alias);
if (current === undefined || current === previousFolderAliases.get(alias)) {
folderAliasIndex.set(alias, canonical);
@@ -3456,8 +3809,16 @@ async function recoverDiskChanges(
await countingOnDiskEvent(event);
}
}
- onRawBatch?.([...new Set([...observed.paths, ...knownPaths])]);
- return { completed: true, candidates: deduplicated.length };
+ if (handed) {
+ if (deduplicated.length > 0) onRawBatch?.(deduplicated.map((event) => event.path));
+ } else {
+ onRawBatch?.([...new Set([...observed.paths, ...knownPaths])]);
+ }
+ return {
+ completed: true,
+ candidates: deduplicated.length,
+ partial: observed.unreadable.size > 0,
+ };
}
export function toParcelIgnorePaths(
@@ -3484,6 +3845,8 @@ interface RecoveryDriverDeps {
recover: (
mutations: RecoveryPublicMutations,
counter: DispatchCounter,
+ mode: RecoveryMode,
+ handed: HandedObservation | undefined,
) => Promise<RecoveryOutcome>;
setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void;
onAfterMutation: () => void;
@@ -3491,15 +3854,24 @@ interface RecoveryDriverDeps {
backendName: WatcherBackend;
}
-function createRecoveryDriver(deps: RecoveryDriverDeps): {
- requestRecovery: () => Promise<RecheckResult>;
-} {
+interface RecoveryDriver {
+ requestRecovery: (mode: RecoveryMode, handed?: HandedObservation) => Promise<RecheckResult>;
+ isIdle: () => boolean;
+}
+
+function createRecoveryDriver(deps: RecoveryDriverDeps): RecoveryDriver {
let recoveryRunning = false;
let nextRecovery: Promise<RecheckResult> | undefined;
+ let nextMode: RecoveryMode = 'metadata';
+ let nextHanded: HandedObservation | undefined;
let followUpRecovery: PromiseWithResolvers<RecheckResult> | undefined;
const closed = (): RecheckResult => ({ status: 'closed', candidates: 0, dispatched: 0 });
const recoverOnce = async (): Promise<RecheckResult> => {
+ const mode = nextMode;
+ const handed = nextHanded;
nextRecovery = undefined;
+ nextMode = 'metadata';
+ nextHanded = undefined;
if (deps.isClosed()) return closed();
recoveryRunning = true;
const recoveryMutations: RecoveryPublicMutations = {
@@ -3513,19 +3885,26 @@ function createRecoveryDriver(deps: RecoveryDriverDeps): {
let result: RecheckResult = { status: 'failed', candidates: 0, dispatched: 0 };
try {
deps.setActivePublicMutations(recoveryMutations);
- const outcome = await deps.recover(recoveryMutations, counter);
+ const outcome = await deps.recover(recoveryMutations, counter, mode, handed);
deps.onAfterMutation();
if (outcome.completed) {
- await deps.onRecoveryComplete?.();
+ if (!outcome.partial) await deps.onRecoveryComplete?.();
result = {
status: 'completed',
candidates: outcome.candidates,
dispatched: counter.dispatched,
};
- log.info(
- { candidates: outcome.candidates, dispatched: counter.dispatched },
- `${deps.backendName} watcher recovery finished`,
- );
+ const finished = {
+ mode: handed ? 'timed' : mode,
+ candidates: outcome.candidates,
+ dispatched: counter.dispatched,
+ partial: outcome.partial === true,
+ };
+ if (mode === 'full' || counter.dispatched > 0) {
+ log.info(finished, `${deps.backendName} watcher recovery finished`);
+ } else {
+ log.debug(finished, `${deps.backendName} watcher recovery finished`);
+ }
} else {
result = { status: 'incomplete', candidates: 0, dispatched: counter.dispatched };
log.warn(
@@ -3560,8 +3939,16 @@ function createRecoveryDriver(deps: RecoveryDriverDeps): {
})
.then(() => settled);
};
- const requestRecovery = (): Promise<RecheckResult> => {
- if (nextRecovery) return nextRecovery;
+ const requestRecovery = (
+ mode: RecoveryMode,
+ handed?: HandedObservation,
+ ): Promise<RecheckResult> => {
+ if (mode === 'full') nextMode = 'full';
+ if (nextRecovery) {
+ nextHanded = undefined;
+ return nextRecovery;
+ }
+ nextHanded = mode === 'full' ? undefined : handed;
if (recoveryRunning) {
followUpRecovery = Promise.withResolvers<RecheckResult>();
nextRecovery = followUpRecovery.promise;
@@ -3570,7 +3957,14 @@ function createRecoveryDriver(deps: RecoveryDriverDeps): {
}
return nextRecovery;
};
- return { requestRecovery };
+ return { requestRecovery, isIdle: () => !recoveryRunning && nextRecovery === undefined };
+}
+
+interface WatcherBackendSubscription extends AsyncSubscription {
+ runOrdered(work: () => Promise<void>): Promise<void>;
+ requestRecovery(mode: RecoveryMode, handed?: HandedObservation): Promise<RecheckResult>;
+ isRecoveryIdle(): boolean;
+ noticeRootMissing(): void;
}
async function startParcelWatcher(
@@ -3592,14 +3986,9 @@ async function startParcelWatcher(
onRecoveryComplete?: () => Promise<void>,
clock: WatcherClock = watcherClock(),
nativeBackend?: NativeWatcherBackend,
-): Promise<
- | (AsyncSubscription & {
- runOrdered(work: () => Promise<void>): Promise<void>;
- requestRecovery(): Promise<RecheckResult>;
- settle(): Promise<void>;
- })
- | null
-> {
+ carryOver: Set<string> = new Set(),
+ coverageWhileLost: string = coverageWithoutWatcher(null),
+): Promise<(WatcherBackendSubscription & { settle(): Promise<void> }) | null> {
let parcel: NativeWatcherBackend;
if (nativeBackend) {
parcel = nativeBackend;
@@ -3616,14 +4005,15 @@ async function startParcelWatcher(
}
try {
- const subscribeOpts = contentFilter
- ? {
- ignore: toParcelIgnorePaths(
- contentFilter.getWatcherIgnoreGlobs(),
- discoveredStructuralDirs ?? [],
- ),
- }
- : undefined;
+ const subscribeOpts = () =>
+ contentFilter
+ ? {
+ ignore: toParcelIgnorePaths(
+ contentFilter.getWatcherIgnoreGlobs(),
+ discoveredStructuralDirs ?? [],
+ ),
+ }
+ : undefined;
let closed = false;
let failure: Error | undefined;
@@ -3642,10 +4032,10 @@ async function startParcelWatcher(
});
return next;
};
- const { requestRecovery } = createRecoveryDriver({
+ const { requestRecovery, isIdle } = createRecoveryDriver({
runOrdered,
isClosed: () => closed,
- recover: (recoveryMutations, counter) =>
+ recover: (recoveryMutations, counter, mode, handed) =>
recoverDiskChanges(
contentDir,
contentFilter,
@@ -3661,24 +4051,27 @@ async function startParcelWatcher(
getFileIndexGeneration,
counter,
onRawBatch,
+ handed ? undefined : beginFreshnessReuse(mode, fileIndex, carryOver),
+ handed,
),
setActivePublicMutations,
onAfterMutation,
onRecoveryComplete,
backendName: 'parcel',
});
- const subscription = await subscribeParcel(
- parcel,
- contentDir,
+ const notificationsFor =
+ (generation: number): Parameters<typeof subscribeParcel>[2] =>
(err, events) => {
- if (closed) return;
+ if (closed || !resubscriber.isLive(generation)) return;
const notification = classifyParcelNotification(err, events);
if (notification.kind === 'error') {
log.error({ err: notification.error }, 'parcel watcher callback error');
failure = new Error(
- `the file watcher failed before it saw its settle fence: ${notification.error.message}`,
+ `the file watcher is subscribing again after an error and cannot settle until it is back: ${notification.error.message}`,
);
for (const fence of settleFences.values()) fence.reject(failure);
+ incrementWatcherErrors();
+ resubscriber.lost('error', generation, notification.error);
return;
}
if (notification.kind === 'rescan') {
@@ -3710,22 +4103,52 @@ async function startParcelWatcher(
for (const event of notification.events)
settleFences.get(basename(event.path))?.resolve();
});
+ if (
+ notification.events.some((event) => event.type === 'delete' && event.path === contentDir)
+ ) {
+ resubscriber.lost('root-delete', generation);
+ }
if (notification.kind !== 'rescan') return records;
const fencesBeforeRescan = [...settleFences.values()];
return Promise.all([
records,
- requestRecovery().then(() => {
+ requestRecovery('full').then(() => {
for (const fence of fencesBeforeRescan) fence.resolve();
}),
]);
+ };
+ const resubscriber = createResubscriber({
+ clock,
+ root: contentDir,
+ backendName: 'parcel',
+ coverageWhileLost,
+ onGaveUp: (err) => {
+ const message = err instanceof Error ? err.message : String(err);
+ failure = new Error(
+ `the file watcher gave up subscribing after repeated failures (${message}); it tries again every ${RESUBSCRIBE_SLOW_RETRY_MS / 60_000} minutes and cannot settle until it is back`,
+ );
+ for (const fence of settleFences.values()) fence.reject(failure);
},
- subscribeOpts,
- platform,
- );
+ subscribe: (generation) =>
+ subscribeParcel(
+ parcel,
+ contentDir,
+ notificationsFor(generation),
+ subscribeOpts(),
+ platform,
+ ),
+ onResubscribed: () => {
+ failure = undefined;
+ void requestRecovery('full');
+ },
+ });
+ await resubscriber.start();
return {
runOrdered,
requestRecovery,
+ isRecoveryIdle: isIdle,
+ noticeRootMissing: () => resubscriber.noticeRootMissing(),
async settle() {
if (closed) return;
if (failure) throw failure;
@@ -3764,7 +4187,7 @@ async function startParcelWatcher(
for (const fence of settleFences.values()) {
fence.reject(new Error('the file watcher stopped before it saw its settle fence'));
}
- await subscription.unsubscribe();
+ await resubscriber.close();
await workTail;
},
};
@@ -3821,6 +4244,14 @@ function waitForPendingFsEventsRegistrations(dir: string, platform: NodeJS.Platf
}
}
+const CHOKIDAR_NON_FATAL_ERROR_CODES: ReadonlySet<string> = new Set([
+ 'EACCES',
+ 'EPERM',
+ 'ENOSPC',
+ 'EMFILE',
+ 'ENFILE',
+]);
+
async function startChokidarWatcher(
contentDir: string,
contentFilter: ContentFilter | undefined,
@@ -3838,20 +4269,11 @@ async function startChokidarWatcher(
clock: WatcherClock,
onRawBatch?: (absPaths: readonly string[]) => void,
onRecoveryComplete?: () => Promise<void>,
-): Promise<AsyncSubscription & { requestRecovery(): Promise<RecheckResult> }> {
+ carryOver: Set<string> = new Set(),
+ coverageWhileLost: string = coverageWithoutWatcher(null),
+): Promise<WatcherBackendSubscription> {
const { watch } = await import('chokidar');
- const watcher = watch(contentDir, {
- ignoreInitial: true,
- followSymlinks: false,
- ignored: contentFilter
- ? (filePath: string, stats?: Stats) =>
- isChokidarPathIgnored(contentDir, contentFilter, filePath, stats)
- : undefined,
- });
-
- watcher.on('error', (err) => log.error({ err }, 'chokidar error'));
-
let closed = false;
let workTail = Promise.resolve();
const runOrdered = (work: () => Promise<void>): Promise<void> => {
@@ -3862,10 +4284,10 @@ async function startChokidarWatcher(
});
return next;
};
- const { requestRecovery } = createRecoveryDriver({
+ const { requestRecovery, isIdle } = createRecoveryDriver({
runOrdered,
isClosed: () => closed,
- recover: (recoveryMutations, counter) =>
+ recover: (recoveryMutations, counter, mode, handed) =>
recoverDiskChanges(
contentDir,
contentFilter,
@@ -3881,6 +4303,8 @@ async function startChokidarWatcher(
getFileIndexGeneration,
counter,
onRawBatch,
+ handed ? undefined : beginFreshnessReuse(mode, fileIndex, carryOver),
+ handed,
),
setActivePublicMutations,
onAfterMutation,
@@ -3920,27 +4344,78 @@ async function startChokidarWatcher(
}, CHOKIDAR_BATCH_WINDOW_MS);
}
- watcher.on('add', (path) => queueEvent({ type: 'create', path }));
- watcher.on('change', (path) => queueEvent({ type: 'update', path }));
- watcher.on('unlink', (path) => queueEvent({ type: 'delete', path, entryKind: 'file' }));
- watcher.on('addDir', (path) => queueEvent({ type: 'create', path }));
- watcher.on('unlinkDir', (path) => queueEvent({ type: 'delete', path, entryKind: 'folder' }));
-
- await new Promise<void>((resolveReady) => {
- let settled = false;
- const settle = () => {
- if (settled) return;
- settled = true;
- resolveReady();
+ const reportedNonFatalCodes = new Set<string>();
+ const watchAs = async (generation: number): Promise<AsyncSubscription> => {
+ const watcher = watch(contentDir, {
+ ignoreInitial: true,
+ followSymlinks: false,
+ ignored: contentFilter
+ ? (filePath: string, stats?: Stats) =>
+ isChokidarPathIgnored(contentDir, contentFilter, filePath, stats)
+ : undefined,
+ });
+ const live = (): boolean => !closed && resubscriber.isLive(generation);
+ const queueLive = (event: InternalRawFileEvent): void => {
+ if (live()) queueEvent(event);
};
- watcher.once('ready', settle);
- clock.setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();
- });
- waitForPendingFsEventsRegistrations(contentDir, platform);
+ watcher.on('error', (err) => {
+ if (!live()) return;
+ incrementWatcherErrors();
+ const code = errnoCode(err);
+ if (code !== undefined && CHOKIDAR_NON_FATAL_ERROR_CODES.has(code)) {
+ if (reportedNonFatalCodes.has(code)) return;
+ reportedNonFatalCodes.add(code);
+ log.warn(
+ { err, code },
+ `chokidar reported ${code}; the watcher stays up, and for what it cannot watch ${coverageWhileLost}`,
+ );
+ return;
+ }
+ log.error({ err }, 'chokidar error');
+ resubscriber.lost('error', generation, err);
+ });
+ watcher.on('add', (path) => queueLive({ type: 'create', path }));
+ watcher.on('change', (path) => queueLive({ type: 'update', path }));
+ watcher.on('unlink', (path) => queueLive({ type: 'delete', path, entryKind: 'file' }));
+ watcher.on('addDir', (path) => queueLive({ type: 'create', path }));
+ watcher.on('unlinkDir', (path) => {
+ queueLive({ type: 'delete', path, entryKind: 'folder' });
+ if (path === contentDir && live()) resubscriber.lost('root-delete', generation);
+ });
+
+ await new Promise<void>((resolveReady) => {
+ let settled = false;
+ const settle = () => {
+ if (settled) return;
+ settled = true;
+ resolveReady();
+ };
+ watcher.once('ready', settle);
+ clock.setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();
+ });
+
+ waitForPendingFsEventsRegistrations(contentDir, platform);
+ return { unsubscribe: () => watcher.close() };
+ };
+
+ const resubscriber = createResubscriber({
+ clock,
+ root: contentDir,
+ backendName: 'chokidar',
+ coverageWhileLost,
+ subscribe: watchAs,
+ onResubscribed: () => {
+ void requestRecovery('full');
+ },
+ });
+ await resubscriber.start();
return {
+ runOrdered,
requestRecovery,
+ isRecoveryIdle: isIdle,
+ noticeRootMissing: () => resubscriber.noticeRootMissing(),
unsubscribe: async () => {
closed = true;
if (batchTimer) {
@@ -3948,12 +4423,21 @@ async function startChokidarWatcher(
batchTimer = null;
pendingEvents = [];
}
- await watcher.close();
+ await resubscriber.close();
await workTail;
},
};
}
+function coverageWithoutWatcher(freshnessIntervalMs: number | null): string {
+ if (freshnessIntervalMs === null) {
+ return 'changes made outside the app are found only by an explicit re-check';
+ }
+ const fastest = freshnessIntervalMs / 1000;
+ const slowest = Math.max(freshnessIntervalMs, FRESHNESS_MAX_INTERVAL_MS) / 1000;
+ return `the timed re-check still finds changes made outside the app, every ${fastest} to ${slowest} seconds`;
+}
+
export interface StartWatcherOptions {
forceBackend?: 'parcel' | 'chokidar';
onRawBatch?: (absPaths: readonly string[]) => void;
@@ -3961,6 +4445,7 @@ export interface StartWatcherOptions {
platform?: NodeJS.Platform;
nativeBackend?: NativeWatcherBackend;
clock?: WatcherClock;
+ freshnessIntervalMs?: number | null;
}
export async function startWatcher(
@@ -4044,8 +4529,10 @@ async function startWatcherWithClock(
WATCHER_DROP_SUMMARY_INTERVAL_MS,
);
- let subscription: AsyncSubscription;
- let requestRecovery: () => Promise<RecheckResult>;
+ const freshnessIntervalMs = resolveFreshnessInterval(opts.freshnessIntervalMs);
+ const coverageWhileLost = coverageWithoutWatcher(freshnessIntervalMs);
+ const carryOver = new Set<string>();
+ let subscription: WatcherBackendSubscription;
let runOrderedRescan: ((work: () => Promise<void>) => Promise<void>) | undefined;
let settleSubscription: (() => Promise<void>) | undefined;
let backend: WatcherBackend;
@@ -4076,10 +4563,11 @@ async function startWatcherWithClock(
onRecoveryComplete,
clock,
opts.nativeBackend,
+ carryOver,
+ coverageWhileLost,
);
if (parcelSub) {
subscription = parcelSub;
- requestRecovery = parcelSub.requestRecovery;
runOrderedRescan = parcelSub.runOrdered;
settleSubscription = () => parcelSub.settle();
backend = 'parcel';
@@ -4106,9 +4594,10 @@ async function startWatcherWithClock(
clock,
onRawBatch,
onRecoveryComplete,
+ carryOver,
+ coverageWhileLost,
);
subscription = chokidarSub;
- requestRecovery = chokidarSub.requestRecovery;
backend = 'chokidar';
}
} catch (e) {
@@ -4117,26 +4606,178 @@ async function startWatcherWithClock(
throw e;
}
- const originalUnsubscribe = subscription.unsubscribe.bind(subscription);
+ const backendSubscription = subscription;
+ let handleClosed = false;
+ let freshnessTimer: WatcherTimer | null = null;
+ let freshnessPassInFlight: Promise<void> | null = null;
+ const scheduleFreshnessPass = (baseMs: number, delayMs: number): void => {
+ if (handleClosed) return;
+ freshnessTimer = clock.setTimeout(() => {
+ freshnessTimer = null;
+ const pass = runFreshnessPass(baseMs);
+ freshnessPassInFlight = pass;
+ void pass.finally(() => {
+ if (freshnessPassInFlight === pass) freshnessPassInFlight = null;
+ });
+ }, delayMs);
+ freshnessTimer.unref();
+ };
+ const walkForFreshness = async (): Promise<DiskObservation | null> => {
+ contentFilter?.refreshInPlaceSkillDirs();
+ const reuse = beginFreshnessReuse('metadata', fileIndex, carryOver);
+ const observed = await observeDisk(contentDir, contentFilter, reuse, () => handleClosed);
+ if (handleClosed) return null;
+ if (reuse) {
+ addFreshnessFilesHashed(reuse.hashed);
+ addFreshnessCarryOver(reuse.deferred.size);
+ }
+ return observed;
+ };
+ const runFreshnessPass = async (baseMs: number): Promise<void> => {
+ if (handleClosed) return;
+ if (!existsSync(contentDir)) {
+ backendSubscription.noticeRootMissing();
+ scheduleFreshnessPass(baseMs, baseMs);
+ return;
+ }
+ if (!backendSubscription.isRecoveryIdle()) {
+ scheduleFreshnessPass(baseMs, baseMs);
+ return;
+ }
+ const startedAt = clock.now();
+ let elapsedMs = 0;
+ const sinceEpoch = openObservationWindow();
+ try {
+ const observed = await walkForFreshness();
+ if (observed === null) return;
+ const result = observed.complete
+ ? await backendSubscription.requestRecovery('metadata', { observed, sinceEpoch })
+ : { status: 'incomplete' as const, candidates: 0, dispatched: 0 };
+ if (handleClosed) return;
+ elapsedMs = clock.now() - startedAt;
+ addFreshnessFinds(result.dispatched);
+ if (result.status === 'incomplete' || result.status === 'failed') {
+ incrementFreshnessPassesIncomplete();
+ }
+ log.debug({ ...result, elapsedMs }, 'freshness re-check pass finished');
+ } catch (err) {
+ if (handleClosed) return;
+ incrementFreshnessPassesIncomplete();
+ log.warn({ err }, 'freshness re-check pass failed');
+ } finally {
+ closeObservationWindow();
+ if (!handleClosed) {
+ incrementFreshnessPasses();
+ scheduleFreshnessPass(baseMs, nextFreshnessDelayMs(elapsedMs, baseMs));
+ }
+ }
+ };
+ if (freshnessIntervalMs !== null) scheduleFreshnessPass(freshnessIntervalMs, freshnessIntervalMs);
- log.info({ contentDir, backend }, 'watching for external .md changes');
+ log.info({ contentDir, backend, freshnessIntervalMs }, 'watching for external .md changes');
return {
async unsubscribe() {
+ handleClosed = true;
+ clock.clearTimeout(freshnessTimer);
+ freshnessTimer = null;
clock.clearInterval(evictionInterval);
clock.clearInterval(dropSummaryInterval);
try {
- await originalUnsubscribe();
+ await backendSubscription.unsubscribe();
} finally {
+ await freshnessPassInFlight;
writeTracker.clear();
removalTracker.clear();
lastKnownHash.clear();
+ lastKnownStat.clear();
+ reportedUnreadable.clear();
resetWatcherDecisionDiagnostics();
restoreClock();
}
},
recheck() {
- return requestRecovery();
+ return backendSubscription.requestRecovery('full');
+ },
+ async recheckPaths(paths) {
+ const unique = [...new Set(paths)];
+ if (unique.length === 0) return { checked: 0, found: 0 };
+ let found = 0;
+ let checked = unique.length;
+ const rootMissing = (): boolean => {
+ if (existsSync(contentDir)) return false;
+ checked = 0;
+ found = 0;
+ log.warn(
+ { contentDir, paths: unique.length },
+ `skipped re-reading ${unique.length} documents: the content folder ${contentDir} is missing`,
+ );
+ backendSubscription.noticeRootMissing();
+ return true;
+ };
+ await backendSubscription.runOrdered(async () => {
+ if (rootMissing()) return;
+ const indexedPaths = new Set<string>();
+ for (const entry of fileIndex.values()) indexedPaths.add(entry.canonicalPath);
+ const changed: InternalRawFileEvent[] = [];
+ for (const path of unique) {
+ const known = lastKnownHash.get(path);
+ const tracked = known !== undefined || indexedPaths.has(path);
+ try {
+ lstatSync(path);
+ } catch (err) {
+ const code = errnoCode(err);
+ if (code === 'ENOENT' || code === 'ENOTDIR') {
+ if (rootMissing()) return;
+ if (tracked) changed.push({ type: 'delete', path, entryKind: 'file' });
+ } else {
+ log.warn({ err, path, code }, `could not re-read ${path}`);
+ }
+ continue;
+ }
+ if (
+ !hasAdmittedMarkdownFile(
+ pathToDocName(path, contentDir),
+ path,
+ contentDir,
+ contentFilter,
+ )
+ ) {
+ log.debug({ path }, `skipped a re-read of ${path}: not an admitted Markdown file`);
+ continue;
+ }
+ let content: string;
+ try {
+ if (statSync(path).size > DOCUMENT_OPEN_BYTE_LIMIT) {
+ log.debug({ path }, `skipped a re-read of ${path}: larger than an open document`);
+ continue;
+ }
+ content = await readFile(path, 'utf-8');
+ } catch (err) {
+ log.warn({ err, path }, `could not re-read ${path}`);
+ continue;
+ }
+ if (contentHash(content) === known) continue;
+ changed.push({ type: tracked ? 'update' : 'create', path });
+ }
+ found = changed.length;
+ if (changed.length === 0) return;
+ onRawBatch?.(changed.map((event) => event.path));
+ await handleRawEventsInternal(
+ changed,
+ contentDir,
+ contentFilter,
+ fileIndex,
+ folderIndex,
+ onDiskEvent,
+ aliasMap,
+ undefined,
+ generalFileIndex,
+ aliasPaths,
+ );
+ bumpFileIndexGeneration();
+ });
+ return { checked, found };
},
getFileIndex() {
if (cachedMarkdownView && cachedMarkdownViewGeneration === fileIndexGeneration) {
@@ -4166,6 +4807,12 @@ async function startWatcherWithClock(
},
mutateFileIndex(reportedEvent) {
const event = declaredDiskEvent(reportedEvent);
+ if (event.kind === 'rename') {
+ noteIntakeMutation(event.oldPath);
+ noteIntakeMutation(event.newPath);
+ } else {
+ noteIntakeMutation(event.path);
+ }
if (event.kind === 'rename') {
activePublicMutations?.markdownNames.add(event.oldDocName);
activePublicMutations?.markdownNames.add(event.newDocName);
@@ -4296,6 +4943,7 @@ async function startWatcherWithClock(
},
pruneFileIndexNowExcluded() {
if (!contentFilter) return 0;
+ noteIntakeMutation(contentDir);
let pruned = 0;
let changed = false;
const removedGeneralPaths = new Set<string>();
@@ -4361,6 +5009,7 @@ async function startWatcherWithClock(
},
pruneFolderIndexNowExcluded() {
if (!contentFilter) return 0;
+ noteIntakeMutation(contentDir);
let pruned = 0;
let aliasesChanged = false;
for (const folderPath of folderIndex.keys()) {
@@ -4388,6 +5037,7 @@ async function startWatcherWithClock(
},
async rescanFromDisk() {
const reseed = async () => {
+ noteIntakeMutation(contentDir);
const mutations: GeneralMutationRecord = {
generalNames: new Set(),
generalEffects: new Map(),
@@ -4426,6 +5076,7 @@ async function startWatcherWithClock(
);
for (const [alias, path] of observation.aliasPaths) aliasPaths.set(alias, path);
bumpFileIndexGeneration();
+ noteIntakeMutation(contentDir);
} finally {
activeReseedMutations = null;
}
diff --git a/packages/server/src/head-watcher.ts b/packages/server/src/head-watcher.ts
index 792c77795..01478841e 100644
--- a/packages/server/src/head-watcher.ts
+++ b/packages/server/src/head-watcher.ts
@@ -3,9 +3,11 @@ import {
type GitRepository,
} from '@inkeep/open-knowledge-core/git-repository';
import { getLogger } from './logger.ts';
+import { incrementWatcherErrors } from './metrics.ts';
import { classifyParcelNotification } from './parcel-recovery.ts';
import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';
import { type WatcherClock, type WatcherTimer, watcherClock } from './watcher-clock.ts';
+import { createResubscriber } from './watcher-freshness.ts';
const log = getLogger('head-watcher');
@@ -28,6 +30,11 @@ interface BatchBeginInfo {
type OnBatchBegin = (info: BatchBeginInfo) => void | Promise<void>;
type OnBatchEnd = (info: BatchEndInfo) => void | Promise<void>;
+interface HeadWatcherCoverage {
+ readonly onGaveUp?: (err: unknown) => void;
+ readonly onBack?: () => void;
+}
+
export interface HeadWatcherHandle {
unsubscribe: () => Promise<void>;
getLastKnownBranch: () => string | null;
@@ -74,6 +81,8 @@ async function tryStartParcelHeadWatcher(
dispatch: (rawPath: string) => boolean,
recover: () => void,
platform: NodeJS.Platform,
+ clock: WatcherClock,
+ coverage: HeadWatcherCoverage,
nativeBackend?: NativeWatcherBackend,
): Promise<(() => Promise<void>) | null> {
let parcel: NativeWatcherBackend;
@@ -91,13 +100,15 @@ async function tryStartParcelHeadWatcher(
}
}
try {
- const subscription = await subscribeParcel(
- parcel,
- gitDir,
+ const notificationsFor =
+ (generation: number): Parameters<typeof subscribeParcel>[2] =>
(err, events) => {
+ if (!resubscriber.isLive(generation)) return;
const notification = classifyParcelNotification(err, events);
if (notification.kind === 'error') {
log.warn({ err: notification.error }, '[head-watcher] parcel subscription error');
+ incrementWatcherErrors();
+ resubscriber.lost('error', generation, notification.error);
return;
}
if (notification.kind === 'rescan') {
@@ -111,11 +122,21 @@ async function tryStartParcelHeadWatcher(
if (dispatch(event.path)) hadWatchedRecord = true;
}
if (notification.kind === 'rescan' && !hadWatchedRecord) recover();
- },
- undefined,
- platform,
- );
- return () => subscription.unsubscribe();
+ };
+ const resubscriber = createResubscriber({
+ clock,
+ root: gitDir,
+ backendName: 'head-parcel',
+ coverageWhileLost:
+ 'branch switches, pulls and other HEAD moves go unnoticed and the server reports head-watcher as degraded',
+ subscribe: (generation) =>
+ subscribeParcel(parcel, gitDir, notificationsFor(generation), undefined, platform),
+ onResubscribed: recover,
+ onGaveUp: coverage.onGaveUp,
+ onBack: coverage.onBack,
+ });
+ await resubscriber.start();
+ return () => resubscriber.close();
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
@@ -151,6 +172,8 @@ export async function startHeadWatcher(
platform?: NodeJS.Platform;
nativeBackend?: NativeWatcherBackend;
clock?: WatcherClock;
+ onGaveUp?: (err: unknown) => void;
+ onBack?: () => void;
subscribeForTest?: (
gitDir: string,
dispatch: HeadEventDispatch,
@@ -304,6 +327,8 @@ export async function startHeadWatcher(
dispatch,
recover,
opts.platform ?? process.platform,
+ clock,
+ { onGaveUp: opts.onGaveUp, onBack: opts.onBack },
opts.nativeBackend,
);
if (resolvedUnsub) backend = 'parcel';
diff --git a/packages/server/src/http/document-refresh-routes.test.ts b/packages/server/src/http/document-refresh-routes.test.ts
new file mode 100644
index 000000000..89eef8bfc
--- /dev/null
+++ b/packages/server/src/http/document-refresh-routes.test.ts
@@ -0,0 +1,69 @@
+import { IncomingMessage } from 'node:http';
+import { Socket } from 'node:net';
+import { describe, expect, test, vi } from 'vitest';
+import { makeCaptureRes, parseProblem } from '../composition-rig.test-helper.ts';
+import type { RefreshDocumentsFromDisk } from '../document-refresh.ts';
+import { buildIngressPolicy } from '../ingress-policy.ts';
+import { loggerFactory } from '../logger.ts';
+import { createApiRequestPipeline } from './api-pipeline.ts';
+import { createDocumentRefreshRoutes } from './document-refresh-routes.ts';
+
+const REFRESH_PATH = '/api/documents/refresh';
+
+function makeRequest(body: unknown, method = 'POST'): IncomingMessage {
+ const req = new IncomingMessage(new Socket());
+ req.push(Buffer.from(JSON.stringify(body)));
+ req.push(null);
+ req.method = method;
+ req.url = REFRESH_PATH;
+ req.headers = { host: '127.0.0.1', 'content-type': 'application/json' };
+ Object.defineProperty(req.socket, 'remoteAddress', { value: '127.0.0.1' });
+ return req;
+}
+
+async function dispatch(refresh: RefreshDocumentsFromDisk, body: unknown, method?: string) {
+ const group = createDocumentRefreshRoutes({ refreshDocumentsFromDisk: refresh });
+ const pipeline = createApiRequestPipeline({
+ log: loggerFactory.getLogger('document-refresh-routes-test'),
+ policy: buildIngressPolicy({}),
+ table: group.table,
+ });
+ const { res, captured } = makeCaptureRes();
+ const handled = await pipeline(makeRequest(body, method), res);
+ return { handled, captured };
+}
+
+describe('createDocumentRefreshRoutes table', () => {
+ test('owns only the refresh path and classifies it as mutating', () => {
+ const group = createDocumentRefreshRoutes({
+ refreshDocumentsFromDisk: async () => ({ checked: 0, found: 0 }),
+ });
+ expect(group.paths).toEqual([REFRESH_PATH]);
+ expect(group.table.resolve(REFRESH_PATH)?.dispatch).toBeDefined();
+ expect(group.table.isMutating(REFRESH_PATH)).toBe(true);
+ for (const path of ['/api/documents', '/api/documents/refresh/', '/api/documents/other']) {
+ expect(group.table.resolve(path), path).toBeNull();
+ expect(group.table.isMutating(path), path).toBe(false);
+ }
+ });
+
+ test('re-reads the named documents as a focus re-read and answers with what it found', async () => {
+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => ({ checked: 2, found: 1 }));
+ const { handled, captured } = await dispatch(refresh, { docNames: ['a', 'folder/b'] });
+
+ expect(handled).toBe(true);
+ expect(captured.status).toBe(200);
+ expect(JSON.parse(captured.body)).toEqual({ checked: 2, found: 1 });
+ expect(refresh).toHaveBeenCalledWith(['a', 'folder/b'], 'focus');
+ });
+
+ test('a failed re-read answers with a problem and no success body', async () => {
+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => {
+ throw new Error('disk unavailable');
+ });
+ const { captured } = await dispatch(refresh, { docNames: ['a'] });
+
+ expect(captured.status).toBe(500);
+ expect(parseProblem(captured.body).type).toBe('urn:ok:error:internal-server-error');
+ });
+});
diff --git a/packages/server/src/http/document-refresh-routes.ts b/packages/server/src/http/document-refresh-routes.ts
new file mode 100644
index 000000000..d315e969d
--- /dev/null
+++ b/packages/server/src/http/document-refresh-routes.ts
@@ -0,0 +1,41 @@
+import {
+ DocumentRefreshRequestSchema,
+ DocumentRefreshSuccessSchema,
+} from '@inkeep/open-knowledge-core';
+import type { RefreshDocumentsFromDisk } from '../document-refresh.ts';
+import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';
+import { errorResponse } from './error-response.ts';
+import { withValidation } from './request-validation.ts';
+import { successResponse } from './success-response.ts';
+
+export interface DocumentRefreshRouteDeps {
+ refreshDocumentsFromDisk: RefreshDocumentsFromDisk;
+}
+
+export function createDocumentRefreshRoutes(deps: DocumentRefreshRouteDeps): ApiRouteGroup {
+ const handleDocumentRefresh = withValidation(
+ DocumentRefreshRequestSchema,
+ async (_req, res, body) => {
+ try {
+ const result = await deps.refreshDocumentsFromDisk(body.docNames, 'focus');
+ successResponse(res, 200, DocumentRefreshSuccessSchema, result, {
+ handler: 'document-refresh',
+ });
+ } catch (err) {
+ errorResponse(
+ res,
+ 500,
+ 'urn:ok:error:internal-server-error',
+ 'Failed to re-read the documents from disk.',
+ { handler: 'document-refresh', cause: err },
+ );
+ }
+ },
+ { handler: 'document-refresh', method: 'POST' },
+ );
+
+ return createApiRouteGroup(
+ { '/api/documents/refresh': handleDocumentRefresh },
+ { mutating: ['/api/documents/refresh'] },
+ );
+}
diff --git a/packages/server/src/http/test-reset-watcher-settle.test.ts b/packages/server/src/http/test-reset-watcher-settle.test.ts
index c5c5baad0..3537fecdb 100644
--- a/packages/server/src/http/test-reset-watcher-settle.test.ts
+++ b/packages/server/src/http/test-reset-watcher-settle.test.ts
@@ -13,7 +13,10 @@ import type { BootedServer } from '../boot.ts';
import { bootCompositionRig } from '../composition-rig.test-helper.ts';
import {
forgetNativeSubscriptions,
+ latestNativeSubscriptionOn,
+ nativeSubscriptionCount,
nativeSubscriptionOn,
+ refuseNativeSubscriptions,
} from '../parcel-watcher-double.test-helper.ts';
vi.mock('@parcel/watcher', async () => {
@@ -73,10 +76,30 @@ test('a server-scope reset runs once the watcher has handled its settle fence',
expect(fencesIn(contentDir)).toEqual([]);
});
-test('after a watcher error ends its subscription, a server-scope reset answers with that error and resets nothing', async () => {
+test('after a watcher error, a server-scope reset runs on the new subscription once it is live', async () => {
const { server, contentDir } = await serverWithTestDoc();
const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;
await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));
+ await expect.poll(() => nativeSubscriptionCount(contentDir)).toBe(2);
+
+ const reset = postServerReset(server);
+ await expect.poll(() => fencesIn(contentDir)).toHaveLength(1);
+ const [fence] = fencesIn(contentDir);
+ await latestNativeSubscriptionOn(contentDir).deliver([
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ const response = await reset;
+
+ expect(response.status).toBe(200);
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe('');
+ expect(fencesIn(contentDir)).toEqual([]);
+});
+
+test('while the watcher cannot subscribe again after an error, a server-scope reset answers with that error and resets nothing', async () => {
+ const { server, contentDir } = await serverWithTestDoc();
+ const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;
+ refuseNativeSubscriptions(contentDir, 3);
+ await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));
const response = await postServerReset(server);
@@ -84,8 +107,9 @@ test('after a watcher error ends its subscription, a server-scope reset answers
expect(await response.json()).toMatchObject({
type: 'urn:ok:error:internal-server-error',
title: 'The server-scope reset did not run, because the file watcher could not settle.',
- detail: `the file watcher failed before it saw its settle fence: ${failure}`,
+ detail: `the file watcher is subscribing again after an error and cannot settle until it is back: ${failure}`,
});
+ expect(nativeSubscriptionCount(contentDir)).toBe(1);
expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe(KEPT);
expect(fencesIn(contentDir)).toEqual([]);
});
diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.ts
index b2350f5ad..3f0ca425d 100644
--- a/packages/server/src/metrics.ts
+++ b/packages/server/src/metrics.ts
@@ -172,6 +172,16 @@ export interface ReconciliationMetrics {
removalRedirectChainCycles: number;
authDocLineageMismatchCount: number;
authDocLineageGuardErrors: number;
+ freshnessPasses: number;
+ freshnessPassesIncomplete: number;
+ freshnessFinds: number;
+ freshnessFilesHashed: number;
+ freshnessCarryOver: number;
+ watcherErrors: number;
+ watcherResubscribes: number;
+ watcherRootMissing: number;
+ watcherUnreadableEntries: number;
+ watcherResubscribesAbandoned: number;
}
const counters: ReconciliationMetrics = {
@@ -294,6 +304,16 @@ const counters: ReconciliationMetrics = {
removalRedirectChainCycles: 0,
authDocLineageMismatchCount: 0,
authDocLineageGuardErrors: 0,
+ freshnessPasses: 0,
+ freshnessPassesIncomplete: 0,
+ freshnessFinds: 0,
+ freshnessFilesHashed: 0,
+ freshnessCarryOver: 0,
+ watcherErrors: 0,
+ watcherResubscribes: 0,
+ watcherRootMissing: 0,
+ watcherUnreadableEntries: 0,
+ watcherResubscribesAbandoned: 0,
};
export function incrementReconcile(): void {
@@ -731,6 +751,46 @@ export function incrementAuthDocLineageGuardError(): void {
counters.authDocLineageGuardErrors++;
}
+export function incrementFreshnessPasses(): void {
+ counters.freshnessPasses++;
+}
+
+export function incrementFreshnessPassesIncomplete(): void {
+ counters.freshnessPassesIncomplete++;
+}
+
+export function addFreshnessFinds(count: number): void {
+ counters.freshnessFinds += count;
+}
+
+export function addFreshnessFilesHashed(count: number): void {
+ counters.freshnessFilesHashed += count;
+}
+
+export function addFreshnessCarryOver(count: number): void {
+ counters.freshnessCarryOver += count;
+}
+
+export function incrementWatcherErrors(): void {
+ counters.watcherErrors++;
+}
+
+export function incrementWatcherResubscribes(): void {
+ counters.watcherResubscribes++;
+}
+
+export function incrementWatcherRootMissing(): void {
+ counters.watcherRootMissing++;
+}
+
+export function incrementWatcherUnreadableEntries(): void {
+ counters.watcherUnreadableEntries++;
+}
+
+export function incrementWatcherResubscribesAbandoned(): void {
+ counters.watcherResubscribesAbandoned++;
+}
+
export function incrementCollabSocketFilteredError(code: 'EPIPE' | 'ECONNRESET'): void {
if (code === 'EPIPE') counters.collabSocketEpipeCount++;
else counters.collabSocketEconnresetCount++;
@@ -912,4 +972,14 @@ export function resetMetrics(): void {
counters.removalRedirectChainCycles = 0;
counters.authDocLineageMismatchCount = 0;
counters.authDocLineageGuardErrors = 0;
+ counters.freshnessPasses = 0;
+ counters.freshnessPassesIncomplete = 0;
+ counters.freshnessFinds = 0;
+ counters.freshnessFilesHashed = 0;
+ counters.freshnessCarryOver = 0;
+ counters.watcherErrors = 0;
+ counters.watcherResubscribes = 0;
+ counters.watcherRootMissing = 0;
+ counters.watcherUnreadableEntries = 0;
+ counters.watcherResubscribesAbandoned = 0;
}
diff --git a/packages/server/src/parcel-drop-notice.test.ts b/packages/server/src/parcel-drop-notice.test.ts
index e957de48e..1e1feffec 100644
--- a/packages/server/src/parcel-drop-notice.test.ts
+++ b/packages/server/src/parcel-drop-notice.test.ts
@@ -712,7 +712,7 @@ test.each(notices)('empty HEAD notice restores an omitted branch change: %s', as
expectHeadSettled(watched);
});
-test('fatal Parcel errors remain observable for both watchers', async () => {
+test('a fatal Parcel error is logged, and both watchers subscribe again and re-read what changed', async () => {
const diagnostics: Array<{ name: string; err: unknown }> = [];
for (const name of ['file-watcher', 'head-watcher']) {
const logger = getLogger(name);
@@ -737,12 +737,15 @@ test('fatal Parcel errors remain observable for both watchers', async () => {
for (const name of ['file-watcher', 'head-watcher']) {
expect(diagnostics).toContainEqual({ name, err: fatal });
}
- expect(events).toEqual([]);
- expect(rawBatches).toEqual([]);
- expect([...watcher.getFileIndex().keys()]).toEqual([]);
- expect(watched.begins).toEqual([]);
- expect(watched.ends).toEqual([]);
- expect(watched.watcher.getLastKnownBranch()).toBe('main');
+ expect(rawBatches).not.toContainEqual([announced]);
+ await vi.waitFor(() => {
+ expect([...watcher.getFileIndex().keys()].sort()).toEqual(['announced', 'hidden']);
+ });
+ expect(
+ events.flatMap((event) => (event.kind === 'create' ? [event.docName] : [])).sort(),
+ ).toEqual(['announced', 'hidden']);
+ await vi.waitFor(() => expect(watched.ends).toHaveLength(1));
+ expectHeadSettled(watched);
});
test('repeated content notices do not repeat effects already applied from a supplied batch and disk scan', async () => {
diff --git a/packages/server/src/parcel-watcher-double.test-helper.ts b/packages/server/src/parcel-watcher-double.test-helper.ts
index 0f89cc683..c67bef534 100644
--- a/packages/server/src/parcel-watcher-double.test-helper.ts
+++ b/packages/server/src/parcel-watcher-double.test-helper.ts
@@ -37,12 +37,19 @@ function register(dir: string, callback: SubscribeCallback): Registration {
};
}
+const refusals = new Map<string, number>();
+
const parcelWatcherDouble = {
subscribe(
dir: string,
callback: SubscribeCallback,
_options?: Options,
): Promise<AsyncSubscription> {
+ const refused = refusals.get(resolve(dir)) ?? 0;
+ if (refused > 0) {
+ refusals.set(resolve(dir), refused - 1);
+ return Promise.reject(new Error(`the watcher double refused to watch ${dir}`));
+ }
const registration = register(resolve(dir), callback);
registrations.push(registration);
return Promise.resolve({ unsubscribe: () => registration.release() });
@@ -59,6 +66,21 @@ export const parcelWatcherModule = { default: parcelWatcherDouble, ...parcelWatc
export function forgetNativeSubscriptions(): void {
registrations.length = 0;
+ refusals.clear();
+}
+
+export function refuseNativeSubscriptions(dir: string, count: number): void {
+ refusals.set(resolve(dir), count);
+}
+
+export function nativeSubscriptionCount(dir: string): number {
+ return registrations.filter((registration) => registration.dir === dir).length;
+}
+
+export function latestNativeSubscriptionOn(dir: string): NativeSubscriptionDouble {
+ const latest = registrations.filter((registration) => registration.dir === dir).at(-1);
+ if (!latest) throw new Error(`no native subscription on ${dir}`);
+ return latest;
}
export function nativeSubscriptionDirs(): string[] {
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index bec310944..a79f8be7f 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -145,6 +145,11 @@ import {
DocumentDurabilityState,
type StaleExternalWriteConflict,
} from './document-durability-state.ts';
+import {
+ createReconnectRefreshExtension,
+ type DocumentRefreshResult,
+ type DocumentRefreshTrigger,
+} from './document-refresh.ts';
import {
type Embedder,
type EmbeddingsKeyStore,
@@ -346,7 +351,8 @@ import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extensio
import { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';
import { trustSystemCertificates } from './trust-system-ca.ts';
import { cleanupOrphanUploadTempfiles } from './upload-streaming.ts';
-import type { WatcherClock } from './watcher-clock.ts';
+import { type WatcherClock, watcherClock } from './watcher-clock.ts';
+import { FRESHNESS_BASE_INTERVAL_MS } from './watcher-freshness.ts';
export interface ServerOptions {
acpRegistryFetchImpl?: typeof fetch;
@@ -395,6 +401,7 @@ export interface ServerOptions {
ephemeral?: boolean;
watcherBackend?: NativeWatcherBackend;
watcherClock?: WatcherClock;
+ freshnessIntervalMs?: number | null;
mergeHistoryWriter?: MergeHistoryWriter;
generatedIndexTestHooks?: {
beforePlan?: (context: { fullSweep: boolean; signal: AbortSignal }) => Promise<void> | void;
@@ -2684,6 +2691,7 @@ export function createServer(options: ServerOptions): ServerInstance {
getSyncEngine: () => syncEngine,
conflicts,
setBatchInProgress: setBatchInProgressAndDrain,
+ refreshDocumentsFromDisk,
localOpCliArgs,
authStreamHeartbeatMs,
projectDir,
@@ -2731,6 +2739,12 @@ export function createServer(options: ServerOptions): ServerInstance {
},
});
hocuspocus.configuration.extensions.push(apiExtension);
+ hocuspocus.configuration.extensions.push(
+ createReconnectRefreshExtension({
+ clock: options.watcherClock ?? watcherClock(),
+ refresh: refreshDocumentsFromDisk,
+ }),
+ );
nativeApi = apiExtension.nativeApi;
localApi = apiExtension.localApi;
shutdownLocalOps = apiExtension.shutdownLocalOps;
@@ -3669,6 +3683,40 @@ export function createServer(options: ServerOptions): ServerInstance {
}
let watcher: WatcherHandle | null = null;
+
+ async function refreshDocumentsFromDisk(
+ docNames: readonly string[],
+ trigger: DocumentRefreshTrigger,
+ ): Promise<DocumentRefreshResult> {
+ if (!watcher) return { checked: 0, found: 0 };
+ const aliases = watcher.getAliasMap();
+ const index = watcher.getFileIndex();
+ const paths: string[] = [];
+ for (const requested of new Set(docNames)) {
+ const docName = aliases.get(requested) ?? requested;
+ if (isReservedForUserTree(docName)) continue;
+ const indexed = index.get(docName)?.canonicalPath;
+ if (indexed !== undefined) {
+ paths.push(indexed);
+ continue;
+ }
+ try {
+ paths.push(safeContentPath(docName, contentDir));
+ } catch (err) {
+ log.debug(
+ { err, trigger },
+ '[reconcile] skipped a re-read for a name outside the content tree',
+ );
+ }
+ }
+ if (paths.length === 0) return { checked: 0, found: 0 };
+ const result = await watcher.recheckPaths(paths);
+ const summary = { trigger, ...result };
+ if (result.found > 0)
+ log.info(summary, `[reconcile] re-read on ${trigger} found outside changes`);
+ else log.debug(summary, `[reconcile] re-read on ${trigger} found nothing new`);
+ return result;
+ }
let syncEngine: SyncEngine | null = null;
let inflightDestroy: Promise<void> | null = null;
@@ -4792,6 +4840,10 @@ export function createServer(options: ServerOptions): ServerInstance {
onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),
nativeBackend: options.watcherBackend,
clock: options.watcherClock,
+ freshnessIntervalMs:
+ options.freshnessIntervalMs === undefined
+ ? FRESHNESS_BASE_INTERVAL_MS
+ : options.freshnessIntervalMs,
}),
);
void resyncRecordedSkillCopies(projectDir, contentDir)
@@ -5286,7 +5338,17 @@ export function createServer(options: ServerOptions): ServerInstance {
}
}
},
- { nativeBackend: options.watcherBackend, clock: options.watcherClock },
+ {
+ nativeBackend: options.watcherBackend,
+ clock: options.watcherClock,
+ onGaveUp: () => {
+ if (!degraded.includes('head-watcher')) degraded.push('head-watcher');
+ },
+ onBack: () => {
+ const at = degraded.indexOf('head-watcher');
+ if (at !== -1) degraded.splice(at, 1);
+ },
+ },
);
} catch (err) {
log.error({ err }, '[server] HEAD watcher failed to start');
diff --git a/packages/server/src/server-freshness-default.test.ts b/packages/server/src/server-freshness-default.test.ts
new file mode 100644
index 000000000..00754ee4b
--- /dev/null
+++ b/packages/server/src/server-freshness-default.test.ts
@@ -0,0 +1,45 @@
+import { mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, expect, test } from 'vitest';
+import type { BootedServer } from './boot.ts';
+import { bootCompositionRig } from './composition-rig.test-helper.ts';
+import { getMetrics } from './metrics.ts';
+import { createManualClock } from './watcher-clock.test-helper.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+import { FRESHNESS_BASE_INTERVAL_MS } from './watcher-freshness.ts';
+
+let root: string | undefined;
+let server: BootedServer | undefined;
+
+afterEach(async () => {
+ await server?.destroy();
+ server = undefined;
+ if (root) rmSync(root, { recursive: true, force: true });
+ root = undefined;
+});
+
+async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
+ const start = performance.now();
+ while (!condition()) {
+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+test('a booted server re-checks the content tree every five seconds unless told otherwise', async () => {
+ root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-freshness-default-')));
+ const clock = createManualClock(Date.now());
+ server = await bootCompositionRig(root, {
+ watcherBackend: createWatcherDouble().backend,
+ watcherClock: clock,
+ });
+ await server.ready;
+ writeFileSync(join(root, 'never-delivered.md'), '# Never delivered\n');
+ const passesBefore = getMetrics().freshnessPasses;
+ const findsBefore = getMetrics().freshnessFinds;
+
+ clock.advance(FRESHNESS_BASE_INTERVAL_MS);
+ await until(() => getMetrics().freshnessPasses > passesBefore, 'the first timed pass');
+ expect(getMetrics().freshnessFinds - findsBefore).toBeGreaterThanOrEqual(1);
+});
diff --git a/packages/server/src/server-head-watcher-degraded.test.ts b/packages/server/src/server-head-watcher-degraded.test.ts
new file mode 100644
index 000000000..1557feb55
--- /dev/null
+++ b/packages/server/src/server-head-watcher-degraded.test.ts
@@ -0,0 +1,66 @@
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, expect, test } from 'vitest';
+import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts';
+import type { BootedServer } from './boot.ts';
+import { bootCompositionRig } from './composition-rig.test-helper.ts';
+import { getMetrics } from './metrics.ts';
+import { createManualClock } from './watcher-clock.test-helper.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+
+let root: string | undefined;
+let server: BootedServer | undefined;
+
+afterEach(async () => {
+ await server?.destroy();
+ server = undefined;
+ if (root) rmSync(root, { recursive: true, force: true });
+ root = undefined;
+});
+
+async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
+ const start = performance.now();
+ while (!condition()) {
+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+test('a server whose HEAD watcher gives up reports head-watcher as degraded until it is back', async () => {
+ root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-head-degraded-')));
+ const gitEnv = {
+ ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_'))),
+ GIT_CONFIG_GLOBAL: '/dev/null',
+ GIT_CONFIG_NOSYSTEM: '1',
+ };
+ execFileSync('git', ['init', '--quiet', '--initial-branch=main'], { cwd: root, env: gitEnv });
+ configureTestGitRepository(root);
+ const clock = createManualClock(Date.now());
+ const double = createWatcherDouble();
+ const booted = await bootCompositionRig(root, {
+ watcherBackend: double.backend,
+ watcherClock: clock,
+ freshnessIntervalMs: null,
+ });
+ server = booted;
+ await booted.ready;
+ const gitDir = join(root, '.git');
+ expect(booted.degraded).not.toContain('head-watcher');
+
+ const gaveUpBefore = getMetrics().watcherResubscribesAbandoned;
+ for (let failure = 1; failure <= 8; failure++) {
+ double.failNextSubscribe(gitDir, new Error(`watch limit ${failure}`));
+ }
+ double.subscriptionFor(gitDir).fail(new Error('the native watcher died'));
+ await until(() => {
+ clock.advance(30_000);
+ return getMetrics().watcherResubscribesAbandoned > gaveUpBefore;
+ }, 'the HEAD watcher to give up');
+ expect(booted.degraded).toContain('head-watcher');
+
+ clock.advance(5 * 60_000);
+ await until(() => !booted.degraded.includes('head-watcher'), 'the HEAD watcher to be back');
+ expect(double.subscriptionCount(gitDir)).toBe(2);
+});
diff --git a/packages/server/src/watcher-double.test-helper.ts b/packages/server/src/watcher-double.test-helper.ts
index a49c80425..72b639593 100644
--- a/packages/server/src/watcher-double.test-helper.ts
+++ b/packages/server/src/watcher-double.test-helper.ts
@@ -8,6 +8,7 @@ type WatcherDoubleEvent = Parameters<SubscribeCallback>[1][number];
interface WatcherDoubleSubscription {
readonly dir: string;
readonly closed: boolean;
+ readonly dead: boolean;
deliver(events: WatcherDoubleEvent[]): void;
fail(error: Error): void;
rescan(events?: WatcherDoubleEvent[]): void;
@@ -16,25 +17,36 @@ interface WatcherDoubleSubscription {
export function createWatcherDouble(): {
readonly backend: NativeWatcherBackend;
subscriptionFor(dir: string): WatcherDoubleSubscription;
+ subscriptionCount(dir: string): number;
+ failNextSubscribe(dir: string, error: Error): void;
} {
const subscriptions: WatcherDoubleSubscription[] = [];
+ const subscribeFailures = new Map<string, Error[]>();
const [rescanNotice] = RESCAN_NOTICES;
const backend: NativeWatcherBackend = {
async subscribe(dir, callback) {
+ const failure = subscribeFailures.get(dir)?.shift();
+ if (failure) throw failure;
let closed = false;
+ let dead = false;
const subscription: WatcherDoubleSubscription = {
dir,
get closed() {
return closed;
},
+ get dead() {
+ return dead;
+ },
deliver(events) {
- if (!closed) callback(null, events);
+ if (!closed && !dead) callback(null, events);
},
fail(error) {
- if (!closed) callback(error, []);
+ if (closed || dead) return;
+ dead = true;
+ callback(error, []);
},
rescan(events = []) {
- if (!closed) callback(new Error(rescanNotice), events);
+ if (!closed && !dead) callback(new Error(rescanNotice), events);
},
};
subscriptions.push(subscription);
@@ -45,10 +57,12 @@ export function createWatcherDouble(): {
};
},
};
+ const forDir = (dir: string): WatcherDoubleSubscription[] =>
+ subscriptions.filter((subscription) => subscription.dir === dir);
return {
backend,
subscriptionFor(dir) {
- const found = subscriptions.find((subscription) => subscription.dir === dir);
+ const found = forDir(dir).at(-1);
if (!found) {
throw new Error(
`no watcher double subscription for ${dir}; have ${subscriptions.map((s) => s.dir).join(', ') || 'none'}`,
@@ -56,5 +70,13 @@ export function createWatcherDouble(): {
}
return found;
},
+ subscriptionCount(dir) {
+ return forDir(dir).length;
+ },
+ failNextSubscribe(dir, error) {
+ const queued = subscribeFailures.get(dir) ?? [];
+ queued.push(error);
+ subscribeFailures.set(dir, queued);
+ },
};
}
diff --git a/packages/server/src/watcher-freshness.ts b/packages/server/src/watcher-freshness.ts
new file mode 100644
index 000000000..31ab78b4d
--- /dev/null
+++ b/packages/server/src/watcher-freshness.ts
@@ -0,0 +1,215 @@
+import { existsSync } from 'node:fs';
+import { getLogger } from './logger.ts';
+import {
+ incrementWatcherResubscribes,
+ incrementWatcherResubscribesAbandoned,
+ incrementWatcherRootMissing,
+} from './metrics.ts';
+import type { WatcherClock, WatcherTimer } from './watcher-clock.ts';
+
+const log = getLogger('watcher-freshness');
+
+export const FRESHNESS_BASE_INTERVAL_MS = 5_000;
+export const FRESHNESS_MAX_INTERVAL_MS = 60_000;
+const FRESHNESS_DUTY_CYCLE_FACTOR = 20;
+
+export const FRESHNESS_HASH_BUDGET = { files: 512, bytes: 64 * 1024 * 1024 } as const;
+
+const RESUBSCRIBE_EARLY_DELAYS_MS: readonly number[] = [0, 1_000, 5_000];
+const RESUBSCRIBE_FINAL_DELAY_MS = 30_000;
+const RESUBSCRIBE_MAX_ATTEMPTS = 8;
+export const RESUBSCRIBE_SLOW_RETRY_MS = 5 * 60_000;
+const RESUBSCRIBE_ROOT_POLL_MS = 1_000;
+const RESUBSCRIBE_STABLE_MS = 60_000;
+
+type ResubscribeReason = 'error' | 'root-delete';
+
+export type RecoveryMode = 'metadata' | 'full';
+
+export function nextFreshnessDelayMs(lastPassMs: number, baseMs: number): number {
+ const ceiling = Math.max(baseMs, FRESHNESS_MAX_INTERVAL_MS);
+ const proportional = Math.ceil(Math.max(0, lastPassMs) * FRESHNESS_DUTY_CYCLE_FACTOR);
+ return Math.min(ceiling, Math.max(baseMs, proportional));
+}
+
+function resubscribeDelayMs(attempt: number): number {
+ return RESUBSCRIBE_EARLY_DELAYS_MS[attempt] ?? RESUBSCRIBE_FINAL_DELAY_MS;
+}
+
+export function resolveFreshnessInterval(option: number | null | undefined): number | null {
+ if (option === null || option === undefined) return null;
+ if (!Number.isFinite(option) || option <= 0) {
+ throw new RangeError('freshnessIntervalMs must be a positive number of milliseconds or null');
+ }
+ return option;
+}
+
+interface WatcherSubscriptionLike {
+ unsubscribe(): Promise<void>;
+}
+
+export interface Resubscriber {
+ start(): Promise<void>;
+ isLive(generation: number): boolean;
+ lost(reason: ResubscribeReason, generation: number, err?: unknown): void;
+ noticeRootMissing(): void;
+ close(): Promise<void>;
+}
+
+export function createResubscriber(deps: {
+ clock: WatcherClock;
+ root: string;
+ backendName: string;
+ coverageWhileLost: string;
+ subscribe: (generation: number) => Promise<WatcherSubscriptionLike>;
+ onResubscribed: () => void;
+ onGaveUp?: (err: unknown) => void;
+ onBack?: () => void;
+}): Resubscriber {
+ let closed = false;
+ let generation = 0;
+ let liveGeneration: number | null = null;
+ let current: WatcherSubscriptionLike | null = null;
+ let liveSince = Number.NEGATIVE_INFINITY;
+ let attempt = 0;
+ let timer: WatcherTimer | null = null;
+ let inFlight: Promise<void> | null = null;
+ let slowRetry = false;
+ const releases = new Set<Promise<void>>();
+ const backend = deps.backendName;
+
+ const release = (subscription: WatcherSubscriptionLike): void => {
+ const released = subscription.unsubscribe().catch((err: unknown) => {
+ log.debug({ err, backend }, 'releasing a lost watcher subscription failed');
+ });
+ releases.add(released);
+ void released.then(() => releases.delete(released));
+ };
+
+ const attemptResubscribe = async (reason: ResubscribeReason): Promise<void> => {
+ if (closed) return;
+ if (!existsSync(deps.root)) {
+ log.debug(
+ { backend, reason },
+ 'the watched directory is missing; waiting before resubscribing',
+ );
+ schedule(RESUBSCRIBE_ROOT_POLL_MS, reason);
+ return;
+ }
+ const next = ++generation;
+ liveGeneration = next;
+ let subscription: WatcherSubscriptionLike;
+ try {
+ subscription = await deps.subscribe(next);
+ } catch (err) {
+ if (liveGeneration === next) liveGeneration = null;
+ if (slowRetry) {
+ log.debug({ err, backend, reason }, 'watcher still cannot subscribe; trying again later');
+ schedule(RESUBSCRIBE_SLOW_RETRY_MS, reason);
+ return;
+ }
+ attempt++;
+ if (attempt >= RESUBSCRIBE_MAX_ATTEMPTS) {
+ giveUp(reason, err);
+ return;
+ }
+ log.warn({ err, backend, reason, attempt }, 'watcher resubscribe failed; retrying');
+ schedule(resubscribeDelayMs(attempt), reason);
+ return;
+ }
+ if (closed || liveGeneration !== next) {
+ release(subscription);
+ return;
+ }
+ current = subscription;
+ liveSince = deps.clock.now();
+ incrementWatcherResubscribes();
+ if (slowRetry) {
+ slowRetry = false;
+ attempt = 0;
+ log.info({ backend, reason }, 'watcher subscribed again after it had given up');
+ deps.onBack?.();
+ } else {
+ log.info({ backend, reason, attempt }, 'watcher resubscribed');
+ }
+ deps.onResubscribed();
+ };
+
+ const giveUp = (reason: ResubscribeReason, err: unknown): void => {
+ slowRetry = true;
+ incrementWatcherResubscribesAbandoned();
+ log.warn(
+ { err, backend, reason, attempts: attempt, retryMs: RESUBSCRIBE_SLOW_RETRY_MS },
+ `watcher subscription failed ${attempt} times in a row; it now tries again every ${RESUBSCRIBE_SLOW_RETRY_MS / 60_000} minutes, and until it is back ${deps.coverageWhileLost}`,
+ );
+ deps.onGaveUp?.(err);
+ schedule(RESUBSCRIBE_SLOW_RETRY_MS, reason);
+ };
+
+ const schedule = (delayMs: number, reason: ResubscribeReason): void => {
+ if (closed) return;
+ timer = deps.clock.setTimeout(() => {
+ timer = null;
+ const work = attemptResubscribe(reason);
+ inFlight = work;
+ void work.finally(() => {
+ if (inFlight === work) inFlight = null;
+ });
+ }, delayMs);
+ timer.unref();
+ };
+
+ const lost = (reason: ResubscribeReason, lostGeneration: number, err?: unknown): void => {
+ if (closed || lostGeneration !== liveGeneration) return;
+ liveGeneration = null;
+ const dead = current;
+ current = null;
+ if (dead) release(dead);
+ if (reason === 'root-delete') incrementWatcherRootMissing();
+ attempt = deps.clock.now() - liveSince < RESUBSCRIBE_STABLE_MS ? attempt + 1 : 0;
+ if (attempt >= RESUBSCRIBE_MAX_ATTEMPTS) {
+ giveUp(
+ reason,
+ err ??
+ new Error(
+ `the ${reason === 'error' ? 'subscription failed' : 'watched directory was deleted'}`,
+ ),
+ );
+ return;
+ }
+ log.warn({ err, backend, reason, attempt }, 'watcher subscription lost; resubscribing');
+ schedule(resubscribeDelayMs(attempt), reason);
+ };
+
+ return {
+ async start() {
+ const first = ++generation;
+ liveGeneration = first;
+ const subscription = await deps.subscribe(first);
+ if (closed || liveGeneration !== first) {
+ release(subscription);
+ return;
+ }
+ current = subscription;
+ },
+ isLive: (candidate) => !closed && candidate === liveGeneration,
+ lost,
+ noticeRootMissing() {
+ if (liveGeneration !== null) lost('root-delete', liveGeneration);
+ },
+ async close() {
+ closed = true;
+ deps.clock.clearTimeout(timer);
+ timer = null;
+ await inFlight;
+ const live = current;
+ current = null;
+ liveGeneration = null;
+ try {
+ if (live) await live.unsubscribe();
+ } finally {
+ await Promise.all(releases);
+ }
+ },
+ };
+}
diff --git a/packages/server/src/watcher-resubscribe-chokidar.test.ts b/packages/server/src/watcher-resubscribe-chokidar.test.ts
new file mode 100644
index 000000000..96a84ae0f
--- /dev/null
+++ b/packages/server/src/watcher-resubscribe-chokidar.test.ts
@@ -0,0 +1,127 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { type DiskEvent, lastKnownHash, startWatcher, type WatcherHandle } from './file-watcher.ts';
+import { getMetrics } from './metrics.ts';
+import { createManualClock, type ManualClock } from './watcher-clock.test-helper.ts';
+
+const fakeChokidar = vi.hoisted(() => {
+ type Listener = (...args: unknown[]) => void;
+ class FakeWatcher {
+ closed = false;
+ private readonly listeners = new Map<string, Listener[]>();
+ on(event: string, listener: Listener): this {
+ this.listeners.set(event, [...(this.listeners.get(event) ?? []), listener]);
+ return this;
+ }
+ once(event: string, listener: Listener): this {
+ const wrapped: Listener = (...args) => {
+ this.listeners.set(
+ event,
+ (this.listeners.get(event) ?? []).filter((candidate) => candidate !== wrapped),
+ );
+ listener(...args);
+ };
+ return this.on(event, wrapped);
+ }
+ emit(event: string, ...args: unknown[]): void {
+ for (const listener of this.listeners.get(event) ?? []) listener(...args);
+ }
+ async close(): Promise<void> {
+ this.closed = true;
+ }
+ }
+ const watchers: FakeWatcher[] = [];
+ return {
+ watchers,
+ watch: () => {
+ const watcher = new FakeWatcher();
+ watchers.push(watcher);
+ queueMicrotask(() => watcher.emit('ready'));
+ return watcher;
+ },
+ };
+});
+
+vi.mock('chokidar', () => ({ watch: fakeChokidar.watch }));
+
+async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
+ const start = performance.now();
+ while (!condition()) {
+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+function errorWithCode(message: string, code: string): NodeJS.ErrnoException {
+ return Object.assign(new Error(message), { code });
+}
+
+describe('chokidar watcher resubscribe', () => {
+ let tmpDir: string;
+ let contentDir: string;
+ let handle: WatcherHandle | undefined;
+ let clock: ManualClock;
+ let events: DiskEvent[];
+
+ beforeEach(async () => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-chokidar-resubscribe-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ contentDir = realpathSync(contentDir);
+ lastKnownHash.clear();
+ fakeChokidar.watchers.length = 0;
+ clock = createManualClock(Date.now());
+ events = [];
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { forceBackend: 'chokidar', clock, freshnessIntervalMs: null, platform: 'linux' },
+ );
+ });
+
+ afterEach(async () => {
+ await handle?.unsubscribe();
+ handle = undefined;
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test('an error that stops chokidar makes the watcher watch again and re-check the tree', async () => {
+ expect(fakeChokidar.watchers).toHaveLength(1);
+ const first = fakeChokidar.watchers[0];
+ writeFileSync(join(contentDir, 'gap.md'), '# Written while chokidar was failing\n');
+
+ first?.emit('error', new Error('the watcher stopped'));
+ clock.advance(0);
+
+ await until(() => fakeChokidar.watchers.length === 2, 'a second chokidar watcher');
+ expect(first?.closed).toBe(true);
+ await until(
+ () => events.some((event) => event.kind === 'create' && event.docName === 'gap'),
+ 'the re-check to find the gap write',
+ );
+ });
+
+ test.each([
+ ['EACCES', 'permission denied'],
+ ['ENOSPC', 'inotify watch limit reached'],
+ ['EMFILE', 'too many open files'],
+ ['ENFILE', 'file table overflow'],
+ ])('a %s error is counted while the watcher stays up', async (code, message) => {
+ const errorsBefore = getMetrics().watcherErrors;
+ const resubscribesBefore = getMetrics().watcherResubscribes;
+ fakeChokidar.watchers[0]?.emit('error', errorWithCode(message, code));
+ fakeChokidar.watchers[0]?.emit('error', errorWithCode(message, code));
+ clock.advance(60_000);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+
+ expect(getMetrics().watcherErrors - errorsBefore).toBe(2);
+ expect(getMetrics().watcherResubscribes - resubscribesBefore).toBe(0);
+ expect(fakeChokidar.watchers).toHaveLength(1);
+ expect(fakeChokidar.watchers[0]?.closed).toBe(false);
+ });
+});
diff --git a/packages/server/src/watcher-resubscribe.test.ts b/packages/server/src/watcher-resubscribe.test.ts
new file mode 100644
index 000000000..24399ba11
--- /dev/null
+++ b/packages/server/src/watcher-resubscribe.test.ts
@@ -0,0 +1,320 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { type DiskEvent, lastKnownHash, startWatcher, type WatcherHandle } from './file-watcher.ts';
+import { type HeadWatcherHandle, startHeadWatcher } from './head-watcher.ts';
+import { getLogger } from './logger.ts';
+import { getMetrics } from './metrics.ts';
+import { createManualClock, type ManualClock } from './watcher-clock.test-helper.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+
+const SLOW_RETRY_MS = 5 * 60_000;
+
+async function flushMicrotasks(): Promise<void> {
+ for (let round = 0; round < 5; round++) {
+ await new Promise((resolve) => setImmediate(resolve));
+ }
+}
+
+async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
+ const start = performance.now();
+ while (!condition()) {
+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+}
+
+function createsOf(events: readonly DiskEvent[], docName: string): DiskEvent[] {
+ return events.filter((event) => event.kind === 'create' && event.docName === docName);
+}
+
+describe('content watcher resubscribe', () => {
+ let tmpDir: string;
+ let contentDir: string;
+ let handle: WatcherHandle | undefined;
+ let clock: ManualClock;
+ let events: DiskEvent[];
+ let double: ReturnType<typeof createWatcherDouble>;
+
+ beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-resubscribe-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ contentDir = realpathSync(contentDir);
+ lastKnownHash.clear();
+ clock = createManualClock(Date.now());
+ events = [];
+ double = createWatcherDouble();
+ });
+
+ afterEach(async () => {
+ await handle?.unsubscribe();
+ handle = undefined;
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ async function watch(freshnessIntervalMs: number | null = null): Promise<void> {
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { nativeBackend: double.backend, clock, freshnessIntervalMs },
+ );
+ }
+
+ test('after a backend error the watcher subscribes again and finds the write made in the gap', async () => {
+ await watch();
+ const errorsBefore = getMetrics().watcherErrors;
+ const resubscribesBefore = getMetrics().watcherResubscribes;
+ const first = double.subscriptionFor(contentDir);
+
+ first.fail(new Error('the native watcher died'));
+ expect(first.dead).toBe(true);
+ expect(getMetrics().watcherErrors - errorsBefore).toBe(1);
+ writeFileSync(join(contentDir, 'gap.md'), '# Written while the watcher was dead\n');
+ first.deliver([{ type: 'create', path: join(contentDir, 'gap.md') }]);
+ expect(createsOf(events, 'gap')).toHaveLength(0);
+
+ clock.advance(0);
+ await until(() => double.subscriptionCount(contentDir) === 2, 'a second subscription');
+ await until(() => createsOf(events, 'gap').length === 1, 'the gap write to be found');
+ expect(getMetrics().watcherResubscribes - resubscribesBefore).toBe(1);
+ expect(first.closed).toBe(true);
+
+ const later = join(contentDir, 'later.md');
+ writeFileSync(later, '# Delivered by the new subscription\n');
+ double.subscriptionFor(contentDir).deliver([{ type: 'create', path: later }]);
+ await until(() => createsOf(events, 'later').length === 1, 'the new subscription to deliver');
+ });
+
+ async function flush(): Promise<void> {
+ for (let round = 0; round < 5; round++) {
+ await new Promise((resolve) => setImmediate(resolve));
+ }
+ }
+
+ test('a resubscribe that keeps failing backs off 0, 1 and 5 seconds, then 30 seconds', async () => {
+ await watch();
+ for (let failure = 0; failure < 3; failure++) {
+ double.failNextSubscribe(contentDir, new Error('the backend is not ready yet'));
+ }
+ double.subscriptionFor(contentDir).fail(new Error('the native watcher died'));
+
+ for (const delayMs of [0, 1_000, 5_000]) {
+ if (delayMs > 0) {
+ clock.advance(delayMs - 1);
+ await flush();
+ }
+ expect(double.subscriptionCount(contentDir)).toBe(1);
+ clock.advance(delayMs === 0 ? 0 : 1);
+ await flush();
+ }
+ clock.advance(29_999);
+ await flush();
+ expect(double.subscriptionCount(contentDir)).toBe(1);
+ clock.advance(1);
+ await until(() => double.subscriptionCount(contentDir) === 2, 'the 30 s retry');
+ });
+
+ test('after eight failed resubscribes the watcher tries again every five minutes until the fault clears', async () => {
+ await watch();
+ const warn = vi.spyOn(getLogger('watcher-freshness'), 'warn');
+ const gaveUpBefore = getMetrics().watcherResubscribesAbandoned;
+ for (let failure = 1; failure <= 8; failure++) {
+ double.failNextSubscribe(contentDir, new Error(`no space left for watches ${failure}`));
+ }
+ double.subscriptionFor(contentDir).fail(new Error('the native watcher died'));
+ for (let step = 0; step < 8; step++) {
+ clock.advance(30_000);
+ await flush();
+ }
+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(1);
+ const gaveUp = warn.mock.calls.find(([, message]) =>
+ String(message).includes('tries again every 5 minutes'),
+ );
+ expect((gaveUp?.[0] as { err?: Error } | undefined)?.err?.message).toBe(
+ 'no space left for watches 8',
+ );
+ await expect((handle as WatcherHandle).settle()).rejects.toThrow(
+ 'gave up subscribing after repeated failures (no space left for watches 8)',
+ );
+
+ double.failNextSubscribe(contentDir, new Error('still no space'));
+ clock.advance(SLOW_RETRY_MS - 1);
+ await flush();
+ clock.advance(1);
+ await flush();
+ expect(double.subscriptionCount(contentDir)).toBe(1);
+
+ writeFileSync(join(contentDir, 'while-down.md'), '# Written while the watcher was down\n');
+ clock.advance(SLOW_RETRY_MS);
+ await until(() => double.subscriptionCount(contentDir) === 2, 'the slow retry to subscribe');
+ await until(() => createsOf(events, 'while-down').length === 1, 'the write made while down');
+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(1);
+ });
+
+ test('a watcher whose new subscriptions keep dying within a minute is given up after eight short lives', async () => {
+ await watch();
+ const gaveUpBefore = getMetrics().watcherResubscribesAbandoned;
+ const resubscribesBefore = getMetrics().watcherResubscribes;
+ double.subscriptionFor(contentDir).fail(new Error('the first subscription died'));
+ for (let life = 1; life <= 8; life++) {
+ clock.advance(30_000);
+ await until(
+ () => getMetrics().watcherResubscribes - resubscribesBefore === life,
+ `resubscribe ${life}`,
+ );
+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(0);
+ double.subscriptionFor(contentDir).fail(new Error(`short life ${life}`));
+ }
+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(1);
+ });
+
+ test('a subscription that lasted a minute is replaced at once, one that died sooner after a delay', async () => {
+ await watch();
+ const resubscribed = (count: number) => () =>
+ double.subscriptionCount(contentDir) === count + 1 &&
+ getMetrics().watcherResubscribes - resubscribesBefore === count;
+ const resubscribesBefore = getMetrics().watcherResubscribes;
+
+ double.subscriptionFor(contentDir).fail(new Error('first death'));
+ clock.advance(0);
+ await until(resubscribed(1), 'the first resubscribe');
+
+ double.subscriptionFor(contentDir).fail(new Error('died within a minute'));
+ clock.advance(999);
+ await flush();
+ expect(double.subscriptionCount(contentDir)).toBe(2);
+ clock.advance(1);
+ await until(resubscribed(2), 'the delayed resubscribe');
+
+ clock.advance(60_000);
+ double.subscriptionFor(contentDir).fail(new Error('died after a stable minute'));
+ clock.advance(0);
+ await until(resubscribed(3), 'the immediate resubscribe');
+ });
+
+ test('a deleted content root is watched again once it is recreated', async () => {
+ await watch();
+ const rootMissingBefore = getMetrics().watcherRootMissing;
+ rmSync(contentDir, { recursive: true, force: true });
+ double.subscriptionFor(contentDir).deliver([{ type: 'delete', path: contentDir }]);
+ expect(getMetrics().watcherRootMissing - rootMissingBefore).toBe(1);
+
+ clock.advance(0);
+ clock.advance(1_000);
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(double.subscriptionCount(contentDir)).toBe(1);
+
+ mkdirSync(contentDir);
+ writeFileSync(join(contentDir, 'after-recreate.md'), '# After\n');
+ clock.advance(1_000);
+ await until(() => double.subscriptionCount(contentDir) === 2, 'the root to be watched again');
+ await until(
+ () => createsOf(events, 'after-recreate').length === 1,
+ 'the write after the recreate to be found',
+ );
+ });
+
+ test('the timed pass notices a content root that vanished without any event', async () => {
+ await watch(5_000);
+ const rootMissingBefore = getMetrics().watcherRootMissing;
+ rmSync(contentDir, { recursive: true, force: true });
+
+ clock.advance(5_000);
+ expect(getMetrics().watcherRootMissing - rootMissingBefore).toBe(1);
+
+ mkdirSync(contentDir);
+ writeFileSync(join(contentDir, 'silent-root.md'), '# Back\n');
+ clock.advance(1_000);
+ await until(() => double.subscriptionCount(contentDir) === 2, 'the root to be watched again');
+ await until(() => createsOf(events, 'silent-root').length === 1, 'the write to be found');
+ });
+});
+
+describe('HEAD watcher resubscribe', () => {
+ let projectRoot: string;
+ let gitDir: string;
+ let handle: HeadWatcherHandle | undefined;
+
+ beforeEach(() => {
+ projectRoot = realpathSync(mkdtempSync(join(tmpdir(), 'ok-head-resubscribe-')));
+ gitDir = join(projectRoot, '.git');
+ mkdirSync(gitDir);
+ writeFileSync(join(gitDir, 'HEAD'), 'ref: refs/heads/main\n', 'utf-8');
+ });
+
+ afterEach(async () => {
+ await handle?.unsubscribe();
+ handle = undefined;
+ rmSync(projectRoot, { recursive: true, force: true });
+ });
+
+ test('after a backend error the HEAD watcher subscribes again and re-reads HEAD in a batch', async () => {
+ const double = createWatcherDouble();
+ const clock = createManualClock(Date.now());
+ const triggers: string[] = [];
+ let batchEnds = 0;
+ handle = await startHeadWatcher(
+ projectRoot,
+ ({ trigger }) => {
+ triggers.push(trigger);
+ },
+ () => {
+ batchEnds++;
+ },
+ { forceBackend: 'parcel', nativeBackend: double.backend, clock },
+ );
+
+ double.subscriptionFor(gitDir).fail(new Error('the native watcher died'));
+ clock.advance(0);
+ await until(() => double.subscriptionCount(gitDir) === 2, 'a second HEAD subscription');
+ await until(() => triggers.includes('rescan'), 'the HEAD re-read batch');
+ clock.advance(100);
+ await until(() => batchEnds === 1, 'the re-read batch to end');
+
+ double.subscriptionFor(gitDir).deliver([{ type: 'update', path: join(gitDir, 'HEAD') }]);
+ await until(() => triggers.includes('HEAD'), 'the new subscription to open a batch');
+ expect(triggers).toEqual(['rescan', 'HEAD']);
+ });
+
+ test('a HEAD watcher that gives up says so, tries again every five minutes and says when it is back', async () => {
+ const double = createWatcherDouble();
+ const clock = createManualClock(Date.now());
+ const coverage: string[] = [];
+ const triggers: string[] = [];
+ handle = await startHeadWatcher(
+ projectRoot,
+ ({ trigger }) => {
+ triggers.push(trigger);
+ },
+ () => {},
+ {
+ forceBackend: 'parcel',
+ nativeBackend: double.backend,
+ clock,
+ onGaveUp: (err) => coverage.push(`gave up: ${(err as Error).message}`),
+ onBack: () => coverage.push('back'),
+ },
+ );
+ for (let failure = 1; failure <= 8; failure++) {
+ double.failNextSubscribe(gitDir, new Error(`watch limit ${failure}`));
+ }
+ double.subscriptionFor(gitDir).fail(new Error('the native watcher died'));
+ for (let step = 0; step < 8; step++) {
+ clock.advance(30_000);
+ await flushMicrotasks();
+ }
+ expect(coverage).toEqual(['gave up: watch limit 8']);
+ expect(double.subscriptionCount(gitDir)).toBe(1);
+
+ clock.advance(SLOW_RETRY_MS);
+ await until(() => coverage.includes('back'), 'the slow retry');
+ expect(coverage).toEqual(['gave up: watch limit 8', 'back']);
+ expect(double.subscriptionCount(gitDir)).toBe(2);
+ await until(() => triggers.includes('rescan'), 'the HEAD re-read once it is back');
+ });
+});