Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
PRD-9738 Re-check the tree on a timer, resubscribe, and re-read on focus (#7133)
mike-inkeepf8500ff0ef4e
diff --git a/.changeset/find-missed-outside-changes.md b/.changeset/find-missed-outside-changes.mdnew file mode 100644index 000000000..8649b3d52--- /dev/null+++ b/.changeset/find-missed-outside-changes.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++Outside edits the file watcher misses now appear within seconds, or about two minutes on very large projects: OpenKnowledge re-checks the project on a timer, watches again after a watcher failure, and re-reads open documents when the window regains focus or reconnects. Large projects use less memory.diff --git a/packages/app/src/editor/EditorLifecycleFlush.tsx b/packages/app/src/editor/EditorLifecycleFlush.tsxindex 619fb1c63..f5a23b0a1 100644--- a/packages/app/src/editor/EditorLifecycleFlush.tsx+++ b/packages/app/src/editor/EditorLifecycleFlush.tsx@@ -2,6 +2,7 @@ import { useEffect } from 'react';import { useConfigContext } from '@/lib/config-context';import { getPool, useDocumentContext } from './DocumentContext';import { installEditorLifecycleFlush } from './install-editor-lifecycle-flush';+import { refreshDocumentsFromDisk } from './refresh-documents-from-disk';export function EditorLifecycleFlush(): null {const { collabUrl } = useDocumentContext();@@ -15,6 +16,9 @@ export function EditorLifecycleFlush(): null {return installEditorLifecycleFlush({onHide: () => pool.flushOnHide(),onVisible: () => pool.resyncOnVisible(),+ onResume: () => {+ void refreshDocumentsFromDisk(pool.openDocNames());+ },});}, [collabUrl, enabled]);diff --git a/packages/app/src/editor/install-editor-lifecycle-flush.test.ts b/packages/app/src/editor/install-editor-lifecycle-flush.test.tsindex df1d50703..5b41b7207 100644--- a/packages/app/src/editor/install-editor-lifecycle-flush.test.ts+++ b/packages/app/src/editor/install-editor-lifecycle-flush.test.ts@@ -1,5 +1,5 @@import { describe, expect, it, vi } from 'vitest';-import { installEditorLifecycleFlush } from './install-editor-lifecycle-flush';+import { installEditorLifecycleFlush, RESUME_THROTTLE_MS } from './install-editor-lifecycle-flush';type Listener = (event: Event) => void;@@ -129,4 +129,75 @@ describe('installEditorLifecycleFlush', () => {expect(() => uninstall()).not.toThrow();expect(onHide).not.toHaveBeenCalled();});++ it('asks for a re-read from disk when the window regains focus', () => {+ const onResume = vi.fn();+ const win = makeFakeTarget();+ const doc = makeFakeDoc('visible');+ let clock = 10_000;+ installEditorLifecycleFlush({+ onHide: vi.fn(),+ onVisible: vi.fn(),+ onResume,+ now: () => clock,+ win,+ doc,+ });++ win.emit('focus');+ expect(onResume).toHaveBeenCalledTimes(1);++ clock += RESUME_THROTTLE_MS;+ win.emit('focus');+ expect(onResume).toHaveBeenCalledTimes(2);+ });++ it('asks once when the tab becomes visible and the window takes focus together', () => {+ const onResume = vi.fn();+ const onVisible = vi.fn();+ const win = makeFakeTarget();+ const doc = makeFakeDoc('visible');+ installEditorLifecycleFlush({+ onHide: vi.fn(),+ onVisible,+ onResume,+ now: () => 10_000,+ win,+ doc,+ });++ doc.emit('visibilitychange');+ win.emit('focus');++ expect(onVisible).toHaveBeenCalledTimes(1);+ expect(onResume).toHaveBeenCalledTimes(1);+ });++ it('does not ask while the page is hidden, and stops asking once uninstalled', () => {+ const onResume = vi.fn();+ const win = makeFakeTarget();+ const doc = makeFakeDoc('hidden');+ let clock = 10_000;+ const uninstall = installEditorLifecycleFlush({+ onHide: vi.fn(),+ onVisible: vi.fn(),+ onResume,+ now: () => clock,+ win,+ doc,+ });++ win.emit('focus');+ expect(onResume).not.toHaveBeenCalled();++ doc.setVisibility('visible');+ win.emit('focus');+ expect(onResume).toHaveBeenCalledTimes(1);++ uninstall();+ clock += RESUME_THROTTLE_MS;+ expect(win.has('focus')).toBe(false);+ win.emit('focus');+ expect(onResume).toHaveBeenCalledTimes(1);+ });});diff --git a/packages/app/src/editor/install-editor-lifecycle-flush.ts b/packages/app/src/editor/install-editor-lifecycle-flush.tsindex d58ac028b..d9464f224 100644--- a/packages/app/src/editor/install-editor-lifecycle-flush.ts+++ b/packages/app/src/editor/install-editor-lifecycle-flush.ts@@ -1,6 +1,10 @@+export const RESUME_THROTTLE_MS = 1_000;+export interface EditorLifecycleFlushDeps {onHide: () => void;onVisible: () => void;+ onResume?: () => void;+ now?: () => number;win?: Pick<Window, 'addEventListener' | 'removeEventListener'>;doc?: Pick<Document, 'addEventListener' | 'removeEventListener'> & {readonly visibilityState: DocumentVisibilityState;@@ -13,6 +17,15 @@ export function installEditorLifecycleFlush(deps: EditorLifecycleFlushDeps): ()if (win === undefined || doc === undefined) return () => {};let hidden = false;+ let lastResumeAt = Number.NEGATIVE_INFINITY;+ const now = deps.now ?? Date.now;+ const fireResume = (): void => {+ if (deps.onResume === undefined) return;+ const at = now();+ if (at - lastResumeAt < RESUME_THROTTLE_MS) return;+ lastResumeAt = at;+ deps.onResume();+ };const fireHide = (): void => {if (hidden) return;hidden = true;@@ -25,14 +38,20 @@ export function installEditorLifecycleFlush(deps: EditorLifecycleFlushDeps): ()else if (doc.visibilityState === 'visible') {hidden = false;deps.onVisible();+ fireResume();}};+ const onFocus = (): void => {+ if (doc.visibilityState === 'visible') fireResume();+ };win.addEventListener('pagehide', onPageHide as (event: Event) => void);+ win.addEventListener('focus', onFocus);doc.addEventListener('visibilitychange', onVisibility);return () => {win.removeEventListener('pagehide', onPageHide as (event: Event) => void);+ win.removeEventListener('focus', onFocus);doc.removeEventListener('visibilitychange', onVisibility);};}diff --git a/packages/app/src/editor/provider-pool.ts b/packages/app/src/editor/provider-pool.tsindex 29364940e..e86964c41 100644--- a/packages/app/src/editor/provider-pool.ts+++ b/packages/app/src/editor/provider-pool.ts@@ -1973,6 +1973,14 @@ export class ProviderPool {return false;}+ openDocNames(): string[] {+ const docNames: string[] = [];+ for (const [docName, entry] of this._entries) {+ if (entry.kind === 'active') docNames.push(docName);+ }+ return docNames;+ }+docNamesToOpenForReplay(): string[] {const docNames: string[] = [];for (const [docName, buffered] of this.bufferedUpdates) {diff --git a/packages/app/src/editor/refresh-documents-from-disk.test.ts b/packages/app/src/editor/refresh-documents-from-disk.test.tsnew file mode 100644index 000000000..05814ad84--- /dev/null+++ b/packages/app/src/editor/refresh-documents-from-disk.test.ts@@ -0,0 +1,66 @@+import { DOCUMENT_REFRESH_MAX_DOCS } from '@inkeep/open-knowledge-core/schemas/api';+import { describe, expect, it, vi } from 'vitest';+import { refreshDocumentsFromDisk } from './refresh-documents-from-disk';++function requestedBatches(fetchImpl: ReturnType<typeof vi.fn<typeof fetch>>): string[][] {+ return fetchImpl.mock.calls.map(([, init]) => {+ const body: { docNames?: unknown } = JSON.parse(String(init?.body));+ return Array.isArray(body.docNames) ? body.docNames.map(String) : [];+ });+}++describe('refreshDocumentsFromDisk', () => {+ it('asks the server to re-read every open document, in batches the route accepts', async () => {+ const fetchImpl = vi.fn<typeof fetch>(async () => new Response('{"checked":0,"found":0}'));+ const docNames = Array.from({ length: DOCUMENT_REFRESH_MAX_DOCS + 6 }, (_, i) => `doc-${i}`);++ await refreshDocumentsFromDisk(docNames, fetchImpl);++ const batches = requestedBatches(fetchImpl);+ expect(batches.map((batch) => batch.length)).toEqual([DOCUMENT_REFRESH_MAX_DOCS, 6]);+ expect(batches.flat()).toEqual(docNames);+ expect(fetchImpl.mock.calls[0]?.[0]).toBe('/api/documents/refresh');+ });++ it('sends nothing when no document is open', async () => {+ const fetchImpl = vi.fn<typeof fetch>(async () => new Response('{}'));+ await refreshDocumentsFromDisk([], fetchImpl);+ expect(fetchImpl).not.toHaveBeenCalled();+ });++ it('keeps going after a failed request', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const fetchImpl = vi+ .fn<typeof fetch>()+ .mockRejectedValueOnce(new Error('offline'))+ .mockResolvedValue(new Response('{}'));+ const docNames = Array.from({ length: DOCUMENT_REFRESH_MAX_DOCS + 1 }, (_, i) => `doc-${i}`);++ await refreshDocumentsFromDisk(docNames, fetchImpl);++ expect(fetchImpl).toHaveBeenCalledTimes(2);+ expect(warn).toHaveBeenCalledTimes(1);+ warn.mockRestore();+ });++ it('logs a refused re-read with its status and still sends the next batch', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ const fetchImpl = vi+ .fn<typeof fetch>()+ .mockResolvedValueOnce(new Response('{}', { status: 500 }))+ .mockResolvedValue(new Response('{"checked":1,"found":0}'));+ const docNames = Array.from({ length: DOCUMENT_REFRESH_MAX_DOCS + 1 }, (_, i) => `doc-${i}`);++ await refreshDocumentsFromDisk(docNames, fetchImpl);++ expect(fetchImpl).toHaveBeenCalledTimes(2);+ expect(warn.mock.calls.map(([line]) => JSON.parse(String(line)))).toEqual([+ {+ event: 'ok-document-refresh-request-failed',+ docCount: DOCUMENT_REFRESH_MAX_DOCS,+ status: 500,+ },+ ]);+ warn.mockRestore();+ });+});diff --git a/packages/app/src/editor/refresh-documents-from-disk.ts b/packages/app/src/editor/refresh-documents-from-disk.tsnew file mode 100644index 000000000..d8b17fb82--- /dev/null+++ b/packages/app/src/editor/refresh-documents-from-disk.ts@@ -0,0 +1,27 @@+import { DOCUMENT_REFRESH_MAX_DOCS } from '@inkeep/open-knowledge-core/schemas/api';++function warnRefreshFailed(details: Record<string, unknown>): void {+ console.warn(JSON.stringify({ event: 'ok-document-refresh-request-failed', ...details }));+}++export async function refreshDocumentsFromDisk(+ docNames: readonly string[],+ fetchImpl: typeof fetch = fetch,+): Promise<void> {+ for (let start = 0; start < docNames.length; start += DOCUMENT_REFRESH_MAX_DOCS) {+ const batch = docNames.slice(start, start + DOCUMENT_REFRESH_MAX_DOCS);+ try {+ const response = await fetchImpl('/api/documents/refresh', {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ docNames: batch }),+ });+ if (!response.ok) warnRefreshFailed({ docCount: batch.length, status: response.status });+ } catch (err) {+ warnRefreshFailed({+ docCount: batch.length,+ message: String(err instanceof Error ? err.message : err),+ });+ }+ }+}diff --git a/packages/app/tests/integration/api-error-envelope/document-refresh.test.ts b/packages/app/tests/integration/api-error-envelope/document-refresh.test.tsnew file mode 100644index 000000000..b2d400be8--- /dev/null+++ b/packages/app/tests/integration/api-error-envelope/document-refresh.test.ts@@ -0,0 +1,56 @@+import { DocumentRefreshSuccessSchema, ProblemDetailsSchema } from '@inkeep/open-knowledge-core';+import { afterAll, beforeAll, describe, expect, test } from 'vitest';+import { HARNESS_BOOT_TIMEOUT_MS } from '../harness-boot-timeout';+import { createTestServer, type TestServer } from '../test-harness';++let server: TestServer;++beforeAll(async () => {+ server = await createTestServer();+}, HARNESS_BOOT_TIMEOUT_MS);++afterAll(async () => {+ await server.cleanup();+});++function refresh(body: unknown, method = 'POST'): Promise<Response> {+ return fetch(`http://127.0.0.1:${server.port}/api/documents/refresh`, {+ method,+ headers: { 'Content-Type': 'application/json' },+ ...(method === 'POST' ? { body: JSON.stringify(body) } : {}),+ });+}++describe('document-refresh envelope (RFC 9457)', () => {+ test('happy path emits a flat counts body with application/json', async () => {+ const res = await refresh({ docNames: ['test-doc'] });+ expect(res.status).toBe(200);+ expect(res.headers.get('content-type')).toBe('application/json');++ const body = await res.json();+ expect(DocumentRefreshSuccessSchema.safeParse(body).success).toBe(true);+ expect((body as Record<string, unknown>).ok).toBeUndefined();+ expect(body).toMatchObject({ checked: 1 });+ });++ test('an empty list emits problem+json invalid-request', async () => {+ const res = await refresh({ docNames: [] });+ expect(res.status).toBe(400);+ expect(res.headers.get('content-type')).toBe('application/problem+json');++ const parsed = ProblemDetailsSchema.safeParse(await res.json());+ expect(parsed.success).toBe(true);+ if (parsed.success) expect(parsed.data.type).toBe('urn:ok:error:invalid-request');+ });++ test('method-not-allowed on GET emits problem+json with Allow: POST', async () => {+ const res = await refresh(undefined, 'GET');+ expect(res.status).toBe(405);+ expect(res.headers.get('allow')).toBe('POST');+ expect(res.headers.get('content-type')).toBe('application/problem+json');++ const parsed = ProblemDetailsSchema.safeParse(await res.json());+ expect(parsed.success).toBe(true);+ if (parsed.success) expect(parsed.data.type).toBe('urn:ok:error:method-not-allowed');+ });+});diff --git a/packages/app/tests/integration/attribution-sweep-coverage.test.ts b/packages/app/tests/integration/attribution-sweep-coverage.test.tsindex eb80fc256..0425804a4 100644--- a/packages/app/tests/integration/attribution-sweep-coverage.test.ts+++ b/packages/app/tests/integration/attribution-sweep-coverage.test.ts@@ -46,6 +46,7 @@ const REQUIRED_HANDLERS = [const EXEMPT_HANDLERS = new Set(['handleDocumentRead','handleDocumentList',+ 'handleDocumentRefresh','handleAcpCatalog','handleAcpModels','handleEmbedDetect',diff --git a/packages/app/tests/integration/conflict-gate-coverage.test.ts b/packages/app/tests/integration/conflict-gate-coverage.test.tsindex 21832b489..5af29c162 100644--- a/packages/app/tests/integration/conflict-gate-coverage.test.ts+++ b/packages/app/tests/integration/conflict-gate-coverage.test.ts@@ -51,6 +51,7 @@ const REQUIRED_HANDLERS = [const EXEMPT_HANDLERS = new Set(['handleDocumentRead','handleDocumentList',+ 'handleDocumentRefresh','handleAcpCatalog','handleAcpModels','handleAsset',diff --git a/packages/app/tests/integration/freshness-recheck.test.ts b/packages/app/tests/integration/freshness-recheck.test.tsnew file mode 100644index 000000000..e95925dec--- /dev/null+++ b/packages/app/tests/integration/freshness-recheck.test.ts@@ -0,0 +1,196 @@+import { randomUUID } from 'node:crypto';+import { existsSync, readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { afterEach, describe, expect, test } from 'vitest';+import { getMetrics } from '../../../server/src/metrics';+import { createManualClock, type ManualClock } from '../../../server/src/watcher-clock.test-helper';+import { createWatcherDouble } from '../../../server/src/watcher-double.test-helper';+import {+ agentWriteMd,+ createTestClient,+ createTestServer,+ pollUntil,+ type TestClient,+ type TestServer,+} from './test-harness';++const SEED = '# Doc\n\noriginal\n';++let server: TestServer | undefined;+const clients: TestClient[] = [];++afterEach(async () => {+ for (const client of clients.splice(0)) await client.cleanup();+ await server?.cleanup();+ server = undefined;+});++interface Rig {+ server: TestServer;+ double: ReturnType<typeof createWatcherDouble>;+ clock: ManualClock;+ docName: string;+ file: string;+ client: TestClient;+}++async function openSeededDocument(options: {+ freshnessIntervalMs: number | null;+ debounce?: number;+ maxDebounce?: number;+}): Promise<Rig> {+ const double = createWatcherDouble();+ const clock = createManualClock(Date.now());+ server = await createTestServer({+ watcherBackend: double.backend,+ watcherClock: clock,+ freshnessIntervalMs: options.freshnessIntervalMs,+ debounce: options.debounce ?? 50,+ maxDebounce: options.maxDebounce ?? 200,+ });+ const docName = `freshness-${randomUUID()}`;+ const file = join(server.contentDir, `${docName}.md`);+ await agentWriteMd(server.port, SEED, { docName, position: 'replace' });+ await pollUntil(+ () => existsSync(file) && readFileSync(file, 'utf-8') === SEED,+ 5_000,+ 25,+ 'the seed write to reach disk',+ );+ const client = await createTestClient(server.port, docName);+ clients.push(client);+ await pollUntil(() => client.ytext.toString() === SEED, 5_000, 25, 'the client to load the seed');+ return { server, double, clock, docName, file, client };+}++async function refresh(port: number, docNames: string[]): Promise<Response> {+ return fetch(`http://127.0.0.1:${port}/api/documents/refresh`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ docNames }),+ });+}++describe('freshness through the server', () => {+ test('with native events silenced, the timed re-check applies an outside write to an open document byte for byte', async () => {+ const rig = await openSeededDocument({ freshnessIntervalMs: 5_000 });+ const outside = '# Doc\n\nchanged outside, never delivered\n';+ writeFileSync(rig.file, outside);+ const findsBefore = getMetrics().freshnessFinds;++ rig.clock.advance(5_000);+ await pollUntil(+ () => rig.client.ytext.toString() === outside,+ 5_000,+ 25,+ 'the timed re-check to apply the outside bytes',+ );+ expect(readFileSync(rig.file, 'utf-8')).toBe(outside);+ expect(getMetrics().freshnessFinds - findsBefore).toBeGreaterThanOrEqual(1);+ });++ test('after a watcher error the server subscribes again, applies the write made in the gap, and takes later events', async () => {+ const rig = await openSeededDocument({ freshnessIntervalMs: null });+ const first = rig.double.subscriptionFor(rig.server.contentDir);+ first.fail(new Error('the native watcher died'));+ const gap = '# Doc\n\nwritten while the watcher was dead\n';+ writeFileSync(rig.file, gap);++ rig.clock.advance(0);+ await pollUntil(+ () => rig.client.ytext.toString() === gap,+ 5_000,+ 25,+ 'the resubscribe re-check to apply the gap write',+ );+ expect(rig.double.subscriptionCount(rig.server.contentDir)).toBe(2);++ const later = '# Doc\n\ndelivered by the new subscription\n';+ writeFileSync(rig.file, later);+ rig.double.subscriptionFor(rig.server.contentDir).deliver([{ type: 'update', path: rig.file }]);+ await pollUntil(+ () => rig.client.ytext.toString() === later,+ 5_000,+ 25,+ 'the new subscription to deliver the later write',+ );+ });++ test('a focus re-read merges an outside change made while the window was unfocused with unsaved typing', async () => {+ const rig = await openSeededDocument({+ freshnessIntervalMs: null,+ debounce: 60_000,+ maxDebounce: 120_000,+ });+ rig.client.ytext.insert(rig.client.ytext.length, '\nlive typing\n');+ await pollUntil(+ () =>+ rig.server.instance.hocuspocus.documents+ .get(rig.docName)+ ?.getText('source')+ .toString()+ .includes('live typing') ?? false,+ 5_000,+ 25,+ 'the typing to reach the server',+ );+ writeFileSync(rig.file, '# Changed outside\n\noriginal\n');++ const response = await refresh(rig.server.port, [rig.docName]);+ expect(response.status).toBe(200);+ expect(await response.json()).toMatchObject({ checked: 1, found: 1 });+ await pollUntil(+ () => rig.client.ytext.toString().includes('# Changed outside'),+ 5_000,+ 25,+ 'the focus re-read to merge the outside change',+ );+ expect(rig.client.ytext.toString()).toContain('live typing');+ expect(rig.client.ytext.toString()).not.toContain('<<<<<<<');+ });++ test('a client that reconnects to an open document gets the outside change made while it was away', async () => {+ const rig = await openSeededDocument({ freshnessIntervalMs: null });+ const away = await createTestClient(rig.server.port, rig.docName);+ clients.push(away);+ await pollUntil(() => away.ytext.toString() === SEED, 5_000, 25, 'the second client to load');++ let disconnected = false;+ away.provider.on('disconnect', () => {+ disconnected = true;+ });+ away.provider.disconnect();+ await pollUntil(() => disconnected, 5_000, 25, 'the second client to disconnect');++ const changed = '# Doc\n\nchanged while the client was away\n';+ writeFileSync(rig.file, changed);+ rig.clock.advance(1_000);+ let resynced = false;+ away.provider.on('synced', () => {+ resynced = true;+ });+ await away.provider.connect();+ await pollUntil(() => resynced, 10_000, 25, 'the second client to sync again');++ await pollUntil(+ () => away.ytext.toString() === changed,+ 5_000,+ 25,+ 'the reconnect re-read to apply the outside change',+ );+ expect(rig.client.ytext.toString()).toBe(changed);+ });++ test('the refresh route skips the system document and names outside the content tree', async () => {+ const rig = await openSeededDocument({ freshnessIntervalMs: null });+ const response = await refresh(rig.server.port, ['__system__', '../outside', 'not-there']);+ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({ checked: 1, found: 0 });+ });++ test('the refresh route refuses an empty list', async () => {+ const rig = await openSeededDocument({ freshnessIntervalMs: null });+ const response = await refresh(rig.server.port, []);+ expect(response.status).toBe(400);+ });+});diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.tsindex 6707743b4..d68bbbcb1 100644--- a/packages/app/tests/integration/test-harness.ts+++ b/packages/app/tests/integration/test-harness.ts@@ -110,6 +110,7 @@ export interface CreateTestServerOptions {seedSkillHostRoot?: boolean;watcherBackend?: ServerOptions['watcherBackend'];watcherClock?: ServerOptions['watcherClock'];+ freshnessIntervalMs?: ServerOptions['freshnessIntervalMs'];mergeHistoryWriter?: ServerOptions['mergeHistoryWriter'];}@@ -189,6 +190,7 @@ export async function createTestServer(options: CreateTestServerOptions = {}): PagentSessionOptions: options.agentSessionOptions,watcherBackend: options.watcherBackend,watcherClock: options.watcherClock,+ freshnessIntervalMs: options.freshnessIntervalMs ?? null,mergeHistoryWriter: options.mergeHistoryWriter,...(ephemeral ? { ephemeral: true, singleDocRelPath: options.singleDocRelPath } : {}),skipStateManifestCheck: true,diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 63b0fba52..dd5cfe5f9 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1332,6 +1332,7 @@ export {DismissDocNoticeSuccessSchema,type DispatchPayload,DispatchPayloadSchema,+ DOCUMENT_REFRESH_MAX_DOCS,DocNoticesSuccessSchema,type DocNoticeWire,DocNoticeWireSchema,@@ -1341,6 +1342,10 @@ export {DocumentListSuccessSchema,type DocumentReadSuccess,DocumentReadSuccessSchema,+ type DocumentRefreshRequest,+ DocumentRefreshRequestSchema,+ type DocumentRefreshSuccess,+ DocumentRefreshSuccessSchema,type DuplicatePathRequest,DuplicatePathRequestSchema,type DuplicatePathSuccess,diff --git a/packages/core/src/schemas/api/document-read.ts b/packages/core/src/schemas/api/document-read.tsindex cb218bc1c..c26ff005d 100644--- a/packages/core/src/schemas/api/document-read.ts+++ b/packages/core/src/schemas/api/document-read.ts@@ -228,3 +228,20 @@ export const LinkGraphSuccessSchema = z}).loose() satisfies StandardSchemaV1;export type LinkGraphSuccess = z.infer<typeof LinkGraphSuccessSchema>;++export const DOCUMENT_REFRESH_MAX_DOCS = 64;++export const DocumentRefreshRequestSchema = z+ .object({+ docNames: z.array(z.string().min(1).max(1024)).min(1).max(DOCUMENT_REFRESH_MAX_DOCS),+ })+ .loose() satisfies StandardSchemaV1;+export type DocumentRefreshRequest = z.infer<typeof DocumentRefreshRequestSchema>;++export const DocumentRefreshSuccessSchema = z+ .object({+ checked: z.number().int().nonnegative(),+ found: z.number().int().nonnegative(),+ })+ .loose() satisfies StandardSchemaV1;+export type DocumentRefreshSuccess = z.infer<typeof DocumentRefreshSuccessSchema>;diff --git a/packages/server/src/api-agent-write-batch-persistence-failure.test.ts b/packages/server/src/api-agent-write-batch-persistence-failure.test.tsindex 02c5ce555..d00005939 100644--- a/packages/server/src/api-agent-write-batch-persistence-failure.test.ts+++ b/packages/server/src/api-agent-write-batch-persistence-failure.test.ts@@ -73,6 +73,7 @@ afterEach(async () => {function createExtension() {return createApiExtension({hocuspocus,+ refreshDocumentsFromDisk: async () => ({ checked: 0, found: 0 }),durabilityState,sessionManager,contentDir,diff --git a/packages/server/src/api-extension.test-helper.ts b/packages/server/src/api-extension.test-helper.tsindex 725909503..2a86226e3 100644--- a/packages/server/src/api-extension.test-helper.ts+++ b/packages/server/src/api-extension.test-helper.ts@@ -252,6 +252,7 @@ export function createApiExtension(| 'conflicts'| 'getProjectConfigEpoch'| 'getLinkAdvisoryPolicy'+ | 'refreshDocumentsFromDisk'> &LegacyIndexOptions & {conflicts?: ApiExtensionOptions['conflicts'];@@ -260,6 +261,7 @@ export function createApiExtension(signalChannel?: LegacySignalChannel;getProjectConfigEpoch?: () => number;getLinkAdvisoryPolicy?: () => LinkAdvisoryPolicy;+ refreshDocumentsFromDisk?: ApiExtensionOptions['refreshDocumentsFromDisk'];},): ReturnType<typeof createApiExtensionBase> {const {@@ -268,6 +270,7 @@ export function createApiExtension(derivedDocumentIndex,durabilityState = new DocumentDurabilityState(),getProjectConfigEpoch = () => 0,+ refreshDocumentsFromDisk = async () => ({ checked: 0, found: 0 }),getLinkAdvisoryPolicy = () => ({links: DEFAULT_LINKS_VALIDATION,suppressLogLinkAdvisories: DEFAULT_SUPPRESS_LOG_LINK_ADVISORIES,@@ -277,6 +280,7 @@ export function createApiExtension(const pendingLegacyWrites = new Set<Promise<unknown>>();const extension = createApiExtensionBase({...apiOptions,+ refreshDocumentsFromDisk,getProjectConfigEpoch,getLinkAdvisoryPolicy,conflicts:diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 9d01d89e3..6aa0d3ae4 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -234,6 +234,7 @@ import {OK_STORE_REFUSED,type StoreFailure,} from './document-durability-state.ts';+import type { RefreshDocumentsFromDisk } from './document-refresh.ts';import {buildReconcileWarning,reconcileDiskBeforeAgentWrite,@@ -272,6 +273,7 @@ import { createAssetRoutes } from './http/asset-routes.ts';import { createCommentRoutes } from './http/comment-routes.ts';import { createConfigSystemRoutes } from './http/config-system-routes.ts';import { createDocNoticeRoutes } from './http/doc-notice-routes.ts';+import { createDocumentRefreshRoutes } from './http/document-refresh-routes.ts';import { createDocumentRoutes } from './http/document-routes.ts';import {type ErrorExtensions,@@ -1334,6 +1336,7 @@ interface ApiExtensionBaseOptions {resolveSyncCredentialConfig?: () => Promise<string[]>;conflicts: ConflictAuthority;setBatchInProgress?: (value: boolean) => void;+ refreshDocumentsFromDisk: RefreshDocumentsFromDisk;localOpCliArgs?: string[];authStreamHeartbeatMs?: number;projectDir?: string;@@ -5476,6 +5479,9 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {agentIntegrations,});const handoffInstallRoutes = createHandoffInstallRoutes({ checkLocalOpSecurity });+ const documentRefreshRoutes = createDocumentRefreshRoutes({+ refreshDocumentsFromDisk: options.refreshDocumentsFromDisk,+ });const skillsInstallRoutes = createSkillsInstallRoutes({resolveSkillsRoot,validateSkillName,@@ -5613,6 +5619,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {linkGraphRoutes,metricsRoutes,documentRoutes,+ documentRefreshRoutes,configSystemRoutes,lintRoutes,lintWriteRoutes,diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex cef298c29..39a77051d 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -156,6 +156,7 @@ export interface BootServerOptions| 'acpRegistryFetchImpl'| 'watcherBackend'| 'watcherClock'+ | 'freshnessIntervalMs'> {config: Config;skipAutoInit?: boolean;@@ -417,6 +418,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {configHomedirOverride: opts.configHomedirOverride,watcherBackend: opts.watcherBackend,watcherClock: opts.watcherClock,+ freshnessIntervalMs: opts.freshnessIntervalMs,});const {diff --git a/packages/server/src/document-refresh.test.ts b/packages/server/src/document-refresh.test.tsnew file mode 100644index 000000000..48a61d1a9--- /dev/null+++ b/packages/server/src/document-refresh.test.ts@@ -0,0 +1,51 @@+import { CONFIG_DOC_NAMES } from '@inkeep/open-knowledge-core';+import { describe, expect, test, vi } from 'vitest';+import { SYSTEM_DOC_NAME } from './cc1-broadcast.ts';+import {+ createReconnectRefreshExtension,+ type RefreshDocumentsFromDisk,+} from './document-refresh.ts';+import { createManualClock } from './watcher-clock.test-helper.ts';++type HookPayload = { documentName: string };++function extensionWith(refresh: RefreshDocumentsFromDisk) {+ const clock = createManualClock(1_000_000);+ const extension = createReconnectRefreshExtension({ clock, refresh });+ const hooks = extension as unknown as {+ afterLoadDocument(payload: HookPayload): Promise<void>;+ connected(payload: HookPayload): Promise<void>;+ };+ return { clock, hooks };+}++describe('re-read on reconnect', () => {+ test('a connection more than a second after the last read re-reads that document', async () => {+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => ({ checked: 1, found: 0 }));+ const { clock, hooks } = extensionWith(refresh);++ await hooks.afterLoadDocument({ documentName: 'notes' });+ await hooks.connected({ documentName: 'notes' });+ expect(refresh).not.toHaveBeenCalled();++ clock.advance(1_000);+ await hooks.connected({ documentName: 'notes' });+ expect(refresh).toHaveBeenCalledWith(['notes'], 'reconnect');++ await hooks.connected({ documentName: 'notes' });+ expect(refresh).toHaveBeenCalledTimes(1);+ });++ test('the system document and config documents are never re-read', async () => {+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => ({ checked: 0, found: 0 }));+ const { clock, hooks } = extensionWith(refresh);+ const reserved = [SYSTEM_DOC_NAME, ...CONFIG_DOC_NAMES];++ for (const documentName of reserved) await hooks.afterLoadDocument({ documentName });+ clock.advance(5_000);+ for (const documentName of reserved) await hooks.connected({ documentName });++ expect(reserved.length).toBeGreaterThan(1);+ expect(refresh).not.toHaveBeenCalled();+ });+});diff --git a/packages/server/src/document-refresh.ts b/packages/server/src/document-refresh.tsnew file mode 100644index 000000000..c377a5ae4--- /dev/null+++ b/packages/server/src/document-refresh.ts@@ -0,0 +1,49 @@+import type { Extension } from '@hocuspocus/server';+import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts';+import { getLogger } from './logger.ts';+import type { WatcherClock } from './watcher-clock.ts';++const log = getLogger('document-refresh');++const RECONNECT_REFRESH_COALESCE_MS = 1_000;++const isRefreshable = (documentName: string): boolean =>+ !isSystemDoc(documentName) && !isConfigDoc(documentName);++export type DocumentRefreshTrigger = 'focus' | 'reconnect';++export interface DocumentRefreshResult {+ checked: number;+ found: number;+}++export type RefreshDocumentsFromDisk = (+ docNames: readonly string[],+ trigger: DocumentRefreshTrigger,+) => Promise<DocumentRefreshResult>;++export function createReconnectRefreshExtension(deps: {+ clock: WatcherClock;+ refresh: RefreshDocumentsFromDisk;+}): Extension {+ const lastReadAt = new Map<string, number>();+ return {+ async afterLoadDocument({ documentName }) {+ if (!isRefreshable(documentName)) return;+ lastReadAt.set(documentName, deps.clock.now());+ },+ async connected({ documentName }) {+ if (!isRefreshable(documentName)) return;+ const now = deps.clock.now();+ const last = lastReadAt.get(documentName);+ if (last !== undefined && now - last < RECONNECT_REFRESH_COALESCE_MS) return;+ lastReadAt.set(documentName, now);+ void deps.refresh([documentName], 'reconnect').catch((err: unknown) => {+ log.warn({ err, docName: documentName }, 'the re-read on reconnect failed');+ });+ },+ async afterUnloadDocument({ documentName }) {+ lastReadAt.delete(documentName);+ },+ };+}diff --git a/packages/server/src/file-watcher-freshness.test.ts b/packages/server/src/file-watcher-freshness.test.tsnew file mode 100644index 000000000..be1ca6747--- /dev/null+++ b/packages/server/src/file-watcher-freshness.test.ts@@ -0,0 +1,636 @@+import { execFileSync } from 'node:child_process';+import {+ chmodSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ realpathSync,+ rmSync,+ statSync,+ symlinkSync,+ utimesSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { createContentFilter } from './content-filter.ts';+import {+ contentHash,+ type DiskEvent,+ lastKnownHash,+ removeLastKnownHash,+ type StartWatcherOptions,+ startWatcher,+ type WatcherHandle,+} from './file-watcher.ts';+import { scanInPlaceSkillDirs, skillRootPathsFor } from './in-place-skills.ts';+import { getMetrics } from './metrics.ts';+import { createManualClock, type ManualClock } from './watcher-clock.test-helper.ts';+import { createWatcherDouble } from './watcher-double.test-helper.ts';+import {+ FRESHNESS_HASH_BUDGET,+ FRESHNESS_MAX_INTERVAL_MS,+ nextFreshnessDelayMs,+} from './watcher-freshness.ts';++const SETTLED_TIMESTAMPS_MS = 10_000;++function readCanBeRevoked(): boolean {+ if (process.platform === 'win32') return false;+ const dir = mkdtempSync(join(tmpdir(), 'ok-revoke-probe-'));+ const probe = join(dir, 'probe');+ try {+ writeFileSync(probe, 'probe');+ chmodSync(probe, 0o000);+ try {+ readFileSync(probe);+ return false;+ } catch (err) {+ if ((err as NodeJS.ErrnoException).code === 'EACCES') return true;+ throw err;+ }+ } finally {+ chmodSync(probe, 0o600);+ rmSync(dir, { recursive: true, force: true });+ }+}++const CAN_REVOKE_READ = readCanBeRevoked();++const walkGate = vi.hoisted(() => ({+ dir: null as string | null,+ afterRead: null as string | null,+ entered: false,+ release: null as (() => void) | null,+ directoryReads: new Map<string, number>(),+}));++vi.mock('node:fs/promises', async (importOriginal) => {+ const actual = await importOriginal<typeof import('node:fs/promises')>();+ const readdir = actual.readdir as (...args: unknown[]) => Promise<unknown>;+ const readFile = actual.readFile as (...args: unknown[]) => Promise<unknown>;+ const hold = async (): Promise<void> => {+ walkGate.entered = true;+ await new Promise<void>((resolve) => {+ walkGate.release = resolve;+ });+ };+ return {+ ...actual,+ readdir: async (...args: unknown[]) => {+ const dir = String(args[0]);+ walkGate.directoryReads.set(dir, (walkGate.directoryReads.get(dir) ?? 0) + 1);+ if (walkGate.dir !== null && dir === walkGate.dir) {+ walkGate.dir = null;+ await hold();+ }+ return readdir(...args);+ },+ readFile: async (...args: unknown[]) => {+ const content = await readFile(...args);+ if (walkGate.afterRead !== null && String(args[0]) === walkGate.afterRead) {+ walkGate.afterRead = null;+ await hold();+ }+ return content;+ },+ };+});++async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {+ const start = performance.now();+ while (!condition()) {+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}++function eventsFor(events: readonly DiskEvent[], kind: DiskEvent['kind'], docName: string) {+ return events.filter(+ (event) => event.kind === kind && 'docName' in event && event.docName === docName,+ );+}++describe('timed freshness re-check', () => {+ let tmpDir: string;+ let contentDir: string;+ let handle: WatcherHandle | undefined;+ let clock: ManualClock;+ let events: DiskEvent[];++ beforeEach(() => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-freshness-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ contentDir = realpathSync(contentDir);+ lastKnownHash.clear();+ clock = createManualClock(Date.now());+ events = [];+ });++ afterEach(async () => {+ walkGate.release?.();+ walkGate.dir = null;+ walkGate.afterRead = null;+ walkGate.entered = false;+ walkGate.release = null;+ walkGate.directoryReads.clear();+ await handle?.unsubscribe();+ handle = undefined;+ vi.useRealTimers();+ for (const locked of revoked.splice(0)) chmodSync(locked, 0o755);+ rmSync(tmpDir, { recursive: true, force: true });+ });++ const revoked: string[] = [];+ function revokeRead(path: string): void {+ chmodSync(path, 0o000);+ revoked.push(path);+ }++ async function flush(): Promise<void> {+ for (let round = 0; round < 5; round++) {+ await new Promise((resolve) => setImmediate(resolve));+ }+ }++ let double: ReturnType<typeof createWatcherDouble>;++ async function watch(+ freshnessIntervalMs?: number | null,+ extra: Pick<StartWatcherOptions, 'onRecoveryComplete'> = {},+ ): Promise<WatcherHandle> {+ double = createWatcherDouble();+ handle = await startWatcher(+ contentDir,+ async (event) => {+ events.push(event);+ },+ undefined,+ {+ nativeBackend: double.backend,+ clock,+ ...(freshnessIntervalMs === undefined ? {} : { freshnessIntervalMs }),+ ...extra,+ },+ );+ return handle;+ }++ function moveModificationTimeLater(path: string): void {+ const { atime, mtime } = statSync(path);+ utimesSync(path, atime, new Date(mtime.getTime() + 2_000));+ }++ function keepModificationTime(path: string, write: () => void): void {+ const { atime, mtime } = statSync(path);+ write();+ utimesSync(path, atime, mtime);+ }++ async function runTimedPass(advanceMs: number): Promise<void> {+ const before = getMetrics().freshnessPasses;+ clock.advance(advanceMs);+ await until(() => getMetrics().freshnessPasses > before, 'the timed pass to finish');+ }++ function settleTimestamps(): void {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(Date.now() + SETTLED_TIMESTAMPS_MS);+ }++ function freezeTimestampsNow(): void {+ vi.useFakeTimers({ toFake: ['Date'] });+ }++ test('a write the native watcher never delivered is applied by the next timed pass', async () => {+ await watch(5_000);+ writeFileSync(join(contentDir, 'silent.md'), '# Silent\n');+ const findsBefore = getMetrics().freshnessFinds;++ clock.advance(4_999);+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(eventsFor(events, 'create', 'silent')).toHaveLength(0);++ await runTimedPass(1);+ expect(eventsFor(events, 'create', 'silent')).toHaveLength(1);+ expect(handle?.getFileIndex().has('silent')).toBe(true);+ expect(getMetrics().freshnessFinds - findsBefore).toBe(1);+ });++ test('a watcher started without an interval option runs no timed pass', async () => {+ await watch();+ writeFileSync(join(contentDir, 'library-default.md'), '# Library default\n');+ const passesBefore = getMetrics().freshnessPasses;+ clock.advance(FRESHNESS_MAX_INTERVAL_MS * 2);+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(getMetrics().freshnessPasses).toBe(passesBefore);+ expect(eventsFor(events, 'create', 'library-default')).toHaveLength(0);+ });++ test('a null interval turns the timed pass off, and an explicit re-check still finds the write', async () => {+ await watch(null);+ writeFileSync(join(contentDir, 'off.md'), '# Off\n');+ const passesBefore = getMetrics().freshnessPasses;+ clock.advance(FRESHNESS_MAX_INTERVAL_MS * 2);+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(getMetrics().freshnessPasses).toBe(passesBefore);+ expect(eventsFor(events, 'create', 'off')).toHaveLength(0);++ const result = await (handle as WatcherHandle).recheck();+ expect(result.status).toBe('completed');+ expect(eventsFor(events, 'create', 'off')).toHaveLength(1);+ });++ test('a pass reads only the files whose size, time or identity changed', async () => {+ writeFileSync(join(contentDir, 'one.md'), '# One\n');+ writeFileSync(join(contentDir, 'two.md'), '# Two\n');+ writeFileSync(join(contentDir, 'three.md'), '# Three\n');+ settleTimestamps();+ await watch(5_000);++ const hashedBefore = getMetrics().freshnessFilesHashed;+ await runTimedPass(5_000);+ expect(getMetrics().freshnessFilesHashed - hashedBefore).toBe(0);+ expect(events).toHaveLength(0);++ writeFileSync(join(contentDir, 'two.md'), '# Two, edited outside\n');+ const hashedBeforeEdit = getMetrics().freshnessFilesHashed;+ await runTimedPass(5_000);+ expect(getMetrics().freshnessFilesHashed - hashedBeforeEdit).toBe(1);+ expect(eventsFor(events, 'update', 'two')).toHaveLength(1);+ expect(events).toHaveLength(1);++ const hashedAfterIntake = getMetrics().freshnessFilesHashed;+ await runTimedPass(5_000);+ expect(getMetrics().freshnessFilesHashed - hashedAfterIntake).toBe(0);+ });++ test('a file changed within the timestamp granularity is read again until its times settle', async () => {+ freezeTimestampsNow();+ writeFileSync(join(contentDir, 'fresh.md'), '# Fresh\n');+ await watch(5_000);++ const hashedBefore = getMetrics().freshnessFilesHashed;+ await runTimedPass(5_000);+ await runTimedPass(5_000);+ expect(getMetrics().freshnessFilesHashed - hashedBefore).toBe(2);++ settleTimestamps();+ await runTimedPass(5_000);+ const hashedAfterSettle = getMetrics().freshnessFilesHashed;+ await runTimedPass(5_000);+ expect(getMetrics().freshnessFilesHashed - hashedAfterSettle).toBe(0);+ expect(events).toHaveLength(0);+ });++ test('changes beyond one pass budget are carried to the next pass', async () => {+ settleTimestamps();+ await watch(5_000);+ const total = FRESHNESS_HASH_BUDGET.files + 40;+ for (let index = 0; index < total; index++) {+ writeFileSync(join(contentDir, `bulk-${String(index).padStart(4, '0')}.md`), `# ${index}\n`);+ }++ const carriedBefore = getMetrics().freshnessCarryOver;+ await runTimedPass(5_000);+ expect(events.filter((event) => event.kind === 'create')).toHaveLength(+ FRESHNESS_HASH_BUDGET.files,+ );+ expect(getMetrics().freshnessCarryOver - carriedBefore).toBe(40);++ await runTimedPass(5_000);+ expect(events.filter((event) => event.kind === 'create')).toHaveLength(total);+ });++ test('an edit that keeps the file size and moves the modification time is found by the timed pass', async () => {+ const same = join(contentDir, 'same.md');+ writeFileSync(same, '# AAAA\n');+ settleTimestamps();+ await watch(5_000);++ writeFileSync(same, '# BBBB\n');+ moveModificationTimeLater(same);+ await runTimedPass(5_000);+ expect(eventsFor(events, 'update', 'same')).toHaveLength(1);+ });++ test('an edit that keeps the file size and the modification time is still found by the timed pass', async () => {+ const same = join(contentDir, 'same.md');+ writeFileSync(same, '# AAAA\n');+ await watch(5_000);++ keepModificationTime(same, () => writeFileSync(same, '# BBBB\n'));+ await runTimedPass(5_000);+ expect(eventsFor(events, 'update', 'same')).toHaveLength(1);+ });++ test.runIf(CAN_REVOKE_READ).each([+ ['a later modification time', true],+ ['an unchanged modification time', false],+ ] as const)(+ 'a file that cannot be read, with %s, does not stop the pass from applying a change beside it',+ async (_label, settled) => {+ const locked = join(contentDir, 'locked.md');+ writeFileSync(locked, '# Locked\n');+ writeFileSync(join(contentDir, 'open.md'), '# Open\n');+ if (settled) settleTimestamps();+ const reconciled = vi.fn(async () => {});+ await watch(5_000, { onRecoveryComplete: reconciled });+ const unreadableBefore = getMetrics().watcherUnreadableEntries;++ if (settled) {+ revokeRead(locked);+ moveModificationTimeLater(locked);+ } else {+ keepModificationTime(locked, () => revokeRead(locked));+ }+ writeFileSync(join(contentDir, 'open.md'), '# Open, edited outside\n');+ reconciled.mockClear();+ await runTimedPass(5_000);+ expect(eventsFor(events, 'update', 'open')).toHaveLength(1);+ expect(eventsFor(events, 'delete', 'locked')).toHaveLength(0);+ expect(handle?.getFileIndex().has('locked')).toBe(true);+ expect(reconciled).not.toHaveBeenCalled();++ writeFileSync(join(contentDir, 'open.md'), '# Open, edited again\n');+ await runTimedPass(5_000);+ expect(eventsFor(events, 'update', 'open')).toHaveLength(2);+ expect(getMetrics().watcherUnreadableEntries - unreadableBefore).toBe(1);++ chmodSync(locked, 0o644);+ moveModificationTimeLater(locked);+ await runTimedPass(5_000);+ expect(reconciled).toHaveBeenCalledTimes(1);+ },+ );++ test.runIf(CAN_REVOKE_READ)(+ 'a folder that cannot be listed keeps its documents while the rest of the tree is checked',+ async () => {+ mkdirSync(join(contentDir, 'private'));+ writeFileSync(join(contentDir, 'private', 'inside.md'), '# Inside\n');+ writeFileSync(join(contentDir, 'open.md'), '# Open\n');+ settleTimestamps();+ await watch(5_000);++ revokeRead(join(contentDir, 'private'));+ writeFileSync(join(contentDir, 'open.md'), '# Open, edited outside\n');+ await runTimedPass(5_000);+ expect(eventsFor(events, 'update', 'open')).toHaveLength(1);+ expect(events.filter((event) => event.kind === 'delete')).toHaveLength(0);+ expect(events.filter((event) => event.kind === 'folder-delete')).toHaveLength(0);+ expect(handle?.getFileIndex().has('private/inside')).toBe(true);+ },+ );++ test('an event the native watcher delivers while a timed pass walks the tree is applied before the pass ends', async () => {+ mkdirSync(join(contentDir, 'slow'));+ writeFileSync(join(contentDir, 'slow', 'a.md'), '# A\n');+ settleTimestamps();+ const double = createWatcherDouble();+ handle = await startWatcher(+ contentDir,+ async (event) => {+ events.push(event);+ },+ undefined,+ { nativeBackend: double.backend, clock, freshnessIntervalMs: 5_000 },+ );+ const passesBefore = getMetrics().freshnessPasses;+ walkGate.dir = join(contentDir, 'slow');+ clock.advance(5_000);+ await until(() => walkGate.entered, 'the timed pass to reach the slow folder');++ const delivered = join(contentDir, 'delivered.md');+ writeFileSync(delivered, '# Delivered\n');+ double.subscriptionFor(contentDir).deliver([{ type: 'create', path: delivered }]);+ await until(+ () => eventsFor(events, 'create', 'delivered').length === 1,+ 'the delivered event while the pass is still walking',+ );+ expect(getMetrics().freshnessPasses).toBe(passesBefore);++ walkGate.release?.();+ await until(() => getMetrics().freshnessPasses > passesBefore, 'the timed pass to finish');+ expect(eventsFor(events, 'create', 'delivered')).toHaveLength(1);+ });++ test('a skill folder added outside the app between passes is admitted by the next timed pass', async () => {+ const filter = createContentFilter({+ projectDir: contentDir,+ contentDir,+ inPlaceSkillDirs: scanInPlaceSkillDirs(contentDir),+ rescanInPlaceSkillDirs: () => scanInPlaceSkillDirs(contentDir),+ skillRootPaths: skillRootPathsFor(contentDir),+ });+ double = createWatcherDouble();+ handle = await startWatcher(+ contentDir,+ async (event) => {+ events.push(event);+ },+ filter,+ { nativeBackend: double.backend, clock, freshnessIntervalMs: 5_000 },+ );+ const bundle = join(contentDir, '.claude', 'skills', 'added-later');+ mkdirSync(bundle, { recursive: true });+ writeFileSync(+ join(bundle, 'SKILL.md'),+ '---\nname: added-later\ndescription: fixture\n---\n\n# Added later\n',+ );+ expect(filter.isExcluded('.claude/skills/added-later/SKILL.md')).toBe(true);++ await runTimedPass(5_000);+ expect(eventsFor(events, 'create', '.claude/skills/added-later/SKILL')).toHaveLength(1);+ expect(handle.getFileIndex().has('.claude/skills/added-later/SKILL')).toBe(true);+ });++ test('a change intake applies while a timed pass walks is kept, and the pass walks the tree once', async () => {+ const raced = join(contentDir, 'raced.md');+ writeFileSync(raced, '# Raced v1\n');+ settleTimestamps();+ await watch(5_000);+ writeFileSync(raced, '# Raced v2, edited outside\n');+ moveModificationTimeLater(raced);+ walkGate.afterRead = raced;+ walkGate.directoryReads.clear();+ const passesBefore = getMetrics().freshnessPasses;+ clock.advance(5_000);+ await until(() => walkGate.entered, 'the timed pass to read the edited file');++ const latest = '# Raced v3, edited again while the pass walks\n';+ writeFileSync(raced, latest);+ double.subscriptionFor(contentDir).deliver([{ type: 'update', path: raced }]);+ await until(() => lastKnownHash.get(raced) === contentHash(latest), 'intake to apply v3');+ walkGate.release?.();+ await until(() => getMetrics().freshnessPasses > passesBefore, 'the timed pass to finish');++ expect(walkGate.directoryReads.get(contentDir)).toBe(1);+ expect(lastKnownHash.get(raced)).toBe(contentHash(latest));+ expect(eventsFor(events, 'update', 'raced')).toHaveLength(1);+ await runTimedPass(5_000);+ expect(eventsFor(events, 'update', 'raced')).toHaveLength(1);+ });++ test('a Markdown link pointed at another file outside the app is followed by the timed pass', async () => {+ writeFileSync(join(contentDir, 'first.md'), '# First\n');+ writeFileSync(join(contentDir, 'second.md'), '# Second\n');+ const pointer = join(contentDir, 'pointer.md');+ symlinkSync(join(contentDir, 'first.md'), pointer);+ settleTimestamps();+ await watch(5_000);+ expect(handle?.getAliasMap().get('pointer')).toBe('first');++ rmSync(pointer);+ symlinkSync(join(contentDir, 'second.md'), pointer);+ await runTimedPass(5_000);+ expect(handle?.getAliasMap().get('pointer')).toBe('second');+ });++ test('a re-read while the content folder is missing reports no document deleted', async () => {+ const kept = join(contentDir, 'kept.md');+ writeFileSync(kept, '# Kept\n');+ await watch(null);++ rmSync(contentDir, { recursive: true, force: true });+ const result = await (handle as WatcherHandle).recheckPaths([kept]);+ expect(result).toEqual({ checked: 0, found: 0 });+ expect(events.filter((event) => event.kind === 'delete')).toHaveLength(0);+ expect(handle?.getFileIndex().has('kept')).toBe(true);+ });++ test('a re-read under an intact content folder reports a file deleted outside the app', async () => {+ const gone = join(contentDir, 'gone.md');+ writeFileSync(gone, '# Gone\n');+ await watch(null);++ rmSync(gone);+ const result = await (handle as WatcherHandle).recheckPaths([gone]);+ expect(result).toEqual({ checked: 1, found: 1 });+ expect(eventsFor(events, 'delete', 'gone')).toHaveLength(1);+ });++ test('stopping the watcher while a timed pass walks the tree waits for that pass, which counts nothing afterwards', async () => {+ mkdirSync(join(contentDir, 'slow'));+ writeFileSync(join(contentDir, 'slow', 'a.md'), '# A\n');+ settleTimestamps();+ const watching = await watch(5_000);+ const passesBefore = getMetrics().freshnessPasses;+ const incompleteBefore = getMetrics().freshnessPassesIncomplete;+ walkGate.dir = join(contentDir, 'slow');+ clock.advance(5_000);+ await until(() => walkGate.entered, 'the timed pass to reach the slow folder');++ let stopped = false;+ handle = undefined;+ const stopping = watching.unsubscribe().then(() => {+ stopped = true;+ });+ await flush();+ expect(stopped).toBe(false);+ walkGate.release?.();+ await stopping;++ expect(getMetrics().freshnessPasses).toBe(passesBefore);+ expect(getMetrics().freshnessPassesIncomplete).toBe(incompleteBefore);+ expect(lastKnownHash.size).toBe(0);+ });++ test('a full re-check requested while a timed recovery is queued runs that recovery in full', async () => {+ settleTimestamps();+ const double = createWatcherDouble();+ let releaseQueue: () => void = () => {};+ const queueHeld = new Promise<void>((resolve) => {+ releaseQueue = resolve;+ });+ handle = await startWatcher(+ contentDir,+ async (event) => {+ events.push(event);+ if (event.kind === 'create' && event.docName === 'blocker') await queueHeld;+ },+ undefined,+ { nativeBackend: double.backend, clock, freshnessIntervalMs: 5_000 },+ );+ const blocker = join(contentDir, 'blocker.md');+ writeFileSync(blocker, '# Blocker\n');+ double.subscriptionFor(contentDir).deliver([{ type: 'create', path: blocker }]);+ await until(() => eventsFor(events, 'create', 'blocker').length === 1, 'the queue to be held');++ const total = FRESHNESS_HASH_BUDGET.files + 40;+ for (let index = 0; index < total; index++) {+ writeFileSync(join(contentDir, `bulk-${String(index).padStart(4, '0')}.md`), `# ${index}\n`);+ }+ const hashedBefore = getMetrics().freshnessFilesHashed;+ clock.advance(5_000);+ await until(() => getMetrics().freshnessFilesHashed > hashedBefore, 'the timed probe');+ await flush();++ const full = (handle as WatcherHandle).recheck();+ releaseQueue();+ expect((await full).status).toBe('completed');+ const bulkCreates = events.filter(+ (event) => event.kind === 'create' && event.docName.startsWith('bulk-'),+ );+ expect(bulkCreates).toHaveLength(total);+ });++ test('an indexed document with no stored hash is read, not reported deleted, when the budget runs out', async () => {+ writeFileSync(join(contentDir, 'zz-kept.md'), '# Kept\n');+ settleTimestamps();+ await watch(5_000);+ removeLastKnownHash(join(contentDir, 'zz-kept.md'));+ for (let index = 0; index < FRESHNESS_HASH_BUDGET.files * 2 + 100; index++) {+ writeFileSync(join(contentDir, `bulk-${String(index).padStart(4, '0')}.md`), `# ${index}\n`);+ }++ await runTimedPass(5_000);+ expect(eventsFor(events, 'delete', 'zz-kept')).toHaveLength(0);+ expect(handle?.getFileIndex().has('zz-kept')).toBe(true);+ });++ test.runIf(process.platform !== 'win32')(+ 'a re-read of a FIFO or a link to a device returns without reading it',+ async () => {+ await watch(null);+ const fifo = join(contentDir, 'pipe.md');+ execFileSync('mkfifo', [fifo]);+ const device = join(contentDir, 'zero.md');+ symlinkSync('/dev/zero', device);++ const result = await (handle as WatcherHandle).recheckPaths([fifo, device]);+ expect(result).toEqual({ checked: 2, found: 0 });+ expect(events).toHaveLength(0);+ },+ );++ test('a content root that is gone leaves the known documents in place instead of deleting them', async () => {+ writeFileSync(join(contentDir, 'kept.md'), '# Kept\n');+ await watch(null);+ expect(handle?.getFileIndex().has('kept')).toBe(true);++ rmSync(contentDir, { recursive: true, force: true });+ const result = await (handle as WatcherHandle).recheck();+ expect(result.status).toBe('incomplete');+ expect(events.filter((event) => event.kind === 'delete')).toHaveLength(0);+ expect(handle?.getFileIndex().has('kept')).toBe(true);+ });+});++describe('freshness pass spacing', () => {+ test('a quick pass keeps the base interval', () => {+ expect(nextFreshnessDelayMs(0, 5_000)).toBe(5_000);+ expect(nextFreshnessDelayMs(200, 5_000)).toBe(5_000);+ });++ test('a slow pass spaces the next one so passes take about a twentieth of the time', () => {+ expect(nextFreshnessDelayMs(1_000, 5_000)).toBe(20_000);+ });++ test('the spacing never exceeds the maximum unless the base interval does', () => {+ expect(nextFreshnessDelayMs(30_000, 5_000)).toBe(FRESHNESS_MAX_INTERVAL_MS);+ expect(nextFreshnessDelayMs(30_000, 90_000)).toBe(90_000);+ });+});diff --git a/packages/server/src/file-watcher-index-retention.test.ts b/packages/server/src/file-watcher-index-retention.test.tsnew file mode 100644index 000000000..03b8b99f5--- /dev/null+++ b/packages/server/src/file-watcher-index-retention.test.ts@@ -0,0 +1,62 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { setFlagsFromString } from 'node:v8';+import { runInNewContext } from 'node:vm';+import { afterEach, beforeEach, expect, test } from 'vitest';+import { lastKnownHash, startWatcher, type WatcherHandle } from './file-watcher.ts';+import { createManualClock } from './watcher-clock.test-helper.ts';+import { createWatcherDouble } from './watcher-double.test-helper.ts';++setFlagsFromString('--expose_gc');+const collectGarbage = runInNewContext('gc') as () => void;++const DOCUMENT_COUNT = 300;+const BODY_BYTES = 64 * 1024;++let tmpDir: string;+let contentDir: string;+let handle: WatcherHandle | undefined;++beforeEach(() => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-retention-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ contentDir = realpathSync(contentDir);+ lastKnownHash.clear();+});++afterEach(async () => {+ await handle?.unsubscribe();+ handle = undefined;+ rmSync(tmpDir, { recursive: true, force: true });+});++function settledHeapBytes(): number {+ collectGarbage();+ collectGarbage();+ return process.memoryUsage().heapUsed;+}++test('the file index keeps titles and descriptions without keeping each document text alive', async () => {+ for (let index = 0; index < DOCUMENT_COUNT; index++) {+ const name = `doc-${String(index).padStart(4, '0')}`;+ writeFileSync(+ join(contentDir, `${name}.md`),+ `---\ntitle: Document title ${index}\ndescription: A description of document ${index}\n---\n\n# Heading of document ${index}\n\n${'x'.repeat(BODY_BYTES)}\n`,+ );+ }++ const before = settledHeapBytes();+ handle = await startWatcher(contentDir, async () => {}, undefined, {+ nativeBackend: createWatcherDouble().backend,+ clock: createManualClock(Date.now()),+ freshnessIntervalMs: null,+ });+ const retained = settledHeapBytes() - before;++ const entry = handle.getFileIndex().get('doc-0007');+ expect(entry?.title).toBe('Document title 7');+ expect(entry?.description).toBe('A description of document 7');+ expect(retained).toBeLessThan((DOCUMENT_COUNT * BODY_BYTES) / 4);+});diff --git a/packages/server/src/file-watcher-settle.test.ts b/packages/server/src/file-watcher-settle.test.tsindex 4dc69318d..96724730c 100644--- a/packages/server/src/file-watcher-settle.test.ts+++ b/packages/server/src/file-watcher-settle.test.ts@@ -183,7 +183,7 @@ test('settle rejects when the watcher stops before its fence arrives, and leavesexpect(fencesOnDisk()).toEqual([]);});-test('a watcher error that ends the subscription fails a pending settle and every later one, and leaves no fence behind', async () => {+test('a watcher error that ends the subscription fails a pending settle and later ones until it is subscribed again, and leaves no fence behind', async () => {const { watcher } = await watchContent('parcel');const pending = observed(watcher.settle());expect(fencesOnDisk()).toHaveLength(1);@@ -191,7 +191,7 @@ test('a watcher error that ends the subscription fails a pending settle and everconst failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));- const refusal = `the file watcher failed before it saw its settle fence: ${failure}`;+ const refusal = `the file watcher is subscribing again after an error and cannot settle until it is back: ${failure}`;expect(pending()).toEqual({ rejected: refusal });expect(fencesOnDisk()).toEqual([]);const later = observed(watcher.settle());diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex 5b1c8c74b..23ea050cc 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -13,7 +13,7 @@ import {} from 'node:fs';import { lstat, readdir, readFile, realpath, stat } from 'node:fs/promises';import { basename, dirname, extname, isAbsolute, join, relative, sep } from 'node:path';-import { LINKABLE_ASSET_EXTENSIONS } from '@inkeep/open-knowledge-core';+import { DOCUMENT_OPEN_BYTE_LIMIT, LINKABLE_ASSET_EXTENSIONS } from '@inkeep/open-knowledge-core';import { filesystemIdentity, statMtime } from '@inkeep/open-knowledge-core/server';import { isConfigDoc, isReservedForUserTree, isSystemDoc } from './cc1-broadcast.ts';import { type ContentFilter, WATCHER_STRUCTURAL_IGNORE_DIRS } from './content-filter.ts';@@ -38,6 +38,15 @@ import {} from './fs-traced.ts';import { errnoCode } from './http/handler-utils.ts';import { getLogger } from './logger.ts';+import {+ addFreshnessCarryOver,+ addFreshnessFilesHashed,+ addFreshnessFinds,+ incrementFreshnessPasses,+ incrementFreshnessPassesIncomplete,+ incrementWatcherErrors,+ incrementWatcherUnreadableEntries,+} from './metrics.ts';import {extractPageDescription,extractPageIcon,@@ -48,6 +57,7 @@ import { classifyParcelNotification } from './parcel-recovery.ts';import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';import { toPosix } from './path-utils.ts';import { containsConflictMarkers } from './reconciliation.ts';+import { isRacyStat, sampleWallClockNs, statSignature } from './stat-signature.ts';import { getMeter, withSpan } from './telemetry.ts';import { contentHash } from './version-hash.ts';import {@@ -56,6 +66,15 @@ import {type WatcherTimer,watcherClock,} from './watcher-clock.ts';+import {+ createResubscriber,+ FRESHNESS_HASH_BUDGET,+ FRESHNESS_MAX_INTERVAL_MS,+ nextFreshnessDelayMs,+ RESUBSCRIBE_SLOW_RETRY_MS,+ type RecoveryMode,+ resolveFreshnessInterval,+} from './watcher-freshness.ts';export type { NativeWatcherBackend } from './parcel-subscription.ts';export { contentHash } from './version-hash.ts';@@ -237,14 +256,21 @@ function derivePageMeta(description: string | undefined;type: string | undefined;} {+ const icon = extractPageIcon(content);+ const description = extractPageDescription(content);+ const type = extractPageType(content);return {- title: extractPageTitle(content, docName),- icon: extractPageIcon(content),- description: extractPageDescription(content),- type: extractPageType(content),+ title: copyDetachedFromSource(extractPageTitle(content, docName)),+ icon: icon === undefined ? undefined : copyDetachedFromSource(icon),+ description: description === undefined ? undefined : copyDetachedFromSource(description),+ type: type === undefined ? undefined : copyDetachedFromSource(type),};}+function copyDetachedFromSource(value: string): string {+ return Buffer.from(value, 'utf16le').toString('utf16le');+}+function markdownIndexView(inner: ReadonlyMap<string, FileIndexEntry>,): ReadonlyMap<string, FileIndexEntry> {@@ -266,6 +292,7 @@ export interface RecheckResult {interface RecoveryOutcome {completed: boolean;candidates: number;+ partial?: boolean;}interface DispatchCounter {@@ -275,6 +302,7 @@ interface DispatchCounter {export interface WatcherHandle {unsubscribe: () => Promise<void>;recheck: () => Promise<RecheckResult>;+ recheckPaths: (paths: readonly string[]) => Promise<{ checked: number; found: number }>;getFileIndex: () => ReadonlyMap<string, FileIndexEntry>;getAllFilesIndex: () => AllFileEntries;getFileIndexGeneration: () => number;@@ -648,16 +676,129 @@ function extractDocExtension(path: string): string | null {export const lastKnownHash = new Map<string, string>();+const lastKnownStat = new Map<string, string>();++let intakeEpoch = 0;+let openObservationWindows = 0;+const intakeEpochByPath = new Map<string, number>();++function noteIntakeMutation(path: string): void {+ intakeEpoch++;+ if (openObservationWindows > 0) intakeEpochByPath.set(path, intakeEpoch);+}++function openObservationWindow(): number {+ openObservationWindows++;+ return intakeEpoch;+}++function closeObservationWindow(): void {+ openObservationWindows--;+ if (openObservationWindows === 0) intakeEpochByPath.clear();+}++function changedSince(epoch: number, contentDir: string): ((path: string) => boolean) | undefined {+ const changed = new Set<string>();+ for (const [path, at] of intakeEpochByPath) if (at > epoch) changed.add(path);+ if (changed.size === 0) return undefined;+ return (path) => {+ let current = path;+ while (true) {+ if (changed.has(current)) return true;+ if (current === contentDir) return false;+ const parent = dirname(current);+ if (parent === current) return false;+ current = parent;+ }+ };+}+export function updateLastKnownHash(filePath: string, hash: string): void {+ noteIntakeMutation(filePath);lastKnownHash.set(filePath, hash);+ lastKnownStat.delete(filePath);}export function removeLastKnownHash(filePath: string): string | undefined {+ noteIntakeMutation(filePath);const hash = lastKnownHash.get(filePath);lastKnownHash.delete(filePath);+ lastKnownStat.delete(filePath);return hash;}+function recordObservedHash(filePath: string, hash: string, signature: string | undefined): void {+ lastKnownHash.set(filePath, hash);+ if (signature === undefined) lastKnownStat.delete(filePath);+ else lastKnownStat.set(filePath, signature);+}++interface FreshnessReuse {+ readonly previousByPath: ReadonlyMap<string, FileIndexEntry>;+ readonly carriedOver: ReadonlySet<string>;+ readonly deferred: Set<string>;+ carriedSlots: number;+ freeSlots: number;+ bytesLeft: number;+ hashed: number;+}++type MarkdownReadDecision =+ | { kind: 'read' }+ | { kind: 'reuse'; entry: FileIndexEntry; hash: string }+ | { kind: 'defer'; entry: FileIndexEntry | undefined; hash: string | undefined };++function beginFreshnessReuse(+ mode: RecoveryMode,+ fileIndex: ReadonlyMap<string, FileIndexEntry>,+ carryOver: Set<string>,+): FreshnessReuse | undefined {+ const carriedOver = new Set(carryOver);+ carryOver.clear();+ if (mode === 'full') return undefined;+ const previousByPath = new Map<string, FileIndexEntry>();+ for (const entry of fileIndex.values()) {+ if (entry.kind === 'markdown') previousByPath.set(entry.canonicalPath, entry);+ }+ const carriedSlots = Math.min(carriedOver.size, FRESHNESS_HASH_BUDGET.files);+ return {+ previousByPath,+ carriedOver,+ deferred: carryOver,+ carriedSlots,+ freeSlots: FRESHNESS_HASH_BUDGET.files - carriedSlots,+ bytesLeft: FRESHNESS_HASH_BUDGET.bytes,+ hashed: 0,+ };+}++function decideMarkdownRead(+ reuse: FreshnessReuse | undefined,+ path: string,+ stats: BigIntStats,+ signature: string,+): MarkdownReadDecision {+ if (reuse === undefined) return { kind: 'read' };+ const previous = reuse.previousByPath.get(path);+ const hash = lastKnownHash.get(path);+ if (previous !== undefined && hash !== undefined && lastKnownStat.get(path) === signature) {+ return { kind: 'reuse', entry: previous, hash };+ }+ const size = Number(stats.size);+ const carried = reuse.carriedOver.has(path);+ const slotFree = carried ? reuse.carriedSlots > 0 : reuse.freeSlots > 0;+ const indexedWithoutHash = previous !== undefined && hash === undefined;+ if (indexedWithoutHash || (slotFree && (reuse.hashed === 0 || size <= reuse.bytesLeft))) {+ if (carried) reuse.carriedSlots--;+ else reuse.freeSlots--;+ reuse.bytesLeft -= size;+ reuse.hashed++;+ return { kind: 'read' };+ }+ reuse.deferred.add(path);+ return { kind: 'defer', entry: previous, hash };+}+interface RawFileEvent {type: 'create' | 'update' | 'delete';path: string;@@ -1070,11 +1211,72 @@ interface DiskObservation {aliasPaths: Map<string, string>;folderAliasIndex: Map<string, string>;hashes: Map<string, string>;+ unreadable: Map<string, string>;+ signatures: Map<string, string>;+ sampledAtNs: bigint;+ reuse: FreshnessReuse | undefined;extensions: Array<{ docName: string; ext: string }>;preferredExtensions: Map<string, string>;structuralIgnoreDirs: Set<string>;paths: Set<string>;complete: boolean;+ stopped?: () => boolean;+}++const reportedUnreadable = new Map<string, string>();++function markUnreadable(+ observation: DiskObservation,+ path: string,+ code: string | undefined,+): void {+ observation.unreadable.set(path, code ?? 'unknown');+}++function reportUnreadableEntries(observation: DiskObservation): void {+ for (const [path, code] of observation.unreadable) {+ if (reportedUnreadable.has(path)) continue;+ reportedUnreadable.set(path, code);+ incrementWatcherUnreadableEntries();+ log.warn(+ { path, code },+ `Cannot read ${path} (${code}); it keeps its last known state until it can be read`,+ );+ }+ for (const path of reportedUnreadable.keys()) {+ if (!observation.unreadable.has(path)) reportedUnreadable.delete(path);+ }+}++function isUnderUnreadable(observation: DiskObservation, path: string): boolean {+ for (const unreadable of observation.unreadable.keys()) {+ if (path === unreadable || path.startsWith(`${unreadable}${sep}`)) return true;+ }+ return false;+}++function observeMarkdownSignature(+ observation: DiskObservation,+ path: string,+ stats: BigIntStats,+ signature: string,+): void {+ if (!isRacyStat(stats, observation.sampledAtNs)) observation.signatures.set(path, signature);+}++function reusedMarkdownEntry(+ decision: Exclude<MarkdownReadDecision, { kind: 'read' }>,+ observation: DiskObservation,+ path: string,+ identity: string,+ aliases: string[],+ signature: string,+): FileIndexEntry | undefined {+ if (decision.entry === undefined || decision.hash === undefined) return undefined;+ observation.hashes.set(path, decision.hash);+ observation.paths.add(path);+ if (decision.kind === 'reuse') observation.signatures.set(path, signature);+ return { ...decision.entry, canonicalPath: path, inode: identity, aliases };}type GeneralIdentity = ReturnType<typeof filesystemIdentity>;@@ -1841,6 +2043,10 @@ async function scanDisk(): Promise<void> {const visited = visitedInodes ?? new Set<string>();const generalVisited = generalVisitedInodes ?? new Set<string>();+ if (observation.stopped?.()) {+ observation.complete = false;+ return;+ }try {const entries = await readdir(dir, { withFileTypes: true });for (const entry of entries) {@@ -1850,10 +2056,7 @@ async function scanDisk(lst = await lstat(fullPath, { bigint: true });} catch (e) {const code = errnoCode(e);- if (code !== 'ENOENT') {- observation.complete = false;- log.warn({ path: fullPath, err: e }, `Failed to lstat ${fullPath}, skipping`);- }+ if (code !== 'ENOENT') markUnreadable(observation, fullPath, code);continue;}const lstIdentity = filesystemIdentity(lst);@@ -1867,8 +2070,7 @@ async function scanDisk(if (code === 'ENOENT' || code === 'ELOOP') {log.warn({ path: fullPath, code }, `Broken/cyclic symlink at ${fullPath}, skipping`);} else {- observation.complete = false;- log.warn({ path: fullPath, err: e }, `Failed to resolve symlink ${fullPath}`);+ markUnreadable(observation, fullPath, code);}continue;}@@ -1972,10 +2174,28 @@ async function scanDisk(observation.aliasPaths.set(aliasDocName, fullPath);observation.paths.add(fullPath);+ const signature = statSignature(canonStat);+ const decision = decideMarkdownRead(observation.reuse, canonical, canonStat, signature);+ if (decision.kind !== 'read') {+ const reused = reusedMarkdownEntry(+ decision,+ observation,+ canonical,+ canonIdentity,+ [aliasDocName],+ signature,+ );+ if (reused === undefined) continue;+ const ext = extractDocExtension(canonical);+ if (ext && !recordObservedExtension(observation, canonicalDocName, ext)) continue;+ observation.fileIndex.set(canonicalDocName, reused);+ continue;+ }try {const content = await readFile(canonical, 'utf-8');const hash = contentHash(content);observation.hashes.set(canonical, hash);+ observeMarkdownSignature(observation, canonical, canonStat, signature);observation.paths.add(canonical);const ext = extractDocExtension(canonical);if (ext && !recordObservedExtension(observation, canonicalDocName, ext)) continue;@@ -1990,18 +2210,16 @@ async function scanDisk(});} catch (err) {const code = errnoCode(err);- if (code !== 'ENOENT') {- observation.complete = false;- log.warn({ path: canonical, err }, `Failed to seed hash for ${canonical}`);- }+ if (code !== 'ENOENT') markUnreadable(observation, canonical, code);}}} catch (e) {const code = errnoCode(e);if (code !== 'ENOENT' && code !== 'ENOTDIR') {- observation.complete = false;+ markUnreadable(observation, fullPath, code);+ } else {+ log.warn({ path: canonical, err: e }, `Failed to stat symlink target ${canonical}`);}- log.warn({ path: canonical, err: e }, `Failed to stat symlink target ${canonical}`);}} else if (lst.isDirectory()) {const relPath = contentRelativePath(contentDir, fullPath);@@ -2031,9 +2249,28 @@ async function scanDisk(const relPath = toPosix(relative(contentDir, fullPath));if (contentFilter.isExcluded(relPath)) continue;}+ const signature = statSignature(lst);+ const decision = decideMarkdownRead(observation.reuse, fullPath, lst, signature);+ if (decision.kind !== 'read') {+ const reused = reusedMarkdownEntry(+ decision,+ observation,+ fullPath,+ lstIdentity,+ [],+ signature,+ );+ if (reused === undefined) continue;+ const docName = pathToDocName(fullPath, contentDir);+ const ext = extractDocExtension(fullPath);+ if (ext && !recordObservedExtension(observation, docName, ext)) continue;+ observation.fileIndex.set(docName, reused);+ continue;+ }try {const content = await readFile(fullPath, 'utf-8');observation.hashes.set(fullPath, contentHash(content));+ observeMarkdownSignature(observation, fullPath, lst, signature);observation.paths.add(fullPath);const docName = pathToDocName(fullPath, contentDir);@@ -2050,16 +2287,7 @@ async function scanDisk(});} catch (err) {const code = errnoCode(err);- if (code === 'EACCES') {- observation.complete = false;- log.warn(- { path: fullPath, code },- `Permission denied reading ${fullPath}, file excluded from index`,- );- } else if (code !== 'ENOENT') {- observation.complete = false;- log.warn({ path: fullPath, err }, `Failed to seed hash for ${fullPath}`);- }+ if (code !== 'ENOENT') markUnreadable(observation, fullPath, code);}} else if (lst.isFile()) {if (contentFilter) {@@ -2088,9 +2316,11 @@ async function scanDisk(}} catch (err) {const code = errnoCode(err);- if (code !== 'ENOENT' && code !== 'ENOTDIR') {+ if (dir === contentDir) {observation.complete = false;- log.warn({ dir, err }, `Failed to read directory ${dir}`);+ log.warn({ dir, err }, `Failed to read the content directory ${dir}`);+ } else if (code !== 'ENOENT' && code !== 'ENOTDIR') {+ markUnreadable(observation, dir, code);}}}@@ -2098,6 +2328,8 @@ async function scanDisk(async function observeDisk(contentDir: string,contentFilter: ContentFilter | undefined,+ reuse?: FreshnessReuse,+ stopped?: () => boolean,): Promise<DiskObservation> {const observation: DiskObservation = {fileIndex: new Map(),@@ -2107,17 +2339,27 @@ async function observeDisk(aliasPaths: new Map(),folderAliasIndex: new Map(),hashes: new Map(),+ unreadable: new Map(),+ signatures: new Map(),+ sampledAtNs: sampleWallClockNs(),+ reuse,extensions: [],preferredExtensions: new Map(),structuralIgnoreDirs: new Set(),paths: new Set(),complete: true,+ stopped,};const generalScan: GeneralScanAccumulator = {facts: new Map(),metadataEffects: new Map(),};await scanDisk(contentDir, contentDir, contentFilter, observation, generalScan);+ if (observation.stopped?.()) {+ observation.complete = false;+ return observation;+ }+ reportUnreadableEntries(observation);materializeScanGeneralIndex(observation, contentDir, generalScan);for (const [name, entry] of new Map(observation.generalFileIndex)) {const state = knownGeneralState(entry);@@ -2202,7 +2444,9 @@ function publishDiskObservation();}}- for (const [path, hash] of observation.hashes) lastKnownHash.set(path, hash);+ for (const [path, hash] of observation.hashes) {+ recordObservedHash(path, hash, observation.signatures.get(path));+ }for (const [docName, entry] of inventory.fileIndex) {if (getDocExtension(docName).toLowerCase() !== extname(entry.canonicalPath).toLowerCase()) {continue;@@ -2507,6 +2751,7 @@ async function handleRawEventsInternal(generalFileIndex?: GeneralFileIndex,aliasPaths?: Map<string, string>,): Promise<void> {+ for (const event of rawEvents) noteIntakeMutation(event.path);const declaredBeforeBatch: ReadonlySet<string> = new Set(removalTracker.keys());const safeEvents = rawEvents.filter((e) => {if (!eventEscapesContentDir(e.path, contentDir)) return true;@@ -3058,6 +3303,43 @@ type RecoveredAliasCandidate =targetIdentity: GeneralIdentity;};+function protectedGeneralNames(+ observed: DiskObservation,+ contentDir: string,+ previousGeneralFacts: ReadonlyMap<string, GeneralPathFact>,+ observedGeneralFacts: ReadonlyMap<string, GeneralPathFact>,+ changedDuringWalk: ((path: string) => boolean) | undefined,+): Set<string> {+ const names = new Set<string>();+ if (observed.unreadable.size === 0 && changedDuringWalk === undefined) return names;+ const keepKnownAt = (name: string): boolean => {+ const path = join(contentDir, name);+ return isUnderUnreadable(observed, path) || changedDuringWalk?.(path) === true;+ };+ const identities = new Set<GeneralIdentity>();+ for (const [name, fact] of previousGeneralFacts) {+ if (fact.kind !== 'unresolved' && keepKnownAt(name)) identities.add(fact.identity);+ }+ for (const [name, fact] of previousGeneralFacts) {+ if (+ keepKnownAt(name) ||+ (fact.kind !== 'unresolved' && identities.has(fact.identity)) ||+ (fact.kind === 'symlink' && keepKnownAt(fact.targetPath))+ ) {+ names.add(name);+ }+ }+ for (const name of observedGeneralFacts.keys()) {+ if (keepKnownAt(name)) names.add(name);+ }+ return names;+}++interface HandedObservation {+ readonly observed: DiskObservation;+ readonly sinceEpoch: number;+}+async function recoverDiskChanges(contentDir: string,contentFilter: ContentFilter | undefined,@@ -3073,6 +3355,8 @@ async function recoverDiskChanges(getFileIndexGeneration: () => number,counter: DispatchCounter,onRawBatch?: (absPaths: readonly string[]) => void,+ reuse?: FreshnessReuse,+ handed?: HandedObservation,): Promise<RecoveryOutcome> {const countingOnDiskEvent = async (event: DiskEvent): Promise<void> => {counter.dispatched++;@@ -3095,18 +3379,35 @@ async function recoverDiskChanges(const previousAliasPaths = new Map(aliasPaths);const previousFolderAliases = new Map(folderAliasIndex);- contentFilter?.refreshInPlaceSkillDirs();- const observed = await observeDisk(contentDir, contentFilter);+ let observed: DiskObservation;+ if (handed) {+ observed = handed.observed;+ } else {+ contentFilter?.refreshInPlaceSkillDirs();+ observed = await observeDisk(contentDir, contentFilter, reuse);+ }if (!observed.complete) return { completed: false, candidates: 0 };+ const changedDuringWalk = handed ? changedSince(handed.sinceEpoch, contentDir) : undefined;+ const intakeOwns = (path: string): boolean => changedDuringWalk?.(path) === true;const markdownPublicMutationFor = (name: string, entry: FileIndexEntry): boolean => {const names = publicMutations.markdownNames;+ if (names.size === 0) return false;if (names.has(name)) return true;const canonicalName = pathToDocName(entry.canonicalPath, contentDir);return names.has(canonicalName) || entry.aliases.some((alias) => names.has(alias));};const previousGeneralFacts = generalPathFacts(previousGeneralFiles);const observedGeneralFacts = generalPathFacts(observed.generalFileIndex);+ const unreadableAt = (path: string): boolean =>+ (observed.unreadable.size > 0 && isUnderUnreadable(observed, path)) || intakeOwns(path);+ const protectedGeneral = protectedGeneralNames(+ observed,+ contentDir,+ previousGeneralFacts,+ observedGeneralFacts,+ changedDuringWalk,+ );const rawEvents: RecoveredFileEvent[] = [];for (const [docName, previous] of previousFiles) {@@ -3116,6 +3417,10 @@ async function recoverDiskChanges((current && markdownPublicMutationFor(docName, current)))continue;+ if (current === undefined && unreadableAt(previous.canonicalPath)) continue;+ if (intakeOwns(previous.canonicalPath) || (current && intakeOwns(current.canonicalPath))) {+ continue;+ }if (current?.canonicalPath !== previous.canonicalPath) {const removal: KnownDeletion = {type: 'delete',@@ -3138,7 +3443,12 @@ async function recoverDiskChanges(}}for (const [name, previous] of previousGeneralFacts) {- if (previous.kind === 'symlink' || publicMutations.generalEffects.has(name)) continue;+ if (+ previous.kind === 'symlink' ||+ publicMutations.generalEffects.has(name) ||+ protectedGeneral.has(name)+ )+ continue;const current = observedGeneralFacts.get(name);if (current === undefined) {rawEvents.push({ type: 'delete', path: join(contentDir, name), entryKind: 'file' });@@ -3157,7 +3467,9 @@ async function recoverDiskChanges(const previous = previousFiles.get(docName);if (markdownPublicMutationFor(docName, current) ||- (previous && markdownPublicMutationFor(docName, previous))+ (previous && markdownPublicMutationFor(docName, previous)) ||+ intakeOwns(current.canonicalPath) ||+ (previous && intakeOwns(previous.canonicalPath)))continue;if (current.canonicalPath !== previous?.canonicalPath) {@@ -3165,7 +3477,12 @@ async function recoverDiskChanges(}}for (const [name, current] of observedGeneralFacts) {- if (current.kind !== 'regular' || publicMutations.generalEffects.has(name)) continue;+ if (+ current.kind !== 'regular' ||+ publicMutations.generalEffects.has(name) ||+ protectedGeneral.has(name)+ )+ continue;const previous = previousGeneralFacts.get(name);if (previous === undefined) {rawEvents.push({ type: 'create', path: join(contentDir, name) });@@ -3175,17 +3492,25 @@ async function recoverDiskChanges(}for (const [relativePath, previous] of folderIndex) {if (!observed.folderIndex.has(relativePath)) {+ if (unreadableAt(join(contentDir, relativePath))) continue;rawEvents.push({ type: 'delete', path: join(contentDir, relativePath), entryKind: 'folder' });- } else if (observed.folderIndex.get(relativePath)?.canonicalPath !== previous.canonicalPath) {+ } else if (+ observed.folderIndex.get(relativePath)?.canonicalPath !== previous.canonicalPath &&+ !intakeOwns(join(contentDir, relativePath))+ ) {rawEvents.push({ type: 'update', path: join(contentDir, relativePath) });}}for (const [relativePath, current] of observed.folderIndex) {- if (!folderIndex.has(relativePath))+ if (+ !folderIndex.has(relativePath) &&+ !intakeOwns(join(contentDir, relativePath)) &&+ !intakeOwns(current.canonicalPath)+ )rawEvents.push({ type: 'create', path: current.canonicalPath });}for (const [alias, path] of aliasPaths) {- if (!observed.aliasPaths.has(alias))+ if (!observed.aliasPaths.has(alias) && !unreadableAt(path))rawEvents.push({ type: 'delete', path, entryKind: 'file' });}@@ -3240,21 +3565,26 @@ async function recoverDiskChanges();}+ const publishedDuringPass = (path: string): boolean =>+ publicMutations.markdownNames.size > 0 &&+ publicMutations.markdownNames.has(pathToDocName(path, contentDir));for (const [path, prior] of previousHashes) {if (- isWithinContentDir(path, contentDir) &&- !publicMutations.markdownNames.has(pathToDocName(path, contentDir)) &&!observed.hashes.has(path) &&- lastKnownHash.get(path) === prior+ lastKnownHash.get(path) === prior &&+ !unreadableAt(path) &&+ isWithinContentDir(path, contentDir) &&+ !publishedDuringPass(path)) {- lastKnownHash.delete(path);+ removeLastKnownHash(path);}}for (const [path, hash] of observed.hashes) {- if (publicMutations.markdownNames.has(pathToDocName(path, contentDir))) continue;+ if (publishedDuringPass(path) || intakeOwns(path)) continue;const current = lastKnownHash.get(path);- if (current === undefined || current === previousHashes.get(path))- lastKnownHash.set(path, hash);+ if (current === undefined || current === previousHashes.get(path) || current === hash) {+ recordObservedHash(path, hash, observed.signatures.get(path));+ }}for (const [name, prior] of previousFiles) {@@ -3262,7 +3592,8 @@ async function recoverDiskChanges(if (!markdownPublicMutationFor(name, prior) &&!observed.fileIndex.has(name) &&- current === prior+ current === prior &&+ !unreadableAt(prior.canonicalPath)) {fileIndex.delete(name);forgetDocExtension(name);@@ -3271,10 +3602,17 @@ async function recoverDiskChanges(const inventory = selectObservedInventory(observed, contentDir, contentFilter);for (const [name, entry] of observed.fileIndex) {if (markdownPublicMutationFor(name, entry)) continue;+ const live = fileIndex.get(name);+ if (intakeOwns(entry.canonicalPath) || (live && intakeOwns(live.canonicalPath))) continue;if (inventory.fileIndex.has(name)) fileIndex.set(name, entry);else if (fileIndex.get(name)?.canonicalPath === entry.canonicalPath) fileIndex.delete(name);}const selectedGeneralFacts = inventory.generalFacts;+ for (const name of protectedGeneral) {+ const fact = previousGeneralFacts.get(name);+ if (fact) selectedGeneralFacts.set(name, fact);+ else selectedGeneralFacts.delete(name);+ }for (const [path, effect] of publicMutations.generalEffects) {if (effect) selectedGeneralFacts.set(path, effect);else selectedGeneralFacts.delete(path);@@ -3288,10 +3626,15 @@ async function recoverDiskChanges();for (const [path, prior] of previousFolders) {- if (!observed.folderIndex.has(path) && folderIndex.get(path) === prior)+ if (+ !observed.folderIndex.has(path) &&+ folderIndex.get(path) === prior &&+ !unreadableAt(join(contentDir, path))+ )folderIndex.delete(path);}for (const [path, entry] of inventory.folderIndex) {+ if (intakeOwns(join(contentDir, path))) continue;const current = folderIndex.get(path);if (current === undefined ||@@ -3304,28 +3647,38 @@ async function recoverDiskChanges(}}for (const [alias, prior] of previousAliases) {+ const aliasPath = previousAliasPaths.get(alias);+ if (aliasPath !== undefined && unreadableAt(aliasPath)) continue;if (!observed.aliasMap.has(alias) && aliasMap.get(alias) === prior) aliasMap.delete(alias);}for (const [alias, canonical] of observed.aliasMap) {+ const aliasPath = observed.aliasPaths.get(alias) ?? previousAliasPaths.get(alias);+ if (aliasPath !== undefined && intakeOwns(aliasPath)) continue;const current = aliasMap.get(alias);if (current === undefined || current === previousAliases.get(alias))aliasMap.set(alias, canonical);}for (const [alias, prior] of previousAliasPaths) {- if (!observed.aliasPaths.has(alias) && aliasPaths.get(alias) === prior)+ if (!observed.aliasPaths.has(alias) && aliasPaths.get(alias) === prior && !unreadableAt(prior))aliasPaths.delete(alias);}for (const [alias, path] of observed.aliasPaths) {+ if (intakeOwns(path)) continue;const current = aliasPaths.get(alias);if (current === undefined || current === previousAliasPaths.get(alias))aliasPaths.set(alias, path);}for (const [alias, prior] of previousFolderAliases) {- if (!observed.folderAliasIndex.has(alias) && folderAliasIndex.get(alias) === prior) {+ if (+ !observed.folderAliasIndex.has(alias) &&+ folderAliasIndex.get(alias) === prior &&+ !unreadableAt(join(contentDir, alias))+ ) {folderAliasIndex.delete(alias);}}for (const [alias, canonical] of inventory.folderAliasIndex) {+ if (intakeOwns(join(contentDir, alias))) continue;const current = folderAliasIndex.get(alias);if (current === undefined || current === previousFolderAliases.get(alias)) {folderAliasIndex.set(alias, canonical);@@ -3456,8 +3809,16 @@ async function recoverDiskChanges(await countingOnDiskEvent(event);}}- onRawBatch?.([...new Set([...observed.paths, ...knownPaths])]);- return { completed: true, candidates: deduplicated.length };+ if (handed) {+ if (deduplicated.length > 0) onRawBatch?.(deduplicated.map((event) => event.path));+ } else {+ onRawBatch?.([...new Set([...observed.paths, ...knownPaths])]);+ }+ return {+ completed: true,+ candidates: deduplicated.length,+ partial: observed.unreadable.size > 0,+ };}export function toParcelIgnorePaths(@@ -3484,6 +3845,8 @@ interface RecoveryDriverDeps {recover: (mutations: RecoveryPublicMutations,counter: DispatchCounter,+ mode: RecoveryMode,+ handed: HandedObservation | undefined,) => Promise<RecoveryOutcome>;setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void;onAfterMutation: () => void;@@ -3491,15 +3854,24 @@ interface RecoveryDriverDeps {backendName: WatcherBackend;}-function createRecoveryDriver(deps: RecoveryDriverDeps): {- requestRecovery: () => Promise<RecheckResult>;-} {+interface RecoveryDriver {+ requestRecovery: (mode: RecoveryMode, handed?: HandedObservation) => Promise<RecheckResult>;+ isIdle: () => boolean;+}++function createRecoveryDriver(deps: RecoveryDriverDeps): RecoveryDriver {let recoveryRunning = false;let nextRecovery: Promise<RecheckResult> | undefined;+ let nextMode: RecoveryMode = 'metadata';+ let nextHanded: HandedObservation | undefined;let followUpRecovery: PromiseWithResolvers<RecheckResult> | undefined;const closed = (): RecheckResult => ({ status: 'closed', candidates: 0, dispatched: 0 });const recoverOnce = async (): Promise<RecheckResult> => {+ const mode = nextMode;+ const handed = nextHanded;nextRecovery = undefined;+ nextMode = 'metadata';+ nextHanded = undefined;if (deps.isClosed()) return closed();recoveryRunning = true;const recoveryMutations: RecoveryPublicMutations = {@@ -3513,19 +3885,26 @@ function createRecoveryDriver(deps: RecoveryDriverDeps): {let result: RecheckResult = { status: 'failed', candidates: 0, dispatched: 0 };try {deps.setActivePublicMutations(recoveryMutations);- const outcome = await deps.recover(recoveryMutations, counter);+ const outcome = await deps.recover(recoveryMutations, counter, mode, handed);deps.onAfterMutation();if (outcome.completed) {- await deps.onRecoveryComplete?.();+ if (!outcome.partial) await deps.onRecoveryComplete?.();result = {status: 'completed',candidates: outcome.candidates,dispatched: counter.dispatched,};- log.info(- { candidates: outcome.candidates, dispatched: counter.dispatched },- `${deps.backendName} watcher recovery finished`,- );+ const finished = {+ mode: handed ? 'timed' : mode,+ candidates: outcome.candidates,+ dispatched: counter.dispatched,+ partial: outcome.partial === true,+ };+ if (mode === 'full' || counter.dispatched > 0) {+ log.info(finished, `${deps.backendName} watcher recovery finished`);+ } else {+ log.debug(finished, `${deps.backendName} watcher recovery finished`);+ }} else {result = { status: 'incomplete', candidates: 0, dispatched: counter.dispatched };log.warn(@@ -3560,8 +3939,16 @@ function createRecoveryDriver(deps: RecoveryDriverDeps): {}).then(() => settled);};- const requestRecovery = (): Promise<RecheckResult> => {- if (nextRecovery) return nextRecovery;+ const requestRecovery = (+ mode: RecoveryMode,+ handed?: HandedObservation,+ ): Promise<RecheckResult> => {+ if (mode === 'full') nextMode = 'full';+ if (nextRecovery) {+ nextHanded = undefined;+ return nextRecovery;+ }+ nextHanded = mode === 'full' ? undefined : handed;if (recoveryRunning) {followUpRecovery = Promise.withResolvers<RecheckResult>();nextRecovery = followUpRecovery.promise;@@ -3570,7 +3957,14 @@ function createRecoveryDriver(deps: RecoveryDriverDeps): {}return nextRecovery;};- return { requestRecovery };+ return { requestRecovery, isIdle: () => !recoveryRunning && nextRecovery === undefined };+}++interface WatcherBackendSubscription extends AsyncSubscription {+ runOrdered(work: () => Promise<void>): Promise<void>;+ requestRecovery(mode: RecoveryMode, handed?: HandedObservation): Promise<RecheckResult>;+ isRecoveryIdle(): boolean;+ noticeRootMissing(): void;}async function startParcelWatcher(@@ -3592,14 +3986,9 @@ async function startParcelWatcher(onRecoveryComplete?: () => Promise<void>,clock: WatcherClock = watcherClock(),nativeBackend?: NativeWatcherBackend,-): Promise<- | (AsyncSubscription & {- runOrdered(work: () => Promise<void>): Promise<void>;- requestRecovery(): Promise<RecheckResult>;- settle(): Promise<void>;- })- | null-> {+ carryOver: Set<string> = new Set(),+ coverageWhileLost: string = coverageWithoutWatcher(null),+): Promise<(WatcherBackendSubscription & { settle(): Promise<void> }) | null> {let parcel: NativeWatcherBackend;if (nativeBackend) {parcel = nativeBackend;@@ -3616,14 +4005,15 @@ async function startParcelWatcher(}try {- const subscribeOpts = contentFilter- ? {- ignore: toParcelIgnorePaths(- contentFilter.getWatcherIgnoreGlobs(),- discoveredStructuralDirs ?? [],- ),- }- : undefined;+ const subscribeOpts = () =>+ contentFilter+ ? {+ ignore: toParcelIgnorePaths(+ contentFilter.getWatcherIgnoreGlobs(),+ discoveredStructuralDirs ?? [],+ ),+ }+ : undefined;let closed = false;let failure: Error | undefined;@@ -3642,10 +4032,10 @@ async function startParcelWatcher(});return next;};- const { requestRecovery } = createRecoveryDriver({+ const { requestRecovery, isIdle } = createRecoveryDriver({runOrdered,isClosed: () => closed,- recover: (recoveryMutations, counter) =>+ recover: (recoveryMutations, counter, mode, handed) =>recoverDiskChanges(contentDir,contentFilter,@@ -3661,24 +4051,27 @@ async function startParcelWatcher(getFileIndexGeneration,counter,onRawBatch,+ handed ? undefined : beginFreshnessReuse(mode, fileIndex, carryOver),+ handed,),setActivePublicMutations,onAfterMutation,onRecoveryComplete,backendName: 'parcel',});- const subscription = await subscribeParcel(- parcel,- contentDir,+ const notificationsFor =+ (generation: number): Parameters<typeof subscribeParcel>[2] =>(err, events) => {- if (closed) return;+ if (closed || !resubscriber.isLive(generation)) return;const notification = classifyParcelNotification(err, events);if (notification.kind === 'error') {log.error({ err: notification.error }, 'parcel watcher callback error');failure = new Error(- `the file watcher failed before it saw its settle fence: ${notification.error.message}`,+ `the file watcher is subscribing again after an error and cannot settle until it is back: ${notification.error.message}`,);for (const fence of settleFences.values()) fence.reject(failure);+ incrementWatcherErrors();+ resubscriber.lost('error', generation, notification.error);return;}if (notification.kind === 'rescan') {@@ -3710,22 +4103,52 @@ async function startParcelWatcher(for (const event of notification.events)settleFences.get(basename(event.path))?.resolve();});+ if (+ notification.events.some((event) => event.type === 'delete' && event.path === contentDir)+ ) {+ resubscriber.lost('root-delete', generation);+ }if (notification.kind !== 'rescan') return records;const fencesBeforeRescan = [...settleFences.values()];return Promise.all([records,- requestRecovery().then(() => {+ requestRecovery('full').then(() => {for (const fence of fencesBeforeRescan) fence.resolve();}),]);+ };+ const resubscriber = createResubscriber({+ clock,+ root: contentDir,+ backendName: 'parcel',+ coverageWhileLost,+ onGaveUp: (err) => {+ const message = err instanceof Error ? err.message : String(err);+ failure = new Error(+ `the file watcher gave up subscribing after repeated failures (${message}); it tries again every ${RESUBSCRIBE_SLOW_RETRY_MS / 60_000} minutes and cannot settle until it is back`,+ );+ for (const fence of settleFences.values()) fence.reject(failure);},- subscribeOpts,- platform,- );+ subscribe: (generation) =>+ subscribeParcel(+ parcel,+ contentDir,+ notificationsFor(generation),+ subscribeOpts(),+ platform,+ ),+ onResubscribed: () => {+ failure = undefined;+ void requestRecovery('full');+ },+ });+ await resubscriber.start();return {runOrdered,requestRecovery,+ isRecoveryIdle: isIdle,+ noticeRootMissing: () => resubscriber.noticeRootMissing(),async settle() {if (closed) return;if (failure) throw failure;@@ -3764,7 +4187,7 @@ async function startParcelWatcher(for (const fence of settleFences.values()) {fence.reject(new Error('the file watcher stopped before it saw its settle fence'));}- await subscription.unsubscribe();+ await resubscriber.close();await workTail;},};@@ -3821,6 +4244,14 @@ function waitForPendingFsEventsRegistrations(dir: string, platform: NodeJS.Platf}}+const CHOKIDAR_NON_FATAL_ERROR_CODES: ReadonlySet<string> = new Set([+ 'EACCES',+ 'EPERM',+ 'ENOSPC',+ 'EMFILE',+ 'ENFILE',+]);+async function startChokidarWatcher(contentDir: string,contentFilter: ContentFilter | undefined,@@ -3838,20 +4269,11 @@ async function startChokidarWatcher(clock: WatcherClock,onRawBatch?: (absPaths: readonly string[]) => void,onRecoveryComplete?: () => Promise<void>,-): Promise<AsyncSubscription & { requestRecovery(): Promise<RecheckResult> }> {+ carryOver: Set<string> = new Set(),+ coverageWhileLost: string = coverageWithoutWatcher(null),+): Promise<WatcherBackendSubscription> {const { watch } = await import('chokidar');- const watcher = watch(contentDir, {- ignoreInitial: true,- followSymlinks: false,- ignored: contentFilter- ? (filePath: string, stats?: Stats) =>- isChokidarPathIgnored(contentDir, contentFilter, filePath, stats)- : undefined,- });-- watcher.on('error', (err) => log.error({ err }, 'chokidar error'));-let closed = false;let workTail = Promise.resolve();const runOrdered = (work: () => Promise<void>): Promise<void> => {@@ -3862,10 +4284,10 @@ async function startChokidarWatcher(});return next;};- const { requestRecovery } = createRecoveryDriver({+ const { requestRecovery, isIdle } = createRecoveryDriver({runOrdered,isClosed: () => closed,- recover: (recoveryMutations, counter) =>+ recover: (recoveryMutations, counter, mode, handed) =>recoverDiskChanges(contentDir,contentFilter,@@ -3881,6 +4303,8 @@ async function startChokidarWatcher(getFileIndexGeneration,counter,onRawBatch,+ handed ? undefined : beginFreshnessReuse(mode, fileIndex, carryOver),+ handed,),setActivePublicMutations,onAfterMutation,@@ -3920,27 +4344,78 @@ async function startChokidarWatcher(}, CHOKIDAR_BATCH_WINDOW_MS);}- watcher.on('add', (path) => queueEvent({ type: 'create', path }));- watcher.on('change', (path) => queueEvent({ type: 'update', path }));- watcher.on('unlink', (path) => queueEvent({ type: 'delete', path, entryKind: 'file' }));- watcher.on('addDir', (path) => queueEvent({ type: 'create', path }));- watcher.on('unlinkDir', (path) => queueEvent({ type: 'delete', path, entryKind: 'folder' }));-- await new Promise<void>((resolveReady) => {- let settled = false;- const settle = () => {- if (settled) return;- settled = true;- resolveReady();+ const reportedNonFatalCodes = new Set<string>();+ const watchAs = async (generation: number): Promise<AsyncSubscription> => {+ const watcher = watch(contentDir, {+ ignoreInitial: true,+ followSymlinks: false,+ ignored: contentFilter+ ? (filePath: string, stats?: Stats) =>+ isChokidarPathIgnored(contentDir, contentFilter, filePath, stats)+ : undefined,+ });+ const live = (): boolean => !closed && resubscriber.isLive(generation);+ const queueLive = (event: InternalRawFileEvent): void => {+ if (live()) queueEvent(event);};- watcher.once('ready', settle);- clock.setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();- });- waitForPendingFsEventsRegistrations(contentDir, platform);+ watcher.on('error', (err) => {+ if (!live()) return;+ incrementWatcherErrors();+ const code = errnoCode(err);+ if (code !== undefined && CHOKIDAR_NON_FATAL_ERROR_CODES.has(code)) {+ if (reportedNonFatalCodes.has(code)) return;+ reportedNonFatalCodes.add(code);+ log.warn(+ { err, code },+ `chokidar reported ${code}; the watcher stays up, and for what it cannot watch ${coverageWhileLost}`,+ );+ return;+ }+ log.error({ err }, 'chokidar error');+ resubscriber.lost('error', generation, err);+ });+ watcher.on('add', (path) => queueLive({ type: 'create', path }));+ watcher.on('change', (path) => queueLive({ type: 'update', path }));+ watcher.on('unlink', (path) => queueLive({ type: 'delete', path, entryKind: 'file' }));+ watcher.on('addDir', (path) => queueLive({ type: 'create', path }));+ watcher.on('unlinkDir', (path) => {+ queueLive({ type: 'delete', path, entryKind: 'folder' });+ if (path === contentDir && live()) resubscriber.lost('root-delete', generation);+ });++ await new Promise<void>((resolveReady) => {+ let settled = false;+ const settle = () => {+ if (settled) return;+ settled = true;+ resolveReady();+ };+ watcher.once('ready', settle);+ clock.setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();+ });++ waitForPendingFsEventsRegistrations(contentDir, platform);+ return { unsubscribe: () => watcher.close() };+ };++ const resubscriber = createResubscriber({+ clock,+ root: contentDir,+ backendName: 'chokidar',+ coverageWhileLost,+ subscribe: watchAs,+ onResubscribed: () => {+ void requestRecovery('full');+ },+ });+ await resubscriber.start();return {+ runOrdered,requestRecovery,+ isRecoveryIdle: isIdle,+ noticeRootMissing: () => resubscriber.noticeRootMissing(),unsubscribe: async () => {closed = true;if (batchTimer) {@@ -3948,12 +4423,21 @@ async function startChokidarWatcher(batchTimer = null;pendingEvents = [];}- await watcher.close();+ await resubscriber.close();await workTail;},};}+function coverageWithoutWatcher(freshnessIntervalMs: number | null): string {+ if (freshnessIntervalMs === null) {+ return 'changes made outside the app are found only by an explicit re-check';+ }+ const fastest = freshnessIntervalMs / 1000;+ const slowest = Math.max(freshnessIntervalMs, FRESHNESS_MAX_INTERVAL_MS) / 1000;+ return `the timed re-check still finds changes made outside the app, every ${fastest} to ${slowest} seconds`;+}+export interface StartWatcherOptions {forceBackend?: 'parcel' | 'chokidar';onRawBatch?: (absPaths: readonly string[]) => void;@@ -3961,6 +4445,7 @@ export interface StartWatcherOptions {platform?: NodeJS.Platform;nativeBackend?: NativeWatcherBackend;clock?: WatcherClock;+ freshnessIntervalMs?: number | null;}export async function startWatcher(@@ -4044,8 +4529,10 @@ async function startWatcherWithClock(WATCHER_DROP_SUMMARY_INTERVAL_MS,);- let subscription: AsyncSubscription;- let requestRecovery: () => Promise<RecheckResult>;+ const freshnessIntervalMs = resolveFreshnessInterval(opts.freshnessIntervalMs);+ const coverageWhileLost = coverageWithoutWatcher(freshnessIntervalMs);+ const carryOver = new Set<string>();+ let subscription: WatcherBackendSubscription;let runOrderedRescan: ((work: () => Promise<void>) => Promise<void>) | undefined;let settleSubscription: (() => Promise<void>) | undefined;let backend: WatcherBackend;@@ -4076,10 +4563,11 @@ async function startWatcherWithClock(onRecoveryComplete,clock,opts.nativeBackend,+ carryOver,+ coverageWhileLost,);if (parcelSub) {subscription = parcelSub;- requestRecovery = parcelSub.requestRecovery;runOrderedRescan = parcelSub.runOrdered;settleSubscription = () => parcelSub.settle();backend = 'parcel';@@ -4106,9 +4594,10 @@ async function startWatcherWithClock(clock,onRawBatch,onRecoveryComplete,+ carryOver,+ coverageWhileLost,);subscription = chokidarSub;- requestRecovery = chokidarSub.requestRecovery;backend = 'chokidar';}} catch (e) {@@ -4117,26 +4606,178 @@ async function startWatcherWithClock(throw e;}- const originalUnsubscribe = subscription.unsubscribe.bind(subscription);+ const backendSubscription = subscription;+ let handleClosed = false;+ let freshnessTimer: WatcherTimer | null = null;+ let freshnessPassInFlight: Promise<void> | null = null;+ const scheduleFreshnessPass = (baseMs: number, delayMs: number): void => {+ if (handleClosed) return;+ freshnessTimer = clock.setTimeout(() => {+ freshnessTimer = null;+ const pass = runFreshnessPass(baseMs);+ freshnessPassInFlight = pass;+ void pass.finally(() => {+ if (freshnessPassInFlight === pass) freshnessPassInFlight = null;+ });+ }, delayMs);+ freshnessTimer.unref();+ };+ const walkForFreshness = async (): Promise<DiskObservation | null> => {+ contentFilter?.refreshInPlaceSkillDirs();+ const reuse = beginFreshnessReuse('metadata', fileIndex, carryOver);+ const observed = await observeDisk(contentDir, contentFilter, reuse, () => handleClosed);+ if (handleClosed) return null;+ if (reuse) {+ addFreshnessFilesHashed(reuse.hashed);+ addFreshnessCarryOver(reuse.deferred.size);+ }+ return observed;+ };+ const runFreshnessPass = async (baseMs: number): Promise<void> => {+ if (handleClosed) return;+ if (!existsSync(contentDir)) {+ backendSubscription.noticeRootMissing();+ scheduleFreshnessPass(baseMs, baseMs);+ return;+ }+ if (!backendSubscription.isRecoveryIdle()) {+ scheduleFreshnessPass(baseMs, baseMs);+ return;+ }+ const startedAt = clock.now();+ let elapsedMs = 0;+ const sinceEpoch = openObservationWindow();+ try {+ const observed = await walkForFreshness();+ if (observed === null) return;+ const result = observed.complete+ ? await backendSubscription.requestRecovery('metadata', { observed, sinceEpoch })+ : { status: 'incomplete' as const, candidates: 0, dispatched: 0 };+ if (handleClosed) return;+ elapsedMs = clock.now() - startedAt;+ addFreshnessFinds(result.dispatched);+ if (result.status === 'incomplete' || result.status === 'failed') {+ incrementFreshnessPassesIncomplete();+ }+ log.debug({ ...result, elapsedMs }, 'freshness re-check pass finished');+ } catch (err) {+ if (handleClosed) return;+ incrementFreshnessPassesIncomplete();+ log.warn({ err }, 'freshness re-check pass failed');+ } finally {+ closeObservationWindow();+ if (!handleClosed) {+ incrementFreshnessPasses();+ scheduleFreshnessPass(baseMs, nextFreshnessDelayMs(elapsedMs, baseMs));+ }+ }+ };+ if (freshnessIntervalMs !== null) scheduleFreshnessPass(freshnessIntervalMs, freshnessIntervalMs);- log.info({ contentDir, backend }, 'watching for external .md changes');+ log.info({ contentDir, backend, freshnessIntervalMs }, 'watching for external .md changes');return {async unsubscribe() {+ handleClosed = true;+ clock.clearTimeout(freshnessTimer);+ freshnessTimer = null;clock.clearInterval(evictionInterval);clock.clearInterval(dropSummaryInterval);try {- await originalUnsubscribe();+ await backendSubscription.unsubscribe();} finally {+ await freshnessPassInFlight;writeTracker.clear();removalTracker.clear();lastKnownHash.clear();+ lastKnownStat.clear();+ reportedUnreadable.clear();resetWatcherDecisionDiagnostics();restoreClock();}},recheck() {- return requestRecovery();+ return backendSubscription.requestRecovery('full');+ },+ async recheckPaths(paths) {+ const unique = [...new Set(paths)];+ if (unique.length === 0) return { checked: 0, found: 0 };+ let found = 0;+ let checked = unique.length;+ const rootMissing = (): boolean => {+ if (existsSync(contentDir)) return false;+ checked = 0;+ found = 0;+ log.warn(+ { contentDir, paths: unique.length },+ `skipped re-reading ${unique.length} documents: the content folder ${contentDir} is missing`,+ );+ backendSubscription.noticeRootMissing();+ return true;+ };+ await backendSubscription.runOrdered(async () => {+ if (rootMissing()) return;+ const indexedPaths = new Set<string>();+ for (const entry of fileIndex.values()) indexedPaths.add(entry.canonicalPath);+ const changed: InternalRawFileEvent[] = [];+ for (const path of unique) {+ const known = lastKnownHash.get(path);+ const tracked = known !== undefined || indexedPaths.has(path);+ try {+ lstatSync(path);+ } catch (err) {+ const code = errnoCode(err);+ if (code === 'ENOENT' || code === 'ENOTDIR') {+ if (rootMissing()) return;+ if (tracked) changed.push({ type: 'delete', path, entryKind: 'file' });+ } else {+ log.warn({ err, path, code }, `could not re-read ${path}`);+ }+ continue;+ }+ if (+ !hasAdmittedMarkdownFile(+ pathToDocName(path, contentDir),+ path,+ contentDir,+ contentFilter,+ )+ ) {+ log.debug({ path }, `skipped a re-read of ${path}: not an admitted Markdown file`);+ continue;+ }+ let content: string;+ try {+ if (statSync(path).size > DOCUMENT_OPEN_BYTE_LIMIT) {+ log.debug({ path }, `skipped a re-read of ${path}: larger than an open document`);+ continue;+ }+ content = await readFile(path, 'utf-8');+ } catch (err) {+ log.warn({ err, path }, `could not re-read ${path}`);+ continue;+ }+ if (contentHash(content) === known) continue;+ changed.push({ type: tracked ? 'update' : 'create', path });+ }+ found = changed.length;+ if (changed.length === 0) return;+ onRawBatch?.(changed.map((event) => event.path));+ await handleRawEventsInternal(+ changed,+ contentDir,+ contentFilter,+ fileIndex,+ folderIndex,+ onDiskEvent,+ aliasMap,+ undefined,+ generalFileIndex,+ aliasPaths,+ );+ bumpFileIndexGeneration();+ });+ return { checked, found };},getFileIndex() {if (cachedMarkdownView && cachedMarkdownViewGeneration === fileIndexGeneration) {@@ -4166,6 +4807,12 @@ async function startWatcherWithClock(},mutateFileIndex(reportedEvent) {const event = declaredDiskEvent(reportedEvent);+ if (event.kind === 'rename') {+ noteIntakeMutation(event.oldPath);+ noteIntakeMutation(event.newPath);+ } else {+ noteIntakeMutation(event.path);+ }if (event.kind === 'rename') {activePublicMutations?.markdownNames.add(event.oldDocName);activePublicMutations?.markdownNames.add(event.newDocName);@@ -4296,6 +4943,7 @@ async function startWatcherWithClock(},pruneFileIndexNowExcluded() {if (!contentFilter) return 0;+ noteIntakeMutation(contentDir);let pruned = 0;let changed = false;const removedGeneralPaths = new Set<string>();@@ -4361,6 +5009,7 @@ async function startWatcherWithClock(},pruneFolderIndexNowExcluded() {if (!contentFilter) return 0;+ noteIntakeMutation(contentDir);let pruned = 0;let aliasesChanged = false;for (const folderPath of folderIndex.keys()) {@@ -4388,6 +5037,7 @@ async function startWatcherWithClock(},async rescanFromDisk() {const reseed = async () => {+ noteIntakeMutation(contentDir);const mutations: GeneralMutationRecord = {generalNames: new Set(),generalEffects: new Map(),@@ -4426,6 +5076,7 @@ async function startWatcherWithClock();for (const [alias, path] of observation.aliasPaths) aliasPaths.set(alias, path);bumpFileIndexGeneration();+ noteIntakeMutation(contentDir);} finally {activeReseedMutations = null;}diff --git a/packages/server/src/head-watcher.ts b/packages/server/src/head-watcher.tsindex 792c77795..01478841e 100644--- a/packages/server/src/head-watcher.ts+++ b/packages/server/src/head-watcher.ts@@ -3,9 +3,11 @@ import {type GitRepository,} from '@inkeep/open-knowledge-core/git-repository';import { getLogger } from './logger.ts';+import { incrementWatcherErrors } from './metrics.ts';import { classifyParcelNotification } from './parcel-recovery.ts';import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';import { type WatcherClock, type WatcherTimer, watcherClock } from './watcher-clock.ts';+import { createResubscriber } from './watcher-freshness.ts';const log = getLogger('head-watcher');@@ -28,6 +30,11 @@ interface BatchBeginInfo {type OnBatchBegin = (info: BatchBeginInfo) => void | Promise<void>;type OnBatchEnd = (info: BatchEndInfo) => void | Promise<void>;+interface HeadWatcherCoverage {+ readonly onGaveUp?: (err: unknown) => void;+ readonly onBack?: () => void;+}+export interface HeadWatcherHandle {unsubscribe: () => Promise<void>;getLastKnownBranch: () => string | null;@@ -74,6 +81,8 @@ async function tryStartParcelHeadWatcher(dispatch: (rawPath: string) => boolean,recover: () => void,platform: NodeJS.Platform,+ clock: WatcherClock,+ coverage: HeadWatcherCoverage,nativeBackend?: NativeWatcherBackend,): Promise<(() => Promise<void>) | null> {let parcel: NativeWatcherBackend;@@ -91,13 +100,15 @@ async function tryStartParcelHeadWatcher(}}try {- const subscription = await subscribeParcel(- parcel,- gitDir,+ const notificationsFor =+ (generation: number): Parameters<typeof subscribeParcel>[2] =>(err, events) => {+ if (!resubscriber.isLive(generation)) return;const notification = classifyParcelNotification(err, events);if (notification.kind === 'error') {log.warn({ err: notification.error }, '[head-watcher] parcel subscription error');+ incrementWatcherErrors();+ resubscriber.lost('error', generation, notification.error);return;}if (notification.kind === 'rescan') {@@ -111,11 +122,21 @@ async function tryStartParcelHeadWatcher(if (dispatch(event.path)) hadWatchedRecord = true;}if (notification.kind === 'rescan' && !hadWatchedRecord) recover();- },- undefined,- platform,- );- return () => subscription.unsubscribe();+ };+ const resubscriber = createResubscriber({+ clock,+ root: gitDir,+ backendName: 'head-parcel',+ coverageWhileLost:+ 'branch switches, pulls and other HEAD moves go unnoticed and the server reports head-watcher as degraded',+ subscribe: (generation) =>+ subscribeParcel(parcel, gitDir, notificationsFor(generation), undefined, platform),+ onResubscribed: recover,+ onGaveUp: coverage.onGaveUp,+ onBack: coverage.onBack,+ });+ await resubscriber.start();+ return () => resubscriber.close();} catch (err) {log.debug({ err: err instanceof Error ? err.message : String(err) },@@ -151,6 +172,8 @@ export async function startHeadWatcher(platform?: NodeJS.Platform;nativeBackend?: NativeWatcherBackend;clock?: WatcherClock;+ onGaveUp?: (err: unknown) => void;+ onBack?: () => void;subscribeForTest?: (gitDir: string,dispatch: HeadEventDispatch,@@ -304,6 +327,8 @@ export async function startHeadWatcher(dispatch,recover,opts.platform ?? process.platform,+ clock,+ { onGaveUp: opts.onGaveUp, onBack: opts.onBack },opts.nativeBackend,);if (resolvedUnsub) backend = 'parcel';diff --git a/packages/server/src/http/document-refresh-routes.test.ts b/packages/server/src/http/document-refresh-routes.test.tsnew file mode 100644index 000000000..89eef8bfc--- /dev/null+++ b/packages/server/src/http/document-refresh-routes.test.ts@@ -0,0 +1,69 @@+import { IncomingMessage } from 'node:http';+import { Socket } from 'node:net';+import { describe, expect, test, vi } from 'vitest';+import { makeCaptureRes, parseProblem } from '../composition-rig.test-helper.ts';+import type { RefreshDocumentsFromDisk } from '../document-refresh.ts';+import { buildIngressPolicy } from '../ingress-policy.ts';+import { loggerFactory } from '../logger.ts';+import { createApiRequestPipeline } from './api-pipeline.ts';+import { createDocumentRefreshRoutes } from './document-refresh-routes.ts';++const REFRESH_PATH = '/api/documents/refresh';++function makeRequest(body: unknown, method = 'POST'): IncomingMessage {+ const req = new IncomingMessage(new Socket());+ req.push(Buffer.from(JSON.stringify(body)));+ req.push(null);+ req.method = method;+ req.url = REFRESH_PATH;+ req.headers = { host: '127.0.0.1', 'content-type': 'application/json' };+ Object.defineProperty(req.socket, 'remoteAddress', { value: '127.0.0.1' });+ return req;+}++async function dispatch(refresh: RefreshDocumentsFromDisk, body: unknown, method?: string) {+ const group = createDocumentRefreshRoutes({ refreshDocumentsFromDisk: refresh });+ const pipeline = createApiRequestPipeline({+ log: loggerFactory.getLogger('document-refresh-routes-test'),+ policy: buildIngressPolicy({}),+ table: group.table,+ });+ const { res, captured } = makeCaptureRes();+ const handled = await pipeline(makeRequest(body, method), res);+ return { handled, captured };+}++describe('createDocumentRefreshRoutes table', () => {+ test('owns only the refresh path and classifies it as mutating', () => {+ const group = createDocumentRefreshRoutes({+ refreshDocumentsFromDisk: async () => ({ checked: 0, found: 0 }),+ });+ expect(group.paths).toEqual([REFRESH_PATH]);+ expect(group.table.resolve(REFRESH_PATH)?.dispatch).toBeDefined();+ expect(group.table.isMutating(REFRESH_PATH)).toBe(true);+ for (const path of ['/api/documents', '/api/documents/refresh/', '/api/documents/other']) {+ expect(group.table.resolve(path), path).toBeNull();+ expect(group.table.isMutating(path), path).toBe(false);+ }+ });++ test('re-reads the named documents as a focus re-read and answers with what it found', async () => {+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => ({ checked: 2, found: 1 }));+ const { handled, captured } = await dispatch(refresh, { docNames: ['a', 'folder/b'] });++ expect(handled).toBe(true);+ expect(captured.status).toBe(200);+ expect(JSON.parse(captured.body)).toEqual({ checked: 2, found: 1 });+ expect(refresh).toHaveBeenCalledWith(['a', 'folder/b'], 'focus');+ });++ test('a failed re-read answers with a problem and no success body', async () => {+ const refresh = vi.fn<RefreshDocumentsFromDisk>(async () => {+ throw new Error('disk unavailable');+ });+ const { captured } = await dispatch(refresh, { docNames: ['a'] });++ expect(captured.status).toBe(500);+ expect(parseProblem(captured.body).type).toBe('urn:ok:error:internal-server-error');+ });+});diff --git a/packages/server/src/http/document-refresh-routes.ts b/packages/server/src/http/document-refresh-routes.tsnew file mode 100644index 000000000..d315e969d--- /dev/null+++ b/packages/server/src/http/document-refresh-routes.ts@@ -0,0 +1,41 @@+import {+ DocumentRefreshRequestSchema,+ DocumentRefreshSuccessSchema,+} from '@inkeep/open-knowledge-core';+import type { RefreshDocumentsFromDisk } from '../document-refresh.ts';+import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';+import { errorResponse } from './error-response.ts';+import { withValidation } from './request-validation.ts';+import { successResponse } from './success-response.ts';++export interface DocumentRefreshRouteDeps {+ refreshDocumentsFromDisk: RefreshDocumentsFromDisk;+}++export function createDocumentRefreshRoutes(deps: DocumentRefreshRouteDeps): ApiRouteGroup {+ const handleDocumentRefresh = withValidation(+ DocumentRefreshRequestSchema,+ async (_req, res, body) => {+ try {+ const result = await deps.refreshDocumentsFromDisk(body.docNames, 'focus');+ successResponse(res, 200, DocumentRefreshSuccessSchema, result, {+ handler: 'document-refresh',+ });+ } catch (err) {+ errorResponse(+ res,+ 500,+ 'urn:ok:error:internal-server-error',+ 'Failed to re-read the documents from disk.',+ { handler: 'document-refresh', cause: err },+ );+ }+ },+ { handler: 'document-refresh', method: 'POST' },+ );++ return createApiRouteGroup(+ { '/api/documents/refresh': handleDocumentRefresh },+ { mutating: ['/api/documents/refresh'] },+ );+}diff --git a/packages/server/src/http/test-reset-watcher-settle.test.ts b/packages/server/src/http/test-reset-watcher-settle.test.tsindex c5c5baad0..3537fecdb 100644--- a/packages/server/src/http/test-reset-watcher-settle.test.ts+++ b/packages/server/src/http/test-reset-watcher-settle.test.ts@@ -13,7 +13,10 @@ import type { BootedServer } from '../boot.ts';import { bootCompositionRig } from '../composition-rig.test-helper.ts';import {forgetNativeSubscriptions,+ latestNativeSubscriptionOn,+ nativeSubscriptionCount,nativeSubscriptionOn,+ refuseNativeSubscriptions,} from '../parcel-watcher-double.test-helper.ts';vi.mock('@parcel/watcher', async () => {@@ -73,10 +76,30 @@ test('a server-scope reset runs once the watcher has handled its settle fence',expect(fencesIn(contentDir)).toEqual([]);});-test('after a watcher error ends its subscription, a server-scope reset answers with that error and resets nothing', async () => {+test('after a watcher error, a server-scope reset runs on the new subscription once it is live', async () => {const { server, contentDir } = await serverWithTestDoc();const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));+ await expect.poll(() => nativeSubscriptionCount(contentDir)).toBe(2);++ const reset = postServerReset(server);+ await expect.poll(() => fencesIn(contentDir)).toHaveLength(1);+ const [fence] = fencesIn(contentDir);+ await latestNativeSubscriptionOn(contentDir).deliver([+ { type: 'create', path: join(contentDir, '.ok', fence) },+ ]);+ const response = await reset;++ expect(response.status).toBe(200);+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe('');+ expect(fencesIn(contentDir)).toEqual([]);+});++test('while the watcher cannot subscribe again after an error, a server-scope reset answers with that error and resets nothing', async () => {+ const { server, contentDir } = await serverWithTestDoc();+ const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;+ refuseNativeSubscriptions(contentDir, 3);+ await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));const response = await postServerReset(server);@@ -84,8 +107,9 @@ test('after a watcher error ends its subscription, a server-scope reset answersexpect(await response.json()).toMatchObject({type: 'urn:ok:error:internal-server-error',title: 'The server-scope reset did not run, because the file watcher could not settle.',- detail: `the file watcher failed before it saw its settle fence: ${failure}`,+ detail: `the file watcher is subscribing again after an error and cannot settle until it is back: ${failure}`,});+ expect(nativeSubscriptionCount(contentDir)).toBe(1);expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe(KEPT);expect(fencesIn(contentDir)).toEqual([]);});diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.tsindex b2350f5ad..3f0ca425d 100644--- a/packages/server/src/metrics.ts+++ b/packages/server/src/metrics.ts@@ -172,6 +172,16 @@ export interface ReconciliationMetrics {removalRedirectChainCycles: number;authDocLineageMismatchCount: number;authDocLineageGuardErrors: number;+ freshnessPasses: number;+ freshnessPassesIncomplete: number;+ freshnessFinds: number;+ freshnessFilesHashed: number;+ freshnessCarryOver: number;+ watcherErrors: number;+ watcherResubscribes: number;+ watcherRootMissing: number;+ watcherUnreadableEntries: number;+ watcherResubscribesAbandoned: number;}const counters: ReconciliationMetrics = {@@ -294,6 +304,16 @@ const counters: ReconciliationMetrics = {removalRedirectChainCycles: 0,authDocLineageMismatchCount: 0,authDocLineageGuardErrors: 0,+ freshnessPasses: 0,+ freshnessPassesIncomplete: 0,+ freshnessFinds: 0,+ freshnessFilesHashed: 0,+ freshnessCarryOver: 0,+ watcherErrors: 0,+ watcherResubscribes: 0,+ watcherRootMissing: 0,+ watcherUnreadableEntries: 0,+ watcherResubscribesAbandoned: 0,};export function incrementReconcile(): void {@@ -731,6 +751,46 @@ export function incrementAuthDocLineageGuardError(): void {counters.authDocLineageGuardErrors++;}+export function incrementFreshnessPasses(): void {+ counters.freshnessPasses++;+}++export function incrementFreshnessPassesIncomplete(): void {+ counters.freshnessPassesIncomplete++;+}++export function addFreshnessFinds(count: number): void {+ counters.freshnessFinds += count;+}++export function addFreshnessFilesHashed(count: number): void {+ counters.freshnessFilesHashed += count;+}++export function addFreshnessCarryOver(count: number): void {+ counters.freshnessCarryOver += count;+}++export function incrementWatcherErrors(): void {+ counters.watcherErrors++;+}++export function incrementWatcherResubscribes(): void {+ counters.watcherResubscribes++;+}++export function incrementWatcherRootMissing(): void {+ counters.watcherRootMissing++;+}++export function incrementWatcherUnreadableEntries(): void {+ counters.watcherUnreadableEntries++;+}++export function incrementWatcherResubscribesAbandoned(): void {+ counters.watcherResubscribesAbandoned++;+}+export function incrementCollabSocketFilteredError(code: 'EPIPE' | 'ECONNRESET'): void {if (code === 'EPIPE') counters.collabSocketEpipeCount++;else counters.collabSocketEconnresetCount++;@@ -912,4 +972,14 @@ export function resetMetrics(): void {counters.removalRedirectChainCycles = 0;counters.authDocLineageMismatchCount = 0;counters.authDocLineageGuardErrors = 0;+ counters.freshnessPasses = 0;+ counters.freshnessPassesIncomplete = 0;+ counters.freshnessFinds = 0;+ counters.freshnessFilesHashed = 0;+ counters.freshnessCarryOver = 0;+ counters.watcherErrors = 0;+ counters.watcherResubscribes = 0;+ counters.watcherRootMissing = 0;+ counters.watcherUnreadableEntries = 0;+ counters.watcherResubscribesAbandoned = 0;}diff --git a/packages/server/src/parcel-drop-notice.test.ts b/packages/server/src/parcel-drop-notice.test.tsindex e957de48e..1e1feffec 100644--- a/packages/server/src/parcel-drop-notice.test.ts+++ b/packages/server/src/parcel-drop-notice.test.ts@@ -712,7 +712,7 @@ test.each(notices)('empty HEAD notice restores an omitted branch change: %s', asexpectHeadSettled(watched);});-test('fatal Parcel errors remain observable for both watchers', async () => {+test('a fatal Parcel error is logged, and both watchers subscribe again and re-read what changed', async () => {const diagnostics: Array<{ name: string; err: unknown }> = [];for (const name of ['file-watcher', 'head-watcher']) {const logger = getLogger(name);@@ -737,12 +737,15 @@ test('fatal Parcel errors remain observable for both watchers', async () => {for (const name of ['file-watcher', 'head-watcher']) {expect(diagnostics).toContainEqual({ name, err: fatal });}- expect(events).toEqual([]);- expect(rawBatches).toEqual([]);- expect([...watcher.getFileIndex().keys()]).toEqual([]);- expect(watched.begins).toEqual([]);- expect(watched.ends).toEqual([]);- expect(watched.watcher.getLastKnownBranch()).toBe('main');+ expect(rawBatches).not.toContainEqual([announced]);+ await vi.waitFor(() => {+ expect([...watcher.getFileIndex().keys()].sort()).toEqual(['announced', 'hidden']);+ });+ expect(+ events.flatMap((event) => (event.kind === 'create' ? [event.docName] : [])).sort(),+ ).toEqual(['announced', 'hidden']);+ await vi.waitFor(() => expect(watched.ends).toHaveLength(1));+ expectHeadSettled(watched);});test('repeated content notices do not repeat effects already applied from a supplied batch and disk scan', async () => {diff --git a/packages/server/src/parcel-watcher-double.test-helper.ts b/packages/server/src/parcel-watcher-double.test-helper.tsindex 0f89cc683..c67bef534 100644--- a/packages/server/src/parcel-watcher-double.test-helper.ts+++ b/packages/server/src/parcel-watcher-double.test-helper.ts@@ -37,12 +37,19 @@ function register(dir: string, callback: SubscribeCallback): Registration {};}+const refusals = new Map<string, number>();+const parcelWatcherDouble = {subscribe(dir: string,callback: SubscribeCallback,_options?: Options,): Promise<AsyncSubscription> {+ const refused = refusals.get(resolve(dir)) ?? 0;+ if (refused > 0) {+ refusals.set(resolve(dir), refused - 1);+ return Promise.reject(new Error(`the watcher double refused to watch ${dir}`));+ }const registration = register(resolve(dir), callback);registrations.push(registration);return Promise.resolve({ unsubscribe: () => registration.release() });@@ -59,6 +66,21 @@ export const parcelWatcherModule = { default: parcelWatcherDouble, ...parcelWatcexport function forgetNativeSubscriptions(): void {registrations.length = 0;+ refusals.clear();+}++export function refuseNativeSubscriptions(dir: string, count: number): void {+ refusals.set(resolve(dir), count);+}++export function nativeSubscriptionCount(dir: string): number {+ return registrations.filter((registration) => registration.dir === dir).length;+}++export function latestNativeSubscriptionOn(dir: string): NativeSubscriptionDouble {+ const latest = registrations.filter((registration) => registration.dir === dir).at(-1);+ if (!latest) throw new Error(`no native subscription on ${dir}`);+ return latest;}export function nativeSubscriptionDirs(): string[] {diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.tsindex bec310944..a79f8be7f 100644--- a/packages/server/src/server-factory.ts+++ b/packages/server/src/server-factory.ts@@ -145,6 +145,11 @@ import {DocumentDurabilityState,type StaleExternalWriteConflict,} from './document-durability-state.ts';+import {+ createReconnectRefreshExtension,+ type DocumentRefreshResult,+ type DocumentRefreshTrigger,+} from './document-refresh.ts';import {type Embedder,type EmbeddingsKeyStore,@@ -346,7 +351,8 @@ import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extensioimport { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';import { trustSystemCertificates } from './trust-system-ca.ts';import { cleanupOrphanUploadTempfiles } from './upload-streaming.ts';-import type { WatcherClock } from './watcher-clock.ts';+import { type WatcherClock, watcherClock } from './watcher-clock.ts';+import { FRESHNESS_BASE_INTERVAL_MS } from './watcher-freshness.ts';export interface ServerOptions {acpRegistryFetchImpl?: typeof fetch;@@ -395,6 +401,7 @@ export interface ServerOptions {ephemeral?: boolean;watcherBackend?: NativeWatcherBackend;watcherClock?: WatcherClock;+ freshnessIntervalMs?: number | null;mergeHistoryWriter?: MergeHistoryWriter;generatedIndexTestHooks?: {beforePlan?: (context: { fullSweep: boolean; signal: AbortSignal }) => Promise<void> | void;@@ -2684,6 +2691,7 @@ export function createServer(options: ServerOptions): ServerInstance {getSyncEngine: () => syncEngine,conflicts,setBatchInProgress: setBatchInProgressAndDrain,+ refreshDocumentsFromDisk,localOpCliArgs,authStreamHeartbeatMs,projectDir,@@ -2731,6 +2739,12 @@ export function createServer(options: ServerOptions): ServerInstance {},});hocuspocus.configuration.extensions.push(apiExtension);+ hocuspocus.configuration.extensions.push(+ createReconnectRefreshExtension({+ clock: options.watcherClock ?? watcherClock(),+ refresh: refreshDocumentsFromDisk,+ }),+ );nativeApi = apiExtension.nativeApi;localApi = apiExtension.localApi;shutdownLocalOps = apiExtension.shutdownLocalOps;@@ -3669,6 +3683,40 @@ export function createServer(options: ServerOptions): ServerInstance {}let watcher: WatcherHandle | null = null;++ async function refreshDocumentsFromDisk(+ docNames: readonly string[],+ trigger: DocumentRefreshTrigger,+ ): Promise<DocumentRefreshResult> {+ if (!watcher) return { checked: 0, found: 0 };+ const aliases = watcher.getAliasMap();+ const index = watcher.getFileIndex();+ const paths: string[] = [];+ for (const requested of new Set(docNames)) {+ const docName = aliases.get(requested) ?? requested;+ if (isReservedForUserTree(docName)) continue;+ const indexed = index.get(docName)?.canonicalPath;+ if (indexed !== undefined) {+ paths.push(indexed);+ continue;+ }+ try {+ paths.push(safeContentPath(docName, contentDir));+ } catch (err) {+ log.debug(+ { err, trigger },+ '[reconcile] skipped a re-read for a name outside the content tree',+ );+ }+ }+ if (paths.length === 0) return { checked: 0, found: 0 };+ const result = await watcher.recheckPaths(paths);+ const summary = { trigger, ...result };+ if (result.found > 0)+ log.info(summary, `[reconcile] re-read on ${trigger} found outside changes`);+ else log.debug(summary, `[reconcile] re-read on ${trigger} found nothing new`);+ return result;+ }let syncEngine: SyncEngine | null = null;let inflightDestroy: Promise<void> | null = null;@@ -4792,6 +4840,10 @@ export function createServer(options: ServerOptions): ServerInstance {onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),nativeBackend: options.watcherBackend,clock: options.watcherClock,+ freshnessIntervalMs:+ options.freshnessIntervalMs === undefined+ ? FRESHNESS_BASE_INTERVAL_MS+ : options.freshnessIntervalMs,}),);void resyncRecordedSkillCopies(projectDir, contentDir)@@ -5286,7 +5338,17 @@ export function createServer(options: ServerOptions): ServerInstance {}}},- { nativeBackend: options.watcherBackend, clock: options.watcherClock },+ {+ nativeBackend: options.watcherBackend,+ clock: options.watcherClock,+ onGaveUp: () => {+ if (!degraded.includes('head-watcher')) degraded.push('head-watcher');+ },+ onBack: () => {+ const at = degraded.indexOf('head-watcher');+ if (at !== -1) degraded.splice(at, 1);+ },+ },);} catch (err) {log.error({ err }, '[server] HEAD watcher failed to start');diff --git a/packages/server/src/server-freshness-default.test.ts b/packages/server/src/server-freshness-default.test.tsnew file mode 100644index 000000000..00754ee4b--- /dev/null+++ b/packages/server/src/server-freshness-default.test.ts@@ -0,0 +1,45 @@+import { mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, expect, test } from 'vitest';+import type { BootedServer } from './boot.ts';+import { bootCompositionRig } from './composition-rig.test-helper.ts';+import { getMetrics } from './metrics.ts';+import { createManualClock } from './watcher-clock.test-helper.ts';+import { createWatcherDouble } from './watcher-double.test-helper.ts';+import { FRESHNESS_BASE_INTERVAL_MS } from './watcher-freshness.ts';++let root: string | undefined;+let server: BootedServer | undefined;++afterEach(async () => {+ await server?.destroy();+ server = undefined;+ if (root) rmSync(root, { recursive: true, force: true });+ root = undefined;+});++async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {+ const start = performance.now();+ while (!condition()) {+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}++test('a booted server re-checks the content tree every five seconds unless told otherwise', async () => {+ root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-freshness-default-')));+ const clock = createManualClock(Date.now());+ server = await bootCompositionRig(root, {+ watcherBackend: createWatcherDouble().backend,+ watcherClock: clock,+ });+ await server.ready;+ writeFileSync(join(root, 'never-delivered.md'), '# Never delivered\n');+ const passesBefore = getMetrics().freshnessPasses;+ const findsBefore = getMetrics().freshnessFinds;++ clock.advance(FRESHNESS_BASE_INTERVAL_MS);+ await until(() => getMetrics().freshnessPasses > passesBefore, 'the first timed pass');+ expect(getMetrics().freshnessFinds - findsBefore).toBeGreaterThanOrEqual(1);+});diff --git a/packages/server/src/server-head-watcher-degraded.test.ts b/packages/server/src/server-head-watcher-degraded.test.tsnew file mode 100644index 000000000..1557feb55--- /dev/null+++ b/packages/server/src/server-head-watcher-degraded.test.ts@@ -0,0 +1,66 @@+import { execFileSync } from 'node:child_process';+import { mkdtempSync, realpathSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, expect, test } from 'vitest';+import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts';+import type { BootedServer } from './boot.ts';+import { bootCompositionRig } from './composition-rig.test-helper.ts';+import { getMetrics } from './metrics.ts';+import { createManualClock } from './watcher-clock.test-helper.ts';+import { createWatcherDouble } from './watcher-double.test-helper.ts';++let root: string | undefined;+let server: BootedServer | undefined;++afterEach(async () => {+ await server?.destroy();+ server = undefined;+ if (root) rmSync(root, { recursive: true, force: true });+ root = undefined;+});++async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {+ const start = performance.now();+ while (!condition()) {+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}++test('a server whose HEAD watcher gives up reports head-watcher as degraded until it is back', async () => {+ root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-head-degraded-')));+ const gitEnv = {+ ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_'))),+ GIT_CONFIG_GLOBAL: '/dev/null',+ GIT_CONFIG_NOSYSTEM: '1',+ };+ execFileSync('git', ['init', '--quiet', '--initial-branch=main'], { cwd: root, env: gitEnv });+ configureTestGitRepository(root);+ const clock = createManualClock(Date.now());+ const double = createWatcherDouble();+ const booted = await bootCompositionRig(root, {+ watcherBackend: double.backend,+ watcherClock: clock,+ freshnessIntervalMs: null,+ });+ server = booted;+ await booted.ready;+ const gitDir = join(root, '.git');+ expect(booted.degraded).not.toContain('head-watcher');++ const gaveUpBefore = getMetrics().watcherResubscribesAbandoned;+ for (let failure = 1; failure <= 8; failure++) {+ double.failNextSubscribe(gitDir, new Error(`watch limit ${failure}`));+ }+ double.subscriptionFor(gitDir).fail(new Error('the native watcher died'));+ await until(() => {+ clock.advance(30_000);+ return getMetrics().watcherResubscribesAbandoned > gaveUpBefore;+ }, 'the HEAD watcher to give up');+ expect(booted.degraded).toContain('head-watcher');++ clock.advance(5 * 60_000);+ await until(() => !booted.degraded.includes('head-watcher'), 'the HEAD watcher to be back');+ expect(double.subscriptionCount(gitDir)).toBe(2);+});diff --git a/packages/server/src/watcher-double.test-helper.ts b/packages/server/src/watcher-double.test-helper.tsindex a49c80425..72b639593 100644--- a/packages/server/src/watcher-double.test-helper.ts+++ b/packages/server/src/watcher-double.test-helper.ts@@ -8,6 +8,7 @@ type WatcherDoubleEvent = Parameters<SubscribeCallback>[1][number];interface WatcherDoubleSubscription {readonly dir: string;readonly closed: boolean;+ readonly dead: boolean;deliver(events: WatcherDoubleEvent[]): void;fail(error: Error): void;rescan(events?: WatcherDoubleEvent[]): void;@@ -16,25 +17,36 @@ interface WatcherDoubleSubscription {export function createWatcherDouble(): {readonly backend: NativeWatcherBackend;subscriptionFor(dir: string): WatcherDoubleSubscription;+ subscriptionCount(dir: string): number;+ failNextSubscribe(dir: string, error: Error): void;} {const subscriptions: WatcherDoubleSubscription[] = [];+ const subscribeFailures = new Map<string, Error[]>();const [rescanNotice] = RESCAN_NOTICES;const backend: NativeWatcherBackend = {async subscribe(dir, callback) {+ const failure = subscribeFailures.get(dir)?.shift();+ if (failure) throw failure;let closed = false;+ let dead = false;const subscription: WatcherDoubleSubscription = {dir,get closed() {return closed;},+ get dead() {+ return dead;+ },deliver(events) {- if (!closed) callback(null, events);+ if (!closed && !dead) callback(null, events);},fail(error) {- if (!closed) callback(error, []);+ if (closed || dead) return;+ dead = true;+ callback(error, []);},rescan(events = []) {- if (!closed) callback(new Error(rescanNotice), events);+ if (!closed && !dead) callback(new Error(rescanNotice), events);},};subscriptions.push(subscription);@@ -45,10 +57,12 @@ export function createWatcherDouble(): {};},};+ const forDir = (dir: string): WatcherDoubleSubscription[] =>+ subscriptions.filter((subscription) => subscription.dir === dir);return {backend,subscriptionFor(dir) {- const found = subscriptions.find((subscription) => subscription.dir === dir);+ const found = forDir(dir).at(-1);if (!found) {throw new Error(`no watcher double subscription for ${dir}; have ${subscriptions.map((s) => s.dir).join(', ') || 'none'}`,@@ -56,5 +70,13 @@ export function createWatcherDouble(): {}return found;},+ subscriptionCount(dir) {+ return forDir(dir).length;+ },+ failNextSubscribe(dir, error) {+ const queued = subscribeFailures.get(dir) ?? [];+ queued.push(error);+ subscribeFailures.set(dir, queued);+ },};}diff --git a/packages/server/src/watcher-freshness.ts b/packages/server/src/watcher-freshness.tsnew file mode 100644index 000000000..31ab78b4d--- /dev/null+++ b/packages/server/src/watcher-freshness.ts@@ -0,0 +1,215 @@+import { existsSync } from 'node:fs';+import { getLogger } from './logger.ts';+import {+ incrementWatcherResubscribes,+ incrementWatcherResubscribesAbandoned,+ incrementWatcherRootMissing,+} from './metrics.ts';+import type { WatcherClock, WatcherTimer } from './watcher-clock.ts';++const log = getLogger('watcher-freshness');++export const FRESHNESS_BASE_INTERVAL_MS = 5_000;+export const FRESHNESS_MAX_INTERVAL_MS = 60_000;+const FRESHNESS_DUTY_CYCLE_FACTOR = 20;++export const FRESHNESS_HASH_BUDGET = { files: 512, bytes: 64 * 1024 * 1024 } as const;++const RESUBSCRIBE_EARLY_DELAYS_MS: readonly number[] = [0, 1_000, 5_000];+const RESUBSCRIBE_FINAL_DELAY_MS = 30_000;+const RESUBSCRIBE_MAX_ATTEMPTS = 8;+export const RESUBSCRIBE_SLOW_RETRY_MS = 5 * 60_000;+const RESUBSCRIBE_ROOT_POLL_MS = 1_000;+const RESUBSCRIBE_STABLE_MS = 60_000;++type ResubscribeReason = 'error' | 'root-delete';++export type RecoveryMode = 'metadata' | 'full';++export function nextFreshnessDelayMs(lastPassMs: number, baseMs: number): number {+ const ceiling = Math.max(baseMs, FRESHNESS_MAX_INTERVAL_MS);+ const proportional = Math.ceil(Math.max(0, lastPassMs) * FRESHNESS_DUTY_CYCLE_FACTOR);+ return Math.min(ceiling, Math.max(baseMs, proportional));+}++function resubscribeDelayMs(attempt: number): number {+ return RESUBSCRIBE_EARLY_DELAYS_MS[attempt] ?? RESUBSCRIBE_FINAL_DELAY_MS;+}++export function resolveFreshnessInterval(option: number | null | undefined): number | null {+ if (option === null || option === undefined) return null;+ if (!Number.isFinite(option) || option <= 0) {+ throw new RangeError('freshnessIntervalMs must be a positive number of milliseconds or null');+ }+ return option;+}++interface WatcherSubscriptionLike {+ unsubscribe(): Promise<void>;+}++export interface Resubscriber {+ start(): Promise<void>;+ isLive(generation: number): boolean;+ lost(reason: ResubscribeReason, generation: number, err?: unknown): void;+ noticeRootMissing(): void;+ close(): Promise<void>;+}++export function createResubscriber(deps: {+ clock: WatcherClock;+ root: string;+ backendName: string;+ coverageWhileLost: string;+ subscribe: (generation: number) => Promise<WatcherSubscriptionLike>;+ onResubscribed: () => void;+ onGaveUp?: (err: unknown) => void;+ onBack?: () => void;+}): Resubscriber {+ let closed = false;+ let generation = 0;+ let liveGeneration: number | null = null;+ let current: WatcherSubscriptionLike | null = null;+ let liveSince = Number.NEGATIVE_INFINITY;+ let attempt = 0;+ let timer: WatcherTimer | null = null;+ let inFlight: Promise<void> | null = null;+ let slowRetry = false;+ const releases = new Set<Promise<void>>();+ const backend = deps.backendName;++ const release = (subscription: WatcherSubscriptionLike): void => {+ const released = subscription.unsubscribe().catch((err: unknown) => {+ log.debug({ err, backend }, 'releasing a lost watcher subscription failed');+ });+ releases.add(released);+ void released.then(() => releases.delete(released));+ };++ const attemptResubscribe = async (reason: ResubscribeReason): Promise<void> => {+ if (closed) return;+ if (!existsSync(deps.root)) {+ log.debug(+ { backend, reason },+ 'the watched directory is missing; waiting before resubscribing',+ );+ schedule(RESUBSCRIBE_ROOT_POLL_MS, reason);+ return;+ }+ const next = ++generation;+ liveGeneration = next;+ let subscription: WatcherSubscriptionLike;+ try {+ subscription = await deps.subscribe(next);+ } catch (err) {+ if (liveGeneration === next) liveGeneration = null;+ if (slowRetry) {+ log.debug({ err, backend, reason }, 'watcher still cannot subscribe; trying again later');+ schedule(RESUBSCRIBE_SLOW_RETRY_MS, reason);+ return;+ }+ attempt++;+ if (attempt >= RESUBSCRIBE_MAX_ATTEMPTS) {+ giveUp(reason, err);+ return;+ }+ log.warn({ err, backend, reason, attempt }, 'watcher resubscribe failed; retrying');+ schedule(resubscribeDelayMs(attempt), reason);+ return;+ }+ if (closed || liveGeneration !== next) {+ release(subscription);+ return;+ }+ current = subscription;+ liveSince = deps.clock.now();+ incrementWatcherResubscribes();+ if (slowRetry) {+ slowRetry = false;+ attempt = 0;+ log.info({ backend, reason }, 'watcher subscribed again after it had given up');+ deps.onBack?.();+ } else {+ log.info({ backend, reason, attempt }, 'watcher resubscribed');+ }+ deps.onResubscribed();+ };++ const giveUp = (reason: ResubscribeReason, err: unknown): void => {+ slowRetry = true;+ incrementWatcherResubscribesAbandoned();+ log.warn(+ { err, backend, reason, attempts: attempt, retryMs: RESUBSCRIBE_SLOW_RETRY_MS },+ `watcher subscription failed ${attempt} times in a row; it now tries again every ${RESUBSCRIBE_SLOW_RETRY_MS / 60_000} minutes, and until it is back ${deps.coverageWhileLost}`,+ );+ deps.onGaveUp?.(err);+ schedule(RESUBSCRIBE_SLOW_RETRY_MS, reason);+ };++ const schedule = (delayMs: number, reason: ResubscribeReason): void => {+ if (closed) return;+ timer = deps.clock.setTimeout(() => {+ timer = null;+ const work = attemptResubscribe(reason);+ inFlight = work;+ void work.finally(() => {+ if (inFlight === work) inFlight = null;+ });+ }, delayMs);+ timer.unref();+ };++ const lost = (reason: ResubscribeReason, lostGeneration: number, err?: unknown): void => {+ if (closed || lostGeneration !== liveGeneration) return;+ liveGeneration = null;+ const dead = current;+ current = null;+ if (dead) release(dead);+ if (reason === 'root-delete') incrementWatcherRootMissing();+ attempt = deps.clock.now() - liveSince < RESUBSCRIBE_STABLE_MS ? attempt + 1 : 0;+ if (attempt >= RESUBSCRIBE_MAX_ATTEMPTS) {+ giveUp(+ reason,+ err ??+ new Error(+ `the ${reason === 'error' ? 'subscription failed' : 'watched directory was deleted'}`,+ ),+ );+ return;+ }+ log.warn({ err, backend, reason, attempt }, 'watcher subscription lost; resubscribing');+ schedule(resubscribeDelayMs(attempt), reason);+ };++ return {+ async start() {+ const first = ++generation;+ liveGeneration = first;+ const subscription = await deps.subscribe(first);+ if (closed || liveGeneration !== first) {+ release(subscription);+ return;+ }+ current = subscription;+ },+ isLive: (candidate) => !closed && candidate === liveGeneration,+ lost,+ noticeRootMissing() {+ if (liveGeneration !== null) lost('root-delete', liveGeneration);+ },+ async close() {+ closed = true;+ deps.clock.clearTimeout(timer);+ timer = null;+ await inFlight;+ const live = current;+ current = null;+ liveGeneration = null;+ try {+ if (live) await live.unsubscribe();+ } finally {+ await Promise.all(releases);+ }+ },+ };+}diff --git a/packages/server/src/watcher-resubscribe-chokidar.test.ts b/packages/server/src/watcher-resubscribe-chokidar.test.tsnew file mode 100644index 000000000..96a84ae0f--- /dev/null+++ b/packages/server/src/watcher-resubscribe-chokidar.test.ts@@ -0,0 +1,127 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { type DiskEvent, lastKnownHash, startWatcher, type WatcherHandle } from './file-watcher.ts';+import { getMetrics } from './metrics.ts';+import { createManualClock, type ManualClock } from './watcher-clock.test-helper.ts';++const fakeChokidar = vi.hoisted(() => {+ type Listener = (...args: unknown[]) => void;+ class FakeWatcher {+ closed = false;+ private readonly listeners = new Map<string, Listener[]>();+ on(event: string, listener: Listener): this {+ this.listeners.set(event, [...(this.listeners.get(event) ?? []), listener]);+ return this;+ }+ once(event: string, listener: Listener): this {+ const wrapped: Listener = (...args) => {+ this.listeners.set(+ event,+ (this.listeners.get(event) ?? []).filter((candidate) => candidate !== wrapped),+ );+ listener(...args);+ };+ return this.on(event, wrapped);+ }+ emit(event: string, ...args: unknown[]): void {+ for (const listener of this.listeners.get(event) ?? []) listener(...args);+ }+ async close(): Promise<void> {+ this.closed = true;+ }+ }+ const watchers: FakeWatcher[] = [];+ return {+ watchers,+ watch: () => {+ const watcher = new FakeWatcher();+ watchers.push(watcher);+ queueMicrotask(() => watcher.emit('ready'));+ return watcher;+ },+ };+});++vi.mock('chokidar', () => ({ watch: fakeChokidar.watch }));++async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {+ const start = performance.now();+ while (!condition()) {+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}++function errorWithCode(message: string, code: string): NodeJS.ErrnoException {+ return Object.assign(new Error(message), { code });+}++describe('chokidar watcher resubscribe', () => {+ let tmpDir: string;+ let contentDir: string;+ let handle: WatcherHandle | undefined;+ let clock: ManualClock;+ let events: DiskEvent[];++ beforeEach(async () => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-chokidar-resubscribe-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ contentDir = realpathSync(contentDir);+ lastKnownHash.clear();+ fakeChokidar.watchers.length = 0;+ clock = createManualClock(Date.now());+ events = [];+ handle = await startWatcher(+ contentDir,+ async (event) => {+ events.push(event);+ },+ undefined,+ { forceBackend: 'chokidar', clock, freshnessIntervalMs: null, platform: 'linux' },+ );+ });++ afterEach(async () => {+ await handle?.unsubscribe();+ handle = undefined;+ rmSync(tmpDir, { recursive: true, force: true });+ });++ test('an error that stops chokidar makes the watcher watch again and re-check the tree', async () => {+ expect(fakeChokidar.watchers).toHaveLength(1);+ const first = fakeChokidar.watchers[0];+ writeFileSync(join(contentDir, 'gap.md'), '# Written while chokidar was failing\n');++ first?.emit('error', new Error('the watcher stopped'));+ clock.advance(0);++ await until(() => fakeChokidar.watchers.length === 2, 'a second chokidar watcher');+ expect(first?.closed).toBe(true);+ await until(+ () => events.some((event) => event.kind === 'create' && event.docName === 'gap'),+ 'the re-check to find the gap write',+ );+ });++ test.each([+ ['EACCES', 'permission denied'],+ ['ENOSPC', 'inotify watch limit reached'],+ ['EMFILE', 'too many open files'],+ ['ENFILE', 'file table overflow'],+ ])('a %s error is counted while the watcher stays up', async (code, message) => {+ const errorsBefore = getMetrics().watcherErrors;+ const resubscribesBefore = getMetrics().watcherResubscribes;+ fakeChokidar.watchers[0]?.emit('error', errorWithCode(message, code));+ fakeChokidar.watchers[0]?.emit('error', errorWithCode(message, code));+ clock.advance(60_000);+ await new Promise((resolve) => setTimeout(resolve, 50));++ expect(getMetrics().watcherErrors - errorsBefore).toBe(2);+ expect(getMetrics().watcherResubscribes - resubscribesBefore).toBe(0);+ expect(fakeChokidar.watchers).toHaveLength(1);+ expect(fakeChokidar.watchers[0]?.closed).toBe(false);+ });+});diff --git a/packages/server/src/watcher-resubscribe.test.ts b/packages/server/src/watcher-resubscribe.test.tsnew file mode 100644index 000000000..24399ba11--- /dev/null+++ b/packages/server/src/watcher-resubscribe.test.ts@@ -0,0 +1,320 @@+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';+import { type DiskEvent, lastKnownHash, startWatcher, type WatcherHandle } from './file-watcher.ts';+import { type HeadWatcherHandle, startHeadWatcher } from './head-watcher.ts';+import { getLogger } from './logger.ts';+import { getMetrics } from './metrics.ts';+import { createManualClock, type ManualClock } from './watcher-clock.test-helper.ts';+import { createWatcherDouble } from './watcher-double.test-helper.ts';++const SLOW_RETRY_MS = 5 * 60_000;++async function flushMicrotasks(): Promise<void> {+ for (let round = 0; round < 5; round++) {+ await new Promise((resolve) => setImmediate(resolve));+ }+}++async function until(condition: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {+ const start = performance.now();+ while (!condition()) {+ if (performance.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}++function createsOf(events: readonly DiskEvent[], docName: string): DiskEvent[] {+ return events.filter((event) => event.kind === 'create' && event.docName === docName);+}++describe('content watcher resubscribe', () => {+ let tmpDir: string;+ let contentDir: string;+ let handle: WatcherHandle | undefined;+ let clock: ManualClock;+ let events: DiskEvent[];+ let double: ReturnType<typeof createWatcherDouble>;++ beforeEach(() => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-resubscribe-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ contentDir = realpathSync(contentDir);+ lastKnownHash.clear();+ clock = createManualClock(Date.now());+ events = [];+ double = createWatcherDouble();+ });++ afterEach(async () => {+ await handle?.unsubscribe();+ handle = undefined;+ rmSync(tmpDir, { recursive: true, force: true });+ });++ async function watch(freshnessIntervalMs: number | null = null): Promise<void> {+ handle = await startWatcher(+ contentDir,+ async (event) => {+ events.push(event);+ },+ undefined,+ { nativeBackend: double.backend, clock, freshnessIntervalMs },+ );+ }++ test('after a backend error the watcher subscribes again and finds the write made in the gap', async () => {+ await watch();+ const errorsBefore = getMetrics().watcherErrors;+ const resubscribesBefore = getMetrics().watcherResubscribes;+ const first = double.subscriptionFor(contentDir);++ first.fail(new Error('the native watcher died'));+ expect(first.dead).toBe(true);+ expect(getMetrics().watcherErrors - errorsBefore).toBe(1);+ writeFileSync(join(contentDir, 'gap.md'), '# Written while the watcher was dead\n');+ first.deliver([{ type: 'create', path: join(contentDir, 'gap.md') }]);+ expect(createsOf(events, 'gap')).toHaveLength(0);++ clock.advance(0);+ await until(() => double.subscriptionCount(contentDir) === 2, 'a second subscription');+ await until(() => createsOf(events, 'gap').length === 1, 'the gap write to be found');+ expect(getMetrics().watcherResubscribes - resubscribesBefore).toBe(1);+ expect(first.closed).toBe(true);++ const later = join(contentDir, 'later.md');+ writeFileSync(later, '# Delivered by the new subscription\n');+ double.subscriptionFor(contentDir).deliver([{ type: 'create', path: later }]);+ await until(() => createsOf(events, 'later').length === 1, 'the new subscription to deliver');+ });++ async function flush(): Promise<void> {+ for (let round = 0; round < 5; round++) {+ await new Promise((resolve) => setImmediate(resolve));+ }+ }++ test('a resubscribe that keeps failing backs off 0, 1 and 5 seconds, then 30 seconds', async () => {+ await watch();+ for (let failure = 0; failure < 3; failure++) {+ double.failNextSubscribe(contentDir, new Error('the backend is not ready yet'));+ }+ double.subscriptionFor(contentDir).fail(new Error('the native watcher died'));++ for (const delayMs of [0, 1_000, 5_000]) {+ if (delayMs > 0) {+ clock.advance(delayMs - 1);+ await flush();+ }+ expect(double.subscriptionCount(contentDir)).toBe(1);+ clock.advance(delayMs === 0 ? 0 : 1);+ await flush();+ }+ clock.advance(29_999);+ await flush();+ expect(double.subscriptionCount(contentDir)).toBe(1);+ clock.advance(1);+ await until(() => double.subscriptionCount(contentDir) === 2, 'the 30 s retry');+ });++ test('after eight failed resubscribes the watcher tries again every five minutes until the fault clears', async () => {+ await watch();+ const warn = vi.spyOn(getLogger('watcher-freshness'), 'warn');+ const gaveUpBefore = getMetrics().watcherResubscribesAbandoned;+ for (let failure = 1; failure <= 8; failure++) {+ double.failNextSubscribe(contentDir, new Error(`no space left for watches ${failure}`));+ }+ double.subscriptionFor(contentDir).fail(new Error('the native watcher died'));+ for (let step = 0; step < 8; step++) {+ clock.advance(30_000);+ await flush();+ }+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(1);+ const gaveUp = warn.mock.calls.find(([, message]) =>+ String(message).includes('tries again every 5 minutes'),+ );+ expect((gaveUp?.[0] as { err?: Error } | undefined)?.err?.message).toBe(+ 'no space left for watches 8',+ );+ await expect((handle as WatcherHandle).settle()).rejects.toThrow(+ 'gave up subscribing after repeated failures (no space left for watches 8)',+ );++ double.failNextSubscribe(contentDir, new Error('still no space'));+ clock.advance(SLOW_RETRY_MS - 1);+ await flush();+ clock.advance(1);+ await flush();+ expect(double.subscriptionCount(contentDir)).toBe(1);++ writeFileSync(join(contentDir, 'while-down.md'), '# Written while the watcher was down\n');+ clock.advance(SLOW_RETRY_MS);+ await until(() => double.subscriptionCount(contentDir) === 2, 'the slow retry to subscribe');+ await until(() => createsOf(events, 'while-down').length === 1, 'the write made while down');+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(1);+ });++ test('a watcher whose new subscriptions keep dying within a minute is given up after eight short lives', async () => {+ await watch();+ const gaveUpBefore = getMetrics().watcherResubscribesAbandoned;+ const resubscribesBefore = getMetrics().watcherResubscribes;+ double.subscriptionFor(contentDir).fail(new Error('the first subscription died'));+ for (let life = 1; life <= 8; life++) {+ clock.advance(30_000);+ await until(+ () => getMetrics().watcherResubscribes - resubscribesBefore === life,+ `resubscribe ${life}`,+ );+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(0);+ double.subscriptionFor(contentDir).fail(new Error(`short life ${life}`));+ }+ expect(getMetrics().watcherResubscribesAbandoned - gaveUpBefore).toBe(1);+ });++ test('a subscription that lasted a minute is replaced at once, one that died sooner after a delay', async () => {+ await watch();+ const resubscribed = (count: number) => () =>+ double.subscriptionCount(contentDir) === count + 1 &&+ getMetrics().watcherResubscribes - resubscribesBefore === count;+ const resubscribesBefore = getMetrics().watcherResubscribes;++ double.subscriptionFor(contentDir).fail(new Error('first death'));+ clock.advance(0);+ await until(resubscribed(1), 'the first resubscribe');++ double.subscriptionFor(contentDir).fail(new Error('died within a minute'));+ clock.advance(999);+ await flush();+ expect(double.subscriptionCount(contentDir)).toBe(2);+ clock.advance(1);+ await until(resubscribed(2), 'the delayed resubscribe');++ clock.advance(60_000);+ double.subscriptionFor(contentDir).fail(new Error('died after a stable minute'));+ clock.advance(0);+ await until(resubscribed(3), 'the immediate resubscribe');+ });++ test('a deleted content root is watched again once it is recreated', async () => {+ await watch();+ const rootMissingBefore = getMetrics().watcherRootMissing;+ rmSync(contentDir, { recursive: true, force: true });+ double.subscriptionFor(contentDir).deliver([{ type: 'delete', path: contentDir }]);+ expect(getMetrics().watcherRootMissing - rootMissingBefore).toBe(1);++ clock.advance(0);+ clock.advance(1_000);+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(double.subscriptionCount(contentDir)).toBe(1);++ mkdirSync(contentDir);+ writeFileSync(join(contentDir, 'after-recreate.md'), '# After\n');+ clock.advance(1_000);+ await until(() => double.subscriptionCount(contentDir) === 2, 'the root to be watched again');+ await until(+ () => createsOf(events, 'after-recreate').length === 1,+ 'the write after the recreate to be found',+ );+ });++ test('the timed pass notices a content root that vanished without any event', async () => {+ await watch(5_000);+ const rootMissingBefore = getMetrics().watcherRootMissing;+ rmSync(contentDir, { recursive: true, force: true });++ clock.advance(5_000);+ expect(getMetrics().watcherRootMissing - rootMissingBefore).toBe(1);++ mkdirSync(contentDir);+ writeFileSync(join(contentDir, 'silent-root.md'), '# Back\n');+ clock.advance(1_000);+ await until(() => double.subscriptionCount(contentDir) === 2, 'the root to be watched again');+ await until(() => createsOf(events, 'silent-root').length === 1, 'the write to be found');+ });+});++describe('HEAD watcher resubscribe', () => {+ let projectRoot: string;+ let gitDir: string;+ let handle: HeadWatcherHandle | undefined;++ beforeEach(() => {+ projectRoot = realpathSync(mkdtempSync(join(tmpdir(), 'ok-head-resubscribe-')));+ gitDir = join(projectRoot, '.git');+ mkdirSync(gitDir);+ writeFileSync(join(gitDir, 'HEAD'), 'ref: refs/heads/main\n', 'utf-8');+ });++ afterEach(async () => {+ await handle?.unsubscribe();+ handle = undefined;+ rmSync(projectRoot, { recursive: true, force: true });+ });++ test('after a backend error the HEAD watcher subscribes again and re-reads HEAD in a batch', async () => {+ const double = createWatcherDouble();+ const clock = createManualClock(Date.now());+ const triggers: string[] = [];+ let batchEnds = 0;+ handle = await startHeadWatcher(+ projectRoot,+ ({ trigger }) => {+ triggers.push(trigger);+ },+ () => {+ batchEnds++;+ },+ { forceBackend: 'parcel', nativeBackend: double.backend, clock },+ );++ double.subscriptionFor(gitDir).fail(new Error('the native watcher died'));+ clock.advance(0);+ await until(() => double.subscriptionCount(gitDir) === 2, 'a second HEAD subscription');+ await until(() => triggers.includes('rescan'), 'the HEAD re-read batch');+ clock.advance(100);+ await until(() => batchEnds === 1, 'the re-read batch to end');++ double.subscriptionFor(gitDir).deliver([{ type: 'update', path: join(gitDir, 'HEAD') }]);+ await until(() => triggers.includes('HEAD'), 'the new subscription to open a batch');+ expect(triggers).toEqual(['rescan', 'HEAD']);+ });++ test('a HEAD watcher that gives up says so, tries again every five minutes and says when it is back', async () => {+ const double = createWatcherDouble();+ const clock = createManualClock(Date.now());+ const coverage: string[] = [];+ const triggers: string[] = [];+ handle = await startHeadWatcher(+ projectRoot,+ ({ trigger }) => {+ triggers.push(trigger);+ },+ () => {},+ {+ forceBackend: 'parcel',+ nativeBackend: double.backend,+ clock,+ onGaveUp: (err) => coverage.push(`gave up: ${(err as Error).message}`),+ onBack: () => coverage.push('back'),+ },+ );+ for (let failure = 1; failure <= 8; failure++) {+ double.failNextSubscribe(gitDir, new Error(`watch limit ${failure}`));+ }+ double.subscriptionFor(gitDir).fail(new Error('the native watcher died'));+ for (let step = 0; step < 8; step++) {+ clock.advance(30_000);+ await flushMicrotasks();+ }+ expect(coverage).toEqual(['gave up: watch limit 8']);+ expect(double.subscriptionCount(gitDir)).toBe(1);++ clock.advance(SLOW_RETRY_MS);+ await until(() => coverage.includes('back'), 'the slow retry');+ expect(coverage).toEqual(['gave up: watch limit 8', 'back']);+ expect(double.subscriptionCount(gitDir)).toBe(2);+ await until(() => triggers.includes('rescan'), 'the HEAD re-read once it is back');+ });+});