Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

PRD-9681 Add a watcher re-check call, injectable backend and test clock (#6946)

mike-inkeepe145bad51fe2
diff --git a/packages/app/tests/integration/test-harness.ts b/packages/app/tests/integration/test-harness.ts
index 975cc654f..2db7269dc 100644
--- a/packages/app/tests/integration/test-harness.ts
+++ b/packages/app/tests/integration/test-harness.ts
@@ -108,6 +108,8 @@ export interface CreateTestServerOptions {
pushIntervalSeconds?: number;
agentSessionOptions?: ServerOptions['agentSessionOptions'];
seedSkillHostRoot?: boolean;
+ watcherBackend?: ServerOptions['watcherBackend'];
+ watcherClock?: ServerOptions['watcherClock'];
}
export async function createTestServer(options: CreateTestServerOptions = {}): Promise<TestServer> {
@@ -184,6 +186,8 @@ export async function createTestServer(options: CreateTestServerOptions = {}): P
pullIntervalSeconds: options.pullIntervalSeconds,
pushIntervalSeconds: options.pushIntervalSeconds,
agentSessionOptions: options.agentSessionOptions,
+ watcherBackend: options.watcherBackend,
+ watcherClock: options.watcherClock,
...(ephemeral ? { ephemeral: true, singleDocRelPath: options.singleDocRelPath } : {}),
skipStateManifestCheck: true,
});
diff --git a/packages/app/tests/integration/watcher-recheck.test.ts b/packages/app/tests/integration/watcher-recheck.test.ts
new file mode 100644
index 000000000..c0fd42d6e
--- /dev/null
+++ b/packages/app/tests/integration/watcher-recheck.test.ts
@@ -0,0 +1,100 @@
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import {
+ getWatcherDecisionRingSnapshot,
+ resetWatcherDecisionDiagnostics,
+} from '../../../server/src/file-watcher';
+import { createWatcherDouble } from '../../../server/src/watcher-double.test-helper';
+import {
+ agentWriteMd,
+ createTestClient,
+ createTestServer,
+ pollUntil,
+ type TestClient,
+ type TestServer,
+} from './test-harness';
+
+let server: TestServer | undefined;
+let client: TestClient | undefined;
+
+afterEach(async () => {
+ await client?.cleanup();
+ client = undefined;
+ await server?.cleanup();
+ server = undefined;
+});
+
+describe('disk re-check through the server', () => {
+ test('a write the watcher never delivered is found, applied to the live document and reported', async () => {
+ const double = createWatcherDouble();
+ server = await createTestServer({
+ watcherBackend: double.backend,
+ debounce: 50,
+ maxDebounce: 200,
+ });
+ const docName = `recheck-${randomUUID()}`;
+ const file = join(server.contentDir, `${docName}.md`);
+ await agentWriteMd(server.port, '# Doc\n\noriginal\n', { docName, position: 'replace' });
+ await pollUntil(
+ () => existsSync(file) && readFileSync(file, 'utf-8').includes('original'),
+ 5_000,
+ 25,
+ 'the agent write to reach disk',
+ );
+ client = await createTestClient(server.port, docName);
+ await pollUntil(() => (client as TestClient).ytext.toString().includes('original'));
+
+ writeFileSync(file, '# Doc\n\nchanged outside\n');
+ const result = await server.instance.recheckDisk();
+ expect(result.status).toBe('completed');
+ expect(result.candidates).toBeGreaterThanOrEqual(1);
+ expect(result.dispatched).toBeGreaterThanOrEqual(1);
+
+ await pollUntil(
+ () => (client as TestClient).ytext.toString().includes('changed outside'),
+ 5_000,
+ 25,
+ 'the re-checked bytes to reach the live document',
+ );
+
+ const again = await server.instance.recheckDisk();
+ expect(again).toEqual({ status: 'completed', candidates: 0, dispatched: 0 });
+ });
+
+ test('the injected backend delivers events the same way the native watcher does', async () => {
+ const double = createWatcherDouble();
+ server = await createTestServer({
+ watcherBackend: double.backend,
+ debounce: 50,
+ maxDebounce: 200,
+ });
+ const docName = `delivered-${randomUUID()}`;
+ const file = join(server.contentDir, `${docName}.md`);
+ writeFileSync(file, '# Delivered\n\nfrom outside\n');
+ resetWatcherDecisionDiagnostics();
+ double.subscriptionFor(server.contentDir).deliver([{ type: 'create', path: file }]);
+ await pollUntil(
+ () =>
+ getWatcherDecisionRingSnapshot().some(
+ (record) =>
+ record.decision === 'dispatched' &&
+ record.kind === 'create' &&
+ record.path.endsWith(`${docName}.md`),
+ ),
+ 5_000,
+ 25,
+ 'the watcher to dispatch the delivered create',
+ );
+ client = await createTestClient(server.port, docName);
+ await pollUntil(
+ () => (client as TestClient).ytext.toString().includes('from outside'),
+ 5_000,
+ 25,
+ 'the delivered create to reach the live document',
+ );
+ expect(client.ytext.toString()).toContain('from outside');
+ expect(double.subscriptionFor(server.contentDir).closed).toBe(false);
+ });
+});
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index 9c5bfda62..cef298c29 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -154,6 +154,8 @@ export interface BootServerOptions
| 'ephemeral'
| 'configHomedirOverride'
| 'acpRegistryFetchImpl'
+ | 'watcherBackend'
+ | 'watcherClock'
> {
config: Config;
skipAutoInit?: boolean;
@@ -413,6 +415,8 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {
singleDocRelPath: opts.singleDocRelPath,
ephemeral: opts.ephemeral,
configHomedirOverride: opts.configHomedirOverride,
+ watcherBackend: opts.watcherBackend,
+ watcherClock: opts.watcherClock,
});
const {
diff --git a/packages/server/src/file-watcher-backend.test.ts b/packages/server/src/file-watcher-backend.test.ts
new file mode 100644
index 000000000..26b2b16ae
--- /dev/null
+++ b/packages/server/src/file-watcher-backend.test.ts
@@ -0,0 +1,219 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+import {
+ CHOKIDAR_BATCH_WINDOW_MS,
+ type DiskEvent,
+ lastKnownHash,
+ startWatcher,
+ type WatcherHandle,
+} from './file-watcher.ts';
+import { createManualClock } from './watcher-clock.test-helper.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+
+async function until(condition: () => boolean, what: string, timeoutMs = 5_000): Promise<void> {
+ const start = Date.now();
+ while (!condition()) {
+ if (Date.now() - start > timeoutMs) throw new Error(`timed out waiting for ${what}`);
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ }
+}
+
+function createsOf(events: readonly DiskEvent[], docName: string): DiskEvent[] {
+ return events.filter((event) => event.kind === 'create' && event.docName === docName);
+}
+
+describe('file watcher with an injected backend', () => {
+ let tmpDir: string;
+ let contentDir: string;
+ let handle: WatcherHandle | undefined;
+
+ beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-backend-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ lastKnownHash.clear();
+ });
+
+ afterEach(async () => {
+ await handle?.unsubscribe();
+ handle = undefined;
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test('the backend is subscribed on the content directory and its events reach the disk-event callback', async () => {
+ const double = createWatcherDouble();
+ const events: DiskEvent[] = [];
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { nativeBackend: double.backend },
+ );
+ const canonicalDir = realpathSync(contentDir);
+ const subscription = double.subscriptionFor(canonicalDir);
+ expect(subscription.closed).toBe(false);
+
+ const file = join(canonicalDir, 'delivered.md');
+ writeFileSync(file, '# Delivered\n');
+ subscription.deliver([{ type: 'create', path: file }]);
+ await until(
+ () => createsOf(events, 'delivered').length > 0,
+ 'the delivered create to reach the callback',
+ );
+ expect(handle.getFileIndex().has('delivered')).toBe(true);
+
+ await handle.unsubscribe();
+ handle = undefined;
+ expect(subscription.closed).toBe(true);
+ });
+
+ test('a re-check finds and applies a write the backend never delivered, and reports what it found', async () => {
+ const double = createWatcherDouble();
+ const events: DiskEvent[] = [];
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { nativeBackend: double.backend },
+ );
+ const canonicalDir = realpathSync(contentDir);
+ writeFileSync(join(canonicalDir, 'silent.md'), '# Silent\n');
+
+ const before = events.length;
+ const result = await handle.recheck();
+ const pass = events.slice(before);
+ expect(result.status).toBe('completed');
+ expect(result.candidates).toBeGreaterThanOrEqual(1);
+ expect(result.dispatched).toBeGreaterThanOrEqual(1);
+ expect(pass.filter((event) => event.kind === 'create')).toHaveLength(result.dispatched);
+ expect(createsOf(pass, 'silent')).toHaveLength(1);
+ expect(handle.getFileIndex().has('silent')).toBe(true);
+
+ const again = await handle.recheck();
+ expect(again).toEqual({ status: 'completed', candidates: 0, dispatched: 0 });
+ });
+
+ test('concurrent re-check requests coalesce into one pass', async () => {
+ const double = createWatcherDouble();
+ handle = await startWatcher(contentDir, async () => {}, undefined, {
+ nativeBackend: double.backend,
+ });
+ writeFileSync(join(realpathSync(contentDir), 'once.md'), '# Once\n');
+ const first = handle.recheck();
+ const second = handle.recheck();
+ expect(second).toBe(first);
+ const result = await first;
+ expect(result.status).toBe('completed');
+ expect(result.candidates).toBeGreaterThanOrEqual(1);
+ });
+
+ test('a backend error does not stop a later re-check', async () => {
+ const double = createWatcherDouble();
+ const events: DiskEvent[] = [];
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { nativeBackend: double.backend },
+ );
+ const canonicalDir = realpathSync(contentDir);
+ double.subscriptionFor(canonicalDir).fail(new Error('boom'));
+ writeFileSync(join(canonicalDir, 'after-error.md'), '# After\n');
+ const result = await handle.recheck();
+ expect(result.status).toBe('completed');
+ expect(createsOf(events, 'after-error')).toHaveLength(1);
+ });
+
+ test('a rescan notice from the backend runs a recovery that finds undelivered writes', async () => {
+ const double = createWatcherDouble();
+ const events: DiskEvent[] = [];
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ { nativeBackend: double.backend },
+ );
+ const canonicalDir = realpathSync(contentDir);
+ writeFileSync(join(canonicalDir, 'rescanned.md'), '# Rescanned\n');
+ double.subscriptionFor(canonicalDir).rescan();
+ await until(
+ () => createsOf(events, 'rescanned').length > 0,
+ 'the rescan recovery to deliver the create',
+ );
+ expect(handle.getFileIndex().has('rescanned')).toBe(true);
+ });
+});
+
+describe('file watcher re-check on the chokidar backend', () => {
+ let tmpDir: string;
+ let contentDir: string;
+ let handle: WatcherHandle | undefined;
+
+ beforeEach(() => {
+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-watcher-chokidar-'));
+ contentDir = join(tmpDir, 'content');
+ mkdirSync(contentDir, { recursive: true });
+ lastKnownHash.clear();
+ });
+
+ afterEach(async () => {
+ await handle?.unsubscribe();
+ handle = undefined;
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test('a write the re-check applied before the chokidar batch fired is created once', async () => {
+ const clock = createManualClock();
+ const events: DiskEvent[] = [];
+ const rawBatches: (readonly string[])[] = [];
+ handle = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ },
+ undefined,
+ {
+ forceBackend: 'chokidar',
+ clock,
+ onRawBatch: (paths) => {
+ rawBatches.push(paths);
+ },
+ },
+ );
+ const baseline = clock.pendingTimers();
+ const file = join(realpathSync(contentDir), 'chokidar.md');
+ writeFileSync(file, '# Chokidar\n');
+ await until(
+ () => clock.pendingTimers() > baseline,
+ 'chokidar to park its batch timer on the manual clock',
+ );
+ expect(clock.pendingTimers()).toBe(baseline + 1);
+
+ const result = await handle.recheck();
+ expect(result.status).toBe('completed');
+ expect(result.dispatched).toBe(1);
+ expect(createsOf(events, 'chokidar')).toHaveLength(1);
+
+ rawBatches.length = 0;
+ clock.advance(CHOKIDAR_BATCH_WINDOW_MS);
+ await until(
+ () => rawBatches.some((paths) => paths.includes(file)),
+ 'the parked chokidar batch to start',
+ );
+
+ const again = await handle.recheck();
+ expect(again.status).toBe('completed');
+ expect(again.candidates).toBe(0);
+ expect(createsOf(events, 'chokidar')).toHaveLength(1);
+ });
+});
diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts
index e65963ba3..5b1c8c74b 100644
--- a/packages/server/src/file-watcher.ts
+++ b/packages/server/src/file-watcher.ts
@@ -45,12 +45,19 @@ import {
extractPageType,
} from './page-identity.ts';
import { classifyParcelNotification } from './parcel-recovery.ts';
-import { subscribeParcel } from './parcel-subscription.ts';
+import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';
import { toPosix } from './path-utils.ts';
import { containsConflictMarkers } from './reconciliation.ts';
import { getMeter, withSpan } from './telemetry.ts';
import { contentHash } from './version-hash.ts';
+import {
+ installWatcherClock,
+ type WatcherClock,
+ type WatcherTimer,
+ watcherClock,
+} from './watcher-clock.ts';
+export type { NativeWatcherBackend } from './parcel-subscription.ts';
export { contentHash } from './version-hash.ts';
const log = getLogger('file-watcher');
@@ -248,8 +255,26 @@ function markdownIndexView(
return snapshot;
}
+type RecheckStatus = 'completed' | 'incomplete' | 'failed' | 'closed';
+
+export interface RecheckResult {
+ status: RecheckStatus;
+ candidates: number;
+ dispatched: number;
+}
+
+interface RecoveryOutcome {
+ completed: boolean;
+ candidates: number;
+}
+
+interface DispatchCounter {
+ dispatched: number;
+}
+
export interface WatcherHandle {
unsubscribe: () => Promise<void>;
+ recheck: () => Promise<RecheckResult>;
getFileIndex: () => ReadonlyMap<string, FileIndexEntry>;
getAllFilesIndex: () => AllFileEntries;
getFileIndexGeneration: () => number;
@@ -279,7 +304,7 @@ const activeOwnWrites = new Map<string, Set<OwnWriteObservation>>();
export function registerWrite(filePath: string, hash: string): void {
const key = declaredPathKey(filePath);
const queue = writeTracker.get(key) ?? [];
- queue.push({ hash, timestamp: Date.now() });
+ queue.push({ hash, timestamp: watcherClock().now() });
writeTracker.set(key, queue);
for (const observation of activeOwnWrites.get(key) ?? []) {
observation.savedDuringObservation = true;
@@ -386,7 +411,7 @@ type _RawStringIsNotARemovalDeclaration = Assert<string extends RemovalDeclarati
leaf; the watcher reports a removed alias at the alias, not at the target it pointed to. */
export function registerRemoval(filePath: string): RemovalDeclaration {
const key = declaredPathKey(filePath);
- removalTracker.set(key, Date.now());
+ removalTracker.set(key, watcherClock().now());
return key as RemovalDeclaration;
}
@@ -408,11 +433,11 @@ export function isSelfRemoval(filePath: string): boolean {
const timestamp = removalTracker.get(key);
if (timestamp === undefined) return false;
removalTracker.delete(key);
- return Date.now() - timestamp <= REMOVAL_TRACKER_TTL_MS;
+ return watcherClock().now() - timestamp <= REMOVAL_TRACKER_TTL_MS;
}
export function evictStaleTrackerEntries(): void {
- const now = Date.now();
+ const now = watcherClock().now();
for (const [path, queue] of writeTracker) {
const fresh = queue.filter((e) => now - e.timestamp <= WRITE_TRACKER_TTL_MS);
if (fresh.length === 0) {
@@ -460,7 +485,7 @@ let watcherDropsSinceLastSummary = 0;
function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath: string): void {
watcherDecisionRing.push({
- ts: Date.now(),
+ ts: watcherClock().now(),
decision,
kind,
path: normalizeFsPath(rawPath),
@@ -3046,8 +3071,13 @@ async function recoverDiskChanges(
onDiskEvent: (event: DiskEvent) => Promise<void>,
onAfterMutation: () => void,
getFileIndexGeneration: () => number,
+ counter: DispatchCounter,
onRawBatch?: (absPaths: readonly string[]) => void,
-): Promise<boolean> {
+): Promise<RecoveryOutcome> {
+ const countingOnDiskEvent = async (event: DiskEvent): Promise<void> => {
+ counter.dispatched++;
+ await onDiskEvent(event);
+ };
const knownPaths = new Set<string>();
for (const entry of fileIndex.values()) knownPaths.add(entry.canonicalPath);
for (const name of generalMemberPaths(contentDir, generalFileIndex)) {
@@ -3067,7 +3097,7 @@ async function recoverDiskChanges(
contentFilter?.refreshInPlaceSkillDirs();
const observed = await observeDisk(contentDir, contentFilter);
- if (!observed.complete) return false;
+ if (!observed.complete) return { completed: false, candidates: 0 };
const markdownPublicMutationFor = (name: string, entry: FileIndexEntry): boolean => {
const names = publicMutations.markdownNames;
@@ -3202,7 +3232,7 @@ async function recoverDiskChanges(
contentFilter,
fileIndex,
folderIndex,
- onDiskEvent,
+ countingOnDiskEvent,
aliasMap,
admitRecoveredEvent,
generalFileIndex,
@@ -3423,11 +3453,11 @@ async function recoverDiskChanges(
};
}
recordWatcherDecision('dispatched', event.kind, event.path);
- await onDiskEvent(event);
+ await countingOnDiskEvent(event);
}
}
onRawBatch?.([...new Set([...observed.paths, ...knownPaths])]);
- return true;
+ return { completed: true, candidates: deduplicated.length };
}
export function toParcelIgnorePaths(
@@ -3448,6 +3478,101 @@ type RecoveryPublicMutations = GeneralMutationRecord & {
/* WARN: LEASE_RESET_MS in packages/app/tests/stress/_helpers/reuse/lease-fixture.ts must exceed this bound plus the rest of a slot reset. */
export const INTERIM_SETTLE_NO_PROGRESS_MS = 20_000;
+interface RecoveryDriverDeps {
+ runOrdered: (work: () => Promise<void>) => Promise<void>;
+ isClosed: () => boolean;
+ recover: (
+ mutations: RecoveryPublicMutations,
+ counter: DispatchCounter,
+ ) => Promise<RecoveryOutcome>;
+ setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void;
+ onAfterMutation: () => void;
+ onRecoveryComplete?: () => Promise<void>;
+ backendName: WatcherBackend;
+}
+
+function createRecoveryDriver(deps: RecoveryDriverDeps): {
+ requestRecovery: () => Promise<RecheckResult>;
+} {
+ let recoveryRunning = false;
+ let nextRecovery: Promise<RecheckResult> | undefined;
+ let followUpRecovery: PromiseWithResolvers<RecheckResult> | undefined;
+ const closed = (): RecheckResult => ({ status: 'closed', candidates: 0, dispatched: 0 });
+ const recoverOnce = async (): Promise<RecheckResult> => {
+ nextRecovery = undefined;
+ if (deps.isClosed()) return closed();
+ recoveryRunning = true;
+ const recoveryMutations: RecoveryPublicMutations = {
+ markdownNames: new Set(),
+ generalNames: new Set(),
+ generalEffects: new Map(),
+ generalMetadata: new Map(),
+ hashEffects: new Map(),
+ };
+ const counter: DispatchCounter = { dispatched: 0 };
+ let result: RecheckResult = { status: 'failed', candidates: 0, dispatched: 0 };
+ try {
+ deps.setActivePublicMutations(recoveryMutations);
+ const outcome = await deps.recover(recoveryMutations, counter);
+ deps.onAfterMutation();
+ if (outcome.completed) {
+ await deps.onRecoveryComplete?.();
+ result = {
+ status: 'completed',
+ candidates: outcome.candidates,
+ dispatched: counter.dispatched,
+ };
+ log.info(
+ { candidates: outcome.candidates, dispatched: counter.dispatched },
+ `${deps.backendName} watcher recovery finished`,
+ );
+ } else {
+ result = { status: 'incomplete', candidates: 0, dispatched: counter.dispatched };
+ log.warn(
+ { dispatched: counter.dispatched },
+ `${deps.backendName} watcher recovery incomplete; known state retained`,
+ );
+ }
+ } catch (handleErr) {
+ log.error({ err: handleErr }, `${deps.backendName} batch error`);
+ result = { status: 'failed', candidates: 0, dispatched: counter.dispatched };
+ } finally {
+ try {
+ for (const [path, hash] of recoveryMutations.hashEffects) {
+ if (hash === null) removeLastKnownHash(path);
+ else updateLastKnownHash(path, hash);
+ }
+ } finally {
+ deps.setActivePublicMutations(null);
+ recoveryRunning = false;
+ const followUp = followUpRecovery;
+ followUpRecovery = undefined;
+ if (followUp) followUp.resolve(runRecovery());
+ }
+ }
+ return result;
+ };
+ const runRecovery = (): Promise<RecheckResult> => {
+ let settled = closed();
+ return deps
+ .runOrdered(async () => {
+ settled = await recoverOnce();
+ })
+ .then(() => settled);
+ };
+ const requestRecovery = (): Promise<RecheckResult> => {
+ if (nextRecovery) return nextRecovery;
+ if (recoveryRunning) {
+ followUpRecovery = Promise.withResolvers<RecheckResult>();
+ nextRecovery = followUpRecovery.promise;
+ } else {
+ nextRecovery = runRecovery();
+ }
+ return nextRecovery;
+ };
+ return { requestRecovery };
+}
+
async function startParcelWatcher(
contentDir: string,
contentFilter: ContentFilter | undefined,
@@ -3465,22 +3590,29 @@ async function startParcelWatcher(
onRawBatch?: (absPaths: readonly string[]) => void,
discoveredStructuralDirs?: ReadonlySet<string>,
onRecoveryComplete?: () => Promise<void>,
+ clock: WatcherClock = watcherClock(),
+ nativeBackend?: NativeWatcherBackend,
): Promise<
| (AsyncSubscription & {
runOrdered(work: () => Promise<void>): Promise<void>;
+ requestRecovery(): Promise<RecheckResult>;
settle(): Promise<void>;
})
| null
> {
- let parcel: typeof import('@parcel/watcher');
- try {
- parcel = await import('@parcel/watcher');
- } catch (err) {
- log.debug(
- { err: err instanceof Error ? err.message : String(err) },
- '[file-watcher] @parcel/watcher import failed; falling back to chokidar',
- );
- return null;
+ let parcel: NativeWatcherBackend;
+ if (nativeBackend) {
+ parcel = nativeBackend;
+ } else {
+ try {
+ parcel = await import('@parcel/watcher');
+ } catch (err) {
+ log.debug(
+ { err: err instanceof Error ? err.message : String(err) },
+ '[file-watcher] @parcel/watcher import failed; falling back to chokidar',
+ );
+ return null;
+ }
}
try {
@@ -3510,23 +3642,11 @@ async function startParcelWatcher(
});
return next;
};
- let recoveryRunning = false;
- let nextRecovery: Promise<void> | undefined;
- let followUpRecovery: PromiseWithResolvers<void> | undefined;
- const recoverOnce = async (): Promise<void> => {
- nextRecovery = undefined;
- if (closed) return;
- recoveryRunning = true;
- const recoveryMutations: RecoveryPublicMutations = {
- markdownNames: new Set(),
- generalNames: new Set(),
- generalEffects: new Map(),
- generalMetadata: new Map(),
- hashEffects: new Map(),
- };
- try {
- setActivePublicMutations(recoveryMutations);
- const recovered = await recoverDiskChanges(
+ const { requestRecovery } = createRecoveryDriver({
+ runOrdered,
+ isClosed: () => closed,
+ recover: (recoveryMutations, counter) =>
+ recoverDiskChanges(
contentDir,
contentFilter,
fileIndex,
@@ -3539,45 +3659,14 @@ async function startParcelWatcher(
onDiskEvent,
onAfterMutation,
getFileIndexGeneration,
+ counter,
onRawBatch,
- );
- onAfterMutation();
- if (recovered) {
- await onRecoveryComplete?.();
- log.info({ completed: true }, 'parcel watcher recovery finished');
- } else {
- log.warn(
- { completed: false },
- 'parcel watcher recovery incomplete; known state retained',
- );
- }
- } catch (handleErr) {
- log.error({ err: handleErr }, 'parcel batch error');
- } finally {
- try {
- for (const [path, hash] of recoveryMutations.hashEffects) {
- if (hash === null) removeLastKnownHash(path);
- else updateLastKnownHash(path, hash);
- }
- } finally {
- setActivePublicMutations(null);
- recoveryRunning = false;
- const followUp = followUpRecovery;
- followUpRecovery = undefined;
- if (followUp) followUp.resolve(runOrdered(recoverOnce));
- }
- }
- };
- const requestRecovery = (): Promise<void> => {
- if (nextRecovery) return nextRecovery;
- if (recoveryRunning) {
- followUpRecovery = Promise.withResolvers<void>();
- nextRecovery = followUpRecovery.promise;
- } else {
- nextRecovery = runOrdered(recoverOnce);
- }
- return nextRecovery;
- };
+ ),
+ setActivePublicMutations,
+ onAfterMutation,
+ onRecoveryComplete,
+ backendName: 'parcel',
+ });
const subscription = await subscribeParcel(
parcel,
contentDir,
@@ -3636,16 +3725,17 @@ async function startParcelWatcher(
return {
runOrdered,
+ requestRecovery,
async settle() {
if (closed) return;
if (failure) throw failure;
const name = `watcher-settle-fence-${randomUUID()}`;
const fencePath = join(contentDir, '.ok', name);
const reached = Promise.withResolvers<void>();
- let quiet: ReturnType<typeof setTimeout> | undefined;
+ let quiet: WatcherTimer | undefined;
const progressed = (): void => {
- clearTimeout(quiet);
- quiet = setTimeout(() => {
+ clock.clearTimeout(quiet);
+ quiet = clock.setTimeout(() => {
reached.reject(
new Error(
`the file watcher handled no batch for ${INTERIM_SETTLE_NO_PROGRESS_MS} ms before it saw its settle fence`,
@@ -3663,7 +3753,7 @@ async function startParcelWatcher(
progressed();
await reached.promise;
} finally {
- clearTimeout(quiet);
+ clock.clearTimeout(quiet);
settleProgress.delete(progressed);
settleFences.delete(name);
if (fenceWritten) tracedRmSync(fencePath, { force: true });
@@ -3714,6 +3804,7 @@ export function isChokidarPathIgnored(
}
const CHOKIDAR_READY_TIMEOUT_MS = 10_000;
+export const CHOKIDAR_BATCH_WINDOW_MS = 50;
/* UPSTREAM(nodejs/node#52601, libuv/libuv#3866, libuv/libuv#637): a macOS directory fs.watch
returns before FSEvents covers its path; closing one waits for the stream rebuild that covers
@@ -3739,10 +3830,15 @@ async function startChokidarWatcher(
onDiskEvent: (event: DiskEvent) => Promise<void>,
aliasMap: Map<string, string>,
aliasPaths: Map<string, string>,
+ folderAliasIndex: Map<string, string>,
+ setActivePublicMutations: (mutations: RecoveryPublicMutations | null) => void,
onAfterMutation: () => void,
+ getFileIndexGeneration: () => number,
platform: NodeJS.Platform,
+ clock: WatcherClock,
onRawBatch?: (absPaths: readonly string[]) => void,
-): Promise<AsyncSubscription> {
+ onRecoveryComplete?: () => Promise<void>,
+): Promise<AsyncSubscription & { requestRecovery(): Promise<RecheckResult> }> {
const { watch } = await import('chokidar');
const watcher = watch(contentDir, {
@@ -3756,32 +3852,72 @@ async function startChokidarWatcher(
watcher.on('error', (err) => log.error({ err }, 'chokidar error'));
- const BATCH_WINDOW_MS = 50;
+ let closed = false;
+ let workTail = Promise.resolve();
+ const runOrdered = (work: () => Promise<void>): Promise<void> => {
+ if (closed) return Promise.resolve();
+ const next = workTail.then(work);
+ workTail = next.catch((err) => {
+ log.error({ err }, 'chokidar batch error');
+ });
+ return next;
+ };
+ const { requestRecovery } = createRecoveryDriver({
+ runOrdered,
+ isClosed: () => closed,
+ recover: (recoveryMutations, counter) =>
+ recoverDiskChanges(
+ contentDir,
+ contentFilter,
+ fileIndex,
+ generalFileIndex,
+ folderIndex,
+ aliasMap,
+ aliasPaths,
+ folderAliasIndex,
+ recoveryMutations,
+ onDiskEvent,
+ onAfterMutation,
+ getFileIndexGeneration,
+ counter,
+ onRawBatch,
+ ),
+ setActivePublicMutations,
+ onAfterMutation,
+ onRecoveryComplete,
+ backendName: 'chokidar',
+ });
+
let pendingEvents: InternalRawFileEvent[] = [];
- let batchTimer: ReturnType<typeof setTimeout> | null = null;
+ let batchTimer: WatcherTimer | null = null;
function queueEvent(event: InternalRawFileEvent) {
pendingEvents.push(event);
- batchTimer ||= setTimeout(() => {
+ batchTimer ||= clock.setTimeout(() => {
const batch = pendingEvents;
pendingEvents = [];
batchTimer = null;
- onRawBatch?.(batch.map((e) => e.path));
- handleRawEventsInternal(
- batch,
- contentDir,
- contentFilter,
- fileIndex,
- folderIndex,
- onDiskEvent,
- aliasMap,
- undefined,
- generalFileIndex,
- aliasPaths,
- )
- .then(onAfterMutation)
- .catch((err) => log.error({ err }, 'chokidar batch error'));
- }, BATCH_WINDOW_MS);
+ void runOrdered(async () => {
+ try {
+ onRawBatch?.(batch.map((e) => e.path));
+ await handleRawEventsInternal(
+ batch,
+ contentDir,
+ contentFilter,
+ fileIndex,
+ folderIndex,
+ onDiskEvent,
+ aliasMap,
+ undefined,
+ generalFileIndex,
+ aliasPaths,
+ );
+ onAfterMutation();
+ } catch (err) {
+ log.error({ err }, 'chokidar batch error');
+ }
+ });
+ }, CHOKIDAR_BATCH_WINDOW_MS);
}
watcher.on('add', (path) => queueEvent({ type: 'create', path }));
@@ -3798,33 +3934,65 @@ async function startChokidarWatcher(
resolveReady();
};
watcher.once('ready', settle);
- setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();
+ clock.setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();
});
waitForPendingFsEventsRegistrations(contentDir, platform);
return {
- unsubscribe: () => {
+ requestRecovery,
+ unsubscribe: async () => {
+ closed = true;
if (batchTimer) {
- clearTimeout(batchTimer);
+ clock.clearTimeout(batchTimer);
batchTimer = null;
pendingEvents = [];
}
- return watcher.close();
+ await watcher.close();
+ await workTail;
},
};
}
+export interface StartWatcherOptions {
+ forceBackend?: 'parcel' | 'chokidar';
+ onRawBatch?: (absPaths: readonly string[]) => void;
+ onRecoveryComplete?: () => Promise<void>;
+ platform?: NodeJS.Platform;
+ nativeBackend?: NativeWatcherBackend;
+ clock?: WatcherClock;
+}
+
export async function startWatcher(
contentDirRaw: string,
onDiskEvent: (event: DiskEvent) => Promise<void>,
contentFilter?: ContentFilter,
- opts: {
- forceBackend?: 'parcel' | 'chokidar';
- onRawBatch?: (absPaths: readonly string[]) => void;
- onRecoveryComplete?: () => Promise<void>;
- platform?: NodeJS.Platform;
- } = {},
+ opts: StartWatcherOptions = {},
+): Promise<WatcherHandle> {
+ const restoreClock = opts.clock ? installWatcherClock(opts.clock) : (): void => {};
+ const clock = opts.clock ?? watcherClock();
+ try {
+ return await startWatcherWithClock(
+ contentDirRaw,
+ onDiskEvent,
+ contentFilter,
+ opts,
+ clock,
+ restoreClock,
+ );
+ } catch (e) {
+ restoreClock();
+ throw e;
+ }
+}
+
+async function startWatcherWithClock(
+ contentDirRaw: string,
+ onDiskEvent: (event: DiskEvent) => Promise<void>,
+ contentFilter: ContentFilter | undefined,
+ opts: StartWatcherOptions,
+ clock: WatcherClock,
+ restoreClock: () => void,
): Promise<WatcherHandle> {
const { onRawBatch, onRecoveryComplete } = opts;
const contentDir = resolveDirectoryRoot(contentDirRaw, {
@@ -3870,10 +4038,14 @@ export async function startWatcher(
for (const [alias, path] of initialObservation.aliasPaths) aliasPaths.set(alias, path);
bumpFileIndexGeneration();
- const evictionInterval = setInterval(evictStaleTrackerEntries, WRITE_TRACKER_TTL_MS);
- const dropSummaryInterval = setInterval(logWatcherDropSummary, WATCHER_DROP_SUMMARY_INTERVAL_MS);
+ const evictionInterval = clock.setInterval(evictStaleTrackerEntries, WRITE_TRACKER_TTL_MS);
+ const dropSummaryInterval = clock.setInterval(
+ logWatcherDropSummary,
+ WATCHER_DROP_SUMMARY_INTERVAL_MS,
+ );
let subscription: AsyncSubscription;
+ let requestRecovery: () => Promise<RecheckResult>;
let runOrderedRescan: ((work: () => Promise<void>) => Promise<void>) | undefined;
let settleSubscription: (() => Promise<void>) | undefined;
let backend: WatcherBackend;
@@ -3902,9 +4074,12 @@ export async function startWatcher(
onRawBatch,
structuralIgnoreDirs,
onRecoveryComplete,
+ clock,
+ opts.nativeBackend,
);
if (parcelSub) {
subscription = parcelSub;
+ requestRecovery = parcelSub.requestRecovery;
runOrderedRescan = parcelSub.runOrdered;
settleSubscription = () => parcelSub.settle();
backend = 'parcel';
@@ -3912,7 +4087,7 @@ export async function startWatcher(
if (opts.forceBackend === 'parcel') {
throw new Error('@parcel/watcher unavailable (forced backend)');
}
- subscription = await startChokidarWatcher(
+ const chokidarSub = await startChokidarWatcher(
contentDir,
contentFilter,
fileIndex,
@@ -3921,15 +4096,24 @@ export async function startWatcher(
onDiskEvent,
aliasMap,
aliasPaths,
+ folderAliasIndex,
+ (mutations) => {
+ activePublicMutations = mutations;
+ },
bumpFileIndexGeneration,
+ () => fileIndexGeneration,
platform,
+ clock,
onRawBatch,
+ onRecoveryComplete,
);
+ subscription = chokidarSub;
+ requestRecovery = chokidarSub.requestRecovery;
backend = 'chokidar';
}
} catch (e) {
- clearInterval(evictionInterval);
- clearInterval(dropSummaryInterval);
+ clock.clearInterval(evictionInterval);
+ clock.clearInterval(dropSummaryInterval);
throw e;
}
@@ -3939,13 +4123,20 @@ export async function startWatcher(
return {
async unsubscribe() {
- clearInterval(evictionInterval);
- clearInterval(dropSummaryInterval);
- await originalUnsubscribe();
- writeTracker.clear();
- removalTracker.clear();
- lastKnownHash.clear();
- resetWatcherDecisionDiagnostics();
+ clock.clearInterval(evictionInterval);
+ clock.clearInterval(dropSummaryInterval);
+ try {
+ await originalUnsubscribe();
+ } finally {
+ writeTracker.clear();
+ removalTracker.clear();
+ lastKnownHash.clear();
+ resetWatcherDecisionDiagnostics();
+ restoreClock();
+ }
+ },
+ recheck() {
+ return requestRecovery();
},
getFileIndex() {
if (cachedMarkdownView && cachedMarkdownViewGeneration === fileIndexGeneration) {
diff --git a/packages/server/src/head-watcher.ts b/packages/server/src/head-watcher.ts
index 0021004fd..792c77795 100644
--- a/packages/server/src/head-watcher.ts
+++ b/packages/server/src/head-watcher.ts
@@ -4,7 +4,8 @@ import {
} from '@inkeep/open-knowledge-core/git-repository';
import { getLogger } from './logger.ts';
import { classifyParcelNotification } from './parcel-recovery.ts';
-import { subscribeParcel } from './parcel-subscription.ts';
+import { type NativeWatcherBackend, subscribeParcel } from './parcel-subscription.ts';
+import { type WatcherClock, type WatcherTimer, watcherClock } from './watcher-clock.ts';
const log = getLogger('head-watcher');
@@ -73,16 +74,21 @@ async function tryStartParcelHeadWatcher(
dispatch: (rawPath: string) => boolean,
recover: () => void,
platform: NodeJS.Platform,
+ nativeBackend?: NativeWatcherBackend,
): Promise<(() => Promise<void>) | null> {
- let parcel: typeof import('@parcel/watcher');
- try {
- parcel = await import('@parcel/watcher');
- } catch (err) {
- log.debug(
- { err: err instanceof Error ? err.message : String(err) },
- '[head-watcher] @parcel/watcher unavailable; falling back to chokidar',
- );
- return null;
+ let parcel: NativeWatcherBackend;
+ if (nativeBackend) {
+ parcel = nativeBackend;
+ } else {
+ try {
+ parcel = await import('@parcel/watcher');
+ } catch (err) {
+ log.debug(
+ { err: err instanceof Error ? err.message : String(err) },
+ '[head-watcher] @parcel/watcher unavailable; falling back to chokidar',
+ );
+ return null;
+ }
}
try {
const subscription = await subscribeParcel(
@@ -143,6 +149,8 @@ export async function startHeadWatcher(
opts: {
forceBackend?: 'parcel' | 'chokidar';
platform?: NodeJS.Platform;
+ nativeBackend?: NativeWatcherBackend;
+ clock?: WatcherClock;
subscribeForTest?: (
gitDir: string,
dispatch: HeadEventDispatch,
@@ -155,10 +163,11 @@ export async function startHeadWatcher(
}
const repository = inspected.repository;
const gitDir = repository.gitDir;
+ const clock = opts.clock ?? watcherClock();
let inBatch = false;
- let quietTimer: ReturnType<typeof setTimeout> | null = null;
- let timeoutTimer: ReturnType<typeof setTimeout> | null = null;
+ let quietTimer: WatcherTimer | null = null;
+ let timeoutTimer: WatcherTimer | null = null;
let oldHead: string | null = null;
let lastKnownBranch: string | null = null;
let batchEndInFlight: Promise<void> | null = null;
@@ -174,11 +183,11 @@ export async function startHeadWatcher(
if (!inBatch) return;
if (quietTimer) {
- clearTimeout(quietTimer);
+ clock.clearTimeout(quietTimer);
quietTimer = null;
}
if (timeoutTimer) {
- clearTimeout(timeoutTimer);
+ clock.clearTimeout(timeoutTimer);
timeoutTimer = null;
}
@@ -226,8 +235,8 @@ export async function startHeadWatcher(
}
function resetQuietWindow(): void {
- if (quietTimer) clearTimeout(quietTimer);
- quietTimer = setTimeout(() => {
+ if (quietTimer) clock.clearTimeout(quietTimer);
+ quietTimer = clock.setTimeout(() => {
quietTimer = null;
void emitBatchEnd(false);
}, QUIET_WINDOW_MS);
@@ -251,7 +260,7 @@ export async function startHeadWatcher(
await beginPromise;
beginInFlight = null;
- timeoutTimer = setTimeout(() => {
+ timeoutTimer = clock.setTimeout(() => {
timeoutTimer = null;
void emitBatchEnd(true);
}, BATCH_TIMEOUT_MS);
@@ -295,6 +304,7 @@ export async function startHeadWatcher(
dispatch,
recover,
opts.platform ?? process.platform,
+ opts.nativeBackend,
);
if (resolvedUnsub) backend = 'parcel';
}
@@ -321,8 +331,8 @@ export async function startHeadWatcher(
} finally {
await headWork;
if (inBatch) await emitBatchEnd(false);
- if (quietTimer) clearTimeout(quietTimer);
- if (timeoutTimer) clearTimeout(timeoutTimer);
+ if (quietTimer) clock.clearTimeout(quietTimer);
+ if (timeoutTimer) clock.clearTimeout(timeoutTimer);
}
},
getLastKnownBranch: () => lastKnownBranch,
diff --git a/packages/server/src/parcel-recovery.ts b/packages/server/src/parcel-recovery.ts
index d28bfbed8..1f70a7798 100644
--- a/packages/server/src/parcel-recovery.ts
+++ b/packages/server/src/parcel-recovery.ts
@@ -1,6 +1,6 @@
import type { Event } from '@parcel/watcher';
-const RESCAN_NOTICES = new Set([
+export const RESCAN_NOTICES: ReadonlySet<string> = new Set([
'Events were dropped by the FSEvents client. File system must be re-scanned.',
'Too many events. File system must be re-scanned.',
'Events were dropped by the kernel. File system must be re-scanned.',
diff --git a/packages/server/src/parcel-subscription.ts b/packages/server/src/parcel-subscription.ts
index 680526e8e..bc046a66e 100644
--- a/packages/server/src/parcel-subscription.ts
+++ b/packages/server/src/parcel-subscription.ts
@@ -1,7 +1,9 @@
import type { AsyncSubscription, Options, SubscribeCallback } from '@parcel/watcher';
+export type NativeWatcherBackend = Pick<typeof import('@parcel/watcher'), 'subscribe'>;
+
export async function subscribeParcel(
- parcel: Pick<typeof import('@parcel/watcher'), 'subscribe'>,
+ parcel: NativeWatcherBackend,
dir: string,
onEvents: SubscribeCallback,
options?: Options,
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 7556a78f9..0c1ffd0ed 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -162,7 +162,9 @@ import {
type DiskEvent,
getMarkdownRemovalCurrentness,
hasAdmittedMarkdownFile,
+ type NativeWatcherBackend,
pathToDocName,
+ type RecheckResult,
reconcileFileIndexAfterFilterRebuild,
registerRemoval,
registerWrite,
@@ -328,6 +330,7 @@ import { createSyncHandshakeSpanExtension } from './sync-handshake-span-extensio
import { initTelemetry, shutdownTelemetry, withSpan } from './telemetry.ts';
import { trustSystemCertificates } from './trust-system-ca.ts';
import { cleanupOrphanUploadTempfiles } from './upload-streaming.ts';
+import type { WatcherClock } from './watcher-clock.ts';
export interface ServerOptions {
acpRegistryFetchImpl?: typeof fetch;
@@ -374,6 +377,8 @@ export interface ServerOptions {
embedderLoader?: (input: LoadOpenAiEmbedderInput) => Promise<Embedder | null>;
singleDocRelPath?: string;
ephemeral?: boolean;
+ watcherBackend?: NativeWatcherBackend;
+ watcherClock?: WatcherClock;
generatedIndexTestHooks?: {
beforePlan?: (context: { fullSweep: boolean; signal: AbortSignal }) => Promise<void> | void;
beforeDecision?: (context: {
@@ -423,6 +428,7 @@ export interface ServerInstance {
readonly resolveTrackedFile: (relativePath: string) => string | undefined;
readonly acpRegistry: AcpRegistry;
readonly acpPermissions: AcpPermissionStore;
+ readonly recheckDisk: () => Promise<RecheckResult>;
}
export function observeShutdownDocumentRetirements(
@@ -4642,6 +4648,8 @@ export function createServer(options: ServerOptions): ServerInstance {
startWatcher(canonicalContentDir, onDiskEvent, contentFilter, {
onRawBatch,
onRecoveryComplete: () => derivedDocumentIndex.recordInventoryReconciled(),
+ nativeBackend: options.watcherBackend,
+ clock: options.watcherClock,
}),
);
void resyncRecordedSkillCopies(projectDir, contentDir)
@@ -5119,6 +5127,7 @@ export function createServer(options: ServerOptions): ServerInstance {
}
}
},
+ { nativeBackend: options.watcherBackend, clock: options.watcherClock },
);
} catch (err) {
log.error({ err }, '[server] HEAD watcher failed to start');
@@ -5281,5 +5290,13 @@ export function createServer(options: ServerOptions): ServerInstance {
resolveTrackedFile,
acpRegistry,
acpPermissions,
+ recheckDisk: () => {
+ if (!watcher) {
+ return Promise.reject(
+ new Error('the file watcher is not running, so there is nothing to re-check'),
+ );
+ }
+ return watcher.recheck();
+ },
};
}
diff --git a/packages/server/src/watcher-clock.test-helper.ts b/packages/server/src/watcher-clock.test-helper.ts
new file mode 100644
index 000000000..4cd8d5a22
--- /dev/null
+++ b/packages/server/src/watcher-clock.test-helper.ts
@@ -0,0 +1,59 @@
+import type { WatcherClock, WatcherTimer } from './watcher-clock.ts';
+
+interface ScheduledTimer extends WatcherTimer {
+ due: number;
+ interval: number | null;
+ callback: () => void;
+}
+
+export interface ManualClock extends WatcherClock {
+ advance(ms: number): void;
+ pendingTimers(): number;
+}
+
+export function createManualClock(startMs = 1_700_000_000_000): ManualClock {
+ let current = startMs;
+ const timers: ScheduledTimer[] = [];
+
+ const schedule = (callback: () => void, delayMs: number, interval: number | null) => {
+ const timer: ScheduledTimer = {
+ due: current + Math.max(0, delayMs),
+ interval,
+ callback,
+ unref: () => timer,
+ };
+ timers.push(timer);
+ return timer;
+ };
+
+ const cancel = (timer: WatcherTimer | null | undefined) => {
+ if (!timer) return;
+ const index = timers.indexOf(timer as ScheduledTimer);
+ if (index >= 0) timers.splice(index, 1);
+ };
+
+ return {
+ now: () => current,
+ setTimeout: (callback, delayMs) => schedule(callback, delayMs, null),
+ clearTimeout: cancel,
+ setInterval: (callback, intervalMs) => schedule(callback, intervalMs, Math.max(1, intervalMs)),
+ clearInterval: cancel,
+ advance(ms) {
+ const target = current + ms;
+ for (;;) {
+ const due = timers.filter((timer) => timer.due <= target).sort((a, b) => a.due - b.due);
+ const next = due[0];
+ if (!next) break;
+ current = Math.max(current, next.due);
+ if (next.interval === null) {
+ timers.splice(timers.indexOf(next), 1);
+ } else {
+ next.due = current + next.interval;
+ }
+ next.callback();
+ }
+ current = target;
+ },
+ pendingTimers: () => timers.length,
+ };
+}
diff --git a/packages/server/src/watcher-clock.test.ts b/packages/server/src/watcher-clock.test.ts
new file mode 100644
index 000000000..dc578db12
--- /dev/null
+++ b/packages/server/src/watcher-clock.test.ts
@@ -0,0 +1,169 @@
+import { mkdirSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import {
+ evictStaleTrackerEntries,
+ isSelfRemoval,
+ registerRemoval,
+ registerWrite,
+ removalTracker,
+ startWatcher,
+ type WatcherHandle,
+ writeTracker,
+} from './file-watcher.ts';
+import type { NativeWatcherBackend } from './parcel-subscription.ts';
+import { createManualClock } from './watcher-clock.test-helper.ts';
+import {
+ installWatcherClock,
+ systemWatcherClock,
+ type WatcherClock,
+ watcherClock,
+} from './watcher-clock.ts';
+import { createWatcherDouble } from './watcher-double.test-helper.ts';
+
+describe('watcher clock', () => {
+ let restore: (() => void) | undefined;
+ let dir: string;
+ const running = new Set<WatcherHandle>();
+
+ async function start(contentDir: string, clock: WatcherClock): Promise<WatcherHandle> {
+ mkdirSync(contentDir, { recursive: true });
+ const handle = await startWatcher(contentDir, async () => {}, undefined, {
+ nativeBackend: createWatcherDouble().backend,
+ clock,
+ });
+ running.add(handle);
+ return handle;
+ }
+
+ async function stop(handle: WatcherHandle): Promise<void> {
+ running.delete(handle);
+ await handle.unsubscribe();
+ }
+
+ afterEach(async () => {
+ for (const handle of running) await handle.unsubscribe();
+ running.clear();
+ restore?.();
+ restore = undefined;
+ writeTracker.clear();
+ removalTracker.clear();
+ if (dir) rmSync(dir, { recursive: true, force: true });
+ });
+
+ test('the system clock is active until one is installed, and the restore puts it back', () => {
+ expect(watcherClock()).toBe(systemWatcherClock);
+ const manual = createManualClock();
+ restore = installWatcherClock(manual);
+ expect(watcherClock()).toBe(manual);
+ restore();
+ restore = undefined;
+ expect(watcherClock()).toBe(systemWatcherClock);
+ });
+
+ test('write-tracker eviction reads the installed clock, not the wall clock', () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-watcher-clock-'));
+ const manual = createManualClock();
+ restore = installWatcherClock(manual);
+ const file = join(dir, 'doc.md');
+ registerWrite(file, 'hash-1');
+ expect(writeTracker.size).toBe(1);
+
+ manual.advance(9_999);
+ evictStaleTrackerEntries();
+ expect(writeTracker.size).toBe(1);
+
+ manual.advance(2);
+ evictStaleTrackerEntries();
+ expect(writeTracker.size).toBe(0);
+ });
+
+ test('a declared removal is a self removal only inside its window on the installed clock', () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-watcher-clock-'));
+ const manual = createManualClock();
+ restore = installWatcherClock(manual);
+ const file = join(dir, 'gone.md');
+
+ registerRemoval(file);
+ manual.advance(30_001);
+ expect(isSelfRemoval(file)).toBe(false);
+
+ registerRemoval(file);
+ manual.advance(29_999);
+ expect(isSelfRemoval(file)).toBe(true);
+ });
+
+ test('the manual clock runs timers in due order and reschedules intervals', () => {
+ const manual = createManualClock(1_000);
+ const order: string[] = [];
+ manual.setTimeout(() => order.push('late'), 50);
+ manual.setTimeout(() => order.push('early'), 10);
+ const interval = manual.setInterval(() => order.push('tick'), 20);
+ manual.advance(55);
+ expect(order).toEqual(['early', 'tick', 'tick', 'late']);
+ expect(manual.now()).toBe(1_055);
+ manual.clearInterval(interval);
+ manual.advance(100);
+ expect(order).toEqual(['early', 'tick', 'tick', 'late']);
+ expect(manual.pendingTimers()).toBe(0);
+ });
+
+ test('a started watcher installs its clock until it unsubscribes', async () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-watcher-clock-'));
+ const manual = createManualClock();
+ const handle = await start(join(dir, 'content'), manual);
+ expect(watcherClock()).toBe(manual);
+
+ await stop(handle);
+ expect(watcherClock()).toBe(systemWatcherClock);
+ });
+
+ test('a watcher that fails to start leaves the system clock installed and no timers behind', async () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-watcher-clock-'));
+ const manual = createManualClock();
+ const refusingBackend: NativeWatcherBackend = {
+ subscribe: async () => {
+ throw new Error('subscribe refused');
+ },
+ };
+ await expect(
+ startWatcher(dir, async () => {}, undefined, {
+ forceBackend: 'parcel',
+ nativeBackend: refusingBackend,
+ clock: manual,
+ }),
+ ).rejects.toThrow('forced backend');
+ expect(watcherClock()).toBe(systemWatcherClock);
+ expect(manual.pendingTimers()).toBe(0);
+ });
+
+ test('two watchers on two clocks unwind to the system clock when stopped in start order', async () => {
+ dir = mkdtempSync(join(tmpdir(), 'ok-watcher-clock-'));
+ const first = createManualClock();
+ const second = createManualClock();
+ const firstHandle = await start(join(dir, 'first'), first);
+ expect(watcherClock()).toBe(first);
+ const secondHandle = await start(join(dir, 'second'), second);
+ expect(watcherClock()).toBe(second);
+
+ await stop(firstHandle);
+ expect(watcherClock()).toBe(second);
+
+ await stop(secondHandle);
+ expect(watcherClock()).toBe(systemWatcherClock);
+ });
+
+ test('calling a restore twice removes only the entry it installed', () => {
+ const manual = createManualClock();
+ const restoreFirst = installWatcherClock(manual);
+ const restoreSecond = installWatcherClock(manual);
+
+ restoreFirst();
+ restoreFirst();
+ expect(watcherClock()).toBe(manual);
+
+ restoreSecond();
+ expect(watcherClock()).toBe(systemWatcherClock);
+ });
+});
diff --git a/packages/server/src/watcher-clock.ts b/packages/server/src/watcher-clock.ts
new file mode 100644
index 000000000..9669c470c
--- /dev/null
+++ b/packages/server/src/watcher-clock.ts
@@ -0,0 +1,40 @@
+export interface WatcherTimer {
+ unref(): unknown;
+}
+
+export interface WatcherClock {
+ now(): number;
+ setTimeout(callback: () => void, delayMs: number): WatcherTimer;
+ clearTimeout(timer: WatcherTimer | null | undefined): void;
+ setInterval(callback: () => void, intervalMs: number): WatcherTimer;
+ clearInterval(timer: WatcherTimer | null | undefined): void;
+}
+
+export const systemWatcherClock: WatcherClock = {
+ now: () => Date.now(),
+ setTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
+ clearTimeout: (timer) => {
+ if (timer) clearTimeout(timer as NodeJS.Timeout);
+ },
+ setInterval: (callback, intervalMs) => setInterval(callback, intervalMs),
+ clearInterval: (timer) => {
+ if (timer) clearInterval(timer as NodeJS.Timeout);
+ },
+};
+
+const installed: WatcherClock[] = [];
+
+export function watcherClock(): WatcherClock {
+ return installed[installed.length - 1] ?? systemWatcherClock;
+}
+
+export function installWatcherClock(clock: WatcherClock): () => void {
+ installed.push(clock);
+ let restored = false;
+ return () => {
+ if (restored) return;
+ restored = true;
+ const index = installed.lastIndexOf(clock);
+ if (index >= 0) installed.splice(index, 1);
+ };
+}
diff --git a/packages/server/src/watcher-double.test-helper.ts b/packages/server/src/watcher-double.test-helper.ts
new file mode 100644
index 000000000..a49c80425
--- /dev/null
+++ b/packages/server/src/watcher-double.test-helper.ts
@@ -0,0 +1,60 @@
+import { RESCAN_NOTICES } from './parcel-recovery.ts';
+import type { NativeWatcherBackend } from './parcel-subscription.ts';
+
+type SubscribeArgs = Parameters<NativeWatcherBackend['subscribe']>;
+type SubscribeCallback = SubscribeArgs[1];
+type WatcherDoubleEvent = Parameters<SubscribeCallback>[1][number];
+
+interface WatcherDoubleSubscription {
+ readonly dir: string;
+ readonly closed: boolean;
+ deliver(events: WatcherDoubleEvent[]): void;
+ fail(error: Error): void;
+ rescan(events?: WatcherDoubleEvent[]): void;
+}
+
+export function createWatcherDouble(): {
+ readonly backend: NativeWatcherBackend;
+ subscriptionFor(dir: string): WatcherDoubleSubscription;
+} {
+ const subscriptions: WatcherDoubleSubscription[] = [];
+ const [rescanNotice] = RESCAN_NOTICES;
+ const backend: NativeWatcherBackend = {
+ async subscribe(dir, callback) {
+ let closed = false;
+ const subscription: WatcherDoubleSubscription = {
+ dir,
+ get closed() {
+ return closed;
+ },
+ deliver(events) {
+ if (!closed) callback(null, events);
+ },
+ fail(error) {
+ if (!closed) callback(error, []);
+ },
+ rescan(events = []) {
+ if (!closed) callback(new Error(rescanNotice), events);
+ },
+ };
+ subscriptions.push(subscription);
+ return {
+ async unsubscribe() {
+ closed = true;
+ },
+ };
+ },
+ };
+ return {
+ backend,
+ subscriptionFor(dir) {
+ const found = subscriptions.find((subscription) => subscription.dir === dir);
+ if (!found) {
+ throw new Error(
+ `no watcher double subscription for ${dir}; have ${subscriptions.map((s) => s.dir).join(', ') || 'none'}`,
+ );
+ }
+ return found;
+ },
+ };
+}