Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Report the chokidar watcher ready only once FSEvents is live (#4976)
Nick Gomez22dd06ba4f59
diff --git a/.changeset/chokidar-watcher-macos-startup-creations.md b/.changeset/chokidar-watcher-macos-startup-creations.mdnew file mode 100644index 000000000..6817550b3--- /dev/null+++ b/.changeset/chokidar-watcher-macos-startup-creations.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++On macOS, files and folders created right after OpenKnowledge starts are no longer missed when it runs its fallback file watcher.diff --git a/packages/server/src/file-watcher-chokidar-readiness.test.ts b/packages/server/src/file-watcher-chokidar-readiness.test.tsnew file mode 100644index 000000000..b75f74053--- /dev/null+++ b/packages/server/src/file-watcher-chokidar-readiness.test.ts@@ -0,0 +1,230 @@+import fs, {+ type FSWatcher,+ lstatSync,+ mkdirSync,+ readdirSync,+ realpathSync,+ statSync,+ writeFileSync,+} from 'node:fs';+import { mkdtemp, rm } from 'node:fs/promises';+import { syncBuiltinESMExports } from 'node:module';+import { tmpdir } from 'node:os';+import { join, relative, resolve } from 'node:path';+import { afterEach, beforeEach, describe, test } from 'vitest';+import { createContentFilter } from './content-filter.ts';+import { type DiskEvent, lastKnownHash, startWatcher, writeTracker } from './file-watcher.ts';+import { waitWithinTestBudget } from './wait-within-test-budget.test-helper.ts';++const DELIVERY_LIVENESS_BOUND_MS = 6_000;+const DELIVERY_POLL_MS = 40;+const CONFLICTED = '# Daily\n\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> branch\n';++interface ModelledDirectoryWatch {+ readonly path: string;+ live: boolean;+ closed: boolean;+ sinceNow: string | null;+}++interface MacosDirectoryWatchModel {+ heldDirectories(): string[];+ directoriesNotYetLive(): string[];+ restore(): void;+}++function directoryFingerprint(dir: string): string | null {+ if (statSync(dir, { throwIfNoEntry: false }) === undefined) return null;+ return readdirSync(dir)+ .sort()+ .map((name) => {+ const entry = lstatSync(join(dir, name), { bigint: true, throwIfNoEntry: false });+ if (entry === undefined) return name;+ return [name, entry.ino, entry.mtimeNs, entry.ctimeNs, entry.size].join('\0');+ })+ .join('/');+}++function installMacosDirectoryWatchModel(): MacosDirectoryWatchModel {+ const descriptor = Object.getOwnPropertyDescriptor(fs, 'watch');+ if (descriptor === undefined) throw new Error('fs.watch descriptor is unavailable');+ const realWatch = fs.watch;+ const directoryWatches: ModelledDirectoryWatch[] = [];++ const completePendingReschedule = (): void => {+ for (const watch of directoryWatches) {+ if (watch.closed || watch.live) continue;+ watch.live = true;+ watch.sinceNow = directoryFingerprint(watch.path);+ }+ };++ const modelledWatch = (...args: unknown[]): FSWatcher => {+ const watcher: FSWatcher = Reflect.apply(realWatch, fs, args);+ const path = resolve(String(args[0]));+ if (statSync(path, { throwIfNoEntry: false })?.isDirectory() !== true) return watcher;++ const watch: ModelledDirectoryWatch = { path, live: false, closed: false, sinceNow: null };+ directoryWatches.push(watch);+ const emit = watcher.emit.bind(watcher);+ const close = watcher.close.bind(watcher);+ Object.defineProperty(watcher, 'emit', {+ configurable: true,+ writable: true,+ value: (event: string | symbol, ...rest: unknown[]): boolean => {+ if (event !== 'change') return emit(event, ...rest);+ if (!watch.live || directoryFingerprint(path) === watch.sinceNow) return false;+ return emit(event, ...rest);+ },+ });+ Object.defineProperty(watcher, 'close', {+ configurable: true,+ writable: true,+ value: (): void => {+ const firstClose = !watch.closed;+ watch.closed = true;+ close();+ if (firstClose) completePendingReschedule();+ },+ });+ return watcher;+ };++ Object.defineProperty(fs, 'watch', { ...descriptor, value: modelledWatch });+ syncBuiltinESMExports();++ return {+ heldDirectories: () =>+ directoryWatches.filter((watch) => !watch.closed).map((watch) => watch.path),+ directoriesNotYetLive: () =>+ directoryWatches.filter((watch) => !watch.closed && !watch.live).map((watch) => watch.path),+ restore: () => {+ Object.defineProperty(fs, 'watch', descriptor);+ syncBuiltinESMExports();+ },+ };+}++const STANDUP_TEMPLATE =+ '---\ntitle: Standup\ndescription: a standup template\n---\n\n# {{date}}\n';++type BeforeCreation = 'nothing' | 'close-a-directory-watch';++interface ExposedCreation {+ readonly creation: string;+ readonly expected: string;+ readonly fixture?: (contentDir: string) => void;+ readonly create: (contentDir: string) => void;+ readonly matches: (event: DiskEvent) => boolean;+}++const EXPOSED_CREATIONS: readonly ExposedCreation[] = [+ {+ creation: 'a conflicted file created in an already-watched directory',+ expected: "a conflict DiskEvent for '.ok/templates/daily'",+ fixture: (contentDir) =>+ mkdirSync(resolve(contentDir, '.ok', 'templates'), { recursive: true }),+ create: (contentDir) =>+ writeFileSync(resolve(contentDir, '.ok', 'templates', 'daily.md'), CONFLICTED),+ matches: (e) => e.kind === 'conflict' && e.docName === '.ok/templates/daily',+ },+ {+ creation: 'a directory created in the content root',+ expected: "a folder-create DiskEvent for 'fresh'",+ fixture: (contentDir) => {+ mkdirSync(resolve(contentDir, 'sub'), { recursive: true });+ writeFileSync(resolve(contentDir, 'root.md'), '# Root\n');+ writeFileSync(resolve(contentDir, 'sub', 'note.md'), '# Note\n\n[Root](./root)\n');+ },+ create: (contentDir) => mkdirSync(resolve(contentDir, 'fresh')),+ matches: (e) => e.kind === 'folder-create' && e.relativePath === 'fresh',+ },+ {+ creation: 'a template created under a brand-new nested folder',+ expected: "a create or update DiskEvent for 'notes/.ok/templates/standup'",+ create: (contentDir) => {+ const target = resolve(contentDir, 'notes', '.ok', 'templates', 'standup.md');+ mkdirSync(resolve(target, '..'), { recursive: true });+ writeFileSync(target, STANDUP_TEMPLATE);+ },+ matches: (e) =>+ (e.kind === 'create' || e.kind === 'update') && e.docName === 'notes/.ok/templates/standup',+ },+];++async function expectDiskEventForCreationAfterStart(+ site: ExposedCreation,+ dirs: { projectDir: string; contentDir: string },+ beforeCreation: BeforeCreation,+): Promise<void> {+ const { projectDir, contentDir } = dirs;+ site.fixture?.(contentDir);+ const contentRoot = realpathSync(contentDir);+ const filter = createContentFilter({ projectDir, contentDir });+ const events: DiskEvent[] = [];+ const model = installMacosDirectoryWatchModel();+ try {+ const handle = await startWatcher(contentDir, async (e) => void events.push(e), filter, {+ forceBackend: 'chokidar',+ platform: 'darwin',+ });+ try {+ if (!model.heldDirectories().includes(contentRoot)) {+ throw new Error(+ `chokidar holds no node:fs watch on the content root ${contentRoot} when startWatcher resolves, so this test is not exercising the directory-watch readiness it models`,+ );+ }+ if (beforeCreation === 'close-a-directory-watch') fs.watch(contentRoot).close();+ const notYetLive = model+ .directoriesNotYetLive()+ .map((dir) => relative(contentRoot, dir) || '.');+ site.create(contentDir);+ await waitWithinTestBudget(+ `${site.expected} (directory watches not yet live when it was created: ${+ notYetLive.length === 0 ? 'none' : notYetLive.join(', ')+ })`,+ () => events.some(site.matches),+ { timeoutMs: DELIVERY_LIVENESS_BOUND_MS, pollMs: DELIVERY_POLL_MS },+ );+ } finally {+ await handle.unsubscribe();+ }+ } finally {+ model.restore();+ }+}++describe('chokidar backend: creations made right after startWatcher resolves, with macOS directory watches registered asynchronously', () => {+ let tmpDir: string;+ let contentDir: string;++ beforeEach(async () => {+ tmpDir = await mkdtemp(resolve(tmpdir(), 'ok-chokidar-readiness-'));+ contentDir = resolve(tmpDir, 'content');+ mkdirSync(contentDir, { recursive: true });+ lastKnownHash.clear();+ writeTracker.clear();+ });++ afterEach(async () => {+ await rm(tmpDir, { recursive: true, force: true });+ });++ for (const site of EXPOSED_CREATIONS) {+ test(`${site.creation} right after startWatcher resolves dispatches ${site.expected}`, async () => {+ await expectDiskEventForCreationAfterStart(+ site,+ { projectDir: tmpDir, contentDir },+ 'nothing',+ );+ });++ test(`${site.creation} after a directory watch close has completed the pending FSEvents reschedule dispatches ${site.expected}`, async () => {+ await expectDiskEventForCreationAfterStart(+ site,+ { projectDir: tmpDir, contentDir },+ 'close-a-directory-watch',+ );+ });+ }+});diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex 0b131a21c..16db67578 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -6,6 +6,7 @@ import {realpathSync,type Stats,statSync,+ watch as watchFsPath,} from 'node:fs';import { lstat, readdir, readFile, realpath, stat } from 'node:fs/promises';import { basename, dirname, extname, join, relative } from 'node:path';@@ -1537,6 +1538,21 @@ export function isChokidarPathIgnored(const CHOKIDAR_READY_TIMEOUT_MS = 10_000;+/* UPSTREAM(nodejs/node#52601, libuv/libuv#3866, libuv/libuv#637): a macOS directory fs.watch+ returns before FSEvents covers its path; closing one waits for the stream rebuild that covers+ every earlier registration. */+function waitForPendingFsEventsRegistrations(dir: string, platform: NodeJS.Platform): void {+ if (platform !== 'darwin') return;+ try {+ watchFsPath(dir).close();+ } catch (err) {+ log.warn(+ { err },+ 'could not wait for FSEvents to cover the chokidar-watched directories; a change made right after start may be missed',+ );+ }+}+async function startChokidarWatcher(contentDir: string,contentFilter: ContentFilter | undefined,@@ -1545,6 +1561,7 @@ async function startChokidarWatcher(onDiskEvent: (event: DiskEvent) => Promise<void>,aliasMap: Map<string, string>,onAfterMutation: () => void,+ platform: NodeJS.Platform,onRawBatch?: (absPaths: readonly string[]) => void,): Promise<AsyncSubscription> {const { watch } = await import('chokidar');@@ -1602,6 +1619,8 @@ async function startChokidarWatcher(setTimeout(settle, CHOKIDAR_READY_TIMEOUT_MS).unref();});+ waitForPendingFsEventsRegistrations(contentDir, platform);+return {unsubscribe: () => {if (batchTimer) {@@ -1621,6 +1640,7 @@ export async function startWatcher(opts: {forceBackend?: 'parcel' | 'chokidar';onRawBatch?: (absPaths: readonly string[]) => void;+ platform?: NodeJS.Platform;} = {},): Promise<WatcherHandle> {const { onRawBatch } = opts;@@ -1694,6 +1714,7 @@ export async function startWatcher(onDiskEvent,aliasMap,bumpFileIndexGeneration,+ opts.platform ?? process.platform,onRawBatch,);backend = 'chokidar';diff --git a/packages/server/tsconfig.test-helper-callers.json b/packages/server/tsconfig.test-helper-callers.jsonindex 270181f8b..c59240a4e 100644--- a/packages/server/tsconfig.test-helper-callers.json+++ b/packages/server/tsconfig.test-helper-callers.json@@ -4,6 +4,7 @@"files": ["src/expect-stable.test.ts","src/file-watcher-chokidar-fallback.test.ts",+ "src/file-watcher-chokidar-readiness.test.ts","src/managed-artifact-watcher.test.ts","src/persistence-tripwire-block.test.ts","src/persistence-tripwire-paste.test.ts",