Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Run two skipped server suites in CI and fix the watcher race they hid (#4980)
Nick Gomez4dbbd2344914
diff --git a/.changeset/stat-poll-watcher-startup-changes.md b/.changeset/stat-poll-watcher-startup-changes.mdnew file mode 100644index 000000000..efc484639--- /dev/null+++ b/.changeset/stat-poll-watcher-startup-changes.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge-server": patch+---++Edits to skill and config files made just after file watching starts are no longer missed.diff --git a/packages/app/package.json b/packages/app/package.jsonindex 219a2326a..10b1dcafc 100644--- a/packages/app/package.json+++ b/packages/app/package.json@@ -210,7 +210,6 @@"@vitejs/plugin-react": "^6.0.1","axe-core": "~4.12.1","babel-plugin-react-compiler": "0.0.0-experimental-a8e64ef-20260402",- "chokidar": "^5.0.0","commonmark.json": "0.31.0","diff": "^8.0.4","fake-indexeddb": "^6.0.0",diff --git a/packages/app/tests/integration/config-hot-apply-without-watcher-echo.test.ts b/packages/app/tests/integration/config-hot-apply-without-watcher-echo.test.tsindex 42b301a59..651ffffd1 100644--- a/packages/app/tests/integration/config-hot-apply-without-watcher-echo.test.ts+++ b/packages/app/tests/integration/config-hot-apply-without-watcher-echo.test.ts@@ -1,34 +1,25 @@import { execFile } from 'node:child_process';-import { EventEmitter } from 'node:events';import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { dirname, join } from 'node:path';import { promisify } from 'node:util';import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest';-class SwallowingWatcher extends EventEmitter {- constructor() {- super();- setTimeout(() => this.emit('ready'), 10);- }- add(): this {- return this;- }- unwatch(): this {- return this;- }- async close(): Promise<void> {}-}-const { watch: realChokidarWatch } = await import('chokidar');-let swallowChokidarEvents = true;+const statPollWatchers = vi.hoisted(() => ({ swallowEvents: true, swallowed: 0 }));afterAll(() => {- swallowChokidarEvents = false;+ statPollWatchers.swallowEvents = false;+});+vi.mock('../../../server/src/polled-path-watcher.ts', async (importOriginal) => {+ const actual =+ await importOriginal<typeof import('../../../server/src/polled-path-watcher.ts')>();+ return {+ ...actual,+ startPolledPathWatcher: (...args: Parameters<typeof actual.startPolledPathWatcher>) => {+ if (!statPollWatchers.swallowEvents) return actual.startPolledPathWatcher(...args);+ statPollWatchers.swallowed += 1;+ return Promise.resolve(async () => {});+ },+ };});-vi.doMock('chokidar', () => ({- watch: (...args: Parameters<typeof realChokidarWatch>) =>- swallowChokidarEvents- ? (new SwallowingWatcher() as unknown as ReturnType<typeof realChokidarWatch>)- : realChokidarWatch(...args),-}));import { HocuspocusProvider } from '@hocuspocus/provider';import {@@ -44,13 +35,10 @@ import {pollUntil,type SyncWiredTestServer,type TestServer,- wait,} from './test-harness';const execFileAsync = promisify(execFile);-const FALLBACK_POLL_WINDOW_MS = 11_000;-describe('PRD-7260 — persisted config change reaches in-process consumers without the watcher echo', () => {let srv: TestServer;let ydoc: Y.Doc;@@ -61,8 +49,9 @@ describe('PRD-7260 — persisted config change reaches in-process consumers withlet projectBinding: ConfigBinding;beforeAll(async () => {+ const swallowedBefore = statPollWatchers.swallowed;srv = await createTestServer({ seedProjectConfigYml: '{}\n' });- await wait(FALLBACK_POLL_WINDOW_MS);+ expect(statPollWatchers.swallowed).toBeGreaterThan(swallowedBefore);ydoc = new Y.Doc();provider = new HocuspocusProvider({@@ -176,11 +165,12 @@ describe('configured attachment folder composes into real Git sync without the w};beforeAll(async () => {+ const swallowedBefore = statPollWatchers.swallowed;srv = await createSyncWiredTestServer({originSeed: { 'seed.txt': 'seed\n' },projectConfigYml: '{}\n',});- await wait(FALLBACK_POLL_WINDOW_MS);+ expect(statPollWatchers.swallowed).toBeGreaterThan(swallowedBefore);ydoc = new Y.Doc();provider = new HocuspocusProvider({@@ -240,8 +230,9 @@ describe('PRD-7260 — reconciled config change reaches in-process consumers witlet binding: ConfigBinding;beforeAll(async () => {+ const swallowedBefore = statPollWatchers.swallowed;srv = await createTestServer();- await wait(FALLBACK_POLL_WINDOW_MS);+ expect(statPollWatchers.swallowed).toBeGreaterThan(swallowedBefore);ydoc = new Y.Doc();provider = new HocuspocusProvider({diff --git a/packages/server/src/config-file-watcher-readiness.test.ts b/packages/server/src/config-file-watcher-readiness.test.tsnew file mode 100644index 000000000..a4c2aab5c--- /dev/null+++ b/packages/server/src/config-file-watcher-readiness.test.ts@@ -0,0 +1,183 @@+import { execFileSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { basename, dirname, join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import {+ delivered,+ type ForkedWatcherProcess,+ forkWatcherProcess,+ type MutationTiming,+ type ReadinessScenario,+ reportOf,+ type WatcherEntry,+} from './forked-watcher-process.test-helper.ts';++interface ConfigReadinessCase {+ watcher: string;+ entry: WatcherEntry;+ scenario: ReadinessScenario;+ change: string;+}++const CASES: ConfigReadinessCase[] = [+ { watcher: 'startConfigFileWatcher', entry: 'config', scenario: 'create', change: 'creation' },+ { watcher: 'startConfigFileWatcher', entry: 'config', scenario: 'edit', change: 'edit' },+ {+ watcher: 'startMultiPathConfigFileWatcher',+ entry: 'multi-config',+ scenario: 'create',+ change: 'creation',+ },+ {+ watcher: 'startMultiPathConfigFileWatcher',+ entry: 'multi-config',+ scenario: 'edit',+ change: 'edit',+ },+];++const INITIAL = 'theme: light\n';+const CHANGED = 'theme: dark\nfont: mono\n';++const WATCH_FILE_DEADLOCK =+ 'A first sample fs.watchFile takes of the file stays held until the harness changes the file, ' ++ 'so this deadlocks, by design, a watcher that registers fs.watchFile on the path and then ' ++ 'awaits threadpool work before resolving';++const START_WAIT: Record<MutationTiming, string> = {+ 'as-start-resolves':+ 'the watcher to resolve its start. The harness holds its first baseline sample of the file ' ++ `until start waits on it. ${WATCH_FILE_DEADLOCK}`,+ 'while-first-baseline-stat-held':+ 'the watcher to resolve its start. The harness changes the file while start waits on its held ' ++ `first baseline sample of the file, then releases that sample. ${WATCH_FILE_DEADLOCK}`,+ 'after-first-baseline-stat': 'the watcher to resolve its start',+};++describe.skipIf(process.platform === 'win32')(+ 'config file watcher readiness: a change made after start is called reaches onChange, however late',+ () => {+ let root: string;+ let harness: ForkedWatcherProcess | undefined;++ beforeEach(() => {+ root = mkdtempSync(join(tmpdir(), 'ok-config-readiness-'));+ execFileSync('mkfifo', [join(root, 'threadpool-barrier')]);+ });+ afterEach(async () => {+ await harness?.close();+ harness = undefined;+ rmSync(root, { recursive: true, force: true });+ });++ async function changeTheFile({ entry, scenario }: ConfigReadinessCase, timing: MutationTiming) {+ const project = join(root, 'project');+ const target =+ entry === 'config' ? join(project, '.ok', 'config.yml') : join(project, '.okignore');+ const watch = entry === 'config' ? [target] : [target, join(project, '.gitignore')];+ mkdirSync(dirname(target), { recursive: true });+ if (entry === 'multi-config') writeFileSync(join(project, '.gitignore'), 'dist/\n', 'utf-8');+ if (scenario === 'edit') writeFileSync(target, INITIAL, 'utf-8');+ const run = forkWatcherProcess({+ mode: 'readiness',+ entry,+ watch,+ scenario,+ timing,+ target,+ held: target,+ content: CHANGED,+ barrier: join(root, 'threadpool-barrier'),+ });+ harness = run;+ await run.until(START_WAIT[timing], (reports) => reportOf(reports, 'started') !== undefined);+ await run.until(+ timing === 'while-first-baseline-stat-held'+ ? `the harness to report the change it made to ${basename(target)} while the watcher's first baseline sample of it was held`+ : `the harness to change ${basename(target)} once every repeating timer the watcher armed ` ++ `while starting had stopped (${reportOf(run.reports(), 'started')?.startupTimers} armed). ` ++ 'A watcher that polls on a repeating timer armed during start never lets this happen, ' ++ 'by design: the change must land after the startup compensation such a timer provides',+ (reports) => reportOf(reports, 'mutated') !== undefined,+ );+ return {+ run,+ target,+ started: reportOf(run.reports(), 'started'),+ mutated: reportOf(run.reports(), 'mutated'),+ };+ }++ test.each(CASES)(+ "$watcher resolves start only once its first baseline sample of the file has landed, and reports the file's $change made once start resolved and its startup compensation ended",+ async (readinessCase) => {+ const { run, target, started, mutated } = await changeTheFile(+ readinessCase,+ 'as-start-resolves',+ );+ expect(+ started?.barrierEngaged,+ `the watcher's start took no sample of ${basename(target)} for the harness to hold`,+ ).toBe(true);+ expect(+ started?.firstBaselineStatHeldAtResolve,+ `start resolved before its first baseline sample of ${basename(target)} landed, so a change made then becomes part of the baseline and is never reported`,+ ).toBe(false);++ await run.until(+ `onChange to report the ${readinessCase.change} of ${basename(target)} made once start resolved ` ++ `and the watcher's startup compensation ended (${mutated?.startupTimersWaitedOut} startup timer(s) waited out)`,+ (reports) => delivered(reports, target, CHANGED),+ );+ },+ );++ test.each(CASES)(+ "$watcher reports the file's $change made while start waits on its held first baseline sample of the file",+ async (readinessCase) => {+ const { run, target, started, mutated } = await changeTheFile(+ readinessCase,+ 'while-first-baseline-stat-held',+ );+ expect(+ mutated?.firstBaselineStatHeldAtMutation,+ `the harness changed ${basename(target)} while the watcher's first baseline sample of it was held`,+ ).toBe(true);+ expect(+ started?.firstBaselineStatHeldAtResolve,+ `start resolved before its first baseline sample of ${basename(target)} landed`,+ ).toBe(false);++ await run.until(+ `onChange to report the ${readinessCase.change} of ${basename(target)} made after start read the file ` ++ 'and before its first baseline sample of it landed',+ (reports) => delivered(reports, target, CHANGED),+ );+ },+ );++ test.each(CASES)(+ "$watcher reports the file's $change made once the first baseline stat has run and its startup compensation ended (ordering control)",+ async (readinessCase) => {+ const { run, target, started, mutated } = await changeTheFile(+ readinessCase,+ 'after-first-baseline-stat',+ );+ expect(+ started?.barrierEngaged,+ `the watcher's start took no sample of ${basename(target)} for the harness to hold`,+ ).toBe(true);+ expect(+ mutated?.firstBaselineStatHeldAtMutation,+ 'the control changes the file only after the held first baseline stat has run',+ ).toBe(false);++ await run.until(+ `onChange to report the ${readinessCase.change} of ${basename(target)} made after the first baseline stat`,+ (reports) => delivered(reports, target, CHANGED),+ );+ },+ );+ },+);diff --git a/packages/server/src/config-file-watcher.test.ts b/packages/server/src/config-file-watcher.test.tsindex f8a385248..dcb2aee3a 100644--- a/packages/server/src/config-file-watcher.test.ts+++ b/packages/server/src/config-file-watcher.test.ts@@ -67,15 +67,9 @@ describe('startConfigFileWatcher', () => {expect(existsSync(fx.absPath)).toBe(false);writeFileSync(fx.absPath, 'theme: dark\n', 'utf-8');- let attempt = 0;- const fired = await waitFor(() => {- if (events.length > 0) return true;- attempt++;- writeFileSync(fx.absPath, `theme: dark\nattempt: ${attempt}\n`, 'utf-8');- return false;- }, 20_000);+ const fired = await waitFor(() => events.length > 0, 20_000);expect(fired).toBe(true);- expect(events[0]?.startsWith('theme: dark\n')).toBe(true);+ expect(events[0]).toBe('theme: dark\n');}, 25_000);test('fires onChange when an existing file is modified', async () => {@@ -94,7 +88,7 @@ describe('startConfigFileWatcher', () => {expect(events.at(-1)).toBe('theme: dark\n');});- test('does NOT fire onChange on the initial scan (ignoreInitial)', async () => {+ test('does NOT fire onChange for the content present when the watcher starts', async () => {writeFileSync(fx.absPath, 'theme: light\n', 'utf-8');const events: string[] = [];@@ -107,7 +101,7 @@ describe('startConfigFileWatcher', () => {expect(events).toEqual([]);});- test('atomic tmp+rename produces a single change event (awaitWriteFinish)', async () => {+ test('atomic tmp+rename reports the renamed-in content, in at most two change events', async () => {writeFileSync(fx.absPath, 'theme: light\n', 'utf-8');const events: string[] = [];@@ -220,17 +214,11 @@ describe('startMultiPathConfigFileWatcher', () => {writeFileSync(multiFx.pathA, 'drafts/\n', 'utf-8');- let attempt = 0;- const fired = await waitFor(() => {- if (events.length > 0) return true;- attempt++;- writeFileSync(multiFx.pathA, `drafts/\nattempt: ${attempt}\n`, 'utf-8');- return false;- }, 20_000);+ const fired = await waitFor(() => events.length > 0, 20_000);expect(fired).toBe(true);const matched = events.find((e) => e.path === multiFx.pathA);expect(matched).toBeDefined();- expect(matched?.content.startsWith('drafts/\n')).toBe(true);+ expect(matched?.content).toBe('drafts/\n');expect(events.some((e) => e.path === multiFx.pathB)).toBe(false);}, 25_000);@@ -258,7 +246,7 @@ describe('startMultiPathConfigFileWatcher', () => {expect(fired).toBe(true);});- test('does NOT fire on the initial scan (ignoreInitial honored across both paths)', async () => {+ test('does NOT fire for the content present on either path when the watcher starts', async () => {writeFileSync(multiFx.pathA, 'drafts/\n', 'utf-8');writeFileSync(multiFx.pathB, 'node_modules/\n', 'utf-8');diff --git a/packages/server/src/config-file-watcher.ts b/packages/server/src/config-file-watcher.tsindex b3aa5f839..73dd2654e 100644--- a/packages/server/src/config-file-watcher.ts+++ b/packages/server/src/config-file-watcher.ts@@ -3,6 +3,7 @@ import { dirname } from 'node:path';import { tracedMkdirSync } from './fs-traced.ts';import { errnoCode } from './http/handler-utils.ts';import { getLogger } from './logger.ts';+import { startPolledPathWatcher } from './polled-path-watcher.ts';export type ConfigFileWatcherUnsubscribe = () => Promise<void>;@@ -11,7 +12,6 @@ export async function startConfigFileWatcher(onChange: (content: string) => void,): Promise<ConfigFileWatcherUnsubscribe> {const log = getLogger('config-file-watcher');- const { watch } = await import('chokidar');const watchDir = dirname(absPath);try {@@ -23,19 +23,6 @@ export async function startConfigFileWatcher(}}- const watcher = watch(watchDir, {- ignoreInitial: true,- depth: 0,- usePolling: true,- interval: 200,- awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 },- ignored: (p) => p !== watchDir && p !== absPath,- });-- await new Promise<void>((resolve) => {- watcher.once('ready', resolve);- });-let lastContent: string | null = null;try {lastContent = readFileSync(absPath, 'utf-8');@@ -48,8 +35,9 @@ export async function startConfigFileWatcher(} catch (err) {const code = errnoCode(err);if (code === 'ENOENT') {- if (logMissing)+ if (logMissing) {log.debug({ path }, 'config file disappeared between event and read; dropping');+ }return;}log.warn({ err, path }, 'config file read failed; dropping event');@@ -63,35 +51,20 @@ export async function startConfigFileWatcher(log.warn({ err, path }, 'config file change handler threw');}};- const handler = (path: string): void => handlePath(path);-- watcher.on('add', handler);- watcher.on('change', handler);- watcher.on('unlink', (path) => {+ const handleUnlink = (path: string): void => {if (path !== absPath) return;log.debug({ path }, 'config file unlinked; Y.Text retained at current state');- });- watcher.on('error', (err) => {- log.warn(- { err, watchDir, absPath },- `[config-file-watcher] chokidar error while watching ${absPath}`,- );- });- let fallbackAttempts = 0;- const fallbackPoll = setInterval(() => {- fallbackAttempts++;- handlePath(absPath, false);- if (fallbackAttempts >= 20) clearInterval(fallbackPoll);- }, 500);- fallbackPoll.unref?.();-- let closed = false;- return async () => {- if (closed) return;- closed = true;- clearInterval(fallbackPoll);- await watcher.close();};++ const stop = await startPolledPathWatcher({+ listPaths: async () => [absPath],+ onEvent: (event, path) => (event === 'unlink' ? handleUnlink(path) : handlePath(path)),+ onError: (err, path) => {+ log.warn({ err, path }, '[config-file-watcher] poll error');+ },+ });+ handlePath(absPath, false);+ return stop;}export async function startMultiPathConfigFileWatcher(@@ -102,7 +75,6 @@ export async function startMultiPathConfigFileWatcher(throw new Error('startMultiPathConfigFileWatcher requires at least one absolute path');}const log = getLogger('config-file-watcher');- const { watch } = await import('chokidar');const watchedPaths = new Set(absPaths);const watchDirs = Array.from(new Set(Array.from(watchedPaths, (p) => dirname(p))));@@ -118,19 +90,6 @@ export async function startMultiPathConfigFileWatcher(}}- const watcher = watch(watchDirs, {- ignoreInitial: true,- depth: 0,- usePolling: true,- interval: 200,- awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 },- ignored: (p) => !watchedPaths.has(p) && !watchDirs.includes(p),- });-- await new Promise<void>((resolve) => {- watcher.once('ready', resolve);- });-const lastContent = new Map<string, string | null>();for (const path of watchedPaths) {try {@@ -148,8 +107,9 @@ export async function startMultiPathConfigFileWatcher(} catch (err) {const code = errnoCode(err);if (code === 'ENOENT') {- if (logMissing)+ if (logMissing) {log.debug({ path }, 'config file disappeared between event and read; dropping');+ }return;}log.warn({ err, path }, 'config file read failed; dropping event');@@ -163,36 +123,18 @@ export async function startMultiPathConfigFileWatcher(log.warn({ err, path }, 'config file change handler threw');}};- const handler = (path: string): void => handlePath(path);-- watcher.on('add', handler);- watcher.on('change', handler);- watcher.on('unlink', (path) => {+ const handleUnlink = (path: string): void => {if (!watchedPaths.has(path)) return;log.debug({ path }, 'config file unlinked; downstream state retained');- });- watcher.on('error', (err) => {- log.warn(- { err, watchDirs, paths: Array.from(watchedPaths) },- '[config-file-watcher] chokidar error in multi-path watcher',- );- });-- let fallbackAttempts = 0;- const fallbackPoll = setInterval(() => {- fallbackAttempts++;- for (const path of watchedPaths) {- handlePath(path, false);- }- if (fallbackAttempts >= 20) clearInterval(fallbackPoll);- }, 500);- fallbackPoll.unref?.();-- let closed = false;- return async () => {- if (closed) return;- closed = true;- clearInterval(fallbackPoll);- await watcher.close();};++ const stop = await startPolledPathWatcher({+ listPaths: async () => Array.from(watchedPaths),+ onEvent: (event, path) => (event === 'unlink' ? handleUnlink(path) : handlePath(path)),+ onError: (err, path) => {+ log.warn({ err, path }, '[config-file-watcher] poll error');+ },+ });+ for (const path of watchedPaths) handlePath(path, false);+ return stop;}diff --git a/packages/server/src/forked-watcher-process.test-helper.ts b/packages/server/src/forked-watcher-process.test-helper.tsnew file mode 100644index 000000000..2eecdb3ce--- /dev/null+++ b/packages/server/src/forked-watcher-process.test-helper.ts@@ -0,0 +1,334 @@+import { fork } from 'node:child_process';+import { fileURLToPath } from 'node:url';+import { waitWithinTestBudget } from './wait-within-test-budget.test-helper.ts';++export type WatcherEntry = 'managed' | 'config' | 'multi-config';++export type ReadinessScenario = 'create' | 'edit';++export type MutationTiming =+ | 'as-start-resolves'+ | 'while-first-baseline-stat-held'+ | 'after-first-baseline-stat';++export interface ReadinessSpec {+ mode: 'readiness';+ entry: WatcherEntry;+ watch: string[];+ scenario: ReadinessScenario;+ timing: MutationTiming;+ target: string;+ held: string;+ content: string;+ barrier: string;+}++export interface ContractSpec {+ mode: 'contract';+ entry: WatcherEntry;+ watch: string[];+ coarseTimestampsAt?: string;+}++export type WatcherProcessSpec = ReadinessSpec | ContractSpec;++export type Recovery = 'readable' | 'removed' | 'replaced-by-file';++export type ContractCommand =+ | { op: 'write-in-two-steps'; path: string; partial: string; complete: string }+ | { op: 'unreadable-until-sampled'; dir: string; leaf: string }+ | { op: 'close-while-sampling'; path: string; content: string }+ | { op: 'hold-next-sample'; path: string; content: string; barrier: string }+ | { op: 'unreadable-across-polls'; dir: string; path: string }+ | {+ op: 'unreadable-again-after-recovery';+ dir: string;+ failingAt: string;+ sampledEveryPoll: string;+ recovery: Recovery;+ aside: string;+ }+ | { op: 'error-code-changes-while-failing'; dir: string; failingAt: string; aside: string }+ | { op: 'close-between-polls'; path: string }+ | { op: 'close-while-sample-held'; path: string; dir: string; barrier: string }+ | { op: 'rewrite-in-place'; path: string; content: string }+ | {+ op: 'two-step-write-across-interruption';+ path: string;+ dir: string;+ partial: string;+ complete: string;+ interruption: 'unreadable' | 'removed';+ };++export type WatcherEvent =+ | { kind: 'change'; path: string; content: string }+ | { kind: 'unlink'; path: string };++export interface LoggedError {+ level: string;+ code: string;+ path: string;+ message: string;+}++export type WatcherProcessReport =+ | WatcherEvent+ | {+ kind: 'started';+ barrierEngaged: boolean;+ firstBaselineStatHeldAtResolve: boolean;+ startupTimers: number;+ }+ | {+ kind: 'mutated';+ barrierEngaged: boolean;+ firstBaselineStatHeldAtResolve: boolean;+ firstBaselineStatHeldAtMutation: boolean;+ startupTimersWaitedOut: number;+ }+ | { kind: 'settled'; startupTimers: number }+ | { kind: 'partial-written'; atStartupTimerTick: boolean }+ | { kind: 'partial-sampled' }+ | { kind: 'completed' }+ | { kind: 'unreadable' }+ | { kind: 'readable-again' }+ | { kind: 'closing' }+ | { kind: 'closed' }+ | { kind: 'quiesced'; eventsAfterClose: WatcherEvent[] }+ | { kind: 'sample-held' }+ | { kind: 'overlap-checked'; samplesBegunWhileHeld: number; timersWaitedOut: number }+ | { kind: 'error-reports'; onceEstablished: LoggedError[]; afterAnotherPoll: LoggedError[] }+ | { kind: 'removed' }+ | {+ kind: 'error-reports-across-recovery';+ firstEpisode: LoggedError[];+ afterRecovery: LoggedError[];+ }+ | {+ kind: 'error-reports-across-code-change';+ beforeChange: LoggedError[];+ afterChange: LoggedError[];+ whileItPersists: LoggedError[];+ }+ | { kind: 'idle-after-close'; samplesAfterClose: number }+ | {+ kind: 'idle-after-held-close';+ resolvedWhileHeld: boolean;+ eventsAfterClose: WatcherEvent[];+ loggedAfterClose: LoggedError[];+ }+ | { kind: 'interrupted' }+ | { kind: 'restored' }+ | { kind: 'rewritten-in-place'; statUnchanged: boolean };++type ReportKind = WatcherProcessReport['kind'];++type FieldType = 'string' | 'boolean' | 'number' | 'events' | 'logged';++type FieldTypeOf<Value> = Value extends string+ ? 'string'+ : Value extends boolean+ ? 'boolean'+ : Value extends number+ ? 'number'+ : Value extends WatcherEvent[]+ ? 'events'+ : Value extends LoggedError[]+ ? 'logged'+ : never;++type FieldTypes<Shape> = {+ [Field in Exclude<keyof Shape, 'kind'>]-?: FieldTypeOf<Shape[Field]>;+};++const REPORT_FIELDS: {+ [Kind in ReportKind]: FieldTypes<Extract<WatcherProcessReport, { kind: Kind }>>;+} = {+ change: { path: 'string', content: 'string' },+ unlink: { path: 'string' },+ started: {+ barrierEngaged: 'boolean',+ firstBaselineStatHeldAtResolve: 'boolean',+ startupTimers: 'number',+ },+ mutated: {+ barrierEngaged: 'boolean',+ firstBaselineStatHeldAtResolve: 'boolean',+ firstBaselineStatHeldAtMutation: 'boolean',+ startupTimersWaitedOut: 'number',+ },+ settled: { startupTimers: 'number' },+ 'partial-written': { atStartupTimerTick: 'boolean' },+ 'partial-sampled': {},+ completed: {},+ unreadable: {},+ 'readable-again': {},+ closing: {},+ closed: {},+ quiesced: { eventsAfterClose: 'events' },+ 'sample-held': {},+ 'overlap-checked': { samplesBegunWhileHeld: 'number', timersWaitedOut: 'number' },+ 'error-reports': { onceEstablished: 'logged', afterAnotherPoll: 'logged' },+ removed: {},+ 'error-reports-across-recovery': { firstEpisode: 'logged', afterRecovery: 'logged' },+ 'error-reports-across-code-change': {+ beforeChange: 'logged',+ afterChange: 'logged',+ whileItPersists: 'logged',+ },+ 'idle-after-close': { samplesAfterClose: 'number' },+ 'idle-after-held-close': {+ resolvedWhileHeld: 'boolean',+ eventsAfterClose: 'events',+ loggedAfterClose: 'logged',+ },+ interrupted: {},+ restored: {},+ 'rewritten-in-place': { statUnchanged: 'boolean' },+};++function isReportKind(kind: unknown): kind is ReportKind {+ return typeof kind === 'string' && Object.hasOwn(REPORT_FIELDS, kind);+}++const LOGGED_ERROR_FIELDS: FieldTypes<LoggedError> = {+ level: 'string',+ code: 'string',+ path: 'string',+ message: 'string',+};++function isLoggedError(value: unknown): value is LoggedError {+ if (typeof value !== 'object' || value === null) return false;+ const entry = value as Record<string, unknown>;+ return Object.entries(LOGGED_ERROR_FIELDS).every(([field, type]) => typeof entry[field] === type);+}++function isWatcherProcessReport(value: unknown): value is WatcherProcessReport {+ if (typeof value !== 'object' || value === null) return false;+ const report = value as Record<string, unknown>;+ if (!isReportKind(report.kind)) return false;+ const fields: Readonly<Record<string, FieldType>> = REPORT_FIELDS[report.kind];+ return Object.entries(fields).every(([field, type]) => {+ const fieldValue = report[field];+ if (type === 'events') return Array.isArray(fieldValue) && fieldValue.every(isWatcherEvent);+ if (type === 'logged') return Array.isArray(fieldValue) && fieldValue.every(isLoggedError);+ return typeof fieldValue === type;+ });+}++function isWatcherEvent(value: unknown): value is WatcherEvent {+ return isWatcherProcessReport(value) && (value.kind === 'change' || value.kind === 'unlink');+}++export function reportOf<Kind extends ReportKind>(+ reports: readonly WatcherProcessReport[],+ kind: Kind,+): Extract<WatcherProcessReport, { kind: Kind }> | undefined {+ return reports.find(+ (report): report is Extract<WatcherProcessReport, { kind: Kind }> => report.kind === kind,+ );+}++export function deliveriesFor(reports: readonly WatcherProcessReport[], path: string): string[] {+ return reports.flatMap((report) =>+ report.kind === 'change' && report.path === path ? [report.content] : [],+ );+}++export function delivered(+ reports: readonly WatcherProcessReport[],+ path: string,+ content: string,+): boolean {+ return deliveriesFor(reports, path).includes(content);+}++export function unlinked(reports: readonly WatcherProcessReport[], path: string): boolean {+ return reports.some((report) => report.kind === 'unlink' && report.path === path);+}++interface WatcherProcessExit {+ code: number | null;+ signal: NodeJS.Signals | null;+}++export interface ForkedWatcherProcess {+ reports(): readonly WatcherProcessReport[];+ exitStatus(): WatcherProcessExit | undefined;+ until(+ what: string,+ condition: (reports: readonly WatcherProcessReport[]) => boolean,+ ): Promise<void>;+ close(): Promise<void>;+}++export interface ForkedContractProcess extends ForkedWatcherProcess {+ send(command: ContractCommand): void;+}++const WATCHER_PROCESS = fileURLToPath(new URL('./watcher-process.test-helper.ts', import.meta.url));++const BOUNDED_BY_THE_TEST_BUDGET = { timeoutMs: Number.POSITIVE_INFINITY };++export function forkWatcherProcess(spec: ContractSpec): ForkedContractProcess;+export function forkWatcherProcess(spec: ReadinessSpec): ForkedWatcherProcess;+export function forkWatcherProcess(spec: WatcherProcessSpec): ForkedContractProcess {+ const child = fork(WATCHER_PROCESS, [JSON.stringify(spec)], {+ execArgv: ['--import', 'tsx', '--conditions=@inkeep/source'],+ silent: true,+ env: { ...process.env, UV_THREADPOOL_SIZE: '1' },+ });+ const reports: WatcherProcessReport[] = [];+ let stderr = '';+ let protocolError: string | undefined;+ let exitStatus: WatcherProcessExit | undefined;+ child.stderr?.on('data', (chunk) => {+ stderr += String(chunk);+ });+ child.stdout?.resume();+ child.on('message', (message) => {+ if (isWatcherProcessReport(message)) reports.push(message);+ else protocolError ??= `sent an unrecognized message ${JSON.stringify(message)}`;+ });+ const exited = new Promise<void>((resolveExit) => {+ child.once('exit', (code, signal) => {+ exitStatus = { code, signal };+ resolveExit();+ });+ child.once('error', (error) => {+ protocolError ??= `failed: ${error.message}`;+ exitStatus ??= { code: null, signal: null };+ resolveExit();+ });+ });+ return {+ reports: () => reports,+ exitStatus: () => exitStatus,+ until: (what, condition) =>+ waitWithinTestBudget(+ what,+ () => {+ if (protocolError !== undefined) {+ throw new Error(`the watcher process ${protocolError}; stderr: ${stderr}`);+ }+ if (condition(reports)) return true;+ if (exitStatus !== undefined) {+ throw new Error(+ `the watcher process exited (code ${exitStatus.code}, signal ${exitStatus.signal}) before the wait was met; stderr: ${stderr}`,+ );+ }+ return false;+ },+ BOUNDED_BY_THE_TEST_BUDGET,+ ),+ send: (command) => {+ child.send(command);+ },+ close: async () => {+ if (child.connected) child.disconnect();+ await exited;+ },+ };+}diff --git a/packages/server/src/managed-artifact-watcher-readiness.test.ts b/packages/server/src/managed-artifact-watcher-readiness.test.tsnew file mode 100644index 000000000..50deddc36--- /dev/null+++ b/packages/server/src/managed-artifact-watcher-readiness.test.ts@@ -0,0 +1,132 @@+import { execFileSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import {+ delivered,+ type ForkedWatcherProcess,+ forkWatcherProcess,+ type MutationTiming,+ type ReadinessScenario,+ reportOf,+} from './forked-watcher-process.test-helper.ts';++interface ReadinessCase {+ scenario: ReadinessScenario;+ change: string;+ initialContent: string | null;+ content: string;+}++type ManagedTiming = Exclude<MutationTiming, 'while-first-baseline-stat-held'>;++const CASES: ReadinessCase[] = [+ { scenario: 'create', change: 'creation', initialContent: null, content: 'v1' },+ { scenario: 'edit', change: 'edit', initialContent: 'v1', content: 'v2' },+];++const START_WAIT: Record<ManagedTiming, string> = {+ 'as-start-resolves':+ 'startManagedArtifactWatcher to resolve so the harness can change SKILL.md. The harness holds ' ++ 'its first baseline sample of the held path until start waits on it, and a first sample ' ++ 'fs.watchFile takes of that path until the change, so this deadlocks, by design, a watcher ' ++ 'that registers fs.watchFile on the path and then awaits threadpool work before resolving',+ 'after-first-baseline-stat': 'startManagedArtifactWatcher to resolve',+};++describe.skipIf(process.platform === 'win32')(+ 'startManagedArtifactWatcher readiness: a SKILL.md change made after start resolves reaches onChange',+ () => {+ let root: string;+ let harness: ForkedWatcherProcess | undefined;++ beforeEach(() => {+ root = mkdtempSync(join(tmpdir(), 'ok-ma-readiness-'));+ execFileSync('mkfifo', [join(root, 'threadpool-barrier')]);+ });+ afterEach(async () => {+ await harness?.close();+ harness = undefined;+ rmSync(root, { recursive: true, force: true });+ });++ async function changeAfterStart(+ { scenario, initialContent, content }: ReadinessCase,+ timing: ManagedTiming,+ ) {+ const skillsRoot = resolve(root, '.ok', 'skills');+ const leaf = resolve(skillsRoot, 'demo', 'SKILL.md');+ if (initialContent !== null) {+ mkdirSync(dirname(leaf), { recursive: true });+ writeFileSync(leaf, initialContent, 'utf-8');+ }+ const run = forkWatcherProcess({+ mode: 'readiness',+ entry: 'managed',+ watch: [skillsRoot],+ scenario,+ timing,+ target: leaf,+ held: scenario === 'create' ? skillsRoot : leaf,+ content,+ barrier: join(root, 'threadpool-barrier'),+ });+ harness = run;+ await run.until(START_WAIT[timing], (reports) => reportOf(reports, 'started') !== undefined);+ await run.until(+ 'the harness to change SKILL.md once startManagedArtifactWatcher resolved',+ (reports) => reportOf(reports, 'mutated') !== undefined,+ );+ return {+ run,+ leaf,+ started: reportOf(run.reports(), 'started'),+ mutated: reportOf(run.reports(), 'mutated'),+ };+ }++ test.each(CASES)(+ 'resolves start only once its first baseline sample has landed, and reports a SKILL.md $change made the moment start resolves',+ async (readinessCase) => {+ const { run, leaf, started } = await changeAfterStart(readinessCase, 'as-start-resolves');+ expect(+ started?.barrierEngaged,+ "the watcher's start took no sample of the held path for the harness to hold",+ ).toBe(true);+ expect(+ started?.firstBaselineStatHeldAtResolve,+ 'start resolved before its first baseline sample landed, so a SKILL.md change made then becomes part of the baseline and is never reported',+ ).toBe(false);++ await run.until(+ `onChange to report the SKILL.md ${readinessCase.change} made the moment start resolved`,+ (reports) => delivered(reports, leaf, readinessCase.content),+ );+ },+ );++ test.each(CASES)(+ 'reports a SKILL.md $change made once the first baseline stat has run (ordering control)',+ async (readinessCase) => {+ const { run, leaf, started, mutated } = await changeAfterStart(+ readinessCase,+ 'after-first-baseline-stat',+ );+ expect(+ started?.barrierEngaged,+ "the watcher's start took no sample of the held path for the harness to hold",+ ).toBe(true);+ expect(+ mutated?.firstBaselineStatHeldAtMutation,+ 'the control changes SKILL.md only after the held first baseline stat has run',+ ).toBe(false);++ await run.until(+ `onChange to report the SKILL.md ${readinessCase.change} made after the first baseline stat`,+ (reports) => delivered(reports, leaf, readinessCase.content),+ );+ },+ );+ },+);diff --git a/packages/server/src/managed-artifact-watcher.test.ts b/packages/server/src/managed-artifact-watcher.test.tsindex 5aa0232f9..4e20c0270 100644--- a/packages/server/src/managed-artifact-watcher.test.ts+++ b/packages/server/src/managed-artifact-watcher.test.ts@@ -21,9 +21,7 @@ afterEach(async () => {rmSync(root, { recursive: true, force: true });});-const RUNNING_IN_CI = Boolean(process.env.CI);--describe.skipIf(RUNNING_IN_CI)('startManagedArtifactWatcher', () => {+describe('startManagedArtifactWatcher', () => {test('fires onChange for a SKILL.md created after start', async () => {const skillsRoot = resolve(root, '.ok', 'skills');const seen: Array<[string, string]> = [];@@ -86,6 +84,36 @@ describe.skipIf(RUNNING_IN_CI)('startManagedArtifactWatcher', () => {);expect(contents).not.toContain('noise');}, 25_000);++ test('fires onChange for a SKILL.md present at start only once it is edited', async () => {+ const skillsRoot = resolve(root, '.ok', 'skills');+ const skillDir = resolve(skillsRoot, 'demo');+ mkdirSync(skillDir, { recursive: true });+ const leaf = resolve(skillDir, 'SKILL.md');+ writeFileSync(leaf, 'v1', 'utf-8');++ const seen: Array<[string, string]> = [];+ cleanup = await startManagedArtifactWatcher([skillsRoot], (p, c) => seen.push([p, c]));++ const addedDir = resolve(skillsRoot, 'added');+ mkdirSync(addedDir, { recursive: true });+ const addedLeaf = resolve(addedDir, 'SKILL.md');+ writeFileSync(addedLeaf, 'added after start', 'utf-8');++ await waitWithinTestBudget(+ "the watcher to report the SKILL.md created after start with its 'added after start' contents",+ () => seen.some(([p, c]) => p === addedLeaf && c === 'added after start'),+ { timeoutMs: 20_000, pollMs: 50 },+ );+ writeFileSync(leaf, 'v2, edited after start', 'utf-8');++ await waitWithinTestBudget(+ "the watcher to report the SKILL.md present at start with its 'v2, edited after start' contents",+ () => seen.some(([p, c]) => p === leaf && c === 'v2, edited after start'),+ { timeoutMs: 20_000, pollMs: 50 },+ );+ expect(seen.filter(([p]) => p === leaf).map(([, c]) => c)).toEqual(['v2, edited after start']);+ }, 25_000);});describe('managed-artifact watch roots never conjure a harness home', () => {diff --git a/packages/server/src/managed-artifact-watcher.ts b/packages/server/src/managed-artifact-watcher.tsindex 1b08758fb..4c9a01139 100644--- a/packages/server/src/managed-artifact-watcher.ts+++ b/packages/server/src/managed-artifact-watcher.ts@@ -1,20 +1,19 @@import { readFileSync } from 'node:fs';-import { basename } from 'node:path';+import { readdir } from 'node:fs/promises';+import { join } from 'node:path';import { tracedMkdirSync } from './fs-traced.ts';import { errnoCode } from './http/handler-utils.ts';import { getLogger } from './logger.ts';+import { startPolledPathWatcher } from './polled-path-watcher.ts';export type ManagedArtifactWatcherUnsubscribe = () => Promise<void>;-const isSkillLeaf = (absPath: string): boolean => basename(absPath) === 'SKILL.md';-export async function startManagedArtifactWatcher(roots: ReadonlyArray<string>,onChange: (absPath: string, content: string) => void,onUnlink?: (absPath: string) => void,): Promise<ManagedArtifactWatcherUnsubscribe> {const log = getLogger('managed-artifact-watcher');- const { watch } = await import('chokidar');const watchRoots = Array.from(new Set(roots));for (const dir of watchRoots) {@@ -28,22 +27,9 @@ export async function startManagedArtifactWatcher(}}- const watcher = watch(watchRoots, {- ignoreInitial: true,- depth: 1,- usePolling: true,- interval: 200,- awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 },- });-- await new Promise<void>((resolve) => {- watcher.once('ready', resolve);- });-const lastContent = new Map<string, string | null>();const handlePath = (path: string): void => {- if (!isSkillLeaf(path)) return;let content: string;try {content = readFileSync(path, 'utf-8');@@ -64,12 +50,7 @@ export async function startManagedArtifactWatcher(log.warn({ err, path }, 'managed-artifact change handler threw');}};- const handler = (path: string): void => handlePath(path);-- watcher.on('add', handler);- watcher.on('change', handler);- watcher.on('unlink', (path) => {- if (!isSkillLeaf(path)) return;+ const handleUnlink = (path: string): void => {lastContent.delete(path);log.debug({ path }, 'managed-artifact leaf unlinked; live doc retained at current state');if (onUnlink) {@@ -79,15 +60,41 @@ export async function startManagedArtifactWatcher(log.warn({ err, path }, 'managed-artifact unlink handler threw');}}- });- watcher.on('error', (err) => {- log.warn({ err, watchRoots }, '[managed-artifact-watcher] chokidar error');- });+ };- let closed = false;- return async () => {- if (closed) return;- closed = true;- await watcher.close();+ const listedLeaves = new Map<string, ReadonlyArray<string>>();+ const failingRoots = new Map<string, string | undefined>();+ const listLeaves = async (root: string): Promise<ReadonlyArray<string>> => {+ let entries: string[];+ try {+ entries = await readdir(root);+ } catch (err) {+ const code = errnoCode(err);+ if (code === 'ENOENT') {+ failingRoots.delete(root);+ listedLeaves.set(root, []);+ return [];+ }+ if (!failingRoots.has(root) || failingRoots.get(root) !== code) {+ failingRoots.set(root, code);+ log.warn({ err, root }, 'failed to list watch root; keeping its previous listing');+ }+ return listedLeaves.get(root) ?? [];+ }+ failingRoots.delete(root);+ const leaves = [+ join(root, 'SKILL.md'),+ ...entries.map((entry) => join(root, entry, 'SKILL.md')),+ ];+ listedLeaves.set(root, leaves);+ return leaves;};++ return startPolledPathWatcher({+ listPaths: async () => (await Promise.all(watchRoots.map(listLeaves))).flat(),+ onEvent: (event, path) => (event === 'unlink' ? handleUnlink(path) : handlePath(path)),+ onError: (err, path) => {+ log.warn({ err, path }, '[managed-artifact-watcher] poll error');+ },+ });}diff --git a/packages/server/src/polled-path-watcher.test.ts b/packages/server/src/polled-path-watcher.test.tsnew file mode 100644index 000000000..ebba98c67--- /dev/null+++ b/packages/server/src/polled-path-watcher.test.ts@@ -0,0 +1,289 @@+import { spawn } from 'node:child_process';+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { logsCurrentPath } from './telemetry-file-sink.ts';++const PRIMITIVE = new URL('./polled-path-watcher.ts', import.meta.url).href;+const LOGGER = new URL('./logger.ts', import.meta.url).href;++const WATCHER_WITH_A_THROWING_ON_ERROR = `+const [primitive, logger, dir, listFailure] = process.argv.slice(1);+const { startPolledPathWatcher } = await import(primitive);+const { loggerFactory } = await import(logger);+const { writeFileSync } = await import('node:fs');+const { join } = await import('node:path');+loggerFactory.configure({ pinoConfig: { fileSink: { projectDir: dir, maxBytes: 1_000_000 } } });+const watched = join(dir, 'watched.txt');+const unstatable = join(dir, 'x'.repeat(300));+writeFileSync(watched, 'before');+const listPathsFailure = () => {+ const err = new Error('listPaths failed');+ if (listFailure === 'unserializable') {+ Object.defineProperty(err, 'detail', {+ enumerable: true,+ get() {+ throw new Error('detail cannot be read');+ },+ });+ }+ return err;+};+let phase = 'starting';+const say = (line) => process.stdout.write(line + '\\n');+const stop = await startPolledPathWatcher({+ listPaths: async () => {+ if (phase === 'list-failing') throw listPathsFailure();+ return phase === 'path-failing' ? [watched, unstatable] : [watched];+ },+ onEvent: (event, path) => {+ if (phase !== 'editing' || event !== 'change' || path !== watched) return;+ say('edit-delivered');+ void stop().then(() => loggerFactory.flushAllFileSinks());+ },+ onError: (err, path) => {+ if (phase === 'path-failing' && path === unstatable) {+ say('path-error-reported ' + err.code);+ phase = 'list-failing';+ } else if (phase === 'list-failing' && path === undefined) {+ say('list-error-reported ' + err.message);+ phase = 'editing';+ writeFileSync(watched, 'after, and longer');+ }+ throw new Error('onError threw');+ },+});+phase = 'path-failing';+say('started');+`;++const WATCHER_OF_A_FILE_STAMPED_AHEAD = `+const [primitive, dir, stampsAhead] = process.argv.slice(1);+const fs = await import('node:fs');+const { syncBuiltinESMExports } = await import('node:module');+const { join } = await import('node:path');+const DAY_MS = 86_400_000;+const DAY_NS = 86_400_000_000_000n;+const NS_PER_S = 1_000_000_000n;+const SAMPLES = 30;+const watched = join(dir, 'watched.txt');+let reads = 0;+const readFile = fs.promises.readFile;+fs.promises.readFile = function (path, ...rest) {+ if (String(path) === watched) reads += 1;+ return Reflect.apply(readFile, this, [path, ...rest]);+};+if (stampsAhead === 'mtime and ctime') {+ const stat = fs.promises.stat;+ fs.promises.stat = async function (path, ...rest) {+ const stats = await Reflect.apply(stat, this, [path, ...rest]);+ if (String(path) === watched && typeof stats.ctimeNs === 'bigint') {+ Object.assign(stats, { mtimeNs: stats.mtimeNs + DAY_NS, ctimeNs: stats.ctimeNs + DAY_NS });+ }+ return stats;+ };+}+syncBuiltinESMExports();+const { startPolledPathWatcher } = await import(primitive);+fs.writeFileSync(watched, 'before');+if (stampsAhead === 'mtime') {+ const ahead = new Date(Date.now() + DAY_MS);+ fs.utimesSync(watched, ahead, ahead);+}+const stamped = await fs.promises.stat(watched, { bigint: true });+const nowNs = BigInt(Date.now()) * 1_000_000n;+const readsAtSample = [];+const stop = await startPolledPathWatcher({+ listPaths: async () => {+ readsAtSample.push(reads);+ if (readsAtSample.length === SAMPLES + 1) fs.writeFileSync(watched, 'BEFORE');+ return [watched];+ },+ onEvent: (event, path) => {+ if (event !== 'change' || path !== watched || readsAtSample.length <= SAMPLES) return;+ process.stdout.write(+ JSON.stringify({+ mtimeAheadOfNowS: Number((stamped.mtimeNs - nowNs) / NS_PER_S),+ mtimeAheadOfCtimeS: Number((stamped.mtimeNs - stamped.ctimeNs) / NS_PER_S),+ readsInEarlySamples: readsAtSample[6] - readsAtSample[1],+ readsInLateSamples: readsAtSample[SAMPLES] - readsAtSample[SAMPLES - 5],+ }) + '\\n',+ );+ void stop();+ },+ onError: (err) => {+ process.stderr.write(String(err) + '\\n');+ },+});+`;++type ListFailure = 'serializable' | 'unserializable';++type StampsAhead = 'mtime' | 'mtime and ctime';++interface StampedAheadReport {+ mtimeAheadOfNowS: number;+ mtimeAheadOfCtimeS: number;+ readsInEarlySamples: number;+ readsInLateSamples: number;+}++interface ChildOutcome {+ code: number | null;+ stdout: string[];+ stderr: string;+}++interface LoggedError {+ code?: string;+ message?: string;+}++interface PrimitiveLogRecord {+ name?: string;+ path?: string;+ err?: LoggedError;+ error?: LoggedError;+}++function runChild(script: string, args: readonly string[]): Promise<ChildOutcome> {+ const child = spawn(+ process.execPath,+ [+ '--import',+ 'tsx',+ '--conditions=@inkeep/source',+ '--input-type=module',+ '--eval',+ script,+ ...args,+ ],+ {+ stdio: ['ignore', 'pipe', 'pipe'],+ env: { ...process.env, OK_CONSOLE_LEVEL: 'silent', OK_FILE_LEVEL: 'trace' },+ },+ );+ let stdout = '';+ let stderr = '';+ child.stdout.on('data', (chunk) => {+ stdout += String(chunk);+ });+ child.stderr.on('data', (chunk) => {+ stderr += String(chunk);+ });+ return new Promise((resolveOutcome) => {+ child.once('close', (code) => {+ resolveOutcome({ code, stdout: stdout.split('\n').filter(Boolean), stderr });+ });+ });+}++function runWatcherWithAThrowingOnError(+ dir: string,+ listFailure: ListFailure,+): Promise<ChildOutcome> {+ return runChild(WATCHER_WITH_A_THROWING_ON_ERROR, [PRIMITIVE, LOGGER, dir, listFailure]);+}++async function watchAFileStampedAhead(+ dir: string,+ stampsAhead: StampsAhead,+): Promise<StampedAheadReport> {+ const outcome = await runChild(WATCHER_OF_A_FILE_STAMPED_AHEAD, [PRIMITIVE, dir, stampsAhead]);+ expect(outcome.code, `the watcher process exit code; stderr: ${outcome.stderr}`).toBe(0);+ expect(+ outcome.stdout,+ `the watcher process must report once, after the same-size rewrite is delivered; stderr: ${outcome.stderr}`,+ ).toHaveLength(1);+ return JSON.parse(outcome.stdout[0] ?? 'null');+}++function primitiveLogRecords(dir: string): PrimitiveLogRecord[] {+ const logPath = logsCurrentPath(dir);+ if (!existsSync(logPath)) return [];+ return readFileSync(logPath, 'utf-8')+ .split('\n')+ .filter(Boolean)+ .map((line): PrimitiveLogRecord => JSON.parse(line))+ .filter((record) => record.name === 'polled-path-watcher');+}++let dir: string;++beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), 'ok-polled-watcher-'));+});+afterEach(() => {+ rmSync(dir, { recursive: true, force: true });+});++describe('startPolledPathWatcher when its onError callback throws', () => {+ test.each<[string, ListFailure]>([+ ['a failing path and a failing listPaths', 'serializable'],+ ['a failing path and a failing listPaths whose error cannot be serialized', 'unserializable'],+ ])('keeps the process alive and keeps polling through %s', async (_, listFailure) => {+ const outcome = await runWatcherWithAThrowingOnError(dir, listFailure);++ expect(+ outcome.stdout,+ `the watcher process must report both errors and then deliver the later edit; stderr: ${outcome.stderr}`,+ ).toEqual([+ 'started',+ 'path-error-reported ENAMETOOLONG',+ 'list-error-reported listPaths failed',+ 'edit-delivered',+ ]);+ expect(outcome.code, `the watcher process exit code; stderr: ${outcome.stderr}`).toBe(0);+ });++ test('logs each error it was reporting together with the exception its onError threw', async () => {+ const outcome = await runWatcherWithAThrowingOnError(dir, 'serializable');+ expect(outcome.code, `the watcher process exit code; stderr: ${outcome.stderr}`).toBe(0);++ expect(+ primitiveLogRecords(dir).map(({ err, error, path }) => ({+ reported: err?.code ?? err?.message,+ path,+ thrown: error?.message,+ })),+ `the records the primitive wrote to ${logsCurrentPath(dir)}`,+ ).toEqual([+ { reported: 'ENAMETOOLONG', path: join(dir, 'x'.repeat(300)), thrown: 'onError threw' },+ { reported: 'listPaths failed', path: undefined, thrown: 'onError threw' },+ ]);+ });+});++describe('startPolledPathWatcher on a file whose timestamps lie ahead of the host clock', () => {+ test('stops reading the file once its change time is past the timestamp bound when only its modification time was set ahead, and still reports a later same-size rewrite', async () => {+ const report = await watchAFileStampedAhead(dir, 'mtime');++ expect(+ report.mtimeAheadOfNowS,+ 'seconds the modification time lies ahead of the host clock',+ ).toBeGreaterThan(3);+ expect(+ report.mtimeAheadOfCtimeS,+ 'seconds the modification time lies ahead of the change time',+ ).toBeGreaterThan(3);+ expect(+ report.readsInEarlySamples,+ "reads of the file in samples 1-5, inside the change time's bound",+ ).toBeGreaterThan(0);+ expect(+ report.readsInLateSamples,+ "reads of the file in samples 25-29, past the change time's bound",+ ).toBe(0);+ });++ test('keeps reading the file while its change time lies ahead as well (both timestamps modelled a day ahead at the stat seam)', async () => {+ const report = await watchAFileStampedAhead(dir, 'mtime and ctime');++ expect(+ report.mtimeAheadOfNowS,+ 'seconds the modification time lies ahead of the host clock',+ ).toBeGreaterThan(3);+ expect(report.readsInLateSamples, 'reads of the file in samples 25-29').toBeGreaterThan(0);+ });+});diff --git a/packages/server/src/polled-path-watcher.ts b/packages/server/src/polled-path-watcher.tsnew file mode 100644index 000000000..e3152f8ac--- /dev/null+++ b/packages/server/src/polled-path-watcher.ts@@ -0,0 +1,196 @@+import { createHash } from 'node:crypto';+import type { BigIntStats } from 'node:fs';+import { readFile, stat } from 'node:fs/promises';+import { errnoCode } from './http/handler-utils.ts';+import { getLogger } from './logger.ts';++const POLL_INTERVAL_MS = 200;++const TIMESTAMP_GRANULARITY_BOUND_NS = 3_000_000_000n;++const NS_PER_MS = 1_000_000n;++type PathEvent = 'add' | 'change' | 'unlink';++interface PolledPathWatcherOptions {+ listPaths: () => Promise<ReadonlyArray<string>>;+ onEvent: (event: PathEvent, path: string) => void;+ onError: (err: unknown, path?: string) => void;+}++interface Observation {+ readonly size: bigint;+ readonly signature: string;+ readonly racy: boolean;+ readonly digest: string | undefined;+}++const readsAsAbsent = (err: unknown): boolean => {+ const code = errnoCode(err);+ return code === 'ENOENT' || code === 'ENOTDIR';+};++const signatureOf = (stats: BigIntStats): string =>+ `${stats.dev}:${stats.ino}:${stats.size}:${stats.mtimeNs}:${stats.ctimeNs}`;++const isRacy = (stats: BigIntStats, sampledAtNs: bigint): boolean =>+ stats.ctimeNs + TIMESTAMP_GRANULARITY_BOUND_NS > sampledAtNs ||+ (stats.mtimeNs + TIMESTAMP_GRANULARITY_BOUND_NS > sampledAtNs &&+ stats.mtimeNs <= sampledAtNs + TIMESTAMP_GRANULARITY_BOUND_NS);++const unchanged = (known: Observation, observed: Observation): boolean =>+ known.signature === observed.signature && (!known.racy || known.digest === observed.digest);++const retained = (observation: Observation): Observation =>+ observation.racy ? observation : { ...observation, digest: undefined };++async function fileStats(path: string): Promise<BigIntStats | undefined> {+ let stats: BigIntStats;+ try {+ stats = await stat(path, { bigint: true });+ } catch (err) {+ if (readsAsAbsent(err)) return undefined;+ throw err;+ }+ return stats.isFile() ? stats : undefined;+}++async function contentDigest(path: string): Promise<string | undefined> {+ try {+ return createHash('sha256')+ .update(await readFile(path))+ .digest('hex');+ } catch (err) {+ if (readsAsAbsent(err)) return undefined;+ throw err;+ }+}++async function observe(+ path: string,+ known: Observation | undefined,+ isClosed: () => boolean,+): Promise<Observation | undefined> {+ const sampledAtNs = BigInt(Date.now()) * NS_PER_MS;+ const first = await fileStats(path);+ if (first === undefined) return undefined;+ if (!known?.racy && !isRacy(first, sampledAtNs)) {+ return { size: first.size, signature: signatureOf(first), racy: false, digest: undefined };+ }+ if (isClosed()) return undefined;+ const digest = await contentDigest(path);+ if (digest === undefined || isClosed()) return undefined;+ const second = await fileStats(path);+ if (second === undefined) return undefined;+ return {+ size: second.size,+ signature: signatureOf(second),+ racy: isRacy(second, sampledAtNs),+ digest,+ };+}++export async function startPolledPathWatcher({+ listPaths,+ onEvent,+ onError,+}: PolledPathWatcherOptions): Promise<() => Promise<void>> {+ const log = getLogger('polled-path-watcher');+ const failingCodes = new Map<string, string | undefined>();+ let closed = false;++ const report = (err: unknown, path?: string): void => {+ if (closed) return;+ try {+ onError(err, path);+ } catch (thrown) {+ try {+ log.error({ err, error: thrown, path }, 'onError threw while reporting a watcher error');+ } catch {}+ }+ };++ const sample = async (+ kept: ReadonlyMap<string, Observation>,+ ): Promise<Map<string, Observation>> => {+ const listed = new Set(await listPaths());+ for (const path of failingCodes.keys()) {+ if (!listed.has(path)) failingCodes.delete(path);+ }+ const observed = new Map<string, Observation>();+ await Promise.all(+ Array.from(listed, async (path) => {+ let observation: Observation | undefined;+ try {+ observation = await observe(path, kept.get(path), () => closed);+ } catch (err) {+ const code = errnoCode(err);+ if (!failingCodes.has(path) || failingCodes.get(path) !== code) {+ failingCodes.set(path, code);+ report(err, path);+ }+ const last = kept.get(path);+ if (last) observed.set(path, last);+ return;+ }+ failingCodes.delete(path);+ if (observation) observed.set(path, observation);+ }),+ );+ return observed;+ };++ const reference = await sample(new Map());+ const pendingSizes = new Map<string, bigint>();++ const compare = (observed: ReadonlyMap<string, Observation>): void => {+ for (const [path, observation] of observed) {+ const known = reference.get(path);+ if (known !== undefined && unchanged(known, observation)) {+ pendingSizes.delete(path);+ if (known.racy) reference.set(path, retained(observation));+ } else if (pendingSizes.get(path) === observation.size) {+ pendingSizes.delete(path);+ reference.set(path, retained(observation));+ onEvent(known ? 'change' : 'add', path);+ } else {+ pendingSizes.set(path, observation.size);+ }+ }+ for (const path of reference.keys()) {+ if (observed.has(path)) continue;+ reference.delete(path);+ onEvent('unlink', path);+ }+ for (const path of pendingSizes.keys()) {+ if (!observed.has(path)) pendingSizes.delete(path);+ }+ };++ let timer: ReturnType<typeof setTimeout> | undefined;++ const poll = async (): Promise<void> => {+ try {+ const observed = await sample(reference);+ if (!closed) compare(observed);+ } catch (err) {+ report(err);+ }+ };++ const schedule = (): void => {+ timer = setTimeout(() => {+ void poll().finally(() => {+ if (!closed) schedule();+ });+ }, POLL_INTERVAL_MS);+ };++ schedule();++ return async () => {+ if (closed) return;+ closed = true;+ clearTimeout(timer);+ };+}diff --git a/packages/server/src/polled-watcher-contract.test.ts b/packages/server/src/polled-watcher-contract.test.tsnew file mode 100644index 000000000..2bd4eccc8--- /dev/null+++ b/packages/server/src/polled-watcher-contract.test.ts@@ -0,0 +1,856 @@+import { execFileSync } from 'node:child_process';+import {+ chmodSync,+ mkdirSync,+ mkdtempSync,+ renameSync,+ rmSync,+ statSync,+ symlinkSync,+ utimesSync,+ writeFileSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import {+ type ContractSpec,+ delivered,+ deliveriesFor,+ type ForkedContractProcess,+ type ForkedWatcherProcess,+ forkWatcherProcess,+ type LoggedError,+ type ReadinessScenario,+ type Recovery,+ reportOf,+ unlinked,+ type WatcherEntry,+ type WatcherProcessReport,+} from './forked-watcher-process.test-helper.ts';++interface WatchedFile {+ watch: string[];+ target: string;+}++interface EntryCase {+ watcher: string;+ entry: WatcherEntry;+}++interface UnreadableCase extends EntryCase {+ unreadable: 'skill directory' | 'skills root' | 'directory';+}++interface RecoveryCase extends UnreadableCase {+ recovery: Recovery;+}++const ENTRIES: EntryCase[] = [+ { watcher: 'startManagedArtifactWatcher', entry: 'managed' },+ { watcher: 'startConfigFileWatcher', entry: 'config' },+ { watcher: 'startMultiPathConfigFileWatcher', entry: 'multi-config' },+];++const UNREADABLE_CASES: UnreadableCase[] = [+ { watcher: 'startManagedArtifactWatcher', entry: 'managed', unreadable: 'skill directory' },+ { watcher: 'startManagedArtifactWatcher', entry: 'managed', unreadable: 'skills root' },+ { watcher: 'startConfigFileWatcher', entry: 'config', unreadable: 'directory' },+ { watcher: 'startMultiPathConfigFileWatcher', entry: 'multi-config', unreadable: 'directory' },+];++const RECOVERY_CASES: RecoveryCase[] = UNREADABLE_CASES.flatMap((unreadableCase) =>+ (['readable', 'removed'] as const).map((recovery) => ({ ...unreadableCase, recovery })),+);++const INITIAL: Record<WatcherEntry, string> = {+ managed: 'demo skill',+ config: 'theme: light\n',+ 'multi-config': '*.tmp\n',+};++const PARTIAL = 'half-writ';+const COMPLETE = 'half-written, then completed\n';+const HELD_CHANGE = 'changed while the next sample was held\n';+const READABLE_AGAIN = 'edited once readable\n';+const SAME_SIZE_REPLACEMENT = 'theme: dark!\n';+const IN_PLACE_REWRITE = 'DEMO SKILL';++function write(path: string, content: string): void {+ mkdirSync(dirname(path), { recursive: true });+ writeFileSync(path, content, 'utf-8');+}++let fixture: string;+let harness: ForkedWatcherProcess | undefined;+const madeUnreadable: string[] = [];++beforeEach(() => {+ fixture = mkdtempSync(join(tmpdir(), 'ok-polled-contract-'));+});+afterEach(async () => {+ await harness?.close();+ harness = undefined;+ for (const dir of madeUnreadable.splice(0)) chmodSync(dir, 0o755);+ rmSync(fixture, { recursive: true, force: true });+});++function skillsRoot(): string {+ return join(fixture, 'skills');+}++function leaf(...segments: string[]): string {+ return join(skillsRoot(), ...segments, 'SKILL.md');+}++function watchedFile(entry: WatcherEntry, scenario: ReadinessScenario): WatchedFile {+ const project = join(fixture, 'project');+ const file: WatchedFile =+ entry === 'managed'+ ? { watch: [skillsRoot()], target: leaf('demo') }+ : entry === 'config'+ ? {+ watch: [join(project, '.ok', 'config.yml')],+ target: join(project, '.ok', 'config.yml'),+ }+ : {+ watch: [join(project, '.okignore'), join(project, '.gitignore')],+ target: join(project, '.okignore'),+ };+ mkdirSync(dirname(file.target), { recursive: true });+ if (entry === 'multi-config') write(join(project, '.gitignore'), 'dist/\n');+ if (scenario === 'edit') write(file.target, INITIAL[entry]);+ return file;+}++async function watchFrom(+ entry: WatcherEntry,+ watch: string[],+ coarseTimestampsAt?: string,+): Promise<ForkedContractProcess> {+ const spec: ContractSpec = { mode: 'contract', entry, watch, coarseTimestampsAt };+ const run = forkWatcherProcess(spec);+ harness = run;+ await run.until(+ 'the watcher to start and every baseline it registered to be taken',+ (reports) => reportOf(reports, 'settled') !== undefined,+ );+ return run;+}++function unreadableScope(+ { entry, unreadable }: UnreadableCase,+ { watch, target }: WatchedFile,+): { dir: string; failing: string[] } {+ if (entry !== 'managed') return { dir: dirname(target), failing: watch };+ if (unreadable === 'skills root') return { dir: skillsRoot(), failing: [skillsRoot(), target] };+ return { dir: dirname(target), failing: [target] };+}++function accessDenials(logged: readonly LoggedError[] = []): LoggedError[] {+ return logged.filter((entry) => entry.code === 'EACCES');+}++function asLogLines(logged: readonly LoggedError[]): string[] {+ return logged.map((entry) => `${entry.level} ${entry.code} ${entry.path}: ${entry.message}`);+}++function reportsFor(+ logged: readonly LoggedError[] | undefined,+ code: string,+ path: string,+): LoggedError[] {+ return (logged ?? []).filter((entry) => entry.code === code && entry.path === path);+}++function reportsBefore(+ reports: readonly WatcherProcessReport[],+ kind: WatcherProcessReport['kind'],+): readonly WatcherProcessReport[] {+ const at = reports.findIndex((report) => report.kind === kind);+ return at === -1 ? reports : reports.slice(0, at);+}++function failingSampleOf({ unreadable }: UnreadableCase, { target }: WatchedFile): string {+ return unreadable === 'skills root' ? skillsRoot() : target;+}++function sampledEveryPollOf({ entry }: UnreadableCase, { target }: WatchedFile): string {+ return entry === 'managed' ? skillsRoot() : target;+}++function failingOnceMore({ unreadable, recovery }: RecoveryCase, failing: string[]): string[] {+ const recreatedRootListsNothing = unreadable === 'skills root' && recovery === 'removed';+ return recreatedRootListsNothing ? [skillsRoot()] : failing;+}++describe('startManagedArtifactWatcher reports SKILL.md at a skills root and one directory down', () => {+ test('reports the root SKILL.md, a skill directory SKILL.md and one reached through a symlinked skill directory, and nothing deeper or beside them', async () => {+ const outside = join(fixture, 'outside');+ const nestedTwoDown = leaf('demo', 'nested');+ const notesBesideASkill = join(skillsRoot(), 'demo', 'NOTES.md');+ const fileAtTheRoot = join(skillsRoot(), 'README.md');+ const outsideTheLeafSet = [nestedTwoDown, notesBesideASkill, fileAtTheRoot];+ write(leaf(), 'root skill');+ write(leaf('demo'), 'demo skill');+ write(nestedTwoDown, 'nested skill');+ write(notesBesideASkill, 'notes');+ write(fileAtTheRoot, 'readme');+ write(join(outside, 'linked-skill', 'SKILL.md'), 'linked skill');+ write(join(outside, 'late-skill', 'SKILL.md'), 'late skill');+ symlinkSync(join(outside, 'linked-skill'), join(skillsRoot(), 'linked'), 'junction');+ const run = await watchFrom('managed', [skillsRoot()]);++ for (const path of outsideTheLeafSet) {+ writeFileSync(path, 'edited, never to be reported', 'utf-8');+ }+ writeFileSync(leaf(), 'root skill, edited', 'utf-8');+ writeFileSync(leaf('demo'), 'demo skill, edited', 'utf-8');+ writeFileSync(join(outside, 'linked-skill', 'SKILL.md'), 'linked skill, edited', 'utf-8');+ symlinkSync(join(outside, 'late-skill'), join(skillsRoot(), 'late'), 'junction');++ const nothingOutsideTheLeafSet = (reports: readonly WatcherProcessReport[]): void => {+ expect(+ reports.filter(+ (report) =>+ (report.kind === 'change' || report.kind === 'unlink') &&+ outsideTheLeafSet.includes(report.path),+ ),+ 'the watcher reported a path other than <root>/SKILL.md or <root>/<entry>/SKILL.md',+ ).toEqual([]);+ };+ const expected: Array<[string, string]> = [+ [leaf(), 'root skill, edited'],+ [leaf('demo'), 'demo skill, edited'],+ [leaf('linked'), 'linked skill, edited'],+ [leaf('late'), 'late skill'],+ ];+ await run.until(+ 'the watcher to report the root SKILL.md, the skill directory SKILL.md, the edited one behind a symlink and the one behind a symlink made after start',+ (reports) => {+ nothingOutsideTheLeafSet(reports);+ return expected.every(([path, content]) => delivered(reports, path, content));+ },+ );++ writeFileSync(leaf('demo'), 'demo skill, edited again', 'utf-8');+ await run.until(+ 'a later report, after which every report the earlier changes could produce has arrived',+ (reports) => {+ nothingOutsideTheLeafSet(reports);+ return delivered(reports, leaf('demo'), 'demo skill, edited again');+ },+ );+ });+});++describe('startManagedArtifactWatcher when a skill directory stops being a readable directory', () => {+ test('reports a SKILL.md unlinked when its skill directory is replaced by a regular file, and keeps reporting the others', async () => {+ write(leaf('demo'), 'demo skill');+ write(leaf('kept'), 'kept skill');+ const run = await watchFrom('managed', [skillsRoot()]);++ rmSync(join(skillsRoot(), 'demo'), { recursive: true, force: true });+ writeFileSync(+ join(skillsRoot(), 'demo'),+ 'a regular file where the skill directory was',+ 'utf-8',+ );+ await run.until(`onUnlink to report ${leaf('demo')}`, (reports) => {+ expect(+ deliveriesFor(reports, leaf('demo')),+ 'the watcher reported the SKILL.md under a directory turned into a file as changed, not unlinked',+ ).toEqual([]);+ return unlinked(reports, leaf('demo'));+ });++ writeFileSync(leaf('kept'), 'kept skill, edited', 'utf-8');+ await run.until('the watcher to keep reporting the other skill', (reports) =>+ delivered(reports, leaf('kept'), 'kept skill, edited'),+ );+ });++ describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'while a directory on the path is unreadable (EACCES; chmod does not bind root or Windows)',+ () => {+ test.each([{ unreadable: 'skill directory' }, { unreadable: 'skills root' }])(+ 'keeps a SKILL.md, reporting it neither unlinked nor changed, while its $unreadable is briefly unreadable, then reports its next edit',+ async ({ unreadable }) => {+ write(leaf('demo'), 'demo skill');+ const dir = unreadable === 'skills root' ? skillsRoot() : join(skillsRoot(), 'demo');+ const run = await watchFrom('managed', [skillsRoot()]);++ madeUnreadable.push(dir);+ run.send({ op: 'unreadable-until-sampled', dir, leaf: leaf('demo') });+ const onlyTheEdit = (reports: readonly WatcherProcessReport[]): void => {+ expect(+ unlinked(reports, leaf('demo')),+ `onUnlink reported ${leaf('demo')} although only its ${unreadable} was unreadable: ` ++ 'a sample that fails with EACCES must keep the last observation, not read as a deletion',+ ).toBe(false);+ expect(+ deliveriesFor(reports, leaf('demo')),+ 'the watcher reported the untouched SKILL.md as changed after the directory was readable again',+ ).not.toContain('demo skill');+ };+ await run.until(+ `the watcher to sample ${leaf('demo')} while its ${unreadable} was unreadable, and the harness to restore it`,+ (reports) => {+ onlyTheEdit(reports);+ return reportOf(reports, 'readable-again') !== undefined;+ },+ );++ writeFileSync(leaf('demo'), 'demo skill, edited once readable again', 'utf-8');+ await run.until(+ `onChange to report the edit made once the ${unreadable} was readable again`,+ (reports) => {+ onlyTheEdit(reports);+ return delivered(reports, leaf('demo'), 'demo skill, edited once readable again');+ },+ );+ },+ );+ },+ );+});++describe('a file written in two steps is reported only once its size holds', () => {+ test.each(+ ENTRIES.flatMap((entryCase) =>+ (['create', 'edit'] as const).map((scenario) => ({ ...entryCase, scenario })),+ ),+ )(+ '$watcher never reports a $scenario half-written, and reports it once the second half lands',+ async ({ entry, scenario }) => {+ const { watch, target } = watchedFile(entry, scenario);+ const run = await watchFrom(entry, watch);++ run.send({ op: 'write-in-two-steps', path: target, partial: PARTIAL, complete: COMPLETE });+ await run.until(+ `the watcher to report ${target} with the contents of both halves, never the first half alone`,+ (reports) => {+ expect(+ deliveriesFor(reports, target),+ `the watcher reported ${target} holding only the first half of a two-step write ` ++ `(first half written at a startup timer tick: ${reportOf(reports, 'partial-written')?.atStartupTimerTick})`,+ ).not.toContain(PARTIAL);+ return delivered(reports, target, COMPLETE);+ },+ );+ expect(+ reportOf(run.reports(), 'partial-sampled'),+ 'the second half is written only after the watcher has sampled the first',+ ).toBeDefined();+ },+ );+});++describe('closing a watcher', () => {+ test.each(ENTRIES)(+ '$watcher cleanup, called while the watcher samples a pending change, resolves when called twice, reports nothing once resolved, and leaves nothing that keeps the process alive',+ async ({ entry }) => {+ const { watch, target } = watchedFile(entry, 'edit');+ const run = await watchFrom(entry, watch);++ run.send({+ op: 'close-while-sampling',+ path: target,+ content: 'changed just before cleanup',+ });+ await run.until(+ `the watcher to sample the change to ${target} once and begin sampling it again, when cleanup is called`,+ (reports) => reportOf(reports, 'closing') !== undefined,+ );+ await run.until(+ 'cleanup to resolve, twice',+ (reports) => reportOf(reports, 'closed') !== undefined,+ );+ await run.until(+ 'the watcher process to fall idle on its own once cleanup resolved (a timer or handle the watcher left armed keeps it running)',+ (reports) => reportOf(reports, 'quiesced') !== undefined,+ );+ expect(+ reportOf(run.reports(), 'quiesced')?.eventsAfterClose,+ 'the watcher called back after its cleanup resolved',+ ).toEqual([]);+ await run.until('the idle watcher process to exit', () => run.exitStatus() !== undefined);+ expect(run.exitStatus(), 'the watcher process exits by itself once idle').toEqual({+ code: 0,+ signal: null,+ });+ },+ );+});++describe.skipIf(process.platform === 'win32')(+ 'a watcher never begins a sample of a path while its previous sample is in flight (the threadpool barrier is a POSIX FIFO)',+ () => {+ test.each(ENTRIES)(+ '$watcher begins no further sample of a changed file while that sample is held on the only libuv worker, then reports the change',+ async ({ entry }) => {+ const { watch, target } = watchedFile(entry, 'edit');+ const barrier = join(fixture, 'threadpool-barrier');+ execFileSync('mkfifo', [barrier]);+ const run = await watchFrom(entry, watch);++ run.send({ op: 'hold-next-sample', path: target, content: HELD_CHANGE, barrier });+ await run.until(+ `the watcher to begin sampling ${target} after it changed`,+ (reports) => reportOf(reports, 'sample-held') !== undefined,+ );+ await run.until(+ 'every timer the watcher had pending while that sample was held to fire or be cleared (this over-constrains, by design, a watcher that keeps a timer pending for longer than the test budget)',+ (reports) => reportOf(reports, 'overlap-checked') !== undefined,+ );+ const checked = reportOf(run.reports(), 'overlap-checked');+ expect(+ checked?.samplesBegunWhileHeld,+ `the watcher began another sample of ${target} (or another listing of its skills root) while the previous sample was still in flight ` ++ `(${checked?.timersWaitedOut} pending timer(s) waited out): a slow filesystem would pile samples up behind the libuv threadpool the server shares`,+ ).toBe(0);++ await run.until(+ `onChange to report the change to ${target} once the held sample was released`,+ (reports) => delivered(reports, target, HELD_CHANGE),+ );+ },+ );+ },+);++describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'an error the watcher keeps silent to its callers is reported at some logger level, and not again while it persists (EACCES; chmod does not bind root or Windows)',+ () => {+ test.each(UNREADABLE_CASES)(+ '$watcher reports the EACCES of every path it cannot read while its $unreadable is unreadable, at least once, and not again in a further poll',+ async (unreadableCase) => {+ const file = watchedFile(unreadableCase.entry, 'edit');+ const { dir, failing } = unreadableScope(unreadableCase, file);+ const run = await watchFrom(unreadableCase.entry, file.watch);++ madeUnreadable.push(dir);+ run.send({ op: 'unreadable-across-polls', dir, path: file.target });+ await run.until(+ `the watcher to sample ${file.target} in polls that fail with EACCES, until a poll begun with the ${unreadableCase.unreadable} already unreadable and one more poll have both settled`,+ (reports) => {+ expect(+ unlinked(reports, file.target),+ `onUnlink reported ${file.target} although only its ${unreadableCase.unreadable} was unreadable: ` ++ 'the watcher read the EACCES as a deletion instead of keeping the last observation and reporting the error',+ ).toBe(false);+ return reportOf(reports, 'error-reports') !== undefined;+ },+ );++ const errorReports = reportOf(run.reports(), 'error-reports');+ const onceEstablished = accessDenials(errorReports?.onceEstablished);+ const afterAnotherPoll = accessDenials(errorReports?.afterAnotherPoll);+ for (const path of failing) {+ expect(+ afterAnotherPoll.filter((entry) => entry.path === path),+ `the watcher never reported the EACCES of ${path} at any logger level: an error it keeps silent to its callers must be reported, not swallowed`,+ ).not.toEqual([]);+ }+ expect(+ asLogLines(afterAnotherPoll),+ 'the watcher reported the EACCES again in a further poll while it persisted, instead of once per path and error code',+ ).toEqual(asLogLines(onceEstablished));+ },+ );+ },+);++describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'an error the watcher reported is reported again once the path has read as present or absent and then fails once more (EACCES; chmod does not bind root or Windows)',+ () => {+ test.each(RECOVERY_CASES)(+ '$watcher reports the EACCES of every path it cannot read in each episode when its $unreadable is unreadable, then $recovery, then unreadable once more',+ async (recoveryCase) => {+ const { unreadable, recovery } = recoveryCase;+ const file = watchedFile(recoveryCase.entry, 'edit');+ const { dir, failing } = unreadableScope(recoveryCase, file);+ const run = await watchFrom(recoveryCase.entry, file.watch);++ madeUnreadable.push(dir);+ run.send({+ op: 'unreadable-again-after-recovery',+ dir,+ failingAt: failingSampleOf(recoveryCase, file),+ sampledEveryPoll: sampledEveryPollOf(recoveryCase, file),+ recovery,+ aside: join(fixture, 'aside'),+ });+ await run.until(+ `the watcher to sample through an unreadable episode of its ${unreadable}, its recovery (${recovery}) and a second unreadable episode, each established by a poll begun with it in place`,+ (reports) => {+ expect(+ unlinked(reportsBefore(reports, 'removed'), file.target),+ `onUnlink reported ${file.target} while its ${unreadable} was only unreadable: ` ++ 'the watcher read the EACCES as a deletion instead of keeping the last observation and reporting the error',+ ).toBe(false);+ return reportOf(reports, 'error-reports-across-recovery') !== undefined;+ },+ );++ const episodes = reportOf(run.reports(), 'error-reports-across-recovery');+ for (const path of failing) {+ expect(+ reportsFor(episodes?.firstEpisode, 'EACCES', path),+ `the watcher never reported the EACCES of ${path} in the first unreadable episode, at any logger level`,+ ).not.toEqual([]);+ }+ for (const path of failingOnceMore(recoveryCase, failing)) {+ expect(+ reportsFor(episodes?.afterRecovery, 'EACCES', path),+ `the watcher reported the EACCES of ${path} in the first unreadable episode but not in the one after its ${unreadable} was ${recovery}: ` ++ 'a path that reads as present or absent again must have its next failure reported, or a second episode is kept silent',+ ).not.toEqual([]);+ }+ },+ );+ },+);++describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'an error the watcher keeps silent to its callers is reported once per error code, so a path that keeps failing with a new code is reported again (EACCES, then ELOOP; chmod does not bind root or Windows)',+ () => {+ test.each(UNREADABLE_CASES)(+ '$watcher reports the ELOOP of every path it cannot read once its $unreadable, unreadable until then, becomes a symlink loop, and not again while it stays one',+ async (unreadableCase) => {+ const { unreadable } = unreadableCase;+ const file = watchedFile(unreadableCase.entry, 'edit');+ const { dir, failing } = unreadableScope(unreadableCase, file);+ const failingAt = failingSampleOf(unreadableCase, file);+ const run = await watchFrom(unreadableCase.entry, file.watch);++ madeUnreadable.push(dir);+ run.send({+ op: 'error-code-changes-while-failing',+ dir,+ failingAt,+ aside: join(fixture, 'aside'),+ });+ await run.until(+ `the watcher to sample ${failingAt} in polls that fail with EACCES and then, once the ${unreadable} is a symlink loop, with ELOOP, each established by a poll begun with it in place`,+ (reports) => {+ expect(+ unlinked(reports, file.target),+ `onUnlink reported ${file.target} while its ${unreadable} was only unreadable and then a symlink loop: ` ++ 'the watcher read EACCES or ELOOP as a deletion instead of keeping the last observation and reporting the error',+ ).toBe(false);+ return reportOf(reports, 'error-reports-across-code-change') !== undefined;+ },+ );++ const codes = reportOf(run.reports(), 'error-reports-across-code-change');+ for (const path of failing) {+ expect(+ reportsFor(codes?.beforeChange, 'EACCES', path),+ `the watcher never reported the EACCES of ${path} before its error code changed, at any logger level`,+ ).not.toEqual([]);+ expect(+ reportsFor(codes?.afterChange, 'ELOOP', path),+ `the watcher reported the EACCES of ${path} but not the ELOOP it failed with next: ` ++ 'reports are once per path and error code, so a new code on a path that keeps failing must be reported',+ ).not.toEqual([]);+ expect(+ asLogLines(reportsFor(codes?.whileItPersists, 'ELOOP', path)),+ `the watcher reported the ELOOP of ${path} again in further polls while it persisted, instead of once per path and error code`,+ ).toEqual([]);+ }+ },+ );+ },+);++describe('startManagedArtifactWatcher when a SKILL.md or its skill directory goes away', () => {+ test('reports a SKILL.md unlinked, never changed, once a directory takes its place', async () => {+ write(leaf('demo'), 'demo skill');+ const run = await watchFrom('managed', [skillsRoot()]);++ rmSync(leaf('demo'));+ mkdirSync(leaf('demo'));+ await run.until(`onUnlink to report ${leaf('demo')}, now a directory`, (reports) => {+ expect(+ deliveriesFor(reports, leaf('demo')),+ 'the watcher reported a SKILL.md that became a directory as changed: only a regular file reads as present',+ ).toEqual([]);+ return unlinked(reports, leaf('demo'));+ });+ });++ test('reports a deleted SKILL.md unlinked once, and not again while it stays deleted', async () => {+ write(leaf('demo'), 'demo skill');+ write(leaf('kept'), 'kept skill');+ const run = await watchFrom('managed', [skillsRoot()]);++ rmSync(leaf('demo'));+ await run.until(`onUnlink to report ${leaf('demo')}`, (reports) =>+ unlinked(reports, leaf('demo')),+ );+ for (const content of ['kept skill, edited once', 'kept skill, edited twice']) {+ writeFileSync(leaf('kept'), content, 'utf-8');+ await run.until(+ `a later report of ${leaf('kept')}, polls after the deletion was reported`,+ (reports) => delivered(reports, leaf('kept'), content),+ );+ }+ expect(+ run.reports().filter((report) => report.kind === 'unlink' && report.path === leaf('demo')),+ 'the watcher reported the deleted SKILL.md unlinked again in later polls while it stayed deleted',+ ).toHaveLength(1);+ });++ test('reports a SKILL.md unlinked when its whole skill directory is deleted', async () => {+ write(leaf('demo'), 'demo skill');+ const run = await watchFrom('managed', [skillsRoot()]);++ rmSync(join(skillsRoot(), 'demo'), { recursive: true, force: true });+ await run.until(+ `onUnlink to report ${leaf('demo')} once its skill directory was deleted, which drops it from the listing`,+ (reports) => unlinked(reports, leaf('demo')),+ );+ });+});++describe('closing a watcher between polls', () => {+ test('startConfigFileWatcher cleanup, called while no sample is in flight, leaves no poll armed to sample again', async () => {+ const { watch, target } = watchedFile('config', 'edit');+ const run = await watchFrom('config', watch);++ run.send({ op: 'close-between-polls', path: target });+ await run.until(+ 'the watcher process to fall idle on its own once cleanup resolved between two polls',+ (reports) => reportOf(reports, 'idle-after-close') !== undefined,+ );+ expect(+ reportOf(run.reports(), 'idle-after-close')?.samplesAfterClose,+ `the watcher sampled ${target} again after its cleanup resolved: cleanup left its next poll armed`,+ ).toBe(0);+ });+});++describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'a watcher started while its file is unreadable (EACCES; chmod does not bind root or Windows)',+ () => {+ test.each(ENTRIES.filter(({ entry }) => entry !== 'multi-config'))(+ '$watcher resolves its start and reports an edit made once the file is readable',+ async ({ entry }) => {+ const { watch, target } = watchedFile(entry, 'edit');+ const dir = dirname(target);+ const mode = statSync(dir).mode & 0o7777;+ madeUnreadable.push(dir);+ chmodSync(dir, 0o000);+ const run = await watchFrom(entry, watch);++ chmodSync(dir, mode);+ writeFileSync(target, READABLE_AGAIN, 'utf-8');+ await run.until(+ `onChange to report the edit of ${target} made once its directory was readable, after a start that met it unreadable`,+ (reports) => delivered(reports, target, READABLE_AGAIN),+ );+ },+ );+ },+);++describe('a file whose two-step write is interrupted between its halves is reported only once its size holds after the interruption', () => {+ test.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'startConfigFileWatcher never reports an edit half-written when a sample between the halves fails with EACCES (chmod does not bind root or Windows)',+ async () => {+ const { watch, target } = watchedFile('config', 'edit');+ const dir = dirname(target);+ madeUnreadable.push(dir);+ const run = await watchFrom('config', watch);++ run.send({+ op: 'two-step-write-across-interruption',+ path: target,+ dir,+ partial: PARTIAL,+ complete: COMPLETE,+ interruption: 'unreadable',+ });+ await run.until(+ `the watcher to report ${target} with both halves after a sample between them failed with EACCES, never the first half alone`,+ (reports) => {+ expect(+ deliveriesFor(reports, target),+ `the watcher reported ${target} holding only the first half once a sample between the halves had failed: a failed sample must drop the size it was waiting to see hold`,+ ).not.toContain(PARTIAL);+ return delivered(reports, target, COMPLETE);+ },+ );+ expect(+ reportOf(run.reports(), 'interrupted'),+ 'a watcher sample between the two halves failed with EACCES',+ ).toBeDefined();+ },+ );++ test('startConfigFileWatcher never reports a creation half-written when the file is deleted and created again with its first half', async () => {+ const { watch, target } = watchedFile('config', 'create');+ const run = await watchFrom('config', watch);++ run.send({+ op: 'two-step-write-across-interruption',+ path: target,+ dir: dirname(target),+ partial: PARTIAL,+ complete: COMPLETE,+ interruption: 'removed',+ });+ await run.until(+ `the watcher to report ${target} with both halves after a sample between them found it deleted, never the first half alone`,+ (reports) => {+ expect(+ deliveriesFor(reports, target),+ `the watcher reported ${target} holding only the first half once a sample between the halves had found it absent: an absent sample must drop the size it was waiting to see hold`,+ ).not.toContain(PARTIAL);+ return delivered(reports, target, COMPLETE);+ },+ );+ expect(+ reportOf(run.reports(), 'interrupted'),+ 'a watcher sample between the two halves found the file absent',+ ).toBeDefined();+ });+});++describe('a watched file replaced by one of the same size and modification time', () => {+ test('startConfigFileWatcher reports the replacement', async () => {+ const { watch, target } = watchedFile('config', 'edit');+ const { mtime } = statSync(target);+ utimesSync(target, mtime, mtime);+ const run = await watchFrom('config', watch);++ const replacement = `${target}.replacement`;+ write(replacement, SAME_SIZE_REPLACEMENT);+ utimesSync(replacement, mtime, mtime);+ expect(+ statSync(replacement, { bigint: true }).size,+ 'the replacement has the size of the file it replaces',+ ).toBe(statSync(target, { bigint: true }).size);+ expect(+ statSync(replacement, { bigint: true }).mtimeNs,+ 'the replacement has the modification time of the file it replaces',+ ).toBe(statSync(target, { bigint: true }).mtimeNs);+ renameSync(replacement, target);+ await run.until(+ `onChange to report the replacement of ${target}, which kept its size and modification time but not its inode or change time`,+ (reports) => delivered(reports, target, SAME_SIZE_REPLACEMENT),+ );+ });+});++describe('a watched file rewritten in place at its size inside the timestamp granule its baseline was sampled in (whole-second timestamps modelled at the stat seam)', () => {+ test('startManagedArtifactWatcher reports the rewrite, which leaves every stat field as the baseline sampled it', async () => {+ const { watch, target } = watchedFile('managed', 'edit');+ const run = await watchFrom('managed', watch, target);++ run.send({ op: 'rewrite-in-place', path: target, content: IN_PLACE_REWRITE });+ await run.until(+ `the watcher process to rewrite ${target} in place`,+ (reports) => reportOf(reports, 'rewritten-in-place') !== undefined,+ );+ expect(+ reportOf(run.reports(), 'rewritten-in-place')?.statUnchanged,+ `the rewrite of ${target} changed its device, inode, size, modification time or change time, so it does not reproduce a rewrite inside one timestamp granule`,+ ).toBe(true);+ await run.until(+ `onChange to report the in-place rewrite of ${target}, which kept its size inside the timestamp granule of the baseline sample`,+ (reports) => delivered(reports, target, IN_PLACE_REWRITE),+ );+ });+});++describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'an error the watcher reported is reported again once its directory has been a regular file (EACCES, then ENOTDIR, then EACCES; chmod does not bind root or Windows)',+ () => {+ test.each(+ UNREADABLE_CASES.filter(+ ({ entry, unreadable }) => entry !== 'multi-config' && unreadable !== 'skills root',+ ),+ )(+ '$watcher reports the EACCES of every path it cannot read in each episode when its $unreadable is unreadable, then a regular file, then unreadable once more',+ async (unreadableCase) => {+ const { unreadable } = unreadableCase;+ const file = watchedFile(unreadableCase.entry, 'edit');+ const { dir, failing } = unreadableScope(unreadableCase, file);+ const run = await watchFrom(unreadableCase.entry, file.watch);++ madeUnreadable.push(dir);+ run.send({+ op: 'unreadable-again-after-recovery',+ dir,+ failingAt: failingSampleOf(unreadableCase, file),+ sampledEveryPoll: sampledEveryPollOf(unreadableCase, file),+ recovery: 'replaced-by-file',+ aside: join(fixture, 'aside'),+ });+ await run.until(+ `the watcher to sample through an unreadable episode of its ${unreadable}, a spell as a regular file and a second unreadable episode, each established by a poll begun with it in place`,+ (reports) => {+ expect(+ unlinked(reportsBefore(reports, 'removed'), file.target),+ `onUnlink reported ${file.target} while its ${unreadable} was only unreadable: ` ++ 'the watcher read the EACCES as a deletion instead of keeping the last observation and reporting the error',+ ).toBe(false);+ return reportOf(reports, 'error-reports-across-recovery') !== undefined;+ },+ );++ const episodes = reportOf(run.reports(), 'error-reports-across-recovery');+ for (const path of failing) {+ expect(+ reportsFor(episodes?.firstEpisode, 'EACCES', path),+ `the watcher never reported the EACCES of ${path} in the first unreadable episode, at any logger level`,+ ).not.toEqual([]);+ expect(+ reportsFor(episodes?.afterRecovery, 'EACCES', path),+ `the watcher reported the EACCES of ${path} before its ${unreadable} was a regular file but not after: ` ++ 'a sample that fails with ENOTDIR reads as absent, so the next failure must be reported again',+ ).not.toEqual([]);+ }+ },+ );+ },+);++describe.skipIf(process.platform === 'win32' || process.geteuid?.() === 0)(+ 'closing a watcher whose in-flight sample has not settled (the threadpool barrier is a POSIX FIFO; chmod does not bind root or Windows)',+ () => {+ test.each(ENTRIES)(+ '$watcher cleanup resolves while its sample of a watched file is held on the only libuv worker, and reports nothing once that sample fails',+ async ({ entry }) => {+ const { watch, target } = watchedFile(entry, 'edit');+ const dir = dirname(target);+ const barrier = join(fixture, 'threadpool-barrier');+ execFileSync('mkfifo', [barrier]);+ madeUnreadable.push(dir);+ const run = await watchFrom(entry, watch);++ run.send({ op: 'close-while-sample-held', path: target, dir, barrier });+ await run.until(+ `the watcher process to fall idle once cleanup was called while its sample of ${target} was held, and that sample then failed with EACCES`,+ (reports) => reportOf(reports, 'idle-after-held-close') !== undefined,+ );+ const idle = reportOf(run.reports(), 'idle-after-held-close');+ expect(+ idle?.resolvedWhileHeld,+ `cleanup waited on the watcher's in-flight sample of ${target}: a sample that never settles, on a stalled mount, keeps server shutdown from reaching its document flush`,+ ).toBe(true);+ expect(+ idle?.eventsAfterClose,+ 'the watcher called back after its cleanup resolved',+ ).toEqual([]);+ expect(+ asLogLines(idle?.loggedAfterClose ?? []),+ 'the watcher reported an error from a sample that landed after its cleanup resolved',+ ).toEqual([]);+ },+ );+ },+);diff --git a/packages/server/src/serialize-doc-ytext.test.ts b/packages/server/src/serialize-doc-ytext.test.tsindex a7f843668..4277bed86 100644--- a/packages/server/src/serialize-doc-ytext.test.ts+++ b/packages/server/src/serialize-doc-ytext.test.ts@@ -2,14 +2,12 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'noimport { tmpdir } from 'node:os';import { join } from 'node:path';import simpleGit from 'simple-git';-import { describe as _vitestDescribe, afterEach, beforeEach, expect, test, vi } from 'vitest';+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import { __resetQuiescenceForTests } from './bridge-quiescence.ts';import { resetMetrics } from './metrics.ts';import { createServer } from './server-factory.ts';import { waitWithinTestBudget } from './wait-within-test-budget.test-helper.ts';-const describe = process.env.CI ? _vitestDescribe.skip : _vitestDescribe;-vi.setConfig({ testTimeout: 20_000, hookTimeout: 20_000 });interface Fixture {diff --git a/packages/server/src/test-helper-caller-typecheck-coverage.test.ts b/packages/server/src/test-helper-caller-typecheck-coverage.test.tsindex 8e18660c7..a0e94a9c4 100644--- a/packages/server/src/test-helper-caller-typecheck-coverage.test.ts+++ b/packages/server/src/test-helper-caller-typecheck-coverage.test.ts@@ -9,7 +9,11 @@ const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');const TSCONFIG = 'tsconfig.test-helper-callers.json';const TYPECHECK_SCRIPT = 'typecheck:test-helper-callers';-const HELPERS = ['wait-within-test-budget.test-helper.ts', 'expect-stable.test-helper.ts'];+const HELPERS = [+ 'wait-within-test-budget.test-helper.ts',+ 'expect-stable.test-helper.ts',+ 'forked-watcher-process.test-helper.ts',+];const UNSCANNED_DIRS = new Set(['node_modules', 'dist', '.turbo']);diff --git a/packages/server/src/watcher-process.test-helper.ts b/packages/server/src/watcher-process.test-helper.tsnew file mode 100644index 000000000..c06c0fb58--- /dev/null+++ b/packages/server/src/watcher-process.test-helper.ts@@ -0,0 +1,1236 @@+import fs, {+ chmodSync,+ closeSync,+ existsSync,+ lstatSync,+ mkdirSync,+ open,+ openSync,+ renameSync,+ type StatWatcher,+ statSync,+ symlinkSync,+ unlinkSync,+ writeFileSync,+} from 'node:fs';+import { stat } from 'node:fs/promises';+import { syncBuiltinESMExports } from 'node:module';+import { dirname } from 'node:path';+import timers from 'node:timers';+import { promisify } from 'node:util';+import type {+ ContractCommand,+ ContractSpec,+ LoggedError,+ MutationTiming,+ ReadinessScenario,+ ReadinessSpec,+ Recovery,+ WatcherEntry,+ WatcherEvent,+ WatcherProcessReport,+ WatcherProcessSpec,+} from './forked-watcher-process.test-helper.ts';+import { loggerFactory, PinoLogger } from './logger.ts';++type StopWatcher = () => Promise<void>;++type StartWatcher = (+ watch: readonly string[],+ deliver: (event: WatcherEvent) => void,+) => Promise<StopWatcher>;++type AnyFunction = (...args: never[]) => unknown;++type BarrierState = 'not-engaged' | 'held' | 'released';++type SampleOutcome = { size: number } | { listed: number } | { code: string };++type SampleForm = 'callback' | 'promise' | 'sync';++interface SampleHook {+ before?(form: SampleForm): void;+ hold?(form: SampleForm): Promise<void> | undefined;+ after?(outcome: SampleOutcome): void;+}++type ErrorPhase = 'landing' | 'establishing' | 'persisting';++type StagePhase = 'entering' | 'landing' | 'establishing';++type OutcomeTest = (outcome: SampleOutcome) => boolean;++interface ErrorStage {+ enteredAt: string;+ observedAt: string;+ enter?(): void;+ holds: OutcomeTest;+}++type CommandOf<Op extends ContractCommand['op']> = Extract<ContractCommand, { op: Op }>;++const NS_PER_MS = 1_000_000n;++const TIMESTAMP_GRANULE_NS = 1_000_000_000n;++function report(message: WatcherProcessReport): void {+ if (process.connected) process.send?.(message);+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null;+}++function text(record: Record<string, unknown>, name: string): string {+ const value = record[name];+ if (typeof value !== 'string' || value === '') {+ throw new Error(`watcher process: ${name} must be a non-empty string`);+ }+ return value;+}++function oneOf<Choice extends string>(value: string, choices: readonly Choice[]): Choice {+ const choice = choices.find((candidate) => candidate === value);+ if (choice === undefined) throw new Error(`watcher process: unexpected ${value}`);+ return choice;+}++function paths(record: Record<string, unknown>, name: string): string[] {+ const value = record[name];+ if (!Array.isArray(value) || value.length === 0) {+ throw new Error(`watcher process: ${name} must list at least one path`);+ }+ return value.map((path) => {+ if (typeof path !== 'string' || path === '') {+ throw new Error(`watcher process: ${name} must list paths`);+ }+ return path;+ });+}++function parseSpec(raw: string | undefined): WatcherProcessSpec {+ const spec: unknown = JSON.parse(raw ?? 'null');+ if (!isRecord(spec)) throw new Error('watcher process: missing spec');+ const entry = oneOf<WatcherEntry>(text(spec, 'entry'), ['managed', 'config', 'multi-config']);+ const watch = paths(spec, 'watch');+ if (oneOf(text(spec, 'mode'), ['readiness', 'contract']) === 'contract') {+ if (spec.coarseTimestampsAt === undefined) return { mode: 'contract', entry, watch };+ return { mode: 'contract', entry, watch, coarseTimestampsAt: text(spec, 'coarseTimestampsAt') };+ }+ return {+ mode: 'readiness',+ entry,+ watch,+ scenario: oneOf<ReadinessScenario>(text(spec, 'scenario'), ['create', 'edit']),+ timing: oneOf<MutationTiming>(text(spec, 'timing'), [+ 'as-start-resolves',+ 'while-first-baseline-stat-held',+ 'after-first-baseline-stat',+ ]),+ target: text(spec, 'target'),+ held: text(spec, 'held'),+ content: text(spec, 'content'),+ barrier: text(spec, 'barrier'),+ };+}++function parseCommand(message: unknown): ContractCommand {+ if (!isRecord(message)) throw new Error('watcher process: malformed command');+ const op = oneOf(text(message, 'op'), [+ 'write-in-two-steps',+ 'unreadable-until-sampled',+ 'close-while-sampling',+ 'hold-next-sample',+ 'unreadable-across-polls',+ 'unreadable-again-after-recovery',+ 'error-code-changes-while-failing',+ 'close-between-polls',+ 'close-while-sample-held',+ 'two-step-write-across-interruption',+ 'rewrite-in-place',+ ]);+ if (op === 'close-between-polls') return { op, path: text(message, 'path') };+ if (op === 'close-while-sample-held') {+ return {+ op,+ path: text(message, 'path'),+ dir: text(message, 'dir'),+ barrier: text(message, 'barrier'),+ };+ }+ if (op === 'two-step-write-across-interruption') {+ return {+ op,+ path: text(message, 'path'),+ dir: text(message, 'dir'),+ partial: text(message, 'partial'),+ complete: text(message, 'complete'),+ interruption: oneOf(text(message, 'interruption'), ['unreadable', 'removed'] as const),+ };+ }+ if (op === 'write-in-two-steps') {+ return {+ op,+ path: text(message, 'path'),+ partial: text(message, 'partial'),+ complete: text(message, 'complete'),+ };+ }+ if (op === 'unreadable-until-sampled') {+ return { op, dir: text(message, 'dir'), leaf: text(message, 'leaf') };+ }+ if (op === 'hold-next-sample') {+ return {+ op,+ path: text(message, 'path'),+ content: text(message, 'content'),+ barrier: text(message, 'barrier'),+ };+ }+ if (op === 'unreadable-across-polls') {+ return { op, dir: text(message, 'dir'), path: text(message, 'path') };+ }+ if (op === 'unreadable-again-after-recovery') {+ return {+ op,+ dir: text(message, 'dir'),+ failingAt: text(message, 'failingAt'),+ sampledEveryPoll: text(message, 'sampledEveryPoll'),+ recovery: oneOf<Recovery>(text(message, 'recovery'), [+ 'readable',+ 'removed',+ 'replaced-by-file',+ ]),+ aside: text(message, 'aside'),+ };+ }+ if (op === 'error-code-changes-while-failing') {+ return {+ op,+ dir: text(message, 'dir'),+ failingAt: text(message, 'failingAt'),+ aside: text(message, 'aside'),+ };+ }+ return { op, path: text(message, 'path'), content: text(message, 'content') };+}++async function loadWatcher(entry: WatcherEntry): Promise<StartWatcher> {+ if (entry === 'managed') {+ const { startManagedArtifactWatcher } = await import('./managed-artifact-watcher.ts');+ return (watch, deliver) =>+ startManagedArtifactWatcher(+ watch,+ (path, content) => deliver({ kind: 'change', path, content }),+ (path) => deliver({ kind: 'unlink', path }),+ );+ }+ const { startConfigFileWatcher, startMultiPathConfigFileWatcher } = await import(+ './config-file-watcher.ts'+ );+ if (entry === 'multi-config') {+ return (watch, deliver) =>+ startMultiPathConfigFileWatcher(watch, (path, content) =>+ deliver({ kind: 'change', path, content }),+ );+ }+ return (watch, deliver) => {+ const [path] = watch;+ if (path === undefined || watch.length !== 1) {+ throw new Error('watcher process: startConfigFileWatcher watches exactly one path');+ }+ return startConfigFileWatcher(path, (content) => deliver({ kind: 'change', path, content }));+ };+}++function assertFifo(fifo: string): void {+ if (!statSync(fifo).isFIFO()) throw new Error(`watcher process: ${fifo} is not a FIFO`);+}++function blockTheOnlyWorker(fifo: string): () => void {+ let releaseFd: number | undefined;+ let released = false;+ open(fifo, 'r', (error, readFd) => {+ if (error) throw error;+ closeSync(readFd);+ if (releaseFd !== undefined) closeSync(releaseFd);+ releaseFd = undefined;+ });+ const release = (): void => {+ if (released) return;+ released = true;+ releaseFd = openSync(fifo, 'r+');+ };+ process.on('exit', release);+ return release;+}++function holdFirstBaselineStat(+ samples: ReturnType<typeof observeWatcherSamples>,+ heldPath: string,+ fifo: string,+ onHeld: (heldBy: 'watchFile' | 'sample') => void,+) {+ assertFifo(fifo);+ let watchFileState: BarrierState = 'not-engaged';+ let sampleState: BarrierState = 'not-engaged';+ let sampleLanded = false;+ let starting = true;+ let releaseWorker: () => void = () => {};+ let releaseSample: () => void = () => {};+ let markReleased: () => void = () => {};+ const released = new Promise<void>((resolve) => {+ markReleased = resolve;+ });+ const letSampleRun = (): void => {+ if (sampleState !== 'held') return;+ releaseSample();+ sampleState = 'released';+ };+ const release = (): void => {+ letSampleRun();+ if (watchFileState === 'held') {+ releaseWorker();+ watchFileState = 'released';+ }+ markReleased();+ };+ const originalWatchFile = fs.watchFile;+ function watchFileBehindBarrier(this: unknown, ...args: unknown[]): StatWatcher {+ if (watchFileState === 'not-engaged' && args[0] === heldPath) {+ watchFileState = 'held';+ releaseWorker = blockTheOnlyWorker(fifo);+ onHeld('watchFile');+ }+ return Reflect.apply(originalWatchFile, this, args);+ }+ Object.assign(fs, { watchFile: watchFileBehindBarrier });+ syncBuiltinESMExports();+ samples.watch(heldPath, {+ hold(form) {+ samples.unwatch(heldPath);+ if (form === 'sync' || !starting) return undefined;+ sampleState = 'held';+ const gate = new Promise<void>((resolve) => {+ releaseSample = resolve;+ });+ onHeld('sample');+ return gate;+ },+ after() {+ sampleLanded = true;+ },+ });+ process.on('exit', release);+ return {+ engaged: (): boolean => watchFileState !== 'not-engaged' || sampleState !== 'not-engaged',+ held: (): boolean =>+ watchFileState === 'held' || (sampleState !== 'not-engaged' && !sampleLanded),+ starting: (): boolean => starting,+ startResolved: (): void => {+ starting = false;+ },+ letSampleRun,+ release,+ released,+ };+}++function trackStartupTimers() {+ const running = new Set<unknown>();+ const beforeNextTick: Array<() => void> = [];+ let armedWhileStarting = 0;+ let starting = false;+ let onStopped: () => void = () => {};+ const originalSetInterval = globalThis.setInterval;+ const originalClearInterval = globalThis.clearInterval;+ const originalClearTimeout = globalThis.clearTimeout;+ function setIntervalTracked(this: unknown, callback: AnyFunction, ...rest: unknown[]): unknown {+ let handle: unknown;+ function tick(this: unknown, ...args: unknown[]): void {+ if (running.has(handle)) for (const action of beforeNextTick.splice(0)) action();+ Reflect.apply(callback, this, args);+ }+ handle = Reflect.apply(originalSetInterval, this, [tick, ...rest]);+ if (starting) {+ armedWhileStarting += 1;+ running.add(handle);+ }+ return handle;+ }+ const clearTracked = (original: AnyFunction) =>+ function (this: unknown, handle: unknown): void {+ if (running.delete(handle)) onStopped();+ Reflect.apply(original, this, [handle]);+ };+ const replacements = {+ setInterval: setIntervalTracked,+ clearInterval: clearTracked(originalClearInterval),+ clearTimeout: clearTracked(originalClearTimeout),+ };+ Object.assign(globalThis, replacements);+ Object.assign(timers, replacements);+ syncBuiltinESMExports();+ return {+ async whileStarting<Started>(start: () => Promise<Started>): Promise<Started> {+ starting = true;+ try {+ return await start();+ } finally {+ starting = false;+ }+ },+ armedWhileStarting: () => armedWhileStarting,+ running: () => running.size,+ allStopped: () =>+ new Promise<void>((resolve) => {+ onStopped = () => {+ if (running.size === 0) resolve();+ };+ onStopped();+ }),+ beforeNextTick(action: () => void): boolean {+ if (running.size === 0) return false;+ beforeNextTick.push(action);+ return true;+ },+ };+}++function trackPendingTimers() {+ const pending = new Set<unknown>();+ const settledListeners = new Set<(handle: unknown) => void>();+ const originalSetTimeout = globalThis.setTimeout;+ const settled = (handle: unknown): void => {+ for (const listener of settledListeners) listener(handle);+ };+ const armTracked = (original: AnyFunction, repeats: boolean) =>+ function (this: unknown, callback: AnyFunction, ...rest: unknown[]): unknown {+ let handle: unknown;+ function fire(this: unknown, ...args: unknown[]): void {+ if (repeats ? pending.has(handle) : pending.delete(handle)) settled(handle);+ Reflect.apply(callback, this, args);+ }+ handle = Reflect.apply(original, this, [fire, ...rest]);+ pending.add(handle);+ return handle;+ };+ const clearTracked = (original: AnyFunction) =>+ function (this: unknown, handle: unknown): void {+ if (pending.delete(handle)) settled(handle);+ Reflect.apply(original, this, [handle]);+ };+ const setTimeoutTracked = armTracked(originalSetTimeout, false);+ Object.defineProperty(setTimeoutTracked, promisify.custom, {+ value: Reflect.get(originalSetTimeout, promisify.custom),+ });+ const replacements = {+ setTimeout: setTimeoutTracked,+ setInterval: armTracked(globalThis.setInterval, true),+ clearTimeout: clearTracked(globalThis.clearTimeout),+ clearInterval: clearTracked(globalThis.clearInterval),+ };+ Object.assign(globalThis, replacements);+ Object.assign(timers, replacements);+ syncBuiltinESMExports();+ return {+ eachPendingTimerSettled(): Promise<number> {+ const awaited = new Set(pending);+ const waitedOut = awaited.size;+ return new Promise((resolve) => {+ let done = false;+ const onSettled = (handle: unknown): void => {+ if (awaited.delete(handle)) setImmediate(check);+ };+ function check(): void {+ if (done || awaited.size > 0) return;+ done = true;+ settledListeners.delete(onSettled);+ resolve(waitedOut);+ }+ settledListeners.add(onSettled);+ setImmediate(check);+ });+ },+ };+}++function errnoOf(error: unknown): string {+ if (isRecord(error) && typeof error.code === 'string') return error.code;+ return 'unknown';+}++type OutcomeOf = (error: unknown, result: unknown) => SampleOutcome;++function observeWatcherSamples() {+ const hooks = new Map<string, SampleHook>();+ const statOutcome: OutcomeOf = (error, stats) => {+ if (error) return { code: errnoOf(error) };+ if (isRecord(stats) && (typeof stats.size === 'number' || typeof stats.size === 'bigint')) {+ return { size: Number(stats.size) };+ }+ return { code: 'ENOENT' };+ };+ const listingOutcome: OutcomeOf = (error, entries) => {+ if (error) return { code: errnoOf(error) };+ return { listed: Array.isArray(entries) ? entries.length : 0 };+ };+ const callbackForm = (original: AnyFunction, outcomeOf: OutcomeOf = statOutcome) =>+ function (this: unknown, ...args: unknown[]): unknown {+ const hook = hooks.get(String(args[0]));+ const callback = args.at(-1);+ if (hook === undefined || typeof callback !== 'function') {+ return Reflect.apply(original, this, args);+ }+ hook.before?.('callback');+ const gate = hook.hold?.('callback');+ const sample = (): unknown =>+ Reflect.apply(original, this, [+ ...args.slice(0, -1),+ (error: unknown, stats: unknown) => {+ hook.after?.(outcomeOf(error, stats));+ Reflect.apply(callback, undefined, [error, stats]);+ },+ ]);+ if (gate === undefined) return sample();+ void gate.then(sample);+ return undefined;+ };+ const promiseForm = (original: AnyFunction, outcomeOf: OutcomeOf = statOutcome) =>+ function (this: unknown, ...args: unknown[]): unknown {+ const hook = hooks.get(String(args[0]));+ if (hook === undefined) return Reflect.apply(original, this, args);+ hook.before?.('promise');+ const gate = hook.hold?.('promise');+ const sampled: Promise<unknown> =+ gate === undefined+ ? Reflect.apply(original, this, args)+ : gate.then(() => Reflect.apply(original, this, args));+ return sampled.then(+ (stats) => {+ hook.after?.(outcomeOf(null, stats));+ return stats;+ },+ (error: unknown) => {+ hook.after?.(outcomeOf(error, undefined));+ throw error;+ },+ );+ };+ const syncForm = (original: AnyFunction, outcomeOf: OutcomeOf = statOutcome) =>+ function (this: unknown, ...args: unknown[]): unknown {+ const hook = hooks.get(String(args[0]));+ if (hook === undefined) return Reflect.apply(original, this, args);+ hook.before?.('sync');+ let stats: unknown;+ try {+ stats = Reflect.apply(original, this, args);+ } catch (error) {+ hook.after?.(outcomeOf(error, undefined));+ throw error;+ }+ hook.after?.(outcomeOf(null, stats));+ return stats;+ };+ Object.assign(fs, {+ stat: callbackForm(fs.stat),+ lstat: callbackForm(fs.lstat),+ statSync: syncForm(fs.statSync),+ lstatSync: syncForm(fs.lstatSync),+ readdir: callbackForm(fs.readdir, listingOutcome),+ readdirSync: syncForm(fs.readdirSync, listingOutcome),+ });+ Object.assign(fs.promises, {+ stat: promiseForm(fs.promises.stat),+ lstat: promiseForm(fs.promises.lstat),+ readdir: promiseForm(fs.promises.readdir, listingOutcome),+ });+ syncBuiltinESMExports();+ return {+ watch: (path: string, hook: SampleHook): void => {+ hooks.set(path, hook);+ },+ unwatch: (path: string): void => {+ hooks.delete(path);+ },+ };+}++function wallClockNs(): bigint {+ return BigInt(Date.now()) * NS_PER_MS;+}++function changedAtNs(stats: Record<string, unknown>): bigint | undefined {+ if (typeof stats.ctimeNs === 'bigint') return stats.ctimeNs;+ if (typeof stats.ctimeMs === 'number') return BigInt(Math.floor(stats.ctimeMs)) * NS_PER_MS;+ return undefined;+}++function restampIntoGranule(stats: unknown, granuleStartNs: bigint): unknown {+ if (!isRecord(stats)) return stats;+ const changedNs = changedAtNs(stats);+ if (changedNs === undefined || changedNs >= granuleStartNs + TIMESTAMP_GRANULE_NS) return stats;+ const granuleStartMs = granuleStartNs / NS_PER_MS;+ const stamp = new Date(Number(granuleStartMs));+ if (typeof stats.ctimeNs === 'bigint') {+ Object.assign(stats, {+ mtimeNs: granuleStartNs,+ ctimeNs: granuleStartNs,+ mtimeMs: granuleStartMs,+ ctimeMs: granuleStartMs,+ });+ } else {+ Object.assign(stats, { mtimeMs: Number(granuleStartMs), ctimeMs: Number(granuleStartMs) });+ }+ Object.assign(stats, { mtime: stamp, ctime: stamp });+ return stats;+}++function coarsenTimestampsOf(path: string): void {+ let granuleStartNs: bigint | undefined;+ const issuedFor = (args: readonly unknown[]): bigint | undefined => {+ if (String(args[0]) !== path) return undefined;+ granuleStartNs ??= wallClockNs();+ return granuleStartNs;+ };+ const callbackForm = (original: AnyFunction) =>+ function (this: unknown, ...args: unknown[]): unknown {+ const startNs = issuedFor(args);+ const callback = args.at(-1);+ if (startNs === undefined || typeof callback !== 'function') {+ return Reflect.apply(original, this, args);+ }+ return Reflect.apply(original, this, [+ ...args.slice(0, -1),+ (error: unknown, stats: unknown) =>+ Reflect.apply(callback, undefined, [+ error,+ error ? stats : restampIntoGranule(stats, startNs),+ ]),+ ]);+ };+ const promiseForm = (original: AnyFunction) =>+ function (this: unknown, ...args: unknown[]): unknown {+ const startNs = issuedFor(args);+ if (startNs === undefined) return Reflect.apply(original, this, args);+ const sampled: Promise<unknown> = Reflect.apply(original, this, args);+ return sampled.then((stats) => restampIntoGranule(stats, startNs));+ };+ const syncForm = (original: AnyFunction) =>+ function (this: unknown, ...args: unknown[]): unknown {+ const startNs = issuedFor(args);+ if (startNs === undefined) return Reflect.apply(original, this, args);+ return restampIntoGranule(Reflect.apply(original, this, args), startNs);+ };+ Object.assign(fs, {+ stat: callbackForm(fs.stat),+ lstat: callbackForm(fs.lstat),+ statSync: syncForm(fs.statSync),+ lstatSync: syncForm(fs.lstatSync),+ });+ Object.assign(fs.promises, {+ stat: promiseForm(fs.promises.stat),+ lstat: promiseForm(fs.promises.lstat),+ });+ syncBuiltinESMExports();+}++function carriedError(data: unknown): { code: string; path: string } | undefined {+ const carried = isRecord(data) ? [data, ...Object.values(data)] : [data];+ for (const candidate of carried.flatMap((item) => [item, isRecord(item) ? item.cause : null])) {+ if (isRecord(candidate) && typeof candidate.code === 'string') {+ return {+ code: candidate.code,+ path: typeof candidate.path === 'string' ? candidate.path : '',+ };+ }+ }+ return undefined;+}++function recordLoggedErrors(): () => LoggedError[] {+ const logged: LoggedError[] = [];+ const record = (level: string, data: unknown, message: string): void => {+ const error = carriedError(data);+ if (error !== undefined) logged.push({ level, ...error, message });+ };+ class ErrorRecordingLogger extends PinoLogger {+ override error(data: unknown, message: string): void {+ record('error', data, message);+ super.error(data, message);+ }+ override warn(data: unknown, message: string): void {+ record('warn', data, message);+ super.warn(data, message);+ }+ override info(data: unknown, message: string): void {+ record('info', data, message);+ super.info(data, message);+ }+ override debug(data: unknown, message: string): void {+ record('debug', data, message);+ super.debug(data, message);+ }+ }+ loggerFactory.configure({ loggerFactory: (name) => new ErrorRecordingLogger(name) });+ return () => logged.map((entry) => ({ ...entry }));+}++function failsWith(code: string): OutcomeTest {+ return (outcome) => 'code' in outcome && outcome.code === code;+}++function presentOrAbsent(outcome: SampleOutcome): boolean {+ return !('code' in outcome) || outcome.code === 'ENOENT';+}++function putDirectoryBack(dir: string, mode: number, aside: string): void {+ const atDir = lstatSync(dir, { throwIfNoEntry: false });+ if (atDir?.isSymbolicLink() || (atDir?.isFile() && existsSync(aside))) unlinkSync(dir);+ if (existsSync(aside)) renameSync(aside, dir);+ if (existsSync(dir)) chmodSync(dir, mode);+}++let stopWatcher: StopWatcher | undefined;+let releaseBarrier: () => void = () => {};++async function shutDown(): Promise<void> {+ releaseBarrier();+ await stopWatcher?.();+ process.exit(0);+}++async function runReadiness(spec: ReadinessSpec): Promise<void> {+ let firstBaselineStatHeldAtMutation: boolean | undefined;+ const barrier = holdFirstBaselineStat(+ observeWatcherSamples(),+ spec.held,+ spec.barrier,+ (heldBy) => {+ if (spec.timing === 'after-first-baseline-stat') setImmediate(barrier.release);+ else if (heldBy === 'sample') setImmediate(whileStartWaits);+ },+ );+ const mutate = (): boolean => {+ const heldAtMutation = barrier.held();+ if (spec.scenario === 'create') mkdirSync(dirname(spec.target), { recursive: true });+ writeFileSync(spec.target, spec.content, 'utf-8');+ barrier.release();+ return heldAtMutation;+ };+ const whileStartWaits = (): void => {+ if (!barrier.starting()) return;+ if (spec.timing === 'while-first-baseline-stat-held') {+ firstBaselineStatHeldAtMutation = mutate();+ } else {+ barrier.letSampleRun();+ }+ };+ releaseBarrier = barrier.release;+ const startupTimers = trackStartupTimers();+ const start = await loadWatcher(spec.entry);+ stopWatcher = await startupTimers.whileStarting(() => start(spec.watch, report));+ barrier.startResolved();+ const firstBaselineStatHeldAtResolve = barrier.held();+ report({+ kind: 'started',+ barrierEngaged: barrier.engaged(),+ firstBaselineStatHeldAtResolve,+ startupTimers: startupTimers.armedWhileStarting(),+ });++ if (spec.timing === 'after-first-baseline-stat') {+ if (barrier.engaged()) await barrier.released;+ await stat(spec.held).catch(() => undefined);+ }+ if (firstBaselineStatHeldAtMutation === undefined) {+ await startupTimers.allStopped();+ firstBaselineStatHeldAtMutation = mutate();+ }+ report({+ kind: 'mutated',+ barrierEngaged: barrier.engaged(),+ firstBaselineStatHeldAtResolve,+ firstBaselineStatHeldAtMutation,+ startupTimersWaitedOut: startupTimers.armedWhileStarting(),+ });+}++async function runContract(spec: ContractSpec): Promise<void> {+ const samples = observeWatcherSamples();+ if (spec.coarseTimestampsAt !== undefined) coarsenTimestampsOf(spec.coarseTimestampsAt);+ const startupTimers = trackStartupTimers();+ const pendingTimers = trackPendingTimers();+ const loggedErrors = recordLoggedErrors();+ const eventsAfterClose: WatcherEvent[] = [];+ let closed = false;+ const deliver = (event: WatcherEvent): void => {+ if (closed) eventsAfterClose.push(event);+ else report(event);+ };++ const writeInTwoSteps = (path: string, partial: string, complete: string): void => {+ let stage: 'pending' | 'written' | 'sampled' = 'pending';+ samples.watch(path, {+ before() {+ if (stage !== 'sampled') return;+ samples.unwatch(path);+ writeFileSync(path, complete, 'utf-8');+ report({ kind: 'completed' });+ },+ after(outcome) {+ if (stage !== 'written' || !('size' in outcome)) return;+ if (outcome.size !== Buffer.byteLength(partial)) return;+ stage = 'sampled';+ report({ kind: 'partial-sampled' });+ },+ });+ const writePartial = (atStartupTimerTick: boolean): void => {+ writeFileSync(path, partial, 'utf-8');+ stage = 'written';+ report({ kind: 'partial-written', atStartupTimerTick });+ };+ if (!startupTimers.beforeNextTick(() => writePartial(true))) writePartial(false);+ };++ const unreadableUntilSampled = (dir: string, leaf: string): void => {+ const mode = statSync(dir).mode & 0o7777;+ samples.watch(leaf, {+ after(outcome) {+ if (!('code' in outcome) || outcome.code !== 'EACCES') return;+ samples.unwatch(leaf);+ chmodSync(dir, mode);+ report({ kind: 'readable-again' });+ },+ });+ chmodSync(dir, 0o000);+ report({ kind: 'unreadable' });+ };++ const closeWatcher = async (): Promise<void> => {+ await stopWatcher?.();+ closed = true;+ await stopWatcher?.();+ report({ kind: 'closed' });+ process.once('beforeExit', () => {+ report({ kind: 'quiesced', eventsAfterClose });+ });+ process.channel?.unref();+ };++ const closeWhileSampling = (path: string, content: string): void => {+ let sampledOnce = false;+ samples.watch(path, {+ before() {+ if (!sampledOnce) return;+ samples.unwatch(path);+ report({ kind: 'closing' });+ void closeWatcher();+ },+ after(outcome) {+ if ('size' in outcome && outcome.size === Buffer.byteLength(content)) sampledOnce = true;+ },+ });+ writeFileSync(path, content, 'utf-8');+ };++ const holdNextSample = (path: string, content: string, barrier: string): void => {+ assertFifo(barrier);+ const listedRoots = spec.entry === 'managed' ? spec.watch : [];+ let stage: 'armed' | 'held' | 'checked' = 'armed';+ let samplesBegunWhileHeld = 0;+ const began = (): void => {+ if (stage === 'held') samplesBegunWhileHeld += 1;+ };+ const checked = (timersWaitedOut: number): void => {+ stage = 'checked';+ samples.unwatch(path);+ for (const root of listedRoots) samples.unwatch(root);+ report({ kind: 'overlap-checked', samplesBegunWhileHeld, timersWaitedOut });+ releaseBarrier();+ };+ for (const root of listedRoots) samples.watch(root, { before: began });+ samples.watch(path, {+ before(form) {+ if (stage !== 'armed') {+ began();+ return;+ }+ report({ kind: 'sample-held' });+ if (form === 'sync') {+ checked(0);+ return;+ }+ stage = 'held';+ releaseBarrier = blockTheOnlyWorker(barrier);+ setImmediate(() => {+ void pendingTimers.eachPendingTimerSettled().then(checked);+ });+ },+ });+ writeFileSync(path, content, 'utf-8');+ };++ const unreadableAcrossPolls = (dir: string, path: string): void => {+ const mode = statSync(dir).mode & 0o7777;+ let phase: ErrorPhase = 'landing';+ let failedIn: ErrorPhase | undefined;+ let onceEstablished: LoggedError[] = [];+ samples.watch(path, {+ before() {+ if (failedIn !== phase) return;+ if (phase === 'landing') {+ phase = 'establishing';+ return;+ }+ if (phase === 'establishing') {+ onceEstablished = loggedErrors();+ phase = 'persisting';+ return;+ }+ samples.unwatch(path);+ const afterAnotherPoll = loggedErrors();+ chmodSync(dir, mode);+ report({ kind: 'error-reports', onceEstablished, afterAnotherPoll });+ },+ after(outcome) {+ if ('code' in outcome && outcome.code === 'EACCES') failedIn = phase;+ },+ });+ chmodSync(dir, 0o000);+ report({ kind: 'unreadable' });+ };++ const stepThroughStages = (+ stages: readonly ErrorStage[],+ whenAllEstablished: (snapshots: LoggedError[][]) => void,+ ): (() => void) => {+ const watched = new Set(stages.flatMap((stage) => [stage.enteredAt, stage.observedAt]));+ const snapshots: LoggedError[][] = [];+ let index = 0;+ let phase: StagePhase = 'landing';+ let held = false;+ const stop = (): void => {+ for (const path of watched) samples.unwatch(path);+ };+ const before = (path: string): void => {+ const stage = stages[index];+ if (stage === undefined) return;+ if (phase === 'entering') {+ if (path !== stage.enteredAt) return;+ stage.enter?.();+ phase = 'landing';+ held = false;+ return;+ }+ if (path !== stage.observedAt || !held) return;+ held = false;+ if (phase === 'landing') {+ phase = 'establishing';+ return;+ }+ snapshots.push(loggedErrors());+ index += 1;+ if (index === stages.length) {+ stop();+ whenAllEstablished(snapshots);+ return;+ }+ phase = 'entering';+ before(path);+ };+ for (const path of watched) {+ samples.watch(path, {+ before: () => before(path),+ after(outcome) {+ const stage = stages[index];+ if (stage === undefined || phase === 'entering' || path !== stage.observedAt) return;+ if (stage.holds(outcome)) held = true;+ },+ });+ }+ stages[0]?.enter?.();+ return stop;+ };++ const unreadableAgainAfterRecovery = ({+ dir,+ failingAt,+ sampledEveryPoll,+ recovery,+ aside,+ }: CommandOf<'unreadable-again-after-recovery'>): void => {+ const mode = statSync(dir).mode & 0o7777;+ const stop = stepThroughStages(+ [+ {+ enteredAt: failingAt,+ observedAt: failingAt,+ enter: () => chmodSync(dir, 0o000),+ holds: failsWith('EACCES'),+ },+ {+ enteredAt: sampledEveryPoll,+ observedAt: sampledEveryPoll,+ enter() {+ chmodSync(dir, mode);+ if (recovery !== 'readable') {+ renameSync(dir, aside);+ if (recovery === 'replaced-by-file')+ writeFileSync(dir, 'a regular file where the directory was', 'utf-8');+ report({ kind: 'removed' });+ }+ },+ holds: (outcome) => presentOrAbsent(outcome) || failsWith('ENOTDIR')(outcome),+ },+ {+ enteredAt: sampledEveryPoll,+ observedAt: failingAt,+ enter() {+ if (recovery === 'replaced-by-file') unlinkSync(dir);+ if (recovery !== 'readable') renameSync(aside, dir);+ chmodSync(dir, 0o000);+ },+ holds: failsWith('EACCES'),+ },+ ],+ ([firstEpisode = [], onceRecovered = [], secondEpisode = []]) => {+ putDirectoryBack(dir, mode, aside);+ report({+ kind: 'error-reports-across-recovery',+ firstEpisode,+ afterRecovery: secondEpisode.slice(onceRecovered.length),+ });+ },+ );+ process.on('exit', () => {+ stop();+ putDirectoryBack(dir, mode, aside);+ });+ };++ const errorCodeChangesWhileFailing = ({+ dir,+ failingAt,+ aside,+ }: CommandOf<'error-code-changes-while-failing'>): void => {+ const mode = statSync(dir).mode & 0o7777;+ const stop = stepThroughStages(+ [+ {+ enteredAt: failingAt,+ observedAt: failingAt,+ enter: () => chmodSync(dir, 0o000),+ holds: failsWith('EACCES'),+ },+ {+ enteredAt: failingAt,+ observedAt: failingAt,+ enter() {+ chmodSync(dir, mode);+ renameSync(dir, aside);+ symlinkSync(dir, dir);+ },+ holds: failsWith('ELOOP'),+ },+ { enteredAt: failingAt, observedAt: failingAt, holds: failsWith('ELOOP') },+ ],+ ([beforeChange = [], onceChanged = [], afterFurtherPolls = []]) => {+ putDirectoryBack(dir, mode, aside);+ report({+ kind: 'error-reports-across-code-change',+ beforeChange,+ afterChange: onceChanged.slice(beforeChange.length),+ whileItPersists: afterFurtherPolls.slice(onceChanged.length),+ });+ },+ );+ process.on('exit', () => {+ stop();+ putDirectoryBack(dir, mode, aside);+ });+ };++ const closeBetweenPolls = (path: string): void => {+ let closing = false;+ let samplesAfterClose = 0;+ samples.watch(path, {+ before() {+ if (closing) samplesAfterClose += 1;+ },+ after(outcome) {+ if (closing || !('size' in outcome)) return;+ closing = true;+ setImmediate(() => {+ void (async () => {+ await stopWatcher?.();+ closed = true;+ report({ kind: 'closed' });+ process.once('beforeExit', () => {+ report({ kind: 'idle-after-close', samplesAfterClose });+ });+ process.channel?.unref();+ })();+ });+ },+ });+ };++ const closeWhileSampleHeld = ({+ path,+ dir,+ barrier,+ }: CommandOf<'close-while-sample-held'>): void => {+ assertFifo(barrier);+ const mode = statSync(dir).mode & 0o7777;+ samples.watch(path, {+ before(form) {+ if (form === 'sync') return;+ samples.unwatch(path);+ releaseBarrier = blockTheOnlyWorker(barrier);+ let closeResolved = false;+ let loggedBeforeClose = 0;+ void stopWatcher?.().then(() => {+ closeResolved = true;+ closed = true;+ loggedBeforeClose = loggedErrors().length;+ });+ setImmediate(() => {+ const resolvedWhileHeld = closeResolved;+ chmodSync(dir, 0o000);+ releaseBarrier();+ process.once('beforeExit', () => {+ chmodSync(dir, mode);+ report({+ kind: 'idle-after-held-close',+ resolvedWhileHeld,+ eventsAfterClose,+ loggedAfterClose: loggedErrors().slice(loggedBeforeClose),+ });+ });+ process.channel?.unref();+ });+ },+ });+ };++ const twoStepWriteAcrossInterruption = ({+ path,+ dir,+ partial,+ complete,+ interruption,+ }: CommandOf<'two-step-write-across-interruption'>): void => {+ const mode = statSync(dir).mode & 0o7777;+ const partialSize = Buffer.byteLength(partial);+ let stage:+ | 'written'+ | 'partial-sampled'+ | 'interrupting'+ | 'interrupted'+ | 'restoring'+ | 'partial-resampled'+ | 'done' = 'written';+ const restore = (): void => {+ if (interruption === 'unreadable') chmodSync(dir, mode);+ else if (!existsSync(path)) writeFileSync(path, partial, 'utf-8');+ };+ samples.watch(path, {+ before() {+ if (stage === 'partial-sampled') {+ if (interruption === 'unreadable') chmodSync(dir, 0o000);+ else unlinkSync(path);+ stage = 'interrupting';+ } else if (stage === 'interrupted') {+ restore();+ stage = 'restoring';+ report({ kind: 'restored' });+ } else if (stage === 'partial-resampled') {+ samples.unwatch(path);+ stage = 'done';+ writeFileSync(path, complete, 'utf-8');+ report({ kind: 'completed' });+ }+ },+ after(outcome) {+ if (stage === 'written' && 'size' in outcome && outcome.size === partialSize) {+ stage = 'partial-sampled';+ report({ kind: 'partial-sampled' });+ } else if (+ stage === 'interrupting' &&+ 'code' in outcome &&+ outcome.code === (interruption === 'unreadable' ? 'EACCES' : 'ENOENT')+ ) {+ stage = 'interrupted';+ report({ kind: 'interrupted' });+ } else if (stage === 'restoring' && 'size' in outcome && outcome.size === partialSize) {+ stage = 'partial-resampled';+ }+ },+ });+ process.on('exit', restore);+ writeFileSync(path, partial, 'utf-8');+ };++ const rewriteInPlace = (path: string, content: string): void => {+ const before = statSync(path, { bigint: true });+ writeFileSync(path, content, 'utf-8');+ const after = statSync(path, { bigint: true });+ report({+ kind: 'rewritten-in-place',+ statUnchanged:+ before.dev === after.dev &&+ before.ino === after.ino &&+ before.size === after.size &&+ before.mtimeNs === after.mtimeNs &&+ before.ctimeNs === after.ctimeNs,+ });+ };++ const [firstWatched] = spec.watch;+ if (firstWatched === undefined) throw new Error('watcher process: nothing to watch');+ const start = await loadWatcher(spec.entry);+ stopWatcher = await startupTimers.whileStarting(() => start(spec.watch, deliver));+ await stat(spec.entry === 'managed' ? firstWatched : dirname(firstWatched));+ process.on('message', (message) => {+ const command = parseCommand(message);+ if (command.op === 'write-in-two-steps') {+ writeInTwoSteps(command.path, command.partial, command.complete);+ } else if (command.op === 'unreadable-until-sampled') {+ unreadableUntilSampled(command.dir, command.leaf);+ } else if (command.op === 'hold-next-sample') {+ holdNextSample(command.path, command.content, command.barrier);+ } else if (command.op === 'unreadable-across-polls') {+ unreadableAcrossPolls(command.dir, command.path);+ } else if (command.op === 'unreadable-again-after-recovery') {+ unreadableAgainAfterRecovery(command);+ } else if (command.op === 'error-code-changes-while-failing') {+ errorCodeChangesWhileFailing(command);+ } else if (command.op === 'close-between-polls') {+ closeBetweenPolls(command.path);+ } else if (command.op === 'close-while-sample-held') {+ closeWhileSampleHeld(command);+ } else if (command.op === 'two-step-write-across-interruption') {+ twoStepWriteAcrossInterruption(command);+ } else if (command.op === 'rewrite-in-place') {+ rewriteInPlace(command.path, command.content);+ } else {+ closeWhileSampling(command.path, command.content);+ }+ });+ report({ kind: 'settled', startupTimers: startupTimers.running() });+}++const spec = parseSpec(process.argv[2]);+if (process.env.UV_THREADPOOL_SIZE !== '1') {+ throw new Error(+ `watcher process: needs UV_THREADPOOL_SIZE=1 so ordering on the only libuv worker is first in, first out, got ${String(process.env.UV_THREADPOOL_SIZE)}`,+ );+}+process.once('disconnect', () => {+ void shutDown();+});+if (!process.connected) await shutDown();++if (spec.mode === 'readiness') await runReadiness(spec);+else await runContract(spec);diff --git a/packages/server/tsconfig.test-helper-callers.json b/packages/server/tsconfig.test-helper-callers.jsonindex c59240a4e..4d178fcca 100644--- a/packages/server/tsconfig.test-helper-callers.json+++ b/packages/server/tsconfig.test-helper-callers.json@@ -2,12 +2,15 @@"extends": "./tsconfig.json","include": [],"files": [+ "src/config-file-watcher-readiness.test.ts","src/expect-stable.test.ts","src/file-watcher-chokidar-fallback.test.ts","src/file-watcher-chokidar-readiness.test.ts",+ "src/managed-artifact-watcher-readiness.test.ts","src/managed-artifact-watcher.test.ts","src/persistence-tripwire-block.test.ts","src/persistence-tripwire-paste.test.ts",+ "src/polled-watcher-contract.test.ts","src/serialize-doc-vs-git-show-byte-equality.test.ts","src/serialize-doc-ytext.test.ts","src/wait-within-test-budget.test.ts"diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 96d7470dd..5ab9b7686 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -778,9 +778,6 @@ importers:babel-plugin-react-compiler:specifier: 0.0.0-experimental-a8e64ef-20260402version: 0.0.0-experimental-a8e64ef-20260402- chokidar:- specifier: ^5.0.0- version: 5.0.0commonmark.json:specifier: 0.31.0version: 0.31.0