Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Fix stale own-write observations blocking document saves (#5670)

Nick Gomez9466cb31a9d5
diff --git a/.changeset/young-words-drive.md b/.changeset/young-words-drive.md
new file mode 100644
index 000000000..9d883b3df
--- /dev/null
+++ b/.changeset/young-words-drive.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Prevent delayed observations of earlier saves from incorrectly blocking newer document writes with a conflict.
diff --git a/packages/app/tests/integration/older-own-write-echo.test.ts b/packages/app/tests/integration/older-own-write-echo.test.ts
new file mode 100644
index 000000000..0ba5395cc
--- /dev/null
+++ b/packages/app/tests/integration/older-own-write-echo.test.ts
@@ -0,0 +1,570 @@
+import { randomUUID } from 'node:crypto';
+import { readFileSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import * as agentSessions from '../../../server/src/agent-sessions';
+import * as watcherModule from '../../../server/src/file-watcher';
+import { contentHash } from '../../../server/src/version-hash';
+import { createTestServer, type TestServer } from './test-harness';
+
+const readOrder = vi.hoisted(() => ({
+ path: '',
+ hold: false,
+ beforeRead: false,
+ entered: Promise.withResolvers<void>(),
+ release: Promise.withResolvers<void>(),
+}));
+
+vi.mock('node:fs/promises', async (importOriginal) => {
+ const original = await importOriginal<typeof import('node:fs/promises')>();
+ return {
+ ...original,
+ readFile: async (...args: Parameters<typeof original.readFile>) => {
+ const held = readOrder.hold && String(args[0]) === readOrder.path;
+ if (held) readOrder.hold = false;
+ if (held && readOrder.beforeRead) {
+ readOrder.entered.resolve();
+ await readOrder.release.promise;
+ }
+ const bytes = await original.readFile(...args);
+ if (held && !readOrder.beforeRead) {
+ readOrder.entered.resolve();
+ await readOrder.release.promise;
+ }
+ return bytes;
+ },
+ };
+});
+
+let server: TestServer | undefined;
+let pending: Promise<unknown>[] = [];
+let gates: PromiseWithResolvers<void>[] = [];
+
+function gate() {
+ const opened = Promise.withResolvers<void>();
+ gates.push(opened);
+ return opened;
+}
+
+function watcherDecisionsFor(file: string) {
+ return watcherModule
+ .getWatcherDecisionRingSnapshot()
+ .filter((record) => basename(record.path) === basename(file))
+ .map((record) => record.decision);
+}
+
+afterEach(async () => {
+ readOrder.hold = false;
+ readOrder.beforeRead = false;
+ readOrder.release.resolve();
+ for (const opened of gates) opened.resolve();
+ gates = [];
+ await Promise.allSettled(pending);
+ pending = [];
+ vi.restoreAllMocks();
+ await server?.cleanup();
+ server = undefined;
+});
+
+async function startOrderedServer() {
+ let dispatch: ((event: watcherModule.DiskEvent) => Promise<void>) | undefined;
+ const startWatcher = watcherModule.startWatcher;
+ vi.spyOn(watcherModule, 'startWatcher').mockImplementation(async (...args) => {
+ dispatch = args[1];
+ const handle = await startWatcher(...args);
+ await handle.unsubscribe();
+ return handle;
+ });
+ server = await createTestServer();
+ const currentServer = server;
+ if (!dispatch) throw new Error('Server watcher callback was not registered');
+ const onDiskEvent = dispatch;
+ const docName = `own-observation-${randomUUID()}`;
+ const file = join(server.contentDir, `${docName}.md`);
+ const fileIndex = new Map<string, watcherModule.FileIndexEntry>();
+ const folderIndex = new Map<string, watcherModule.FolderIndexEntry>();
+ return {
+ server: currentServer,
+ docName,
+ file,
+ fileIndex,
+ folderIndex,
+ onDiskEvent,
+ post: (markdown: string) =>
+ fetch(`${currentServer.baseUrl}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docName, agentId: 'ordered-writer', markdown, position: 'append' }),
+ }),
+ observe: () =>
+ watcherModule.handleRawEvents(
+ [{ type: 'update', path: file }],
+ currentServer.contentDir,
+ undefined,
+ fileIndex,
+ folderIndex,
+ onDiskEvent,
+ ),
+ };
+}
+
+describe('older own-write observations', () => {
+ test.each([
+ {
+ label: 'distinct text',
+ first: 'Before\n',
+ next: 'After\n',
+ final: 'Final\n',
+ acknowledged: 'Before\n\nAfter\n',
+ expected: 'Before\n\nAfter\n\nFinal\n',
+ },
+ {
+ label: 'repeated text',
+ first: 'concurrent agent line\n',
+ next: 'concurrent agent line\n',
+ final: 'concurrent agent line\n',
+ acknowledged: 'concurrent agent line\n\nconcurrent agent line\n',
+ expected: 'concurrent agent line\n\nconcurrent agent line\n\nconcurrent agent line\n',
+ },
+ {
+ label: 'unchanged content',
+ first: 'Before\n',
+ next: '',
+ final: 'Final\n',
+ acknowledged: 'Before\n',
+ expected: 'Before\n\nFinal\n',
+ },
+ ])('an accepted write ignores an older observation with $label', async (contents) => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ expect((await post(contents.first)).status).toBe(200);
+ expect(readFileSync(file, 'utf8')).toBe(contents.first);
+ readOrder.path = file;
+ readOrder.entered = Promise.withResolvers<void>();
+ readOrder.release = Promise.withResolvers<void>();
+ readOrder.hold = true;
+ const earlierBatch = observe();
+ pending.push(earlierBatch);
+ await readOrder.entered.promise;
+ if (contents.next) expect((await post(contents.next)).status).toBe(200);
+ await observe();
+ expect(readFileSync(file, 'utf8')).toBe(contents.acknowledged);
+ const raised = vi.spyOn(active.instance.conflicts, 'raise');
+ const prepare = agentSessions.prepareAgentMarkdownParse;
+ vi.spyOn(agentSessions, 'prepareAgentMarkdownParse').mockImplementation((...args) => {
+ if (args[0].name === docName) readOrder.release.resolve();
+ return prepare(...args);
+ });
+ const writing = post(contents.final);
+ pending.push(writing);
+ const response = await writing;
+ await earlierBatch;
+ expect.soft(response.status, JSON.stringify(await response.json())).toBe(200);
+ expect.soft(raised.mock.calls.map(([input]) => input)).toEqual([]);
+ expect.soft(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ expect.soft(readFileSync(file, 'utf8')).toBe(contents.expected);
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ contents.expected,
+ );
+ });
+
+ test('a stale external write of bytes the server never saved still refuses a write', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ const external = 'External seed\n';
+ writeFileSync(file, external);
+ const registrations = vi.spyOn(watcherModule, 'registerWrite');
+ expect((await post('Acknowledged\n')).status).toBe(200);
+ await observe();
+ const acknowledged = 'External seed\n\nAcknowledged\n';
+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);
+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(acknowledged));
+ expect(registrations.mock.calls.map(([, hash]) => hash)).not.toContain(contentHash(external));
+ writeFileSync(file, external);
+ await observe();
+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({
+ kind: 'reconcile',
+ reason: 'stale-external-write',
+ stages: { base: acknowledged, ours: acknowledged, theirs: external },
+ });
+ const response = await post('Final\n');
+ expect(response.status).toBe(409);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:doc-in-conflict',
+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },
+ });
+ expect(readFileSync(file, 'utf8')).toBe(external);
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ acknowledged,
+ );
+ });
+
+ test('an external rollback to an already observed own save still refuses a write', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ expect((await post('Before\n')).status).toBe(200);
+ expect((await post('After\n')).status).toBe(200);
+ await observe();
+ writeFileSync(file, 'Before\n');
+ await observe();
+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({
+ kind: 'reconcile',
+ reason: 'stale-external-write',
+ stages: { base: 'Before\n\nAfter\n', ours: 'Before\n\nAfter\n', theirs: 'Before\n' },
+ });
+ const response = await post('Final\n');
+ expect(response.status).toBe(409);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:doc-in-conflict',
+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },
+ });
+ expect(readFileSync(file, 'utf8')).toBe('Before\n');
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ 'Before\n\nAfter\n',
+ );
+ });
+
+ test('a newer external edit still reconciles before the next write', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ expect((await post('Before\n')).status).toBe(200);
+ await observe();
+ const raised = vi.spyOn(active.instance.conflicts, 'raise');
+ writeFileSync(file, 'Edited outside\n');
+ await observe();
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ 'Edited outside\n',
+ );
+ const response = await post('Final\n');
+ expect(response.status).toBe(200);
+ expect(raised.mock.calls.map(([input]) => input)).toEqual([]);
+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ expect(readFileSync(file, 'utf8')).toBe('Edited outside\n\nFinal\n');
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ 'Edited outside\n\nFinal\n',
+ );
+ });
+
+ test('an in-flight read still detects an external rollback after the own save was consumed', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ expect((await post('Before\n')).status).toBe(200);
+ readOrder.path = file;
+ readOrder.entered = Promise.withResolvers<void>();
+ readOrder.release = Promise.withResolvers<void>();
+ readOrder.beforeRead = true;
+ readOrder.hold = true;
+ const earlierBatch = observe();
+ pending.push(earlierBatch);
+ await readOrder.entered.promise;
+ expect((await post('After\n')).status).toBe(200);
+ await observe();
+ writeFileSync(file, 'Before\n');
+ readOrder.release.resolve();
+ await earlierBatch;
+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({
+ kind: 'reconcile',
+ reason: 'stale-external-write',
+ stages: { base: 'Before\n\nAfter\n', ours: 'Before\n\nAfter\n', theirs: 'Before\n' },
+ });
+ const response = await post('Final\n');
+ expect(response.status).toBe(409);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:doc-in-conflict',
+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },
+ });
+ expect(readFileSync(file, 'utf8')).toBe('Before\n');
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ 'Before\n\nAfter\n',
+ );
+ });
+
+ test('an outside edit observed before an own save retains its conflict outcome', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ const registrations = vi.spyOn(watcherModule, 'registerWrite');
+ expect((await post('Before\n')).status).toBe(200);
+ await observe();
+ const external = 'Outside edit\n';
+ const acknowledged = 'OK save\n';
+ writeFileSync(file, external);
+ readOrder.path = file;
+ readOrder.entered = Promise.withResolvers<void>();
+ readOrder.release = Promise.withResolvers<void>();
+ readOrder.hold = true;
+ const earlierBatch = observe();
+ pending.push(earlierBatch);
+ await readOrder.entered.promise;
+ const saved = await fetch(`${active.baseUrl}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ docName,
+ agentId: 'ordered-writer',
+ markdown: acknowledged,
+ position: 'replace',
+ }),
+ });
+ expect(saved.status, JSON.stringify(await saved.json())).toBe(200);
+ await observe();
+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);
+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(acknowledged));
+ expect(registrations.mock.calls.map(([, hash]) => hash)).not.toContain(contentHash(external));
+ readOrder.release.resolve();
+ await earlierBatch;
+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({
+ kind: 'reconcile',
+ reason: 'stale-external-write',
+ stages: { base: acknowledged, ours: acknowledged, theirs: external },
+ });
+ const response = await post('Final\n');
+ expect(response.status).toBe(409);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:doc-in-conflict',
+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },
+ });
+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ acknowledged,
+ );
+ });
+ test('a superseded identical-byte outside rewrite is treated as an own echo', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ const registrations = vi.spyOn(watcherModule, 'registerWrite');
+ const raised = vi.spyOn(active.instance.conflicts, 'raise');
+ expect((await post('Before\n')).status).toBe(200);
+ const external = 'Before\n';
+ const acknowledged = 'OK save\n';
+ writeFileSync(file, external);
+ readOrder.path = file;
+ readOrder.entered = Promise.withResolvers<void>();
+ readOrder.release = Promise.withResolvers<void>();
+ readOrder.hold = true;
+ const earlierBatch = observe();
+ pending.push(earlierBatch);
+ await readOrder.entered.promise;
+ const saved = await fetch(`${active.baseUrl}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ docName,
+ agentId: 'ordered-writer',
+ markdown: acknowledged,
+ position: 'replace',
+ }),
+ });
+ expect(saved.status, JSON.stringify(await saved.json())).toBe(200);
+ await observe();
+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);
+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(acknowledged));
+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(external));
+ readOrder.release.resolve();
+ await earlierBatch;
+ expect(watcherDecisionsFor(file)).toEqual(['self-write-skip', 'superseded-own-write-skip']);
+ expect(raised.mock.calls.map(([input]) => input)).toEqual([]);
+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ acknowledged,
+ );
+ });
+
+ test('a retargeted alias keeps the new target when an old-target save overlaps its read', async () => {
+ const { server: active, docName, file, post } = await startOrderedServer();
+ expect((await post('Before\n')).status).toBe(200);
+ const targetName = `target-${randomUUID()}`;
+ const target = join(active.contentDir, `${targetName}.md`);
+ const aliasName = `alias-${randomUUID()}`;
+ const alias = join(active.contentDir, `${aliasName}.md`);
+ writeFileSync(target, 'Before\n');
+ symlinkSync(file, alias);
+ const aliases = new Map([[aliasName, docName]]);
+ readOrder.path = alias;
+ readOrder.entered = Promise.withResolvers<void>();
+ readOrder.release = Promise.withResolvers<void>();
+ readOrder.beforeRead = true;
+ readOrder.hold = true;
+ const observing = watcherModule.classifyEvents(
+ [{ type: 'update', path: alias }],
+ active.contentDir,
+ undefined,
+ aliases,
+ );
+ pending.push(observing);
+ await readOrder.entered.promise;
+ expect((await post('After\n')).status).toBe(200);
+ unlinkSync(alias);
+ symlinkSync(target, alias);
+ expect(readFileSync(alias, 'utf8')).toBe('Before\n');
+ expect(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n');
+ readOrder.release.resolve();
+ expect(await observing).toEqual([
+ { kind: 'update', path: alias, docName: targetName, content: 'Before\n' },
+ ]);
+ expect(aliases.get(aliasName)).toBe(targetName);
+ expect(watcherModule.lastKnownHash.get(alias)).toBe(contentHash('Before\n'));
+ });
+
+ test('an external rollback after a discarded older observation still refuses a write', async () => {
+ const { server: active, docName, file, post, observe } = await startOrderedServer();
+ expect((await post('Before\n')).status).toBe(200);
+ readOrder.path = file;
+ readOrder.entered = Promise.withResolvers<void>();
+ readOrder.release = Promise.withResolvers<void>();
+ readOrder.hold = true;
+ const earlierBatch = observe();
+ pending.push(earlierBatch);
+ await readOrder.entered.promise;
+ expect((await post('After\n')).status).toBe(200);
+ expect(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n');
+ readOrder.release.resolve();
+ await earlierBatch;
+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ writeFileSync(file, 'Before\n');
+ await observe();
+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({
+ kind: 'reconcile',
+ reason: 'stale-external-write',
+ stages: { base: 'Before\n\nAfter\n', ours: 'Before\n\nAfter\n', theirs: 'Before\n' },
+ });
+ const response = await post('Final\n');
+ expect(response.status).toBe(409);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:doc-in-conflict',
+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },
+ });
+ expect(readFileSync(file, 'utf8')).toBe('Before\n');
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ 'Before\n\nAfter\n',
+ );
+ });
+
+ test('an own rename saved again during an earlier dispatch still moves its index entry', async () => {
+ const { server: active } = await startOrderedServer();
+ const firstOld = `rename-${randomUUID()}`;
+ const firstNew = `rename-${randomUUID()}`;
+ const secondOld = `rename-${randomUUID()}`;
+ const secondNew = `rename-${randomUUID()}`;
+ const path = (name: string) => join(active.contentDir, `${name}.md`);
+ const moved = 'Moved outside\n';
+ const ownMove = 'Moved by OK\n';
+ const ownSave = 'Saved after the move\n';
+ writeFileSync(path(firstNew), moved);
+ writeFileSync(path(secondNew), ownMove);
+ watcherModule.updateLastKnownHash(path(firstOld), contentHash(moved));
+ watcherModule.updateLastKnownHash(path(secondOld), contentHash(ownMove));
+ watcherModule.registerWrite(path(secondNew), contentHash(ownMove));
+ const fileIndex = new Map<string, watcherModule.FileIndexEntry>(
+ [firstOld, secondOld].map((name) => [
+ name,
+ {
+ size: 0,
+ modified: new Date(0).toISOString(),
+ canonicalPath: path(name),
+ inode: 0,
+ aliases: [],
+ kind: 'markdown',
+ },
+ ]),
+ );
+ const entered = gate();
+ const release = gate();
+ const dispatched: watcherModule.DiskEvent[] = [];
+ const batch = watcherModule.handleRawEvents(
+ [
+ { type: 'delete', path: path(firstOld) },
+ { type: 'create', path: path(firstNew) },
+ { type: 'delete', path: path(secondOld) },
+ { type: 'create', path: path(secondNew) },
+ ],
+ active.contentDir,
+ undefined,
+ fileIndex,
+ new Map(),
+ async (event) => {
+ dispatched.push(event);
+ if (event.kind === 'rename' && event.newDocName === firstNew) {
+ entered.resolve();
+ await release.promise;
+ }
+ },
+ );
+ pending.push(batch);
+ await entered.promise;
+ watcherModule.registerWrite(path(secondNew), contentHash(ownSave));
+ writeFileSync(path(secondNew), ownSave);
+ release.resolve();
+ await batch;
+ expect(dispatched).toEqual([
+ {
+ kind: 'rename',
+ oldPath: path(firstOld),
+ newPath: path(firstNew),
+ oldDocName: firstOld,
+ newDocName: firstNew,
+ content: moved,
+ },
+ ]);
+ const labels = new Map([
+ [firstOld, 'first source'],
+ [firstNew, 'first destination'],
+ [secondOld, 'second source'],
+ [secondNew, 'second destination'],
+ ]);
+ expect([...fileIndex.keys()].map((name) => labels.get(name) ?? name).sort()).toEqual([
+ 'first destination',
+ 'second destination',
+ ]);
+ expect(fileIndex.get(secondNew)).toMatchObject({ kind: 'markdown' });
+ });
+
+ test('an older observation held behind another dispatch does not refuse a later write', async () => {
+ const {
+ server: active,
+ docName,
+ file,
+ fileIndex,
+ folderIndex,
+ onDiskEvent,
+ post,
+ observe,
+ } = await startOrderedServer();
+ const otherFile = join(active.contentDir, `outside-${randomUUID()}.md`);
+ writeFileSync(otherFile, 'Outside\n');
+ const raised = vi.spyOn(active.instance.conflicts, 'raise');
+ expect((await post('Before\n')).status).toBe(200);
+ const entered = gate();
+ const release = gate();
+ const earlierBatch = watcherModule.handleRawEvents(
+ [
+ { type: 'update', path: otherFile },
+ { type: 'update', path: file },
+ ],
+ active.contentDir,
+ undefined,
+ fileIndex,
+ folderIndex,
+ async (event) => {
+ if (event.kind === 'update' && event.path === otherFile) {
+ entered.resolve();
+ await release.promise;
+ }
+ await onDiskEvent(event);
+ },
+ );
+ pending.push(earlierBatch);
+ await entered.promise;
+ expect((await post('After\n')).status).toBe(200);
+ await observe();
+ expect(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n');
+ release.resolve();
+ await earlierBatch;
+ expect
+ .soft(watcherDecisionsFor(file))
+ .toEqual(['self-write-skip', 'superseded-own-write-skip']);
+ const response = await post('Final\n');
+ expect.soft(response.status, JSON.stringify(await response.json())).toBe(200);
+ expect.soft(raised.mock.calls.map(([input]) => input)).toEqual([]);
+ expect.soft(active.instance.conflicts.findByDocName(docName)).toBeUndefined();
+ expect.soft(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n\nFinal\n');
+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(
+ 'Before\n\nAfter\n\nFinal\n',
+ );
+ });
+});
diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts
index 9ed9d17d1..fa7b8b0e2 100644
--- a/packages/server/src/file-watcher.ts
+++ b/packages/server/src/file-watcher.ts
@@ -3,6 +3,7 @@ import {
existsSync,
lstatSync,
readdirSync,
+ readFileSync,
realpathSync,
type Stats,
statSync,
@@ -257,11 +258,81 @@ export const writeTracker = new Map<string, Array<{ hash: string; timestamp: num
const WRITE_TRACKER_TTL_MS = 10_000;
const REMOVAL_TRACKER_TTL_MS = 30_000;
+interface OwnWriteObservation {
+ key: string;
+ hashes: Set<string>;
+ savedDuringObservation: boolean;
+}
+
+const activeOwnWrites = new Map<string, Set<OwnWriteObservation>>();
+
export function registerWrite(filePath: string, hash: string): void {
const key = declaredPathKey(filePath);
const queue = writeTracker.get(key) ?? [];
queue.push({ hash, timestamp: Date.now() });
writeTracker.set(key, queue);
+ for (const observation of activeOwnWrites.get(key) ?? []) {
+ observation.savedDuringObservation = true;
+ observation.hashes.add(hash);
+ }
+}
+
+function beginOwnWriteObservation(
+ filePath: string,
+ observations: Map<string, OwnWriteObservation>,
+): void {
+ if (observations.has(filePath)) return;
+ const key = observedPathKey(filePath);
+ const observation = {
+ key,
+ hashes: new Set((writeTracker.get(key) ?? []).map(({ hash }) => hash)),
+ savedDuringObservation: false,
+ };
+ const live = activeOwnWrites.get(key) ?? new Set<OwnWriteObservation>();
+ live.add(observation);
+ activeOwnWrites.set(key, live);
+ observations.set(filePath, observation);
+}
+
+function releaseOwnWriteObservations(observations: Map<string, OwnWriteObservation>): void {
+ for (const observation of observations.values()) {
+ const live = activeOwnWrites.get(observation.key);
+ live?.delete(observation);
+ if (live?.size === 0) activeOwnWrites.delete(observation.key);
+ }
+}
+
+function discardSupersededOwnWrite(
+ filePath: string,
+ content: string,
+ observations: ReadonlyMap<string, OwnWriteObservation>,
+): boolean {
+ const observation = observations.get(filePath);
+ if (!observation || !isSupersededOwnWrite(filePath, content, observation)) return false;
+ consumeOwnWrite(observation.key, contentHash(content));
+ return true;
+}
+
+function isSupersededOwnWrite(
+ filePath: string,
+ content: string,
+ observation: OwnWriteObservation,
+): boolean {
+ if (!observation.savedDuringObservation || !observation.hashes.has(contentHash(content))) {
+ return false;
+ }
+ if (observedPathKey(filePath) !== observation.key) return false;
+ try {
+ return readFileSync(observation.key, 'utf-8') !== content;
+ } catch (e) {
+ if (errnoCode(e) !== 'ENOENT') {
+ log.warn(
+ { path: observation.key, eventPath: filePath, err: e },
+ `Supersession recheck failed to read ${observation.key}`,
+ );
+ }
+ return false;
+ }
}
export const removalTracker = new Map<string, number>();
@@ -355,6 +426,7 @@ type WatcherDropReason =
type WatcherDecision =
| 'dispatched'
| 'self-write-skip'
+ | 'superseded-own-write-skip'
| 'self-removal-skip'
| `drop-${WatcherDropReason}`;
@@ -372,6 +444,7 @@ const watcherDecisionRing: WatcherDecisionRecord[] = [];
const watcherDropCounts = new Map<WatcherDropReason, number>();
let watcherDispatchedCount = 0;
let watcherSelfWriteSkipCount = 0;
+let watcherSupersededOwnWriteSkipCount = 0;
let watcherSelfRemovalSkipCount = 0;
let watcherDropsSinceLastSummary = 0;
@@ -394,6 +467,10 @@ function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath:
watcherSelfWriteSkipCount++;
return;
}
+ if (decision === 'superseded-own-write-skip') {
+ watcherSupersededOwnWriteSkipCount++;
+ return;
+ }
if (decision === 'self-removal-skip') {
watcherSelfRemovalSkipCount++;
return;
@@ -404,6 +481,15 @@ function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath:
_fileWatcherDropsCounter().add(1, { 'disk.drop_reason': reason });
}
+function recordSupersededOwnWriteSkip(kind: DiskEvent['kind'], filePath: string): void {
+ log.debug(
+ { kind, path: filePath, self: true, superseded: true },
+ `[file-watcher] Skipped superseded own write: ${kind}`,
+ );
+ _fileWatcherEventsCounter().add(1, { 'disk.kind': kind, self: true, 'disk.superseded': true });
+ recordWatcherDecision('superseded-own-write-skip', kind, filePath);
+}
+
export function getWatcherDecisionRingSnapshot(): WatcherDecisionRecord[] {
return watcherDecisionRing.map((record) => ({ ...record }));
}
@@ -424,6 +510,7 @@ export function logWatcherDropSummary(): void {
dropTotals,
dispatched: watcherDispatchedCount,
selfWriteSkips: watcherSelfWriteSkipCount,
+ supersededOwnWriteSkips: watcherSupersededOwnWriteSkipCount,
selfRemovalSkips: watcherSelfRemovalSkipCount,
},
`[file-watcher] drop summary: ${droppedSinceLastSummary} event(s) dropped since last summary`,
@@ -435,6 +522,7 @@ export function resetWatcherDecisionDiagnostics(): void {
watcherDropCounts.clear();
watcherDispatchedCount = 0;
watcherSelfWriteSkipCount = 0;
+ watcherSupersededOwnWriteSkipCount = 0;
watcherSelfRemovalSkipCount = 0;
watcherDropsSinceLastSummary = 0;
}
@@ -555,12 +643,24 @@ export async function classifyEvents(
contentFilter?: ContentFilter,
aliasMap?: Map<string, string>,
): Promise<MarkdownDiskEvent[]> {
- return classifyEventsInternal(rawEvents, contentDir, contentFilter, aliasMap);
+ const observations = new Map<string, OwnWriteObservation>();
+ try {
+ return await classifyEventsInternal(
+ rawEvents,
+ contentDir,
+ observations,
+ contentFilter,
+ aliasMap,
+ );
+ } finally {
+ releaseOwnWriteObservations(observations);
+ }
}
async function classifyEventsInternal(
rawEvents: RawFileEvent[],
contentDir: string,
+ observations: Map<string, OwnWriteObservation>,
contentFilter?: ContentFilter,
aliasMap?: Map<string, string>,
aliasContext?: {
@@ -619,6 +719,7 @@ async function classifyEventsInternal(
const createContents = new Map<string, string>();
const updateContents = new Map<string, string>();
for (const event of creates) {
+ beginOwnWriteObservation(event.path, observations);
try {
createContents.set(event.path, await readFile(event.path, 'utf-8'));
} catch (e) {
@@ -629,6 +730,7 @@ async function classifyEventsInternal(
}
}
for (const event of updates) {
+ beginOwnWriteObservation(event.path, observations);
try {
updateContents.set(event.path, await readFile(event.path, 'utf-8'));
} catch (e) {
@@ -639,6 +741,17 @@ async function classifyEventsInternal(
}
}
+ for (const [contents, type] of [
+ [createContents, 'create'],
+ [updateContents, 'update'],
+ ] as const) {
+ for (const [filePath, content] of contents) {
+ if (!discardSupersededOwnWrite(filePath, content, observations)) continue;
+ contents.delete(filePath);
+ recordSupersededOwnWriteSkip(type, filePath);
+ }
+ }
+
function resolveDocName(event: RawFileEvent): string {
const rawPath = event.path;
const raw = pathToDocName(rawPath, contentDir);
@@ -801,15 +914,18 @@ function declaredDiskEvent(event: DiskEvent): DiskEvent {
}
export function isSelfWrite(filePath: string, hash: string): boolean {
- const queue = writeTracker.get(filePath);
+ if (consumeOwnWrite(filePath, hash)) return true;
+ writeTracker.delete(filePath);
+ return false;
+}
+
+function consumeOwnWrite(key: string, hash: string): boolean {
+ const queue = writeTracker.get(key);
if (!queue) return false;
const idx = queue.findIndex((e) => e.hash === hash);
- if (idx < 0) {
- writeTracker.delete(filePath);
- return false;
- }
+ if (idx < 0) return false;
queue.splice(0, idx + 1);
- if (queue.length === 0) writeTracker.delete(filePath);
+ if (queue.length === 0) writeTracker.delete(key);
return true;
}
@@ -2399,127 +2515,141 @@ async function handleRawEventsInternal(
}
const canonicalAliasRecords = new Set<RawFileEvent>();
- const diskEvents =
- mdEvents.length > 0
- ? await classifyEventsInternal(
- mdEvents,
- contentDir,
- contentFilter,
- aliasMap,
- aliasPaths
- ? { fileIndex, aliasPaths, canonicalRecords: canonicalAliasRecords }
- : undefined,
- )
- : [];
-
+ const observations = new Map<string, OwnWriteObservation>();
const admittedMarkdownPaths = new Set<string>();
- for (const event of diskEvents) {
- if (admitEvent && !admitEvent(event)) continue;
- if (event.kind === 'rename') {
- admittedMarkdownPaths.add(event.oldPath);
- admittedMarkdownPaths.add(event.newPath);
- } else {
- admittedMarkdownPaths.add(event.path);
- }
- let isSelf = false;
- let indexEvent = event;
-
- const previousIndexedFields =
- event.kind === 'update'
- ? (() => {
- const previous = fileIndex.get(event.docName);
- if (previous?.kind !== 'markdown') return undefined;
- return {
- title: previous.title,
- description: previous.description,
- type: previous.type,
- };
- })()
- : undefined;
-
- if (event.kind !== 'delete' && event.kind !== 'rename') {
- const hash = contentHash(event.content);
- const checkPath = observedPathKey(event.path);
- isSelf = isSelfWrite(checkPath, hash);
- voidRemoval(event.path, declaredBeforeBatch);
- } else if (event.kind === 'rename') {
- const hash = contentHash(event.content);
- const checkPath = observedPathKey(event.newPath);
- isSelf = isSelfWrite(checkPath, hash);
- indexEvent = { ...event, newPath: checkPath };
- voidRemoval(event.oldPath, declaredBeforeBatch);
- voidRemoval(event.newPath, declaredBeforeBatch);
- } else {
- isSelf = isSelfRemoval(event.path);
- }
+ try {
+ const diskEvents =
+ mdEvents.length > 0
+ ? await classifyEventsInternal(
+ mdEvents,
+ contentDir,
+ observations,
+ contentFilter,
+ aliasMap,
+ aliasPaths
+ ? { fileIndex, aliasPaths, canonicalRecords: canonicalAliasRecords }
+ : undefined,
+ )
+ : [];
- const dispatchedEvent =
- event.kind === 'update' && previousIndexedFields !== undefined
- ? { ...event, previousIndexedFields }
- : event;
+ for (const event of diskEvents) {
+ if (
+ event.kind !== 'delete' &&
+ event.kind !== 'rename' &&
+ discardSupersededOwnWrite(event.path, event.content, observations)
+ ) {
+ recordSupersededOwnWriteSkip(event.kind, event.path);
+ continue;
+ }
+ if (admitEvent && !admitEvent(event)) continue;
+ if (event.kind === 'rename') {
+ admittedMarkdownPaths.add(event.oldPath);
+ admittedMarkdownPaths.add(event.newPath);
+ } else {
+ admittedMarkdownPaths.add(event.path);
+ }
+ let isSelf = false;
+ let indexEvent = event;
+
+ const previousIndexedFields =
+ event.kind === 'update'
+ ? (() => {
+ const previous = fileIndex.get(event.docName);
+ if (previous?.kind !== 'markdown') return undefined;
+ return {
+ title: previous.title,
+ description: previous.description,
+ type: previous.type,
+ };
+ })()
+ : undefined;
+
+ if (event.kind !== 'delete' && event.kind !== 'rename') {
+ const hash = contentHash(event.content);
+ const checkPath = observedPathKey(event.path);
+ isSelf = isSelfWrite(checkPath, hash);
+ voidRemoval(event.path, declaredBeforeBatch);
+ } else if (event.kind === 'rename') {
+ const hash = contentHash(event.content);
+ const checkPath = observedPathKey(event.newPath);
+ isSelf = isSelfWrite(checkPath, hash);
+ indexEvent = { ...event, newPath: checkPath };
+ voidRemoval(event.oldPath, declaredBeforeBatch);
+ voidRemoval(event.newPath, declaredBeforeBatch);
+ } else {
+ isSelf = isSelfRemoval(event.path);
+ }
- updateFileIndex(indexEvent, fileIndex);
+ const dispatchedEvent =
+ event.kind === 'update' && previousIndexedFields !== undefined
+ ? { ...event, previousIndexedFields }
+ : event;
- if (contentFilter && !isSelf) {
- switch (event.kind) {
- case 'create':
- contentFilter.incrementMdDir(dirname(event.docName));
- break;
- case 'delete':
- contentFilter.decrementMdDir(dirname(event.docName));
- break;
- case 'rename':
- contentFilter.decrementMdDir(dirname(event.oldDocName));
- contentFilter.incrementMdDir(dirname(event.newDocName));
- break;
- case 'update':
- case 'conflict':
- break;
- default:
- assertNeverDiskEvent(event);
+ updateFileIndex(indexEvent, fileIndex);
+
+ if (contentFilter && !isSelf) {
+ switch (event.kind) {
+ case 'create':
+ contentFilter.incrementMdDir(dirname(event.docName));
+ break;
+ case 'delete':
+ contentFilter.decrementMdDir(dirname(event.docName));
+ break;
+ case 'rename':
+ contentFilter.decrementMdDir(dirname(event.oldDocName));
+ contentFilter.incrementMdDir(dirname(event.newDocName));
+ break;
+ case 'update':
+ case 'conflict':
+ break;
+ default:
+ assertNeverDiskEvent(event);
+ }
+ }
+
+ if (isSelf) {
+ const selfKind = event.kind === 'delete' ? 'self-removal' : 'self-write';
+ log.debug(
+ {
+ kind: event.kind,
+ path: event.kind === 'rename' ? event.newPath : event.path,
+ self: true,
+ },
+ `[file-watcher] Skipped ${selfKind}: ${event.kind}`,
+ );
+ _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: true });
+ recordWatcherDecision(
+ `${selfKind}-skip`,
+ event.kind,
+ event.kind === 'rename' ? event.newPath : event.path,
+ );
+ continue;
}
- }
- if (isSelf) {
- const selfKind = event.kind === 'delete' ? 'self-removal' : 'self-write';
log.debug(
{
kind: event.kind,
path: event.kind === 'rename' ? event.newPath : event.path,
- self: true,
},
- `[file-watcher] Skipped ${selfKind}: ${event.kind}`,
+ `[file-watcher] Dispatching: ${event.kind}`,
);
- _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: true });
- recordWatcherDecision(
- `${selfKind}-skip`,
- event.kind,
- event.kind === 'rename' ? event.newPath : event.path,
+ _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: false });
+ const rawPath = event.kind === 'rename' ? event.newPath : event.path;
+ recordWatcherDecision('dispatched', event.kind, rawPath);
+ await withSpan(
+ 'file_watcher.process_event',
+ {
+ attributes: {
+ 'disk.kind': event.kind,
+ 'disk.path': normalizeFsPath(rawPath),
+ 'disk.path.role': classifyFsPath(rawPath),
+ },
+ },
+ async () => onDiskEvent(dispatchedEvent),
);
- continue;
}
-
- log.debug(
- {
- kind: event.kind,
- path: event.kind === 'rename' ? event.newPath : event.path,
- },
- `[file-watcher] Dispatching: ${event.kind}`,
- );
- _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: false });
- const rawPath = event.kind === 'rename' ? event.newPath : event.path;
- recordWatcherDecision('dispatched', event.kind, rawPath);
- await withSpan(
- 'file_watcher.process_event',
- {
- attributes: {
- 'disk.kind': event.kind,
- 'disk.path': normalizeFsPath(rawPath),
- 'disk.path.role': classifyFsPath(rawPath),
- },
- },
- async () => onDiskEvent(dispatchedEvent),
- );
+ } finally {
+ releaseOwnWriteObservations(observations);
}
for (const event of folderEvents) {