Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Fix stale own-write observations blocking document saves (#5670)
Nick Gomez9466cb31a9d5
diff --git a/.changeset/young-words-drive.md b/.changeset/young-words-drive.mdnew file mode 100644index 000000000..9d883b3df--- /dev/null+++ b/.changeset/young-words-drive.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Prevent delayed observations of earlier saves from incorrectly blocking newer document writes with a conflict.diff --git a/packages/app/tests/integration/older-own-write-echo.test.ts b/packages/app/tests/integration/older-own-write-echo.test.tsnew file mode 100644index 000000000..0ba5395cc--- /dev/null+++ b/packages/app/tests/integration/older-own-write-echo.test.ts@@ -0,0 +1,570 @@+import { randomUUID } from 'node:crypto';+import { readFileSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs';+import { basename, join } from 'node:path';+import { afterEach, describe, expect, test, vi } from 'vitest';+import * as agentSessions from '../../../server/src/agent-sessions';+import * as watcherModule from '../../../server/src/file-watcher';+import { contentHash } from '../../../server/src/version-hash';+import { createTestServer, type TestServer } from './test-harness';++const readOrder = vi.hoisted(() => ({+ path: '',+ hold: false,+ beforeRead: false,+ entered: Promise.withResolvers<void>(),+ release: Promise.withResolvers<void>(),+}));++vi.mock('node:fs/promises', async (importOriginal) => {+ const original = await importOriginal<typeof import('node:fs/promises')>();+ return {+ ...original,+ readFile: async (...args: Parameters<typeof original.readFile>) => {+ const held = readOrder.hold && String(args[0]) === readOrder.path;+ if (held) readOrder.hold = false;+ if (held && readOrder.beforeRead) {+ readOrder.entered.resolve();+ await readOrder.release.promise;+ }+ const bytes = await original.readFile(...args);+ if (held && !readOrder.beforeRead) {+ readOrder.entered.resolve();+ await readOrder.release.promise;+ }+ return bytes;+ },+ };+});++let server: TestServer | undefined;+let pending: Promise<unknown>[] = [];+let gates: PromiseWithResolvers<void>[] = [];++function gate() {+ const opened = Promise.withResolvers<void>();+ gates.push(opened);+ return opened;+}++function watcherDecisionsFor(file: string) {+ return watcherModule+ .getWatcherDecisionRingSnapshot()+ .filter((record) => basename(record.path) === basename(file))+ .map((record) => record.decision);+}++afterEach(async () => {+ readOrder.hold = false;+ readOrder.beforeRead = false;+ readOrder.release.resolve();+ for (const opened of gates) opened.resolve();+ gates = [];+ await Promise.allSettled(pending);+ pending = [];+ vi.restoreAllMocks();+ await server?.cleanup();+ server = undefined;+});++async function startOrderedServer() {+ let dispatch: ((event: watcherModule.DiskEvent) => Promise<void>) | undefined;+ const startWatcher = watcherModule.startWatcher;+ vi.spyOn(watcherModule, 'startWatcher').mockImplementation(async (...args) => {+ dispatch = args[1];+ const handle = await startWatcher(...args);+ await handle.unsubscribe();+ return handle;+ });+ server = await createTestServer();+ const currentServer = server;+ if (!dispatch) throw new Error('Server watcher callback was not registered');+ const onDiskEvent = dispatch;+ const docName = `own-observation-${randomUUID()}`;+ const file = join(server.contentDir, `${docName}.md`);+ const fileIndex = new Map<string, watcherModule.FileIndexEntry>();+ const folderIndex = new Map<string, watcherModule.FolderIndexEntry>();+ return {+ server: currentServer,+ docName,+ file,+ fileIndex,+ folderIndex,+ onDiskEvent,+ post: (markdown: string) =>+ fetch(`${currentServer.baseUrl}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ docName, agentId: 'ordered-writer', markdown, position: 'append' }),+ }),+ observe: () =>+ watcherModule.handleRawEvents(+ [{ type: 'update', path: file }],+ currentServer.contentDir,+ undefined,+ fileIndex,+ folderIndex,+ onDiskEvent,+ ),+ };+}++describe('older own-write observations', () => {+ test.each([+ {+ label: 'distinct text',+ first: 'Before\n',+ next: 'After\n',+ final: 'Final\n',+ acknowledged: 'Before\n\nAfter\n',+ expected: 'Before\n\nAfter\n\nFinal\n',+ },+ {+ label: 'repeated text',+ first: 'concurrent agent line\n',+ next: 'concurrent agent line\n',+ final: 'concurrent agent line\n',+ acknowledged: 'concurrent agent line\n\nconcurrent agent line\n',+ expected: 'concurrent agent line\n\nconcurrent agent line\n\nconcurrent agent line\n',+ },+ {+ label: 'unchanged content',+ first: 'Before\n',+ next: '',+ final: 'Final\n',+ acknowledged: 'Before\n',+ expected: 'Before\n\nFinal\n',+ },+ ])('an accepted write ignores an older observation with $label', async (contents) => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ expect((await post(contents.first)).status).toBe(200);+ expect(readFileSync(file, 'utf8')).toBe(contents.first);+ readOrder.path = file;+ readOrder.entered = Promise.withResolvers<void>();+ readOrder.release = Promise.withResolvers<void>();+ readOrder.hold = true;+ const earlierBatch = observe();+ pending.push(earlierBatch);+ await readOrder.entered.promise;+ if (contents.next) expect((await post(contents.next)).status).toBe(200);+ await observe();+ expect(readFileSync(file, 'utf8')).toBe(contents.acknowledged);+ const raised = vi.spyOn(active.instance.conflicts, 'raise');+ const prepare = agentSessions.prepareAgentMarkdownParse;+ vi.spyOn(agentSessions, 'prepareAgentMarkdownParse').mockImplementation((...args) => {+ if (args[0].name === docName) readOrder.release.resolve();+ return prepare(...args);+ });+ const writing = post(contents.final);+ pending.push(writing);+ const response = await writing;+ await earlierBatch;+ expect.soft(response.status, JSON.stringify(await response.json())).toBe(200);+ expect.soft(raised.mock.calls.map(([input]) => input)).toEqual([]);+ expect.soft(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ expect.soft(readFileSync(file, 'utf8')).toBe(contents.expected);+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ contents.expected,+ );+ });++ test('a stale external write of bytes the server never saved still refuses a write', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ const external = 'External seed\n';+ writeFileSync(file, external);+ const registrations = vi.spyOn(watcherModule, 'registerWrite');+ expect((await post('Acknowledged\n')).status).toBe(200);+ await observe();+ const acknowledged = 'External seed\n\nAcknowledged\n';+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(acknowledged));+ expect(registrations.mock.calls.map(([, hash]) => hash)).not.toContain(contentHash(external));+ writeFileSync(file, external);+ await observe();+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({+ kind: 'reconcile',+ reason: 'stale-external-write',+ stages: { base: acknowledged, ours: acknowledged, theirs: external },+ });+ const response = await post('Final\n');+ expect(response.status).toBe(409);+ expect(await response.json()).toMatchObject({+ type: 'urn:ok:error:doc-in-conflict',+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },+ });+ expect(readFileSync(file, 'utf8')).toBe(external);+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ acknowledged,+ );+ });++ test('an external rollback to an already observed own save still refuses a write', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ expect((await post('Before\n')).status).toBe(200);+ expect((await post('After\n')).status).toBe(200);+ await observe();+ writeFileSync(file, 'Before\n');+ await observe();+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({+ kind: 'reconcile',+ reason: 'stale-external-write',+ stages: { base: 'Before\n\nAfter\n', ours: 'Before\n\nAfter\n', theirs: 'Before\n' },+ });+ const response = await post('Final\n');+ expect(response.status).toBe(409);+ expect(await response.json()).toMatchObject({+ type: 'urn:ok:error:doc-in-conflict',+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },+ });+ expect(readFileSync(file, 'utf8')).toBe('Before\n');+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ 'Before\n\nAfter\n',+ );+ });++ test('a newer external edit still reconciles before the next write', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ expect((await post('Before\n')).status).toBe(200);+ await observe();+ const raised = vi.spyOn(active.instance.conflicts, 'raise');+ writeFileSync(file, 'Edited outside\n');+ await observe();+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ 'Edited outside\n',+ );+ const response = await post('Final\n');+ expect(response.status).toBe(200);+ expect(raised.mock.calls.map(([input]) => input)).toEqual([]);+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ expect(readFileSync(file, 'utf8')).toBe('Edited outside\n\nFinal\n');+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ 'Edited outside\n\nFinal\n',+ );+ });++ test('an in-flight read still detects an external rollback after the own save was consumed', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ expect((await post('Before\n')).status).toBe(200);+ readOrder.path = file;+ readOrder.entered = Promise.withResolvers<void>();+ readOrder.release = Promise.withResolvers<void>();+ readOrder.beforeRead = true;+ readOrder.hold = true;+ const earlierBatch = observe();+ pending.push(earlierBatch);+ await readOrder.entered.promise;+ expect((await post('After\n')).status).toBe(200);+ await observe();+ writeFileSync(file, 'Before\n');+ readOrder.release.resolve();+ await earlierBatch;+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({+ kind: 'reconcile',+ reason: 'stale-external-write',+ stages: { base: 'Before\n\nAfter\n', ours: 'Before\n\nAfter\n', theirs: 'Before\n' },+ });+ const response = await post('Final\n');+ expect(response.status).toBe(409);+ expect(await response.json()).toMatchObject({+ type: 'urn:ok:error:doc-in-conflict',+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },+ });+ expect(readFileSync(file, 'utf8')).toBe('Before\n');+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ 'Before\n\nAfter\n',+ );+ });++ test('an outside edit observed before an own save retains its conflict outcome', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ const registrations = vi.spyOn(watcherModule, 'registerWrite');+ expect((await post('Before\n')).status).toBe(200);+ await observe();+ const external = 'Outside edit\n';+ const acknowledged = 'OK save\n';+ writeFileSync(file, external);+ readOrder.path = file;+ readOrder.entered = Promise.withResolvers<void>();+ readOrder.release = Promise.withResolvers<void>();+ readOrder.hold = true;+ const earlierBatch = observe();+ pending.push(earlierBatch);+ await readOrder.entered.promise;+ const saved = await fetch(`${active.baseUrl}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ docName,+ agentId: 'ordered-writer',+ markdown: acknowledged,+ position: 'replace',+ }),+ });+ expect(saved.status, JSON.stringify(await saved.json())).toBe(200);+ await observe();+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(acknowledged));+ expect(registrations.mock.calls.map(([, hash]) => hash)).not.toContain(contentHash(external));+ readOrder.release.resolve();+ await earlierBatch;+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({+ kind: 'reconcile',+ reason: 'stale-external-write',+ stages: { base: acknowledged, ours: acknowledged, theirs: external },+ });+ const response = await post('Final\n');+ expect(response.status).toBe(409);+ expect(await response.json()).toMatchObject({+ type: 'urn:ok:error:doc-in-conflict',+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },+ });+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ acknowledged,+ );+ });+ test('a superseded identical-byte outside rewrite is treated as an own echo', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ const registrations = vi.spyOn(watcherModule, 'registerWrite');+ const raised = vi.spyOn(active.instance.conflicts, 'raise');+ expect((await post('Before\n')).status).toBe(200);+ const external = 'Before\n';+ const acknowledged = 'OK save\n';+ writeFileSync(file, external);+ readOrder.path = file;+ readOrder.entered = Promise.withResolvers<void>();+ readOrder.release = Promise.withResolvers<void>();+ readOrder.hold = true;+ const earlierBatch = observe();+ pending.push(earlierBatch);+ await readOrder.entered.promise;+ const saved = await fetch(`${active.baseUrl}/api/agent-write-md`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ docName,+ agentId: 'ordered-writer',+ markdown: acknowledged,+ position: 'replace',+ }),+ });+ expect(saved.status, JSON.stringify(await saved.json())).toBe(200);+ await observe();+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(acknowledged));+ expect(registrations.mock.calls.map(([, hash]) => hash)).toContain(contentHash(external));+ readOrder.release.resolve();+ await earlierBatch;+ expect(watcherDecisionsFor(file)).toEqual(['self-write-skip', 'superseded-own-write-skip']);+ expect(raised.mock.calls.map(([input]) => input)).toEqual([]);+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ expect(readFileSync(file, 'utf8')).toBe(acknowledged);+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ acknowledged,+ );+ });++ test('a retargeted alias keeps the new target when an old-target save overlaps its read', async () => {+ const { server: active, docName, file, post } = await startOrderedServer();+ expect((await post('Before\n')).status).toBe(200);+ const targetName = `target-${randomUUID()}`;+ const target = join(active.contentDir, `${targetName}.md`);+ const aliasName = `alias-${randomUUID()}`;+ const alias = join(active.contentDir, `${aliasName}.md`);+ writeFileSync(target, 'Before\n');+ symlinkSync(file, alias);+ const aliases = new Map([[aliasName, docName]]);+ readOrder.path = alias;+ readOrder.entered = Promise.withResolvers<void>();+ readOrder.release = Promise.withResolvers<void>();+ readOrder.beforeRead = true;+ readOrder.hold = true;+ const observing = watcherModule.classifyEvents(+ [{ type: 'update', path: alias }],+ active.contentDir,+ undefined,+ aliases,+ );+ pending.push(observing);+ await readOrder.entered.promise;+ expect((await post('After\n')).status).toBe(200);+ unlinkSync(alias);+ symlinkSync(target, alias);+ expect(readFileSync(alias, 'utf8')).toBe('Before\n');+ expect(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n');+ readOrder.release.resolve();+ expect(await observing).toEqual([+ { kind: 'update', path: alias, docName: targetName, content: 'Before\n' },+ ]);+ expect(aliases.get(aliasName)).toBe(targetName);+ expect(watcherModule.lastKnownHash.get(alias)).toBe(contentHash('Before\n'));+ });++ test('an external rollback after a discarded older observation still refuses a write', async () => {+ const { server: active, docName, file, post, observe } = await startOrderedServer();+ expect((await post('Before\n')).status).toBe(200);+ readOrder.path = file;+ readOrder.entered = Promise.withResolvers<void>();+ readOrder.release = Promise.withResolvers<void>();+ readOrder.hold = true;+ const earlierBatch = observe();+ pending.push(earlierBatch);+ await readOrder.entered.promise;+ expect((await post('After\n')).status).toBe(200);+ expect(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n');+ readOrder.release.resolve();+ await earlierBatch;+ expect(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ writeFileSync(file, 'Before\n');+ await observe();+ expect(active.instance.conflicts.findByDocName(docName)).toMatchObject({+ kind: 'reconcile',+ reason: 'stale-external-write',+ stages: { base: 'Before\n\nAfter\n', ours: 'Before\n\nAfter\n', theirs: 'Before\n' },+ });+ const response = await post('Final\n');+ expect(response.status).toBe(409);+ expect(await response.json()).toMatchObject({+ type: 'urn:ok:error:doc-in-conflict',+ conflict: { kind: 'reconcile', reason: 'stale-external-write' },+ });+ expect(readFileSync(file, 'utf8')).toBe('Before\n');+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ 'Before\n\nAfter\n',+ );+ });++ test('an own rename saved again during an earlier dispatch still moves its index entry', async () => {+ const { server: active } = await startOrderedServer();+ const firstOld = `rename-${randomUUID()}`;+ const firstNew = `rename-${randomUUID()}`;+ const secondOld = `rename-${randomUUID()}`;+ const secondNew = `rename-${randomUUID()}`;+ const path = (name: string) => join(active.contentDir, `${name}.md`);+ const moved = 'Moved outside\n';+ const ownMove = 'Moved by OK\n';+ const ownSave = 'Saved after the move\n';+ writeFileSync(path(firstNew), moved);+ writeFileSync(path(secondNew), ownMove);+ watcherModule.updateLastKnownHash(path(firstOld), contentHash(moved));+ watcherModule.updateLastKnownHash(path(secondOld), contentHash(ownMove));+ watcherModule.registerWrite(path(secondNew), contentHash(ownMove));+ const fileIndex = new Map<string, watcherModule.FileIndexEntry>(+ [firstOld, secondOld].map((name) => [+ name,+ {+ size: 0,+ modified: new Date(0).toISOString(),+ canonicalPath: path(name),+ inode: 0,+ aliases: [],+ kind: 'markdown',+ },+ ]),+ );+ const entered = gate();+ const release = gate();+ const dispatched: watcherModule.DiskEvent[] = [];+ const batch = watcherModule.handleRawEvents(+ [+ { type: 'delete', path: path(firstOld) },+ { type: 'create', path: path(firstNew) },+ { type: 'delete', path: path(secondOld) },+ { type: 'create', path: path(secondNew) },+ ],+ active.contentDir,+ undefined,+ fileIndex,+ new Map(),+ async (event) => {+ dispatched.push(event);+ if (event.kind === 'rename' && event.newDocName === firstNew) {+ entered.resolve();+ await release.promise;+ }+ },+ );+ pending.push(batch);+ await entered.promise;+ watcherModule.registerWrite(path(secondNew), contentHash(ownSave));+ writeFileSync(path(secondNew), ownSave);+ release.resolve();+ await batch;+ expect(dispatched).toEqual([+ {+ kind: 'rename',+ oldPath: path(firstOld),+ newPath: path(firstNew),+ oldDocName: firstOld,+ newDocName: firstNew,+ content: moved,+ },+ ]);+ const labels = new Map([+ [firstOld, 'first source'],+ [firstNew, 'first destination'],+ [secondOld, 'second source'],+ [secondNew, 'second destination'],+ ]);+ expect([...fileIndex.keys()].map((name) => labels.get(name) ?? name).sort()).toEqual([+ 'first destination',+ 'second destination',+ ]);+ expect(fileIndex.get(secondNew)).toMatchObject({ kind: 'markdown' });+ });++ test('an older observation held behind another dispatch does not refuse a later write', async () => {+ const {+ server: active,+ docName,+ file,+ fileIndex,+ folderIndex,+ onDiskEvent,+ post,+ observe,+ } = await startOrderedServer();+ const otherFile = join(active.contentDir, `outside-${randomUUID()}.md`);+ writeFileSync(otherFile, 'Outside\n');+ const raised = vi.spyOn(active.instance.conflicts, 'raise');+ expect((await post('Before\n')).status).toBe(200);+ const entered = gate();+ const release = gate();+ const earlierBatch = watcherModule.handleRawEvents(+ [+ { type: 'update', path: otherFile },+ { type: 'update', path: file },+ ],+ active.contentDir,+ undefined,+ fileIndex,+ folderIndex,+ async (event) => {+ if (event.kind === 'update' && event.path === otherFile) {+ entered.resolve();+ await release.promise;+ }+ await onDiskEvent(event);+ },+ );+ pending.push(earlierBatch);+ await entered.promise;+ expect((await post('After\n')).status).toBe(200);+ await observe();+ expect(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n');+ release.resolve();+ await earlierBatch;+ expect+ .soft(watcherDecisionsFor(file))+ .toEqual(['self-write-skip', 'superseded-own-write-skip']);+ const response = await post('Final\n');+ expect.soft(response.status, JSON.stringify(await response.json())).toBe(200);+ expect.soft(raised.mock.calls.map(([input]) => input)).toEqual([]);+ expect.soft(active.instance.conflicts.findByDocName(docName)).toBeUndefined();+ expect.soft(readFileSync(file, 'utf8')).toBe('Before\n\nAfter\n\nFinal\n');+ expect(active.instance.hocuspocus.documents.get(docName)?.getText('source').toString()).toBe(+ 'Before\n\nAfter\n\nFinal\n',+ );+ });+});diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.tsindex 9ed9d17d1..fa7b8b0e2 100644--- a/packages/server/src/file-watcher.ts+++ b/packages/server/src/file-watcher.ts@@ -3,6 +3,7 @@ import {existsSync,lstatSync,readdirSync,+ readFileSync,realpathSync,type Stats,statSync,@@ -257,11 +258,81 @@ export const writeTracker = new Map<string, Array<{ hash: string; timestamp: numconst WRITE_TRACKER_TTL_MS = 10_000;const REMOVAL_TRACKER_TTL_MS = 30_000;+interface OwnWriteObservation {+ key: string;+ hashes: Set<string>;+ savedDuringObservation: boolean;+}++const activeOwnWrites = new Map<string, Set<OwnWriteObservation>>();+export function registerWrite(filePath: string, hash: string): void {const key = declaredPathKey(filePath);const queue = writeTracker.get(key) ?? [];queue.push({ hash, timestamp: Date.now() });writeTracker.set(key, queue);+ for (const observation of activeOwnWrites.get(key) ?? []) {+ observation.savedDuringObservation = true;+ observation.hashes.add(hash);+ }+}++function beginOwnWriteObservation(+ filePath: string,+ observations: Map<string, OwnWriteObservation>,+): void {+ if (observations.has(filePath)) return;+ const key = observedPathKey(filePath);+ const observation = {+ key,+ hashes: new Set((writeTracker.get(key) ?? []).map(({ hash }) => hash)),+ savedDuringObservation: false,+ };+ const live = activeOwnWrites.get(key) ?? new Set<OwnWriteObservation>();+ live.add(observation);+ activeOwnWrites.set(key, live);+ observations.set(filePath, observation);+}++function releaseOwnWriteObservations(observations: Map<string, OwnWriteObservation>): void {+ for (const observation of observations.values()) {+ const live = activeOwnWrites.get(observation.key);+ live?.delete(observation);+ if (live?.size === 0) activeOwnWrites.delete(observation.key);+ }+}++function discardSupersededOwnWrite(+ filePath: string,+ content: string,+ observations: ReadonlyMap<string, OwnWriteObservation>,+): boolean {+ const observation = observations.get(filePath);+ if (!observation || !isSupersededOwnWrite(filePath, content, observation)) return false;+ consumeOwnWrite(observation.key, contentHash(content));+ return true;+}++function isSupersededOwnWrite(+ filePath: string,+ content: string,+ observation: OwnWriteObservation,+): boolean {+ if (!observation.savedDuringObservation || !observation.hashes.has(contentHash(content))) {+ return false;+ }+ if (observedPathKey(filePath) !== observation.key) return false;+ try {+ return readFileSync(observation.key, 'utf-8') !== content;+ } catch (e) {+ if (errnoCode(e) !== 'ENOENT') {+ log.warn(+ { path: observation.key, eventPath: filePath, err: e },+ `Supersession recheck failed to read ${observation.key}`,+ );+ }+ return false;+ }}export const removalTracker = new Map<string, number>();@@ -355,6 +426,7 @@ type WatcherDropReason =type WatcherDecision =| 'dispatched'| 'self-write-skip'+ | 'superseded-own-write-skip'| 'self-removal-skip'| `drop-${WatcherDropReason}`;@@ -372,6 +444,7 @@ const watcherDecisionRing: WatcherDecisionRecord[] = [];const watcherDropCounts = new Map<WatcherDropReason, number>();let watcherDispatchedCount = 0;let watcherSelfWriteSkipCount = 0;+let watcherSupersededOwnWriteSkipCount = 0;let watcherSelfRemovalSkipCount = 0;let watcherDropsSinceLastSummary = 0;@@ -394,6 +467,10 @@ function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath:watcherSelfWriteSkipCount++;return;}+ if (decision === 'superseded-own-write-skip') {+ watcherSupersededOwnWriteSkipCount++;+ return;+ }if (decision === 'self-removal-skip') {watcherSelfRemovalSkipCount++;return;@@ -404,6 +481,15 @@ function recordWatcherDecision(decision: WatcherDecision, kind: string, rawPath:_fileWatcherDropsCounter().add(1, { 'disk.drop_reason': reason });}+function recordSupersededOwnWriteSkip(kind: DiskEvent['kind'], filePath: string): void {+ log.debug(+ { kind, path: filePath, self: true, superseded: true },+ `[file-watcher] Skipped superseded own write: ${kind}`,+ );+ _fileWatcherEventsCounter().add(1, { 'disk.kind': kind, self: true, 'disk.superseded': true });+ recordWatcherDecision('superseded-own-write-skip', kind, filePath);+}+export function getWatcherDecisionRingSnapshot(): WatcherDecisionRecord[] {return watcherDecisionRing.map((record) => ({ ...record }));}@@ -424,6 +510,7 @@ export function logWatcherDropSummary(): void {dropTotals,dispatched: watcherDispatchedCount,selfWriteSkips: watcherSelfWriteSkipCount,+ supersededOwnWriteSkips: watcherSupersededOwnWriteSkipCount,selfRemovalSkips: watcherSelfRemovalSkipCount,},`[file-watcher] drop summary: ${droppedSinceLastSummary} event(s) dropped since last summary`,@@ -435,6 +522,7 @@ export function resetWatcherDecisionDiagnostics(): void {watcherDropCounts.clear();watcherDispatchedCount = 0;watcherSelfWriteSkipCount = 0;+ watcherSupersededOwnWriteSkipCount = 0;watcherSelfRemovalSkipCount = 0;watcherDropsSinceLastSummary = 0;}@@ -555,12 +643,24 @@ export async function classifyEvents(contentFilter?: ContentFilter,aliasMap?: Map<string, string>,): Promise<MarkdownDiskEvent[]> {- return classifyEventsInternal(rawEvents, contentDir, contentFilter, aliasMap);+ const observations = new Map<string, OwnWriteObservation>();+ try {+ return await classifyEventsInternal(+ rawEvents,+ contentDir,+ observations,+ contentFilter,+ aliasMap,+ );+ } finally {+ releaseOwnWriteObservations(observations);+ }}async function classifyEventsInternal(rawEvents: RawFileEvent[],contentDir: string,+ observations: Map<string, OwnWriteObservation>,contentFilter?: ContentFilter,aliasMap?: Map<string, string>,aliasContext?: {@@ -619,6 +719,7 @@ async function classifyEventsInternal(const createContents = new Map<string, string>();const updateContents = new Map<string, string>();for (const event of creates) {+ beginOwnWriteObservation(event.path, observations);try {createContents.set(event.path, await readFile(event.path, 'utf-8'));} catch (e) {@@ -629,6 +730,7 @@ async function classifyEventsInternal(}}for (const event of updates) {+ beginOwnWriteObservation(event.path, observations);try {updateContents.set(event.path, await readFile(event.path, 'utf-8'));} catch (e) {@@ -639,6 +741,17 @@ async function classifyEventsInternal(}}+ for (const [contents, type] of [+ [createContents, 'create'],+ [updateContents, 'update'],+ ] as const) {+ for (const [filePath, content] of contents) {+ if (!discardSupersededOwnWrite(filePath, content, observations)) continue;+ contents.delete(filePath);+ recordSupersededOwnWriteSkip(type, filePath);+ }+ }+function resolveDocName(event: RawFileEvent): string {const rawPath = event.path;const raw = pathToDocName(rawPath, contentDir);@@ -801,15 +914,18 @@ function declaredDiskEvent(event: DiskEvent): DiskEvent {}export function isSelfWrite(filePath: string, hash: string): boolean {- const queue = writeTracker.get(filePath);+ if (consumeOwnWrite(filePath, hash)) return true;+ writeTracker.delete(filePath);+ return false;+}++function consumeOwnWrite(key: string, hash: string): boolean {+ const queue = writeTracker.get(key);if (!queue) return false;const idx = queue.findIndex((e) => e.hash === hash);- if (idx < 0) {- writeTracker.delete(filePath);- return false;- }+ if (idx < 0) return false;queue.splice(0, idx + 1);- if (queue.length === 0) writeTracker.delete(filePath);+ if (queue.length === 0) writeTracker.delete(key);return true;}@@ -2399,127 +2515,141 @@ async function handleRawEventsInternal(}const canonicalAliasRecords = new Set<RawFileEvent>();- const diskEvents =- mdEvents.length > 0- ? await classifyEventsInternal(- mdEvents,- contentDir,- contentFilter,- aliasMap,- aliasPaths- ? { fileIndex, aliasPaths, canonicalRecords: canonicalAliasRecords }- : undefined,- )- : [];-+ const observations = new Map<string, OwnWriteObservation>();const admittedMarkdownPaths = new Set<string>();- for (const event of diskEvents) {- if (admitEvent && !admitEvent(event)) continue;- if (event.kind === 'rename') {- admittedMarkdownPaths.add(event.oldPath);- admittedMarkdownPaths.add(event.newPath);- } else {- admittedMarkdownPaths.add(event.path);- }- let isSelf = false;- let indexEvent = event;-- const previousIndexedFields =- event.kind === 'update'- ? (() => {- const previous = fileIndex.get(event.docName);- if (previous?.kind !== 'markdown') return undefined;- return {- title: previous.title,- description: previous.description,- type: previous.type,- };- })()- : undefined;-- if (event.kind !== 'delete' && event.kind !== 'rename') {- const hash = contentHash(event.content);- const checkPath = observedPathKey(event.path);- isSelf = isSelfWrite(checkPath, hash);- voidRemoval(event.path, declaredBeforeBatch);- } else if (event.kind === 'rename') {- const hash = contentHash(event.content);- const checkPath = observedPathKey(event.newPath);- isSelf = isSelfWrite(checkPath, hash);- indexEvent = { ...event, newPath: checkPath };- voidRemoval(event.oldPath, declaredBeforeBatch);- voidRemoval(event.newPath, declaredBeforeBatch);- } else {- isSelf = isSelfRemoval(event.path);- }+ try {+ const diskEvents =+ mdEvents.length > 0+ ? await classifyEventsInternal(+ mdEvents,+ contentDir,+ observations,+ contentFilter,+ aliasMap,+ aliasPaths+ ? { fileIndex, aliasPaths, canonicalRecords: canonicalAliasRecords }+ : undefined,+ )+ : [];- const dispatchedEvent =- event.kind === 'update' && previousIndexedFields !== undefined- ? { ...event, previousIndexedFields }- : event;+ for (const event of diskEvents) {+ if (+ event.kind !== 'delete' &&+ event.kind !== 'rename' &&+ discardSupersededOwnWrite(event.path, event.content, observations)+ ) {+ recordSupersededOwnWriteSkip(event.kind, event.path);+ continue;+ }+ if (admitEvent && !admitEvent(event)) continue;+ if (event.kind === 'rename') {+ admittedMarkdownPaths.add(event.oldPath);+ admittedMarkdownPaths.add(event.newPath);+ } else {+ admittedMarkdownPaths.add(event.path);+ }+ let isSelf = false;+ let indexEvent = event;++ const previousIndexedFields =+ event.kind === 'update'+ ? (() => {+ const previous = fileIndex.get(event.docName);+ if (previous?.kind !== 'markdown') return undefined;+ return {+ title: previous.title,+ description: previous.description,+ type: previous.type,+ };+ })()+ : undefined;++ if (event.kind !== 'delete' && event.kind !== 'rename') {+ const hash = contentHash(event.content);+ const checkPath = observedPathKey(event.path);+ isSelf = isSelfWrite(checkPath, hash);+ voidRemoval(event.path, declaredBeforeBatch);+ } else if (event.kind === 'rename') {+ const hash = contentHash(event.content);+ const checkPath = observedPathKey(event.newPath);+ isSelf = isSelfWrite(checkPath, hash);+ indexEvent = { ...event, newPath: checkPath };+ voidRemoval(event.oldPath, declaredBeforeBatch);+ voidRemoval(event.newPath, declaredBeforeBatch);+ } else {+ isSelf = isSelfRemoval(event.path);+ }- updateFileIndex(indexEvent, fileIndex);+ const dispatchedEvent =+ event.kind === 'update' && previousIndexedFields !== undefined+ ? { ...event, previousIndexedFields }+ : event;- if (contentFilter && !isSelf) {- switch (event.kind) {- case 'create':- contentFilter.incrementMdDir(dirname(event.docName));- break;- case 'delete':- contentFilter.decrementMdDir(dirname(event.docName));- break;- case 'rename':- contentFilter.decrementMdDir(dirname(event.oldDocName));- contentFilter.incrementMdDir(dirname(event.newDocName));- break;- case 'update':- case 'conflict':- break;- default:- assertNeverDiskEvent(event);+ updateFileIndex(indexEvent, fileIndex);++ if (contentFilter && !isSelf) {+ switch (event.kind) {+ case 'create':+ contentFilter.incrementMdDir(dirname(event.docName));+ break;+ case 'delete':+ contentFilter.decrementMdDir(dirname(event.docName));+ break;+ case 'rename':+ contentFilter.decrementMdDir(dirname(event.oldDocName));+ contentFilter.incrementMdDir(dirname(event.newDocName));+ break;+ case 'update':+ case 'conflict':+ break;+ default:+ assertNeverDiskEvent(event);+ }+ }++ if (isSelf) {+ const selfKind = event.kind === 'delete' ? 'self-removal' : 'self-write';+ log.debug(+ {+ kind: event.kind,+ path: event.kind === 'rename' ? event.newPath : event.path,+ self: true,+ },+ `[file-watcher] Skipped ${selfKind}: ${event.kind}`,+ );+ _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: true });+ recordWatcherDecision(+ `${selfKind}-skip`,+ event.kind,+ event.kind === 'rename' ? event.newPath : event.path,+ );+ continue;}- }- if (isSelf) {- const selfKind = event.kind === 'delete' ? 'self-removal' : 'self-write';log.debug({kind: event.kind,path: event.kind === 'rename' ? event.newPath : event.path,- self: true,},- `[file-watcher] Skipped ${selfKind}: ${event.kind}`,+ `[file-watcher] Dispatching: ${event.kind}`,);- _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: true });- recordWatcherDecision(- `${selfKind}-skip`,- event.kind,- event.kind === 'rename' ? event.newPath : event.path,+ _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: false });+ const rawPath = event.kind === 'rename' ? event.newPath : event.path;+ recordWatcherDecision('dispatched', event.kind, rawPath);+ await withSpan(+ 'file_watcher.process_event',+ {+ attributes: {+ 'disk.kind': event.kind,+ 'disk.path': normalizeFsPath(rawPath),+ 'disk.path.role': classifyFsPath(rawPath),+ },+ },+ async () => onDiskEvent(dispatchedEvent),);- continue;}-- log.debug(- {- kind: event.kind,- path: event.kind === 'rename' ? event.newPath : event.path,- },- `[file-watcher] Dispatching: ${event.kind}`,- );- _fileWatcherEventsCounter().add(1, { 'disk.kind': event.kind, self: false });- const rawPath = event.kind === 'rename' ? event.newPath : event.path;- recordWatcherDecision('dispatched', event.kind, rawPath);- await withSpan(- 'file_watcher.process_event',- {- attributes: {- 'disk.kind': event.kind,- 'disk.path': normalizeFsPath(rawPath),- 'disk.path.role': classifyFsPath(rawPath),- },- },- async () => onDiskEvent(dispatchedEvent),- );+ } finally {+ releaseOwnWriteObservations(observations);}for (const event of folderEvents) {