Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): re-land disk-reconcile three-way merge without false 409s (#1826)

Nick Gomez9a790f047c33
diff --git a/.changeset/own-flush-reconcile-discrimination.md b/.changeset/own-flush-reconcile-discrimination.md
new file mode 100644
index 000000000..d2694efad
--- /dev/null
+++ b/.changeset/own-flush-reconcile-discrimination.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge-server": patch
+---
+
+Re-land the three-way disk-reconcile guard for agent writes with own-write discrimination. The guard that merges divergent on-disk edits before an agent write now recognizes the server's own in-flight persistence flush (disk content matching the flush snapshot mid-commit) and skips reconciliation instead of misclassifying it as foreign divergence. This fixes spurious 409 doc-in-conflict refusals — including a permanent wedge where every subsequent agent write was refused — that could occur when an agent write raced the server's own debounced disk flush on large documents. Genuinely foreign disk edits still reconcile (or refuse) exactly as before.
diff --git a/.changeset/reconcile-before-agent-write-merge.md b/.changeset/reconcile-before-agent-write-merge.md
new file mode 100644
index 000000000..00493b9cd
--- /dev/null
+++ b/.changeset/reconcile-before-agent-write-merge.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Agent writes no longer drop concurrent un-flushed collaborative edits when an out-of-band disk edit races them. The pre-write reconcile used to ingest the divergent disk content wholesale, reverting any CRDT edit still inside the persistence debounce window; it now routes through the same three-way merge the file watcher uses, so both the disk edit and the concurrent edit survive (the success response's `disk-edit-reconciled` warning gains a `mergeOutcome` field discriminating `clean` from `merged`). Overlapping-block conflicts and conflict-marker disk content now refuse the agent write with the standard 409 doc-in-conflict envelope instead of silently picking a winner. The file watcher's own merge now advances the reconciled base to the on-disk bytes rather than the memory-only merged content, so a subsequent agent write no longer misreads the merge as a fresh divergence and reverts it (arrival-order independence), and the merged content actually reaches disk on the next store.
diff --git a/packages/app/tests/integration/disk-authority-reconcile.test.ts b/packages/app/tests/integration/disk-authority-reconcile.test.ts
index 786b9942e..9042a6050 100644
--- a/packages/app/tests/integration/disk-authority-reconcile.test.ts
+++ b/packages/app/tests/integration/disk-authority-reconcile.test.ts
@@ -110,7 +110,7 @@ describe('PRD-6832 β L1: agent write reconciles a newer out-of-band disk edit',
expect(after).toContain('native-body-line'); // out-of-band body edit preserved
});
- test('concurrent un-flushed CRDT edit: the L1 wholesale ingest drops it (current behavior, recorded honestly)', async () => {
+ test('concurrent un-flushed CRDT edit survives: L1 three-way merges disk + CRDT, agent edit lands on top (agent write first)', async () => {
server = await createTestServer({ debounce: 300_000, maxDebounce: 600_000 });
const { port, contentDir } = server;
const docName = `reconcile-concurrent-${randomUUID()}`;
@@ -141,13 +141,170 @@ describe('PRD-6832 β L1: agent write reconciles a newer out-of-band disk edit',
body: JSON.stringify({ docName, markdown: 'agent-line\n', position: 'append' }),
});
expect(res.status).toBe(200);
- const body = (await res.json()) as { warning?: { kind?: string } };
+ const body = (await res.json()) as {
+ warning?: { kind?: string; mergeOutcome?: string };
+ };
expect(body.warning?.kind).toBe('disk-edit-reconciled');
+ expect(body.warning?.mergeOutcome).toBe('merged');
await pollUntil(() => serverYtext().includes('agent-line'));
expect(serverYtext()).toContain('disk-oob-line');
+ expect(serverYtext()).toContain('crdt-unflushed-line');
+
+ await pollUntil(() => {
+ const d = readTestDoc(contentDir, docName);
+ return (
+ d.includes('agent-line') &&
+ d.includes('crdt-unflushed-line') &&
+ d.includes('disk-oob-line')
+ );
+ });
+ } finally {
+ for (const c of clients) {
+ await c.cleanup();
+ }
+ }
+ });
+
+ test('arrival-order independence: file-watcher merges first, agent write does not re-ingest and revert it', async () => {
+ server = await createTestServer({ debounce: 300_000, maxDebounce: 600_000 });
+ const { port, contentDir } = server;
+ const docName = `reconcile-watcher-first-${randomUUID()}`;
+ const filePath = join(contentDir, `${docName}.md`);
+
+ writeFileSync(filePath, '# Doc\n\nseed-body\n', 'utf-8');
+
+ let clients: TestClient[] = [];
+ try {
+ clients = await createTestClients(port, { count: 1, docName });
+ const client = clients[0];
+ if (!client) throw new Error('client setup failed');
+ await pollUntil(() => client.ytext.toString().includes('seed-body'));
+
+ client.doc.transact(() => {
+ client.ytext.insert(client.ytext.length, '\ncrdt-unflushed-line\n');
+ });
+ const serverYtext = () =>
+ server?.instance.hocuspocus.documents.get(docName)?.getText('source').toString() ?? '';
+ await pollUntil(() => serverYtext().includes('crdt-unflushed-line'));
+
+ writeFileSync(filePath, '# Doc\n\nseed-body\n\ndisk-oob-line\n', 'utf-8');
+ await pollUntil(() => serverYtext().includes('disk-oob-line'), 10_000);
+ expect(serverYtext()).toContain('crdt-unflushed-line');
+
+ const res = await fetch(`http://127.0.0.1:${port}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docName, markdown: 'agent-line\n', position: 'append' }),
+ });
+ expect(res.status).toBe(200);
+ const body = (await res.json()) as { warning?: { kind?: string } };
+ expect(body.warning).toBeUndefined();
+
+ await pollUntil(() => serverYtext().includes('agent-line'));
+ expect(serverYtext()).toContain('disk-oob-line');
+ expect(serverYtext()).toContain('crdt-unflushed-line');
+
+ await pollUntil(() => {
+ const d = readTestDoc(contentDir, docName);
+ return (
+ d.includes('agent-line') &&
+ d.includes('crdt-unflushed-line') &&
+ d.includes('disk-oob-line')
+ );
+ });
+ } finally {
+ for (const c of clients) {
+ await c.cleanup();
+ }
+ }
+ });
+
+ test('overlapping-block conflict: agent write is refused 409 doc-in-conflict, neither side is silently dropped', async () => {
+ server = await createTestServer({ debounce: 300_000, maxDebounce: 600_000 });
+ const { port, contentDir } = server;
+ const docName = `reconcile-conflict-${randomUUID()}`;
+ const filePath = join(contentDir, `${docName}.md`);
+
+ writeFileSync(filePath, '# Doc\n\nshared-line\n', 'utf-8');
+
+ let clients: TestClient[] = [];
+ try {
+ clients = await createTestClients(port, { count: 1, docName });
+ const client = clients[0];
+ if (!client) throw new Error('client setup failed');
+ await pollUntil(() => client.ytext.toString().includes('shared-line'));
+
+ client.doc.transact(() => {
+ const text = client.ytext.toString();
+ const at = text.indexOf('shared-line') + 'shared-line'.length;
+ client.ytext.insert(at, ' crdt-version');
+ });
+ const serverYtext = () =>
+ server?.instance.hocuspocus.documents.get(docName)?.getText('source').toString() ?? '';
+ await pollUntil(() => serverYtext().includes('crdt-version'));
+
+ writeFileSync(filePath, '# Doc\n\nshared-line disk-version\n', 'utf-8');
+
+ const res = await fetch(`http://127.0.0.1:${port}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docName, markdown: 'agent-line\n', position: 'append' }),
+ });
+ expect(res.status).toBe(409);
+ const body = (await res.json()) as { type?: string };
+ expect(body.type).toBe('urn:ok:error:doc-in-conflict');
+
+ expect(serverYtext()).toContain('crdt-version');
+ expect(serverYtext()).not.toContain('agent-line');
+ expect(readTestDoc(contentDir, docName)).toContain('disk-version');
+ } finally {
+ for (const c of clients) {
+ await c.cleanup();
+ }
+ }
+ });
+
+ test('conflict markers on disk: L1 refuses to ingest them and the agent write is refused 409', async () => {
+ server = await createTestServer({ debounce: 300_000, maxDebounce: 600_000 });
+ const { port, contentDir } = server;
+ const docName = `reconcile-markers-${randomUUID()}`;
+ const filePath = join(contentDir, `${docName}.md`);
+
+ writeFileSync(filePath, '# Doc\n\nseed-body\n', 'utf-8');
+
+ let clients: TestClient[] = [];
+ try {
+ clients = await createTestClients(port, { count: 1, docName });
+ const client = clients[0];
+ if (!client) throw new Error('client setup failed');
+ await pollUntil(() => client.ytext.toString().includes('seed-body'));
+
+ client.doc.transact(() => {
+ client.ytext.insert(client.ytext.length, '\ncrdt-unflushed-line\n');
+ });
+ const serverYtext = () =>
+ server?.instance.hocuspocus.documents.get(docName)?.getText('source').toString() ?? '';
+ await pollUntil(() => serverYtext().includes('crdt-unflushed-line'));
+
+ writeFileSync(
+ filePath,
+ '# Doc\n\n<<<<<<< HEAD\nseed-body\n=======\nother-side\n>>>>>>> theirs\n',
+ 'utf-8',
+ );
+
+ const res = await fetch(`http://127.0.0.1:${port}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ docName, markdown: 'agent-line\n', position: 'append' }),
+ });
+ expect(res.status).toBe(409);
+ const body = (await res.json()) as { type?: string };
+ expect(body.type).toBe('urn:ok:error:doc-in-conflict');
- expect(serverYtext()).not.toContain('crdt-unflushed-line');
+ expect(serverYtext()).not.toContain('<<<<<<<');
+ expect(serverYtext()).toContain('crdt-unflushed-line');
+ expect(serverYtext()).not.toContain('agent-line');
} finally {
for (const c of clients) {
await c.cleanup();
diff --git a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
index 773be947a..2da2c53c3 100644
--- a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
+++ b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
@@ -71,6 +71,10 @@ import {
type TestClient,
type TestServer,
} from '../integration/test-harness';
+import {
+ buildOracleEExpectations,
+ markerPrefixOf as prefixOf,
+} from './oracle-e-expectations.test-helper';
function createPRNG(seed: number) {
let state = seed | 0 || 1;
@@ -514,10 +518,6 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
);
const livePrefixes = new Set<string>();
- const prefixOf = (marker: string): string => {
- const dashIdx = marker.indexOf('-');
- return dashIdx === -1 ? marker : marker.slice(0, dashIdx + 1);
- };
let expectedBody = 'seed paragraph'; // post-seed, pre-op initial state
const updateExpectedBody = (op: Op): void => {
@@ -566,9 +566,13 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
try {
const ops = generateOps(rng, clientCount, opCount);
- for (const op of ops) {
+ const notAppliedOpIndices = new Set<number>();
+ for (const [opIdx, op] of ops.entries()) {
const applied = await applyOp(op, clients, server, docName);
- if (!applied) continue;
+ if (!applied) {
+ notAppliedOpIndices.add(opIdx);
+ continue;
+ }
if (
op.kind === 'wysiwyg-type' ||
@@ -640,30 +644,7 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
);
}
- const preMarkerLines = new Map<string, string>(); // prefix → pre-patch line
- const patches: Array<{ find: string; replace: string }> = [];
- for (const op of ops) {
- switch (op.kind) {
- case 'wysiwyg-type':
- case 'source-type':
- preMarkerLines.set(prefixOf(op.marker), op.marker);
- break;
- case 'agent-write':
- if (op.position === 'replace') preMarkerLines.clear();
- preMarkerLines.set(prefixOf(op.marker), op.marker);
- break;
- case 'agent-patch':
- patches.push({ find: op.find, replace: op.replace });
- break;
- case 'external-change':
- preMarkerLines.clear();
- preMarkerLines.set(prefixOf(op.marker), op.marker);
- break;
- case 'agent-undo':
- preMarkerLines.clear();
- break;
- }
- }
+ const { preMarkerLines, patches } = buildOracleEExpectations(ops, notAppliedOpIndices);
if (preMarkerLines.size > 0) {
const acceptableForPrefix = new Map<string, Set<string>>();
@@ -719,8 +700,9 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
if (missingContent.length > 0) {
throw new Error(
`Oracle (e) content-set violation — ${missingContent.length} marker prefixes ` +
- `with no acceptable line form (zero tolerance: tail corruption that can't be ` +
- `explained by any applied agent-patch).\n` +
+ `with no acceptable line form. Either content diverged in a way no applied ` +
+ `agent-patch explains, or the expectation walk demanded an op the run never ` +
+ `applied (check refusal counts before assuming corruption).\n` +
missingContent
.slice(0, 5)
.map(
diff --git a/packages/app/tests/stress/oracle-e-expectations.test-helper.ts b/packages/app/tests/stress/oracle-e-expectations.test-helper.ts
new file mode 100644
index 000000000..80f692794
--- /dev/null
+++ b/packages/app/tests/stress/oracle-e-expectations.test-helper.ts
@@ -0,0 +1,55 @@
+export type OracleEOp =
+ | { kind: 'wysiwyg-type'; marker: string }
+ | { kind: 'source-type'; marker: string }
+ | { kind: 'agent-write'; position: 'append' | 'prepend' | 'replace'; marker: string }
+ | { kind: 'agent-patch'; find: string; replace: string; marker: string }
+ | { kind: 'agent-undo' }
+ | { kind: 'external-change'; marker: string }
+ | { kind: 'chunked-source-paste'; marker: string }
+ | { kind: 'sync-pause' }
+ | { kind: 'sync-resume' }
+ | { kind: 'wait' };
+
+export function markerPrefixOf(marker: string): string {
+ const dashIdx = marker.indexOf('-');
+ return dashIdx === -1 ? marker : marker.slice(0, dashIdx + 1);
+}
+
+export interface OracleEExpectations {
+ preMarkerLines: Map<string, string>;
+ patches: Array<{ find: string; replace: string }>;
+}
+
+export function buildOracleEExpectations(
+ ops: readonly OracleEOp[],
+ notAppliedOpIndices: ReadonlySet<number>,
+): OracleEExpectations {
+ const preMarkerLines = new Map<string, string>(); // prefix → pre-patch line
+ const patches: Array<{ find: string; replace: string }> = [];
+ for (let i = 0; i < ops.length; i++) {
+ if (notAppliedOpIndices.has(i)) continue;
+ const op = ops[i];
+ if (op === undefined) continue;
+ switch (op.kind) {
+ case 'wysiwyg-type':
+ case 'source-type':
+ preMarkerLines.set(markerPrefixOf(op.marker), op.marker);
+ break;
+ case 'agent-write':
+ if (op.position === 'replace') preMarkerLines.clear();
+ preMarkerLines.set(markerPrefixOf(op.marker), op.marker);
+ break;
+ case 'agent-patch':
+ patches.push({ find: op.find, replace: op.replace });
+ break;
+ case 'external-change':
+ preMarkerLines.clear();
+ preMarkerLines.set(markerPrefixOf(op.marker), op.marker);
+ break;
+ case 'agent-undo':
+ preMarkerLines.clear();
+ break;
+ }
+ }
+ return { preMarkerLines, patches };
+}
diff --git a/packages/app/tests/stress/oracle-e-expectations.test.ts b/packages/app/tests/stress/oracle-e-expectations.test.ts
new file mode 100644
index 000000000..edbd19b0e
--- /dev/null
+++ b/packages/app/tests/stress/oracle-e-expectations.test.ts
@@ -0,0 +1,60 @@
+import { describe, expect, test } from 'bun:test';
+import {
+ buildOracleEExpectations,
+ markerPrefixOf,
+ type OracleEOp,
+} from './oracle-e-expectations.test-helper';
+
+describe('oracle (e) expectation walk — refused-op accounting', () => {
+ test('a refused (never-applied) agent write does not contribute its marker to the expectations', () => {
+ const ops: OracleEOp[] = [
+ { kind: 'agent-write', position: 'replace', marker: 'M0-anchor words' },
+ { kind: 'source-type', marker: 'M1-typed line' },
+ { kind: 'agent-write', position: 'append', marker: 'M4-foxtrot bravo' },
+ ];
+ const notAppliedOpIndices = new Set([2]); // the M4 write was refused
+
+ const { preMarkerLines } = buildOracleEExpectations(ops, notAppliedOpIndices);
+
+ expect(preMarkerLines.get(markerPrefixOf('M0-anchor words'))).toBe('M0-anchor words');
+ expect(preMarkerLines.get(markerPrefixOf('M1-typed line'))).toBe('M1-typed line');
+ expect(preMarkerLines.has(markerPrefixOf('M4-foxtrot bravo'))).toBe(false);
+ });
+
+ test('a chunked-source-paste op is intentionally excluded from the walk', () => {
+ const ops: OracleEOp[] = [
+ { kind: 'agent-write', position: 'replace', marker: 'M0-anchor words' },
+ { kind: 'chunked-source-paste', marker: 'M2-pasted chunk' },
+ ];
+
+ const { preMarkerLines } = buildOracleEExpectations(ops, new Set());
+
+ expect(preMarkerLines.get(markerPrefixOf('M0-anchor words'))).toBe('M0-anchor words');
+ expect(preMarkerLines.has(markerPrefixOf('M2-pasted chunk'))).toBe(false);
+ });
+
+ test('a refused replace-position agent write does not clear previously applied markers', () => {
+ const ops: OracleEOp[] = [
+ { kind: 'agent-write', position: 'replace', marker: 'M0-anchor words' },
+ { kind: 'agent-write', position: 'replace', marker: 'M9-delta' },
+ ];
+ const notAppliedOpIndices = new Set([1]); // the M9 replace was refused
+
+ const { preMarkerLines } = buildOracleEExpectations(ops, notAppliedOpIndices);
+
+ expect(preMarkerLines.get(markerPrefixOf('M0-anchor words'))).toBe('M0-anchor words');
+ expect(preMarkerLines.has(markerPrefixOf('M9-delta'))).toBe(false);
+ });
+
+ test('a refused agent-undo does not clear previously applied markers', () => {
+ const ops: OracleEOp[] = [
+ { kind: 'agent-write', position: 'replace', marker: 'M0-anchor words' },
+ { kind: 'agent-undo' },
+ ];
+ const notAppliedOpIndices = new Set([1]); // the undo was refused
+
+ const { preMarkerLines } = buildOracleEExpectations(ops, notAppliedOpIndices);
+
+ expect(preMarkerLines.get(markerPrefixOf('M0-anchor words'))).toBe('M0-anchor words');
+ });
+});
diff --git a/packages/core/src/schemas/api/agent-write.ts b/packages/core/src/schemas/api/agent-write.ts
index 7a96cb9dd..db32c5080 100644
--- a/packages/core/src/schemas/api/agent-write.ts
+++ b/packages/core/src/schemas/api/agent-write.ts
@@ -95,6 +95,7 @@ export const DiskEditReconciledWarningSchema = z
intendedBytes: z.number().int().nonnegative(),
actualBytes: z.number().int().nonnegative(),
byteDelta: z.number().int(),
+ mergeOutcome: z.string().optional(),
hint: z.string().optional(),
})
.loose() satisfies StandardSchemaV1;
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index 9b0621cba..8afe4072f 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -1934,7 +1934,11 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
intendedBytes: reconcile.baseBytes,
actualBytes: reconcile.diskBytes,
byteDelta: reconcile.diskBytes - reconcile.baseBytes,
- hint: 'An out-of-band edit was reconciled into this document before your edit was applied on top; the document now reflects that edit plus yours. Re-read it (e.g. `exec("cat <path>")`) to see the combined result before continuing.',
+ ...(reconcile.mergeOutcome ? { mergeOutcome: reconcile.mergeOutcome } : {}),
+ hint:
+ reconcile.mergeOutcome === 'merged'
+ ? 'An out-of-band edit was three-way merged into this document before your edit was applied on top; the merge may have interleaved content blocks. Re-read it (e.g. `exec("cat <path>")`) and review the combined result carefully before continuing.'
+ : 'An out-of-band edit was reconciled into this document before your edit was applied on top; the document now reflects that edit plus yours. Re-read it (e.g. `exec("cat <path>")`) to see the combined result before continuing.',
};
}
diff --git a/packages/server/src/external-change.ts b/packages/server/src/external-change.ts
index 6fedbc0ad..902a6643d 100644
--- a/packages/server/src/external-change.ts
+++ b/packages/server/src/external-change.ts
@@ -4,6 +4,7 @@ import {
BridgeInvariantViolationError,
BridgeMergeContentLossError,
normalizeBridge,
+ prependFrontmatter,
stripFrontmatter,
} from '@inkeep/open-knowledge-core';
import { formatReconcileSubject } from '@inkeep/open-knowledge-core/shadow-repo-layout';
@@ -14,13 +15,19 @@ import { isDocInConflict } from './conflict-errors.ts';
import { recordContributor } from './contributor-tracker.ts';
import { recordFrontmatterEditSurface } from './frontmatter-telemetry.ts';
import { getLogger } from './logger.ts';
-import { incrementExternalChangeHandlerErrors } from './metrics.ts';
+import {
+ incrementExternalChangeHandlerErrors,
+ incrementReconcileInFlightFallthroughs,
+ incrementReconcileOwnFlushSkips,
+} from './metrics.ts';
import {
getReconciledBase,
isWithinContentDir,
+ peekInFlightFlush,
safeContentPath,
setReconciledBase,
} from './persistence.ts';
+import { reconcile } from './reconciliation.ts';
import type { PairedWriteOrigin } from './server-observers.ts';
import { FILE_SYSTEM_WRITER } from './shadow-repo.ts';
@@ -106,6 +113,7 @@ export interface ReconcileBeforeWriteResult {
reconciled: boolean;
baseBytes: number;
diskBytes: number;
+ mergeOutcome?: 'clean' | 'merged';
}
const NOT_RECONCILED: ReconcileBeforeWriteResult = {
@@ -114,6 +122,14 @@ const NOT_RECONCILED: ReconcileBeforeWriteResult = {
diskBytes: 0,
};
+export function serializeYDocSource(document: {
+ getText(name: string): { toString(): string };
+}): string {
+ const ytextSnapshot = document.getText('source').toString();
+ const { frontmatter, body } = stripFrontmatter(ytextSnapshot);
+ return prependFrontmatter(frontmatter, body);
+}
+
export function reconcileDiskBeforeAgentWrite(
hocuspocus: Hocuspocus,
docName: string,
@@ -148,16 +164,74 @@ export function reconcileDiskBeforeAgentWrite(
let diskContent: string;
try {
diskContent = readFileSync(canonical, 'utf-8');
- } catch {
+ } catch (err) {
+ const code = (err as NodeJS.ErrnoException | null)?.code;
+ if (code !== 'ENOENT') {
+ getLogger('reconcile').warn(
+ { docName, canonical, code },
+ `[reconcile] disk read failed for ${docName} (${code ?? 'unknown'}); skipping reconcile`,
+ );
+ }
return NOT_RECONCILED;
}
if (normalizeBridge(diskContent) === normalizeBridge(base)) return NOT_RECONCILED;
- applyExternalChange(hocuspocus, docName, diskContent, resolveEmbed);
- return {
- reconciled: true,
- baseBytes: Buffer.byteLength(base, 'utf8'),
- diskBytes: Buffer.byteLength(diskContent, 'utf8'),
- };
+ const inFlightFlush = peekInFlightFlush(docName);
+ if (inFlightFlush !== undefined) {
+ if (normalizeBridge(diskContent) === inFlightFlush) {
+ incrementReconcileOwnFlushSkips();
+ getLogger('reconcile').debug(
+ { docName, diskBytes: diskContent.length },
+ `[reconcile] disk matches own in-flight flush for ${docName}; skipping reconcile`,
+ );
+ return NOT_RECONCILED;
+ }
+ incrementReconcileInFlightFallthroughs();
+ getLogger('reconcile').warn(
+ { docName, diskBytes: diskContent.length },
+ `[reconcile] in-flight flush present but disk differs from snapshot for ${docName}; falling through to merge`,
+ );
+ }
+
+ if (!document) return NOT_RECONCILED;
+
+ const ours = serializeYDocSource(document);
+
+ const outcome = reconcile({ docName, base, ours, theirs: diskContent });
+ getLogger('reconcile').info(
+ { docName, result: outcome.kind, baseBytes: base.length, diskBytes: diskContent.length },
+ `[reconcile] before-agent-write ${docName} result=${outcome.kind}`,
+ );
+
+ switch (outcome.kind) {
+ case 'noop':
+ return NOT_RECONCILED;
+
+ case 'conflicts':
+ case 'refused': {
+ const lifecycleMap = document.getMap('lifecycle');
+ lifecycleMap.set('status', 'conflict');
+ lifecycleMap.set(
+ 'reason',
+ outcome.kind === 'refused' ? outcome.reason : 'reconcile-conflicts',
+ );
+ return NOT_RECONCILED;
+ }
+
+ case 'clean':
+ case 'merged': {
+ const ingest = outcome.kind === 'clean' ? diskContent : outcome.newContent;
+ applyExternalChange(hocuspocus, docName, ingest, resolveEmbed);
+ if (outcome.kind === 'merged') {
+ setReconciledBase(docName, diskContent);
+ }
+ return {
+ reconciled: true,
+ baseBytes: Buffer.byteLength(base, 'utf8'),
+ diskBytes: Buffer.byteLength(diskContent, 'utf8'),
+ mergeOutcome: outcome.kind,
+ };
+ }
+ }
}
diff --git a/packages/server/src/metrics.ts b/packages/server/src/metrics.ts
index 200916587..9fcf7427e 100644
--- a/packages/server/src/metrics.ts
+++ b/packages/server/src/metrics.ts
@@ -241,6 +241,16 @@ export interface ReconciliationMetrics {
* indicates `parseWithFallback`'s paragraph fallback isn't catching some
* malformed-bytes class. */
externalChangeHandlerErrors: number;
+ /** Disk-reconcile guard outcomes inside the server's own flush-commit
+ * window. `reconcileOwnFlushSkips` counts agent writes where disk matched
+ * the in-flight flush snapshot (guard skipped; the flush continuation owns
+ * the base advance). `reconcileInFlightFallthroughs` counts the rare case
+ * where a flush was in flight but disk held OTHER bytes — a genuinely
+ * foreign edit landing inside the window, routed to the three-way merge.
+ * Both bounded; growth in fallthroughs under zero external editors would
+ * indicate the own-write discrimination is mis-matching. */
+ reconcileOwnFlushSkips: number;
+ reconcileInFlightFallthroughs: number;
/** Y.Text-is-truth contract — count of persistence pre-write
* sanity-check cycles where `mdManager.serialize` itself threw, blocking
* the bridge invariant assertion from running. Distinct from
@@ -379,6 +389,8 @@ const counters: ReconciliationMetrics = {
bridgeSplitBrainRederivesSuppressed: 0,
persistenceReconciliationFailures: 0,
externalChangeHandlerErrors: 0,
+ reconcileOwnFlushSkips: 0,
+ reconcileInFlightFallthroughs: 0,
persistenceSanityCheckSerializeFailures: 0,
deferredStoreFailures: 0,
authRenameRedirectCount: 0,
@@ -534,6 +546,14 @@ export function incrementExternalChangeHandlerErrors(): void {
counters.externalChangeHandlerErrors++;
}
+export function incrementReconcileOwnFlushSkips(): void {
+ counters.reconcileOwnFlushSkips++;
+}
+
+export function incrementReconcileInFlightFallthroughs(): void {
+ counters.reconcileInFlightFallthroughs++;
+}
+
export function incrementPersistenceSanityCheckSerializeFailures(): void {
counters.persistenceSanityCheckSerializeFailures++;
}
@@ -661,6 +681,8 @@ export function resetMetrics(): void {
counters.bridgeSplitBrainRederivesSuppressed = 0;
counters.persistenceReconciliationFailures = 0;
counters.externalChangeHandlerErrors = 0;
+ counters.reconcileOwnFlushSkips = 0;
+ counters.reconcileInFlightFallthroughs = 0;
counters.persistenceSanityCheckSerializeFailures = 0;
counters.deferredStoreFailures = 0;
counters.authRenameRedirectCount = 0;
diff --git a/packages/server/src/persistence.ts b/packages/server/src/persistence.ts
index 94526a438..236c2069e 100644
--- a/packages/server/src/persistence.ts
+++ b/packages/server/src/persistence.ts
@@ -264,6 +264,12 @@ export function deleteReconciledBase(docName: string): void {
reconciledBaseByBranch.get(activeBranch)?.delete(docName);
}
+const inFlightFlushByDoc = new Map<string, string>();
+
+export function peekInFlightFlush(docName: string): string | undefined {
+ return inFlightFlushByDoc.get(docName);
+}
+
let batchInProgress = false;
export function setBatchInProgress(value: boolean): void {
@@ -714,6 +720,7 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
}): Promise<void> {
ensureHistograms();
const started = Date.now();
+ let inFlightFlushValue: string | undefined;
return withSpan(
'persistence.onStoreDocument',
{ attributes: { 'doc.name': documentName } },
@@ -908,6 +915,9 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
}
}
+ inFlightFlushValue = normalizeBridge(markdown);
+ inFlightFlushByDoc.set(documentName, inFlightFlushValue);
+
const requestedPath = safeContentPath(documentName, contentDir);
await tracedMkdir(dirname(requestedPath), { recursive: true });
@@ -1054,6 +1064,12 @@ export function createPersistenceExtension(options?: PersistenceOptions): Persis
scheduleGitCommit();
},
).finally(() => {
+ if (
+ inFlightFlushValue !== undefined &&
+ inFlightFlushByDoc.get(documentName) === inFlightFlushValue
+ ) {
+ inFlightFlushByDoc.delete(documentName);
+ }
storeDurationHist?.record((Date.now() - started) / 1000);
});
}
diff --git a/packages/server/src/reconcile-own-flush-window.test.ts b/packages/server/src/reconcile-own-flush-window.test.ts
new file mode 100644
index 000000000..9685f8aa8
--- /dev/null
+++ b/packages/server/src/reconcile-own-flush-window.test.ts
@@ -0,0 +1,259 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { Hocuspocus } from '@hocuspocus/server';
+import { normalizeBridge } from '@inkeep/open-knowledge-core';
+import * as Y from 'yjs';
+import { isDocInConflict } from './conflict-errors.ts';
+import {
+ type ReconcileBeforeWriteResult,
+ reconcileDiskBeforeAgentWrite,
+} from './external-change.ts';
+import {
+ createPersistenceExtension,
+ getReconciledBase,
+ peekInFlightFlush,
+ setBatchInProgress,
+ switchReconciledBaseScope,
+} from './persistence.ts';
+
+const BROWSER_ORIGIN = {
+ source: 'connection',
+ connection: { context: { principalId: 'principal-test' } },
+};
+
+function replaceDocParagraphs(document: Y.Doc, texts: string[]): void {
+ const body = `${texts.join('\n\n')}\n`;
+ const fragment = document.getXmlFragment('default');
+ const ytext = document.getText('source');
+ if (fragment.length > 0) {
+ fragment.delete(0, fragment.length);
+ }
+ fragment.insert(
+ 0,
+ texts.map((text) => {
+ const paragraph = new Y.XmlElement('paragraph');
+ paragraph.insert(0, [new Y.XmlText(text)]);
+ return paragraph;
+ }),
+ );
+ if (ytext.length > 0) {
+ ytext.delete(0, ytext.length);
+ }
+ ytext.insert(0, body);
+}
+
+async function loadDocument(
+ persistence: ReturnType<typeof createPersistenceExtension>,
+ document: Y.Doc,
+ documentName: string,
+): Promise<void> {
+ await persistence.extension.onLoadDocument?.({
+ document,
+ documentName,
+ context: {},
+ } as never);
+}
+
+async function storeDocument(
+ persistence: ReturnType<typeof createPersistenceExtension>,
+ document: Y.Doc,
+ documentName: string,
+): Promise<void> {
+ await persistence.extension.onStoreDocument?.({
+ document,
+ documentName,
+ lastTransactionOrigin: BROWSER_ORIGIN,
+ lastContext: {},
+ } as never);
+}
+
+function fakeHocuspocusWith(docName: string, document: Y.Doc): Hocuspocus {
+ return { documents: new Map([[docName, document]]) } as unknown as Hocuspocus;
+}
+
+const BASE_CONTENT = 'alpha\n\nbeta\n'; // prior flush (stale base in-window)
+const FLUSHED_PARAGRAPHS = ['alpha', 'beta gamma']; // own flush snapshot (theirs)
+const FLUSHED_CONTENT = 'alpha\n\nbeta gamma\n';
+const LIVE_PARAGRAPHS = ['alpha', 'beta gamma delta']; // live Y.Text moved past the snapshot (ours)
+
+interface WindowProbe {
+ windowResult: ReconcileBeforeWriteResult | undefined;
+ baseSeenInWindow: string | undefined;
+ inFlightSeenInWindow: string | undefined;
+ conflictAfterGuard: boolean | undefined;
+}
+
+async function drivePhantomDivergence(
+ tmpDir: string,
+ docName: string,
+ document: Y.Doc,
+ options: { diskContentInWindow?: string } = {},
+): Promise<WindowProbe> {
+ const docPath = join(tmpDir, `${docName}.md`);
+ writeFileSync(docPath, BASE_CONTENT, 'utf-8');
+
+ const probe: WindowProbe = {
+ windowResult: undefined,
+ baseSeenInWindow: undefined,
+ inFlightSeenInWindow: undefined,
+ conflictAfterGuard: undefined,
+ };
+
+ let windowFired = false;
+ const persistence = createPersistenceExtension({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ gitEnabled: false,
+ onDiskFlush: (name) => {
+ if (name !== docName || windowFired) return;
+ windowFired = true;
+ document.transact(() => replaceDocParagraphs(document, LIVE_PARAGRAPHS), BROWSER_ORIGIN);
+ probe.baseSeenInWindow = getReconciledBase(docName);
+ probe.inFlightSeenInWindow = peekInFlightFlush(docName);
+ if (options.diskContentInWindow !== undefined) {
+ writeFileSync(docPath, options.diskContentInWindow, 'utf-8');
+ }
+ probe.windowResult = reconcileDiskBeforeAgentWrite(
+ fakeHocuspocusWith(docName, document),
+ docName,
+ tmpDir,
+ );
+ probe.conflictAfterGuard = isDocInConflict(document as never);
+ },
+ });
+
+ await loadDocument(persistence, document, docName);
+ document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
+ await storeDocument(persistence, document, docName);
+
+ expect(windowFired).toBe(true);
+ expect(probe.baseSeenInWindow).toBe(BASE_CONTENT);
+ return probe;
+}
+
+describe('reconcileDiskBeforeAgentWrite — own persistence flush is not foreign divergence', () => {
+ let tmpDir: string;
+ let document: Y.Doc;
+
+ beforeEach(() => {
+ tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-own-flush-window-')));
+ mkdirSync(tmpDir, { recursive: true });
+ setBatchInProgress(false);
+ switchReconciledBaseScope('main');
+ document = new Y.Doc();
+ });
+
+ afterEach(() => {
+ document.destroy();
+ setBatchInProgress(false);
+ switchReconciledBaseScope('main');
+ rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test("an agent write inside the server's own flush-commit window is not refused and does not latch lifecycle conflict", async () => {
+ const docName = 'own-flush-window';
+ const probe = await drivePhantomDivergence(tmpDir, docName, document);
+
+ expect(probe.conflictAfterGuard).toBe(false);
+ expect(isDocInConflict(document as never)).toBe(false);
+ expect(probe.windowResult?.reconciled).toBe(false);
+ expect(probe.inFlightSeenInWindow).toBe(normalizeBridge(FLUSHED_CONTENT));
+ });
+
+ test('a FOREIGN disk edit landing inside the flush window still reconciles (narrow-equality safety boundary)', async () => {
+ const docName = 'own-flush-foreign-in-window';
+ const FOREIGN_CONTENT = 'alpha FOREIGN EDIT\n\nbeta\n';
+ const probe = await drivePhantomDivergence(tmpDir, docName, document, {
+ diskContentInWindow: FOREIGN_CONTENT,
+ });
+
+ expect(probe.inFlightSeenInWindow).toBe(normalizeBridge(FLUSHED_CONTENT));
+ expect(probe.windowResult?.reconciled).toBe(true);
+ expect(probe.windowResult?.mergeOutcome).toBe('merged');
+ });
+
+ test('no permanent 409 wedge: after the flush settles, subsequent agent writes are not refused', async () => {
+ const docName = 'own-flush-wedge';
+ await drivePhantomDivergence(tmpDir, docName, document);
+
+ expect(getReconciledBase(docName)).toBe(FLUSHED_CONTENT);
+
+ const laterGuard = reconcileDiskBeforeAgentWrite(
+ fakeHocuspocusWith(docName, document),
+ docName,
+ tmpDir,
+ );
+ expect(laterGuard.reconciled).toBe(false);
+
+ expect(isDocInConflict(document as never)).toBe(false);
+ });
+
+ test('a failed disk flush does not leave the in-flight flush signal stuck set', async () => {
+ const docName = 'own-flush-fault';
+ const docPath = join(tmpDir, `${docName}.md`);
+ writeFileSync(docPath, BASE_CONTENT, 'utf-8');
+
+ const persistence = createPersistenceExtension({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ gitEnabled: false,
+ });
+ await loadDocument(persistence, document, docName);
+ document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
+
+ const prevFault = process.env.OK_TEST_STORE_FAULT;
+ process.env.OK_TEST_STORE_FAULT = docName;
+ try {
+ await expect(storeDocument(persistence, document, docName)).rejects.toThrow(
+ 'OK_TEST_STORE_FAULT',
+ );
+ } finally {
+ if (prevFault === undefined) {
+ delete process.env.OK_TEST_STORE_FAULT;
+ } else {
+ process.env.OK_TEST_STORE_FAULT = prevFault;
+ }
+ }
+
+ expect(peekInFlightFlush(docName)).toBeUndefined();
+ expect(getReconciledBase(docName)).toBe(BASE_CONTENT);
+ });
+
+ test("an earlier overlapping flush settling does not clear a later flush's in-flight signal", async () => {
+ const docName = 'own-flush-overlap';
+ const docPath = join(tmpDir, `${docName}.md`);
+ writeFileSync(docPath, BASE_CONTENT, 'utf-8');
+
+ const OVERLAP_PARAGRAPHS = ['alpha', 'beta gamma epsilon'];
+ const OVERLAP_CONTENT = 'alpha\n\nbeta gamma epsilon\n';
+
+ let windowFired = false;
+ let laterFlush: Promise<void> | undefined;
+ let peekAfterLaterStart: string | undefined;
+ const persistence = createPersistenceExtension({
+ contentDir: tmpDir,
+ projectDir: tmpDir,
+ gitEnabled: false,
+ onDiskFlush: (name) => {
+ if (name !== docName || windowFired) return;
+ windowFired = true;
+ document.transact(() => replaceDocParagraphs(document, OVERLAP_PARAGRAPHS), BROWSER_ORIGIN);
+ laterFlush = storeDocument(persistence, document, docName);
+ peekAfterLaterStart = peekInFlightFlush(docName);
+ },
+ });
+
+ await loadDocument(persistence, document, docName);
+ document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
+ await storeDocument(persistence, document, docName);
+
+ expect(windowFired).toBe(true);
+ expect(peekAfterLaterStart).toBe(normalizeBridge(OVERLAP_CONTENT));
+ expect(peekInFlightFlush(docName)).toBe(normalizeBridge(OVERLAP_CONTENT));
+
+ await laterFlush;
+ expect(peekInFlightFlush(docName)).toBeUndefined();
+ });
+});
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 8871038c7..3a7a61395 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -15,8 +15,6 @@ import {
humanFormat,
type MarkdownManager,
type Principal,
- prependFrontmatter,
- stripFrontmatter,
} from '@inkeep/open-knowledge-core';
import {
readConfigSafely,
@@ -62,6 +60,7 @@ import {
applyDiskContentToDoc,
applyExternalChange,
FILE_WATCHER_ORIGIN,
+ serializeYDocSource,
} from './external-change.ts';
import {
assertNeverDiskEvent,
@@ -768,9 +767,7 @@ export function createServer(options: ServerOptions): ServerInstance {
function serializeDoc(docName: string): string | null {
const document = hocuspocus.documents.get(docName);
if (!document) return null;
- const ytextSnapshot = document.getText('source').toString();
- const { frontmatter, body } = stripFrontmatter(ytextSnapshot);
- return prependFrontmatter(frontmatter, body);
+ return serializeYDocSource(document);
}
const applyToDoc = (docName: string, content: string): void =>
@@ -921,7 +918,7 @@ export function createServer(options: ServerOptions): ServerInstance {
case 'merged':
try {
applyToDoc(docName, result.newContent);
- setReconciledBase(docName, result.newContent);
+ setReconciledBase(docName, theirs);
incrementReconcile();
clearLifecycleConflict(document);
backlinkIndex.updateDocumentFromMarkdown(docName, theirs);