Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { Hocuspocus } from '@hocuspocus/server';
import { normalizeBridge } from '@inkeep/open-knowledge-core';
import { afterEach, beforeEach, describe, expect, test } from 'vitest';
import * as Y from 'yjs';
import { RECONCILE_TEST_CONFLICTS } from './conflict-authority.test-helper.ts';
import { isDocInConflict } from './conflict-authority.ts';
import { DocumentDurabilityState } from './document-durability-state.ts';
import {
type ReconcileBeforeWriteResult,
reconcileDiskBeforeAgentWrite,
} from './external-change.ts';
import { NO_SHADOW_MERGE_HISTORY } from './merge-history.test-helper.ts';
import { createPersistenceExtension } from './persistence.ts';
const BROWSER_ORIGIN = {
source: 'connection',
connection: { context: { principalId: 'principal-test' } },
};
function replaceDocParagraphs(document: Y.Doc, texts: string[]): void {
const body = `${texts.join('\n\n')}\n`;
const fragment = document.getXmlFragment('default');
const ytext = document.getText('source');
if (fragment.length > 0) {
fragment.delete(0, fragment.length);
}
fragment.insert(
0,
texts.map((text) => {
const paragraph = new Y.XmlElement('paragraph');
paragraph.insert(0, [new Y.XmlText(text)]);
return paragraph;
}),
);
if (ytext.length > 0) {
ytext.delete(0, ytext.length);
}
ytext.insert(0, body);
}
async function loadDocument(
persistence: ReturnType<typeof createPersistenceExtension>,
document: Y.Doc,
documentName: string,
): Promise<void> {
await persistence.extension.onLoadDocument?.({
document,
documentName,
context: {},
} as never);
}
async function storeDocument(
persistence: ReturnType<typeof createPersistenceExtension>,
document: Y.Doc,
documentName: string,
): Promise<void> {
await persistence.extension.onStoreDocument?.({
document,
documentName,
lastTransactionOrigin: BROWSER_ORIGIN,
lastContext: {},
} as never);
}
function fakeHocuspocusWith(docName: string, document: Y.Doc): Hocuspocus {
return { documents: new Map([[docName, document]]) } as unknown as Hocuspocus;
}
const BASE_CONTENT = 'alpha\n\nbeta\n';
const FLUSHED_PARAGRAPHS = ['alpha', 'beta gamma'];
const FLUSHED_CONTENT = 'alpha\n\nbeta gamma\n';
const LIVE_PARAGRAPHS = ['alpha', 'beta gamma delta'];
interface WindowProbe {
windowResult: ReconcileBeforeWriteResult | undefined;
baseSeenInWindow: string | undefined;
inFlightSeenInWindow: string | undefined;
conflictAfterGuard: boolean | undefined;
}
async function drivePhantomDivergence(
tmpDir: string,
docName: string,
document: Y.Doc,
durabilityState: DocumentDurabilityState,
options: { diskContentInWindow?: string } = {},
): Promise<WindowProbe> {
const docPath = join(tmpDir, `${docName}.md`);
writeFileSync(docPath, BASE_CONTENT, 'utf-8');
const probe: WindowProbe = {
windowResult: undefined,
baseSeenInWindow: undefined,
inFlightSeenInWindow: undefined,
conflictAfterGuard: undefined,
};
let windowFired = false;
let windowGuard: Promise<ReconcileBeforeWriteResult> | undefined;
const persistence = createPersistenceExtension({
mergeHistory: null,
ingestDiskAtStore: null,
contentDir: tmpDir,
projectDir: tmpDir,
gitEnabled: false,
durabilityState,
onDiskFlush: (name) => {
if (name !== docName || windowFired) return;
windowFired = true;
document.transact(() => replaceDocParagraphs(document, LIVE_PARAGRAPHS), BROWSER_ORIGIN);
probe.baseSeenInWindow = durabilityState.getReconciledBase(docName);
probe.inFlightSeenInWindow = durabilityState.peekInFlightFlush(docName);
if (options.diskContentInWindow !== undefined) {
writeFileSync(docPath, options.diskContentInWindow, 'utf-8');
}
windowGuard = reconcileDiskBeforeAgentWrite(
durabilityState,
fakeHocuspocusWith(docName, document),
docName,
tmpDir,
undefined,
undefined,
RECONCILE_TEST_CONFLICTS,
NO_SHADOW_MERGE_HISTORY,
);
probe.conflictAfterGuard = isDocInConflict(document as never);
},
});
await loadDocument(persistence, document, docName);
document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
await storeDocument(persistence, document, docName);
probe.windowResult = await windowGuard;
expect(windowFired).toBe(true);
expect(probe.baseSeenInWindow).toBe(BASE_CONTENT);
return probe;
}
describe('reconcileDiskBeforeAgentWrite — own persistence flush is not foreign divergence', () => {
let tmpDir: string;
let document: Y.Doc;
let durabilityState: DocumentDurabilityState;
beforeEach(() => {
tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-own-flush-window-')));
mkdirSync(tmpDir, { recursive: true });
durabilityState = new DocumentDurabilityState();
document = new Y.Doc();
});
afterEach(() => {
document.destroy();
rmSync(tmpDir, { recursive: true, force: true });
});
test("an agent write inside the server's own flush-commit window is not refused and does not latch lifecycle conflict", async () => {
const docName = 'own-flush-window';
const probe = await drivePhantomDivergence(tmpDir, docName, document, durabilityState);
expect(probe.conflictAfterGuard).toBe(false);
expect(isDocInConflict(document as never)).toBe(false);
expect(probe.windowResult?.reconciled).toBe(false);
expect(probe.inFlightSeenInWindow).toBe(normalizeBridge(FLUSHED_CONTENT));
});
test('a FOREIGN disk edit landing inside the flush window still reconciles (narrow-equality safety boundary)', async () => {
const docName = 'own-flush-foreign-in-window';
const FOREIGN_CONTENT = 'alpha FOREIGN EDIT\n\nbeta\n';
const probe = await drivePhantomDivergence(tmpDir, docName, document, durabilityState, {
diskContentInWindow: FOREIGN_CONTENT,
});
expect(probe.inFlightSeenInWindow).toBe(normalizeBridge(FLUSHED_CONTENT));
expect(probe.windowResult?.reconciled).toBe(true);
expect(probe.windowResult?.mergeOutcome).toBe('merged');
});
test('no permanent 409 wedge: after the flush settles, subsequent agent writes are not refused', async () => {
const docName = 'own-flush-wedge';
await drivePhantomDivergence(tmpDir, docName, document, durabilityState);
expect(durabilityState.getReconciledBase(docName)).toBe(FLUSHED_CONTENT);
const laterGuard = await reconcileDiskBeforeAgentWrite(
durabilityState,
fakeHocuspocusWith(docName, document),
docName,
tmpDir,
undefined,
undefined,
RECONCILE_TEST_CONFLICTS,
NO_SHADOW_MERGE_HISTORY,
);
expect(laterGuard.reconciled).toBe(false);
expect(isDocInConflict(document as never)).toBe(false);
});
test('a failed disk flush does not leave the in-flight flush signal stuck set', async () => {
const docName = 'own-flush-fault';
const docPath = join(tmpDir, `${docName}.md`);
writeFileSync(docPath, BASE_CONTENT, 'utf-8');
const persistence = createPersistenceExtension({
mergeHistory: null,
ingestDiskAtStore: null,
contentDir: tmpDir,
projectDir: tmpDir,
gitEnabled: false,
durabilityState,
});
await loadDocument(persistence, document, docName);
document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
const prevFault = process.env.OK_TEST_STORE_FAULT;
process.env.OK_TEST_STORE_FAULT = docName;
try {
await expect(storeDocument(persistence, document, docName)).rejects.toThrow(
'OK_TEST_STORE_FAULT',
);
} finally {
if (prevFault === undefined) {
delete process.env.OK_TEST_STORE_FAULT;
} else {
process.env.OK_TEST_STORE_FAULT = prevFault;
}
}
expect(durabilityState.peekInFlightFlush(docName)).toBeUndefined();
expect(durabilityState.getReconciledBase(docName)).toBe(BASE_CONTENT);
});
test("an earlier overlapping flush settling does not clear a later flush's in-flight signal", async () => {
const docName = 'own-flush-overlap';
const docPath = join(tmpDir, `${docName}.md`);
writeFileSync(docPath, BASE_CONTENT, 'utf-8');
const OVERLAP_PARAGRAPHS = ['alpha', 'beta gamma epsilon'];
const OVERLAP_CONTENT = 'alpha\n\nbeta gamma epsilon\n';
let windowFired = false;
let laterFlush: Promise<void> | undefined;
let peekAfterLaterStart: string | undefined;
const persistence = createPersistenceExtension({
mergeHistory: null,
ingestDiskAtStore: null,
contentDir: tmpDir,
projectDir: tmpDir,
gitEnabled: false,
durabilityState,
onDiskFlush: (name) => {
if (name !== docName || windowFired) return;
windowFired = true;
document.transact(() => replaceDocParagraphs(document, OVERLAP_PARAGRAPHS), BROWSER_ORIGIN);
laterFlush = storeDocument(persistence, document, docName);
peekAfterLaterStart = durabilityState.peekInFlightFlush(docName);
},
});
await loadDocument(persistence, document, docName);
document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
await storeDocument(persistence, document, docName);
expect(windowFired).toBe(true);
expect(peekAfterLaterStart).toBe(normalizeBridge(OVERLAP_CONTENT));
expect(durabilityState.peekInFlightFlush(docName)).toBe(normalizeBridge(OVERLAP_CONTENT));
await laterFlush;
expect(durabilityState.peekInFlightFlush(docName)).toBeUndefined();
});
});
describe('reconcileDiskBeforeAgentWrite — overlapping own flushes are not foreign divergence', () => {
let tmpDir: string;
let document: Y.Doc;
let durabilityState: DocumentDurabilityState;
beforeEach(() => {
tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-own-flush-overlap-')));
mkdirSync(tmpDir, { recursive: true });
durabilityState = new DocumentDurabilityState();
document = new Y.Doc();
});
afterEach(() => {
document.destroy();
rmSync(tmpDir, { recursive: true, force: true });
});
async function driveOverlapWindow(
docName: string,
options: { diskContentInWindow?: string } = {},
): Promise<{
diskAtWindow: string;
pendingAtWindow: number;
headAtWindow: string | undefined;
diskMatchedPendingAtWindow: boolean;
result: ReconcileBeforeWriteResult | undefined;
conflictAfterGuard: boolean | undefined;
}> {
const docPath = join(tmpDir, `${docName}.md`);
writeFileSync(docPath, BASE_CONTENT, 'utf-8');
const OVERLAP_PARAGRAPHS = ['alpha', 'beta gamma epsilon'];
let windowFired = false;
let laterFlush: Promise<void> | undefined;
let diskAtWindow = '';
let pendingAtWindow = 0;
let headAtWindow: string | undefined;
let diskMatchedPendingAtWindow = false;
let pendingResult: Promise<ReconcileBeforeWriteResult> | undefined;
let conflictAfterGuard: boolean | undefined;
const persistence = createPersistenceExtension({
mergeHistory: null,
ingestDiskAtStore: async (name, _path, content) => {
const taken = await reconcileDiskBeforeAgentWrite(
durabilityState,
fakeHocuspocusWith(name, document),
name,
tmpDir,
undefined,
undefined,
RECONCILE_TEST_CONFLICTS,
NO_SHADOW_MERGE_HISTORY,
);
return taken.reconciled || durabilityState.getReconciledBase(name) === content
? { kind: 'taken-in' }
: {
kind: 'not-taken-in',
error: new Error('the pre-write merge did not take the disk version in'),
};
},
contentDir: tmpDir,
projectDir: tmpDir,
gitEnabled: false,
durabilityState,
onDiskFlush: (name) => {
if (name !== docName || windowFired) return;
windowFired = true;
document.transact(() => replaceDocParagraphs(document, OVERLAP_PARAGRAPHS), BROWSER_ORIGIN);
laterFlush = storeDocument(persistence, document, docName);
if (options.diskContentInWindow !== undefined) {
writeFileSync(docPath, options.diskContentInWindow, 'utf-8');
}
diskAtWindow = readFileSync(docPath, 'utf-8');
pendingAtWindow = durabilityState.inFlightFlushCount(docName);
headAtWindow = durabilityState.peekInFlightFlush(docName);
diskMatchedPendingAtWindow = durabilityState.hasInFlightFlush(
docName,
normalizeBridge(diskAtWindow),
);
pendingResult = reconcileDiskBeforeAgentWrite(
durabilityState,
fakeHocuspocusWith(docName, document),
docName,
tmpDir,
undefined,
undefined,
RECONCILE_TEST_CONFLICTS,
NO_SHADOW_MERGE_HISTORY,
);
conflictAfterGuard = isDocInConflict(document as never);
},
});
await loadDocument(persistence, document, docName);
document.transact(() => replaceDocParagraphs(document, FLUSHED_PARAGRAPHS), BROWSER_ORIGIN);
await storeDocument(persistence, document, docName);
await laterFlush;
const result = await pendingResult;
expect(windowFired).toBe(true);
expect(pendingAtWindow).toBeGreaterThanOrEqual(2);
return {
diskAtWindow,
pendingAtWindow,
headAtWindow,
diskMatchedPendingAtWindow,
result,
conflictAfterGuard,
};
}
test('an agent write landing between two overlapping own flushes does not latch lifecycle conflict', async () => {
const probe = await driveOverlapWindow('overlap-window-clean');
expect(normalizeBridge(probe.diskAtWindow)).toBe(normalizeBridge(FLUSHED_CONTENT));
expect(probe.diskMatchedPendingAtWindow).toBe(true);
expect(probe.headAtWindow).not.toBe(normalizeBridge(probe.diskAtWindow));
expect(probe.conflictAfterGuard).toBe(false);
expect(isDocInConflict(document as never)).toBe(false);
expect(document.getMap('lifecycle').get('reason')).toBeUndefined();
expect(probe.result?.reconciled).toBe(false);
});
test('a FOREIGN disk edit landing in the overlap window still reconciles', async () => {
const probe = await driveOverlapWindow('overlap-window-foreign', {
diskContentInWindow: 'alpha FOREIGN EDIT\n\nbeta\n',
});
expect(probe.diskMatchedPendingAtWindow).toBe(false);
expect(probe.result?.reconciled).toBe(true);
expect(probe.result?.mergeOutcome).toBe('merged');
expect(probe.conflictAfterGuard).toBe(false);
expect(isDocInConflict(document as never)).toBe(false);
});
});