Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

22.6 KB
Raw Blame
import { mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { Hocuspocus } from '@hocuspocus/server';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
import { RECONCILE_TEST_CONFLICTS } from './conflict-authority.test-helper.ts';
import { ConflictAuthority } from './conflict-authority.ts';
import { DISPLACED_VERSION_TTL_MS, DocumentDurabilityState } from './document-durability-state.ts';
import {
  reconcileDiskBeforeAgentWrite,
  refuseStaleExternalWrite,
  serializeYDocSource,
} from './external-change.ts';
import { getLogger } from './logger.ts';
import { NO_SHADOW_MERGE_HISTORY } from './merge-history.test-helper.ts';
import { getMetrics } from './metrics.ts';
import { createPersistenceExtension } from './persistence.ts';
import { MAX_LCS_CELLS } from './reconciliation.ts';

const BROWSER_ORIGIN = {
  source: 'connection',
  connection: { context: { principalId: 'principal-test' } },
};

const STALE_CONTENT = 'alpha\n\nbeta\n';
const ACKNOWLEDGED_CONTENT = 'alpha\n\nbeta gamma\n';

function replaceDocParagraphs(document: Y.Doc, texts: string[]): void {
  const fragment = document.getXmlFragment('default');
  const ytext = document.getText('source');
  if (fragment.length > 0) fragment.delete(0, fragment.length);
  fragment.insert(
    0,
    texts.map((text) => {
      const paragraph = new Y.XmlElement('paragraph');
      paragraph.insert(0, [new Y.XmlText(text)]);
      return paragraph;
    }),
  );
  if (ytext.length > 0) ytext.delete(0, ytext.length);
  ytext.insert(0, `${texts.join('\n\n')}\n`);
}

function fakeHocuspocusWith(docName: string, document: Y.Doc): Hocuspocus {
  return { documents: new Map([[docName, document]]) } as unknown as Hocuspocus;
}

describe('reconcileDiskBeforeAgentWrite — stale external write gate', () => {
  let tmpDir: string;
  let docName: string;
  let document: Y.Doc;
  let durabilityState: DocumentDurabilityState;

  async function settleWrite(agentTriggered = true): Promise<void> {
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');
    const persistence = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
    });
    await persistence.extension.onLoadDocument?.({
      document,
      documentName: docName,
      context: {},
    } as never);
    document.transact(
      () => replaceDocParagraphs(document, ['alpha', 'beta gamma']),
      BROWSER_ORIGIN,
    );
    if (agentTriggered) durabilityState.markAgentWriteStore(docName);
    await persistence.extension.onStoreDocument?.({
      document,
      documentName: docName,
      lastTransactionOrigin: BROWSER_ORIGIN,
      lastContext: {},
    } as never);
    expect(durabilityState.getReconciledBase(docName)).toBe(ACKNOWLEDGED_CONTENT);
  }

  beforeEach(() => {
    tmpDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-stale-gate-')));
    docName = 'gated-doc';
    document = new Y.Doc();
    durabilityState = new DocumentDurabilityState();
  });

  afterEach(() => {
    document.destroy();
    rmSync(tmpDir, { recursive: true, force: true });
  });

  test('refuses disk bytes the server itself displaced and conflicts the document', async () => {
    await settleWrite();
    const before = getMetrics();
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');

    const result = await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      RECONCILE_TEST_CONFLICTS,
      NO_SHADOW_MERGE_HISTORY,
    );

    expect(result.reconciled).toBe(false);
    expect(durabilityState.getStaleExternalWrite(docName)).toBeDefined();
    expect(document.getText('source').toString()).toBe(ACKNOWLEDGED_CONTENT);
    expect(durabilityState.getReconciledBase(docName)).toBe(ACKNOWLEDGED_CONTENT);
    refuseStaleExternalWrite(durabilityState, document, docName, STALE_CONTENT);
    const after = getMetrics();
    expect(after.staleExternalWriteRefused).toBe(before.staleExternalWriteRefused + 1);
    expect(after.conflictCount).toBe(before.conflictCount);
    expect(after.storeDiskCompare).toEqual(before.storeDiskCompare);
    expect(after.bridgeMergeContentLoss).toBe(before.bridgeMergeContentLoss);
  });

  test('captures live document bytes before stale-write hydration raises the conflict', () => {
    const liveContent = `${ACKNOWLEDGED_CONTENT}\nnext live edit\n`;
    let authority: ConflictAuthority;
    durabilityState = new DocumentDurabilityState('main', {
      onStaleExternalWriteChange: () => {
        for (const stale of durabilityState.listStaleExternalWrites()) {
          const base = durabilityState.getReconciledBase(stale.docName) ?? stale.diskContent;
          authority.raise({
            kind: 'reconcile',
            file: stale.file,
            reason: 'stale-external-write',
            detectedAt: stale.detectedAt,
            stages: {
              base,
              ours: stale.retainedContent ?? base,
              theirs: stale.diskContent,
            },
          });
        }
      },
    });
    authority = new ConflictAuthority({
      projectDir: tmpDir,
      contentDir: tmpDir,
      io: {
        gitRaw: async () => '',
        writeProjectFileUntracked: () => undefined,
        unlinkProjectFileUndeclared: () => undefined,
        deleteResolvedContent: () => undefined,
        applyResolvedContent: async () => undefined,
      },
    });
    durabilityState.setReconciledBase(docName, ACKNOWLEDGED_CONTENT);
    durabilityState.recordDisplacedVersion(docName, STALE_CONTENT);
    replaceDocParagraphs(document, ['alpha', 'beta gamma', 'next live edit']);

    expect(
      refuseStaleExternalWrite(
        durabilityState,
        document,
        docName,
        STALE_CONTENT,
        authority,
        serializeYDocSource(document),
      ),
    ).toBe(true);

    const conflict = authority.findByDocName(docName);
    expect(conflict?.kind).toBe('reconcile');
    if (conflict?.kind !== 'reconcile') throw new Error('expected a reconcile conflict');
    expect(conflict.stages.ours).toBe(liveContent);
  });

  test('refuses to load rejected bytes when the acknowledged base is missing', async () => {
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');
    durabilityState.recordDisplacedVersion(docName, STALE_CONTENT);
    const persistence = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
    });
    await expect(
      persistence.extension.onLoadDocument?.({
        document,
        documentName: docName,
        context: {},
      } as never),
    ).rejects.toThrow('Missing acknowledged content');
    expect(document.getText('source').toString()).toBe('');
  });

  test('lets an external edit that matches no displaced version reconcile normally', async () => {
    await settleWrite();
    writeFileSync(join(tmpDir, `${docName}.md`), 'alpha\n\nbeta gamma\n\ndelta\n', 'utf-8');

    const result = await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      RECONCILE_TEST_CONFLICTS,
      NO_SHADOW_MERGE_HISTORY,
    );

    expect(result.reconciled).toBe(true);
    expect(document.getMap('lifecycle').get('status')).toBeUndefined();
    expect(document.getText('source').toString()).toContain('delta');
  });

  test('does not classify bytes displaced by a non-agent save as a stale agent rollback', async () => {
    await settleWrite(false);
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');

    const result = await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      RECONCILE_TEST_CONFLICTS,
      NO_SHADOW_MERGE_HISTORY,
    );

    expect(result.reconciled).toBe(true);
    expect(document.getMap('lifecycle').get('status')).toBeUndefined();
    expect(document.getText('source').toString()).toBe(STALE_CONTENT);
  });

  test('keeps refusing a detected stale write after its history entry expires', async () => {
    vi.useFakeTimers();
    try {
      await settleWrite();
      expect(refuseStaleExternalWrite(durabilityState, document, docName, STALE_CONTENT)).toBe(
        true,
      );

      vi.advanceTimersByTime(DISPLACED_VERSION_TTL_MS + 1_000);

      expect(refuseStaleExternalWrite(durabilityState, document, docName, STALE_CONTENT)).toBe(
        true,
      );
      expect(document.getText('source').toString()).toBe(ACKNOWLEDGED_CONTENT);
    } finally {
      vi.useRealTimers();
    }
  });

  test('clears a stale conflict when disk catches up to the acknowledged version', async () => {
    await settleWrite();
    expect(refuseStaleExternalWrite(durabilityState, document, docName, STALE_CONTENT)).toBe(true);

    expect(refuseStaleExternalWrite(durabilityState, document, docName, ACKNOWLEDGED_CONTENT)).toBe(
      false,
    );
    expect(document.getMap('lifecycle').get('status')).toBeUndefined();
    expect(durabilityState.listStaleExternalWrites()).toEqual([]);
  });

  test('pre-write reconciliation can clear a stale conflict after disk catches up', async () => {
    await settleWrite();
    expect(refuseStaleExternalWrite(durabilityState, document, docName, STALE_CONTENT)).toBe(true);
    writeFileSync(join(tmpDir, `${docName}.md`), ACKNOWLEDGED_CONTENT, 'utf-8');

    await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      RECONCILE_TEST_CONFLICTS,
      NO_SHADOW_MERGE_HISTORY,
    );

    expect(document.getMap('lifecycle').get('status')).toBeUndefined();
    expect(durabilityState.listStaleExternalWrites()).toEqual([]);
  });

  test('lets a genuinely new external edit replace a stale conflict', async () => {
    await settleWrite();
    expect(refuseStaleExternalWrite(durabilityState, document, docName, STALE_CONTENT)).toBe(true);

    const externalEdit = `${ACKNOWLEDGED_CONTENT}\ndelta\n`;
    expect(refuseStaleExternalWrite(durabilityState, document, docName, externalEdit)).toBe(false);
    expect(document.getMap('lifecycle').get('status')).toBeUndefined();
    expect(durabilityState.listStaleExternalWrites()).toEqual([]);
  });

  test('restores the acknowledged base instead of stale disk bytes when a document reloads', async () => {
    await settleWrite();
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');
    expect(refuseStaleExternalWrite(durabilityState, undefined, docName, STALE_CONTENT)).toBe(true);

    document.destroy();
    document = new Y.Doc();
    const persistence = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
    });
    await persistence.extension.onLoadDocument?.({
      document,
      documentName: docName,
      context: {},
    } as never);

    expect(document.getText('source').toString()).toBe(ACKNOWLEDGED_CONTENT);
    expect(durabilityState.getStaleExternalWrite(docName)).toBeDefined();
    expect(durabilityState.getReconciledBase(docName)).toBe(ACKNOWLEDGED_CONTENT);
  });

  test('restores the acknowledged base instead of stale disk bytes after a server restart', async () => {
    const persistencePath = join(tmpDir, 'stale-external-writes.json');
    durabilityState = new DocumentDurabilityState('main', { persistencePath });
    await settleWrite();
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');

    document.destroy();
    document = new Y.Doc();
    durabilityState = new DocumentDurabilityState('main', { persistencePath });
    const persistence = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
    });
    await persistence.extension.onLoadDocument?.({
      document,
      documentName: docName,
      context: {},
    } as never);

    expect(document.getText('source').toString()).toBe(ACKNOWLEDGED_CONTENT);
    expect(durabilityState.getStaleExternalWrite(docName)).toBeDefined();
  });

  test('the final persistence disk read refuses a stale version that arrives after preflight', async () => {
    const persistencePath = join(tmpDir, 'state.json');
    durabilityState = new DocumentDurabilityState('main', { persistencePath });
    durabilityState.setReconciledBase(docName, ACKNOWLEDGED_CONTENT);
    durabilityState.recordDisplacedVersion(docName, STALE_CONTENT);
    replaceDocParagraphs(document, ['alpha', 'beta gamma', 'next agent edit']);
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');
    durabilityState.markAgentWriteStore(docName);
    const persistence = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
    });

    await persistence.extension.onStoreDocument?.({
      document,
      documentName: docName,
      lastTransactionOrigin: BROWSER_ORIGIN,
      lastContext: {},
    } as never);

    expect(document.getText('source').toString()).toContain('next agent edit');
    expect(durabilityState.getStaleExternalWrite(docName)).toBeDefined();
    expect(durabilityState.getReconciledBase(docName)).toBe(ACKNOWLEDGED_CONTENT);
    expect(durabilityState.takeStaleExternalWriteFreeze(docName)).toBe(true);
    const retained = document.getText('source').toString();
    durabilityState.recordDisplacedVersion(docName, 'an older displaced version\n');
    writeFileSync(join(tmpDir, `${docName}.md`), 'an older displaced version\n', 'utf-8');
    expect(
      refuseStaleExternalWrite(durabilityState, document, docName, 'an older displaced version\n'),
    ).toBe(true);
    document.destroy();
    document = new Y.Doc();
    const restoredState = new DocumentDurabilityState('main', { persistencePath });
    const restarted = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState: restoredState,
    });
    await restarted.extension.onLoadDocument?.({
      document,
      documentName: docName,
      context: {},
    } as never);
    expect(document.getText('source').toString()).toBe(retained);
    expect(restoredState.getReconciledBase(docName)).toBe(ACKNOWLEDGED_CONTENT);
    expect(restoredState.getStaleExternalWrite(docName)).toBeDefined();
  });

  test('normalized-equal displaced bytes remain a conflict until exact acknowledged bytes return', async () => {
    const crlfStale = ACKNOWLEDGED_CONTENT.replaceAll('\n', '\r\n');
    durabilityState.setReconciledBase(docName, ACKNOWLEDGED_CONTENT);
    durabilityState.recordDisplacedVersion(docName, crlfStale);
    replaceDocParagraphs(document, ['alpha', 'beta gamma']);
    writeFileSync(join(tmpDir, `${docName}.md`), crlfStale, 'utf-8');
    await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      RECONCILE_TEST_CONFLICTS,
      NO_SHADOW_MERGE_HISTORY,
    );
    expect(durabilityState.getStaleExternalWrite(docName)).toBeDefined();
    expect(document.getText('source').toString()).toBe(ACKNOWLEDGED_CONTENT);
    writeFileSync(join(tmpDir, `${docName}.md`), ACKNOWLEDGED_CONTENT, 'utf-8');
    await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      RECONCILE_TEST_CONFLICTS,
      NO_SHADOW_MERGE_HISTORY,
    );
    expect(durabilityState.getStaleExternalWrite(docName)).toBeUndefined();
  });

  test.each([ACKNOWLEDGED_CONTENT, 'genuinely new external content\n'])(
    'a pending retained edit survives disk replacement with %s before restart',
    async (diskContent) => {
      const persistencePath = join(tmpDir, 'state.json');
      durabilityState = new DocumentDurabilityState('main', { persistencePath });
      const retained = `${ACKNOWLEDGED_CONTENT}\nnext retained edit\n`;
      durabilityState.setReconciledBase(docName, ACKNOWLEDGED_CONTENT);
      durabilityState.recordDisplacedVersion(docName, STALE_CONTENT);
      durabilityState.recordStaleExternalWrite(docName, STALE_CONTENT, retained);
      const beforeReplacement = getMetrics();
      replaceDocParagraphs(document, ['alpha', 'beta gamma', 'next retained edit']);
      writeFileSync(join(tmpDir, `${docName}.md`), diskContent, 'utf-8');
      await reconcileDiskBeforeAgentWrite(
        durabilityState,
        fakeHocuspocusWith(docName, document),
        docName,
        tmpDir,
        undefined,
        undefined,
        RECONCILE_TEST_CONFLICTS,
        NO_SHADOW_MERGE_HISTORY,
      );
      expect(durabilityState.getStaleExternalWrite(docName)).toBeDefined();
      expect(refuseStaleExternalWrite(durabilityState, undefined, docName, diskContent)).toBe(true);
      expect(getMetrics().staleExternalWriteRefused).toBe(
        beforeReplacement.staleExternalWriteRefused + 1,
      );
      expect(getMetrics().conflictCount).toBe(beforeReplacement.conflictCount);
      document.destroy();
      document = new Y.Doc();
      const restored = new DocumentDurabilityState('main', { persistencePath });
      const persistence = createPersistenceExtension({
        mergeHistory: null,
        ingestDiskAtStore: null,
        contentDir: tmpDir,
        projectDir: tmpDir,
        gitEnabled: false,
        durabilityState: restored,
      });
      await persistence.extension.onLoadDocument?.({
        document,
        documentName: docName,
        context: {},
      } as never);
      expect(document.getText('source').toString()).toBe(retained);
      expect(restored.getStaleExternalWrite(docName)).toBeDefined();
      expect(restored.getStaleExternalWrite(docName)?.diskContent).toBe(diskContent);
      expect(getMetrics().staleExternalWriteRefused).toBe(
        beforeReplacement.staleExternalWriteRefused + 1,
      );
    },
  );

  test('records the exact disk spelling displaced by an agent write', async () => {
    const crlfStale = STALE_CONTENT.replaceAll('\n', '\r\n');
    writeFileSync(join(tmpDir, `${docName}.md`), crlfStale, 'utf-8');
    durabilityState.setReconciledBase(docName, STALE_CONTENT);
    replaceDocParagraphs(document, ['alpha', 'beta gamma']);
    durabilityState.markAgentWriteStore(docName);
    const persistence = createPersistenceExtension({
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
      mergeHistory: null,
      ingestDiskAtStore: async (name, _path, content) => {
        durabilityState.setReconciledBase(name, content);
        return { kind: 'taken-in' };
      },
    });

    await persistence.extension.onStoreDocument?.({
      document,
      documentName: docName,
      lastTransactionOrigin: BROWSER_ORIGIN,
      lastContext: {},
    } as never);

    expect(durabilityState.isDisplacedVersion(docName, crlfStale)).toBe(true);
  });

  test('a blockless acknowledged base conflicts the pre-write reconcile and preserves the live doc', async () => {
    writeFileSync(join(tmpDir, `${docName}.md`), STALE_CONTENT, 'utf-8');
    const persistence = createPersistenceExtension({
      mergeHistory: null,
      ingestDiskAtStore: null,
      contentDir: tmpDir,
      projectDir: tmpDir,
      gitEnabled: false,
      durabilityState,
    });
    await persistence.extension.onLoadDocument?.({
      document,
      documentName: docName,
      context: {},
    } as never);
    expect(durabilityState.getReconciledBase(docName)).toBe(STALE_CONTENT);

    durabilityState.setReconciledBase(docName, '');
    replaceDocParagraphs(document, ['alpha', 'beta gamma']);

    const authority = new ConflictAuthority({
      projectDir: tmpDir,
      contentDir: tmpDir,
      io: {
        gitRaw: async () => '',
        writeProjectFileUntracked: () => undefined,
        unlinkProjectFileUndeclared: () => undefined,
        deleteResolvedContent: () => undefined,
        applyResolvedContent: async () => undefined,
      },
    });

    const result = await reconcileDiskBeforeAgentWrite(
      durabilityState,
      fakeHocuspocusWith(docName, document),
      docName,
      tmpDir,
      undefined,
      undefined,
      authority,
      NO_SHADOW_MERGE_HISTORY,
    );

    expect(result.reconciled).toBe(false);
    const conflict = authority.findByDocName(docName);
    expect(conflict?.kind).toBe('reconcile');
    if (conflict?.kind !== 'reconcile') throw new Error('expected a reconcile conflict');
    expect(conflict.reason).toBe('refused-no-base');
    expect(document.getText('source').toString()).toBe(ACKNOWLEDGED_CONTENT);
    expect(durabilityState.getReconciledBase(docName)).toBe('');
  });

  test('an insert-group merge past the LCS cap bumps the dedup-skipped counter on the pre-write path', async () => {
    await settleWrite();
    const before = getMetrics();
    const perSide = Math.ceil(Math.sqrt(MAX_LCS_CELLS)) + 10;
    const shared = Array.from({ length: 10 }, (_, i) => `shared ${i}.`);
    const ourOnly = Array.from({ length: perSide }, (_, i) => `ours ${i}.`);
    const theirOnly = Array.from({ length: perSide }, (_, i) => `theirs ${i}.`);
    const baseBody = ACKNOWLEDGED_CONTENT.replace(/\n$/, '');
    const theirsText = `${[baseBody, ...shared, ...theirOnly].join('\n\n')}\n`;
    replaceDocParagraphs(document, ['alpha', 'beta gamma', ...shared, ...ourOnly]);
    writeFileSync(join(tmpDir, `${docName}.md`), theirsText, 'utf-8');

    const warnSpy = vi.spyOn(getLogger('reconcile'), 'warn');
    try {
      const result = await reconcileDiskBeforeAgentWrite(
        durabilityState,
        fakeHocuspocusWith(docName, document),
        docName,
        tmpDir,
        undefined,
        undefined,
        RECONCILE_TEST_CONFLICTS,
        NO_SHADOW_MERGE_HISTORY,
      );

      expect(result.reconciled).toBe(true);
      expect(result.mergeOutcome).toBe('merged');
      expect(getMetrics().reconcileInsertDedupSkipped).toBe(before.reconcileInsertDedupSkipped + 1);
      expect(
        warnSpy.mock.calls.some((call) => String(call[1] ?? '').includes('dedup skipped')),
      ).toBe(true);
      const mergedText = document.getText('source').toString();
      expect(mergedText).toContain('shared 0.');
      expect(mergedText).toContain('ours 0.');
      expect(mergedText).toContain('theirs 0.');
    } finally {
      warnSpy.mockRestore();
    }
  }, 60_000);
});