Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

21.1 KB
Raw Blame
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  readdirSync,
  readFileSync,
  rmSync,
  writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fnv1aDigest, LINEAGE_EPOCH_KEY } from '@inkeep/open-knowledge-core';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import * as Y from 'yjs';
import { DiskReceiptLog } from './disk-receipts.ts';
import * as fsTraced from './fs-traced.ts';
import { tracedAtomicFs } from './fs-traced.ts';
import type { EnginelessFileType, KeptCopy } from './merge-history.ts';
import {
  applyExternalEnginelessChange,
  loadMermaidDoc,
  MERMAID_SOURCE_ORIGIN,
  type MermaidPersistenceCtx,
  storeMermaidDoc,
} from './mermaid-persistence.ts';
import { getMetrics, resetMetrics } from './metrics.ts';
import { assertServerContentPath, snapshotServerContentScope } from './server-content-policy.ts';
import { contentHash } from './version-hash.ts';

let contentDir: string;
let home: string;

interface MovedVersion {
  docName: string;
  side: 'ours' | 'theirs';
  bytes: string;
  fileType: string;
}

interface RecordedNotice {
  docName: string;
  side: 'ours' | 'theirs';
  fileType: string;
  copy: KeptCopy;
}

type RecordingCtx = MermaidPersistenceCtx & {
  receipts: DiskReceiptLog;
  moved: MovedVersion[];
  notices: RecordedNotice[];
};

function makeCtx(
  options: { historySaves?: boolean; noHistory?: boolean; duringSave?: () => void } = {},
): RecordingCtx {
  const moved: MovedVersion[] = [];
  const notices: RecordedNotice[] = [];
  return {
    contentDir,
    home,
    lkgCache: new Map<string, string>(),
    receipts: new DiskReceiptLog(),
    moved,
    notices,
    history: options.noHistory
      ? null
      : {
          saveNoEngine: async (docName, displaced, fileType) => {
            options.duringSave?.();
            if (options.historySaves === false) return null;
            moved.push({ docName, ...displaced, fileType });
            return `entry-${moved.length}`;
          },
          noticeNoEngine: (docName, side, fileType, copy) => {
            notices.push({ docName, side, fileType, copy });
          },
        },
  };
}

function backupsFor(docName: string): string[] {
  const dir = join(
    home,
    '.ok',
    'edit-backups',
    'discarded',
    'projects',
    fnv1aDigest(contentDir),
    docName.replace(/[^A-Za-z0-9._-]+/g, '__'),
  );
  if (!existsSync(dir)) return [];
  return readdirSync(dir).map((name) => readFileSync(join(dir, name), 'utf-8'));
}

function blockBackups(): void {
  mkdirSync(join(home, '.ok'), { recursive: true });
  writeFileSync(join(home, '.ok', 'edit-backups'), 'not a folder');
}

const DOC = 'assets/flow.mmd';
const ABS = () => resolve(contentDir, DOC);
const SRC = 'graph TD;\n  A[Start] --> B{Choice};\n  B -->|Yes| C[Go];\n';

function writeMmd(content: string): void {
  mkdirSync(dirname(ABS()), { recursive: true });
  writeFileSync(ABS(), content, 'utf-8');
}

beforeEach(() => {
  contentDir = mkdtempSync(join(tmpdir(), 'ok-mermaid-'));
  home = mkdtempSync(join(tmpdir(), 'ok-mermaid-home-'));
  resetMetrics();
});
afterEach(() => {
  vi.restoreAllMocks();
  rmSync(contentDir, { recursive: true, force: true });
  rmSync(home, { recursive: true, force: true });
});

describe('storeMermaidDoc', () => {
  test('refuses a nested project created between staging and publishing verbatim content', async () => {
    writeMmd(SRC);
    const ctx = makeCtx();
    const scope = snapshotServerContentScope(contentDir);
    ctx.assertContentPath = (path) => assertServerContentPath(scope, path);
    ctx.lkgCache.set(DOC, SRC);
    const doc = new Y.Doc();
    doc.getText('source').insert(0, 'graph TD; After-->Change;\n');
    const write = tracedAtomicFs.writeFile;
    const staged = vi.spyOn(tracedAtomicFs, 'writeFile').mockImplementation(async (...args) => {
      await write(...args);
      mkdirSync(join(contentDir, 'assets', '.ok'));
      writeFileSync(join(contentDir, 'assets', '.ok', 'config.yml'), '');
    });
    try {
      expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('write-failed');
      expect(readFileSync(ABS(), 'utf8')).toBe(SRC);
      expect(ctx.lkgCache.get(DOC)).toBe(SRC);
      expect(
        readdirSync(join(contentDir, 'assets')).filter((name) => name.endsWith('.tmp')),
      ).toEqual([]);
    } finally {
      staged.mockRestore();
      doc.destroy();
    }
  });

  test('persists Y.Text("source") verbatim to the .mmd file', async () => {
    const ctx = makeCtx();
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, SRC), 'agent');
    expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('persisted');
    expect(readFileSync(ABS(), 'utf-8')).toBe(SRC);
  });

  test('stores source with markdown-syntactic + blank-line content byte-for-byte (no remark round-trip)', async () => {
    const ctx = makeCtx();
    const awkward = '## not a heading\n\n\n> not a quote  \ngraph LR; A-->B;\n';
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, awkward), 'agent');
    expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('persisted');
    expect(readFileSync(ABS(), 'utf-8')).toBe(awkward);
  });

  test('no-op when the store originates from a seed/reconcile import (skip-store origin)', async () => {
    const ctx = makeCtx();
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, SRC), 'agent');
    expect(await storeMermaidDoc(doc, DOC, MERMAID_SOURCE_ORIGIN, ctx)).toBe('no-op');
  });

  test('no-op on the second identical store (LKG equality guard)', async () => {
    const ctx = makeCtx();
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, SRC), 'agent');
    expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('persisted');
    expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('no-op');
  });

  test('records an own-write receipt for the bytes it wrote', async () => {
    const ctx = makeCtx();
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, SRC), 'agent');
    expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('persisted');
    expect(ctx.receipts.get(DOC)).toMatchObject({ hash: contentHash(SRC), via: 'own-write' });
    expect(ctx.receipts.isRecentOwnWrite(DOC, contentHash(SRC))).toBe(true);
  });

  test('writes when disk still holds the last saved version though the receipt moved on', async () => {
    writeMmd(SRC);
    const ctx = makeCtx();
    const doc = new Y.Doc();
    loadMermaidDoc(doc, DOC, ctx);
    ctx.receipts.recordAgreed(DOC, 'graph TD; Elsewhere;\n');
    doc.transact(() => doc.getText('source').insert(0, '%% edited\n'), 'agent');
    expect(await storeMermaidDoc(doc, DOC, 'agent', ctx)).toBe('persisted');
    expect(readFileSync(ABS(), 'utf-8')).toBe(`%% edited\n${SRC}`);
    expect(ctx.moved).toEqual([]);
  });
});

const ENGINELESS_CASES: ReadonlyArray<{
  docName: string;
  fileType: EnginelessFileType;
  saved: string;
  here: string;
  outside: string;
}> = [
  {
    docName: 'diagrams/flow.mmd',
    fileType: 'mermaid',
    saved: 'graph TD\n  A-->B\n',
    here: 'graph TD\n  A-->C\n',
    outside: 'graph TD\n  A-->D\n',
  },
  {
    docName: 'boards/plan.excalidraw',
    fileType: 'excalidraw',
    saved: '{"type":"excalidraw","elements":[{"id":"a","x":0}]}\n',
    here: '{"type":"excalidraw","elements":[{"id":"a","x":10}]}\n',
    outside: '{"type":"excalidraw","elements":[{"id":"a","x":20}]}\n',
  },
  {
    docName: 'src/value.ts',
    fileType: 'text',
    saved: 'export const value = 1;\n',
    here: 'export const value = 2;\n',
    outside: 'export const value = 3;\n',
  },
];

function writeDocFile(docName: string, content: string): void {
  const abs = resolve(contentDir, docName);
  mkdirSync(dirname(abs), { recursive: true });
  writeFileSync(abs, content, 'utf-8');
}

function readDocFile(docName: string): string {
  return readFileSync(resolve(contentDir, docName), 'utf-8');
}

function editTo(doc: Y.Doc, text: string): void {
  doc.transact(() => {
    const ytext = doc.getText('source');
    ytext.delete(0, ytext.length);
    ytext.insert(0, text);
  }, 'person');
}

function openWithUnsavedEdit(
  kase: (typeof ENGINELESS_CASES)[number],
  ctx: RecordingCtx,
): { doc: Y.Doc; undo: Y.UndoManager } {
  writeDocFile(kase.docName, kase.saved);
  const doc = new Y.Doc();
  loadMermaidDoc(doc, kase.docName, ctx);
  const undo = new Y.UndoManager(doc.getText('source'), { trackedOrigins: new Set(['person']) });
  editTo(doc, kase.here);
  return { doc, undo };
}

describe.each(ENGINELESS_CASES)('an outside write under unsaved edits to $docName', (kase) => {
  test('keeps the version here live and on disk and moves the outside version to history', async () => {
    const ctx = makeCtx();
    const { doc, undo } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('kept-live');

    expect(doc.getText('source').toString()).toBe(kase.here);
    expect(readDocFile(kase.docName)).toBe(kase.here);
    expect(ctx.moved).toEqual([
      { docName: kase.docName, side: 'theirs', bytes: kase.outside, fileType: kase.fileType },
    ]);
    expect(ctx.notices).toEqual([
      {
        docName: kase.docName,
        side: 'theirs',
        fileType: kase.fileType,
        copy: { where: 'history', entry: 'entry-1' },
      },
    ]);
    expect(backupsFor(kase.docName)).toEqual([]);
    expect(ctx.lkgCache.get(kase.docName)).toBe(kase.here);
    expect(ctx.receipts.get(kase.docName)?.hash).toBe(contentHash(kase.here));
    expect(getMetrics().enginelessStore).toEqual({ 'kept-live': 1 });
    expect(undo.undoStack).toHaveLength(1);
    undo.undo();
    expect(doc.getText('source').toString()).toBe(kase.saved);
  });

  test('when history cannot take the outside version it goes to the backup folder and the edits are written', async () => {
    const ctx = makeCtx({ historySaves: false });
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('kept-live');

    expect(doc.getText('source').toString()).toBe(kase.here);
    expect(readDocFile(kase.docName)).toBe(kase.here);
    expect(backupsFor(kase.docName)).toEqual([kase.outside]);
    expect(ctx.notices).toHaveLength(1);
    const [notice] = ctx.notices;
    expect(notice?.copy.where).toBe('backup');
    expect(notice?.copy.where === 'backup' && notice.copy.backup.display).toMatch(
      /^~\/\.ok\/edit-backups\/discarded\/projects\/[0-9a-f]{8}\/.+/,
    );
    expect(getMetrics().displacedVersionBackups).toBe(1);
  });

  test('writes nothing and keeps the version here live when neither history nor the backup folder can take the outside version', async () => {
    blockBackups();
    const ctx = makeCtx({ historySaves: false });
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toEqual({
      kind: 'held',
      cause: 'no-history-or-backup',
    });

    expect(doc.getText('source').toString()).toBe(kase.here);
    expect(readDocFile(kase.docName)).toBe(kase.outside);
    expect(ctx.lkgCache.get(kase.docName)).toBe(kase.saved);
    expect(ctx.notices).toEqual([]);
    expect(getMetrics().enginelessStore).toEqual({ 'held-no-history-or-backup': 1 });
    expect(getMetrics().displacedVersionBackupFailures).toBe(1);
  });
});

describe('the compare read before an engine-less write', () => {
  test('a persistence with no history recorder keeps the outside version in the backup folder before writing', async () => {
    const kase = ENGINELESS_CASES[0];
    const ctx = makeCtx({ noHistory: true });
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('kept-live');
    expect(readDocFile(kase.docName)).toBe(kase.here);
    expect(backupsFor(kase.docName)).toEqual([kase.outside]);
  });

  test('a persistence with no history recorder and no backup folder never writes over an outside change', async () => {
    blockBackups();
    const kase = ENGINELESS_CASES[0];
    const ctx = makeCtx({ noHistory: true });
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toEqual({
      kind: 'held',
      cause: 'no-history-or-backup',
    });
    expect(readDocFile(kase.docName)).toBe(kase.outside);
  });

  test('retries a busy read, then writes nothing when the file stays unreadable', async () => {
    const kase = ENGINELESS_CASES[0];
    const ctx = makeCtx();
    const { doc } = openWithUnsavedEdit(kase, ctx);
    vi.spyOn(fsTraced, 'tracedReadObservedSync').mockImplementation(() => {
      throw Object.assign(new Error('busy'), { code: 'EBUSY' });
    });

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toEqual({
      kind: 'held',
      cause: 'unreadable',
    });

    vi.restoreAllMocks();
    expect(readDocFile(kase.docName)).toBe(kase.saved);
    expect(getMetrics().enginelessStore).toEqual({ 'held-unreadable': 1 });
    expect(getMetrics().storeDiskCompareReadRetries).toBe(3);
  });
});

describe('an outside write that lands while an engine-less save runs', () => {
  test('a write during the history save is also moved to history before the edits are written', async () => {
    const kase = ENGINELESS_CASES[0];
    const later = 'graph TD\n  A-->E\n';
    let planted = false;
    const ctx = makeCtx({
      duringSave: () => {
        if (planted) return;
        planted = true;
        writeDocFile(kase.docName, later);
      },
    });
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('kept-live');

    expect(readDocFile(kase.docName)).toBe(kase.here);
    expect(ctx.moved.map((version) => version.bytes)).toEqual([kase.outside, later]);
  });

  test('a write between the compare and the rename is refused, moved to history, and the edits land after it', async () => {
    const kase = ENGINELESS_CASES[2];
    const ctx = makeCtx();
    const { doc } = openWithUnsavedEdit(kase, ctx);
    const between = 'export const value = 44;\n';
    const write = tracedAtomicFs.writeFile;
    let planted = false;
    vi.spyOn(tracedAtomicFs, 'writeFile').mockImplementation(async (...args) => {
      await write(...args);
      if (planted) return;
      planted = true;
      writeDocFile(kase.docName, between);
    });

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('kept-live');

    expect(readDocFile(kase.docName)).toBe(kase.here);
    expect(ctx.moved.map((version) => version.bytes)).toEqual([between]);
    expect(getMetrics().storeRestatRefused).toBe(1);
  });

  test('gives up after three outside versions in a row, each in history, and writes nothing', async () => {
    const kase = ENGINELESS_CASES[0];
    let next = 0;
    const ctx = makeCtx({
      duringSave: () => {
        next += 1;
        writeDocFile(kase.docName, `graph TD\n  A-->V${next}\n`);
      },
    });
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toEqual({
      kind: 'held',
      cause: 'keeps-changing',
    });

    expect(ctx.moved).toHaveLength(3);
    expect(readDocFile(kase.docName)).not.toBe(kase.here);
    expect(getMetrics().enginelessStore).toEqual({ 'held-exhausted': 1 });
  });

  test('an outside version that is not valid UTF-8 is never written over, at the save or the intake', async () => {
    const kase = ENGINELESS_CASES[2];
    const ctx = makeCtx();
    const { doc } = openWithUnsavedEdit(kase, ctx);
    const utf16 = Buffer.from('\ufeffexport const value = 5;\r\n', 'utf16le');
    writeFileSync(resolve(contentDir, kase.docName), utf16);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toMatchObject({
      kind: 'held',
      cause: 'not-utf8',
      path: resolve(contentDir, kase.docName),
    });
    expect(await applyExternalEnginelessChange(doc, kase.docName, ctx)).toBe('held');

    expect(readFileSync(resolve(contentDir, kase.docName)).equals(utf16)).toBe(true);
    expect(doc.getText('source').toString()).toBe(kase.here);
    expect(ctx.moved).toEqual([]);
    expect(backupsFor(kase.docName)).toEqual([]);
    expect(getMetrics().enginelessStore).toEqual({ 'held-not-utf8': 1 });
    expect(getMetrics().enginelessIntake).toEqual({ 'held-not-utf8': 1 });
  });
});

describe('applyExternalEnginelessChange', () => {
  test('applies the disk version live when nothing is unsaved', async () => {
    const kase = ENGINELESS_CASES[0];
    const ctx = makeCtx();
    writeDocFile(kase.docName, kase.saved);
    const doc = new Y.Doc();
    loadMermaidDoc(doc, kase.docName, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await applyExternalEnginelessChange(doc, kase.docName, ctx)).toBe('applied');

    expect(doc.getText('source').toString()).toBe(kase.outside);
    expect(ctx.lkgCache.get(kase.docName)).toBe(kase.outside);
    expect(ctx.receipts.get(kase.docName)).toMatchObject({
      hash: contentHash(kase.outside),
      via: 'agreed',
    });
    expect(ctx.moved).toEqual([]);
  });

  test('keeps unsaved edits live, moves the outside version to history once, and the next store writes the edits', async () => {
    const kase = ENGINELESS_CASES[1];
    const ctx = makeCtx();
    const { doc } = openWithUnsavedEdit(kase, ctx);
    writeDocFile(kase.docName, kase.outside);

    expect(await applyExternalEnginelessChange(doc, kase.docName, ctx)).toBe('kept-live');
    expect(doc.getText('source').toString()).toBe(kase.here);
    expect(readDocFile(kase.docName)).toBe(kase.outside);

    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('persisted');
    expect(readDocFile(kase.docName)).toBe(kase.here);
    expect(ctx.moved).toEqual([
      { docName: kase.docName, side: 'theirs', bytes: kase.outside, fileType: kase.fileType },
    ]);
  });

  test('takes nothing in when the file holds its own last write', async () => {
    const kase = ENGINELESS_CASES[2];
    const ctx = makeCtx();
    const doc = new Y.Doc();
    editTo(doc, kase.here);
    expect(await storeMermaidDoc(doc, kase.docName, 'person', ctx)).toBe('persisted');
    editTo(doc, `${kase.here}// more\n`);

    expect(await applyExternalEnginelessChange(doc, kase.docName, ctx)).toBe('unchanged');
    expect(doc.getText('source').toString()).toBe(`${kase.here}// more\n`);
    expect(ctx.moved).toEqual([]);
  });
});

describe('loadMermaidDoc', () => {
  test('seeds Y.Text("source") from disk and mints a lineage epoch', () => {
    writeMmd(SRC);
    const ctx = makeCtx();
    const doc = new Y.Doc();
    loadMermaidDoc(doc, DOC, ctx);
    expect(doc.getText('source').toString()).toBe(SRC);
    expect(typeof doc.getMap('lifecycle').get(LINEAGE_EPOCH_KEY)).toBe('string');
    expect(doc.getXmlFragment('default').length).toBe(0);
  });

  test('records an agreed receipt for the bytes it read', () => {
    writeMmd(SRC);
    const ctx = makeCtx();
    loadMermaidDoc(new Y.Doc(), DOC, ctx);
    expect(ctx.receipts.get(DOC)).toMatchObject({ hash: contentHash(SRC), via: 'agreed' });
  });

  test('lazy: a missing file seeds nothing (admitting a doc never creates disk)', () => {
    const ctx = makeCtx();
    const doc = new Y.Doc();
    loadMermaidDoc(doc, DOC, ctx);
    expect(doc.getText('source').toString()).toBe('');
  });

  test('idempotent: does not re-seed when Y.Text is already populated', () => {
    writeMmd(SRC);
    const ctx = makeCtx();
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, 'pre-existing'), 'agent');
    loadMermaidDoc(doc, DOC, ctx);
    expect(doc.getText('source').toString()).toBe('pre-existing');
  });

  test('round-trips store → load through disk unchanged', async () => {
    const ctx = makeCtx();
    const writeDoc = new Y.Doc();
    writeDoc.transact(() => writeDoc.getText('source').insert(0, SRC), 'agent');
    await storeMermaidDoc(writeDoc, DOC, 'agent', ctx);

    const readDoc = new Y.Doc();
    loadMermaidDoc(readDoc, DOC, makeCtx());
    expect(readDoc.getText('source').toString()).toBe(SRC);
  });
});

describe('editable text docs on the verbatim path', () => {
  const TS_DOC = 'src/util.ts';
  const TS_SRC = `export function greet(name: string): string {\n\n  return \`hello $\{name}\`;\n}\n`;

  test('loads a .ts file verbatim into Y.Text("source")', () => {
    mkdirSync(dirname(resolve(contentDir, TS_DOC)), { recursive: true });
    writeFileSync(resolve(contentDir, TS_DOC), TS_SRC, 'utf-8');
    const ctx = makeCtx();
    const doc = new Y.Doc();
    loadMermaidDoc(doc, TS_DOC, ctx);
    expect(doc.getText('source').toString()).toBe(TS_SRC);
  });

  test('stores edited .ts bytes back verbatim (no markdown canonicalization)', async () => {
    const ctx = makeCtx();
    const doc = new Y.Doc();
    doc.transact(() => doc.getText('source').insert(0, TS_SRC), 'agent');
    expect(await storeMermaidDoc(doc, TS_DOC, 'agent', ctx)).toBe('persisted');
    expect(readFileSync(resolve(contentDir, TS_DOC), 'utf-8')).toBe(TS_SRC);
  });
});