Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.5 KB
Raw Blame
/**
 * Cathedral 5 (WI4) — corpus-segments unit contract: content-addressed
 * idempotency, hash-keyed ledger, boundary slicing/splitting, exact-set
 * coverage (duplicate-boundary can't mask a missed one), crash-order
 * fallback, GC of orphaned sidecars + aged ledgers.
 */
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import {
  appendSegmentLedger,
  readOpenclawBoundaryTail,
  bankCompactSegment,
  coverageComplete,
  decideCorpusMode,
  gcCorpusArtifacts,
  ledgerFileName,
  parseSegmentFileName,
  readSegmentLedger,
  renderSegmentText,
  segmentFileName,
  segmentHash,
  sliceBoundaryWindow,
  splitByBoundaries,
  writeSegment,
} from '../src/core/context/corpus-segments.ts';
import type { WindowTurn } from '../src/core/context/entity-salience.ts';

let dir: string;
beforeEach(() => {
  dir = mkdtempSync(join(tmpdir(), 'gb-seg-'));
});
afterEach(() => {
  rmSync(dir, { recursive: true, force: true });
});

const t = (text: string, role: 'user' | 'assistant' = 'user'): WindowTurn => ({ role, text });

describe('content addressing + ledger', () => {
  test('same text ⇒ same name ⇒ existed=true, no rewrite; ledger append is idempotent', () => {
    const w1 = writeSegment(dir, 's', 'hello world');
    expect(w1.existed).toBe(false);
    expect(w1.file.endsWith(segmentFileName('s', segmentHash('hello world')))).toBe(true);
    const k1 = appendSegmentLedger(dir, 's', w1.hash);
    expect(k1).toBe(1);

    const w2 = writeSegment(dir, 's', 'hello world');
    expect(w2.existed).toBe(true);
    expect(w2.file).toBe(w1.file);
    expect(appendSegmentLedger(dir, 's', w2.hash)).toBe(1); // same ordinal, no dup entry
    expect(readSegmentLedger(dir, 's')).toHaveLength(1);

    const w3 = writeSegment(dir, 's', 'different content');
    expect(appendSegmentLedger(dir, 's', w3.hash)).toBe(2);
    expect(readSegmentLedger(dir, 's')).toHaveLength(2);
  });

  test('unreadable/malformed ledger reads as absent (fail-open [])', () => {
    writeFileSync(join(dir, ledgerFileName('s')), 'not json at all');
    expect(readSegmentLedger(dir, 's')).toEqual([]);
    writeFileSync(join(dir, ledgerFileName('s2')), JSON.stringify({ nope: true }));
    expect(readSegmentLedger(dir, 's2')).toEqual([]);
  });
});

describe('boundary slicing', () => {
  const turns = [t('a'), t('b'), t('c'), t('d'), t('e')];

  test('sliceBoundaryWindow: since last boundary; no boundary ⇒ all; maxTurns caps from the newest end', () => {
    expect(sliceBoundaryWindow(turns, [])).toEqual(turns);
    expect(sliceBoundaryWindow(turns, [2])).toEqual([t('c'), t('d'), t('e')]);
    expect(sliceBoundaryWindow(turns, [1, 3])).toEqual([t('d'), t('e')]);
    expect(sliceBoundaryWindow(turns, [], { maxTurns: 2 })).toEqual([t('d'), t('e')]);
  });

  test('splitByBoundaries: windows partition the pre-boundary turns; remainder is the tail', () => {
    const { windows, remainder } = splitByBoundaries(turns, [2, 4]);
    expect(windows).toEqual([[t('a'), t('b')], [t('c'), t('d')]]);
    expect(remainder).toEqual([t('e')]);
  });
});

describe('exact-set coverage', () => {
  async function bankWindow(win: WindowTurn[]): Promise<string> {
    const rendered = await renderSegmentText(win);
    expect(rendered).not.toBeNull();
    const w = writeSegment(dir, 's', rendered!.text);
    appendSegmentLedger(dir, 's', w.hash);
    return w.hash;
  }

  test('covered when every non-empty window hash is banked; remainder-only decision follows', async () => {
    const turns = [t('w1-a'), t('w1-b'), t('w2-a'), t('rem-a')];
    await bankWindow(turns.slice(0, 2)); // window 1
    await bankWindow(turns.slice(2, 3)); // window 2
    expect(await coverageComplete(dir, 's', turns, [2, 3])).toBe(true);
    const d = await decideCorpusMode(dir, 's', turns, [2, 3]);
    expect(d.mode).toBe('remainder');
    expect(d.turns).toEqual([t('rem-a')]);
  });

  test('a missed window fails coverage even when a duplicated boundary keeps the COUNT equal', async () => {
    const turns = [t('w1-a'), t('w2-a')];
    // Bank window 1 TWICE conceptually (same content = one ledger entry) plus
    // an unrelated hash so ledger COUNT (2) equals the boundary count (2) —
    // but window 2's hash is absent: exact-set must fail.
    await bankWindow(turns.slice(0, 1));
    appendSegmentLedger(dir, 's', 'deadbeef0000');
    expect(readSegmentLedger(dir, 's')).toHaveLength(2);
    expect(await coverageComplete(dir, 's', turns, [1, 2])).toBe(false);
    expect((await decideCorpusMode(dir, 's', turns, [1, 2])).mode).toBe('full_fallback');
  });

  test('empty windows are covered by construction; empty remainder ⇒ skip_covered', async () => {
    const turns = [t('w1-a')];
    await bankWindow(turns);
    // Boundary at 1 then a duplicate boundary at 1: second window is empty.
    expect(await coverageComplete(dir, 's', turns, [1, 1])).toBe(true);
    expect((await decideCorpusMode(dir, 's', turns, [1, 1])).mode).toBe('skip_covered');
  });

  test('no boundaries ⇒ mode full (today behavior); empty ledger with boundaries ⇒ full_fallback', async () => {
    const turns = [t('a')];
    expect((await decideCorpusMode(dir, 's', turns, [])).mode).toBe('full');
    expect((await decideCorpusMode(dir, 's-none', turns, [1])).mode).toBe('full_fallback');
  });
});

describe('bankCompactSegment (compact-time step)', () => {
  test('banks the since-last-boundary window; identical retry is segment_dup with the same ordinal', async () => {
    const turns = [t('old-a'), t('new-a'), t('new-b')];
    const budget = { remainingMs: () => 5000, minScanMs: 600, minWriteMs: 300 };
    const r1 = await bankCompactSegment(dir, 's', turns, [1], budget);
    expect(r1.segment).toBe('segment_banked');
    expect(r1.ordinal).toBe(1);
    expect(existsSync(join(dir, r1.flushCorpusFile!))).toBe(true);
    const body = readFileSync(join(dir, r1.flushCorpusFile!), 'utf8');
    expect(body).toContain('new-a');
    expect(body).not.toContain('old-a');

    const r2 = await bankCompactSegment(dir, 's', turns, [1], budget);
    expect(r2.segment).toBe('segment_dup');
    expect(r2.ordinal).toBe(1);
    expect(readSegmentLedger(dir, 's')).toHaveLength(1);
  });

  test('empty window ⇒ empty_window, nothing written; tight budget ⇒ deadline_scan, nothing written', async () => {
    const turns = [t('a')];
    const empty = await bankCompactSegment(dir, 's', turns, [1], { remainingMs: () => 5000, minScanMs: 600, minWriteMs: 300 });
    expect(empty.segment).toBe('empty_window');
    const tight = await bankCompactSegment(dir, 's', turns, [], { remainingMs: () => 100, minScanMs: 600, minWriteMs: 300 });
    expect(tight.segment).toBe('deadline_scan');
    expect(readdirSync(dir).filter((f) => f.endsWith('.txt'))).toEqual([]);
    expect(readSegmentLedger(dir, 's')).toEqual([]);
  });
});

describe('gcCorpusArtifacts', () => {
  test('removes orphaned sidecars (base .txt gone) and aged ledgers; keeps live pairs', () => {
    // Live pair: base + sidecar both stay.
    writeFileSync(join(dir, 'live.txt'), 'x');
    writeFileSync(join(dir, 'live.txt.ingested'), '');
    // Orphan sidecars: base missing.
    writeFileSync(join(dir, 'gone.txt.ingested'), '');
    writeFileSync(join(dir, 'gone.txt.in-progress'), '');
    // Fresh ledger stays; aged ledger goes.
    writeFileSync(join(dir, 'fresh.ledger.json'), '[]');
    writeFileSync(join(dir, 'old.ledger.json'), '[]');
    const past = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
    utimesSync(join(dir, 'old.ledger.json'), past, past);

    gcCorpusArtifacts(dir, 30 * 24 * 60 * 60 * 1000, ['.ingested', '.in-progress']);

    const names = readdirSync(dir).sort();
    expect(names).toContain('live.txt');
    expect(names).toContain('live.txt.ingested');
    expect(names).toContain('fresh.ledger.json');
    expect(names).not.toContain('gone.txt.ingested');
    expect(names).not.toContain('gone.txt.in-progress');
    expect(names).not.toContain('old.ledger.json');
  });

  test('reaps orphaned receipts and aged tmp files from crashed writers (adversarial review)', () => {
    // Orphaned receipt (base .txt gone) — previously lived forever.
    writeFileSync(join(dir, 'gone.txt.receipt.json'), '{}');
    // Live receipt (base present, harvest mid-retry) stays.
    writeFileSync(join(dir, 'live.txt'), 'x');
    writeFileSync(join(dir, 'live.txt.receipt.json'), '{}');
    // Aged tmp from a crashed atomic writer goes; fresh tmp stays.
    writeFileSync(join(dir, 's.ledger.json.tmp-12345'), '[]');
    const past = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000);
    utimesSync(join(dir, 's.ledger.json.tmp-12345'), past, past);
    writeFileSync(join(dir, 's.txt.tmp-99999'), 'x');

    gcCorpusArtifacts(dir, 30 * 24 * 60 * 60 * 1000, ['.ingested', '.in-progress', '.receipt.json']);

    const names = readdirSync(dir).sort();
    expect(names).not.toContain('gone.txt.receipt.json');
    expect(names).toContain('live.txt.receipt.json');
    expect(names).not.toContain('s.ledger.json.tmp-12345');
    expect(names).toContain('s.txt.tmp-99999');
  });
});

describe('parseSegmentFileName (sweep-lane link publish)', () => {
  test('round-trips segmentFileName, accepts legacy 12-hex, rejects non-segments', () => {
    const hash = segmentHash('some window text');
    expect(hash).toHaveLength(24); // 96-bit width (adversarial review)
    const name = segmentFileName('sess-a', hash);
    expect(parseSegmentFileName(name)).toEqual({ sessionId: 'sess-a', hash });
    // Pre-widening 12-hex segments still parse during upgrade.
    expect(parseSegmentFileName('s.seg-abc123def456.txt')).toEqual({
      sessionId: 's', hash: 'abc123def456',
    });
    expect(parseSegmentFileName('plain-corpus-file.txt')).toBeNull();
    expect(parseSegmentFileName('s.ledger.json')).toBeNull();
    expect(parseSegmentFileName('s.seg-XYZ.txt')).toBeNull();
  });
});

describe('pre-landing review pins', () => {
  test('filename builders enforce the safe charset structurally (traversal-shaped ids cannot escape the dir)', () => {
    const w = writeSegment(dir, '../../etc/evil', 'contained content');
    // Separators are stripped, so the name can never traverse — the file
    // lands INSIDE dir (a literal '..' SUBSTRING without separators is inert).
    expect(w.file.startsWith(dir + '/')).toBe(true);
    expect(w.file.slice(dir.length + 1)).not.toContain('/');
    expect(existsSync(w.file)).toBe(true);
    expect(ledgerFileName('../../x')).not.toContain('/');
    const hostile = segmentFileName('a/b\\c', 'ff00ff00ff00');
    expect(hostile).not.toContain('/');
    expect(hostile).not.toContain('\\');
  });

  test('bankCompactSegment deadline_write: scan budget passes, write budget does not — nothing written', async () => {
    const turns = [t('window that renders fine')];
    let calls = 0;
    // First check (minScanMs) sees plenty; second check (minWriteMs) sees a
    // drained budget — the render happened, the write must not.
    const remainingMs = () => (calls++ === 0 ? 5000 : 100);
    const r = await bankCompactSegment(dir, 's-dw', turns, [], { remainingMs, minScanMs: 600, minWriteMs: 300 });
    expect(r.segment).toBe('deadline_write');
    expect(readdirSync(dir).filter((f) => f.startsWith('s-dw'))).toEqual([]);
  });

  test('readOpenclawBoundaryTail tail-reads over maxBytes: partial first line dropped, boundary positions window-relative', () => {
    const sessionLine = JSON.stringify({ type: 'session', id: 's', cwd: '/w', timestamp: 't' });
    const msgLine = (text: string) =>
      JSON.stringify({ type: 'message', timestamp: 't', message: { role: 'user', content: [{ type: 'text', text }] } });
    const boundaryLine = JSON.stringify({ type: 'compaction', timestamp: 't' });
    const pad = 'x'.repeat(400);
    const lines = [sessionLine];
    for (let i = 0; i < 20; i++) lines.push(msgLine(`early-${i}-${pad}`));
    lines.push(boundaryLine);
    lines.push(msgLine('tail-a'), msgLine('tail-b'));
    const p = join(dir, 'oc-tail.jsonl');
    writeFileSync(p, lines.join('\n') + '\n');
    const full = statSync(p).size;
    // Cap the read to roughly the last few lines: the cut lands mid-line and
    // that partial first line must be silently dropped, not corrupt a turn.
    const parsed = readOpenclawBoundaryTail(p, { maxBytes: 1200 });
    expect(parsed).not.toBeNull();
    expect(full).toBeGreaterThan(1200);
    const texts = parsed!.turns.map((x) => x.text);
    expect(texts).toContain('tail-a');
    expect(texts).toContain('tail-b');
    for (const x of texts) expect(x.startsWith('early-0')).toBe(false); // scrolled out
    // Boundary position is relative to THIS window's turns array.
    expect(parsed!.boundaryTurnIndexes.length).toBeGreaterThanOrEqual(1);
    const b = parsed!.boundaryTurnIndexes.at(-1)!;
    expect(parsed!.turns.slice(b).map((x) => x.text)).toEqual(['tail-a', 'tail-b']);
  });
});

describe('decideCorpusMode reports where the written span begins', () => {
  /** Anything else derived from the transcript — tool calls, hashes — must
   * use the same origin as the corpus, or the receipt describes two spans. */
  test('full and full_fallback modes start at zero', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gbrain-start-'));
    try {
      const turns = [{ role: 'user', text: 'a' }, { role: 'assistant', text: 'b' }] as never;
      const d = await decideCorpusMode(dir, 'sess', turns, []);
      expect(d.mode).toBe('full');
      expect(d.startTurnIndex).toBe(0);
    } finally { rmSync(dir, { recursive: true, force: true }); }
  });

  test('an uncovered boundary falls back to full, still from zero', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'gbrain-start-'));
    try {
      const turns = [{ role: 'user', text: 'a' }, { role: 'assistant', text: 'b' }, { role: 'user', text: 'c' }] as never;
      const d = await decideCorpusMode(dir, 'sess', turns, [1]);
      expect(d.mode).toBe('full_fallback');
      expect(d.startTurnIndex).toBe(0);
      // the whole window is written, so nothing may be filtered out of it
      expect(d.turns.length).toBe(3);
    } finally { rmSync(dir, { recursive: true, force: true }); }
  });
});

describe('readOpenclawBoundaryTail — tool-call extraction (memorable, name-only v1)', () => {
  test('fixture toolCall block extracted with input:null, stamped at its own turn slot', () => {
    const parsed = readOpenclawBoundaryTail('test/fixtures/transcripts/agent-session.jsonl');
    expect(parsed).not.toBeNull();
    // The fixture's one toolCall rides the assistant message at turn index 1
    // (m-1 user = 0, m-2 assistant = 1) — stamped BEFORE its turn is pushed.
    expect(parsed!.toolCalls).toEqual([{ name: 'search_brain', input: null }]);
    expect(parsed!.toolCallTurnIndexes).toEqual([1]);
    expect(parsed!.toolCalls.length).toBe(parsed!.toolCallTurnIndexes.length);
  });

  test('a placeholder-only message (toolCall, no text) keeps its calls; stamps stay non-decreasing', () => {
    const lines = [
      JSON.stringify({ type: 'session', id: 's-tc', cwd: '/w', timestamp: 't0', version: 3 }),
      JSON.stringify({ type: 'message', message: { role: 'user', content: [{ type: 'text', text: 'run the tests' }] } }),
      // No text at all — today's mapper skips the TURN but the call is real work.
      JSON.stringify({ type: 'message', message: { role: 'assistant', content: [{ type: 'toolCall', id: 'tc-a', name: 'exec' }] } }),
      JSON.stringify({ type: 'compaction', summary: 'x' }),
      JSON.stringify({ type: 'message', message: { role: 'assistant', content: [{ type: 'text', text: 'done' }, { type: 'toolCall', id: 'tc-b', name: 'read_file' }] } }),
    ];
    const p = join(dir, 'tc.jsonl');
    writeFileSync(p, lines.join('\n') + '\n');
    const parsed = readOpenclawBoundaryTail(p);
    expect(parsed).not.toBeNull();
    expect(parsed!.turns.map((x) => x.text)).toEqual(['run the tests', 'done']);
    expect(parsed!.toolCalls.map((c) => c.name)).toEqual(['exec', 'read_file']);
    // exec sits between turn 0 and turn 1 (stamped 1 — the slot it precedes);
    // read_file rides the turn at index 1. Non-decreasing, parallel arrays.
    expect(parsed!.toolCallTurnIndexes).toEqual([1, 1]);
    const idx = parsed!.toolCallTurnIndexes;
    for (let i = 1; i < idx.length; i++) expect(idx[i]!).toBeGreaterThanOrEqual(idx[i - 1]!);
    // Boundary position unchanged by call extraction.
    expect(parsed!.boundaryTurnIndexes).toEqual([1]);
    // Filtering at the boundary keeps only the post-boundary call — the
    // span rule the receipt writer applies.
    const startTurnIndex = parsed!.boundaryTurnIndexes[0]!;
    const span = parsed!.toolCalls.filter((_c, i) => (parsed!.toolCallTurnIndexes[i] ?? 0) >= startTurnIndex);
    expect(span.map((c) => c.name)).toEqual(['exec', 'read_file']);
  });

  test('no speculative args keys: an args-bearing block still extracts input:null (name-only v1)', () => {
    const lines = [
      JSON.stringify({ type: 'session', id: 's-args', cwd: '/w', timestamp: 't0', version: 3 }),
      JSON.stringify({
        type: 'message',
        message: {
          role: 'assistant',
          content: [
            { type: 'text', text: 'ok' },
            // Whatever field the real format uses for args, v1 must NOT guess
            // it into an egress payload.
            { type: 'toolCall', id: 'tc-x', name: 'exec', arguments: { command: 'rm -rf /' }, args: { c: 1 }, input: { c: 2 } },
          ],
        },
      }),
    ];
    const p = join(dir, 'tc-args.jsonl');
    writeFileSync(p, lines.join('\n') + '\n');
    const parsed = readOpenclawBoundaryTail(p);
    expect(parsed!.toolCalls).toEqual([{ name: 'exec', input: null }]);
  });
});