Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.0 KB
Raw Blame
/**
 * Wave B PR 2 (spec 7.4): Markdown export of segmented pages.
 *
 * Protects:
 * - Round trip at 2 and 20 MiB, both engines: a page stored inline (switch off)
 *   and the same page stored segmented (switch on) export the same bytes, the
 *   segmented one streamed (exportStreamWork: the pieces add up to the file and
 *   none is over one segment, so a whole-body join fails at 2 MiB); that
 *   file F is canonical (sha256(F) == sha256(serializePageToMarkdown) of the
 *   inline page) and export(import(F)) == F for both storage classes. With a
 *   withdrawn fact both classes export the same struck file.
 * - Streamed ingest (a file over PAGE_MAX_BYTES) reproduces the importer's trim:
 *   leading and trailing body whitespace is dropped, so export(import(F')) has
 *   F's body (above 24 MiB the import stamps content-sanity keys into the
 *   frontmatter, so only the body round-trips byte for byte).
 * - Abort semantics: a segmented page over the payload limit refuses the whole
 *   export (its estimate is `body_bytes`, not NULL); a streamed page extends the
 *   snapshot deadline by its bytes.
 * - Backup: a PGLite backup restores a segmented page that exports the same bytes.
 * - 128 MiB (GBRAIN_TEST_LARGE_EXPORT=1, PGLite): the export of a 128 MiB page
 *   has F's body and its peak memory (VmHWM) stays under 80 MiB above exporting a
 *   2 MiB page. Run it under a cgroup limit as well (the 20 MiB arms import a
 *   fence-heavy page on the string path, which peaks far above 1G on its own):
 *     GBRAIN_TEST_LARGE_EXPORT=1 sudo -E systemd-run --scope -p MemoryMax=1G -p MemorySwapMax=0 \
 *       "$(command -v bun)" test --timeout=1800000 -t '128 MiB' test/e2e/large-pages-export.test.ts
 * Fails when: export reads a segmented body through the throwing getters
 * (page_body_segmented), joins its segments into one string or Buffer, publishes
 * it as one Buffer, keeps the 32 MiB per-page limit, or the stream and the inline
 * serializer disagree on the frontmatter head, timeline separator or overlay.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { closeSync, mkdirSync, mkdtempSync, openSync, readFileSync, realpathSync, rmSync, writeFileSync, writeSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
import type { BrainEngine } from '../../src/core/engine.ts';
import { importFromFile } from '../../src/core/import-file.ts';
import { runExport } from '../../src/commands/export.ts';
import { serializeMarkdown, serializePageToMarkdown } from '../../src/core/markdown.ts';
import { EXPORT_SNAPSHOT_MS, exportStreamWork } from '../../src/core/export-stage.ts';
import { PAGE_MAX_BYTES } from '../../src/core/page-size-policy.ts';
import { readPageBodyPolicy, SEGMENT_MAX_BYTES } from '../../src/core/page-body/limits.ts';
import { createPgliteBackup, restorePgliteBackup } from '../../src/core/backup/snapshot.ts';
import { bigPage } from '../helpers/segment-bodies.ts';
import { conversationBody } from '../helpers/conversation-page.ts';
import { requirePostgresTestDatabase, testBackends } from '../helpers/test-backends.ts';

const REPO = resolve(import.meta.dir, '../..');
const MiB = 1024 * 1024;
const SLUG = 'notes/large-export';
const TIMELINE = '- **2024-01-02** | src — first entry\n  detail line\n- **2024-02-03** | src — second entry';
const sha = (bytes: Buffer | string) => createHash('sha256').update(bytes).digest('hex');
/**
 * The file after its frontmatter block. Import stamps `embed_skip` / `content_flag` (with an `assessed_at` time) into
 * the frontmatter of a page over content_sanity.bytes_block (24 MiB), so above it only the body round-trips exactly.
 */
const bodyOf = (file: Buffer) => file.subarray(file.indexOf('\n---\n\n', 4) + 6);
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'large-pages-export-')));
afterAll(() => rmSync(dir, { recursive: true, force: true }));

function seeded(seed: number): () => number {
  let state = seed;
  return () => { state = (state * 1103515245 + 12345) & 0x7fffffff; return state / 0x7fffffff; };
}

/** A page file with frontmatter, facts, takes, keys, code and a timeline; not yet in canonical form. */
function pageFile(bytes: number, seed: number): string {
  return `---\ntype: note\ntitle: Large export\n---\n\n${bigPage(seeded(seed), bytes)}\n\n<!-- timeline -->\n\n${TIMELINE}\n`;
}

async function quietly<T>(fn: () => Promise<T>): Promise<T> {
  const { log, error } = console;
  const exit = process.exit;
  const messages: string[] = [];
  console.log = console.error = (...args: unknown[]) => { messages.push(args.join(' ')); };
  process.exit = ((code: number) => { throw new Error(`EXIT:${code} ${messages.join('\n')}`); }) as never;
  try { return await fn(); } finally { console.log = log; console.error = error; process.exit = exit; }
}

let exports = 0;
async function exportPage(engine: BrainEngine, args: string[] = []): Promise<Buffer> {
  const out = join(dir, `out-${++exports}`);
  Object.assign(exportStreamWork, { files: 0, pieces: 0, bytes: 0, largestPiece: 0 });
  await quietly(() => runExport(engine, ['--dir', out, ...args]));
  expect(readFileSync(join(out, '.gbrain-export-status'), 'utf8')).toEndWith('COMPLETE\n');
  return readFileSync(join(out, `${SLUG}.md`));
}

/** The export streamed the segmented page as one file: its pieces add up to the file and none is over one segment. */
function expectStreamed(file: Buffer): void {
  expect(exportStreamWork.files).toBe(1);
  expect(exportStreamWork.bytes).toBe(file.length);
  expect(exportStreamWork.largestPiece).toBeLessThanOrEqual(SEGMENT_MAX_BYTES);
}

async function importAs(engine: BrainEngine, storage: 'on' | 'off', file: string): Promise<string> {
  await engine.executeRaw('DELETE FROM pages');
  await engine.setConfig('pages.segmented_storage', storage);
  const bodyPolicy = await readPageBodyPolicy(engine);
  const result = await quietly(() => importFromFile(engine, file, `${SLUG}.md`, { noEmbed: true, bodyPolicy }));
  expect(result.error ?? null).toBeNull();
  expect(result.status).toBe("imported");
  const [row] = await engine.executeRaw<{ body_storage: string }>('SELECT body_storage FROM pages WHERE slug=$1', [SLUG]);
  return row!.body_storage;
}

for (const backend of testBackends()) describe(`large-page export: ${backend}`, () => {
  let engine: BrainEngine;
  beforeAll(async () => {
    engine = backend === 'postgres' ? new PostgresEngine() : new PGLiteEngine();
    await engine.connect(backend === 'postgres' ? { database_url: requirePostgresTestDatabase() } : {});
    await engine.initSchema();
  }, 120_000);
  afterAll(async () => {
    await engine.executeRaw("DELETE FROM fact_withdrawals WHERE source_id='default'");
    await engine.executeRaw('DELETE FROM pages');
    await engine.setConfig('pages.segmented_storage', 'off');
    await engine.disconnect();
  });

  test.each([2, 20])('%i MiB: inline and segmented export the same canonical file, and export(import(F)) == F', async size => {
    const raw = join(dir, `${backend}-${size}-raw.md`);
    writeFileSync(raw, pageFile(size * MiB, size));
    expect(await importAs(engine, 'off', raw)).toBe('inline');
    const canonical = await exportPage(engine);
    const page = (await engine.getPage(SLUG))!;
    expect(sha(canonical)).toBe(sha(serializePageToMarkdown(page, await engine.getTags(SLUG))));
    const file = join(dir, `${backend}-${size}-canonical.md`);
    writeFileSync(file, canonical);

    expect(await importAs(engine, 'on', file)).toBe('segmented');
    const segmented = await exportPage(engine);
    expectStreamed(segmented);
    expect(sha(segmented)).toBe(sha(canonical));
    expect(await importAs(engine, 'off', file)).toBe('inline');
    expect(sha(await exportPage(engine))).toBe(sha(canonical));

    await engine.executeRaw("INSERT INTO fact_withdrawals (source_id, visibility, fact_hash) VALUES ('default', 'world', gbrain_fact_fingerprint('likes tea'))");
    try {
      const struck = await exportPage(engine);
      expect(struck.equals(canonical)).toBe(false);
      expect(await importAs(engine, 'on', file)).toBe('segmented');
      const segmentedStruck = await exportPage(engine);
      expectStreamed(segmentedStruck);
      expect(sha(segmentedStruck)).toBe(sha(struck));
    } finally { await engine.executeRaw("DELETE FROM fact_withdrawals WHERE source_id='default'"); }
  }, 600_000);

  test('a segmented page over the payload limit refuses the whole export; a streamed page extends the snapshot deadline', async () => {
    const file = join(dir, `${backend}-limit.md`);
    writeFileSync(file, pageFile(2 * MiB, 3));
    expect(await importAs(engine, 'on', file)).toBe('segmented');
    const [{ body_bytes: stored }] = await engine.executeRaw<{ body_bytes: number }>('SELECT body_bytes FROM pages WHERE slug=$1', [SLUG]);
    await engine.executeRaw('UPDATE pages SET body_bytes=$2 WHERE slug=$1', [SLUG, 300 * MiB]);
    try {
      await expect(exportPage(engine)).rejects.toThrow('Export page payload limit exceeded');
    } finally { await engine.executeRaw('UPDATE pages SET body_bytes=$2 WHERE slug=$1', [SLUG, stored]); }

    const query = engine.executeRaw, now = Date.now;
    engine.executeRaw = async function (sql, params, options) {
      if (sql.includes('AS export_tags')) {
        const late = now() + EXPORT_SNAPSHOT_MS + 1000;
        Date.now = () => late;
      }
      return query.call(this, sql, params, options) as never;
    };
    try {
      const extended = await exportPage(engine);
      expectStreamed(extended);
      expect(extended.length).toBeGreaterThan(2 * MiB);
    } finally { engine.executeRaw = query; Date.now = now; }
  }, 300_000);

  test.skipIf(backend !== 'pglite')('streamed ingest drops leading and trailing body whitespace like parseMarkdown, so export(import(F + whitespace)) has the body of F', async () => {
    const head = serializeMarkdown({}, '', '', { type: 'note', title: 'Large export', tags: [] }).slice(0, -1);
    const body = conversationBody(PAGE_MAX_BYTES + MiB, { seed: 5 });
    const canonical = `${head}${body}\n\n<!-- timeline -->\n\n${TIMELINE}\n`;
    const padded = join(dir, 'padded.md');
    writeFileSync(padded, `${head}\n  \n\t\n${body} \t\n\n  \n\n<!-- timeline -->\n\n${TIMELINE}\n\n \n`);
    expect(await importAs(engine, 'on', padded)).toBe('segmented');
    const exported = await exportPage(engine);
    expectStreamed(exported);
    expect(sha(bodyOf(exported))).toBe(sha(bodyOf(Buffer.from(canonical))));
  }, 600_000);
});

describe.skipIf(testBackends().includes('postgres') && !testBackends().includes('pglite'))('PGLite backup of a segmented page', () => {
  test('restores the page so it exports the same bytes', async () => {
    const root = join(dir, 'backup-root');
    mkdirSync(join(root, '.gbrain'), { recursive: true });
    const database = join(root, '.gbrain', 'brain.pglite');
    writeFileSync(join(root, '.gbrain', 'config.json'), JSON.stringify({ engine: 'pglite', database_path: database }));
    const file = join(dir, 'backup-page.md');
    writeFileSync(file, pageFile(2 * MiB, 11));
    const withBrain = async <T>(path: string, fn: (engine: PGLiteEngine) => Promise<T>): Promise<T> => {
      const engine = new PGLiteEngine();
      await engine.connect({ engine: 'pglite', database_path: path });
      try { return await fn(engine); } finally { await engine.disconnect(); }
    };
    const before = await withBrain(database, async engine => {
      await engine.initSchema();
      expect(await importAs(engine, 'on', file)).toBe('segmented');
      const exported = await exportPage(engine);
      expectStreamed(exported);
      return exported;
    });
    const archive = join(dir, 'segmented.gbrain-backup');
    await createPgliteBackup({ root, output: archive });
    const restored = join(dir, 'restored-root');
    await restorePgliteBackup({ archive, into: restored });
    const after = await withBrain(join(restored, '.gbrain', 'brain.pglite'), async engine => {
      const [row] = await engine.executeRaw<{ body_storage: string }>('SELECT body_storage FROM pages WHERE slug=$1', [SLUG]);
      expect(row?.body_storage).toBe('segmented');
      const exported = await exportPage(engine);
      expectStreamed(exported);
      return exported;
    });
    expect(sha(after)).toBe(sha(before));
  }, 600_000);
});

const CHILD = `
import { readFileSync, writeFileSync } from 'node:fs';
import { PGLiteEngine } from ${JSON.stringify(join(REPO, 'src/core/pglite-engine.ts'))};
import { importFromFile } from ${JSON.stringify(join(REPO, 'src/core/import-file.ts'))};
import { readPageBodyPolicy } from ${JSON.stringify(join(REPO, 'src/core/page-body/limits.ts'))};
import { runExport } from ${JSON.stringify(join(REPO, 'src/commands/export.ts'))};
const [mode, brain, file, out] = process.argv.slice(-4);
const engine = new PGLiteEngine();
await engine.connect({ engine: 'pglite', database_path: brain });
await engine.initSchema();
let result = null;
if (mode === 'import') {
  await engine.setConfig('pages.segmented_storage', 'on');
  result = await importFromFile(engine, file, ${JSON.stringify(`${SLUG}.md`)}, { noEmbed: true, bodyPolicy: await readPageBodyPolicy(engine) });
  result = { status: result.status, error: result.error ?? null };
} else {
  console.log = () => {};
  writeFileSync('/proc/self/clear_refs', '5');
  await runExport(engine, ['--dir', out]);
}
const peak = Number(/^VmHWM:\\s+(\\d+) kB$/m.exec(readFileSync('/proc/self/status', 'utf8'))[1]) / 1024;
await engine.disconnect();
process.stdout.write(JSON.stringify({ result, peak }) + '\\n');
`;

/** A canonical page file whose body is at most `bytes`, written in pieces; returns the sha256 of its body (bodyOf). */
function writeCanonical(path: string, bytes: number): string {
  const hash = createHash('sha256');
  const fd = openSync(path, 'w');
  const write = (text: string) => { hash.update(text); writeSync(fd, text); };
  try {
    writeSync(fd, serializeMarkdown({}, '', '', { type: 'note', title: 'Large export', tags: [] }).slice(0, -1));
    for (let at = 0, part = 0; ; part++) {
      const piece = (part ? '\n\n' : '') + conversationBody(Math.min(4 * MiB, bytes - at - 16), { seed: part + 1 });
      if (at + Buffer.byteLength(piece) > bytes) break;
      write(piece);
      at += Buffer.byteLength(piece);
    }
    write(`\n\n<!-- timeline -->\n\n${TIMELINE}\n`);
  } finally { closeSync(fd); }
  return hash.digest('hex');
}

function child(mode: 'import' | 'export', brain: string, file: string, out: string): { result: { status: string; error: string | null } | null; peak: number } {
  const script = join(dir, 'child.ts');
  writeFileSync(script, CHILD);
  const run = spawnSync(process.execPath, [script, mode, brain, file, out], { cwd: REPO, encoding: 'utf8', timeout: 1_500_000, maxBuffer: 16 * MiB });
  if (run.status !== 0) throw new Error(`${mode} child failed (${run.status} ${run.signal}): ${run.stderr.slice(-4000)}`);
  return JSON.parse(run.stdout.trim().split('\n').pop()!);
}

describe.skipIf(process.env.GBRAIN_TEST_LARGE_EXPORT !== '1' || process.platform !== 'linux')('128 MiB export memory (PGLite)', () => {
  test('exports a 128 MiB page with the body of F and peak memory independent of the page size', () => {
    const measure = (name: string, bytes: number) => {
      const file = join(dir, `${name}.md`);
      const expected = writeCanonical(file, bytes);
      const brain = join(dir, `${name}.pglite`);
      expect(child('import', brain, file, '').result).toEqual({ status: 'imported', error: null });
      rmSync(file);
      const out = join(dir, `${name}-out`);
      const { peak } = child('export', brain, file, out);
      expect(sha(bodyOf(readFileSync(join(out, `${SLUG}.md`))))).toBe(expected);
      rmSync(out, { recursive: true, force: true });
      rmSync(brain, { recursive: true, force: true });
      return peak;
    };
    const small = measure('small', 2 * MiB);
    const large = measure('large', 128 * MiB - 64 * 1024);
    console.log(JSON.stringify({ smallPeakMiB: Math.round(small), largePeakMiB: Math.round(large) }));
    expect(large - small).toBeLessThan(80);
  }, 1_800_000);
});