Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.2 KB
Raw Blame
/**
 * Wave B managed import of a Markdown file over the single-string ceiling
 * (decision 4, lane 1d).
 *
 * 1. Protects: on a managed brain with `pages.segmented_storage` on, a file
 *    over PAGE_MAX_BYTES inside the canonical root imports through its owner
 *    with an intent that holds a `body_ref` (sha256, bytes) and no content; it
 *    replies at commit with `searchable: 'pending'` (#7106: a new page over 4 MiB
 *    is projected after commit) and seals once the build the owner tracks ends
 *    (the consumer's stop waits for it); the page round-trips byte-identical
 *    from its manifest, `content_hash` equals `contentHash`, and the canonical
 *    file is left untouched. A file within
 *    the ceiling stages its segments in the owner's preparation (the writer
 *    guard admits it) and publishes segmented. Segment GC on a managed brain
 *    collects an abandoned preparation's staged rows under the guard. A file that
 *    changes between admission and preparation publishes nothing; a request_id
 *    replayed with other bytes is `idempotency_conflict`; with the switch off
 *    the refusal is the one before Wave B. With the switch on a file-backed
 *    source takes a page over 24 MiB only by import of its own canonical
 *    file, never by a managed write of new content.
 * 2. Fails when: the journal carries the body, the preparer skips the hash
 *    check, the publication rewrites the canonical file, or the switch-off
 *    refusal changes.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import type { BrainEngine } from '../../src/core/engine.ts';
import type { OperationContext } from '../../src/core/ops/contract.ts';
import { parseMarkdown } from '../../src/core/markdown.ts';
import { PAGE_MAX_BYTES } from '../../src/core/page-size-policy.ts';
import { importManagedFile } from '../../src/core/persistence/import-mutations.ts';
import { prepareManagedImportMutation, type ManagedImportIntent } from '../../src/core/persistence/import-prepare.ts';
import { claimWorktree, getWorktreeBinding } from '../../src/core/persistence/ownership.ts';
import { localHostId, registerLocalWriter } from '../../src/core/persistence/identity.ts';
import { admitWrite, claimNextWrite, sweepSegmentStorage } from '../../src/core/persistence/journal.ts';
import { preparePageAdmission } from '../../src/core/persistence/page-mutations.ts';
import { submissionAuthority } from '../../src/core/persistence/authority.ts';
import { disposePersistenceConsumer } from '../../src/core/persistence/service.ts';
import { sha256 } from '../../src/core/persistence/digest.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { withEnv } from '../helpers/with-env.ts';
import { conversationPage } from '../helpers/conversation-page.ts';
import { reconstructedPage } from '../helpers/segmented-page-checks.ts';

const home = mkdtempSync(join(tmpdir(), 'gbrain-managed-streamed-'));
const engines: BrainEngine[] = [];
let closePostgres: (() => Promise<void>) | undefined;
const env = { GBRAIN_HOME: home, OPENAI_API_KEY: undefined, VOYAGE_API_KEY: undefined, ANTHROPIC_API_KEY: undefined };
const content = conversationPage(PAGE_MAX_BYTES + 2 * 1024 * 1024, { seed: 51 });

beforeAll(async () => {
  const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engines.push(lite);
  if (process.env.DATABASE_URL) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL); engines.push(pg.engine); closePostgres = pg.close; }
}, 120_000);

afterAll(async () => {
  await withEnv(env, async () => {
    for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); }
    await closePostgres?.();
  });
  rmSync(home, { recursive: true, force: true });
});

async function fixture(engine: BrainEngine) {
  const sourceId = `streamed-${randomUUID().slice(0, 8)}`;
  const root = join(home, sourceId);
  mkdirSync(join(root, 'notes'), { recursive: true });
  await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
  await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', [sourceId, root]);
  await registerLocalWriter(engine, 'cli');
  await claimWorktree(engine, sourceId, root);
  await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
  return { sourceId, root };
}

/** A body_ref intent for `file`, admitted and claimed (the owner's next request). */
async function admitted(engine: BrainEngine, sourceId: string, file: string, sourcePath: string, bytes: Buffer, base?: { pageId: number; revision: string }) {
  const binding = (await getWorktreeBinding(engine, sourceId))!;
  const slug = sourcePath.replace(/\.md$/, '');
  const ctx = { engine, remote: false, sourceId } as OperationContext;
  const authority = await submissionAuthority(ctx, 'put_page', sourceId, binding.source_incarnation, slug);
  const hash = sha256(bytes);
  const intent: ManagedImportIntent = { kind: 'managed_file_import', slug, sourcePath, path: sourcePath, inputPath: file, inputHash: hash, targetHash: hash,
    body_ref: { kind: 'input_file', input_path: file, sha256: hash, bytes: bytes.length }, ownerEpoch: String(binding.owner_epoch), noEmbed: true,
    ...(base ? { expected_revision: base.revision } : {}) };
  await admitWrite(engine, { principal: authority.principal, operation: 'put_page', sourceId, sourceIncarnation: binding.source_incarnation,
    slug, pageId: base?.pageId ?? null, requestId: randomUUID(), callerIntent: intent, intent, authority, worktreeId: binding.worktree_id, topologyGeneration: binding.topology_generation });
  return (await claimNextWrite(engine, localHostId()))!;
}

describe('managed import over the single-string ceiling', () => {
  test('switch on: imports by reference, round-trips byte-identical, leaves the canonical file and keeps the intent small', async () => withEnv(env, async () => {
    for (const engine of engines) {
      await disposePersistenceConsumer(engine);
      await engine.setConfig('pages.segmented_storage', 'on');
      const f = await fixture(engine);
      const file = join(f.root, 'notes/huge.md');
      writeFileSync(file, content);
      const before = statSync(file);
      const result = await importManagedFile(engine, file, 'notes/huge.md', { sourceId: f.sourceId, noEmbed: true });
      expect(result).toMatchObject({ slug: 'notes/huge', status: 'imported', searchable: 'pending', projection: 'pending' });
      await disposePersistenceConsumer(engine);
      const parsed = parseMarkdown(content, 'notes/huge.md');
      const { page, body, hash } = await reconstructedPage(engine, 'notes/huge', f.sourceId);
      expect(page.body_storage).toBe('segmented');
      expect(body.compiled_truth).toBe(parsed.compiled_truth);
      expect(body.timeline).toBe(parsed.timeline);
      expect(page.content_hash).toBe(hash);
      expect(page.text_projection_revision).toBe(page.knowledge_revision);
      const after = statSync(file);
      expect(after.mtimeMs).toBe(before.mtimeMs);
      expect(sha256(readFileSync(file))).toBe(sha256(Buffer.from(content)));
      const [request] = await engine.executeRaw<{ intent_bytes: string; intent: Record<string, unknown>; state: string; outcome: Record<string, unknown> }>(
        `SELECT intent_bytes::text AS intent_bytes, intent, state, outcome FROM persistence_requests WHERE source_id=$1 AND slug=$2`, [f.sourceId, 'notes/huge']);
      expect(request!.state).toBe('committed');
      expect(request!.outcome).toMatchObject({ searchable: 'pending', projection: 'pending' });
      expect(Number(request!.intent_bytes)).toBeLessThan(16 * 1024);
      expect(request!.intent.content).toBeUndefined();
      expect(request!.intent.body_ref).toMatchObject({ kind: 'input_file', sha256: sha256(Buffer.from(content)), bytes: Buffer.byteLength(content) });

      const again = await importManagedFile(engine, file, 'notes/huge.md', { sourceId: f.sourceId, noEmbed: true });
      expect(again.status).toBe('skipped');
      expect((await reconstructedPage(engine, 'notes/huge', f.sourceId)).page.body_generation).toBe(page.body_generation);

      // An owner preparation that never publishes leaves staged rows; segment GC collects them under the writer guard.
      await disposePersistenceConsumer(engine);
      const appended = Buffer.from(`${content}\nA closing note.\n`);
      writeFileSync(file, appended);
      const snapshot = (await engine.readPageSnapshot('notes/huge', { sourceId: f.sourceId }))!;
      await prepareManagedImportMutation(engine, await admitted(engine, f.sourceId, file, 'notes/huge.md', appended,
        { pageId: Number(snapshot.page.id), revision: snapshot.revision }), { engine: engine.kind });
      const staged = async () => Number((await engine.executeRaw<{ n: string }>(
        'SELECT count(*)::text AS n FROM page_body_manifest WHERE page_id=$1 AND staging_id IS NOT NULL', [page.id]))[0]!.n);
      expect(await staged()).toBeGreaterThan(0);
      await engine.executeRaw(`UPDATE page_body_stagings SET state='aborted' WHERE page_id=$1 AND state IN ('open','prepared')`, [page.id]);
      await sweepSegmentStorage(engine);
      expect(await staged()).toBe(0);
    }
  }), 600_000);

  test('a file within the ceiling stages segments in the owner\'s preparation and publishes segmented', async () => withEnv(env, async () => {
    for (const engine of engines) {
      await disposePersistenceConsumer(engine);
      await engine.setConfig('pages.segmented_storage', 'on');
      const f = await fixture(engine);
      const file = join(f.root, 'notes/mid.md');
      const mid = conversationPage(3 * 1024 * 1024, { seed: 52 });
      writeFileSync(file, mid);
      expect(await importManagedFile(engine, file, 'notes/mid.md', { sourceId: f.sourceId, noEmbed: true })).toMatchObject({ slug: 'notes/mid', status: 'imported' });
      const parsed = parseMarkdown(mid, 'notes/mid.md');
      const { page, body, hash } = await reconstructedPage(engine, 'notes/mid', f.sourceId);
      expect(page.body_storage).toBe('segmented');
      expect(body.compiled_truth).toBe(parsed.compiled_truth);
      expect(page.content_hash).toBe(hash);
    }
  }), 600_000);

  test('a file changed between admission and preparation is refused', async () => withEnv(env, async () => {
    for (const engine of engines) {
      await disposePersistenceConsumer(engine);
      await engine.setConfig('pages.segmented_storage', 'on');
      const f = await fixture(engine);
      const file = join(f.root, 'notes/edited.md');
      const bytes = Buffer.from(content);
      writeFileSync(file, bytes);
      const row = await admitted(engine, f.sourceId, file, 'notes/edited.md', bytes);
      const edited = Buffer.from(bytes);
      edited.write('Edited', bytes.length - 64, 'utf8');
      writeFileSync(file, edited);
      await expect(prepareManagedImportMutation(engine, row, { engine: engine.kind })).rejects.toMatchObject({ code: 'source_changed' });
      expect(await engine.getPage('notes/edited', { sourceId: f.sourceId })).toBeNull();
    }
  }), 600_000);

  test('a request_id replayed with other bytes is idempotency_conflict', async () => withEnv(env, async () => {
    for (const engine of engines) {
      await disposePersistenceConsumer(engine);
      await engine.setConfig('pages.segmented_storage', 'on');
      const f = await fixture(engine);
      const binding = (await getWorktreeBinding(engine, f.sourceId))!;
      const file = join(f.root, 'notes/replay.md');
      const ctx = { engine, remote: false, sourceId: f.sourceId } as OperationContext;
      const requestId = randomUUID();
      const params = (hash: string) => ({ kind: 'managed_file_import', slug: 'notes/replay', sourcePath: 'notes/replay.md', path: 'notes/replay.md', inputPath: file,
        inputHash: hash, targetHash: hash, body_ref: { kind: 'input_file', input_path: file, sha256: hash, bytes: 1 }, ownerEpoch: String(binding.owner_epoch),
        noEmbed: true, source_id: f.sourceId, request_id: requestId });
      const first = await preparePageAdmission(ctx, { operation: 'put_page', params: params('a'.repeat(64)), managedFileImport: true });
      await admitWrite(engine, first.admission!);
      await expect(preparePageAdmission(ctx, { operation: 'put_page', params: params('b'.repeat(64)), managedFileImport: true }))
        .rejects.toMatchObject({ code: 'idempotency_conflict' });
      expect((await preparePageAdmission(ctx, { operation: 'put_page', params: params('a'.repeat(64)), managedFileImport: true })).prior?.request_id).toBe(requestId);
    }
  }), 120_000);

  test('switch on: a file over 24 MiB imports only from its place in the canonical root; put_page content stays within 24 MiB', async () => withEnv(env, async () => {
    for (const engine of engines) {
      await disposePersistenceConsumer(engine);
      await engine.setConfig('pages.segmented_storage', 'on');
      const f = await fixture(engine);
      const outside = join(home, `${f.sourceId}-outside.md`);
      writeFileSync(outside, content);
      await expect(importManagedFile(engine, outside, 'notes/outside.md', { sourceId: f.sourceId, noEmbed: true }))
        .rejects.toMatchObject({ code: 'invalid_params', message: `File too large (${Buffer.byteLength(content)} bytes, max ${PAGE_MAX_BYTES}): a file over the single-string import limit `
          + `imports only from its place inside the source root. Move it to ${join(f.root, 'notes/outside.md')} and import it there.` });
      expect(await engine.getPage('notes/outside', { sourceId: f.sourceId })).toBeNull();

      const file = join(f.root, 'notes/inside.md');
      writeFileSync(file, content);
      const before = statSync(file);
      expect(await importManagedFile(engine, file, 'notes/inside.md', { sourceId: f.sourceId, noEmbed: true })).toMatchObject({ slug: 'notes/inside', status: 'imported' });
      expect((await reconstructedPage(engine, 'notes/inside', f.sourceId)).page.body_storage).toBe('segmented');
      expect(statSync(file).mtimeMs).toBe(before.mtimeMs);

      const ctx = { engine, remote: false, sourceId: f.sourceId } as OperationContext;
      await expect(preparePageAdmission(ctx, { operation: 'put_page', params: { slug: 'notes/put-huge', content, source_id: f.sourceId, request_id: randomUUID() } }))
        .rejects.toMatchObject({ code: 'request_too_large' });
      expect(await engine.getPage('notes/put-huge', { sourceId: f.sourceId })).toBeNull();
    }
  }), 600_000);

  test('switch off: the refusal is the one before segmented storage', async () => withEnv(env, async () => {
    for (const engine of engines) {
      await disposePersistenceConsumer(engine);
      await engine.setConfig('pages.segmented_storage', 'off');
      const f = await fixture(engine);
      const file = join(f.root, 'notes/off.md');
      writeFileSync(file, content);
      await expect(importManagedFile(engine, file, 'notes/off.md', { sourceId: f.sourceId, noEmbed: true }))
        .rejects.toMatchObject({ code: 'invalid_params', message: `File too large (max ${PAGE_MAX_BYTES} bytes).` });
      expect(await engine.getPage('notes/off', { sourceId: f.sourceId })).toBeNull();
    }
  }), 120_000);
});