Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.7 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { mkdtempSync, realpathSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { PostgresEngine } from '../src/core/postgres-engine.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { runExport } from '../src/commands/export.ts';
import { parseFactsFence, renderFactsTable } from '../src/core/facts-fence.ts';
import { ExportStage } from '../src/core/export-stage.ts';
import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';

const PAGES = Number(process.env.GBRAIN_TEST_EXPORT_SCALE_PAGES ?? 100001);
if (!Number.isInteger(PAGES) || PAGES < 1001 || PAGES % 100 !== 1 || PAGES > 999999) {
  throw new Error(`GBRAIN_TEST_EXPORT_SCALE_PAGES must be a multiple of 100 plus 1, between 1001 and 999901, got ${process.env.GBRAIN_TEST_EXPORT_SCALE_PAGES}`);
}
const FENCED = (PAGES - 1) / 100;
const FOLDER_POLICY_READS = 1;
const NONEMPTY_BATCHES = Math.ceil(PAGES / 256);
const pageFile = (n: number) => `scale/p${String(n).padStart(6, '0')}.md`;

for (const backend of testBackends()) describe(`export scale: ${backend}`, () => {
  let engine: BrainEngine;
  beforeAll(async () => {
    engine = backend === 'postgres' ? new PostgresEngine() : new PGLiteEngine();
    await engine.connect(backend === 'postgres' ? { database_url: requirePostgresTestDatabase() } : {});
    await engine.initSchema();
  }, 60000);
  afterAll(async () => { await engine.disconnect(); });
  test(`exports all ${PAGES} pages with bounded keyset enumeration`, async () => {
    const dir = realpathSync(mkdtempSync(join(tmpdir(), 'export-scale-')));
    const started = performance.now();
    let batches = 0, largestBatch = 0, ledgerReads = 0, normalizations = 0;
    let executeRawQueries = 0, rawQueries = 0, transactions = 0, stagingPayloadBytes = 0, stagingManifestBytes = 0;
    const real = engine.executeRaw, raw = engine.getRawData, transaction = engine.transaction, close = ExportStage.prototype.close;
    try {
      await engine.executeRaw('DELETE FROM pages');
      await engine.executeRaw("DELETE FROM fact_withdrawals WHERE source_id='default'");
      const fenced = renderFactsTable([{ rowNum: 1, claim: 'Synthetic withdrawn claim', kind: 'fact', confidence: 1,
        visibility: 'world', notability: 'medium', active: true, context: '' }]);
      await engine.executeRaw('ALTER TABLE pages DISABLE TRIGGER bump_page_generation_trg');
      try {
        await engine.executeRaw(`INSERT INTO pages (source_id,slug,type,title,compiled_truth,timeline,frontmatter,generation)
          SELECT 'default', 'scale/p' || lpad(n::text,6,'0'), 'note', 'Synthetic ' || n,
            'Body ' || n || CASE WHEN n%100=0 THEN chr(10)||$1 ELSE '' END, '', '{}'::jsonb, n
          FROM generate_series(1,$2::int) n`, [fenced, PAGES]);
      } finally { await engine.executeRaw('ALTER TABLE pages ENABLE TRIGGER bump_page_generation_trg'); }
      await engine.executeRaw(`INSERT INTO fact_withdrawals(source_id,visibility,fact_hash)
        SELECT 'default','world',gbrain_fact_fingerprint(CASE WHEN n=1 THEN 'Synthetic withdrawn claim'
          ELSE 'Unrelated synthetic claim '||n END) FROM generate_series(1,1000) n`);
      await engine.executeRaw("INSERT INTO tags(page_id,tag) SELECT id,'scale-tag' FROM pages WHERE right(slug,2)='00'");
      await engine.executeRaw("INSERT INTO raw_data(page_id,source,data) SELECT id,'feed',jsonb_build_object('slug',slug) FROM pages WHERE right(slug,2)='00'");
      const seededMs = performance.now() - started;
      console.log(JSON.stringify({ backend, seeded: PAGES, seededMs: Math.round(seededMs) }));
      engine.executeRaw = async function (query: string, params?: unknown[], options?: never) {
        executeRawQueries++;
        const rows = await real.call(this, query, params, options);
        if (query.includes('ORDER BY p.id LIMIT 256')) { batches++; largestBatch = Math.max(largestBatch, rows.length); }
        if (query.includes('SELECT visibility,fact_hash,withdrawn_at')) ledgerReads++;
        if (query.includes('AS lines(line,ord)')) normalizations++;
        return rows as never;
      };
      engine.getRawData = async function (...args) { rawQueries++; return raw.apply(this, args); };
      engine.transaction = async function (fn) { transactions++; return transaction.call(this, fn) as never; };
      ExportStage.prototype.close = function () {
        stagingPayloadBytes = this.bytes;
        stagingManifestBytes = statSync(join(this.directory, 'manifest.sqlite')).size;
        return close.call(this);
      };
      await runExport(engine, ['--dir', dir]);
      expect(readdirSync(join(dir, 'scale')).filter(name => name.endsWith('.md')).length).toBe(PAGES);
      expect(readdirSync(join(dir, 'scale/.raw')).length).toBe(FENCED);
      expect(readFileSync(join(dir, pageFile(PAGES)), 'utf8')).toContain(`Body ${PAGES}`);
      expect(readFileSync(join(dir, pageFile(PAGES - 1)), 'utf8')).toContain('scale-tag');
      expect(parseFactsFence(readFileSync(join(dir, pageFile(PAGES - 1)), 'utf8')).facts[0].active).toBe(false);
      expect(parseFactsFence(readFileSync(join(dir, 'scale/p000100.md'), 'utf8')).facts[0].active).toBe(false);
      expect(JSON.parse(readFileSync(join(dir, `scale/.raw/p${String(PAGES - 1).padStart(6, '0')}.json`), 'utf8')))
        .toEqual({ feed: { slug: `scale/p${String(PAGES - 1).padStart(6, '0')}` } });
      expect(batches).toBe(NONEMPTY_BATCHES + 1); expect(largestBatch).toBe(256);
      expect(ledgerReads).toBe(NONEMPTY_BATCHES); expect(normalizations).toBe(FENCED);
      expect(transactions).toBe(1); expect(rawQueries).toBe(FENCED);
      expect(executeRawQueries).toBe(PAGES + (NONEMPTY_BATCHES + 1) + NONEMPTY_BATCHES * 2 + FENCED + 2 + FOLDER_POLICY_READS);
      expect(stagingPayloadBytes).toBeGreaterThan(PAGES);
      expect(stagingManifestBytes).toBeGreaterThan(stagingPayloadBytes);
      console.log(JSON.stringify({ backend, selected: PAGES, processed: PAGES, rawSidecars: FENCED, taggedPages: FENCED,
        withdrawals: 1000, fencedPages: FENCED, ledgerReads, normalizations, blocked: 0, failed: 0, retries: 0,
        stagingPayloadBytes, stagingManifestBytes, executeRawQueries, rawQueries, transactions,
        totalSqlStatements: executeRawQueries + rawQueries + transactions * 2,
        batches, largestBatch, durationMs: Math.round(performance.now() - started), exportMs: Math.round(performance.now() - started - seededMs), peakRssKb: process.resourceUsage().maxRSS }));
    } finally {
      engine.executeRaw = real; engine.getRawData = raw; engine.transaction = transaction; ExportStage.prototype.close = close;
      rmSync(dir, { recursive: true, force: true });
    }
  }, 900000);
});