Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
18.0 KB
Raw Blame
/**
 * D2 — whole-brain `runMigrateEngine` journey: real PGLite brain → real Postgres.
 *
 * Exercises the production engine-transfer path end to end against the live
 * DATABASE_URL: multi-source pages (including a same-slug collision across
 * sources), jsonb frontmatter, chunks with real embedding vectors, a facts
 * chain with a superseded row, tags / timeline / raw data / links, and DB-plane
 * config rows — then asserts the Postgres side row-for-row, that the file-plane
 * config flipped to postgres, and that the resume manifest was cleared.
 *
 * Failure arm runs FIRST (before the config flips) as a spawned child process,
 * because `runMigrateEngine` reaches `process.exit(1)`-adjacent paths and a
 * target-connect failure throws out of the CLI: the child must die, not the
 * test runner. It asserts non-zero exit, config NOT flipped, PGLite intact.
 *
 * Environment notes (why the env juggling below exists):
 *  - `loadConfig()` infers `engine: 'postgres'` whenever a DATABASE_URL-style
 *    env var is exported, which would trip runMigrateEngine's "Already using
 *    postgres engine" `process.exit(1)`. helpers.ts captures DATABASE_URL at
 *    module load, so this file deletes it from process.env for the duration
 *    (restored in afterAll) and passes the target URL explicitly via --url.
 *  - Both fixture engines explicitly use the legacy test embedding shape.
 *    The target is a fresh database so permanent receipts from other suites
 *    remain intact; gateway sizing is pinned before either schema is created.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join, resolve } from 'path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { runMigrateEngine } from '../../src/commands/migrate-engine.ts';
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
import type { BrainEngine } from '../../src/core/engine.ts';
import { hasDatabase } from './helpers.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { LEGACY_EMBEDDING_CONFIG } from '../helpers/legacy-embedding-config.ts';

const describePg = hasDatabase() ? describe : describe.skip;

// Captured at module load, before beforeAll deletes it from process.env.
const DB_URL = process.env.DATABASE_URL ?? '';
const REPO_ROOT = resolve(import.meta.dir, '../..');
const EMBED_DIMS = LEGACY_EMBEDDING_CONFIG.embedding_dimensions;

/** Deterministic 1536-d vector; v[0] = seed/8 is float4-exact for the
 * round-trip spot check on the Postgres side. */
function vec(seed: number): Float32Array {
  const v = new Float32Array(EMBED_DIMS);
  for (let i = 0; i < EMBED_DIMS; i++) v[i] = ((seed * 31 + i) % 97) / 97;
  v[0] = seed / 8;
  return v;
}

interface FixtureCounts {
  pages: number;
  chunks: number;
  chunks_embedded: number;
  facts: number;
  sources: number;
  tags: number;
  timeline_entries: number;
  raw_data: number;
  links: number;
}

async function tableCounts(engine: BrainEngine): Promise<FixtureCounts> {
  const one = async (sql: string): Promise<number> => {
    const rows = await engine.executeRaw<{ n: number | string }>(sql);
    return Number(rows[0]?.n ?? -1);
  };
  return {
    pages: await one('SELECT count(*)::int AS n FROM pages'),
    chunks: await one('SELECT count(*)::int AS n FROM content_chunks'),
    chunks_embedded: await one('SELECT count(*)::int AS n FROM content_chunks WHERE embedding IS NOT NULL'),
    facts: await one('SELECT count(*)::int AS n FROM facts'),
    sources: await one('SELECT count(*)::int AS n FROM sources'),
    tags: await one('SELECT count(*)::int AS n FROM tags'),
    timeline_entries: await one('SELECT count(*)::int AS n FROM timeline_entries'),
    raw_data: await one('SELECT count(*)::int AS n FROM raw_data'),
    links: await one('SELECT count(*)::int AS n FROM links'),
  };
}

describePg('migrate-engine whole-brain PGLite to Postgres (D2)', () => {
  const tmpBase = mkdtempSync(join(tmpdir(), 'gbrain-d2-'));
  const gbrainDir = join(tmpBase, '.gbrain');
  const configFile = join(gbrainDir, 'config.json');
  const manifestFile = join(gbrainDir, 'migrate-manifest.json');
  const pgliteDir = join(gbrainDir, 'brain.pglite');

  const origEnv = {
    DATABASE_URL: process.env.DATABASE_URL,
    GBRAIN_DATABASE_URL: process.env.GBRAIN_DATABASE_URL,
    GBRAIN_HOME: process.env.GBRAIN_HOME,
  };

  let source: PGLiteEngine | null = null;
  let targetFixture: Awaited<ReturnType<typeof isolatedPersistencePostgres>>;
  let targetUrl: string;
  const getEngine = () => targetFixture.engine;
  let seeded: FixtureCounts;
  let factId1 = 0; // superseded by factId2
  let factId2 = 0;
  let factMaxId = 0;

  beforeAll(async () => {
    if (!DB_URL) throw new Error('DATABASE_URL must be set for this e2e file');

    // Permanent receipt IDs from other files cannot be truncated for a copy.
    // Give this legacy-migration journey a fresh target brain and pin both
    // engines to the legacy vector shape before either schema is initialized.
    configureGateway({ ...LEGACY_EMBEDDING_CONFIG, env: {} });
    targetFixture = await isolatedPersistencePostgres(DB_URL);
    const [{ name }] = await targetFixture.engine.executeRaw<{ name: string }>('SELECT current_database() AS name');
    const url = new URL(DB_URL); url.pathname = `/${name}`; targetUrl = url.toString();

    // Isolated gbrain home with a real pglite file config — the SOURCE brain.
    mkdirSync(gbrainDir, { recursive: true });
    writeFileSync(configFile, JSON.stringify({
      engine: 'pglite',
      database_path: pgliteDir,
      ...LEGACY_EMBEDDING_CONFIG,
    }, null, 2));
    process.env.GBRAIN_HOME = tmpBase;
    // See header: an exported DATABASE_URL makes loadConfig() infer postgres,
    // which would hit runMigrateEngine's "Already using postgres" exit(1).
    delete process.env.DATABASE_URL;
    delete process.env.GBRAIN_DATABASE_URL;

    // ---- Seed the PGLite source brain ----
    source = new PGLiteEngine();
    await source.connect({ engine: 'pglite', database_path: pgliteDir });
    await source.initSchema();

    await source.executeRaw(`INSERT INTO sources (id, name, config)
      VALUES ('alpha', 'Source Alpha', '{"federated":true,"note":"a"}'::jsonb),
             ('beta',  'Source Beta',  '{}'::jsonb)`);

    // alpha: person page with jsonb frontmatter, 2 embedded chunks, tag,
    // timeline entry, raw data, and an outgoing link.
    await source.putPage('people/alice-example', {
      type: 'person',
      title: 'Alice Example',
      compiled_truth: 'Alice founded acme-example.',
      frontmatter: { tags: ['founder'], score: 2, nested: { deep: true } },
    }, { sourceId: 'alpha' });
    await source.putPage('companies/acme-example', {
      type: 'company',
      title: 'Acme Example',
      compiled_truth: 'Acme makes widgets.',
    }, { sourceId: 'alpha' });
    await source.putPage('notes/plain', {
      type: 'note',
      title: 'Plain note',
      compiled_truth: 'No chunks here.',
    }, { sourceId: 'alpha' });
    // beta: same slug as the alpha person page — multi-source collision.
    await source.putPage('people/alice-example', {
      type: 'person',
      title: 'Alice (beta view)',
      compiled_truth: 'Beta-source view of Alice.',
      frontmatter: { origin: 'beta' },
    }, { sourceId: 'beta' });
    await source.putPage('notes/beta-only', {
      type: 'note',
      title: 'Beta only',
      compiled_truth: 'Beta note with an unembedded chunk.',
    }, { sourceId: 'beta' });
    await source.putPage('index', {
      type: 'note',
      title: 'Index',
      compiled_truth: 'Default-source landing page.',
    }, { sourceId: 'default' });

    await source.upsertChunks('people/alice-example', [
      { chunk_index: 0, chunk_text: 'Alice founded acme-example.', chunk_source: 'compiled_truth', embedding: vec(1), model: 'test-model', token_count: 6 },
      { chunk_index: 1, chunk_text: 'She prefers espresso.', chunk_source: 'compiled_truth', embedding: vec(2), model: 'test-model', token_count: 4 },
    ], { sourceId: 'alpha' });
    await source.upsertChunks('companies/acme-example', [
      { chunk_index: 0, chunk_text: 'Acme makes widgets.', chunk_source: 'compiled_truth', embedding: vec(3), model: 'test-model', token_count: 4 },
    ], { sourceId: 'alpha' });
    await source.upsertChunks('people/alice-example', [
      { chunk_index: 0, chunk_text: 'Beta-source view of Alice.', chunk_source: 'compiled_truth', embedding: vec(4), model: 'test-model', token_count: 5 },
    ], { sourceId: 'beta' });
    await source.upsertChunks('notes/beta-only', [
      { chunk_index: 0, chunk_text: 'Beta note with an unembedded chunk.', chunk_source: 'compiled_truth' },
    ], { sourceId: 'beta' });

    await source.addTag('people/alice-example', 'person', { sourceId: 'alpha' });
    await source.addTimelineEntry('people/alice-example', {
      date: '2026-01-15', source: 'meeting', summary: 'Kickoff', detail: 'Discussed roadmap',
    }, { sourceId: 'alpha' });
    await source.putRawData('people/alice-example', 'crm', { stage: 'seed' }, { sourceId: 'alpha' });
    await source.addLink('people/alice-example', 'companies/acme-example', 'founder of', 'manual',
      undefined, undefined, undefined, { fromSourceId: 'alpha', toSourceId: 'alpha' });

    // Facts chain: f1 superseded by f2, plus a standalone f3 on another source.
    const f1 = await source.insertFact(
      { fact: 'User prefers dark roast coffee', source: 'cli:test-session', kind: 'preference' },
      { source_id: 'default' },
    );
    const f2 = await source.insertFact(
      { fact: 'User prefers light roast coffee', source: 'cli:test-session', kind: 'preference' },
      { source_id: 'default', supersedeId: f1.id },
    );
    expect(f2.status).toBe('superseded');
    const f3 = await source.insertFact(
      { fact: 'Acme raised a seed round', source: 'cli:test-session', entity_slug: 'companies/acme-example' },
      { source_id: 'alpha' },
    );
    factId1 = f1.id;
    factId2 = f2.id;
    factMaxId = Math.max(f1.id, f2.id, f3.id);

    // DB-plane config rows that MUST follow the data to the target.
    await source.setConfig('sync.repo_path', '/tmp/example-repo');
    await source.setConfig('search.mode', 'balanced');

    seeded = await tableCounts(source);
    expect(seeded.pages).toBe(6);
    expect(seeded.chunks).toBe(5);
    expect(seeded.chunks_embedded).toBe(4);
    expect(seeded.facts).toBe(3);
    expect(seeded.sources).toBe(3); // default + alpha + beta
    expect(seeded.links).toBe(1);

    // Release the PGLite file lock so the failure-arm child can open the brain.
    await source.disconnect();
    source = null;
  }, 150_000);

  afterAll(async () => {
    if (source) await source.disconnect().catch(() => {});
    await targetFixture?.close();
    resetGateway();
    for (const [k, v] of Object.entries(origEnv)) {
      if (v === undefined) delete process.env[k];
      else process.env[k] = v;
    }
    rmSync(tmpBase, { recursive: true, force: true });
  });

  test('failure arm (child process): invalid target URL exits non-zero, config stays pglite, no manifest', async () => {
    const badUrl = 'postgresql://postgres@localhost:5432/gbrain_d2_no_such_db';
    const childEnv: Record<string, string> = {};
    for (const [k, v] of Object.entries(process.env)) {
      if (v !== undefined) childEnv[k] = v;
    }
    delete childEnv.DATABASE_URL; // see header — would flip loadConfig to postgres
    delete childEnv.GBRAIN_DATABASE_URL;
    childEnv.GBRAIN_HOME = tmpBase;

    const proc = Bun.spawn({
      cmd: [process.execPath, 'src/cli.ts', 'migrate', '--to', 'supabase', '--url', badUrl],
      cwd: REPO_ROOT,
      env: childEnv,
      stdout: 'pipe',
      stderr: 'pipe',
    });
    const exitCode = await proc.exited;
    const stdout = await new Response(proc.stdout).text();
    const stderr = await new Response(proc.stderr).text();

    if (exitCode === 0) {
      console.error(`child unexpectedly succeeded\nstdout:\n${stdout}\nstderr:\n${stderr}`);
    }
    expect(exitCode).not.toBe(0);

    // The file-plane config did NOT flip.
    const cfg = JSON.parse(readFileSync(configFile, 'utf-8'));
    expect(cfg.engine).toBe('pglite');
    expect(cfg.database_url).toBeUndefined();
    // No resume manifest was left behind (failure happened before any copy).
    expect(existsSync(manifestFile)).toBe(false);
  }, 120_000);

  test('happy arm: runMigrateEngine copies the whole brain, flips config, clears the manifest', async () => {
    // Reconnect the source and prove the failed arm left the PGLite brain intact.
    source = new PGLiteEngine();
    await source.connect({ engine: 'pglite', database_path: pgliteDir });
    expect(await tableCounts(source)).toEqual(seeded);

    // Real argv contract: `gbrain migrate --to supabase --url <url>`.
    await runMigrateEngine(source, ['--to', 'supabase', '--url', targetUrl]);
    await source.disconnect();
    source = null;

    // Per-table row counts on the POSTGRES side match the seeded PGLite counts.
    const target = getEngine();
    expect(await tableCounts(target)).toEqual(seeded);

    // Local config flipped to the postgres engine, preserving non-engine keys.
    const cfg = JSON.parse(readFileSync(configFile, 'utf-8'));
    expect(cfg.engine).toBe('postgres');
    expect(cfg.database_url).toBe(targetUrl);
    expect(cfg.database_path).toBeUndefined();
    expect(cfg.embedding_dimensions).toBe(EMBED_DIMS); // pre-existing file keys preserved

    // Migration manifest cleared on a clean run.
    expect(existsSync(manifestFile)).toBe(false);
  }, 150_000);

  test('jsonb survives as objects (never double-encoded strings)', async () => {
    const target = getEngine();

    const fm = await target.executeRaw<{ t: string; deep: string }>(`
      SELECT jsonb_typeof(frontmatter) AS t, frontmatter->'nested'->>'deep' AS deep
        FROM pages WHERE slug = 'people/alice-example' AND source_id = 'alpha'`);
    expect(fm[0]?.t).toBe('object');
    expect(fm[0]?.deep).toBe('true');

    const betaFm = await target.executeRaw<{ t: string; origin: string }>(`
      SELECT jsonb_typeof(frontmatter) AS t, frontmatter->>'origin' AS origin
        FROM pages WHERE slug = 'people/alice-example' AND source_id = 'beta'`);
    expect(betaFm[0]?.t).toBe('object');
    expect(betaFm[0]?.origin).toBe('beta');

    const srcCfg = await target.executeRaw<{ t: string; federated: string }>(`
      SELECT jsonb_typeof(config) AS t, config->>'federated' AS federated
        FROM sources WHERE id = 'alpha'`);
    expect(srcCfg[0]?.t).toBe('object');
    expect(srcCfg[0]?.federated).toBe('true');
  });

  test('embedding vectors survive with the right dims and values', async () => {
    const target = getEngine();

    const dims = await target.executeRaw<{ d: number; n: number | string }>(`
      SELECT vector_dims(embedding)::int AS d, count(*)::int AS n
        FROM content_chunks WHERE embedding IS NOT NULL GROUP BY 1`);
    expect(dims).toHaveLength(1);
    expect(Number(dims[0].d)).toBe(EMBED_DIMS);
    expect(Number(dims[0].n)).toBe(seeded.chunks_embedded);

    // Float4-exact first component of the alpha person page's chunk 0 (seed 1 → 0.125).
    const spot = await target.executeRaw<{ e: string }>(`
      SELECT c.embedding::text AS e
        FROM content_chunks c JOIN pages p ON p.id = c.page_id
       WHERE p.slug = 'people/alice-example' AND p.source_id = 'alpha' AND c.chunk_index = 0`);
    expect(spot[0]?.e.startsWith('[0.125,')).toBe(true);

    // The deliberately unembedded chunk stayed NULL.
    const nulls = await target.executeRaw<{ n: number | string }>(`
      SELECT count(*)::int AS n FROM content_chunks WHERE embedding IS NULL`);
    expect(Number(nulls[0].n)).toBe(seeded.chunks - seeded.chunks_embedded);
  });

  test('facts chain survives verbatim and the id sequence does not collide', async () => {
    const target = getEngine();

    const rows = await target.executeRaw<{
      id: number | string; fact: string; source_id: string;
      superseded_by: number | string | null; expired: boolean;
    }>(`SELECT id, fact, source_id, superseded_by, (expired_at IS NOT NULL) AS expired
          FROM facts ORDER BY id`);
    expect(rows).toHaveLength(3);
    const byId = new Map(rows.map(r => [Number(r.id), r]));

    const old = byId.get(factId1)!;
    expect(old.fact).toBe('User prefers dark roast coffee');
    expect(Number(old.superseded_by)).toBe(factId2);
    expect(old.expired).toBe(true);

    const current = byId.get(factId2)!;
    expect(current.fact).toBe('User prefers light roast coffee');
    expect(current.superseded_by).toBeNull();
    expect(current.expired).toBe(false);

    const acme = rows.find(r => r.fact === 'Acme raised a seed round');
    expect(acme?.source_id).toBe('alpha');

    // BIGSERIAL was bumped past MAX(id): a fresh insert must not collide.
    const fresh = await target.insertFact(
      { fact: 'Post-migration fact lands cleanly', source: 'cli:test-session' },
      { source_id: 'default' },
    );
    expect(fresh.status).toBe('inserted');
    expect(fresh.id).toBeGreaterThan(factMaxId);
  });

  test('pages the source had not sealed arrive queued with their verbatim chunks, and do not block the flip (#6286)', async () => {
    const target = getEngine();
    const jobs = await target.executeRaw<{ reason: string; n: number | string }>(
      'SELECT reason, count(*)::int AS n FROM page_projection_jobs GROUP BY reason ORDER BY reason');
    expect(jobs.map(j => [j.reason, Number(j.n)])).toEqual([['engine_migration_source_unsealed', seeded.pages]]);
  });

  test('DB-plane config rows migrated; engine-local keys did not', async () => {
    const target = getEngine();
    const get = async (key: string): Promise<string | null> => {
      const rows = await target.executeRaw<{ value: string }>(
        'SELECT value FROM config WHERE key = $1', [key]);
      return rows[0]?.value ?? null;
    };
    expect(await get('sync.repo_path')).toBe('/tmp/example-repo');
    expect(await get('search.mode')).toBe('balanced');
    // Denylisted engine-identity row was NOT overwritten with the source's.
    expect(await get('engine')).not.toBe('pglite');
  });
});