Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.2 KB
Raw Blame
/**
 * embeddings doctor check reports the embedding backlog (Foundations 1, F4d).
 *
 * Protects: when chunks lack embeddings, the `embeddings` check names the
 * backlog count and the command that finishes it (`gbrain embed --stale
 * --catch-up`), says it makes paid calls, and carries the same facts in
 * `details` for agents. A clean brain keeps the golden-pinned message.
 * Fails when: the check goes back to recommending a plain `embed --stale`,
 * which stops after its 30-minute budget and on a 52k-document brain left
 * 51,910 chunks behind without saying so.
 * Why new: no test exercised the embeddings check's messages; the doctor
 * goldens only pin the fully embedded case.
 * A keyless brain reports "not applicable" instead of a paid fix.
 * Seam: none (a stub engine answering getHealth and getConfig).
 *
 * Backlog age (E-B): a large backlog (over 1,000 chunks or 1%) warns only when
 * its oldest pending chunk has waited longer than EMBEDDING_BACKLOG_MAX_AGE_S,
 * read from content_chunks.embedding_pending_since, never created_at.
 * Fails when: the check ages the backlog from created_at (an edited old page
 * or a model swap reads as a months-old backlog and asks for paid spend on a
 * brain draining normally), or goes back to "ok at >= 90% coverage whatever
 * the backlog" (forced probe: 136,314 chunks missing at 90% read ok on master).
 * The stub probes answer the age query through executeRaw; the PGLite probes
 * run the real write paths that set and clear the column.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { EMBEDDING_BACKLOG_MAX_AGE_S, embeddingsEntry } from '../src/commands/doctor/checks/schema-health.ts';
import type { DoctorContext } from '../src/commands/doctor/context.ts';
import type { Check } from '../src/commands/doctor.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { invalidateStaleSignatureEmbeddingsGuarded } from '../src/core/embedding-invalidation.ts';
import { runSchemaTransition } from '../src/core/embedding-migration.ts';
import { runMigrations } from '../src/core/migrate.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { installFixtureChunks } from './helpers/page-projection.ts';

async function runCheck(ctx: DoctorContext): Promise<Check[]> {
  return await embeddingsEntry.run(ctx) as Check[];
}

function ctxFor(coverage: number, missing: number, config: Record<string, string> = {}, age?: PendingAge): DoctorContext {
  const engine = {
    getHealth: async () => ({ embed_coverage: coverage, missing_embeddings: missing }),
    getConfig: async (key: string) => config[key] ?? null,
    ...(age ? {
      executeRaw: async (sql: string) => sql.includes('embedding_pending_since')
        ? [{ age_s: age.ageS, unaged: age.unaged ?? 0, backfilled: age.backfilled ?? false }]
        : [],
    } : {}),
  };
  return { engine, progress: { heartbeat() {} } } as unknown as DoctorContext;
}

/** What the age query answers: the oldest pending chunk's wait, unstamped pending rows, backfill label. */
interface PendingAge { ageS: number | null; unaged?: number; backfilled?: boolean }

const HOUR = 3600;
const DAY = 24 * HOUR;

describe('embeddings doctor check: backlog', () => {
  test('a large backlog warns with the count and the catch-up command', async () => {
    const [check] = await runCheck(ctxFor(0.002, 51_910));
    expect(check.status).toBe('warn');
    expect(check.message).toContain('Backlog: 51910 chunk(s) without embeddings');
    expect(check.message).toContain('Fix: gbrain embed --stale --catch-up');
    expect(check.message).toContain('paid embedding calls');
    expect(check.details).toMatchObject({ code: 'embedding_backlog', backlog: 51_910, fix: 'gbrain embed --stale --catch-up' });
  });

  test('a brain with no embeddings yet names the backlog too', async () => {
    const [check] = await runCheck(ctxFor(0, 1200));
    expect(check.status).toBe('warn');
    expect(check.message).toContain('Backlog: 1200 chunk(s)');
  });

  test('a mostly embedded brain stays ok but still reports its backlog', async () => {
    const [check] = await runCheck(ctxFor(0.95, 2600));
    expect(check.status).toBe('ok');
    expect(check.message).toContain('Backlog: 2600 chunk(s)');
    expect(check.details).toMatchObject({ backlog: 2600 });
  });

  test('a keyless brain is not applicable, never told to make paid calls', async () => {
    const [check] = await runCheck(ctxFor(0, 200, { embedding_disabled: 'true' }));
    expect(check.status).toBe('ok');
    expect(check.message).toContain('Not applicable: embeddings are disabled');
    expect(check.message).not.toContain('catch-up');
  });

  test('a fully embedded brain keeps its message and carries no backlog', async () => {
    const [check] = await runCheck(ctxFor(1, 0));
    expect(check).toEqual({ name: 'embeddings', status: 'ok', message: '100% coverage, 0 missing',
      details: { backlog: 0, oldest_pending_age_s: null, age_threshold_s: EMBEDDING_BACKLOG_MAX_AGE_S } });
  });
});

describe('embeddings doctor check: backlog age', () => {
  test('95% coverage with 50k missing and an old pending chunk warns, with backlog and age in details', async () => {
    const [check] = await runCheck(ctxFor(0.95, 50_000, {}, { ageS: 3 * DAY }));
    expect(check.status).toBe('warn');
    expect(check.message).toContain('Backlog: 50000 chunk(s) without embeddings');
    expect(check.message).toContain('Oldest pending chunk has waited 3d');
    expect(check.details).toMatchObject({ code: 'embedding_backlog', backlog: 50_000, oldest_pending_age_s: 3 * DAY,
      age_threshold_s: EMBEDDING_BACKLOG_MAX_AGE_S, fix: 'gbrain embed --stale --catch-up' });
    expect(check.fix?.consent).toEqual(['paid']);
    expect(check.fix?.argv).toEqual(['gbrain', 'embed', '--stale', '--catch-up', '--yes']);
  });

  test('forced probe: 136,314 chunks missing at 90% coverage for days warns (ok on master)', async () => {
    const [check] = await runCheck(ctxFor(0.9, 136_314, {}, { ageS: 9 * DAY }));
    expect(check.status).toBe('warn');
    expect(check.details).toMatchObject({ backlog: 136_314, oldest_pending_age_s: 9 * DAY });
  });

  test('the audited production shape (89%, 136,314 missing, days old) warns', async () => {
    const [check] = await runCheck(ctxFor(0.893, 136_314, {}, { ageS: 9 * DAY }));
    expect(check.status).toBe('warn');
  });

  test('a fresh brain mid-drain stays ok and still reports backlog, age and threshold', async () => {
    const [check] = await runCheck(ctxFor(0.5, 40_000, {}, { ageS: 2 * HOUR }));
    expect(check.status).toBe('ok');
    expect(check.message).toContain('Backlog: 40000 chunk(s)');
    expect(check.details).toMatchObject({ backlog: 40_000, oldest_pending_age_s: 2 * HOUR, age_threshold_s: EMBEDDING_BACKLOG_MAX_AGE_S });
    expect(check.fix?.consent).toEqual(['paid']);
  });

  test('an old backlog under both size floors stays ok', async () => {
    const [check] = await runCheck(ctxFor(0.9995, 500, {}, { ageS: 30 * DAY }));
    expect(check.status).toBe('ok');
    expect(check.details).toMatchObject({ backlog: 500, oldest_pending_age_s: 30 * DAY });
  });

  test('a small brain mostly missing for days warns on the 1% floor', async () => {
    const [check] = await runCheck(ctxFor(0.2, 400, {}, { ageS: 2 * DAY }));
    expect(check.status).toBe('warn');
  });

  test('an age backfilled from created_at is labelled as such', async () => {
    const [check] = await runCheck(ctxFor(0.95, 50_000, {}, { ageS: 40 * DAY, backfilled: true }));
    expect(check.status).toBe('warn');
    expect(check.details).toMatchObject({ age_source: 'created_at_backfill' });
    expect(check.message).toContain('backfilled from its creation time');
  });

  test('pending chunks without a stamp leave the age unknown and the coverage rule decides', async () => {
    const low = (await runCheck(ctxFor(0.5, 40_000, {}, { ageS: 2 * HOUR, unaged: 3 })))[0];
    expect(low.status).toBe('warn');
    expect(low.details).toMatchObject({ backlog: 40_000, oldest_pending_age_s: null, age_threshold_s: EMBEDDING_BACKLOG_MAX_AGE_S });
    const high = (await runCheck(ctxFor(0.95, 50_000, {}, { ageS: null })))[0];
    expect(high.status).toBe('ok');
  });

  test('a keyless brain is info even with an old backlog', async () => {
    const [check] = await runCheck(ctxFor(0, 50_000, { embedding_disabled: 'true' }, { ageS: 90 * DAY }));
    expect(check.status).toBe('ok');
    expect(check.severity).toBe('info');
    expect(check.readiness_state).toBe('disabled_by_choice');
    expect(check.fix?.consent ?? []).not.toContain('paid');
  });
});

describe('embeddings doctor check: pending age on a real brain [pglite]', () => {
  let engine: PGLiteEngine;
  let dims = 0;
  const MODEL = 'openai:text-embedding-3-large';

  beforeAll(async () => {
    engine = new PGLiteEngine();
    await engine.connect({});
    await engine.initSchema();
  }, 60_000);

  afterAll(async () => {
    await engine.disconnect();
  });

  beforeEach(async () => {
    await resetPgliteState(engine);
    const rows = await engine.executeRaw<{ dim: number }>(
      `SELECT atttypmod AS dim FROM pg_attribute WHERE attrelid = 'content_chunks'::regclass AND attname = 'embedding' AND attnum > 0`);
    dims = Number(rows[0]?.dim);
  });

  const doctor = async () => (await runCheck({ engine, progress: { heartbeat() {} } } as unknown as DoctorContext))[0];
  const vector = () => { const v = new Float32Array(dims); v[0] = 1; return v; };

  /** One sealed page per slug, its chunk embedded through the real upsert, created 90 days ago. */
  async function seedEmbeddedPages(slugs: string[]): Promise<void> {
    for (const slug of slugs) {
      await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: `# ${slug}` });
      await installFixtureChunks(engine, slug, [{ chunk_index: 0, chunk_text: `${slug} body`, chunk_source: 'compiled_truth', model: MODEL }]);
      await engine.upsertChunks(slug, [{ chunk_index: 0, chunk_text: `${slug} body`, chunk_source: 'compiled_truth', embedding: vector(), model: MODEL }]);
    }
    await engine.executeRaw(`UPDATE content_chunks SET created_at = now() - interval '90 days'`);
  }

  test('an embedded chunk carries no pending stamp; a fresh insert without a vector does', async () => {
    await seedEmbeddedPages(['notes/embedded-alice-example']);
    await engine.putPage('notes/fresh-alice-example', { type: 'note', title: 'fresh', compiled_truth: '# fresh' });
    await installFixtureChunks(engine, 'notes/fresh-alice-example', [{ chunk_index: 0, chunk_text: 'fresh body', chunk_source: 'compiled_truth' }]);
    const rows = await engine.executeRaw<{ slug: string; pending: boolean }>(
      `SELECT p.slug, cc.embedding_pending_since IS NOT NULL AS pending FROM content_chunks cc JOIN pages p ON p.id = cc.page_id ORDER BY p.slug`);
    expect(rows.map(r => [r.slug, r.pending])).toEqual([['notes/embedded-alice-example', false], ['notes/fresh-alice-example', true]]);
  });

  test('a fresh brain mid-drain stays ok', async () => {
    for (const slug of ['notes/a-alice-example', 'notes/b-alice-example', 'notes/c-alice-example']) {
      await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: `# ${slug}` });
      await installFixtureChunks(engine, slug, [{ chunk_index: 0, chunk_text: `${slug} body`, chunk_source: 'compiled_truth' }]);
    }
    await engine.upsertChunks('notes/a-alice-example', [{ chunk_index: 0, chunk_text: 'notes/a-alice-example body', chunk_source: 'compiled_truth', embedding: vector(), model: MODEL }]);
    const check = await doctor();
    expect(check.status).toBe('ok');
    expect(check.details).toMatchObject({ backlog: 2, age_threshold_s: EMBEDDING_BACKLOG_MAX_AGE_S });
    expect((check.details as { oldest_pending_age_s: number }).oldest_pending_age_s).toBeLessThan(HOUR);
  });

  test('an edited old page stays ok: its wait starts when the edit cleared the vector, not at created_at', async () => {
    await seedEmbeddedPages(['notes/edited-alice-example']);
    expect((await doctor()).details).toMatchObject({ backlog: 0 });
    await engine.upsertChunks('notes/edited-alice-example', [{ chunk_index: 0, chunk_text: 'edited body', chunk_source: 'compiled_truth' }]);
    const check = await doctor();
    expect(check.status).toBe('ok');
    expect(check.details).toMatchObject({ backlog: 1 });
    expect((check.details as { oldest_pending_age_s: number }).oldest_pending_age_s).toBeLessThan(HOUR);
    await engine.executeRaw(`UPDATE content_chunks SET embedding_pending_since = now() - interval '2 days'`);
    expect((await doctor()).status).toBe('warn');
  });

  test('a model-swap invalidation actively draining stays ok', async () => {
    await seedEmbeddedPages(['notes/swap-a-alice-example', 'notes/swap-b-alice-example']);
    const invalidated = await invalidateStaleSignatureEmbeddingsGuarded(engine, { signature: `voyage:voyage-4:${dims}`, includeNullSignature: true });
    expect(invalidated).toBe(2);
    let check = await doctor();
    expect(check.status).toBe('ok');
    expect(check.details).toMatchObject({ backlog: 2 });
    expect((check.details as { oldest_pending_age_s: number }).oldest_pending_age_s).toBeLessThan(HOUR);
    await engine.upsertChunks('notes/swap-a-alice-example', [{ chunk_index: 0, chunk_text: 'notes/swap-a-alice-example body', chunk_source: 'compiled_truth', embedding: vector(), model: MODEL }]);
    check = await doctor();
    expect(check.status).toBe('ok');
    expect(check.details).toMatchObject({ backlog: 1 });
  });

  test('the migration backfills chunks already missing a vector to created_at once, and doctor labels that age', async () => {
    await seedEmbeddedPages(['notes/backfill-embedded-alice-example']);
    await engine.putPage('notes/backfill-pending-alice-example', { type: 'note', title: 'pending', compiled_truth: '# pending' });
    await installFixtureChunks(engine, 'notes/backfill-pending-alice-example', [{ chunk_index: 0, chunk_text: 'pending body', chunk_source: 'compiled_truth' }]);
    await engine.executeRaw(`UPDATE content_chunks SET embedding_pending_since = NULL, created_at = now() - interval '40 days'`);
    await engine.executeRaw(`DELETE FROM config WHERE key = 'embedding_pending_since_backfilled_at'`);
    await engine.setConfig('version', '219');
    await runMigrations(engine);
    const rows = await engine.executeRaw<{ slug: string; backfilled: boolean | null }>(
      `SELECT p.slug, cc.embedding_pending_since = cc.created_at AS backfilled FROM content_chunks cc JOIN pages p ON p.id = cc.page_id ORDER BY p.slug`);
    expect(rows.map(r => [r.slug, r.backfilled])).toEqual([['notes/backfill-embedded-alice-example', null], ['notes/backfill-pending-alice-example', true]]);
    const stamp = await engine.getConfig('embedding_pending_since_backfilled_at');
    expect(stamp).not.toBeNull();
    const check = await doctor();
    expect(check.status).toBe('warn');
    expect(check.details).toMatchObject({ backlog: 1, age_source: 'created_at_backfill' });
    await engine.setConfig('version', '219');
    await runMigrations(engine);
    expect(await engine.getConfig('embedding_pending_since_backfilled_at')).toBe(stamp);
  });

  test('a column rebuild stamps every chunk it emptied, keeping an older pending stamp', async () => {
    await seedEmbeddedPages(['notes/rebuilt-alice-example']);
    await engine.putPage('notes/waiting-alice-example', { type: 'note', title: 'waiting', compiled_truth: '# waiting' });
    await installFixtureChunks(engine, 'notes/waiting-alice-example', [{ chunk_index: 0, chunk_text: 'waiting body', chunk_source: 'compiled_truth' }]);
    await engine.executeRaw(`UPDATE content_chunks SET embedding_pending_since = now() - interval '3 days' WHERE embedding IS NULL`);
    await runSchemaTransition(engine, dims === 512 ? 256 : 512);
    const rows = await engine.executeRaw<{ slug: string; waited_s: number; stamped_at_null: boolean }>(
      `SELECT p.slug, EXTRACT(EPOCH FROM now() - cc.embedding_pending_since)::float8 AS waited_s,
              cc.embedded_at IS NULL AS stamped_at_null
         FROM content_chunks cc JOIN pages p ON p.id = cc.page_id ORDER BY p.slug`);
    expect(rows.map(r => r.slug)).toEqual(['notes/rebuilt-alice-example', 'notes/waiting-alice-example']);
    expect(rows.every(r => r.stamped_at_null)).toBe(true);
    expect(Number(rows[0].waited_s)).toBeLessThan(HOUR);
    expect(Number(rows[1].waited_s)).toBeGreaterThan(2 * DAY);
  });
});