Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
18.6 KB
Raw Blame
import { installFixtureChunks } from './helpers/page-projection.ts';
/**
 * Chunkless-page safety net for `embed --stale`.
 *
 * `embedAllStale` (and the underlying `listStaleChunks`/`countStaleChunks`)
 * only ever scan `content_chunks` rows where `embedding IS NULL`. A page
 * written directly via `putPage` that never went through chunking (e.g. an
 * enrichment-generated entity stub — the dogfooding case this fix targets)
 * has NO `content_chunks` row at all, so it is invisible to that scan
 * forever: there is no row to go stale, no matter how many times
 * `embed --stale` runs.
 *
 * Two layers tested here:
 *   1. The detection primitive itself (`countChunklessPagesWithContent` /
 *      `listChunklessPagesWithContent`) — including that quarantined pages
 *      are excluded (chunkless by design) while a chunkless `embed_skip` page
 *      with a body is included (#7174: writes keep its chunks since #6722).
 *   2. The end-to-end `embed --stale` wiring — a chunkless page gets
 *      chunked AND embedded in the SAME pass, while pre-existing stale
 *      chunks and intentionally-chunkless pages are unaffected.
 *
 * Named `.serial.test.ts` (mirrors v0_37_gap_fill.serial.test.ts Lane D.2):
 * configures the AI gateway + a fake embed transport for its whole
 * lifecycle, which withEnv() can't wrap.
 */
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { configureGateway, resetGateway, __setEmbedTransportForTests } from '../src/core/ai/gateway.ts';
import { runEmbedCore } from '../src/commands/embed.ts';
import { EMBED_SKIP_KEY, buildEmbedSkipMarker } from '../src/core/embed-skip.ts';
import { QUARANTINE_KEY, buildQuarantineMarker } from '../src/core/quarantine.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { recordFactWithdrawal } from '../src/core/facts/withdrawal.ts';
import { installPageEmbeddings, readProjectionSnapshot, type ProjectionSnapshot } from '../src/core/page-state/projections.ts';

const DIMS = 1536;
let engine: PGLiteEngine;

beforeAll(async () => {
  configureGateway({
    embedding_model: 'openai:text-embedding-3-large',
    embedding_dimensions: DIMS,
    env: { ...process.env, OPENAI_API_KEY: 'sk-test-fake' },
  });
  __setEmbedTransportForTests(async ({ values }: { values: string[] }) => ({
    embeddings: values.map(() => new Array(DIMS).fill(0.001)),
    usage: { tokens: values.length * 4 },
  } as never));

  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
}, 30000);

afterAll(async () => {
  __setEmbedTransportForTests(null);
  resetGateway();
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
});

describe('countChunklessPagesWithContent / listChunklessPagesWithContent', () => {
  test('detects a page written via putPage that was never chunked', async () => {
    await engine.putPage('stub/entity-a', {
      type: 'person',
      title: 'Entity A',
      compiled_truth: 'Entity A is a stub written directly via putPage.',
    });

    expect(await engine.countChunklessPagesWithContent()).toBe(1);
    const rows = await engine.listChunklessPagesWithContent();
    expect(rows).toHaveLength(1);
    expect(rows[0].slug).toBe('stub/entity-a');
    expect(rows[0].compiled_truth).toContain('stub written directly');
  });

  test('excludes pages that already have chunk rows', async () => {
    await engine.putPage('normal/page', { type: 'note', title: 'Normal', compiled_truth: 'hello world' });
    await installFixtureChunks(engine, 'normal/page', [
      { chunk_index: 0, chunk_text: 'hello world', chunk_source: 'compiled_truth' },
    ]);

    expect(await engine.countChunklessPagesWithContent()).toBe(0);
    expect(await engine.listChunklessPagesWithContent()).toEqual([]);
  });

  test('excludes pages with empty content', async () => {
    // Matches pages.compiled_truth's schema DEFAULT '' — the #2822 empty-put
    // case. Empty content has nothing to chunk; it is not this bug class.
    await engine.putPage('empty/page', { type: 'note', title: 'Empty', compiled_truth: '' });

    expect(await engine.countChunklessPagesWithContent()).toBe(0);
  });

  test('detects a timeline-only page (empty compiled_truth, non-empty timeline)', async () => {
    // healChunklessPages chunks compiled_truth AND timeline independently
    // (mirrors embedPage) — the SQL predicate must not require
    // compiled_truth alone or this class of page is never even detected.
    await engine.putPage('timeline-only/page', {
      type: 'note',
      title: 'Timeline Only',
      compiled_truth: '',
      timeline: '2026-01-01: something happened',
    });

    expect(await engine.countChunklessPagesWithContent()).toBe(1);
    const rows = await engine.listChunklessPagesWithContent();
    expect(rows.map(r => r.slug)).toEqual(['timeline-only/page']);
  });

  test('excludes quarantined pages (intentionally chunkless by design)', async () => {
    await engine.putPage('junk/page', {
      type: 'note',
      title: 'Junk',
      compiled_truth: 'Cloudflare interstitial junk content',
      frontmatter: { [QUARANTINE_KEY]: buildQuarantineMarker('junk_pattern', 'test fixture') },
    });

    expect(await engine.countChunklessPagesWithContent()).toBe(0);
  });

  test('includes a chunkless embed_skip page with a body (#7174: keyword search cannot see it)', async () => {
    await engine.putPage('oversized/page', {
      type: 'note',
      title: 'Oversized',
      compiled_truth: 'x'.repeat(1000),
      frontmatter: { [EMBED_SKIP_KEY]: buildEmbedSkipMarker(1000) },
    });

    expect(await engine.countChunklessPagesWithContent()).toBe(1);
    expect((await engine.listChunklessPagesWithContent()).map(page => page.slug)).toEqual(['oversized/page']);
  });

});

describe('embed --stale chunkless-page safety net (end-to-end)', () => {
  test('a putPage-only page gets chunked AND embedded in the same --stale pass', async () => {
    await engine.putPage('stub/heal-me', {
      type: 'person',
      title: 'Heal Me',
      compiled_truth: 'This entity stub was written directly via putPage and never chunked.',
    });
    expect(await engine.getChunks('stub/heal-me')).toEqual([]);

    const result = await runEmbedCore(engine, { stale: true, quiet: true });

    expect(result.chunkless_pages_healed).toBe(1);
    const chunks = await engine.getChunks('stub/heal-me');
    expect(chunks.length).toBeGreaterThan(0);
    for (const c of chunks) {
      expect(c.embedded_at).not.toBeNull();
    }
  });

  test('pre-existing NULL-embedding chunks on other pages still get embedded (no regression)', async () => {
    await engine.putPage('normal/pre-chunked', { type: 'note', title: 'Pre-chunked', compiled_truth: 'hello world' });
    await installFixtureChunks(engine, 'normal/pre-chunked', [
      { chunk_index: 0, chunk_text: 'hello world', chunk_source: 'compiled_truth' },
    ]);
    await engine.putPage('stub/heal-me-2', {
      type: 'person',
      title: 'Heal Me Two',
      compiled_truth: 'Another chunkless stub.',
    });

    const result = await runEmbedCore(engine, { stale: true, quiet: true });

    expect(result.chunkless_pages_healed).toBe(1);
    expect(result.embedded).toBeGreaterThanOrEqual(2); // 1 pre-existing + >=1 healed
    const preChunked = await engine.getChunks('normal/pre-chunked');
    expect(preChunked[0]?.embedded_at).not.toBeNull();
  });

  test('healing sanitizes active, withdrawn and private fences before chunking either body column', async () => {
    const slug = 'stub/fenced-history';
    const fence = (column: string) => `<!--- gbrain:facts:begin -->
| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |
|---|---|---|---|---|---|---|---|---|---|
| 1 | activeworld${column} remains searchable | fact | 1.0 | world | medium | 2026-01-01 | | test | |
| 2 | withdrawnsentinel${column} retained history | fact | 1.0 | world | medium | 2026-01-01 | | test | |
| 3 | privatesentinel${column} hidden detail | fact | 1.0 | private | medium | 2026-01-01 | | test | |
<!--- gbrain:facts:end -->`;
    await engine.putPage(slug, { type: 'note', title: 'Fenced history',
      compiled_truth: `Safe body prose.\n${fence('body')}`, timeline: `Safe timeline prose.\n${fence('timeline')}` });
    for (const column of ['body', 'timeline']) {
      const fact = await engine.insertFact({ fact: `withdrawnsentinel${column} retained history`, source: 'test', visibility: 'world' }, { source_id: 'default' });
      expect((await recordFactWithdrawal(engine, fact.id, 'default', true)).withdrawn).toBe(true);
    }
    expect(await engine.getChunks(slug, { includeUnsealed: true })).toEqual([]);
    const historical = (await engine.readPageSnapshot(slug, { sourceId: 'default' }))!;
    expect(historical.page.compiled_truth).toContain('~~withdrawnsentinelbody retained history~~');
    expect(historical.page.timeline).toContain('~~withdrawnsentineltimeline retained history~~');

    const result = await runEmbedCore(engine, { stale: true, quiet: true });

    expect(result.chunkless_pages_healed).toBe(1);
    const chunks = await engine.getChunks(slug);
    for (const [field, column] of [['compiled_truth', 'body'], ['timeline', 'timeline']]) {
      const text = chunks.filter(c => c.chunk_source === field).map(c => c.chunk_text).join('\n');
      expect(text).toContain(`activeworld${column}`);
      expect(text).not.toContain(`withdrawnsentinel${column}`);
      expect(text).not.toContain(`privatesentinel${column}`);
      expect(await engine.searchKeyword(`withdrawnsentinel${column}`)).toEqual([]);
    }
    expect(chunks.every(c => c.embedded_at !== null)).toBe(true);
    const after = (await engine.readPageSnapshot(slug, { sourceId: 'default' }))!;
    expect(after.revision).toBe(historical.revision);
    expect(after.page.compiled_truth).toBe(historical.page.compiled_truth);
    expect(after.page.timeline).toBe(historical.page.timeline);
    expect(after.page.text_projection_revision).toBe(after.revision);
  });

  test('a quarantined page stays chunkless; a chunkless embed_skip page is chunked for keyword search and gets no vector (#7174)', async () => {
    await engine.putPage('junk/quarantined', {
      type: 'note',
      title: 'Junk',
      compiled_truth: 'junk content',
      frontmatter: { [QUARANTINE_KEY]: buildQuarantineMarker('junk_pattern', 'test fixture') },
    });
    await engine.putPage('oversized/skipped', {
      type: 'note',
      title: 'Oversized',
      compiled_truth: 'x'.repeat(1000),
      frontmatter: { [EMBED_SKIP_KEY]: buildEmbedSkipMarker(1000) },
    });

    const result = await runEmbedCore(engine, { stale: true, quiet: true });

    expect(result.chunkless_pages_healed).toBe(1);
    expect(await engine.getChunks('junk/quarantined')).toEqual([]);
    const [skipped] = await engine.executeRaw<{ chunks: number; vectors: number }>(`SELECT count(*)::int AS chunks, count(cc.embedding)::int AS vectors
      FROM content_chunks cc JOIN pages p ON p.id = cc.page_id WHERE p.slug = 'oversized/skipped'`);
    expect(skipped).toEqual({ chunks: 1, vectors: 0 });
    expect((await engine.searchKeyword('x'.repeat(1000), { limit: 5 })).map(hit => hit.slug)).toContain('oversized/skipped');
  });

  test('dry-run reports the would-be-healed count without writing any chunks', async () => {
    await engine.putPage('stub/dry-run-only', {
      type: 'person',
      title: 'Dry Run Only',
      compiled_truth: 'This stub must NOT be chunked by a dry run.',
    });

    const result = await runEmbedCore(engine, { stale: true, dryRun: true, quiet: true });

    expect(result.chunkless_pages_healed).toBe(1);
    expect(result.would_embed).toBeGreaterThan(0);
    expect(await engine.getChunks('stub/dry-run-only')).toEqual([]); // no mutation
  });

  test('healthy brain (no chunkless pages) pays no extra cost and behaves exactly as before', async () => {
    await engine.putPage('normal/only-page', { type: 'note', title: 'Only', compiled_truth: 'hello' });
    await installFixtureChunks(engine, 'normal/only-page', [
      { chunk_index: 0, chunk_text: 'hello', chunk_source: 'compiled_truth' },
    ]);

    const result = await runEmbedCore(engine, { stale: true, quiet: true });

    expect(result.chunkless_pages_healed).toBe(0);
    expect(result.embedded).toBe(1);
  });

  test('race mitigation: a page chunked by a concurrent writer between list and write is not clobbered', async () => {
    // Review catch: healChunklessPages lists chunkless pages, chunks them
    // in memory, then writes. If a concurrent writer (sync, another
    // put_page/embed) chunks the SAME page in between, a naive write would
    // overwrite the concurrent writer's (newer) chunks with this sweep's
    // stale-content snapshot. Simulate that race by injecting a write
    // immediately after listChunklessPagesWithContent returns — i.e. AFTER
    // the sweep has decided to heal this page but BEFORE its own write.
    await engine.putPage('stub/raced', {
      type: 'person',
      title: 'Raced',
      compiled_truth: 'Original content the sweep read.',
    });

    let injected = false;
    const realList = engine.listChunklessPagesWithContent.bind(engine);
    const raceEngine = new Proxy(engine, {
      get(target, prop, receiver) {
        if (prop === 'listChunklessPagesWithContent') {
          return async (opts?: Parameters<typeof realList>[0]) => {
            const rows = await realList(opts);
            if (!injected && rows.some(r => r.slug === 'stub/raced')) {
              injected = true;
              // Simulate the concurrent writer: chunks the page with DIFFERENT
              // content than what the sweep just read.
              await installFixtureChunks(engine, 'stub/raced', [
                { chunk_index: 0, chunk_text: 'concurrently-written chunk', chunk_source: 'compiled_truth' },
              ]);
            }
            return rows;
          };
        }
        const value = Reflect.get(target, prop, receiver);
        return typeof value === 'function' ? value.bind(target) : value;
      },
    }) as unknown as BrainEngine;

    const result = await runEmbedCore(raceEngine, { stale: true, quiet: true });

    // The race was detected and the page was skipped this pass (not
    // counted as healed by this sweep) rather than clobbered.
    expect(result.chunkless_pages_healed).toBe(0);
    const chunks = await engine.getChunks('stub/raced');
    expect(chunks).toHaveLength(1);
    expect(chunks[0].chunk_text).toBe('concurrently-written chunk');
  });

  test('late chunkless healing preserves a projection completed after its guarded capture', async () => {
    const slug = 'stub/late-heal';
    await engine.putPage(slug, { type: 'note', title: 'Late heal', compiled_truth: 'First sentence. Second sentence.' });
    let injected = false;
    let current: Awaited<ReturnType<BrainEngine['getChunks']>> = [];
    const raced = new Proxy(engine, {
      get(target, key) {
        if (key === 'transaction') return async <T>(run: (tx: BrainEngine) => Promise<T>) => {
          const result = await target.transaction(run);
          if (!injected && (result as ProjectionSnapshot | null)?.snapshot?.page.slug === slug) {
            injected = true;
            await installFixtureChunks(engine, slug, ['First sentence.', 'Second sentence.'].map((text, index) => ({
              chunk_index: index, chunk_text: text, chunk_source: 'compiled_truth',
            })));
            const newer = (await readProjectionSnapshot(engine, slug, 'default'))!;
            const vector = new Float32Array(DIMS); vector[0] = 0.75;
            expect((await installPageEmbeddings(engine, newer, newer.chunks.map(c => ({
              chunk_index: c.chunk_index, chunk_source: c.chunk_source, chunk_text: c.chunk_text, embedding: vector,
            })))).complete).toBe(true);
            current = await engine.getChunks(slug, { includeEmbedding: true });
          }
          return result;
        };
        const value = Reflect.get(target, key, target);
        return typeof value === 'function' ? value.bind(target) : value;
      },
    });

    const result = await runEmbedCore(raced, { stale: true, quiet: true });

    expect(injected).toBe(true);
    expect(result.chunkless_pages_healed).toBe(0);
    expect(result.failures).toBe(0);
    expect(await engine.getChunks(slug, { includeEmbedding: true })).toEqual(current);
  });

  test('one broken chunkless page does not abort the whole --stale run (per-page failure isolation)', async () => {
    // Review catch: healChunklessPages must try/catch per page. Before the
    // fix, an exception from getPage/getChunks/upsertChunks for ONE
    // chunkless page propagated out of healChunklessPages entirely,
    // aborting embedAllStale before it even reached the normal
    // NULL-embedding pass — making the safety net worse than the bug.
    await engine.putPage('stub/broken', {
      type: 'person',
      title: 'Broken',
      compiled_truth: 'This page will fail to heal.',
    });
    await engine.putPage('normal/unrelated', { type: 'note', title: 'Unrelated', compiled_truth: 'fine' });
    await installFixtureChunks(engine, 'normal/unrelated', [
      { chunk_index: 0, chunk_text: 'fine', chunk_source: 'compiled_truth' },
    ]);

    const brokenEngine = new Proxy(engine, {
      get(target, prop, receiver) {
        if (prop === 'transaction') {
          return <T>(run: (tx: BrainEngine) => Promise<T>) => engine.transaction(tx => run(new Proxy(tx, {
            get(inner, key) {
              if (key === 'readPageSnapshot') return async (slug: string, opts?: Parameters<BrainEngine['readPageSnapshot']>[1]) => {
                if (slug === 'stub/broken') throw new Error('simulated page snapshot failure');
                return inner.readPageSnapshot(slug, opts);
              };
              const value = Reflect.get(inner, key, inner);
              return typeof value === 'function' ? value.bind(inner) : value;
            },
          })));
        }
        const value = Reflect.get(target, prop, receiver);
        return typeof value === 'function' ? value.bind(target) : value;
      },
    }) as unknown as BrainEngine;

    const result = await runEmbedCore(brokenEngine, { stale: true, quiet: true });

    // The broken page's failure is recorded, not swallowed silently...
    expect(result.failures).toBeGreaterThanOrEqual(1);
    expect(result.failure_samples.some(s => s.includes('stub/broken'))).toBe(true);
    // ...but did NOT abort the run: the unrelated pre-existing stale chunk
    // still got embedded in the SAME pass.
    expect(result.embedded).toBeGreaterThanOrEqual(1);
    const unrelatedChunks = await engine.getChunks('normal/unrelated');
    expect(unrelatedChunks[0]?.embedded_at).not.toBeNull();
  });
});