Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
31.5 KB
Raw Blame
/**
 * Tests for src/core/embed-stale.ts (v0.40 D15.2).
 *
 * Hermetic — uses an injected `embedFn` so no network call lands. Validates:
 *   - empty stale set → done:true, embedded:0
 *   - multi-batch run → embed every stale chunk, advance cursor correctly
 *   - kill mid-flight (signal.aborted) → aborted:true, partial progress preserved
 *   - resume from cursor → picks up where prior call left off (DB predicate)
 *   - per-page embedFn throw → logged + skipped, NOT propagated; chunks stay NULL
 *
 * Why PGLite: validates the engine.listStaleChunks/getChunks/upsertChunks
 * roundtrip the helper depends on, not just the loop control flow.
 */
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { installFixtureChunks } from './helpers/page-projection.ts';
import { embedStaleForSource } from '../src/core/embed-stale.ts';
import type { ChunkInput } from '../src/core/types.ts';

let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
}, 30000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
});

/** Seed a page with N stale chunks (no embedding) into the default source. */
async function seedPageWithStaleChunks(slug: string, chunkCount: number): Promise<void> {
  await engine.putPage(slug, {
    type: 'note',
    title: slug,
    compiled_truth: `# ${slug}\n\nseeded`,
  });
  const chunks: ChunkInput[] = Array.from({ length: chunkCount }, (_, i) => ({
    chunk_index: i,
    chunk_text: `chunk ${i} of ${slug}`,
    chunk_source: 'compiled_truth',
    token_count: 4,
    embedding: undefined, // NULL = stale
  }));
  await installFixtureChunks(engine, slug, chunks);
}

/** Deterministic fake embedder — returns unit-length 1536-dim vectors with
 *  first dim = text length, so we can assert specific chunks got embedded. */
function fakeEmbedFn(texts: string[]): Promise<Float32Array[]> {
  return Promise.resolve(
    texts.map((t) => {
      const v = new Float32Array(1536);
      v[0] = t.length;
      v[1] = 1;
      return v;
    }),
  );
}

/** The model `upsertChunks` recorded for `slug`'s chunks (gateway-resolved in
 *  this process). A signature must name it for the provenance stamp to land. */
async function recordedModel(slug: string): Promise<string> {
  const rows = await engine.executeRaw<{ model: string }>(
    `SELECT cc.model FROM content_chunks cc JOIN pages p ON p.id = cc.page_id WHERE p.slug = $1 LIMIT 1`,
    [slug],
  );
  return rows[0]!.model;
}

describe('embedStaleForSource', () => {
  test('empty stale set returns done:true with zero embedded', async () => {
    const result = await embedStaleForSource(engine, 'default', {
      embedFn: fakeEmbedFn,
    });
    expect(result).toEqual({
      embedded: 0,
      chunksProcessed: 0,
      pagesProcessed: 0,
      invalidated: 0,
      contentDrift: expect.objectContaining({ scope: 'source:default', state: 'complete', checked: 0, drifted: 0, batches: 1 }),
      lastCursor: null,
      done: true,
      aborted: false,
    });
  });

  test('embeds every stale chunk across multiple pages in one call', async () => {
    await seedPageWithStaleChunks('a', 5);
    await seedPageWithStaleChunks('b', 3);

    const result = await embedStaleForSource(engine, 'default', {
      embedFn: fakeEmbedFn,
    });
    expect(result.done).toBe(true);
    expect(result.aborted).toBe(false);
    expect(result.embedded).toBe(8);
    expect(result.pagesProcessed).toBe(2);

    // Verify DB: zero stale remaining for default.
    const stale = await engine.countStaleChunks({ sourceId: 'default' });
    expect(stale).toBe(0);
  });

  test('respects batchSize for cursor pagination', async () => {
    await seedPageWithStaleChunks('a', 3);
    await seedPageWithStaleChunks('b', 3);
    let batchCount = 0;
    const result = await embedStaleForSource(engine, 'default', {
      embedFn: fakeEmbedFn,
      batchSize: 2,
      onProgress: () => {
        batchCount++;
      },
    });
    expect(result.embedded).toBe(6);
    // 2-chunk batches across 6 stale rows = at least 3 progress callbacks.
    expect(batchCount).toBeGreaterThanOrEqual(3);
  });

  test('IRON-RULE: aborted mid-flight → aborted:true, partial progress preserved', async () => {
    await seedPageWithStaleChunks('a', 4);
    await seedPageWithStaleChunks('b', 4);
    await seedPageWithStaleChunks('c', 4);
    const controller = new AbortController();
    // Batch size 4 = one page per batch. concurrency 1 = serialize keys.
    // Abort fires inside embedFn for page 'b', so 'a' lands, 'b' aborts mid-call,
    // and the third batch ('c') never starts.
    const result = await embedStaleForSource(engine, 'default', {
      batchSize: 4,
      concurrency: 1,
      signal: controller.signal,
      embedFn: async (texts) => {
        if (texts.some((t) => t.includes(' of b'))) {
          controller.abort();
          throw new Error('aborted'); // simulates HTTP abort throw
        }
        return fakeEmbedFn(texts);
      },
    });
    expect(result.aborted).toBe(true);
    expect(result.done).toBe(false);
    expect(result.embedded).toBe(4); // only 'a' landed
    // 'b' and 'c' (8 chunks) remain stale
    const stale = await engine.countStaleChunks({ sourceId: 'default' });
    expect(stale).toBe(8);
  });

  test('IRON-RULE: kill + resume — second call picks up via embedding-IS-NULL predicate', async () => {
    await seedPageWithStaleChunks('a', 4);
    await seedPageWithStaleChunks('b', 4);

    // First call aborts when 'b' is reached
    const controller = new AbortController();
    const first = await embedStaleForSource(engine, 'default', {
      batchSize: 4,
      concurrency: 1,
      signal: controller.signal,
      embedFn: async (texts) => {
        if (texts.some((t) => t.includes(' of b'))) {
          controller.abort();
          throw new Error('aborted');
        }
        return fakeEmbedFn(texts);
      },
    });
    expect(first.aborted).toBe(true);
    expect(first.embedded).toBe(4); // 'a' landed

    // Second call with NO cursor — predicate excludes already-embedded chunks
    const second = await embedStaleForSource(engine, 'default', {
      embedFn: fakeEmbedFn,
    });
    expect(second.done).toBe(true);
    expect(first.embedded + second.embedded).toBe(8);

    const stale = await engine.countStaleChunks({ sourceId: 'default' });
    expect(stale).toBe(0);
  });

  test('per-page embedFn throw is logged but does NOT propagate', async () => {
    await seedPageWithStaleChunks('good', 2);
    await seedPageWithStaleChunks('bad', 2);

    let badCount = 0;
    const result = await embedStaleForSource(engine, 'default', {
      embedFn: async (texts) => {
        if (texts.some((t) => t.includes('bad'))) {
          badCount++;
          throw new Error('intentional embed failure');
        }
        return fakeEmbedFn(texts);
      },
    });

    // The helper itself didn't throw
    expect(result.done).toBe(true);
    expect(badCount).toBe(1);

    // 'good' chunks got embedded; 'bad' chunks stayed NULL
    expect(result.embedded).toBe(2);
    const stale = await engine.countStaleChunks({ sourceId: 'default' });
    expect(stale).toBe(2);
  });

  test('source-scoped: does not touch other sources', async () => {
    await engine.executeRaw(
      `INSERT INTO sources (id, name, config) VALUES ('other', 'other', '{"federated":true}'::jsonb) ON CONFLICT (id) DO NOTHING`,
    );
    await seedPageWithStaleChunks('a', 3);
    await engine.putPage('b', {
      type: 'note',
      title: 'b',
      compiled_truth: '# b\n\nseeded',
    }, { sourceId: 'other' });
    await installFixtureChunks(engine,
      'b',
      Array.from({ length: 3 }, (_, i) => ({
        chunk_index: i,
        chunk_text: `other ${i}`,
        chunk_source: 'compiled_truth',
        token_count: 4,
        embedding: undefined,
      })),
      { sourceId: 'other' },
    );

    const result = await embedStaleForSource(engine, 'default', {
      embedFn: fakeEmbedFn,
    });
    expect(result.embedded).toBe(3);

    // 'other' source still has 3 stale chunks
    const otherStale = await engine.countStaleChunks({ sourceId: 'other' });
    expect(otherStale).toBe(3);
  });

  test('preserves modality and code-symbol metadata across the merge round-trip', async () => {
    // Regression: the merged ChunkInput[] used to rebuild rows with only 5
    // fields; upsertChunks writes modality/symbol columns as EXCLUDED.<col>,
    // so an image page with one stale TEXT chunk got its image row reset to
    // modality='text' — permanently invisible to the image search arm.
    await engine.putPage('media/mixed-page', {
      type: 'image',
      title: 'mixed',
      compiled_truth: 'mixed modality page',
    });
    const imgVec = new Float32Array(1024).fill(0.03);
    await installFixtureChunks(engine, 'media/mixed-page', [
      {
        chunk_index: 0,
        chunk_text: 'field-photo.jpg',
        chunk_source: 'image_asset',
        modality: 'image',
        embedding_image: imgVec,
        // embedding intentionally present so this row is NOT stale.
        embedding: new Float32Array(1536).fill(0.01),
        token_count: 4,
      },
      {
        chunk_index: 1,
        chunk_text: 'ocr caption text needing embed',
        chunk_source: 'compiled_truth',
        language: 'python',
        symbol_name: 'kept_symbol',
        symbol_type: 'function',
        symbol_name_qualified: 'mod::kept_symbol',
        token_count: 6,
        embedding: undefined, // stale — triggers the merge path
      },
    ]);

    const result = await embedStaleForSource(engine, 'default', { embedFn: fakeEmbedFn });
    expect(result.embedded).toBe(1);

    const after = await engine.getChunks('media/mixed-page');
    const imgRow = after.find((c) => c.chunk_index === 0)!;
    const txtRow = after.find((c) => c.chunk_index === 1)!;
    expect(imgRow.modality).toBe('image');
    expect(txtRow.language).toBe('python');
    expect(txtRow.symbol_name).toBe('kept_symbol');
    expect(txtRow.symbol_name_qualified).toBe('mod::kept_symbol');
    // The stale text row actually got its embedding.
    expect(txtRow.embedded_at).not.toBeNull();
  });
});

// ────────────────────────────────────────────────────────────────
// #3507 — re-embed must reproduce the page's STORED contextual-retrieval
// wrapping convention. Before the fix, every plain re-embed (including the
// normal post-model-migration `embed --stale`) embedded raw chunk_text,
// silently replacing context-wrapped vectors with unwrapped ones.
// ────────────────────────────────────────────────────────────────

describe('contextual-retrieval wrapping on re-embed (#3507)', () => {
  /** embedFn that records every text it is asked to embed. */
  function capturingEmbedFn(seen: string[]) {
    return (texts: string[]): Promise<Float32Array[]> => {
      seen.push(...texts);
      return fakeEmbedFn(texts);
    };
  }

  async function seedWrappablePage(slug: string, title: string): Promise<void> {
    await engine.putPage(slug, { type: 'note', title, compiled_truth: 'seeded' });
    await installFixtureChunks(engine, slug, [
      { chunk_index: 0, chunk_text: 'prose chunk about widgets', chunk_source: 'compiled_truth', token_count: 4 },
      { chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code', token_count: 4 },
    ]);
  }

  test('title-mode page: stale re-embed sends title-wrapped texts; fenced_code stays raw', async () => {
    await seedWrappablePage('wrapped-page', 'Widget Notes');
    await engine.updatePageContextualRetrievalState('wrapped-page', 'default', 'title', 'gen-title');

    const seen: string[] = [];
    const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
    expect(result.embedded).toBe(2);

    expect(seen).toContain('<context>Widget Notes\n</context>\nprose chunk about widgets');
    expect(seen).toContain('const x = 1;'); // fenced_code is NEVER wrapped (D20-T4)

    // D20-T1: the canonical chunk_text is NOT rewritten — wrapping is embed-input-only.
    const chunks = await engine.getChunks('wrapped-page');
    expect(chunks.map((c) => c.chunk_text).sort()).toEqual(['const x = 1;', 'prose chunk about widgets']);
    // Mode stamp unchanged for title-tier pages.
    const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
      `SELECT contextual_retrieval_mode FROM pages WHERE slug = 'wrapped-page'`,
    );
    expect(rows[0].contextual_retrieval_mode).toBe('title');
  });

  test('per_chunk_synopsis page: re-embed applies the title-tier wrapper and restamps honestly', async () => {
    await seedWrappablePage('synopsis-page', 'Synopsis Notes');
    await engine.updatePageContextualRetrievalState('synopsis-page', 'default', 'per_chunk_synopsis', 'gen-synopsis');

    const seen: string[] = [];
    const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
    expect(result.embedded).toBe(2);

    // Synopsis re-generation is a paid backfill concern; the plain re-embed
    // lands at the title tier (the service's own D14 fallback tier)…
    expect(seen).toContain('<context>Synopsis Notes\n</context>\nprose chunk about widgets');
    // …and the stamped mode is updated so it keeps describing the vectors.
    const rows = await engine.executeRaw<{ contextual_retrieval_mode: string }>(
      `SELECT contextual_retrieval_mode FROM pages WHERE slug = 'synopsis-page'`,
    );
    expect(rows[0].contextual_retrieval_mode).toBe('title');
  });

  test('unstamped page (NULL mode) embeds raw chunk_text — convention preserved', async () => {
    await seedWrappablePage('plain-page', 'Plain Notes');
    // No updatePageContextualRetrievalState call: pre-CR page.

    const seen: string[] = [];
    const result = await embedStaleForSource(engine, 'default', { embedFn: capturingEmbedFn(seen) });
    expect(result.embedded).toBe(2);

    expect(seen).toContain('prose chunk about widgets');
    expect(seen.some((t) => t.startsWith('<context>'))).toBe(false);
  });
});

// ────────────────────────────────────────────────────────────────
// #4283 — a backfill must NEVER NULL more embeddings than it can write.
// The incident shape: a worker with a misresolved embedding config (temp
// GBRAIN_HOME → compile-time default model, no API key) saw every page's
// signature as drifted, NULLed 60k embeddings, embedded 0, and reported
// success — 12 runs in a row.
// ────────────────────────────────────────────────────────────────

describe('signature invalidation is probe-gated (#4283)', () => {
  /** Seed a page with N EMBEDDED chunks stamped under `signature`. */
  async function seedEmbeddedPage(slug: string, chunkCount: number, signature: string): Promise<void> {
    await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: `# ${slug}` });
    await installFixtureChunks(engine, slug, Array.from({ length: chunkCount }, (_, i) => ({
      chunk_index: i,
      chunk_text: `chunk ${i} of ${slug}`,
      chunk_source: 'compiled_truth',
      token_count: 4,
      embedding: new Float32Array(1536).fill(0.1),
    })));
    await engine.setPageEmbeddingSignature(slug, { sourceId: 'default', signature });
  }

  async function embeddedCount(): Promise<number> {
    const rows = await engine.executeRaw<{ n: number }>(
      `SELECT count(*)::int AS n FROM content_chunks WHERE embedding IS NOT NULL`,
    );
    return rows[0]!.n;
  }

  test('IRON-RULE: broken embedder + drifted signature → nothing is NULLed, run degrades to NULL-only', async () => {
    await seedEmbeddedPage('p1', 2, 'old:model:1536');
    await seedEmbeddedPage('p2', 2, 'old:model:1536');
    expect(await embeddedCount()).toBe(4);

    const result = await embedStaleForSource(engine, 'default', {
      embeddingSignature: 'fixture-provider:embedding-v1:1280',
      embedFn: async () => { throw new Error('Fixture embedding requires FIXTURE_API_KEY.'); },
    });

    // Pre-fix: all 4 embeddings were stripped and the run reported done.
    expect(await embeddedCount()).toBe(4);
    expect(result.invalidated).toBe(0);
    expect(result.invalidationSkipped).toBe('embedder_probe_failed');
    expect(result.embedded).toBe(0);
  });

  test('wrong-dims probe result (signature lies about the vectors) also skips invalidation', async () => {
    await seedEmbeddedPage('p1', 2, 'old:model:1536');
    const result = await embedStaleForSource(engine, 'default', {
      // Signature claims 1280 dims but the embedder returns 1536-d vectors:
      // every post-NULL upsert would fail, so refuse to NULL at all.
      embeddingSignature: 'fixture-provider:embedding-v1:1280',
      embedFn: fakeEmbedFn,
    });
    expect(await embeddedCount()).toBe(2);
    expect(result.invalidated).toBe(0);
    expect(result.invalidationSkipped).toBe('embedder_probe_failed');
  });

  test('working embedder → probe fires once, drifted chunks are invalidated, re-embedded, and counted', async () => {
    const { EMBED_PROBE_TEXT } = await import('../src/core/embed-stale.ts');
    await seedEmbeddedPage('p1', 3, 'old:model:1536');
    // The re-embed stamps only a signature naming the model the vectors were
    // actually written under (#4825), so the target names the recorded model.
    const target = `${await recordedModel('p1')}:1536`;
    await engine.executeRaw(`UPDATE content_chunks SET model = 'old:model'
      WHERE page_id = (SELECT id FROM pages WHERE slug = 'p1' AND source_id = 'default')`);
    const seen: string[] = [];
    const result = await embedStaleForSource(engine, 'default', {
      embeddingSignature: target,
      embedFn: async (texts) => {
        seen.push(...texts);
        return fakeEmbedFn(texts);
      },
    });
    expect(seen.filter((t) => t === EMBED_PROBE_TEXT).length).toBe(1);
    expect(result.invalidated).toBe(3);
    expect(result.embedded).toBe(3);
    expect(result.invalidationSkipped).toBeUndefined();
    expect(await embeddedCount()).toBe(3);
    const sig = await engine.executeRaw<{ s: string | null }>(
      `SELECT embedding_signature AS s FROM pages WHERE slug = 'p1'`,
    );
    expect(sig[0]!.s).toBe(target);
  });

  test('a drift pre-check that cannot count still invalidates (#7148)', async () => {
    await seedEmbeddedPage('p1', 3, 'old:model:1536');
    const target = (await recordedModel('p1')) + ':1536';
    await engine.executeRaw("UPDATE content_chunks SET model = 'old:model' WHERE page_id = (SELECT id FROM pages WHERE slug = 'p1' AND source_id = 'default')");
    let timedOut = 0;
    let armed = true;
    const timingOut = new Proxy(engine, {
      get(target, key) {
        if (key === 'countStaleChunks') return async (...args: Parameters<PGLiteEngine['countStaleChunks']>) => {
          if (armed) { timedOut++; throw new Error('canceling statement due to statement timeout'); }
          return target.countStaleChunks(...args);
        };
        if (key === 'transaction') armed = false;
        const value = Reflect.get(target, key, target);
        return typeof value === 'function' ? value.bind(target) : value;
      },
    });
    const result = await embedStaleForSource(timingOut, 'default', { embeddingSignature: target, embedFn: fakeEmbedFn });
    expect(result.invalidated).toBe(3);
    expect(result.embedded).toBe(3);
    expect(timedOut).toBe(0);
  });

  test('no signature drift → no probe call (no embed spend on the common path)', async () => {
    const { EMBED_PROBE_TEXT } = await import('../src/core/embed-stale.ts');
    await seedPageWithStaleChunks('a', 2); // NULL embeddings only, no drift
    const seen: string[] = [];
    const result = await embedStaleForSource(engine, 'default', {
      embeddingSignature: 'new:model:1536',
      embedFn: async (texts) => {
        seen.push(...texts);
        return fakeEmbedFn(texts);
      },
    });
    expect(seen.some((t) => t === EMBED_PROBE_TEXT)).toBe(false);
    expect(result.embedded).toBe(2);
    expect(result.invalidated).toBe(0);
  });
});

// ────────────────────────────────────────────────────────────────
// #4246 — staleness keys on the content revision the vector was computed
// from. A chunk whose text was rewritten after embedding (historical writer
// bugs, races) kept its old vector forever: `embedding IS NULL` never
// matched, coverage read 100%, retrieval served the old body.
// ────────────────────────────────────────────────────────────────

describe('content-drift staleness via embedded_text_hash (#4246)', () => {
  test('upsert stamps md5(chunk_text) when an embedding lands; NULL when text changes without one', async () => {
    await engine.putPage('h1', { type: 'note', title: 'h1', compiled_truth: 'seeded' });
    await engine.upsertChunks('h1', [{
      chunk_index: 0, chunk_text: 'original text', chunk_source: 'compiled_truth',
      token_count: 4, embedding: new Float32Array(1536).fill(0.2),
    }]);
    const stamped = await engine.executeRaw<{ h: string | null; m: string }>(
      `SELECT embedded_text_hash AS h, md5(chunk_text) AS m FROM content_chunks
        WHERE page_id = (SELECT id FROM pages WHERE slug = 'h1')`,
    );
    expect(stamped[0]!.h).toBe(stamped[0]!.m);

    // Deferred-embed rewrite (sync noEmbed path): embedding AND hash reset.
    await engine.upsertChunks('h1', [{
      chunk_index: 0, chunk_text: 'rewritten text', chunk_source: 'compiled_truth', token_count: 4,
    }]);
    const reset = await engine.executeRaw<{ h: string | null }>(
      `SELECT embedded_text_hash AS h FROM content_chunks
        WHERE page_id = (SELECT id FROM pages WHERE slug = 'h1')`,
    );
    expect(reset[0]!.h).toBeNull();
  });

  /** Manufacture the damaged state: text rewritten under a kept vector. */
  async function seedDriftedPage(slug: string): Promise<void> {
    await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: 'seeded' });
    await installFixtureChunks(engine, slug, [{
      chunk_index: 0, chunk_text: `old body of ${slug}`, chunk_source: 'compiled_truth',
      token_count: 4, embedding: new Float32Array(1536).fill(0.3),
    }]);
    // Simulate a historical writer that changed text without touching the
    // vector (the reporter's production state: 253 such chunks).
    await engine.executeRaw(
      `UPDATE content_chunks SET chunk_text = 'NEW body, never re-embedded'
        WHERE page_id = (SELECT id FROM pages WHERE slug = $1)`,
      [slug],
    );
  }

  test('IRON-RULE: a drifted chunk is re-embedded from its CURRENT text; coverage stops lying', async () => {
    await seedDriftedPage('drifted');
    // Pre-fix: countStaleChunks = 0 and embed --stale had nothing to do.
    const seen: string[] = [];
    const result = await embedStaleForSource(engine, 'default', {
      embedFn: async (texts) => {
        seen.push(...texts);
        return fakeEmbedFn(texts);
      },
    });
    expect(result.invalidated).toBe(1);
    expect(result.embedded).toBe(1);
    expect(seen).toContain('NEW body, never re-embedded');
    const rows = await engine.executeRaw<{ h: string | null; m: string; is_null: boolean }>(
      `SELECT embedded_text_hash AS h, md5(chunk_text) AS m, (embedding IS NULL) AS is_null
         FROM content_chunks WHERE page_id = (SELECT id FROM pages WHERE slug = 'drifted')`,
    );
    expect(rows[0]!.is_null).toBe(false);
    expect(rows[0]!.h).toBe(rows[0]!.m); // hash restamped against the new text
  });

  test('NULL hash (pre-v133 rows) is grandfathered — no upgrade re-embed spike', async () => {
    await engine.putPage('legacy', { type: 'note', title: 'legacy', compiled_truth: 'seeded' });
    await installFixtureChunks(engine, 'legacy', [{
      chunk_index: 0, chunk_text: 'legacy text', chunk_source: 'compiled_truth',
      token_count: 4, embedding: new Float32Array(1536).fill(0.4),
    }]);
    await engine.executeRaw(`UPDATE content_chunks SET embedded_text_hash = NULL`);
    const n = await engine.invalidateContentDriftEmbeddings({ sourceId: 'default' });
    expect(n).toBe(0);
    const rows = await engine.executeRaw<{ is_null: boolean }>(
      `SELECT (embedding IS NULL) AS is_null FROM content_chunks
        WHERE page_id = (SELECT id FROM pages WHERE slug = 'legacy')`,
    );
    expect(rows[0]!.is_null).toBe(false);
  });

  test('embed_skip pages are never invalidated (nothing would re-embed them)', async () => {
    await seedDriftedPage('skipped');
    await engine.executeRaw(
      `UPDATE pages SET frontmatter = frontmatter || '{"embed_skip": true}'::jsonb WHERE slug = 'skipped'`,
    );
    const n = await engine.invalidateContentDriftEmbeddings({ sourceId: 'default' });
    expect(n).toBe(0);
  });

  test('source-scoped: drift in another source is untouched', async () => {
    await engine.executeRaw(
      `INSERT INTO sources (id, name, config) VALUES ('other-drift', 'other-drift', '{"federated":true}'::jsonb) ON CONFLICT (id) DO NOTHING`,
    );
    await engine.putPage('od', { type: 'note', title: 'od', compiled_truth: 'seeded' }, { sourceId: 'other-drift' });
    await installFixtureChunks(engine, 'od', [{
      chunk_index: 0, chunk_text: 'other old', chunk_source: 'compiled_truth',
      token_count: 4, embedding: new Float32Array(1536).fill(0.5),
    }], { sourceId: 'other-drift' });
    await engine.executeRaw(
      `UPDATE content_chunks SET chunk_text = 'other NEW'
        WHERE page_id = (SELECT id FROM pages WHERE slug = 'od' AND source_id = 'other-drift')`,
    );
    const n = await engine.invalidateContentDriftEmbeddings({ sourceId: 'default' });
    expect(n).toBe(0);
    const m = await engine.invalidateContentDriftEmbeddings({ sourceId: 'other-drift' });
    expect(m).toBe(1);
  });
});

describe('embedStalePages (#4216 phase-end closure)', () => {
  test('embeds ONLY the listed pages, stamps signature on full re-embeds, leaves the backlog alone', async () => {
    const { embedStalePages } = await import('../src/core/embed-stale.ts');
    await seedPageWithStaleChunks('wiki/target-page', 3);
    await seedPageWithStaleChunks('wiki/backlog-page', 2);
    const fakeEmbed = async (texts: string[]) => texts.map(() => new Float32Array(1536).fill(0.1));
    const res = await embedStalePages(engine, ['wiki/target-page'], 'default', {
      embedFn: fakeEmbed,
      embeddingSignature: 'test:model:1536',
    });
    expect(res.pagesProcessed).toBe(1);
    expect(res.embedded).toBeGreaterThan(0);
    const target = await engine.executeRaw<{ n: number }>(
      `SELECT count(*)::int AS n FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
        WHERE p.slug = 'wiki/target-page' AND cc.embedding IS NULL`);
    expect(target[0]!.n).toBe(0);
    // The rest of the backlog is untouched — this is NOT a source sweep.
    const backlog = await engine.executeRaw<{ n: number }>(
      `SELECT count(*)::int AS n FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
        WHERE p.slug = 'wiki/backlog-page' AND cc.embedding IS NULL`);
    expect(backlog[0]!.n).toBeGreaterThan(0);
    const sig = await engine.executeRaw<{ s: string | null }>(
      `SELECT embedding_signature AS s FROM pages WHERE slug = 'wiki/target-page'`);
    expect(sig[0]!.s).toBe('test:model:1536');
  });
});

// ────────────────────────────────────────────────────────────────
// #4825 — listStaleChunks pages by ROW with no page alignment, so a page
// whose chunks straddle a batch boundary is never wholly in one batch. The
// old stamp gate compared the batch subset against the whole page and left
// such pages unstamped forever (a NULL signature is grandfathered).
// ────────────────────────────────────────────────────────────────

describe('signature stamp for a page split across a cursor batch (#4825)', () => {
  test('page straddling the batch boundary is stamped once its last chunk lands', async () => {
    await engine.putPage('probe', { type: 'note', title: 'probe', compiled_truth: 'probe' });
    await installFixtureChunks(engine, 'probe', [{
      chunk_index: 0, chunk_text: 'probe', chunk_source: 'compiled_truth',
      token_count: 1, embedding: new Float32Array(1536).fill(0.1),
    }]);
    const signature = `${await recordedModel('probe')}:1536`;

    await seedPageWithStaleChunks('a', 1);
    await seedPageWithStaleChunks('b', 2);
    // Stale rows drain as (a.0, b.0) then (b.1): 'b' is never whole in one batch.
    const result = await embedStaleForSource(engine, 'default', {
      embedFn: fakeEmbedFn,
      batchSize: 2,
      embeddingSignature: signature,
    });
    expect(result.embedded).toBe(3);

    const sigs = await engine.executeRaw<{ slug: string; s: string | null }>(
      `SELECT slug, embedding_signature AS s FROM pages WHERE slug IN ('a', 'b') ORDER BY slug`,
    );
    expect(sigs.map((r) => r.s)).toEqual([signature, signature]);
  });
});

// ────────────────────────────────────────────────────────────────
// wave review — the provenance stamp resolved the registry-active column on
// EVERY page it stamped (a config round-trip per page per batch, from both
// keyset drains). The drain now resolves it once (`resolveProvenanceStamp`)
// and hands the stamp helper a ProvenanceStamp.
// ────────────────────────────────────────────────────────────────
describe('provenance stamp resolves the active embedding column once per drain', () => {
  test('the config lookups the signature ADDS do not grow with the page count', async () => {
    const orig = engine.executeRaw;
    const calls: string[] = [];
    engine.executeRaw = (async function(this: PGLiteEngine, sql: string, ...rest: any[]) {
      calls.push(sql);
      return orig.call(this, sql, ...(rest as []));
    }) as any;
    try {
      // getChunks & co. resolve the column per call whether or not a
      // signature is set, so measure what the signature adds at a FIXED
      // page count: pre-fix that was one lookup per stamped page.
      const configLookups = async (pages: number, signature?: string): Promise<number> => {
        await resetPgliteState(engine);
        for (let i = 0; i < pages; i++) await seedPageWithStaleChunks(`p${i}`, 1);
        calls.length = 0;
        await embedStaleForSource(engine, 'default', { embedFn: fakeEmbedFn, embeddingSignature: signature });
        return calls.filter((sql) => sql.includes("'search_embedding_column'")).length;
      };
      const sig = 'new:model:1536';
      const oneSigCost = (await configLookups(1, sig)) - (await configLookups(1));
      const fourSigCost = (await configLookups(4, sig)) - (await configLookups(4));
      expect(fourSigCost).toBe(oneSigCost);
    } finally {
      engine.executeRaw = orig;
    }
  });
});