Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.4 KB
Raw Blame
/**
 * #7172: a managed timeline entry keeps the passages it did not change, and its publication stays within a bounded
 * number of vector-index writes.
 *
 * 1. Protects: an entry appended at the end of a page whose stored page signature is stale (NULL, or an older format)
 *    keeps every unchanged passage row and its vector, and the embedding effect embeds only the passages whose text
 *    changed (effects.ts readEmbeddingEffectProjection accepts a vector whose recorded input is current and restamps
 *    the page); an entry at the top of a descending timeline keeps every passage whose text is unchanged under its
 *    old id, renumbered (import-file.ts, page-state/chunk-moves.ts), and the page ends with the passages a fresh
 *    import of the same body gives; a publication whose replacement needs more than CHUNK_VECTOR_WRITE_BUDGET
 *    vector-index writes commits with its projection pending and finishes in bounded post-commit steps
 *    (page-state/projections.ts continueDeferredProjection), resuming after an interrupted step.
 * 2. Fails when: a stale page signature re-embeds every passage; a passage whose index moves is deleted and
 *    re-inserted; a top insert publishes every moved passage inside the publication's 5 s statement bound (the
 *    PostgreSQL probe charges each vector-index write 0.6 s and shows the write ending 57014 / write_interrupted).
 * 3. PGLite always; PostgreSQL when DATABASE_URL is set (test/postgres-unit-arms.txt). Work is counted, never timed,
 *    except the PostgreSQL probe's injected per-row delay.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { openAppendBrain, type AppendBrain } from './helpers/append-brain.ts';
import { testBackends } from './helpers/test-backends.ts';
import { runEmbedCore } from '../src/commands/embed.ts';
import { currentEmbeddingSignature } from '../src/core/embedding.ts';
import { DEFERRED_CHUNK_CLAIM, rebuildPendingPageProjections } from '../src/core/page-state/projections.ts';
import { CHUNK_VECTOR_WRITE_BUDGET } from '../src/core/page-state/chunk-moves.ts';

const WORDS = 'alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa quebec romeo sierra tango uniform victor whiskey xray yankee zulu'.split(' ');
function prose(seed: number, words: number): string {
  const out: string[] = [];
  for (let i = 0; i < words; i++) out.push(WORDS[(seed * 7 + i * 13 + (i % 5)) % WORDS.length]! + (i % 17 === 16 ? '.' : ''));
  return out.join(' ');
}
/** Compiled truth of `truth` paragraphs and a timeline of `bullets` dated bullets of 15 to 84 words, ascending or descending. */
function page(truth: number, bullets: number, order: 'ascending' | 'descending'): string {
  const body = Array.from({ length: truth }, (_, i) => prose(i + 1, 120)).join('\n\n');
  const lines = Array.from({ length: bullets }, (_, i) => {
    const day = order === 'ascending' ? i : bullets - 1 - i;
    return `- **${new Date(Date.UTC(2025, 0, 1 + day)).toISOString().slice(0, 10)}** | manual — Example event ${day} ${prose(day + 100, 15 + (day * 37) % 70)}`;
  });
  return `---\ntitle: Example page\ntype: note\n---\n\n${body}\n\n<!-- timeline -->\n\n## Timeline\n\n${lines.join('\n')}\n`;
}

interface Row { id: number; chunk_index: number; chunk_source: string; chunk_text: string; embedded_at: string | null; vector: boolean }

for (const kind of testBackends()) {
  describe(`#7172 timeline chunk reuse (${kind})`, () => {
    let brain: AppendBrain;
    beforeAll(async () => { brain = await openAppendBrain(kind); }, 600_000);
    afterAll(() => brain.close(), 120_000);

    const rows = (slug: string) => brain.engine.executeRaw<Row>(`SELECT c.id, c.chunk_index, c.chunk_source, c.chunk_text,
        c.embedded_at::text AS embedded_at, c.embedding IS NOT NULL AS vector
      FROM content_chunks c JOIN pages p ON p.id=c.page_id WHERE p.source_id='default' AND p.slug=$1 ORDER BY c.chunk_index`, [slug]);
    const pageState = async (slug: string) => (await brain.engine.executeRaw<{ sealed: boolean; signature: string | null }>(
      `SELECT text_projection_revision IS NOT DISTINCT FROM knowledge_revision AS sealed, embedding_signature AS signature
       FROM pages WHERE source_id='default' AND slug=$1`, [slug]))[0]!;
    /** Until no persistence effect is queued or running (the background consumer drains them). */
    const settle = async () => {
      for (let i = 0; i < 600; i++) {
        const [row] = await brain.engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM persistence_effects WHERE state IN ('queued','running')");
        if (Number(row!.n) === 0) return;
        await new Promise(resolve => setTimeout(resolve, 100));
      }
      throw new Error('persistence effects did not drain');
    };
    /** A page published through put_page, every passage embedded and the effects drained. */
    const publish = async (slug: string, content: string) => {
      const put = await brain.call('put_page', { slug, content, request_id: randomUUID() });
      expect(put.isError).toBe(false);
      await settle();
      await runEmbedCore(brain.engine, { stale: true, quiet: true } as never);
      await settle();
      const stored = await rows(slug);
      expect(stored.every(row => row.vector)).toBe(true);
      expect((await pageState(slug)).sealed).toBe(true);
      return stored;
    };
    const addEntry = (slug: string, date: string, summary: string) =>
      brain.call('add_timeline_entry', { slug, date, summary, source: 'manual', request_id: randomUUID() });
    const bodyOf = async (slug: string) => {
      const stored = await brain.body(slug);
      return `---\ntitle: Example page\ntype: note\n---\n\n${stored.truth}\n\n<!-- timeline -->\n\n${stored.timeline}\n`;
    };
    /** The passages a fresh import of the page's current body gives, as (index, source, text). */
    const freshPassages = async (slug: string) => {
      const twin = `notes/twin-${randomUUID().slice(0, 8)}`;
      await brain.seed(twin, await bodyOf(slug));
      return (await rows(twin)).map(row => [row.chunk_index, row.chunk_source, row.chunk_text]);
    };
    const passages = (stored: Row[]) => stored.map(row => [row.chunk_index, row.chunk_source, row.chunk_text]);
    /** A long entry: at the top of these timelines it adds one passage there and shifts every passage below by one. */
    const LONG = `Example long entry ${prose(7, 450)}`;

    for (const stale of ['null', 'older-format'] as const) {
      test(`an entry at the end keeps every unchanged passage and its vector when the page signature is ${stale}`, async () => {
        const slug = `notes/example-stale-${stale}`;
        const before = await publish(slug, page(24, 60, 'ascending'));
        await brain.engine.executeRaw('UPDATE pages SET embedding_signature=$2 WHERE source_id=$3 AND slug=$1',
          [slug, stale === 'null' ? null : 'text-embedding-3-large:1536', 'default']);
        brain.embedded.length = 0;
        const added = await addEntry(slug, '2026-10-19', 'Example entry at the end');
        expect(added.isError).toBe(false);
        await settle();
        const after = await rows(slug);
        const old = new Map(before.map(row => [row.id, row]));
        const kept = after.filter(row => old.has(row.id));
        const unchanged = after.filter(row => before.some(prior => prior.chunk_source === row.chunk_source && prior.chunk_text === row.chunk_text));
        expect(kept.length).toBe(unchanged.length);
        expect(kept.length).toBeGreaterThanOrEqual(before.length - 1);
        // A kept passage keeps its vector: never embedded again, so its embedding time is the old one.
        for (const row of kept) expect(row.embedded_at).toBe(old.get(row.id)!.embedded_at);
        const oldTexts = new Set(before.map(row => row.chunk_text));
        expect(brain.embedded.filter(text => oldTexts.has(text))).toEqual([]);
        expect(brain.embedded.length).toBe(after.length - kept.length);
        expect(after.every(row => row.vector)).toBe(true);
        const state = await pageState(slug);
        expect(state.sealed).toBe(true);
        expect(state.signature).toBe(currentEmbeddingSignature());
      }, 600_000);
    }

    test('an entry at the top of a descending timeline keeps every unchanged passage under its id, renumbered', async () => {
      const slug = 'notes/example-descending';
      const before = await publish(slug, page(12, 380, 'descending'));
      expect(before.filter(row => row.chunk_source === 'timeline').length).toBeGreaterThan(40);
      brain.embedded.length = 0;
      const added = await addEntry(slug, '2026-10-19', LONG);
      expect(added.isError).toBe(false);
      await settle();
      const after = await rows(slug);
      const old = new Map(before.map(row => [row.id, row]));
      const kept = after.filter(row => old.has(row.id));
      const key = (row: Row) => `${row.chunk_source}\0${row.chunk_text}`;
      const beforeKeys = new Set(before.map(key));
      const unchanged = after.filter(row => beforeKeys.has(key(row)));
      console.log(`[${kind}] top insert: ${before.length} -> ${after.length} passages, ${unchanged.length} unchanged, ${kept.length} kept, ${brain.embedded.length} embedded`);
      // Every passage whose text is unchanged keeps its row (id, vector, embedding time); only the others are new.
      expect(kept.map(key).sort()).toEqual(unchanged.map(key).sort());
      expect(kept.filter(row => row.chunk_index !== old.get(row.id)!.chunk_index).length).toBeGreaterThan(CHUNK_VECTOR_WRITE_BUDGET);
      for (const row of kept) expect(row.embedded_at).toBe(old.get(row.id)!.embedded_at);
      expect(brain.embedded.length).toBe(after.length - kept.length);
      expect(after.every(row => row.vector)).toBe(true);
      expect((await pageState(slug)).sealed).toBe(true);
      expect(passages(after)).toEqual(await freshPassages(slug));
    }, 600_000);

    test('an interrupted continuation leaves the entry committed and the drain finishes it', async () => {
      const slug = 'notes/example-interrupted';
      await publish(slug, page(12, 380, 'descending'));
      // Every chunk_index move past the first CHUNK_VECTOR_WRITE_BUDGET fails, as a crash after the first step would, and
      // the reason each continuation leaves when it hands its claim back is recorded.
      await brain.engine.executeRaw(`CREATE TABLE IF NOT EXISTS probe_7172_moves (n int)`);
      await brain.engine.executeRaw(`CREATE TABLE IF NOT EXISTS probe_7172_handbacks (reason text)`);
      await brain.engine.executeRaw(`CREATE OR REPLACE FUNCTION probe_7172_interrupt() RETURNS trigger LANGUAGE plpgsql AS $$
        BEGIN
          IF NEW.chunk_index IS DISTINCT FROM OLD.chunk_index THEN
            INSERT INTO probe_7172_moves VALUES (1);
            IF (SELECT count(*) FROM probe_7172_moves) > ${CHUNK_VECTOR_WRITE_BUDGET} THEN RAISE EXCEPTION 'probe_7172 interrupted'; END IF;
          END IF;
          RETURN NEW;
        END $$`);
      await brain.engine.executeRaw('CREATE TRIGGER probe_7172_interrupt BEFORE UPDATE ON content_chunks FOR EACH ROW EXECUTE FUNCTION probe_7172_interrupt()');
      await brain.engine.executeRaw(`CREATE OR REPLACE FUNCTION probe_7172_handback() RETURNS trigger LANGUAGE plpgsql AS $$
        BEGIN
          IF split_part(OLD.reason, ':', 1) = '${DEFERRED_CHUNK_CLAIM}' AND NEW.reason IS DISTINCT FROM OLD.reason THEN
            INSERT INTO probe_7172_handbacks VALUES (NEW.reason);
          END IF;
          RETURN NEW;
        END $$`);
      await brain.engine.executeRaw('CREATE TRIGGER probe_7172_handback AFTER UPDATE ON page_projection_jobs FOR EACH ROW EXECUTE FUNCTION probe_7172_handback()');
      let added;
      let sealed: boolean | undefined;
      let handbacks: Array<{ reason: string }> = [];
      try {
        added = await addEntry(slug, '2026-10-19', LONG);
        // The resident's drain resumes the job once the interruption is gone, so the page is read while it is in place,
        // after the publication's continuation has handed its claim back (a reply settled at commit precedes it).
        for (let i = 0; i < 600 && !(handbacks = await brain.engine.executeRaw<{ reason: string }>('SELECT reason FROM probe_7172_handbacks')).length; i++) {
          await new Promise(resolve => setTimeout(resolve, 100));
        }
        sealed = (await pageState(slug)).sealed;
      } finally {
        await brain.engine.executeRaw('DROP TRIGGER IF EXISTS probe_7172_interrupt ON content_chunks');
        await brain.engine.executeRaw('DROP FUNCTION IF EXISTS probe_7172_interrupt()');
        await brain.engine.executeRaw('DROP TABLE IF EXISTS probe_7172_moves');
        await brain.engine.executeRaw('DROP TRIGGER IF EXISTS probe_7172_handback ON page_projection_jobs');
        await brain.engine.executeRaw('DROP FUNCTION IF EXISTS probe_7172_handback()');
        await brain.engine.executeRaw('DROP TABLE IF EXISTS probe_7172_handbacks');
      }
      expect(added!.isError).toBe(false);
      // The entry is committed; the passages are not sealed yet, and the job names the deferred replacement.
      expect((await brain.body(slug)).timeline).toContain('Example long entry');
      expect(sealed).toBe(false);
      expect(handbacks.map(row => row.reason)).toEqual(['chunk_replacement_deferred']);
      // The drain resumes it from the stored rows: this call, or the resident's drain when it got there first (it holds the claim).
      await rebuildPendingPageProjections(brain.engine, 10);
      for (let i = 0; i < 600 && !(await pageState(slug)).sealed; i++) await new Promise(resolve => setTimeout(resolve, 100));
      await settle();
      await runEmbedCore(brain.engine, { stale: true, quiet: true } as never);
      const after = await rows(slug);
      expect((await pageState(slug)).sealed).toBe(true);
      expect(after.every(row => row.vector)).toBe(true);
      expect(passages(after)).toEqual(await freshPassages(slug));
    }, 600_000);

    if (kind === 'postgres') {
      test('probe: with each vector-index write charged 0.6 s, a top insert commits and no transaction makes more than the budget', async () => {
        const slug = 'notes/example-slow-index';
        await publish(slug, page(12, 380, 'descending'));
        // Each new row version that adds an HNSW entry (one carrying a vector: an insert, or an update of chunk_index, the
        // text or the vector, none of them HOT) costs 0.6 s, as on a large uncached index, and is counted per transaction.
        // A version with a NULL vector adds no HNSW entry and is not charged (chunk-moves.ts measures both).
        await brain.engine.executeRaw('CREATE TABLE probe_7172_writes (tx bigint)');
        await brain.engine.executeRaw(`CREATE OR REPLACE FUNCTION probe_7172_slow() RETURNS trigger LANGUAGE plpgsql AS $$
          BEGIN
            IF NEW.embedding IS NOT NULL AND (TG_OP = 'INSERT' OR NEW.chunk_index IS DISTINCT FROM OLD.chunk_index
              OR NEW.chunk_text IS DISTINCT FROM OLD.chunk_text OR NEW.embedding IS DISTINCT FROM OLD.embedding) THEN
              INSERT INTO probe_7172_writes VALUES (txid_current());
              PERFORM pg_sleep(0.6);
            END IF;
            RETURN NEW;
          END $$`);
        await brain.engine.executeRaw('CREATE TRIGGER probe_7172_slow BEFORE INSERT OR UPDATE ON content_chunks FOR EACH ROW EXECUTE FUNCTION probe_7172_slow()');
        let receipt: Record<string, any> = {};
        let interruptions: number | undefined;
        let writes: Array<{ n: number }> = [];
        try {
          const added = await addEntry(slug, '2026-10-19', LONG);
          const requestId = added.body.write_request?.request_id ?? added.body.request_id;
          // The reply waits a bounded time; the receipt says how the write ended.
          receipt = added.body;
          const state = () => receipt.write_request?.state ?? receipt.state;
          for (let i = 0; i < 1500 && state() !== 'committed' && state() !== 'failed'; i++) {
            await new Promise(resolve => setTimeout(resolve, 200));
            receipt = (await brain.call('get_write_request', { request_id: requestId })).body;
          }
          for (let i = 0; i < 1500 && state() === 'committed' && !(await pageState(slug)).sealed; i++) await new Promise(resolve => setTimeout(resolve, 200));
          [{ transient_interruptions: interruptions }] = await brain.engine.executeRaw<{ transient_interruptions: number }>(
            'SELECT transient_interruptions FROM persistence_requests WHERE request_id=$1', [requestId]);
          writes = await brain.engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM probe_7172_writes GROUP BY tx ORDER BY 1 DESC');
        } finally {
          await brain.engine.executeRaw('DROP TRIGGER IF EXISTS probe_7172_slow ON content_chunks');
          await brain.engine.executeRaw('DROP FUNCTION IF EXISTS probe_7172_slow()');
          await brain.engine.executeRaw('DROP TABLE IF EXISTS probe_7172_writes');
        }
        console.log(`[probe] state ${receipt.write_request?.state ?? receipt.state}, interruptions ${interruptions}, vector-index writes per transaction ${JSON.stringify(writes.map(w => w.n))}`);
        if ((receipt.write_request?.state ?? receipt.state) !== 'committed') console.log(JSON.stringify(receipt).slice(0, 1200));
        expect(receipt.write_request?.state ?? receipt.state).toBe('committed');
        expect(Number(interruptions)).toBe(0);
        expect(Number(writes[0]?.n ?? 0)).toBeLessThanOrEqual(CHUNK_VECTOR_WRITE_BUDGET);
        await settle();
        expect((await pageState(slug)).sealed).toBe(true);
        expect(passages(await rows(slug))).toEqual(await freshPassages(slug));
      }, 900_000);
    }
  });
}