Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.3 KB
Raw Blame
/**
 * PR 4 E round 2 (gbrain-evals#180): the per-page embedding lease (src/core/page-embedding-lease.ts) under contention.
 *
 * 1. Protects: a path that finds a page's lease held gives up at its deadline (or after one TTL) with a busy lease
 *    instead of waiting forever; a holder whose lease was taken over during its provider call installs nothing;
 *    `embedStalePages` reads the page under the lease; a release deletes the expired `embed-page:` rows a dead
 *    holder left (doctor `stale_locks` clears); a holder that throws releases its lease; and activation's legacy-lock
 *    inspection and deactivation's lease blockers leave page embedding leases out, live or expired.
 * 2. Fails when: the wait loop has no bound, the install skips the lease renewal, the drain reads its snapshot before
 *    the lease, expired lease rows stay, or an error path skips the release. Each test holds the lease itself or
 *    steals it inside the provider call, so every contention is forced, never timed.
 * 3. The provider is an injected embedFn or the gateway's transport seam: no network, no spend. PGLite always;
 *    PostgreSQL when DATABASE_URL is set (test/postgres-unit-arms.txt). Serial: the transport and the lease probe are
 *    process-wide.
 */
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { openAppendBrain, type AppendBrain } from './helpers/append-brain.ts';
import { pendingEmbedPage } from './helpers/pending-embed-page.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';
import { __setEmbedTransportForTests } from '../src/core/ai/gateway.ts';
import { resolveActiveEmbeddingColumnFromEngine } from '../src/core/search/embedding-column.ts';
import { chunkGenerationFilter } from '../src/core/search/safe-chunks.ts';
import { acquirePageEmbeddingLease, pageEmbeddingLeaseId, pageEmbeddingLeaseProbe } from '../src/core/page-embedding-lease.ts';
import { embedStaleForSource, embedStalePages } from '../src/core/embed-stale.ts';
import { runPersistenceEffects } from '../src/core/persistence/effects.ts';
import { localHostId } from '../src/core/persistence/identity.ts';
import { installPageProjection, readProjectionSnapshot } from '../src/core/page-state/projections.ts';
import { listStaleLocks, tryAcquireDbLock } from '../src/core/db-lock.ts';
import { inspectLegacyWriterLocks } from '../src/core/persistence/legacy-locks.ts';
import { deactivationBlockers } from '../src/core/persistence/deactivation.ts';
import { checkStaleLocks } from '../src/commands/doctor/checks/routing-federation.ts';

const CHUNKS = ['Alpha example chunk about the acme-example ledger.', 'Beta example chunk about the quarterly report.', 'Gamma example chunk about the roadmap.'];

for (const kind of testBackends()) {
  describe(`page embedding lease under contention (${kind})`, () => {
    let brain: AppendBrain;
    let home: string;
    let dims: number;
    const sent: string[] = [];
    const vector = () => Float32Array.from({ length: dims }, (_, j) => (j === 0 ? 1 : 0));
    /** The injected provider: records each input and answers unit vectors. */
    const embedFn = async (texts: string[]) => { sent.push(...texts); return texts.map(vector); };
    /** The gateway provider the embedding effect calls; `during` runs inside the call, before it answers. */
    const transport = (during?: () => Promise<void>) => __setEmbedTransportForTests((async ({ values }: { values: string[] }) => {
      sent.push(...values);
      await during?.();
      return { embeddings: values.map(() => Array.from(vector())), usage: { tokens: values.length } };
    }) as never);
    const vectorless = async (sourceId: string, slug = 'page') => (await brain.engine.executeRaw<{ chunk_text: string }>(
      `SELECT cc.chunk_text FROM content_chunks cc JOIN pages p ON p.id=cc.page_id
        WHERE p.source_id=$1 AND p.slug=$2 AND cc.embedding IS NULL AND ${chunkGenerationFilter('p', 'cc')} ORDER BY cc.chunk_index`, [sourceId, slug])).map(r => r.chunk_text);
    const leaseRows = async (pageId: number) => brain.engine.executeRaw<{ acquisition_token: string }>(
      'SELECT acquisition_token FROM gbrain_cycle_locks WHERE id=$1', [pageEmbeddingLeaseId(pageId)]);
    const steal = (pageId: number) => brain.engine.executeRaw('UPDATE gbrain_cycle_locks SET acquisition_token=$2::uuid WHERE id=$1', [pageEmbeddingLeaseId(pageId), randomUUID()]);
    const runEffect = async (effectId: string) => {
      // An earlier test's requeued effect (700-1300 ms jittered backoff) can be due again on a slow runner, and the
      // drain claims the oldest due effect first; only the effect under test may be due.
      await brain.engine.executeRaw("UPDATE persistence_effects SET next_attempt_at=now()+interval '1 hour' WHERE state='queued' AND id<>$1", [effectId]);
      await brain.engine.executeRaw('UPDATE persistence_effects SET next_attempt_at=now() WHERE id=$1', [effectId]);
      await runPersistenceEffects(brain.engine, { engine: brain.engine.kind }, { hostId: localHostId(), limit: 1 });
      return (await brain.engine.executeRaw<{ state: string; error_code: string | null }>('SELECT state,error_code FROM persistence_effects WHERE id=$1', [effectId]))[0]!;
    };
    const pending = () => withEnv({ GBRAIN_HOME: home }, () => pendingEmbedPage(brain.engine, CHUNKS));

    beforeAll(async () => {
      brain = await openAppendBrain(kind);
      dims = (await resolveActiveEmbeddingColumnFromEngine(brain.engine)).dimensions;
      home = mkdtempSync(join(tmpdir(), 'gbrain-embed-lease-'));
    }, 600_000);
    afterEach(async () => {
      // A stolen lease row carries a token no holder releases; it would outlive its test until the TTL.
      await brain.engine.executeRaw("DELETE FROM gbrain_cycle_locks WHERE id LIKE 'embed-page:%'");
      sent.length = 0;
      pageEmbeddingLeaseProbe.contended = undefined;
      pageEmbeddingLeaseProbe.waitBoundMs = undefined;
      transport();
    });
    afterAll(async () => {
      await brain.close();
      rmSync(home, { recursive: true, force: true });
    }, 120_000);

    test('a waiter gives up at its deadline with a busy lease, and the holder keeps the page', async () => {
      const { pageId } = await pending();
      await using _holder = await acquirePageEmbeddingLease(brain.engine, pageId);
      const [held] = await leaseRows(pageId);
      let contended = 0;
      pageEmbeddingLeaseProbe.contended = () => { contended++; };
      const waiter = await acquirePageEmbeddingLease(brain.engine, pageId, { deadline: Date.now() + 300 });
      expect(waiter.busy).toBe(true);
      expect(contended).toBeGreaterThan(0);
      expect(await leaseRows(pageId)).toEqual([held!]);
    }, 30_000);

    test('a stale drain skips a page another path holds past the drain deadline: nothing sent, no failure', async () => {
      const { sourceId, pageId } = await pending();
      await using _holder = await acquirePageEmbeddingLease(brain.engine, pageId);
      let contended = 0;
      pageEmbeddingLeaseProbe.contended = () => { contended++; };
      const result = await embedStaleForSource(brain.engine, sourceId, { embedFn, concurrency: 1, deadline: Date.now() + 2_000 });
      expect(contended).toBeGreaterThan(0);
      expect(sent).toEqual([]);
      expect(result.failures ?? 0).toBe(0);
      expect(await vectorless(sourceId)).toEqual(CHUNKS);
    }, 30_000);

    test('the embedding effect gives up after its wait bound and retries with lock_busy, sending nothing', async () => {
      const { sourceId, effectId, pageId } = await pending();
      await using _holder = await acquirePageEmbeddingLease(brain.engine, pageId);
      pageEmbeddingLeaseProbe.waitBoundMs = 300;
      let contended = 0;
      pageEmbeddingLeaseProbe.contended = () => { contended++; };
      const effect = await withEnv({ GBRAIN_HOME: home }, () => runEffect(effectId));
      expect(contended).toBeGreaterThan(0);
      expect(effect).toMatchObject({ state: 'queued', error_code: 'lock_busy' });
      expect(sent).toEqual([]);
      expect(await vectorless(sourceId)).toEqual(CHUNKS);
    }, 30_000);

    test('a stale drain whose lease is taken over during the provider call installs nothing', async () => {
      const { sourceId, pageId } = await pending();
      const result = await embedStalePages(brain.engine, ['page'], sourceId, { embedFn: async texts => { await steal(pageId); return embedFn(texts); } });
      expect(sent.length).toBe(CHUNKS.length);
      expect(result.embedded).toBe(0);
      expect(await vectorless(sourceId)).toEqual(CHUNKS);
    }, 30_000);

    test('the embedding effect whose lease is taken over during the provider call installs nothing and retries', async () => {
      const { sourceId, effectId, pageId } = await pending();
      transport(async () => { await steal(pageId); });
      const effect = await withEnv({ GBRAIN_HOME: home }, () => runEffect(effectId));
      expect(sent.length).toBe(CHUNKS.length);
      expect(effect).toMatchObject({ state: 'queued', error_code: 'lock_busy' });
      expect(await vectorless(sourceId)).toEqual(CHUNKS);
    }, 30_000);

    test('#7998: an earlier effect due first does not run in place of the effect under test', async () => {
      // An earlier test's lock_busy retry is due again after its 0.7-1.3 s backoff, and the drain claims the oldest due effect first.
      const earlier = await pending();
      await brain.engine.executeRaw("UPDATE persistence_effects SET next_attempt_at=now()-interval '1 minute' WHERE id=$1", [earlier.effectId]);
      const { sourceId, effectId, pageId } = await pending();
      transport(async () => { await steal(pageId); });
      const effect = await withEnv({ GBRAIN_HOME: home }, () => runEffect(effectId));
      expect(sent.length).toBe(CHUNKS.length);
      expect(effect).toMatchObject({ state: 'queued', error_code: 'lock_busy' });
      expect(await vectorless(sourceId)).toEqual(CHUNKS);
      expect(await vectorless(earlier.sourceId)).toEqual(CHUNKS);
    }, 30_000);

    test('embedStalePages reads the page under its lease: a rewrite while it waits is what it sends', async () => {
      const { sourceId, pageId } = await pending();
      const REWRITTEN = CHUNKS.map(text => text.replace('example chunk', 'rewritten chunk'));
      const holder = await acquirePageEmbeddingLease(brain.engine, pageId);
      const waiting = new Promise<void>(resolve => { pageEmbeddingLeaseProbe.contended = () => resolve(); });
      const drained = embedStalePages(brain.engine, ['page'], sourceId, { embedFn });
      await waiting;
      const prepared = (await readProjectionSnapshot(brain.engine, 'page', sourceId))!;
      await installPageProjection(brain.engine, prepared, REWRITTEN.map((chunk_text, chunk_index) => ({ chunk_index, chunk_source: 'compiled_truth' as const, chunk_text })), { seal: true });
      await holder[Symbol.asyncDispose]();
      const result = await drained;
      expect(sent.filter(input => CHUNKS.some(text => input.endsWith(text)))).toEqual([]);
      expect(sent.length).toBe(REWRITTEN.length);
      expect(REWRITTEN.every(text => sent.some(input => input.endsWith(text)))).toBe(true);
      expect(result.embedded).toBe(REWRITTEN.length);
      expect(await vectorless(sourceId)).toEqual([]);
    }, 30_000);

    test('a release deletes the expired embed-page rows a dead holder left, so doctor stale_locks clears; other locks stay', async () => {
      const { sourceId } = await pending();
      const dead = 'embed-page:987654321';
      const sync = `gbrain-sync:lease-test-${randomUUID()}`;
      for (const id of [dead, sync]) {
        await brain.engine.executeRaw(`INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at, acquisition_token)
          VALUES ($1, 999999, 'host-a', now() - interval '2 hours', now() - interval '1 hour', now() - interval '1 hour', $2::uuid)`, [id, randomUUID()]);
      }
      expect((await listStaleLocks(brain.engine)).map(l => l.id)).toContain(dead);
      const result = await embedStalePages(brain.engine, ['page'], sourceId, { embedFn });
      expect(result.embedded).toBe(CHUNKS.length);
      const stale = (await listStaleLocks(brain.engine)).map(l => l.id);
      expect(stale.filter(id => id.startsWith('embed-page:'))).toEqual([]);
      expect(stale).toContain(sync);
      await brain.engine.executeRaw('DELETE FROM gbrain_cycle_locks WHERE id=$1', [sync]);
      expect((await checkStaleLocks(brain.engine)).status).toBe('ok');
    }, 30_000);

    test('activation legacy locks and deactivation lease blockers leave page embedding leases out, live or expired; a writer lock still shows', async () => {
      const { pageId } = await pending();
      const expired = 'embed-page:987654322';
      await brain.engine.executeRaw(`INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at, acquisition_token)
        VALUES ($1, 999999, 'host-a', now() - interval '2 hours', now() - interval '1 hour', now() - interval '1 hour', $2::uuid)`, [expired, randomUUID()]);
      const writer = (await tryAcquireDbLock(brain.engine, `gbrain-sync:lease-test-${randomUUID()}`))!;
      try {
        await using _live = await acquirePageEmbeddingLease(brain.engine, pageId);
        expect((await brain.engine.executeRaw<{ id: string }>("SELECT id FROM gbrain_cycle_locks WHERE id LIKE 'embed-page:%' ORDER BY id")).map(r => r.id))
          .toEqual([pageEmbeddingLeaseId(pageId), expired].sort());
        expect((await inspectLegacyWriterLocks(brain.engine)).map(l => l.id)).toEqual([writer.id]);
        expect((await deactivationBlockers(brain.engine)).filter(b => b.kind === 'lease').map(b => b.id)).toEqual([writer.id]);
      } finally {
        await writer.release();
        await brain.engine.executeRaw('DELETE FROM gbrain_cycle_locks WHERE id=$1', [expired]);
      }
    }, 30_000);

    test('a holder that throws releases its lease: the drain, the effect and a bare holder', async () => {
      const failing = async (): Promise<Float32Array[]> => { throw new Error('provider refused'); };
      const drainPage = await pending();
      expect((await embedStalePages(brain.engine, ['page'], drainPage.sourceId, { embedFn: failing })).embedded).toBe(0);
      expect(await leaseRows(drainPage.pageId)).toEqual([]);

      const effectPage = await pending();
      __setEmbedTransportForTests((async () => { throw new Error('provider refused'); }) as never);
      const effect = await withEnv({ GBRAIN_HOME: home }, () => runEffect(effectPage.effectId));
      expect(effect.state).not.toBe('committed');
      expect(await leaseRows(effectPage.pageId)).toEqual([]);

      const barePage = await pending();
      await expect((async () => {
        await using _lease = await acquirePageEmbeddingLease(brain.engine, barePage.pageId);
        throw new Error('holder failed');
      })()).rejects.toThrow('holder failed');
      expect(await leaseRows(barePage.pageId)).toEqual([]);

      let contended = 0;
      pageEmbeddingLeaseProbe.contended = () => { contended++; };
      for (const { pageId } of [drainPage, effectPage, barePage]) {
        await using _next = await acquirePageEmbeddingLease(brain.engine, pageId, { deadline: Date.now() });
        expect((await leaseRows(pageId)).length).toBe(1);
      }
      expect(contended).toBe(0);
    }, 30_000);
  });
}