Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.2 KB
Raw Blame
/**
 * src/eval/shared/embed-cache.ts — hermetic pins.
 *
 * No network, no API spend: the "real" transport is a counting fake. bun:sqlite
 * files live under a per-test tmp dir. The gateway is configured explicitly
 * (openai:text-embedding-3-large @ 4 dims) so the install path resolves a
 * known model@dims without touching the preload's baseline beyond resetGateway.
 */

import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { Database } from 'bun:sqlite';
import { mkdtempSync, rmSync, existsSync, statSync, readFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
  EmbeddingCache,
  EmbedCacheIntegrityError,
  installEmbedCache,
  sideFromParams,
} from '../src/eval/shared/embed-cache.ts';
import {
  configureGateway,
  resetGateway,
  embed,
  embedQuery,
  __setEmbedTransportForTests,
} from '../src/core/ai/gateway.ts';

const MODEL = 'openai:text-embedding-3-large';
const DIMS = 4;

let dir: string;
beforeEach(() => {
  dir = mkdtempSync(join(tmpdir(), 'gbrain-embed-cache-'));
});
afterEach(() => {
  __setEmbedTransportForTests(null);
  resetGateway();
  rmSync(dir, { recursive: true, force: true });
});

/** Binary-exact values (multiples of 1/8) so the Float32 round-trip is lossless. */
function vec(seed: number, dims = DIMS): number[] {
  return Array.from({ length: dims }, (_, i) => (seed + i) / 8);
}

describe('EmbeddingCache — get/put/stats', () => {
  test('miss then hit, stats count both, path reported', () => {
    const c = new EmbeddingCache(join(dir, 'a.sqlite'));
    c.open();
    expect(c.get(MODEL, DIMS, 'hello')).toBeNull();
    c.put(MODEL, DIMS, 'hello', vec(1));
    expect(c.get(MODEL, DIMS, 'hello')).toEqual(vec(1));
    const s = c.stats();
    expect(s).toEqual({ hits: 1, misses: 1, bypassed: 0, infra_faults: 0, path: join(dir, 'a.sqlite') });
    expect(c.size()).toBe(1);
    c.close();
  });

  test('key is model@dims:sha256(text); dims/model/side changes are different keys', () => {
    const c = new EmbeddingCache(join(dir, 'k.sqlite'));
    const k = c.key(MODEL, 1536, 'x');
    expect(k).toMatch(new RegExp(`^${MODEL.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}@1536:[0-9a-f]{64}$`));
    expect(c.key(MODEL, 1024, 'x')).not.toBe(k);
    expect(c.key('voyage:voyage-4', 1536, 'x')).not.toBe(k);
    expect(c.key(MODEL, 1536, 'y')).not.toBe(k);
    expect(c.key(MODEL, 1536, 'x', 'query')).toContain('#query:');
    expect(c.key(MODEL, 1536, 'x', 'query')).not.toBe(k);
    // side-aware storage: a document-side vector is never served for a query-side lookup
    c.open();
    c.put(MODEL, DIMS, 'x', vec(1), 'document');
    expect(c.get(MODEL, DIMS, 'x', 'query')).toBeNull();
    expect(c.get(MODEL, DIMS, 'x', 'document')).toEqual(vec(1));
    c.close();
  });

  test('WAL journal mode + synchronous=NORMAL are set', () => {
    const c = new EmbeddingCache(join(dir, 'w.sqlite'));
    c.open();
    const raw = new Database(join(dir, 'w.sqlite'));
    expect((raw.query<{ journal_mode: string }, []>('PRAGMA journal_mode').get()?.journal_mode ?? '').toLowerCase()).toBe('wal');
    raw.close();
    c.close();
  });

  test('operations before open() throw a clear error', () => {
    const c = new EmbeddingCache(join(dir, 'closed.sqlite'));
    expect(() => c.get(MODEL, DIMS, 'x')).toThrow(/not open/);
  });
});

describe('EmbeddingCache — integrity (hard errors)', () => {
  test('put with a vector whose length != dims throws naming file and key', () => {
    const path = join(dir, 'i.sqlite');
    const c = new EmbeddingCache(path);
    c.open();
    let err: unknown;
    try {
      c.put(MODEL, DIMS, 'short', [0.1, 0.2]);
    } catch (e) {
      err = e;
    }
    expect(err).toBeInstanceOf(EmbedCacheIntegrityError);
    expect((err as Error).message).toContain(path);
    expect((err as Error).message).toContain(c.key(MODEL, DIMS, 'short'));
    expect(c.size()).toBe(0);
    c.close();
  });

  test('a stored row whose byte length disagrees with its dims is a hard error on read', () => {
    const path = join(dir, 'corrupt.sqlite');
    const c = new EmbeddingCache(path);
    c.open();
    c.put(MODEL, DIMS, 'ok', vec(2));
    // Tamper: declare 8 dims on a 4-dim blob, under the 8-dim key.
    const badKey = c.key(MODEL, 8, 'ok');
    const raw = new Database(path);
    raw
      .query('INSERT INTO embed_cache (key, model, dims, byte_len, vector) VALUES (?, ?, ?, ?, ?)')
      .run(badKey, MODEL, 8, 32, new Uint8Array(16));
    raw.close();
    expect(() => c.get(MODEL, 8, 'ok')).toThrow(EmbedCacheIntegrityError);
    try {
      c.get(MODEL, 8, 'ok');
    } catch (e) {
      expect((e as Error).message).toContain(path);
      expect((e as Error).message).toContain(badKey);
    }
    // the healthy row is still fine
    expect(c.get(MODEL, DIMS, 'ok')).toEqual(vec(2));
    c.close();
  });
});

describe('EmbeddingCache — canonical hash', () => {
  test('stable across close/reopen and independent of WAL state + insertion order', () => {
    const pathA = join(dir, 'h1.sqlite');
    const a = new EmbeddingCache(pathA);
    a.open();
    a.put(MODEL, DIMS, 't1', vec(1));
    a.put(MODEL, DIMS, 't2', vec(2));
    a.put(MODEL, DIMS, 't3', vec(3));
    // Before any checkpoint the WAL sidecar carries the rows.
    const walBefore = existsSync(`${pathA}-wal`) ? statSync(`${pathA}-wal`).size : 0;
    const h1 = a.canonicalSha256();
    // checkpoint(TRUNCATE) folded the WAL into the main file
    const walAfter = existsSync(`${pathA}-wal`) ? statSync(`${pathA}-wal`).size : 0;
    expect(walAfter).toBeLessThanOrEqual(walBefore);
    a.close();

    const a2 = new EmbeddingCache(pathA);
    a2.open();
    expect(a2.canonicalSha256()).toBe(h1);
    a2.close();

    // Same rows, different insertion order, different file → same canonical hash.
    const b = new EmbeddingCache(join(dir, 'h2.sqlite'));
    b.open();
    b.put(MODEL, DIMS, 't3', vec(3));
    b.put(MODEL, DIMS, 't1', vec(1));
    b.put(MODEL, DIMS, 't2', vec(2));
    expect(b.canonicalSha256()).toBe(h1);
    // Any content change moves it.
    b.put(MODEL, DIMS, 't4', vec(4));
    expect(b.canonicalSha256()).not.toBe(h1);
    b.close();
  });

  test('canonical hash VALUE is pinned: streaming rows (iterate) hashes exactly what the eager form did', () => {
    // Literal computed on the pre-streaming implementation (`.all()` +
    // readFileSync) over this exact fixture. The streaming rewrite must not
    // move it: same ORDER BY key, same `key \0 dims \0 sha256(vector) \n`
    // input, one row at a time.
    const pinVec = (seed: number): number[] => [seed, seed / 2, -seed, 0.25 * seed];
    const c = new EmbeddingCache(join(dir, 'pin.sqlite'));
    c.open();
    c.put(MODEL, DIMS, 't1', pinVec(1));
    c.put(MODEL, DIMS, 't2', pinVec(2));
    c.put(MODEL, DIMS, 't3', pinVec(3));
    c.put(MODEL, DIMS, 'q1', pinVec(1), 'query');
    expect(c.canonicalSha256()).toBe('4465dd82365a2ec400a62f8825ec1fa2a40325bbfed764ecd36e94fb8510ccd9');
    // …and the documented formula recomputed independently agrees.
    const rows = new Database(c.path)
      .query<{ key: string; dims: number; vector: Uint8Array }, []>('SELECT key, dims, vector FROM embed_cache ORDER BY key')
      .all();
    expect(rows.length).toBe(4);
    const h = createHash('sha256');
    for (const r of rows) {
      const vh = createHash('sha256').update(r.vector).digest('hex');
      h.update(`${r.key}\0${r.dims}\0${vh}\n`);
    }
    expect(h.digest('hex')).toBe('4465dd82365a2ec400a62f8825ec1fa2a40325bbfed764ecd36e94fb8510ccd9');
    // fileSha256 streams the file in chunks; it must equal a whole-file digest.
    expect(c.fileSha256()).toBe(createHash('sha256').update(readFileSync(c.path)).digest('hex'));
    c.close();
  });

  test('the module source carries no raw NUL bytes (the separators are the \\0 escape)', () => {
    // A literal 0x00 inside a string literal made file(1) classify the module
    // as data and `grep -I` skip it. Same hash input, escaped spelling.
    const src = readFileSync(join(import.meta.dir, '..', 'src', 'eval', 'shared', 'embed-cache.ts'), 'utf-8');
    expect(src.includes('\0')).toBe(false);
    expect(src).toContain(".update('\\0')");
  });

  test('empty cache hashes deterministically; fileSha256 is a hex digest', () => {
    const c = new EmbeddingCache(join(dir, 'e.sqlite'));
    c.open();
    const h = c.canonicalSha256();
    expect(h).toMatch(/^[0-9a-f]{64}$/);
    expect(c.fileSha256()).toMatch(/^[0-9a-f]{64}$/);
    c.close();
    const d = new EmbeddingCache(join(dir, 'e2.sqlite'));
    d.open();
    expect(d.canonicalSha256()).toBe(h);
    d.close();
  });
});

describe('EmbeddingCache — withTransaction', () => {
  test('batches puts; a throwing body rolls back; nesting uses savepoints', async () => {
    const c = new EmbeddingCache(join(dir, 'tx.sqlite'));
    c.open();
    await c.withTransaction(() => {
      for (let i = 0; i < 20; i++) c.put(MODEL, DIMS, `q${i}`, vec(i));
    });
    expect(c.size()).toBe(20);

    await expect(
      c.withTransaction(async () => {
        c.put(MODEL, DIMS, 'rolled', vec(99));
        await Promise.resolve();
        throw new Error('boom');
      }),
    ).rejects.toThrow('boom');
    expect(c.size()).toBe(20);
    expect(c.get(MODEL, DIMS, 'rolled')).toBeNull();

    // nested: inner rollback does not lose the outer's writes
    await c.withTransaction(async () => {
      c.put(MODEL, DIMS, 'outer', vec(5));
      try {
        await c.withTransaction(() => {
          c.put(MODEL, DIMS, 'inner', vec(6));
          throw new Error('inner-fail');
        });
      } catch {
        /* expected */
      }
    });
    expect(c.get(MODEL, DIMS, 'outer')).toEqual(vec(5));
    expect(c.get(MODEL, DIMS, 'inner')).toBeNull();
    c.close();
  });
});

describe('installEmbedCache — gateway seam', () => {
  function fakeTransport(dims = DIMS) {
    const calls: string[][] = [];
    const fn = (async (params: { values: string[] }) => {
      calls.push([...params.values]);
      return {
        embeddings: params.values.map((v) => vec(v.length, dims)),
        values: params.values,
        warnings: [],
        usage: { tokens: params.values.length },
      };
    }) as unknown as Parameters<typeof __setEmbedTransportForTests>[0];
    return { calls, fn: fn! };
  }

  beforeEach(() => {
    configureGateway({
      embedding_model: MODEL,
      embedding_dimensions: DIMS,
      env: { OPENAI_API_KEY: 'sk-fake' },
    });
  });

  test('serves hits from the cache and batches only misses to the real transport', async () => {
    const cache = new EmbeddingCache(join(dir, 'install.sqlite'));
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    expect(inst.model).toBe(MODEL);
    expect(inst.dims).toBe(DIMS);

    const first = await embed(['aa', 'bbb']);
    expect(real.calls).toEqual([['aa', 'bbb']]);
    expect(first.map((f) => Array.from(f))).toEqual([vec(2), vec(3)]);

    const second = await embed(['aa', 'bbb']);
    expect(real.calls.length).toBe(1); // all hits, no transport call
    expect(second.map((f) => Array.from(f))).toEqual([vec(2), vec(3)]);

    const third = await embed(['aa', 'cccc', 'bbb']);
    expect(real.calls.length).toBe(2);
    expect(real.calls[1]).toEqual(['cccc']); // only the miss went out
    expect(third.map((f) => Array.from(f))).toEqual([vec(2), vec(4), vec(3)]);

    const s = cache.stats();
    expect(s.hits).toBe(4);
    expect(s.misses).toBe(3);
    expect(s.bypassed).toBe(0);
    expect(cache.size()).toBe(3);
    inst.uninstall();
    cache.close();
  });

  test('vectors are keyed under the resolved model@dims (a different embedder cannot read them)', async () => {
    const cache = new EmbeddingCache(join(dir, 'keyed.sqlite'));
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    await embed(['aa']);
    inst.uninstall();
    expect(cache.get(MODEL, DIMS, 'aa')).toEqual(vec(2));
    expect(cache.get('voyage:voyage-4', DIMS, 'aa')).toBeNull();
    expect(cache.get(MODEL, 1536, 'aa')).toBeNull();
    cache.close();
  });

  test('query-side embeds are cached under the query key, not the document key', async () => {
    const cache = new EmbeddingCache(join(dir, 'side.sqlite'));
    // Use an asymmetric provider so the gateway emits input_type (Voyage's
    // smallest supported Matryoshka dims is 256).
    const VOYAGE_DIMS = 256;
    const real = fakeTransport(VOYAGE_DIMS);
    configureGateway({
      embedding_model: 'voyage:voyage-4',
      embedding_dimensions: VOYAGE_DIMS,
      env: { VOYAGE_API_KEY: 'sk-fake' },
    });
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    expect(inst.model).toBe('voyage:voyage-4');
    expect(inst.dims).toBe(VOYAGE_DIMS);
    await embedQuery('aa');
    await embed(['aa']);
    expect(real.calls.length).toBe(2); // the two sides never alias
    expect(cache.get('voyage:voyage-4', VOYAGE_DIMS, 'aa', 'query')).toEqual(vec(2, VOYAGE_DIMS));
    expect(cache.get('voyage:voyage-4', VOYAGE_DIMS, 'aa', 'document')).toEqual(vec(2, VOYAGE_DIMS));
    await embedQuery('aa');
    expect(real.calls.length).toBe(2); // query-side hit
    inst.uninstall();
    cache.close();
  });

  test('sideFromParams reads providerOptions.openaiCompatible.input_type', () => {
    expect(sideFromParams({})).toBe('document');
    expect(sideFromParams({ providerOptions: { openaiCompatible: { input_type: 'query' } } })).toBe('query');
    expect(sideFromParams({ providerOptions: { openaiCompatible: { input_type: 'document' } } })).toBe('document');
  });

  test('a batch whose SDK model id is not the resolved model bypasses the cache', async () => {
    const cache = new EmbeddingCache(join(dir, 'bypass.sqlite'));
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn, model: 'voyage:voyage-4', dims: DIMS });
    await embed(['aa']); // gateway resolves openai:text-embedding-3-large → modelId mismatch
    await embed(['aa']);
    expect(real.calls.length).toBe(2);
    expect(cache.size()).toBe(0);
    expect(cache.stats().bypassed).toBe(2);
    inst.uninstall();
    cache.close();
  });

  test('uninstall restores the prior transport (every embed reaches it directly again)', async () => {
    const cache = new EmbeddingCache(join(dir, 'restore.sqlite'));
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    await embed(['aa']);
    await embed(['aa']);
    expect(real.calls.length).toBe(1);
    inst.uninstall();
    inst.uninstall(); // idempotent
    await embed(['aa']);
    expect(real.calls.length).toBe(2); // cache no longer in the path
    expect(cache.stats().hits).toBe(1);
    cache.close();
  });

  test('a cache infrastructure fault mid-run re-embeds uncached AND is accounted: misses grow by the batch, infra_faults > 0', async () => {
    const path = join(dir, 'infra.sqlite');
    const cache = new EmbeddingCache(path);
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    // Warm two rows so the fault batch has genuine hits to retract.
    await embed(['aa', 'bbb']);
    expect(cache.stats()).toMatchObject({ hits: 0, misses: 2, infra_faults: 0 });
    // Infrastructure fault: the table vanishes under the open handle (a
    // deleted/replaced cache file behaves the same way) — NOT an integrity
    // error, so the batch falls open to the real transport.
    const raw = new Database(path);
    raw.exec('DROP TABLE embed_cache');
    raw.close();
    const out = await embed(['aa', 'bbb', 'cccc']);
    expect(out.map((f) => Array.from(f))).toEqual([vec(2), vec(3), vec(4)]);
    // The whole 3-value batch went to the transport.
    expect(real.calls.length).toBe(2);
    expect(real.calls[1]).toEqual(['aa', 'bbb', 'cccc']);
    const s = cache.stats();
    // Pre-fix: misses stayed 2 (the run looked "served from cache" for the
    // re-embedded batch). Now: 2 genuine + 3 re-embedded = 5, no hits kept
    // for a batch the cache never served, and the fault is named.
    expect(s.misses).toBe(5);
    expect(s.hits).toBe(0);
    expect(s.infra_faults).toBe(1);
    // A second faulting batch keeps accounting (2 faults, +1 miss).
    await embed(['dd']);
    expect(cache.stats()).toMatchObject({ misses: 6, hits: 0, infra_faults: 2 });
    inst.uninstall();
    // A run that only ever fell open can never show a clean misses:0 receipt.
    expect(cache.stats().misses).toBeGreaterThan(0);
    // close() must not throw on the faulted handle.
    cache.close();
  });

  test('concurrent miss batches inside one outer withTransaction commit cleanly (no savepoint interleave, infra_faults 0)', async () => {
    // Expansion embeds the query + its variants in PARALLEL; each is a separate
    // caching-transport batch. Pre-fix the write-back used the async
    // withTransaction, whose savepoint depth interleaved across the concurrent
    // bodies (`no such savepoint: embed_cache_sp_2`) and was mis-reported as a
    // cache infrastructure fault on every expansion question.
    const cache = new EmbeddingCache(join(dir, 'concurrent.sqlite'));
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    const results = await cache.withTransaction(async () =>
      Promise.all([embed(['q-original']), embed(['q-variant-1']), embed(['q-variant-2']), embed(['doc-a', 'doc-b'])]),
    );
    expect(results.map((r) => r.length)).toEqual([1, 1, 1, 2]);
    expect(cache.stats()).toMatchObject({ hits: 0, misses: 5, infra_faults: 0 });
    expect(cache.size()).toBe(5); // every write-back committed with the outer tx
    // And with NO outer transaction (each batch opens its own BEGIN/COMMIT).
    await Promise.all([embed(['x1']), embed(['x2']), embed(['x3'])]);
    expect(cache.stats()).toMatchObject({ misses: 8, infra_faults: 0 });
    expect(cache.size()).toBe(8);
    inst.uninstall();
    cache.close();
  });

  test('a fault while WRITING back (reads fine) bumps infra_faults without double-counting misses', async () => {
    const path = join(dir, 'infra-write.sqlite');
    const cache = new EmbeddingCache(path);
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    // Make the write fail but the read succeed: an INSERT trigger that aborts.
    const raw = new Database(path);
    raw.exec(`CREATE TRIGGER block_insert BEFORE INSERT ON embed_cache BEGIN SELECT RAISE(ABORT, 'disk full (simulated)'); END`);
    raw.close();
    await embed(['aa', 'bbb']);
    const s = cache.stats();
    expect(s.misses).toBe(2); // counted once, by the reads
    expect(s.hits).toBe(0);
    expect(s.infra_faults).toBe(1);
    expect(cache.size()).toBe(0); // nothing was written
    inst.uninstall();
    cache.close();
  });

  test('a corrupt row surfaced during a real embed is a hard error, not a silent re-embed', async () => {
    const path = join(dir, 'hard.sqlite');
    const cache = new EmbeddingCache(path);
    const real = fakeTransport();
    const inst = installEmbedCache(cache, { realTransport: real.fn });
    await embed(['aa']);
    const raw = new Database(path);
    raw.query('UPDATE embed_cache SET byte_len = 3').run();
    raw.close();
    await expect(embed(['aa'])).rejects.toThrow(/integrity/);
    inst.uninstall();
    cache.close();
  });
});

describe('EmbeddingCache — concurrent runs sharing one cache file', () => {
  test('withTransaction does not hold the SQLite write lock across its async body', async () => {
    const path = join(dir, 'shared.sqlite');
    const a = new EmbeddingCache(path);
    a.open();
    let lockedDuringBody: string | null = 'not-probed';
    await a.withTransaction(async () => {
      a.put(MODEL, DIMS, 'a-1', vec(1));
      await Promise.resolve();
      const other = new Database(path);
      try {
        other.exec('PRAGMA busy_timeout = 0');
        other.exec('BEGIN IMMEDIATE');
        other.exec('ROLLBACK');
        lockedDuringBody = null;
      } catch (err) {
        lockedDuringBody = (err as Error).message;
      } finally {
        other.close();
      }
    });
    expect(lockedDuringBody).toBeNull();
    expect(a.get(MODEL, DIMS, 'a-1')).toEqual(vec(1));
    a.close();
  });

  test('a run whose read snapshot predates another run\'s commit still lands its writes', async () => {
    const path = join(dir, 'shared.sqlite');
    const a = new EmbeddingCache(path);
    const b = new EmbeddingCache(path);
    a.open();
    b.open();
    let releaseA!: () => void;
    const gateA = new Promise<void>((r) => { releaseA = r; });
    const runA = a.withTransaction(async () => {
      a.put(MODEL, DIMS, 'a-1', vec(1));
      await gateA;
    });
    await b.withTransaction(async () => {
      expect(b.get(MODEL, DIMS, 'b-1')).toBeNull();
      releaseA();
      await runA;
      b.put(MODEL, DIMS, 'b-1', vec(2));
    });
    expect(b.get(MODEL, DIMS, 'a-1')).toEqual(vec(1));
    expect(a.get(MODEL, DIMS, 'b-1')).toEqual(vec(2));
    expect(a.stats().infra_faults + b.stats().infra_faults).toBe(0);
    a.close();
    b.close();
  });

  test('reads inside a transaction see that transaction\'s own buffered writes', async () => {
    const c = new EmbeddingCache(join(dir, 'own.sqlite'));
    c.open();
    await c.withTransaction(async () => {
      c.put(MODEL, DIMS, 'x', vec(3));
      expect(c.get(MODEL, DIMS, 'x')).toEqual(vec(3));
    });
    expect(c.size()).toBe(1);
    c.close();
  });
});