Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.2 KB
Raw Blame
/**
 * Entity-anchored query retrieval, gate 1 (docs/eval/decisions/entity-anchoring-query/).
 * The appended-corrections workload of the pinned-questions run 3, read
 * through the real `query` op with `search.entity_anchoring` on and off in the
 * same brain, by the same reader, at equal tokens (the top rows in rank order
 * up to 200 tokens) and at full evidence (16 rows). Expansion is off so the
 * only model call per read is the reader.
 *
 *   bun evals/entity-anchoring/query-gate.ts --plan --seeds 42,7,1234
 *   bun evals/entity-anchoring/query-gate.ts --offline --json --seeds 42 --entities 2
 *   bun evals/entity-anchoring/query-gate.ts --run --yes --max-usd 4 --seeds 42,7,1234 --embeddings voyage:voyage-4 --json
 */
import { writeFileSync } from 'node:fs';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { handleToolCall } from '../../src/mcp/server.ts';
import { normalizeModelId } from '../../src/core/model-id.ts';
import { ENTITY_ANCHORING_KEY } from '../../src/core/search/entity-anchor.ts';
import type { SearchResult } from '../../src/core/types.ts';
import { BATCHES, RATIOS, aggregate, appendedHash, applyEventsForTest, fit, generateAppendedWorkload, question, summarize,
  type AppendedWorkload, type ArmRun } from '../pinned-questions/appended-gate.ts';
import { SpendGuard, correctFor, offlineArms, paidArms, priced, readerAnswer, staleFor, type GateOpts } from '../pinned-questions/benefit-gate.ts';

export const ARMS = ['anchoring_on', 'anchoring_off', 'anchoring_on_full', 'anchoring_off_full'] as const;
export const BUDGET_TOKENS = 200;
const FULL_ROWS = 16;

export interface QueryGateOpts { workload: AppendedWorkload; readerModel: string; embed: boolean; reader: GateOpts['reader'] }

export async function runQueryGate(opts: QueryGateOpts): Promise<{ arms: ArmRun[]; triggered: number; states: number }> {
  const w = opts.workload;
  const engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  const arms = Object.fromEntries(ARMS.map(a => [a, { arm: a, reads: [], lifecycle_usd: 0, refresh_attempts: 0 } as ArmRun])) as Record<typeof ARMS[number], ArmRun>;
  const remembered = new Map<string, string>();
  let triggered = 0;
  try {
    const { runEmbedCore } = await import('../../src/commands/embed.ts');
    const rows = async (q: string, on: boolean) => {
      await engine.setConfig(ENTITY_ANCHORING_KEY, on ? 'true' : 'false');
      return await handleToolCall(engine, 'query', { query: q, limit: FULL_ROWS, expand: false, use_cache: false }) as SearchResult[];
    };
    for (let b = 0; b < BATCHES; b++) {
      await applyEventsForTest(engine, w.batches[b]!, remembered);
      if (opts.embed) {
        const r = await runEmbedCore(engine, { stale: true, sourceId: 'default', quiet: true });
        if (r.failures > 0) throw new Error(`embedding failed for ${r.failures} page(s)`);
      }
      for (const e of w.entities) {
        const gold = w.gold[e.slug]![b]!;
        const past = new Set(w.history[e.slug]!);
        const q = question(e.name);
        const read = async (arm: ArmRun, context: string) => {
          const r = await readerAnswer(opts.reader, opts.readerModel, q, context);
          arm.reads.push({ entity: e.slug, batch: b, gold, correct: correctFor(r.text, gold), stale: staleFor(r.text, gold, past), usd: priced(r.model, r.input_tokens, r.output_tokens) });
        };
        const on = await rows(q, true);
        const off = await rows(q, false);
        if (on.some(r => r.entity_anchored)) triggered++;
        const parts = (rs: SearchResult[]) => rs.map(r => `[${r.slug}] ${r.chunk_text}`);
        await read(arms.anchoring_on, fit(parts(on), BUDGET_TOKENS));
        await read(arms.anchoring_off, fit(parts(off), BUDGET_TOKENS));
        await read(arms.anchoring_on_full, parts(on).join('\n'));
        await read(arms.anchoring_off_full, parts(off).join('\n'));
      }
    }
  } finally {
    await engine.disconnect();
  }
  return { arms: Object.values(arms), triggered, states: w.entities.length * BATCHES };
}

type Summary = ReturnType<typeof summarize>;

/** Preregistered gate 1: in every seed the key-on arm has higher accuracy or lower freshness lag, and lower accuracy in no seed. */
export function decideGate(perSeed: Array<{ seed: number; summary: Summary[] }>) {
  const seeds = perSeed.map(({ seed, summary }) => {
    const at = (arm: string) => summary.find(s => s.reads_per_write === 1 && s.arm === arm)!;
    const on = at('anchoring_on'), off = at('anchoring_off');
    const higher = on.accuracy > off.accuracy, fresher = on.freshness.mean_lag_batches < off.freshness.mean_lag_batches;
    return { seed, on_accuracy: on.accuracy, off_accuracy: off.accuracy, on_lag: on.freshness.mean_lag_batches, off_lag: off.freshness.mean_lag_batches,
      on_stale: on.stale_wrong_rate, off_stale: off.stale_wrong_rate, wins: higher || fresher, accuracy_lower: on.accuracy < off.accuracy };
  });
  return { seeds, pass: seeds.every(s => s.wins && !s.accuracy_lower) };
}

if (import.meta.main) {
  const args = process.argv.slice(2);
  const flag = (n: string) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; };
  const json = args.includes('--json');
  const seeds = (flag('--seeds') ?? '42').split(',').map(Number);
  const readerModel = flag('--reader-model') ?? 'anthropic:claude-sonnet-5-5';
  const embeddingModel = flag('--embeddings');
  const entities = Number(flag('--entities') ?? 6);
  const workloads = seeds.map(seed => generateAppendedWorkload(seed, entities));
  const states = entities * BATCHES;
  // Two budgeted reads (~650 in / 50 out) and two full-evidence reads (~2k in / 50 out) per state, run 3's sizes x1.25.
  const estimate = seeds.length * states * (2 * priced(readerModel, 650, 50) + 2 * priced(readerModel, 2_000, 50));
  if (args.includes('--plan') || (!args.includes('--offline') && !args.includes('--run'))) {
    console.log(JSON.stringify({ mode: 'plan', seeds, workload_hashes: workloads.map(appendedHash), states_per_seed: states, reader_model: readerModel, embedding_model: embeddingModel ?? null, est_usd: estimate }, null, 2));
    process.exit(0);
  }
  const paid = args.includes('--run');
  const cap = Number(flag('--max-usd'));
  if (paid && (!args.includes('--yes') || !Number.isFinite(cap) || cap < estimate)) {
    console.error(`Refusing a paid run: pass --yes and --max-usd >= the estimate ($${estimate.toFixed(2)}).`);
    process.exit(3);
  }
  if (embeddingModel && !paid) { console.error('--embeddings needs --run (offline mode makes no network calls).'); process.exit(2); }
  const guard = new SpendGuard(paid ? cap : Infinity);
  if (paid) {
    const { configureGateway } = await import('../../src/core/ai/gateway.ts');
    configureGateway({ chat_model: normalizeModelId(readerModel), ...(embeddingModel ? { embedding_model: embeddingModel, embedding_dimensions: 1024 } : {}), env: { ...process.env } as Record<string, string> });
  }
  const reader = paid ? paidArms(readerModel, guard).reader : offlineArms(readerModel, readerModel).reader;
  const perSeed: Array<{ seed: number; workload_hash: string; triggered: number; states: number; summary: Summary[]; reads: unknown[] }> = [];
  for (const workload of workloads) {
    const out = await runQueryGate({ workload, readerModel, embed: !!embeddingModel, reader });
    perSeed.push({ seed: workload.seed, workload_hash: appendedHash(workload), triggered: out.triggered, states: out.states,
      summary: RATIOS.flatMap(ratio => out.arms.map(r => summarize(r, workload, ratio, null))), reads: out.arms.map(r => ({ arm: r.arm, reads: r.reads })) });
    const partial = flag('--partial');
    if (partial) writeFileSync(partial, JSON.stringify({ metered_spend_usd: guard.spent, per_seed: perSeed }, null, 2));
    if (!json) console.error(`[query-gate] seed ${workload.seed} done; metered spend $${guard.spent.toFixed(4)}`);
  }
  const out = { mode: paid ? 'run' : 'offline', plumbing_only: !paid, seeds, entities, batches: BATCHES, reader_model: readerModel, embedding_model: embeddingModel ?? null,
    budget_tokens: BUDGET_TOKENS, full_rows: FULL_ROWS, metered_spend_usd: Number.isFinite(guard.spent) ? guard.spent : null,
    gate: decideGate(perSeed), aggregate: aggregate(perSeed.map(s => s.summary)), per_seed: perSeed };
  console.log(json ? JSON.stringify(out, null, 2) : JSON.stringify(out));
}