Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.1 KB
Raw Blame
/**
 * synopsis-tier.ts — makes `gbrain eval longmemeval --mode tokenmax` measure
 * the vectors production builds for that mode.
 *
 * Production imports every page at the free title tier and leaves the
 * `per_chunk_synopsis` tier to the contextual-reindex backfill, which calls
 * `reembedPageWithContextualRetrieval` per page with the resolved
 * contextual-synopsis model. The benchmark imports the same way, so without
 * this step a tokenmax run scores title-tier vectors. When the resolved mode's
 * tier is `per_chunk_synopsis`, the harness runs that same service on every
 * imported session (same model resolution, prompt, page-level fall-back and
 * guarded install), with two eval-only additions:
 *
 *   - a content-addressed on-disk synopsis cache keyed by the service's own
 *     cache-key shape (`buildSynopsisCacheKey`), so shards and reruns never
 *     pay twice; only answers the model actually returned are cached;
 *   - a spend meter and cap over uncached synopsis calls, priced from the
 *     provider-reported usage.
 */

import { createHash } from 'node:crypto';
import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { homedir } from 'node:os';
import type { BrainEngine } from '../../core/engine.ts';
import {
  computeCorpusGeneration,
  computeSourceTextHash,
  reembedPageWithContextualRetrieval,
  resolveContextualChunkConcurrency,
} from '../../core/contextual-retrieval-service.ts';
import { resolveContextualSynopsisModel } from '../../core/minions/handlers/contextual-reindex-per-chunk.ts';
import {
  buildSynopsisCacheKey,
  generatePerChunkSynopsis,
  SYNOPSIS_DOC_MAX_CHARS,
  SYNOPSIS_MAX_TOKENS,
  SYNOPSIS_PROMPT_VERSION,
  type GeneratePerChunkSynopsisArgs,
  type GeneratePerChunkSynopsisResult,
} from '../../core/page-summary.ts';
import { canonicalLookup } from '../../core/model-pricing.ts';
import type { ResolvedSearchKnobs } from '../../core/search/mode.ts';
import type { RetrievalPins } from './run-config.ts';

const TRANSIENT_ATTEMPTS = 5;

export interface SynopsisTierOptions {
  maxUsd: number;
  concurrency: number;
  cachePath?: string;
}

export function newSynopsisTierOptions(): SynopsisTierOptions {
  return { maxUsd: 20, concurrency: 8 };
}

export const LME_SYNOPSIS_FLAGS: Array<{ name: string; arg?: string; help: string[]; apply: (o: { synopsis: SynopsisTierOptions }, value: string) => void }> = [
  { name: '--synopsis-max-usd', arg: 'N', help: [
      'Spend cap for uncached per-chunk synopses when the resolved mode\'s tier is',
      'per_chunk_synopsis (tokenmax; default 20). At the cap the run stops after',
      'the current question; cached synopses are free.'],
    apply: (o, v) => { const n = Number(v); if (!Number.isFinite(n) || n <= 0) throw new Error(`--synopsis-max-usd must be a positive number (got: ${v})`); o.synopsis.maxUsd = n; } },
  { name: '--synopsis-concurrency', arg: 'N', help: ['Sessions synopsized in parallel per question (default 8; chunks within a session follow GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY).'],
    apply: (o, v) => { const n = Number(v); if (!Number.isInteger(n) || n < 1 || n > 64) throw new Error(`--synopsis-concurrency must be an integer from 1 to 64 (got: ${v})`); o.synopsis.concurrency = n; } },
  { name: '--synopsis-cache', arg: 'FILE', help: ['Synopsis cache (JSONL). Default: ~/.cache/gbrain-eval/synopsis-<model>.jsonl.'],
    apply: (o, v) => { o.synopsis.cachePath = v; } },
];

export function defaultSynopsisCachePath(model: string): string {
  return join(homedir(), '.cache', 'gbrain-eval', `synopsis-${model.replace(/[^a-z0-9.-]+/gi, '_')}.jsonl`);
}

export class SynopsisSpendCapReached extends Error {
  constructor(readonly maxUsd: number, readonly cachePath: string) {
    super(`synopsis spend reached its cap ($${maxUsd.toFixed(2)}); raise --synopsis-max-usd or reuse the cache at ${cachePath}`);
    this.name = 'SynopsisSpendCapReached';
  }
}

export class SynopsisSpend {
  usd = 0;
  calls = 0;
  cacheHits = 0;
  inputTokens = 0;
  outputTokens = 0;
  constructor(readonly maxUsd: number) {}
  add(model: string, usage: { input_tokens?: number; output_tokens?: number } | undefined): number {
    this.calls++;
    if (!usage) return 0;
    this.inputTokens += usage.input_tokens ?? 0;
    this.outputTokens += usage.output_tokens ?? 0;
    const p = canonicalLookup(model);
    if (!p) return 0;
    const usd = ((usage.input_tokens ?? 0) * p.input + (usage.output_tokens ?? 0) * p.output) / 1_000_000;
    this.usd += usd;
    return usd;
  }
  exhausted(): boolean {
    return this.usd >= this.maxUsd;
  }
}

type CachedSynopsis = { kind: GeneratePerChunkSynopsisResult['kind']; synopsis?: string; detail?: string };

class SynopsisCache {
  private readonly map = new Map<string, CachedSynopsis>();
  constructor(readonly path: string) {
    if (!existsSync(path)) return;
    for (const line of readFileSync(path, 'utf8').split('\n')) {
      if (!line.trim()) continue;
      try {
        const row = JSON.parse(line) as { key: string; value: CachedSynopsis };
        if (row && typeof row.key === 'string' && row.value && typeof row.value.kind === 'string') this.map.set(row.key, row.value);
      } catch { /* a torn final line is skipped */ }
    }
  }
  get(key: string): CachedSynopsis | undefined { return this.map.get(key); }
  set(key: string, value: CachedSynopsis): void {
    this.map.set(key, value);
    mkdirSync(dirname(this.path), { recursive: true });
    appendFileSync(this.path, `${JSON.stringify({ key, value })}\n`);
  }
}

const caches = new Map<string, SynopsisCache>();
function cacheFor(path: string): SynopsisCache {
  let c = caches.get(path);
  if (!c) { c = new SynopsisCache(path); caches.set(path, c); }
  return c;
}

export interface SynopsisTier {
  model: string;
  maxTokens: number;
  cachePath: string;
  concurrency: number;
  chunkConcurrency: number;
  spend: SynopsisSpend;
  /** Set when the run stopped early because the spend cap was reached. */
  stoppedAtCap: boolean;
}

/** Pins folded into retrieval_config_hash for a synopsis-tier run. */
export function synopsisPins(tier: SynopsisTier): Record<string, unknown> {
  return { model: tier.model, prompt_version: SYNOPSIS_PROMPT_VERSION, doc_max_chars: SYNOPSIS_DOC_MAX_CHARS, max_tokens: tier.maxTokens };
}

/**
 * The synopsis tier for this run, or null when the resolved mode does not
 * build synopsis vectors (or the run never embeds). The model resolves the
 * way the production backfill resolves it on a brain with no override, and
 * its pins join the run's retrieval pins (so retrieval_config_hash covers it).
 */
export async function resolveSynopsisTier(knobs: ResolvedSearchKnobs, keywordOnly: boolean, o: SynopsisTierOptions, pins: RetrievalPins): Promise<SynopsisTier | null> {
  if (keywordOnly || knobs.contextual_retrieval_disabled || knobs.contextual_retrieval !== 'per_chunk_synopsis') return null;
  const model = await resolveContextualSynopsisModel(null);
  const tier: SynopsisTier = {
    model,
    maxTokens: SYNOPSIS_MAX_TOKENS,
    cachePath: o.cachePath ?? defaultSynopsisCachePath(model),
    concurrency: o.concurrency,
    chunkConcurrency: resolveContextualChunkConcurrency(),
    spend: new SynopsisSpend(o.maxUsd),
    stoppedAtCap: false,
  };
  pins.contextual_synopsis = synopsisPins(tier);
  return tier;
}

/** Run-end stderr lines for the synopsis tier; true when the run stopped at the spend cap (a failed run). */
export function reportSynopsisRunEnd(tier: SynopsisTier | null, questionsRun: number): boolean {
  if (!tier || questionsRun === 0) return false;
  const sp = tier.spend;
  process.stderr.write(`[longmemeval] synopsis tier: ${tier.model}, ${sp.calls} call(s), ${sp.cacheHits} cache hit(s), ${sp.inputTokens} in / ${sp.outputTokens} out tokens, $${sp.usd.toFixed(4)} (cap $${sp.maxUsd.toFixed(2)}; cache ${tier.cachePath})\n`);
  if (!tier.stoppedAtCap) return false;
  process.stderr.write(`[longmemeval] FAIL --synopsis-max-usd: synopsis spend reached $${sp.maxUsd.toFixed(2)}; the run stopped early. Resume with a higher cap (cached synopses are free).\n`);
  return true;
}

function cacheKey(args: GeneratePerChunkSynopsisArgs, model: string, maxTokens: number): string {
  return buildSynopsisCacheKey({
    contentHash: createHash('sha256').update(`${args.pageTitle}\n${maxTokens}\n${args.chunkText}`).digest('hex'),
    chunkIndex: args.chunkIndex,
    corpusGeneration: computeCorpusGeneration({ crMode: 'per_chunk_synopsis', synopsisModel: model, synopsisDocMaxChars: SYNOPSIS_DOC_MAX_CHARS }),
    sourceTextHash: computeSourceTextHash(args.documentText),
  });
}

export interface SynopsisTierResult {
  pages: number;
  pages_synopsis: number;
  pages_fallback: number;
  chunks: number;
  calls: number;
  cache_hits: number;
  usd: number;
}

/**
 * Re-embed every imported session at the synopsis tier through the
 * production service. A transient provider failure retries with backoff; one
 * that persists, a permanent failure, or the spend cap fails the question.
 */
export async function applySynopsisTier(
  engine: BrainEngine,
  pages: ReadonlyArray<{ slug: string }>,
  tier: SynopsisTier,
  generate: typeof generatePerChunkSynopsis = generatePerChunkSynopsis,
): Promise<SynopsisTierResult> {
  const cache = cacheFor(tier.cachePath);
  const out: SynopsisTierResult = { pages: pages.length, pages_synopsis: 0, pages_fallback: 0, chunks: 0, calls: 0, cache_hits: 0, usd: 0 };
  const cachedGenerate: typeof generatePerChunkSynopsis = async (args) => {
    const key = cacheKey(args, tier.model, tier.maxTokens);
    const hit = cache.get(key);
    if (hit) {
      tier.spend.cacheHits++;
      out.cache_hits++;
      return hit.kind === 'success' ? { kind: 'success', synopsis: hit.synopsis ?? '' } : { kind: hit.kind, detail: hit.detail };
    }
    if (tier.spend.exhausted()) throw new SynopsisSpendCapReached(tier.spend.maxUsd, tier.cachePath);
    const res = await generate(args);
    if (!res.usage) return res;
    out.calls++;
    out.usd += tier.spend.add(tier.model, res.usage);
    cache.set(key, res.kind === 'success' ? { kind: 'success', synopsis: res.synopsis } : { kind: res.kind, detail: res.detail });
    return res;
  };
  const queue = [...pages];
  await Promise.all(Array.from({ length: Math.min(tier.concurrency, queue.length) }, async () => {
    for (let page = queue.shift(); page; page = queue.shift()) {
      for (let attempt = 1; ; attempt++) {
        const r = await reembedPageWithContextualRetrieval({
          engine, pageSlug: page.slug, sourceId: 'default', globalMode: 'per_chunk_synopsis',
          synopsisModel: tier.model, synopsisMaxTokens: tier.maxTokens, chunkConcurrency: tier.chunkConcurrency, generateSynopsis: cachedGenerate,
        });
        if (r.kind === 'success') { out.pages_synopsis++; out.chunks += r.chunks_embedded; break; }
        if (r.kind === 'page_fallback') { out.pages_fallback++; out.chunks += r.chunks_embedded; break; }
        if (r.kind === 'skipped') break;
        if (r.kind === 'transient_error' && attempt < TRANSIENT_ATTEMPTS) {
          await new Promise(res => setTimeout(res, 1000 * 2 ** attempt));
          continue;
        }
        throw new Error(`synopsis tier ${r.kind} for ${page.slug} (${r.cause}): ${r.detail}`);
      }
    }
  }));
  return out;
}