Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.2 KB
Raw Blame
/**
 * run-config.ts — resolved retrieval pins, the `retrieval_config_hash`, the
 * `run_config` receipt block, secret redaction for ledger/error text, and the
 * dev-slice (`--question-ids`) loader.
 *
 * INVARIANT: pure given its inputs. No engine, no gateway read — the harness
 * (src/commands/eval-longmemeval.ts) resolves the mode bundle and the embedder
 * string and passes them in. Two runs with identical pins AND an identical
 * resolved `knobs_hash` produce an identical `retrieval_config_hash`
 * regardless of flag order, dataset, output path or clock, so a resume file
 * written under different pins — or under a snapshot that differs in any
 * non-pin knob (rrf k, autocut jump, token budget, ...) — is refused (plan
 * D33) and the hash never covers anything a row cannot reproduce. The
 * human-readable pins block stays alongside for the receipt reader.
 *
 * INVARIANT: `redactSecrets` is applied to every error string BEFORE it is
 * written to a receipt row or the eval ledger (plan 0k / TODOS 1914) —
 * provider keys, bearer tokens and DB connection strings never land on disk.
 */

import { createHash } from 'node:crypto';
import { existsSync, readFileSync } from 'node:fs';
import type { SearchMode } from '../../core/search/mode.ts';
import type { LongMemEvalQuestion } from './adapter.ts';

/** The pins that change fused results for one `gbrain eval longmemeval` run. */
export interface RetrievalPins {
  /** Resolved search mode (flag > config snapshot > balanced). */
  mode: SearchMode;
  /** `--keyword-only`: engine.searchKeyword, no vector arm, no reranker. */
  keyword_only: boolean;
  reranker: { enabled: boolean; model: string };
  autocut: boolean;
  /** Expansion fires only with `--expansion` (per-call wins over the bundle). */
  expansion: boolean;
  /** `null` = legacy weighting (every RRF list weight 1). */
  expansion_variant_budget: number | null;
  /** `model@dims` of the embedder, or `unconfigured` when no gateway is set up. */
  embedder: string;
  top_k: number;
  trajectory: boolean;
  /**
   * Raw `--search-pin KEY=VALUE` map (sorted by key), present ONLY when at
   * least one pin was given. A pin the mode resolver does not parse (e.g.
   * `search.adaptive_return`) still changes ranking but never reaches
   * `knobs_hash`; folding the raw map here keeps two differently-pinned runs
   * from merging on resume. Omitted when empty so every pre-existing
   * receipt's hash is unchanged.
   */
  search_pins?: Record<string, string>;
  /** `--eval-pool-depth N` (recall experiment): present only when set, so every other receipt's hash is unchanged. */
  eval_pool_depth?: number;
  /** System One arm (`--decide`): slots, provider, calibrations, thresholds, force_on, split. Present only when a slot is on or shadow. */
  decide?: Record<string, unknown>;
  /** Eval-only fact-key / time-scope arms (retrieval-arms.ts). Present only when an arm is on. */
  retrieval_arms?: Record<string, unknown>;
  /** Per-chunk synopsis tier pins (model, prompt version, doc cap, max tokens). Present only when the resolved mode builds synopsis vectors. */
  contextual_synopsis?: Record<string, unknown>;
  /** `--via-operation`: the operation, the `expand` sent and the agent-shaped context (via-operation.ts). Present only on that path. */
  via_operation?: Record<string, unknown>;
}

/** Stable JSON: sorted keys at every level so key order can never move the hash. */
export function stableStringify(value: unknown): string {
  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
  if (value && typeof value === 'object') {
    const o = value as Record<string, unknown>;
    return `{${Object.keys(o).sort().map(k => `${JSON.stringify(k)}:${stableStringify(o[k])}`).join(',')}}`;
  }
  return JSON.stringify(value);
}

/** The resolved-knobs fingerprint folded into `retrievalConfigHash` (D15/D33). */
export interface KnobsFingerprint {
  /** `knobsHash(resolveSearchMode(...))` — every search knob the run resolved. */
  knobs_hash: string;
  /** `KNOBS_HASH_VERSION` at run time (the hash's vocabulary version). */
  knobs_hash_version: number;
}

/**
 * sha256 (hex) over the stable JSON of `{ pins, knobs_hash, knobs_hash_version }`.
 * The eight pins alone would let a resume merge two runs whose injected
 * config snapshot differs in a non-pin knob; folding the resolved knobs hash
 * closes that (the knobs hash already covers every pin, so the pins block is
 * kept for readability, not for coverage). `pins.search_pins` (raw
 * `--search-pin` map) covers the keys the knobs hash does not parse.
 */
export function retrievalConfigHash(pins: RetrievalPins, knobs: KnobsFingerprint): string {
  return createHash('sha256')
    .update(stableStringify({ pins, knobs_hash: knobs.knobs_hash, knobs_hash_version: knobs.knobs_hash_version }))
    .digest('hex');
}

export function sha256Hex(data: string | Uint8Array): string {
  return createHash('sha256').update(data).digest('hex');
}

/**
 * Scrub provider/DB secrets from free text. Conservative patterns: URL
 * userinfo (`scheme://user:pass@`), bearer tokens, `key=value` pairs whose
 * key smells like a secret, and the common provider key prefixes. The
 * surrounding message (which stage failed, which host) survives so the
 * error stays diagnosable.
 */
export function redactSecrets(text: string): string {
  return text
    // scheme://user:password@host → scheme://<redacted>@host
    .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+:[^\s/@]+@/gi, '$1<redacted>@')
    // Authorization: Bearer <token>
    .replace(/\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}/gi, '$1 <redacted>')
    // api_key=..., apiKey: ..., token=..., password=..., secret=...
    .replace(/\b(api[_-]?key|access[_-]?token|auth[_-]?token|token|password|passwd|secret)(\s*[=:]\s*)["']?[^\s"'&,;]+/gi, '$1$2<redacted>')
    // Provider key shapes: sk-..., sk-ant-..., pa-..., AKIA..., ghp_..., xoxb-...
    .replace(/\b(sk-(?:ant-)?|pa-|ghp_|gho_|xox[abpr]-)[A-Za-z0-9_-]{8,}/g, '$1<redacted>')
    .replace(/\bAKIA[0-9A-Z]{12,}/g, 'AKIA<redacted>')
    // postgres://host/db?password=... is covered above; also scrub ?sslpassword= style
    .replace(/([?&](?:ssl)?password=)[^&\s]+/gi, '$1<redacted>');
}

/**
 * `--question-ids FILE`: one question_id per line; blank lines and `#`
 * comments ignored. Throws when the file is missing or yields no ids.
 */
export function loadQuestionIds(path: string): string[] {
  if (!existsSync(path)) throw new Error(`--question-ids file not found: ${path}`);
  const ids: string[] = [];
  const seen = new Set<string>();
  for (const raw of readFileSync(path, 'utf8').split('\n')) {
    const line = raw.trim();
    if (!line || line.startsWith('#')) continue;
    if (!seen.has(line)) { seen.add(line); ids.push(line); }
  }
  if (ids.length === 0) throw new Error(`--question-ids file is empty: ${path}`);
  return ids;
}

/** `run_config.cache` receipt block (plan D9/D18). */
export interface CacheReceipt {
  path: string;
  hits: number;
  misses: number;
  bypassed: number;
  /** Cache infrastructure faults (re-embedded uncached / write-back lost). Must be 0 for a receipt. */
  infra_faults: number;
  canonical_sha256: string;
  sha256: string;
}

export interface RunConfigInput {
  pins: RetrievalPins;
  retrieval_config_hash: string;
  dataset_sha256: string;
  dataset_questions: number;
  knobs_hash: string;
  knobs_hash_version: number;
  cache: CacheReceipt | null;
  /** Why the cache block is null (`disabled`, `keyword_only`, `gateway_unconfigured`, ...). */
  cache_skipped?: string;
  reranker_skipped_rows: number;
  /** Rows whose vector arm silently degraded (vector_enabled=false / embed_unavailable / embed_timeout) on a non-keyword-only run. */
  vector_degraded_rows: number;
  /** Rows whose --expansion did not run as configured (expansion_failed / expansion_partial). */
  expansion_failed_rows: number;
  expansion_replay_miss: number;
  expansion_replay: string | null;
  gold_missing_from_haystack: number;
  slug_collisions: number;
  excluded_abstention: number;
  question_ids_file: string | null;
  errors: number;
  reader?: { mode: string; prompt_version: string; prompt_sha: string; max_tokens: number; model: string; config_hash: string };
  /** Run-level roll-up of the rows' `decide` receipts (System One arms only). */
  decide_summary?: Record<string, unknown>;
  /** `--via-operation` per-arm roll-up: tree, latency p50/p95, model calls per query, CRAG grades, expansion outcomes. */
  via_operation_summary?: Record<string, unknown>;
}

/** The `run_config` object stamped on the by_type_summary (schema v2). */
export function buildRunConfig(input: RunConfigInput): Record<string, unknown> {
  const p = input.pins;
  return {
    mode: p.mode,
    keyword_only: p.keyword_only,
    reranker: { enabled: p.reranker.enabled, model: p.reranker.model },
    autocut: p.autocut,
    expansion: p.expansion,
    expansion_variant_budget: p.expansion_variant_budget,
    expansion_replay: input.expansion_replay,
    embedder: p.embedder,
    topK: p.top_k,
    trajectory: p.trajectory,
    ...(p.search_pins && Object.keys(p.search_pins).length > 0 ? { search_pins: p.search_pins } : {}),
    ...(p.eval_pool_depth ? { eval_pool_depth: p.eval_pool_depth } : {}),
    ...(p.decide ? { decide: p.decide } : {}),
    ...(p.contextual_synopsis ? { contextual_synopsis: p.contextual_synopsis } : {}),
    ...(p.via_operation ? { via_operation: p.via_operation } : {}),
    dataset_sha256: input.dataset_sha256,
    dataset_questions: input.dataset_questions,
    question_ids_file: input.question_ids_file,
    retrieval_config_hash: input.retrieval_config_hash,
    knobs_hash: input.knobs_hash,
    knobs_hash_version: input.knobs_hash_version,
    cache: input.cache,
    ...(input.cache === null && input.cache_skipped ? { cache_skipped: input.cache_skipped } : {}),
    reranker_skipped_rows: input.reranker_skipped_rows,
    vector_degraded_rows: input.vector_degraded_rows,
    expansion_failed_rows: input.expansion_failed_rows,
    expansion_replay_miss: input.expansion_replay_miss,
    gold_missing_from_haystack: input.gold_missing_from_haystack,
    slug_collisions: input.slug_collisions,
    excluded_abstention: input.excluded_abstention,
    errors: input.errors,
    ...(input.reader ? { reader: input.reader } : {}),
    ...(input.decide_summary ? { decide_summary: input.decide_summary } : {}),
    ...(input.via_operation_summary ? { via_operation_summary: input.via_operation_summary } : {}),
  };
}

/** D12: duplicate question_ids keep the first occurrence; `warn` gets one line per dropped duplicate. */
export function dedupeQuestions<Q extends { question_id: string }>(questions: readonly Q[], warn: (line: string) => void): Q[] {
  const seen = new Set<string>();
  const out: Q[] = [];
  for (const q of questions) {
    if (seen.has(q.question_id)) { warn(`[longmemeval] WARN duplicate question_id ${q.question_id} — keeping the first\n`); continue; }
    seen.add(q.question_id);
    out.push(q);
  }
  return out;
}

/** The `question_date` field is optional on disk; the reader emits `Current Date:` only when present. */
export type DatasetQuestion = LongMemEvalQuestion & { question_date?: string };

/**
 * Load a LongMemEval dataset (JSONL, or a JSON array) and its sha256 (the
 * receipt's dataset pin). Throws with a download hint when missing and with
 * the line number on a parse failure — the harness maps both to exit 1.
 */
export function loadDataset(datasetPath: string, downloadUrl: string): { questions: DatasetQuestion[]; sha256: string } {
  if (!existsSync(datasetPath)) {
    throw new Error(`dataset not found: ${datasetPath}\nDownload from ${downloadUrl}`);
  }
  const bytes = readFileSync(datasetPath);
  const sha256 = sha256Hex(bytes);
  const raw = bytes.toString('utf8');
  if (raw.trimStart().startsWith('[')) {
    const arr = JSON.parse(raw);
    if (!Array.isArray(arr)) throw new Error(`dataset ${datasetPath} parsed as JSON but is not an array`);
    return { questions: arr as DatasetQuestion[], sha256 };
  }
  const out: DatasetQuestion[] = [];
  let lineNo = 0;
  for (const line of raw.split('\n')) {
    lineNo++;
    if (!line.trim()) continue;
    try {
      out.push(JSON.parse(line) as DatasetQuestion);
    } catch (err: any) {
      throw new Error(`dataset ${datasetPath}:${lineNo}: ${err.message ?? err}`);
    }
  }
  return { questions: out, sha256 };
}