Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* `gbrain eval longmemeval <dataset.jsonl>` — public LongMemEval benchmark
* adapter and the repo's receipt producer for the strict retrieval metric.
* Spins up an in-memory PGLite, imports each question's haystack, runs
* hybridSearch, optionally generates an answer via the configured gateway
* chat model, emits one JSONL row per question (plus an optional schema-v2
* `by_type_summary`) for downstream `evaluate_qa.py` / replay tooling.
*
* Hermetic by design: cli.ts skips connectEngine() when this subcommand is
* invoked, so the user's ~/.gbrain brain is never opened. Tests stub the
* ThinkLLMClient / expandFn / readiness / embed transport seams so the full
* pipeline runs without any API key.
*
* INVARIANTS:
* - Retrieval scores join raw session IDs and exclude abstentions by default.
* Retrieval and reader configuration are independently pinned; resume
* refuses changed reader pins even when retrieval mixing is allowed.
* - The reranker preflight (exit 2) and the un-reranked-rows gate (exit 1)
* key on the RESOLVED reranker pin — flag, --search-pin, snapshot or
* bundle — so a configured-but-silently-skipped reranker never exits 0.
* A run in which every question errored and no row was scored exits 1.
* - Reranker, vector and expansion degradation fail at run end, including
* no-op resumes; `--record` uses the same finish path.
* - The embed-cache transaction wraps only the embed-producing section of a
* question (import + search); a reader/LLM failure afterwards never rolls
* back committed vectors and never holds the cache write lock across a
* network round-trip.
* - `LME_FLAGS` drives parsing and help from the same table.
* - `--call-plan FILE` returns right after the run's flags, pins and models
* resolve: before the resume skip, any client, engine or embed call
* (src/eval/longmemeval/call-plan.ts writes the plan).
* - `--judge` (Phase D) judges the reader's answer with the official
* LongMemEval prompts (src/eval/longmemeval/judge.ts) through the shared
* judge runner; a judge malfunction is a `judge_error`, never an
* `incorrect`, and the headline scores it as incorrect. Every judged row
* carries `judge_config_hash` (judge + reader pins); `--judge
* --resume-from` is a judge-only backfill that re-judges rows lacking a
* settled verdict from their stored hypothesis and rebuilds `qa_accuracy`
* from ALL rows. Incomplete judgments (judge_errors / skipped_budget > 0)
* are not publishable → exit 1 unless --allow-incomplete-judgments.
*/
import { homedir } from 'os';
import { join } from 'path';
import { execFileSync } from '../core/spawn.ts';
import { brainRecycler, createBenchmarkBrain, LME_BRAIN_RECYCLE_EVERY, resetTables } from '../eval/longmemeval/harness.ts';
export { LME_BRAIN_RECYCLE_EVERY };
import { haystackToPages, normalizeSessions } from '../eval/longmemeval/adapter.ts';
import {
generateAnswer,
rawSessionId,
readerConfigHash,
renderRetrievedAsHypothesis,
resolveReaderConfig,
type ReaderConfig,
} from '../eval/longmemeval/reader.ts';
import {
DEFAULT_JUDGE_MODEL,
JUDGE_MAX_TOKENS,
JUDGE_METHODOLOGY_NOTE,
JUDGE_PROMPT_VERSION,
JUDGE_RAW_MAX_CHARS,
judgeRow,
stripJudgeFields,
type JudgeLaneContext,
} from '../eval/longmemeval/judge.ts';
import {
judgePreflight,
makeJudgeConfigHasher,
parseMaxUsd,
runJudgeBackfill,
selectBackfillRows,
type RowLike,
} from '../eval/longmemeval/judge-lane.ts';
import { anyRowJudged, buildQaAccuracy, type QaAccuracyBlock } from '../eval/longmemeval/qa-accuracy.ts';
import { emitByTypeSummary, makeEmitter, compactJsonlByQuestionId } from '../eval/longmemeval/emit.ts';
import type { BudgetLedger, JudgeChatFn } from '../eval/shared/judge-runner.ts';
import {
addRowToBucket,
buildByTypeSummaryV2,
buildRow,
buildSlugToRawMap,
collisionsTouchingGold,
detectSlugCollisions,
floorBreaches,
isAbstentionQuestion,
newBucket,
sessionIdFromSlug,
type ByTypeSummaryContext,
type ByTypeSummaryV2,
type LongMemEvalRow,
type RecallBucket,
type SlugToRawMap,
} from '../eval/longmemeval/metrics.ts';
import {
buildRunConfig,
dedupeQuestions,
loadDataset,
loadQuestionIds,
redactSecrets,
retrievalConfigHash,
type CacheReceipt,
type DatasetQuestion,
type RetrievalPins,
} from '../eval/longmemeval/run-config.ts';
import {
checkResumeConfigHash,
checkResumeReaderConfig,
classifyDegradation,
countDegradation,
isScoredQuestionRow,
loadExpansionReplay,
loadResumeSet,
readJsonlRows,
seedBucketsFromRows,
type RowSearchMeta,
} from '../eval/longmemeval/resume.ts';
import { EmbeddingCache, installEmbedCache, type EmbedTransportFn, type InstalledEmbedCache } from '../eval/shared/embed-cache.ts';
import { importFromContent } from '../core/import-file.ts';
import { hybridSearch, type HybridSearchOpts } from '../core/search/hybrid.ts';
import { expandQuery } from '../core/search/expansion.ts';
import { normalizeExpansionVariantBudget } from '../core/search/fusion-lists.ts';
import {
KNOBS_HASH_VERSION,
SEARCH_MODES,
isSearchMode,
knobsHash,
loadOverridesFromConfig,
resolveSearchMode,
type ResolvedSearchKnobs,
type SearchMode,
} from '../core/search/mode.ts';
import { buildCaptureExtras } from '../eval/longmemeval/capture.ts';
import * as decideLane from '../eval/longmemeval/decide-lane.ts';
import * as retrievalArms from '../eval/longmemeval/retrieval-arms.ts';
import * as viaOperation from '../eval/longmemeval/via-operation.ts';
import * as synopsisTier from '../eval/longmemeval/synopsis-tier.ts';
import * as callPlan from '../eval/longmemeval/call-plan.ts';
import { resolveModel } from '../core/model-config.ts';
import type { ThinkLLMClient } from '../core/think/index.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import type { BrainEngine } from '../core/engine.ts';
import type { HybridSearchMeta, SearchResult } from '../core/types.ts';
import { classifyIntent, type Intent } from '../eval/longmemeval/intent.ts';
import { makeGatewayThinkClient } from '../eval/longmemeval/gateway-client.ts';
import { EMPTY_TRAJECTORY_ROUTE, routeTrajectory, type TrajectoryRoute } from '../eval/longmemeval/trajectory-route.ts';
import {
extractAndInsertClaims,
makeAliasMap,
resetExtractorState,
getCacheStats,
trajectoryMethodologyNote,
type AliasMap,
} from '../eval/longmemeval/extract.ts';
import { normalizeModelId } from '../core/model-id.ts';
import { chat as gatewayChat, getEmbeddingDimensions, getEmbeddingModel, isAvailable } from '../core/ai/gateway.ts';
import { rerankerReadinessForEngine, type EngineReadiness } from '../core/ai/reranker-readiness-engine.ts';
import { describeRerankerFix } from '../core/ai/reranker-readiness.ts';
import { persistRunRecord, type EvalRunRecord } from './eval-run-all.ts';
// Back-compat re-exports (these used to live here; tests + consumers import from the harness).
export { loadResumeSet } from '../eval/longmemeval/resume.ts';
export { emitByTypeSummary } from '../eval/longmemeval/emit.ts';
const HUGGINGFACE_URL = 'https://huggingface.co/datasets/xiaowu0162/longmemeval';
const DEFAULT_EMBED_CACHE_PATH = join(homedir(), '.cache', 'gbrain-eval', 'longmemeval-embed.sqlite');
type FloorMetric = 'recall_all' | 'recall_any';
interface ParsedArgs {
help: boolean;
datasetPath?: string;
limit?: number;
model?: string;
readerMode?: string;
readerMaxTokens?: number;
retrievalOnly: boolean;
keywordOnly: boolean;
expansion: boolean;
/** `--expansion-replay FILE`: serve recorded `expansion_variants` per question_id. Implies --expansion. */
expansionReplayPath?: string;
/** undefined = not pinned (config/bundle decides); null = legacy weighting. */
expansionVariantBudget?: number | null;
/** --search-pin key=value (repeatable): extra `search.*` config pins written before the explicit flags and folded into retrieval_config_hash. */
searchPins?: Record<string, string>;
topK: number;
outputPath?: string;
mode?: SearchMode;
/** `--reranker on|off` → `search.reranker.enabled` pin. undefined = not pinned. */
reranker?: boolean;
/** `--autocut on|off` → `search.autocut` pin. undefined = not pinned. */
autocut?: boolean;
resumeFromPath?: string;
allowMixedRunConfig: boolean;
questionIdsPath?: string;
noTrajectory: boolean;
byType: boolean;
byTypeFloor?: number;
byTypeFloorMetric: FloorMetric;
includeAbstention: boolean;
embedCache: boolean;
embedCachePath: string;
capturePool: boolean;
record: boolean;
/** `--judge`: LLM-judge each answer (implies --by-type: qa_accuracy lands on the summary line). */
judge: boolean;
judgeModel: string;
/** Judge spend cap in USD; null = `--max-usd off`. */
maxUsd: number | null;
yes: boolean;
judgeConcurrency: number;
allowIncompleteJudgments: boolean;
/** System One arm (`--decide*`, src/eval/decide-eval-flags.ts) and the eval-only `--eval-pool-depth`. */
decide: decideLane.DecideEvalOptions; evalPoolDepth?: number; arms: retrievalArms.RetrievalArmOptions; synopsis: synopsisTier.SynopsisTierOptions; viaOp: viaOperation.ViaOperationOptions;
/** `--call-plan FILE` / `--call-plan-gate CMD`: write the gstack eval call plan instead of running (src/eval/longmemeval/call-plan.ts). */
callPlanPath?: string; callPlanGate?: string;
}
interface LmeFlag {
name: string;
/** Value placeholder for help (`N`, `FILE`, `on|off`); absent = boolean flag. */
arg?: string;
help: string[];
apply: (out: ParsedArgs, value: string) => void;
}
function parseOnOff(flag: string, v: string): boolean {
const s = v.trim().toLowerCase();
if (s === 'on' || s === 'true' || s === '1') return true;
if (s === 'off' || s === 'false' || s === '0') return false;
throw new Error(`${flag} must be on|off (got: ${v})`);
}
/**
* The ONE flag table (eng D4): renders `printHelp` AND drives `parseArgs`.
* Every `--flag` literal the command accepts appears here, so the
* flag-registry generator (which scans this file's text) sees the full
* surface and the help text can never drift from the parser.
*/
const LME_FLAGS: LmeFlag[] = [
{ name: '--limit', arg: 'N', help: ['Run only the first N questions (after --question-ids filtering).'],
apply: (o, v) => { o.limit = Number(v); if (!Number.isInteger(o.limit) || o.limit < 1) throw new Error(`--limit must be a positive integer (got: ${v})`); } },
{ name: '--model', arg: 'M', help: ['Override answer-generation model (default: resolveModel).'],
apply: (o, v) => { o.model = v; } },
{ name: '--reader-mode', arg: 'direct|notes', help: ['Reader instruction: notes (default, 1024 tokens) extracts evidence before a concise answer; direct preserves the prior prompt (512 tokens).'],
apply: (o, v) => { o.readerMode = v; resolveReaderConfig({ mode: v }); } },
{ name: '--reader-max-tokens', arg: 'N', help: ['Override the reader output budget (positive integer; use 512 to reproduce the original notes treatment).'],
apply: (o, v) => { o.readerMaxTokens = Number(v); resolveReaderConfig({ maxTokens: o.readerMaxTokens }); } },
{ name: '--retrieval-only', help: ['Skip LLM answer generation; emit retrieved sessions instead.'],
apply: (o) => { o.retrievalOnly = true; } },
{ name: '--keyword-only', help: ['Skip vector embedding; pure keyword retrieval (no reranker, no cache).'],
apply: (o) => { o.keywordOnly = true; } },
{ name: '--expansion', help: [
'Enable multi-query expansion. OFF by default for EVERY mode — the per-call',
'setting wins over the bundle, so `--mode tokenmax` alone does NOT expand;',
'expansion fires only with this flag. One Haiku call per question, non-',
'deterministic; each row records `expansion_variants` for replay.'],
apply: (o) => { o.expansion = true; } },
{ name: '--expansion-replay', arg: 'FILE', help: [
'Serve the `expansion_variants` recorded in FILE (a prior --expansion run)',
'instead of calling the LLM, so every cell differs only in its knobs.',
'Implies --expansion. A question_id missing from FILE is an error row',
'(`expansion_replay_miss`) and the run exits 1 at the end.'],
apply: (o, v) => { o.expansionReplayPath = v; o.expansion = true; } },
{ name: '--search-pin', arg: 'KEY=VALUE', help: [
'Pin an arbitrary `search.*` config key for the run (repeatable; last wins).',
'Generic pass-through for knob A/Bs (e.g. search.metadata_boost_gate=lexical);',
'the value is written via engine.setConfig BEFORE the explicit flags, so',
'--mode/--reranker/--autocut/--expansion-variant-budget beat a pin of the',
'same key. Every pin folds into retrieval_config_hash (keys the mode',
'resolver parses also reach knobs_hash); unknown keys are set verbatim.'],
apply: (o, v) => {
const eq = v.indexOf('=');
if (eq <= 0) throw new Error(`--search-pin takes KEY=VALUE (got: ${v})`);
const key = v.slice(0, eq).trim(); const val = v.slice(eq + 1).trim();
if (!key.startsWith('search.') || key === 'search.' || val.length === 0) throw new Error(`--search-pin key must start with "search." and carry a non-empty value (got: ${v})`);
o.searchPins = { ...(o.searchPins ?? {}), [key]: val };
} },
{ name: '--expansion-variant-budget', arg: 'B', help: [
'Pin `search.expansion_variant_budget`: `legacy` (every RRF list weight 1)',
'or a number in (0, 4] — the total RRF weight shared equally by all',
'expansion variant lists (the original list always keeps weight 1).'],
apply: (o, v) => {
const b = normalizeExpansionVariantBudget(v);
if (b === undefined) throw new Error(`--expansion-variant-budget must be legacy or a number in (0, 4] (got: ${v})`);
o.expansionVariantBudget = b;
} },
{ name: '--top-k', arg: 'K', help: [
'Retrieve K chunk rows per question (default: 8). recall_*@k is scored over',
'the distinct sessions among those K rows (k = K).'],
apply: (o, v) => { o.topK = Number(v); if (!Number.isInteger(o.topK) || o.topK < 1) throw new Error(`--top-k must be a positive integer (got: ${v})`); } },
{ name: '--mode', arg: 'M', help: [
`Search mode: ${SEARCH_MODES.join('|')}. Resolves through`,
'src/core/search/mode.ts so retrieval matches production under that mode.',
'NOTE: no mode implies --expansion (see --expansion).'],
apply: (o, v) => {
if (!isSearchMode(v)) throw new Error(`--mode must be one of ${SEARCH_MODES.join('|')} (got: ${v})`);
o.mode = v;
} },
{ name: '--reranker', arg: 'on|off', help: [
'Pin `search.reranker.enabled` for the run (beats any injected snapshot or',
'--search-pin). Whenever the RESOLVED pin is on (this flag, a --search-pin,',
'the snapshot or the mode bundle): readiness preflight (exit 2 with the fix',
'if the reranker cannot run) and the run exits non-zero if any row fell',
'through un-reranked (`reranker_skipped_rows` in the summary).'],
apply: (o, v) => { o.reranker = parseOnOff('--reranker', v); } },
{ name: '--autocut', arg: 'on|off', help: ['Pin `search.autocut` for the run (beats any injected snapshot).'],
apply: (o, v) => { o.autocut = parseOnOff('--autocut', v); } },
{ name: '--output', arg: 'FILE', help: ['Write JSONL to FILE instead of stdout.'],
apply: (o, v) => { o.outputPath = v; } },
{ name: '--resume-from', arg: 'FILE', help: [
'Skip question_ids already present in FILE; resume the remaining',
'questions. Typically the same path as --output so the run continues in',
'append mode. Rows are re-scored from their retrieved ids + the dataset',
'gold; a file written under different retrieval pins is refused.'],
apply: (o, v) => { o.resumeFromPath = v; } },
{ name: '--allow-mixed-run-config', help: ['Resume even when FILE rows carry a different retrieval_config_hash.'],
apply: (o) => { o.allowMixedRunConfig = true; } },
{ name: '--question-ids', arg: 'FILE', help: [
'Run only the question_ids listed in FILE (one per line, # comments).',
'Unknown ids or an empty file exit 1. Dev-slice / held-out discipline.'],
apply: (o, v) => { o.questionIdsPath = v; } },
{ name: '--no-trajectory', help: [
'Opt out of trajectory routing (skips the Haiku claim extractor AND the',
'per-question intent routing). Use for like-for-like retrieval receipts.'],
apply: (o) => { o.noTrajectory = true; } },
{ name: '--by-type', help: [
'Emit a final schema_version:2 by_type_summary line: per question_type',
'{total, all_hit, all_rate, any_hit, any_rate} + aggregate, excluded_abstention,',
'mean_distinct_sessions, run_config (pins, hashes, cache stats). Resume-safe:',
'a prior summary at the tail is REPLACED, not appended.'],
apply: (o) => { o.byType = true; } },
{ name: '--by-type-floor', arg: 'F', help: [
'Exit non-zero if any question_type rate < F (range [0, 1]). Gates on',
'recall_all by default (see --by-type-floor-metric). Implies --by-type.'],
apply: (o, v) => {
const f = Number(v);
if (!Number.isFinite(f) || f < 0 || f > 1) throw new Error(`--by-type-floor must be a number in [0, 1] (got: ${v})`);
o.byTypeFloor = f; o.byType = true;
} },
{ name: '--by-type-floor-metric', arg: 'M', help: ['Which rate --by-type-floor gates on: recall_all (default) | recall_any.'],
apply: (o, v) => {
if (v !== 'recall_all' && v !== 'recall_any') throw new Error(`--by-type-floor-metric must be recall_all|recall_any (got: ${v})`);
o.byTypeFloorMetric = v;
} },
{ name: '--include-abstention', help: [
'Count `_abs` (abstention) questions in the recall denominators. Default:',
'emitted with abstention:true but excluded (summary.excluded_abstention).'],
apply: (o) => { o.includeAbstention = true; } },
{ name: '--embed-cache', arg: 'FILE', help: [
`Content-addressed embedding cache (bun:sqlite). Default:`,
`~/.cache/gbrain-eval/longmemeval-embed.sqlite. Hits/misses land in`,
`run_config.cache; misses must be 0 for a like-for-like arm.`],
apply: (o, v) => { o.embedCachePath = v; o.embedCache = true; } },
{ name: '--no-embed-cache', help: ['Disable the embedding cache for this run.'],
apply: (o) => { o.embedCache = false; } },
{ name: '--capture-pool', help: [
'Record `rerank_pool` per row: the post-rerank candidate pool BEFORE',
'autocut / the limit slice ({slug, chunk_id, session_id, rrf_rank,',
'rerank_score, alias_hit, est_tokens}) for the autocut-floor replay.'],
apply: (o) => { o.capturePool = true; } },
{ name: '--record', help: [
'Append an EvalRunRecord (suite longmemeval, params = run_config) to',
'.gbrain-evals/eval-results.jsonl. Error text is secret-redacted.'],
apply: (o) => { o.record = true; } },
{ name: '--judge', help: [
'LLM-judge each answer against the gold with the official LongMemEval',
`evaluate_qa.py prompts (temperature 0, max_tokens ${JUDGE_MAX_TOKENS}). Implies --by-type:`,
'the summary line gains a qa_accuracy block whose headline scores judge',
'errors as incorrect. Incompatible with --retrieval-only. With',
'--resume-from FILE: judge-only backfill of rows lacking a settled verdict',
'(no reader call; rows with judge_error are re-judged), then qa_accuracy',
'is rebuilt from ALL rows and FILE is rewritten with the judged rows.'],
apply: (o) => { o.judge = true; o.byType = true; } },
{ name: '--judge-model', arg: 'M', help: [`Judge model (default: ${DEFAULT_JUDGE_MODEL}, the official scorer's model).`],
apply: (o, v) => { o.judgeModel = normalizeModelId(v, 'openai'); } },
{ name: '--max-usd', arg: 'N|off', help: [
'Cap on JUDGE spend only, in USD (default: 5) — the reader / extractor',
'lanes are not metered here. Preflight refuses an estimate over the cap',
'without --yes (exit 2); at run time the lane soft-stops at the cap and',
'stamps the remaining rows judge_skipped:"budget" (not publishable). An',
'unpriced judge model requires `off` (no cap).'],
apply: (o, v) => { o.maxUsd = parseMaxUsd('--max-usd', v); } },
{ name: '--yes', help: ['Proceed when the judge estimate exceeds --max-usd (the cap still soft-stops the run).'],
apply: (o) => { o.yes = true; } },
{ name: '--judge-concurrency', arg: 'N', help: [
'Parallel judge calls during a --resume-from backfill (default: 1; live',
'rows are judged inline after each reader call).'],
apply: (o, v) => { o.judgeConcurrency = Number(v); if (!Number.isInteger(o.judgeConcurrency) || o.judgeConcurrency < 1) throw new Error(`--judge-concurrency must be a positive integer (got: ${v})`); } },
{ name: '--allow-incomplete-judgments', help: [
'Exit 0 even when judge_errors, skipped_budget or unjudged > 0. Default:',
'such a run is NOT publishable (stderr FAIL line + exit 1) — re-run with',
'--judge --resume-from FILE until all three are 0.'],
apply: (o) => { o.allowIncompleteJudgments = true; } },
{ name: '--call-plan', arg: 'FILE', help: [
'Write the gstack eval call plan (GSTACK_EVAL_CALLS v1) for the run these',
'flags describe to FILE and exit 0: per-lane calls and token bounds, no',
'provider call, no brain. A lane it cannot bound refuses (exit 1).'],
apply: (o, v) => { o.callPlanPath = v; } },
{ name: '--call-plan-gate', arg: 'CMD', help: ['The GSTACK_EVAL_GATE line for --call-plan (default: this command without --call-plan).'],
apply: (o, v) => { o.callPlanGate = v; } },
...decideLane.LME_DECIDE_FLAGS,
...retrievalArms.LME_RETRIEVAL_ARM_FLAGS, ...synopsisTier.LME_SYNOPSIS_FLAGS, ...viaOperation.LME_VIA_OPERATION_FLAGS,
];
function parseArgs(args: string[]): ParsedArgs {
const out: ParsedArgs = {
help: false,
retrievalOnly: false,
keywordOnly: false,
expansion: false,
topK: 8,
allowMixedRunConfig: false,
noTrajectory: false,
byType: false,
byTypeFloorMetric: 'recall_all',
includeAbstention: false,
embedCache: true,
embedCachePath: DEFAULT_EMBED_CACHE_PATH,
capturePool: false,
record: false,
judge: false,
judgeModel: DEFAULT_JUDGE_MODEL,
maxUsd: 5,
yes: false,
judgeConcurrency: 1,
allowIncompleteJudgments: false,
decide: decideLane.newDecideEvalOptions(), arms: retrievalArms.newRetrievalArmOptions(), synopsis: synopsisTier.newSynopsisTierOptions(), viaOp: viaOperation.newViaOperationOptions(),
};
const byName = new Map(LME_FLAGS.map(f => [f.name, f]));
for (let i = 0; i < args.length; i++) {
const a = args[i];
if (a === '--help' || a === '-h') { out.help = true; continue; }
if (a.startsWith('--')) {
const eq = a.indexOf('=');
const name = eq >= 0 ? a.slice(0, eq) : a;
const flag = byName.get(name);
if (!flag) throw new Error(`unknown flag ${name} for 'gbrain eval longmemeval' (see --help)`);
if (flag.arg === undefined && eq >= 0) throw new Error(`${name} is a switch and takes no value (write ${name} on its own)`);
let value = '';
if (flag.arg !== undefined) {
value = eq >= 0 ? a.slice(eq + 1) : (args[++i] ?? '');
if (value === '' || (eq < 0 && value.startsWith('--'))) throw new Error(`${name} requires a value (${flag.arg})`);
}
flag.apply(out, value);
continue;
}
if (!out.datasetPath) { out.datasetPath = a; continue; }
throw new Error(`unexpected extra argument "${a}" (only one <dataset.jsonl> positional is accepted; see --help)`);
}
if (out.judge && out.retrievalOnly) {
throw new Error('--judge cannot be combined with --retrieval-only (there is no reader hypothesis to judge)');
}
viaOperation.validateViaOperation(out);
if (out.callPlanGate !== undefined && !out.callPlanPath) throw new Error('--call-plan-gate needs --call-plan FILE');
return out;
}
function printHelp(): void {
const lines: string[] = [
`gbrain eval longmemeval <dataset.jsonl> [options]`,
``,
`Run the LongMemEval benchmark against gbrain's hybrid retrieval. Spins up an`,
`in-memory PGLite per benchmark run; the user's brain is never opened.`,
``,
`Arguments:`,
` <dataset.jsonl> LongMemEval dataset file (one question per line).`,
` Download from ${HUGGINGFACE_URL}`,
``,
`Options:`,
];
const pad = 28;
for (const f of LME_FLAGS) {
const head = ` ${f.name}${f.arg ? ' ' + f.arg : ''}`;
if (head.length >= pad - 1) {
lines.push(head);
for (const h of f.help) lines.push(' '.repeat(pad) + h);
} else {
lines.push(head.padEnd(pad) + f.help[0]);
for (const h of f.help.slice(1)) lines.push(' '.repeat(pad) + h);
}
}
lines.push(` -h, --help Show this help.`);
lines.push(``);
lines.push(`Row fields: recall_all_hit (every gold session in the top-k distinct sessions),`);
lines.push(`recall_any_hit (at least one), recall_hit (DEPRECATED alias of recall_any_hit),`);
lines.push(`abstention, distinct_sessions_in_top_k, retrieved[] (every returned chunk row),`);
lines.push(`retrieved_session_ids, search_meta, retrieval_config_hash, reader_model, reader_mode, reader_prompt_sha, reader_finish_reason;`);
lines.push(`with --judge: judge_correct | judge_error | judge_skipped, judge_model, judge_raw, judge_cost_usd,`);
lines.push(`judge_config_hash (summary: qa_accuracy).`);
lines.push(``);
lines.push(`Note: a full 500-question run takes ~20-60 minutes depending on flags. Use`);
lines.push(`--limit or --question-ids during development.`);
process.stderr.write(lines.join('\n') + '\n');
}
export interface RunOpts {
exitOnError?: boolean;
/** Inject a chat client for tests; defaults to the gateway-routed client (#4636). */
client?: ThinkLLMClient;
/** Separate stub for the claim extractor (defaults to the same gateway client). */
extractorClient?: ThinkLLMClient;
/** Model id for the claim extractor. Defaults to the utility tier via resolveModel. */
extractorModel?: string;
/**
* Inject a pre-built benchmark brain instead of creating one inside this
* call (tests amortize the PGLite cold-create across many runs). MUST be
* the result of createBenchmarkBrain(); the caller owns lifecycle.
*/
engine?: BrainEngine;
/** #5092 test seams: brain recycle interval and factory (harness-owned brain only). */
brainRecycleEvery?: number;
createBrain?: () => Promise<BrainEngine>;
/**
* Live nightly-probe search-mode/reranker settings copied into the isolated
* benchmark brain (#3676). Explicit pin flags (--mode/--reranker/--autocut/
* --expansion-variant-budget) beat the snapshot.
*/
searchConfigSnapshot?: Record<string, string>;
/** Test seam: the expansion function behind `--expansion` (default expandQuery). */
expandFn?: (query: string) => Promise<string[]>;
/**
* Test seam: the transport the embed cache serves misses from and restores
* on uninstall (default: the real ai-sdk embedMany). A test that installed
* a fake transport must pass it here or the cache install would replace it.
*/
embedTransport?: EmbedTransportFn | null;
/** Test seam: reranker readiness probe behind `--reranker on` (default rerankerReadinessForEngine). */
rerankerReadiness?: (engine: BrainEngine, model: string) => Promise<EngineReadiness>;
/** Test seam: directory for the `--record` ledger (default <repo>/.gbrain-evals/). */
recordDir?: string;
/** Test seam: the judge chat client behind `--judge` (default: the gateway `chat`). */
judgeClient?: JudgeChatFn;
/** Test seam: base retry backoff for judge timeouts / 429s (default 500ms). */
judgeBackoffMs?: number;
}
/** Abort one question with an error row that carries extra diagnostic fields. */
class QuestionAbort extends Error {
constructor(message: string, readonly extra: Record<string, unknown>) {
super(message);
this.name = 'QuestionAbort';
}
}
interface RunContext {
opts: ParsedArgs;
model: string;
readerConfig: ReaderConfig;
readerHash: string;
client: ThinkLLMClient;
trajectoryEnabled: boolean;
extractorClient: ThinkLLMClient;
extractorModel: string;
expandFn: (query: string) => Promise<string[]>;
replay: Map<string, string[]> | null;
retrievalConfigHash: string;
/** The run's resolved expansion expectation (pins.expansion): the degradation gate's `expansion` input. */
expectExpansion: boolean;
/**
* Wraps the embed-producing section of one question (import + search) in
* ONE embed-cache transaction (eng D7) — and nothing after it, so a reader
* failure cannot roll back committed vectors. Identity when no cache.
*/
embedTxn: <T>(fn: () => Promise<T>) => Promise<T>;
decide: decideLane.DecideEvalRun | null;
factKeys: { arm: retrievalArms.FactKeyArm; spend: retrievalArms.FactKeySpend } | null; synopsis: synopsisTier.SynopsisTier | null;
}
interface QuestionOutcome {
row: LongMemEvalRow;
rerankerSkipped: boolean;
vectorDegraded: boolean;
expansionFailed: boolean;
}
/** Resolve the pins for this run from flags + the injected snapshot (no engine read). */
function resolvePins(opts: ParsedArgs, runOpts: RunOpts, trajectoryEnabled: boolean, decide: decideLane.DecideEvalRun | null): { pins: RetrievalPins; knobs: ResolvedSearchKnobs } {
// Generic --search-pin entries ride the same override plane as the injected snapshot (explicit pins win),
// so knobs_hash / retrieval_config_hash reflect them.
const snapshot = { ...(runOpts.searchConfigSnapshot ?? {}), ...(opts.searchPins ?? {}) };
const knobs = resolveSearchMode({
mode: opts.mode ?? snapshot['search.mode'],
overrides: loadOverridesFromConfig(snapshot),
perCall: {
expansion: opts.viaOp.op ? opts.viaOp.expand : opts.expansion,
...(opts.expansionVariantBudget !== undefined ? { expansion_variant_budget: opts.expansionVariantBudget } : {}),
...(opts.reranker !== undefined ? { reranker_enabled: opts.reranker } : {}),
...(opts.autocut !== undefined ? { autocut: opts.autocut } : {}),
},
});
let embedder = 'unconfigured';
try { embedder = `${getEmbeddingModel()}@${getEmbeddingDimensions()}`; } catch { /* no gateway configured */ }
// Raw --search-pin map, sorted, folded into retrieval_config_hash ONLY when
// non-empty (a pin the mode resolver does not parse never reaches knobs_hash,
// yet still changes ranking — two differently-pinned runs must not merge).
const pinKeys = Object.keys(opts.searchPins ?? {}).sort();
const searchPins = pinKeys.length > 0 ? Object.fromEntries(pinKeys.map(k => [k, opts.searchPins![k]])) : undefined;
const pins: RetrievalPins = {
mode: knobs.resolved_mode,
keyword_only: opts.keywordOnly,
reranker: { enabled: opts.keywordOnly ? false : knobs.reranker_enabled, model: knobs.reranker_model },
autocut: opts.keywordOnly ? false : knobs.autocut,
expansion: opts.viaOp.op ? opts.viaOp.op === 'query' && knobs.expansion : opts.keywordOnly ? false : opts.expansion,
expansion_variant_budget: knobs.expansion_variant_budget,
embedder,
top_k: opts.topK,
trajectory: trajectoryEnabled,
...(searchPins ? { search_pins: searchPins } : {}),
...(opts.evalPoolDepth ? { eval_pool_depth: opts.evalPoolDepth } : {}), ...(decide ? { decide: decide.runConfig } : {}),
...retrievalArms.armPins(opts.arms), ...viaOperation.viaOperationPins(opts.viaOp),
};
return { pins, knobs };
}
/** Run one fixed `git` subcommand (argv form, no shell) and return trimmed stdout, or `fallback`. */
function gitShort(argv: readonly string[], fallback: string): string {
try { return execFileSync('git', argv, { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || fallback; } catch { return fallback; }
}
export async function runEvalLongMemEval(args: string[], runOpts: RunOpts = {}): Promise<void> {
const fail = (code: number): never => {
if (runOpts.exitOnError !== false) process.exit(code);
throw new Error(`longmemeval failed (exit ${code}); see evaluation diagnostics`);
};
let opts: ParsedArgs;
let readerConfig: ReaderConfig;
try {
opts = parseArgs(args);
readerConfig = resolveReaderConfig({ mode: opts.readerMode, maxTokens: opts.readerMaxTokens });
} catch (err: any) {
process.stderr.write(`Error: ${err.message ?? err}\n`);
fail(1);
return;
}
if (opts.help) { printHelp(); return; }
if (!opts.datasetPath) {
process.stderr.write(`Error: <dataset.jsonl> is required.\n\n`);
printHelp();
fail(1);
return;
}
let questions: DatasetQuestion[];
let datasetSha256: string;
try {
({ questions, sha256: datasetSha256 } = loadDataset(opts.datasetPath, HUGGINGFACE_URL));
} catch (err: any) {
process.stderr.write(`Error: ${err.message ?? err}\n`);
fail(1);
return;
}
const datasetQuestionCount = questions.length;
questions = dedupeQuestions(questions, (line) => process.stderr.write(line)); // D12: duplicate question_ids → WARN, first wins
// Gold by question_id for resume re-scoring (dataset is loaded on resume).
const goldByQid = new Map<string, readonly string[]>(questions.map(q => [q.question_id, q.answer_session_ids ?? []]));
// RAW haystack ids by question_id: a pre-stamp resume row carries slug-normalized
// ids, which the re-scorer maps back to raw ids through this map.
const haystackByQid = new Map<string, readonly string[]>(questions.map(q => {
try { return [q.question_id, normalizeSessions(q).map(x => x.session_id)] as const; } catch { return [q.question_id, []] as const; }
}));
const seedCtx = (k: number, includeAbstention: boolean) => ({ goldByQid, haystackByQid, k, includeAbstention });
/** Dataset row by id — the judge backfill takes question_type / answer from the dataset, not the row. */
const questionByQid = new Map<string, DatasetQuestion>(questions.map(q => [q.question_id, q]));
if (opts.questionIdsPath) {
try {
const ids = loadQuestionIds(opts.questionIdsPath);
const known = new Set(questions.map(q => q.question_id));
const unknown = ids.filter(id => !known.has(id));
if (unknown.length > 0) throw new Error(`--question-ids: ${unknown.length} id(s) not in the dataset: ${unknown.slice(0, 5).join(', ')}${unknown.length > 5 ? ', …' : ''}`);
const want = new Set(ids);
questions = questions.filter(q => want.has(q.question_id));
} catch (err: any) {
process.stderr.write(`Error: ${err.message ?? err}\n`);
fail(1);
return;
}
}
let decideRun: decideLane.DecideEvalRun | null;
try { ({ run: decideRun, questions } = decideLane.prepareLmeDecide(opts, questions)); } catch (err: any) { process.stderr.write(`Error: ${err.message ?? err}\n`); return fail(1); }
if (opts.limit && opts.limit < questions.length) questions = questions.slice(0, opts.limit);
if (questions.length === 0) {
process.stderr.write(`Error: dataset contains no questions.\n`);
fail(1);
return;
}
const trajectoryEnabled = !opts.noTrajectory;
const { pins, knobs } = resolvePins(opts, runOpts, trajectoryEnabled, decideRun);
const synopsis = await synopsisTier.resolveSynopsisTier(knobs, opts.keywordOnly, opts.synopsis, pins);
const knobsHashValue = knobsHash(knobs);
// D33 + review: the hash covers the pins AND the resolved knobs hash, so a
// resume cannot merge runs whose injected snapshot differs in a non-pin knob.
const retrievalHash = retrievalConfigHash(pins, { knobs_hash: knobsHashValue, knobs_hash_version: KNOBS_HASH_VERSION });
const degradeOpts = { keywordOnly: opts.keywordOnly, expansion: pins.expansion };
// Resolved BEFORE the resume block: the reader model is half of every
// row's judge_config_hash (D33), which the --judge backfill gate needs.
const model = await resolveModel(null, {
cliFlag: opts.model,
configKey: 'models.eval.longmemeval',
envVar: 'GBRAIN_MODEL',
fallback: 'sonnet',
});
const extractorModel = trajectoryEnabled
? await resolveModel(null, {
cliFlag: runOpts.extractorModel,
tier: 'utility',
fallback: 'haiku',
})
: '';
const readerHash = readerConfigHash(readerConfig, normalizeModelId(model));
const judgeHashFor = makeJudgeConfigHasher(opts.judgeModel, { model, prompt_sha: readerConfig.promptSha, max_tokens: readerConfig.maxTokens, k: opts.topK });
/** This run's judge_config_hash (live rows; prior rows hash from their own recorded reader pins). */
const runJudgeHash = judgeHashFor({});
if (opts.callPlanPath) return callPlan.runLmeCallPlan({ args, opts, questions, questionByQid, pins, knobs, synopsis, model, extractorModel, readerConfig, decide: decideRun, judgeHashFor }, fail);
/** Everything the run-end block needs; filled by the main path or the no-op resume path. */
interface RunEndState {
buckets: Record<string, RecallBucket>;
distinct: number[];
excludedAbstention: number;
goldMissing: number;
slugCollisions: number;
rerankerSkippedRows: number;
vectorDegradedRows: number;
expansionFailedRows: number;
expansionReplayMiss: number;
errorCount: number;
errorMessages: string[];
cacheReceipt: CacheReceipt | null;
cacheSkipped?: string;
questionsRun: number;
runStart: number;
/** Every question row of the run (prior rows, rewritten on a judge backfill, + new rows) for qa_accuracy. */
qaRows: RowLike[];
judgeEstUsd: number | null;
judgeLedger: BudgetLedger | null;
}
const summaryRunConfig = (st: RunEndState): Record<string, unknown> => buildRunConfig({
pins,
retrieval_config_hash: retrievalHash,
dataset_sha256: datasetSha256,
dataset_questions: datasetQuestionCount,
knobs_hash: knobsHashValue,
knobs_hash_version: KNOBS_HASH_VERSION,
cache: st.cacheReceipt,
cache_skipped: st.cacheSkipped,
reranker_skipped_rows: st.rerankerSkippedRows,
vector_degraded_rows: st.vectorDegradedRows,
expansion_failed_rows: st.expansionFailedRows,
expansion_replay_miss: st.expansionReplayMiss,
expansion_replay: opts.expansionReplayPath ?? null,
gold_missing_from_haystack: st.goldMissing,
slug_collisions: st.slugCollisions,
excluded_abstention: st.excludedAbstention,
question_ids_file: opts.questionIdsPath ?? null,
errors: st.errorCount,
reader: { mode: readerConfig.mode, prompt_version: readerConfig.promptVersion, prompt_sha: readerConfig.promptSha, max_tokens: readerConfig.maxTokens, model: normalizeModelId(model), config_hash: readerHash },
...(decideRun ? { decide_summary: decideLane.summarizeDecideReceipts(st.qaRows.map((r) => r.decide as never)) } : {}),
...(opts.viaOp.op ? { via_operation_summary: viaOperation.summarizeViaOperation(st.qaRows as never, viaOperation.gbrainBuild()) } : {}),
});
/**
* The ONE run-end block: stderr summary, --by-type emission + floor gate,
* the reranker / vector-degraded / expansion-failed / replay-miss gates,
* --record, exit. Called from the no-op resume branch AND the main path so
* a resume that has nothing left to do still records and still fails on
* the gates its prior rows breach.
*/
const finishRun = (st: RunEndState): void => {
let exitCode = 0;
const elapsed = Math.round((Date.now() - st.runStart) / 1000);
process.stderr.write(`\n[longmemeval] done. ${st.questionsRun} questions in ${elapsed}s. ${st.errorCount} errors.\n`);
if (Object.keys(st.buckets).length > 0) {
process.stderr.write(`[longmemeval] recall_all@${opts.topK} / recall_any@${opts.topK} by question_type:\n`);
for (const [t, v] of Object.entries(st.buckets).sort()) {
const pa = v.total === 0 ? 0 : (v.all_hit / v.total) * 100;
const pn = v.total === 0 ? 0 : (v.any_hit / v.total) * 100;
process.stderr.write(` ${t}: all ${v.all_hit}/${v.total} (${pa.toFixed(1)}%), any ${v.any_hit}/${v.total} (${pn.toFixed(1)}%)\n`);
}
if (st.excludedAbstention > 0) process.stderr.write(` (excluded ${st.excludedAbstention} abstention question(s); --include-abstention counts them)\n`);
}
if (st.cacheReceipt) {
const c = st.cacheReceipt;
process.stderr.write(`[longmemeval] embed cache: ${c.hits} hits, ${c.misses} misses, ${c.bypassed} bypassed, ${c.infra_faults} infra fault(s) (canonical ${c.canonical_sha256.slice(0, 12)})\n`);
}
if (synopsisTier.reportSynopsisRunEnd(synopsis, st.questionsRun)) exitCode = 1;
const incomplete = st.qaRows.filter(row => typeof row.error === 'string' && String(row.error).startsWith('reader_')).length;
if (incomplete > 0) {
process.stderr.write(`[longmemeval] ${incomplete} incomplete reader completion(s); partial, empty or unknown answers are errors and cannot count as completed.\n`);
exitCode = 1;
}
const runConfig = summaryRunConfig(st);
// qa_accuracy (Phase D): rebuilt from ALL rows whenever the judge lane ran
// or any row carries a verdict. Headline scores judge errors / budget
// skips / reader errors as incorrect (D16); errors-excluded is secondary.
let qa: QaAccuracyBlock | undefined;
if (opts.judge || anyRowJudged(st.qaRows)) {
qa = buildQaAccuracy(st.qaRows, {
judgeModel: opts.judgeModel,
judgePromptVersion: JUDGE_PROMPT_VERSION,
judgeConfigHash: runJudgeHash,
estCostUsd: st.judgeEstUsd,
runCostUsd: st.judgeLedger ? st.judgeLedger.actualUsd : null,
methodologyNote: JUDGE_METHODOLOGY_NOTE,
});
const pct = (x: number | null): string => (x === null ? 'n/a' : `${(x * 100).toFixed(1)}%`);
const ci = qa.ci95_bootstrap;
process.stderr.write(
`[longmemeval] qa_accuracy: headline ${pct(qa.accuracy_headline)} (${qa.correct}/${qa.total_questions}; ` +
`judge_errors ${qa.judge_errors}, skipped_budget ${qa.skipped_budget}, reader_errors ${qa.reader_errors}), ` +
`excluding errors ${pct(qa.accuracy_excluding_errors)} (${qa.correct}/${qa.judged}), ` +
`non-abstention ${pct(qa.accuracy_470)} (n=${qa.non_abstention_total}), abstention ${qa.abstention.correct}/${qa.abstention.total}; ` +
`ci95 [${pct(ci.lower)}, ${pct(ci.upper)}] (${ci.label}); judge ${qa.judge_model} v=${qa.judge_prompt_version}; ` +
`spend est ${qa.est_cost_usd === null ? 'unpriced' : `$${qa.est_cost_usd.toFixed(4)}`}, ` +
`this run $${(qa.run_cost_usd ?? 0).toFixed(4)}, all rows $${qa.actual_cost_usd.toFixed(4)}\n`,
);
if (opts.judge && !qa.complete) {
const line = `[longmemeval] ${opts.allowIncompleteJudgments ? 'WARN' : 'FAIL'} --judge: judgments incomplete ` +
`(judge_errors ${qa.judge_errors}, skipped_budget ${qa.skipped_budget}, unjudged ${qa.unjudged}) — NOT publishable; re-run with ` +
`--judge --resume-from FILE until all three are 0`;
process.stderr.write(opts.allowIncompleteJudgments ? `${line} (continuing: --allow-incomplete-judgments)\n` : `${line}.\n`);
if (!opts.allowIncompleteJudgments) exitCode = 1;
}
}
let summary: ByTypeSummaryV2 | undefined;
if (opts.byType) {
summary = buildByTypeSummaryV2(st.buckets, {
k: opts.topK,
excludedAbstention: st.excludedAbstention,
goldMissingFromHaystack: st.goldMissing,
slugCollisions: st.slugCollisions,
distinctSessionsInTopK: st.distinct,
runConfig,
...(qa ? { qa } : {}),
});
emitByTypeSummary(opts.outputPath, summary);
if (opts.byTypeFloor !== undefined) {
const breaches = floorBreaches(summary, opts.byTypeFloor, opts.byTypeFloorMetric);
if (breaches.length > 0) {
process.stderr.write(`[longmemeval] FAIL --by-type-floor=${opts.byTypeFloor}: ${breaches.join(', ')}\n`);
exitCode = 1;
}
}
}
// Gate on the RESOLVED reranker pin (flag, --search-pin, snapshot or bundle),
// not the flag alone: a configured-but-silently-skipped reranker never exits 0.
if (pins.reranker.enabled && st.rerankerSkippedRows > 0) {
process.stderr.write(`[longmemeval] FAIL reranker on (resolved pin): ${st.rerankerSkippedRows} row(s) fell through un-reranked (reranker_skipped / rerank_passthrough / rerank_failed) — pass --reranker off or set the reranker provider key (e.g. VOYAGE_API_KEY)\n`);
exitCode = 1;
}
// Every question of this run errored AND the output holds no scored row
// (prior resume rows included): the buckets are empty, every rate is null
// and the floor gate is vacuous — such a run is a failure, never 'completed'.
if (st.questionsRun > 0 && st.errorCount === st.questionsRun && !st.qaRows.some(isScoredQuestionRow)) {
process.stderr.write(`[longmemeval] FAIL every question errored (${st.errorCount}/${st.questionsRun}) — no row was scored; see the error rows${st.errorMessages.length > 0 ? ` (first: ${st.errorMessages[0]})` : ''}\n`);
exitCode = 1;
}
if (!opts.keywordOnly && st.vectorDegradedRows > 0) {
process.stderr.write(`[longmemeval] FAIL vector arm: ${st.vectorDegradedRows} row(s) scored keyword-only after a silent embed failure (vector_enabled:false / embed_unavailable / embed_timeout) — vector_degraded_rows\n`);
exitCode = 1;
}
if (!opts.keywordOnly && pins.expansion && st.expansionFailedRows > 0) {
process.stderr.write(opts.viaOp.op ? viaOperation.expansionFailLine(st.expansionFailedRows)
: `[longmemeval] FAIL --expansion: ${st.expansionFailedRows} row(s) did not expand as configured (expansion_failed / expansion_partial) — expansion_failed_rows\n`);
exitCode = 1;
}
if (st.expansionReplayMiss > 0) {
process.stderr.write(`[longmemeval] FAIL --expansion-replay: ${st.expansionReplayMiss} question(s) had no recorded variants (expansion_replay_miss)\n`);
exitCode = 1;
}
if (opts.record) {
const commit = gitShort(['rev-parse', '--short', 'HEAD'], 'unknown');
const repoRoot = gitShort(['rev-parse', '--show-toplevel'], process.cwd());
const record: EvalRunRecord = {
schema_version: 3,
run_id: `${commit}-longmemeval-${pins.mode}-${Date.now().toString(36)}`,
ran_at: new Date().toISOString(),
suite: 'longmemeval',
mode: pins.mode,
commit,
seed: 0,
...(opts.limit !== undefined ? { limit: opts.limit } : {}),
params: {
...runConfig,
questions_run: st.questionsRun,
output: opts.outputPath ?? null,
aggregate: summary?.aggregate ?? null,
mean_distinct_sessions: summary?.mean_distinct_sessions ?? null,
qa_accuracy: qa
? {
accuracy_headline: qa.accuracy_headline, accuracy_excluding_errors: qa.accuracy_excluding_errors,
judged: qa.judged, correct: qa.correct, judge_errors: qa.judge_errors, skipped_budget: qa.skipped_budget,
judge_model: qa.judge_model, judge_config_hash: qa.judge_config_hash, complete: qa.complete,
}
: null,
},
status: exitCode === 0 ? 'completed' : 'failed',
duration_ms: Date.now() - st.runStart,
// Belt-and-braces: persistRunRecord redacts too; keep the caller-side scrub.
...(st.errorMessages.length > 0 || exitCode !== 0
? { error: redactSecrets([...(exitCode !== 0 ? [`exit ${exitCode}`] : []), ...st.errorMessages].join(' | ')) }
: {}),
};
persistRunRecord(repoRoot, record, runOpts.recordDir);
process.stderr.write(`[longmemeval] recorded ${record.run_id} (${record.status})\n`);
}
if (exitCode !== 0) fail(exitCode);
};
// --resume-from: filter out already-answered question_ids before any
// model/brain setup so a no-op resume costs ~zero. Refuse a mixed file
// (rows produced under different retrieval pins) unless told otherwise.
let appendOutput = false;
/** Prior rows (resume). With --judge they are judged in place and re-emitted ahead of the new rows. */
let priorRows: RowLike[] = [];
/** Prior rows the --judge backfill judges from their stored hypothesis. */
let backfill: RowLike[] = [];
if (opts.resumeFromPath) {
priorRows = readJsonlRows(opts.resumeFromPath);
const readerCheck = checkResumeReaderConfig(priorRows, readerConfig, normalizeModelId(model), opts.retrievalOnly, opts.judge);
if (readerCheck.mismatched > 0 || readerCheck.unknown > 0) {
process.stderr.write(`[longmemeval] resume: reader configuration mismatch (${readerCheck.mismatched} different, ${readerCheck.unknown} unknown). Refusing to mix reader modes, prompts, models or budgets even with --allow-mixed-run-config.\n`);
fail(1);
return;
}
const check = checkResumeConfigHash(priorRows, retrievalHash);
if (check.mismatched > 0) {
const msg = `[longmemeval] resume: ${check.mismatched} row(s) in ${opts.resumeFromPath} carry a different retrieval_config_hash ` +
`(${check.foreign.map(h => h.slice(0, 12)).join(', ')} vs this run's ${retrievalHash.slice(0, 12)}).`;
if (!opts.allowMixedRunConfig) {
process.stderr.write(`${msg} Refusing to mix runs; pass --allow-mixed-run-config to override.\n`);
fail(1);
return;
}
process.stderr.write(`${msg} Continuing (--allow-mixed-run-config).\n`);
}
if (check.unstamped > 0) {
process.stderr.write(`[longmemeval] resume: ${check.unstamped} row(s) carry no retrieval_config_hash (pre-stamp file) — re-scored, not refused\n`);
}
const done = loadResumeSet(opts.resumeFromPath);
const before = questions.length;
questions = questions.filter(q => !done.has(q.question_id));
process.stderr.write(`[longmemeval] resume: ${done.size} already done; ${questions.length}/${before} remaining\n`);
if (opts.judge) {
// Judge-only backfill (D33): rows lacking a settled verdict are judged
// from their stored hypothesis; rows already judged under a DIFFERENT
// judge_config_hash are refused unless told otherwise. The output is
// REWRITTEN (prior rows + judge fields, then new rows), never appended.
const sel = selectBackfillRows(priorRows, { questionByQid, hashFor: judgeHashFor });
if (sel.mismatched > 0) {
const msg = `[longmemeval] resume: ${sel.mismatched} judged row(s) in ${opts.resumeFromPath} carry a different judge_config_hash ` +
`(${sel.foreign.map(h => h.slice(0, 12)).join(', ')} vs this run's ${runJudgeHash.slice(0, 12)}).`;
if (!opts.allowMixedRunConfig) {
process.stderr.write(`${msg} Refusing to mix judge configs; pass --allow-mixed-run-config to override.\n`);
fail(1);
return;
}
process.stderr.write(`${msg} Continuing (--allow-mixed-run-config).\n`);
}
if (sel.retrievalOnly > 0) {
process.stderr.write(`Error: --judge: ${sel.retrievalOnly} row(s) in ${opts.resumeFromPath} were produced with --retrieval-only and carry no reader hypothesis to judge.\n`);
fail(1);
return;
}
if (sel.missingFromDataset > 0) process.stderr.write(`[longmemeval] WARN judge backfill: ${sel.missingFromDataset} row(s) not in this dataset — left unjudged\n`);
backfill = sel.candidates;
process.stderr.write(`[longmemeval] judge backfill: ${backfill.length} row(s) to judge from their stored hypothesis; ${sel.settled} verdict(s) stand\n`);
}
// Same-file resume ALWAYS appends (judge or not): rows persist as they land, so a
// timeout/kill loses at most the in-flight question; the file is compacted to one
// row per question_id (last wins) before the summary is written.
if (opts.outputPath && opts.resumeFromPath === opts.outputPath) appendOutput = true;
if (questions.length === 0 && backfill.length === 0) {
process.stderr.write(`[longmemeval] resume: nothing to do (all questions already answered${opts.judge ? ' and judged' : ''}).\n`);
if (opts.outputPath && opts.outputPath === opts.resumeFromPath) compactJsonlByQuestionId(opts.outputPath);
if (opts.judge && opts.outputPath && opts.outputPath !== opts.resumeFromPath) {
// A judge resume into a DIFFERENT output still copies the prior rows
// forward, so the new file is complete (rows + the summary below).
const em = makeEmitter(opts.outputPath, false);
for (const row of priorRows) if (row.kind !== 'by_type_summary' && typeof row.question_id === 'string') em.emit(row);
em.close();
}
// Even a no-op resume runs the FULL run-end block against the prior
// rows (CDX-3 + review): --by-type emission, the floor gate, the
// reranker / vector-degraded / expansion gates, and --record.
const buckets: Record<string, RecallBucket> = {};
const seed = seedBucketsFromRows(priorRows, buckets, seedCtx(opts.topK, opts.includeAbstention));
const deg = countDegradation(priorRows, degradeOpts);
finishRun({
buckets,
distinct: seed.distinct,
excludedAbstention: seed.excludedAbstention,
goldMissing: seed.goldMissing,
slugCollisions: seed.collisions,
rerankerSkippedRows: deg.rerankerSkipped,
vectorDegradedRows: deg.vectorDegraded,
expansionFailedRows: deg.expansionFailed,
expansionReplayMiss: 0,
errorCount: 0,
errorMessages: [],
cacheReceipt: null,
cacheSkipped: 'resume_noop',
questionsRun: 0,
runStart: Date.now(),
qaRows: priorRows,
judgeEstUsd: null,
judgeLedger: null,
});
return;
}
}
let replay: Map<string, string[]> | null = null;
if (opts.expansionReplayPath) {
try {
replay = loadExpansionReplay(opts.expansionReplayPath);
} catch (err: any) {
process.stderr.write(`Error: ${err.message ?? err}\n`);
fail(1);
return;
}
process.stderr.write(`[longmemeval] expansion replay: ${replay.size} recorded variant set(s) from ${opts.expansionReplayPath}\n`);
}
// #4636: BOTH chat lanes (answer generation + trajectory claim extractor)
// route through the configured AI gateway (gateway-client.ts).
const gatewayClient: ThinkLLMClient = makeGatewayThinkClient();
const client: ThinkLLMClient = runOpts.client ?? gatewayClient;
const extractorClient: ThinkLLMClient = runOpts.extractorClient ?? gatewayClient;
if (runOpts.exitOnError === false) {
if (!opts.retrievalOnly && !runOpts.client && !isAvailable('chat', model)) {
throw new Error(`longmemeval reader chat provider not configured for ${model}; configure this model's provider or models.eval.longmemeval`);
}
if (trajectoryEnabled && !runOpts.extractorClient && !isAvailable('chat', extractorModel)) {
throw new Error(`longmemeval extractor chat provider not configured for ${extractorModel}; configure this model's provider or its utility tier`);
}
}
// --judge preflight (Phase D): availability → pricing → estimate → cap.
// Runs before the brain connects so a refused run costs nothing.
let judgeCtx: JudgeLaneContext | null = null;
let judgeEstUsd: number | null = null;
let judgeLedger: BudgetLedger | null = null;
if (opts.judge) {
const pre = judgePreflight({
judgeModel: opts.judgeModel,
maxUsd: opts.maxUsd,
yes: opts.yes,
available: runOpts.judgeClient ? true : isAvailable('chat', opts.judgeModel),
live: questions,
readerMaxTokens: readerConfig.maxTokens,
backfill,
});
if (!pre.ok) {
process.stderr.write(`Error: ${pre.message}\n`);
fail(pre.exitCode);
return;
}
for (const line of pre.lines) process.stderr.write(`${line}\n`);
judgeEstUsd = pre.estUsd;
judgeLedger = pre.ledger;
judgeCtx = {
client: runOpts.judgeClient ?? gatewayChat,
model: opts.judgeModel,
ledger: pre.ledger,
configHashFor: judgeHashFor,
...(runOpts.judgeBackoffMs !== undefined ? { backoffMs: runOpts.judgeBackoffMs } : {}),
};
}
process.stderr.write(`[longmemeval] estimated 20-60 minutes for ${questions.length} questions; use --limit N for shorter runs\n`);
process.stderr.write(`[longmemeval] connecting in-memory brain...\n`);
process.stderr.write(
`[longmemeval] starting (questions: ${questions.length}, model: ${model}, mode: ${pins.mode}, ` +
`reranker: ${pins.reranker.enabled ? 'on' : 'off'}, autocut: ${pins.autocut ? 'on' : 'off'}, ` +
`expansion: ${pins.expansion ? (replay ? 'replay' : 'on') : 'off'}, budget: ${pins.expansion_variant_budget ?? 'legacy'}, ` +
`embedder: ${pins.embedder}, top-k: ${pins.top_k}, trajectory: ${trajectoryEnabled ? 'on' : 'off'}` +
`${trajectoryEnabled ? `, extractor: ${extractorModel}` : ''}, retrieval_config_hash: ${retrievalHash.slice(0, 12)})\n`,
);
if (trajectoryEnabled) resetExtractorState();
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
const buckets: Record<string, RecallBucket> = {};
const distinct: number[] = [];
let excludedAbstention = 0;
let goldMissing = 0;
let slugCollisions = 0;
let rerankerSkippedRows = 0;
let vectorDegradedRows = 0;
let expansionFailedRows = 0;
let expansionReplayMiss = 0;
// Prior rows that survive into the output: a prior error row for a question
// about to be re-run is dropped (its retry row follows it).
const rerun = new Set(questions.map(q => q.question_id));
const keptPrior = priorRows.filter(r => !(typeof r.error === 'string' && !r.hypothesis && rerun.has(r.question_id as string)));
// --resume-from: prior rows feed the degradation gates whether or not
// --by-type is set (a partial resume must still fail on rows the prior run
// degraded). Bucket seeding (re-scored) only feeds the --by-type summary.
if (opts.resumeFromPath) {
const deg = countDegradation(keptPrior, degradeOpts);
rerankerSkippedRows += deg.rerankerSkipped;
vectorDegradedRows += deg.vectorDegraded;
expansionFailedRows += deg.expansionFailed;
}
if (opts.byType && opts.resumeFromPath) {
const seed = seedBucketsFromRows(keptPrior, buckets, seedCtx(opts.topK, opts.includeAbstention));
distinct.push(...seed.distinct);
excludedAbstention += seed.excludedAbstention;
goldMissing += seed.goldMissing;
slugCollisions += seed.collisions;
}
const runStart = Date.now();
let errorCount = 0;
const errorMessages: string[] = [];
let cacheReceipt: CacheReceipt | null = null;
let cacheSkipped: string | undefined;
const qaRows: RowLike[] = [];
const foldRow = (row: LongMemEvalRow): void => {
// gold_missing / slug_collisions count EVERY question row (abstention + no-gold rows here,
// collision-abort error rows in the catch below) — the set seedBucketsFromRows counts on resume.
if (row.gold_missing_from_haystack.length > 0) goldMissing++;
if (row.slug_collision > 0) slugCollisions++;
if (row.abstention && !opts.includeAbstention) { excludedAbstention++; return; }
if (typeof row.recall_all_hit !== 'boolean') return; // no gold → not in the denominator
const bucket = buckets[row.question_type] ?? (buckets[row.question_type] = newBucket());
if (addRowToBucket(bucket, row) !== 'skipped') distinct.push(row.distinct_sessions_in_top_k);
};
const ctx: RunContext = {
opts, model, readerConfig, readerHash, client, trajectoryEnabled, extractorClient, extractorModel, decide: decideRun,
factKeys: await retrievalArms.resolveFactKeyArm(opts.arms, extractorClient), synopsis,
expandFn: runOpts.expandFn ?? expandQuery,
replay,
retrievalConfigHash: retrievalHash, expectExpansion: pins.expansion,
embedTxn: (fn) => fn(),
};
const configureBrain = async (engine: BrainEngine): Promise<void> => {
// #3676: nightly probe callers may copy audited live search config into
// this isolated engine. Data tables stay hermetic.
for (const [key, value] of Object.entries(runOpts.searchConfigSnapshot ?? {})) {
if (key.trim().length > 0) await engine.setConfig(key, value);
}
// Raw --search-pin entries land next, then the explicit flags, so an
// explicit --mode/--reranker/--autocut/--expansion-variant-budget beats a
// --search-pin of the same key — the SAME precedence resolvePins used for
// retrieval_config_hash / run_config / the gates. resetTables preserves
// `config` between questions, so these fire once for the run; hybridSearch
// resolves them through the standard chain.
for (const [k, v] of Object.entries(opts.searchPins ?? {})) await engine.setConfig(k, v);
if (opts.mode) await engine.setConfig('search.mode', opts.mode);
if (opts.reranker !== undefined) await engine.setConfig('search.reranker.enabled', opts.reranker ? 'true' : 'false');
if (opts.autocut !== undefined) await engine.setConfig('search.autocut', opts.autocut ? 'true' : 'false');
if (opts.expansionVariantBudget !== undefined) {
await engine.setConfig('search.expansion_variant_budget', opts.expansionVariantBudget === null ? 'legacy' : String(opts.expansionVariantBudget));
}
if (decideRun) await decideLane.configureDecideBrain(engine, decideRun);
};
const work = async (engine: BrainEngine): Promise<void> => {
await configureBrain(engine);
// Resolved reranker pin on (flag, --search-pin, snapshot or bundle):
// preflight. A reranker that cannot run would silently turn every row
// into RRF order (reranker_skipped) — refuse up front.
if (pins.reranker.enabled) {
const probe = runOpts.rerankerReadiness ?? rerankerReadinessForEngine;
const r = await probe(engine, pins.reranker.model);
if (!r.readiness.ready) {
process.stderr.write(`[longmemeval] reranker not ready (${r.plane} plane): ${describeRerankerFix(r.readiness) ?? 'not ready'}\n`);
fail(2);
return;
}
}
// Embedding cache (plan 0c): byte-identical vectors across arms. Never
// installed on --keyword-only (no embeds happen).
let cache: EmbeddingCache | null = null;
let installed: InstalledEmbedCache | null = null;
if (!opts.embedCache) cacheSkipped = 'disabled';
else if (opts.keywordOnly) cacheSkipped = 'keyword_only';
else {
try {
cache = new EmbeddingCache(opts.embedCachePath);
installed = installEmbedCache(cache, { realTransport: runOpts.embedTransport ?? null });
process.stderr.write(`[longmemeval] embed cache: ${cache.path} (${installed.model}@${installed.dims}, ${cache.size()} rows)\n`);
} catch (err: any) {
cache = null;
cacheSkipped = `install_failed: ${redactSecrets(String(err?.message ?? err))}`;
process.stderr.write(`[longmemeval] embed cache not installed: ${cacheSkipped}\n`);
}
}
// The transaction is scoped INSIDE runOneQuestion to the import + search
// section (the only embed producers). Wrapping the whole question here
// would roll back a question's vectors on a reader failure and hold the
// sqlite write lock across the LLM round-trip.
const c = cache;
ctx.embedTxn = c ? (fn) => c.withTransaction(fn) : (fn) => fn();
// Same-file resume APPENDS every new / judged / retried row as it lands
// (a kill mid-run loses at most the in-flight question, never the paid
// reader rows) and compacts to one row per question_id in the finally
// below; a fresh output path (no resume, or a resume into a different
// file) is truncated and the prior rows are carried forward explicitly.
const emitter = makeEmitter(opts.outputPath, appendOutput);
// #5092: a harness-owned brain is replaced on a fixed cadence (see brainRecycler).
const brains = brainRecycler(engine, runOpts.engine ? 0 : (runOpts.brainRecycleEvery ?? LME_BRAIN_RECYCLE_EVERY),
runOpts.createBrain ?? createBenchmarkBrain, configureBrain);
progress.start('eval.longmemeval', backfill.length + questions.length);
try {
// Judge-only backfill first (prior rows, no reader call), then re-emit
// EVERY prior question row (judged or not) so the rewritten output is
// complete; prior error rows for questions about to be re-run are
// dropped (their retry row follows), prior summary lines always are.
if (judgeCtx && backfill.length > 0) {
await runJudgeBackfill(backfill, judgeCtx, {
concurrency: opts.judgeConcurrency,
questionByQid,
onRow: (row) => {
progress.tick(1, `${String(row.question_id)} (judge)`);
// Append mode: persist the judged row NOW as a newer duplicate (compacted at run end).
if (appendOutput) emitter.emit(row);
},
});
}
for (const row of keptPrior) {
if (row.kind === 'by_type_summary' || typeof row.question_id !== 'string') continue;
qaRows.push(row);
// Rewrite into a DIFFERENT file: carry every prior row forward. Same-file
// append: the rows are already on disk.
if (opts.judge && !appendOutput) emitter.emit(row);
}
for (const q of questions) {
const brain = await brains.next();
const qStart = Date.now();
try {
const outcome = await runOneQuestion(brain, q, ctx);
if (judgeCtx && typeof outcome.row.error !== 'string') {
// Judge inline from the row's hypothesis (the same path the backfill
// takes). A judge THROW (judgeRow never throws for a transport
// failure, but a malformed provider result / hasher bug can) is a
// judge_error on the row — the paid reader row is kept, mirroring
// runJudgeBackfill's failure stamp — never an error row.
try {
Object.assign(outcome.row, await judgeRow({
...outcome.row,
question_id: q.question_id, question_type: q.question_type, question: q.question,
answer: String(q.answer ?? ''), hypothesis: outcome.row.hypothesis, // 32 LongMemEval golds are integers
}, judgeCtx));
} catch (judgeErr: any) {
stripJudgeFields(outcome.row as unknown as Record<string, unknown>);
Object.assign(outcome.row, {
judge_error: 'provider_error',
judge_error_detail: redactSecrets(String(judgeErr?.message ?? judgeErr)).slice(0, JUDGE_RAW_MAX_CHARS),
judge_model: judgeCtx.model,
judge_prompt_version: JUDGE_PROMPT_VERSION,
judge_config_hash: runJudgeHash,
});
}
}
emitter.emit(outcome.row);
qaRows.push(outcome.row);
if (typeof outcome.row.error === 'string') {
errorCount++;
if (errorMessages.length < 3) errorMessages.push(`${q.question_id}: ${outcome.row.error}`);
}
if (outcome.rerankerSkipped) rerankerSkippedRows++;
if (outcome.vectorDegraded) vectorDegradedRows++;
if (outcome.expansionFailed) expansionFailedRows++;
foldRow(outcome.row);
progress.tick(1, q.question_id);
} catch (err: any) {
errorCount++;
const message = redactSecrets(String(err?.message ?? err));
if (errorMessages.length < 3) errorMessages.push(`${q.question_id}: ${message}`);
const extra = err instanceof QuestionAbort ? err.extra : {};
if (extra.expansion_replay_miss === true) expansionReplayMiss++;
if (typeof extra.slug_collision === 'number' && extra.slug_collision > 0) slugCollisions++;
// Error rows carry question + question_type so the cross-modal
// --batch consumer can flag them as upstream errors instead of
// silently dropping them from the denominator.
const errorRow: RowLike = {
question_id: q.question_id,
question: q.question,
question_type: q.question_type,
hypothesis: '',
error: message,
retrieval_config_hash: retrievalHash,
...extra,
};
emitter.emit(errorRow);
qaRows.push(errorRow);
progress.tick(1, `${q.question_id} (error)`);
}
if (process.env.GBRAIN_LME_DEBUG === '1') {
process.stderr.write(`[longmemeval] ${q.question_id} ${Date.now() - qStart}ms\n`);
}
if (synopsis?.spend.exhausted()) { synopsis.stoppedAtCap = true; break; }
}
} finally {
await brains.close();
progress.finish();
emitter.close();
if (appendOutput && opts.outputPath) {
const c = compactJsonlByQuestionId(opts.outputPath);
if (c.superseded > 0 || c.summaries_dropped > 0) {
process.stderr.write(`[longmemeval] resume: compacted ${opts.outputPath} to ${c.rows} row(s) (${c.superseded} superseded, ${c.summaries_dropped} stale summary line(s))\n`);
}
}
if (cache) {
try {
const s = cache.stats();
cacheReceipt = { path: cache.path, hits: s.hits, misses: s.misses, bypassed: s.bypassed, infra_faults: s.infra_faults, canonical_sha256: cache.canonicalSha256(), sha256: cache.fileSha256() };
} catch (err: any) {
cacheSkipped = `receipt_failed: ${redactSecrets(String(err?.message ?? err))}`;
}
installed?.uninstall();
cache.close();
}
}
};
if (runOpts.engine) {
// Caller owns engine lifecycle (typically a test beforeAll/afterAll).
await work(runOpts.engine);
} else {
const first = await (runOpts.createBrain ?? createBenchmarkBrain)();
try {
await work(first);
} finally {
await first.disconnect();
}
}
if (trajectoryEnabled) {
const xc = getCacheStats();
const total = xc.hits + xc.misses;
const pct = total === 0 ? 0 : (xc.hits / total) * 100;
process.stderr.write(`[longmemeval] extractor.cache_hits: ${xc.hits} / ${total} sessions (${pct.toFixed(1)}%, cached_bodies=${xc.size})\n`);
process.stderr.write(`[longmemeval] methodology_note: ${trajectoryMethodologyNote(extractorModel)}\n`);
}
finishRun({
buckets, distinct, excludedAbstention, goldMissing, slugCollisions,
rerankerSkippedRows, vectorDegradedRows, expansionFailedRows, expansionReplayMiss,
errorCount, errorMessages, cacheReceipt, cacheSkipped,
questionsRun: questions.length, runStart,
qaRows, judgeEstUsd, judgeLedger,
});
}
async function runOneQuestion(
engine: BrainEngine,
q: DatasetQuestion,
ctx: RunContext,
): Promise<QuestionOutcome> {
const { opts } = ctx;
const slugToRaw = buildSlugToRawMap(q);
const gold = q.answer_session_ids ?? [];
// Pre-import aborts (pure, zero cost): both fire BEFORE resetTables /
// import / any embed call so an abort never spends an embedding.
//
// Plan D32: a slug collision that touches a gold id makes the join
// ambiguous — abort the question with an error row instead of scoring it.
const goldCollisions = collisionsTouchingGold(slugToRaw, gold);
if (goldCollisions.length > 0) {
throw new QuestionAbort(
`slug_collision touches a gold session id (${goldCollisions.map(s => `${s} <- ${(slugToRaw.get(s) ?? []).join(' / ')}`).join('; ')})`,
{ slug_collision: detectSlugCollisions(slugToRaw).length, slug_collision_gold: goldCollisions, abstention: isAbstentionQuestion(q.question_id) },
);
}
// --expansion-replay: a question with no recorded variants is an error row
// (exit 1 at the end) — detected here, not after the haystack was imported.
let expansionVariants: string[] | undefined;
let expansionReplayed = false;
let expandFn: ((query: string) => Promise<string[]>) | undefined;
if (!opts.keywordOnly && opts.expansion) {
if (ctx.replay) {
const recorded = ctx.replay.get(q.question_id);
if (!recorded) {
throw new QuestionAbort(`expansion_replay_miss: no recorded expansion_variants for ${q.question_id}`, { expansion_replay_miss: true });
}
expansionVariants = recorded;
expansionReplayed = true;
expandFn = async () => recorded;
} else {
const base = ctx.expandFn;
expandFn = async (query) => {
const v = await base(query);
expansionVariants = v;
return v;
};
}
}
await resetTables(engine);
const adapterPages = haystackToPages(q);
const dates = q.haystack_dates ?? [];
const pageMeta: { slug: string; content: string; date?: string }[] = [];
// Per-question alias map for the extractor so canonical-slug aliases never
// leak across questions.
const aliasMap: AliasMap = makeAliasMap();
let meta: HybridSearchMeta | undefined;
const extra0: Record<string, unknown> = {};
const decideBefore = await decideLane.lmeSpendBefore(engine, ctx.decide);
let pool: SearchResult[] | undefined;
let preRerank: SearchResult[] | undefined;
let viaRow: viaOperation.ViaOperationRow | undefined;
// The embed-producing section (import + search) runs in ONE embed-cache
// transaction and COMMITS before the trajectory lookup / reader call.
const results: SearchResult[] = await ctx.embedTxn(async () => {
for (let i = 0; i < adapterPages.length; i++) {
const p = adapterPages[i];
pageMeta.push({ slug: p.slug, content: p.content, date: dates[i] });
await importFromContent(engine, p.slug, p.content, { noEmbed: opts.keywordOnly });
// Inline claim extractor populates the facts table so trajectory routing
// has data to retrieve. Fail-open per session.
if (ctx.trajectoryEnabled) {
await extractAndInsertClaims({
engine,
client: ctx.extractorClient,
model: ctx.extractorModel,
sessionSlug: p.slug,
sessionId: sessionIdFromSlug(p.slug),
sessionBody: p.content,
sourceId: 'default',
aliasMap,
});
}
}
if (ctx.synopsis) Object.assign(extra0, { contextual_synopsis: await synopsisTier.applySynopsisTier(engine, adapterPages, ctx.synopsis) });
if (ctx.factKeys) Object.assign(extra0, { fact_keys: await retrievalArms.applyFactKeyArm(engine, adapterPages, ctx.factKeys.arm, ctx.factKeys.spend) });
if (opts.viaOp.op) {
const via = await viaOperation.runViaOperation(engine, q.question, { ...opts.viaOp, op: opts.viaOp.op }, ctx.expectExpansion);
meta = via.meta; viaRow = via.row;
return via.results;
}
if (opts.keywordOnly) return engine.searchKeyword(q.question, { limit: opts.topK });
const searchOpts: HybridSearchOpts = {
limit: opts.arms.timeScope ? Math.max(opts.topK, opts.arms.timeScopePool) : opts.topK,
// Per-call wins over the bundle: expansion fires ONLY with --expansion.
expansion: opts.expansion,
...(expandFn ? { expandFn } : {}),
...(opts.expansionVariantBudget !== undefined ? { expansionVariantBudget: opts.expansionVariantBudget } : {}),
onMeta: (m) => { meta = m; }, ...decideLane.lmeDecideSearchOpts(ctx.decide),
...(opts.capturePool
? { onRerankPool: (p: readonly SearchResult[], pre?: readonly SearchResult[]) => { pool = [...p]; preRerank = pre ? [...pre] : undefined; } }
: {}),
};
return retrievalArms.scopeResults(await hybridSearch(engine, q.question, searchOpts), opts.arms, { q, pageMeta, slugToRaw, gold, k: opts.topK, extra: extra0 });
});
// Trajectory routing for temporal / knowledge_update intents. Skips for
// 'other' or when --no-trajectory. Best-effort (trajectory-route.ts).
const intent: Intent = ctx.trajectoryEnabled ? classifyIntent(q) : 'other';
const route: TrajectoryRoute = ctx.trajectoryEnabled && intent !== 'other'
? await routeTrajectory(engine, q.question, results.map(r => r.slug), intent)
: EMPTY_TRAJECTORY_ROUTE;
// Reader pins (D30) ride every answered row: requested model, the
// provider-reported snapshot when it differs, the system-prompt sha and the
// output cap. --retrieval-only rows are marked so a --judge backfill can
// refuse them (no reader hypothesis to grade).
let hypothesis: string;
let readerFields: Record<string, unknown>;
if (opts.retrievalOnly) {
hypothesis = renderRetrievedAsHypothesis(results, slugToRaw);
readerFields = { retrieval_only: true };
} else {
// The gateway is called with the NORMALIZED id (gatewayClient normalizes
// again, idempotently), so the snapshot comparison inside generateAnswer
// sees the same string the provider echoes — a bare alias never shows up
// as a fake `reader_model_snapshot`.
const answer = decideLane.lmeAbstention(ctx.decide, meta) ?? await generateAnswer(ctx.client, q, results, pageMeta, slugToRaw, normalizeModelId(ctx.model), route.block, ctx.readerConfig);
const readerError = answer.finish_reason === 'max_tokens' ? 'reader_max_tokens'
: answer.finish_reason !== 'end_turn' ? 'reader_unknown_finish_reason'
: answer.text === '' ? 'reader_empty_response' : null;
hypothesis = readerError ? '' : answer.text;
readerFields = {
reader_model: ctx.model,
reader_model_snapshot: answer.response_model,
reader_mode: ctx.readerConfig.mode,
reader_prompt_version: ctx.readerConfig.promptVersion,
reader_prompt_sha: ctx.readerConfig.promptSha,
reader_max_tokens: ctx.readerConfig.maxTokens,
reader_config_hash: ctx.readerHash,
reader_finish_reason: answer.finish_reason, reader_usage: answer.usage,
...(readerError ? { error: readerError } : {}),
...(readerError && answer.text ? { reader_partial_output: answer.text } : {}),
reader_context_chars: answer.context_chars,
reader_context_sessions: answer.context_sessions,
reader_sessions_truncated: answer.sessions_truncated,
};
}
// search_meta (plan 0e): `reranked` is derived — no such meta field exists.
const degraded = meta?.degraded ?? [];
const searchMeta: Record<string, unknown> & RowSearchMeta = {
vector_enabled: meta?.vector_enabled ?? false,
expansion_applied: meta?.expansion_applied ?? false,
degraded,
reranked: false,
...(meta?.autocut ? { autocut: meta.autocut } : {}),
};
const { rerankerSkipped, vectorDegraded, expansionFailed } = classifyDegradation(searchMeta, { keywordOnly: opts.keywordOnly, expansion: ctx.expectExpansion });
searchMeta.reranked = (viaRow ? viaOperation.rerankCalled(viaRow) : results.some(r => Number.isFinite(r.rerank_score))) && !rerankerSkipped;
const extra: Record<string, unknown> = {
...extra0,
retrieval_config_hash: ctx.retrievalConfigHash,
...readerFields,
search_meta: searchMeta,
...(expansionVariants ? { expansion_variants: expansionVariants } : {}),
...(expansionReplayed ? { expansion_replayed: true } : {}), ...(viaRow ? { via_operation: viaRow } : {}),
...(ctx.trajectoryEnabled ? {
intent,
trajectory_points: route.points,
entity_resolved: route.entityResolved,
resolution_source: route.resolutionSource,
methodology_note: trajectoryMethodologyNote(ctx.extractorModel),
} : {}),
};
Object.assign(extra, buildCaptureExtras({ pool, preRerank, meta, results, slugToRaw, gold }), await decideLane.lmeDecideRow(engine, ctx.decide, meta, decideBefore));
const row = buildRow({ question: q, hypothesis, results, k: opts.topK, slugToRaw, mode: opts.mode, extra });
return { row, rerankerSkipped, vectorDegraded, expansionFailed };
}
/** Schema-v2 by_type_summary (metrics.ts builder) — the ONE summary shape. */
export type ByTypeSummary = ByTypeSummaryV2;
export { buildByTypeSummaryV2 as buildByTypeSummary } from '../eval/longmemeval/metrics.ts';
export { seedRecallByTypeFromFile } from '../eval/longmemeval/resume.ts';