Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* resume.ts — reading a prior run's JSONL back: row scanning, the
* `retrieval_config_hash` mixed-run check, expansion-variant replay maps, and
* re-scoring per-type buckets from rows + the dataset's gold.
*
* INVARIANT: recall is RECOMPUTED, never trusted. On resume every row's
* recall_all/any is re-derived from its `retrieved[]` (top-k chunk rows →
* distinct RAW session ids) or, for older rows, `retrieved_session_ids`,
* joined against the dataset's gold. A pre-stamp row whose ids are slug-tail
* normalized (lowercased, `_`/`.` → `-`) is mapped back to the RAW haystack id
* through `SeedContext.haystackByQid` when the harness supplies it (the
* dataset is loaded on resume); without that map such a row scores false
* instead of poisoning the summary. There is no legacy any-only counter
* (plan 0g).
*
* INVARIANT: pure given its inputs (file reads are the only I/O; no engine).
*/
import { existsSync, readFileSync } from 'node:fs';
import {
addRowToBucket,
isAbstentionQuestion,
newBucket,
normalizeSessionId,
scoreRecall,
type RecallBucket,
} from './metrics.ts';
import { readerConfigHash, READER_MAX_TOKENS, READER_PROMPT_SHA, READER_PROMPT_VERSION, type ReaderConfig } from './reader.ts';
import { normalizeModelId } from '../../core/model-id.ts';
/**
* Parse a JSONL file into rows; corrupt lines are skipped (SIGKILL tail).
* Question rows are deduped LAST-WINS per question_id (a same-file resume
* APPENDS judged-backfill rows and retries as newer duplicates and compacts
* only at run end — see emit.ts compactJsonlByQuestionId), keeping the
* first-seen position; non-question rows (summary) pass through in place.
*/
export function readJsonlRows(path: string): Array<Record<string, unknown>> {
if (!existsSync(path)) return [];
const out: Array<Record<string, unknown>> = [];
const slot = new Map<string, number>();
for (const line of readFileSync(path, 'utf8').split('\n')) {
if (!line.trim()) continue;
try {
const row = JSON.parse(line);
if (!(row && typeof row === 'object' && !Array.isArray(row))) continue;
const r = row as Record<string, unknown>;
if (typeof r.question_id === 'string' && r.kind !== 'by_type_summary') {
const at = slot.get(r.question_id);
if (at === undefined) { slot.set(r.question_id, out.length); out.push(r); }
else out[at] = r;
} else out.push(r);
} catch {
// corrupt line — the resume loader logs these; here we just skip
}
}
return out;
}
/** A per-question row (not a summary, not an error row awaiting retry). */
export function isScoredQuestionRow(row: Record<string, unknown>): boolean {
if (row.kind === 'by_type_summary') return false;
if (typeof row.question_id !== 'string') return false;
if (typeof row.error === 'string' && (!row.hypothesis || row.hypothesis === '')) return false;
return true;
}
export interface ResumeHashCheck {
/** Rows stamped with a DIFFERENT retrieval_config_hash than this run's. */
mismatched: number;
/** Rows with no hash at all (written before the stamp existed). */
unstamped: number;
/** Distinct foreign hashes seen (for the refusal message). */
foreign: string[];
}
/**
* Plan D33: a resume file whose rows were produced under different retrieval
* pins must not be silently merged with this run's rows. Unstamped rows are
* reported but tolerated (pre-stamp files carry no evidence either way).
*/
export function checkResumeConfigHash(
rows: ReadonlyArray<Record<string, unknown>>,
currentHash: string,
): ResumeHashCheck {
let mismatched = 0;
let unstamped = 0;
const foreign = new Set<string>();
for (const row of rows) {
if (!isScoredQuestionRow(row)) continue;
const h = row.retrieval_config_hash;
if (typeof h !== 'string') { unstamped++; continue; }
if (h !== currentHash) { mismatched++; foreign.add(h); }
}
return { mismatched, unstamped, foreign: [...foreign].sort() };
}
/** Reader runs are separate even when retrieval pins match. Historical direct rows retain their old receipt shape. */
export function checkResumeReaderConfig(
rows: ReadonlyArray<Record<string, unknown>>,
config: ReaderConfig,
model: string,
retrievalOnly = false,
judge = false,
): { mismatched: number; unknown: number } {
const expected = readerConfigHash(config, model);
let mismatched = 0;
let unknown = 0;
for (const row of rows) {
if (row.kind === 'by_type_summary' || typeof row.question_id !== 'string') continue;
if (row.retrieval_only === true) {
if (!retrievalOnly && !judge) mismatched++;
continue;
}
if (retrievalOnly && isScoredQuestionRow(row)) { mismatched++; continue; }
const hasReader = typeof row.reader_config_hash === 'string' || typeof row.reader_prompt_sha === 'string';
if (!hasReader) {
// Pre-reader failures have no hypothesis and are retried, not merged.
if (isScoredQuestionRow(row)) unknown++;
continue;
}
if (typeof row.reader_config_hash === 'string') {
if (row.reader_config_hash !== expected) mismatched++;
continue;
}
const legacyDirect = config.mode === 'direct' && config.promptSha === READER_PROMPT_SHA &&
config.promptVersion === READER_PROMPT_VERSION && config.maxTokens === READER_MAX_TOKENS &&
row.reader_prompt_sha === READER_PROMPT_SHA && row.reader_max_tokens === READER_MAX_TOKENS &&
typeof row.reader_model === 'string' && normalizeModelId(row.reader_model) === normalizeModelId(model);
if (!legacyDirect) mismatched++;
}
return { mismatched, unknown };
}
/**
* `--expansion-replay FILE`: question_id → recorded `expansion_variants`
* (the full list the live `expandFn` returned, original included). Rows
* without a variants array (keyword-only runs, error rows) are ignored.
*/
export function loadExpansionReplay(path: string): Map<string, string[]> {
if (!existsSync(path)) throw new Error(`--expansion-replay file not found: ${path}`);
const map = new Map<string, string[]>();
for (const row of readJsonlRows(path)) {
if (typeof row.question_id !== 'string') continue;
const v = row.expansion_variants;
if (!Array.isArray(v) || !v.every(x => typeof x === 'string')) continue;
map.set(row.question_id, v as string[]);
}
return map;
}
/**
* Distinct session ids over the row's top-k CHUNK rows. Prefers `retrieved[]`
* (rank-ordered chunk rows with RAW session ids); falls back to
* `retrieved_session_ids` (already distinct, first-occurrence order — the
* pre-`retrieved[]` row shape, where every returned row was a top-k row).
*/
export function retrievedIdsAtK(row: Record<string, unknown>, k: number): string[] {
const retrieved = row.retrieved;
if (Array.isArray(retrieved)) {
const ids: string[] = [];
for (const r of retrieved.slice(0, k)) {
const sid = (r as { session_id?: unknown })?.session_id;
if (typeof sid === 'string' && !ids.includes(sid)) ids.push(sid);
}
return ids;
}
const flat = row.retrieved_session_ids;
if (Array.isArray(flat)) return (flat.filter(x => typeof x === 'string') as string[]).slice(0, k);
return [];
}
export interface SeedContext {
/** question_id → RAW gold session ids (from the dataset loaded on resume). */
goldByQid: ReadonlyMap<string, readonly string[]>;
k: number;
includeAbstention: boolean;
/**
* question_id → RAW haystack session ids. When present, a retrieved id that
* is not a raw id but equals the slug-normalized form of exactly one raw id
* is scored as that raw id (legacy pre-stamp rows carried normalized ids).
*/
haystackByQid?: ReadonlyMap<string, readonly string[]>;
}
/**
* Map legacy normalized ids back to RAW haystack ids. An id already present in
* the haystack passes through; otherwise the unique raw id whose normalized
* form equals it is used; an ambiguous (colliding) or unknown id is kept as-is
* (and so scores as a miss, never as a false hit).
*/
export function rawifyRetrievedIds(ids: readonly string[], haystack: readonly string[] | undefined): string[] {
if (!haystack || haystack.length === 0) return [...ids];
const raw = new Set(haystack);
const byNormalized = new Map<string, string[]>();
for (const h of haystack) {
const n = normalizeSessionId(h);
const list = byNormalized.get(n);
if (list) { if (!list.includes(h)) list.push(h); } else byNormalized.set(n, [h]);
}
const out: string[] = [];
for (const id of ids) {
if (raw.has(id)) { if (!out.includes(id)) out.push(id); continue; }
const cands = byNormalized.get(id);
const mapped = cands && cands.length === 1 ? cands[0] : id;
if (!out.includes(mapped)) out.push(mapped);
}
return out;
}
export interface SeedResult {
/** Rows folded into a bucket. */
seeded: number;
excludedAbstention: number;
/** Rows skipped: not in the dataset, no gold, error/summary rows. */
skipped: number;
/** Per-seeded-row distinct_sessions_in_top_k (for mean_distinct_sessions). */
distinct: number[];
/**
* Rows whose gold names a session absent from their haystack. Counted over
* EVERY question row (abstention, no-gold and error rows included) — the
* same set the live harness counts — so `run_config.gold_missing_from_haystack`
* is identical for a fresh run and a resume of the same file.
*/
goldMissing: number;
/** Rows with at least one slug collision (same row set as `goldMissing`; a collision-abort error row counts). */
collisions: number;
}
/**
* Fold prior rows into per-type buckets by RECOMPUTING both metrics from the
* row's retrieved ids + the dataset's gold. Abstention rows (`_abs` id or
* `abstention: true`) stay out of the denominators unless `includeAbstention`.
*/
export function seedBucketsFromRows(
rows: ReadonlyArray<Record<string, unknown>>,
buckets: Record<string, RecallBucket>,
ctx: SeedContext,
): SeedResult {
const res: SeedResult = { seeded: 0, excludedAbstention: 0, skipped: 0, distinct: [], goldMissing: 0, collisions: 0 };
for (const row of rows) {
if (row.kind === 'by_type_summary' || typeof row.question_id !== 'string') { res.skipped++; continue; }
if (Array.isArray(row.gold_missing_from_haystack) && row.gold_missing_from_haystack.length > 0) res.goldMissing++;
if (typeof row.slug_collision === 'number' && row.slug_collision > 0) res.collisions++;
if (!isScoredQuestionRow(row) || typeof row.question_type !== 'string') { res.skipped++; continue; }
const qid = row.question_id as string;
const gold = ctx.goldByQid.get(qid);
if (!gold) { res.skipped++; continue; }
const abstention = isAbstentionQuestion(qid) || row.abstention === true;
if (abstention && !ctx.includeAbstention) { res.excludedAbstention++; continue; }
if (gold.length === 0) { res.skipped++; continue; }
const score = scoreRecall(rawifyRetrievedIds(retrievedIdsAtK(row, ctx.k), ctx.haystackByQid?.get(qid)), gold, ctx.k);
const bucket = buckets[row.question_type] ?? (buckets[row.question_type] = newBucket());
addRowToBucket(bucket, { recall_all_hit: score.recall_all_hit, recall_any_hit: score.recall_any_hit });
res.seeded++;
res.distinct.push(score.distinct_sessions_in_top_k);
}
return res;
}
/**
* `--resume-from`: the question_ids already answered in `resumePath`. Rows
* whose `hypothesis` is empty AND carry an `error` are NOT counted — those
* are previous-run failures that should be retried. Missing file → empty set
* (a first run with the flag behaves exactly like no flag).
*/
export function loadResumeSet(resumePath: string): Set<string> {
const done = new Set<string>();
if (!existsSync(resumePath)) return done;
let lineNo = 0;
// Last row per question_id decides (an appended retry supersedes an error row).
const last = new Map<string, boolean>();
for (const line of readFileSync(resumePath, 'utf8').split('\n')) {
lineNo++;
if (!line.trim()) continue;
let row: { question_id?: string; hypothesis?: string; error?: string; kind?: string };
try {
row = JSON.parse(line);
} catch {
process.stderr.write(`[longmemeval] resume: skipping corrupt line ${lineNo}\n`);
continue;
}
if (typeof row.question_id !== 'string' || row.kind === 'by_type_summary') continue;
last.set(row.question_id, !(row.error && (!row.hypothesis || row.hypothesis === '')));
}
for (const [qid, ok] of last) if (ok) done.add(qid);
return done;
}
/**
* Seed per-type buckets from an existing output file so the by_type_summary
* is cumulative across resume runs. Both metrics are RECOMPUTED from each
* row's retrieved ids + the dataset's gold (`goldByQid`); nothing is trusted
* from the row's own recall fields. Summary rows and error rows are skipped.
*/
export function seedRecallByTypeFromFile(
outputPath: string,
buckets: Record<string, RecallBucket>,
ctx: { goldByQid: ReadonlyMap<string, readonly string[]>; k: number; includeAbstention?: boolean; haystackByQid?: ReadonlyMap<string, readonly string[]> },
): SeedResult {
return seedBucketsFromRows(readJsonlRows(outputPath), buckets, {
goldByQid: ctx.goldByQid,
k: ctx.k,
includeAbstention: ctx.includeAbstention ?? false,
...(ctx.haystackByQid ? { haystackByQid: ctx.haystackByQid } : {}),
});
}
// ---------------------------------------------------------------------------
// Silent-degradation classification (shared by the live row and the resume re-scan)
// ---------------------------------------------------------------------------
/** The `search_meta` shape the harness stamps on every row (also read back on resume). */
export interface RowSearchMeta {
vector_enabled?: boolean;
degraded?: Array<{ stage?: string }>;
}
const VECTOR_DEGRADED_STAGES: ReadonlySet<string> = new Set(['embed_unavailable', 'embed_timeout']);
/** `expansion_not_applied`: a `--via-operation query` row the run expected to expand that did not (via-operation.ts). */
const EXPANSION_FAILED_STAGES: ReadonlySet<string> = new Set(['expansion_failed', 'expansion_partial', 'expansion_not_applied']);
const RERANKER_SKIPPED_STAGES: ReadonlySet<string> = new Set(['reranker_skipped', 'rerank_passthrough', 'rerank_failed']);
/**
* Silent-degradation classifier shared by the live row and the resume
* re-scan. hybridSearch swallows an embed/expansion failure into `degraded`
* and scores the row keyword-only — for a like-for-like receipt that row is
* NOT the configured arm, so the run must not exit 0.
*/
export function classifyDegradation(meta: RowSearchMeta | undefined, opts: { keywordOnly: boolean; expansion: boolean }): {
rerankerSkipped: boolean; vectorDegraded: boolean; expansionFailed: boolean;
} {
const stages = new Set((meta?.degraded ?? []).map(d => d.stage).filter((x): x is string => typeof x === 'string'));
const has = (set: ReadonlySet<string>) => [...stages].some(st => set.has(st));
return {
rerankerSkipped: has(RERANKER_SKIPPED_STAGES),
vectorDegraded: !opts.keywordOnly && (meta?.vector_enabled === false || has(VECTOR_DEGRADED_STAGES)),
expansionFailed: !opts.keywordOnly && opts.expansion && has(EXPANSION_FAILED_STAGES),
};
}
/** Re-scan prior rows (resume) for the three silent-degradation counters. */
export function countDegradation(
rows: ReadonlyArray<Record<string, unknown>>,
opts: { keywordOnly: boolean; expansion: boolean },
): { rerankerSkipped: number; vectorDegraded: number; expansionFailed: number } {
const out = { rerankerSkipped: 0, vectorDegraded: 0, expansionFailed: 0 };
for (const row of rows) {
if (row.kind === 'by_type_summary' || typeof row.question_id !== 'string') continue;
const c = classifyDegradation(row.search_meta as RowSearchMeta | undefined, opts);
if (c.rerankerSkipped) out.rerankerSkipped++;
if (c.vectorDegraded) out.vectorDegraded++;
if (c.expansionFailed) out.expansionFailed++;
}
return out;
}