Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* BrainBench write-back fidelity — grades the PRODUCTION conversation→memory
* pipeline (decision 15), not a bench-only path.
*
* Flow per fixture:
* 1. Render the fixture's turns into a conversation page in the
* imessage-slack line format the conversation-parser ships
* (`**Speaker** (YYYY-MM-DD H:MM AM): text`), import it (noEmbed).
* 2. Run `runExtractConversationFactsCore` over that page with an injected
* GOLD extractor: for each segment, it emits exactly the gold facts whose
* source turn text appears in the segment. Zero LLM calls; segmentation,
* insertFacts batching, dedup, provenance stamping, terminal-audit rows
* all execute the shipped code.
* With opts.llm=true the injection is skipped — the real Haiku extractor
* runs and extraction_recall / extraction_precision are additionally
* scored against the gold keyword probes.
* 3. Read back via the facts table and score:
* write_back_fidelity — gold facts that survived (keyword probe) with
* the right entity attribution
* provenance_accuracy — surviving facts carrying correct
* {source, source_session, source_markdown_slug}
*/
import type { BudgetTracker } from '../../../core/budget/budget-tracker.ts';
import type { ExtractInput, ExtractedFact } from '../../../core/facts/extract.ts';
import {
PER_SEGMENT_SOURCE_PREFIX,
TERMINAL_AUDIT_SOURCE,
runExtractConversationFactsCore,
} from '../../../commands/extract-conversation-facts.ts';
import { importFromContent } from '../../../core/import-file.ts';
import type { BrainEngine } from '../../../core/engine.ts';
import type { BrainBenchFixture, FixtureGold, GoldFactSpec } from '../types.ts';
import { SeedError } from '../seed.ts';
export interface WriteBackScore {
gold_total: number;
gold_failed: number;
/** Raw counters so the harness can aggregate across fixtures (Σ-based). */
survived: number;
provenance_ok: number;
/** Total non-audit rows stored (extraction_precision denominator in --llm mode). */
stored_rows: number;
/** Stored rows matching ANY gold probe (extraction_precision numerator). */
matched_any_gold: number;
metrics: Record<string, number>;
failed_items: string[];
}
/** Slug the rendered conversation page imports under. */
export function conversationSlug(fixtureId: string): string {
return `conversations/bench-${fixtureId}`;
}
/** Render a fixture turn timestamp as the imessage-slack inline form (UTC). */
function renderInlineTime(iso: string): string {
const d = new Date(iso);
// Date and time MUST come from the same UTC instant (red-team finding: a
// string-sliced date + UTC-converted time straddles midnight for non-Z
// offsets, silently reordering segments for foreign corpora).
const date = d.toISOString().slice(0, 10);
let h = d.getUTCHours();
const ampm = h >= 12 ? 'PM' : 'AM';
h = h % 12;
if (h === 0) h = 12;
const mm = String(d.getUTCMinutes()).padStart(2, '0');
return `(${date} ${h}:${mm} ${ampm})`;
}
export function renderConversationPage(fixture: BrainBenchFixture): string {
const lines: string[] = [
'---',
`title: Bench conversation ${fixture.fixture_id}`,
'type: conversation',
'---',
'',
];
for (const turn of fixture.turns) {
if (!turn.ts) {
throw new Error(`write-back fixture ${fixture.fixture_id} turn ${turn.turn_id} missing ts`);
}
const speaker = turn.role === 'user' ? 'You' : 'Assistant';
lines.push(`**${speaker}** ${renderInlineTime(turn.ts)}: ${turn.text}`);
}
lines.push('');
return lines.join('\n');
}
/** All gold facts of a fixture, with the turn text that produces each. */
function goldFactsWithSourceText(
fixture: BrainBenchFixture,
gold: FixtureGold,
): Array<{ turnText: string; spec: GoldFactSpec }> {
const out: Array<{ turnText: string; spec: GoldFactSpec }> = [];
for (const turn of fixture.turns) {
const tg = gold.turns[String(turn.turn_id)];
if (!tg?.gold_facts) continue;
for (const spec of tg.gold_facts) out.push({ turnText: turn.text, spec });
}
return out;
}
/**
* Deterministic gold extractor: emits the gold facts whose source turn text
* appears verbatim inside the segment the production pipeline hands it.
*/
export function makeGoldExtractor(
fixture: BrainBenchFixture,
gold: FixtureGold,
): (input: ExtractInput) => Promise<ExtractedFact[]> {
const items = goldFactsWithSourceText(fixture, gold);
return async (input: ExtractInput): Promise<ExtractedFact[]> => {
const out: ExtractedFact[] = [];
for (const { turnText, spec } of items) {
if (!input.turnText.includes(turnText)) continue;
out.push({
fact: spec.fact,
kind: spec.kind ?? 'fact',
entity_slug: spec.entity_slug,
source: input.source,
confidence: 1.0,
notability: 'medium',
embedding: null,
});
}
return out;
};
}
interface StoredFactRow {
fact: string;
entity_slug: string | null;
source: string;
source_session: string | null;
source_markdown_slug: string | null;
}
export async function runWriteBack(
engine: BrainEngine,
fixture: BrainBenchFixture,
gold: FixtureGold,
opts: {
llm: boolean;
budgetUsd?: number;
/**
* RUN-scOPED tracker for --llm mode (review finding: a per-invocation cap
* would multiply by fixture count). The harness owns one tracker for the
* whole run and threads it here; the pipeline uses it as-is.
*/
budgetTracker?: BudgetTracker;
},
): Promise<WriteBackScore> {
const sourceId = fixture.active_source ?? 'default';
const slug = conversationSlug(fixture.fixture_id);
const body = renderConversationPage(fixture);
const imported = await importFromContent(engine, slug, body, { noEmbed: true, sourceId });
if (imported.status !== 'imported') {
throw new SeedError(
fixture.fixture_id,
slug,
`conversation page import status=${imported.status}${imported.error ? ` (${imported.error})` : ''}`,
);
}
const extractResult = await runExtractConversationFactsCore(engine, {
sourceId,
slug,
types: ['conversation'],
overrideDisabled: true,
sleepMs: 0,
// Deterministic CI mode injects the gold extractor; --llm runs the real
// one under the RUN-scoped tracker (cost-guard pattern, decision 2).
...(opts.llm
? { budgetTracker: opts.budgetTracker, maxCostUsd: opts.budgetUsd }
: { extractor: makeGoldExtractor(fixture, gold) }),
});
if (extractResult.budget_exhausted) {
// Partial extraction would silently corrupt every downstream score —
// abort the run loudly instead (CLI maps this to exit 2).
throw new Error(
`brainbench --llm budget exhausted during ${fixture.fixture_id} (spent ~$${(extractResult.spent_usd ?? 0).toFixed(2)}) — raise --budget-usd or drop --llm`,
);
}
const stored = await engine.executeRaw<StoredFactRow>(
`SELECT fact, entity_slug, source, source_session, source_markdown_slug
FROM facts
WHERE source_id = $1
AND source_markdown_slug = $2
AND source <> $3`,
[sourceId, slug, TERMINAL_AUDIT_SOURCE],
);
const expectedSession = `${PER_SEGMENT_SOURCE_PREFIX}:${slug}`;
const goldItems = goldFactsWithSourceText(fixture, gold);
let survived = 0;
let provenanceOk = 0;
const failed: string[] = [];
// Each stored row may satisfy AT MOST one gold fact (codex adversarial
// finding: unconstrained find() let one merged/overly-broad extracted row
// inflate fidelity by matching several gold probes).
const consumed = new Set<number>();
for (const { spec } of goldItems) {
const matchIdx = stored.findIndex(
(row, idx) =>
!consumed.has(idx) &&
spec.match_keywords.every((kw) => row.fact.toLowerCase().includes(kw.toLowerCase())) &&
(spec.entity_slug === null || row.entity_slug === spec.entity_slug),
);
if (matchIdx === -1) {
failed.push(`${fixture.fixture_id} (gold fact lost: ${spec.gist})`);
continue;
}
consumed.add(matchIdx);
const match = stored[matchIdx];
survived++;
if (
match.source === PER_SEGMENT_SOURCE_PREFIX &&
match.source_session === expectedSession &&
match.source_markdown_slug === slug
) {
provenanceOk++;
} else {
failed.push(`${fixture.fixture_id} (provenance wrong on: ${spec.gist})`);
}
}
const metrics: Record<string, number> = {
write_back_fidelity: goldItems.length > 0 ? survived / goldItems.length : 1,
provenance_accuracy: survived > 0 ? provenanceOk / survived : 1,
};
// Computed in both modes so the harness can aggregate; only SCORED as
// extraction metrics when the real extractor ran (--llm).
const matchedAnyGold = stored.filter((row) =>
goldItems.some(({ spec }) =>
spec.match_keywords.every((kw) => row.fact.toLowerCase().includes(kw.toLowerCase())),
),
).length;
if (opts.llm) {
metrics.extraction_recall = goldItems.length > 0 ? survived / goldItems.length : 1;
metrics.extraction_precision = stored.length > 0 ? matchedAnyGold / stored.length : 1;
}
// gold_failed counts BOTH lost facts and provenance failures — a fact that
// survived with wrong provenance is a failed gold item for the count gate.
return {
gold_total: goldItems.length,
gold_failed: goldItems.length - survived + (survived - provenanceOk),
survived,
provenance_ok: provenanceOk,
stored_rows: stored.length,
matched_any_gold: matchedAnyGold,
metrics,
failed_items: failed,
};
}