Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.42 — Extract Receipt writer.
*
* Every LLM-backed extraction run writes a receipt page recording the
* round's outcome: total rows, cost, model id, eval verdict. Receipts
* are first-class brain memory — queryable via gbrain search, citable
* in takes, surfaced in cross-modal contradiction probes.
*
* Two structural protections against extraction loops (D-EXTRACT-19,
* belt + suspenders):
* 1. the page `type` column `extract_receipt` — eligibility predicate's
* type filter rejects this type because it's not in ELIGIBLE_TYPES.
* 2. `dream_generated: true` — the anti-loop guard at
* `src/core/facts/eligibility.ts:62` rejects this flag.
*
* Search-rank: receipts get factor 0.3 demote via the `extracts/` slug
* prefix entry in DEFAULT_SOURCE_BOOSTS (D-EXTRACT-42). They surface
* when specifically relevant (extraction-related queries) but never
* dominate user content.
*
* Slug shape (D-EXTRACT-17):
* extracts/{date}/{kind}/{source_id}/{run_id_short}/round-{N}.md
*
* Where:
* - date = YYYY-MM-DD of extraction
* - kind = extractor kind ("facts.conversation", "atoms", ...)
* - source_id = brain source the extraction targeted
* - run_id_short = first 8 chars of the op-checkpoint id (or
* progressive-batch operation_id) — groups all
* rounds of one run under one directory
* - N = round identifier ("trial" | "ramp_100" | "ramp_500" |
* "full" | "single")
*
* Resume rounds land under the same run_id_short so the audit trail
* for a halted-then-resumed run stays coherent.
*/
import type { BrainEngine } from '../engine.ts';
import type { Page } from '../types.ts';
import { maintenanceTransaction } from '../persistence/attribution.ts';
import { isNeverFiledDerivedPage } from '../persistence/git-declared.ts';
import { resealSafeChunks } from '../page-state/projections.ts';
import { SAFE_FENCE_CHUNKER_VERSION } from '../search/safe-chunks.ts';
/**
* Round identifier. Matches the progressive-batch primitive's Stage
* union from src/core/progressive-batch/types.ts:42, plus 'single' for
* extractors that don't ramp (deterministic / one-shot).
*/
export type ExtractReceiptRound =
| 'trial'
| 'ramp_100'
| 'ramp_500'
| 'full'
| 'single';
/**
* Input to writeReceipt. Optional fields are recorded in frontmatter
* only when the caller provides them, so receipts stay clean for
* deterministic extractors (no LLM cost or eval to record).
*/
export interface ExtractReceiptInput {
/** Kind of extraction (matches the conceptual handler name). */
kind: string;
/** Brain source the extraction targeted. */
source_id: string;
/** Op-checkpoint id or progressive-batch operation_id for trace continuity. */
run_id: string;
/** Which round this receipt covers. */
round: ExtractReceiptRound;
/** ISO timestamp of round completion. */
extracted_at: string;
/** Rows committed to the target store this round. */
total_rows: number;
/** Cumulative cost across the round in USD. 0 for deterministic extractors. */
cost_usd: number;
/** LLM model id (optional; only for LLM-backed extractors). */
model_id?: string;
/** Eval gate verdict (optional; only when a gate fired). */
eval_pass?: boolean;
/** Eval gate score (optional; companion to eval_pass). */
eval_score?: number;
/** Human-readable summary line (1-2 sentences). */
summary?: string;
}
const RUN_ID_SHORT_LEN = 8;
/**
* Truncate a run id to the standard 8-char short form used in slug
* paths. Idempotent — passing an already-short id returns it unchanged.
* Non-hex / non-alphanumeric chars survive INSIDE the short form
* (op-checkpoint ids may include dashes or other separators), but
* boundary hyphens are trimmed (#3443): `slugifySegment()` strips
* leading/trailing hyphens during repo sync, so a short form like
* 'propose-' (from propose-<timestamp> run ids) made the DB receipt
* slug and its Git-backed slug disagree — writing the receipt through
* to the repo created a normalized sibling instead of materializing
* the existing page. Invariant: slugifySegment(shortRunId(x)) ===
* shortRunId(x) for slug-safe run ids.
*/
export function shortRunId(runId: string): string {
// ponytail: truncation-based discrimination is only as good as the run id's
// first 8 chars; families that need per-run uniqueness must front-load it.
const short = runId.slice(0, RUN_ID_SHORT_LEN).replace(/^-+|-+$/g, '');
return short || (runId ? 'run' : '');
}
/**
* Derive a YYYY-MM-DD date string from an ISO timestamp.
* Falls back to UTC slice; locale-independent.
*/
export function dateFromIso(iso: string): string {
// ISO 8601 always starts YYYY-MM-DD. Slice the first 10 chars.
// If the caller passes a non-ISO shape, slice still returns something;
// the receipt slug is best-effort labeling, not load-bearing parsing.
return iso.slice(0, 10);
}
/**
* Compute the canonical slug for a receipt. Exported for tests +
* symmetry with read-side code that needs to compose the same slug
* (e.g. doctor reading receipts by run_id).
*/
export function receiptSlug(input: ExtractReceiptInput): string {
const date = dateFromIso(input.extracted_at);
const short = shortRunId(input.run_id);
return `extracts/${date}/${input.kind}/${input.source_id}/${short}/round-${input.round}`;
}
/**
* Build the receipt page body. Operator-readable. Frontmatter is
* machine-readable.
*/
function buildReceiptBody(input: ExtractReceiptInput): string {
const lines: string[] = [];
lines.push(`# ${input.kind} — round ${input.round}`);
lines.push('');
if (input.summary) {
lines.push(input.summary);
lines.push('');
}
lines.push(`Source: \`${input.source_id}\``);
lines.push(`Run: \`${input.run_id}\``);
lines.push(`Round: \`${input.round}\``);
lines.push(`Extracted at: ${input.extracted_at}`);
lines.push('');
lines.push(`Rows extracted: **${input.total_rows}**`);
if (typeof input.cost_usd === 'number' && input.cost_usd > 0) {
lines.push(`Cost: $${input.cost_usd.toFixed(4)}`);
}
if (input.model_id) {
lines.push(`Model: \`${input.model_id}\``);
}
if (typeof input.eval_pass === 'boolean') {
const verdict = input.eval_pass ? 'PASS' : 'FAIL';
const score = typeof input.eval_score === 'number'
? ` (score ${input.eval_score.toFixed(2)})`
: '';
lines.push(`Eval gate: **${verdict}**${score}`);
}
return lines.join('\n') + '\n';
}
/**
* Build the receipt frontmatter. `dream_generated: true` is stamped by every
* writeReceipt call regardless of caller; the other anti-loop flag is the
* page-level `type` writeReceipt passes to putPage, never a frontmatter key
* (`type` is parser-reserved, #7058). Per D-EXTRACT-19.
*/
function buildReceiptFrontmatter(input: ExtractReceiptInput): Record<string, unknown> {
const fm: Record<string, unknown> = {
dream_generated: true,
// #1978: receipts record an operation, not a source document — the
// run_id/round fields ARE the provenance. Explicit exemption keeps the
// doctor raw_provenance check quiet.
raw_trace_exempt: true,
raw_trace_exempt_reason: 'operation receipt; provenance is run_id + round',
kind: input.kind,
source_id: input.source_id,
run_id: input.run_id,
round: input.round,
extracted_at: input.extracted_at,
total_rows: input.total_rows,
cost_usd: input.cost_usd,
};
if (input.model_id) fm.model_id = input.model_id;
if (typeof input.eval_pass === 'boolean') fm.eval_pass = input.eval_pass;
if (typeof input.eval_score === 'number') fm.eval_score = input.eval_score;
return fm;
}
/**
* Write an extract receipt page. Returns the slug of the written page
* for the caller's audit/logging needs.
*
* Side-effects: calls engine.putPage with the receipt's compiled body +
* frontmatter. Threads sourceId so federated brains route the receipt
* to the same source the extraction targeted.
*
* #7620: a receipt is database-only derived bookkeeping, never content: a
* never-filed page under the derive-phase `extracts/` folder (the
* `derive_folder` class of docs/designs/GIT_CANONICAL_MEMORY.md), so the
* write refuses a slug that holds a content page or a page with a file.
* Its chunks are installed and sealed at the safe-chunk fence in the same
* transaction as the page (resealSafeChunks), so no reader sees it unsealed
* (remote search would report safe_index_pending) and a rollback leaves
* neither.
*
* Re-running with the same run_id + round overwrites the prior receipt
* (idempotent on resume). The op-checkpoint id is stable across
* resumes per src/core/op-checkpoint.ts, so this is the desired
* semantic.
*/
export async function writeReceipt(
engine: BrainEngine,
input: ExtractReceiptInput,
): Promise<{ slug: string; page: Page }> {
const slug = receiptSlug(input);
const title = `${input.kind} — ${input.round} — ${input.source_id}`;
const frontmatter = buildReceiptFrontmatter(input);
const compiled_truth = buildReceiptBody(input);
const page = await maintenanceTransaction(engine, async tx => {
await tx.lockPageKeys([{ sourceId: input.source_id, slug }]);
const [prior] = await tx.executeRaw<{ type: string | null; source_path: string | null; source_uri: string | null; withheld_source_path: string | null }>(
'SELECT type, source_path, source_uri, withheld_source_path FROM pages WHERE source_id=$1 AND slug=$2', [input.source_id, slug]);
if (!isNeverFiledDerivedPage(slug, prior ?? {}) || (prior && prior.type !== 'extract_receipt')) {
throw new Error(`Extract receipt ${slug} (source ${input.source_id}) was not written: that slug holds a content page or a page with a file in Git, and a receipt is database-only bookkeeping.`);
}
const written = await tx.putPage(
slug,
{
type: 'extract_receipt',
title,
compiled_truth,
frontmatter,
},
{ sourceId: input.source_id },
);
await resealSafeChunks(tx, slug, input.source_id);
const [sealed] = await tx.executeRaw<{ sealed: boolean }>(
`SELECT chunker_version >= ${SAFE_FENCE_CHUNKER_VERSION} AND text_projection_revision = knowledge_revision AS sealed FROM pages WHERE source_id=$1 AND slug=$2`,
[input.source_id, slug]);
if (sealed?.sealed !== true) throw new Error(`Extract receipt ${slug} (source ${input.source_id}) was not written: its chunks could not be sealed in the writing transaction.`);
// #4009: receipts are audit artifacts — deliberately never run through
// the contextual-retrieval ladder. Born with a NULL
// contextual_retrieval_mode they tripped doctor's
// contextual_retrieval_coverage warn on every extraction run right
// after a reindex. Stamp mode 'none' so receipts are born CR-evaluated.
// NOT permanent: a reindex that takes the DB fallback clears the stamp
// and it recurs per receipt — the doctor check also excludes
// type='extract_receipt' from mode_null (belt+braces).
await tx.updatePageContextualRetrievalState(slug, input.source_id, 'none', null);
return written;
});
return { slug, page };
}