Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* corpus-segments.ts — Cathedral 5 (checkpoint compaction): content-addressed
* pre-compaction corpus segments + the per-session ledger + the session-end
* exact-set coverage decision.
*
* ENGINE-FREE, pure fs given a corpus dir — usable from the engine-free hook
* command AND (lazily) from the OpenClaw context engine's compact() step.
*
* Identity model (codex round 2):
* - Segment filename = `<sessionId>.seg-<sha256_24(redactedText)>.txt` —
* PURELY content-addressed, no ordinal. A retry of identical content maps
* to the same name (idempotent; a stale `.ingested` sidecar on that name
* means the same bytes were already extracted — a correct skip, never a
* collision). Segments end in `.txt` ON PURPOSE: the serve sweep's corpus
* pass picks them up with zero changes (the extraction backstop).
* - Ledger `<sessionId>.ledger.json` = append-only array of {hash, ts},
* entry order is the only ordinal (manifest metadata, never filenames).
* Writes are atomic tmp+rename; unreadable/unparseable ⇒ treated as
* absent (fail-open — the consumer falls back to full-transcript
* behavior, at-least-once).
* - Crash ordering: segment rename FIRST, ledger append SECOND. A crash
* between leaves an orphan segment (sweep still extracts it; session-end
* sees the ledger gap and falls back to full-file).
*
* Coverage (session-end): for each non-empty boundary window in the caller's
* OWN parse, recompute the window's redacted-segment hash and require it in
* the ledger SET (count equality alone can be fooled by a duplicated boundary
* masking a missed one). Ledger-superset is fine — a banked segment whose
* window scrolled out of the parse tail is still covered.
*/
import { createHash } from 'node:crypto';
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
import { basename, join } from 'node:path';
import type { WindowTurn } from './entity-salience.ts';
import type { CorpusFileStat, CorpusProgress } from './corpus-windows.ts';
import { toCorpusText, type ToolCallRecord } from '../transcripts/claude-code-jsonl.ts';
import { mapOpenclawLine } from '../transcripts/openclaw.ts';
import { stripPastedContent, stripPastedContentFromCorpus } from '../transcripts/pasted-content.ts';
/** Length of the hex content-hash slice in segment filenames. */
/**
* 24 hex chars = 96 bits (adversarial review: the original 48-bit truncation
* let an attacker who influences conversation content birthday-collide two
* windows offline in ~2^24 hashes — the second window would map to an
* existing filename and silently never be banked. 96 bits puts the collision
* work at ~2^48 hashes; filenames stay well under SAFE_CORPUS_BASENAME's 240
* cap). Parsers accept 12–64 hex so pre-widening segments still match.
*/
const SEGMENT_HASH_LEN = 24;
/**
* Filename-component sanitizer (pre-landing review, security): session ids
* reach the filename builders from MULTIPLE trust planes (the hook sanitizes
* its own, but the OpenClaw engine's params.sessionId and IPC-supplied ids
* are host-shaped) — enforce the safe charset HERE so every caller is
* covered structurally. Matches hook.ts's sanitizeSessionId charset.
*/
function safeIdComponent(id: string): string {
const s = String(id).replace(/[^A-Za-z0-9._-]/g, '-').slice(0, 120);
return s && !/^\.+$/.test(s) ? s : 'unknown';
}
export interface SegmentLedgerEntry {
hash: string;
ts: string;
}
export function segmentHash(text: string): string {
return createHash('sha256').update(text, 'utf8').digest('hex').slice(0, SEGMENT_HASH_LEN);
}
/**
* #6268 (R14): the spool subdirectory every source-stamped corpus file lives
* in. A pre-stamp sweeper reads only the corpus dir's top level and files any
* non-writeback `.txt` there under its own pass source, so a stamped file it
* could see would be misfiled; inside this subdirectory it sees nothing.
* Top-level files are the legacy (unstamped) corpus. No leading dot: dream
* synthesis walks into it exactly as it walked the top level.
*/
export const CORPUS_SPOOL_SUBDIR = 'sourced';
/**
* The stamp of a file whose writer could not resolve the session's source.
* Underscores are never valid in a source id (`SOURCE_ID_RE`), so no real
* source can be named this.
*/
export const CORPUS_UNRESOLVED_STAMP = '_unresolved';
const SOURCE_STAMP_RE = /^[a-z0-9](?:[a-z0-9-]{0,30}[a-z0-9])?$/;
/** A valid corpus source stamp: a source id or the unresolved marker. */
export function isCorpusSourceStamp(stamp: unknown): stamp is string {
return typeof stamp === 'string' && (stamp === CORPUS_UNRESOLVED_STAMP || SOURCE_STAMP_RE.test(stamp));
}
export function corpusSpoolDir(corpusDir: string): string {
return join(corpusDir, CORPUS_SPOOL_SUBDIR);
}
/** The spool dir, created 0700. Throws on fs failure. */
export function ensureCorpusSpoolDir(corpusDir: string): string {
const dir = corpusSpoolDir(corpusDir);
mkdirSync(dir, { recursive: true, mode: 0o700 });
return dir;
}
function stampPart(stamp: string | null | undefined): string {
if (stamp === undefined || stamp === null) return '';
if (!isCorpusSourceStamp(stamp)) throw new Error(`invalid corpus source stamp ${JSON.stringify(stamp)}`);
return `.src-${stamp}`;
}
/** `<sessionId>.seg-<hash>[.src-<stamp>].txt`; spool writers always pass the stamp. */
export function segmentFileName(sessionId: string, hash: string, stamp?: string | null): string {
return `${safeIdComponent(sessionId)}.seg-${hash.replace(/[^0-9a-f]/g, '')}${stampPart(stamp)}.txt`;
}
/** The session-end corpus file: `<sessionId>[.src-<stamp>].txt`. */
export function sessionCorpusFileName(sessionId: string, stamp?: string | null): string {
return `${safeIdComponent(sessionId)}${stampPart(stamp)}.txt`;
}
export function ledgerFileName(sessionId: string): string {
return `${safeIdComponent(sessionId)}.ledger.json`;
}
/** #4618 per-session seat sidecar (context/seat.ts owns its content). One
* per SESSION, not per corpus file: the session-end `.txt`, its checkpoint
* segments and its writeback turns all share it. */
export const SEAT_SIDECAR_SUFFIX = '.seat.json';
export function seatFileName(sessionId: string): string {
return `${safeIdComponent(sessionId)}${SEAT_SIDECAR_SUFFIX}`;
}
/** #8064 per-session speaker sidecar (context/session-speaker.ts owns its content): present only for agent-launched sessions. */
export const SPEAKER_SIDECAR_SUFFIX = '.speaker.json';
export function speakerFileName(sessionId: string): string {
return `${safeIdComponent(sessionId)}${SPEAKER_SIDECAR_SUFFIX}`;
}
/** #6268: the per-session source record in the spool (context/corpus-source.ts owns its content and lock). */
export const SESSION_SOURCE_SUFFIX = '.source.json';
export const SESSION_SOURCE_LOCK_SUFFIX = '.source.lock';
export function sessionSourceFileName(sessionId: string): string {
return `${safeIdComponent(sessionId)}${SESSION_SOURCE_SUFFIX}`;
}
/** A seat sidecar written ahead of its corpus file is never reaped inside
* this window (the writer renames the corpus file within seconds). */
const SEAT_ORPHAN_GRACE_MS = 10 * 60 * 1000;
/** Receipt sidecar suffix (harvest link candidates persisted before manifest
* publish). Lives HERE so the engine-free hook lane can GC orphaned receipts
* without importing the engine-typed harvest module. */
export const HARVEST_RECEIPT_SUFFIX = '.receipt.json';
/** #6091: capture-time consent record (capture-consent.ts); reaped with its corpus file after a grace period.
* `<file>.capture-off.json` is the pre-wave-12 single record; W4.2 records are `<file>.capture-off.<brain>.json`. */
export const CAPTURE_OFF_SUFFIX = '.capture-off.json';
export const CAPTURE_OFF_INFIX = '.capture-off.';
/** #5887 window-progress sidecar (context/corpus-windows.ts) and its CAS
* lock. Engine-free home so the hook's GC reaps both with the `.txt`; the
* hook's resume rewrite never deletes them. */
export const CORPUS_PROGRESS_SUFFIX = '.progress';
export const CORPUS_PROGRESS_LOCK_SUFFIX = '.progress.lock';
/**
* Inverse of `segmentFileName`: `{sessionId, hash}` when `name` is a corpus
* checkpoint segment, null for any other corpus file. The returned sessionId
* is the SANITIZED filename component — exactly the key the compact lanes
* bank the manifest under. Accepts 12–64 hex (pre-widening segments parse).
*/
export function parseSegmentFileName(
name: string,
): { sessionId: string; hash: string; stamp?: string } | null {
const m = /^(.+)\.seg-([0-9a-f]{12,64})(?:\.src-([a-z0-9_-]{1,40}))?\.txt$/.exec(name);
if (!m) return null;
return { sessionId: m[1], hash: m[2], ...(m[3] && isCorpusSourceStamp(m[3]) ? { stamp: m[3] } : {}) };
}
/** `{sessionId, stamp}` of a stamped session-end file; null for any other name. */
export function parseSessionCorpusFileName(name: string): { sessionId: string; stamp: string } | null {
if (parseSegmentFileName(name) || parseWbFileName(name)) return null;
const m = /^(.+)\.src-([a-z0-9_-]{1,40})\.txt$/.exec(name);
return m && isCorpusSourceStamp(m[2]) ? { sessionId: m[1], stamp: m[2] } : null;
}
/**
* The source stamp a corpus file NAME carries (any kind), or null. A spool
* file without one is malformed; at the top level only the legacy
* writeback grammar ever carried a stamp.
*/
export function corpusFileStamp(name: string): string | null {
return parseSegmentFileName(name)?.stamp ?? parseWbFileName(name)?.sourceId ?? parseSessionCorpusFileName(name)?.stamp ?? null;
}
/**
* The pre-compaction window: turns since the last PRIOR boundary (at compact
* time the current boundary is not yet in the file), optionally capped to the
* newest `maxTurns` (the OpenClaw arm caps at 40; the claude lane passes no
* cap — its window is already bounded by the parse tail).
*/
export function sliceBoundaryWindow(
turns: WindowTurn[],
boundaryTurnIndexes: number[],
opts: { maxTurns?: number } = {},
): WindowTurn[] {
const from = boundaryTurnIndexes.length ? boundaryTurnIndexes[boundaryTurnIndexes.length - 1] : 0;
const win = turns.slice(from);
return opts.maxTurns && win.length > opts.maxTurns ? win.slice(-opts.maxTurns) : win;
}
/** All boundary windows + the post-last-boundary remainder, in turns-index space. */
export function splitByBoundaries(
turns: WindowTurn[],
boundaryTurnIndexes: number[],
): { windows: WindowTurn[][]; remainder: WindowTurn[] } {
const windows: WindowTurn[][] = [];
let prev = 0;
for (const idx of boundaryTurnIndexes) {
windows.push(turns.slice(prev, idx));
prev = idx;
}
return { windows, remainder: turns.slice(prev) };
}
/**
* Render a window to redacted corpus text. Returns null when the secret
* scanner is unavailable — a SEGMENT is never written unscanned (unlike the
* session-end full-corpus write, which degrades-and-writes because the corpus
* dir is 0700-local; a segment skip is safe — session-end's coverage check
* fails for that window and the full fallback carries the text).
*/
export async function renderSegmentText(
turns: WindowTurn[],
): Promise<{ text: string; redactions: number } | null> {
if (!turns.length) return { text: '', redactions: 0 };
const text = toCorpusText(turns);
try {
const scan = await import('../secret-scan.ts');
const redacted = scan.redactFindings(text);
return { text: redacted.text, redactions: redacted.redactions.length };
} catch {
return null;
}
}
/** Read the ledger. Fail-open: absent/unreadable/malformed ⇒ []. */
export function readSegmentLedger(dir: string, sessionId: string): SegmentLedgerEntry[] {
try {
const raw = readFileSync(join(dir, ledgerFileName(sessionId)), 'utf8');
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) return [];
return parsed.filter(
(e): e is SegmentLedgerEntry =>
!!e && typeof e === 'object' &&
typeof (e as SegmentLedgerEntry).hash === 'string' &&
typeof (e as SegmentLedgerEntry).ts === 'string',
);
} catch {
return [];
}
}
/**
* Append a hash to the ledger (atomic tmp+rename; single writer per session ⇒
* last-writer-wins is safe). Idempotent: an already-present hash is NOT
* re-appended. Returns the entry's 1-based ordinal (the manifest's `n`).
* Throws only on write failure — callers treat that as a degraded segment.
*/
export function appendSegmentLedger(dir: string, sessionId: string, hash: string): number {
const entries = readSegmentLedger(dir, sessionId);
const existing = entries.findIndex((e) => e.hash === hash);
if (existing !== -1) return existing + 1;
entries.push({ hash, ts: new Date().toISOString() });
const p = join(dir, ledgerFileName(sessionId));
const tmp = `${p}.tmp-${process.pid}`;
writeFileSync(tmp, JSON.stringify(entries) + '\n', { mode: 0o600 });
renameSync(tmp, p);
return entries.length;
}
/**
* Atomically write a content-addressed segment. Same content ⇒ same name ⇒
* `existed: true` and NO rewrite (sidecar lifecycle stays untouched by
* construction). Throws on fs failure.
*/
export function writeSegment(
dir: string,
sessionId: string,
text: string,
stamp?: string | null,
): { file: string; hash: string; existed: boolean } {
const hash = segmentHash(text);
const file = join(dir, segmentFileName(sessionId, hash, stamp));
if (existsSync(file)) return { file, hash, existed: true };
const tmp = `${file}.tmp-${process.pid}`;
writeFileSync(tmp, text, { mode: 0o600 });
renameSync(tmp, file);
return { file, hash, existed: false };
}
/**
* Exact-set coverage: TRUE iff every NON-EMPTY boundary window's recomputed
* redacted hash is present in the ledger (empty windows were skipped at
* compact time by design — covered by construction). Scanner unavailable or
* no boundaries-with-ledger-mismatch ⇒ FALSE (caller falls back to the
* full-transcript write; at-least-once, never loss).
*/
export async function coverageComplete(
dir: string,
sessionId: string,
turns: WindowTurn[],
boundaryTurnIndexes: number[],
): Promise<boolean> {
if (!boundaryTurnIndexes.length) return false;
const ledger = new Set(readSegmentLedger(dir, sessionId).map((e) => e.hash));
if (!ledger.size) return false;
const { windows } = splitByBoundaries(turns, boundaryTurnIndexes);
for (const win of windows) {
if (!win.length) continue;
const rendered = await renderSegmentText(win);
if (!rendered) return false;
if (!rendered.text.trim()) continue;
if (!ledger.has(segmentHash(rendered.text))) return false;
}
return true;
}
/**
* Tail-capable openclaw boundary reader (cathedral 5): the import adapter
* REJECTS over-cap files and whole-file-reads under the cap — too heavy for
* a compaction boundary — so this reads the newest `maxBytes`, drops the
* partial first line on tail reads, and delegates line→message mapping to
* the mapper EXPORTED from the adapter (the dated SPEC_TARGET stays the
* single source of truth). Boundary positions in turns-index space, exactly
* like the claude-code parser. Returns null when nothing maps (not an
* openclaw session file) or on fs errors — callers fail open.
*/
export function readOpenclawBoundaryTail(
path: string,
opts: { maxBytes?: number } = {},
): OpenclawBoundaryTail | null {
const maxBytes = Math.max(1, Math.floor(opts.maxBytes ?? 2 * 1024 * 1024));
let raw: string;
try {
const size = statSync(path).size;
if (size <= maxBytes) {
raw = readFileSync(path, 'utf8');
} else {
const fd = openSync(path, 'r');
try {
const buf = Buffer.alloc(maxBytes);
const n = readSync(fd, buf, 0, maxBytes, size - maxBytes);
raw = buf.subarray(0, n).toString('utf8');
} finally {
closeSync(fd);
}
}
} catch {
return null;
}
const entries: unknown[] = [];
for (const line of raw.split('\n')) {
const t = line.trim();
if (!t) continue;
try {
entries.push(JSON.parse(t));
} catch {
continue; // includes a tail read's partial first line
}
}
return mapOpenclawEntries(entries);
}
export type OpenclawBoundaryTail = { turns: WindowTurn[]; boundaryTurnIndexes: number[]; toolCalls: ToolCallRecord[]; toolCallTurnIndexes: number[] };
/**
* #6316: OpenClaw 2026.9.x keeps transcripts in its SQLite store and passes
* compact() a `sessionTarget` instead of a `sessionFile`. The host's
* `session-transcript-runtime` returns the same event objects the JSONL file
* holds, so this keeps the newest events up to `maxBytes` of serialized JSON
* (the file reader's tail bound) and maps them with the same mapper.
*/
export function readOpenclawEventsTail(events: unknown[], opts: { maxBytes?: number } = {}): OpenclawBoundaryTail | null {
const maxBytes = Math.max(1, Math.floor(opts.maxBytes ?? 2 * 1024 * 1024));
let bytes = 0;
let start = events.length;
while (start > 0) {
let size: number;
try {
size = Buffer.byteLength(JSON.stringify(events[start - 1]) ?? '', 'utf8') + 1;
} catch {
break;
}
if (bytes + size > maxBytes) break;
bytes += size;
start--;
}
return mapOpenclawEntries(events.slice(start));
}
function mapOpenclawEntries(entries: unknown[]): OpenclawBoundaryTail | null {
const turns: WindowTurn[] = [];
const boundaryTurnIndexes: number[] = [];
// Memorable integration: tool calls stamped with the turn slot they sit at
// (turns.length BEFORE the line's own turn is pushed — same semantics as the
// claude-code parser), so a caller writing only a window span can filter the
// calls to the same origin. A `skip` line can still carry calls (a
// placeholder-only message has no text but its calls are real work).
const toolCalls: ToolCallRecord[] = [];
const toolCallTurnIndexes: number[] = [];
let mappedAnything = false;
for (const entry of entries) {
const mapped = mapOpenclawLine(entry);
if (mapped.kind === 'message' || mapped.kind === 'skip') {
for (const c of mapped.toolCalls ?? []) {
toolCalls.push(c);
toolCallTurnIndexes.push(turns.length);
}
}
if (mapped.kind === 'session') {
mappedAnything = true;
} else if (mapped.kind === 'boundary') {
mappedAnything = true;
boundaryTurnIndexes.push(turns.length);
} else if (mapped.kind === 'message') {
mappedAnything = true;
turns.push({ role: mapped.message.role, text: mapped.message.text });
}
}
return mappedAnything ? { turns, boundaryTurnIndexes, toolCalls, toolCallTurnIndexes } : null;
}
/**
* Ambient-writeback turn files (WP4) — the Stop-hook backstop's siblings of
* checkpoint segments. `<sessionId>.wb-<hash24>[.src-<stamp>].txt` where
* the hash is the GATE's normalized-turn hash (turn identity, computed in
* writeback-gate.ts) — a re-fired Stop hook for the same turn re-derives the
* same name and short-circuits (requirement 9's deterministic idempotency,
* embedding-free). The `.src-` segment banks the session's source so the
* SWEEP fallback files the turn into the same source the prompt-time IPC
* lane would have (source-isolation invariant). #6268: spool writers stamp
* every file, `default` included; a top-level name without one is a legacy
* file written before stamps (old writers omitted `default` and an unset
* GBRAIN_SOURCE alike, so it proves nothing). They end `.txt` ON PURPOSE: the sweep's
* corpus pass is the free backstop when serve/IPC is down (the sweep applies
* the writeback gate before extracting them — OV2-11).
*/
export function wbFileName(sessionId: string, hash24: string, stamp?: string | null): string {
return `${safeIdComponent(sessionId)}.wb-${hash24.replace(/[^0-9a-f]/g, '')}${stampPart(stamp)}.txt`;
}
/** `{sessionId, hash, sourceId?}` when `name` is a writeback turn file, else
* null. `sourceId` is the RAW banked segment — consumers validate it
* (isValidSourceId) before any DB use. */
export function parseWbFileName(name: string): { sessionId: string; hash: string; sourceId?: string } | null {
const m = /^(.+)\.wb-([0-9a-f]{12,64})(?:\.src-([A-Za-z0-9._-]+))?\.txt$/.exec(name);
return m ? { sessionId: m[1], hash: m[2], ...(m[3] ? { sourceId: m[3] } : {}) } : null;
}
/**
* The harness session a `.txt` corpus file belongs to: the session-end
* `<sessionId>.txt`, a checkpoint segment or a writeback turn file. Used to
* classify gbrain's own claude-cli sessions (#5413) across every form.
*/
export function corpusFileSessionId(name: string): string {
return parseSegmentFileName(name)?.sessionId ?? parseWbFileName(name)?.sessionId
?? parseSessionCorpusFileName(name)?.sessionId ?? name.replace(/\.txt$/, '');
}
/**
* #5812 — the text a fact extractor may see from a session-corpus file: paste
* blocks removed (someone else's words), the file itself unchanged. A
* writeback turn file is one user turn; every other corpus file is
* `toCorpusText` blocks, stripped per `[user]` block.
*/
export function corpusTextForExtraction(path: string, text: string): string {
return parseWbFileName(basename(path)) ? stripPastedContent(text).text : stripPastedContentFromCorpus(text);
}
/** The TERMINAL writeback_off `.ingested` sidecar payload — the wb state
* machine's one terminal skip, written identically by the serve harvest and
* the sweep backstop (shape drift between the two writers would be invisible
* until a consumer disagrees). */
export function writebackOffSidecarJson(): string {
return JSON.stringify({ ingested_at: new Date().toISOString(), skipped: 'writeback_off' }) + '\n';
}
/** The TERMINAL self_capture `.ingested` sidecar payload (#5413/#5820) —
* written identically by the serve harvest and the sweep, so neither retries
* a corpus file captured from gbrain's own claude-cli session. */
export function selfCaptureSidecarJson(): string {
return JSON.stringify({ ingested_at: new Date().toISOString(), skipped: 'self_capture' }) + '\n';
}
/**
* Bank one gated user turn as a writeback corpus file: redacted render
* (NEVER written unscanned — scanner unavailable is a typed skip, matching
* the segment posture) → idempotent existence check (same turn or its
* `.ingested` sidecar ⇒ dup skip, zero LLM) → atomic write. Returns typed
* status codes for the heartbeat. Never throws.
*/
export async function bankWritebackTurn(
dir: string,
sessionId: string,
normalizedTurn: string,
hash24: string,
stamp?: string | null,
): Promise<{ status: 'wb_banked' | 'wb_dup' | 'wb_scan_unavailable' | 'wb_empty' | `wb_${string}`; flushCorpusFile?: string }> {
try {
const name = wbFileName(sessionId, hash24, stamp);
const file = join(dir, name);
if (existsSync(file) || existsSync(file + '.ingested')) return { status: 'wb_dup', flushCorpusFile: name };
let redacted: string;
try {
const scan = await import('../secret-scan.ts');
redacted = scan.redactFindings(normalizedTurn).text;
} catch {
return { status: 'wb_scan_unavailable' };
}
if (!redacted.trim()) return { status: 'wb_empty' };
const tmp = `${file}.tmp-${process.pid}`;
writeFileSync(tmp, redacted + '\n', { mode: 0o600 });
renameSync(tmp, file);
return { status: 'wb_banked', flushCorpusFile: name };
} catch (e) {
const name = e instanceof Error ? e.name || 'Error' : 'Error';
return { status: `wb_${name.toLowerCase()}` };
}
}
/**
* The whole compact-time segment step (cathedral 5, durability first): slice
* the since-last-boundary window → per-step deadline degrades → redacted
* render (never written unscanned) → content-addressed write → ledger append
* (segment FIRST, ledger SECOND — crash between leaves a sweep-harvestable
* orphan and a session-end full-fallback). Returns a typed status CODE for
* the heartbeat plus the segment basename to flush over IPC when banked.
* Never throws.
*/
export async function bankCompactSegment(
dir: string,
sessionId: string,
allTurns: WindowTurn[],
boundaryTurnIndexes: number[],
opts: {
remainingMs: () => number; minScanMs: number; minWriteMs: number; maxTurns?: number;
/** #6091: runs before the segment is renamed into place (capture-time consent record); a throw banks nothing. */
beforeWrite?: (file: string, text: string) => void;
/** #6268: the session's source stamp (spool writers always pass one). */
stamp?: string | null;
},
): Promise<{ segment: string; flushCorpusFile?: string; hash?: string; ordinal?: number }> {
try {
const windowTurns = sliceBoundaryWindow(allTurns, boundaryTurnIndexes, {
...(opts.maxTurns ? { maxTurns: opts.maxTurns } : {}),
});
if (windowTurns.length === 0) return { segment: 'empty_window' };
if (opts.remainingMs() < opts.minScanMs) return { segment: 'deadline_scan' };
const rendered = await renderSegmentText(windowTurns);
if (!rendered) return { segment: 'scan_unavailable' };
if (!rendered.text.trim()) return { segment: 'empty_window' };
if (opts.remainingMs() < opts.minWriteMs) return { segment: 'deadline_write' };
opts.beforeWrite?.(join(dir, segmentFileName(sessionId, segmentHash(rendered.text), opts.stamp)), rendered.text);
const w = writeSegment(dir, sessionId, rendered.text, opts.stamp);
const ordinal = appendSegmentLedger(dir, sessionId, w.hash);
return {
segment: w.existed ? 'segment_dup' : 'segment_banked',
flushCorpusFile: segmentFileName(sessionId, w.hash, opts.stamp),
hash: w.hash,
ordinal,
};
} catch (e) {
const name = e instanceof Error ? e.name || 'Error' : 'Error';
return { segment: `segment_${name.toLowerCase()}` };
}
}
/**
* Session-end corpus-mode decision (cathedral 5 dedup contract): when every
* non-empty boundary window is ledger-covered, only the post-last-boundary
* remainder needs the corpus write; an EMPTY remainder means everything is
* already segment-banked (skip the write). Any mismatch ⇒ full fallback.
* Never throws.
*/
export async function decideCorpusMode(
dir: string,
sessionId: string,
turns: WindowTurn[],
boundaryTurnIndexes: number[],
): Promise<{ mode: 'full' | 'full_fallback' | 'remainder' | 'skip_covered'; turns: WindowTurn[]; startTurnIndex: number }> {
if (!boundaryTurnIndexes.length) return { mode: 'full', turns, startTurnIndex: 0 };
try {
if (await coverageComplete(dir, sessionId, turns, boundaryTurnIndexes)) {
const remainder = splitByBoundaries(turns, boundaryTurnIndexes).remainder;
// splitByBoundaries takes the remainder from the LAST boundary, so that
// index is where the written span begins. Callers that derive anything
// else from the transcript (tool calls, hashes) must use the same
// origin, or their artifacts describe a different span than the corpus.
const startTurnIndex = boundaryTurnIndexes[boundaryTurnIndexes.length - 1] ?? 0;
return remainder.length === 0
? { mode: 'skip_covered', turns: remainder, startTurnIndex }
: { mode: 'remainder', turns: remainder, startTurnIndex };
}
} catch {
/* fall through to full_fallback */
}
return { mode: 'full_fallback', turns, startTurnIndex: 0 };
}
/**
* E-N1: a corpus `.txt` with no `.ingested` sidecar holds turns nothing has
* extracted yet (an HTTP serve with no sweep, a refused harvest). Retention
* removes it only at this multiple of the configured retention, and doctor
* `memory_writeback` warns while it waits.
*/
export const CORPUS_UNINGESTED_RETENTION_FACTOR = 3;
/** Completion sidecar suffix (sweep.ts CORPUS_INGESTED_SUFFIX; duplicated to stay engine-free). */
const INGESTED_SUFFIX = '.ingested';
/**
* #6786: `.ingested` skip reasons that record a state of the extracting
* process (no servable extraction model), not anything about the file. A
* sidecar carrying one is not a completion: selection, the HTTP drain, the
* doctor backlog and retention GC all treat the file as pending, and the
* sweep extracts it once a model serves (pre-#6786 builds wrote these).
*/
export const CORPUS_NON_TERMINAL_SKIPS: readonly string[] = ['chat_unavailable'];
/**
* #6786: what `<full>.ingested` says. `absent` (no sidecar), `unavailable`
* (a non-terminal skip), or `final` (a completion; an unreadable or
* non-JSON sidecar counts as one, as every build before #6786 did). Never throws.
*/
export function corpusSidecarState(full: string): 'absent' | 'unavailable' | 'final' {
let text: string;
try {
text = readFileSync(full + INGESTED_SUFFIX, 'utf8');
} catch (e) {
return (e as NodeJS.ErrnoException).code === 'ENOENT' ? 'absent' : 'final';
}
try {
const skipped = (JSON.parse(text) as { skipped?: unknown } | null)?.skipped;
return typeof skipped === 'string' && CORPUS_NON_TERMINAL_SKIPS.includes(skipped) ? 'unavailable' : 'final';
} catch {
return 'final';
}
}
function isFileStat(v: unknown): v is CorpusFileStat {
const s = v as CorpusFileStat;
return !!s && typeof s === 'object' && Number.isFinite(s.size) && Number.isFinite(s.mtime_ms) && Number.isFinite(s.ino);
}
/** #5887 `.progress` parse (context/corpus-windows.ts); null when unparseable (replayed safely). */
export function parseCorpusProgress(text: string): CorpusProgress | null {
let p: CorpusProgress;
try { p = JSON.parse(text) as CorpusProgress; } catch { return null; }
if (!p || typeof p !== 'object' || p.version !== 1 || !Number.isSafeInteger(p.generation)) return null;
if (!Array.isArray(p.turns) || !p.turns.every((t) => t && typeof t.sha256 === 'string')) return null;
if (p.continuation !== null && !(p.continuation && typeof p.continuation.turn_sha256 === 'string'
&& Number.isSafeInteger(p.continuation.stripped_offset))) return null;
if (p.finished !== null && !isFileStat(p.finished)) return null;
if (!p.totals || !Array.isArray(p.entity_slugs)) return null;
if (p.source != null && !(typeof p.source.id === 'string' && typeof p.source.incarnation === 'string')) return null;
return p;
}
/**
* How a corpus file stands (#6786: the one classifier for sweep selection,
* the under-claim re-check and retention GC):
* 'pending' — no `.ingested` sidecar;
* 'unavailable' — the sidecar records a non-terminal skip: pending;
* 'done' — its `.progress` finished this exact file (stat match);
* 'changed' — rewritten since; re-enter (turn hashes decide resume);
* 'legacy' — finished before `.progress` existed and not rewritten
* since (the sidecar is newer than the file): adopt;
* 'legacy_changed' — finished before `.progress` existed, rewritten after.
* Writeback turn files are immutable single turns: a final sidecar is 'done'.
* Only 'done' and 'legacy' are finished. Stat and progress read errors throw.
*/
export type CorpusFileCompletion = 'pending' | 'unavailable' | 'done' | 'changed' | 'legacy' | 'legacy_changed';
export function corpusFileCompletion(full: string, name: string): CorpusFileCompletion {
const sidecar = corpusSidecarState(full);
if (sidecar !== 'final') return sidecar === 'absent' ? 'pending' : 'unavailable';
if (parseWbFileName(name)) return 'done';
const st = statSync(full);
let progress: CorpusProgress | null = null;
try {
progress = parseCorpusProgress(readFileSync(full + CORPUS_PROGRESS_SUFFIX, 'utf8'));
} catch (e) {
if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e;
return statSync(full + INGESTED_SUFFIX).mtimeMs >= st.mtimeMs ? 'legacy' : 'legacy_changed';
}
if (!progress) return statSync(full + INGESTED_SUFFIX).mtimeMs >= st.mtimeMs ? 'legacy' : 'legacy_changed';
const f = progress.finished;
return f && f.size === st.size && f.mtime_ms === st.mtimeMs && f.ino === st.ino ? 'done' : 'changed';
}
export function corpusFileFinished(state: CorpusFileCompletion): boolean {
return state === 'done' || state === 'legacy';
}
/** Whether a corpus file still waits for extraction, by `corpusFileCompletion`; a file that is gone does not. */
export function corpusFileAwaitsExtraction(full: string, name: string): boolean {
try {
return !corpusFileFinished(corpusFileCompletion(full, name));
} catch {
return existsSync(full);
}
}
/**
* #6785: per-file record of the one corpus window (or writeback turn) whose
* extraction keeps failing after the model call. Written under the file's
* `.in-progress` claim by context/corpus-windows.ts; removed when that
* window commits, or by `gbrain sweep --retry-failed --apply`.
*/
export const CORPUS_FAILURE_SUFFIX = '.failures.json';
/** Failed attempts after which a window is not retried until an operator retry or one extraction-config change. */
export const CORPUS_WINDOW_FAILURE_CAP = 5;
/**
* #9279: failure codes no retry on the same build can change (the write refuses the same way every time). One such
* failure stops the window at once; the first sweep on another gbrain build retries it once; retention keeps the file.
*/
export const CORPUS_DETERMINISTIC_FAILURE_CODES: readonly string[] = ['page_body_segmented'];
export interface CorpusFailureRecord {
version: 1;
/** The failing window's start (`<turn sha256>@<offset>`), or `turn` for a writeback turn file. */
key: string;
source_incarnation: string | null;
count: number;
first_at: number;
last_at: number;
/** Error code or class of the last failure; never its message. */
last_code: string;
/** extraction-config fingerprint (model, provider, key presence) at the last failure. */
fingerprint: string | null;
/** True once a fingerprint change re-armed a capped window; a second change does not. */
rearmed: boolean;
/** #9279: the gbrain version that recorded a deterministic failure (CORPUS_DETERMINISTIC_FAILURE_CODES). */
build?: string;
/** #9279: true once the automatic retry on a newer build ran; only an operator retry runs the window after that. */
upgrade_retried?: boolean;
/** #9279: true once a deterministic failure held the window; retention keeps its file until it commits or an operator retries it. */
held?: boolean;
}
/** The failure record, or null when absent or unparseable. Never throws. */
export function readCorpusFailure(full: string): CorpusFailureRecord | null {
try {
const r = JSON.parse(readFileSync(full + CORPUS_FAILURE_SUFFIX, 'utf8')) as CorpusFailureRecord;
if (!r || r.version !== 1 || typeof r.key !== 'string' || !Number.isSafeInteger(r.count) || r.count < 1
|| !Number.isFinite(r.last_at) || !Number.isFinite(r.first_at) || typeof r.last_code !== 'string'
|| !(r.source_incarnation === null || typeof r.source_incarnation === 'string')
|| !(r.fingerprint === null || typeof r.fingerprint === 'string') || typeof r.rearmed !== 'boolean'
|| !(r.build === undefined || typeof r.build === 'string')
|| !(r.upgrade_retried === undefined || typeof r.upgrade_retried === 'boolean')
|| !(r.held === undefined || typeof r.held === 'boolean')) return null;
return r;
} catch {
return null;
}
}
/** #9279: a window stopped by a deterministic failure: its file is never extracted until a retry, so retention keeps it. */
export function corpusFailureHeld(record: CorpusFailureRecord | null): boolean {
return !!record && (record.held === true || (record.count >= CORPUS_WINDOW_FAILURE_CAP && CORPUS_DETERMINISTIC_FAILURE_CODES.includes(record.last_code)));
}
/** Every corpus `.txt` in `dir` with a failure record. Missing dir = empty; never throws. */
export function listCorpusFailures(dir: string): Array<{ name: string; full: string; record: CorpusFailureRecord }> {
let names: string[];
try {
names = readdirSync(dir);
} catch {
return [];
}
const present = new Set(names);
const out: Array<{ name: string; full: string; record: CorpusFailureRecord }> = [];
for (const name of names.filter((n) => n.endsWith('.txt') && present.has(n + CORPUS_FAILURE_SUFFIX)).sort()) {
const record = readCorpusFailure(join(dir, name));
if (record) out.push({ name, full: join(dir, name), record });
}
return out;
}
export interface CorpusBacklog {
/** `.txt` files not finished by `corpusFileCompletion` (no final sidecar, or windows left). */
pending: number;
/** mtime of the oldest pending file, or null with none pending. */
oldestPendingMtimeMs: number | null;
/** Pending files older than retention: kept only by the un-ingested rule, deleted at the ceiling. */
pastRetention: number;
/** #9279: pending files past retention held by a deterministic failure: kept past the ceiling (corpusFailureHeld). */
held: number;
}
/** Count corpus turn files still waiting for extraction. Missing dir = empty backlog; never throws. */
export function corpusBacklog(dir: string, retentionMs: number, now = Date.now()): CorpusBacklog {
const backlog: CorpusBacklog = { pending: 0, oldestPendingMtimeMs: null, pastRetention: 0, held: 0 };
let names: string[];
try {
names = readdirSync(dir);
} catch {
return backlog;
}
const present = new Set(names);
for (const name of names) {
if (!name.endsWith('.txt')) continue;
if (present.has(name + INGESTED_SUFFIX) && !corpusFileAwaitsExtraction(join(dir, name), name)) continue;
let mtimeMs: number;
try {
mtimeMs = statSync(join(dir, name)).mtimeMs;
} catch {
continue;
}
backlog.pending++;
if (backlog.oldestPendingMtimeMs === null || mtimeMs < backlog.oldestPendingMtimeMs) backlog.oldestPendingMtimeMs = mtimeMs;
if (mtimeMs < now - retentionMs) {
backlog.pastRetention++;
if (corpusFailureHeld(readCorpusFailure(join(dir, name)))) backlog.held++;
}
}
return backlog;
}
/**
* Retention GC for corpus turn files: a finished file (corpusFileCompletion
* 'done' or 'legacy') goes at `retentionMs`; one nothing has finished
* extracting (no sidecar, a non-terminal skip such as `chat_unavailable`
* #6786, windows still pending, or rewritten since) stays until
* `CORPUS_UNINGESTED_RETENTION_FACTOR` x `retentionMs` (E-N1); one held by a deterministic failure
* (corpusFailureHeld, #9279) is never extracted until a retry, so it stays. Best-effort per file; never throws.
*/
export function gcCorpusTurnFiles(dir: string, retentionMs: number, now = Date.now()): void {
try {
for (const name of readdirSync(dir)) {
if (!name.endsWith('.txt')) continue;
const p = join(dir, name);
try {
const age = now - statSync(p).mtimeMs;
if (age <= retentionMs) continue;
const finished = corpusFileFinished(corpusFileCompletion(p, name));
const ageLimit = finished ? retentionMs : retentionMs * CORPUS_UNINGESTED_RETENTION_FACTOR;
if (age > ageLimit && (finished || !corpusFailureHeld(readCorpusFailure(p)))) rmSync(p, { force: true });
} catch {
/* per-file best effort */
}
}
} catch {
/* GC never breaks the caller */
}
}
/**
* GC companion for the corpus dir (extends the hook's `.txt`-only GC): remove
* ledgers past the retention window, ORPHANED sidecars whose base `.txt` is
* gone (previously they lived forever), and a session's seat sidecar once no
* `.txt` of that session remains (after a grace window, so a sidecar written
* just ahead of its corpus file survives). Best-effort per file; never throws.
* #6611: a session is live while a `.txt` of it exists in ANY of
* `liveSessionDirs` (the hook passes the spool and the legacy top level: the
* source record is always frozen into the spool, a legacy session's `.txt`
* lives in the top level).
*/
export function gcCorpusArtifacts(
dir: string,
maxAgeMs: number,
sidecarSuffixes: string[],
liveSessionDirs: string[] = [dir],
): void {
try {
const cutoff = Date.now() - maxAgeMs;
const names = readdirSync(dir);
const liveSessions = new Set(liveSessionDirs.flatMap((d) => {
try { return d === dir ? names : readdirSync(d); } catch { return []; }
}).filter((n) => n.endsWith('.txt')).map(corpusFileSessionId));
for (const name of names) {
const p = join(dir, name);
try {
const perSession = [SEAT_SIDECAR_SUFFIX, SPEAKER_SIDECAR_SUFFIX, SESSION_SOURCE_SUFFIX].find((suffix) => name.endsWith(suffix));
if (perSession) {
if (!liveSessions.has(name.slice(0, -perSession.length)) && statSync(p).mtimeMs < Date.now() - SEAT_ORPHAN_GRACE_MS) {
rmSync(p, { force: true });
}
continue;
}
if (name.endsWith(SESSION_SOURCE_LOCK_SUFFIX)) {
if (statSync(p).mtimeMs < Date.now() - SEAT_ORPHAN_GRACE_MS) rmSync(p, { force: true });
continue;
}
const offAt = name.endsWith('.json') ? name.lastIndexOf(CAPTURE_OFF_INFIX) : -1;
if (offAt > 0) {
// Written BEFORE its corpus file lands: only an orphan past the grace period is reaped (every brain's record alike).
if (!existsSync(join(dir, name.slice(0, offAt))) && statSync(p).mtimeMs < Date.now() - SEAT_ORPHAN_GRACE_MS) rmSync(p, { force: true });
continue;
}
if (name.endsWith('.ledger.json')) {
if (statSync(p).mtimeMs < cutoff) rmSync(p, { force: true });
continue;
}
// Crashed atomic writers strand `*.tmp-<pid>` files nothing else
// reaps (adversarial review) — age them out with the ledgers.
if (/\.tmp-\d+$/.test(name)) {
if (statSync(p).mtimeMs < cutoff) rmSync(p, { force: true });
continue;
}
for (const suffix of sidecarSuffixes) {
if (!name.endsWith(suffix)) continue;
const base = p.slice(0, -suffix.length);
if (!existsSync(base)) rmSync(p, { force: true });
break;
}
} catch {
/* per-file best effort */
}
}
} catch {
/* GC never breaks the caller */
}
}