Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* render.ts — session → conversation page(s) for the transcripts-import lane
* (cathedral-4).
*
* Pipeline per session (all BEFORE any engine write; fail-closed — a throw
* here means the caller aborts the SESSION, never writes a partial page):
*
* redact (secret-scan + user patterns + imperative count)
* → render body lines (imessage-slack, REAL timestamps, anchor-escape)
* → group at message boundaries under the layout's part bytes
* → frontmatter (YAML serializer, mandatory type+date).
*
* Body format is the conversation-parser `imessage-slack` builtin — the
* REGEX IS SHARED (imported from builtins.ts), never re-declared: the line
* we emit must match it (round-trip guarantee) and any BODY line that would
* match it is escaped so hostile message content cannot forge speakers or
* timestamps on re-parse.
*
* Layouts (Wave B; the mode comes from what is stored, see ingest.ts
* `storedTranscriptLayout`):
* - single: a new session is one page, grouped under the page ceiling
* (`pages.max_bytes` or the content-sanity block line, whichever is
* lower, times PAGE_CEILING_FILL). Since Wave A a page up to that ceiling
* is stored, chunked and embedded like any other, so there is no smaller
* size a session has to be split at. Its frontmatter is `part: 1, of: 1`,
* as before.
* - split: a session imported before Wave B as parts of PART_TARGET_BYTES
* keeps them and their greedy grouping, so a grown session leaves parts
* 1..N-1 byte-identical and only the last part (or a new one) changes.
* The part count is not in hashed content: the title is
* `<title> (part X)` and `transcript_import` carries `part` and the
* `part_bytes` the parts are grouped at, never `of`. A single-layout
* session that overflows the ceiling (a message larger than the ceiling,
* or a session past it) is written the same way, pinned at that ceiling.
* Groups break at message boundaries with OVERLAP_MESSAGES carried into the
* next part (cross-boundary decision/answer pairs can still ground facts;
* extraction dedup absorbs the duplicates). No message is truncated (#6388):
* one too large for a part becomes continuation blocks that repeat its
* speaker and timestamp anchor, cut only at code-point boundaries, after
* redaction and escaping ran over the whole message; continuation blocks are
* never repeated as overlap. Part slugs: part 1 keeps the base slug; parts
* 2..N get `-pN`. frontmatter.id is UNIQUE PER PART (`<id8>-pN`) — a
* shared per-session id would make parts 2..N skip as cross-slug duplicates.
*/
import { gbrainPath } from '../config.ts';
import { dumpFrontmatterYaml } from '../data-frontmatter.ts';
import { join } from 'node:path';
import { DEFAULT_BYTES_WARN } from '../content-sanity.ts';
import { TRANSCRIPT_LANE_SCAN_OPTS, applyRedaction, planRedaction, trailingDanglingLabel, type EchoDictionary, type RedactionPlan } from '../secret-scan.ts';
import type { DanglingForm } from '../secret-scan-labeled.ts';
import { loadPatterns } from '../skillpack/harvest-lint.ts';
import { sanitizeForJsonb } from '../batch-rows.ts';
import { ensureWellFormed } from '../text-safe.ts';
import { neutralizeOutsideText } from '../outside-content.ts';
import type { ParsedSession, TranscriptMessage } from './types.ts';
import { buildTranscriptSlug, transcriptFullId, utcTimestamp } from './types.ts';
// ── Shared line format (imessage-slack builtin; escaping lives in outside-content.ts) ──
export { escapeAnchorLines, MESSAGE_ANCHOR_RE } from '../outside-content.ts';
/**
* The part size sessions were split at before Wave B (#5427 set it under the
* content-sanity WARN line), and the grouping a stored split session without
* `part_bytes` keeps. New sessions no longer split at it.
*/
export const PART_TARGET_BYTES = Math.floor(DEFAULT_BYTES_WARN * 0.9);
/**
* Most part pages one session renders to. A session over it is refused before
* any write (the session is errored and the scan is not clean): 2,000 parts is
* 90 MB at the pre-Wave-B part size, and a new session splits only past the
* page ceiling.
*/
export const TRANSCRIPT_MAX_PARTS = 2_000;
/** Share of the page ceiling a single-layout page's body may fill; the rest is frontmatter headroom. */
export const PAGE_CEILING_FILL = 0.9;
/** `single`: a new session, one page under the ceiling; `split`: a stored split session keeps its parts. */
export type TranscriptLayout = 'single' | 'split';
/** Messages repeated at each part boundary for cross-part fact grounding. */
export const OVERLAP_MESSAGES = 2;
/** Adapter-schema version stamped into transcript_import frontmatter. */
export const TRANSCRIPT_IMPORT_VERSION = 1;
// ── Redaction ────────────────────────────────────────────────────────────────
/** Default user-pattern file — the same convention skillpack harvest uses. */
export function defaultUserPatternsPath(): string {
return gbrainPath('harvest-private-patterns.txt');
}
/**
* Agent-directed imperative shapes. Detection only STAMPS A COUNT into the
* page's transcript_import frontmatter (hash-covered, idempotent) so readers
* and future triage can see the page carries instruction-shaped content —
* it never hides or rewrites the text. The write gate
* (`src/core/write-gate-patterns.ts`) reuses them as `override` patterns.
*/
export const IMPERATIVE_RES: readonly RegExp[] = [
/\b(ignore|disregard|forget)\s+(all\s+|any\s+)?(previous|prior|above|earlier)\s+(instructions|context|rules)\b/i,
/\byou\s+(must|should)\s+now\s+(act|behave|respond)\b/i,
/\bnew\s+system\s+prompt\b/i,
];
export interface RedactedSession {
session: ParsedSession;
redactionCount: number;
imperativesFlagged: number;
/** Per message: whether it matched an imperative shape (a split part counts its own messages). */
imperativeFlags?: readonly boolean[];
}
export type ImportRedactionPattern = { regex: RegExp; source: string };
/**
* Compile the import-lane redaction pattern set ONCE per run. The harvest
* defaults include a slack-channel pattern that also matches issue/PR refs
* (a token like a hash-prefixed number) — ubiquitous in coding transcripts
* and NOT private — so it is excluded; the other defaults (private names,
* emails) plus every user-file pattern stay.
*/
export function loadImportRedactionPatterns(userPatternsPath?: string): ImportRedactionPattern[] {
return loadPatterns(userPatternsPath ?? defaultUserPatternsPath()).filter(
(p) => !p.source.includes('(?:^|\\s)#'),
);
}
/**
* Secret-scan + user-pattern redaction over every text surface that will be
* persisted (message text, SPEAKER labels, title, raw-meta string fields).
* Throws on scanner or pattern failure — page writes are FAIL-CLOSED (unlike
* the hook corpus lane, these pages are searchable and synced).
*
* Detector classes (all from `secret-scan.ts`, format-based — not just vendor
* prefixes): vendor key shapes (anthropic/openai/voyage/github/gitlab/slack/
* aws/google/stripe/sendgrid/twilio/supabase/npm/huggingface/digitalocean/
* openrouter/groq/xai/perplexity/replicate/gbrain), JWTs,
* whole-block PEM private keys, `Bearer <token>` headers, connection strings
* carrying inline credentials, and — opted in for THIS lane — the
* high-entropy `KEY=`/`TOKEN=`/`PASSWORD=` assignment heuristic (value must
* carry a digit and clear the entropy gate). Transcripts are the corpus where
* a pasted `.env` line is most likely, so recall wins over the false-positive
* cost here; the push gate and compiled-context scan keep the heuristic off.
* Also opted in for this lane only: the `labeled_credential` detector
* (`secret-scan-labeled.ts`), which claims a low-entropy value typed after a
* password / login / credentials label (`password: hunter2`, `login alice /
* hunter2`); a claimed value of 8+ characters that is not a stoplisted word
* joins the echo dictionary below.
* Also transcript-lane only (`TRANSCRIPT_LANE_SCAN_OPTS.transcriptRules`,
* GST-2768): `cookie_header` claims cookie values after a `Cookie:` /
* `Set-Cookie:` header or a curl `-b` argument (names kept), a `.netrc`
* password is claimed, and a label left dangling at the end of one message
* (`password:`, `here is my API key:`) claims the value alone on the first
* line of the NEXT message, never further.
* Every match becomes `<REDACTED:pattern>`; user patterns become
* `<REDACTED:user-pattern>`.
*
* TWO PHASES, ONE SESSION-WIDE ECHO DICTIONARY. Phase 1 PLANS every persisted
* field (scan + claimed spans); the eligible bearer / high-entropy values
* every field claims accumulate in one `echoValues` map under the scanner's
* shared floors and single cap. Phase 2 APPLIES each plan, and the splice
* reads the FULL map — so a token claimed in a tool message's
* `Authorization: Bearer …` header is also scrubbed where the assistant
* echoed it bare in ANOTHER message (or the title, a speaker label, a
* metadata value), whichever order the two appear in. Redacting each field
* independently left those cross-field echoes in the page. The per-field
* `redactions` that feed `redactionCount` (the receipt) are unchanged by the
* echo pass: an echo is not a claim and adds no finding.
*/
export function redactSession(
session: ParsedSession,
opts: { userPatternsPath?: string; patterns?: ImportRedactionPattern[] } = {},
): RedactedSession {
const patterns = opts.patterns ?? loadImportRedactionPatterns(opts.userPatternsPath);
let redactionCount = 0;
let imperativesFlagged = 0;
const echoValues: EchoDictionary = new Map();
const plan = (text: string, leadingDangling: DanglingForm | null = null): RedactionPlan =>
// sanitizeForJsonb (NUL-strip + well-form): transcripts capture raw tool
// output that legitimately carries U+0000, which Postgres text/jsonb
// reject at the write boundary (#4392).
// TRANSCRIPT_LANE_SCAN_OPTS: transcripts opt into the assignment
// heuristic, the labeled-credential detector and the transcript-only
// rules (see doc comment above); the shared scanner keeps all off by default.
planRedaction(sanitizeForJsonb(text), { ...TRANSCRIPT_LANE_SCAN_OPTS, leadingDangling, echoValues });
const apply = (p: RedactionPlan): string => {
redactionCount += p.redactions.length;
let out = applyRedaction(p);
for (const { regex } of patterns) {
out = out.replace(regex, () => {
redactionCount++;
return '<REDACTED:user-pattern>';
});
}
return out;
};
// Phase 1: plan every field.
const imperativeFlags: boolean[] = [];
const planned = session.messages.map((m, i) => {
const imperative = IMPERATIVE_RES.some((re) => re.test(m.text));
imperativeFlags.push(imperative);
if (imperative) imperativesFlagged++;
// Speaker labels are persisted into the anchor line, so they get the
// same redaction as bodies (a secret or private name in a display name
// must not bypass the scan).
// Cross-message join: a label the previous message left dangling reads this message's first line as its value.
const leading = i > 0 ? trailingDanglingLabel(sanitizeForJsonb(session.messages[i - 1]!.text), TRANSCRIPT_LANE_SCAN_OPTS) : null;
return { m, text: plan(m.text, leading), speaker: m.speaker ? plan(m.speaker) : undefined };
});
const titlePlan = session.meta.title ? plan(session.meta.title) : undefined;
// Flatness is ENFORCED, not assumed: strings are planned; primitive
// scalars pass; anything nested (arrays/objects an adapter let through
// from hostile export data) is DROPPED — it would reach putRawData
// unscanned otherwise.
const rawEntries: Array<{ key: string; plan: RedactionPlan | null; scalar: unknown }> = [];
if (session.meta.raw) {
for (const [key, v] of Object.entries(session.meta.raw)) {
if (typeof v === 'string') rawEntries.push({ key, plan: plan(v), scalar: undefined });
else if (v === null || typeof v === 'number' || typeof v === 'boolean') {
rawEntries.push({ key, plan: null, scalar: v });
}
}
}
// Phase 2: apply with the complete dictionary.
const messages = planned.map(({ m, text, speaker }) => ({
...m,
text: apply(text),
...(speaker ? { speaker: apply(speaker) } : {}),
}));
const meta = { ...session.meta };
if (titlePlan) meta.title = apply(titlePlan);
if (meta.raw) {
const raw: Record<string, unknown> = {};
for (const e of rawEntries) raw[e.key] = e.plan ? apply(e.plan) : e.scalar;
meta.raw = raw;
}
return { session: { meta, messages }, redactionCount, imperativesFlagged, imperativeFlags };
}
// ── Rendering ────────────────────────────────────────────────────────────────
/** `2026-08-01T10:00:05.000Z` → `(2026-08-01 10:00 AM)` (UTC), matching the builtin. */
function anchorTimestamp(iso: string): string {
const d = new Date(iso);
const day = Number.isNaN(d.getTime()) ? iso.slice(0, 10) : d.toISOString().slice(0, 10);
let h = d.getUTCHours();
const ampm = h >= 12 ? 'PM' : 'AM';
h = h % 12 || 12;
const mm = String(d.getUTCMinutes()).padStart(2, '0');
return `${day} ${h}:${mm} ${ampm}`;
}
export interface RenderedPart {
slug: string;
/** Full page content: YAML frontmatter + body. */
content: string;
/** UNIQUE per part — the import-dedup identity. */
frontmatterId: string;
part: number;
/** The keys this collector renders (and so owns on re-ingest), and the body. */
frontmatter: Record<string, unknown>;
body: string;
}
export function renderPartContent(frontmatter: Record<string, unknown>, body: string): string {
return `---\n${dumpFrontmatterYaml(frontmatter, { lineWidth: 1000 })}---\n\n${body}\n`;
}
export interface RenderSessionResult {
parts: RenderedPart[];
/** Base slug (part 1's slug) — putRawData and reconciliation key off it. */
baseSlug: string;
dateIso: string;
}
/**
* Speaker label for the anchor line. Anchor-forming characters are stripped
* (never escaped — the label sits INSIDE the anchor, so a speaker containing
* `**` or `(date):` shapes could otherwise forge message boundaries on
* round-trip; hostile BODY lines are handled by neutralizeOutsideText).
*/
function speakerLabel(m: TranscriptMessage): string {
const raw = m.speaker?.trim();
if (!raw) return m.role === 'user' ? 'User' : 'Assistant';
const cleaned = ensureWellFormed(raw).replace(/\*/g, '').replace(/[()\n:]/g, ' ').trim();
return cleaned || (m.role === 'user' ? 'User' : 'Assistant');
}
const utf8Bytes = (cp: number): number => (cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4);
/**
* One escaped message → 1..N blocks of at most `maxBytes` UTF-8 bytes, each
* starting with `anchor` (#6388: no message text is dropped). Fragments break
* at line breaks when a line fits, and inside an over-long line only at a
* code-point boundary. A cut piece of a line is always a fragment's FIRST
* line, which sits on the generated anchor line, so every body line is a
* whole line that neutralizeOutsideText already saw.
*/
function messageBlocks(anchor: string, text: string, maxBytes: number): string[] {
const whole = `${anchor}${text}`;
if (Buffer.byteLength(whole, 'utf8') <= maxBytes) return [whole];
const room = maxBytes - Buffer.byteLength(anchor, 'utf8');
if (room < 4) {
throw new Error(`renderSessionParts: partTargetBytes ${maxBytes + 2} leaves no room for message text after its ${anchor.length}-character anchor`);
}
const fragments: string[] = [];
let current: string | null = null;
let currentBytes = 0;
for (const line of text.split('\n')) {
const lineBytes = Buffer.byteLength(line, 'utf8');
if (current !== null && currentBytes + 1 + lineBytes <= room) {
current += `\n${line}`;
currentBytes += 1 + lineBytes;
continue;
}
if (current !== null) fragments.push(current);
let start = 0;
let bytes = 0;
for (let i = 0; i < line.length;) {
const cp = line.codePointAt(i)!;
const b = utf8Bytes(cp);
if (bytes + b > room) {
fragments.push(line.slice(start, i));
start = i;
bytes = 0;
}
bytes += b;
i += cp > 0xffff ? 2 : 1;
}
current = line.slice(start);
currentBytes = bytes;
}
if (current !== null) fragments.push(current);
return fragments.map((f) => `${anchor}${f}`);
}
/**
* The dedup identity of one part (`frontmatter.id`): a HARNESS-NAMESPACED
* 64-bit hash of the session id plus `-pN`. importFromContent skips any
* cross-slug frontmatter-id match as a duplicate, so it must be
* collision-proof across harnesses, days, and fallback session ids.
*/
export function transcriptPartId(meta: Pick<ParsedSession['meta'], 'harness' | 'sessionId'>, part: number): string {
return `${meta.harness}-${transcriptFullId(meta.sessionId)}-p${part}`;
}
/**
* Render one redacted session into 1..N part pages. `layout` comes from what
* is stored (default `single`); `partTargetBytes` is the page ceiling for a
* single-layout session and the stored part size for a split one. Timestamps:
* each message uses its own REAL timestamp; a message missing one carries the
* previous message's timestamp forward (carried, never fabricated — documented
* in the page header note); a session with NO timestamps at all is
* unrenderable and throws (the adapter contract requires real times).
*/
export function renderSessionParts(
redacted: RedactedSession,
opts: { sourcePath: string; partTargetBytes?: number; layout?: TranscriptLayout } = { sourcePath: '' },
): RenderSessionResult {
const partTargetBytes = opts.partTargetBytes ?? PART_TARGET_BYTES;
if (!Number.isSafeInteger(partTargetBytes) || partTargetBytes <= 0) {
throw new Error(`renderSessionParts: partTargetBytes must be a positive integer (got ${partTargetBytes})`);
}
const { session, imperativesFlagged } = redacted;
const { meta, messages } = session;
if (!messages.length) throw new Error('renderSessionParts: session has no messages');
const firstTs = meta.startedAt || messages.find((m) => m.timestamp)?.timestamp;
if (!firstTs) {
throw new Error(
`session ${meta.sessionId} carries no timestamps — refusing to fabricate provenance`,
);
}
const dateIso = utcTimestamp(firstTs);
const baseSlug = buildTranscriptSlug(meta.harness, dateIso, {
sessionId: meta.sessionId,
title: meta.title,
});
// One or more rendered blocks per message: the whole message is escaped,
// then split into continuation blocks (same anchor) when it cannot fit a part.
let lastTs = firstTs;
const blocks: Array<{ text: string; bytes: number; continued: boolean; message: number }> = messages.flatMap((m, message) => {
const ts = m.timestamp || lastTs;
lastTs = ts;
const text = neutralizeOutsideText(m.text);
const fragments = messageBlocks(`**${speakerLabel(m)}** (${anchorTimestamp(ts)}): `, text, partTargetBytes - 2);
return fragments.map((f) => ({ text: f, bytes: Buffer.byteLength(f, 'utf8') + 2, continued: fragments.length > 1, message }));
});
// Split at block boundaries under the part target. Overlap repeats up to
// OVERLAP_MESSAGES whole messages, never a continuation block, and only
// as many as still fit beside the next block.
const groups: Array<typeof blocks> = [];
let current: typeof blocks = [];
let currentBytes = 0;
for (const b of blocks) {
if (current.length > 0 && currentBytes + b.bytes > partTargetBytes) {
groups.push(current);
current = current.slice(-OVERLAP_MESSAGES).filter((c) => !c.continued);
currentBytes = current.reduce((n, c) => n + c.bytes, 0);
while (current.length > 0 && currentBytes + b.bytes > partTargetBytes) currentBytes -= current.shift()!.bytes;
}
current.push(b);
currentBytes += b.bytes;
}
if (current.length) groups.push(current);
if (groups.length > TRANSCRIPT_MAX_PARTS) {
throw new Error(`session ${meta.sessionId} renders ${groups.length} parts at ${partTargetBytes} bytes per part, over the ${TRANSCRIPT_MAX_PARTS}-part cap; nothing was written for it`);
}
const single = (opts.layout ?? 'single') === 'single' && groups.length === 1;
const title = meta.title?.trim() || `${meta.harness} session ${meta.sessionId.slice(0, 12)}`;
const parts: RenderedPart[] = groups.map((group, idx) => {
const part = idx + 1;
const slug = part === 1 ? baseSlug : `${baseSlug}-p${part}`;
const frontmatterId = transcriptPartId(meta, part);
const imperatives = single || !redacted.imperativeFlags
? imperativesFlagged
: new Set(group.filter((b) => redacted.imperativeFlags![b.message]).map((b) => b.message)).size;
const fm: Record<string, unknown> = {
type: 'conversation',
title: single ? title : `${title} (part ${part})`,
date: dateIso.slice(0, 10),
id: frontmatterId,
transcript_import: {
harness: meta.harness,
session_id: meta.sessionId,
version: TRANSCRIPT_IMPORT_VERSION,
part,
...(single ? { of: 1 } : { part_bytes: partTargetBytes }),
...(imperatives > 0 ? { imperatives_flagged: imperatives } : {}),
},
};
const body = group.map((b) => b.text).join('\n\n');
return { slug, content: renderPartContent(fm, body), frontmatterId, part, frontmatter: fm, body };
});
return { parts, baseSlug, dateIso };
}