Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* ingest.ts — the transcripts-import core (cathedral-4).
*
* Engine-facing, CLI-free: `gbrain transcripts ingest` parses flags and
* calls runTranscriptsIngest; e2e tests call it directly. Pipeline per
* session (ATOMICITY = SESSION, never file — a multi-session file commits
* the sessions that pass and skips the ones that fail; idempotent re-runs
* complete the rest):
*
* detect → adapter.parse (AsyncGenerator, per-session) → since/limit
* filters → redactSession (fail-closed) → renderSessionParts →
* importFromContent per part (embed OFF unless opted in) →
* putRawData(baseSlug) → stale-part reconciliation (delete parts past the
* rendered count).
*
* Layout (Wave B): the stored pages decide how a session renders
* (`storedTranscriptLayout`). A session with no split pages is one page
* under the page ceiling; a session already split keeps its parts and their
* grouping. Nothing merges a split session implicitly.
*
* Error taxonomy:
* - per-FILE: unreadable / unknown format / symlink → counted, run continues.
* - per-SESSION: every failure `classifyItemFailure` calls the item's
* (a coordinator refusal, an oversize part, an encoding error) is
* recorded on the session with its typed code; the file continues.
* - PENDING: a part whose write the coordinator accepted but has not
* published within the wait is a pending session: never an error, never
* imported, and it keeps the scan unclean. Its request id is derived from
* the source, slug, revision and content, so a rerun waits on the same
* write instead of admitting a duplicate; a write that ended without
* committing is retried under the next attempt's id.
* - RUN-LEVEL: run-scoped failures (connection lost, a failed transaction,
* the writer unavailable, an abort) stop the run unchanged, and so does
* the ItemRun breaker (`item_failures_systemic`) when most sessions fail.
*
* Watermark: the RESULT carries `cleanScan` (no errors anywhere, no limit
* truncation) + `maxSessionTs`; the COMMAND advances the `--since last`
* checkpoint only on a clean scan — a truncated or partially-failed run
* must never skip work permanently.
*/
import type { BrainEngine } from '../engine.ts';
import { randomUUID } from 'node:crypto';
import { importFromContent } from '../import-file.ts';
import type { OperationContext } from '../ops/contract.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import { initializeLocalPersistence, pendingAwareResponse, requestPrincipalForContext, submitPageMutation } from '../persistence/page-mutations.ts';
import { waitForWrite } from '../persistence/service.ts';
import { isTerminal, type WriteRequest } from '../persistence/model.ts';
import { clientRequestUuid } from '../persistence/preconditions.ts';
import { digest } from '../persistence/digest.ts';
import { classifyItemFailure, ItemRun } from '../item-failure.ts';
import { unchangedPutPage } from '../persistence/put-page-screen.ts';
import { canonicalJson } from '../remediation-step.ts';
import type { FileDiagnostics, ParsedSession, TranscriptAdapter, TranscriptFormat } from './types.ts';
import { detectAdapter, type DetectResult } from './detect.ts';
import {
loadImportRedactionPatterns,
PAGE_CEILING_FILL,
PART_TARGET_BYTES,
redactSession,
renderPartContent,
renderSessionParts,
transcriptPartId,
type RedactedSession,
type RenderedPart,
type RenderSessionResult,
type TranscriptLayout,
} from './render.ts';
import { readPageBodyPolicy } from '../page-body/limits.ts';
import { PAGE_MAX_BYTES } from '../page-size-policy.ts';
import { effectiveBytesBlock } from '../import-screen.ts';
import { RECONCILE_SAFETY_KEYS } from '../persistence/reconcile-safety.ts';
import { ATOMS_SCAN_HASH_KEY } from '../utils.ts';
export interface IngestActivePack {
page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string> }>;
}
export interface TranscriptsIngestOpts {
/** Files to import (post-glob, pre-detection). */
paths: string[];
/** Explicit format wins over detection. */
format?: TranscriptFormat;
/** Per-file format harness discovery already established; used when no explicit format is set. */
discoveredFormats?: ReadonlyMap<string, TranscriptFormat>;
/** Parse + redact + render + report; ZERO engine writes. */
dryRun?: boolean;
/** Max sessions imported this run (session granularity; truncation ⇒ not a clean scan). */
limit?: number;
/** Only sessions whose LAST message is strictly newer than this ISO. */
sinceIso?: string;
/** Resolved source id — threads through import, raw-data, reconciliation. */
sourceId: string;
/** Embedding opt-in (default OFF: bulk imports defer to the embed backfill). */
embed?: boolean;
/**
* gbrain#4149: explicit byte-cap OVERRIDE threaded to every adapter's
* parse. Undefined = each adapter keeps its own format-specific default
* (Hermes 512MB store guard, 50MB jsonl cap, ...) — the override exists
* for legitimate oversized stores, not to replace the defaults.
*/
maxBytes?: number;
activePack?: IngestActivePack;
/**
* The CLI's trusted-local operation context. On a writer-claimed brain each
* rendered part is submitted as a `put_page` write request through it; the
* direct importer is refused there. Without it the direct importer runs.
*/
context?: OperationContext;
/** Test seam for the redaction user-pattern file. */
userPatternsPath?: string;
/** Adapter registry override (tests). */
adapters?: TranscriptAdapter[];
/** Called once per processed file (progress ticks). */
onFileDone?: (done: number, total: number, path: string) => void;
/**
* Called once per SESSION — the liveness signal for multi-session stores
* (one hermes state.db can hold thousands of sessions between file ticks).
*/
onSession?: (sessionId: string) => void;
}
export interface IngestSessionOutcome {
sessionId: string;
harness: TranscriptFormat;
baseSlug: string;
parts: number;
/** Per-part import statuses (dry-run: 'planned'; 'pending': accepted, not yet published). */
statuses: Array<'imported' | 'skipped' | 'error' | 'planned' | 'pending'>;
redactions: number;
imperatives: number;
error?: string;
/** The typed code of `error` (classifyItemFailure). */
code?: string;
/** Request ids of accepted writes still publishing: the session lands on a later run without a new admission. */
pending?: string[];
/** C-19: a session with no timestamps is skipped (never fabricated), not errored. */
skipped?: 'no_timestamp';
}
export interface IngestFileOutcome {
path: string;
format?: TranscriptFormat;
sessions: IngestSessionOutcome[];
skippedLines: number;
drift: boolean;
/** E-N4: assistant turns parsed with zero user turns (counted in driftFiles). */
userTurnsMissing?: boolean;
/** #6741: a Codex sub-agent rollout with no user turn by design (counted in subagentRollouts, not drift). */
subagentRollout?: boolean;
/** Adapter degraded to a bounded read (e.g. codex head+tail) — part of the file was never scanned. */
truncated: boolean;
error?: string;
/** The filesystem error code when the file could not be read (ENOENT, EACCES, ...). */
errno?: string;
}
/** The per-file refusal: what was refused and the next step. Nothing of a refused file is imported; other files in the run are unaffected. */
function detectRefusal(detected: Extract<DetectResult, { ok: false }>): string {
if (detected.reason === 'unknown_format') return `unknown format (tried: ${detected.tried.join(', ')}); pass an explicit format flag`;
if (detected.reason === 'symlink') return 'symlink: transcript import never follows a symlink; pass the real file path instead. This file was not imported.';
const errno = detected.errno ?? 'unknown error';
const step = errno === 'ENOENT' ? 'the path does not exist; check it and retry'
: errno === 'EACCES' || errno === 'EPERM' ? 'ask the file owner to grant read access, then retry this file with the same format'
: 'retry this file once the filesystem error clears';
return `unreadable (${errno}): ${step}. This file was not imported.`;
}
export interface TranscriptsIngestResult {
files: IngestFileOutcome[];
pages: { imported: number; skipped: number; errored: number; planned: number; pending: number };
sessionsSeen: number;
sessionsImported: number;
sessionsFiltered: number;
sessionsErrored: number;
/** Sessions with an accepted write still publishing (not imported, not errored; the scan is not clean). */
sessionsPending: number;
/** Sessions with no timestamps at all: reported, not imported, not an error (C-19). */
sessionsSkippedNoTimestamp: number;
redactions: number;
imperatives: number;
partsDeleted: number;
driftFiles: number;
/** #6741: Codex sub-agent rollouts with no user turns by design; understood, so they keep the scan clean. */
subagentRollouts: number;
/** Files whose adapter reported a truncated (partially-unscanned) read. */
truncatedFiles: number;
erroredFiles: number;
/** EVERY slug the run touched — imported AND hash-skipped (--facts targets all). */
slugsTouched: string[];
/** True ⇔ no file/session errors and no limit truncation: watermark may advance. */
cleanScan: boolean;
/** Newest session last-message ISO seen (imported or filtered). */
maxSessionTs: string;
}
/**
* Session's last message timestamp, NORMALIZED to Z-form ISO ('' when none
* carry one). Normalization matters because since/watermark comparisons are
* lexicographic: an offset-form ISO (+07:00) string-sorts after a real-time
* newer Z-form and would poison the watermark. UNPARSEABLE timestamps are
* SKIPPED, never passed through — a single hostile/corrupt value like a
* letter-leading string would otherwise become the watermark and since-filter
* every real session forever.
*/
function lastMessageTs(messages: Array<{ timestamp: string }>): string {
for (let i = messages.length - 1; i >= 0; i--) {
const raw = messages[i].timestamp;
if (!raw) continue;
const d = new Date(raw);
if (Number.isNaN(d.getTime())) continue;
return d.toISOString();
}
return '';
}
export async function runTranscriptsIngest(
engine: BrainEngine,
opts: TranscriptsIngestOpts,
): Promise<TranscriptsIngestResult> {
const result: TranscriptsIngestResult = {
files: [],
pages: { imported: 0, skipped: 0, errored: 0, planned: 0, pending: 0 },
sessionsSeen: 0,
sessionsImported: 0,
sessionsFiltered: 0,
sessionsErrored: 0,
sessionsPending: 0,
sessionsSkippedNoTimestamp: 0,
redactions: 0,
imperatives: 0,
partsDeleted: 0,
driftFiles: 0,
subagentRollouts: 0,
truncatedFiles: 0,
erroredFiles: 0,
slugsTouched: [],
cleanScan: true,
maxSessionTs: '',
};
let limitTruncated = false;
// Redaction patterns compile ONCE per run — loadPatterns re-reads and
// recompiles the pattern file on every call, which a bulk import would
// otherwise repeat thousands of times.
const redactionPatterns = loadImportRedactionPatterns(opts.userPatternsPath);
const coordinator = !opts.dryRun && opts.context && await managedPersistenceEnabled(engine) ? opts.context : null;
const items = new ItemRun('transcripts.ingest_session');
// Read with the first session that renders. A dry run reads nothing from the brain: it plans every session as a
// new one-page session at the default ceiling.
let pageCeiling: number | undefined;
const total = opts.paths.length;
let done = 0;
let newWorkSessions = 0;
for (const path of opts.paths) {
if (limitTruncated) break;
const fileOutcome: IngestFileOutcome = {
path,
sessions: [],
skippedLines: 0,
drift: false,
truncated: false,
};
result.files.push(fileOutcome);
const detected = detectAdapter(path, {
explicitFormat: opts.format ?? opts.discoveredFormats?.get(path),
adapters: opts.adapters,
});
if (!detected.ok) {
fileOutcome.error = detectRefusal(detected);
if (detected.errno) fileOutcome.errno = detected.errno;
result.erroredFiles++;
result.cleanScan = false;
done++;
opts.onFileDone?.(done, total, path);
continue;
}
fileOutcome.format = detected.adapter.format;
// gbrain#4149: thread the explicit cap override; omit the opts object
// entirely when unset so adapters keep their native defaults.
const gen = opts.maxBytes != null
? detected.adapter.parse(path, { maxBytes: opts.maxBytes })
: detected.adapter.parse(path);
try {
let step = await gen.next();
while (!step.done) {
if (limitTruncated) {
// Stop consuming; the generator's finally blocks clean up.
await gen.return?.(undefined as never);
break;
}
const session = step.value;
result.sessionsSeen++;
opts.onSession?.(session.meta.sessionId);
const lastTs = lastMessageTs(session.messages);
if (lastTs && lastTs > result.maxSessionTs) result.maxSessionTs = lastTs;
if (opts.sinceIso && lastTs && lastTs <= opts.sinceIso) {
result.sessionsFiltered++;
step = await gen.next();
continue;
}
// The limit counts NEW WORK only (sessions with a non-skipped part).
// Counting hash-skipped re-scans would make batched backfill loop
// over the same already-imported prefix forever: every run would
// burn the limit on free re-scans and truncate before new sessions.
if (opts.limit !== undefined && newWorkSessions >= opts.limit) {
limitTruncated = true;
result.cleanScan = false;
await gen.return?.(undefined as never);
break;
}
const outcome: IngestSessionOutcome = {
sessionId: session.meta.sessionId,
harness: session.meta.harness,
baseSlug: '',
parts: 0,
statuses: [],
redactions: 0,
imperatives: 0,
};
fileOutcome.sessions.push(outcome);
// C-19: a session with no timestamps can never render (provenance is
// never fabricated), so retrying it is pointless; report the skip and
// keep the scan clean instead of freezing the --since checkpoint.
if (!session.meta.startedAt && !session.messages.some(m => m.timestamp)) {
outcome.skipped = 'no_timestamp';
result.sessionsSkippedNoTimestamp++;
step = await gen.next();
continue;
}
try {
const redacted = redactSession(session, {
userPatternsPath: opts.userPatternsPath,
patterns: redactionPatterns,
});
outcome.redactions = redacted.redactionCount;
outcome.imperatives = redacted.imperativesFlagged;
pageCeiling ??= opts.dryRun ? Math.floor(PAGE_MAX_BYTES * PAGE_CEILING_FILL) : await transcriptPageCeiling(engine);
const stored = opts.dryRun
? { layout: 'single' as const, partTargetBytes: pageCeiling, existingSlug: null }
: await storedTranscriptLayout(engine, opts.sourceId ?? 'default', session.meta, pageCeiling);
const rendered = renderSessionParts(redacted, { sourcePath: path, layout: stored.layout, partTargetBytes: stored.partTargetBytes });
if (stored.existingSlug) adoptExistingBaseSlug(rendered, stored.existingSlug);
outcome.baseSlug = rendered.baseSlug;
outcome.parts = rendered.parts.length;
if (opts.dryRun) {
outcome.statuses = rendered.parts.map(() => 'planned' as const);
result.pages.planned += rendered.parts.length;
// #4762: a planned session is new work too, so the --limit gate
// above truncates the preview like the real run. A dry run cannot
// see hash-skips or a stored split layout (no engine reads), so `--dry-run --limit N` shows
// the first N eligible sessions — an upper bound on what the write
// path would import.
newWorkSessions++;
} else if (await importSession(engine, opts, { coordinator, items, path, session, redacted, rendered, outcome, result })) {
newWorkSessions++;
}
if (outcome.pending) {
result.sessionsPending++;
result.cleanScan = false;
} else {
items.succeed();
result.sessionsImported++;
result.redactions += outcome.redactions;
result.imperatives += outcome.imperatives;
}
} catch (err) {
// Run-scoped failures and a tripped breaker leave through here unchanged.
const failure = items.fail(session.meta.sessionId, err);
if (failure.scope === 'pending') {
(outcome.pending ??= []).push(failure.requestId!);
result.sessionsPending++;
} else {
outcome.error = failure.message;
outcome.code = failure.code;
result.sessionsErrored++;
}
result.cleanScan = false;
}
step = await gen.next();
}
if (step.done && step.value) applyFileDiagnostics(step.value, fileOutcome, result);
} catch (err) {
if (classifyItemFailure(err).scope === 'run') throw err;
fileOutcome.error = err instanceof Error ? err.message : String(err);
result.erroredFiles++;
result.cleanScan = false;
}
done++;
opts.onFileDone?.(done, total, path);
}
if (opts.dryRun) result.cleanScan = false; // dry-runs never advance watermarks
return result;
}
/**
* Writes one rendered session: every part (through the coordinator or the
* direct importer), then the session's raw metadata and its stale
* higher-numbered parts. A part the coordinator accepted but has not
* published leaves the session pending while its other parts still go; the
* metadata and the reconciliation wait for a run where every part landed.
* Returns whether the session was new work (a part that was not skipped).
*/
async function importSession(engine: BrainEngine, opts: TranscriptsIngestOpts, run: {
coordinator: OperationContext | null; items: ItemRun; path: string; session: ParsedSession; redacted: RedactedSession;
rendered: RenderSessionResult; outcome: IngestSessionOutcome; result: TranscriptsIngestResult;
}): Promise<boolean> {
const { coordinator, items, path, session, redacted, rendered, outcome, result } = run;
// The RESOLVED base slug: identity dedup can resolve part 1 to an
// EXISTING page under a different slug (same session id, changed
// title or corrected start date) — raw-data writes and stale-part
// reconciliation must follow the page that actually exists, or
// every re-run aborts on a nonexistent slug. The caller adopted the
// stored slug (storedTranscriptLayout read it).
let resolvedBaseSlug = rendered.baseSlug;
for (const part of rendered.parts) {
let r: { slug: string; status: 'imported' | 'skipped' | 'error' };
try {
await preserveForeignFrontmatter(engine, opts.sourceId ?? 'default', part);
const provenance = { source_kind: `transcript:${session.meta.harness}`, source_uri: path, ingested_via: 'cli:transcripts-ingest' };
r = coordinator
? await submitTranscriptPart(coordinator, engine, opts.sourceId, part, provenance)
: await importFromContent(engine, part.slug, part.content, {
noEmbed: !opts.embed,
sourceId: opts.sourceId,
activePack: opts.activePack,
...provenance,
});
} catch (err) {
if (classifyItemFailure(err).scope !== 'pending') throw err;
(outcome.pending ??= []).push(items.fail(`${session.meta.sessionId}#p${part.part}`, err).requestId!);
outcome.statuses.push('pending');
result.pages.pending++;
continue;
}
outcome.statuses.push(r.status);
if (r.status === 'imported') result.pages.imported++;
else if (r.status === 'skipped') result.pages.skipped++;
else result.pages.errored++;
const actualSlug = r.slug || part.slug;
if (part.part === 1 && actualSlug) resolvedBaseSlug = actualSlug;
result.slugsTouched.push(actualSlug);
}
if (outcome.pending) return true;
// importFromContent RETURNS status 'error' (it does not throw)
// for e.g. frontmatter-parse failures. A page that never landed
// is a session error and must freeze the watermark — otherwise
// a since-last run permanently skips content that never imported.
if (outcome.statuses.includes('error')) {
throw new Error(
`page import returned error status for session ${session.meta.sessionId}`,
);
}
const allSkipped =
outcome.statuses.length > 0 && outcome.statuses.every((s) => s === 'skipped');
// Session metadata rides the base page's raw_data — the REDACTED
// copy, never the original (secrets in titles/cwd would otherwise
// bypass the page-body redaction). On all-skipped re-runs the
// write is HEALED, not assumed: a prior run can have committed
// the pages and then died before putRawData, and hash-skips
// would otherwise make that hole permanent.
if (redacted.session.meta.raw) {
const rawSource = `transcript:${session.meta.harness}`;
// Skipped re-runs COMPARE, never assume: existence alone is
// not freshness — a private pattern added AFTER the first
// import must refresh the stored copy, and a prior run can
// have died before this write. Content-equal rows skip the
// write so healthy re-runs stay write-free.
let needsRaw = true;
if (allSkipped) {
// Active rows only: `allSkipped` means the import hash check
// (which reads ACTIVE rows) just matched every page, so the
// base page is alive here by construction — a tombstoned
// page never reaches this branch (it reads as missing and is
// re-imported, see the "resurrects the page" e2e). No
// includeDeleted flag: the probe must never read through a
// soft-delete the hash check did not.
const existing = await engine.getRawData(resolvedBaseSlug, rawSource, {
sourceId: opts.sourceId,
});
// Key-order-insensitive compare: JSONB hands keys back in
// its own canonical order, so a plain JSON.stringify never
// matched the freshly built object and every healthy re-run
// rewrote the row.
needsRaw =
existing.length === 0 ||
canonicalJson(existing[0].data) !==
canonicalJson(JSON.parse(JSON.stringify(redacted.session.meta.raw)));
}
if (needsRaw) {
await engine.putRawData(resolvedBaseSlug, rawSource, redacted.session.meta.raw, {
sourceId: opts.sourceId,
});
}
}
// Stale-part reconciliation: a session whose content shrank leaves
// higher-numbered part pages behind — delete them, or a stale part
// stays searchable forever. It never merges a split session: the
// layout came from the stored pages, so a split session renders as
// parts at its stored part size and growth never lowers their count. ENUMERATED via one SQL
// query (never a sequential probe: a crash mid-delete leaves
// holes that a first-miss or bounded-miss probe walks past) and
// run on EVERY pass including all-skipped re-runs, because a
// prior run can have died between the page writes and this step.
const partRows = await engine.executeRaw<{ slug: string }>(
`SELECT slug FROM pages
WHERE source_id = $1 AND deleted_at IS NULL AND slug LIKE $2`,
[opts.sourceId, `${resolvedBaseSlug}-p%`],
);
for (const row of partRows) {
const suffix = row.slug.slice(resolvedBaseSlug.length);
const m = /^-p(\d+)$/.exec(suffix);
const num = m ? Number(m[1]) : NaN;
if (Number.isFinite(num) && num > rendered.parts.length) {
// #5235: on a managed brain the stale part goes through the coordinator like the parts
// themselves, bound to its revision; the direct delete is refused there.
if (coordinator) await deleteStaleTranscriptPart(coordinator, engine, opts.sourceId, row.slug);
else await engine.deletePages([row.slug], { sourceId: opts.sourceId });
result.partsDeleted++;
}
}
return !allSkipped;
}
/**
* A writer-claimed brain refuses the direct importer, so a rendered part is
* submitted as a `put_page` write request: the same importer runs behind the
* coordinator, naming the current revision so a replacement is never a blind
* overwrite (no revision means create-only). Embedding is deferred there, as
* for every coordinated page write. #6741 B: a part whose publication would
* change nothing takes no admission (put-page-screen.ts), so an unchanged
* re-ingest burns no permanent request id. A part whose earlier write is
* still publishing waits on that write (transcriptPartRequest) instead of
* admitting a second one.
*/
export async function submitTranscriptPart(
ctx: OperationContext,
engine: BrainEngine,
sourceId: string,
part: RenderedPart,
provenance: { source_kind: string; source_uri: string; ingested_via: string },
): Promise<{ slug: string; status: 'imported' | 'skipped' | 'error' }> {
const snapshot = await engine.readPageSnapshot(part.slug, { sourceId, includeDeleted: true });
const params = { slug: part.slug, content: part.content, source_id: sourceId, ...provenance,
...(snapshot?.revision ? { expected_revision: snapshot.revision } : {}) };
if (await unchangedPutPage(ctx, params, snapshot)) return { slug: part.slug, status: 'skipped' };
const { requestId, prior } = await transcriptPartRequest(ctx, sourceId, part.slug, params);
const receipt = prior
? pendingAwareResponse(ctx, await waitForWrite(ctx.engine, prior, ctx.config, ctx.writeWaitMs))
: await submitPageMutation(ctx, { operation: 'put_page', params: { ...params, request_id: requestId } });
const slug = typeof receipt.slug === 'string' && receipt.slug ? receipt.slug : part.slug;
if (receipt.state !== 'committed') return { slug, status: 'error' };
return { slug, status: receipt.noop === true || receipt.status === 'skipped' ? 'skipped' : 'imported' };
}
/** Attempts read per journal lookup while finding a part's current request. */
const PART_ATTEMPT_PAGE = 8;
/**
* One part's write keeps one request identity across runs. The id is derived
* from the source, slug, expected revision and content, so a rerun while that
* write is still queued or publishing gets it back (`prior`) and waits on it;
* a committed one is replayed only while the live page is still the one it
* produced (same source incarnation, same revision). A committed write whose
* page has since been deleted and purged, or rewritten, is passed over like a
* write that ended without committing (conflict, failed, cancelled): the part
* is submitted under the next attempt's id, so a cleared cause or an absent
* page gets a fresh, normally admitted write instead of an old receipt.
*/
async function transcriptPartRequest(ctx: OperationContext, sourceId: string, slug: string, params: Record<string, unknown>):
Promise<{ requestId: string; prior: WriteRequest | null }> {
await initializeLocalPersistence(ctx);
const principal = await requestPrincipalForContext(ctx);
const base = `transcript-${digest({ source_id: sourceId, slug, expected_revision: params.expected_revision ?? null, content: params.content }).slice(0, 40)}`;
for (let first = 0; ; first += PART_ATTEMPT_PAGE) {
const names = Array.from({ length: PART_ATTEMPT_PAGE }, (_, i) => `${base}-${first + i}`);
const rows = await ctx.engine.executeRaw<WriteRequest>(
'SELECT * FROM persistence_requests WHERE principal_kind=$1 AND principal_id=$2 AND request_id=ANY($3::uuid[])',
[principal.kind, principal.id, names.map(clientRequestUuid)]);
const byId = new Map(rows.map(row => [String(row.request_id).toLowerCase(), row]));
for (const requestId of names) {
const prior = byId.get(clientRequestUuid(requestId)) ?? null;
if (!prior || !isTerminal(prior)) return { requestId, prior };
if (prior.state !== 'committed') continue;
const live = await ctx.engine.readPageSnapshot(slug, { sourceId });
if (live?.sourceIncarnation === prior.source_incarnation && live.revision === prior.outcome?.revision) return { requestId, prior };
}
}
}
/**
* A stale higher-numbered part on a managed brain: a revision-bound `delete_page` write request (soft delete, recoverable).
* #6741 C: an internal reconciliation, so it is admitted as background work: against a worktree in recovery it is
* refused before any receipt (admission-origin.ts) and the session errors, instead of each run taking a new request id.
* That refusal (owner_unavailable) pauses background work only, so it is this session's error, never the run's:
* foreground parts of every other session are still admitted.
*/
export async function deleteStaleTranscriptPart(ctx: OperationContext, engine: BrainEngine, sourceId: string, slug: string): Promise<void> {
const snapshot = await engine.readPageSnapshot(slug, { sourceId });
if (!snapshot) return;
const receipt = await submitPageMutation({ ...ctx, admissionOrigin: 'background' }, { operation: 'delete_page', params: { slug, source_id: sourceId, expected_revision: snapshot.revision, request_id: randomUUID() } })
.catch((err: unknown) => {
if (classifyItemFailure(err).code !== 'owner_unavailable') throw err;
throw new Error(`stale part ${slug} was not deleted: ${err instanceof Error ? err.message : String(err)}`);
});
if (receipt.state !== 'committed') {
throw new Error(`stale part ${slug} was not deleted: the coordinator returned ${String(receipt.state)}${receipt.error_code ? ` (${String(receipt.error_code)})` : ''}`);
}
}
/**
* The slug embeds the title (and start day) for export formats, so a rename
* renders a new slug for the same session. Import at the slug the session's
* pages already live at: the content is then an in-place update. At a new
* slug, the importer's cross-slug identity dedup skipped it as a duplicate
* and every message added since the last import was silently dropped.
*/
/**
* Re-ingest replaces only what the collector renders (#5431): keys another
* tool or the operator added to an imported conversation page (a review flag,
* say) are carried onto the re-rendered part. Gate- and phase-owned markers
* are re-derived from the new content, so they are never carried.
*/
async function preserveForeignFrontmatter(engine: BrainEngine, sourceId: string, part: RenderedPart): Promise<void> {
const existing = await engine.getPage(part.slug, { sourceId });
const foreign = Object.entries(existing?.frontmatter ?? {})
.filter(([key]) => !Object.hasOwn(part.frontmatter, key) && !RE_DERIVED_KEYS.has(key));
if (foreign.length === 0) return;
part.content = renderPartContent({ ...part.frontmatter, ...Object.fromEntries(foreign) }, part.body);
}
const RE_DERIVED_KEYS = new Set([...RECONCILE_SAFETY_KEYS, ATOMS_SCAN_HASH_KEY]);
function adoptExistingBaseSlug(rendered: RenderSessionResult, existingSlug: string): void {
if (existingSlug === rendered.baseSlug) return;
rendered.baseSlug = existingSlug;
for (const part of rendered.parts) part.slug = part.part === 1 ? existingSlug : `${existingSlug}-p${part.part}`;
}
/**
* The body bytes a single-layout session may fill: the page ceiling
* (`pages.max_bytes`, or the inline ceiling while segmented storage is off)
* or the content-sanity block line, whichever is lower (a page over the block
* line would land with embedding skipped), times PAGE_CEILING_FILL for the
* frontmatter. Read once per run.
*/
export async function transcriptPageCeiling(engine: BrainEngine): Promise<number> {
const [policy, block] = await Promise.all([readPageBodyPolicy(engine), effectiveBytesBlock(engine)]);
return Math.floor(Math.min(policy.maxBytes, block) * PAGE_CEILING_FILL);
}
/** The provenance the importer stamps on every part it writes (importFromContent and the coordinated put_page). */
const IMPORTER_INGESTED_VIA = 'cli:transcripts-ingest';
/**
* How a session renders, from what is stored (one query by the part 1 and
* part 2 identities, which also finds part 1's slug for adoption). The
* session is split when part 2 is live, or part 1 is live without
* `of <= 1` (a legacy `of > 1`, or a split part 1 that no longer carries
* `of`); it then keeps the part size its parts carry (`part_bytes`, else
* the pre-Wave-B PART_TARGET_BYTES). Otherwise it is one page under the page
* ceiling. A split session is never re-rendered as one page.
*
* `transcript_import` is ordinary frontmatter that any writer can set, so its
* `of` and `part_bytes` are read only from a part whose LIVE revision the
* importer wrote. `pages.ingested_via` can't say that: a managed write keeps
* the first write's stamp (persistence/page-prepare.ts), so a remote put_page
* over an imported page still reads `cli:transcripts-ingest`. The live
* revision's writer is the write attribution (`revision_write_request_id`):
* - a journaled request wrote it: trusted when that request is the local
* CLI's and its own intent carries the importer's provenance (a remote
* request's intent is always server-stamped `mcp:<operation>`);
* - its request has left the journal: trusted when the revision's principal
* is the local CLI and the page carries the importer's stamp;
* - no request wrote it (the direct importer on an unmanaged brain, a write
* from before attribution): trusted when `ingested_via` is the importer's
* or NULL; a remote write is always a journaled request, and on this path
* it also replaces the stamp with `mcp:<operation>`.
* So a remote edit can't choose the layout or the part size. A part 2 is
* split evidence only under the same rule: its id is derivable from part 1's,
* so any writer can put a page carrying it anywhere. A split session the
* importer rewrote still reads as split from part 1 (no `of`), or from a
* part 2 the importer wrote. One exception keeps a remote edit from merging a
* split session (deleting its parts): when part 1's own fields are not
* trusted, a part 2 at part 1's slug plus `-p2` counts whoever wrote it. A `part_bytes` is accepted only between PART_TARGET_BYTES
* (or the ceiling, when that is lower) and the current ceiling.
*/
async function storedTranscriptLayout(engine: BrainEngine, sourceId: string, meta: ParsedSession['meta'], pageCeiling: number):
Promise<{ layout: TranscriptLayout; partTargetBytes: number; existingSlug: string | null }> {
const [first, second] = [transcriptPartId(meta, 1), transcriptPartId(meta, 2)];
const rows = await engine.executeRaw<{ slug: string; fid: string; ti: unknown; trusted: boolean }>(
`SELECT p.slug, p.frontmatter->>'id' AS fid, p.frontmatter->'transcript_import' AS ti,
CASE WHEN p.revision_write_request_id IS NULL THEN p.ingested_via IS NULL OR p.ingested_via = $4
WHEN r.id IS NOT NULL THEN r.principal_kind = 'local_cli' AND r.intent->>'ingested_via' IS NOT DISTINCT FROM $4
ELSE p.revision_principal_kind IS NOT DISTINCT FROM 'local_cli' AND p.ingested_via IS NOT DISTINCT FROM $4 END AS trusted
FROM pages p LEFT JOIN persistence_requests r ON r.id = p.revision_write_request_id
WHERE p.source_id = $1 AND p.deleted_at IS NULL AND p.frontmatter->>'id' IN ($2, $3) ORDER BY p.id`,
[sourceId, first, second, IMPORTER_INGESTED_VIA],
);
const importMeta = (row: { ti: unknown; trusted: boolean } | undefined) => {
if (!row || row.trusted !== true) return null;
const ti = typeof row.ti === 'string' ? JSON.parse(row.ti) : row.ti;
return ti && typeof ti === 'object' ? ti as { of?: unknown; part_bytes?: unknown } : null;
};
const partOne = rows.find(row => row.fid === first);
const partTwo = rows.find(row => row.fid === second && (row.trusted === true
|| (partOne !== undefined && partOne.trusted !== true && row.slug === `${partOne.slug}-p2`)));
const one = importMeta(partOne);
const split = partTwo !== undefined || (one !== null && !(Number(one.of) <= 1));
if (!split) return { layout: 'single', partTargetBytes: pageCeiling, existingSlug: partOne?.slug ?? null };
const floor = Math.min(PART_TARGET_BYTES, pageCeiling);
const pinned = [one?.part_bytes, importMeta(partTwo)?.part_bytes]
.find(v => Number.isSafeInteger(v) && (v as number) >= floor && (v as number) <= pageCeiling) as number | undefined;
return { layout: 'split', partTargetBytes: pinned ?? floor, existingSlug: partOne?.slug ?? null };
}
/** Fold one file's adapter diagnostics into its outcome and the run result (drift, skipped lines, truncation). */
function applyFileDiagnostics(diag: FileDiagnostics, fileOutcome: IngestFileOutcome, result: TranscriptsIngestResult): void {
fileOutcome.skippedLines = diag.skippedLines;
if (diag.bytesRead > 0 && diag.sessions === 0 && !diag.expectedEmpty) {
fileOutcome.drift = true;
result.driftFiles++;
// A drifting file may hold sessions a fixed parser will surface
// later (torn hermes copy, transient format break) — the shared
// watermark must not advance past it. expectedEmpty (a grok
// tool/reasoning-only session) is understood, not drifted.
result.cleanScan = false;
}
if (diag.userTurnsMissing && !fileOutcome.drift) {
// E-N4: assistant turns parsed but not one user turn — the shape
// #5163 hid behind (a host renamed its user-turn record). The
// session still imports, but the file is drift: the watermark holds
// so a fixed parser re-reads it.
fileOutcome.drift = true;
fileOutcome.userTurnsMissing = true;
result.driftFiles++;
result.cleanScan = false;
}
if (diag.subagentRollout) {
fileOutcome.subagentRollout = true;
result.subagentRollouts++;
}
if (diag.skippedLines > 0) {
// Malformed lines can be DROPPED RECORDS (an actively-appended
// file read mid-write, corruption) — freeze the watermark so a
// later repair with an older timestamp is still picked up.
// Re-scans stay cheap via content-hash skip.
result.cleanScan = false;
}
if (diag.truncated) {
// A bounded read (codex head+tail over an over-budget rollout)
// skipped a window of the file — advancing the since-watermark
// over that unscanned window would drop its sessions permanently.
fileOutcome.truncated = true;
result.truncatedFiles++;
result.cleanScan = false;
}
}