Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* gbrain extract-conversation-facts — batch fact extraction for
* conversation pages (and adjacent long-form types).
*
* Background
* ----------
* Long-running conversation pages (imported chat logs, transcripts,
* etc.) can be very large — tens of thousands of messages spanning
* years. The default embedding pipeline chunks them into ~300-word
* blocks and prepends a tiny page-title hint. Enough for short pages,
* but it falls apart on long-running conversations:
*
* - A user searches for "mountain cabin lock code" but the chunk
* that contains the literal code reads only "Locker 93 code 9494"
* — no mention of "cabin", "mountain", or any topical anchor.
* - Retrieval misses, because the chunk-level embedding can't see
* the surrounding 50K messages of context that establish the topic.
*
* The facts table doesn't have this problem. Each row is a discrete
* claim with its own embedding and entity linkage, and `gbrain search`
* blends facts into the result set. The extraction pipeline that
* builds facts (src/core/facts/extract.ts) is already wired into
* real-time MCP turns and the post-sync backstop — but had never been
* run as a bulk backfill over imported chat history.
*
* This command closes that gap.
*
* Architecture decisions (locked by CEO + 3-round spec review + 2-round
* Codex outside voice + 2-pass eng review):
*
* - Strict per-source core. `runExtractConversationFactsCore` ALWAYS
* takes one sourceId. Multi-source iteration lives in the CLI
* wrapper (and in the cycle phase wrapper, separately).
* - Two-phase memory-bounded enumeration. Use paginated
* `listPages({type, sourceId, limit: PAGE_LIST_BATCH})` so worst
* case is BATCH × 25MB per batch (currently 10 × 25MB = 250MB
* bounded). Per-page body cap drops oversize before parsing.
* - Body read prefers frontmatter.raw_transcript when present, then
* falls back to compiled_truth + timeline. Meeting pages often
* store the real turn-by-turn transcript in a sidecar file while
* compiled_truth is just the human summary.
* - Page-global row_num accumulator. facts table unique index is
* (source_id, source_markdown_slug, row_num); per-segment row_num
* would collide on segment 2. Per-page counter increments across
* segments.
* - Snapshot-bound terminal audit row on completion. After all segments
* commit, one v2 row binds completion to the exact page version or raw
* transcript digest. Partial extraction has no matching terminal and the
* next claim performs a delete-first full replay.
* - Optional budgetTracker via opts. If a tracker is in opts, use it
* as-is (NO `withBudgetTracker` wrap, which would REPLACE the active
* tracker per gateway.ts AsyncLocalStorage semantics, defeating an
* outer brain-wide cap). If absent, auto-create from `maxCostUsd`
* and wrap. Callers explicitly own lifecycle.
* - Op-checkpoint string-encoded resume state. Entries are
* "<sourceId>|<slug>|<endIso>" strings (op_checkpoints stores
* string[] only and is GC'd at 7 days; durable audit is the facts
* table itself via the terminal row).
* - Fingerprint on sourceId only. Widening cycle.types config does
* NOT invalidate completed-page state.
*
* Honor brain-wide kill-switch:
* `facts.extraction_enabled=false` config blocks. Pass
* `--override-disabled` to force-run.
*/
import { extractExitCode, extractRunStatus, recordFailedPage } from '../core/facts/conversation-run-status.ts';
import type { BrainEngine, NewFact } from '../core/engine.ts';
import type { Page } from '../core/types.ts';
import {
isFactsExtractionEnabled,
type ExtractFailureReason,
type ExtractInput,
type ExtractedFact,
} from '../core/facts/extract.ts';
import { configureGatewayIfUninitialized, isAvailable, withBudgetTracker } from '../core/ai/gateway.ts';
import { writeDerivedFacts } from '../core/persistence/derived-facts.ts';
import { managedPersistenceEnabled } from '../core/persistence/ownership.ts';
import { countConversationFacts, enqueueManagedPage, flushManagedBatch, managedConversationPublisher, notePurgedSkips, publishConversationFacts, startManagedPage, stopsRun, type ConversationScope, type ManagedConversationPublisher } from '../core/facts/conversation-publication.ts';
import { ConversationRunClaims, listConversationPagesBySlug } from '../core/facts/conversation-enumeration.ts';
import { BudgetTracker, BudgetExhausted, loadPricingOverrides, type BudgetReason, type NoPricingGuidance } from '../core/budget/budget-tracker.ts';
import { noPricingMessage } from '../core/budget/no-pricing.ts';
import { conversationEstimateLine, conversationFactsCostCap, conversationRunEstimate } from '../core/facts/conversation-budget.ts';
import { listSources } from '../core/sources-ops.ts';
import {
loadOpCheckpoint,
recordCompleted,
type OpCheckpointKey,
} from '../core/op-checkpoint.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions, maybeBackground } from '../core/cli-options.ts';
import { createHash } from 'crypto';
// v0.41.15.0 (T5): worker-pool primitive + per-source-clamp wrapper +
// per-page advisory lock + delete-orphans-first replay safety. See plan
// `~/.claude/plans/system-instruction-you-are-working-fancy-creek.md`
// decisions D2, D6, D9, D11, D12, D13, D15.
import { runSlidingPool } from '../core/worker-pool.ts';
import { parseWorkers, resolveWorkersWithClamp } from '../core/sync-concurrency.ts';
import { withRefreshingLock, LockUnavailableError } from '../core/db-lock.ts';
import { assertFactsEmbeddingDimMatchesConfig } from '../core/embedding-dim-check.ts';
import { writeReceipt, shortRunId } from '../core/extract/receipt-writer.ts';
import { upsertExtractRollup, classifyRunStop } from '../core/extract/rollup-writer.ts';
import { ALLOWED_TYPES, ALLOWED_TYPE_ALIASES, isConversationFactsEligiblePage, pageTypesForAllowed, requireParseableConversationFlag, type AllowedType } from '../core/facts/conversation-types.ts';
import { TERMINAL_AUDIT_SOURCE, NON_EXTRACTABLE_AUDIT_SOURCE } from '../core/facts/audit-sources.ts';
import { resolveDefaultVisibility, tightenToStoredPage, type FactVisibility } from '../core/facts/visibility.ts';
import {
emptySaveTimeResolutionCounts,
formatSaveTimeResolutionCounts,
mergeSaveTimeResolutionCounts,
} from '../core/entities/resolve-on-save.ts';
import { conversationEntityHints } from '../core/facts/conversation-entity-hints.ts';
import { extractConversationWindow, nonExtractableAuditFact, outcomeSession, regularPageVersionToken, terminalAuditFact } from '../core/facts/conversation-window-extract.ts';
export { segmentObservationDate } from '../core/facts/conversation-window-extract.ts';
import { processSegmentedConversationPage, type GrammarPreview } from '../core/facts/conversation-incremental.ts';
import { BUILTIN_PATTERNS } from '../core/conversation-parser/builtins.ts';
import { isSegmentedPage } from '../core/page-body/page-shape.ts';
// Re-exported verbatim so existing importers (this file's own helpers below
// and this file's tests) keep working unchanged; doctor.ts, jobs.ts,
// sources.ts, and the cycle backfill phase import the leaf directly. Moved to
// src/core/facts/conversation-types.ts (see that file for why) so a
// consumer that only needs the six values doesn't also pull in this file's
// own CLI flag surface.
export { ALLOWED_TYPES };
export { ALLOWED_TYPE_ALIASES, pageTypesForAllowed };
export type { AllowedType };
// Re-exported for existing importers (test/extract-conversation-facts.test.ts,
// test/doctor-conversation-facts-backlog.test.ts, src/eval/brainbench/metrics/write-back.ts).
// The values themselves now live in ../core/facts/audit-sources.ts — see that
// leaf module's docstring for why (engine-live static-import requirement).
export { TERMINAL_AUDIT_SOURCE, NON_EXTRACTABLE_AUDIT_SOURCE };
// ---------------------------------------------------------------------------
// Tunables (exported for tests).
// ---------------------------------------------------------------------------
// Segment gap/cap/budget tunables and the windowing itself live in
// src/core/facts/conversation-windows.ts (#6267) and are re-exported below.
// #4136 — labels that are common DOCUMENT section headings, never lost
// speakers. Gate the DECLINE only (a miss here is warn-noise on healthy
// pages, never data loss — fail-open by construction).
const DOC_HEADING_STOPLIST = new Set([
'summary', 'results', 'notes', 'context', 'overview', 'background',
'example', 'examples', 'usage', 'steps', 'details', 'references',
'sources', 'appendix', 'conclusion', 'introduction', 'todo', 'tasks',
'output', 'input', 'goals', 'plan', 'ideas', 'agenda', 'decisions',
'actions', 'findings',
]);
/** #4136 — does a folded heading label LOOK like a speaker (1-2 title-cased
* words, not a stoplisted doc heading)? Only speaker-shaped folds can
* decline a page; everything else is warn-only. */
function isSpeakerShapedHeadingLabel(label: string): boolean {
if (!/^[A-Z][A-Za-z0-9._-]*( [A-Z][A-Za-z0-9._-]*)?$/.test(label)) return false;
return !DOC_HEADING_STOPLIST.has(label.toLowerCase());
}
/** Delay between extractor calls so we don't burst the chat provider. */
export const DEFAULT_INTER_CALL_SLEEP_MS = 200;
/**
* Hard cap on the parser input's UTF-8 bytes (the summary body, or the
* `raw_transcript` sidecar, which is refused from its size before any read).
* Pages over it are skipped to bound worker memory (Eng A2) and get a durable
* `skipped_too_large` receipt in page_extraction_state that doctor's
* `extraction_input_truncated` reports. Derived from PAGE_MAX_BYTES
* (conversation-parser/body.ts), so it never sits below the page ceiling.
*/
export const MAX_PAGE_BODY_BYTES = CONVERSATION_BODY_MAX_BYTES;
/** Default cost cap when no tracker is passed explicitly. */
export const DEFAULT_MAX_COST_USD = 5.0;
// ALLOWED_TYPES / AllowedType now live in
// ../core/facts/conversation-types.ts (imported + re-exported above).
// Mirrors cycle.conversation_facts_backfill.types config default. CLI's
// `--types` flag is an explicit per-run override; cycle config is the
// single source of truth.
/**
* Pagination batch size for listPages enumeration. Per-batch memory
* worst case = BATCH × MAX_PAGE_BODY_BYTES = 250MB at default 10
* (Eng-v2 C8 — bounded vs PR's unbounded listPages limit:500 = 12.5GB).
*/
export const PAGE_LIST_BATCH = 10;
/** Op name for the checkpoint primitive. */
export const CHECKPOINT_OP = 'extract-conversation-facts';
/**
* Source string written on per-segment facts. Doctor queries the
* TERMINAL variant below; this variant marks individual fact provenance.
*/
export const PER_SEGMENT_SOURCE_PREFIX = 'cli:extract-conversation-facts';
// TERMINAL_AUDIT_SOURCE / NON_EXTRACTABLE_AUDIT_SOURCE: defined in
// ../core/facts/audit-sources.ts, imported + re-exported above. (Doctor's
// backlog query matches TERMINAL_AUDIT_SOURCE + source_session, not the
// per-segment source; partial extraction = no terminal row = page stays in
// backlog. NON_EXTRACTABLE_AUDIT_SOURCE is kept distinct from successful
// extraction so operator surfaces can report the truth without rescanning
// the page forever.)
// ---------------------------------------------------------------------------
// Public types.
// ---------------------------------------------------------------------------
export type { ConversationMessage, ConversationSegment, SplitSegmentsOpts } from '../core/facts/conversation-types.ts';
export {
DEFAULT_SEGMENT_GAP_MINUTES, DEFAULT_SEGMENT_MAX_MESSAGES, MIN_SEGMENT_MESSAGES, SEGMENT_TEXT_CHAR_LIMIT,
MAX_PAGE_SEGMENT_GAP_MINUTES, splitIntoSegments, renderSegmentForExtraction,
} from '../core/facts/conversation-windows.ts';
import type { ConversationMessage } from '../core/facts/conversation-types.ts';
import {
DEFAULT_SEGMENT_GAP_MINUTES, MAX_PAGE_SEGMENT_GAP_MINUTES, splitIntoSegments, renderSegmentForExtraction,
} from '../core/facts/conversation-windows.ts';
/**
* Core function opts. Strict — `sourceId` is always required (Eng-v2 A1).
* Multi-source iteration is the caller's job.
*/
export interface ExtractConversationFactsCoreOpts {
/** REQUIRED. Strict per-source contract. */
sourceId: string;
/**
* Page types to walk. Reads cycle config when omitted.
* Allowlist enforced via ALLOWED_TYPES.
*/
types?: AllowedType[];
/** Process a single page; otherwise iterate all matching pages in the source. */
slug?: string;
/**
* cathedral-4 batch selector: process exactly these pages (serial, with
* the same per-page advisory lock + durable-outcome gates as enumeration).
* ONE core invocation per caller run — per-slug invocations multiply
* config resolution, checkpoint IO, and receipt writes by page count.
* Takes precedence over `slug`.
*/
slugs?: string[];
/** Show would-do counts without writing facts or advancing checkpoint. */
dryRun?: boolean;
/** Cap pages processed in this invocation (enumeration path only; ignored when `slugs` is set). */
limit?: number;
/** ISO watermark; messages older than this are filtered out. */
sinceIso?: string;
/** Clear this page's resume entry before processing. */
force?: boolean;
/** Delay between extractor calls. */
sleepMs?: number;
/** Max segments to process per page (0 = unlimited). */
segmentLimit?: number;
/**
* Cost cap (USD). Used when budgetTracker is NOT passed; core
* creates a fresh tracker. Default DEFAULT_MAX_COST_USD.
*/
maxCostUsd?: number;
/**
* Externally-managed BudgetTracker (Eng-v2 C5). If present, core
* uses it as-is — no `withBudgetTracker` wrap. Cycle phase passes
* a brain-wide tracker; CLI/Minion pass nothing.
*/
budgetTracker?: BudgetTracker;
/** Bypass `facts.extraction_enabled=false`. Power-user escape. */
overrideDisabled?: boolean;
/**
* v0.41.15.0 (D9 wrapper): in-process worker count for the per-page
* fan-out. Default 1 (back-compat). Recommended 5-20 for LLM-bound
* work; PGLite engines silently clamp to 1 with a stderr warn. Cross-
* process safety is structurally guaranteed by D2's per-page advisory
* lock + D11's delete-orphans-first replay.
*
* Worst-case overshoot on `--max-cost-usd`: D3 documented overshoot is
* `N × avg_per_call_cost` over the configured cap because per-worker
* `reserve()` calls aren't serialized. At workers=20 × ~$0.02/page,
* expect up to ~$0.40 over the cap. Tighten the cap or pin workers=1
* if you need exact-ceiling compliance.
*/
workers?: number;
/**
* Injectable per-segment extractor (BrainBench decision 15). When unset,
* the production path is `extractFactsFromTurnWithOutcome` (fail-hard: a
* per-segment extraction failure aborts the page). The bench's deterministic
* CI mode injects a gold-facts extractor here so segmentation → insertFacts →
* dedup → provenance all execute THIS production pipeline with zero LLM calls.
*/
extractor?: (input: ExtractInput) => Promise<ExtractedFact[]>;
/**
* PR 4 (Wave B incremental derivation, transcripts): the operator's conversation format for the page (--grammar; null:
* none, the page is not a transcript) for a page stored in segments on an unmanaged brain (the CLI refuses it otherwise:
* resolveGrammarSource). Recorded for the page's current version (later plain runs follow it until the page changes)
* unless dryRun; bypasses the fresh-outcome skip.
*/
grammar?: string | null;
/**
* #7068: given a copy of the run's counters when the run ends in a thrown error, taken after the managed flush published
* the pages that finished, so a caller can report work that committed. The error is still thrown.
*/
onInterrupted?: (partial: ExtractConversationFactsResult) => void;
}
export { extractExitCode, extractRunStatus };
export interface ExtractConversationFactsResult {
pages_considered: number;
pages_processed: number;
pages_skipped: number;
pages_skipped_unparsed: number;
pages_skipped_type_mismatch: number;
pages_skipped_insufficient_turns: number;
pages_skipped_since: number;
pages_skipped_too_large: number;
pages_skipped_disappeared: number;
/** Fresh terminal outcomes skipped before parsing or model work. */
pages_skipped_completed: number;
/** Fresh scanned-not-extractable outcomes skipped before parser work. */
pages_skipped_non_extractable: number;
/** Durable scanned-not-extractable outcomes written by this run. */
pages_marked_non_extractable: number;
/** #4136 — pages declined because the winning heading pattern folded a
* speaker-shaped unrecognized heading and the parse had fewer than two
* distinct speakers (attribution would be wrong). Non-terminal: no
* durable audit row is written, so a future parser/pattern fix retries. */
pages_skipped_unrecognized_speaker: number;
/** Pages whose claim reached extraction but failed before durable outcome. */
pages_failed: number;
/** The failed pages (first 50) with the error each one hit; they stay unfinished and retry next run. */
failed_pages: Array<{ slug: string; error: string }>;
/**
* Pages whose built-in parse returned `no_match` and whose messages were
* recovered by the explicitly enabled LLM fallback.
*/
pages_llm_fallback: number;
/**
* v0.41.15.0 (D6): pages we attempted to claim but skipped because
* another worker / parallel process held the advisory lock. The pages
* stay in the backlog; the next enumeration cycle picks them up once
* the holder writes its terminal row. Operator surfaces this via the
* exit summary; exit code 3 when non-zero AND no hard failures.
*/
pages_lock_skipped: number;
/**
* v0.41.15.0 (D11): facts deleted by the per-page delete-orphans-first
* replay safety pass. Non-zero means a prior run crashed mid-extract;
* the current worker cleaned up and re-extracted from scratch. Always
* safe; surfaced for operator observability.
*/
orphan_facts_cleaned: number;
segments_processed: number;
facts_extracted: number;
facts_inserted: number;
/** #7438: extracted claims left out because they match a purge tombstone (the rest of the page still publishes). */
facts_skipped_purged?: number;
/** Entity values that reached the shipped deterministic fallback slug path; GBRA-574 `unbacked_keys` of them kept a name key with no page. */
fallback_slugify_count: number; unbacked_keys?: number;
/** Entity values kept raw after a best-effort resolution failure. */
resolution_errors: number;
/** GBRA-574: windows whose slot-tagging call failed; their facts were kept without slots (--force tags them again). */ slot_tagging_failed?: number;
/** Managed brains: pages whose publication was accepted but is still pending (retried next run). */
pages_pending?: number;
/** Managed brains: pages whose extraction generation is blocked until the page changes (no model call). */
pages_blocked?: number;
/** Pages the run listed more than once and extracted once (#6500). */
pages_skipped_duplicate_listing?: number;
/** PR 4 (Wave B incremental derivation, transcripts): pages held because no format is detected over them yet some format parses a message. */
pages_detection_ambiguous?: number;
/** PR 4 (Wave B incremental derivation, transcripts): --grammar --dry-run, what a run under the chosen format would extract and collect. */
grammar_previews?: GrammarPreview[];
budget_exhausted?: boolean;
budget_reason?: BudgetReason;
budget_model?: string;
/** no_pricing halt: the lookup-and-register guidance (model, provider, kind, units, command, docs). */
budget_pricing?: NoPricingGuidance;
spent_usd?: number;
}
// ---------------------------------------------------------------------------
// Message parsing — v0.41.13.0 delegates to the new
// `src/core/conversation-parser/parse.ts` orchestrator (12+ built-in
// formats + opt-IN LLM polish/fallback). PR #1461's Telegram bracket-time
// shape is the `telegram-bracket` built-in pattern. PR #1461's existing
// `MESSAGE_LINE_RX` is the `imessage-slack` built-in pattern.
//
// This wrapper preserves the historical `parseConversationMessages(body,
// opts)` shape for back-compat with the test suite + any direct callers.
// `processPage` below threads a full Page through `parseConversation` so
// frontmatter date / timezone / effective_date precedence per D8 takes
// effect.
// ---------------------------------------------------------------------------
import {
deriveDateContext,
parseConversation,
type ParseConversationOpts as OrchestratorParseOpts,
} from '../core/conversation-parser/parse.ts';
import { ConversationBodyTooLargeError, CONVERSATION_BODY_MAX_BYTES, readConversationInput } from '../core/conversation-parser/body.ts';
import { CONVERSATION_WINDOW_BYTES, type ConversationWindow } from '../core/page-body/segment-conversation.ts';
import { setExtractionReceipt } from '../core/extraction-watermark.ts';
import { conversationSkip } from '../core/facts/conversation-skip.ts';
import { runLlmFallback } from '../core/conversation-parser/llm-fallback.ts';
import { resolveModel, resolveTierDefault } from '../core/model-config.ts';
import { FAILED_EXIT_CODE } from '../core/exit-codes.ts';
import { usageError } from '../cli/cli-error.ts';
import { intFlagValue } from '../cli/flag-values.ts';
const ECF_HELP_HINT = 'Run `gbrain extract-conversation-facts --help` for the accepted flags and examples.';
/**
* v0.41.13.0 — back-compat shape for direct callers + the existing
* test suite. Delegates to the new orchestrator.
*
* Per D8: callers with a full Page should pass `opts.page` instead of
* `opts.fallbackDate` so the orchestrator's date-derivation chain
* (frontmatter.date > effective_date > '1970-01-01') applies. The
* `fallbackDate` field is preserved for PR #1461's test cases that
* pass it explicitly.
*/
export function parseConversationMessages(
body: string,
opts: { fallbackDate?: string } = {},
): ConversationMessage[] {
const result = parseConversation(body, {
fallbackDate: opts.fallbackDate,
} as OrchestratorParseOpts);
return result.messages;
}
// ---------------------------------------------------------------------------
// Segment gap override (windowing: src/core/facts/conversation-windows.ts).
// ---------------------------------------------------------------------------
/**
* A page's own segmentation gap: frontmatter `conversation_segment_gap_minutes`,
* set by a collector that knows its message cadence. Absent means the global
* default. Any value other than an integer from 1 to
* MAX_PAGE_SEGMENT_GAP_MINUTES is ignored with a warning naming the accepted
* range, so the page still splits on the default instead of failing.
*/
export function pageSegmentGapMinutes(page: Pick<Page, 'slug' | 'frontmatter'>): number | undefined {
const raw = page.frontmatter?.conversation_segment_gap_minutes;
if (raw === undefined || raw === null) return undefined;
if (typeof raw === 'number' && Number.isInteger(raw) && raw >= 1 && raw <= MAX_PAGE_SEGMENT_GAP_MINUTES) return raw;
process.stderr.write(
`[extract-conversation-facts] ${page.slug}: ignoring frontmatter conversation_segment_gap_minutes=${JSON.stringify(raw)?.slice(0, 80)}; ` +
`it must be a whole number of minutes from 1 to ${MAX_PAGE_SEGMENT_GAP_MINUTES} (unquoted). ` +
`Splitting on the default ${DEFAULT_SEGMENT_GAP_MINUTES}-minute gap instead. ` +
`To fix: set the key to an integer in that range, or remove it, then rerun gbrain extract-conversation-facts --slug ${page.slug}\n`,
);
return undefined;
}
// ---------------------------------------------------------------------------
// Fingerprint — sourceId-only (Eng-v2 A3). Widening types config does NOT
// invalidate prior completion state.
// ---------------------------------------------------------------------------
export function extractConversationFactsFingerprint(opts: { sourceId: string }): string {
const canonical = JSON.stringify({ sourceId: opts.sourceId });
return createHash('sha256').update(canonical).digest('hex').slice(0, 8);
}
function checkpointKey(sourceId: string): OpCheckpointKey {
return { op: CHECKPOINT_OP, fingerprint: extractConversationFactsFingerprint({ sourceId }) };
}
// ---------------------------------------------------------------------------
// Op-checkpoint helpers — string-encoded "<sourceId>|<slug>|<endIso>" entries.
// ---------------------------------------------------------------------------
interface DecodedEntry {
sourceId: string;
slug: string;
endIso: string;
}
export function encodeCheckpointEntry(sourceId: string, slug: string, endIso: string): string {
// Slugs are validated to [a-z0-9_/-] + CJK; sourceId is [a-z0-9_-].
// Neither contains the pipe character, so the delimiter is safe.
return `${sourceId}|${slug}|${endIso}`;
}
export function decodeCheckpointEntry(entry: string): DecodedEntry | null {
// Split on first two pipes only — endIso has no pipes either.
const i1 = entry.indexOf('|');
if (i1 < 0) return null;
const i2 = entry.indexOf('|', i1 + 1);
if (i2 < 0) return null;
return {
sourceId: entry.slice(0, i1),
slug: entry.slice(i1 + 1, i2),
endIso: entry.slice(i2 + 1),
};
}
/** Returns the newest endIso for a given (sourceId, slug), or null if absent. */
function findCompletedEndIso(
entries: string[],
sourceId: string,
slug: string,
): string | null {
let best: string | null = null;
for (const e of entries) {
const d = decodeCheckpointEntry(e);
if (!d) continue;
if (d.sourceId !== sourceId) continue;
if (d.slug !== slug) continue;
if (best === null || d.endIso > best) best = d.endIso;
}
return best;
}
/** Returns entries with all (sourceId, slug)-matching rows stripped. */
function filterOutSlug(entries: string[], sourceId: string, slug: string): string[] {
return entries.filter((e) => {
const d = decodeCheckpointEntry(e);
if (!d) return true;
return !(d.sourceId === sourceId && d.slug === slug);
});
}
// ---------------------------------------------------------------------------
// Body cap (Eng A2).
// ---------------------------------------------------------------------------
/**
* The body cap, pre-parse: a skip leaves a durable receipt (doctor: extraction_input_truncated); a page under the cap
* clears an earlier one. A segmented page over the window cap is extracted from its newest window only when it has no
* conversation facts (the run clears them all; lossless until PR 4's anchors): one with any arrives unread (window.priorFacts).
*/
async function skippedForSize(state: ExtractCoreState, snapshot: ConversationPageSnapshot): Promise<boolean> {
const origin = snapshot.oversizedSidecarBytes === undefined ? 'page' : 'sidecar';
const bytes = snapshot.oversizedSidecarBytes ?? Buffer.byteLength(snapshot.body, 'utf8');
const over = bytes > MAX_PAGE_BODY_BYTES;
const window = snapshot.window;
const priorFacts = window?.priorFacts ?? 0;
if (over || priorFacts > 0) {
state.result.pages_skipped_too_large++;
process.stderr.write(`[extract-conversation-facts] SKIP ${snapshot.page.slug}: ${over ? `${origin === 'sidecar' ? 'raw_transcript sidecar ' : ''}`
+ `${(bytes / 1024 / 1024).toFixed(1)}MB exceeds ${MAX_PAGE_BODY_BYTES / 1024 / 1024}MB cap` : `${(window!.totalBytes / 1024 / 1024).toFixed(1)}MB is over the `
+ `${CONVERSATION_WINDOW_BYTES / 1024 / 1024}MB window and its ${priorFacts} conversation fact(s) may lie outside it; they were kept`}\n`);
}
if (!state.dryRun) await setExtractionReceipt(state.engine, snapshot.page.id, 'conversation_facts', over
? { kind: 'skipped_too_large', pipeline: 'conversation_facts', processed_chars: 0, total_bytes: bytes, cap_bytes: MAX_PAGE_BODY_BYTES, origin }
: !window ? null : priorFacts > 0
? { kind: 'skipped_too_large', pipeline: 'conversation_facts', processed_chars: 0, total_bytes: window.totalBytes, cap_bytes: CONVERSATION_WINDOW_BYTES,
origin: 'page', window_rule: 'prior_facts_kept', prior_facts: priorFacts }
: { kind: 'tail_window', pipeline: 'conversation_facts', processed_chars: snapshot.body.length, total_bytes: window.totalBytes,
cap_bytes: CONVERSATION_WINDOW_BYTES, origin: 'page', window_rule: 'no_prior_facts' });
return over || priorFacts > 0;
}
// ---------------------------------------------------------------------------
// Types config resolver (Eng-v2 A2 — unified single source of truth).
// ---------------------------------------------------------------------------
const TYPES_CONFIG_KEY = 'cycle.conversation_facts_backfill.types';
async function resolveTypesFromConfig(
engine: BrainEngine,
explicit?: AllowedType[],
): Promise<AllowedType[]> {
if (explicit && explicit.length > 0) return explicit;
const raw = await engine.getConfig(TYPES_CONFIG_KEY);
if (raw) {
try {
const parsed = JSON.parse(raw);
if (Array.isArray(parsed)) {
const filtered = parsed
.filter((t): t is string => typeof t === 'string')
.filter((t): t is AllowedType => (ALLOWED_TYPES as readonly string[]).includes(t));
if (filtered.length > 0) return filtered;
}
} catch {
// fall through to default
}
}
// Default: full allowlist when no config and no explicit override.
// Mirrors cycle.conversation_facts_backfill.types default.
return [...ALLOWED_TYPES];
}
// ---------------------------------------------------------------------------
// v0.41.15.0 helpers (D2 lock + D6 rate-limited log + D11 delete-orphans).
// ---------------------------------------------------------------------------
/**
* Lock id for the per-page advisory lock (D2). Includes the source so
* two pages with the same slug in different sources don't false-share.
*/
export function extractConversationFactsLockId(sourceId: string, slug: string): string {
return `extract-conversation-facts:${sourceId}:${slug}`;
}
/**
* Per-page lock TTL (D12). 2 minutes — `withRefreshingLock` refreshes
* at 1/6 the TTL (`Math.max(15000, 120_000/6) = 20s`) so a long page
* (50 segments × Haiku ~3s) gets ~6 refreshes per minute. If the holder
* process dies, the lock auto-expires within 2 minutes regardless.
*/
export const PER_PAGE_LOCK_TTL_MINUTES = 2;
/**
* D6: in-memory rate-limit cache for lock-busy log lines. Keyed on
* (source_id, minute-bucket) so we log at most once per source per
* minute even under heavy contention. Pure process-local state.
*/
const _lockBusyLogCache = new Map<string, number>();
function logLockBusyRateLimited(sourceId: string, slug: string): void {
const minuteBucket = Math.floor(Date.now() / 60_000);
const key = `${sourceId}:${minuteBucket}`;
if (_lockBusyLogCache.has(key)) return;
_lockBusyLogCache.set(key, minuteBucket);
// Best-effort cleanup: when the map grows past 100 entries, drop ones
// older than 10 minutes. Avoids unbounded growth on multi-hour runs.
if (_lockBusyLogCache.size > 100) {
const cutoff = minuteBucket - 10;
for (const [k, v] of _lockBusyLogCache) {
if (v < cutoff) _lockBusyLogCache.delete(k);
}
}
process.stderr.write(
`[extract-conversation-facts] lock-busy for ${sourceId}:${slug} (and possibly more); skipping — another worker holds it. Page will be retried on next enumeration.\n`,
);
}
/**
* Unmanaged brains: publishes one page's buffered rows (facts and its outcome
* row, when the run reached one) in one maintenance transaction, which
* retires the prior extractor batch only for a complete replacement (#7013,
* `publishConversationFacts`). Returns the rows removed from the page's
* active batch and the facts inserted.
*/
function publishPageFacts(state: ExtractCoreState, slug: string, rows: Array<NewFact & { row_num: number; source_markdown_slug: string }>,
scope: ConversationScope): Promise<{ deleted: number; facts_inserted: number; duplicates_skipped: number; purged_skipped: number }> {
return writeDerivedFacts(state.engine, state.sourceId, slug, db => publishConversationFacts(db, state.sourceId, slug, rows, { scope }));
}
// ---------------------------------------------------------------------------
// Core extraction loop (single source).
// ---------------------------------------------------------------------------
interface ExtractCoreState {
result: ExtractConversationFactsResult;
engine: BrainEngine;
sourceId: string;
/** Managed brain (not a dry run): a page's rows are buffered and replace the prior batch in one receipted request. */
managed: ManagedConversationPublisher | null;
dryRun: boolean;
sleepMs: number;
segmentLimit: number;
types: AllowedType[];
factVisibility: FactVisibility;
signal: AbortSignal | undefined;
/**
* Injected per-segment extractor (BrainBench decision 15). ONLY set when a
* caller overrides; when undefined the production fail-hard
* `extractFactsFromTurnWithOutcome` path runs.
*/
extractor?: (input: ExtractInput) => Promise<ExtractedFact[]>;
/**
* v0.41.15.0 (D11): shared per-(sourceId, slug) checkpoint map mutated
* in place from processPage callers. Map.set is atomic in JS's single-
* threaded event loop so parallel workers (D9) don't clobber each
* other. Serialized to op-checkpoint string[] via recordCompleted at
* batch boundaries + final flush.
*/
cpMap: Map<string, string>;
/**
* Opt-in LLM parser state, resolved once per source run. A null model means
* the fallback is disabled and no chat content leaves the deterministic
* parser path.
*/
llmFallbackModel: string | null;
}
function cpMapKey(sourceId: string, slug: string): string {
return `${sourceId}|${slug}`;
}
function cpMapToEntries(map: Map<string, string>): string[] {
const out: string[] = [];
for (const [key, endIso] of map) {
const i = key.indexOf('|');
if (i < 0) continue;
const sourceId = key.slice(0, i);
const slug = key.slice(i + 1);
out.push(encodeCheckpointEntry(sourceId, slug, endIso));
}
return out;
}
function cpEntriesToMap(entries: string[]): Map<string, string> {
const map = new Map<string, string>();
for (const e of entries) {
const d = decodeCheckpointEntry(e);
if (!d) continue;
// Newest-endIso wins on duplicates (defensive against pre-fix
// entries that may have stacked).
const key = cpMapKey(d.sourceId, d.slug);
const prior = map.get(key);
if (prior === undefined || d.endIso > prior) map.set(key, d.endIso);
}
return map;
}
export type DurableExtractionOutcome = 'complete' | 'non_extractable';
interface ConversationPageSnapshot {
page: Page;
body: string;
/** A segmented page over the window cap: `body` is its newest window. */
window?: ConversationWindow;
versionToken: string;
/** The `raw_transcript` sidecar was over the cap and not read; `body` is empty. */
oversizedSidecarBytes?: number;
}
function hasRawTranscriptSidecar(page: Page): boolean {
const raw = page.frontmatter?.raw_transcript;
return typeof raw === 'string' && raw.trim().length > 0;
}
function snapshotVersionToken(page: Page, input: { body: string; sidecar?: true }): string {
if (!hasRawTranscriptSidecar(page)) return regularPageVersionToken(page);
// Sidecar contents can change without touching pages.updated_at. Hash the exact parser input plus parser-relevant page metadata so those
// edits reopen the page without a schema migration. A segmented page parsing its own body hashes content_hash, never the input the prior-fact guard empties.
const parserInput = page.body_storage === 'segmented' && !input.sidecar ? { page: regularPageVersionToken(page) } : { body: input.body };
return `sidecar-${createHash('sha256')
.update(
JSON.stringify({
...parserInput,
title: page.title,
type: page.type,
frontmatter: page.frontmatter,
effective_date: page.effective_date ?? null,
}),
)
.digest('hex')}`;
}
async function preparePageSnapshot(
engine: BrainEngine,
page: Page,
): Promise<ConversationPageSnapshot> {
try {
const input = await readConversationInput(engine, page, { priorFacts: () => countConversationFacts(engine, page.source_id, page.slug) });
return { page, body: input.body, ...(input.window ? { window: input.window } : {}), versionToken: snapshotVersionToken(page, input) };
} catch (error) {
if (!(error instanceof ConversationBodyTooLargeError)) throw error;
return { page, body: '', versionToken: `sidecar-oversized-${error.bytes}-${error.mtimeMs}`, oversizedSidecarBytes: error.bytes };
}
}
/**
* Find v2 outcomes bound to the exact parser input snapshot. Legacy outcome
* rows deliberately do not match and are replayed once under the strict v2
* protocol. Sidecar files are hashed because pages.updated_at cannot see them.
*/
export async function findFreshExtractionOutcomes(
engine: BrainEngine,
sourceId: string,
pages: readonly Page[],
): Promise<Map<string, DurableExtractionOutcome>> {
if (pages.length === 0) return new Map();
const expected = new Map<string, string>();
for (const page of pages) {
// Batch enumeration can already be stale. Refresh before deciding to skip
// so an edit between listPages and this check cannot match an old marker.
const current = await engine.getPage(page.slug, { sourceId });
if (!current) continue;
const token = hasRawTranscriptSidecar(current)
? (await preparePageSnapshot(engine, current)).versionToken
: regularPageVersionToken(current);
expected.set(current.slug, token);
}
const rows = await engine.executeRaw<{
slug: string;
source: string;
source_session: string | null;
}>(
`SELECT source_markdown_slug AS slug, source, source_session
FROM facts
WHERE source_id = $1
AND source_markdown_slug = ANY($2::text[])
AND source = ANY($3::text[]) AND expired_at IS NULL
ORDER BY source_markdown_slug,
CASE WHEN source = $4 THEN 0 ELSE 1 END`,
[
sourceId,
pages.map((page) => page.slug),
[TERMINAL_AUDIT_SOURCE, NON_EXTRACTABLE_AUDIT_SOURCE],
TERMINAL_AUDIT_SOURCE,
],
);
const outcomes = new Map<string, DurableExtractionOutcome>();
for (const row of rows) {
if (outcomes.has(row.slug)) continue;
const token = expected.get(row.slug);
if (!token || row.source_session !== outcomeSession(row.source, row.slug, token)) {
continue;
}
outcomes.set(
row.slug,
row.source === TERMINAL_AUDIT_SOURCE ? 'complete' : 'non_extractable',
);
}
return outcomes;
}
function recordDurableOutcomeSkip(
state: ExtractCoreState,
outcome: DurableExtractionOutcome,
): void {
state.result.pages_considered++;
if (outcome === 'complete') state.result.pages_skipped_completed++;
else state.result.pages_skipped_non_extractable++;
}
async function snapshotIsCurrent(
engine: BrainEngine,
sourceId: string,
snapshot: ConversationPageSnapshot,
): Promise<boolean> {
const current = await engine.getPage(snapshot.page.slug, { sourceId });
if (!current) return false;
const currentSnapshot = await preparePageSnapshot(engine, current);
return currentSnapshot.versionToken === snapshot.versionToken;
}
/** The parser-input version token of `page` as it is now (the managed preparer's recheck). */
export async function currentConversationVersionToken(engine: BrainEngine, page: Page): Promise<string> {
return (await preparePageSnapshot(engine, page)).versionToken;
}
async function processPage(
state: ExtractCoreState,
snapshot: ConversationPageSnapshot,
sinceIso: string | undefined,
): Promise<{ newEndIso: string | null }> {
const { page, body } = snapshot;
state.result.pages_considered++;
// Body cap check first — pre-parse, pre-segment, pre-extraction.
if (await skippedForSize(state, snapshot)) return { newEndIso: null };
// v0.41.13.0: thread the full Page through the orchestrator so D8
// date-derivation chain (frontmatter.date > effective_date >
// '1970-01-01') AND timezone_policy warnings apply. The historical
// `parseConversationMessages(body)` shape only saw the body, which
// meant Telegram-bracket pages with frontmatter dates landed at
// 1970-01-01. Now they pick up the correct date.
const fallbackDate = snapshot.window?.fallbackDate ?? undefined;
const parseResult = parseConversation(body, { page, ...(fallbackDate ? { fallbackDate } : {}) });
let messages = parseResult.messages;
if (parseResult.timezone_warning) {
process.stderr.write(parseResult.timezone_warning + '\n');
}
// #4136 — the winning heading pattern folded heading-shaped lines into the
// previous turn's body instead of anchoring them. Decline ONLY when a
// folded label is speaker-shaped (title-cased, not a doc heading) AND the
// parse produced fewer than two distinct speakers — the reported repro is
// exactly this shape ([User, User] with the assistant's reply swallowed).
// A multi-speaker page with folds is warn-only (residual risk, visible).
// phase stays 'regex_match', so the LLM fallback gate below stays closed.
const foldedHeadings = parseResult.unrecognized_headings ?? [];
const speakerShapedFolds = foldedHeadings.filter(isSpeakerShapedHeadingLabel);
const declinedUnrecognizedSpeaker =
speakerShapedFolds.length > 0 &&
new Set(messages.map((m) => m.speaker)).size < 2;
if (foldedHeadings.length > 0) {
const detail =
`pattern=${parseResult.matched_pattern_id} folded unrecognized heading(s) ` +
`[${foldedHeadings.join(', ')}] into the previous turn`;
if (declinedUnrecognizedSpeaker) {
process.stderr.write(
`[extract-conversation-facts] ${page.slug}: ${detail}; declining extraction (speaker attribution would be wrong)\n`,
);
state.result.pages_skipped_unrecognized_speaker++;
messages = [];
} else if (speakerShapedFolds.length > 0) {
process.stderr.write(
`[extract-conversation-facts] ${page.slug}: ${detail}; proceeding (speakers alternate) — facts near those headings may be misattributed\n`,
);
}
}
// The fallback runs only for a true built-in miss. It never replaces or
// polishes a deterministic parse, and it remains unreachable unless the
// operator explicitly enables conversation_parser.llm_fallback_enabled.
if (
!state.dryRun &&
messages.length === 0 &&
parseResult.phase === 'no_match' &&
state.llmFallbackModel
) {
const fallbackMessages = await runLlmFallback({
modelStr: state.llmFallbackModel,
body,
engine: state.engine,
signal: state.signal,
fallbackDate: deriveDateContext({ page, ...(fallbackDate ? { fallbackDate } : {}) }).fallbackDate,
propagateError: (error) =>
error instanceof BudgetExhausted ||
(state.signal?.aborted === true && isAbortError(error)),
});
if (fallbackMessages && fallbackMessages.length > 0) {
messages = fallbackMessages;
state.result.pages_llm_fallback++;
process.stderr.write(
`[extract-conversation-facts] LLM fallback parsed ${fallbackMessages.length} message(s) for ${page.slug}\n`,
);
}
}
// #5025 / N2: undated time-only turns, a single email or a prose
// meeting/email page end in a not-extractable outcome instead of
// epoch-dated facts or a rescan every run.
const skip = conversationSkip(page, body, parseResult, messages, { llmFallback: Boolean(state.llmFallbackModel) });
if (skip) {
process.stderr.write(`[extract-conversation-facts] SKIP ${page.slug}: ${skip.message}\n`);
messages = [];
}
// An email thread is one conversation even when replies are hours apart;
// a frontmatter conversation_segment_gap_minutes still wins.
const gapMinutes = pageSegmentGapMinutes(page) ??
(parseResult.matched_pattern_id === 'email-thread-heading' ? MAX_PAGE_SEGMENT_GAP_MINUTES : undefined);
const allSegments = splitIntoSegments(messages, { gapMinutes, pageTitle: page.title || page.slug });
const segments = splitIntoSegments(messages, { gapMinutes, sinceIso, pageTitle: page.title || page.slug });
if (segments.length === 0) {
state.result.pages_skipped++;
if (!declinedUnrecognizedSpeaker && !skip) {
if (messages.length === 0) state.result.pages_skipped_unparsed++;
else if (allSegments.length === 0) state.result.pages_skipped_insufficient_turns++;
else state.result.pages_skipped_since++;
}
if (
!state.dryRun &&
(parseResult.phase !== 'no_match' || skip !== null) &&
allSegments.length === 0 &&
// #4136 — a decline must stay NON-TERMINAL. The audit row is keyed by
// a content versionToken and skips the page on every future run; a
// declined page must retry once the parser learns the label instead.
// (Trade, stated: pre-existing wrong-speaker facts also skip the
// orphan cleanup below until the page re-extracts.)
!declinedUnrecognizedSpeaker
) {
const reason = skip?.reason ?? (messages.length === 0
? 'no conversation messages found'
: 'fewer than two eligible messages');
if (await snapshotIsCurrent(state.engine, state.sourceId, snapshot)) {
if (state.managed) {
// Counted when its batch publishes.
const managed = await startManagedPage(state, snapshot, sinceIso, 0);
if ('start' in managed) await enqueueManagedPage(state, snapshot, managed.start, [nonExtractableAuditFact(page.slug, 0, snapshot.versionToken, reason)], null, 'full');
return { newEndIso: null };
} else {
// #7013: the durable not-extractable outcome stops repeated scans; the page's prior facts stay.
state.result.orphan_facts_cleaned += (await publishPageFacts(state, page.slug, [nonExtractableAuditFact(page.slug, 0, snapshot.versionToken, reason)], 'full')).deleted;
}
state.result.pages_marked_non_extractable++;
}
}
return { newEndIso: null };
}
if (state.dryRun) {
state.result.segments_processed += state.segmentLimit > 0
? Math.min(segments.length, state.segmentLimit)
: segments.length;
state.result.pages_processed++;
return { newEndIso: null };
}
const managed = state.managed ? await startManagedPage(state, snapshot, sinceIso, segments.length) : null;
if (managed && 'done' in managed) return managed.done;
// #7013: the page's rows are buffered and published once, after every
// segment's extraction (a managed brain in its receipted batch, an
// unmanaged one in one transaction below); the prior batch stays until a
// complete replacement retires it, so a failed, partial or empty run never
// removes it. Row numbers are reassigned under the page lock on publish.
let rowNum = 0;
let newestEnd: string | null = null;
let segmentsThisPage = 0;
const pageRows: Array<NewFact & { row_num: number; source_markdown_slug: string }> = [];
const pageResolution = emptySaveTimeResolutionCounts();
const factVisibility = await tightenToStoredPage(state.engine, state.sourceId, state.factVisibility, page);
const entityHints = await conversationEntityHints(state.engine, state.sourceId, page.slug, factVisibility); // #6731
for (const seg of segments) {
if (state.segmentLimit > 0 && segmentsThisPage >= state.segmentLimit) break;
if (state.signal?.aborted) throw new Error('aborted');
const text = renderSegmentForExtraction(page.title || page.slug, seg);
const window = await extractConversationWindow(state, page, seg, text, rowNum, entityHints);
state.result.segments_processed++;
segmentsThisPage++;
state.result.facts_extracted += window.extracted;
pageRows.push(...window.rows);
rowNum += window.extracted;
mergeSaveTimeResolutionCounts(pageResolution, window.resolution);
state.result.fallback_slugify_count += window.resolution.fallback_slugify_count; if (window.resolution.unbacked_key_count) state.result.unbacked_keys = (state.result.unbacked_keys ?? 0) + window.resolution.unbacked_key_count;
state.result.resolution_errors += window.resolution.resolution_errors;
newestEnd = seg.endIso;
if (state.sleepMs > 0) await sleep(state.sleepMs);
}
// Eng-v2 C7 / E16: write terminal audit row after all segments commit
// successfully. Only run when every segment was processed — including a
// page whose segment count equals --segment-limit exactly, which
// previously missed the terminal row and re-extracted on every run.
const fullyProcessed = segmentsThisPage === segments.length;
const inScope = (list: typeof segments) => list.reduce((n, seg) => n + seg.messages.length, 0);
const scope: ConversationScope = !fullyProcessed ? 'partial' : inScope(segments) < inScope(allSegments) ? 'window' : 'full';
if (
fullyProcessed &&
newestEnd !== null &&
await snapshotIsCurrent(state.engine, state.sourceId, snapshot)
) {
// The terminal row publishes in the same transaction as the page's facts.
// A publication failure propagates, so bulk accounting, CLI exit status,
// cycle status, and rollups all report the page as unfinished.
pageRows.push(terminalAuditFact(page.slug, rowNum, snapshot.versionToken));
} else if (fullyProcessed && newestEnd !== null) {
process.stderr.write(
`[extract-conversation-facts] ${page.slug} changed during extraction; leaving it unfinished for replay\n`,
);
// An unmanaged brain keeps what the run extracted as a partial batch, which never retires prior facts (#7013).
if (!state.managed) state.result.orphan_facts_cleaned += (await publishPageFacts(state, page.slug, pageRows, 'partial')).deleted;
// #4869: no terminal, no checkpoint — this claim did not reach a durable
// outcome, so it counts as failed (cycle 'warn'), not processed.
state.result.pages_failed++; recordFailedPage(state.result, page.slug, 'the page changed during extraction; left unfinished for replay');
return { newEndIso: null };
}
// A managed page is counted, and its checkpoint set, when its batch publishes.
if (managed && newestEnd !== null) { await enqueueManagedPage(state, snapshot, managed.start, pageRows, newestEnd, scope); return { newEndIso: null }; }
const published = await publishPageFacts(state, page.slug, pageRows, scope);
state.result.facts_inserted += published.facts_inserted;
state.result.orphan_facts_cleaned += published.deleted;
if (published.duplicates_skipped) process.stderr.write(`[extract-conversation-facts] ${page.slug}: skipped ${published.duplicates_skipped} fact(s) an active fact of the source already states (#7043)\n`);
notePurgedSkips(state.result, page.slug, published.purged_skipped);
if (published.facts_inserted + published.duplicates_skipped + published.purged_skipped < pageRows.filter(row => row.source !== TERMINAL_AUDIT_SOURCE).length) {
process.stderr.write(`[extract-conversation-facts] ${page.slug}: kept its prior facts; this ${scope} run's facts were not published over them (#7013)\n`);
}
if (newestEnd !== null) {
// v0.41.15.0 (codex #5/#6): per-page atomic checkpoint write. Mutate
// the shared Map in place — JS single-threaded event loop makes
// Map.set atomic across parallel workers; we don't need a load-mutate-
// flush race. Map serializes back to op-checkpoint string[] at batch
// boundaries via the caller's periodic recordCompleted call.
state.cpMap.set(cpMapKey(state.sourceId, page.slug), newestEnd);
}
process.stderr.write(
`[extract-conversation-facts] ${page.slug}: ${published.facts_inserted} facts inserted across ${segmentsThisPage} segments ` +
`entity_resolution_counts=${formatSaveTimeResolutionCounts(pageResolution.counts)}\n`,
);
state.result.pages_processed++;
return { newEndIso: newestEnd };
}
/** A result with every counter at zero, the starting point of a run or an aggregate. */
export function emptyExtractConversationFactsResult(): ExtractConversationFactsResult {
return {
pages_considered: 0, pages_processed: 0, pages_skipped: 0, pages_skipped_unparsed: 0, pages_skipped_type_mismatch: 0,
pages_skipped_insufficient_turns: 0, pages_skipped_since: 0, pages_skipped_too_large: 0, pages_skipped_disappeared: 0,
pages_skipped_completed: 0, pages_skipped_non_extractable: 0, pages_marked_non_extractable: 0, pages_skipped_unrecognized_speaker: 0,
pages_failed: 0, failed_pages: [], pages_llm_fallback: 0, pages_lock_skipped: 0, orphan_facts_cleaned: 0, segments_processed: 0,
facts_extracted: 0, facts_inserted: 0, fallback_slugify_count: 0, resolution_errors: 0,
};
}
/**
* Core entry point — one source per call. Caller (CLI / Minion / cycle
* phase) handles multi-source iteration externally.
*
* Budget tracker semantics:
* - If `opts.budgetTracker` is set: use it as-is (no wrap). Caller
* owns lifecycle; nested wrap would REPLACE the active tracker.
* - If absent: create a fresh tracker scoped to `opts.maxCostUsd`
* and run the body inside `withBudgetTracker`.
*/
export async function runExtractConversationFactsCore(
engine: BrainEngine,
opts: ExtractConversationFactsCoreOpts,
signal?: AbortSignal,
): Promise<ExtractConversationFactsResult> {
const sourceId = opts.sourceId;
if (!sourceId) {
throw new Error('runExtractConversationFactsCore: opts.sourceId is required');
}
const managed = await managedConversationPublisher(engine, sourceId, opts);
const result = emptyExtractConversationFactsResult();
// F2: honor brain-wide kill-switch unless overridden.
if (!opts.overrideDisabled) {
const enabled = await isFactsExtractionEnabled(engine);
if (!enabled) {
throw new Error(
'facts.extraction_enabled=false; pass --override-disabled to force-run',
);
}
}
// v0.41.15.0 (D15): preflight facts.embedding dim check. Throws a
// paste-ready ALTER hint BEFORE the first insert if the configured
// embedding_dimensions differs from the facts column width. Doctor
// also warns, but doctor-only doesn't close the bug class: new users
// who skip doctor crash on first insert with the opaque pgvector
// error. Preflight catches them up-front. Result cached per process.
if (!opts.dryRun) {
await assertFactsEmbeddingDimMatchesConfig(engine);
}
const types = await resolveTypesFromConfig(engine, opts.types);
const strictEligibility = await requireParseableConversationFlag(engine);
const factVisibility = await resolveDefaultVisibility(engine);
const dryRun = !!opts.dryRun;
const sleepMs = opts.sleepMs ?? DEFAULT_INTER_CALL_SLEEP_MS;
const segmentLimit = opts.segmentLimit ?? 0;
// v0.41.15.0 (D9): resolve effective worker count via the PGLite-clamp
// wrapper. Embedded engines silently become serial; the explicit
// override + auto-concurrency rules from sync-concurrency.ts apply on
// Postgres. Page count for the auto-path is unknown ahead of
// enumeration, so pass 0 — the wrapper falls back to override-or-1.
const workersResolved = resolveWorkersWithClamp(
engine,
opts.workers,
'extract-conversation-facts',
0,
);
const workers = workersResolved.workers;
// Privacy boundary: the parser never sends page content to an LLM unless
// this exact DB-plane key is explicitly true. Resolve the model once rather
// than probing configuration for every page.
const llmFallbackEnabled =
(await engine.getConfig('conversation_parser.llm_fallback_enabled')) === 'true';
const llmFallbackModel = llmFallbackEnabled
? await resolveModel(engine, {
tier: 'utility',
// #3813: last-resort fallback stays key-aware, never hardcoded Anthropic.
fallback: resolveTierDefault('utility'),
})
: null;
const state: ExtractCoreState = {
result,
engine,
sourceId,
managed,
dryRun,
sleepMs,
segmentLimit,
types,
factVisibility,
signal,
extractor: opts.extractor,
cpMap: new Map(),
llmFallbackModel,
};
// Run body. Either inside the externally-provided tracker scope (no
// wrap; opts.budgetTracker is in scope upstream OR caller passes it
// explicitly via withBudgetTracker), or inside a fresh local wrap.
const body = async () => {
const cpKey = checkpointKey(sourceId);
const initialEntries = await loadOpCheckpoint(engine, cpKey);
// v0.41.15.0 (codex #5/#6): hold checkpoint state as a Map so
// parallel workers (D9) can mutate it atomically per-page. Serialize
// back to op-checkpoint string[] at batch boundaries + final flush.
state.cpMap = cpEntriesToMap(initialEntries);
/**
* Wrap processPage in the per-page advisory lock (D2 + D12). The
* pool's onItem closes over this so D6's skip-and-continue semantics
* land at the right level: a lock-busy page increments the counter,
* logs once per (source, minute), and the worker claims the next
* page rather than blocking. A hard error from processPage propagates
* up; the pool's onError='continue' captures it into failures[].
*/
const claims = new ConversationRunClaims();
const processPageWithLock = async (page: Page): Promise<void> => {
if (!claims.claim(sourceId, page.slug)) {
state.result.pages_skipped_duplicate_listing = (state.result.pages_skipped_duplicate_listing ?? 0) + 1;
return;
}
const lockId = extractConversationFactsLockId(sourceId, page.slug);
if (opts.force) {
state.cpMap.delete(cpMapKey(sourceId, page.slug));
}
try {
await withRefreshingLock(
engine,
lockId,
async () => {
// Re-fetch under the advisory lock. Batch enumeration is only a
// candidate list; it must never become the snapshot we certify.
const currentPage = await engine.getPage(page.slug, { sourceId });
if (!currentPage) {
state.result.pages_skipped_disappeared++;
return { newEndIso: null };
}
// Close the race between batch selection and lock acquisition.
if (!opts.force && opts.grammar === undefined) {
const outcome = (
await findFreshExtractionOutcomes(engine, sourceId, [currentPage])
).get(currentPage.slug);
if (outcome) {
recordDurableOutcomeSkip(state, outcome);
return { newEndIso: null };
}
}
// A checkpoint without a matching durable v2 outcome cannot prove
// which page snapshot it describes. Clear it and replay safely;
// delete-orphans-first makes that replay deterministic.
state.cpMap.delete(cpMapKey(sourceId, currentPage.slug));
if (isSegmentedPage(currentPage) && !state.managed && !hasRawTranscriptSidecar(currentPage)) {
const { newestEnd } = await processSegmentedConversationPage(state, currentPage, { gapMinutes: pageSegmentGapMinutes(currentPage) ?? null, sinceIso: opts.sinceIso, retryFailed: opts.force, grammar: opts.grammar });
if (newestEnd) state.cpMap.set(cpMapKey(sourceId, currentPage.slug), newestEnd);
return { newEndIso: newestEnd };
}
const snapshot = await preparePageSnapshot(engine, currentPage);
return processPage(state, snapshot, opts.sinceIso);
},
{ ttlMinutes: PER_PAGE_LOCK_TTL_MINUTES },
).then(() => undefined);
} catch (err) {
if (err instanceof LockUnavailableError) {
// D6: skip-and-continue. Page stays in backlog; next
// enumeration picks it up after the holder's terminal row
// lands. Rate-limited log so contention doesn't spam stderr.
state.result.pages_lock_skipped++;
logLockBusyRateLimited(sourceId, page.slug);
return;
}
throw err;
}
};
// Expand logical types (conversation/meeting/slack/email) to the concrete
// `pages.type` values to enumerate, so brains on the granular collector
// types are not silently skipped (see ALLOWED_TYPE_ALIASES).
const concreteTypes = pageTypesForAllowed(types);
if (opts.slugs !== undefined) {
// Batch mode is selected by the PRESENCE of the selector: an empty
// list means "process exactly these zero pages" (a no-op), never a
// fall-through to full-corpus enumeration and its LLM spend.
for (const slug of opts.slugs) {
if (signal?.aborted) throw new Error('aborted');
const page = await engine.getPage(slug, { sourceId });
if (!page) {
result.pages_skipped_disappeared++;
continue;
}
if (!isConversationFactsEligiblePage(page, concreteTypes, strictEligibility)) {
result.pages_skipped++;
result.pages_skipped_type_mismatch++;
continue;
}
try {
await processPageWithLock(page);
} catch (error) {
if (isAbortError(error) || error instanceof BudgetExhausted || stopsRun(error)) throw error;
recordPageFailure(result, sourceId, slug, error);
}
}
} else if (opts.slug) {
const page = await engine.getPage(opts.slug, { sourceId });
if (!page) {
result.pages_skipped_disappeared++;
return;
}
if (!isConversationFactsEligiblePage(page, concreteTypes, strictEligibility)) {
result.pages_skipped++;
result.pages_skipped_type_mismatch++;
return;
}
await processPageWithLock(page);
} else {
// Multi-page enumeration: paginate per-type at small batch size to
// bound memory (Eng-v2 C8 — 10 × 25MB = 250MB worst case), by slug
// keyset so every page is listed once (#6500, conversation-enumeration.ts).
// v0.41.15.0 (D9): inner per-page loop replaced with runSlidingPool
// so parallel workers can claim pages from the batch. The pool
// honors AbortSignal at each claim boundary and threads
// BudgetExhausted abort (D13) automatically.
let processedPagesCount = 0;
pageLoop: for (const type of concreteTypes) {
for await (const batch of listConversationPagesBySlug(engine, { type, sourceId, batchSize: PAGE_LIST_BATCH })) {
if (signal?.aborted) throw new Error('aborted');
if (opts.limit && processedPagesCount >= opts.limit) break pageLoop;
let claimable = batch.filter(page => isConversationFactsEligiblePage(page, concreteTypes, strictEligibility));
result.pages_skipped += batch.length - claimable.length;
// Checkpoints are an intra-page cursor; fresh durable outcomes are
// the page-level selection authority and survive checkpoint GC.
if (!opts.force && claimable.length > 0) {
const fresh = await findFreshExtractionOutcomes(
engine,
sourceId,
claimable,
);
claimable = claimable.filter((page) => {
const outcome = fresh.get(page.slug);
if (!outcome) return true;
recordDurableOutcomeSkip(state, outcome);
return false;
});
}
// Apply --limit after durable filtering. The limit caps pages that
// need work, not already-completed pages scanned to find that work.
if (opts.limit) {
const remaining = opts.limit - processedPagesCount;
if (remaining < claimable.length) {
claimable = claimable.slice(0, remaining);
}
}
const poolResult = await runSlidingPool({
items: claimable,
workers,
signal,
onItem: (page) => processPageWithLock(page),
onError: (error) => (isAbortError(error) || stopsRun(error) ? 'abort' : 'continue'),
failureLabel: (page) => page.slug,
});
const cancellation = poolResult.failures.find((failure) =>
isAbortError(failure.error) || stopsRun(failure.error),
);
if (cancellation) throw cancellation.error;
if (signal?.aborted) {
if (signal.reason instanceof Error) throw signal.reason;
throw Object.assign(new Error('caller cancelled'), {
name: 'AbortError',
});
}
for (const failure of poolResult.failures) recordPageFailure(result, sourceId, failure.label, failure.error);
processedPagesCount += claimable.length;
// Persist checkpoint between batches so a crash mid-walk
// doesn't lose all progress.
if (!dryRun) {
await recordCompleted(engine, checkpointKey(sourceId), cpMapToEntries(state.cpMap));
}
}
}
}
// The last partial batch publishes before the final checkpoint flush.
await flushManagedBatch(state);
if (!dryRun) {
await recordCompleted(engine, checkpointKey(sourceId), cpMapToEntries(state.cpMap));
}
};
let ownedTracker: BudgetTracker | null = null;
try {
if (opts.budgetTracker) {
// Caller-managed scope — use as-is, no wrap (nested wrap REPLACES
// tracker per gateway.ts AsyncLocalStorage semantics).
await body();
} else {
const pricingOverrides = await loadPricingOverrides(engine);
const tracker = new BudgetTracker({
maxCostUsd: await conversationFactsCostCap(engine, opts.maxCostUsd ?? DEFAULT_MAX_COST_USD, opts.maxCostUsd !== undefined, pricingOverrides),
label: `extract-conversation-facts:${sourceId}`,
pricingOverrides,
});
ownedTracker = tracker;
try {
await withBudgetTracker(tracker, body);
} finally {
result.spent_usd = tracker.totalSpent;
}
}
} catch (err) {
await flushManagedBatch(state, { afterError: true }); // spent model work still publishes
if (err instanceof BudgetExhausted) {
Object.assign(result, { budget_exhausted: true, budget_reason: err.reason, budget_model: err.modelId });
if (err.pricing) result.budget_pricing = err.pricing;
if (opts.budgetTracker) {
result.spent_usd = opts.budgetTracker.totalSpent;
}
// Fall through to receipt+rollup write so the partial run is
// still observable in extract_health doctor + extracts/ pages.
// ...but not under --dry-run: a preview must not persist cache state.
if (!dryRun) await writeRunReceiptAndRollup(engine, sourceId, result, /* halted */ true);
// Return partial result — caller (CLI / Minion) decides how to
// surface. NOT a thrown failure.
return result;
}
opts.onInterrupted?.(structuredClone(result));
throw err;
}
// gateway.chat preserves a successful provider result when the final
// tracker.record() discovers an underestimated overage. Usually the next
// reserve surfaces it, but a fallback that yields fewer than two messages
// has no next call. Detect that terminal overage so the result and rollup
// remain honest.
const effectiveTracker = opts.budgetTracker ?? ownedTracker;
if (
effectiveTracker?.cap !== undefined &&
effectiveTracker.totalSpent > effectiveTracker.cap
) {
result.budget_exhausted = true;
result.budget_reason = 'cost';
result.spent_usd = effectiveTracker.totalSpent;
}
// v0.42 — Wave B1: extract-conversation-facts writes a receipt page
// (queryable + citable per D-EXTRACT-17/19) AND UPSERTs the per-day
// rollup row (best-effort cache per F-OUT-19). Both are best-effort —
// failures stderr-warn but never fail the parent operation.
// --dry-run must not persist cache/knowledge state: skip the rollup UPSERT +
// receipt-page write so a preview leaves no extract cache row behind.
if (!dryRun) {
await writeRunReceiptAndRollup(
engine,
sourceId,
result,
/* halted */ result.budget_exhausted === true,
);
}
return result;
}
/**
* v0.42 — Wave B1: best-effort receipt + rollup writes at the end of an
* extract-conversation-facts run. Skips the receipt page when the run
* extracted ZERO facts (no-op runs don't need brain memory) but always
* UPSERTs the rollup row so doctor sees the cycle ran.
*
* `halted` true means the run hit a budget cap mid-flight; receipt
* carries that state in its frontmatter (round='full' regardless; the
* halt is recorded as a halt_delta=1 in the rollup table).
*/
async function writeRunReceiptAndRollup(
engine: BrainEngine,
sourceId: string,
result: ExtractConversationFactsResult,
halted: boolean,
): Promise<void> {
const now = new Date().toISOString();
// run_id: stable-ish identifier for this run. Includes day so multiple
// runs of the same source on different days don't collide on the
// receipt slug. shortRunId() truncates to 8 chars.
const runId = `ecf-${Date.now().toString(36)}-${sourceId.slice(0, 4)}`;
// Receipt write: only when the run actually inserted facts (receipt pages are unmanaged-only, like extract_atoms).
if (result.facts_inserted > 0 && !await managedPersistenceEnabled(engine)) {
try {
await writeReceipt(engine, {
kind: 'facts.conversation',
source_id: sourceId,
run_id: runId,
round: 'full',
extracted_at: now,
total_rows: result.facts_inserted,
cost_usd: result.spent_usd ?? 0,
summary:
`Extracted ${result.facts_inserted} facts from ` +
`${result.pages_processed}/${result.pages_considered} eligible pages` +
(result.pages_failed > 0
? `; ${result.pages_failed} page(s) failed and remain unfinished.`
: '.'),
});
} catch (err) {
// Best-effort: receipt write failure shouldn't kill the run.
// The audit trail lives in the facts table (terminal rows) +
// optionally the new audit JSONL once wired.
const msg = (err as Error).message || String(err);
console.error(`[extract-conversation-facts] receipt write failed: ${msg}`);
}
}
// Rollup UPSERT: ALWAYS fire so doctor's extract_health sees the
// cycle ran (even no-op runs are signal — they prove the extractor
// was alive). Best-effort per F-OUT-19.
//
// #4482: a run that stopped ONLY because it hit its per-source budget cap
// is working as designed (partial progress banked; the backlog drains over
// future runs) — record it as expected_limit_delta, not halt_delta, so
// doctor's extract_health failure rate stops warning on normal
// bigger-backlog-than-budget operation. Per-page failures stay error halts.
await upsertExtractRollup(engine, {
kind: 'facts.conversation',
source_id: sourceId,
cost_delta: result.spent_usd ?? 0,
...classifyRunStop({
budget_exhausted: halted,
error: result.budget_reason === 'no_pricing' ? 'no_pricing' : result.pages_failed > 0 ? 'item_error' : false,
}),
});
}
// ---------------------------------------------------------------------------
// CLI parsing + handler.
// ---------------------------------------------------------------------------
interface ParsedArgs {
sourceId?: string;
types?: AllowedType[];
slug?: string;
/** Exactly these pages (comma-separated `--slugs`; `gbrain repair conversation-labels` prints it). */
slugs?: string[];
dryRun?: boolean;
limit?: number;
sinceIso?: string;
force?: boolean;
sleepMs?: number;
segmentLimit?: number;
maxCostUsd?: number;
overrideDisabled?: boolean;
/** v0.41.15.0 (D9): in-process parallel workers per source. */
workers?: number;
/** PR 4 (Wave B incremental derivation, transcripts): --grammar <format|none> for the one --slug page. */
grammar?: string | null;
yes?: boolean;
json?: boolean;
help?: boolean;
error?: string;
}
export function parseArgs(args: string[]): ParsedArgs {
const out: ParsedArgs = {};
for (let i = 0; i < args.length; i++) {
const a = args[i];
if (a === '--help' || a === '-h') { out.help = true; continue; }
if (a === '--dry-run') { out.dryRun = true; continue; }
if (a === '--json') { out.json = true; continue; }
if (a === '--force') { out.force = true; continue; }
if (a === '--yes' || a === '-y') { out.yes = true; continue; }
if (a === '--override-disabled') { out.overrideDisabled = true; continue; }
if (a === '--slug') { out.slug = args[++i]; continue; }
if (a === '--slugs') { out.slugs = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean); continue; }
if (a === '--grammar') {
const v = args[++i] ?? '';
if (v !== 'none' && !BUILTIN_PATTERNS.some(p => p.id === v)) {
out.error = `--grammar takes none or a conversation format: ${BUILTIN_PATTERNS.map(p => p.id).join(', ')}`;
return out;
}
out.grammar = v === 'none' ? null : v;
continue;
}
if (a === '--source-id') { out.sourceId = args[++i]; continue; }
if (a === '--since') { out.sinceIso = args[++i]; continue; }
if (a === '--types') {
const v = args[++i] ?? '';
const parts = v.split(',').map((s) => s.trim()).filter(Boolean);
const bad = parts.filter((p) => !(ALLOWED_TYPES as readonly string[]).includes(p));
if (bad.length > 0) {
out.error = `Unknown type(s) in --types: ${bad.join(', ')}. Allowed: ${ALLOWED_TYPES.join(', ')}`;
return out;
}
out.types = parts as AllowedType[];
continue;
}
// #5934 (D4): strict values; a bad one is a usage error (exit 2), never silently ignored.
if (a === '--limit') { out.limit = intFlagValue(args[++i], '--limit', { min: 1, example: 100 }); continue; }
if (a === '--sleep') { out.sleepMs = intFlagValue(args[++i], '--sleep', { min: 0, example: 500 }); continue; }
if (a === '--segment-limit') { out.segmentLimit = intFlagValue(args[++i], '--segment-limit', { min: 0, example: 50 }); continue; }
if (a === '--max-cost-usd') {
const n = Number(args[++i]);
if (!Number.isFinite(n) || n <= 0) {
out.error = '--max-cost-usd requires a positive finite number.';
return out;
}
out.maxCostUsd = n;
continue;
}
if (a === '--workers' || a === '--concurrency') {
try {
out.workers = parseWorkers(args[++i]);
} catch (e) {
out.error = (e as Error).message;
return out;
}
continue;
}
if (a.startsWith('--')) {
out.error = `Unknown flag: ${a}`;
return out;
}
}
if (out.grammar !== undefined && (!out.slug || out.slugs?.length)) out.error = '--grammar chooses the format of one page: pass it with --slug <slug>';
if (out.sinceIso) {
const ms = Date.parse(out.sinceIso);
if (!Number.isFinite(ms)) {
out.error = `Invalid --since: ${out.sinceIso}`;
}
}
return out;
}
const HELP = `Usage: gbrain extract-conversation-facts [options]
Batch-extract facts from conversation pages (and adjacent long-form
types: meeting, slack, email) into the facts table. Each page is parsed
into time-windowed segments and passed through the shared fact extractor
with a topical/temporal context header so the resulting facts retain
anchor terms ("Conversation between A and B on DATE …") that the
chunk-level embedding loses on long conversations.
Options:
--source-id <id> Source to operate on (default: 'default').
--types <list> Comma-separated subset of: ${ALLOWED_TYPES.join(', ')}.
Default: reads cycle.conversation_facts_backfill.types config
(falls back to the full allowlist).
--slug <slug> Process a single page (overrides multi-page enumeration).
--slugs <a,b,...> Process exactly these pages (comma-separated).
--grammar <format|none> With --slug (and --source-id when the slug is in more than one
source): extract that one page under this conversation format
(none: the page is not a transcript; it settles as not
extractable and its conversation facts are removed). For a page
held because no format is detected over it (gbrain doctor
conversation_detection_ambiguous). Preview with --dry-run first.
The choice holds until the page changes; then it is detected
again and, when still undetected, held for a new choice.
--dry-run Show segmentation + counts; no model calls, DB writes, or checkpoint advance.
--limit <N> Cap pages processed (default: all).
--since <iso> Only consider messages newer than this ISO timestamp.
--force Re-process the target page (clears its resume entry).
--sleep <ms> Delay between extractor calls (default ${DEFAULT_INTER_CALL_SLEEP_MS}).
--segment-limit <N> Max segments per page (0 = unlimited).
--max-cost-usd <FLOAT> Cost cap for this run (default ${DEFAULT_MAX_COST_USD}).
NOTE: under --workers N, the cap can be exceeded by up to
N × per-page-cost because per-worker reserve() calls aren't
serialized. At workers=20 × ~$0.02/page that's ~$0.40 over.
Pin --workers 1 if you need exact-ceiling compliance.
--workers N Parallel page workers within a single source. Default 1.
Recommended 5-20 for LLM-bound work on Postgres. PGLite
silently clamps to 1 (single-writer engine). Cross-process
safety is guaranteed by the per-page advisory lock + replay
safety (delete-orphans-first on each page claim).
--override-disabled Bypass facts.extraction_enabled=false brain-wide kill-switch.
--background Submit as a Minion job; print job_id; exit (use 'gbrain jobs follow').
--json Emit the run counters as one JSON object on stdout (diagnostics stay on stderr).
--yes Auto-confirm cost preview in non-TTY contexts.
--help, -h Show this help.
Exit status: 0 when the run finished, including a partial run where some pages
failed (they are listed, stay unfinished and retry on the next run; --json
reports status "partial" with failed_pages). 1 when every attempted page failed,
a model has no pricing, or the gateway is unavailable.
Multi-source: when --source-id is omitted, the command iterates ALL
sources from gbrain sources list. Per-source budget cap defaults to
--max-cost-usd; the brain-wide cap when running via the autopilot cycle
phase is cycle.conversation_facts_backfill.max_total_cost_usd.
Default USD caps are not enforced for unpriced chat models (a warning is emitted).
Explicit caps remain fail-closed; register a missing price with gbrain pricing set <model> --input <usd-per-1M> --output <usd-per-1M>.
Resumability: per-page completion is durable via a terminal audit row
in the facts table (source='${TERMINAL_AUDIT_SOURCE}'). gbrain doctor's
conversation_facts_backlog check counts pages without this row.
`;
/**
* PR 4 (Wave B incremental derivation, transcripts): the one source whose page --grammar applies to: --source-id, else the
* only source holding the slug (a slug is not a page identity across sources, so two refuse). The page must be stored in
* segments on an unmanaged brain. Anything else is refused before any work.
*/
async function resolveGrammarSource(engine: BrainEngine, explicit: string | undefined, slug: string): Promise<string> {
if (await managedPersistenceEnabled(engine)) throw usageError('--grammar applies to an unmanaged brain: a managed brain extracts a conversation page whole', ECF_HELP_HINT);
const holders = explicit ? [explicit] : (await engine.executeRaw<{ source_id: string }>(
'SELECT source_id FROM pages WHERE slug = $1 AND deleted_at IS NULL ORDER BY source_id', [slug])).map(r => r.source_id);
if (holders.length !== 1) {
const commands = holders.map(s => `gbrain extract-conversation-facts --source-id ${s} --slug ${slug} --grammar <format|none> --dry-run`);
throw usageError(holders.length ? `--grammar chooses the format of one page, and ${slug} is in ${holders.length} sources (${holders.join(', ')}): pass --source-id`
: `--grammar: no page ${slug} in any source`, holders.length ? commands.join('; ') : ECF_HELP_HINT, { reason: holders.length ? 'grammar_source_ambiguous' : 'grammar_page_not_found' });
}
const page = await engine.getPage(slug, { sourceId: holders[0]! });
if (page && (!isSegmentedPage(page) || hasRawTranscriptSidecar(page))) throw usageError(`--grammar applies to a conversation page stored in segments; ${holders[0]}:${slug} is extracted whole`, ECF_HELP_HINT);
return holders[0]!;
}
function buildJobParams(args: string[]): Record<string, unknown> {
const parsed = parseArgs(args);
if (parsed.error) throw usageError(parsed.error, ECF_HELP_HINT);
if (parsed.grammar !== undefined) throw usageError('--grammar runs in the foreground: drop --background', ECF_HELP_HINT);
return {
sourceId: parsed.sourceId,
types: parsed.types,
slug: parsed.slug,
slugs: parsed.slugs,
dryRun: parsed.dryRun,
limit: parsed.limit,
sinceIso: parsed.sinceIso,
force: parsed.force,
sleepMs: parsed.sleepMs,
segmentLimit: parsed.segmentLimit,
maxCostUsd: parsed.maxCostUsd,
overrideDisabled: parsed.overrideDisabled,
// v0.41.15.0 (D9): thread workers through the Minion job envelope
// so `gbrain extract-conversation-facts --background --workers 20`
// round-trips. The handler in src/commands/jobs.ts reads
// job.data.workers and passes to runExtractConversationFactsCore.
workers: parsed.workers,
};
}
export async function runExtractConversationFacts(
engine: BrainEngine,
args: string[],
): Promise<void> {
// --help short-circuit.
if (args.includes('--help') || args.includes('-h')) {
console.log(HELP);
return;
}
// --background path.
const backgrounded = await maybeBackground({
engine,
args,
jobName: 'extract-conversation-facts',
paramBuilder: buildJobParams,
});
if (backgrounded) return;
const parsed = parseArgs(args);
if (parsed.error) throw usageError(parsed.error, ECF_HELP_HINT);
// Chat gateway is required for non-dry-run. Recover a cold singleton before
// reporting an availability error (#2590).
if (!parsed.dryRun && parsed.grammar !== null && !isAvailable('chat')) configureGatewayIfUninitialized();
if (!parsed.dryRun && parsed.grammar !== null && !isAvailable('chat')) {
console.error(
'Chat gateway unavailable. Set a provider key (OPENAI_API_KEY or ANTHROPIC_API_KEY — ' +
'extraction routes to whichever is present), or configure a model explicitly ' +
'(`gbrain config set facts.extraction_model <provider:model>`), or pass --dry-run to ' +
'preview segmentation. Keyless brains capture memory via agent-authored `## Facts` ' +
'fences and the `remember` verb instead.',
);
process.exit(1);
}
// Aggregate result across all sources.
const aggregate = emptyExtractConversationFactsResult();
let totalSpent = 0;
let anyBudgetExhausted = false;
const unpricedModels = new Set<string>();
const pricingGuidance = new Map<string, NoPricingGuidance>();
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
// Multi-source enumeration when --source-id NOT set.
const sourceIds: string[] = parsed.grammar !== undefined ? [await resolveGrammarSource(engine, parsed.sourceId, parsed.slug!)]
: parsed.sourceId ? [parsed.sourceId] : (await listSources(engine)).map((s) => s.id);
progress.start('extract.conversation_facts', sourceIds.length);
try {
for (const sourceId of sourceIds) {
const perSource = await runExtractConversationFactsCore(engine, {
sourceId,
types: parsed.types,
slug: parsed.slug,
slugs: parsed.slugs,
dryRun: parsed.dryRun,
limit: parsed.limit,
sinceIso: parsed.sinceIso,
force: parsed.force,
sleepMs: parsed.sleepMs,
segmentLimit: parsed.segmentLimit,
maxCostUsd: parsed.maxCostUsd,
overrideDisabled: parsed.overrideDisabled,
workers: parsed.workers,
grammar: parsed.grammar,
});
aggregate.pages_considered += perSource.pages_considered;
aggregate.pages_processed += perSource.pages_processed;
aggregate.pages_skipped += perSource.pages_skipped;
aggregate.pages_skipped_unparsed += perSource.pages_skipped_unparsed;
aggregate.pages_skipped_type_mismatch += perSource.pages_skipped_type_mismatch;
aggregate.pages_skipped_insufficient_turns += perSource.pages_skipped_insufficient_turns;
aggregate.pages_skipped_since += perSource.pages_skipped_since;
aggregate.pages_skipped_too_large += perSource.pages_skipped_too_large;
aggregate.pages_skipped_disappeared += perSource.pages_skipped_disappeared;
aggregate.pages_skipped_completed += perSource.pages_skipped_completed;
aggregate.pages_skipped_non_extractable += perSource.pages_skipped_non_extractable;
aggregate.pages_marked_non_extractable += perSource.pages_marked_non_extractable;
aggregate.pages_skipped_unrecognized_speaker += perSource.pages_skipped_unrecognized_speaker;
aggregate.pages_failed += perSource.pages_failed;
for (const failed of perSource.failed_pages) recordFailedPage(aggregate, failed.slug, failed.error);
aggregate.pages_llm_fallback += perSource.pages_llm_fallback;
aggregate.pages_lock_skipped += perSource.pages_lock_skipped;
aggregate.orphan_facts_cleaned += perSource.orphan_facts_cleaned;
aggregate.segments_processed += perSource.segments_processed;
aggregate.facts_extracted += perSource.facts_extracted;
aggregate.facts_inserted += perSource.facts_inserted;
aggregate.fallback_slugify_count += perSource.fallback_slugify_count; if (perSource.unbacked_keys) aggregate.unbacked_keys = (aggregate.unbacked_keys ?? 0) + perSource.unbacked_keys;
aggregate.resolution_errors += perSource.resolution_errors;
if (perSource.slot_tagging_failed) aggregate.slot_tagging_failed = (aggregate.slot_tagging_failed ?? 0) + perSource.slot_tagging_failed;
if (perSource.pages_pending) aggregate.pages_pending = (aggregate.pages_pending ?? 0) + perSource.pages_pending;
if (perSource.pages_blocked) aggregate.pages_blocked = (aggregate.pages_blocked ?? 0) + perSource.pages_blocked;
if (perSource.pages_detection_ambiguous) aggregate.pages_detection_ambiguous = (aggregate.pages_detection_ambiguous ?? 0) + perSource.pages_detection_ambiguous;
if (perSource.grammar_previews) (aggregate.grammar_previews ??= []).push(...perSource.grammar_previews);
if (perSource.budget_exhausted) anyBudgetExhausted = true;
if (perSource.budget_reason === 'no_pricing') unpricedModels.add(perSource.budget_model ?? 'unknown model');
if (perSource.budget_pricing) pricingGuidance.set(perSource.budget_pricing.model, perSource.budget_pricing);
if (perSource.spent_usd) totalSpent += perSource.spent_usd;
progress.tick(1, `${sourceId}: ${perSource.facts_inserted} facts inserted`);
}
} finally {
progress.finish();
}
const outcome = parsed.dryRun
? '(dry run) segmentation only; no facts extracted'
: `extracted ${aggregate.facts_extracted} facts (${aggregate.facts_inserted} inserted)`;
// #5448: --json is a universal cli-flag-registry flag, so scripts add it
// expecting an envelope. Emit the same counters as the Done: summary as one
// JSON object on stdout; progress and diagnostics stay on stderr.
const runStatus = extractRunStatus(aggregate);
const estimate = parsed.dryRun ? await conversationRunEstimate(engine, aggregate.segments_processed) : null;
if (parsed.json) {
console.log(JSON.stringify({
...aggregate, ...(estimate ? { estimate } : {}),
status: runStatus,
...(aggregate.pages_failed > 0 ? { next: { action: 'rerun', why: 'Failed pages stay unfinished and are retried by re-running the same command; pages that succeeded are not redone.' } } : {}),
sources: sourceIds,
dry_run: parsed.dryRun ?? false,
outcome,
spent_usd: totalSpent,
budget_exhausted: anyBudgetExhausted,
no_pricing_models: [...unpricedModels],
no_pricing: [...pricingGuidance.values()],
}, null, 2));
} else {
console.log(
`\nDone: ${outcome} across ${aggregate.segments_processed} segments ` +
`from ${aggregate.pages_processed}/${aggregate.pages_considered} pages ` +
`in ${sourceIds.length} source(s). ` +
`Spent ~$${totalSpent.toFixed(4)}.`,
);
if (parsed.dryRun && parsed.slugs?.length) console.log(` Targets: ${parsed.slugs.join(', ')}`); // #6764: the preview names what the paid run would re-extract
if (estimate) console.log(conversationEstimateLine(estimate));
if (aggregate.pages_skipped > 0) {
console.log(` Skipped ${aggregate.pages_skipped} page(s) without eligible segments or outside the selected types:`);
console.log(` ${aggregate.pages_skipped_unparsed} with no parseable speaker turns (retryable); ${aggregate.pages_skipped_type_mismatch} with a type mismatch; ${aggregate.pages_skipped_insufficient_turns} with insufficient turns; ${aggregate.pages_skipped_since} with no eligible segments after --since; ${aggregate.pages_skipped_unrecognized_speaker} declined for speaker attribution.`);
}
if (aggregate.pages_skipped_too_large > 0) {
console.log(` Skipped ${aggregate.pages_skipped_too_large} page(s) exceeding ${MAX_PAGE_BODY_BYTES / 1024 / 1024}MB body cap.`);
}
if (aggregate.pages_skipped_disappeared > 0) {
console.log(` Skipped ${aggregate.pages_skipped_disappeared} page(s) that disappeared between enumeration and fetch.`);
}
if (aggregate.pages_skipped_completed > 0) {
console.log(` Skipped ${aggregate.pages_skipped_completed} page(s) with fresh durable completion outcomes.`);
}
if (aggregate.pages_skipped_non_extractable > 0) {
console.log(` Skipped ${aggregate.pages_skipped_non_extractable} page(s) previously scanned as not extractable.`);
}
if (aggregate.pages_skipped_unrecognized_speaker > 0) {
console.log(` Declined ${aggregate.pages_skipped_unrecognized_speaker} page(s) with unrecognized speaker headings (attribution would be wrong; retried next run).`);
}
if (aggregate.pages_marked_non_extractable > 0) {
console.log(` Marked ${aggregate.pages_marked_non_extractable} page(s) as scanned, not extractable.`);
}
if (aggregate.pages_failed > 0) {
console.error(` Failed ${aggregate.pages_failed} page(s); they remain unfinished and will retry. Re-run the same command to retry them:`);
for (const failed of aggregate.failed_pages) console.error(` ${failed.slug}: ${failed.error}`);
}
if (aggregate.pages_pending) console.log(` ${aggregate.pages_pending} page(s) were accepted and are still pending; rerun the same command to confirm them (no model call).`);
if (aggregate.pages_detection_ambiguous) console.log(` ${aggregate.pages_detection_ambiguous} page(s) are held: no conversation format is detected over them, yet some parse messages, so their facts were kept and nothing was extracted; gbrain doctor conversation_detection_ambiguous lists each with its formats, and --source-id <source> --slug <slug> --grammar <format|none> --dry-run previews a choice.`);
for (const p of aggregate.grammar_previews ?? []) console.log(` Preview of ${p.source_id}:${p.slug} under --grammar ${p.grammar ?? 'none'}: ${p.messages} message(s) in ${p.windows} window(s), ${p.windows_to_extract} to extract; a settled run removes ${p.facts_to_collect} conversation fact(s).`);
if (aggregate.pages_blocked) console.log(` ${aggregate.pages_blocked} page(s) are blocked at their current version after a request failed; they are retried only once the page changes (each SKIP line names the receipt).`);
if (aggregate.pages_llm_fallback > 0) {
console.log(` Parsed ${aggregate.pages_llm_fallback} page(s) with the opt-in LLM fallback.`);
}
if (aggregate.pages_lock_skipped > 0) {
console.log(` Skipped ${aggregate.pages_lock_skipped} page(s) held by another worker / process (will retry next run).`);
}
if (aggregate.orphan_facts_cleaned > 0) {
console.log(` Cleaned ${aggregate.orphan_facts_cleaned} orphan fact(s) from prior partial runs (D11 replay safety).`);
}
if (aggregate.fallback_slugify_count > 0) {
console.log(` Preserved ${aggregate.fallback_slugify_count - (aggregate.unbacked_keys ?? 0)} fact(s) without an entity target after unresolved fallback_slugify${aggregate.unbacked_keys ? `; kept ${aggregate.unbacked_keys} under a name key with no page (facts.unbacked_entity_keys)` : ''}.`);
}
if (aggregate.resolution_errors > 0) {
console.log(` Preserved ${aggregate.resolution_errors} fact(s) without an entity target after best-effort resolution errors.`);
}
if (aggregate.slot_tagging_failed) console.log(` Kept the facts of ${aggregate.slot_tagging_failed} window(s) without slots after slot tagging failed; --force tags them again.`);
if (anyBudgetExhausted) {
console.log(unpricedModels.size > 0
? ` no_pricing: ${[...unpricedModels].join(', ')}. Raising --max-cost-usd cannot resolve missing pricing.`
: ` Budget cap reached. Re-run with a higher --max-cost-usd to continue.`);
for (const g of pricingGuidance.values()) console.log(` ${noPricingMessage(g)}`);
}
}
// v0.41.15.0 (codex #3): pages skipped due to lock-busy AND no hard
// failures fired: "incomplete run, please re-run". Exit 1 (retryable) —
// 3 is reserved for confirmation_required under the agent contract v1.
// anyBudgetExhausted doesn't trigger exit 3; the budget message
// above already tells the user what to do, and exit 0 is the right
// signal for "ran to the cap intentionally."
// Partial success (some pages extracted, some failed) exits 0: the failed
// pages are listed above / in the JSON envelope and retry on the next run.
// Nonzero is reserved for a run where every attempted page failed, or for
// missing model pricing (a configuration problem a re-run cannot fix).
if (extractExitCode(aggregate, unpricedModels.size) !== 0) {
process.exit(1);
}
if (aggregate.pages_lock_skipped > 0 && !anyBudgetExhausted) {
console.error(`${aggregate.pages_lock_skipped} page(s) were skipped because another writer held their lock; re-run the same command to finish them (retryable).`);
process.exit(FAILED_EXIT_CODE);
}
}
// ---------------------------------------------------------------------------
// Helpers.
// ---------------------------------------------------------------------------
function pickLaterIso(
a: string | null | undefined,
b: string | null | undefined,
): string | undefined {
const av = a ? Date.parse(a) : NaN;
const bv = b ? Date.parse(b) : NaN;
if (Number.isFinite(av) && Number.isFinite(bv)) return av >= bv ? a! : b!;
if (Number.isFinite(av)) return a!;
if (Number.isFinite(bv)) return b!;
return undefined;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// The log names only the closed extraction reason, never provider or error text.
function recordPageFailure(result: ExtractConversationFactsResult, sourceId: string, slug: string, error: unknown): void {
result.pages_failed++;
recordFailedPage(result, slug, error instanceof Error ? error.message : String(error));
const reason = (error as { extractionReason?: ExtractFailureReason } | null)?.extractionReason ?? 'page_error';
process.stderr.write(`[extract-conversation-facts] ${slug} failed (${reason}) and stays unfinished; retry: gbrain extract-conversation-facts --source-id ${sourceId} --slug ${slug}\n`);
}
export function isAbortError(err: unknown): boolean {
if (!(err instanceof Error)) return false;
return err.name === 'AbortError' || /aborted|cancell?ed/i.test(err.message);
}