Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.40.3.0 — contextual retrieval service (D27 P1-1 + D26 P0-2 +
* D26 P0-3 + D27 P1-3 + D27 P2-2).
*
* Single source of truth for "re-embed one page with the active CR mode."
* Composed by THREE consumers:
*
* 1. `src/core/import-file.ts` — sync-time re-embed during `gbrain sync`
* (synchronous, inline).
* 2. `src/commands/reindex.ts` — `gbrain reindex --markdown` batch sweep
* (per-page loop, synchronous within the command process).
* 3. `src/core/minions/handlers/contextual-reindex-per-chunk.ts` — Minion
* job handler for the post-upgrade-prompt-accepted backfill (async,
* worker-pool concurrency with global rate-leasing).
*
* The service runs the D26 P0-2 two-phase build pattern. PHASE 1 collects
* all synopses + embeddings IN MEMORY. If any chunk triggers a refusal /
* empty / malformed response (per D14 + D27 P1-2), PHASE 1 RESTARTS at
* the lower 'title' tier — discarding the in-progress synopsis vectors so
* the page's chunks NEVER end up mid-state on disk. Only after PHASE 1
* completes successfully does PHASE 2 run a single DB transaction to
* update the captured chunks' vectors + stamp `contextual_retrieval_mode` +
* `corpus_generation`. Crash anywhere in PHASE 1 = page-level retry with
* zero half-state. Crash during PHASE 2 = standard transaction rollback.
*
* Transient failures (auth / rate_limit / timeout / network / provider_5xx
* per D27 P1-2) are NOT swallowed into a fall-back — they throw to the
* caller's retry loop (Minion handler retries the job; inline callers
* propagate the throw up).
*
* Per D27 P2-2 the embedBatch call runs ONCE per page after the per-chunk
* synopsis loop completes, not per-chunk. Saves per-call overhead + reduces
* failure surface.
*
* Rate-leasing is the caller's responsibility (D26 P0-3 — the Minion
* handler acquires a shared, resolved-model-specific synopsis lease per chunk
* before invoking the service's optional `acquireSynopsisLease`
* / `releaseSynopsisLease` hooks). Inline callers (import-file, reindex
* command) pass no hooks and rely on the gateway's own rate-limit retry.
*/
import { createHash } from 'crypto';
import * as fs from 'node:fs';
import { embedBatch } from './embedding.ts';
import { resolveContextualRetrievalMode } from './contextual-retrieval-resolver.ts';
import {
buildContextualPrefix,
modeRequiresSynopsis,
modeRequiresWrapper,
wrapChunkForEmbedding,
} from './embedding-context.ts';
import {
DEFAULT_SYNOPSIS_MODEL,
estimateSynopsisCallUsd,
generatePerChunkSynopsis,
segmentSynopsisWindows,
SYNOPSIS_MAX_TOKENS,
SYNOPSIS_PROMPT_VERSION,
SYNOPSIS_DOC_MAX_CHARS,
type GeneratePerChunkSynopsisResult,
} from './page-summary.ts';
import type { SynopsisFailureKind } from './audit-synopsis.ts';
import { runSlidingPool } from './worker-pool.ts';
import type { BrainEngine } from './engine.ts';
import type { ChunkInput, CRMode, Page } from './types.ts';
import type { SourceRow } from './sources-ops.ts';
import { embeddingInputContext, embeddingWriteTarget, installPageEmbeddings, readProjectionSnapshot, type ProjectionSnapshot } from './page-state/projections.ts';
import { reconcileContextualEmbeddingInputs } from './page-state/contextual-proof.ts';
import { digest, sha256 } from './persistence/digest.ts';
import { isLargePageBytes } from './page-size-policy.ts';
import { isSegmentedPage, pageBodyBytes } from './page-body/page-shape.ts';
import { installSegmentedVectors, listLiveEmbedSegments, listSegmentChunkRows, readSegmentedEmbedPage } from './page-body/segmented-embedding.ts';
import { acquirePageEmbeddingLease, pageEmbeddingLeaseUnavailable } from './page-embedding-lease.ts';
import { planSegmentDerivations, segmentDerivationContextHash } from './page-body/derivations.ts';
import { embeddingInputHash, synopsisDocumentHash, type EmbeddingTier } from './embedding-input-hash.ts';
import { currentEmbeddingSignature } from './embedding.ts';
import { estimatePageEmbedUsd, meterPaidCall } from './page-embedding-budget.ts';
import { segmentSynopsisLead } from './page-body/segment-synopsis.ts';
/**
* v3 = chunks embed with optional contextual retrieval wrapper. The
* wrapper text shape itself is also versioned so future format tweaks
* (e.g. switching to `<page_context>` instead of `<context>`, or adding
* a structured `metadata:` block) invalidate prior embeddings via the
* corpus_generation hash.
*/
export const TITLE_WRAPPER_VERSION = 1;
export const DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY = 4;
export const MAX_CONTEXTUAL_CHUNK_CONCURRENCY = 16;
export function resolveContextualChunkConcurrency(
env: Record<string, string | undefined> = process.env,
): number {
const raw = env.GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY;
if (raw === undefined || raw.trim() === '') return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
const n = Number(raw);
if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
return clampContextualChunkConcurrency(n);
}
function clampContextualChunkConcurrency(n: number): number {
if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
return Math.max(1, Math.min(MAX_CONTEXTUAL_CHUNK_CONCURRENCY, Math.trunc(n)));
}
/**
* Embedding model placeholder. The actual model name lands here from
* `gateway.getEmbeddingModelName()` at runtime; for hash composition we
* just need the SAME string the gateway threads into embedBatch.
*/
function getEmbeddingModelTag(): string {
// Lazy — keeps the hash composition pure-ish without forcing the
// gateway init at module load. Falls back to a sentinel when the
// gateway hasn't been configured (test paths).
try {
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { getEmbeddingModelName } = require('./ai/gateway.ts');
return String(getEmbeddingModelName() ?? 'unknown');
} catch {
return 'unknown';
}
}
/**
* Compose the corpus_generation hash per D27 P1-5. Per-chunk synopsis
* generations fold in the prompt version + synopsis model; title-only
* generations retain the historical default-model identity because they do
* not call a synopsis model. Both include the wrapper version + embedding
* model so a relevant tweak invalidates prior cache rows via the
* `query_cache.page_generations` LEFT JOIN.
*
* Pure function — `embedding_dimensions` and `embedding_column` stay in
* the existing KNOBS_HASH_VERSION space per A6 in the eng-review pass.
*/
export type ComputeCorpusGenerationArgs = {
crMode: CRMode;
/**
* Resolved `SYNOPSIS_DOC_MAX_CHARS` for per_chunk_synopsis runs. When
* present, folded into the hash so changes to
* `GBRAIN_SYNOPSIS_DOC_MAX_CHARS` invalidate the prior cache cleanly.
* Omit for `crMode !== 'per_chunk_synopsis'` — title / none modes
* don't consult the cap and the field stays out of the hash for
* back-compat with pre-cap embeddings.
*/
synopsisDocMaxChars?: number;
} & (
| {
/** Canonical provider-neutral synopsis model identifier. */
synopsisModel: string;
/** @deprecated Use `synopsisModel`. Ignored when both are present. */
haikuModel?: string;
}
| {
synopsisModel?: undefined;
/** @deprecated Use `synopsisModel`. */
haikuModel: string;
}
);
export function computeCorpusGeneration(args: ComputeCorpusGenerationArgs): string {
const synopsisModel =
args.crMode === 'per_chunk_synopsis'
? args.synopsisModel ?? args.haikuModel
: DEFAULT_SYNOPSIS_MODEL;
const h = createHash('sha256')
.update(args.crMode)
.update('|')
.update(String(SYNOPSIS_PROMPT_VERSION))
.update('|')
.update(synopsisModel)
.update('|')
.update(String(TITLE_WRAPPER_VERSION))
.update('|')
.update(getEmbeddingModelTag());
if (args.synopsisDocMaxChars !== undefined) {
h.update('|doc_cap=').update(String(args.synopsisDocMaxChars));
}
return h.digest('hex').slice(0, 16);
}
/**
* #3507 — the corpus_generation a page lands on when a plain re-embed path
* (`embed --stale` and friends) re-embeds a `per_chunk_synopsis` page at the
* title-only tier (the D14 fallback tier; synopsis re-generation is a paid
* backfill concern). Callers restamp
* `updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration())`
* so the stamped mode keeps describing the vectors actually in the column.
* Matches what the inline import path writes for its title-tier pages.
*/
export function titleTierCorpusGeneration(): string {
return computeCorpusGeneration({ crMode: 'title', synopsisModel: DEFAULT_SYNOPSIS_MODEL });
}
/**
* #3883: resolve the synopsis output-token cap from the
* `models.synopsis_max_tokens` config key. Returns undefined when unset or
* invalid so generatePerChunkSynopsis falls back to its SYNOPSIS_MAX_TOKENS
* default (200). Clamped to a sane range (16..8192); fail-open on config
* errors (a config read failure must never block a re-embed).
*/
export async function resolveSynopsisMaxTokens(engine: BrainEngine): Promise<number | undefined> {
try {
const raw = await engine.getConfig('models.synopsis_max_tokens');
if (raw == null || raw === '') return undefined;
const n = Number(raw);
if (Number.isFinite(n) && n >= 16 && n <= 8192) return Math.floor(n);
} catch { /* fail-open to the page-summary default */ }
return undefined;
}
/**
* Compute source_text_hash for D27 P1-4 cache key composition. The
* synopsis cache invalidates correctly when adjacent text changes (page
* content edit, frontmatter change, fallback chain different source).
*/
export function computeSourceTextHash(sourceText: string): string {
return createHash('sha256').update(sourceText).digest('hex').slice(0, 16);
}
/**
* Result of `reembedPageWithContextualRetrieval`. Caller dispatches on
* `kind`:
*
* success → all chunks embedded + page stamped, DB transaction committed
* skipped → missing/deleted, or mode='none'/no chunks with a guarded context stamp
* page_fallback → PHASE 1 detected a refusal/empty/malformed; restarted
* at lower tier; final tier in `mode_applied` is what landed
* transient_error → caller should retry (Minion job retries; inline
* callers propagate)
* permanent_error → won't be fixed by retry; doctor surfaces
*/
export type ReembedPageResult =
| {
kind: 'success';
mode_applied: CRMode;
chunks_embedded: number;
corpus_generation: string;
}
| {
kind: 'skipped';
reason: 'mode_none' | 'no_chunks' | 'page_missing' | 'soft_deleted' | 'budget_limited' | 'purge_sweep_pending' | 'page_body_segmented';
}
| {
kind: 'page_fallback';
mode_attempted: CRMode;
mode_applied: CRMode;
chunks_embedded: number;
corpus_generation: string;
fallback_kind: SynopsisFailureKind;
}
| {
kind: 'transient_error';
cause: SynopsisFailureKind | 'embed' | 'db';
detail: string;
}
| {
kind: 'permanent_error';
cause: SynopsisFailureKind | 'embed' | 'db';
detail: string;
};
export interface ReembedPageArgs {
engine: BrainEngine;
/** Page slug (within `sourceId`). */
pageSlug: string;
/** Source that owns the page (multi-source brains). */
sourceId: string;
/** Global effective mode from the active ModeBundle. */
globalMode: CRMode;
/**
* Soft kill switch from D18. When true, the resolver short-circuits to
* 'none' regardless of frontmatter / source / global. Wrapped vectors
* already in `content_chunks` continue serving queries.
*/
killSwitchDisabled?: boolean;
/** Resolved provider-neutral model used for per-chunk synopsis generation. */
synopsisModel?: string;
/**
* #3883: output-token cap for each synopsis call. When omitted, the
* `models.synopsis_max_tokens` config is read once per page; when that is
* unset too, page-summary's SYNOPSIS_MAX_TOKENS default (200) applies.
*/
synopsisMaxTokens?: number;
/**
* @deprecated Use `synopsisModel`. Retained for callers compiled against
* the pre-provider-neutral service shape.
*/
haikuModel?: string;
/** Optional abort signal threaded into gateway.chat + embedBatch. */
abortSignal?: AbortSignal;
/**
* Optional rate-leaser hooks (D26 P0-3). The Minion handler wires
* src/core/minions/rate-leases.ts here; inline callers (import-file,
* reindex command) pass undefined and rely on gateway-level retry.
*/
acquireSynopsisLease?: () => Promise<unknown>;
releaseSynopsisLease?: (lease?: unknown) => Promise<void>;
/**
* Intra-page per-chunk synopsis concurrency. 1 preserves the legacy
* sequential loop exactly; higher values only parallelize synopsis
* calls. Embedding remains one batch after all synopses succeed.
*/
chunkConcurrency?: number;
/** @internal Synopsis generator seam (benchmark caching/metering); production callers omit this. */
generateSynopsis?: typeof generatePerChunkSynopsis;
}
/**
* Re-embed one page through the active CR mode. Implements the D26 P0-2
* two-phase build pattern.
*/
export async function reembedPageWithContextualRetrieval(
args: ReembedPageArgs,
): Promise<ReembedPageResult> {
// ── Load page + source + chunks ────────────────────────────────────
const initial = await args.engine.getPage(args.pageSlug, { sourceId: args.sourceId });
if (!initial) {
return { kind: 'skipped', reason: 'page_missing' };
}
if (initial.deleted_at != null) {
return { kind: 'skipped', reason: 'soft_deleted' };
}
if (isSegmentedPage(initial)) return reembedSegmentedPage(args, initial);
const prepared = await readProjectionSnapshot(args.engine, args.pageSlug, args.sourceId, { allowUnsealed: true });
if (!prepared) return { kind: 'skipped', reason: 'page_missing' };
const page = prepared.snapshot.page;
const source = await loadSourceRow(args.engine, page.source_id);
const superseded: ReembedPageResult = { kind: 'transient_error', cause: 'db',
detail: 'The page projection or contextual policy changed during preparation; retry from its current snapshot.' };
// The same guarded completion publishes vectors and their context together.
// Metadata-only branches also compare the raw inventory, including empty or
// intentionally unsealed pages; those chunks never reach a provider.
const publish = async (mode: CRMode, generation: string | null, embedded?: ChunkInput[]): Promise<ReembedPageResult | null> => {
try {
const installed = await args.engine.transaction(async tx => {
await tx.lockPageKeys([{ sourceId: page.source_id, slug: page.slug }]);
if (digest(sourcePolicy(await loadSourceRow(tx, page.source_id))) !== digest(sourcePolicy(source))) return false;
if (embedded) {
if (!(await installPageEmbeddings(tx, prepared, embedded.map(chunk => ({ ...chunk, model: prepared.embeddingModel ?? undefined })),
undefined, { tier: mode, corpusGeneration: generation })).complete) return false;
} else {
const current = await readProjectionSnapshot(tx, page.slug, page.source_id, { allowUnsealed: true });
if (!sameProjection(prepared, current)) return false;
}
await tx.updatePageContextualRetrievalState(page.slug, page.source_id, mode, generation);
if (!embedded && mode === 'none' && page.contextual_retrieval_mode == null
&& !Object.hasOwn(page.frontmatter ?? {}, 'embed_skip')) {
await reconcileContextualEmbeddingInputs(tx, prepared.snapshot, mode, generation);
}
return true;
});
return installed ? null : superseded;
} catch (err) {
return { kind: 'transient_error', cause: 'db', detail: err instanceof Error ? err.message : String(err) };
}
};
// ── Resolve effective mode (D5+D6+D15+D18) ─────────────────────────
const resolution = resolveContextualRetrievalMode({
pageFrontmatter: page.frontmatter ?? {},
source: {
id: source.id,
contextual_retrieval_mode: source.contextual_retrieval_mode ?? null,
trust_frontmatter_overrides: source.trust_frontmatter_overrides ?? false,
},
globalMode: args.globalMode,
killSwitchDisabled: args.killSwitchDisabled ?? false,
});
// 'none' mode skips wrapping entirely. Still stamp the column to record
// the page is up-to-date relative to current global state — prevents
// the reindex sweep from re-walking pages that are already aligned.
if (resolution.mode === 'none') {
const failure = await publish('none', null);
if (failure) return failure;
return { kind: 'skipped', reason: 'mode_none' };
}
const chunks = prepared.chunks;
if (chunks.length === 0) {
// No chunks but page exists (frontmatter-only or empty). Stamp the
// column anyway so subsequent reindex sweeps don't keep visiting.
const failure = await publish(resolution.mode,
computeCorpusGeneration({
crMode: resolution.mode,
synopsisModel: args.synopsisModel ?? args.haikuModel ?? DEFAULT_SYNOPSIS_MODEL,
synopsisDocMaxChars: resolution.mode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
}),
);
if (failure) return failure;
return { kind: 'skipped', reason: 'no_chunks' };
}
if (page.text_projection_revision !== prepared.snapshot.revision) return superseded;
// ── PHASE 1: in-memory build (no DB writes) ────────────────────────
// Iterate at most twice — once at the requested tier, once at the
// fall-back 'title' tier if a refusal/empty/malformed fires. The
// fall-back path is the D14 page-level consistency guarantee: a
// single bad chunk demotes the whole page to title-only so all
// chunks on the page share the same wrapper shape.
const synopsisModel = args.synopsisModel ?? args.haikuModel ?? DEFAULT_SYNOPSIS_MODEL;
// #3883: explicit arg wins; else the models.synopsis_max_tokens config;
// else undefined → page-summary's SYNOPSIS_MAX_TOKENS default (200).
const synopsisMaxTokens =
args.synopsisMaxTokens ?? (await resolveSynopsisMaxTokens(args.engine));
// Large pages: the title tier until per-chunk synopses are metered per page
// (plan B8), and no re-embed while the per-page cap holds or capped the page.
const large = isLargePageBytes(pageBodyBytes(page));
if (large && (await args.engine.executeRaw(`SELECT 1 FROM page_embedding_budget WHERE page_id = $1 AND state <> 'open'`, [page.id])).length) {
return { kind: 'skipped', reason: 'budget_limited' };
}
let attemptMode: CRMode = large && resolution.mode === 'per_chunk_synopsis' ? 'title' : resolution.mode;
let fallbackReason: SynopsisFailureKind | null = null;
// eslint-disable-next-line no-constant-condition
while (true) {
const phase1 = await tryBuildPhase1({
attemptMode,
page,
chunks: chunks as ChunkInput[],
args,
synopsisModel,
synopsisMaxTokens,
});
if (phase1.kind === 'success') {
const corpus_generation = computeCorpusGeneration({
crMode: attemptMode,
synopsisModel,
synopsisDocMaxChars: attemptMode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
});
// ── PHASE 2: single DB transaction ───────────────────────────
const failure = await publish(attemptMode, corpus_generation, phase1.embeddedChunks);
if (failure) return failure;
if (fallbackReason != null) {
return {
kind: 'page_fallback',
mode_attempted: resolution.mode,
mode_applied: attemptMode,
chunks_embedded: phase1.embeddedChunks.length,
corpus_generation,
fallback_kind: fallbackReason,
};
}
return {
kind: 'success',
mode_applied: attemptMode,
chunks_embedded: phase1.embeddedChunks.length,
corpus_generation,
};
}
// PHASE 1 failed. Decide whether to fall back or surface.
if (phase1.kind === 'page_level_fallback_requested') {
if (attemptMode === 'title') {
// Already at the lowest non-none tier; can't fall back further.
// Stamp as 'none' and treat as permanent so doctor flags.
return {
kind: 'permanent_error',
cause: phase1.cause,
detail: 'Even title-only tier failed; pages_summary call surfaced unrecoverable error.',
};
}
// Restart PHASE 1 at the lower tier. fallbackReason captures the
// original failure kind for the result envelope.
fallbackReason = phase1.cause;
attemptMode = 'title';
continue;
}
// Transient (rate_limit / timeout / network / provider_5xx) or auth
// failure — caller decides retry policy.
return {
kind: phase1.kind === 'transient' ? 'transient_error' : 'permanent_error',
cause: phase1.cause,
detail: phase1.detail,
};
}
}
/** Embedding-input chars a synopsis adds to a title-wrapped chunk at most (embedding-context's synopsis cap plus its markup). */
const SYNOPSIS_WRAPPER_CHARS = 320;
/** page_segment_derivations.pipeline_version of the synopsis pipeline (wave-b spec 8.6). */
export const SYNOPSIS_DERIVATION_VERSION = 1;
/**
* Wave B spec 8.6: per-chunk synopsis on a segmented page, segment by segment, never reading the body whole. Only a
* page whose resolved mode is `per_chunk_synopsis` runs (the title and none tiers of a segmented page come from the
* segmented embedding path, so those are skipped as `page_body_segmented`).
*
* Each segment's chunks are packed into windows (segmentSynopsisWindows); a chunk's synopsis document is the page title,
* the first 8 KiB of the projected lead and its window, and its recorded input names that document's hash
* (synopsisDocumentHash), so a chunk whose recorded input already matches keeps its vector. The pipeline's
* page_segment_derivations rows (kind synopsis, one per occurrence, output = the window anchors) record each segment
* done under a context hash over its entry state, the title, the lead's hash, the synopsis model, its output cap, the
* chunker version the page's rows were cut at (so a re-chunk unsettles its segments), the corpus generation and the write target: a rerun reads no chunk row of a settled
* segment, and an append pays only for the windows of its new segments. Segments run newest first, so a per-page cap
* spends on the newest text, as the embedding path's grants do.
*
* Each window with work reserves its synopsis calls and their embedding against the page's spend cap
* (`synopsis:<model>`) first: a held page or a refused reservation leaves the title tier in place (`budget_limited`, or
* `purge_sweep_pending` when a fact purge's pending derivation sweep covers the page), the
* segment `partial` and the windows already installed kept. A window whose synopsis is refused, empty or malformed
* embeds at the title tier, which a synopsis page accepts (page_fallback; the segment is settled with the fallback in
* its receipt, so it is not paid for again until its inputs change). Each segment runs under the page's embedding lease:
* a lease another path holds through the wait returns a retryable result before any provider call, and a lease lost
* before an install (assertHeld) installs nothing and returns one too. Vectors install through installSegmentedVectors;
* the page is stamped `per_chunk_synopsis` once every segment is settled.
*/
async function reembedSegmentedPage(args: ReembedPageArgs, initial: Page): Promise<ReembedPageResult> {
const { engine } = args;
const page = await readSegmentedEmbedPage(engine, { sourceId: args.sourceId, slug: args.pageSlug });
if (!page) return { kind: 'skipped', reason: 'page_body_segmented' };
const source = await loadSourceRow(engine, page.source_id);
const resolution = resolveContextualRetrievalMode({
pageFrontmatter: initial.frontmatter ?? {},
source: {
id: source.id,
contextual_retrieval_mode: source.contextual_retrieval_mode ?? null,
trust_frontmatter_overrides: source.trust_frontmatter_overrides ?? false,
},
globalMode: args.globalMode,
killSwitchDisabled: args.killSwitchDisabled ?? false,
});
if (resolution.mode !== 'per_chunk_synopsis') return { kind: 'skipped', reason: 'page_body_segmented' };
if ((await engine.executeRaw(`SELECT 1 FROM page_embedding_budget WHERE page_id = $1 AND state <> 'open'`, [page.id])).length) {
return { kind: 'skipped', reason: 'budget_limited' };
}
const superseded: ReembedPageResult = { kind: 'transient_error', cause: 'db',
detail: 'The page, its title or its contextual mode changed during the synopsis pass; retry from its current state.' };
const synopsisModel = args.synopsisModel ?? args.haikuModel ?? DEFAULT_SYNOPSIS_MODEL;
const synopsisMaxTokens = args.synopsisMaxTokens ?? (await resolveSynopsisMaxTokens(engine));
const corpusGeneration = computeCorpusGeneration({ crMode: 'per_chunk_synopsis', synopsisModel, synopsisDocMaxChars: SYNOPSIS_DOC_MAX_CHARS });
const target = await embeddingWriteTarget(engine);
const base = embeddingInputContext(target, page.title, corpusGeneration, []);
const titlePrefix = buildContextualPrefix(page.title, null);
const embedSignature = currentEmbeddingSignature();
const lead = await segmentSynopsisLead(engine, page);
const [chunked] = await engine.executeRaw<{ chunker_version: number | string | null }>('SELECT chunker_version FROM pages WHERE id = $1', [page.id]);
const inputs = { title: page.title, lead_sha256: sha256(lead), synopsis_model: synopsisModel, synopsis_max_tokens: synopsisMaxTokens ?? SYNOPSIS_MAX_TOKENS,
chunker_version: chunked?.chunker_version == null ? null : Number(chunked.chunker_version), corpus_generation: corpusGeneration,
embedding: { column: target.column.name, model: target.provenanceModel, dimensions: target.column.dimensions } };
const segments = await listLiveEmbedSegments(engine, page.id);
const live = segments.map(s => ({ occurrenceId: s.occurrenceId, sha256: s.sha256,
contextHash: segmentDerivationContextHash('synopsis', { entryState: s.entryState, page: inputs }) }));
const plan = planSegmentDerivations('synopsis', SYNOPSIS_DERIVATION_VERSION, live, await engine.readSegmentDerivations(page.id, { kind: 'synopsis' }));
const derivation = (segment: typeof live[number], state: 'done' | 'partial', windows: unknown[], receipt: unknown) =>
engine.upsertSegmentDerivations([{ pageId: page.id, kind: 'synopsis', occurrenceId: segment.occurrenceId, version: SYNOPSIS_DERIVATION_VERSION,
segmentSha256: segment.sha256, contextHash: segment.contextHash, state, output: { windows }, receipt }]);
let embedded = 0;
let fallbackKind: SynopsisFailureKind | null = null;
let unpricedModel: string | null = null;
for (const segment of plan.todo) {
await using lease = await acquirePageEmbeddingLease(engine, page.id, { signal: args.abortSignal });
if (lease.busy) return { kind: 'transient_error', cause: 'db', detail: pageEmbeddingLeaseUnavailable(page.id, false).message };
const anchors: Array<{ first_chunk: number; last_chunk: number; chunks: number; document_sha256: string; tier: EmbeddingTier }> = [];
let segmentFallback: SynopsisFailureKind | null = null;
for (const window of segmentSynopsisWindows(await listSegmentChunkRows(engine, page.id, segment.occurrenceId), lead)) {
const provenance = { ...base, bodyHash: synopsisDocumentHash(window.document) };
const anchor = { first_chunk: window.chunks[0]!.chunk_index, last_chunk: window.chunks.at(-1)!.chunk_index, chunks: window.chunks.length,
document_sha256: provenance.bodyHash, tier: 'per_chunk_synopsis' as EmbeddingTier };
const todo = window.chunks.filter(c => c.embedding_input_hash !== embeddingInputHash(provenance, 'per_chunk_synopsis', c));
if (todo.length === 0) { anchors.push(anchor); continue; }
const estimateUsd = todo.reduce((usd, c) => usd
+ (c.chunk_source === 'fenced_code' ? 0 : estimateSynopsisCallUsd(synopsisModel, window.document, c.chunk_text, synopsisMaxTokens)), 0)
+ estimatePageEmbedUsd(todo.reduce((n, c) => n + wrapChunkForEmbedding(c.chunk_text, titlePrefix, c.chunk_source).length + SYNOPSIS_WRAPPER_CHARS, 0), embedSignature);
const metered = await meterPaidCall(engine, page, `synopsis:${synopsisModel}`, estimateUsd, async () => {
const synopsized = await synopsisWrappedTexts({ chunks: todo, sourceText: window.document, title: page.title, args, synopsisModel, synopsisMaxTokens });
if (!Array.isArray(synopsized) && synopsized.kind !== 'page_level_fallback_requested') return synopsized;
const tier: EmbeddingTier = Array.isArray(synopsized) ? 'per_chunk_synopsis' : 'title';
const inputTexts = Array.isArray(synopsized) ? synopsized : todo.map(c => wrapChunkForEmbedding(c.chunk_text, titlePrefix, c.chunk_source));
try {
return { kind: 'built' as const, tier, fallback: Array.isArray(synopsized) ? null : synopsized.cause, vectors: await embedBatch(inputTexts, { abortSignal: args.abortSignal }) };
} catch (err) {
return classifyEmbedError(err, err instanceof Error ? err.message : String(err));
}
});
unpricedModel = metered.unpricedModel ?? unpricedModel;
if ('refused' in metered) {
await derivation(segment, 'partial', anchors, { reason: unpricedModel ? 'unpriced_model' : metered.refused.reason === 'purge_sweep_pending' ? 'purge_sweep_pending' : 'budget_limited', ...(unpricedModel ? { model: unpricedModel } : {}),
cap_usd: metered.refused.capUsd, estimate_usd: metered.refused.estimateUsd });
return { kind: 'skipped', reason: metered.refused.reason === 'purge_sweep_pending' ? 'purge_sweep_pending' : 'budget_limited' };
}
const built = metered.value;
if (built.kind !== 'built') {
return { kind: built.kind === 'permanent' ? 'permanent_error' : 'transient_error', cause: built.cause, detail: built.detail };
}
if (built.vectors.length !== todo.length || built.vectors.some(v => !v?.length || !v.every(Number.isFinite))) {
return { kind: 'transient_error', cause: 'embed', detail: 'Embedding provider returned an incomplete or invalid batch' };
}
let held = true;
const installed = await installSegmentedVectors(engine, page, target, todo.map((c, i) => ({ chunk_index: c.chunk_index, chunk_text: c.chunk_text,
vector: built.vectors[i]!, hash: embeddingInputHash(built.tier === 'title' ? base : provenance, built.tier, c), expected: c.embedding_input_hash ?? null })),
async tx => (held = await lease.assertHeld(tx).then(() => true, () => false)));
if (!held) return { kind: 'transient_error', cause: 'db', detail: pageEmbeddingLeaseUnavailable(page.id, true).message };
if (installed < todo.length) return superseded;
embedded += installed;
if (built.fallback) segmentFallback ??= built.fallback;
anchors.push({ ...anchor, tier: built.tier });
}
await derivation(segment, 'done', anchors, segmentFallback || unpricedModel
? { ...(unpricedModel ? { kind: 'unpriced_model', pipeline: 'synopsis', model: unpricedModel } : {}), ...(segmentFallback ? { fallback_kind: segmentFallback } : {}) } : null);
fallbackKind ??= segmentFallback;
}
await engine.retireSegmentDerivations(page.id, live.map(s => s.occurrenceId), { kind: 'synopsis' });
const stamped = await engine.transaction(async tx => {
await tx.lockPageKeys([{ sourceId: page.source_id, slug: page.slug }]);
const current = await readSegmentedEmbedPage(tx, { id: page.id });
if (!current || current.title !== page.title || current.contextual_retrieval_mode !== page.contextual_retrieval_mode) return false;
await tx.updatePageContextualRetrievalState(page.slug, page.source_id, 'per_chunk_synopsis', corpusGeneration);
return true;
});
if (!stamped) return superseded;
if (fallbackKind !== null) {
return { kind: 'page_fallback', mode_attempted: 'per_chunk_synopsis', mode_applied: 'per_chunk_synopsis',
chunks_embedded: embedded, corpus_generation: corpusGeneration, fallback_kind: fallbackKind };
}
return { kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: embedded, corpus_generation: corpusGeneration };
}
// ── Internal helpers ────────────────────────────────────────────────
interface Phase1Success {
kind: 'success';
embeddedChunks: ChunkInput[];
}
interface Phase1FallbackRequest {
kind: 'page_level_fallback_requested';
cause: SynopsisFailureKind;
}
interface Phase1Transient {
kind: 'transient';
cause: SynopsisFailureKind | 'embed';
detail: string;
}
interface Phase1Permanent {
kind: 'permanent';
cause: SynopsisFailureKind | 'embed';
detail: string;
}
type Phase1Result = Phase1Success | Phase1FallbackRequest | Phase1Transient | Phase1Permanent;
async function tryBuildPhase1(opts: {
attemptMode: CRMode;
page: Page;
chunks: ChunkInput[];
args: ReembedPageArgs;
synopsisModel: string;
/** #3883: resolved output-token cap (undefined → page-summary default). */
synopsisMaxTokens?: number;
}): Promise<Phase1Result> {
const { attemptMode, page, chunks, args, synopsisModel, synopsisMaxTokens } = opts;
// Build the wrapper prefix for THIS page. Title-only tier: one prefix
// reused across all chunks. per_chunk_synopsis tier: prefix is built
// per-chunk with the chunk-specific generated synopsis.
// buildContextualPrefix sanitizes once, as every embedding writer and embedding-input-hash.ts do.
const title = page.title;
if (attemptMode === 'title' || !modeRequiresSynopsis(attemptMode)) {
// Title-only path. No synopsis-model calls; pure string concat.
// Use compiled_truth first sentences as a free pseudo-summary when
// the title tier wants slightly more context — but per D2 the
// balanced default is title-only without summary. Keep it pure for
// now; the title block alone is what 'balanced' ships.
const prefix = buildContextualPrefix(title, null);
const wrappedTexts = chunks.map((c) =>
modeRequiresWrapper(attemptMode)
? wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source)
: c.chunk_text,
);
try {
const embeddings = await embedBatch(wrappedTexts, { abortSignal: args.abortSignal });
return {
kind: 'success',
embeddedChunks: chunks.map((c, i) => ({
...c,
chunk_text: c.chunk_text, // canonical, NOT the wrapped string (D20-T1)
embedding: embeddings[i],
token_count: Math.ceil(wrappedTexts[i].length / 4),
})),
};
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
return classifyEmbedError(err, detail);
}
}
// per_chunk_synopsis path. Read source text via fallback chain,
// generate synopsis per chunk through a bounded sliding pool, then
// batch embed at the end (D27 P2-2).
const sourceText = readSourceTextWithFallback(page, chunks);
const wrappedTexts = await synopsisWrappedTexts({ chunks, sourceText, title, args, synopsisModel, synopsisMaxTokens });
if (!Array.isArray(wrappedTexts)) return wrappedTexts;
// All chunks synthesized successfully. Single batch embed (D27 P2-2).
try {
const embeddings = await embedBatch(wrappedTexts, { abortSignal: args.abortSignal });
return {
kind: 'success',
embeddedChunks: chunks.map((c, i) => ({
...c,
chunk_text: c.chunk_text, // canonical (D20-T1)
embedding: embeddings[i],
token_count: Math.ceil(wrappedTexts[i].length / 4),
})),
};
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
return classifyEmbedError(err, detail);
}
}
type SynopsisChunk = Pick<ChunkInput, 'chunk_index' | 'chunk_text' | 'chunk_source'>;
/**
* The embedding input of each chunk under per-chunk synopsis over `sourceText`: a synopsis per chunk through a bounded
* sliding pool (fenced code passes through raw), or the first failing chunk's Phase 1 result.
*/
async function synopsisWrappedTexts(opts: {
chunks: readonly SynopsisChunk[];
sourceText: string;
title: string;
args: ReembedPageArgs;
synopsisModel: string;
synopsisMaxTokens?: number;
}): Promise<string[] | Exclude<Phase1Result, Phase1Success>> {
const { chunks, args } = opts;
const wrappedTexts: string[] = new Array(chunks.length);
const poolResult = await runSlidingPool({
items: [...chunks],
workers: clampContextualChunkConcurrency(args.chunkConcurrency ?? resolveContextualChunkConcurrency()),
signal: args.abortSignal,
onError: 'abort',
failureLabel: (c) => String(c.chunk_index),
onItem: async (c, i) => {
wrappedTexts[i] = await buildWrappedChunkText({ ...opts, chunk: c });
},
});
if (poolResult.failures.length > 0) {
const failure = [...poolResult.failures].sort((a, b) => a.idx - b.idx)[0].error;
if (failure instanceof ChunkSynopsisPhase1Error) {
return failure.result;
}
throw failure;
}
if (poolResult.aborted || args.abortSignal?.aborted) {
return { kind: 'transient', cause: 'timeout', detail: 'aborted' };
}
return wrappedTexts;
}
class ChunkSynopsisPhase1Error extends Error {
constructor(readonly result: Exclude<Phase1Result, Phase1Success>) {
super(`chunk synopsis failed: ${result.kind}`);
this.name = 'ChunkSynopsisPhase1Error';
}
}
async function buildWrappedChunkText(opts: {
chunk: SynopsisChunk;
sourceText: string;
title: string;
args: ReembedPageArgs;
synopsisModel: string;
/** #3883: resolved output-token cap (undefined → page-summary default). */
synopsisMaxTokens?: number;
}): Promise<string> {
const { chunk: c, sourceText, title, args, synopsisModel, synopsisMaxTokens } = opts;
// Code chunks always bypass the wrapper (D20-T4) — pass through.
if (c.chunk_source === 'fenced_code') {
return c.chunk_text;
}
// Acquire rate-lease per chunk (D26 P0-3). Inline callers pass no
// hooks; only the Minion handler wires through rate-leases.ts.
let lease: unknown;
let leaseAcquired = false;
let synopsisResult: GeneratePerChunkSynopsisResult;
try {
if (args.acquireSynopsisLease) {
try {
lease = await args.acquireSynopsisLease();
} catch (err) {
if (args.abortSignal?.aborted || isAbortError(err)) {
throw new ChunkSynopsisPhase1Error({
kind: 'transient',
cause: 'timeout',
detail: 'aborted',
});
}
throw err;
}
leaseAcquired = true;
}
synopsisResult = await (args.generateSynopsis ?? generatePerChunkSynopsis)({
documentText: sourceText,
chunkText: c.chunk_text,
pageTitle: title,
pageSlug: args.pageSlug,
sourceId: args.sourceId,
chunkIndex: c.chunk_index,
model: synopsisModel,
abortSignal: args.abortSignal,
maxTokens: synopsisMaxTokens,
});
} finally {
if (leaseAcquired && args.releaseSynopsisLease) {
try {
await args.releaseSynopsisLease(lease);
} catch {
// Lease release failure shouldn't abort the page; surfacing it
// would race with the synopsis result. Audit-only.
}
}
}
if (synopsisResult.kind === 'success') {
const prefix = buildContextualPrefix(title, synopsisResult.synopsis);
return wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source);
}
// Failure classification per D27 P1-2:
// refusal | empty | malformed → page-level fall-back to title-only
// auth_failure → permanent (won't fix with retry)
// rate_limit | timeout | network | provider_5xx → transient
// source_missing → walked into fallback already; would be 'malformed'
// from generatePerChunkSynopsis if we ever propagated it here
if (
synopsisResult.kind === 'refusal' ||
synopsisResult.kind === 'empty' ||
synopsisResult.kind === 'malformed'
) {
throw new ChunkSynopsisPhase1Error({
kind: 'page_level_fallback_requested',
cause: synopsisResult.kind,
});
}
if (synopsisResult.kind === 'auth_failure') {
throw new ChunkSynopsisPhase1Error({
kind: 'permanent',
cause: synopsisResult.kind,
detail: synopsisResult.detail ?? 'auth failure',
});
}
throw new ChunkSynopsisPhase1Error({
kind: 'transient',
cause: synopsisResult.kind,
detail: synopsisResult.detail ?? 'transient',
});
}
function isAbortError(err: unknown): boolean {
return (
typeof err === 'object' &&
err !== null &&
(err as { name?: unknown }).name === 'AbortError'
);
}
/**
* Source-text fallback chain per D11:
* 1. read page.source_path from disk (truest "document")
* 2. concat all chunks (loses frontmatter context but in-memory)
* 3. just the chunks one-by-one (degraded; loses cross-chunk context)
*
* For step 2/3 we just concatenate chunk_text — synopsis quality
* degrades but stays valid.
*/
function readSourceTextWithFallback(page: Page, chunks: ChunkInput[]): string {
// Step 1: try filesystem read
// Note: Page type doesn't carry source_path in the canonical shape;
// it lives on PageInput / DB column. For now we attempt the chunk-concat
// path directly — a v0.41+ wave can thread source_path through Page if
// the synopsis quality benchmark shows the filesystem read materially
// helps.
if (chunks.length === 0) return page.title;
// Step 2: chunk-concat (in-memory).
return chunks
.filter((c) => c.chunk_source !== 'image_asset')
.map((c) => c.chunk_text)
.join('\n\n');
}
/**
* Load one source row with the CR-relevant columns. Exported for reuse by
* the import path (#3885: stored `sources set-cr-mode` must apply on
* capture/reindex, not just the Minion backfill) and the conversation-parser
* body reader (#3911: relative raw_transcript resolves against the OWNING
* source's local_path). Throws SourceRowNotFoundError when the source id is
* unknown, so callers can tell a missing row from a failed read.
*/
export class SourceRowNotFoundError extends Error {}
export async function loadSourceRow(engine: BrainEngine, sourceId: string): Promise<SourceRow> {
const rows = await engine.executeRaw<SourceRow>(
`SELECT id, name, local_path, last_commit, last_sync_at, config, created_at,
contextual_retrieval_mode, trust_frontmatter_overrides
FROM sources WHERE id = $1`,
[sourceId],
);
// A missing result set (unit-test engine doubles) reads as no row.
if (!rows?.length) {
throw new SourceRowNotFoundError(`Source not found: ${sourceId}`);
}
return rows[0];
}
function sourcePolicy(source: SourceRow) {
return { id: source.id, mode: source.contextual_retrieval_mode ?? null,
trustFrontmatter: source.trust_frontmatter_overrides ?? false };
}
function sameProjection(prepared: ProjectionSnapshot, current: ProjectionSnapshot | null): boolean {
return current !== null && current.snapshot.revision === prepared.snapshot.revision
&& current.snapshot.sourceIncarnation === prepared.snapshot.sourceIncarnation
&& current.snapshot.page.id === prepared.snapshot.page.id
&& current.snapshot.page.text_projection_revision === prepared.snapshot.page.text_projection_revision
&& current.indexingContext === prepared.indexingContext
&& digest(current.chunks) === digest(prepared.chunks);
}
function classifyEmbedError(err: unknown, detail: string): Phase1Transient | Phase1Permanent {
const e = err as { status?: number; code?: string; name?: string };
if (e.status === 401 || e.status === 403) {
return { kind: 'permanent', cause: 'embed', detail };
}
if (
e.status === 429 ||
(e.status != null && e.status >= 500 && e.status < 600) ||
e.name === 'AbortError' ||
/timeout|network|fetch/i.test(detail)
) {
return { kind: 'transient', cause: 'embed', detail };
}
return { kind: 'transient', cause: 'embed', detail };
}
/**
* Public helper for the eng-review T9 reindex sweep: compute the
* expected effective mode for one page given (page row, source row,
* global mode, kill switch). Used by the reindex predicate to decide
* whether a page is drifted relative to its expected state.
*
* Pure — no DB / network. Doesn't read frontmatter; the reindex
* sweep parses frontmatter separately in the candidate path that
* resolves overrides.
*/
export function expectedModeForPageSourceOnly(args: {
source: { id: string; contextual_retrieval_mode?: string | null };
globalMode: CRMode;
killSwitchDisabled?: boolean;
}): CRMode {
if (args.killSwitchDisabled) return 'none';
const sm = args.source.contextual_retrieval_mode;
if (sm === 'none' || sm === 'title' || sm === 'per_chunk_synopsis') {
return sm;
}
return args.globalMode;
}
// fs imported above for the future filesystem-read path.
void fs;