Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Evidence delivery (`return_unit`) — the authorization-safe assembler that
* turns ranked chunk hits into whole evidence: a window of neighbor chunks,
* the enclosing section, or the page, packed into a token budget.
*
* Runs at the op layer AFTER `hybridSearchCached` returns and before output
* redaction / snippet capping. It never mutates the hit rows (they may be
* shared with cache or capture) and never caches expanded text. Opt-in: a
* resolved unit of `chunk` means the stage does not run at all, so the
* off path is byte-identical.
*
* Text source: the page's stored body, sanitized WHOLE with the strict
* protected-body boundary before any slicing (a slice can cut a fence and
* read as a protected tail), fetched with the hit chunks in one batched
* `getChunkWindows` call that re-authorizes every page under the caller's
* current read scope. Chunks only anchor hits in that text. A segmented page
* has no such text: its windows are read around the hits through the segment
* reader (segmented-evidence.ts) and joined as separate runs.
*
* The normative algorithm (page text, units, allocation, spans) is specified
* in docs/evidence-delivery.md; gbrain-evals measures exactly this code.
*/
import { createHash } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import type { PageReadScope, SearchResult } from '../types.ts';
import type { ChunkWindowPage, ChunkWindowRequest } from './chunk-windows.ts';
import { estimateTokens as cl100kTokens, cl100kAvailable } from '../chunkers/token-estimate.ts';
import { estimateTokens as heuristicTokens } from './token-budget.ts';
import { redactRetrievalOutput } from './output-redaction.ts';
import { buildSnippetMarker } from './snippet-cap.ts';
import { deliveredQuoteText, deliveredQuoteTitle, fenceTranscriptRows, neutralizeTranscriptRows } from './transcript-quote.ts';
import { stampPageTrust } from '../eligibility/stamp.ts';
import { pageReadFilter } from './read-policy-sql.ts';
import { chunkGenerationFilter, chunkManifestReads, safeChunksFilter, requiresSafeChunks, servedTextProjectionFilter } from './safe-chunks.ts';
import { resolveExcludePrivatePages, worldOnlyFacts } from './private-visibility.ts';
import { safeSplitIndex } from '../text-safe.ts';
import { loadFenceChunkOverlay, type FenceChunkOverlay } from '../eligibility/fence-overlay.ts';
import { DEFAULT_INLINE_BODY_BYTES, locateChunksByStoredSpans, projectPageText, TIMELINE_SEPARATOR } from '../page-window/document.ts';
import { readSegmentedWindows } from './segmented-evidence.ts';
import { stripChunkHeader } from '../chunkers/code.ts';
import { OperationError } from '../ops/contract.ts';
import { blockDateHeaders, evidenceDateHeaderEnabled, evidenceTitlesEnabled } from './evidence-date.ts';
import { cutWholePages, resolvePageRankCut, type PageRankCut, type PageRankCutMeta } from './page-rank-cut.ts';
import { evidenceTailTrimEnabled, trimEvidenceTail, type TailTrimMeta } from './evidence-tail-trim.ts';
export { pagePlanHits } from './page-plan-hits.ts';
import { applyEffectiveDate } from '../utils.ts';
import { configPacking, MIN_EXPLICIT_AUTO_BUDGET, parseAutoPacking, type AutoPacking } from './evidence-packing.ts';
export { AUTO_PACKINGS, AUTO_PACKING_CONFIG_KEY, DEFAULT_AUTO_PACKING, MIN_EXPLICIT_AUTO_BUDGET, QUERY_BUDGET_UNIT_CONFIG_KEY, autoPackingValueProblem, parseAutoPacking, queryBudgetUsesAuto, type AutoPacking } from './evidence-packing.ts';
export const RETURN_UNITS = ['chunk', 'window', 'section', 'page', 'auto'] as const;
export type ReturnUnit = typeof RETURN_UNITS[number];
export type DeliveredUnit = 'chunk' | 'window' | 'section' | 'page';
export const RETURN_UNIT_CONFIG_KEY = 'search.return_unit';
export const THINK_RETURN_UNIT_CONFIG_KEY = 'think.return_unit';
export const RETURN_WINDOW_CONFIG_KEY = 'search.return_window';
export const RETURN_BUDGET_DEFAULT_KEY = 'search.return_budget_default';
export const RETURN_BUDGET_MAX_REMOTE_KEY = 'search.return_budget_max_remote';
export const RETURN_BUDGET_CONVERSATION_KEY = 'search.return_budget_conversation';
export const DEFAULT_RETURN_BUDGET = 6000;
/** `auto`'s default budget: whole chat sessions run about 15K tokens, and 16K still trimmed some sessions. */
export const DEFAULT_CONVERSATION_BUDGET = 24000;
export const DEFAULT_REMOTE_BUDGET_MAX = 32000;
export const EVIDENCE_BLOCK_CHAR_CAP = 60_000;
export const EVIDENCE_OMISSION = '\n\n[…]\n\n';
export const EVIDENCE_FETCH_TIMEOUT_MS = 5000;
/** First server release that understands `return_unit` (thin-client skew warning). */
export const EVIDENCE_DELIVERY_MIN_SERVER_VERSION = '0.60.13.0';
/** Ends a block or chunk the cap cut short (the omission line, as emitted at a block's end). */
export const EVIDENCE_CUT_MARKER = EVIDENCE_OMISSION.trimEnd();
/** Body tokens a cut always keeps: a cut title never takes them. */
const CUT_BODY_MIN = 8;
const PIECE_MAX_CHARS = 400;
const MAX_ROWS = 1024;
const MAX_WINDOW = 3;
export interface MatchSpan { chunk_id: number; start: number; end: number }
/**
* Why `auto` chose a result's unit: a conversation page (by type or slug
* prefix) gets the whole page; anything else keeps its ranked chunk exactly;
* a conversation whose matching span no longer fits the budget keeps its
* ranked chunks too.
*/
export type AutoReason = 'conversation_type' | 'conversation_slug' | 'not_conversation' | 'conversation_over_budget' | 'saved_fact'
/** `search.long_page_unit` delivered this long page in its unit; `long_page_chunk`: it kept its ranked chunk (`delivery.fallbacks` says why). */
| 'long_page' | 'long_page_chunk';
export interface DeliveredEvidence {
unit: DeliveredUnit;
chunk_ids: number[];
match_spans: MatchSpan[];
tokens: number;
truncated: boolean;
revision?: string;
unmapped_chunk_ids?: number[];
fallback_reason?: string;
/** Present only under `auto`: why this result got its unit. */
reason?: AutoReason;
}
export interface DeliveryMeta {
requested_unit: ReturnUnit;
applied_unit: ReturnUnit;
return_window: number;
budget_tokens: number;
budget_used: number;
tokens_delivered: number;
tokenizer: 'cl100k' | 'heuristic';
coordinates: 'utf16';
blocks: number;
dropped: number;
dropped_reasons: Record<string, number>;
fallbacks: string[];
budget_clamped?: { requested: number; max: number };
/** Present (true) when every block starts with its one-line date header (`search.evidence_date_header`). */
date_header?: true;
/** Present (true) when page block headers also carry the page title (`search.evidence_titles`). */
title_header?: true;
/** Present only when an explicit budget engaged the cap: the packing that ran. */
auto_packing?: AutoPacking;
/** W5: present when `search.page_rank_cut` cut the page candidates (page-rank-cut.ts). */
page_rank_cut?: PageRankCutMeta;
/** Present when `search.evidence_tail_trim` ran on this delivery (evidence-tail-trim.ts). */
tail_trim?: TailTrimMeta;
}
export type DeliveredSearchResult = SearchResult & { delivered: DeliveredEvidence };
export interface EvidencePlan {
requestedUnit: ReturnUnit;
unit: Exclude<ReturnUnit, 'chunk'>;
window: number;
budgetTokens: number;
/** return_unit came from the call, not config (snippet precedence). */
explicitUnit: boolean;
/** The caller passed the budget; a config or default budget never engages the cap. */
budgetExplicit: boolean;
/** search.auto_packing (or the library override), resolved once; allocation never rereads config. */
packing: AutoPacking;
budgetClamped?: { requested: number; max: number };
/** `search.evidence_date_header` is on: each block starts with its date header line. */
dateHeader?: true;
/** `search.evidence_titles` is on with the date header: page block header lines end with the page title. */
titleHeader?: true;
/** F2: set only by the search/query caller (withLongPagePolicy); recall, think and assemble_evidence never carry it. */
longPage?: LongPagePolicy;
/** W5: page unit + caller budget + `search.page_rank_cut: whole_pages` (page-rank-cut.ts). */
rankCut?: PageRankCut;
/** `auto` with `search.evidence_tail_trim` on: low-scoring conversation blocks past the top 3 are dropped (evidence-tail-trim.ts). */
tailTrim?: true;
}
/** The cap runs only under `auto`, with a budget the caller passed and a packing other than `off`. */
export function capEngaged(plan: EvidencePlan): boolean {
return plan.unit === 'auto' && plan.budgetExplicit && plan.packing !== 'off';
}
export interface DeliveryScope extends PageReadScope {
detail?: 'low' | 'medium' | 'high';
}
// ---------------------------------------------------------------------------
// Parameter validation + plan resolution
// ---------------------------------------------------------------------------
function exampleCall(op: string, unit: string, extra = ''): string {
const head = op === 'recall' ? '"query": "renewal terms"' : '"query": "launch date"';
return `Example: {${head}, "return_unit": "${unit}"${extra}}`;
}
export function parseReturnUnit(raw: unknown, op: string): ReturnUnit | undefined {
if (raw === undefined || raw === null) return undefined;
if (typeof raw === 'string' && (RETURN_UNITS as readonly string[]).includes(raw)) return raw as ReturnUnit;
const shown = typeof raw === 'string' ? JSON.stringify(raw.slice(0, 40)) : typeof raw;
throw new OperationError(
'invalid_params',
`return_unit must be one of ${RETURN_UNITS.join(', ')} (got ${shown}).`,
`${exampleCall(op, 'page')}. Use "window" for local context, "page" for whole conversations.`,
);
}
export function parseReturnWindow(raw: unknown, op: string): number | undefined {
if (raw === undefined || raw === null) return undefined;
if (typeof raw === 'number' && Number.isInteger(raw) && raw >= 1 && raw <= MAX_WINDOW) return raw;
const shown = typeof raw === 'number' ? String(raw) : typeof raw;
throw new OperationError(
'invalid_params',
`return_window must be an integer from 1 to ${MAX_WINDOW} (got ${shown}).`,
exampleCall(op, 'window', ', "return_window": 2'),
);
}
async function configNumber(engine: BrainEngine, key: string): Promise<number | null> {
try {
const raw = await engine.getConfig(key);
if (raw == null || raw === '') return null;
const n = Number(raw);
return Number.isFinite(n) ? n : null;
} catch {
return null;
}
}
export interface ResolvePlanInput {
remote: boolean | undefined;
viaSubagent?: boolean;
returnUnit: unknown;
returnWindow: unknown;
/** The op's own budget param (token_budget / budget_tokens). */
budget: unknown;
/** Explicit snippet_chars param (number) or undefined. */
snippetChars: unknown;
/** The resolved snippet cap for this call (0 = none). */
snippetCap: number;
configKey?: string;
op: string;
/**
* The caller passed a chunk-mode budget knob (query's token_budget, recall's
* budget_policy): an implied unit stays `chunk` so that legacy budgeting
* keeps its meaning.
*/
legacyBudget?: boolean;
/** Library-only per-call packing (gbrain-evals); wins over search.auto_packing. Never an MCP param. */
autoPacking?: unknown;
}
/**
* Resolve the per-call evidence plan, or null when the resolved unit is
* `chunk` (the stage stays inert and the response is unchanged). The unit is
* the call's `return_unit`, else the config unit, else `auto`. Throws
* invalid_params for a bad `return_unit` / `return_window`.
*/
export async function resolveEvidencePlan(engine: BrainEngine, input: ResolvePlanInput): Promise<EvidencePlan | null> {
const explicit = parseReturnUnit(input.returnUnit, input.op);
const explicitWindow = parseReturnWindow(input.returnWindow, input.op);
let unit: ReturnUnit = explicit ?? 'auto';
if (explicit === undefined) {
try {
const raw = await engine.getConfig(input.configKey ?? RETURN_UNIT_CONFIG_KEY);
if (typeof raw === 'string' && (RETURN_UNITS as readonly string[]).includes(raw)) unit = raw as ReturnUnit;
} catch { /* config unreadable: stay on the default */ }
// Snippet precedence: an implied or config-level unit never overrides an
// explicit snippet cap, the subagent default token economy, or a legacy
// chunk-mode budget knob.
const explicitSnippet = typeof input.snippetChars === 'number' && Number.isFinite(input.snippetChars);
if (unit !== 'chunk' && !explicitSnippet && input.viaSubagent === true && input.snippetCap > 0) unit = 'chunk';
if (input.legacyBudget === true) unit = 'chunk';
}
if (unit === 'chunk') return null;
// A budget that is not a positive number keeps its old meaning (no budget)
// unless the call also named its unit; then it is validated below.
const budgetExplicit = typeof input.budget === 'number' && (explicit !== undefined || (Number.isFinite(input.budget) && input.budget > 0));
const packing = parseAutoPacking(input.autoPacking) ?? await configPacking(engine);
if (unit === 'auto' && budgetExplicit && packing !== 'off') {
const raw = input.budget as number;
if (!Number.isFinite(raw) || Math.floor(raw) < MIN_EXPLICIT_AUTO_BUDGET) {
const name = input.op === 'recall' ? 'budget_tokens' : 'token_budget';
throw new OperationError(
'invalid_params',
`${name} must be at least ${MIN_EXPLICIT_AUTO_BUDGET} tokens under return_unit auto (got ${String(raw)}): an explicit budget is a hard cap, and a smaller one cannot hold a title, a cut marker and any evidence.`,
`${exampleCall(input.op, 'auto', `, "${name}": ${MIN_EXPLICIT_AUTO_BUDGET * 50}`)}. Omit ${name} for the default budget.`,
);
}
}
let window = explicitWindow ?? 1;
if (explicitWindow === undefined) {
const w = await configNumber(engine, RETURN_WINDOW_CONFIG_KEY);
if (w !== null) window = Math.min(MAX_WINDOW, Math.max(1, Math.floor(w)));
}
const fallbackBudget = unit === 'auto' ? DEFAULT_CONVERSATION_BUDGET : DEFAULT_RETURN_BUDGET;
let budget = typeof input.budget === 'number' && Number.isFinite(input.budget) && input.budget > 0
? Math.floor(input.budget)
: Math.floor((await configNumber(engine, unit === 'auto' ? RETURN_BUDGET_CONVERSATION_KEY : RETURN_BUDGET_DEFAULT_KEY)) ?? fallbackBudget);
if (budget <= 0) budget = fallbackBudget;
let budgetClamped: EvidencePlan['budgetClamped'];
if (input.remote !== false) {
const cfgMax = await configNumber(engine, RETURN_BUDGET_MAX_REMOTE_KEY);
const max = cfgMax !== null && cfgMax > 0 ? Math.floor(cfgMax) : DEFAULT_REMOTE_BUDGET_MAX;
if (budget > max) {
budgetClamped = { requested: budget, max };
budget = max;
}
}
const rankCut = unit === 'page' && budgetExplicit ? await resolvePageRankCut(engine, budget) : undefined;
const dateHeader = await evidenceDateHeaderEnabled(engine);
return {
requestedUnit: unit,
unit,
window,
budgetTokens: budget,
explicitUnit: explicit !== undefined,
budgetExplicit,
packing,
...(budgetClamped ? { budgetClamped } : {}),
...(dateHeader ? { dateHeader: true as const } : {}),
...(dateHeader && await evidenceTitlesEnabled(engine) ? { titleHeader: true as const } : {}),
...(rankCut ? { rankCut } : {}),
...(unit === 'auto' && await evidenceTailTrimEnabled(engine) ? { tailTrim: true as const } : {}),
};
}
// ---------------------------------------------------------------------------
// auto: conversation detection
// ---------------------------------------------------------------------------
/**
* Page types that hold conversations: imported sessions (`conversation`, what
* the transcripts and chat-connector ingest paths write), `transcript` and
* `chat` pages, meetings, and the Slack / iMessage chat-log types the
* conversation-facts pipeline reads (including the collector's granular
* Slack types before type consolidation).
*/
const CONVERSATION_PAGE_TYPES: ReadonlySet<string> = new Set([
'conversation', 'transcript', 'chat', 'meeting',
'slack', 'slack-dm-day', 'slack-thread', 'imessage', 'imessage-daily',
]);
/** `chat/` (LongMemEval and chat imports) and `conversations/` (transcripts + connectors ingest). */
const CONVERSATION_SLUG_PREFIXES: readonly string[] = ['chat/', 'conversations/'];
/** Deterministic, row-local conversation signal for `auto` (no query, no model). */
export function conversationSignal(hit: { type?: string | null; slug: string }): 'conversation_type' | 'conversation_slug' | null {
if (typeof hit.type === 'string' && CONVERSATION_PAGE_TYPES.has(hit.type.toLowerCase())) return 'conversation_type';
if (CONVERSATION_SLUG_PREFIXES.some(prefix => hit.slug.startsWith(prefix))) return 'conversation_slug';
return null;
}
/**
* The plan to run for these hits: an implied or config-level `auto` with no
* conversation hit is the chunk path (null), so responses without
* conversations stay byte-identical. An explicit `auto`, or an `auto` under
* an explicit budget the cap enforces, always runs and reports its
* per-result decision.
*/
export function effectivePlan(plan: EvidencePlan | null, hits: SearchResult[]): EvidencePlan | null {
if (!plan || plan.unit !== 'auto' || plan.explicitUnit || capEngaged(plan)) return plan;
return hits.some(h => conversationSignal(h) !== null) ? plan : null;
}
// ---------------------------------------------------------------------------
// auto: long non-conversation pages (search.long_page_unit, F2)
// ---------------------------------------------------------------------------
export const LONG_PAGE_UNIT_CONFIG_KEY = 'search.long_page_unit';
export const LONG_PAGE_UNITS = ['off', 'window', 'section', 'page'] as const;
export type LongPageUnit = Exclude<typeof LONG_PAGE_UNITS[number], 'off'>;
/** A page is long when its rendered body (compiled truth, plus timeline unless `detail: "low"`) holds at least this many characters. */
export const LONG_PAGE_MIN_CHARS_CONFIG_KEY = 'search.long_page_min_chars';
export const DEFAULT_LONG_PAGE_MIN_CHARS = 20_000;
/** Long pages per call that get the unit, in rank order; later ones keep their chunk (`long_page_pages`). */
export const LONG_PAGE_MAX_PAGES = 5;
/** Stored-body bytes those pages may bring in; a page that would cross it keeps its chunk (`long_page_bytes`), since bodies are read whole and never sliced. */
export const LONG_PAGE_MAX_INPUT_BYTES = 1_000_000;
/** Cooperative deadline for the size preflight plus the long-page fetch (`long_page_deadline`). */
export const LONG_PAGE_DEADLINE_MS = 2_000;
export interface LongPagePolicy { unit: LongPageUnit; minChars: number }
/** The long pages one call delivers in the policy's unit, and the ones that keep their chunk (page id → fallback code). */
export interface LongPageSelection {
unit: LongPageUnit;
minChars: number;
pages: ReadonlySet<number>;
kept: ReadonlyMap<number, string>;
deadlineAt: number;
fallbacks: string[];
}
export function longPageUnitValueProblem(value: string): string | null {
return (LONG_PAGE_UNITS as readonly string[]).includes(value.trim()) ? null : `${LONG_PAGE_UNIT_CONFIG_KEY} must be one of ${LONG_PAGE_UNITS.join(', ')} (got ${JSON.stringify(value.slice(0, 40))}).`;
}
export function longPageMinCharsValueProblem(value: string): string | null {
return /^[1-9][0-9]{0,8}$/.test(value.trim()) ? null : `${LONG_PAGE_MIN_CHARS_CONFIG_KEY} must be a positive whole number of characters (got ${JSON.stringify(value.slice(0, 40))}).`;
}
/**
* The F2 policy for a search/query call: only under an implied `auto` (no
* `return_unit`, `search.return_unit` unset or `auto`, no explicit budget).
* `snippet_chars` leaves it on; capDeliveredSnippets exempts widened rows.
* Read errors and unknown values are `off`; an unusable threshold is the default.
*/
export async function withLongPagePolicy(engine: BrainEngine, plan: EvidencePlan | null): Promise<EvidencePlan | null> {
if (!plan || plan.unit !== 'auto' || plan.explicitUnit || plan.budgetExplicit) return plan;
const raw = (await engine.getConfig(LONG_PAGE_UNIT_CONFIG_KEY).catch(() => null))?.trim();
if (raw !== 'window' && raw !== 'section' && raw !== 'page') return plan;
const min = (await engine.getConfig(LONG_PAGE_MIN_CHARS_CONFIG_KEY).catch(() => null)) ?? '';
return { ...plan, longPage: { unit: raw, minChars: longPageMinCharsValueProblem(min) ? DEFAULT_LONG_PAGE_MIN_CHARS : Number(min.trim()) } };
}
/**
* The plan to run for these hits once the long-page policy is applied: a
* size preflight (one query, the caller's read scope) finds the
* non-conversation pages whose stored body reaches the character threshold
* and keeps the rank-order ones inside the page and byte limits; delivery
* confirms the threshold on the rendered body. With no long page this is exactly effectivePlan, so a
* response without one is unchanged; a failed or late preflight is too.
*/
export async function selectLongPages(engine: BrainEngine, plan: EvidencePlan | null, hits: SearchResult[], scope: DeliveryScope): Promise<{ plan: EvidencePlan | null; longPages?: LongPageSelection }> {
const base = effectivePlan(plan, hits);
if (!plan?.longPage) return { plan: base };
const deadlineAt = Date.now() + LONG_PAGE_DEADLINE_MS;
const ids: number[] = [];
for (const h of hits) if (!h.fact_row && conversationSignal(h) === null && Number.isFinite(h.page_id) && !ids.includes(h.page_id)) ids.push(h.page_id);
if (ids.length === 0) return { plan: base };
const { unit, minChars } = plan.longPage;
let sizes: Map<number, { chars: number; bytes: number }>;
try {
const params: unknown[] = [ids];
const filter = pageReadFilter('p', scope, params, true);
const tl = scope.detail === 'low' ? null : `CASE WHEN COALESCE(p.timeline, '') !~ '\\S' THEN 0 ELSE ${TIMELINE_SEPARATOR.length} + `;
const rows = await withTimeout(engine.executeRaw<{ id: number; chars: number; bytes: number }>(
`SELECT p.id, char_length(COALESCE(p.compiled_truth, ''))${tl ? ` + ${tl}char_length(p.timeline) END` : ''} AS chars,
octet_length(COALESCE(p.compiled_truth, ''))${tl ? ` + ${tl}octet_length(p.timeline) END` : ''} AS bytes FROM pages p
WHERE p.id = ANY($1::int[]) AND ${filter} AND ${servedTextProjectionFilter('p')}${requiresSafeChunks(scope) ? ` AND ${safeChunksFilter('p')}` : ''}`,
params), LONG_PAGE_DEADLINE_MS);
sizes = new Map(rows.map(r => [Number(r.id), { chars: Number(r.chars), bytes: Number(r.bytes) }]));
} catch (e) {
return { plan: base, ...(base ? { longPages: { unit, minChars, pages: new Set(), kept: new Map(), deadlineAt, fallbacks: [e instanceof EvidenceTimeout ? 'long_page_deadline' : 'long_page_preflight_failed'] } } : {}) };
}
const pages = new Set<number>();
const kept = new Map<number, string>();
let total = 0;
for (const id of ids) {
const size = sizes.get(id);
if (size === undefined || size.chars < minChars) continue;
if (pages.size >= LONG_PAGE_MAX_PAGES) kept.set(id, 'long_page_pages');
else if (total + size.bytes > LONG_PAGE_MAX_INPUT_BYTES) kept.set(id, 'long_page_bytes');
else { pages.add(id); total += size.bytes; }
}
if (pages.size === 0 && kept.size === 0) return { plan: base };
return { plan, longPages: { unit, minChars, pages, kept, deadlineAt, fallbacks: [] } };
}
// ---------------------------------------------------------------------------
// Tokens
// ---------------------------------------------------------------------------
export function evidenceTokenizer(): 'cl100k' | 'heuristic' {
return cl100kAvailable() ? 'cl100k' : 'heuristic';
}
function pieceTokens(text: string, tokenizer: 'cl100k' | 'heuristic'): number {
if (!text) return 0;
return tokenizer === 'cl100k' ? cl100kTokens(text) : heuristicTokens(text);
}
/** Split into lines (newline kept); lines over PIECE_MAX_CHARS split at whitespace. */
export function splitPieces(text: string): Array<{ start: number; end: number; lineStart: boolean }> {
const out: Array<{ start: number; end: number; lineStart: boolean }> = [];
let i = 0;
while (i < text.length) {
const nl = text.indexOf('\n', i);
const lineEnd = nl === -1 ? text.length : nl + 1;
let s = i;
let first = true;
while (s < lineEnd) {
let e = lineEnd;
if (e - s > PIECE_MAX_CHARS) {
e = s + PIECE_MAX_CHARS;
const ws = text.slice(s + PIECE_MAX_CHARS - 80, e).search(/\s\S*$/);
e = ws >= 0 ? s + PIECE_MAX_CHARS - 80 + ws + 1 : safeSplitIndex(text, e);
if (e <= s) e = Math.min(lineEnd, s + PIECE_MAX_CHARS);
}
out.push({ start: s, end: e, lineStart: first });
first = false;
s = e;
}
i = lineEnd;
}
return out;
}
/** Deterministic token count: the sum of per-piece counts (see the doc). */
export function countEvidenceTokens(text: string, tokenizer: 'cl100k' | 'heuristic' = evidenceTokenizer()): number {
let n = 0;
for (const p of splitPieces(text)) n += pieceTokens(text.slice(p.start, p.end), tokenizer);
return n;
}
function sliceToTokenCount(text: string, max: number, tokenizer: 'cl100k' | 'heuristic'): string {
if (max <= 0) return '';
if (countEvidenceTokens(text, tokenizer) <= max) return text;
let lo = 0;
let hi = text.length;
while (lo < hi) {
const mid = Math.ceil((lo + hi) / 2);
if (countEvidenceTokens(text.slice(0, safeSplitIndex(text, mid)), tokenizer) <= max) lo = mid;
else hi = mid - 1;
}
return text.slice(0, safeSplitIndex(text, lo));
}
// ---------------------------------------------------------------------------
// Locating chunks in the sanitized body
// ---------------------------------------------------------------------------
export { TIMELINE_SEPARATOR };
/** Whitespace runs collapsed to one space, with a map back to original offsets. */
function squeeze(text: string): { s: string; map: number[] } {
const parts: string[] = [];
const map: number[] = [];
let runStart = 0;
const WS = /\s+/g;
for (const m of text.matchAll(WS)) {
const at = m.index!;
if (at > runStart) {
parts.push(text.slice(runStart, at));
for (let i = runStart; i < at; i++) map.push(i);
}
parts.push(' ');
map.push(at);
runStart = at + m[0].length;
}
if (runStart < text.length) {
parts.push(text.slice(runStart));
for (let i = runStart; i < text.length; i++) map.push(i);
}
return { s: parts.join(''), map };
}
/**
* Locate each chunk (chunk_index order) in the page text the chunks were cut
* from. The chunker trims chunks and can fold whitespace-only runs, so the
* match ignores whitespace differences; consecutive chunks overlap, so each
* search starts just after the previous chunk's start. Unlocated chunks are absent.
*/
export function locateChunks(text: string, chunks: Array<{ id: number; chunk_index: number; chunk_text: string }>, fold = true): Array<{ id: number; chunk_index: number; start: number; end: number }> {
let body: ReturnType<typeof squeeze> | null = null;
const out: Array<{ id: number; chunk_index: number; start: number; end: number }> = [];
let from = 0;
for (const c of [...chunks].sort((a, b) => a.chunk_index - b.chunk_index)) {
const exact = c.chunk_text.trim();
if (exact.length === 0) continue;
let at = text.indexOf(exact, from);
if (at < 0) at = text.indexOf(exact);
if (at >= 0) {
out.push({ id: c.id, chunk_index: c.chunk_index, start: at, end: at + exact.length });
from = at + 1;
continue;
}
if (!fold) continue;
body ??= squeeze(text);
const needle = squeeze(exact).s;
let lo = 0;
let hi = body.map.length;
while (lo < hi) { const mid = (lo + hi) >> 1; if (body.map[mid] < from) lo = mid + 1; else hi = mid; }
let sq = body.s.indexOf(needle, lo);
if (sq < 0) sq = body.s.indexOf(needle);
if (sq < 0) continue;
out.push({ id: c.id, chunk_index: c.chunk_index, start: body.map[sq], end: body.map[sq + needle.length - 1] + 1 });
from = body.map[sq] + 1;
}
return out;
}
// ---------------------------------------------------------------------------
// Document model
// ---------------------------------------------------------------------------
interface Piece {
run: number;
start: number;
end: number;
text: string;
lineStart: boolean;
heading: number;
speaker: string | null;
tok: number;
}
interface Doc {
text: string;
pieces: Piece[];
spans: Array<{ id: number; chunk_index: number; start: number; end: number }>;
}
const HEADING = /^ {0,3}(#{1,6})[ \t]+\S/;
const FENCE = /^ {0,3}(```|~~~)/;
const SPEAKER = /^\s{0,3}(?:\*\*([^*\n:]{1,40}):\s*\*\*|\*\*([^*\n:]{1,40})\*\*\s*:|(user|assistant|human|ai|system|bot|speaker\s*\d{1,3})\s*:)/i;
function buildDoc(text: string, spans: Doc['spans']): Doc {
const pieces: Piece[] = [];
let inFence = false;
for (const p of splitPieces(text)) {
const t = text.slice(p.start, p.end);
let heading = 0;
let speaker: string | null = null;
if (p.lineStart) {
if (FENCE.test(t)) inFence = !inFence;
else if (!inFence) {
const h = HEADING.exec(t);
if (h) heading = h[1].length;
const sp = SPEAKER.exec(t);
if (sp) speaker = (sp[1] ?? sp[2] ?? sp[3]).trim().toLowerCase().replace(/\s+/g, ' ');
}
}
pieces.push({ run: 0, start: p.start, end: p.end, text: t, lineStart: p.lineStart, heading, speaker, tok: -1 });
}
return { text, pieces, spans };
}
/** ≥ 4 speaker-turn lines, ≥ 2 distinct labels, ≥ 3 label changes. */
export function isConversationLabels(labels: string[]): boolean {
if (labels.length < 4 || new Set(labels).size < 2) return false;
let changes = 0;
for (let i = 1; i < labels.length; i++) if (labels[i] !== labels[i - 1]) changes++;
return changes >= 3;
}
function piecesOverlapping(doc: Doc, start: number, end: number): number[] {
let lo = 0;
let hi = doc.pieces.length;
while (lo < hi) {
const mid = (lo + hi) >> 1;
if (doc.pieces[mid].end <= start) lo = mid + 1;
else hi = mid;
}
const out: number[] = [];
for (let i = lo; i < doc.pieces.length && doc.pieces[i].start < end; i++) out.push(i);
return out;
}
// ---------------------------------------------------------------------------
// Blocks
// ---------------------------------------------------------------------------
interface Anchor {
chunk_id: number;
mapped: boolean;
pieces: number[];
start: number;
end: number;
chunkIndex: number;
}
interface Block {
hit: SearchResult;
hits: SearchResult[];
doc: Doc;
unit: DeliveredUnit;
candidates: number[];
anchors: Anchor[];
revision?: string;
fallbackReason?: string;
titleTok: number;
/** The date header line plus its newline ('' when off), counted in the block's floor. */
header: string;
headerTok: number;
selected: Set<number>;
cut: boolean;
title: string;
/** Index of the block's best hit in the ranked input. */
rank: number;
reason?: AutoReason;
/** The cap cut this block: its text ends with EVIDENCE_CUT_MARKER. */
capMarker?: boolean;
}
type PlannedBlock = Omit<Block, 'titleTok' | 'title' | 'rank' | 'reason' | 'capMarker' | 'header' | 'headerTok'>;
/**
* The hit's text as it appears in the page: a fenced_code chunk carries the
* code chunker's synthesized `[Lang] fence.ts:N-M symbol` header, which is
* not page text, so it is stripped before the chunk anchors or is delivered.
*/
function hitPageText(hit: SearchResult): string {
const text = hit.chunk_text ?? '';
return hit.chunk_source === 'fenced_code' ? stripChunkHeader(text) : text;
}
function fallbackBlock(hit: SearchResult, hits: SearchResult[], reason: string): PlannedBlock {
const text = hitPageText(hit);
const doc = buildDoc(text, []);
const all = doc.pieces.map((_, i) => i);
return {
hit, hits, doc, unit: 'chunk', candidates: all,
anchors: [
{ chunk_id: hit.chunk_id, mapped: true, pieces: all, start: 0, end: text.length, chunkIndex: hit.chunk_index },
...hits.slice(1).map(h => ({ chunk_id: h.chunk_id, mapped: false, pieces: [], start: 0, end: 0, chunkIndex: h.chunk_index })),
],
fallbackReason: reason, selected: new Set(), cut: false,
};
}
/**
* The page text evidence is cut from: the one projected page text every
* windowed read slices (page-window/document.ts): the complete stored body,
* sanitized with the strict protected-body boundary BEFORE any slicing (Takes,
* non-world and withdrawn Facts rows, malformed protected tails, materialized
* markers), compiled truth and timeline each sanitized whole, run through the
* same credential-safe projection the chunkers cut from, and joined the way
* serializeMarkdown joins them. Frontmatter is not part of it.
*/
export function pageEvidenceText(page: { compiled_truth: string; timeline: string; fenceOverlay?: FenceChunkOverlay }, includeTimeline: boolean): { text: string; timelineAt: number } {
return projectPageText(page, includeTimeline);
}
/** `placed` (segmented pages): where the hits without a stored span were found; the text is then never searched. */
function locateAnchor(doc: Doc, hit: SearchResult, fold: boolean, placed?: ReadonlyMap<number, { start: number; end: number }>): Anchor {
const byId = hit.chunk_id > 0 ? doc.spans.find(x => x.id === hit.chunk_id) : undefined;
if (byId) return { chunk_id: hit.chunk_id, mapped: true, pieces: piecesOverlapping(doc, byId.start, byId.end), start: byId.start, end: byId.end, chunkIndex: byId.chunk_index };
const text = hitPageText(hit).trim();
if (placed || text.length >= 16) {
const found = placed ? placed.get(hit.chunk_id) : locateChunks(doc.text, [{ id: hit.chunk_id, chunk_index: hit.chunk_index, chunk_text: text }], fold)[0];
if (found) {
// A fenced_code chunk_index follows every prose chunk, so it says nothing
// about position: the window grows from the prose chunk holding the fence,
// or not at all when none was fetched.
const chunkIndex = hit.chunk_source !== 'fenced_code' ? hit.chunk_index
: doc.spans.find(sp => sp.start <= found.start && found.end <= sp.end)?.chunk_index ?? Number.NEGATIVE_INFINITY;
return { chunk_id: hit.chunk_id, mapped: true, pieces: piecesOverlapping(doc, found.start, found.end), start: found.start, end: found.end, chunkIndex };
}
}
// Positional anchor (the current revision no longer holds the hit chunk, or
// a synthetic chunk_id 0 row): the nearest located chunk drives the unit
// cut, but no match span is reported.
let best: Doc['spans'][number] | null = null;
for (const sp of doc.spans) if (!best || Math.abs(sp.chunk_index - hit.chunk_index) < Math.abs(best.chunk_index - hit.chunk_index)) best = sp;
if (!best) return { chunk_id: hit.chunk_id, mapped: false, pieces: [], start: 0, end: 0, chunkIndex: hit.chunk_index };
const mapped = hit.chunk_id === 0 && best.chunk_index === hit.chunk_index;
return { chunk_id: hit.chunk_id, mapped, pieces: piecesOverlapping(doc, best.start, best.end), start: best.start, end: best.end, chunkIndex: best.chunk_index };
}
function windowCandidates(doc: Doc, anchors: Anchor[], n: number): number[] {
const set = new Set<number>();
for (const a of anchors) {
for (const sp of doc.spans) {
if (Math.abs(sp.chunk_index - a.chunkIndex) <= n) for (const p of piecesOverlapping(doc, sp.start, sp.end)) set.add(p);
}
for (const p of a.pieces) set.add(p);
}
return [...set].sort((x, y) => x - y);
}
function sectionCandidates(doc: Doc, anchors: Anchor[], conversation: boolean): number[] | null {
const set = new Set<number>();
let opener: string | null = null;
if (conversation) {
const labels = doc.pieces.map(p => p.speaker).filter((x): x is string => x !== null);
opener = labels.find(l => l === 'user' || l === 'human') ?? labels[0] ?? null;
}
for (const a of anchors) {
if (a.pieces.length === 0) continue;
const first = a.pieces[0];
const last = a.pieces[a.pieces.length - 1];
// A section never leaves its run (a segmented page's window): the text between runs was not read.
let lo = first;
while (lo > 0 && doc.pieces[lo - 1].run === doc.pieces[first].run) lo--;
let hi = last;
while (hi + 1 < doc.pieces.length && doc.pieces[hi + 1].run === doc.pieces[last].run) hi++;
let start = lo;
let level = 0;
for (let i = first; i >= lo; i--) {
const p = doc.pieces[i];
if (conversation ? p.speaker === opener : p.heading > 0) { start = i; level = conversation ? 0 : p.heading; break; }
}
let end = hi;
for (let i = last + 1; i <= hi; i++) {
const p = doc.pieces[i];
if (conversation ? p.speaker === opener : p.heading > 0 && (level === 0 || p.heading <= level)) { end = i - 1; break; }
}
for (let i = start; i <= end; i++) set.add(i);
}
return set.size === 0 ? null : [...set].sort((x, y) => x - y);
}
/** Large pages: above this size hits are located by their stored spans only (no whole-text whitespace folding). */
const SPAN_ONLY_LOCATE_BYTES = 1024 * 1024;
/** fallback_reason when a 'page' (or auto) unit was cut to a window because the page is over the inline read size. */
export const PAGE_OVER_INLINE_BYTES = 'page_over_inline_bytes';
function planBlock(page: ChunkWindowPage, hits: SearchResult[], requested: Exclude<ReturnUnit, 'chunk' | 'auto'>, radius: number, includeTimeline: boolean): PlannedBlock {
const best = hits[0];
if (!page.sealed) return fallbackBlock(best, hits, 'unsealed_page');
if (page.chunks.length === 0 && page.row_limited) return fallbackBlock(best, hits, 'row_limit');
const { text, timelineAt } = pageEvidenceText(page, includeTimeline);
const textBytes = Buffer.byteLength(text, 'utf8');
const truthChunks = page.chunks.filter(c => c.chunk_source === 'compiled_truth');
const timelineChunks = page.chunks.filter(c => c.chunk_source === 'timeline');
// Large pages: above 1 MiB the text is never whitespace-folded whole; hits are anchored by their stored spans only.
const spans = textBytes > SPAN_ONLY_LOCATE_BYTES ? locateChunksByStoredSpans(text, timelineAt, page.chunks) : [
...locateChunks(timelineAt < 0 ? text : text.slice(0, timelineAt - TIMELINE_SEPARATOR.length), truthChunks),
...(timelineAt < 0 ? [] : locateChunks(text.slice(timelineAt), timelineChunks).map(sp => ({ ...sp, start: sp.start + timelineAt, end: sp.end + timelineAt }))),
];
const doc = buildDoc(text, spans);
return blockFromDoc(page, hits, doc, hits.map(h => locateAnchor(doc, h, textBytes <= SPAN_ONLY_LOCATE_BYTES)), requested, radius, textBytes > DEFAULT_INLINE_BODY_BYTES);
}
/**
* planBlock for a segmented page (spec 7.4): the same units over the windows readSegmentedWindows reads around the
* hits, never a page text, a whole-page projection or a text search. Each window is a run of the document: emission
* marks the gap between two runs and a section never crosses one.
*/
async function planSegmentedBlock(engine: BrainEngine, page: ChunkWindowPage, hits: SearchResult[], requested: Exclude<ReturnUnit, 'chunk' | 'auto'>, radius: number, includeTimeline: boolean): Promise<PlannedBlock> {
const best = hits[0];
if (!page.sealed) return fallbackBlock(best, hits, 'unsealed_page');
if (page.chunks.length === 0 && page.row_limited) return fallbackBlock(best, hits, 'row_limit');
const read = await readSegmentedWindows(engine, page, hits.map(h => ({ chunk_id: h.chunk_id, chunk_index: h.chunk_index, text: hitPageText(h) })),
{ unit: requested, radius, includeTimeline });
const doc: Doc = { text: '', pieces: [], spans: [] };
const placed = new Map<number, { start: number; end: number }>();
read.runs.forEach((run, k) => {
const base = doc.text.length;
for (const p of buildDoc(run.text, []).pieces) doc.pieces.push({ ...p, run: k, start: p.start + base, end: p.end + base });
for (const sp of run.spans) doc.spans.push({ ...sp, start: sp.start + base, end: sp.end + base });
for (const [id, at] of read.placed) if (at.run === k) placed.set(id, { start: at.start + base, end: at.end + base });
doc.text += run.text;
});
return blockFromDoc(page, hits, doc, hits.map(h => locateAnchor(doc, h, false, placed)), requested, radius, read.overInline);
}
function blockFromDoc(page: ChunkWindowPage, hits: SearchResult[], doc: Doc, anchors: Anchor[], requested: Exclude<ReturnUnit, 'chunk' | 'auto'>, radius: number, overInline: boolean): PlannedBlock {
const best = hits[0];
if (anchors.every(a => a.pieces.length === 0)) {
return fallbackBlock(best, hits, page.row_limited ? 'row_limit' : page.chunks.length === 0 ? 'no_text_chunks' : 'anchor_not_located');
}
const labels = doc.pieces.map(p => p.speaker).filter((x): x is string => x !== null);
const conversation = isConversationLabels(labels);
const hasHeadings = doc.pieces.some(p => p.heading > 0);
let unit: DeliveredUnit = requested;
let fallbackReason: string | undefined;
let candidates: number[];
// Large pages (T18): a page over the inline read size is never delivered whole; it becomes the window around its hits.
if (unit === 'page' && overInline) { unit = 'window'; fallbackReason = PAGE_OVER_INLINE_BYTES; }
if (unit === 'page') {
candidates = doc.pieces.map((_, i) => i);
} else if (unit === 'section') {
const sec = conversation || hasHeadings ? sectionCandidates(doc, anchors, conversation) : null;
if (sec) candidates = sec;
else {
unit = 'window';
fallbackReason = 'no_section_structure';
candidates = windowCandidates(doc, anchors, radius);
}
} else {
candidates = windowCandidates(doc, anchors, radius);
}
return {
hit: best, hits, doc, unit, candidates, anchors,
revision: page.revision,
...(fallbackReason ? { fallbackReason } : {}),
selected: new Set(), cut: false,
};
}
// ---------------------------------------------------------------------------
// Allocation + emission
// ---------------------------------------------------------------------------
function tok(b: Block, i: number, tokenizer: 'cl100k' | 'heuristic'): number {
const p = b.doc.pieces[i];
if (p.tok < 0) p.tok = pieceTokens(p.text, tokenizer);
return p.tok;
}
function adjacent(b: Block, i: number): boolean {
const p = b.doc.pieces[i];
return (b.selected.has(i - 1) && b.doc.pieces[i - 1].run === p.run) || (b.selected.has(i + 1) && b.doc.pieces[i + 1]?.run === p.run);
}
function enrichmentOrder(b: Block): number[] {
const core = new Set(b.anchors[0]?.pieces ?? []);
const candPos = new Map(b.candidates.map((c, i) => [c, i]));
const coreRank = new Map<number, number>();
b.anchors.slice(1).forEach((a, r) => { for (const p of a.pieces) if (!coreRank.has(p)) coreRank.set(p, r); });
const seeds = b.candidates.map((c, i) => (core.has(c) || coreRank.has(c) ? i : -1)).filter(i => i >= 0);
const dist = new Array<number>(b.candidates.length).fill(Number.MAX_SAFE_INTEGER);
for (const s of seeds) dist[s] = 0;
for (let i = 1; i < dist.length; i++) dist[i] = Math.min(dist[i], dist[i - 1] + 1);
for (let i = dist.length - 2; i >= 0; i--) dist[i] = Math.min(dist[i], dist[i + 1] + 1);
return b.candidates
.filter(c => !core.has(c))
.sort((x, y) => {
const ox = coreRank.has(x) ? 0 : 1;
const oy = coreRank.has(y) ? 0 : 1;
if (ox !== oy) return ox - oy;
if (ox === 0 && coreRank.get(x)! !== coreRank.get(y)!) return coreRank.get(x)! - coreRank.get(y)!;
const dx = dist[candPos.get(x)!];
const dy = dist[candPos.get(y)!];
return dx - dy || x - y;
});
}
/** Rank one alone exceeds the budget: cut its title to `titleMax`, then keep core pieces (slicing the first) to fit. */
function cutToFit(b: Block, remaining: number, titleMax: number, tokenizer: 'cl100k' | 'heuristic'): number {
const core = b.anchors[0]?.pieces ?? [];
// The date header line is kept whole and paid for first.
remaining = Math.max(0, remaining - b.headerTok);
titleMax = Math.max(0, titleMax - b.headerTok);
if (b.titleTok > titleMax) {
b.title = sliceToTokenCount(b.title, titleMax, tokenizer);
b.titleTok = countEvidenceTokens(b.title, tokenizer);
}
remaining -= b.titleTok;
let chars = 0;
for (const i of core) {
const t = tok(b, i, tokenizer);
const len = b.doc.pieces[i].text.length;
if (t <= remaining && chars + len <= EVIDENCE_BLOCK_CHAR_CAP) {
b.selected.add(i);
remaining -= t;
chars += len;
continue;
}
if (b.selected.size === 0) {
const p = b.doc.pieces[i];
p.text = sliceToTokenCount(p.text.slice(0, EVIDENCE_BLOCK_CHAR_CAP), remaining, tokenizer);
p.end = p.start + p.text.length;
p.tok = pieceTokens(p.text, tokenizer);
b.selected.add(i);
remaining -= p.tok;
}
break;
}
b.cut = true;
return remaining;
}
/** Grow a kept block in enrichment order until the next piece does not fit; returns what is left. */
function enrichBlock(b: Block, remaining: number, tokenizer: 'cl100k' | 'heuristic', omitTok: number): number {
let chars = [...b.selected].reduce((n, i) => n + b.doc.pieces[i].text.length, 0);
for (const i of enrichmentOrder(b)) {
if (b.selected.has(i)) continue;
const adj = adjacent(b, i);
const cost = tok(b, i, tokenizer) + (adj ? 0 : omitTok);
const len = b.doc.pieces[i].text.length + (adj ? 0 : EVIDENCE_OMISSION.length);
if (cost > remaining || chars + len > EVIDENCE_BLOCK_CHAR_CAP) break;
b.selected.add(i);
remaining -= cost;
chars += len;
}
return remaining;
}
function floorOf(b: Block, tokenizer: 'cl100k' | 'heuristic'): { tokens: number; chars: number } {
const core = b.anchors[0]?.pieces ?? [];
return {
tokens: b.titleTok + b.headerTok + core.reduce((n, i) => n + tok(b, i, tokenizer), 0),
chars: core.reduce((n, i) => n + b.doc.pieces[i].text.length, 0),
};
}
/**
* Reserve each block's floor in rank order, then enrich. `spill` (auto) takes
* a block whose floor does not fit instead of dropping or cutting it.
*/
/** What delivery emits for `b` with an unlimited budget: its floor, then every enrichment piece the block char cap allows. */
function wholeCost(b: Block, tokenizer: 'cl100k' | 'heuristic', omitTok: number): number {
const selected = b.selected;
b.selected = new Set(b.anchors[0]?.pieces ?? []);
const cost = floorOf(b, tokenizer).tokens + Number.MAX_SAFE_INTEGER - enrichBlock(b, Number.MAX_SAFE_INTEGER, tokenizer, omitTok);
b.selected = selected;
return cost;
}
function allocate(blocks: Block[], budget: number, tokenizer: 'cl100k' | 'heuristic', dropped: Record<string, number>, spill?: (b: Block) => void): Block[] {
const omitTok = pieceTokens(EVIDENCE_OMISSION, tokenizer);
let remaining = budget;
const kept: Block[] = [];
let droppedAny = false;
for (const b of blocks) {
const core = b.anchors[0]?.pieces ?? [];
const floor = floorOf(b, tokenizer);
if (floor.tokens <= remaining && floor.chars <= EVIDENCE_BLOCK_CHAR_CAP) {
for (const i of core) b.selected.add(i);
remaining -= floor.tokens;
kept.push(b);
continue;
}
if (spill) { spill(b); continue; }
if (kept.length === 0 && !droppedAny) {
// Rank one alone exceeds the budget: cut it to fit (minKeep).
remaining = cutToFit(b, remaining, remaining, tokenizer);
kept.push(b);
continue;
}
droppedAny = true;
dropped.budget_floor = (dropped.budget_floor ?? 0) + 1;
}
for (const b of kept) {
if (b.cut) continue;
remaining = enrichBlock(b, remaining, tokenizer, omitTok);
}
return kept;
}
/** A non-conversation hit under `auto`: its ranked chunk, or a cut prefix of it under the cap. */
/** `text` starts with the hit's date header line (`header`, '' when off); a cut keeps the header. */
interface ChunkItem { rank: number; hit: SearchResult; reason: AutoReason; title: string; text: string; cut: boolean; header: string }
/** A selection as emit writes it: the pieces plus one omission line per gap. */
function selectionCost(b: Block, sel: Set<number>, tokenizer: 'cl100k' | 'heuristic', omitTok: number): { tokens: number; chars: number } {
const order = [...sel].sort((x, y) => x - y);
let tokens = 0;
let chars = 0;
order.forEach((i, k) => {
const gap = k > 0 && !(order[k - 1] === i - 1 && b.doc.pieces[i - 1].run === b.doc.pieces[i].run);
tokens += tok(b, i, tokenizer) + (gap ? omitTok : 0);
chars += b.doc.pieces[i].text.length + (gap ? EVIDENCE_OMISSION.length : 0);
});
return { tokens, chars };
}
/**
* The cap (explicit budget under `auto`, docs/evidence-delivery.md
* "Explicit budgets"): global rank one first, cut to fit if it alone exceeds
* the budget; then the rank-order prefix of the other non-conversation chunks
* that fits; then conversations by the plan's packing. Nothing is spilled
* outside the budget; what does not fit is counted in `dropped`.
*/
function allocateCapped(blocks: Block[], chunks: ChunkItem[], plan: EvidencePlan, tokenizer: 'cl100k' | 'heuristic', dropped: Record<string, number>): { kept: Block[]; chunks: ChunkItem[] } {
const omitTok = pieceTokens(EVIDENCE_OMISSION, tokenizer);
const cutTok = countEvidenceTokens(EVIDENCE_CUT_MARKER, tokenizer);
const drop = (reason: string) => { dropped[reason] = (dropped[reason] ?? 0) + 1; };
const chunkCost = (c: ChunkItem) => countEvidenceTokens(c.title, tokenizer) + countEvidenceTokens(c.text, tokenizer);
let remaining = plan.budgetTokens;
const keptChunks: ChunkItem[] = [];
const kept: Block[] = [];
let lead: Block | null = null;
const firstChunk = chunks[0];
const firstBlock = blocks[0];
let rest = chunks;
let others = blocks;
if (firstChunk && (!firstBlock || firstChunk.rank < firstBlock.rank)) {
rest = chunks.slice(1);
const cost = chunkCost(firstChunk);
if (cost <= remaining) remaining -= cost;
else {
const room = remaining - cutTok;
let title = firstChunk.title;
if (countEvidenceTokens(title, tokenizer) > room - CUT_BODY_MIN) title = sliceToTokenCount(title, Math.max(0, room - CUT_BODY_MIN), tokenizer);
const bodyRoom = room - countEvidenceTokens(title, tokenizer);
const source = firstChunk.text.slice(0, EVIDENCE_BLOCK_CHAR_CAP);
let end = 0;
let used = 0;
for (const piece of splitPieces(source)) {
const t = pieceTokens(source.slice(piece.start, piece.end), tokenizer);
if (used + t > bodyRoom) break;
used += t;
end = piece.end;
}
const body = end > 0 ? source.slice(0, end).replace(/\s+$/, '') : sliceToTokenCount(source, Math.max(0, bodyRoom), tokenizer);
Object.assign(firstChunk, { title, text: body + EVIDENCE_CUT_MARKER, cut: true });
remaining -= chunkCost(firstChunk);
}
keptChunks.push(firstChunk);
} else if (firstBlock) {
others = blocks.slice(1);
const floor = floorOf(firstBlock, tokenizer);
if (floor.tokens <= remaining && floor.chars <= EVIDENCE_BLOCK_CHAR_CAP) {
for (const i of firstBlock.anchors[0]?.pieces ?? []) firstBlock.selected.add(i);
remaining -= floor.tokens;
lead = firstBlock;
} else {
remaining = cutToFit(firstBlock, remaining - cutTok, Math.max(0, remaining - cutTok - CUT_BODY_MIN), tokenizer);
firstBlock.capMarker = true;
}
kept.push(firstBlock);
}
let stopped = false;
for (const c of rest) {
const cost = chunkCost(c);
if (!stopped && cost <= remaining) { remaining -= cost; keptChunks.push(c); continue; }
stopped = true;
drop('budget_note');
}
const reserveFloor = (b: Block): boolean => {
const floor = floorOf(b, tokenizer);
if (floor.tokens > remaining || floor.chars > EVIDENCE_BLOCK_CHAR_CAP) return false;
for (const i of b.anchors[0]?.pieces ?? []) b.selected.add(i);
remaining -= floor.tokens;
return true;
};
if (plan.packing === 'depth_first') {
if (lead) remaining = enrichBlock(lead, remaining, tokenizer, omitTok);
for (const b of others) {
if (!reserveFloor(b)) { drop('budget_floor'); continue; }
kept.push(b);
remaining = enrichBlock(b, remaining, tokenizer, omitTok);
}
} else if (plan.packing === 'breadth_capped') {
// k: the longest rank-order prefix of conversations whose title, floor
// and target window (the window unit's candidates at return_window) fit.
let capped = false;
for (const b of lead ? [lead, ...others] : others) {
if (capped) { drop('breadth_cap'); continue; }
const window = new Set(windowCandidates(b.doc, b.anchors, plan.window));
const target = new Set(b.anchors[0]?.pieces ?? []);
for (const i of enrichmentOrder(b)) {
if (!window.has(i)) continue;
target.add(i);
if (selectionCost(b, target, tokenizer, omitTok).chars > EVIDENCE_BLOCK_CHAR_CAP) { target.delete(i); break; }
}
const sel = selectionCost(b, target, tokenizer, omitTok);
const cost = b.titleTok + b.headerTok + sel.tokens - (b === lead ? floorOf(b, tokenizer).tokens : 0);
if (cost <= remaining && sel.chars <= EVIDENCE_BLOCK_CHAR_CAP) {
for (const i of target) b.selected.add(i);
remaining -= cost;
if (b !== lead) kept.push(b);
continue;
}
capped = true;
if (b !== lead) drop('breadth_cap');
}
for (const b of kept) if (!b.cut) remaining = enrichBlock(b, remaining, tokenizer, omitTok);
} else {
for (const b of others) {
if (reserveFloor(b)) kept.push(b);
else drop('budget_floor');
}
for (const b of kept) if (!b.cut) remaining = enrichBlock(b, remaining, tokenizer, omitTok);
}
return { kept, chunks: keptChunks };
}
function emit(b: Block, tokenizer: 'cl100k' | 'heuristic'): { text: string; spans: MatchSpan[]; unmapped: number[]; tokens: number } {
const order = [...b.selected].sort((x, y) => x - y);
const emitStart = new Map<number, number>();
let text = '';
let gaps = 0;
let tokens = 0;
order.forEach((i, k) => {
const prev = order[k - 1];
if (k > 0 && !(prev === i - 1 && b.doc.pieces[prev].run === b.doc.pieces[i].run)) {
text += EVIDENCE_OMISSION;
gaps++;
}
emitStart.set(i, text.length);
text += b.doc.pieces[i].text;
tokens += tok(b, i, tokenizer);
});
tokens += gaps * pieceTokens(EVIDENCE_OMISSION, tokenizer);
const trimmed = text.replace(/\s+$/, '');
const spans: MatchSpan[] = [];
const unmapped: number[] = [];
for (const a of b.anchors) {
if (!a.mapped) { unmapped.push(a.chunk_id); continue; }
const sel = a.pieces.filter(i => b.selected.has(i));
if (sel.length === 0) { unmapped.push(a.chunk_id); continue; }
let segStart = sel[0];
for (let k = 0; k < sel.length; k++) {
const i = sel[k];
const next = sel[k + 1];
if (next === i + 1 && emitStart.get(next)! === emitStart.get(i)! + b.doc.pieces[i].text.length) continue;
const first = b.doc.pieces[segStart];
const last = b.doc.pieces[i];
const start = emitStart.get(segStart)! + Math.max(0, a.start - first.start);
const end = Math.min(trimmed.length, emitStart.get(i)! + Math.min(last.text.length, a.end - last.start));
if (end > start) spans.push({ chunk_id: a.chunk_id, start, end });
segStart = next;
}
}
return { text: trimmed, spans, unmapped, tokens };
}
// ---------------------------------------------------------------------------
// The stage
// ---------------------------------------------------------------------------
export interface DeliverOptions {
/**
* The hits came from this request's live, authorized retrieval: a fetch
* failure may fall back to the hit chunk text. When false (cached hits),
* a fetch failure drops the result instead — uncertain authorization
* never fails open to cached text.
*/
liveHits?: boolean;
timeoutMs?: number;
/** F2: the long pages selectLongPages chose for these hits (search/query only). */
longPages?: LongPageSelection;
/** The request's query: the tail trim's query-term guard reads it, and the trim never runs without it. */
query?: string;
}
function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
const t = new Promise<never>((_, reject) => { timer = setTimeout(() => reject(new EvidenceTimeout()), ms); });
return Promise.race([p, t]).finally(() => clearTimeout(timer));
}
class EvidenceTimeout extends Error {}
/**
* Assemble delivered evidence for ranked hits. Returns NEW result objects
* (one per page, at the page's best rank) plus the delivery meta. The hit
* array and its objects are never mutated.
*/
export async function deliverEvidence(
engine: BrainEngine,
hits: SearchResult[],
plan: EvidencePlan,
scope: DeliveryScope,
opts: DeliverOptions = {},
): Promise<{ results: DeliveredSearchResult[]; delivery: DeliveryMeta }> {
const tokenizer = evidenceTokenizer();
const fallbacks = new Set<string>();
const dropped: Record<string, number> = {};
if (plan.budgetClamped) fallbacks.add('budget_clamped');
if (tokenizer === 'heuristic') fallbacks.add('tokenizer_heuristic');
// auto keeps every non-conversation hit as its ranked chunk, unchanged;
// conversation hits are grouped by page like every other unit.
const auto = plan.unit === 'auto';
const passthrough: Array<{ rank: number; hit: SearchResult; reason: AutoReason }> = [];
type Group = { pageId: number | null; hits: SearchResult[]; rank: number; reason?: AutoReason };
const groups: Group[] = [];
const byPage = new Map<number, Group>();
const long = auto ? opts.longPages : undefined;
for (const code of long?.fallbacks ?? []) fallbacks.add(code);
hits.forEach((h, rank) => {
// A saved-fact row (facts arm) has no page to expand: it is delivered as written, paid for first.
if (h.fact_row) { passthrough.push({ rank, hit: h, reason: 'saved_fact' }); return; }
const signal = auto ? conversationSignal(h) : null;
const longPage = !signal && long?.pages.has(h.page_id) === true;
if (auto && !signal && !longPage) {
const code = long?.kept.get(h.page_id);
if (code) fallbacks.add(code);
passthrough.push({ rank, hit: h, reason: code ? 'long_page_chunk' : 'not_conversation' });
return;
}
const id = Number.isFinite(h.page_id) ? h.page_id : null;
const g = id === null ? undefined : byPage.get(id);
if (g) { g.hits.push(h); return; }
const ng: Group = { pageId: id, hits: [h], rank, ...(signal ? { reason: signal } : longPage ? { reason: 'long_page' as const } : {}) };
if (id !== null) byPage.set(id, ng);
groups.push(ng);
});
// Chunks only anchor hits in the page text; every unit but window (and the
// section fallback to window) needs just the hit chunks themselves. Long
// pages (F2) are read in their own bounded call under the long-page
// deadline, so they can neither slow nor fail the conversation fetch.
const fetchPages = async (want: Group[], timeoutMs: number): Promise<Map<number, ChunkWindowPage> | string> => {
const requests: ChunkWindowRequest[] = [];
want.forEach((g, rank) => {
if (g.pageId === null) return;
for (const h of g.hits) {
const at = Number.isFinite(h.chunk_index) ? h.chunk_index : 0;
requests.push({ page_id: g.pageId, from_index: Math.max(0, at - plan.window), to_index: at + plan.window, priority: rank });
}
});
if (requests.length === 0) return new Map();
if (timeoutMs <= 0) return 'fetch_timeout';
try {
const rows = await withTimeout(engine.getChunkWindows(requests, {
...(scope.sourceIds && scope.sourceIds.length > 0 ? { sourceIds: scope.sourceIds } : scope.sourceId ? { sourceId: scope.sourceId } : {}),
excludePrivate: scope.excludePrivate,
requireSafeChunks: requiresSafeChunks(scope),
chunkSources: scope.detail === 'low' ? ['compiled_truth'] : ['compiled_truth', 'timeline'],
maxRows: Math.min(MAX_ROWS, requests.reduce((n, r) => n + (r.to_index - r.from_index + 1), 0)),
}), timeoutMs);
for (const p of rows) if (!p.segmented) p.fenceOverlay = await loadFenceChunkOverlay(engine, { sourceId: p.source_id, slug: p.slug, compiled_truth: p.compiled_truth, timeline: p.timeline });
return new Map(rows.map(p => [p.page_id, p]));
} catch (e) {
return e instanceof EvidenceTimeout ? 'fetch_timeout' : 'fetch_failed';
}
};
const longGroups = groups.filter(g => g.reason === 'long_page');
const [mainFetch, longFetch] = await Promise.all([
fetchPages(groups.filter(g => g.reason !== 'long_page'), opts.timeoutMs ?? EVIDENCE_FETCH_TIMEOUT_MS),
long && longGroups.length > 0 ? fetchPages(longGroups, long.deadlineAt - Date.now()) : Promise.resolve(new Map<number, ChunkWindowPage>()),
]);
const pages = typeof mainFetch === 'string' ? null : mainFetch;
const fetchFailure = typeof mainFetch === 'string' ? mainFetch : null;
if (fetchFailure) fallbacks.add(fetchFailure);
// A long page that cannot be delivered in its unit keeps its ranked chunks, exactly as with the policy off.
const keepChunks = (g: Group, code: string) => {
fallbacks.add(code);
for (const h of g.hits) passthrough.push({ rank: hits.indexOf(h), hit: h, reason: 'long_page_chunk' });
};
const unit = plan.unit === 'auto' ? 'page' : plan.unit;
const segmentedReads = new Map(groups.flatMap(g => {
const page = !fetchFailure && g.reason !== 'long_page' && g.pageId !== null ? pages?.get(g.pageId) : undefined;
return page?.segmented ? [[g, planSegmentedBlock(engine, page, g.hits, unit, plan.window, scope.detail !== 'low').catch(() => null)] as const] : [];
}));
// C1 date headers: one line per block (passthrough rows included), paid for inside the budget like the title.
const headerFor = plan.dateHeader ? await blockDateHeaders(engine, hits, () => fallbacks.add('date_header_unavailable'), { titles: plan.titleHeader === true }) : () => '';
const planned: Block[] = [];
for (const g of groups) {
let b: PlannedBlock;
if (g.reason === 'long_page') {
if (typeof longFetch === 'string') { keepChunks(g, longFetch === 'fetch_timeout' ? 'long_page_deadline' : 'long_page_fetch_failed'); continue; }
const page = longFetch.get(g.pageId!);
if (!page) { dropped.not_readable = (dropped.not_readable ?? 0) + 1; continue; }
if (pageEvidenceText(page, scope.detail !== 'low').text.length < long!.minChars) {
for (const h of g.hits) passthrough.push({ rank: hits.indexOf(h), hit: h, reason: 'not_conversation' });
continue;
}
b = planBlock(page, g.hits, long!.unit, plan.window, scope.detail !== 'low');
if (b.unit === 'chunk') { keepChunks(g, b.fallbackReason ?? 'long_page_fallback'); continue; }
} else if (g.pageId === null) {
b = fallbackBlock(g.hits[0], g.hits, 'page_missing');
} else if (fetchFailure) {
if (opts.liveHits === false) { dropped.not_readable = (dropped.not_readable ?? 0) + 1; continue; }
b = fallbackBlock(g.hits[0], g.hits, fetchFailure);
} else {
const page = pages?.get(g.pageId);
if (!page) { dropped.not_readable = (dropped.not_readable ?? 0) + 1; continue; }
const read = page.segmented ? await segmentedReads.get(g)! : planBlock(page, g.hits, unit, plan.window, scope.detail !== 'low');
if (!read && opts.liveHits === false) { dropped.not_readable = (dropped.not_readable ?? 0) + 1; continue; }
b = read ?? fallbackBlock(g.hits[0], g.hits, 'segment_read_failed');
}
if (b.fallbackReason) fallbacks.add(b.fallbackReason);
const title = b.hit.title ?? '';
const header = headerFor(b.hit);
planned.push({ ...b, title, titleTok: countEvidenceTokens(title, tokenizer), header, headerTok: countEvidenceTokens(header, tokenizer), rank: g.rank, ...(g.reason ? { reason: g.reason } : {}) });
}
const capped = capEngaged(plan);
let kept: Block[];
let rankCutMeta: PageRankCutMeta | undefined;
let chunkItems: ChunkItem[] = [];
if (capped) {
({ kept, chunks: chunkItems } = allocateCapped(planned,
passthrough.map(p => chunkItem(p, headerFor(p.hit))), plan, tokenizer, dropped));
} else {
// Unchanged chunks are paid for first; conversations share the rest, and one
// whose matching span no longer fits keeps its ranked chunks instead.
const reserved = passthrough.reduce((n, p) => n + countEvidenceTokens(headerFor(p.hit) + (p.hit.chunk_text ?? ''), tokenizer) + countEvidenceTokens(p.hit.title ?? '', tokenizer), 0);
const shared = Math.max(0, plan.budgetTokens - reserved);
let candidates = planned;
if (plan.rankCut) {
const omitTok = pieceTokens(EVIDENCE_OMISSION, tokenizer);
({ kept: candidates, meta: rankCutMeta } = cutWholePages(planned, shared, plan.rankCut, b => wholeCost(b, tokenizer, omitTok), b => floorOf(b, tokenizer).tokens));
if (planned.length > candidates.length) dropped.rank_cut = planned.length - candidates.length;
}
kept = allocate(candidates, shared, tokenizer, dropped, auto
? b => {
if (b.reason === 'long_page') fallbacks.add('long_page_over_budget');
for (const h of b.hits) passthrough.push({ rank: hits.indexOf(h), hit: h, reason: b.reason === 'long_page' ? 'long_page_chunk' : 'conversation_over_budget' });
}
: undefined);
}
let results: DeliveredSearchResult[] = kept.map(b => {
const out = emit(b, tokenizer);
const text = b.header + (b.capMarker ? out.text + EVIDENCE_CUT_MARKER : out.text);
const truncated = b.cut || b.selected.size < b.candidates.length;
const delivered: DeliveredEvidence = {
unit: b.unit,
chunk_ids: b.hits.map(h => h.chunk_id),
match_spans: out.spans.map(sp => ({ ...sp, start: sp.start + b.header.length, end: sp.end + b.header.length })),
tokens: b.capMarker ? countEvidenceTokens(text, tokenizer) : out.tokens + b.headerTok,
truncated,
...(b.revision ? { revision: b.revision } : {}),
...(out.unmapped.length > 0 ? { unmapped_chunk_ids: out.unmapped } : {}),
...(b.fallbackReason && (b.unit === 'chunk' || b.fallbackReason === PAGE_OVER_INLINE_BYTES) ? { fallback_reason: b.fallbackReason } : {}),
...(b.reason ? { reason: b.reason } : {}),
};
return { ...b.hit, title: b.title, chunk_text: text, delivered };
});
// Under the cap the kept chunks join before redaction, so every evidence
// field is redacted and recounted inside the budget.
if (capped) {
const ranked = results.map((r, i) => ({ rank: kept[i].rank, r }));
for (const c of chunkItems) ranked.push({ rank: c.rank, r: chunkResult(c, tokenizer) });
results = ranked.sort((a, b) => a.rank - b.rank).map(x => x.r);
}
results = redactDelivered(results, tokenizer, fallbacks);
if (!capped && passthrough.length > 0) {
const ranked = results.map((r, i) => ({ rank: kept[i].rank, r }));
for (const p of passthrough) ranked.push({ rank: p.rank, r: chunkResult(chunkItem(p, headerFor(p.hit)), tokenizer) });
results = ranked.sort((a, b) => a.rank - b.rank).map(x => x.r);
}
let tailTrim: TailTrimMeta | undefined;
if (plan.tailTrim && opts.query) {
({ results, meta: tailTrim } = trimEvidenceTail(results, opts.query));
if (tailTrim.dropped > 0) dropped.tail_trim = tailTrim.dropped;
}
const tokensDelivered = results.reduce((n, r) => n + r.delivered.tokens, 0);
const budgetUsed = tokensDelivered + results.reduce((n, r) => n + countEvidenceTokens(r.title ?? '', tokenizer), 0);
const droppedTotal = Object.values(dropped).reduce((n, v) => n + v, 0);
const delivery: DeliveryMeta = {
requested_unit: plan.requestedUnit,
applied_unit: fetchFailure ? 'chunk' : plan.unit,
return_window: plan.window,
budget_tokens: plan.budgetTokens,
budget_used: budgetUsed,
tokens_delivered: tokensDelivered,
tokenizer,
coordinates: 'utf16',
blocks: results.length,
dropped: droppedTotal,
dropped_reasons: dropped,
fallbacks: [...fallbacks].sort(),
...(plan.budgetClamped ? { budget_clamped: plan.budgetClamped } : {}),
...(capped ? { auto_packing: plan.packing } : {}),
...(plan.dateHeader ? { date_header: true as const } : {}),
...(plan.titleHeader ? { title_header: true as const } : {}),
...(rankCutMeta ? { page_rank_cut: rankCutMeta } : {}),
...(tailTrim ? { tail_trim: tailTrim } : {}),
};
return { results: capped ? capEvidenceToBudget(results, delivery) : results, delivery };
}
function chunkItem(p: { rank: number; hit: SearchResult; reason: AutoReason }, header: string): ChunkItem {
return { ...p, title: p.hit.title ?? '', text: header + (p.hit.chunk_text ?? ''), cut: false, header };
}
/** A non-conversation hit's delivered row: its date header and the ranked chunk unchanged, or the cap's cut prefix of them. */
function chunkResult(c: ChunkItem, tokenizer: 'cl100k' | 'heuristic'): DeliveredSearchResult {
const text = c.text;
const matched = c.cut ? text.length - EVIDENCE_CUT_MARKER.length : text.length;
return { ...c.hit, ...(c.cut ? { title: c.title } : {}), ...(c.cut || c.header ? { chunk_text: text } : {}), delivered: {
unit: 'chunk',
chunk_ids: [c.hit.chunk_id],
match_spans: matched > c.header.length ? [{ chunk_id: c.hit.chunk_id, start: c.header.length, end: matched }] : [],
tokens: countEvidenceTokens(text, tokenizer),
truncated: c.cut,
reason: c.reason,
} };
}
/**
* Budget after redaction: blocks pass through the same secret redaction as
* every search response before spans and tokens are final. A block the
* redactor changed loses its spans (explicit unmapped state) and is
* re-counted, never allowed to grow past its allocation.
*/
function redactDelivered(results: DeliveredSearchResult[], tokenizer: 'cl100k' | 'heuristic', fallbacks: Set<string>): DeliveredSearchResult[] {
const redacted = redactRetrievalOutput(results, {}).results;
return results.map((r, i) => {
const rr = redacted[i];
if (rr.chunk_text === r.chunk_text && rr.title === r.title) return r;
fallbacks.add('redaction_unmapped');
let text = rr.chunk_text;
const titleTok = countEvidenceTokens(rr.title ?? '', tokenizer);
const allowed = r.delivered.tokens + countEvidenceTokens(r.title ?? '', tokenizer) - titleTok;
if (countEvidenceTokens(text, tokenizer) > allowed) text = sliceToTokenCount(text, allowed, tokenizer);
const unmapped = [...new Set([...(r.delivered.unmapped_chunk_ids ?? []), ...r.delivered.match_spans.map(s => s.chunk_id)])];
return {
...r,
title: rr.title,
chunk_text: text,
delivered: {
...r.delivered,
match_spans: [],
tokens: countEvidenceTokens(text, tokenizer),
...(unmapped.length > 0 ? { unmapped_chunk_ids: unmapped } : {}),
},
};
});
}
/**
* The cap's final evidence boundary: recount every row's evidence fields
* (title and chunk_text, with every marker in it) and keep the rank-order
* rows that fit `budget_tokens`. A row that would cross it is cut with
* EVIDENCE_CUT_MARKER when a body still fits (rank one always is), otherwise
* dropped as `budget_recount`. `budget_used`, `tokens_delivered`, `blocks`
* and `dropped` are rewritten from the recount. Runs only when the delivery
* carries `auto_packing`; everything else is returned as is. With `quote`
* (search/query and assemble_evidence) a conversation row counts inside its
* quote fence (#7600), and with `untrusted` (a remote reader) so does a row below the owner's tier (GST-687), its title included (#8384);
* deliverEvidence's own cap (recall) counts the text as returned.
*/
export function capEvidenceToBudget<T extends SearchResult & { delivered?: DeliveredEvidence }>(results: T[], delivery: DeliveryMeta, quote = false, untrusted = false): T[] {
if (!delivery.auto_packing) return results;
const tokenizer = delivery.tokenizer;
const count = (text: string | undefined) => countEvidenceTokens(text ?? '', tokenizer);
const cutTok = count(EVIDENCE_CUT_MARKER);
let remaining = delivery.budget_tokens;
let lost = 0;
const out: T[] = [];
for (const r of results) {
const asDelivered = (text: string) => (quote ? deliveredQuoteText(r, text, delivery.date_header === true, untrusted) : text), asTitle = (title: string) => (quote ? deliveredQuoteTitle(r, title, untrusted) : title);
const titleTok = count(asTitle(r.title ?? ''));
const textTok = count(asDelivered(r.chunk_text));
if (titleTok + textTok <= remaining) {
remaining -= titleTok + textTok;
out.push(!r.delivered || r.delivered.tokens === textTok ? r : { ...r, delivered: { ...r.delivered, tokens: textTok } });
continue;
}
const source = (r.chunk_text ?? '').endsWith(EVIDENCE_CUT_MARKER) ? (r.chunk_text ?? '').slice(0, -EVIDENCE_CUT_MARKER.length) : r.chunk_text ?? '';
const quoted = asDelivered(source) !== source;
const fenceTok = quoted ? count(asDelivered(source)) - count(source) : 0;
if (out.length > 0 && remaining - titleTok - cutTok - fenceTok < CUT_BODY_MIN) { lost++; continue; }
const titleRoom = Math.max(0, remaining - cutTok - fenceTok - (titleTok - count(r.title)) - CUT_BODY_MIN);
const title = count(r.title) > titleRoom ? sliceToTokenCount(r.title ?? '', titleRoom, tokenizer) : r.title ?? '';
let room = remaining - count(asTitle(title)) - cutTok - fenceTok;
let body = '';
let text = '';
for (;;) {
body = sliceToTokenCount(source, Math.max(0, room), tokenizer).replace(/\s+$/, '');
text = body + EVIDENCE_CUT_MARKER;
const over = count(asTitle(title)) + count(asDelivered(text)) - remaining;
if (!quoted || over <= 0 || room <= 0) break;
room -= over;
}
remaining -= count(asTitle(title)) + count(asDelivered(text));
const row = { ...r, title, chunk_text: text } as T;
if (r.delivered) {
const spans = r.delivered.match_spans.filter(s => s.start < body.length).map(s => ({ ...s, end: Math.min(s.end, body.length) }));
const unmapped = [...new Set([...(r.delivered.unmapped_chunk_ids ?? []), ...r.delivered.match_spans.map(s => s.chunk_id).filter(id => !spans.some(s => s.chunk_id === id))])];
row.delivered = { ...r.delivered, match_spans: spans, tokens: count(asDelivered(text)), truncated: true, ...(unmapped.length > 0 ? { unmapped_chunk_ids: unmapped } : {}) };
}
out.push(row);
}
if (lost > 0) {
delivery.dropped_reasons = { ...delivery.dropped_reasons, budget_recount: (delivery.dropped_reasons.budget_recount ?? 0) + lost };
delivery.dropped += lost;
}
delivery.blocks = out.length;
delivery.tokens_delivered = out.reduce((n, r) => n + (r.delivered?.tokens ?? count(r.chunk_text)), 0);
delivery.budget_used = delivery.tokens_delivered + out.reduce((n, r) => n + count(r.title), 0);
return out;
}
/**
* Explicit `snippet_chars` precedence: cap delivered blocks at `cap`
* characters (marker names the get_page recovery move), clip spans, mark
* truncated and re-count. A long-page row widened by the F2 policy
* (`delivered.reason: 'long_page'`) is exempt: it already fits the delivery
* budget and the page limits, and a head cut would drop the passage it was
* widened for. Non-mutating.
*/
export function capDeliveredSnippets<T extends SearchResult & { delivered?: DeliveredEvidence }>(
results: T[],
cap: number,
delivery: DeliveryMeta,
): T[] {
if (!Number.isFinite(cap) || cap <= 0) return results;
let any = false;
const tokenizer = delivery.tokenizer;
let markerOmitted = false;
const out = results.map(r => {
if (typeof r.chunk_text !== 'string' || r.chunk_text.length <= cap || r.delivered?.reason === 'long_page') return r;
any = true;
let keep = cap;
let text = r.chunk_text.slice(0, cap) + buildSnippetMarker(r.slug, r.chunk_text.length - cap, r.follow_up);
if (delivery.auto_packing) {
// Under the cap the marker is paid from the row's own allocation: a
// capped row never grows past what it held before the snippet cap.
const allowed = countEvidenceTokens(r.chunk_text, tokenizer);
if (countEvidenceTokens(text, tokenizer) > allowed) {
const markerTok = countEvidenceTokens(buildSnippetMarker(r.slug, r.chunk_text.length, r.follow_up), tokenizer);
const withMarker = allowed - markerTok >= 1;
const body = sliceToTokenCount(r.chunk_text.slice(0, cap), withMarker ? allowed - markerTok : allowed, tokenizer);
keep = body.length;
text = withMarker ? body + buildSnippetMarker(r.slug, r.chunk_text.length - keep, r.follow_up) : body;
if (!withMarker) markerOmitted = true;
}
}
if (!r.delivered) return { ...r, chunk_text: text };
const spans = r.delivered.match_spans.filter(s => s.start < keep).map(s => ({ ...s, end: Math.min(s.end, keep) }));
const lost = r.delivered.match_spans.filter(s => s.start >= keep).map(s => s.chunk_id);
const unmapped = [...new Set([...(r.delivered.unmapped_chunk_ids ?? []), ...lost.filter(id => !spans.some(s => s.chunk_id === id))])];
return {
...r,
chunk_text: text,
delivered: {
...r.delivered,
match_spans: spans,
tokens: countEvidenceTokens(text, tokenizer),
truncated: true,
...(unmapped.length > 0 ? { unmapped_chunk_ids: unmapped } : {}),
},
};
});
if (!any) return results;
if (!delivery.fallbacks.includes('snippet_cap')) delivery.fallbacks = [...delivery.fallbacks, 'snippet_cap'].sort();
if (markerOmitted && !delivery.fallbacks.includes('snippet_marker_omitted')) delivery.fallbacks = [...delivery.fallbacks, 'snippet_marker_omitted'].sort();
delivery.tokens_delivered = out.reduce((n, r) => n + (r.delivered?.tokens ?? 0), 0);
return capEvidenceToBudget(out, delivery);
}
/** Image-query branches never expand; the meta says so instead of staying silent. */
export function unsupportedDelivery(plan: EvidencePlan, reason: string): DeliveryMeta {
return {
requested_unit: plan.requestedUnit,
applied_unit: 'chunk',
return_window: plan.window,
budget_tokens: plan.budgetTokens,
budget_used: 0,
tokens_delivered: 0,
tokenizer: evidenceTokenizer(),
coordinates: 'utf16',
blocks: 0,
dropped: 0,
dropped_reasons: {},
fallbacks: [reason],
...(plan.budgetClamped ? { budget_clamped: plan.budgetClamped } : {}),
};
}
// ---------------------------------------------------------------------------
// Frozen-candidate interface (gbrain-evals E1/E3 parity)
// ---------------------------------------------------------------------------
/** `score` (library only): the hit's search score, so the tail trim can run on a frozen list. */
export interface FrozenHit { source_id: string; slug: string; chunk_id: number; score?: number }
export interface AssembleEvidenceInput {
hits: FrozenHit[];
return_unit: ReturnUnit;
return_window?: number;
budget_tokens?: number;
detail?: 'low' | 'medium' | 'high';
caller?: { remote?: boolean; sourceId?: string; sourceIds?: string[]; excludePrivate?: boolean; minTrust?: import('../trust/tier.ts').TrustTier };
/**
* Library only (trusted local, never an MCP param): the packing for this
* call's explicit `auto` budget, winning over search.auto_packing, so an
* evaluation can compare packings on one frozen hit list.
*/
auto_packing?: AutoPacking;
/** Library only: the query the hits answered (the tail trim's guard reads it). */
query?: string;
}
export interface AssembleEvidenceOutput {
results: Array<SearchResult & { delivered?: DeliveredEvidence }>;
delivery: DeliveryMeta | null;
unresolved: number[];
}
export const MAX_ASSEMBLE_HITS = 50;
/**
* Resolve frozen (source_id, slug, chunk_id) hits into search rows under the
* caller's read scope (one query). Hits outside the scope, on deleted or
* private-to-this-caller pages, or naming a chunk the page no longer holds
* are unresolved. chunk_id 0 addresses the page's first chunk.
*/
export async function resolveFrozenHits(
engine: BrainEngine,
hits: FrozenHit[],
scope: PageReadScope,
): Promise<{ rows: SearchResult[]; unresolved: number[] }> {
if (hits.length === 0) return { rows: [], unresolved: [] };
const params: unknown[] = [
hits.map(h => String(h.source_id)),
hits.map(h => String(h.slug)),
hits.map(h => (Number.isInteger(h.chunk_id) ? h.chunk_id : -1)),
hits.map((_, i) => i),
];
const filter = pageReadFilter('p', scope, params, true);
const safe = requiresSafeChunks(scope) ? `AND ${safeChunksFilter('p')}` : '';
const rows = await engine.executeRaw<Record<string, unknown>>(
`SELECT h.ord, p.id AS page_id, p.slug, p.source_id, p.title, p.type,
p.effective_date, p.effective_date_source,
cc.id AS chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source
FROM unnest($1::text[], $2::text[], $3::int[], $4::int[]) AS h(source_id, slug, chunk_id, ord)
JOIN pages p ON p.source_id = h.source_id AND p.slug = h.slug
LEFT JOIN content_chunks cc ON cc.page_id = p.id AND ${chunkGenerationFilter('p', 'cc', await chunkManifestReads(engine))}
AND CASE WHEN h.chunk_id > 0 THEN cc.id = h.chunk_id ELSE cc.chunk_index = 0 END
WHERE ${filter} AND ${servedTextProjectionFilter('p')} ${safe}`,
params,
);
const byOrd = new Map<number, Record<string, unknown>>();
for (const r of rows) byOrd.set(Number(r.ord), r);
const out: SearchResult[] = [];
const unresolved: number[] = [];
hits.forEach((h, i) => {
const r = byOrd.get(i);
if (!r || (h.chunk_id > 0 && r.chunk_id == null) || h.chunk_id < 0) { unresolved.push(i); return; }
const row: SearchResult = {
slug: String(r.slug),
page_id: Number(r.page_id),
source_id: String(r.source_id),
title: String(r.title ?? ''),
type: r.type as SearchResult['type'],
chunk_text: r.chunk_text == null ? '' : String(r.chunk_text),
chunk_source: (r.chunk_source ?? 'compiled_truth') as SearchResult['chunk_source'],
chunk_id: r.chunk_id == null ? 0 : Number(r.chunk_id),
chunk_index: r.chunk_index == null ? 0 : Number(r.chunk_index),
score: Number.isFinite(h.score) ? h.score! : 0,
stale: false,
};
// The page date, normalized exactly as live search rows are, so frozen
// and live delivery hand a reader the same date fields.
applyEffectiveDate(row, r);
out.push(row);
});
return { rows: out, unresolved };
}
/**
* The library entry gbrain-evals calls for a frozen candidate list: the
* same plan resolution, assembler and output redaction the `query` op
* applies, for these hits. `caller.remote` selects the trust class
* (default trusted local).
*/
export async function assembleEvidenceForHits(engine: BrainEngine, input: AssembleEvidenceInput): Promise<AssembleEvidenceOutput> {
if (!Array.isArray(input.hits)) throw new OperationError('invalid_params', 'hits must be an array of { source_id, slug, chunk_id }.',
'Pass hits as an array of { source_id, slug, chunk_id } objects, e.g. [{ "source_id": "default", "slug": "notes/a", "chunk_id": 12 }], ranked best first.');
if (input.hits.length > MAX_ASSEMBLE_HITS) {
throw new OperationError('invalid_params', `hits holds at most ${MAX_ASSEMBLE_HITS} entries (got ${input.hits.length}).`, 'Pass the top-ranked hits only.');
}
const remote = input.caller?.remote === true;
const excludePrivate = input.caller?.excludePrivate ?? await resolveExcludePrivatePages(engine, remote ? true : false);
const scope: DeliveryScope = {
...(input.caller?.sourceIds && input.caller.sourceIds.length > 0 ? { sourceIds: input.caller.sourceIds } : input.caller?.sourceId ? { sourceId: input.caller.sourceId } : {}),
excludePrivate,
requireSafeChunks: remote,
...(input.caller?.minTrust ? { minTrust: input.caller.minTrust } : {}),
...(input.detail ? { detail: input.detail } : {}),
};
const plan = await resolveEvidencePlan(engine, {
remote,
returnUnit: input.return_unit ?? 'page',
returnWindow: input.return_window,
budget: input.budget_tokens,
snippetChars: undefined,
snippetCap: 0,
op: 'assemble_evidence',
autoPacking: input.auto_packing,
});
const { rows, unresolved } = await resolveFrozenHits(engine, input.hits, scope);
// GST-687: a remote reader's rows carry their page trust before the fence and the budget read it (quoteLabel's tier arm).
const trusted = <R extends SearchResult>(results: R[]) => (remote ? stampPageTrust(engine, results, input.caller?.minTrust, undefined, worldOnlyFacts(remote)) : Promise.resolve(results));
if (!plan) {
const out = redactRetrievalOutput(neutralizeTranscriptRows(await trusted(rows)), {});
return { results: fenceTranscriptRows(out.results, undefined, remote), delivery: null, unresolved };
}
const d = await deliverEvidence(engine, rows, plan, scope, { liveHits: true, ...(input.query ? { query: input.query } : {}) });
const out = redactRetrievalOutput(neutralizeTranscriptRows(await trusted(d.results), d.delivery.date_header === true), d.delivery);
return { results: fenceTranscriptRows(capEvidenceToBudget(out.results, out.meta, true, remote), { count: text => countEvidenceTokens(text, out.meta.tokenizer), delivery: out.meta }, remote), delivery: out.meta, unresolved };
}
/** SHA-256 over the delivered evidence (see docs/evidence-delivery.md). */
export function evidenceFingerprint(results: Array<{ source_id?: string; slug: string; chunk_text: string; chunk_id: number; delivered?: { unit: string; chunk_ids: number[] } }>): string {
const canon = results.map(r => [r.source_id ?? 'default', r.slug, r.chunk_text, r.delivered?.unit ?? 'chunk', r.delivered?.chunk_ids ?? [r.chunk_id]]);
return createHash('sha256').update(JSON.stringify(canon)).digest('hex');
}
/**
* Thin-client version skew: `return_unit` was requested but the response
* carries no delivery meta (an older server ignored the param). Returns the
* one-line warning, or null.
*/
export function deliveryVersionSkewWarning(
opName: string,
params: Record<string, unknown>,
retrievalMeta: Record<string, unknown> | null | undefined,
result: unknown,
): string | null {
if (!['search', 'query', 'recall'].includes(opName)) return null;
const unit = params.return_unit;
if (typeof unit !== 'string' || unit === 'chunk') return null;
const present = opName === 'recall'
? typeof result === 'object' && result !== null && 'delivery' in result
: !!retrievalMeta && 'delivery' in retrievalMeta;
if (present) return null;
return `warning: the server ignored return_unit (evidence delivery needs gbrain server v${EVIDENCE_DELIVERY_MIN_SERVER_VERSION} or newer); results are plain chunks.`;
}