Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Embedding for segmented pages (Wave B, spec 4.4). The inline path
* (`readProjectionSnapshot` / `installPageEmbeddings`) reads the page body, its
* fence overlay and every chunk of the page; a segmented page is embedded from
* its stale chunk rows alone: keyset pages of live, vector-less rows with their
* text, the page title and the plain tier (no input depends on the rest of the
* body; per-chunk synopsis on a segmented page, whose input is the lead and one
* window of the chunk's segment, is the contextual retrieval service's, spec
* 8.6, and installs through installSegmentedVectors), metered by the
* per-page budget and installed by chunk_index and exact text onto rows still
* in the page's live body generation. It never reads a segment.
*/
import type { BrainEngine } from '../engine.ts';
import type { ChunkInput, CRMode } from '../types.ts';
import { jsonRowBatches } from '../batch-rows.ts';
import { wrapChunkTextsForStoredMode } from '../embedding-context.ts';
import { embeddingInputHash, plainEmbeddingTier } from '../embedding-input-hash.ts';
import { meterPageBatch, spendGrant } from '../page-embedding-budget.ts';
import { acquirePageEmbeddingLease, vectorlessChunkIndexes } from '../page-embedding-lease.ts';
import { embeddingInputContext, embeddingWriteTarget } from '../page-state/projections.ts';
import { quoteIdentifier, vectorCastSuffix } from '../search/embedding-column.ts';
import { chunkGenerationFilter, currentTextProjectionFilter } from '../search/safe-chunks.ts';
import { buildSegmentedProjection } from './projection.ts';
import { isSegmentedPage } from './page-shape.ts';
import type { BodyStorage } from './types.ts';
/** Stale rows read and embedded per provider round trip. */
export const SEGMENTED_EMBED_BATCH_ROWS = 500;
/** The page fields a segmented embed reads: metadata only, never the body. */
export interface SegmentedEmbedPage {
id: number;
slug: string;
source_id: string;
title: string;
contextual_retrieval_mode: CRMode | null;
body_storage: 'segmented';
body_bytes: number;
}
export interface SegmentedStaleRow { chunk_index: number; chunk_text: string; chunk_source: string }
export type SegmentedEmbedFn = (texts: string[], opts: { abortSignal?: AbortSignal }) => Promise<Float32Array[]>;
export interface SegmentedEmbedOpts {
embedFn: SegmentedEmbedFn;
signal?: AbortSignal;
signature?: string | null;
approvedCapUsd?: number;
/** Runs inside the install transaction after the page guard (ownership, effect claim); throwing installs nothing. */
beforeInstall?: (tx: BrainEngine) => Promise<void>;
/** Runs before each wait for the page's embedding lease (page-embedding-lease.ts); an effect renews its claim there. */
whileWaiting?: () => Promise<void>;
}
/** A segmented page that is live, sealed and in a live source, by id or by source and slug; null otherwise (inline pages included). */
export async function readSegmentedEmbedPage(engine: Pick<BrainEngine, 'executeRaw'>,
ref: { id: number } | { sourceId: string; slug: string }): Promise<SegmentedEmbedPage | null> {
const [where, params] = 'id' in ref ? ['p.id = $1', [ref.id]] : ['p.source_id = $1 AND p.slug = $2', [ref.sourceId, ref.slug]];
const [row] = await engine.executeRaw<{ id: number; slug: string; source_id: string; title: string; contextual_retrieval_mode: CRMode | null; body_bytes: number | string }>(
`SELECT p.id, p.slug, p.source_id, p.title, p.contextual_retrieval_mode, p.body_bytes
FROM pages p JOIN sources s ON s.id = p.source_id
WHERE ${where} AND p.body_storage = 'segmented' AND p.deleted_at IS NULL AND NOT s.archived AND ${currentTextProjectionFilter('p')}`, params);
return row ? { ...row, id: Number(row.id), body_storage: 'segmented', body_bytes: Number(row.body_bytes) } : null;
}
/**
* `readSegmentedEmbedPage` for a page row; with `build`, a live segmented page
* in a live source that is not sealed gets its projection built first (a
* create whose post-commit projection never ran or failed has no chunk rows
* to embed). Null for an inline page or one still not sealed.
*/
export async function readSealedSegmentedEmbedPage(engine: BrainEngine,
page: { slug: string; source_id: string; body_storage?: BodyStorage | null }, build: boolean): Promise<SegmentedEmbedPage | null> {
const ref = { sourceId: page.source_id, slug: page.slug };
const sealed = await readSegmentedEmbedPage(engine, ref);
if (sealed || !build || !isSegmentedPage(page)) return sealed;
const [unsealed] = await engine.executeRaw(`SELECT 1 FROM pages p JOIN sources s ON s.id = p.source_id
WHERE p.source_id = $1 AND p.slug = $2 AND p.body_storage = 'segmented' AND p.deleted_at IS NULL AND NOT s.archived`, [ref.sourceId, ref.slug]);
return unsealed && (await buildSegmentedProjection(engine, ref)).sealed ? readSegmentedEmbedPage(engine, ref) : null;
}
/** A dry run's counts: the page's live rows and those without a vector in the write column. */
export async function countSegmentedEmbedRows(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number): Promise<{ total: number; stale: number }> {
const col = quoteIdentifier((await embeddingWriteTarget(engine)).column.name);
const [row] = await engine.executeRaw<{ total: number; stale: number }>(
`SELECT count(*)::int AS total, count(*) FILTER (WHERE cc.${col} IS NULL)::int AS stale
FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
WHERE cc.page_id = $1 AND ${currentTextProjectionFilter('p')} AND ${chunkGenerationFilter('p', 'cc')}`, [pageId]);
return { total: Number(row?.total ?? 0), stale: Number(row?.stale ?? 0) };
}
/**
* Vector-less live rows of a segmented page below `beforeChunkIndex` (null: from the newest), newest first, so a
* per-page cap spends on the newest rows; `occurrenceIds` limits them to the occurrences a write added,
* `includeEmbedded` takes rows that already hold a vector too (`gbrain embed --all`).
*/
export async function listSegmentedStaleRows(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number, column: string,
opts: { beforeChunkIndex: number | null; limit: number; occurrenceIds?: readonly number[]; includeEmbedded?: boolean }): Promise<SegmentedStaleRow[]> {
const col = quoteIdentifier(column);
return engine.executeRaw<SegmentedStaleRow>(
`SELECT cc.chunk_index, cc.chunk_text, cc.chunk_source
FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
WHERE cc.page_id = $1 AND ($5::boolean OR cc.${col} IS NULL) AND ($2::int IS NULL OR cc.chunk_index < $2::int)
AND ($4::bigint[] IS NULL OR cc.occurrence_id = ANY($4::bigint[]))
AND ${currentTextProjectionFilter('p')} AND ${chunkGenerationFilter('p', 'cc')}
ORDER BY cc.chunk_index DESC LIMIT $3`,
[pageId, opts.beforeChunkIndex, opts.limit, opts.occurrenceIds ? [...opts.occurrenceIds] : null, opts.includeEmbedded ?? false]);
}
/**
* Embed `listed` rows of one segmented page under the page's embedding lease
* (page-embedding-lease.ts): keep the rows still without a vector once the
* lease is held (every row with `includeEmbedded`), meter them under the
* per-page budget, call the provider once for the granted rows, and install
* each vector with its recorded input on the row with the same chunk_index and
* text while that row is in the live generation and the page's title, mode and
* write target are unchanged. Returns how many rows were still pending and how
* many took a vector; null when the budget holds the page or the cap left
* nothing to embed.
*/
export async function embedSegmentedRows(engine: BrainEngine, page: SegmentedEmbedPage, listed: readonly SegmentedStaleRow[],
opts: SegmentedEmbedOpts & { includeEmbedded?: boolean }): Promise<{ pending: number; installed: number; busy?: true } | null> {
if (listed.length === 0) return { pending: 0, installed: 0 };
await using lease = await acquirePageEmbeddingLease(engine, page.id, { signal: opts.signal, whileWaiting: opts.whileWaiting });
if (lease.busy) return { pending: listed.length, installed: 0, busy: true };
const target = await embeddingWriteTarget(engine);
const missing = opts.includeEmbedded ? null : await vectorlessChunkIndexes(engine, page.id, target.column.name);
const rows = missing ? listed.filter(row => missing.has(row.chunk_index)) : listed;
if (rows.length === 0) return { pending: 0, installed: 0 };
const asRead = await readChunkInputHashes(engine, page.id, rows.map(row => row.chunk_index));
const metered = await meterPageBatch(engine, page, rows, { signature: opts.signature ?? null, approvedCapUsd: opts.approvedCapUsd });
if (!metered) return null;
const chunks = metered.chunks;
const vectors = await spendGrant(metered.grant, () => opts.embedFn(wrapChunkTextsForStoredMode(page, chunks), { abortSignal: opts.signal }));
if (vectors.length !== chunks.length || vectors.some(v => !v?.length || !v.every(Number.isFinite))) {
throw new Error('Embedding provider returned an incomplete or invalid batch');
}
const tier = plainEmbeddingTier(page.contextual_retrieval_mode);
const provenance = embeddingInputContext(target, page.title, null, []);
return { pending: rows.length, installed: await installSegmentedVectors(engine, page, target, chunks.map((chunk, i) => ({ chunk_index: chunk.chunk_index,
chunk_text: chunk.chunk_text, vector: vectors[i]!, hash: embeddingInputHash(provenance, tier, chunk), expected: asRead.get(chunk.chunk_index) ?? null })),
async tx => { await lease.assertHeld(tx); await opts.beforeInstall?.(tx); }) };
}
/** The recorded embedding input of a segmented page's live rows at `chunkIndexes`, as read now (null: none recorded). */
async function readChunkInputHashes(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number, chunkIndexes: readonly number[]): Promise<Map<number, string | null>> {
const rows = await engine.executeRaw<{ chunk_index: number; embedding_input_hash: string | null }>(
`SELECT cc.chunk_index, cc.embedding_input_hash FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
WHERE cc.page_id = $1 AND cc.chunk_index = ANY($2::int[]) AND ${chunkGenerationFilter('p', 'cc')}`, [pageId, [...chunkIndexes]]);
return new Map(rows.map(row => [Number(row.chunk_index), row.embedding_input_hash]));
}
/**
* One vector to install on a segmented page: the row's chunk_index and exact text, the recorded embedding input, and
* `expected`, the row's recorded input as the writer read it before building the vector (null: none).
*/
export interface SegmentedVectorUpdate { chunk_index: number; chunk_text: string; vector: Float32Array; hash: string; expected: string | null }
/**
* Install vectors with their recorded inputs on the live rows of a segmented page with the same chunk_index and text,
* under the page lock, while the page's slug, title and contextual mode and the write target (column, provenance
* model) are those the inputs were built under; 0 when any moved. A row whose recorded input is no longer `expected`
* (another writer installed since the read) is left as it is: a stale install is a no-op. `beforeInstall` runs after
* the lock; throwing or returning false installs nothing. Returns the rows that took a vector.
*/
export async function installSegmentedVectors(engine: BrainEngine, page: SegmentedEmbedPage, target: Awaited<ReturnType<typeof embeddingWriteTarget>>,
vectors: readonly SegmentedVectorUpdate[], beforeInstall?: (tx: BrainEngine) => Promise<void | boolean>): Promise<number> {
const updates = vectors.map(v => ({ chunk_index: v.chunk_index, chunk_text: v.chunk_text, embedding: `[${Array.from(v.vector).join(',')}]`, hash: v.hash, expected: v.expected }));
const col = quoteIdentifier(target.column.name);
return engine.transaction(async tx => {
await tx.lockPageKeys([{ sourceId: page.source_id, slug: page.slug }]);
if (await beforeInstall?.(tx) === false) return 0;
const current = await readSegmentedEmbedPage(tx, { id: page.id });
const now = await embeddingWriteTarget(tx);
if (!current || current.slug !== page.slug || current.title !== page.title
|| current.contextual_retrieval_mode !== page.contextual_retrieval_mode
|| now.column.name !== target.column.name || now.provenanceModel !== target.provenanceModel) return 0;
let installed = 0;
for (const batch of jsonRowBatches(updates, update => update)) {
installed += (await tx.executeRaw(`UPDATE content_chunks cc SET ${col} = v.embedding${vectorCastSuffix(target.column)},
embedding_input_hash = v.hash, embedded_at = now(), embedded_text_hash = md5(cc.chunk_text), model = COALESCE($3, cc.model),
embedding_pending_since = NULL
FROM jsonb_to_recordset($2::text::jsonb) AS v(chunk_index int, chunk_text text, embedding text, hash text, expected text), pages p
WHERE cc.page_id = $1 AND p.id = cc.page_id AND cc.chunk_index = v.chunk_index AND cc.chunk_text = v.chunk_text
AND cc.embedding_input_hash IS NOT DISTINCT FROM v.expected AND ${chunkGenerationFilter('p', 'cc')}
RETURNING cc.id`, [page.id, batch, target.model])).length;
}
return installed;
});
}
/** A live segment of a segmented page, newest first: its occurrence, text hash and entry state (manifest metadata only). */
export interface LiveEmbedSegment { occurrenceId: number; sha256: Uint8Array; entryState: unknown }
export async function listLiveEmbedSegments(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number): Promise<LiveEmbedSegment[]> {
const rows = await engine.executeRaw<{ occurrence_id: number | string; sha256: Uint8Array; entry_state: unknown }>(
`SELECT m.occurrence_id, m.sha256, m.entry_state FROM page_body_manifest m JOIN pages p ON p.id = m.page_id
WHERE m.page_id = $1 AND m.staging_id IS NULL AND m.gen_from <= p.body_generation AND (m.gen_to IS NULL OR m.gen_to > p.body_generation)
ORDER BY m.seq DESC`, [pageId]);
return rows.map(row => ({ occurrenceId: Number(row.occurrence_id), sha256: new Uint8Array(row.sha256), entryState: row.entry_state }));
}
export interface SegmentChunkRow { chunk_index: number; chunk_text: string; chunk_source: NonNullable<ChunkInput['chunk_source']>; embedding_input_hash: string | null }
/** The live chunk rows of one segment of a segmented page in text order, with their recorded embedding inputs. */
export async function listSegmentChunkRows(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number, occurrenceId: number): Promise<SegmentChunkRow[]> {
const rows = await engine.executeRaw<SegmentChunkRow>(
`SELECT cc.chunk_index, cc.chunk_text, cc.chunk_source, cc.embedding_input_hash
FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
WHERE cc.page_id = $1 AND cc.occurrence_id = $2 AND ${currentTextProjectionFilter('p')} AND ${chunkGenerationFilter('p', 'cc')}
ORDER BY cc.chunk_index`, [pageId, occurrenceId]);
return rows.map(row => ({ ...row, chunk_index: Number(row.chunk_index) }));
}
/**
* Embed every vector-less live row of a segmented page (or only the rows of
* `occurrenceIds`, the occurrences an append added; every live row with
* `all`), newest first, one keyset batch at a time, until the rows run out
* (`complete`), the budget holds the page or leaves no room (`stopped:
* 'budget'`), or the page moved under an install (`stopped: 'page_moved'`).
* An approval (`approvedCapUsd`) raises the page's cap once per run: later
* batches keep the approval's lifted hold but spend under the cap it set.
*/
export async function embedSegmentedPage(engine: BrainEngine, page: SegmentedEmbedPage,
opts: SegmentedEmbedOpts & { occurrenceIds?: readonly number[]; batchRows?: number; all?: boolean }): Promise<{ embedded: number; complete: boolean; stopped?: 'budget' | 'page_moved' | 'busy' }> {
const column = (await embeddingWriteTarget(engine)).column.name;
const limit = opts.batchRows ?? SEGMENTED_EMBED_BATCH_ROWS;
let embedded = 0;
for (let before: number | null = null; ;) {
opts.signal?.throwIfAborted();
const rows = await listSegmentedStaleRows(engine, page.id, column, { beforeChunkIndex: before, limit, occurrenceIds: opts.occurrenceIds, includeEmbedded: opts.all });
if (rows.length === 0) return { embedded, complete: true };
const approvedCapUsd = opts.approvedCapUsd === undefined || before === null ? opts.approvedCapUsd : 0;
const done = await embedSegmentedRows(engine, page, rows, { ...opts, approvedCapUsd, includeEmbedded: opts.all });
if (done === null) return { embedded, complete: false, stopped: 'budget' };
if (done.busy) return { embedded, complete: false, stopped: 'busy' };
embedded += done.installed;
if (done.pending > 0 && done.installed === 0) return { embedded, complete: false, stopped: 'page_moved' };
if (done.installed < done.pending) return { embedded, complete: false, stopped: 'budget' };
before = rows[rows.length - 1]!.chunk_index;
}
}