Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import type { BrainEngine } from '../engine.ts';
import type { ChunkInput, PageReadScope } from '../types.ts';
import { jsonRowBatches } from '../batch-rows.ts';
/** First markdown index built from the strict full-body fence sanitizer. */
export const SAFE_FENCE_CHUNKER_VERSION = 4;
const PROTECTED_MARKER = '<!--- gbrain:(facts|takes):(begin|end) -->';
export function hasProtectedBody(text: string): boolean {
return new RegExp(PROTECTED_MARKER).test(text);
}
/** Expression and aliases are internal SQL, never caller-supplied values. */
export function protectedTextFilter(expression: string): string {
return `COALESCE(${expression}, '') ~ '${PROTECTED_MARKER}'`;
}
/**
* Fixed SQL aliases only. Check full bodies because fragments can lose markers.
* A segmented page has no body columns: its `body_flags.has_protected` (set by
* the ingest from the segmenter's regions) decides, and a page without the
* flag counts as protected (fail closed).
*/
export function protectedBodyFilter(alias: string): string {
return `(CASE WHEN ${alias}.body_storage = 'segmented' THEN ${alias}.body_flags->>'has_protected' IS DISTINCT FROM 'false'
ELSE (${protectedTextFilter(`${alias}.compiled_truth`)} OR ${protectedTextFilter(`${alias}.timeline`)}) END)`;
}
/** A body-only write cannot certify old fragments, even after fence removal. */
export function bodyWriteChunkVersion(compiledTruth: string, timeline: string): string {
return `CASE WHEN pages.compiled_truth IS NOT DISTINCT FROM ${compiledTruth}
AND pages.timeline IS NOT DISTINCT FROM ${timeline} THEN pages.chunker_version
WHEN pages.chunker_version < 0 OR ${protectedBodyFilter('pages')}
OR ${protectedTextFilter(compiledTruth)} OR ${protectedTextFilter(timeline)} THEN -1
ELSE LEAST(COALESCE(pages.chunker_version, 0), ${SAFE_FENCE_CHUNKER_VERSION - 1}) END`;
}
export function requiresSafeChunks(scope?: PageReadScope): boolean {
return scope?.requireSafeChunks ?? scope?.excludePrivate ?? false;
}
/** Every user sees projections only from the current canonical revision. */
export function currentTextProjectionFilter(alias: string): string {
return `(${alias}.text_projection_revision = ${alias}.knowledge_revision) IS TRUE`;
}
/**
* #9192: the `text_projection_revision` v153's activation leaves on a page whose pre-upgrade index it keeps serving
* (PAGE_PROJECTION_ACTIVATION_SQL says which). The page stays unsealed (its rebuild is queued); any revision change
* clears the column (pages_knowledge_revision trigger), and so does the rebuild. A version-0 UUID: every revision is
* gen_random_uuid() (version 4), so the marker never equals one and every strict `= knowledge_revision` read treats
* the page as not current.
*/
export const ACTIVATION_SERVED_REVISION = '00000000-0000-0000-0000-000000000153';
/**
* What search serves: the current projection, or the pre-activation index of a page unchanged since (#9192). The
* current-projection term stays its own disjunct so the planner keeps reading pages_text_projection_current_stats.
*/
export function servedTextProjectionFilter(alias: string): string {
return `(${currentTextProjectionFilter(alias)} OR (${alias}.text_projection_revision = '${ACTIVATION_SERVED_REVISION}'::uuid) IS TRUE)`;
}
/**
* Wave B: the chunk row belongs to its page's live body: an inline row
* (`body_generation IS NULL`) of a page that is not segmented, or a segmented
* row at or below the page's generation whose occurrence has a live manifest
* row. Staged rows (above the page's generation), rows of retired occurrences
* (closed by a flip or a demotion, deleted later when the flip deferred their
* retirement) and rows of a preparation that never published are invisible
* whether or not they were deleted yet. The manifest probe is one primary-key
* lookup per segmented row; `page` must expose `body_storage` and `body_generation`.
* `manifest: false` (chunkManifestReads) drops the segmented branch, which then
* matches no row the reader can see.
*/
export function chunkGenerationFilter(page: string, chunk: string, manifest = true): string {
if (!manifest) return `(${chunk}.body_generation IS NULL AND ${page}.body_storage <> 'segmented')`;
return `((${chunk}.body_generation IS NULL AND ${page}.body_storage <> 'segmented')
OR (${chunk}.body_generation <= ${page}.body_generation
AND EXISTS (SELECT 1 FROM page_body_manifest lm WHERE lm.page_id = ${chunk}.page_id AND lm.occurrence_id = ${chunk}.occurrence_id
AND lm.staging_id IS NULL AND lm.gen_from <= ${page}.body_generation AND (lm.gen_to IS NULL OR lm.gen_to > ${page}.body_generation))))`;
}
/** `chunkGenerationFilter` for a statement that has no page row joined. */
export function chunkGenerationProbe(chunk: string, manifest = true): string {
return `EXISTS (SELECT 1 FROM pages gp WHERE gp.id = ${chunk}.page_id AND ${chunkGenerationFilter('gp', chunk, manifest)})`;
}
const CHUNK_MANIFEST_ACCESS_SQL = `-- chunk-visibility-ok: whether any chunk row of a segmented body exists, no row is returned
SELECT has_table_privilege('page_body_manifest', 'SELECT') AS readable,
NOT has_table_privilege('page_body_manifest', 'SELECT')
AND (EXISTS (SELECT 1 FROM content_chunks WHERE body_generation IS NOT NULL)
OR EXISTS (SELECT 1 FROM content_chunks WHERE occurrence_id IS NOT NULL)) AS segmented`;
const manifestReadable = new WeakSet<object>();
/**
* Whether a read's chunk statements name `page_body_manifest`. Postgres checks the privilege on every relation a
* statement names, matched or not, so a role without SELECT on it (docs/ENGINES.md, "Segmented page tables") would
* fail every chunk read with 42501 although no page is segmented. True on PGLite, for a role that may read the
* manifest (kept per engine: a transaction clone probes again, so its SET LOCAL ROLE counts) and for a role that sees
* a chunk row of a segmented body (`body_generation` or `occurrence_id` set, both partial-indexed), which then gets
* 42501 as documented. Otherwise every chunk row the role sees is an inline row, and the manifest-free forms of
* chunkGenerationFilter, chunkGenerationProbe and chunkManifestJoin select exactly the rows the full forms would.
* `db` runs the probe: a scoped read passes its transaction, so the probe sees the rows that read sees.
*/
export async function chunkManifestReads(engine: Pick<BrainEngine, 'kind' | 'executeRaw'>, db: Pick<BrainEngine, 'executeRaw'> = engine): Promise<boolean> {
if (engine.kind !== 'postgres' || manifestReadable.has(engine)) return true;
const [access] = await db.executeRaw<{ readable: boolean; segmented: boolean }>(CHUNK_MANIFEST_ACCESS_SQL);
if (access?.readable) manifestReadable.add(engine);
return access?.readable === true || access?.segmented === true;
}
/**
* Search-facing chunk visibility: the served text projection and the live body generation; maintenance reads pass
* currentTextProjectionFilter. A sealed page holds no
* rows of a retired occurrence (the flip deletes them in its transaction, or leaves the page unsealed until the
* projection job has), so the row predicate needs no manifest probe, whose per-row cost moved vector search off its
* HNSW plan (test/e2e/vector-chunk-statistics-postgres.test.ts).
*/
export function chunkVisibleFilter(page: string, chunk: string, projection = servedTextProjectionFilter): string {
return `(${projection(page)}
AND (${chunk}.body_generation IS NULL OR ${chunk}.body_generation <= ${page}.body_generation))`;
}
/**
* The live manifest row of a segmented chunk (no row for an inline chunk), for
* `chunkDocumentOrder`. LEFT JOIN text; `page` must expose `body_generation`.
* `readsManifest: false` (chunkManifestReads) joins no row, as for an inline chunk.
*/
export function chunkManifestJoin(page: string, chunk: string, manifest: string, readsManifest = true): string {
if (!readsManifest) return `LEFT JOIN (SELECT NULL::bigint AS byte_start) ${manifest} ON false`;
return `LEFT JOIN page_body_manifest ${manifest} ON ${chunk}.occurrence_id IS NOT NULL
AND ${manifest}.page_id = ${chunk}.page_id AND ${manifest}.occurrence_id = ${chunk}.occurrence_id
AND ${manifest}.staging_id IS NULL AND ${manifest}.gen_from <= ${page}.body_generation
AND (${manifest}.gen_to IS NULL OR ${manifest}.gen_to > ${page}.body_generation)`;
}
/** Document order: (manifest byte_start, local_ordinal) for segmented rows; inline rows keep chunk_index order. */
export function chunkDocumentOrder(chunk: string, manifest: string): string {
return `${manifest}.byte_start, ${chunk}.local_ordinal, ${chunk}.chunk_index`;
}
/** A page of any kind chunked before the fence; remote reads withhold it until it is re-sealed. */
export function belowSafeChunkFence(chunkerVersion: number | null | undefined): boolean {
return (chunkerVersion ?? 0) < SAFE_FENCE_CHUNKER_VERSION;
}
/**
* A body write that removes a protected fence leaves `chunker_version < 0`
* (bodyWriteChunkVersion): the stored text lost its marker, but the source
* behind it (an image's pixels) did not, so only a fresh attestation counts.
*/
/** #6803: the importer's refusal of a distrusted image without a successful fresh OCR pass; the sweep counts it as `ocr_required`. */
export const OCR_REQUIRED_ERROR = 'This image had protected OCR, so indexing it requires successful fresh OCR. Enable OCR and reimport the image.';
export async function provenanceDistrusted(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number): Promise<boolean> {
const [row] = await engine.executeRaw<{ chunker_version: number | null }>('SELECT chunker_version FROM pages WHERE id = $1', [pageId]);
return (row?.chunker_version ?? 0) < 0;
}
/**
* Older fragments lack trustworthy provenance, even if markers were removed
* later. `pages.chunker_version` is `smallint NOT NULL` at head on both
* engines (v054), so the bare column compares and keeps its statistics (#6632:
* a COALESCE gets the default 1/3 selectivity and misleads the count plan).
*/
export function safeChunksFilter(alias: string): string {
return `${alias}.chunker_version >= ${SAFE_FENCE_CHUNKER_VERSION}`;
}
/**
* Preserve an existing seal for an embedding-only refresh of the exact chunk
* snapshot. Additions, removals and textual metadata changes invalidate before
* the first mutation, including partial failures outside an import transaction.
* Both engines canonicalize chunk_text before calling, and persist/hash those
* same bytes. Identity and enum fields retain their existing rejection policy.
*
* One statement per encoded batch of the incoming set (`jsonRowBatches`), so a
* large page is compared without one whole-page JSON document; each statement
* also checks for stored chunks missing from the complete incoming index set.
* The caller stops at the first statement that invalidates (RETURNING id).
*/
export function* chunkWriteInvalidations(pageId: number, chunks: ChunkInput[]): Generator<{ sql: string; params: unknown[] }> {
const metadata = ['language', 'symbol_name', 'symbol_type', 'start_line', 'end_line', 'parent_symbol_path', 'doc_comment', 'symbol_name_qualified'] as const;
// chunk-visibility-ok: inline chunk write invalidation; segmented pages stage rows through stageSegmentChunks
const sql = `UPDATE pages SET chunker_version = -1 WHERE id = $1 AND (
NOT (${safeChunksFilter('pages')})
OR EXISTS (SELECT 1 FROM content_chunks cc WHERE cc.page_id = $1 AND cc.chunk_index <> ALL($3::int[]))
OR EXISTS (
SELECT 1 FROM jsonb_to_recordset($2::text::jsonb) AS incoming(
chunk_index int, chunk_text text, chunk_source text, modality text,
language text, symbol_name text, symbol_type text, start_line int,
end_line int, parent_symbol_path text[], doc_comment text, symbol_name_qualified text)
LEFT JOIN content_chunks cc ON cc.page_id = $1 AND cc.chunk_index = incoming.chunk_index
WHERE cc.id IS NULL
OR cc.chunk_text IS DISTINCT FROM incoming.chunk_text
OR cc.chunk_source IS DISTINCT FROM incoming.chunk_source
OR cc.modality IS DISTINCT FROM COALESCE(incoming.modality, 'text')
${metadata.map(key => `OR cc.${key} IS DISTINCT FROM COALESCE(incoming.${key}, cc.${key})`).join('\n ')}
)) RETURNING id`;
const indices = chunks.map(chunk => chunk.chunk_index);
const batches = jsonRowBatches(chunks, chunk => Object.fromEntries(
['chunk_index', 'chunk_text', 'chunk_source', 'modality', ...metadata].map(key => [key, chunk[key as keyof ChunkInput] ?? null])));
let sent = false;
for (const batch of batches) {
sent = true;
yield { sql, params: [pageId, batch, indices] };
}
if (!sent) yield { sql, params: [pageId, '[]', indices] };
}
/**
* A stored row whose vector provably describes its own text (#4246), the only
* rows a content-keyed rebuild moves to a new index. `col` is a quoted column.
*/
export function movableChunkSql(col: string): string {
return `(${col} IS NOT NULL AND embedded_text_hash IS NOT DISTINCT FROM md5(chunk_text))`;
}