Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Shared batch-insert row builders (gbrain#1861).
*
* WHY THIS FILE EXISTS
* --------------------
* `addLinksBatch` / `addTimelineEntriesBatch` / `addTakesBatch` used to bind
* free text through `unnest(${arr}::text[])`. postgres.js serializes a JS
* string[] into a Postgres `text[]` literal (`{"...","..."}`); calendar/Zoom
* context strings (commas, quotes, braces, em-dashes) produced a literal that
* Postgres `array_in` rejected -> "malformed array literal", which aborted the
* whole `extract links --stale` sweep. The fix passes the batch as a single
* JSONB document via `jsonb_to_recordset((($1::jsonb)->'rows'))`, which encodes
* arbitrary free text safely and dodges the 65535-bind-param cap.
*
* Both engines (postgres.js and PGLite) must build the SAME row objects or they
* drift, so the object construction lives here once and both engines import it.
*
* LinkBatchInput[] ---+
* TimelineInput[] ----+--> build*Rows() --> [{...}, ...] --> { rows } wrapper
* TakeBatchInput[] ---+ | |
* sanitizeForJsonb executeRawJsonb
* free-text fields only $1::jsonb -> 'rows'
* jsonb_to_recordset(...)
*
* NUL + SURROGATE POLICY (codex P0 hardening; #2011): Postgres `jsonb` rejects
* both the Unicode NUL escape AND an unpaired UTF-16 surrogate half, and Postgres
* `text` cannot store a NUL either, so the OLD `unnest(::text[])` path rejected
* (errored) any row carrying either. We deliberately PRESERVE that reject
* semantics for IDENTITY and security-relevant fields: slugs, source_ids,
* `holder`, `kind`, dates, and the enum-ish `link_type` / `link_source` /
* `origin_slug` / `origin_field`. Those are left UN-sanitized, so junk in them
* still errors the batch and can never silently retarget a row to a different
* page/source or normalize a `holder` past the read-side `holder = ANY(allowlist)`
* privacy filter.
*
* `sanitizeForJsonb` (strip NUL + well-form lone surrogates) is applied ONLY to
* genuinely free-prose body fields where junk plausibly arrives from
* calendar/meeting/LLM/OCR content and where dropping the whole batch would be
* the worse outcome: `context` (links), `summary` + `detail` + `source`
* (timeline), `claim` + `source` (takes). A lone surrogate is normalized to
* U+FFFD; NUL is removed. Timeline `source` is free-text here (it round-trips
* arbitrary provenance labels and is treated as free text by both engines), so it
* is sanitized too even though it participates in `idx_timeline_dedup` — replacing
* a malformed half-char before dedup is correct, not lossy (#2011). Takes `source`
* is likewise free-text LLM-extracted provenance (not an identity/dedup field —
* the takes key is `(page_id, row_num)`), so it is sanitized as well. Commas, quotes,
* braces, and em-dashes are exactly what JSONB encodes correctly and are never
* touched; stripping them would corrupt user data.
*
* DEFAULTING NOTE: the builders reproduce each method's exact pre-#1861
* defaulting. `|| ''` / `|| 'markdown'` / `|| 'default'` collapse empty strings;
* `origin_slug` / `origin_field` use truthy-`|| null` (empty string -> null,
* which the LEFT JOIN treats as no-match); `link_kind` uses `?? null` (empty
* string preserved). Do NOT "simplify" `||` to `??`; it changes empty-string
* behavior.
*
* BATCH SIZE: one JSONB parameter dodges the 65535-param cap but is not
* unbounded; it has a server-side datum/parse-memory ceiling. In-tree callers
* batch small (extract links ~100/batch, NER ~500), well within budget. Direct
* engine callers passing arbitrarily large batches should chunk (~1-5K rows).
*/
import type { LinkBatchInput, TimelineBatchInput, TakeBatchInput } from './engine.ts';
import { normalizeWeightForStorage } from './takes-fence.ts';
import { ensureWellFormed } from './text-safe.ts';
/**
* Strip Unicode NUL (U+0000) from a free-text body field. Fast-path the common
* case (no NUL) so the regex replace only runs when a NUL is actually present.
* Only call this on free-prose columns, never on identity/security fields (see
* the NUL POLICY note above).
*
* This is the NUL-only primitive. For anything serialized into a `::jsonb` cast,
* prefer `sanitizeForJsonb` below — NUL alone does not protect against the lone
* surrogate that aborts a jsonb batch (#2011). Exported only as the building
* block for `sanitizeForJsonb`; don't reach for it directly on a jsonb path.
*/
export const stripNul = (s: string): string => (s.includes('\0') ? s.replace(/\0/g, '') : s);
/**
* Sanitize a free-text body field for JSONB serialization: strip NUL (#1861)
* AND well-form unpaired UTF-16 surrogates (#2011). A lone surrogate half (left
* behind when a slicer like `excerpt()` cuts a window boundary through an emoji)
* is rejected by Postgres inside a `::jsonb` cast and aborts the whole batch;
* `ensureWellFormed` replaces it with U+FFFD. Like `stripNul`, call this ONLY on
* free-prose body columns, NEVER on identity/security fields (see NUL POLICY).
*/
export const sanitizeForJsonb = (s: string): string => ensureWellFormed(stripNul(s));
/**
* Null-safe free-prose sanitizer for plain TEXT / TEXT[] columns (title,
* compiled_truth, timeline, chunk_text). Same NUL + lone-surrogate cleanup as
* `sanitizeForJsonb` — a raw 0x00 in a page body otherwise aborts the INSERT
* with `invalid byte sequence for encoding "UTF8": 0x00` (Postgres text cannot
* store NUL), and a lone surrogate is likewise rejected. Passes null/undefined
* through unchanged so nullable columns keep their semantics. Same policy as
* sanitizeForJsonb: free-prose body only, NEVER identity/security fields.
*/
export const sanitizeText = <T extends string | null | undefined>(s: T): T =>
(typeof s === 'string' ? (sanitizeForJsonb(s) as T) : s);
/** One links row, keys === the jsonb_to_recordset column list. */
export interface LinkRow {
from_slug: string;
to_slug: string;
link_type: string;
context: string;
link_source: string;
origin_slug: string | null;
origin_field: string | null;
from_source_id: string;
to_source_id: string;
origin_source_id: string;
link_kind: string | null;
resolution_type: string | null;
}
/** One timeline row, keys === the jsonb_to_recordset column list. */
export interface TimelineRow {
slug: string;
date: string;
source: string;
summary: string;
detail: string;
source_id: string;
}
/** One takes row, keys === the jsonb_to_recordset column list. Numbers/booleans
* stay JSON-native so the recordset can declare native column types. */
export interface TakeRow {
page_id: number;
row_num: number;
claim: string;
kind: string;
holder: string;
weight: number;
since_date: string | null;
until_date: string | null;
source: string | null;
superseded_by: number | null;
active: boolean;
}
export function buildLinkRows(links: LinkBatchInput[]): LinkRow[] {
return links.map(l => ({
from_slug: l.from_slug,
to_slug: l.to_slug,
link_type: l.link_type || '',
context: sanitizeForJsonb(l.context || ''), // free-text body: NUL + lone-surrogate sanitized
link_source: l.link_source || 'markdown',
origin_slug: l.origin_slug || null,
origin_field: l.origin_field || null,
from_source_id: l.from_source_id || 'default',
to_source_id: l.to_source_id || 'default',
origin_source_id: l.origin_source_id || 'default',
link_kind: l.link_kind ?? null,
resolution_type: l.resolution_type ?? null,
}));
}
export function buildTimelineRows(entries: TimelineBatchInput[]): TimelineRow[] {
return entries.map(e => ({
slug: e.slug,
date: e.date,
source: sanitizeForJsonb(e.source || ''), // free-text body: NUL + lone-surrogate sanitized (#2011)
summary: sanitizeForJsonb(e.summary), // free-text body: NUL + lone-surrogate sanitized
detail: sanitizeForJsonb(e.detail || ''), // free-text body: NUL + lone-surrogate sanitized
source_id: e.source_id || 'default',
}));
}
/**
* Build takes rows AND report how many weights were clamped, so the caller can
* emit the TAKES_WEIGHT_CLAMPED stderr counter exactly as before. Weight
* normalization (clamp to [0,1] + round to 0.05 grid) stays centralized here.
*/
export function buildTakeRows(rowsIn: TakeBatchInput[]): { rows: TakeRow[]; weightClamped: number } {
let weightClamped = 0;
const rows = rowsIn.map(r => {
const { weight, clamped } = normalizeWeightForStorage(r.weight);
if (clamped) weightClamped++;
return {
page_id: r.page_id,
row_num: r.row_num,
claim: sanitizeForJsonb(r.claim), // free-text body: NUL + lone-surrogate sanitized
kind: r.kind,
holder: r.holder,
weight,
since_date: r.since_date ?? null,
until_date: r.until_date ?? null,
// free-text provenance → NUL + lone-surrogate sanitized (#2011); same
// jsonb crash class as claim. null/undefined stay null; '' stays ''.
source: r.source == null ? null : sanitizeForJsonb(r.source),
superseded_by: r.superseded_by ?? null,
active: r.active ?? true,
};
});
return { rows, weightClamped };
}
/** Rows per encoded batch of a page-scale chunk write (A3). */
export const ROW_BATCH_MAX_ROWS = 500;
/** UTF-8 bytes per encoded batch, measured on the batch's own JSON document. */
export const ROW_BATCH_MAX_BYTES = 16 * 1024 ** 2;
/**
* Encode `items` as JSON array documents for `jsonb_to_recordset`, one batch at
* a time, each at most ROW_BATCH_MAX_ROWS rows and ROW_BATCH_MAX_BYTES bytes (a
* single larger row travels alone). Only the batch being built is held as
* JSON, so a page's statement size and staging memory stay bounded whatever
* the page size. `maxBytes` lowers the byte bound (segmented-page staging
* uses STAGE_BATCH_BYTES, 4 MiB on PGLite).
*/
export function* jsonRowBatches<T>(items: Iterable<T>, encode: (item: T) => unknown, maxBytes = ROW_BATCH_MAX_BYTES): Generator<string> {
let rows: string[] = [];
let bytes = 2;
for (const item of items) {
const row = JSON.stringify(encode(item));
const size = Buffer.byteLength(row) + 1;
if (rows.length > 0 && (rows.length === ROW_BATCH_MAX_ROWS || bytes + size > maxBytes)) {
yield `[${rows.join(',')}]`;
rows = [];
bytes = 2;
}
rows.push(row);
bytes += size;
}
if (rows.length > 0) yield `[${rows.join(',')}]`;
}