Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Embedding state, derived (#6696 B3): where every piece of indexed text
* stands against the registry-ACTIVE embedding column, read from the rows
* themselves (no stored status column, no migration).
*
* Unit: one `content_chunks` row, plus one unit per page that has no chunk
* row. Each unit is in exactly one state; the classifier SQL lives once in
* `src/core/engine-sql/chunks.ts` (`chunkEmbeddingState` /
* `chunklessPageState`, next to `staleChunkWhere`) and both engines delegate
* to it. Precedence for a chunk: deleted page, quarantined page, stored
* vector, embed job predicate, else embed_skip. For a chunkless live page:
* quarantined, healable body, else empty.
*
* - `embedded`: a chunk on a live, unquarantined page whose active column
* holds a vector (vectors retained on an embed_skip page count: vector
* search reads them).
* - `pending`: a chunk the embed job selects (`staleChunkWhere`: active vector
* NULL, page live, no `embed_skip`) on an unquarantined page.
* `gbrain embed --stale` drains it. On a brain keyless by choice keyword
* search is the configured retrieval and reads these chunks, so they are
* not counted as unsearchable there.
* - `keyword_only`: a chunk with no vector on a live `embed_skip` page. By design (#4306: never purged; its
* text feeds keyword search). The import gate sets the marker for a body
* over the effective `content_sanity.bytes_block`, so removing the marker
* alone is undone on the next import; the exits are splitting the page or
* an operator-approved threshold override followed by a re-import.
* - `quarantined`: the chunks of a live page carrying the `quarantine` marker,
* or the page itself when it has none (the gate writes zero chunks). Hidden
* from search; doctor `quarantined_pages` owns the warning.
* - `empty`: a live page with no chunk rows and nothing to index: an empty
* body (`compiled_truth` and `timeline` both ''), or a current revision
* whose sealed text projection has no chunks (#8473: a body holding only
* a protected takes fence).
* - `deleted_page_chunks`: chunks on a soft-deleted page. The purge lifecycle
* (`gbrain pages purge-deleted`, the autopilot purge phase, 72 h) removes
* them; past that window they are overdue.
* - `chunkless_healable`: a live, unquarantined page with body text, no
* chunk rows and no sealed projection of its current revision
* (`listChunklessPagesWithContent`'s predicate): `gbrain embed
* --stale` chunks it before embedding. An `embed_skip` page is included
* (#7174): writes keep its chunks (#6722), so none means keyword search
* cannot find it; it is chunked and still gets no vectors.
*
* Unstamped pending chunks (`embedding_pending_since` NULL: written by a binary
* older than the v221 stamp) carry no age; the summary reports their count,
* this writer's version and a `created_at` lower bound, and never warns on it.
*/
import type { BrainEngine } from './engine.ts';
import type { Action } from './agent-output.ts';
import { embeddingsDisabled } from './embedding-disabled.ts';
import { SOFT_DELETE_TTL_HOURS } from './destructive-guard.ts';
import { DEFAULT_BYTES_BLOCK } from './content-sanity.ts';
import { loadConfig } from './config.ts';
import { embeddingEnablement } from './readiness.ts';
import { VERSION } from '../version.ts';
import { chunkGenerationFilter } from './search/safe-chunks.ts';
export type EmbeddingState = 'embedded' | 'pending' | 'keyword_only' | 'quarantined' | 'empty' | 'deleted_page_chunks' | 'chunkless_healable';
export { EMBEDDING_STATES } from './engine-sql/chunks.ts';
/** The states `unsearchable` sums (pending drops out on a brain keyless by choice). */
export const UNSEARCHABLE_STATES: readonly EmbeddingState[] = ['pending', 'quarantined', 'empty', 'chunkless_healable'];
/** Hours a soft-deleted page waits before the purge lifecycle removes it (`pages purge-deleted` default). */
export const DELETED_PAGE_PURGE_WINDOW_HOURS = SOFT_DELETE_TTL_HOURS;
export const EMBEDDING_STATE_DOCS = 'docs/guides/troubleshooting.md#embedding-state';
/** What the engine's classifier returns (`BrainEngine.readEmbeddingStateCounts`). */
export interface EmbeddingStateCounts {
active_column: string;
/** `unknown` only in ops mode without a covering partial index on the active column. */
basis: 'exact' | 'estimated' | 'unknown';
/** The partial index (`WHERE <active column> IS NULL`) ops mode reads through; null when none (or full mode). */
covering_index: string | null;
counts: Partial<Record<EmbeddingState, number>>;
/** States this mode does not count (ops: the anti-join states no index backs). */
omitted: EmbeddingState[];
/** Embedded chunks on pages the embed job serves (no embed_skip); null in ops mode. */
eligible_embedded: number | null;
deleted_overdue: number;
embed_skip_pages: number;
embed_skip_marker_bytes: number;
unstamped_pending: number;
unstamped_created_at_min: Date | null;
}
/** One `(page, state)` row (`BrainEngine.listEmbeddingStateRows`). */
export interface EmbeddingStateRow {
slug: string;
source_id: string;
state: EmbeddingState;
chunks: number;
/** Chunk text bytes, or the body bytes of a page with no chunk rows. */
bytes: number;
embed_skip_reason: string | null;
marker_bytes: number | null;
quarantine_reason: string | null;
deleted_at: Date | null;
overdue: boolean;
}
export interface EmbeddingStateSummary {
status: 'known' | 'unknown';
basis: 'exact' | 'estimated' | 'unknown';
reason?: string;
active_column: string;
counts: Partial<Record<EmbeddingState, number>>;
/** pending + quarantined + empty + chunkless_healable (never keyword_only or deleted_page_chunks); null when unknown. */
unsearchable: number | null;
[k: string]: unknown;
}
/**
* The brain's embedding state. `mode: 'ops'` counts only what an index backs
* (Postgres and PGLite alike: the covering partial index for the active
* column, the frontmatter GIN index, the deleted-at purge index; `embedded` is
* a planner estimate) and reports `status: 'unknown'` with
* `reason: 'no_covering_index'` rather than scan; `empty` and
* `chunkless_healable` are omitted there (`unsearchable_complete: false`).
* `mode: 'full'` counts every state exactly. `sourceIds` is the caller's
* resolved scope (sourceScopeOpts); absent means brain-wide.
*/
export async function readEmbeddingState(engine: BrainEngine, opts: { mode: 'ops' | 'full'; sourceIds?: string[] }): Promise<EmbeddingStateSummary> {
const raw = await engine.readEmbeddingStateCounts({ mode: opts.mode, sourceIds: opts.sourceIds, purgeWindowHours: DELETED_PAGE_PURGE_WINDOW_HOURS });
if (raw.basis === 'unknown') {
return { status: 'unknown', basis: 'unknown', reason: 'no_covering_index', active_column: raw.active_column, counts: {}, unsearchable: null, mode: opts.mode };
}
const disabled = await embeddingsDisabled(engine);
const parts = UNSEARCHABLE_STATES.filter(state => !(disabled && state === 'pending'));
const counts = raw.counts;
const pending = counts.pending ?? 0;
const threshold = await effectiveBlockThreshold(engine);
const purgeable = raw.deleted_overdue ? Math.min(raw.deleted_overdue, await purgeableDeletedChunks(engine, opts)) : 0;
return {
status: 'known',
basis: raw.basis,
...(disabled ? { reason: 'embeddings_disabled' } : {}),
active_column: raw.active_column,
counts,
unsearchable: parts.reduce((sum, state) => sum + (counts[state] ?? 0), 0),
unsearchable_complete: parts.every(state => counts[state] !== undefined),
omitted_states: raw.omitted,
mode: opts.mode,
embeddings_enabled: !disabled,
units: 'content_chunks rows, plus one unit per page with no chunk rows',
covering_index: raw.covering_index,
coverage: raw.eligible_embedded === null
? { basis: 'estimated', ratio: ratio(counts.embedded, pending) }
: { basis: 'exact', embedded: raw.eligible_embedded, eligible: raw.eligible_embedded + pending, ratio: ratio(raw.eligible_embedded, pending) },
keyword_only: { reason: 'embed_skip', pages: raw.embed_skip_pages, marker_bytes: raw.embed_skip_marker_bytes, threshold_bytes: threshold },
deleted_page_chunks: { within_window: (counts.deleted_page_chunks ?? 0) - raw.deleted_overdue, past_window: raw.deleted_overdue,
purgeable, held: raw.deleted_overdue - purgeable, window_hours: DELETED_PAGE_PURGE_WINDOW_HOURS },
unstamped_pending: { count: raw.unstamped_pending, created_at_lower_bound: raw.unstamped_created_at_min?.toISOString() ?? null, writer_version: VERSION },
};
}
function ratio(embedded: number | undefined, pending: number): number | null {
const total = (embedded ?? 0) + pending;
return total === 0 ? null : Math.round(((embedded ?? 0) / total) * 10_000) / 10_000;
}
/** #7825: chunks past the window that `gbrain pages purge-deleted` would remove, under the purge's own holds (`purgeHoldsSql`); counted as the classifier counts them. */
async function purgeableDeletedChunks(engine: BrainEngine, opts: { mode: 'ops' | 'full'; sourceIds?: string[] }): Promise<number> {
const { purgeHoldsSql } = await import('./persistence/purge-deleted.ts');
const [row] = await engine.executeRaw<{ n: string | number }>(`SELECT count(*)::bigint AS n
FROM pages p JOIN sources s ON s.id = p.source_id JOIN content_chunks cc ON cc.page_id = p.id
WHERE p.deleted_at < now() - make_interval(hours => $1::int) AND ${purgeHoldsSql('p', 's')}
${opts.mode === 'full' ? `AND ${chunkGenerationFilter('p', 'cc')}` : ''}${opts.sourceIds ? ' AND p.source_id = ANY($2::text[])' : ''}`,
opts.sourceIds ? [DELETED_PAGE_PURGE_WINDOW_HOURS, opts.sourceIds] : [DELETED_PAGE_PURGE_WINDOW_HOURS]);
return Number(row?.n ?? 0);
}
/** The effective `content_sanity.bytes_block` the import gate applies (env > file > DB > default, capped at the page ceiling). */
async function effectiveBlockThreshold(engine: BrainEngine): Promise<number> {
try {
return await (await import('./import-screen.ts')).effectiveBytesBlock(engine);
} catch {
return DEFAULT_BYTES_BLOCK;
}
}
/** `gbrain pages unsearchable` for one state: the read-only check every fix verifies with. */
export function embeddingStateVerify(sourceId: string, state: EmbeddingState): { argv: string[] } {
return { argv: ['gbrain', 'pages', 'unsearchable', '--source', sourceId, '--state', state, '--json'] };
}
/**
* The exit for one state, built only from existing commands (agent operator
* protocol: argv, consent, actor, why, requires_exclusive, read-only verify,
* docs). Null for `embedded`, which needs none. Without `sourceId` the
* commands run brain-wide; pass `verify` then (doctor verifies with itself).
*/
export function embeddingStateFix(state: EmbeddingState, ctx: { sourceId?: string; slug?: string; markerBytes?: number | null; blockBytes?: number; embeddingsEnabled: boolean; verify?: { argv: string[] } }): Action | null {
const verify = ctx.verify ?? embeddingStateVerify(ctx.sourceId ?? 'default', state);
const base = { verify, docs: EMBEDDING_STATE_DOCS, requires_exclusive: false };
const source = ctx.sourceId ? ['--source', ctx.sourceId] : [];
switch (state) {
case 'embedded':
return null;
case 'pending':
case 'chunkless_healable': {
if (!ctx.embeddingsEnabled) return keylessFix(verify);
const run = ['gbrain', 'embed', '--stale', '--catch-up', ...source];
return { ...base, argv: [...run, '--yes'], preview_argv: [...run, '--dry-run'], consent: ['paid'], actor: 'agent',
why: state === 'pending'
? 'These chunks have no vector in the active embedding column, so vector search cannot see them. The catch-up drain embeds them; it sends their text to the embedding provider and costs money.'
: 'These pages have body text but no chunk rows, so neither keyword nor vector search can read their body. `embed --stale` chunks them first, then embeds the new chunks; it sends their text to the embedding provider and costs money.',
user_message: 'Some notes are not indexed for search yet. Indexing them sends their text to your embedding provider and costs a little money. OK to run it?' };
}
case 'keyword_only': {
// A marker written under an older, lower limit: the free backfill clears it; raising the limit to the marker would lower it.
if (ctx.markerBytes && ctx.blockBytes && ctx.markerBytes <= ctx.blockBytes) {
const reindex = ['gbrain', 'reindex', '--oversized', ...source];
return { ...base, argv: reindex, preview_argv: [...reindex, '--dry-run'], consent: [], actor: 'agent',
why: `This page was marked embed_skip under an older, lower size limit; at ${ctx.markerBytes} bytes it is under the current content_sanity.bytes_block (${ctx.blockBytes}). The backfill removes the stale marker and re-chunks it on the CPU at no cost; a page over 1 MiB then waits for the user's approval before paid embedding.` };
}
const threshold = ctx.markerBytes && ctx.markerBytes > 0 ? String(ctx.markerBytes + 1) : null;
const why = 'The import gate marked this page embed_skip because its body exceeds content_sanity.bytes_block; keyword search still reads its chunks. Removing the marker alone is undone on the next import. Two exits: split the page into smaller pages, or raise the threshold (a brain-wide setting the operator approves) and re-import, which embeds it.';
const reimport: Action = { ...base, argv: ['gbrain', 'sync', ...source, '--full'], consent: ['paid'], actor: 'agent',
why: 'Re-imports the source so the gate re-assesses the page under the new threshold, drops the marker and embeds the page.' };
if (!threshold) return { ...base, ask_user: true, argv: ['gbrain', 'config', 'get', 'content_sanity.bytes_block'], consent: [], actor: 'user', why,
user_message: 'An oversized page is searchable by keyword only. Split it into smaller pages, or raise the size limit for embedding and re-import it?' };
return { ...base, ask_user: true, argv: ['gbrain', 'config', 'set', 'content_sanity.bytes_block', threshold], consent: [], actor: 'user', why,
user_message: `A page of ${ctx.markerBytes} bytes is searchable by keyword only. Split it into smaller pages, or raise the size limit for embedding to ${threshold} bytes and re-import it (embedding costs a little money)?`,
then: reimport };
}
case 'quarantined':
return { ...base, ask_user: true, argv: ctx.slug ? ['gbrain', 'quarantine', 'clear', ctx.slug, ...(ctx.sourceId ? ['--source-id', ctx.sourceId] : []), '--force'] : ['gbrain', 'quarantine', 'list', '--json'],
preview_argv: ['gbrain', 'quarantine', 'list', '--json'], consent: ctx.slug ? ['paid'] : [], actor: 'user',
why: 'The content-quality gate hid this page as junk, so it has no searchable chunks. Clearing is the owner\'s judgement for a false positive; it re-imports and embeds the page.',
user_message: 'A page was hidden from search as junk. Review it and clear it if it is a false positive?' };
case 'empty':
return { ...base, ask_user: true, argv: ctx.slug ? ['gbrain', 'delete', ctx.slug, ...source, '--force'] : ['gbrain', 'list', '--json'],
consent: ['destructive'], actor: 'user',
why: 'The page has no indexable body text: its body is empty, or holds only content kept out of search (such as a takes fence). Writing content to it chunks it on the next write; delete it only if it is a stub nobody needs (soft delete, restorable).',
user_message: 'A page is empty, so there is nothing to search. Add content to it, or delete the empty stub?' };
case 'deleted_page_chunks':
return { ...base, argv: ['gbrain', 'pages', 'purge-deleted', '--yes'], preview_argv: ['gbrain', 'pages', 'purge-deleted', '--dry-run', '--json'],
consent: ['destructive'], actor: 'agent',
why: `These chunks belong to soft-deleted pages; search never reads them. The purge lifecycle hard-deletes pages soft-deleted more than ${DELETED_PAGE_PURGE_WINDOW_HOURS}h ago (brain-wide, irreversible), taking their chunks with them.` };
}
}
function keylessFix(verify: { argv: string[] }): Action {
try {
const cfg = loadConfig();
if (cfg) {
return { ...embeddingEnablement(cfg), verify, docs: EMBEDDING_STATE_DOCS };
}
} catch { /* unreadable config: the read-only fallback below */ }
return { argv: ['gbrain', 'doctor', '--only', 'embeddings', '--json'], consent: [], actor: 'agent', verify, docs: EMBEDDING_STATE_DOCS, requires_exclusive: false,
why: 'Embeddings are disabled on this brain by choice; keyword search reads these chunks. The embeddings check names the command that turns embeddings on, if the user wants them.' };
}