Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.6 KB
Raw Blame
/**
 * The Wave A oversized-page backfill (`gbrain reindex --oversized`, and a
 * time-boxed pass in `gbrain post-upgrade`).
 *
 * Before the 24 MiB page ceiling, the import gate stored any page over
 * `content_sanity.bytes_block` (500 KB) with a gate-written
 * `embed_skip: { reason: oversized }` marker and zero chunks, so it was
 * invisible to keyword and vector search. Removing the marker alone does not
 * help: import returns early on an unchanged content hash, so the page would
 * stay chunkless. This backfill rewrites each such page through the normal
 * write path (the canonical owner on a managed brain, exactly like
 * `gbrain quarantine clear`) with the marker and its `content_flag: oversized`
 * removed and a forced re-chunk. That is CPU-only and runs with embeddings
 * disabled: chunks land with NULL vectors and become keyword-searchable at
 * once; the embedding sweep fills vectors later.
 *
 * Limits are preserved: a page still above the effective `bytes_block`
 * (env `GBRAIN_PAGE_BLOCK_BYTES`, file or DB `content_sanity.bytes_block`)
 * keeps its marker and gets no vectors, and only the gate-written marker shape
 * (`reason: oversized`) is touched. A chunkless one is still re-chunked with
 * the marker kept (#6722), so every page up to the ceiling is keyword-searchable. A large page (over `pages.inline_max_bytes`) is held in
 * `page_embedding_budget` before its rewrite, so no paid embedding starts
 * until the user approves the previewed cost (`gbrain embed --held --source S --max-usd N`).
 *
 * Idempotent and resumable: the work list is recomputed from the markers on
 * every run, so a page done is never redone and an interrupted run continues.
 */
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from './engine.ts';
import { EMBED_SKIP_KEY } from './embed-skip.ts';
import { CONTENT_FLAG_KEY } from './quarantine.ts';
import { serializeMarkdown } from './markdown.ts';
import { importFromContent } from './import-file.ts';
import { loadConfig } from './config.ts';
import type { OperationContext } from './ops/contract.ts';
import { managedPersistenceEnabled } from './persistence/ownership.ts';
import { submitPageMutation } from './persistence/page-mutations.ts';
import { resolveCliWriteWaitMs } from './persistence/write-wait.ts';
import { effectiveBytesBlock } from './import-screen.ts';
import { PAGE_MAX_BYTES, isLargePageBytes } from './page-size-policy.ts';
import { estimatePageEmbedUsd, holdPageEmbedding } from './page-embedding-budget.ts';
import { currentEmbeddingSignature } from './embedding.ts';
import { budgetLimitedReport } from './held-embedding.ts';
import type { PageType } from './types.ts';
import { materializedSnapshot } from './persistence/segmented-apply.ts';

/** The `page_embedding_budget.reason` a backfilled large page waits under. */
export const BACKFILL_HOLD_REASON = 'upgrade_backfill';

const OVERSIZED_MARKER_SQL = `p.deleted_at IS NULL AND p.frontmatter->'${EMBED_SKIP_KEY}'->>'reason' = 'oversized'`;
/** A segmented page (Wave B) has no body columns: its size is `body_bytes`. */
const PAGE_BYTES_SQL = `(CASE WHEN p.body_storage = 'segmented' THEN p.body_bytes ELSE octet_length(p.compiled_truth) + octet_length(COALESCE(p.timeline, '')) END)`;
/** Under the block (marker removed), or over it, under the ceiling and chunkless (re-chunked, marker kept). $1 is the block. */
const CANDIDATE_SQL = `(${PAGE_BYTES_SQL} <= $1 OR (${PAGE_BYTES_SQL} <= ${PAGE_MAX_BYTES} AND NOT EXISTS (SELECT 1 FROM content_chunks cc WHERE cc.page_id = p.id)))`;

export interface OversizedBackfillReceipt {
  dry_run: boolean;
  /** The bytes_block in effect: pages at or under it (and the ceiling) are re-indexed. */
  block_bytes: number;
  /** Size-skipped pages under the block when the run started. */
  candidates: number;
  reindexed: number;
  /** Large pages whose embedding waits for approval. */
  held_for_approval: number;
  failed: Array<{ slug: string; source_id: string; error: string }>;
  /** Candidates this run did not reach (limit or time budget); the same command continues. */
  remaining: number;
  /** Size-skipped pages still over the block: they keep the marker and get no vectors. */
  over_block: number;
  /** Chunkless pages over the block this run re-chunked for keyword search, marker kept (#6722). */
  rechunked_over_block: number;
  chunks_pending_embedding: number;
  /** null when embeddings are off on this brain. */
  estimated_usd: number | null;
  embedding_enabled: boolean;
  /** Exact commands: finish the backfill; preview held embedding; approve it for the source with the most waiting chunks (null until a run holds pages). */
  commands: { finish: string[]; approve: string[] | null; preview: string[] };
  /** Every source's approve command (a concrete --max-usd ceiling), one approval per source (#6803). */
  approvals: string[];
}

interface Candidate { id: number; slug: string; source_id: string; bytes: number }

async function listCandidates(engine: BrainEngine, block: number, opts: { sourceId?: string; afterId: number; limit: number }): Promise<Candidate[]> {
  const params: unknown[] = [block, opts.afterId, opts.limit];
  const source = opts.sourceId ? ` AND p.source_id = $${params.push(opts.sourceId)}` : '';
  const rows = await engine.executeRaw<Candidate>(`SELECT p.id, p.slug, p.source_id, ${PAGE_BYTES_SQL}::bigint AS bytes FROM pages p
    WHERE ${OVERSIZED_MARKER_SQL} AND ${CANDIDATE_SQL} AND p.id > $2${source} ORDER BY p.id LIMIT $3`, params);
  return rows.map(row => ({ ...row, id: Number(row.id), bytes: Number(row.bytes) }));
}

async function countMarked(engine: BrainEngine, block: number, sourceId: string | undefined): Promise<{ candidates: number; over: number }> {
  const params: unknown[] = [block];
  const source = sourceId ? ` AND p.source_id = $${params.push(sourceId)}` : '';
  const [row] = await engine.executeRaw<{ candidates: number; over: number }>(`SELECT
      COUNT(*) FILTER (WHERE ${CANDIDATE_SQL})::int AS candidates, COUNT(*) FILTER (WHERE ${PAGE_BYTES_SQL} > $1)::int AS over
    FROM pages p WHERE ${OVERSIZED_MARKER_SQL}${source}`, params);
  return { candidates: Number(row?.candidates ?? 0), over: Number(row?.over ?? 0) };
}

/** Rewrite one page re-chunked with no embedding: without its size-gate markers under the block, with them kept over it. */
async function reindexPage(engine: BrainEngine, page: Candidate, managed: boolean, overBlock: boolean): Promise<void> {
  // Every candidate is within the single-string ceiling, so a segmented body is read back whole and rewritten through the segmented write path.
  const snapshot = await materializedSnapshot(engine, await engine.readPageSnapshot(page.slug, { sourceId: page.source_id }));
  if (!snapshot || snapshot.page.id !== page.id) return;
  const fm = { ...((snapshot.page.frontmatter ?? {}) as Record<string, unknown>) };
  if ((fm[EMBED_SKIP_KEY] as { reason?: unknown } | undefined)?.reason !== 'oversized') return;
  let prior: { state: string; reason: string | null } | null | undefined;
  if (!overBlock) {
    delete fm[EMBED_SKIP_KEY];
    if ((fm[CONTENT_FLAG_KEY] as { reason?: unknown } | undefined)?.reason === 'oversized') delete fm[CONTENT_FLAG_KEY];
    if (isLargePageBytes(page.bytes)) {
      [prior = null] = await engine.executeRaw<{ state: string; reason: string | null }>('SELECT state, reason FROM page_embedding_budget WHERE page_id = $1', [page.id]);
      await holdPageEmbedding(engine, page.id, BACKFILL_HOLD_REASON);
    }
  }
  try {
    await rewritePage(engine, page, snapshot, fm, managed);
  } catch (error) {
    // #7185: a rewrite that did not land keeps the marker, so the hold placed for it would wait on nothing; put the row back.
    if (prior === null) await engine.executeRaw('DELETE FROM page_embedding_budget WHERE page_id = $1', [page.id]);
    else if (prior) await engine.executeRaw('UPDATE page_embedding_budget SET state = $2, reason = $3, updated_at = now() WHERE page_id = $1', [page.id, prior.state, prior.reason]);
    throw error;
  }
}

async function rewritePage(engine: BrainEngine, page: Candidate, snapshot: NonNullable<Awaited<ReturnType<typeof materializedSnapshot>>>, fm: Record<string, unknown>, managed: boolean): Promise<void> {
  const markdown = serializeMarkdown(fm, snapshot.page.compiled_truth ?? '', snapshot.page.timeline ?? '', {
    type: (snapshot.page.type as PageType) ?? 'note', title: snapshot.page.title ?? '', tags: snapshot.tags,
  });
  if (!managed) {
    await importFromContent(engine, page.slug, markdown, { sourceId: page.source_id, noEmbed: true, forceRechunk: true, preserveGateMarkers: true });
    return;
  }
  const config = loadConfig() ?? { engine: engine.kind };
  const ctx = { engine, config, remote: false, dryRun: false, sourceId: page.source_id, writeWaitMs: resolveCliWriteWaitMs({ config }),
    logger: { info: () => {}, warn: (m: string) => console.error(m), error: (m: string) => console.error(m) } } as unknown as OperationContext;
  await submitPageMutation(ctx, { operation: 'put_page', managedFileImport: true, params: { slug: page.slug, source_id: page.source_id, content: markdown,
    kind: 'managed_quarantine_clear', expected_revision: snapshot.revision, request_id: randomUUID() } });
}

export async function runOversizedBackfill(engine: BrainEngine, opts: {
  dryRun?: boolean; sourceId?: string; limit?: number; deadlineMs?: number; now?: () => number;
  onPage?: (page: { slug: string; source_id: string; bytes: number }) => void;
} = {}): Promise<OversizedBackfillReceipt> {
  const now = opts.now ?? Date.now;
  const started = now();
  const block = await effectiveBytesBlock(engine);
  const before = await countMarked(engine, block, opts.sourceId);
  const embeddingEnabled = loadConfig()?.embedding_disabled !== true;
  const scope = opts.sourceId ? ['--source', opts.sourceId] : [];
  const receipt: OversizedBackfillReceipt = {
    dry_run: opts.dryRun === true, block_bytes: block, candidates: before.candidates, reindexed: 0, held_for_approval: 0, failed: [],
    remaining: before.candidates, over_block: before.over, rechunked_over_block: 0, chunks_pending_embedding: 0, estimated_usd: null, embedding_enabled: embeddingEnabled,
    commands: { finish: ['gbrain', 'reindex', '--oversized', ...scope], approve: null,
      preview: ['gbrain', 'embed', '--held', ...scope, '--dry-run'] }, approvals: [],
  };
  const managed = !receipt.dry_run && await managedPersistenceEnabled(engine);
  const touched: number[] = [];
  let pendingChars = 0;
  let afterId = 0;
  let attempted = 0;
  while (opts.limit === undefined || attempted < opts.limit) {
    const batch = await listCandidates(engine, block, { sourceId: opts.sourceId, afterId, limit: Math.min(100, (opts.limit ?? Infinity) - attempted) });
    if (batch.length === 0) break;
    for (const page of batch) {
      if (attempted > 0 && opts.deadlineMs !== undefined && now() - started >= opts.deadlineMs) break;
      afterId = page.id;
      attempted++;
      opts.onPage?.(page);
      const overBlock = page.bytes > block;
      if (receipt.dry_run) {
        if (overBlock) { receipt.rechunked_over_block++; continue; }
        if (isLargePageBytes(page.bytes)) receipt.held_for_approval++;
        pendingChars += page.bytes;
        continue;
      }
      try {
        await reindexPage(engine, page, managed, overBlock);
        receipt.reindexed++;
        if (overBlock) { receipt.rechunked_over_block++; continue; }
        touched.push(page.id);
        if (isLargePageBytes(page.bytes)) receipt.held_for_approval++;
      } catch (error) {
        receipt.failed.push({ slug: page.slug, source_id: page.source_id, error: (error instanceof Error ? error.message : String(error)).split('\n')[0].slice(0, 300) });
      }
    }
    if (opts.deadlineMs !== undefined && now() - started >= opts.deadlineMs) break;
  }
  if (touched.length) {
    const [pending] = await engine.executeRaw<{ chunks: number; chars: number }>(`SELECT COUNT(*)::int AS chunks, COALESCE(SUM(length(chunk_text)), 0)::bigint AS chars
      FROM content_chunks WHERE page_id = ANY($1::int[]) AND embedded_at IS NULL`, [touched]);
    receipt.chunks_pending_embedding = Number(pending?.chunks ?? 0);
    pendingChars = Number(pending?.chars ?? 0);
  }
  receipt.remaining = receipt.dry_run ? before.candidates : (await countMarked(engine, block, opts.sourceId)).candidates;
  if (embeddingEnabled) receipt.estimated_usd = estimatePageEmbedUsd(pendingChars, currentEmbeddingSignature());
  const found = embeddingEnabled && receipt.held_for_approval && !receipt.dry_run
    ? await budgetLimitedReport(engine, { ...(opts.sourceId ? { sourceId: opts.sourceId } : {}), signature: currentEmbeddingSignature() }) : null;
  if (found) {
    receipt.commands.approve = found.report.by_source[0]!.fix.argv!;
    receipt.approvals = found.report.by_source.map(s => s.fix.command!);
  }
  return receipt;
}

/** Human lines for a receipt (stdout of `gbrain reindex --oversized` and post-upgrade). */
export function formatOversizedBackfillReceipt(r: OversizedBackfillReceipt): string[] {
  const lines = [`[reindex --oversized] ${r.dry_run ? 'Would re-index' : 'Re-indexed'} ${r.dry_run ? r.candidates : r.reindexed} size-skipped page(s)`
    + `${r.dry_run ? '' : `; ${r.chunks_pending_embedding} chunk(s) are keyword-searchable now and wait for embedding`}.`];
  if (r.failed.length) lines.push(`  ${r.failed.length} page(s) failed (first: ${r.failed[0].source_id}/${r.failed[0].slug}: ${r.failed[0].error}); rerun: ${r.commands.finish.join(' ')}`);
  if (!r.dry_run && r.remaining > r.failed.length) lines.push(`  ${r.remaining} page(s) remain; continue with: ${r.commands.finish.join(' ')}`);
  if (r.over_block) lines.push(`  ${r.over_block} page(s) get no vectors: they are over content_sanity.bytes_block (${r.block_bytes} bytes)`
    + `${r.rechunked_over_block ? `; ${r.rechunked_over_block} of them ${r.dry_run ? 'would be' : 'were'} re-chunked for keyword search` : ''}. Chunks keep them in keyword search.`);
  if (!r.embedding_enabled) lines.push('  Embeddings are off on this brain, so keyword search is the whole result; nothing is spent.');
  else if (r.held_for_approval) {
    lines.push(`  ${r.held_for_approval} large page(s) wait for approval before embedding (about $${(r.estimated_usd ?? 0).toFixed(2)} in total). `
      + `Preview: ${r.commands.preview.join(' ')}${r.approvals.length ? `; with the user's agreement, approve one source at a time: ${r.approvals.join('; ')}` : ''}`);
  } else if (r.estimated_usd !== null && r.chunks_pending_embedding) {
    lines.push(`  The embedding sweep embeds them under the existing caps (about $${r.estimated_usd.toFixed(2)}).`);
  }
  return lines;
}

/**
 * `gbrain post-upgrade`: a two-minute backfill pass, then the receipt. When
 * large pages wait for paid embedding, an [AGENT] block asks the user with the
 * previewed cost before anything is spent. Best-effort: never blocks upgrade.
 */
export async function printOversizedBackfillUpgradeNotice(engine: BrainEngine): Promise<void> {
  try {
    const block = await effectiveBytesBlock(engine);
    if ((await countMarked(engine, block, undefined)).candidates === 0) return;
    const receipt = await runOversizedBackfill(engine, { deadlineMs: 120_000 });
    const lines = ['', ...formatOversizedBackfillReceipt(receipt)];
    if (receipt.embedding_enabled && receipt.held_for_approval > 0) {
      const { agentBlock } = await import('./agent-markers.ts');
      lines.push(...agentBlock({
        ask: `Embed the ${receipt.held_for_approval} large page(s) that are searchable by keyword now?`,
        why: 'Pages an older gbrain skipped for size are re-indexed and keyword-searchable. Embedding them adds semantic search and calls the paid embedding provider.',
        risk: `About $${(receipt.estimated_usd ?? 0).toFixed(2)} in total; each page is capped by embed.max_usd_per_page and the run by --max-usd (the lowest cap wins).`,
        consent: 'paid, egress',
        actor: 'user',
        next: 'ask_user',
        if_yes: `Preview: ${receipt.commands.preview.join(' ')}. Then, one source at a time, run ${receipt.approvals.join('; ') || 'the approve command the preview prints'}.`,
        if_no: 'Nothing to do: the pages stay keyword-searchable and doctor lists them under partial_embedding.',
        verify: 'gbrain doctor --only partial_embedding --json',
      }, {
        showUser: `${receipt.held_for_approval} large page(s) that older gbrain skipped are searchable by keyword now. `
          + `Embedding them for semantic search costs about $${(receipt.estimated_usd ?? 0).toFixed(2)}. Embed them?`,
      }).trimEnd().split('\n'));
    }
    for (const line of lines) console.log(line);
  } catch (error) {
    console.error(`[gbrain] The oversized-page backfill did not finish (${error instanceof Error ? error.message.split('\n')[0] : String(error)}); run gbrain reindex --oversized.`);
  }
}