Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.0 KB
Raw Blame
/**
 * Keyset-batched rebuilds of stored keyword vectors, shared by
 * `gbrain reindex-search-vector` (every row, new language) and the
 * fts-cjk-boundary migration (only rows whose text has a CJK/ASCII boundary).
 *
 * Each batch is its own statement, never one transaction: on Postgres one
 * transaction would hold every row lock for the whole run, and a hang inside
 * it would bank zero progress. The keyset cursor persists after every batch
 * under `backfill.<checkpoint>.last_id`, so a killed run resumes with at most
 * one batch redone; the caller clears it when the run finishes.
 *
 * Every batch is bounded by an id range as well as a row count: it scans at
 * most `BACKFILL_RANGE` ids past the cursor and rebuilds at most
 * `BACKFILL_BATCH_SIZE` matching rows, so a filtered run (the migration) costs
 * one bounded scan per statement even when nothing matches, and the cursor
 * advances to the range end either way. The run stops at the table's highest
 * id read by the first batch; rows written later go through the new triggers.
 */

import type { BrainEngine } from './engine.ts';
import { checkpointKey } from './backfill-base.ts';
import { chunkSearchVectorSql, pageSearchVectorSql } from './fts-language.ts';
import { sanitizeRemoteBody } from './remote-body.ts';

/** Rows per backfill UPDATE. Keyset-batched so one statement never locks the whole table. */
export const BACKFILL_BATCH_SIZE = 5000;
/** Ids scanned per backfill statement, matched or not. */
export const BACKFILL_RANGE = 50_000;

export interface VectorBackfillOpts {
  lang: string;
  /** Checkpoint name (→ `backfill.<name>.last_id`). */
  checkpoint: string;
  /** Extra SQL predicate over the table's own columns; only matching rows are rebuilt. */
  where?: string;
  /** Ids scanned per statement; defaults to `BACKFILL_RANGE`. */
  range?: number;
  tick?: (n: number) => void;
}

async function savedCursor(engine: BrainEngine, checkpoint: string): Promise<number> {
  const saved = Number(await engine.getConfig(checkpointKey(checkpoint)));
  return Number.isFinite(saved) && saved > 0 ? saved : 0;
}

const batchIds = (table: string, cursor: number, end: number, where: string | undefined) => `
  SELECT id FROM ${table}
  WHERE search_vector IS NOT NULL AND id > ${cursor} AND id <= ${end}${where ? ` AND (${where})` : ''}
  ORDER BY id
  LIMIT ${BACKFILL_BATCH_SIZE}`;

/** Where the next batch starts: the last row rebuilt when the row cap was hit, else the range end. */
const nextCursor = (matched: number, lastId: number, end: number) => (matched >= BACKFILL_BATCH_SIZE ? lastId : end);

/** Chunk vectors are a pure function of the row's own columns, so one UPDATE per batch. Returns rows rebuilt. */
export async function backfillChunkVectors(engine: BrainEngine, opts: VectorBackfillOpts): Promise<number> {
  const range = opts.range ?? BACKFILL_RANGE;
  let cursor = await savedCursor(engine, opts.checkpoint);
  let hi: number | undefined;
  let total = 0;
  for (;;) {
    const end = cursor + range;
    const [batch] = await engine.executeRaw<{ hi: number; n: number; last: number }>(`
      WITH bound AS (SELECT coalesce(max(id), 0) AS hi FROM content_chunks),
      upd AS (
        UPDATE content_chunks SET search_vector = ${chunkSearchVectorSql(opts.lang)}
        WHERE id IN (${batchIds('content_chunks', cursor, end, opts.where)})
        RETURNING id
      )
      SELECT (SELECT hi FROM bound) AS hi, count(*)::int AS n, coalesce(max(upd.id), 0) AS last FROM upd
    `);
    if (!batch) break;
    hi ??= Number(batch.hi);
    const n = Number(batch.n);
    total += n;
    if (n > 0) opts.tick?.(n);
    cursor = nextCursor(n, Number(batch.last), end);
    await engine.setConfig(checkpointKey(opts.checkpoint), String(cursor));
    if (cursor >= hi) break;
  }
  return total;
}

/**
 * #6374: page vectors rebuilt with the seal's own expression. The pages
 * trigger fires only on `UPDATE OF title,timeline` (and a segmented page's
 * timeline_lead) and indexes the RAW
 * timeline, so neither an UPDATE-to-self nor re-firing it is right. Each batch
 * reads its rows through an UPDATE-to-self (row locks held for that statement
 * only, never across the TypeScript sanitize), sanitizes the timeline exactly
 * as the seal does, then writes `search_vector` guarded by the knowledge
 * revision it read: a page edited in between is skipped, and its own seal
 * indexes it under the current trigger. The timeline is the seal's input: a
 * page with a database withdrawal or purge (its own or source-wide) is read
 * through `readPageSnapshot`, whose timeline carries that overlay, so a
 * withdrawn fact never returns to the vector; other pages' raw timeline is
 * already their effective one. Returns rows read.
 */
export async function backfillPageVectors(engine: BrainEngine, opts: VectorBackfillOpts): Promise<number> {
  const range = opts.range ?? BACKFILL_RANGE;
  let cursor = await savedCursor(engine, opts.checkpoint);
  let hi: number | undefined;
  let total = 0;
  for (;;) {
    const end = cursor + range;
    const batch = await engine.executeRaw<{ hi: number; id: number | null; slug: string; source_id: string; timeline: string | null; knowledge_revision: string; withdrawn: boolean }>(`
      WITH bound AS (SELECT coalesce(max(id), 0) AS hi FROM pages),
      upd AS (
        UPDATE pages SET id = id
        WHERE id IN (${batchIds('pages', cursor, end, opts.where)})
        RETURNING id, slug, source_id, timeline, knowledge_revision::text AS knowledge_revision,
          (EXISTS (SELECT 1 FROM fact_withdrawals w WHERE w.source_id = pages.source_id AND (w.subject = '*' OR w.subject = pages.slug))
            OR EXISTS (SELECT 1 FROM fact_purges x WHERE x.source_id = pages.source_id AND (x.subject = '*' OR x.subject = pages.slug))) AS withdrawn
      )
      SELECT bound.hi, upd.* FROM bound LEFT JOIN upd ON true
    `);
    if (batch.length === 0) break;
    hi ??= Number(batch[0]!.hi);
    const rows = batch.filter(r => r.id !== null);
    const ids: number[] = [];
    const timelines: string[] = [];
    const revisions: string[] = [];
    for (const r of rows) {
      let timeline = r.timeline ?? '';
      if (r.withdrawn) {
        const snapshot = await engine.readPageSnapshot(r.slug, { sourceId: r.source_id, includeDeleted: true });
        if (!snapshot) continue;
        timeline = snapshot.page.timeline ?? '';
      }
      ids.push(Number(r.id));
      timelines.push(sanitizeRemoteBody(timeline));
      revisions.push(r.knowledge_revision);
    }
    if (ids.length > 0) await engine.executeRaw(`
      UPDATE pages p SET search_vector = ${pageSearchVectorSql('p.title', 'v.timeline', opts.lang)}
      FROM unnest($1::int[], $2::text[], $3::uuid[]) AS v(id, timeline, revision)
      WHERE p.id = v.id AND p.knowledge_revision = v.revision
    `, [ids, timelines, revisions]);
    total += rows.length;
    if (rows.length > 0) opts.tick?.(rows.length);
    cursor = nextCursor(rows.length, rows.reduce((m, r) => Math.max(m, Number(r.id)), cursor), end);
    await engine.setConfig(checkpointKey(opts.checkpoint), String(cursor));
    if (cursor >= hi) break;
  }
  return total;
}