Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.6 KB
Raw Blame
/**
 * #6803 part 2: `gbrain projections drain` restores the importer's own seal on
 * `sealable` image pages (page-state/projection-class.ts). Migration v153 only
 * removed their seal marker: each chunk was written by the importer at or after
 * the safe-chunk fence with safeChunks true, and the page's revision has not
 * moved since. Explicit only: the resident never seals images.
 */
import type { BrainEngine } from '../engine.ts';
import { TEXT_REBUILD_PAGES_PER_SECOND } from './rebuild-estimate.ts';
import { quoteIdentifier } from '../search/embedding-column.ts';
import { chunkGenerationFilter, chunkGenerationProbe, movableChunkSql } from '../search/safe-chunks.ts';
import { sealableImageSql } from './projection-class.ts';
import { jsonRowBatches } from '../batch-rows.ts';
import { acceptedEmbeddingInputHashes } from '../embedding-input-hash.ts';
import { embeddingInputContext, embeddingWriteTarget } from './projections.ts';
import { AGENT_TOOL_LIMIT_MS } from '../agent-output.ts';

/**
 * Measured `sealImageProjections` throughput, the basis of doctor's drain
 * `--limit` and the post-upgrade estimate. Measured 2026-10-16 on a 4-vCPU
 * cloud machine, a full `drainProjections({ media: true })` of 10,000 queued
 * sealable images (statistics refresh included): 2,047 pages/s on PGLite,
 * 1,809 pages/s on local Postgres 16; this is the slower, rounded down.
 */
export const IMAGE_SEAL_PAGES_PER_SECOND = 1500;

export { TEXT_REBUILD_PAGES_PER_SECOND } from './rebuild-estimate.ts';

/**
 * Pages one drain run takes in doctor's fix: half the agent tool limit at the
 * measured rate, so a run fits the limit with room to spare. The drain takes
 * queued text first, so while `textPending` rows wait the slower text rate sizes it.
 */
export function projectionDrainLimit(textPending = 0): number {
  const rate = textPending > 0 ? TEXT_REBUILD_PAGES_PER_SECOND : IMAGE_SEAL_PAGES_PER_SECOND;
  return Math.max(100, Math.floor(rate * AGENT_TOOL_LIMIT_MS / 1000 / 2 / 100) * 100);
}

/** Whole minutes (at least 1) a drain of `pages` takes at the measured rate. */
export function projectionDrainMinutes(pages: number): number {
  return Math.max(1, Math.ceil(pages / IMAGE_SEAL_PAGES_PER_SECOND / 60));
}

/** Statement group of `sealImageProjections`: pages locked, read and sealed together. */
export const IMAGE_SEAL_GROUP = 500;

export interface ImageSealResult {
  sealed: number;
  /** Sealed pages whose chunk carries a visual vector: image search ranks them again. */
  sealed_with_visual: number;
  /** Sealed pages with no visual vector yet (imported with --no-embed): keyword-only until `gbrain embed --stale --images`. */
  sealed_text_only: number;
}

type Candidate = { source_id: string; source_incarnation: string; slug: string; revision: string };

/**
 * Restore the importer's own seal on `sealable` images, set-based: per group
 * of up to 500 pages, under the page guard, a compare-and-set on the queued
 * revision with the class re-checked under the guard. No chunk is written and
 * `embedding_image` and the keyword vector are untouched; a text vector whose recorded embedding input
 * no longer matches the page (#5553) is nulled for `embed --stale`, as
 * `installPageProjection` would. The queue trigger deletes the job of every
 * sealed page. No provider call, no file read. `notAfter` (a database
 * instant, text) leaves rows queued during the run for the next one.
 */
export async function sealImageProjections(engine: BrainEngine, opts: { limit?: number; notAfter?: string } = {}): Promise<ImageSealResult> {
  const result: ImageSealResult = { sealed: 0, sealed_with_visual: 0, sealed_text_only: 0 };
  let after: [string, string] = ['00000000-0000-0000-0000-000000000000', ''];
  let budget = opts.limit ?? Infinity;
  const target = await embeddingWriteTarget(engine);
  const col = quoteIdentifier(target.column.name);
  while (budget > 0) {
    // A primary-key window of queue rows, each probed by its page's unique key only: without planner statistics
    // (a fresh PGLite) a deleted_at filter here lets the planner pick pages_dedup_idx and scan the source per row.
    const params: unknown[] = [after[0], after[1], IMAGE_SEAL_GROUP];
    const notAfter = opts.notAfter === undefined ? '' : `AND j.updated_at<=$${params.push(opts.notAfter)}::text::timestamptz`;
    const window = await engine.executeRaw<Candidate & { sealable: boolean }>(`SELECT s.id AS source_id,j.source_incarnation::text AS source_incarnation,
        j.slug,j.revision::text AS revision,p.sealable IS TRUE AS sealable
      FROM (SELECT source_incarnation,slug,revision FROM page_projection_jobs j WHERE (j.source_incarnation,j.slug) > ($1::uuid,$2) ${notAfter}
        ORDER BY j.source_incarnation,j.slug LIMIT $3) j
      LEFT JOIN LATERAL (SELECT s.id FROM sources s WHERE s.incarnation=j.source_incarnation AND NOT s.archived OFFSET 0) s ON true
      LEFT JOIN LATERAL (SELECT p.deleted_at IS NULL AND p.knowledge_revision=j.revision AND ${sealableImageSql('p')} AS sealable
        FROM pages p WHERE p.source_id=s.id AND p.slug=j.slug OFFSET 0) p ON true
      ORDER BY j.source_incarnation,j.slug`, params);
    if (!window.length) break;
    const group = window.filter(row => row.sealable).slice(0, budget);
    const last = group.length === budget ? group[group.length - 1]! : window[window.length - 1]!;
    after = [last.source_incarnation, last.slug];
    if (!group.length) continue;
    budget -= group.length;
    const sealed = await engine.transaction(async tx => {
      await tx.lockPageKeys(group.map(row => ({ sourceId: row.source_id, slug: row.slug })));
      const pages = await tx.executeRaw<{ id: number; revision: string; title: string; mode: string | null;
        corpus_generation: string | null }>(`SELECT p.id,p.knowledge_revision::text AS revision,p.title,
          p.contextual_retrieval_mode AS mode,p.corpus_generation
        FROM jsonb_to_recordset($1::text::jsonb) AS v(source_id text, slug text, revision uuid)
        JOIN sources s ON s.id=v.source_id AND NOT s.archived
        CROSS JOIN LATERAL (SELECT p.id FROM pages p WHERE p.source_id=v.source_id AND p.slug=v.slug OFFSET 0) k
        JOIN pages p ON p.id=k.id
        WHERE p.deleted_at IS NULL AND p.knowledge_revision=v.revision AND ${sealableImageSql('p')}`, [JSON.stringify(group.map(({ source_id, slug, revision }) => ({ source_id, slug, revision })))]);
      if (!pages.length) return [];
      const chunks = await tx.executeRaw<{ id: number; page_id: number; chunk_text: string; chunk_source: string;
        embedding_input_hash: string | null; current: boolean }>(`SELECT id,page_id,chunk_text,chunk_source,embedding_input_hash,
          ${movableChunkSql(col)} AND model IS NOT DISTINCT FROM $2 AS current
        FROM content_chunks c WHERE c.page_id=ANY($1::int[]) AND c.${col} IS NOT NULL AND ${chunkGenerationProbe('c')}`, [pages.map(page => Number(page.id)), target.provenanceModel]);
      const byId = new Map(pages.map(page => [Number(page.id), page]));
      const stale = chunks.filter(chunk => {
        const page = byId.get(Number(chunk.page_id))!;
        if (!chunk.current) return true;
        if (chunk.embedding_input_hash === null) return page.mode != null;
        const provenance = embeddingInputContext(target, page.title, page.corpus_generation, [chunk]);
        return !acceptedEmbeddingInputHashes(provenance, page.mode, chunk).includes(chunk.embedding_input_hash);
      }).map(chunk => Number(chunk.id));
      if (stale.length) {
        await tx.executeRaw(`UPDATE content_chunks SET ${col}=NULL,embedded_at=NULL,embedded_text_hash=NULL,embedding_input_hash=NULL,
          embedding_pending_since=COALESCE(embedding_pending_since,now()) WHERE id=ANY($1::int[])`, [stale]);
      }
      const done: Array<{ has_visual: boolean }> = [];
      // Only the marker: the importer's keyword vector is still the page's own, since its revision has not moved.
      for (const batch of jsonRowBatches(pages, page => ({ id: Number(page.id), revision: page.revision }))) {
        done.push(...await tx.executeRaw<{ has_visual: boolean }>(`UPDATE pages SET text_projection_revision=pages.knowledge_revision
          FROM jsonb_to_recordset($1::text::jsonb) AS v(id int, revision uuid)
          WHERE pages.id=v.id AND pages.knowledge_revision=v.revision
          RETURNING EXISTS (SELECT 1 FROM content_chunks c WHERE c.page_id=pages.id AND c.embedding_image IS NOT NULL
            AND ${chunkGenerationFilter('pages', 'c')}) AS has_visual`, [batch]));
      }
      return done;
    });
    result.sealed += sealed.length;
    result.sealed_with_visual += sealed.filter(row => row.has_visual).length;
  }
  result.sealed_text_only = result.sealed - result.sealed_with_visual;
  return result;
}