Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.3 KB
Raw Blame
import type { BrainEngine } from './engine.ts';
import { readProjectionSnapshot, queuePageProjection, preparePageProjection, installPageProjection } from './page-state/projections.ts';
import { quoteIdentifier, resolveActiveEmbeddingColumnFromEngine } from './search/embedding-column.ts';
import { QUARANTINE_FILTER_FRAGMENT } from './quarantine.ts';
import { isMultimodalEnabled } from './sync.ts';
import { buildSegmentedProjection } from './page-body/projection.ts';

/** A segmented page with a body: its projection (`buildSegmentedProjection`) owes chunk rows even when it has none yet. */
const SEGMENTED_BODY = `(p.body_storage = 'segmented' AND p.body_bytes > 0)`;

export async function countArchivedEmbeddingWork(engine: BrainEngine, opts: { sourceId?: string; signature?: string; includeNullSignature?: boolean; includeUnsealed?: boolean } = {}, activeColumn?: string | null): Promise<number> {
  const column = activeColumn === undefined ? (await resolveActiveEmbeddingColumnFromEngine(engine)).name : activeColumn;
  const vector = column === null ? 'NULL::vector' : `c.${quoteIdentifier(column)}`;
  const [row] = await engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM pages p
    JOIN sources s ON s.id=p.source_id WHERE s.archived AND p.deleted_at IS NULL
      AND NOT (COALESCE(p.frontmatter,'{}'::jsonb) ? 'embed_skip')
      AND ($1::text IS NULL OR p.source_id=$1)
      AND (($4::boolean AND p.text_projection_revision IS DISTINCT FROM p.knowledge_revision)
        OR EXISTS (SELECT 1 FROM content_chunks c WHERE c.page_id=p.id AND (${vector} IS NULL
          OR c.embedded_text_hash <> md5(c.chunk_text)
          OR ($2::text IS NOT NULL AND (p.embedding_signature <> $2 OR ($3::boolean AND p.embedding_signature IS NULL)))))
        OR ((p.compiled_truth <> '' OR p.timeline <> '' OR ${SEGMENTED_BODY}) AND ${QUARANTINE_FILTER_FRAGMENT}
          AND NOT EXISTS (SELECT 1 FROM content_chunks c WHERE c.page_id=p.id)))`,
  [opts.sourceId ?? null, opts.signature ?? null, opts.includeNullSignature ?? false, opts.includeUnsealed ?? false]);
  if (!Number.isSafeInteger(row?.n) || row.n < 0) throw new Error('Cannot determine archived embedding work.');
  return row.n;
}

type ProjectionScope = { sourceId?: string; existingChunksOnly?: boolean; activeSourcesOnly?: boolean; stale?: { signature?: string; includeNullSignature?: boolean } };

/** The pages still waiting for a text projection under `opts`, as one WHERE clause over `pages p JOIN sources s`. */
async function pendingProjectionWhere(engine: BrainEngine, opts: ProjectionScope): Promise<{ where: string; params: unknown[] }> {
  const params: unknown[] = [opts.sourceId ?? null, opts.existingChunksOnly ?? false];
  let eligible = '';
  if (opts.stale) {
    const column = quoteIdentifier((await resolveActiveEmbeddingColumnFromEngine(engine)).name);
    params.push(opts.stale.signature ?? null, opts.stale.includeNullSignature ?? false);
    eligible = `AND (${SEGMENTED_BODY} OR EXISTS(SELECT 1 FROM content_chunks c WHERE c.page_id=p.id AND (
      c.${column} IS NULL OR c.embedded_text_hash <> md5(c.chunk_text)
      OR ($3::text IS NOT NULL AND (p.embedding_signature <> $3 OR ($4::boolean AND p.embedding_signature IS NULL))))))`;
  }
  const where = `p.deleted_at IS NULL
    AND NOT (COALESCE(p.frontmatter,'{}'::jsonb) ? 'embed_skip')
    AND p.text_projection_revision IS DISTINCT FROM p.knowledge_revision
    AND ($1::text IS NULL OR p.source_id=$1)
    ${opts.activeSourcesOnly ? 'AND NOT s.archived' : ''}
    AND (NOT $2::boolean OR ${SEGMENTED_BODY} OR EXISTS(SELECT 1 FROM content_chunks c WHERE c.page_id=p.id)) ${eligible}`;
  return { where, params };
}

/**
 * #6223: the first pages (at most `limit`, 1000 by default) of active sources
 * still waiting for a projection, for the report a caller makes after bounded
 * recovery left some blocked (image/media pages only their importer rebuilds).
 */
export async function listPendingProjectionPages(engine: BrainEngine, opts: ProjectionScope, limit = 1000) {
  const { where, params } = await pendingProjectionWhere(engine, opts);
  return engine.executeRaw<{ source_id: string; slug: string; page_kind: string }>(`SELECT p.source_id,p.slug,p.page_kind FROM pages p
    JOIN sources s ON s.id=p.source_id WHERE ${where} AND NOT s.archived ORDER BY p.id LIMIT $${params.length + 1}`, [...params, limit]);
}

/**
 * #6223: `embed --stale` keeps going when bounded recovery leaves pages
 * blocked; this adds one failure sample naming the first of them (and the
 * importer route for image/media pages) and returns the keys
 * (`source_id::slug`) of the pages it listed, so the drain counts their chunks
 * without renaming them as pages that changed mid-run.
 */
export async function reportBlockedProjections(engine: BrainEngine, opts: ProjectionScope & { sourceId?: string }, blocked: number, samples: string[]): Promise<Set<string>> {
  const pages = await listPendingProjectionPages(engine, opts);
  const named = pages.slice(0, 5);
  const more = blocked > named.length ? ` and ${blocked - named.length} more` : '';
  const sourceArg = opts.sourceId ? ` --source ${opts.sourceId}` : '';
  samples.push(`${blocked} page(s) still need a text projection, so their chunks were not embedded (${named.map(page => `${page.source_id}:${page.slug}`).join(', ')}${more}); every other stale chunk was embedded. `
    + (pages.some(page => page.page_kind !== 'markdown' && page.page_kind !== 'code')
      ? isMultimodalEnabled()
        ? `Image pages are sealed only by gbrain projections drain (importer-attested images, keyless; preview with gbrain projections drain --dry-run) or by their importer; this run's image sweep (images: missing_samples) could not rebuild the rest: restore their source files, then run gbrain embed --stale${sourceArg} again; gbrain doctor --only text_projection_readiness --json counts each class. `
        : `Image pages are sealed only by gbrain projections drain (importer-attested images, keyless; preview with gbrain projections drain --dry-run) or by their importer: set GBRAIN_EMBEDDING_MULTIMODAL=true so gbrain embed --stale${sourceArg} sweeps the rest (preview with gbrain embed --stale --images --dry-run${sourceArg}); gbrain doctor --only text_projection_readiness --json counts each class. ` : '')
    + 'Rerun gbrain embed --stale to retry bounded projection recovery.');
  return new Set(pages.map(page => `${page.source_id}::${page.slug}`));
}

export async function prepareEmbeddingProjections(engine: BrainEngine, opts: { sourceId?: string; limit?: number; repair?: boolean; existingChunksOnly?: boolean; activeSourcesOnly?: boolean; stale?: { signature?: string; includeNullSignature?: boolean }; signal?: AbortSignal; deadline?: number; assertOwned?: (tx?: BrainEngine) => Promise<void> } = {}) {
  const limit = Math.max(1, Math.min(opts.limit ?? 100, 100));
  const { where, params } = await pendingProjectionWhere(engine, opts);
  const countBlocked = async () => {
    const [remaining] = await engine.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM pages p
      JOIN sources s ON s.id=p.source_id WHERE ${where}`, params);
    if (!Number.isSafeInteger(remaining?.n) || remaining.n < 0) throw new Error('Cannot determine pending embedding projections.');
    return remaining.n;
  };
  let rebuilt = 0;
  let blocked = await countBlocked();
  // #6223: a keyset cursor visits each candidate once, so a catch-up run
  // cannot retry the same batch forever. #6803: it walks only the kinds a
  // rebuild can build; image pages stay counted as blocked (their seal is the
  // importer's or `gbrain projections drain`'s), never queued or re-stamped.
  let afterPageId = 0;
  while (opts.repair && blocked > 0 && !opts.signal?.aborted && Date.now() < (opts.deadline ?? Infinity)) {
    const rows = await engine.executeRaw<{ id: number; slug: string; source_id: string }>(`SELECT p.slug,p.source_id,p.id FROM pages p
      JOIN sources s ON s.id=p.source_id WHERE ${where} AND NOT s.archived AND p.id>$${params.length + 1}
        AND (p.page_kind IN ('markdown','code') OR p.body_storage='segmented')
      ORDER BY p.id LIMIT $${params.length + 2}`, [...params, afterPageId, limit]);
    if (!rows.length) break;
    afterPageId = Number(rows[rows.length - 1]!.id);
    for (const row of rows) {
      if (opts.signal?.aborted || Date.now() >= (opts.deadline ?? Infinity)) break;
      await opts.assertOwned?.();
      try {
        if (await readProjectionSnapshot(engine, row.slug, row.source_id)) continue;
        const [segmented] = await engine.executeRaw(`SELECT 1 FROM pages WHERE id=$1 AND body_storage='segmented'`, [row.id]);
        if (segmented) {
          if ((await buildSegmentedProjection(engine, { sourceId: row.source_id, slug: row.slug })).sealed) rebuilt++;
          continue;
        }
        const queued = await engine.transaction(async tx => {
          await opts.assertOwned?.(tx);
          await tx.lockPageKeys([{ sourceId: row.source_id, slug: row.slug }]);
          if (!await tx.readPageSnapshot(row.slug, { sourceId: row.source_id, requireLiveSource: true })) return false;
          opts.signal?.throwIfAborted();
          await queuePageProjection(tx, row.source_id, row.slug, 'embedding_recovery');
          opts.signal?.throwIfAborted();
          return true;
        });
        if (!queued) continue;
        const prepared = await readProjectionSnapshot(engine, row.slug, row.source_id, { allowUnsealed: true, requireLiveSource: true });
        if (!prepared) continue;
        const projection = await preparePageProjection(prepared);
        await opts.assertOwned?.();
        await engine.transaction(async tx => {
          await opts.assertOwned?.(tx);
          opts.signal?.throwIfAborted();
          if (Date.now() >= (opts.deadline ?? Infinity)) throw new Error('Projection recovery deadline exceeded.');
          await installPageProjection(tx, prepared, projection.chunks, { seal: true, preserveEmbeddings: true, code: projection.code });
          await opts.assertOwned?.(tx);
          opts.signal?.throwIfAborted();
          if (Date.now() >= (opts.deadline ?? Infinity)) throw new Error('Projection recovery deadline exceeded.');
        });
        rebuilt++;
      } catch {
        if (opts.signal?.aborted) break;
        await opts.assertOwned?.();
      }
    }
    await opts.assertOwned?.();
    blocked = await countBlocked();
    if (rows.length < limit) break;
  }
  return { rebuilt, blocked };
}