Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.4 KB
Raw Blame
/**
 * #7191 ask 2: `search_exact` (CLI `gbrain search --exact "<phrase>"`), the
 * exhaustive literal scan agents otherwise run with grep. Unranked: every page
 * with a visible text chunk containing the phrase, in page-id order, plus a
 * `complete` flag that is true only when every page in the caller's scope was
 * scanned and none of them had text the scan could not read.
 *
 * Match: the phrase's words in order with any run of whitespace between them
 * (a line break included), case-insensitive unless `caseSensitive`. Nothing
 * else is normalized: no stemming, punctuation and accents are literal.
 *
 * What it reads: `content_chunks.chunk_text` (an image page's chunk holds its OCR text) under the same
 * visibility predicate as ranked search (live page, unarchived source, not
 * quarantined, current text projection, live body generation, the caller's
 * source scope, private pages and the trust floor for an untrusted caller, safe
 * chunks for a remote caller), so a remote caller never matches fence text.
 * Chunks overlap by a sentence-aware margin, so a phrase inside one sentence is
 * always inside one chunk.
 *
 * What makes it incomplete: the deadline or the page limit (both resumable
 * from `resume_after`), and pages in scope whose text the scan cannot read:
 * a page with body text and no visible chunk (`unchunked`, e.g. a page
 * held from chunking), a page whose text projection is behind its revision
 * (`projection_pending`), and for a remote caller a page indexed before the
 * safe-chunk fence (`unsealed`). Those are listed, never silently skipped.
 *
 * Scan shape: keyset batches of pages by id, each one statement, run with JIT
 * off (the private-visibility subplans cost past `jit_above_cost`, and JIT
 * compilation alone took about 1 s per batch on Postgres 16). A statement
 * that times out is retried with half the batch; a single page that still
 * times out is recorded as `scan_timeout` and passed.
 */
import type { BrainEngine } from '../engine.ts';
import type { TrustTier } from '../trust/tier.ts';
import { quarantineFilterFragment } from '../quarantine.ts';
import { privatePagesFilterForCaller } from './private-visibility.ts';
import { pageEligibleSql } from '../eligibility/sql.ts';
import { buildHardExcludeClause } from './sql-ranking.ts';
import { escapeLikePattern } from './sql-ranking.ts';
import { isStatementTimeoutError } from '../retry-matcher.ts';
import { chunkVisibleFilter, safeChunksFilter, SAFE_FENCE_CHUNKER_VERSION, servedTextProjectionFilter } from './safe-chunks.ts';

export const EXACT_DEFAULT_LIMIT = 100;
export const EXACT_MAX_LIMIT = 1000;
export const EXACT_DEFAULT_SECONDS = 30;
export const EXACT_MAX_SECONDS = 120;
export const EXACT_MAX_PHRASE_CHARS = 500;
export const EXACT_SNIPPETS_PER_PAGE = 3;
export const EXACT_SNIPPET_CONTEXT = 80;
const BATCH_PAGES = 2000;
const GAP_SAMPLE = 20;

export type ExactGapReason = 'unchunked' | 'projection_pending' | 'unsealed' | 'scan_timeout';

export interface ExactScanOpts {
  phrase: string;
  caseSensitive?: boolean;
  sourceId?: string;
  sourceIds?: string[];
  types?: string[];
  slugPrefix?: string;
  excludeSlugPrefixes: string[];
  excludePrivate: boolean;
  remote: boolean;
  minTrust?: TrustTier;
  limit: number;
  afterPageId?: number;
  deadlineMs: number;
  /** Test seam: pages per statement. */
  batchPages?: number;
}

export interface ExactMatch {
  page_id: number;
  slug: string;
  source_id: string;
  title: string;
  type: string;
  matching_chunks: number;
  snippets: Array<{ chunk_index: number; chunk_source: string; text: string }>;
}

export interface ExactScanResult {
  matches: ExactMatch[];
  complete: boolean;
  incomplete_reasons: Array<'deadline' | 'limit' | 'unreadable_pages'>;
  resume_after: number | null;
  pages_scanned: number;
  unreadable: { total: number; by_reason: Partial<Record<ExactGapReason, number>>; sample: Array<{ slug: string; source_id: string; reason: ExactGapReason }> };
}

/** The phrase as a POSIX ARE: each word escaped literally, words joined by one or more whitespace characters. */
export function phrasePattern(phrase: string): { regex: string; longest: string } | null {
  const words = phrase.trim().split(/\s+/).filter(Boolean);
  if (words.length === 0) return null;
  const regex = words.map(w => w.replace(/[\\^$.|?*+()[\]{}]/g, ch => '\\' + ch)).join('[[:space:]]+');
  const longest = words.reduce((a, b) => (b.length > a.length ? b : a));
  return { regex, longest };
}

interface BatchRow {
  page_id: number;
  slug: string | null;
  source_id: string | null;
  title: string | null;
  type: string | null;
  gap: ExactGapReason | null;
  matching_chunks: number | null;
  snippets: unknown;
}

/** One batch statement: every page of the next `batchPages` in scope after `after`, in id order, with its matches or the reason its text could not be read. */
export function buildExactBatchStatement(opts: ExactScanOpts, after: number, batchPages: number): { sql: string; params: unknown[] } | null {
  const pattern = phrasePattern(opts.phrase);
  if (!pattern) return null;
  const params: unknown[] = [];
  const add = (v: unknown) => { params.push(v); return '$' + params.length; };
  const afterP = add(after);
  const limitP = add(batchPages);
  const flags = opts.caseSensitive ? "''" : "'i'";
  const re = add(pattern.regex);
  const snippetRe = add('.{0,' + EXACT_SNIPPET_CONTEXT + '}' + pattern.regex + '.{0,' + EXACT_SNIPPET_CONTEXT + '}');
  const like = add('%' + escapeLikePattern(pattern.longest) + '%');
  const op = opts.caseSensitive ? '~' : '~*';
  const likeOp = opts.caseSensitive ? 'LIKE' : 'ILIKE';
  const pageFilters: string[] = [
    'p.id > ' + afterP + '::int',
    'p.deleted_at IS NULL',
    'NOT s.archived',
    quarantineFilterFragment('p'),
    "NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'pinned_question')",
  ];
  if (opts.sourceIds?.length) pageFilters.push('p.source_id = ANY(' + add(opts.sourceIds) + '::text[])');
  else if (opts.sourceId) pageFilters.push('p.source_id = ' + add(opts.sourceId));
  if (opts.types?.length) pageFilters.push('p.type = ANY(' + add(opts.types) + '::text[])');
  if (opts.slugPrefix) pageFilters.push('p.slug LIKE ' + add(escapeLikePattern(opts.slugPrefix) + '%') + " ESCAPE '\\'");
  if (opts.excludePrivate) pageFilters.push(privatePagesFilterForCaller('p'));
  if (opts.minTrust) pageFilters.push(pageEligibleSql('p', { floor: opts.minTrust }));
  const hardExclude = buildHardExcludeClause('p.slug', opts.excludeSlugPrefixes);
  const safe = opts.remote ? ` AND ${safeChunksFilter('p')}` : '';
  const unsealed = opts.remote ? `WHEN p.chunker_version < ${SAFE_FENCE_CHUNKER_VERSION} THEN 'unsealed'` : '';
  // Sizes only (octet_length reads the stored length, never the body): does the page have text to index?
  const hasBody = `(octet_length(p.compiled_truth) > 0 OR octet_length(p.timeline) > 0 OR (p.body_storage = 'segmented' AND p.body_bytes > 0))`;
  const sql = `WITH batch AS MATERIALIZED (
  SELECT p.id FROM pages p JOIN sources s ON s.id = p.source_id
   WHERE ${pageFilters.join('\n     AND ')} ${hardExclude}
   ORDER BY p.id LIMIT ${limitP}::int
), hits AS (
  SELECT cc.page_id, cc.chunk_index, cc.chunk_source,
         (regexp_match(cc.chunk_text, ${snippetRe}, ${flags}))[1] AS snippet,
         row_number() OVER (PARTITION BY cc.page_id ORDER BY cc.chunk_index) AS rn
    FROM batch b JOIN pages p ON p.id = b.id JOIN content_chunks cc ON cc.page_id = b.id
   WHERE ${chunkVisibleFilter('p', 'cc')}${safe}
     AND cc.chunk_text ${likeOp} ${like} ESCAPE '\\' AND cc.chunk_text ${op} ${re}
), per_page AS (
  SELECT h.page_id, count(*)::int AS matching_chunks,
         json_agg(json_build_object('chunk_index', h.chunk_index, 'chunk_source', h.chunk_source, 'text', h.snippet) ORDER BY h.chunk_index)
           FILTER (WHERE h.rn <= ${EXACT_SNIPPETS_PER_PAGE}) AS snippets
    FROM hits h GROUP BY h.page_id
)
SELECT p.id AS page_id, p.slug, p.source_id, p.title, p.type, pp.matching_chunks, pp.snippets,
       CASE WHEN pp.page_id IS NOT NULL THEN NULL ${unsealed}
            WHEN NOT ${servedTextProjectionFilter('p')} THEN 'projection_pending'
            WHEN NOT EXISTS (SELECT 1 FROM content_chunks vc WHERE vc.page_id = p.id AND ${chunkVisibleFilter('p', 'vc')}) AND ${hasBody} THEN 'unchunked'
            ELSE NULL END AS gap
  FROM batch b JOIN pages p ON p.id = b.id LEFT JOIN per_page pp ON pp.page_id = b.id
 ORDER BY p.id`;
  return { sql, params };
}

/** Runs the scan from `afterPageId` until the scope ends, the limit fills or the deadline passes. */
export async function runExactScan(engine: Pick<BrainEngine, 'executeRaw'>, opts: ExactScanOpts): Promise<ExactScanResult> {
  const matches: ExactMatch[] = [];
  const gaps: Array<{ page_id: number; slug: string; source_id: string; reason: ExactGapReason }> = [];
  let after = opts.afterPageId ?? 0;
  let pagesScanned = 0;
  let batch = opts.batchPages ?? BATCH_PAGES;
  let reason: 'deadline' | 'limit' | null = null;
  let done = false;
  while (!done) {
    const remaining = opts.deadlineMs - Date.now();
    if (remaining <= 0) { reason = 'deadline'; break; }
    const stmt = buildExactBatchStatement(opts, after, batch);
    if (!stmt) { done = true; break; }
    let rows: BatchRow[];
    try {
      rows = await engine.executeRaw<BatchRow>(stmt.sql, stmt.params, { timeoutMs: Math.max(1000, remaining), jitOff: true });
    } catch (e) {
      if (!isStatementTimeoutError(e)) throw e;
      if (Date.now() >= opts.deadlineMs) { reason = 'deadline'; break; }
      if (batch > 1) { batch = Math.max(1, Math.floor(batch / 2)); continue; }
      const [next] = await engine.executeRaw<{ id: number; slug: string; source_id: string }>(
        'SELECT id, slug, source_id FROM pages WHERE id > $1::int AND deleted_at IS NULL ORDER BY id LIMIT 1', [after]);
      if (!next) { done = true; break; }
      gaps.push({ page_id: Number(next.id), slug: next.slug, source_id: next.source_id, reason: 'scan_timeout' });
      after = Number(next.id);
      pagesScanned++;
      continue;
    }
    if (rows.length === 0) { done = true; break; }
    for (const row of rows) {
      const id = Number(row.page_id);
      if (row.matching_chunks) {
        if (matches.length >= opts.limit) { reason = 'limit'; break; }
        const snippets = (typeof row.snippets === 'string' ? JSON.parse(row.snippets) : row.snippets) as ExactMatch['snippets'] | null;
        matches.push({ page_id: id, slug: row.slug!, source_id: row.source_id!, title: row.title ?? row.slug!, type: row.type ?? '',
          matching_chunks: Number(row.matching_chunks), snippets: (snippets ?? []).map(s => ({ chunk_index: Number(s.chunk_index), chunk_source: s.chunk_source, text: s.text })) });
      } else if (row.gap) {
        gaps.push({ page_id: id, slug: row.slug!, source_id: row.source_id!, reason: row.gap });
      }
      after = id;
      pagesScanned++;
    }
    if (reason === 'limit') break;
    if (rows.length < batch) done = true;
  }
  const counted = gaps;
  const byReason: Partial<Record<ExactGapReason, number>> = {};
  for (const g of counted) byReason[g.reason] = (byReason[g.reason] ?? 0) + 1;
  const reasons: ExactScanResult['incomplete_reasons'] = [];
  if (reason) reasons.push(reason);
  if (counted.length) reasons.push('unreadable_pages');
  return {
    matches,
    complete: reasons.length === 0,
    incomplete_reasons: reasons,
    resume_after: reason ? after : null,
    pages_scanned: pagesScanned,
    unreadable: { total: counted.length, by_reason: byReason, sample: counted.slice(0, GAP_SAMPLE).map(({ slug, source_id, reason: r }) => ({ slug, source_id, reason: r })) },
  };
}