Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.4 KB
Raw Blame
/**
 * #7165: query-scoped coverage for search and query. A pending text projection
 * or a held file is brain state; whether it can change THIS answer depends on
 * the query. An item could affect a query when it is one of the returned pages,
 * its page type is one of the result types or the query's types, or its title,
 * slug or file path holds a query word. An empty result counts every item:
 * nothing rules it out, and an empty answer must never read as complete.
 *
 * #7492: a query word counts only as a whole token of the title, slug or path
 * (bounded by characters other than letters and digits), so "412" no longer
 * matches an image frame named `t004120` and short words no longer match
 * inside longer ones. An image page counts only when an image arm ran or the
 * page was returned: text search never reads image chunks, so a pending image
 * cannot change a text-only answer, empty or not. The pending pages that could
 * affect the query are also counted by class (`could_affect.pending_images`:
 * images in the `sealable` or `reimport` class, which wait for an operator,
 * not for indexing), so the notice can say which wait applies.
 *
 * Counted here: pending text projections (the readiness probe's unsealed pages
 * that need work) and pages held from embedding (a held or capped
 * page_embedding_budget row that still applies: no vectors, so the vector arm
 * cannot find them). Held files are counted by persistence/held-reads.ts.
 */
import type { BrainEngine } from '../engine.ts';
import { budgetRowAppliesFilter } from '../page-size-policy.ts';
import { projectionClassSql } from '../page-state/projection-class.ts';
import { pageReadFilter } from './read-policy-sql.ts';
import { servedTextProjectionFilter } from './safe-chunks.ts';
import type { ProjectionReadinessScope } from './projection-readiness.ts';

/** Counts stop here (one bounded scan per call); a capped count reports `capped`. */
export const COVERAGE_COUNT_CAP = 1000;

export interface CoverageRelevance {
  all: boolean;
  types: string[];
  /** POSIX regular expressions matching one query word as a whole token, one per word. */
  patterns: string[];
  pageIds: number[];
  /** The top result's page when it is an exact slug, title or alias match for the query, else null. */
  exactTopPageId: number | null;
  /** Whether an image arm ran: otherwise an image page counts only when it was returned. */
  imageArm: boolean;
}

/** `_meta.retrieval.coverage`: counts in the read scope and how many of them could affect this query. */
export interface RetrievalCoverage {
  pending_projection?: number;
  held_files?: number;
  embedding_held?: number;
  could_affect: {
    pending_projection?: number; held_files?: number; embedding_held?: number;
    /** Of `pending_projection`, image pages in the `sealable` or `reimport` class (absent when none): they wait for an operator. */
    pending_images?: number;
  };
  capped?: true;
  /** The top result is the page the query named (exact slug, title or alias) and its projection is current: pending pages are not counted as able to affect this answer. */
  top_hit_exact?: true;
  /** #9192: while the upgrade's index rebuild (v153 activation) drains, pages in scope done of total and the rate measured since it started. */
  rebuild?: ProjectionRebuildProgress;
}

export interface ProjectionRebuildProgress { done: number; total: number; pages_per_second: number | null }

export interface ScopedCount {
  total: number; could_affect: number; capped: boolean;
  /** With an exact top hit: whether that page is itself counted. */
  top_counted?: boolean;
  /** Pending counts: the could-affect images in the `sealable` or `reimport` class. */
  could_affect_images?: number;
}

/** What could make a pending or held item relevant to this query (see the module comment). */
export function coverageRelevance(
  results: ReadonlyArray<{ type?: unknown; page_id?: unknown; exact_lookup?: unknown; alias_hit?: unknown }>,
  queryText: string,
  queryTypes: readonly string[] | undefined,
  stopwords: ReadonlySet<string>,
  imageArm = false,
): CoverageRelevance {
  const terms = [...new Set(queryText.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}'-]+/gu) ?? [])]
    .filter(t => t.length >= 3 && !stopwords.has(t)).slice(0, 12);
  return {
    all: results.length === 0,
    types: [...new Set([...(queryTypes ?? []), ...results.flatMap(r => (typeof r.type === 'string' ? [r.type] : []))])],
    patterns: terms.map(t => `(^|[^[:alnum:]])${t.replace(/[.*+?^${}()|[\]\\]/g, m => `\\${m}`)}([^[:alnum:]]|$)`),
    pageIds: [...new Set(results.flatMap(r => (typeof r.page_id === 'number' ? [r.page_id] : [])))],
    exactTopPageId: results[0] && (results[0].exact_lookup || results[0].alias_hit === true) && typeof results[0].page_id === 'number' ? results[0].page_id : null,
    imageArm,
  };
}

/** The readiness probe's scope predicates over `pages p` (sources, visibility, kind, types, excluded prefixes), binding into `params`. */
export function coverageScopeFilters(scope: ProjectionReadinessScope, params: unknown[]): string[] {
  const filters = [pageReadFilter('p', scope, params, true)];
  if (scope.sourceIds?.length === 0) filters.push('FALSE');
  if (scope.pageKind) {
    params.push(scope.pageKind);
    filters.push(`p.page_kind = $${params.length}`);
  }
  if (scope.types) {
    params.push(scope.types);
    filters.push(`p.type = ANY($${params.length}::text[])`);
  }
  for (const prefix of scope.excludeSlugPrefixes ?? []) {
    params.push(prefix);
    filters.push(`LEFT(p.slug, LENGTH($${params.length}::text)) <> $${params.length}`);
  }
  return filters;
}

/**
 * Pages in scope matching `where` (over `pages p` joined to `sources s` and `join`), up to the cap, and how many could affect the query.
 * `classed` also counts the could-affect images in the `sealable` or `reimport` class. Null when the count fails: unknown, never zero.
 */
async function countInScope(engine: Pick<BrainEngine, 'executeRaw'>, scope: ProjectionReadinessScope, relevance: CoverageRelevance,
  join: string, where: string, classed = false): Promise<ScopedCount | null> {
  const params: unknown[] = [];
  const filters = coverageScopeFilters(scope, params);
  params.push(relevance.imageArm, relevance.all, relevance.types, relevance.patterns, relevance.pageIds);
  const n = params.length;
  const could = `(($${n - 3}::boolean OR x.type = ANY($${n - 2}::text[]) OR x.haystack ~ ANY($${n - 1}::text[]) OR x.id = ANY($${n}::int[]))
    AND ($${n - 4}::boolean OR x.page_kind IS DISTINCT FROM 'image' OR x.id = ANY($${n}::int[])))`;
  const top = relevance.exactTopPageId;
  if (top !== null) params.push(top);
  try {
    const [row] = await engine.executeRaw<{ total: number | string; could_affect: number | string; could_affect_images?: number | string; top_counted?: boolean | null }>(
      `SELECT count(*)::int AS total,
        count(*) FILTER (WHERE ${could})::int AS could_affect${classed ? `,
        count(*) FILTER (WHERE ${could} AND x.class IN ('sealable','reimport'))::int AS could_affect_images` : ''}${top === null ? '' : `,
        EXISTS (SELECT 1 FROM pages p JOIN sources s ON s.id = p.source_id ${join} WHERE p.id = $${n + 1}::int AND p.deleted_at IS NULL AND ${where}) AS top_counted`}
      FROM (SELECT p.id, p.type, p.page_kind, lower(COALESCE(p.title, '') || ' ' || p.slug) AS haystack${classed ? `, ${projectionClassSql('p', 's')} AS class` : ''}
        FROM pages p JOIN sources s ON s.id = p.source_id ${join}
        WHERE p.deleted_at IS NULL AND ${where} AND ${filters.join(' AND ')}
        LIMIT ${COVERAGE_COUNT_CAP}) x`,
      params,
    );
    const total = Number(row?.total), couldAffect = Number(row?.could_affect);
    if (!Number.isFinite(total) || !Number.isFinite(couldAffect)) return null;
    return { total, could_affect: couldAffect, capped: total >= COVERAGE_COUNT_CAP, ...(top === null ? {} : { top_counted: row?.top_counted === true }),
      ...(classed ? { could_affect_images: Number(row?.could_affect_images ?? 0) } : {}) };
  } catch {
    return null;
  }
}

/** The pending pages the readiness probe found (unsealed, needing work) in the same scope. */
export function countPendingProjections(engine: Pick<BrainEngine, 'executeRaw'>, scope: ProjectionReadinessScope, relevance: CoverageRelevance): Promise<ScopedCount | null> {
  return countInScope(engine, scope, relevance, '',
    `NOT ${servedTextProjectionFilter('p')} AND ${projectionClassSql('p', 's')} NOT IN ('protected','archived')`, true);
}

/** Pages in scope held from embedding: a held or capped budget row that still applies, on a page no `embed_skip` marker keeps out of embedding for good. */
export function countEmbeddingHeld(engine: Pick<BrainEngine, 'executeRaw'>, scope: ProjectionReadinessScope, relevance: CoverageRelevance): Promise<ScopedCount | null> {
  return countInScope(engine, scope, relevance, 'JOIN page_embedding_budget b ON b.page_id = p.id',
    `${budgetRowAppliesFilter('b', 'p')} AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')`);
}

/**
 * #9192: progress of the index rebuild an upgrade's activation queued (rows still queued with reason
 * `protocol_activation`), over the pages in scope: how many are done of the total, and the rate measured from the
 * start of the rebuild. Null when no such rebuild is running or the count fails.
 */
export async function countProjectionRebuild(engine: Pick<BrainEngine, 'executeRaw'>, scope: ProjectionReadinessScope): Promise<ProjectionRebuildProgress | null> {
  const params: unknown[] = [];
  const filters = coverageScopeFilters(scope, params);
  try {
    const [row] = await engine.executeRaw<{ seconds: number | null; total: number; done: number }>(
      `WITH started AS (SELECT EXTRACT(EPOCH FROM now() - min(updated_at))::float8 AS seconds FROM page_projection_jobs WHERE reason = 'protocol_activation')
      SELECT started.seconds, count(p.id)::int AS total,
        count(p.id) FILTER (WHERE p.text_projection_revision IS NOT DISTINCT FROM p.knowledge_revision)::int AS done
      FROM started LEFT JOIN (pages p JOIN sources s ON s.id = p.source_id) ON started.seconds IS NOT NULL AND p.deleted_at IS NULL
        AND ${projectionClassSql('p', 's')} NOT IN ('protected','archived') AND ${filters.join(' AND ')}
      GROUP BY started.seconds`, params);
    if (row?.seconds === null || row?.seconds === undefined) return null;
    const done = Number(row.done), total = Number(row.total), seconds = Number(row.seconds);
    return { done, total, pages_per_second: seconds > 0 && done > 0 ? done / seconds : null };
  } catch {
    return null;
  }
}