Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.4 KB
Raw Blame
/**
 * The title-arm statement (`BrainEngine.searchTitles`), built once for both
 * engines and the plan-proof E2E (test/e2e/title-arm-postgres.test.ts) so the
 * SQL they run or EXPLAIN can never drift. Execution, the per-engine session
 * settings and the OR fallback live in engine-sql/titles.ts; see its module
 * doc for the exact-title key (#5889) and the weight-A remote predicate.
 */
import type { SearchOpts } from '../types.ts';
import { clampSearchLimit } from '../engine.ts';
import { searchLimitCap } from './eval-pool-depth.ts';
import { getFtsLanguage } from '../fts-language.ts';
import { chunkGenerationFilter, requiresSafeChunks } from './safe-chunks.ts';
import { normalizeAlias } from './alias-normalize.ts';
import { titleNameVariants } from './title-match.ts';
import { frontmatterWhereSql } from './frontmatter-where.ts';
import { resolveBoostMap, resolveHardExcludes } from './source-boost.ts';
import {
  boundWebsearchQuery,
  buildHardExcludeClause,
  buildSourceFactorCase,
  buildVisibilityClause,
} from './sql-ranking.ts';

/** Engine behavior the title arm keeps from each engine's original method. */
export interface SearchTitlesDialect {
  /** Per-attempt `statement_timeout` (Postgres: '8s'); unset means none. */
  statementTimeout?: string;
  /** Run the relaxed (OR) local re-run with `enable_seqscan` off (Postgres). */
  relaxedPrefersIndex: boolean;
  /** Project `stale` from timeline_entries (PGLite); otherwise `false`. */
  staleProbe: boolean;
  /** chunkManifestReads; unset reads the manifest. */
  chunkManifest?: boolean;
  /** Match and rank title lexemes only, for local callers too: the OR re-run whose whole-vector matches do not fit the cap (#7135) and the cut strict attempt (#7082). */
  relaxed?: boolean;
  /**
   * A local caller's OR re-run (GST-2899): rank the whole-vector matches only when all of them fit
   * `titleWholeVectorCandidates()`; a set that does not fit returns one row of nulls marked `candidates_cut`, and only then
   * does the caller run the title-only statement. The candidate count has no ORDER BY, since nothing is cut from it: ordered by page id it walked
   * pages_pkey for a word on a few percent of pages (about 0.8 s at 1,000,000 pages against 40 ms from idx_pages_search).
   */
  completeOnly?: boolean;
}

/**
 * #7082: a local caller's strict attempt matches the whole vector (title plus
 * timelines, timeline_entries rows included), and a very common word matched
 * most pages, so ranking read every matching vector until the statement timeout.
 * It now ranks at most this many matching pages, the ones with the lowest page
 * ids, so a cut keeps the same pages on every call for the same data; a term
 * that matches fewer is ranked exactly as before, and a cut attempt is
 * completed by the title-only statement (engine-sql/titles.ts).
 */
let wholeVectorCandidates = 5000;
export function titleWholeVectorCandidates(): number { return wholeVectorCandidates; }
/** Test seam: lower the cap so a small fixture can be cut. */
export function __setTitleWholeVectorCandidatesForTests(n: number): void { wholeVectorCandidates = n; }

export interface SearchTitlesStatement {
  sql: string;
  /** `$1` is the websearch text (rebound by the OR fallback); `$2` the exact-title key. */
  params: unknown[];
  safeChunks: boolean;
}

export function buildSearchTitlesStatement(query: string, opts: SearchOpts | undefined, dialect: SearchTitlesDialect): SearchTitlesStatement {
  const limit = clampSearchLimit(opts?.limit, 20, searchLimitCap());
  const offset = opts?.offset || 0;
  const detailLow = opts?.detail === 'low';
  const safeChunks = requiresSafeChunks(opts);

  const boostMap = opts?.source_boosts ?? resolveBoostMap();
  const sourceFactorCase = buildSourceFactorCase('p.slug', boostMap, opts?.detail);
  const hardExcludeClause = buildHardExcludeClause('p.slug', resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes));
  const visibilityClause = buildVisibilityClause('p', 's', opts);
  // FTS config name (e.g. 'english', 'pt_br'). Validated by getFtsLanguage()
  // — safe to interpolate into raw SQL.
  const ftsLang = getFtsLanguage();
  const params: unknown[] = [boundWebsearchQuery(query), normalizeAlias(query), limit, offset];
  // #7471: each given-name variant ("matthew example" for "Matt Example") is
  // ORed into the query as its own websearch query, so a title that spells the
  // given name another way still matches every word. The OR fallback rebinds
  // only $1; a variant matches nothing there (the strict query found no page).
  const tsqueries = ['$1', ...titleNameVariants(query).map(v => `$${params.push(v)}`)]
    .map(p => `websearch_to_tsquery('${ftsLang}', gbrain_fts_input(${p}))`);
  const tsq = tsqueries.length === 1 ? tsqueries[0] : `(${tsqueries.join(' || ')})`;
  const titleOnly = safeChunks || dialect.relaxed === true;
  const matchSql = titleOnly ? `ts_filter(p.search_vector, '{a}') @@ ${tsq}` : `p.search_vector @@ ${tsq}`;
  const rankSql = titleOnly ? `ts_rank_cd(ts_filter(p.search_vector, '{a}'), ${tsq})` : `ts_rank_cd(p.search_vector, ${tsq})`;

  let filters = '';
  if (opts?.type) {
    params.push(opts.type);
    filters += ` AND p.type = $${params.length}`;
  }
  if (opts?.types && opts.types.length > 0) {
    params.push(opts.types);
    filters += ` AND p.type = ANY($${params.length}::text[])`;
  }
  filters += frontmatterWhereSql('p', opts?.where, json => `$${params.push(json)}`);
  if (opts?.exclude_slugs?.length) {
    params.push(opts.exclude_slugs);
    filters += ` AND p.slug != ALL($${params.length}::text[])`;
  }
  // Date filters read COALESCE(effective_date, …), the keyword arm's
  // effective-date-first convention.
  if (opts?.afterDate) {
    params.push(opts.afterDate);
    filters += ` AND COALESCE(p.effective_date, p.updated_at, p.created_at) ${opts?.afterDateInclusive ? '>=' : '>'} $${params.length}::text::timestamptz`;
  }
  if (opts?.beforeDate) {
    params.push(opts.beforeDate);
    filters += ` AND COALESCE(p.effective_date, p.updated_at, p.created_at) ${opts?.beforeDateInclusive ? '<=' : '<'} $${params.length}::text::timestamptz`;
  }
  if (opts?.sourceIds && opts.sourceIds.length > 0) {
    params.push(opts.sourceIds);
    filters += ` AND p.source_id = ANY($${params.length}::text[])`;
  } else if (opts?.sourceId) {
    params.push(opts.sourceId);
    filters += ` AND p.source_id = $${params.length}`;
  }

  // Page grain: one row per page, so no best_per_page pool. top_pages ranks
  // and limits first; the representative-chunk LATERAL (compiled_truth first,
  // then lowest chunk_index) and the stale probe run per OUTPUT row. Neither
  // affects order. COALESCEs keep chunkless pages retrievable (chunk_id 0,
  // empty text). Accepted limitations (Reviewer F5/F6): the synthetic
  // chunkless row dedups on empty chunk_text (fusion's compiledTruthBoost
  // skips it since #3695); detail='low' filters only the representative.
  const staleSql = dialect.staleProbe
    ? `CASE WHEN tp.updated_at < (SELECT MAX(te.created_at) FROM timeline_entries te
         WHERE te.page_id = tp.page_id) THEN true ELSE false END`
    : 'false';
  const pageFilters = `${filters}
        ${hardExcludeClause}
        ${visibilityClause}`;
  const capped = !titleOnly;
  const sql = `
    WITH ${capped ? `candidates AS MATERIALIZED (
      SELECT p.id FROM pages p
      JOIN sources s ON s.id = p.source_id
      WHERE ${matchSql}
        ${pageFilters}
      ${dialect.completeOnly ? '' : 'ORDER BY p.id'}
      LIMIT ${wholeVectorCandidates + 1}
    ),
    ` : ''}top_pages AS (
      SELECT p.slug, p.id AS page_id, p.title, p.type, p.source_id,
        p.effective_date, p.effective_date_source, p.updated_at, p.body_storage, p.body_generation,
        (lower(btrim(p.title, ' "''' || chr(160))) = $2) AS exact_title,
        ${rankSql} * ${sourceFactorCase} AS score
      FROM ${capped ? `(SELECT id FROM candidates ORDER BY id LIMIT ${wholeVectorCandidates}) c JOIN pages p ON p.id = c.id` : 'pages p'}
      JOIN sources s ON s.id = p.source_id
      WHERE ${capped ? (dialect.completeOnly ? `(SELECT count(*) FROM candidates) <= ${wholeVectorCandidates}` : 'true') : `${matchSql}
        ${pageFilters}`}
      ORDER BY exact_title DESC, score DESC, p.id ASC
      LIMIT $3 OFFSET $4
    )
    SELECT tp.slug, tp.page_id, tp.title, tp.type, tp.source_id,
      tp.effective_date, tp.effective_date_source,
      COALESCE(rep.id, 0) AS chunk_id,
      COALESCE(rep.chunk_index, 0) AS chunk_index,
      COALESCE(rep.chunk_text, '') AS chunk_text,
      COALESCE(rep.chunk_source, 'compiled_truth') AS chunk_source,
      tp.score,
      ${staleSql} AS stale,
      ${capped ? `(SELECT count(*) FROM candidates) > ${wholeVectorCandidates}` : 'false'} AS candidates_cut
    FROM ${dialect.completeOnly ? `(SELECT (SELECT count(*) FROM candidates) > ${wholeVectorCandidates} AS cut) k
    LEFT JOIN top_pages tp ON true` : 'top_pages tp'}
    LEFT JOIN LATERAL (
      SELECT cc.id, cc.chunk_index, cc.chunk_text, cc.chunk_source
      FROM content_chunks cc
      WHERE cc.page_id = tp.page_id AND ${chunkGenerationFilter('tp', 'cc', dialect.chunkManifest ?? true)}
        AND cc.modality = 'text'
        ${detailLow ? `AND cc.chunk_source = 'compiled_truth'` : ''}
      ORDER BY (cc.chunk_source = 'compiled_truth') DESC, cc.chunk_index ASC
      LIMIT 1
    ) rep ON true
    ${dialect.completeOnly ? 'WHERE tp.page_id IS NOT NULL OR k.cut' : ''}
    ORDER BY tp.exact_title DESC, tp.score DESC, tp.page_id ASC
  `;
  return { sql, params, safeChunks };
}