Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.5 KB
Raw Blame
/**
 * Full-text search language configuration.
 *
 * Postgres tsvector/tsquery require a text search configuration name (e.g.
 * 'english', 'portuguese', 'spanish'). Historically GBrain hardcoded
 * 'english' across engines and trigger functions, which broke search
 * quality for non-English brains (no stemming, no stop-word removal).
 *
 * This helper centralizes the choice. Default stays 'english' for backward
 * compatibility — only users who set GBRAIN_FTS_LANGUAGE see different
 * behavior.
 *
 * Custom configs (e.g. accent-insensitive 'pt_br' built with unaccent +
 * portuguese stemmer) are supported as long as the configuration exists
 * in the target Postgres instance. See docs/guides/multi-language-fts.md
 * for setup instructions.
 *
 * Validation: only allow lowercase letters, digits, and underscores. This
 * prevents SQL injection when the value is interpolated into queries
 * (Postgres tsvector functions don't accept parameterized config names —
 * they must be literals or identifiers).
 */

import { CJK_SLUG_CHARS } from './cjk.ts';
import { requireMaintainerPoliciesSql } from './derived-state-maintainers.ts';

const VALID_CONFIG_NAME = /^[a-z][a-z0-9_]*$/;
const DEFAULT_LANGUAGE = 'english';

/**
 * Config-table key `gbrain reindex-search-vector` sets (value = target
 * language) before it flips the trigger functions and clears only after both
 * backfills finish. While present, rows written after the flip and rows not
 * yet backfilled are tokenized under different configurations, so keyword
 * search matches only part of the corpus; doctor's `fts_reindex_incomplete`
 * check fails on it (#4795).
 */
export const FTS_REINDEX_MARKER_KEY = 'fts.reindex_in_progress';

let cachedLanguage: string | null = null;

/**
 * Returns the configured Postgres text search configuration name.
 *
 * Resolution order:
 *   1. process.env.GBRAIN_FTS_LANGUAGE (if set and valid)
 *   2. 'english' (default — preserves existing behavior)
 *
 * The return value is safe to interpolate directly into SQL because it
 * passes the VALID_CONFIG_NAME guard. If validation fails, falls back to
 * the default and emits a one-time warning.
 *
 * Cached on first call; reset with `resetFtsLanguageCache()` (test only).
 */
export function getFtsLanguage(): string {
  if (cachedLanguage !== null) return cachedLanguage;

  const raw = process.env.GBRAIN_FTS_LANGUAGE?.trim();
  if (!raw) {
    cachedLanguage = DEFAULT_LANGUAGE;
    return cachedLanguage;
  }

  if (!VALID_CONFIG_NAME.test(raw)) {
    console.warn(
      `[gbrain] Invalid GBRAIN_FTS_LANGUAGE='${raw}' — must match /^[a-z][a-z0-9_]*$/. ` +
      `Falling back to '${DEFAULT_LANGUAGE}'.`
    );
    cachedLanguage = DEFAULT_LANGUAGE;
    return cachedLanguage;
  }

  cachedLanguage = raw;
  return cachedLanguage;
}

/**
 * Resets the cached language. Tests only — don't use in production code.
 */
export function resetFtsLanguageCache(): void {
  cachedLanguage = null;
}

/**
 * Rewrites a schema template's hardcoded `to_tsvector('english', ...)` calls
 * to the configured text search configuration.
 *
 * Why this exists: migration v123 (`configurable_fts_language`) stamps the
 * two search_vector trigger functions with `getFtsLanguage()` at apply time,
 * but the schema templates that `initSchema()` replays still carry the
 * literal 'english'. Those templates are applied with `CREATE OR REPLACE
 * FUNCTION`, so every `initSchema()` — including the one behind
 * `gbrain init --migrate-only`, which reports "Schema up to date" — silently
 * reverts a non-English brain's trigger functions to english. The write side
 * then tokenizes under a different configuration than the read side, and
 * nothing warns: rows keep being indexed, just unreachable by the queries
 * that were supposed to find them.
 *
 * Mirrors `applyChunkEmbeddingIndexPolicy()`: a runtime-config-driven rewrite
 * of the static schema template, applied at the same seam in both engines.
 *
 * No-op for the default configuration, so English installs get a
 * byte-identical schema string.
 */
export function applyFtsLanguagePolicy(sql: string): string {
  const lang = getFtsLanguage();
  if (lang === DEFAULT_LANGUAGE) return sql;
  // getFtsLanguage() validates against VALID_CONFIG_NAME, so the value is
  // safe to interpolate as a SQL string literal.
  return sql.replaceAll(`to_tsvector('${DEFAULT_LANGUAGE}',`, `to_tsvector('${lang}',`);
}

/**
 * The facts keyword document: fact text plus the entity slug's words, under
 * a literal text search configuration so `idx_facts_fts` can serve it (an
 * expression index matches only the same literal configuration, never a
 * bound parameter). The index (facts/fts-index.ts) and the question-recall
 * keyword arm both build it here.
 */
export function factsFtsDocument(alias: string | null, lang: string = getFtsLanguage()): string {
  const col = (c: string) => alias ? `${alias}.${c}` : c;
  return `to_tsvector('${lang}'::regconfig, ${col('fact')} || ' ' || replace(COALESCE(${col('entity_slug')}, ''), '-', ' '))`;
}

/** The facts keyword GIN index over active facts under `lang`, named `name` (facts/fts-index.ts builds and swaps it). */
export function factsFtsIndexSql(name: string, lang: string): string {
  return `CREATE INDEX IF NOT EXISTS ${name} ON facts USING gin (${factsFtsDocument(null, lang)}) WHERE expired_at IS NULL`;
}

const CJK_CLASS = `[${CJK_SLUG_CHARS}]`;

/**
 * `gbrain_fts_input(text)`: the text every indexing `to_tsvector` and the
 * title query's `websearch_to_tsquery` read (#6370). Postgres' default parser
 * treats Han, Kana and Hangul as letters, so `升级PostgreSQL17后查询Redis7`
 * is one lexeme and keyword search for `Redis7` misses it. Two passes insert
 * a space at every CJK→ASCII-alnum and ASCII-alnum→CJK boundary (two passes,
 * not lookbehind, so both engines run the same regex). A byte-for-byte no-op
 * on text without CJK, and idempotent. One copy of this DDL: the schema
 * build splices it into src/schema.sql and the fts-cjk-boundary migration
 * applies it.
 */
export const FTS_INPUT_FUNCTION_SQL = `CREATE OR REPLACE FUNCTION gbrain_fts_input(t text) RETURNS text
  LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE SET search_path = pg_catalog, pg_temp
AS $fn$
  SELECT regexp_replace(regexp_replace(t, '(${CJK_CLASS})([A-Za-z0-9])', '\\1 \\2', 'g'), '([A-Za-z0-9])(${CJK_CLASS})', '\\1 \\2', 'g')
$fn$;`;

/** POSIX regex (SQL `~`) matching text that holds a CJK/ASCII-alnum boundary `gbrain_fts_input` splits. */
export const FTS_CJK_BOUNDARY_REGEX = `${CJK_CLASS}[A-Za-z0-9]|[A-Za-z0-9]${CJK_CLASS}`;

/**
 * Characters of the page timeline the page keyword vector indexes, and of
 * the joined timeline entries the trigger adds. A tsvector's lexeme strings
 * cap at 1 MB; the densest text measured (hyphenated astral and CJK pairs)
 * yields about 7.5 lexeme bytes per character under Postgres 16, so 65,536 +
 * 32,768 characters stay under 750 KB with room for the title, while a
 * diverse multi-MB timeline used to fail the whole page write. Chunk search
 * still covers the full timeline.
 */
export const PAGE_TIMELINE_FTS_MAX_CHARS = 65_536;
export const PAGE_TIMELINE_ENTRIES_FTS_MAX_CHARS = 32_768;

/**
 * The page keyword vector every sealed writer stores: the title (weight A) and
 * the first PAGE_TIMELINE_FTS_MAX_CHARS characters of the SANITIZED timeline
 * (weight C, bound by the caller; never the raw column, which can hold
 * private or withdrawn fence text). `title` and `timeline` are SQL
 * expressions. Shared by the seal, the import seal, `reindex-search-vector`
 * and the fts-cjk-boundary migration, so they cannot drift.
 */
export function pageSearchVectorSql(title: string, timeline: string, lang: string = getFtsLanguage()): string {
  return `setweight(to_tsvector('${lang}',gbrain_fts_input(COALESCE(${title},''))),'A') || setweight(to_tsvector('${lang}',gbrain_fts_input(left(${timeline}::text,${PAGE_TIMELINE_FTS_MAX_CHARS}))),'C')`;
}

/** The chunk keyword vector, over a row's own columns (`NEW.` in the trigger, bare in a backfill). */
export function chunkSearchVectorSql(lang: string, row = ''): string {
  return `setweight(to_tsvector('${lang}', gbrain_fts_input(COALESCE(${row}doc_comment, ''))), 'A') ||
    setweight(to_tsvector('${lang}', gbrain_fts_input(COALESCE(${row}symbol_name_qualified, ''))), 'A') ||
    setweight(to_tsvector('${lang}', gbrain_fts_input(COALESCE(${row}chunk_text, ''))), 'B')`;
}

/**
 * The two search_vector trigger functions under `lang`, for every runtime
 * rewrite (`reindex-search-vector`, the fts-cjk-boundary migration). Their
 * bodies are byte-identical to src/schema.sql's for 'english' (pinned by
 * test/fts-cjk-boundary.test.ts), so the blob replay that every boot runs
 * after a migration leaves the catalog unchanged. #2704: compiled_truth is not
 * indexed (it overflows the 1MB tsvector cap); the timeline and the joined
 * entries are cut to their FTS caps for the same reason, the entries in id
 * order so the cut is deterministic. A segmented page (Wave B) has a NULL
 * timeline and indexes its stored `timeline_lead` prefix instead, read only
 * then because the column is migration-only (v248). `SET search_path = pg_catalog,
 * public` keeps the v120/#1647 hardening, which CREATE OR REPLACE would
 * otherwise reset.
 */
export function pageSearchVectorTriggerFnSql(lang: string): string {
  return `CREATE OR REPLACE FUNCTION update_page_search_vector() RETURNS trigger SET search_path = pg_catalog, public, pg_temp AS $$
DECLARE
  timeline_text TEXT;
  timeline_lead_text TEXT;
BEGIN
  IF TG_OP = 'UPDATE' THEN
    ${requireMaintainerPoliciesSql('timeline_entries', ['SELECT'], '    ')}
  END IF;
  SELECT coalesce(string_agg(summary || ' ' || detail, ' ' ORDER BY id), '')
  INTO timeline_text
  FROM timeline_entries
  WHERE page_id = NEW.id;

  IF NEW.timeline IS NULL THEN
    timeline_lead_text := NEW.timeline_lead;
  END IF;

  NEW.search_vector :=
    setweight(to_tsvector('${lang}', gbrain_fts_input(coalesce(NEW.title, ''))), 'A') ||
    setweight(to_tsvector('${lang}', gbrain_fts_input(left(coalesce(NEW.timeline, timeline_lead_text, ''), ${PAGE_TIMELINE_FTS_MAX_CHARS}))), 'C') ||
    setweight(to_tsvector('${lang}', gbrain_fts_input(left(coalesce(timeline_text, ''), ${PAGE_TIMELINE_ENTRIES_FTS_MAX_CHARS}))), 'C');

  RETURN NEW;
END;
$$ LANGUAGE plpgsql;`;
}

export function chunkSearchVectorTriggerFnSql(lang: string): string {
  return `CREATE OR REPLACE FUNCTION update_chunk_search_vector() RETURNS TRIGGER SET search_path = pg_catalog, public, pg_temp AS $fn$
BEGIN
  NEW.search_vector :=
    ${chunkSearchVectorSql(lang, 'NEW.')};
  RETURN NEW;
END;
$fn$ LANGUAGE plpgsql;`;
}