Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.1 KB
Raw Blame
/**
 * #8744: the per-chunk synopsis sweep for brains whose configured contextual
 * retrieval tier is `per_chunk_synopsis` (search.mode tokenmax, or a source or
 * page override). Import and sync embed those pages at the free title tier and
 * leave the synopsis tier to the `contextual_reindex_per_chunk` job, which no
 * automatic path submits; `gbrain reindex --synopsis` is the explicit,
 * operator-run sweep. This module holds the reads shared by that command, the
 * doctor check `synopsis_tier_coverage` and the `gbrain search modes` line:
 * which pages resolve to the synopsis tier (the SQL twin of
 * resolveContextualRetrievalMode), how many of their chunks carry synopsis
 * vectors, and what the sweep would cost. Database reads only, so a repo-less
 * brain behaves exactly like a repo brain.
 *
 * Coverage is counted per chunk by the page's stamped mode: a page is stamped
 * `per_chunk_synopsis` only once every chunk is settled. Inline pages over the
 * large-page threshold stay on the title tier by design (contextual-retrieval
 * service, plan B8), so they leave the denominator and are reported apart.
 */
import type { BrainEngine } from './engine.ts';
import { loadSearchModeConfig, resolveSearchMode } from './search/mode.ts';
import { largePageBodyFilter } from './page-size-policy.ts';
import { chunkGenerationFilter } from './search/safe-chunks.ts';
import {
  SEGMENT_SYNOPSIS_LEAD_BYTES,
  SEGMENT_SYNOPSIS_WINDOW_BYTES,
  SYNOPSIS_DOC_MAX_CHARS,
  SYNOPSIS_MAX_TOKENS,
  SYNOPSIS_PROMPT_OVERHEAD_CHARS,
} from './page-summary.ts';
import { reservationCostUsd } from './budget/reservation-cost.ts';
import type { CRMode } from './types.ts';

export const SYNOPSIS_SWEEP_JOB = 'contextual_reindex_per_chunk';
export const SYNOPSIS_SWEEP_ARGV = ['gbrain', 'reindex', '--synopsis'] as const;
/** Below this share of eligible chunks on the synopsis tier, the configured tier is "missing from most chunks". */
export const SYNOPSIS_COVERAGE_WARN_PCT = 50;
/** Embedding-input chars a synopsis adds to a chunk at most (contextual-retrieval-service SYNOPSIS_WRAPPER_CHARS). */
const SYNOPSIS_WRAPPER_CHARS = 320;

export interface SynopsisTierConfig {
  /** The brain-level tier (search.mode bundle or search.contextual_retrieval override). */
  global_tier: CRMode;
  kill_switch: boolean;
  /** Sources whose own contextual_retrieval_mode is per_chunk_synopsis. */
  synopsis_sources: string[];
  /** True when any page can resolve to the synopsis tier (page frontmatter overrides aside). */
  configured: boolean;
}

export async function readSynopsisTierConfig(engine: BrainEngine): Promise<SynopsisTierConfig> {
  const knobs = resolveSearchMode(await loadSearchModeConfig(engine));
  const rows = await engine.executeRaw<{ id: string }>(
    `SELECT id FROM sources WHERE contextual_retrieval_mode = 'per_chunk_synopsis' AND NOT archived ORDER BY id`);
  const synopsisSources = rows.map(r => r.id);
  const killSwitch = knobs.contextual_retrieval_disabled === true;
  return {
    global_tier: knobs.contextual_retrieval,
    kill_switch: killSwitch,
    synopsis_sources: synopsisSources,
    configured: !killSwitch && (knobs.contextual_retrieval === 'per_chunk_synopsis' || synopsisSources.length > 0),
  };
}

/**
 * Markdown pages whose resolved tier is per_chunk_synopsis, with their stamped
 * mode, large-inline flag and chunk counts. Mirrors resolveContextualRetrievalMode:
 * a valid frontmatter value wins on the host source or a trusted mount, then a
 * valid source-row mode, then the global tier. `$1` is the global tier.
 */
function eligiblePagesSql(opts: { sourceParam?: string; withChars: boolean }): string {
  return `
    WITH chunk_stats AS (
      SELECT c.page_id, count(*)::bigint AS chunks${opts.withChars ? ', COALESCE(sum(length(c.chunk_text)), 0)::bigint AS chunk_chars' : ''}
        FROM content_chunks c
        JOIN pages p ON p.id = c.page_id
       WHERE c.chunk_source <> 'image_asset' AND ${chunkGenerationFilter('p', 'c')}
       GROUP BY c.page_id
    ), resolved AS (
      SELECT p.id, p.slug, p.source_id, p.contextual_retrieval_mode AS stamped,
             p.text_projection_revision AS revision,
             p.body_storage = 'segmented' AS segmented,
             (COALESCE(p.body_storage, 'inline') <> 'segmented' AND ${largePageBodyFilter('p')}) AS large_inline,
             CASE
               WHEN p.frontmatter ? 'contextual_retrieval'
                    AND (p.source_id = 'default' OR COALESCE(s.trust_frontmatter_overrides, false))
                    AND p.frontmatter->>'contextual_retrieval' IN ('none', 'title', 'per_chunk_synopsis')
                 THEN p.frontmatter->>'contextual_retrieval'
               WHEN s.contextual_retrieval_mode IN ('none', 'title', 'per_chunk_synopsis') THEN s.contextual_retrieval_mode
               ELSE $1::text
             END AS effective
        FROM pages p
        LEFT JOIN sources s ON s.id = p.source_id
       WHERE p.deleted_at IS NULL AND p.page_kind = 'markdown' AND p.type <> 'extract_receipt'
         AND NOT (p.frontmatter ? 'embed_skip')
         AND NOT COALESCE(s.archived, false)
         ${opts.sourceParam ? `AND p.source_id = ANY(${opts.sourceParam}::text[])` : ''}
    )
    SELECT r.*, cs.chunks${opts.withChars ? ', cs.chunk_chars' : ''}
      FROM resolved r
      JOIN chunk_stats cs ON cs.page_id = r.id
     WHERE r.effective = 'per_chunk_synopsis'`;
}

export interface SynopsisCoverage {
  config: SynopsisTierConfig;
  eligible_pages: number;
  eligible_chunks: number;
  synopsis_pages: number;
  synopsis_chunks: number;
  /** Chunks of eligible pages on the synopsis tier, as a percentage (0..100, one decimal); null with no eligible chunks. */
  coverage_pct: number | null;
  /** Inline pages over the large-page threshold: the title tier by design, outside the denominator. */
  large_title_pages: number;
  large_title_chunks: number;
}

/** Synopsis-tier coverage. `sourceIds` confines every count to those sources (the remote doctor's grant). */
export async function readSynopsisCoverage(engine: BrainEngine, opts: { sourceIds?: string[]; config?: SynopsisTierConfig } = {}): Promise<SynopsisCoverage> {
  const config = opts.config ?? await readSynopsisTierConfig(engine);
  const empty = { eligible_pages: 0, eligible_chunks: 0, synopsis_pages: 0, synopsis_chunks: 0, coverage_pct: null, large_title_pages: 0, large_title_chunks: 0 };
  if (!config.configured) return { config, ...empty };
  const rows = await engine.executeRaw<Record<string, string | number>>(
    `SELECT
       count(*) FILTER (WHERE NOT large_inline)::bigint AS eligible_pages,
       COALESCE(sum(chunks) FILTER (WHERE NOT large_inline), 0)::bigint AS eligible_chunks,
       count(*) FILTER (WHERE NOT large_inline AND stamped = 'per_chunk_synopsis')::bigint AS synopsis_pages,
       COALESCE(sum(chunks) FILTER (WHERE NOT large_inline AND stamped = 'per_chunk_synopsis'), 0)::bigint AS synopsis_chunks,
       count(*) FILTER (WHERE large_inline)::bigint AS large_title_pages,
       COALESCE(sum(chunks) FILTER (WHERE large_inline), 0)::bigint AS large_title_chunks
     FROM (${eligiblePagesSql({ sourceParam: opts.sourceIds ? '$2' : undefined, withChars: false })}) eligible`,
    opts.sourceIds ? [config.global_tier, opts.sourceIds] : [config.global_tier]);
  const n = (key: string) => Number(rows[0]?.[key] ?? 0);
  const eligibleChunks = n('eligible_chunks');
  return {
    config,
    eligible_pages: n('eligible_pages'),
    eligible_chunks: eligibleChunks,
    synopsis_pages: n('synopsis_pages'),
    synopsis_chunks: n('synopsis_chunks'),
    coverage_pct: eligibleChunks > 0 ? Math.floor((n('synopsis_chunks') / eligibleChunks) * 1000) / 10 : null,
    large_title_pages: n('large_title_pages'),
    large_title_chunks: n('large_title_chunks'),
  };
}

export interface SynopsisSweepPage {
  id: number;
  slug: string;
  source_id: string;
  revision: string | null;
  chunks: number;
}

export interface SynopsisSweepPlan {
  config: SynopsisTierConfig;
  synopsis_model: string;
  pages: SynopsisSweepPage[];
  chunks: number;
  /** Ceiling: every synopsis prompt uncached (the prompt cache usually serves all but a page's first call). */
  est_input_tokens: number;
  /** Ceiling: every synopsis at the output-token cap. */
  est_output_tokens: number;
  /** Null when the synopsis model has no known price. */
  est_synopsis_usd: number | null;
  /** Null when the embedding model has no known price (or embedding is not configured). */
  est_embedding_usd: number | null;
  est_usd: number | null;
  /** Pending pages beyond `limit` that this run leaves for the next one. */
  remaining_pages: number;
}

/**
 * The pages a sweep would queue (eligible, not large-inline, not yet stamped
 * per_chunk_synopsis), oldest id first, with the token and dollar estimate on
 * the configured synopsis model. A page's synopsis document is its chunks joined
 * (synopsisDocument) capped at SYNOPSIS_DOC_MAX_CHARS (a segmented page: its lead plus one segment window),
 * sent once per chunk with the chunk and the prompt overhead, at 4 chars per
 * token (estimateSynopsisCallUsd's rate).
 */
export async function planSynopsisSweep(engine: BrainEngine, opts: {
  synopsisModel: string;
  synopsisMaxTokens?: number;
  sourceId?: string;
  limit?: number;
  embeddingPricePerMTok?: number | null;
}): Promise<SynopsisSweepPlan> {
  const config = await readSynopsisTierConfig(engine);
  const base = { config, synopsis_model: opts.synopsisModel, pages: [], chunks: 0, est_input_tokens: 0, est_output_tokens: 0,
    est_synopsis_usd: 0, est_embedding_usd: 0, est_usd: 0, remaining_pages: 0 };
  if (!config.configured) return base;
  const rows = await engine.executeRaw<{ id: number | string; slug: string; source_id: string; revision: string | null; segmented: boolean | null;
    chunks: number | string; chunk_chars: number | string }>(
    `SELECT id, slug, source_id, revision, segmented, chunks, chunk_chars
       FROM (${eligiblePagesSql({ sourceParam: opts.sourceId ? '$2' : undefined, withChars: true })}) eligible
      WHERE NOT large_inline AND stamped IS DISTINCT FROM 'per_chunk_synopsis'
      ORDER BY id`,
    opts.sourceId ? [config.global_tier, [opts.sourceId]] : [config.global_tier]);
  const selected = opts.limit ? rows.slice(0, opts.limit) : rows;
  const maxTokens = opts.synopsisMaxTokens ?? SYNOPSIS_MAX_TOKENS;
  const segmentedDocChars = Math.min(SEGMENT_SYNOPSIS_LEAD_BYTES + SEGMENT_SYNOPSIS_WINDOW_BYTES, SYNOPSIS_DOC_MAX_CHARS);
  let chunks = 0;
  let inputTokens = 0;
  let embeddedChars = 0;
  for (const row of selected) {
    const pageChunks = Number(row.chunks);
    const chunkChars = Number(row.chunk_chars);
    const documentChars = chunkChars + 2 * Math.max(0, pageChunks - 1);
    const docChars = Math.min(documentChars, row.segmented ? segmentedDocChars : SYNOPSIS_DOC_MAX_CHARS);
    chunks += pageChunks;
    inputTokens += Math.ceil((pageChunks * (docChars + SYNOPSIS_PROMPT_OVERHEAD_CHARS) + chunkChars) / 4);
    embeddedChars += chunkChars + pageChunks * SYNOPSIS_WRAPPER_CHARS;
  }
  const outputTokens = chunks * maxTokens;
  const synopsisUsd = chunks === 0 ? 0 : reservationCostUsd(opts.synopsisModel, 'chat', inputTokens, outputTokens);
  const embeddingUsd = opts.embeddingPricePerMTok == null ? null : (Math.ceil(embeddedChars / 3.5) / 1_000_000) * opts.embeddingPricePerMTok;
  return {
    ...base,
    pages: selected.map(r => ({ id: Number(r.id), slug: r.slug, source_id: r.source_id, revision: r.revision ?? null, chunks: Number(r.chunks) })),
    chunks,
    est_input_tokens: inputTokens,
    est_output_tokens: outputTokens,
    est_synopsis_usd: synopsisUsd,
    est_embedding_usd: embeddingUsd,
    est_usd: synopsisUsd === null || embeddingUsd === null ? null : synopsisUsd + embeddingUsd,
    remaining_pages: rows.length - selected.length,
  };
}

/** The newest finished sweep job and how many ever ran (null when the queue table is unreadable). */
export async function readLastSynopsisSweep(engine: BrainEngine): Promise<{ jobs: number; last_finished_at: string | null } | null> {
  try {
    const rows = await engine.executeRaw<{ jobs: number | string; last: string | Date | null }>(
      `SELECT count(*)::bigint AS jobs, max(finished_at) AS last FROM minion_jobs WHERE name = $1`, [SYNOPSIS_SWEEP_JOB]);
    const last = rows[0]?.last ?? null;
    return { jobs: Number(rows[0]?.jobs ?? 0), last_finished_at: last === null ? null : new Date(last).toISOString() };
  } catch {
    return null;
  }
}

/** One line for `gbrain search modes` and the doctor message: the configured tier's chunk coverage. */
export function formatSynopsisCoverageLine(c: SynopsisCoverage): string {
  if (c.eligible_chunks === 0) return 'Synopsis tier: configured (per_chunk_synopsis); no chunked pages resolve to it yet.';
  const large = c.large_title_pages ? ` ${c.large_title_pages} large page(s) (${c.large_title_chunks} chunk(s)) stay on the title tier by design.` : '';
  const pending = c.eligible_chunks - c.synopsis_chunks;
  return `Synopsis tier: ${c.coverage_pct}% of chunks (${c.synopsis_chunks}/${c.eligible_chunks}) and ${c.synopsis_pages}/${c.eligible_pages} page(s) carry per-chunk synopsis vectors`
    + (pending > 0 ? `; ${pending} chunk(s) search on title-tier vectors. Preview the sweep: ${SYNOPSIS_SWEEP_ARGV.join(' ')}` : '')
    + `.${large}`;
}