Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.8 KB
Raw Blame
/**
 * #6803 (D1, D3): the extraction backlog as doctor's `links_extraction_lag`, the end-of-sync nudge and
 * `gbrain extract --stale` all report it. One statement reads the live page count, the actionable stale count
 * (attendance-blocked pages excluded: extraction cannot clear them) and the attendance-blocked count under a short
 * statement deadline; the mention-due read runs only when the mention pass is enabled. One verdict decides whether
 * doctor warns and the nudge prints, and one builder renders the default fix (plain `gbrain extract --stale`, the
 * cron default) plus the decision between both routes, with an estimate stated at its basis.
 */
import type { BrainEngine } from './engine.ts';
import { withRequestConfigSnapshot } from './config-snapshot.ts';
import { AGENT_TOOL_LIMIT_MS, type Action, type Decision } from './agent-output.ts';
import { cliRouting, pinRouting, type FixRouting } from './fix-routing.ts';
import { extractionLagStatement, heldLinkExtractionStatement } from './engine-sql/pages.ts';
import { effectiveLinkExtractorWatermark } from './link-extraction-watermark.ts';
import { previewMentionPass } from './mentions/stale.ts';
import { readMentionPolicy } from './mentions/policy.ts';
import { resolveEnvNumber, warnOnceForEnv } from './env-number.ts';
import { isMissingSchemaObject } from './utils.ts';
import { managedPersistenceEnabled } from './persistence/ownership.ts';
import { resolveIncludeFrontmatter } from './extract-frontmatter.ts';
import { LINK_HOLD_THRESHOLD_RUNS, linkExtractionRetryFix } from './persistence/links-extraction-holds.ts';

/** Warn threshold (percent of live pages actionable-stale); `GBRAIN_EXTRACTION_LAG_WARN_PCT` overrides it. */
export const EXTRACTION_LAG_WARN_PCT_DEFAULT = 20;
/** Below this many live pages a brain-wide (unscoped) read is too small to grade. */
export const EXTRACTION_LAG_MIN_PAGES = 100;
/**
 * Wall-clock budget of one plain `gbrain extract --stale` run (default 30 min); `--catch-up` removes it.
 * The deferred-sweep submitters derive their job timeout from it.
 */
export const STALE_TIME_BUDGET_MS = Math.max(1000, Number(process.env.GBRAIN_EXTRACT_TIME_BUDGET_MS) || 30 * 60 * 1000);
/** Measured managed extraction throughput at 10 ms round-trip time (part-6 investigation); the estimates' basis. */
export const EXTRACTION_PAGES_PER_SECOND = 13.5;
export const EXTRACTION_ESTIMATE_BASIS = 'estimate at 10 ms RTT; measure your pooler RTT';

export interface ExtractionLag {
  total: number;
  /** Stale pages extraction can clear (attendance-blocked excluded). */
  actionable: number;
  attendance_blocked: number;
  /** GBRA-130 A-1: pages whose link extraction is held after repeated failures (excluded from `actionable`). */
  held?: number;
  /** Pages due for the mention pass; null when the pass is off. */
  mention_due: number | null;
  mention_last_pass_at: string | null;
}

/**
 * GBRA-133: one config snapshot answers the watermark, the mention policy and the schema pack (one statement instead
 * of five), and the lag count and the mention preview are issued together, so the whole read is two round trips.
 */
export async function readExtractionLag(engine: BrainEngine, opts: { sourceId?: string; timeoutMs?: number } = {}): Promise<ExtractionLag> {
  return withRequestConfigSnapshot(engine, async () => {
    const [row, mention] = await Promise.all([readLagRow(engine, opts),
      readMentionPolicy(engine).then(policy => policy.enabled ? previewMentionPass(engine, opts.sourceId) : null)]);
    return { total: Number(row?.total ?? 0), actionable: Number(row?.stale ?? 0), attendance_blocked: Number(row?.attendance_blocked ?? 0), held: Number(row?.held ?? 0),
      mention_due: mention ? mention.due : null, mention_last_pass_at: mention?.last_pass_at ?? null };
  });
}

async function readLagRow(engine: BrainEngine, opts: { sourceId?: string; timeoutMs?: number }) {
  // #9295: count the wanted rows the configured sweep re-resolves (a managed or frontmatter sweep: every producer). The
  // managed-mode read needs no config, so it goes out with the config read rather than one round trip before the count.
  const [versionTs, managed] = await Promise.all([effectiveLinkExtractorWatermark(engine), managedPersistenceEnabled(engine)]);
  const wantedProducers = managed || await resolveIncludeFrontmatter(engine) ? undefined : ['body'];
  const read = async (attendance: boolean, held: boolean, guests: boolean) => {
    const { text, params } = extractionLagStatement({ sourceId: opts.sourceId, versionTs, attendance, held, guests, wantedProducers });
    return (await engine.executeRaw<{ total: number; stale: number; attendance_blocked: number; held: number }>(text, params, { timeoutMs: opts.timeoutMs ?? 10_000 }))[0];
  };
  const readShape = async (guests: boolean) => {
    try { return await read(true, true, guests); } catch (e) {
      if (isMissingSchemaObject(e, 'column', 'links_extract_failure')) return read(true, false, guests).catch(async inner => {
        if (!isMissingSchemaObject(inner, 'column', 'links_attendance_blocked_revision')) throw inner;
        return read(false, false, guests);
      });
      if (isMissingSchemaObject(e, 'column', 'links_attendance_blocked_revision')) return read(false, false, guests);
      throw e;
    }
  };
  // GBRA-504: guest pages are never extracted, so they are not lag; a brain behind the guest-notes migration has no guest_sources table.
  try { return await readShape(true); } catch (e) {
    if (!isMissingSchemaObject(e, 'table', 'guest_sources')) throw e;
    return readShape(false);
  }
}

/**
 * GBRA-130 A-1: doctor's note for pages whose managed link extraction is held (failed
 * `LINK_HOLD_THRESHOLD_RUNS` runs in a row), naming the first five with their codes, and the fix that retries them
 * (`gbrain extract --stale [--source-id <id>] --retry-held`). Empty note and no fix when nothing is held.
 */
export async function heldExtractionReport(engine: BrainEngine, held: number, sourceId?: string): Promise<{ note: string; fix?: Action }> {
  if (!held) return { note: '' };
  const { text, params } = heldLinkExtractionStatement({ sourceId, versionTs: await effectiveLinkExtractorWatermark(engine), limit: 5 });
  const sample = (await engine.executeRaw<{ slug: string; source_id: string; code: string }>(text, params, { timeoutMs: 10_000 }))
    .map(row => `${row.source_id}:${row.slug} (${row.code})`);
  const fix = linkExtractionRetryFix(sourceId, true);
  return { fix, note: `. ${held} page(s) failed link extraction ${LINK_HOLD_THRESHOLD_RUNS} runs in a row and are held (${sample.join(', ')}${held > sample.length ? ', ...' : ''}); `
    + `fix the cause the code names, then run ${fix.argv!.join(' ')}` };
}

export interface ExtractionLagVerdict { status: 'ok' | 'warn' | 'fail'; skipped: boolean; pct: number; warn_pct: number; fail_pct: number | null }

/** Doctor warns, and the sync nudge prints, exactly when this says warn or fail. */
export function extractionLagVerdict(lag: Pick<ExtractionLag, 'total' | 'actionable'>, sourceId?: string): ExtractionLagVerdict {
  const warnPct = resolveEnvNumber('GBRAIN_EXTRACTION_LAG_WARN_PCT', EXTRACTION_LAG_WARN_PCT_DEFAULT, { unit: '%' });
  // Hard-fail is off unless GBRAIN_EXTRACTION_LAG_FAIL_PCT is set: a just-upgraded brain warns, it never breaks a cron gate.
  let failPct: number | null = null;
  const failRaw = process.env.GBRAIN_EXTRACTION_LAG_FAIL_PCT;
  if (failRaw !== undefined && failRaw !== '') {
    const n = Number(failRaw);
    if (Number.isFinite(n) && n > 0) failPct = n;
    else warnOnceForEnv('GBRAIN_EXTRACTION_LAG_FAIL_PCT', `[gbrain] Ignoring invalid GBRAIN_EXTRACTION_LAG_FAIL_PCT=${failRaw}; hard-fail stays disabled.`);
  }
  const skipped = lag.total === 0 || (lag.total < EXTRACTION_LAG_MIN_PAGES && !sourceId);
  const pct = lag.total ? (lag.actionable / lag.total) * 100 : 0;
  const status = skipped ? 'ok' : failPct !== null && pct > failPct ? 'fail' : pct > warnPct ? 'warn' : 'ok';
  return { status, skipped, pct, warn_pct: warnPct, fail_pct: failPct };
}

function duration(seconds: number): string {
  if (seconds < 90) return `${Math.max(1, Math.ceil(seconds))} s`;
  if (seconds < 90 * 60) return `${Math.ceil(seconds / 60)} min`;
  return `${(seconds / 3600).toFixed(1)} h`;
}

export interface ExtractionRoute { id: 'plain' | 'catch_up'; argv: string[]; label: string; estimate_seconds: number; runs: number }

/**
 * The default fix (plain `gbrain extract --stale`: 30 minutes per run, resumes; the cron default), verified by the
 * scoped dry run, and one decision between both routes. A fix expected to outlast an agent tool call
 * (`AGENT_TOOL_LIMIT_MS`) is the user's (`host_admin` on a remote brain): `tell_user_to_run`. Option argv are pinned to
 * the selected brain (the CLI's routing unless given; none on a remote brain); a brain-wide run never gets a source
 * pinned onto it.
 */
export function extractionRoutes(lag: Pick<ExtractionLag, 'actionable'>, opts: { sourceId?: string; remote?: boolean; routing?: FixRouting } = {}): { fix: Action; decision: Decision; routes: ExtractionRoute[] } {
  const scope = opts.sourceId ? ['--source-id', opts.sourceId] : [];
  const selected = opts.routing ?? (opts.remote ? undefined : cliRouting());
  const routing = selected ? { ...(selected.brain ? { brain: selected.brain } : {}), ...(opts.sourceId ? { source: opts.sourceId } : {}) } : undefined;
  const catchUpSeconds = lag.actionable / EXTRACTION_PAGES_PER_SECOND;
  const runs = Math.max(1, Math.ceil((catchUpSeconds * 1000) / STALE_TIME_BUDGET_MS));
  const runMinutes = Math.round(STALE_TIME_BUDGET_MS / 60_000);
  const long = Math.min(catchUpSeconds * 1000, STALE_TIME_BUDGET_MS) > AGENT_TOOL_LIMIT_MS;
  const plainArgv = ['gbrain', 'extract', '--stale', ...scope];
  const catchUpArgv = ['gbrain', 'extract', '--stale', '--catch-up', ...scope];
  const routes: ExtractionRoute[] = [
    { id: 'plain', argv: pinRouting(plainArgv, routing), estimate_seconds: Math.ceil(catchUpSeconds), runs,
      label: `${lag.actionable} page(s): about ${runs} run(s) of up to ${runMinutes} min (${EXTRACTION_ESTIMATE_BASIS}); each run resumes where the last stopped (the cron default)` },
    { id: 'catch_up', argv: pinRouting(catchUpArgv, routing), estimate_seconds: Math.ceil(catchUpSeconds), runs: 1,
      label: `${lag.actionable} page(s): about ${duration(catchUpSeconds)} in one run (${EXTRACTION_ESTIMATE_BASIS}); finishes, safe to interrupt, each page commits alone` },
  ];
  const fix: Action = {
    argv: plainArgv, consent: [], actor: long ? (opts.remote ? 'host_admin' : 'user') : 'agent', requires_exclusive: false,
    why: `Derives links and timeline entries for ${lag.actionable} stale page(s); each run stops after ${runMinutes} minutes and the next resumes where it stopped; it changes no page content.`,
    ...(long ? { user_message: `Run on the brain host, in a terminal rather than an agent tool call: ${routes[0]!.argv.join(' ')} (${routes[0]!.label}), or ${routes[1]!.argv.join(' ')} (${routes[1]!.label}).` } : {}),
    verify: { argv: ['gbrain', 'extract', '--stale', '--dry-run', '--json', ...scope] },
    docs: 'docs/guides/troubleshooting.md#symptom-table',
  };
  const decision: Decision = {
    id: 'extraction_route', question: 'How should the link-extraction backlog finish?',
    options: routes.map(route => ({ id: route.id, label: `${route.argv.join(' ')}: ${route.label}`, argv: route.argv })),
    default: 'plain', default_reason: `Bounded runs fit cron and never hold the brain longer than ${runMinutes} minutes; the catch-up run finishes in one go.`,
  };
  return { fix, decision, routes };
}

/** The end-of-sync nudge's lines: the count doctor reports, then both commands with duration and resume text. */
export function extractionNudgeLines(lag: ExtractionLag, opts: { sourceId?: string; routing?: FixRouting } = {}): string[] {
  const { fix, routes } = extractionRoutes(lag, opts);
  const blocked = lag.attendance_blocked ? `; ${lag.attendance_blocked} more wait on unresolved attendees and are not counted` : '';
  const mentions = lag.mention_due ? `; ${lag.mention_due} page(s) await the mention pass` : '';
  const where = fix.actor === 'agent' ? '' : ' Run it on the brain host, not inside an agent tool call.';
  return [`[sync] ${lag.actionable} page(s) have un-extracted edges (doctor: links_extraction_lag${blocked}${mentions}). Two ways to finish:${where}`,
    ...routes.map(route => `[sync]   ${route.argv.join(' ')} — ${route.label}`)];
}

export interface StaleExtractionCounts { actionable: number; attendance_blocked: number; held: number; mention_due: number | null; complete: boolean }

/** The terminal counts `extract --stale` reports: done when `actionable` is 0 and, with mentions on, `mention_due` is 0. */
export async function staleExtractionCounts(engine: BrainEngine, sourceId?: string): Promise<StaleExtractionCounts> {
  const lag = await readExtractionLag(engine, { sourceId });
  return { actionable: lag.actionable, attendance_blocked: lag.attendance_blocked, held: lag.held ?? 0, mention_due: lag.mention_due,
    complete: lag.actionable === 0 && (lag.mention_due ?? 0) === 0 };
}

/** One human line for the counts; raw totals elsewhere in the report include blocked pages and are not a completion signal. */
export function formatStaleCounts(c: StaleExtractionCounts): string {
  return `Actionable: ${c.actionable} page(s) still need link/timeline extraction` +
    (c.attendance_blocked ? `; ${c.attendance_blocked} attendance-blocked page(s) wait on unresolved attendees (extraction cannot clear them)` : '') +
    (c.held ? `; ${c.held} held page(s) failed extraction ${LINK_HOLD_THRESHOLD_RUNS} runs in a row (gbrain extract --stale --retry-held retries them)` : '') +
    (c.mention_due === null ? '' : `; ${c.mention_due} page(s) due for the mention pass`) +
    (c.complete ? '. Extraction is complete (raw stale totals are not a completion signal).' : '. Raw stale totals include blocked pages; done when actionable is 0' + (c.mention_due === null ? '.' : ' and mention-due is 0.'));
}