Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.8 KB
Raw Blame
/**
 * C-9 (#6948 sweep, #6761): the retry set for dead page-write facts-absorb jobs.
 *
 * A put_page's facts-backstop effect submits one facts-absorb job (idempotency key facts-absorb:write:<request id>,
 * 5 attempts) and completes as 'queued' in the same transaction, so nothing dispatches it again. When the job dies
 * on a provider or configuration failure (an outage, a revoked key, #6113), that revision's facts would never land.
 *
 * Each maintenance pass (the facts_drain cycle phase, any engine) reads the dead page-write jobs. A job whose error
 * is a provider or configuration failure is recorded in a durable set keyed by its idempotency key (config row
 * facts.absorb_retry.state). Once the cause has cleared (extraction enabled and its model servable), the pass
 * re-submits it in place (gbrain jobs retry semantics: same row, same key, fresh attempts), at most
 * facts.absorb_retry.max_resubmits times per key (default 3; 0 disables) and at most once an hour per key, and at
 * most MAX_RESUBMITS_PER_PASS keys per pass. A key at its limit is exhausted: it stays dead and visible until an
 * operator resets it (gbrain facts retry-dead --reset <key>) or clears it (--clear <key>: never re-submitted again).
 * A job that died on anything else (a deterministic write refusal, an unclassified error) is not retryable: it stays
 * dead and is counted. Entries settle when their job completes or its row is gone.
 */
import type { BrainEngine } from '../engine.ts';
import { classifyJobError } from '../minions/error-classify.ts';

export const FACTS_ABSORB_RETRY_STATE_KEY = 'facts.absorb_retry.state';
export const FACTS_ABSORB_RETRY_MAX_KEY = 'facts.absorb_retry.max_resubmits';
export const DEFAULT_MAX_RESUBMITS = 3;
export const MAX_RESUBMITS_PER_PASS = 20;
export const RESUBMIT_SPACING_MS = 60 * 60_000;
const WRITE_KEY_PREFIX = 'facts-absorb:write:';

/** The idempotency key of the facts-absorb job a page write's facts-backstop effect submits. */
export function factsAbsorbWriteKey(requestId: string): string {
  return `${WRITE_KEY_PREFIX}${requestId}`;
}

export type DeadAbsorbClass = 'provider' | 'config' | 'not_retryable';

/**
 * Provider and configuration failures clear on their own or with an operator fix; everything else stays dead.
 * Extraction failures carry `[facts-extract] <reason> (model=..) (cause=<class> <global llm class>)` (extract.ts):
 * provider_error and chat_unavailable are retryable; refusal, content filter, malformed or truncated output fail the
 * same way on the same page text, and so does a provider_error whose success response could not be parsed (#9466).
 */
export function classifyDeadFactsAbsorb(errorText: string | null | undefined): DeadAbsorbClass {
  const text = errorText ?? '';
  if (/facts_absorb_write_refused/.test(text)) return 'not_retryable';
  const extract = /^\[facts-extract\] ([a-z_]+)/.exec(text);
  if (extract) {
    if (extract[1] === 'chat_unavailable') return 'config';
    if (extract[1] !== 'provider_error') return 'not_retryable';
    if (/\(cause=[^)]*\bprovider_response_unparseable\b/.test(text)) return 'not_retryable';
    return /\(cause=[^)]*\b(auth|billing|model_not_found)\b/.test(text) ? 'config' : 'provider';
  }
  if (/chat_unavailable|no usable provider key|not servable|model[_ ]not[_ ]found|billing|insufficient[_ ]quota|credit/i.test(text)) return 'config';
  if (/ECONNREFUSED|ECONNRESET|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|EPIPE|fetch failed|socket hang up|network error/i.test(text)) return 'provider';
  const cluster = classifyJobError(text);
  if (cluster === 'auth') return 'config';
  if (cluster === 'rate_limit' || cluster === 'rate_lease_full' || cluster === 'http_5xx' || cluster === 'timeout') return 'provider';
  return 'not_retryable';
}

export interface RetryEntry {
  /** not_retryable only when an operator reset that key by name. */
  job_id: number; slug: string | null; source_id: string; class: DeadAbsorbClass;
  first_dead_at: string; resubmits: number; last_resubmit_at: string | null; cleared_at: string | null;
}
interface RetryState { version: 1; keys: Record<string, RetryEntry> }

export interface DeadAbsorbRetryResult {
  max_resubmits: number;
  /** Retryable dead jobs in the set that the next pass may re-submit (under the limit, not cleared). */
  pending: number;
  resubmitted: number;
  /** Keys at the limit: dead until reset. */
  exhausted: Array<{ key: string; job_id: number; slug: string | null; resubmits: number }>;
  cleared: number;
  not_retryable: Array<{ job_id: number; slug: string | null }>;
  /** Why nothing was re-submitted this pass although keys were pending. */
  waiting_on?: 'extraction_disabled' | 'extraction_unavailable' | 'spacing' | 'disabled' | 'dry_run';
}

export function factsRetryDeadCommand(flag: '--list' | '--reset' | '--clear', key?: string): string {
  return ['gbrain facts retry-dead', flag, ...(key ? [`'${key.replaceAll("'", "'\\''")}'`] : [])].join(' ');
}

export async function loadMaxResubmits(engine: BrainEngine): Promise<number> {
  const raw = await engine.getConfig(FACTS_ABSORB_RETRY_MAX_KEY).catch(() => null);
  if (raw === null || raw === undefined || raw.trim() === '') return DEFAULT_MAX_RESUBMITS;
  const n = Number(raw);
  return Number.isSafeInteger(n) && n >= 0 ? n : DEFAULT_MAX_RESUBMITS;
}

async function readState(engine: BrainEngine): Promise<RetryState> {
  try {
    const raw = await engine.getConfig(FACTS_ABSORB_RETRY_STATE_KEY);
    const parsed = raw ? JSON.parse(raw) as Partial<RetryState> : null;
    if (parsed && parsed.keys && typeof parsed.keys === 'object' && !Array.isArray(parsed.keys)) return { version: 1, keys: parsed.keys };
  } catch { /* an unreadable set is rebuilt from the dead rows */ }
  return { version: 1, keys: {} };
}

const writeState = (engine: BrainEngine, state: RetryState) => engine.setConfig(FACTS_ABSORB_RETRY_STATE_KEY, JSON.stringify(state));

const JOB_COLUMNS = "id, idempotency_key, status, error_text, data->>'slug' AS slug, COALESCE(data->>'sourceId', 'default') AS source_id, finished_at";
const newEntry = (row: JobRow, cls: DeadAbsorbClass, now: number): RetryEntry => ({ job_id: Number(row.id), slug: row.slug, source_id: row.source_id, class: cls,
  first_dead_at: new Date(row.finished_at ?? now).toISOString(), resubmits: 0, last_resubmit_at: null, cleared_at: null });

interface JobRow { id: number | string; idempotency_key: string; status: string; error_text: string | null; slug: string | null; source_id: string; finished_at: string | Date | null }

/** Reconcile the set with the dead rows; with apply, re-submit the eligible keys and persist. */
async function reconcile(engine: BrainEngine, opts: { apply: boolean; record?: boolean; now: number }): Promise<DeadAbsorbRetryResult> {
  const state = await readState(engine);
  const max = await loadMaxResubmits(engine);
  const rows = await engine.executeRaw<JobRow>(
    `SELECT ${JOB_COLUMNS} FROM minion_jobs
      WHERE name = 'facts-absorb' AND idempotency_key LIKE $1 AND (status = 'dead' OR idempotency_key = ANY($2::text[]))`,
    [`${WRITE_KEY_PREFIX}%`, Object.keys(state.keys)]);
  const byKey = new Map(rows.map(r => [r.idempotency_key, r]));
  for (const key of Object.keys(state.keys)) {
    const row = byKey.get(key);
    if (!row || row.status === 'completed' || row.status === 'cancelled') delete state.keys[key];
  }
  const result: DeadAbsorbRetryResult = { max_resubmits: max, pending: 0, resubmitted: 0, exhausted: [], cleared: 0, not_retryable: [] };
  for (const row of rows) {
    if (row.status !== 'dead') continue;
    const cls = classifyDeadFactsAbsorb(row.error_text);
    const known = state.keys[row.idempotency_key];
    if (cls === 'not_retryable' && known?.class !== 'not_retryable') {
      delete state.keys[row.idempotency_key];
      result.not_retryable.push({ job_id: Number(row.id), slug: row.slug });
      continue;
    }
    state.keys[row.idempotency_key] = known ?? newEntry(row, cls, opts.now);
  }
  const due: string[] = [];
  let spaced = false;
  for (const [key, entry] of Object.entries(state.keys)) {
    if (byKey.get(key)?.status !== 'dead') continue;
    if (entry.cleared_at) { result.cleared++; continue; }
    if (entry.resubmits >= max) { result.exhausted.push({ key, job_id: entry.job_id, slug: entry.slug, resubmits: entry.resubmits }); continue; }
    result.pending++;
    if (entry.last_resubmit_at && opts.now - Date.parse(entry.last_resubmit_at) < RESUBMIT_SPACING_MS) { spaced = true; continue; }
    due.push(key);
  }
  if (max === 0 && Object.keys(state.keys).length) result.waiting_on = 'disabled';
  else if (due.length === 0 && result.pending > 0 && spaced) result.waiting_on = 'spacing';
  if (due.length && !opts.apply) result.waiting_on = 'dry_run';
  if (due.length && opts.apply) {
    const { isFactsExtractionEnabled } = await import('./extract.ts');
    const { resolveExtractionAvailability } = await import('./extraction-availability.ts');
    // As the drain does: a key or model fixed since the process started counts as the cause clearing.
    const { refreshGatewayEnvFromFilePlane, reconfigureGatewayWithEngine } = await import('../ai/gateway.ts');
    refreshGatewayEnvFromFilePlane();
    await reconfigureGatewayWithEngine(engine).catch(() => undefined);
    if (!await isFactsExtractionEnabled(engine)) result.waiting_on = 'extraction_disabled';
    else if (!(await resolveExtractionAvailability(engine)).available) result.waiting_on = 'extraction_unavailable';
    else {
      const { MinionQueue } = await import('../minions/queue.ts');
      const queue = new MinionQueue(engine);
      for (const key of due.slice(0, MAX_RESUBMITS_PER_PASS)) {
        const entry = state.keys[key]!;
        if (!await queue.retryJob(entry.job_id)) continue;
        entry.resubmits++;
        entry.last_resubmit_at = new Date(opts.now).toISOString();
        result.resubmitted++;
        result.pending--;
      }
    }
  }
  if (opts.apply || opts.record) await writeState(engine, state);
  return result;
}

/** One maintenance pass: record, settle and re-submit. dryRun reports what would happen and writes nothing. */
export function retryDeadFactsAbsorb(engine: BrainEngine, opts: { dryRun?: boolean; now?: number } = {}): Promise<DeadAbsorbRetryResult> {
  return reconcile(engine, { apply: !opts.dryRun, now: opts.now ?? Date.now() });
}

/** Read-only status for doctor and gbrain facts retry-dead --list (nothing is written or re-submitted). */
export async function readDeadFactsAbsorbRetry(engine: BrainEngine, now = Date.now()): Promise<DeadAbsorbRetryResult> {
  const r = await reconcile(engine, { apply: false, now });
  if (r.waiting_on === 'dry_run') delete r.waiting_on;
  return r;
}

/**
 * Operator controls. reset: the key's resubmits count from zero and it is no longer cleared, so the next pass may
 * re-submit it; naming a dead page-write job the set does not hold (one that died not retryable, after its cause was
 * fixed) adds it. clear: the key is never re-submitted again and stays listed as cleared. Returns the keys changed.
 */
export async function updateDeadFactsAbsorbKeys(engine: BrainEngine, action: 'reset' | 'clear', keys: string[] | 'all', now = Date.now()): Promise<string[]> {
  await reconcile(engine, { apply: false, record: true, now });
  const state = await readState(engine);
  if (keys !== 'all') {
    const rows = await engine.executeRaw<JobRow>(`SELECT ${JOB_COLUMNS} FROM minion_jobs WHERE name = 'facts-absorb' AND status = 'dead' AND idempotency_key = ANY($1::text[])`, [keys]);
    for (const row of rows) state.keys[row.idempotency_key] ??= newEntry(row, classifyDeadFactsAbsorb(row.error_text), now);
  }
  const targets = (keys === 'all' ? Object.keys(state.keys) : keys).filter(k => state.keys[k]);
  for (const key of targets) {
    const entry = state.keys[key]!;
    if (action === 'reset') { entry.resubmits = 0; entry.last_resubmit_at = null; entry.cleared_at = null; }
    else entry.cleared_at = new Date(now).toISOString();
  }
  await writeState(engine, state);
  return targets;
}