Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.3 KB
Raw Blame
/**
 * loop-catchup — recovery passes for the open-loop engine that the Gmail
 * sweep runs besides its normal per-thread detection.
 *
 * Grace holds (#5868). `detectThreadLoop` withholds a loop while its grace
 * window (inbound 24h, outbound 72h) runs, and Gmail history never re-lists a
 * quiet thread, so a held thread used to stay unevaluated forever. The sweep
 * now records each grace hold in connector state (`loop_grace_holds`, capped
 * at GRACE_HOLDS_CAP, oldest pruned and logged) with the loop spec the
 * detection produced. Once due, an unchanged thread (the page still carries
 * the newest message id the spec was computed from) opens its loop from the
 * stored spec with no Gmail call; a changed or missing page, or a backfill
 * seed with no spec, is re-fetched (at most GRACE_REFETCH_PER_SWEEP per
 * sweep). Grace holds are not connector item holds: they never count as
 * failures, `connector_held_items` or `waiting` coverage. A one-shot backfill
 * seeds holds for email pages active in the last GRACE_BACKFILL_DAYS that
 * have no open deterministic loop, from stored pages only.
 *
 * Catch-up (#5867). Managed connector syncs never enqueued `loops_extract`
 * before this wave, so a managed source's recent threads were never analyzed.
 * A one-shot, managed-only pass re-candidates the source's email pages whose
 * newest message falls in the last LOOPS_EXTRACT_WINDOW_DAYS. Evidence that a
 * thread is done is a revision-bound outcome, not a completed job: the
 * `loops_extract` handler records `extracted` or `skipped:<reason>` per page
 * revision (`recordLoopsExtractOutcome`). A candidate is settled when its
 * current revision was `extracted`, when the catch-up itself produced an
 * outcome for it, or when it is not eligible. Dead-lettered catch-up jobs get
 * one retry, then are logged and settled. The done marker is written only
 * when every candidate is settled; the enqueue ceiling defers the rest to the
 * next sweep.
 */

import type { BrainEngine } from '../engine.ts';
import { sanitizeForJsonb } from '../batch-rows.ts';
import { connectorStateKey } from '../persistence/connector-state.ts';
import { LOOPS_EXTRACT_ENQUEUE_CEILING, LOOPS_EXTRACT_JOB, LOOPS_EXTRACT_WINDOW_DAYS, isLoopsExtractionEnabled, loopExtractionEligibility } from './loops-extract.ts';
import { openDueGraceHold, type ThreadLoopVerdict } from './loop-detect.ts';
import { NO_EXCLUSION, type LoopsExclusionPolicy } from './loops-exclusion.ts';
import { muteRecheckThreads } from '../loops/loops-store.ts';
import { loopsSpendGate } from './loops-spend.ts';
import type { GmailThreadData, GoogleSourceState, LoopGraceHold } from './types.ts';

export const GRACE_HOLDS_CAP = 2_000;
export const GRACE_BACKFILL_DAYS = 14;
export const GRACE_REFETCH_PER_SWEEP = 100;
export const LOOPS_OUTCOMES_OP = 'loops-extract-outcomes';
const DAY_MS = 86_400_000;

type Log = (msg: string) => void;

// ── Grace holds (#5868) ──────────────────────────────────────────────────────

/** Records or clears one thread's grace hold from its fresh detection verdict. */
export function recordGraceVerdict(state: GoogleSourceState, thread: GmailThreadData, verdict: ThreadLoopVerdict,
  slug: string | null, myAddresses: Set<string>, log: Log, exclusion: LoopsExclusionPolicy = NO_EXCLUSION): void {
  const holds = state.loop_grace_holds ??= {};
  if (verdict.held && loopExtractionEligibility(thread, myAddresses, exclusion).eligible) {
    // Message prose reaches the connector checkpoint's jsonb: sanitize it as the page render does.
    const spec = verdict.held.spec;
    holds[thread.threadId] = { due_ms: verdict.held.untilMs, slug, rev: thread.messages[thread.messages.length - 1]?.id ?? null,
      spec: { ...spec, summary: sanitizeForJsonb(spec.summary), counterpartyEmail: sanitizeForJsonb(spec.counterpartyEmail),
        evidence: spec.evidence.map(e => ({ ...e, ...(e.quote !== undefined ? { quote: sanitizeForJsonb(e.quote) } : {}) })) } };
    pruneGraceHolds(holds, log);
  } else {
    delete holds[thread.threadId];
  }
}

function pruneGraceHolds(holds: Record<string, LoopGraceHold>, log: Log): void {
  const ids = Object.keys(holds);
  if (ids.length <= GRACE_HOLDS_CAP) return;
  const excess = ids.sort((a, b) => holds[a].due_ms - holds[b].due_ms).slice(0, ids.length - GRACE_HOLDS_CAP);
  for (const id of excess) delete holds[id];
  log(`[google] loop grace holds over the ${GRACE_HOLDS_CAP} cap: pruned ${excess.length} oldest hold(s); those threads are re-evaluated when they change`);
}

/**
 * One-shot backfill: seeds a due hold (no spec, so it is re-fetched) for every
 * email page with activity in the last 14 days and no open deterministic
 * loop. Reads stored pages only. The caller sets `loop_grace_backfill_done`
 * after a non-aborted sweep.
 */
export async function seedGraceBackfill(engine: BrainEngine, sourceId: string, state: GoogleSourceState, log: Log, now = Date.now()): Promise<number> {
  if (state.loop_grace_backfill_done) return 0;
  const rows = await engine.executeRaw<{ slug: string; thread_id: string }>(
    `SELECT p.slug, p.frontmatter->>'thread_id' AS thread_id FROM pages p
      WHERE p.source_id=$1 AND p.type='email' AND p.deleted_at IS NULL
        AND p.frontmatter->>'thread_id' IS NOT NULL AND p.frontmatter->>'date' >= $2
        AND NOT EXISTS (SELECT 1 FROM open_loops l WHERE l.source_id=p.source_id AND l.thread_id=p.frontmatter->>'thread_id'
          AND l.status='open' AND l.loop_type IN ('unanswered_inbound','unanswered_outbound'))`,
    [sourceId, new Date(now - GRACE_BACKFILL_DAYS * DAY_MS).toISOString()]);
  const holds = state.loop_grace_holds ??= {};
  let seeded = 0;
  for (const row of rows) {
    if (holds[row.thread_id]) continue;
    holds[row.thread_id] = { due_ms: now, slug: row.slug, rev: null, spec: null };
    seeded++;
  }
  pruneGraceHolds(holds, log);
  log(`[google] loop grace backfill: ${seeded} thread(s) from the last ${GRACE_BACKFILL_DAYS} days held for re-evaluation`);
  return seeded;
}

/**
 * #5012: queues threads for a re-fetch through the grace-hold path (a due hold
 * with no spec). Every sweep: threads with loops an unmute queued for
 * re-check. Once: the newest thread of each of the WITHHELD_BACKFILL_SENDERS
 * senders with the most open reply-owed loops, so the withheld-senders record
 * covers loops opened before the automated-mail rule existed. `walkFromMs` is
 * the lower bound of an unfinished backfill walk: that walk runs after the
 * grace settle and fetches threads active since then itself, so they are not
 * queued twice.
 */
export async function seedLoopRechecks(engine: BrainEngine, sourceId: string, state: GoogleSourceState, log: Log, now = Date.now(), walkFromMs: number | null = null): Promise<number> {
  const holds = state.loop_grace_holds ??= {};
  const seed = (rows: Array<{ thread_id: string; page_slug: string | null }>) => rows.reduce((n, r) => {
    if (holds[r.thread_id]) return n;
    holds[r.thread_id] = { due_ms: now, slug: r.page_slug, rev: null, spec: null };
    return n + 1;
  }, 0);
  const rechecks = seed(await muteRecheckThreads(engine, sourceId));
  let backfill = 0;
  if (!state.loop_automated_backfill_done) {
    backfill = seed(await engine.executeRaw<{ thread_id: string; page_slug: string | null }>(
      `SELECT DISTINCT ON (counterparty_email) thread_id, page_slug, n, last_activity_at FROM (
         SELECT counterparty_email, thread_id, page_slug, last_activity_at,
                count(*) OVER (PARTITION BY counterparty_email) AS n
           FROM open_loops
          WHERE source_id = $1 AND status = 'open' AND loop_type = 'unanswered_inbound'
            AND counterparty_email IS NOT NULL AND thread_id IS NOT NULL) l
        ORDER BY counterparty_email, last_activity_at DESC`, [sourceId])
      .then((rows) => (rows as Array<{ thread_id: string; page_slug: string | null; n: number; last_activity_at: Date | string | null }>)
        .sort((a, b) => Number(b.n) - Number(a.n)).slice(0, WITHHELD_BACKFILL_SENDERS)
        .filter((r) => walkFromMs == null || r.last_activity_at == null || new Date(r.last_activity_at).getTime() < walkFromMs + DAY_MS)));
    state.loop_automated_backfill_done = true;
  }
  pruneGraceHolds(holds, log);
  if (rechecks + backfill > 0) log(`[google] loop re-checks queued: ${rechecks} unmuted thread(s), ${backfill} sender backfill thread(s)`);
  return rechecks + backfill;
}

export const WITHHELD_BACKFILL_SENDERS = 500;
export const WITHHELD_SENDERS_CAP = 2_000;

/** #5012: keeps the per-sender record of withheld reply-owed loops current with one fresh verdict. */
export function recordWithheldVerdict(state: GoogleSourceState, verdict: ThreadLoopVerdict, now = Date.now()): void {
  const record = state.loop_withheld_senders ??= {};
  for (const spec of verdict.open) if (spec.loopType === 'unanswered_inbound') delete record[spec.counterpartyEmail];
  if (!verdict.withheld) return;
  const key = sanitizeForJsonb(verdict.withheld.counterpartyEmail);
  const prior = record[key];
  record[key] = { reason: verdict.withheld.reason, count: (prior?.count ?? 0) + 1, last_ms: now };
  const keys = Object.keys(record);
  if (keys.length <= WITHHELD_SENDERS_CAP) return;
  for (const k of keys.sort((a, b) => record[a].last_ms - record[b].last_ms).slice(0, keys.length - WITHHELD_SENDERS_CAP)) delete record[k];
}

export type GraceRefetch = (threadId: string) => Promise<'ok' | 'gone' | 'failed'>;

/**
 * Settles due grace holds. Threads the sweep already processed this run were
 * re-detected there. An unchanged thread opens from its stored spec; others
 * are re-fetched through `refetch` (which re-records or clears the hold). A
 * failed re-fetch keeps the hold due for the next sweep.
 */
export async function settleDueGraceHolds(ctx: {
  engine: BrainEngine; sourceId: string; state: GoogleSourceState; log: Log; signal?: AbortSignal;
  processed: Set<string>; refetch: GraceRefetch; now?: number; exclusion?: LoopsExclusionPolicy;
}): Promise<{ opened: number; refetched: number; deferred: number } | 'aborted'> {
  const holds = ctx.state.loop_grace_holds ?? {};
  const now = ctx.now ?? Date.now();
  const due = Object.entries(holds).filter(([tid, h]) => h.due_ms <= now && !ctx.processed.has(tid)).sort((a, b) => a[1].due_ms - b[1].due_ms);
  const result = { opened: 0, refetched: 0, deferred: 0 };
  for (const [tid, hold] of due) {
    if (ctx.signal?.aborted) return 'aborted';
    if (hold.spec && hold.slug && await pageRevision(ctx.engine, ctx.sourceId, hold.slug) === hold.rev) {
      // #5445: an unresolved exclusion keeps the hold (and its spec) for the next sweep; nothing is published.
      const outcome = await openDueGraceHold(ctx.engine, ctx.sourceId, tid, hold.spec, hold.slug, ctx.exclusion ?? NO_EXCLUSION);
      if (outcome === 'deferred') { result.deferred++; continue; }
      if (outcome === 'opened') result.opened++;
      delete holds[tid];
      continue;
    }
    if (result.refetched >= GRACE_REFETCH_PER_SWEEP) { result.deferred++; continue; }
    result.refetched++;
    const outcome = await ctx.refetch(tid);
    if (outcome === 'gone') delete holds[tid];
    else if (outcome === 'ok' && holds[tid] === hold) delete holds[tid];
  }
  if (due.length > 0) {
    ctx.log(`[google] loop grace holds: ${due.length} due — ${result.opened} opened from stored data, ${result.refetched} re-fetched`
      + (result.deferred > 0 ? `, ${result.deferred} deferred to the next sweep` : ''));
  }
  return result;
}

async function pageRevision(engine: BrainEngine, sourceId: string, slug: string): Promise<string | null | undefined> {
  const page = await engine.getPage(slug, { sourceId });
  if (!page) return undefined;
  const id = (page.frontmatter as Record<string, unknown> | undefined)?.message_id;
  return typeof id === 'string' ? id : null;
}

// ── Revision-bound loops_extract outcomes (#5867, E9) ────────────────────────

export interface LoopsExtractOutcome {
  rev: number;
  outcome: string;
  catchup: boolean;
  at: string;
}

async function outcomesKey(engine: BrainEngine, sourceId: string): Promise<string | null> {
  const [row] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [sourceId]);
  return row ? connectorStateKey(sourceId, row.incarnation) : null;
}

/**
 * Records one page revision's extraction outcome in one statement (no race
 * with the sync lease holder), pruning entries older than the extraction
 * window so the row stays bounded.
 */
export async function recordLoopsExtractOutcome(engine: BrainEngine, sourceId: string, slug: string,
  entry: { rev: number; outcome: string; catchup: boolean }): Promise<void> {
  const key = await outcomesKey(engine, sourceId);
  if (!key) return;
  const value = JSON.stringify({ ...entry, at: new Date().toISOString() });
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys)
      VALUES($1,$2,jsonb_build_array(jsonb_build_object('version',1,'outcomes',jsonb_build_object($3::text,$4::text::jsonb))))
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=jsonb_build_array(jsonb_build_object('version',1,'outcomes',
      COALESCE((SELECT jsonb_object_agg(e.key,e.value) FROM jsonb_each(op_checkpoints.completed_keys->0->'outcomes') e
        WHERE e.key<>$3::text AND (e.value->>'at')::timestamptz > now() - ($5::text || ' days')::interval),'{}'::jsonb)
      || jsonb_build_object($3::text,$4::text::jsonb))), updated_at=now()`,
  [LOOPS_OUTCOMES_OP, key, slug, value, String(LOOPS_EXTRACT_WINDOW_DAYS + 1)]);
}

export async function readLoopsExtractOutcomes(engine: BrainEngine, sourceId: string): Promise<Record<string, LoopsExtractOutcome>> {
  const key = await outcomesKey(engine, sourceId);
  if (!key) return {};
  const [row] = await engine.executeRaw<{ outcomes: Record<string, LoopsExtractOutcome> | null }>(
    `SELECT completed_keys->0->'outcomes' AS outcomes FROM op_checkpoints WHERE op=$1 AND fingerprint=$2`, [LOOPS_OUTCOMES_OP, key]);
  return row?.outcomes ?? {};
}

/** The page revision a loops_extract job analyzed: its payload, else the page's newest message date. */
export async function loopsExtractRevision(engine: BrainEngine, sourceId: string, slug: string, payloadRev: unknown): Promise<number | null> {
  if (typeof payloadRev === 'number' && Number.isFinite(payloadRev)) return payloadRev;
  const page = await engine.getPage(slug, { sourceId });
  const date = (page?.frontmatter as Record<string, unknown> | undefined)?.date;
  const ms = typeof date === 'string' ? Date.parse(date) : NaN;
  return Number.isFinite(ms) ? ms : null;
}

// ── Managed 30-day catch-up (#5867) ──────────────────────────────────────────

export interface LoopsEnqueueReport {
  enqueued: number;
  deferred: number;
  skipped_reason: string | null;
}

/** The pending loops_extract depth for one source (waiting, delayed, active). */
export async function pendingLoopsExtractDepth(engine: BrainEngine, sourceId: string): Promise<number> {
  try {
    const rows = await engine.executeRaw<{ n: string }>(
      `SELECT count(*)::text AS n FROM minion_jobs
        WHERE name = $1 AND status IN ('waiting', 'delayed', 'active') AND data->>'sourceId' = $2`, [LOOPS_EXTRACT_JOB, sourceId]);
    return parseInt(rows[0]?.n ?? '0', 10) || 0;
  } catch {
    return 0;
  }
}

/**
 * One catch-up pass for a managed source. Returns null when the catch-up is
 * already done. Never throws for an individual candidate: a failed Gmail read
 * leaves the candidate for the next sweep.
 */
export async function runLoopsCatchup(ctx: {
  engine: BrainEngine; sourceId: string; state: GoogleSourceState; log: Log; signal?: AbortSignal;
  myAddresses: Set<string>; fetchThread: (threadId: string) => Promise<GmailThreadData | null>; now?: number;
  /** Pages this sweep already queues through the normal enqueue. */
  inFlight: Set<string>;
  /** #5445: the sweep's resolved label exclusion; unresolved fails closed for every candidate. */
  exclusion?: LoopsExclusionPolicy;
}): Promise<LoopsEnqueueReport | null> {
  if (ctx.state.loops_catchup?.done) return null;
  const now = ctx.now ?? Date.now();
  const skip = (reason: string, message: string): LoopsEnqueueReport => {
    ctx.log(`[google] loops catch-up skipped (${reason}): ${message}; nothing queued, it runs on a later sweep`);
    return { enqueued: 0, deferred: 0, skipped_reason: reason };
  };
  if (!(await isLoopsExtractionEnabled(ctx.engine))) return skip('extraction_disabled', 'loops.extraction_enabled is off');
  const exclusion = ctx.exclusion ?? NO_EXCLUSION;
  if (exclusion.unresolved.length > 0) {
    return skip('excluded_label_unresolved', `loop exclusion labels ${exclusion.unresolved.map((t) => JSON.stringify(t)).join(', ')} did not resolve`);
  }
  const { isAvailable } = await import('../ai/gateway.ts');
  if (!isAvailable('chat')) return skip('chat_unavailable', 'no configured chat model / API key');
  const spend = await loopsSpendGate(ctx.engine, now);
  if (!spend.ok) return skip(spend.reason, spend.message);

  const cu = ctx.state.loops_catchup ??= { version: 1, floor_ms: now - LOOPS_EXTRACT_WINDOW_DAYS * DAY_MS, retried: [], done: false };
  const pages = await ctx.engine.executeRaw<{ slug: string; thread_id: string; date: string }>(
    `SELECT slug, frontmatter->>'thread_id' AS thread_id, frontmatter->>'date' AS date FROM pages
      WHERE source_id=$1 AND type='email' AND deleted_at IS NULL AND frontmatter->>'thread_id' IS NOT NULL AND frontmatter->>'date' >= $2
      ORDER BY frontmatter->>'date' DESC`, [ctx.sourceId, new Date(cu.floor_ms).toISOString()]);
  const outcomes = await readLoopsExtractOutcomes(ctx.engine, ctx.sourceId);
  const keyOf = (slug: string, rev: number) => `loops-catchup:${ctx.sourceId}:${slug}:${rev}`;
  const candidates = pages.map(p => ({ ...p, rev: Date.parse(p.date) })).filter(p => Number.isFinite(p.rev));
  const jobRows = candidates.length === 0 ? [] : await ctx.engine.executeRaw<{ idempotency_key: string; status: string }>(
    'SELECT idempotency_key, status FROM minion_jobs WHERE idempotency_key = ANY($1::text[])',
    [candidates.flatMap(c => [keyOf(c.slug, c.rev), `${keyOf(c.slug, c.rev)}:retry`, `loops:${ctx.sourceId}:${c.slug}:${c.rev}`])]);
  const jobs = new Map(jobRows.map(r => [r.idempotency_key, r.status]));
  const { MinionQueue } = await import('../minions/queue.ts');
  const queue = new MinionQueue(ctx.engine);
  let budget = Math.max(0, LOOPS_EXTRACT_ENQUEUE_CEILING - await pendingLoopsExtractDepth(ctx.engine, ctx.sourceId));
  const report: LoopsEnqueueReport = { enqueued: 0, deferred: 0, skipped_reason: null };
  let unsettled = 0;
  const settle = (slug: string, rev: number, outcome: string) => recordLoopsExtractOutcome(ctx.engine, ctx.sourceId, slug, { rev, outcome, catchup: true });
  const enqueue = async (c: { slug: string; thread_id: string; rev: number }, key: string) => {
    await queue.add(LOOPS_EXTRACT_JOB, { slug: c.slug, sourceId: ctx.sourceId, threadId: c.thread_id, newestMs: c.rev, catchup: true },
      { priority: 6, idempotency_key: key }, { spendAuthorization: spend.record });
    budget--;
    report.enqueued++;
  };
  for (const c of candidates) {
    if (ctx.signal?.aborted) { unsettled++; break; }
    const prior = outcomes[c.slug];
    if (prior && prior.rev === c.rev && (prior.outcome === 'extracted' || (prior.catchup && prior.outcome !== 'skipped:extraction_disabled'))) continue;
    const sweepStatus = jobs.get(`loops:${ctx.sourceId}:${c.slug}:${c.rev}`);
    if (ctx.inFlight.has(c.slug) || sweepStatus === 'waiting' || sweepStatus === 'delayed' || sweepStatus === 'active') { unsettled++; continue; }
    const key = keyOf(c.slug, c.rev);
    const status = jobs.get(`${key}:retry`) ?? jobs.get(key);
    if (status === 'completed') continue;
    if (status === 'dead' || status === 'failed' || status === 'cancelled') {
      if (cu.retried.includes(c.slug) || jobs.has(`${key}:retry`)) {
        ctx.log(`[google] loops catch-up: ${c.slug} dead-lettered after its retry; giving up on this revision`);
        await settle(c.slug, c.rev, 'skipped:dead_letter');
        continue;
      }
      if (budget <= 0) { report.deferred++; unsettled++; continue; }
      cu.retried.push(c.slug);
      await enqueue(c, `${key}:retry`);
      unsettled++;
      continue;
    }
    if (status !== undefined) { unsettled++; continue; }
    if (budget <= 0) { report.deferred++; unsettled++; continue; }
    let thread: GmailThreadData | null;
    try {
      thread = await ctx.fetchThread(c.thread_id);
    } catch (e) {
      ctx.log(`[google] loops catch-up: could not read thread for ${c.slug} (${e instanceof Error ? e.message : String(e)}); retried next sweep`);
      unsettled++;
      continue;
    }
    if (!thread) { await settle(c.slug, c.rev, 'skipped:thread_not_found'); continue; }
    const verdict = loopExtractionEligibility(thread, ctx.myAddresses, exclusion);
    if (!verdict.eligible) { await settle(c.slug, c.rev, `skipped:ineligible_${verdict.reason}`); continue; }
    await enqueue(c, key);
    unsettled++;
  }
  if (unsettled === 0) cu.done = true;
  ctx.log(`[google] loops catch-up (last ${LOOPS_EXTRACT_WINDOW_DAYS} days): enqueued ${report.enqueued} thread(s)`
    + (report.deferred > 0 ? `, deferred ${report.deferred} to a later sweep (enqueue ceiling ${LOOPS_EXTRACT_ENQUEUE_CEILING})` : '')
    + (cu.done ? '; catch-up complete' : `; ${unsettled} thread(s) not settled yet`));
  return report;
}