Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.3 KB
Raw Blame
/**
 * failed-ledger.ts — the per-(provider, source) record of conversations a
 * connector sync could not archive, and the retry plan built from it (#6387).
 *
 * An entry stays until its conversation imports. The watermark may pass it
 * (a quarantined entry no longer holds the watermark back) and the provider
 * may stop listing it, so a sync retries due entries by id: at most
 * RETRY_BUDGET per run, oldest-due first, with slots reserved under `--limit`
 * so new conversations cannot starve them. A quarantined entry waits
 * HOLD_BACKOFF_MS[n] (then HOLD_DAILY_MS) after each further failure — the
 * connector item-holds schedule, so connector retry timing is one policy.
 * A conversation the provider lists with a new updatedAt, and `--full`,
 * retry at once.
 *
 * Providers that route a detail fetch by organization (`routesByOrg`) record
 * the org with the entry. An entry written before orgs were recorded cannot
 * be retried by id safely (the wrong org returns 404 and would look like a
 * content failure); it is held until a `--full` sync lists it again.
 *
 * No engine imports beyond config reads: status and doctor read this without
 * loading the sync pipeline.
 */

import type { BrainEngine } from '../engine.ts';
import { connectorSourceKey } from './config-keys.ts';
import { HOLD_BACKOFF_MS, HOLD_DAILY_MS } from './item-holds.ts';
import type { ConnectorProviderName, ConversationStub } from './types.ts';

/** Failed fetches or ingests at one updatedAt before a conversation is quarantined. */
export const QUARANTINE_ATTEMPTS = 3;
/** Retained failures retried by id per sync run. */
export const RETRY_BUDGET = 10;

export interface FailedEntry {
  attempts: number;
  updatedAt: string;
  lastAttemptAt?: string;
  /** Absent: due now. Set once the entry is quarantined. */
  nextRetryAt?: string;
  orgId?: string;
  /** The typed code of the last ingest failure (a fetch failure records none). */
  code?: string;
}

export type FailedLedger = Record<string, FailedEntry>;

export function isQuarantined(entry: FailedEntry): boolean {
  return entry.attempts >= QUARANTINE_ATTEMPTS;
}

/** Record one failure; a quarantined entry gets its next retry time. */
export function recordFailedAttempt(ledger: FailedLedger, stub: ConversationStub, nowMs: number, code: string | null = null): FailedEntry {
  const updatedAt = stub.updatedAt ?? '';
  const prior = ledger[stub.id];
  const attempts = (prior?.updatedAt === updatedAt ? prior.attempts : 0) + 1;
  const backoff = attempts >= QUARANTINE_ATTEMPTS ? HOLD_BACKOFF_MS[attempts - QUARANTINE_ATTEMPTS] ?? HOLD_DAILY_MS : 0;
  const orgId = stub.orgId ?? prior?.orgId;
  const entry: FailedEntry = {
    attempts,
    updatedAt,
    lastAttemptAt: new Date(nowMs).toISOString(),
    ...(backoff ? { nextRetryAt: new Date(nowMs + backoff).toISOString() } : {}),
    ...(orgId ? { orgId } : {}),
    ...(code ? { code } : {}),
  };
  ledger[stub.id] = entry;
  return entry;
}

const isDue = (entry: FailedEntry, nowIso: string) => !entry.nextRetryAt || entry.nextRetryAt <= nowIso;

export interface FetchPlan {
  /** Listed conversations to fetch (new, edited, or failed but not quarantined). */
  listed: ConversationStub[];
  /** Due retained failures, fetched by id (oldest-due first). */
  retries: ConversationStub[];
  /** Listed new work the `--limit` cap left for a later run. */
  capped: boolean;
  skippedUnchanged: number;
  /** Quarantined, not due yet. */
  waiting: string[];
  /** Legacy entries without an org on an org-routed provider; need `--full`. */
  held: string[];
}

/**
 * Decide what one run fetches. `--full` fetches every listed conversation;
 * retained failures it does not list are left to the caller (a complete
 * listing that omits them means the provider no longer has them).
 */
export function planConnectorFetch(input: {
  stubs: ConversationStub[];
  synced: Record<string, string>;
  failed: FailedLedger;
  full: boolean;
  limit?: number;
  nowMs: number;
  routesByOrg: boolean;
}): FetchPlan {
  const nowIso = new Date(input.nowMs).toISOString();
  const listedIds = new Set(input.stubs.map((s) => s.id));
  const listed: ConversationStub[] = [];
  const due: Array<{ stub: ConversationStub; at: string }> = [];
  const waiting: string[] = [];
  const held: string[] = [];
  let skippedUnchanged = 0;

  for (const stub of input.stubs) {
    if (input.full) { listed.push(stub); continue; }
    if (stub.updatedAt && input.synced[stub.id] === stub.updatedAt) { skippedUnchanged++; continue; }
    const entry = input.failed[stub.id];
    if (entry && entry.updatedAt === (stub.updatedAt ?? '') && isQuarantined(entry)) {
      if (isDue(entry, nowIso)) due.push({ stub, at: entry.nextRetryAt ?? '' });
      else waiting.push(stub.id);
      continue;
    }
    listed.push(stub);
  }
  if (!input.full) {
    for (const [id, entry] of Object.entries(input.failed)) {
      if (listedIds.has(id)) continue;
      if (input.routesByOrg && !entry.orgId) { held.push(id); continue; }
      if (!isDue(entry, nowIso)) { waiting.push(id); continue; }
      due.push({ stub: { id, updatedAt: entry.updatedAt, ...(entry.orgId ? { orgId: entry.orgId } : {}) }, at: entry.nextRetryAt ?? '' });
    }
  }

  due.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
  let retries = due.slice(0, RETRY_BUDGET).map((d) => d.stub);
  let take = listed.length;
  if (input.limit !== undefined) {
    const reserved = Math.min(retries.length, Math.ceil(input.limit / 2));
    take = Math.min(listed.length, input.limit - reserved);
    retries = retries.slice(0, input.limit - take);
  }
  return { listed: listed.slice(0, take), retries, capped: take < listed.length, skippedUnchanged, waiting, held };
}

export async function readFailedLedger(engine: BrainEngine, provider: ConnectorProviderName, sourceId: string): Promise<FailedLedger> {
  const raw = await engine.getConfig(connectorSourceKey(provider, sourceId, 'failed'));
  if (!raw) return {};
  try {
    const parsed: unknown = JSON.parse(raw);
    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as FailedLedger : {};
  } catch {
    return {};
  }
}

/** The full-repair command a held or pruned failure needs (it re-downloads the whole history; ask first). */
export const fullSyncArgv = (provider: ConnectorProviderName, sourceId: string): string[] =>
  ['gbrain', 'connectors', 'sync', provider, '--full', '--source', sourceId];