Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.0 KB
Raw Blame
/**
 * #6858 ask 3: the managed sync runs that are stopped, not draining. An unfinished cursor whose stop is recorded (a
 * `managed-sync-failure` row under its key) or whose pending page receipt ended without committing resumes nothing by
 * itself; doctor `managed_sync_blocked` and get_brain_health read it. An unfinished cursor waiting on a queued or
 * running request (`sync_incomplete`) is draining and not listed, and neither is a cursor saved after its recorded stop:
 * the failure row stays until the whole run finishes, so a resumed run that is moving again would otherwise still read
 * as blocked. A failure whose cursor is gone is an orphan (`gbrain repair managed-sync-orphans`), not a blocked run.
 * #6712 PR 4b (design v3.2 §3.7): a non-company source has a pass row instead; it is blocked while its passes keep
 * stopping (`passHealth`: the newest ended `blocked`, a `not_converged` streak, a `settle_mismatch` streak), with the
 * newest pass's stop reason as `code`, its pass id as `run_id` and `stuck_since` as `blocked_since`. A source listed by
 * its pass is not listed again by a legacy cursor.
 * GBRA-133 S-61: a pass that threw records its stop under a pass failure key (`passFailureKeys`) with no cursor, and its pass
 * row stays `open`, so neither rule above saw it: a discovery-phase stop (path `<discovery>`) or an entry's stop read as
 * nothing blocked while the source indexed nothing. Such a stop is listed while it is newer than the pass row (no pass has
 * started since; a converged or idle pass clears it).
 * #7596: an open pass whose process stopped without ending it (no live drain-lease holder: killed or crashed) is listed as
 * `pass_interrupted` from its row's last write, and once passes in a row keep stopping that way or (#7982) ending on a transient stop,
 * `pass_abandoned` (status's rule).
 */
import type { BrainEngine } from '../engine.ts';
import { readManagedSyncFailures } from './sync-failures.ts';
import { ABANDONED_REPORT_STREAK, passFingerprint, passReadClock, transientEnding, type PassRow } from './sync-pass-row.ts';
import { passFailureKeys } from './sync-cursor-key.ts';
import type { ManagedSyncFailure } from './sync-failures.ts';

/** A run blocked longer than this fails doctor `managed_sync_blocked` and turns get_brain_health to `attention`. */
export const MANAGED_SYNC_BLOCKED_FAIL_MS = 60 * 60_000;

export interface BlockedManagedSync { source_id: string; code: string; run_id: string; blocked_since: string }

export async function readBlockedManagedSyncs(engine: BrainEngine, sourceIds?: string[]): Promise<BlockedManagedSync[]> {
  const cursors = new Set((await engine.executeRaw<{ fingerprint: string }>(`
    SELECT c.fingerprint FROM op_checkpoints c LEFT JOIN op_checkpoints f ON f.op='managed-sync-failure' AND f.fingerprint=c.fingerprint
    WHERE c.op='managed-sync' AND COALESCE(c.completed_keys->0->>'done','false')<>'true' AND (f.updated_at IS NULL OR f.updated_at>=c.updated_at)`)).map(row => row.fingerprint));
  const { readPassReadings, passStopCode, passStuck, passStuckSince } = await import('./sync-pass-readers.ts');
  const failures = await readManagedSyncFailures(engine, sourceIds);
  // A pass blocked by a failed write names that write's code (its failure row is keyed by the pass fingerprint), as a cursor did.
  const passFailure = (sourceId: string, incarnation: string) => failures.find(failure => failure.cursor_key === passFingerprint(sourceId, incarnation) && failure.code !== 'sync_incomplete');
  const readings = await readPassReadings(engine, { sourceIds, head: false });
  const passes: BlockedManagedSync[] = readings.filter(reading => passStuck(reading.health))
    .map(reading => {
      const failure = passStopCode(reading) === 'blocked_by_failures' ? passFailure(reading.source_id, reading.row.value.incarnation) : undefined;
      return { source_id: reading.source_id, code: failure?.code ?? passStopCode(reading), run_id: reading.row.value.telemetry.pass_id, blocked_since: failure?.first_seen ?? passStuckSince(reading.row) };
    });
  const passSources = new Set(passes.map(blocked => blocked.source_id));
  const passRowAt = new Map(readings.map(reading => [reading.source_id, Date.parse(reading.row.updated_at)]));
  const passStops = (await engine.executeRaw<{ fingerprint: string; updated_at: string | Date; failure: ManagedSyncFailure }>(`
    SELECT f.fingerprint, f.updated_at, f.completed_keys->0 AS failure FROM op_checkpoints f
    JOIN sources s ON s.id=f.completed_keys->0->>'source_id' AND s.incarnation::text=f.completed_keys->0->>'source_incarnation'
    WHERE f.op='managed-sync-failure' AND ($1::text[] IS NULL OR s.id=ANY($1::text[]))
      AND NOT EXISTS(SELECT 1 FROM op_checkpoints c WHERE c.op='managed-sync' AND c.fingerprint=f.fingerprint)`, [sourceIds ?? null]))
    .filter(({ fingerprint, updated_at, failure }) => !passSources.has(failure.source_id) && failure.code !== 'sync_incomplete'
      && passFailureKeys(failure.source_id, failure.source_incarnation).includes(fingerprint) && new Date(updated_at).getTime() >= (passRowAt.get(failure.source_id) ?? 0))
    .map(({ failure }) => ({ source_id: failure.source_id, code: failure.code, run_id: failure.run_id, blocked_since: failure.first_seen }));
  const listed = [...failures
    .filter(failure => failure.code !== 'sync_incomplete' && cursors.has(failure.cursor_key) && !passSources.has(failure.source_id))
    .map(failure => ({ source_id: failure.source_id, code: failure.code, run_id: failure.run_id, blocked_since: failure.first_seen })), ...passes, ...passStops];
  // #7982: passes that keep ending on the same transient stop (status's rule: the newest counts itself).
  const { abandonedPasses, openPassRowsStill, passBanked } = await import('./sync-pass-status.ts');
  // GBRA-542: a pass that stopped after committing work moved the source; it is not listed abandoned.
  const banked = (row: PassRow) => passBanked(engine, row.value.source_id, row.value.incarnation, row.value.telemetry).catch(() => false);
  for (const { source_id, row } of readings) {
    if (transientEnding(row.value.telemetry) && abandonedPasses(row) >= ABANDONED_REPORT_STREAK && !listed.some(blocked => blocked.source_id === source_id) && !await banked(row)) {
      listed.push({ source_id, code: 'pass_abandoned', run_id: row.value.telemetry.pass_id, blocked_since: passStuckSince(row) });
    }
  }
  // #7596: an open pass no recorded stop explains, whose drain lease no process renews: its process was killed or crashed.
  const open = readings.filter(reading => reading.row.value.telemetry.outcome === 'open' && !listed.some(blocked => blocked.source_id === reading.source_id));
  if (!open.length) return listed;
  const [{ inspectLocks, syncLockId }, { drainLeaseLive }] = await Promise.all([import('../db-lock.ts'), import('./sync-lease.ts')]);
  // #7614: one read for every open source's lock. A read that failed decides nothing: the quiet-window rule (status's) still counts a streak.
  const locks = await inspectLocks(engine, open.map(({ source_id }) => syncLockId(source_id))).catch(() => undefined);
  const candidates = (await Promise.all(open.map(async ({ source_id, row }) => {
    const lock = locks?.get(syncLockId(source_id));
    const interrupted = locks === undefined ? undefined : !lock || !drainLeaseLive(lock);
    const abandoned = abandonedPasses(row, passReadClock(row), interrupted) >= ABANDONED_REPORT_STREAK && !await banked(row);
    return { source_id, row, interrupted, abandoned };
  }))).filter(entry => entry.interrupted || entry.abandoned);
  // #7614: a pass that ended (releasing its lease) after the pass read changed its row, so it is not listed; nor is an archived source.
  // A re-read that failed lists nothing new.
  const still = candidates.length ? await openPassRowsStill(engine, candidates).catch(() => new Set<string>()) : new Set<string>();
  for (const { source_id, row, abandoned } of candidates) {
    if (still.has(source_id)) listed.push({ source_id, code: abandoned ? 'pass_abandoned' : 'pass_interrupted', run_id: row.value.telemetry.pass_id, blocked_since: abandoned ? passStuckSince(row) : row.updated_at });
  }
  return listed;
}