Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* src/commands/autopilot-fanout.ts — per-source autopilot dispatch (v0.38).
*
* Replaces the v0.36+ "one autopilot-cycle job per tick" dispatch with a
* fan-out across all sources whose freshness window has elapsed. The
* headline win: a 5-source federated brain refreshes in ~5 min wall-clock
* (parallel via worker pool) instead of ~25 min (sequential across 5 ticks).
*
* Per the codex outside-voice review of this plan:
* - P0-5: each per-source cycle writes `last_full_cycle_at` in its
* `sources.config` JSONB on success (handled in `runCycle` exit hook,
* not here — this module just READS it for freshness gating).
* - P1-2: automatic pull needs a remote and an unmanaged source
* (`automaticSyncPull`), so local-only and managed sources never git-pull.
* - P1-3: PGLite engines default `fanoutMax=1` (PGLite is single-writer;
* parallel fan-out would queue uselessly behind the file lock).
* - P1-4: enumeration keeps sources with a `local_path`, so pure-DB
* sources don't get dispatched (handler would fall back to global
* sync.repo_path, which is wrong for them). Connector sources (#5673)
* are enumerated whatever their `local_path` once they have a recorded
* sync attempt; their cycles run only CONNECTOR_SOURCE_PHASES with no
* brain directory, because the freshness loop owns their sync.
* - P1-5: archive recheck happens in the handler (jobs.ts:1146), not
* here, so a source archived between fan-out and worker claim still
* skips cleanly.
* - Missing checkout paths are skipped at dispatch time; sources.local_path
* is machine-specific shared state and can legitimately point elsewhere.
*
* Phase-scope caveat (codex r1 P0-1): per-source cycle LOCKS let two cycles
* RUN concurrently, but several phases (embed, orphans, purge,
* resolve_symbol_edges, grade_takes, calibration_profile) still walk the
* brain globally inside each cycle. Genuine per-phase per-source isolation
* is the deferred Phase 2 follow-up; THIS wave intentionally accepts that
* two concurrent cycles share embed/orphans work (idempotent at the
* row layer; cost duplication is the visible tradeoff).
*/
import { existsSync } from 'fs';
import type { BrainEngine, SourceRow } from '../core/engine.ts';
import type { MinionQueue } from '../core/minions/queue.ts';
import { SOURCE_FRESHNESS_PHASES, MAINTENANCE_PHASES, LAST_GLOBAL_AT_KEY } from '../core/cycle.ts';
import { GLOBAL_FLOOR_MIN, resolveGlobalFloorMin } from '../core/minions/handlers/autopilot-global-maintenance.ts';
import { CONNECTOR_SOURCE_PHASES } from '../core/cycle/phase-scope.ts';
import { isConnectorSourceKind } from '../core/persistence/connector-identity.ts';
import { attemptedConnectorSourceIds, type ConnectorAccountMismatch } from '../core/persistence/connector-state.ts';
import { connectorAccountMismatches, connectorAccountMismatchSkipWarning } from '../core/persistence/connector-account-mismatch.ts';
import { parseSourceConfig, sourceLocalPathSkipWarning } from '../core/sources-load.ts';
import { automaticSyncPull } from '../core/persistence/automatic-sync-policy.ts';
import { automaticSyncSkip, loadActivationPendingSourceIds, reportAutomaticSyncSkip } from '../core/sync-policy.ts';
import { AUTOPILOT_FULL_CYCLE_FLOOR_MINUTES } from './autopilot-remediation-policy.ts';
// #2194 fix #2: failure cooldown. A source whose autopilot-cycle keeps
// failing/timing-out re-dispatches every tick today (only SUCCESS gates
// dispatch), so the same handful of sources fail and re-fan-out forever — the
// self-perpetuating dead-job storm. Back a failed source off with bounded
// exponential cooldown so a chronically-slow source can't re-dispatch every
// tick. Disabled with autopilot.failure_cooldown_min=0.
const FAILURE_COOLDOWN_BASE_MIN = 10;
const FAILURE_COOLDOWN_CAP_MIN = 120;
const FAILURE_COOLDOWN_EXP_CAP = 4; // 2^4 = 16× base before the cap clamps
/** Recent-failure record for one source (from minion_jobs dead/failed rows). */
export interface SourceFailure { count: number; lastFailedAt: Date; }
/** Resolved cooldown knobs. baseMin <= 0 means the cooldown is disabled. */
export interface CooldownOpts { baseMin: number; capMin: number; }
export interface FanoutOpts {
repoPath: string;
slot: string;
timeoutMs: number;
/**
* Cap on per-tick job submissions. Postgres default 4; PGLite default 1.
* Operator override via `autopilot.fanout_max_per_tick` config.
*/
fanoutMax: number;
jsonMode: boolean;
/** Sink for dispatch events; defaults to process.stderr.write. */
emit?: (line: string) => void;
/** Sink for non-JSON human log lines; defaults to console.log. */
log?: (line: string) => void;
/** Test seam for source checkout availability. */
pathExists?: (path: string) => boolean;
}
/** True only for "the sources table does not exist" (pre-v0.18 brains). */
export function isMissingSourcesTable(e: unknown): boolean {
const code = (e as { code?: unknown } | null)?.code;
if (code === '42P01') return true; // Postgres undefined_table
const message = e instanceof Error ? e.message : String(e);
return /relation "?sources"? does not exist|no such table:? sources|sources table missing/i.test(message);
}
export interface FanoutResult {
/** Source ids whose submission INSERTED a fresh job this tick. */
dispatched: string[];
/** Source ids whose submission coalesced onto an existing pending job
* (maxPending single-flight or same-slot idempotency) — work is in
* flight, but no new row was created. Kept separate so no surface
* claims a dispatch that didn't insert. */
coalesced: string[];
/** Source ids skipped because their last_full_cycle_at is still fresh. */
skipped_fresh: string[];
/** Source ids beyond the fanoutMax cap (will retry next tick). */
skipped_cap: string[];
/** Source ids skipped because they're in failure cooldown (#2194 fix #2). */
skipped_cooldown: string[];
/** Source ids skipped because local_path is unavailable from this machine. */
skipped_unavailable_path: string[];
/** True when this tick fell back to the legacy single-job path
* (no sources rows / engine empty). */
legacy_fallback: boolean;
/** True when every enumerated source is inside the freshness window. */
all_sources_fresh: boolean;
/** True when every enumerated source is either fresh or skipped locally. */
all_sources_handled: boolean;
}
/**
* Resolve `fanoutMax` honoring engine kind + operator override.
*
* Defaults: Postgres = 4, PGLite = 1.
* Override: `autopilot.fanout_max_per_tick` config key (must be >= 1).
* Codex P1-3: PGLite is single-writer; the global cycle.lock serializes
* all source cycles even with per-source DB lock IDs. fanout > 1 on
* PGLite produces no parallelism, only queue pressure. The override is
* still allowed (operator opt-in) but documented as ineffective on PGLite.
*/
export async function resolveFanoutMax(engine: BrainEngine): Promise<number> {
const override = await engine.getConfig('autopilot.fanout_max_per_tick');
if (override) {
const n = parseInt(override, 10);
if (Number.isFinite(n) && n >= 1) return n;
// Invalid override falls through to default — never silently below 1.
}
return engine.kind === 'pglite' ? 1 : 4;
}
/**
* Read the worker concurrency the supervisor most recently STARTED with, from
* its `started` audit event (the lowest-coupling source — no extra lock-row
* column). Filesystem read; returns null when no supervisor has ever started
* (or the event lacks concurrency). Filtered by queue so a `shell`-queue
* supervisor's concurrency doesn't leak into the `default`-queue decision.
*
* ADVISORY use only (doctor warning). Behavior-changing callers (the fanout
* clamp) must additionally gate on a LIVE supervisor — see
* resolveEffectiveFanoutMax — because a stale `started` row can otherwise
* shrink fan-out for a supervisor that isn't running that config (codex #9/D5).
*/
export async function readSupervisorConcurrency(queue = 'default'): Promise<number | null> {
try {
const { readSupervisorEvents } = await import('../core/minions/handlers/supervisor-audit.ts');
const events = readSupervisorEvents({ sinceMs: 24 * 60 * 60 * 1000 });
const started = events
.filter((e) => e.event === 'started' && (e.queue === undefined || e.queue === queue))
.pop();
const c = started?.concurrency;
return typeof c === 'number' && Number.isFinite(c) ? c : null;
} catch {
return null;
}
}
/**
* Resolve fanoutMax CLAMPED to the worker's effective concurrency (#2194 fix #1).
*
* Fanning out more cycles than the worker can run guarantees waiters that then
* race the stalled-sweeper. Clamp to `max(1, concurrency - 1)` — reserving ≥1
* slot for targeted sync/embed jobs that share the `default` queue.
*
* codex #9 / D5: the clamp is BEHAVIOR-changing, so it trusts only a
* proven-alive supervisor (live DB-lock holder, `ttl_expires_at`-gated). With
* no live holder the concurrency is UNKNOWN and we fall back to the unclamped
* default (4 pg / 1 pglite) — the safe direction (never starve on stale data).
* Operators can disable the clamp via `autopilot.fanout_clamp_to_concurrency`.
*/
export async function resolveEffectiveFanoutMax(engine: BrainEngine, queue = 'default'): Promise<number> {
const base = await resolveFanoutMax(engine);
const clampCfg = await engine.getConfig('autopilot.fanout_clamp_to_concurrency');
if (clampCfg === 'false' || clampCfg === '0') return base; // operator opt-out
try {
const { inspectLock, isLockHolderLive } = await import('../core/db-lock.ts');
const { supervisorLockId, SUPERVISOR_LOCK_TTL_MIN } = await import('../core/minions/supervisor.ts');
const snap = await inspectLock(engine, supervisorLockId(queue));
if (!snap || !isLockHolderLive(snap, SUPERVISOR_LOCK_TTL_MIN)) return base; // no live holder → unknown → no clamp
const concurrency = await readSupervisorConcurrency(queue);
if (concurrency === null) return base;
return Math.max(1, Math.min(base, concurrency - 1));
} catch {
return base;
}
}
/**
* Read `last_full_cycle_at` ISO string from a source's config JSONB.
* Returns null when missing or unparseable. Pure function over the row
* shape `listAllSources` returns (config is already a parsed object).
*/
export function readLastFullCycleAt(src: SourceRow): Date | null {
const raw = src.config?.last_full_cycle_at;
if (typeof raw !== 'string') return null;
const d = new Date(raw);
return Number.isFinite(d.getTime()) ? d : null;
}
/**
* A source needs work when either:
* 1. It has never had a full cycle complete (`last_full_cycle_at` null), OR
* 2. The last full cycle is older than the freshness floor.
*
* `last_sync_at` is NOT consulted here — sync is one phase of a cycle, and
* a brain may have fresh sync but stale extract/embed. The 60-min floor on
* full-cycle is the canonical freshness signal for autopilot dispatch.
*/
export function isSourceStale(
src: SourceRow,
now = Date.now(),
floorMin = AUTOPILOT_FULL_CYCLE_FLOOR_MINUTES,
): boolean {
const last = readLastFullCycleAt(src);
if (last === null) return true;
const ageMin = (now - last.getTime()) / 60_000;
return ageMin >= floorMin;
}
/**
* Most recent SUCCESSFUL cycle for a source. Prefers `last_source_cycle_at`
* (per-source phases, written by the split cycle) and falls back to the legacy
* `last_full_cycle_at`, so this works before AND after the cycle split.
*/
export function readLastSuccessAt(src: SourceRow): Date | null {
const c = src.config ?? {};
const raw = (typeof c.last_source_cycle_at === 'string' && c.last_source_cycle_at)
|| (typeof c.last_full_cycle_at === 'string' && c.last_full_cycle_at)
|| null;
if (!raw) return null;
const d = new Date(raw);
return Number.isFinite(d.getTime()) ? d : null;
}
/** Bounded exponential cooldown window (minutes) for a given failure count. */
export function cooldownMinForCount(count: number, opts: CooldownOpts): number {
if (count <= 0 || opts.baseMin <= 0) return 0;
const mult = Math.pow(2, Math.min(count - 1, FAILURE_COOLDOWN_EXP_CAP));
return Math.min(opts.baseMin * mult, opts.capMin);
}
/**
* Is a source currently in failure cooldown? Pure — drives both the dispatch
* gate and the claim-time guard. A SUCCESS at-or-after the most recent failure
* clears the cooldown (codex #7: operator repair / manual cycle re-eligibility),
* so a recovered source is never suppressed by stale failure history.
*/
export function isInFailureCooldown(
failure: SourceFailure | undefined,
lastSuccessAt: Date | null,
now: number,
opts: CooldownOpts,
): boolean {
if (opts.baseMin <= 0) return false; // disabled
if (!failure || failure.count <= 0) return false;
if (lastSuccessAt && lastSuccessAt.getTime() >= failure.lastFailedAt.getTime()) return false;
const cooldownMs = cooldownMinForCount(failure.count, opts) * 60_000;
return (now - failure.lastFailedAt.getTime()) < cooldownMs;
}
/**
* Resolve cooldown knobs from config. `autopilot.failure_cooldown_min` overrides
* the base (0 = disable entirely — exactly today's behavior);
* `autopilot.failure_cooldown_cap_min` overrides the ceiling.
*/
export async function resolveFailureCooldownOpts(engine: BrainEngine): Promise<CooldownOpts> {
let baseMin = FAILURE_COOLDOWN_BASE_MIN;
let capMin = FAILURE_COOLDOWN_CAP_MIN;
const baseCfg = await engine.getConfig('autopilot.failure_cooldown_min');
if (baseCfg !== null && baseCfg !== undefined && baseCfg !== '') {
const n = parseInt(baseCfg, 10);
if (Number.isFinite(n) && n >= 0) baseMin = n;
}
const capCfg = await engine.getConfig('autopilot.failure_cooldown_cap_min');
if (capCfg) {
const n = parseInt(capCfg, 10);
if (Number.isFinite(n) && n >= 1) capMin = n;
}
return { baseMin, capMin };
}
/**
* Read recent dead/failed autopilot-cycle jobs grouped by source. Read-at-
* dispatch (NOT a write hook) because timeouts/RSS-kills/stalls dead-letter via
* SQL in queue.ts and never run handler code — a write-only cooldown would miss
* the exact failures that drive the storm. Engine-parity-safe via executeRaw
* (one query, both engines); cutoff is precomputed in JS to avoid INTERVAL
* portability concerns. codex #6: rows with a null source_id are excluded.
* #7188: a cycle that ran but had a failed phase ends `failed` with
* `job_outcome: 'phase_failed'` and does not count: only a whole-cycle failure
* (a thrown handler, a timeout or stall, `blocked_by_failures`) backs a source off,
* so one failing phase never stops the source's sync.
*/
export async function readRecentSourceFailures(
engine: BrainEngine,
opts: { sinceMin?: number; sourceId?: string } = {},
): Promise<Map<string, SourceFailure>> {
const sinceMin = opts.sinceMin ?? FAILURE_COOLDOWN_CAP_MIN;
const cutoff = new Date(Date.now() - sinceMin * 60_000).toISOString();
const map = new Map<string, SourceFailure>();
try {
const params: unknown[] = [cutoff];
let sql =
`SELECT data->>'source_id' AS source_id,
count(*)::int AS fail_count,
max(finished_at) AS last_failed_at
FROM minion_jobs
WHERE name = 'autopilot-cycle'
AND status IN ('dead','failed')
AND (result->>'job_outcome' IS NULL OR result->>'job_outcome' <> 'phase_failed')
AND data->>'source_id' IS NOT NULL
AND finished_at IS NOT NULL
AND finished_at > $1`;
if (opts.sourceId) { params.push(opts.sourceId); sql += ` AND data->>'source_id' = $${params.length}`; }
sql += ` GROUP BY data->>'source_id'`;
const rows = await engine.executeRaw<{ source_id: string | null; fail_count: number; last_failed_at: string | Date }>(sql, params);
for (const r of rows) {
if (!r.source_id) continue; // codex #6 null-source guard (defensive)
const last = r.last_failed_at instanceof Date ? r.last_failed_at : new Date(r.last_failed_at);
if (!Number.isFinite(last.getTime())) continue;
map.set(r.source_id, { count: Number(r.fail_count) || 0, lastFailedAt: last });
}
} catch {
// Pre-migration / transient DB error → no cooldown data (fail open: dispatch).
}
return map;
}
/**
* Claim-time cooldown guard (codex #5 / D4): a job already queued or retrying
* (max_attempts:2) can reach the worker after the dispatch gate decided. The
* handler calls this immediately before runCycle; an in-cooldown claim becomes
* a no-op skip (NOT a failure — it must not re-arm the cooldown). Shares the
* exact cooldown math with the dispatch gate (DRY).
*/
export async function isSourceInCooldown(engine: BrainEngine, sourceId: string, now = Date.now()): Promise<boolean> {
const opts = await resolveFailureCooldownOpts(engine);
if (opts.baseMin <= 0) return false;
const failures = await readRecentSourceFailures(engine, { sinceMin: opts.capMin, sourceId });
const failure = failures.get(sourceId);
if (!failure) return false;
let lastSuccessAt: Date | null = null;
try {
const rows = await engine.executeRaw<{ config: Record<string, unknown> | null }>(
`SELECT config FROM sources WHERE id = $1`, [sourceId],
);
if (rows[0]) lastSuccessAt = readLastSuccessAt({ config: rows[0].config ?? {} } as SourceRow);
} catch { /* treat as no success */ }
return isInFailureCooldown(failure, lastSuccessAt, now, opts);
}
/**
* Decide which sources to dispatch this tick. Pure function so tests can
* exercise the freshness gate + cap math without an engine.
*
* Returns the ordered list of source ids to fan out:
* - Filters to stale sources (per isSourceStale).
* - Sorts by oldest-first (sources with NULL last_full_cycle_at go first;
* then oldest by ascending date). Deterministic for tests.
* - Caps at fanoutMax. Sources past the cap retry next tick.
*/
export function selectSourcesForDispatch(
sources: SourceRow[],
fanoutMax: number,
now = Date.now(),
floorMin = AUTOPILOT_FULL_CYCLE_FLOOR_MINUTES,
recentFailures: Map<string, SourceFailure> = new Map(),
cooldownOpts: CooldownOpts = { baseMin: FAILURE_COOLDOWN_BASE_MIN, capMin: FAILURE_COOLDOWN_CAP_MIN },
pathExists: (path: string) => boolean = () => true,
connectorIds: ReadonlySet<string> = new Set(),
accountMismatches: ReadonlyMap<string, ConnectorAccountMismatch> = new Map(),
): {
dispatch: SourceRow[];
skippedFresh: SourceRow[];
skippedCap: SourceRow[];
skippedCooldown: SourceRow[];
/** Each row carries the warning that excluded it, so the caller logs it
* directly instead of recomputing (a second existsSync per row). */
skippedUnavailablePath: Array<SourceRow & { skip_warning: string }>;
} {
const stale: SourceRow[] = [];
const fresh: SourceRow[] = [];
const cooldown: SourceRow[] = [];
const unavailablePath: Array<SourceRow & { skip_warning: string }> = [];
for (const s of sources) {
const mismatch = connectorIds.has(s.id) ? accountMismatches.get(s.id) : undefined;
const skipWarning = mismatch ? connectorAccountMismatchSkipWarning(s.id, mismatch)
: s.local_path && !connectorIds.has(s.id) ? sourceLocalPathSkipWarning(s.id, s.local_path, pathExists, s.config) : null;
if (skipWarning) {
unavailablePath.push({ ...s, skip_warning: skipWarning });
continue;
}
if (!isSourceStale(s, now, floorMin)) { fresh.push(s); continue; }
// #2194 fix #2: a stale source that recently failed is held in cooldown so
// it can't re-dispatch every tick (the storm). Success clears it.
if (isInFailureCooldown(recentFailures.get(s.id), readLastSuccessAt(s), now, cooldownOpts)) {
cooldown.push(s);
continue;
}
stale.push(s);
}
// Oldest-first ordering: NULL last_full_cycle_at sorts before any timestamp.
stale.sort((a, b) => {
const la = readLastFullCycleAt(a)?.getTime() ?? -Infinity;
const lb = readLastFullCycleAt(b)?.getTime() ?? -Infinity;
if (la !== lb) return la - lb;
return a.id.localeCompare(b.id); // tiebreaker: stable alphabetical
});
const dispatch = stale.slice(0, fanoutMax);
const skippedCap = stale.slice(fanoutMax);
return { dispatch, skippedFresh: fresh, skippedCap, skippedCooldown: cooldown, skippedUnavailablePath: unavailablePath };
}
/**
* Per-tick autopilot fan-out. Replaces the v0.36+ single autopilot-cycle
* dispatch when `shouldFullCycle` is true.
*
* Fallback path: if `listAllSources` returns 0 rows (fresh install before
* `gbrain sources add`, or `sources` table not migrated yet), submit ONE
* legacy autopilot-cycle with no source_id so the existing single-source
* brain keeps working.
*/
export async function dispatchPerSource(
engine: BrainEngine,
queue: MinionQueue,
opts: FanoutOpts,
): Promise<FanoutResult> {
const emit = opts.emit ?? ((line) => process.stderr.write(line + '\n'));
const log = opts.log ?? ((line) => console.log(line));
let sources: SourceRow[];
let connectorIds = new Set<string>();
let idleConnectors = 0;
try {
sources = await engine.listAllSources({ localPathOnly: true });
// The DX O1 dispatch gate is read whether or not any checkout source
// exists, so a connector whose path was cleared is never folded into the
// legacy cycle (#5673). An unreadable gate fails closed for connectors.
const { connectorAwaitingFirstSync } = await import('./autopilot-dispatch.ts');
const attempted = await attemptedConnectorSourceIds(engine).catch(() => null);
connectorIds = new Set(attempted ?? []);
const checkouts = new Set(sources.filter(s => !isConnectorSourceKind(parseSourceConfig(s.config).kind)).map(s => s.id));
idleConnectors = sources.filter(s => connectorAwaitingFirstSync(s, attempted, opts.jsonMode === true, emit)).length;
const all = connectorIds.size ? await engine.listAllSources() : sources;
sources = all.filter(s => connectorIds.has(s.id) || checkouts.has(s.id));
} catch (e) {
const message = e instanceof Error ? e.message : String(e);
// Brand-new brain without sources table (pre-v0.18) — fall through
// to the legacy single-job path.
//
// Any OTHER error (connection timeout, pooler CONNECTION_CLOSED, …) is
// transient and says nothing about the brain's shape. Falling back then
// dispatched a legacy cycle against repoPath on a multi-source brain,
// which full-imported the repo root into the 'default' source and
// duplicated every page under a new slug prefix. Skip this tick instead;
// the next tick retries once the database is reachable.
if (!isMissingSourcesTable(e)) {
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'fanout_skipped', reason: 'sources_unavailable', error: message }));
} else {
log(`[dispatch] skipped tick: could not list sources (${message}); retrying next tick`);
}
return {
dispatched: [],
coalesced: [],
skipped_fresh: [],
skipped_cap: [],
skipped_cooldown: [],
skipped_unavailable_path: [],
legacy_fallback: false,
all_sources_fresh: false,
all_sources_handled: false,
};
}
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'fanout_unavailable', error: message }));
}
sources = [];
}
// Sources exist but every one is a connector awaiting its first sync: stay
// idle rather than falling back to the legacy cycle on the global repo path.
if (sources.length === 0 && idleConnectors > 0) {
return { dispatched: [], coalesced: [], skipped_fresh: [], skipped_cap: [], skipped_cooldown: [], skipped_unavailable_path: [],
legacy_fallback: false, all_sources_fresh: false, all_sources_handled: false };
}
if (sources.length === 0) {
// Legacy path — preserves today's behavior for single-source brains
// (default source) and pre-v0.18 brains without the sources table.
const job = await queue.add(
'autopilot-cycle',
{ repoPath: opts.repoPath },
{
queue: 'default',
// Slot key dedups repeats within one slot; maxPending: 1 is the
// cross-slot guard — an in-flight (waiting or live-lock active)
// cycle suppresses re-dispatch even after the slot rotates. This
// closes the unbounded-duplicate loop: slot rotation used to mint
// a fresh key every baseInterval while maxWaiting ignored the
// active row, growing the queue forever when a cycle stalled.
idempotency_key: `autopilot-cycle:${opts.slot}`,
max_attempts: 2,
timeout_ms: opts.timeoutMs,
maxPending: 1,
},
);
if (job.coalesced) {
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'dispatch_coalesced', job_id: job.id, mode: 'legacy', slot: opts.slot }));
} else {
log(`[dispatch] coalesced onto job #${job.id} autopilot-cycle (legacy single-source; already in flight)`);
}
} else if (opts.jsonMode) {
emit(JSON.stringify({ event: 'dispatched', job_id: job.id, mode: 'legacy', slot: opts.slot }));
} else {
log(`[dispatch] job #${job.id} autopilot-cycle (legacy single-source)`);
}
return {
dispatched: [],
coalesced: [],
skipped_fresh: [],
skipped_cap: [],
skipped_cooldown: [],
skipped_unavailable_path: [],
legacy_fallback: true,
all_sources_fresh: false,
all_sources_handled: false,
};
}
// #2194 fix #2: load recent per-source failures + cooldown knobs so a
// chronically-failing source is backed off instead of re-dispatched every
// tick. Fail-open: cooldown is an optimization, not a correctness gate — if
// config/job-history reads fail (or the engine lacks them), dispatch proceeds
// with no cooldown rather than blocking.
let cooldownOpts: CooldownOpts = { baseMin: 0, capMin: FAILURE_COOLDOWN_CAP_MIN };
let recentFailures = new Map<string, SourceFailure>();
try {
cooldownOpts = await resolveFailureCooldownOpts(engine);
if (cooldownOpts.baseMin > 0) {
recentFailures = await readRecentSourceFailures(engine, { sinceMin: cooldownOpts.capMin });
}
} catch {
cooldownOpts = { baseMin: 0, capMin: FAILURE_COOLDOWN_CAP_MIN };
}
const pathExists = opts.pathExists ?? existsSync;
// #6803: a connector whose credential resolves to another account is unavailable until the operator repins it.
const accountMismatches = connectorIds.size ? await connectorAccountMismatches(engine).catch(() => new Map<string, ConnectorAccountMismatch>()) : new Map<string, ConnectorAccountMismatch>();
const { dispatch, skippedFresh, skippedCap, skippedCooldown, skippedUnavailablePath } =
selectSourcesForDispatch(
sources,
opts.fanoutMax,
Date.now(),
AUTOPILOT_FULL_CYCLE_FLOOR_MINUTES,
recentFailures,
cooldownOpts,
pathExists,
connectorIds,
accountMismatches,
);
for (const src of skippedUnavailablePath) {
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'fanout_source_path_skipped', source_id: src.id, reason: src.skip_warning }));
} else {
log(src.skip_warning);
}
}
// #5198: claimed-but-not-activated sources refuse sync until activation.
const activationPending = await loadActivationPendingSourceIds(engine);
const dispatched: string[] = [];
const coalesced: string[] = [];
for (const src of dispatch) {
try {
// #4399: config.syncEnabled=false excludes the source from automatic
// sync. It still gets its lint/backlinks/extract cycle and freshness
// stamp; only the sync phase (and the pull that feeds it) is dropped —
// normalizeQueuedSourcePhases passes a freshness subset through as-is.
// #5198: a claimed source awaiting activation is treated the same way,
// so its cycle does not fail on a sync that is refused by contract, and
// so is a gbrain-owned content directory where sync does not apply (P4.2).
const skip = await automaticSyncSkip(engine, src, activationPending);
if (skip?.reason !== 'sync_disabled') reportAutomaticSyncSkip(src.id, skip, 'fanout_sync_skipped', opts.jsonMode === true, opts.jsonMode ? emit : log);
const syncDisabled = !!skip;
const connector = connectorIds.has(src.id);
const shouldPull = !syncDisabled && !connector && await automaticSyncPull(engine, src);
const job = await queue.add(
'autopilot-cycle',
{
repoPath: connector ? null : opts.repoPath,
source_id: src.id,
pull: shouldPull,
// Freshness is stamped by bounded deterministic work only. LLM-backed
// source enrichment (atoms, takes, thin-page development, etc.) is
// explicit/background work and cannot hold source freshness hostage.
phases: connector ? CONNECTOR_SOURCE_PHASES : syncDisabled ? SOURCE_FRESHNESS_PHASES.filter((p) => p !== 'sync') : SOURCE_FRESHNESS_PHASES,
},
{
queue: 'default',
// Per-source idempotency key — two ticks for the same source
// within the same slot coalesce; different sources never collide.
idempotency_key: `autopilot-cycle:${src.id}:${opts.slot}`,
max_attempts: 2,
timeout_ms: opts.timeoutMs,
// Still DELIBERATELY no maxWaiting here (its NULL-as-wildcard
// source scope would coalesce N per-source jobs down to one).
// maxPending is safe: its scope is EXACT on
// COALESCE(data.sourceId, data.source_id), so each source keeps
// an independent single-flight cap — and unlike the slot key, it
// suppresses cross-slot re-dispatch while THIS source's cycle is
// still in flight (waiting or live-lock active).
maxPending: 1,
},
);
if (job.coalesced) {
coalesced.push(src.id);
if (opts.jsonMode) {
emit(JSON.stringify({
event: 'dispatch_coalesced',
job_id: job.id,
mode: 'per_source',
source_id: src.id,
slot: opts.slot,
}));
} else {
log(`[dispatch] coalesced onto job #${job.id} autopilot-cycle source=${src.id} (already in flight)`);
}
} else {
dispatched.push(src.id);
if (opts.jsonMode) {
emit(JSON.stringify({
event: 'dispatched',
job_id: job.id,
mode: 'per_source',
source_id: src.id,
pull: shouldPull,
slot: opts.slot,
}));
} else {
log(`[dispatch] job #${job.id} autopilot-cycle source=${src.id}${shouldPull ? ' pull=yes' : ''}`);
}
}
} catch (e) {
// Per-source submit failure does NOT abort the tick (codex E1 F1
// defensive). Other sources still dispatched; this one retries
// next tick.
if (opts.jsonMode) {
emit(JSON.stringify({
event: 'fanout_submit_failed',
source_id: src.id,
error: e instanceof Error ? e.message : String(e),
}));
} else {
log(`[dispatch] WARN source=${src.id}: ${e instanceof Error ? e.message : String(e)}`);
}
}
}
if (skippedCap.length > 0 && opts.jsonMode) {
emit(JSON.stringify({
event: 'fanout_cap_reached',
cap: opts.fanoutMax,
pending: skippedCap.map(s => s.id),
}));
}
if (skippedCooldown.length > 0 && opts.jsonMode) {
emit(JSON.stringify({
event: 'fanout_cooldown_skipped',
sources: skippedCooldown.map(s => s.id),
}));
}
return {
dispatched,
coalesced,
skipped_fresh: skippedFresh.map(s => s.id),
skipped_cap: skippedCap.map(s => s.id),
skipped_cooldown: skippedCooldown.map(s => s.id),
skipped_unavailable_path: skippedUnavailablePath.map(s => s.id),
legacy_fallback: false,
all_sources_fresh: skippedFresh.length === sources.length,
all_sources_handled: skippedFresh.length + skippedUnavailablePath.length === sources.length,
};
}
/** Is the brain-wide maintenance overdue? Null/unparseable → overdue. */
export function isGlobalMaintenanceStale(lastGlobalAtIso: string | null, now = Date.now(), floorMin = GLOBAL_FLOOR_MIN): boolean {
if (!lastGlobalAtIso) return true;
const d = new Date(lastGlobalAtIso);
if (!Number.isFinite(d.getTime())) return true;
return (now - d.getTime()) / 60_000 >= floorMin;
}
/**
* #4578: the global maintenance job deadline. Precedence:
* GBRAIN_GLOBAL_MAINTENANCE_TIMEOUT_MS > config autopilot.global_maintenance_timeout_ms
* > the autopilot full-cycle default. Values below one minute are ignored.
*/
export async function resolveGlobalMaintenanceTimeoutMs(engine: BrainEngine, fallbackMs: number): Promise<number> {
const parse = (raw: string | null | undefined) => {
const n = raw ? Number(raw) : NaN;
return Number.isSafeInteger(n) && n >= 60_000 ? n : null;
};
return parse(process.env.GBRAIN_GLOBAL_MAINTENANCE_TIMEOUT_MS)
?? parse(await engine.getConfig('autopilot.global_maintenance_timeout_ms'))
?? fallbackMs;
}
/**
* #2194 fix #3 / #2227 bug #3 — dispatch the single brain-wide maintenance job
* that runs the `mixed` + `global` cycle phases ONCE per
* window, instead of N per-source cycles each running them concurrently (the
* RSS blowout). Single-flight is structural: one `idempotency_key` per slot +
* `maxPending:1` (an in-flight waiting/live-lock-active run suppresses
* re-dispatch even across slot rotation), so a slow run never stacks. Gated on
* `autopilot.last_global_at` (stamped by the handler on success). Postgres-only
* fan-out concern; on PGLite the file lock already serializes, but the job is
* still correct there.
*/
export async function dispatchGlobalMaintenance(
engine: BrainEngine,
queue: MinionQueue,
opts: { repoPath: string; slot: string; timeoutMs: number; jsonMode: boolean; emit?: (l: string) => void; log?: (l: string) => void },
): Promise<{ dispatched: boolean; coalesced?: boolean; reason: 'stale' | 'fresh' }> {
const emit = opts.emit ?? ((line) => process.stderr.write(line + '\n'));
const log = opts.log ?? ((line) => console.log(line));
const floorMin = await resolveGlobalFloorMin(engine);
const lastGlobalAt = await engine.getConfig(LAST_GLOBAL_AT_KEY);
if (!isGlobalMaintenanceStale(lastGlobalAt, Date.now(), floorMin)) {
return { dispatched: false, reason: 'fresh' };
}
const timeoutMs = await resolveGlobalMaintenanceTimeoutMs(engine, opts.timeoutMs);
const job = await queue.add(
'autopilot-global-maintenance',
{ repoPath: opts.repoPath, phases: MAINTENANCE_PHASES },
{
queue: 'default',
// Structural single-flight: one global job per slot; maxPending:1
// coalesces any surplus — including across slot rotation while a slow
// brain-wide pass is still in flight — so duplicates never stack.
idempotency_key: `autopilot-global:${opts.slot}`,
max_attempts: 2,
timeout_ms: timeoutMs,
maxPending: 1,
},
);
if (job.coalesced) {
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'dispatch_coalesced', job_id: job.id, mode: 'global_maintenance', slot: opts.slot }));
} else {
log(`[dispatch] coalesced onto job #${job.id} autopilot-global-maintenance (already in flight)`);
}
// dispatched: false — no row was inserted (same honest-dispatch contract
// as dispatchPerSource, where coalesced sources are excluded from
// `dispatched`). The coalesced flag says work is already in flight.
return { dispatched: false, coalesced: true, reason: 'stale' };
}
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'dispatched', job_id: job.id, mode: 'global_maintenance', slot: opts.slot }));
} else {
log(`[dispatch] job #${job.id} autopilot-global-maintenance (brain-wide phases)`);
}
return { dispatched: true, reason: 'stale' };
}
/**
* Opt-in scheduled connector sync (OV#4: default OFF, daily cadence). For each
* provider that has a credential AND `connectors.<p>.auto_sync` truthy AND no
* `auth_error_at` newer than the credential's `savedAt` (a dead cookie must not
* burn job slots), submit a `connector-sync` job when the last sync is older
* than `connectors.sync_floor_min` (default 1440). Idempotency-keyed +
* maxPending:1 so a slow sync never stacks.
*/
export async function maybeDispatchConnectorSyncs(
engine: BrainEngine,
queue: MinionQueue,
opts: { slot: string; timeoutMs: number; jsonMode: boolean; nowMs?: number; emit?: (l: string) => void; log?: (l: string) => void },
): Promise<{ dispatched: string[] }> {
const emit = opts.emit ?? ((line) => process.stderr.write(line + '\n'));
const log = opts.log ?? ((line) => console.log(line));
const nowMs = opts.nowMs ?? Date.now();
const { connectorProviderNames } = await import('../core/connectors/registry.ts');
const { loadCredential } = await import('../core/connectors/credentials.ts');
const {
autoSyncKey,
authErrorAtKey,
readConnectorState,
syncFloorMinKey,
sourceIdKey,
isTruthy,
isConnectorSyncStale,
DEFAULT_SYNC_FLOOR_MIN,
} = await import('../core/connectors/config-keys.ts');
let floorMin = DEFAULT_SYNC_FLOOR_MIN;
const floorCfg = await engine.getConfig(syncFloorMinKey());
if (floorCfg) {
const n = parseInt(floorCfg, 10);
if (Number.isFinite(n) && n >= 1) floorMin = n;
}
const sourceId = (await engine.getConfig(sourceIdKey())) || 'default';
const dispatched: string[] = [];
for (const provider of connectorProviderNames()) {
const cred = loadCredential(provider);
if (!cred) continue; // no credential → nothing to sync
if (!isTruthy(await engine.getConfig(autoSyncKey(provider)))) continue; // opt-in only
// A dead cookie stamps auth_error_at; skip until the user re-auths (a fresh
// credential's savedAt will be newer than the stale error).
const authErrorAt = await engine.getConfig(authErrorAtKey(provider));
if (authErrorAt && cred.savedAt && authErrorAt > cred.savedAt) continue;
const lastSyncAt = await readConnectorState(engine, provider, sourceId, 'last_sync_at');
if (!isConnectorSyncStale(lastSyncAt, nowMs, floorMin)) continue;
const job = await queue.add(
'connector-sync',
{ provider, sourceId },
{
queue: 'default',
idempotency_key: `connector-sync:${provider}:${opts.slot}`,
max_attempts: 2,
timeout_ms: opts.timeoutMs,
maxPending: 1,
},
);
if (job.coalesced) {
if (!opts.jsonMode) log(`[dispatch] coalesced connector-sync ${provider} (already in flight)`);
continue;
}
dispatched.push(provider);
if (opts.jsonMode) {
emit(JSON.stringify({ event: 'dispatched', job_id: job.id, mode: 'connector_sync', provider, slot: opts.slot }));
} else {
log(`[dispatch] job #${job.id} connector-sync ${provider}`);
}
}
return { dispatched };
}