Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

32.6 KB
Raw Blame
/**
 * Proxy Health Check Scheduler
 *
 * Periodically tests all proxy registry entries and automatically
 * removes (or soft-disables) proxies that have been failing consecutively.
 *
 * Config via environment:
 *   PROXY_HEALTH_INTERVAL_MS  — sweep interval (default: 600000 = 10min)
 *   PROXY_HEALTH_RELAY_INTERVAL_MS — probe interval for edge-relay rows (deno / vercel /
 *                               cloudflare), whose probes wake a billed serverless isolate
 *                               (default: 21600000 = 6h, minimum 60000; see relayCadence.ts)
 *   PROXY_HEALTH_ENABLED      — set "false" to disable
 *   PROXY_AUTO_REMOVE         — set "true" to auto-remove dead proxies (destructive)
 *   PROXY_AUTO_DISABLE        — set "true" to auto-disable dead proxies instead of
 *                               deleting them (status → "dead", already excluded from
 *                               pool/rotation resolution by PROXY_ALIVE_PREDICATE). The
 *                               row is never deleted, and the same recovery check that
 *                               re-activates proxies for PROXY_AUTO_REMOVE flips it back
 *                               to "active" once it starts answering probes again — no
 *                               manual re-add needed. If both flags are set, auto-remove
 *                               wins (see decision.ts).
 *   PROXY_AUTO_REMOVE_AFTER   — consecutive failures before the action above fires
 *                               (default: 3). Shared by both PROXY_AUTO_REMOVE and
 *                               PROXY_AUTO_DISABLE — they are alternative actions at the
 *                               same threshold, not independently tunable.
 */

import { deleteProxyById, listProxies, updateProxy } from "@/lib/db/proxies";
import { exportProxyLogsSince } from "@/lib/db/proxyLogs";
import { isProxyLogIncludeIps } from "@/lib/proxyLogger";
import {
  getRecentEgressSharingSummary,
  type EgressSharingSummary,
  type EgressSharingWarning,
} from "@/lib/proxyEgress";
import {
  createProxyDispatcher,
  clearDispatcherCache,
  proxyConfigToUrl,
} from "@omniroute/open-sse/utils/proxyDispatcher";
import { fetch as undiciFetch } from "undici";
import {
  applyCrossProbeEvidence,
  buildTargetEvidenceMap,
  classifyProbeError,
  classifyProbeStatus,
  decideProxyHealthAction,
  type ProxyProbeOutcome,
} from "./decision.ts";
import {
  MAX_PROBE_CONCURRENCY,
  resolveProbeConcurrency,
  resolveProbeStaggerMs,
  resolveProbeTarget,
  waitForProbeSlot,
} from "./probeTarget.ts";
import { selectRelayIdsToSkip } from "./relayCadence.ts";
import { resolveProviderProbeTarget } from "./providerProbeTarget.ts";
import {
  noteProxyRecovered,
  noteProxyRefusal,
  proxyEgressKey,
  type ProxyRefusalKind,
} from "@omniroute/open-sse/utils/proxyRefusalMemory";
import { deleteSweepVerdict, recordSweepVerdict, toSweepVerdict } from "./sweepVerdict.ts";
import {
  deleteBlockedHistory,
  pruneBlockedHistory,
  recordBlockedObservation,
} from "./blockedHistory.ts";
import {
  isProxyHealthBlockedResetsStreakEnabled,
  isProxySkipRecentlyFailedEnabled,
} from "@/shared/utils/featureFlags";
import {
  hasProvenSuccessWithoutAttributedFailure,
  isPassiveSweepSkipEnabled,
  passiveVerdictKey,
  resolvePassiveWindowMs,
  type PassiveLogRow,
} from "./passiveVerdict.ts";

// #6246: a HEAD to the public probe target through a legit (often loaded) proxy
// can exceed a few seconds; the old 5s ceiling produced false negatives that
// flipped healthy proxies to inactive. Raise it and treat our own timeout as
// inconclusive (see testOneProxy) rather than a proxy failure.
const TEST_TIMEOUT_MS = 15000;
// Probe target, batch size and intra-batch spacing come from probeTarget.ts, which the
// auto-test endpoint reads too — one surface to tune instead of two that can drift apart.
// Resolved at module load, as these constants always were.
const TEST_URL = resolveProbeTarget();
const CONCURRENCY = resolveProbeConcurrency();
const STAGGER_MS = resolveProbeStaggerMs();
const INITIAL_DELAY_MS = 60_000;
const DEFAULT_INTERVAL_MS = 600_000;
const DEFAULT_REMOVE_AFTER = 3;
// Opt-in, off by default. When on, a bounded recovery pass re-tests
// proxies the sweep saw refused (429) before their set-aside expires.
const RECOVERY_MAX_CANDIDATES = 10;
const RECOVERY_DURATION_MS = 60_000;
const LOG_PREFIX = "[ProxyHealth]";

/** One entry eligible for a targeted recovery probe. */
export interface RecoveryCandidate {
  /** Egress key of the set-aside proxy; null (opaque entry) is never planned. */
  key: string | null;
  /** Refusal motive recorded for the key — only `ip_quota_429` is ever planned. */
  kind: ProxyRefusalKind;
  /** When the key was set aside (ms epoch); oldest first. */
  setAsideAt: number;
}

/**
 * A targeted recovery probe: given a set-aside egress key, re-test it.
 * Keyed by string (not by registry id): the ledger only holds egress keys.
 */
export type RecoveryProbe = (key: string) => Promise<ProxyProbeOutcome>;

export interface RecoveryProbeLimits {
  maxCandidates: number;
  durationMs: number;
  concurrency: number;
}

/**
 * PURE: pick the bounded set of targeted recovery probes. Motive-filtered,
 * not status-filtered: only `ip_quota_429` entries are planned, so a
 * `proxy_unreachable` entry is excluded even when a 429 was observed for the
 * same key. Null keys are never tested.
 */
export function planRecoveryProbes(
  candidates: RecoveryCandidate[],
  limits: RecoveryProbeLimits
): RecoveryCandidate[] {
  const max =
    Number.isFinite(limits.maxCandidates) && limits.maxCandidates > 0
      ? Math.floor(limits.maxCandidates)
      : 0;
  if (max === 0) return [];
  return candidates
    .filter((c) => c.key !== null && c.kind === "ip_quota_429")
    .sort((a, b) => a.setAsideAt - b.setAsideAt)
    .slice(0, max);
}

/**
 * Owned ledger of sweep-observed refusals. Written ONLY on a 429 the
 * sweep itself observed, with a non-null key — never on fail/hang, never for
 * `proxy_unreachable`. The hot path writes its own motives to the shared
 * memory; this ledger only ever holds `ip_quota_429` by construction, while
 * `planRecoveryProbes` keeps the kind filter as a defensive contract.
 */
const recoveryLedger = new Map<string, { setAsideAt: number }>();

function noteSweepRefusal(key: string | null, status: number): void {
  if (key === null || status !== 429) return;
  if (!recoveryLedger.has(key)) recoveryLedger.set(key, { setAsideAt: Date.now() });
}

/**
 * Answered-target evidence of the immediately previous sweep generation:
 * URL → true when at least one probe received any HTTP status. Replaced
 * wholesale at the end of every sweep (never merged) so a silent generation
 * drops stale proof — an absent URL is no proof (no ghost evidence, no clock).
 */
const priorAnsweredTargetEvidence = new Map<string, boolean>();
const MAX_TARGET_EVIDENCE_ENTRIES = 100;

/** Test-only: forget the previous-generation target evidence. */
export function __resetTargetEvidenceForTesting(): void {
  priorAnsweredTargetEvidence.clear();
}

interface CollectedProbe {
  id: string;
  proxy: { id: string };
  outcome: ProxyProbeOutcome;
  status: number | null;
  target: string | null;
}

/** Phase 1 of the sweep: probe every proxy in batches, decide nothing yet. */
async function collectProbeResults(
  proxies: Array<{ id: string }>,
  probe: (proxy: { id: string }) => Promise<CollectedProbe>
): Promise<CollectedProbe[]> {
  const collected: CollectedProbe[] = [];
  for (let i = 0; i < proxies.length; i += CONCURRENCY) {
    const batch = proxies.slice(i, i + CONCURRENCY);
    const results = await Promise.allSettled(
      batch.map((proxy, indexInBatch) => probeBatchMember(proxy, indexInBatch, probe))
    );
    for (const result of results) {
      if (result.status !== "fulfilled") continue;
      collected.push(result.value);
    }
  }
  return collected;
}

async function probeBatchMember(
  proxy: { id: string },
  indexInBatch: number,
  probe: (proxy: { id: string }) => Promise<CollectedProbe>
): Promise<CollectedProbe> {
  // Spread the departures: without this the whole batch leaves at the same tick and a
  // shared egress IP hits the target with CONCURRENCY simultaneous requests.
  await waitForProbeSlot(indexInBatch, STAGGER_MS);
  return probe(proxy);
}

/** Refresh the previous-generation evidence wholesale (never merged). */
function refreshTargetEvidence(collected: CollectedProbe[]): void {
  priorAnsweredTargetEvidence.clear();
  for (const [target, answered] of buildTargetEvidenceMap(collected)) {
    if (priorAnsweredTargetEvidence.size >= MAX_TARGET_EVIDENCE_ENTRIES) break;
    priorAnsweredTargetEvidence.set(target, answered);
  }
}

export interface SweepTally {
  tested: number;
  alive: number;
  inconclusive: number;
  blocked: number;
  hangs: number;
  removed: number;
  disabled: number;
  promoted: number;
}

/** Phase 2 of the sweep: apply cross-proxy evidence, then decide per proxy. */
async function decideCollectedResults(
  collected: CollectedProbe[],
  ctx: {
    failureMap: Map<string, number>;
    removeAfter: number;
    autoRemove: boolean;
    autoDisable: boolean;
    blockedResetsStreak: boolean;
  }
): Promise<SweepTally> {
  const tally: SweepTally = {
    tested: 0,
    alive: 0,
    inconclusive: 0,
    blocked: 0,
    hangs: 0,
    removed: 0,
    disabled: 0,
    promoted: 0,
  };
  const promoted = new Map<string, boolean>();
  const evidenced = applyCrossProbeEvidence(collected, priorAnsweredTargetEvidence);
  for (let i = 0; i < collected.length; i++) {
    if (evidenced[i].outcome !== collected[i].outcome) {
      promoted.set(collected[i].id, true);
      tally.promoted++;
    }
  }
  for (let i = 0; i < evidenced.length; i++) {
    await decideOneResult(
      collected[i],
      evidenced[i],
      promoted.get(collected[i].id) === true,
      ctx,
      tally
    );
  }
  return tally;
}

export interface CollectedProbeForTesting {
  id: string;
  proxy: { id: string };
  outcome: ProxyProbeOutcome;
  status: number | null;
  target: string | null;
}

/** Test-only: run one per-proxy decision with injected raw/final verdicts. */
export async function __decideOneResultForTesting(
  raw: CollectedProbeForTesting,
  final: CollectedProbeForTesting,
  wasPromoted: boolean,
  ctx: {
    failureMap: Map<string, number>;
    removeAfter: number;
    autoRemove: boolean;
    autoDisable: boolean;
    blockedResetsStreak: boolean;
  },
  tally: SweepTally
): Promise<void> {
  return decideOneResult(raw, final, wasPromoted, ctx, tally);
}

async function decideOneResult(
  raw: CollectedProbe,
  final: CollectedProbe,
  wasPromoted: boolean,
  ctx: {
    failureMap: Map<string, number>;
    removeAfter: number;
    autoRemove: boolean;
    autoDisable: boolean;
    blockedResetsStreak: boolean;
  },
  tally: SweepTally
): Promise<void> {
  const { id, proxy, outcome: rawOutcome } = raw;
  const { outcome, target } = final;
  tally.tested++;
  if (rawOutcome === "ok") tally.alive++;
  else if (rawOutcome === "inconclusive") tally.inconclusive++;
  else if (rawOutcome === "blocked") tally.blocked++;
  else if (rawOutcome === "hang") tally.hangs++;

  const decision = decideProxyHealthAction({
    outcome: rawOutcome,
    priorFailures: ctx.failureMap.get(id) ?? 0,
    autoRemove: ctx.autoRemove,
    autoDisable: ctx.autoDisable,
    removeAfter: ctx.removeAfter,
    blockedResetsStreak: ctx.blockedResetsStreak,
  });

  // Last sweep verdict (memory only, display): recorded for every probe
  // after the decision so the screen explains dead/blocked proxies without
  // triggering probes. Uses the final (possibly promoted) outcome, so a
  // promoted fail shows as fail. The cause stays a sidecar: never branched on.
  // The per-sweep tally is intentionally discarded after logging below — the
  // observable blocked history (blockedHistory.ts) is the persisted aggregate.
  const verdict = toSweepVerdict(outcome, final.status, Date.now());
  recordSweepVerdict(id, verdict);
  if (verdict.verdict === "blocked") {
    recordBlockedObservation(id, verdict.cause, verdict.status, verdict.at);
  } else if (verdict.verdict === "ok") {
    // A healthy sweep ends the refusal streak (see blockedHistory.ts lifecycle).
    deleteBlockedHistory(id);
  }

  if (decision.clearFailures) ctx.failureMap.delete(id);
  else ctx.failureMap.set(id, decision.failures);

  // #6246 (policy C) / auto-disable (policy D): only mutate the operator-owned
  // status when the decision explicitly asks for it. With both flags off,
  // setStatus is null, so a transient probe failure never flips a healthy
  // proxy's status.
  if (decision.setStatus) {
    await updateProxy(id, { status: decision.setStatus }).catch(() => {});
    if (decision.setStatus === "dead") tally.disabled++;
  }

  if (decision.remove) {
    if (await deleteProxyById(id, { force: true }).catch(() => false)) {
      ctx.failureMap.delete(id);
      deleteSweepVerdict(id);
      deleteBlockedHistory(id);
      tally.removed++;
      try {
        clearDispatcherCache();
      } catch {
        /* non-critical */
      }
    }
  }

  // Only a PROMOTED fail follows the existing set-aside path:
  // never a native fail (generic ECONNREFUSED, hang) and never an invalid
  // config (target null). Same gate as the hot path (proxyHealth.ts).
  if (wasPromoted && target !== null && outcome === "fail" && isProxySkipRecentlyFailedEnabled()) {
    noteProxyRefusal(proxyEgressKey(proxy), "proxy_unreachable");
  }
}

function isRecoveryEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
  return env.PROXY_HEALTH_RECOVERY_ENABLED === "true";
}

function resolveRecoveryLimits(env: NodeJS.ProcessEnv = process.env): RecoveryProbeLimits {
  return {
    maxCandidates: RECOVERY_MAX_CANDIDATES,
    durationMs: RECOVERY_DURATION_MS,
    concurrency: Math.min(resolveProbeConcurrency(env), MAX_PROBE_CONCURRENCY),
  };
}

declare global {
  var __proxyHealthInterval: ReturnType<typeof setInterval> | undefined;
  var __proxyHealthConsecutiveFailures: Map<string, number> | undefined;
  // Recovery timer handle for the opt-in pass (own timer, off by default).
  var __proxyHealthRecoveryInterval: ReturnType<typeof setInterval> | undefined;
}

function getFailureMap(): Map<string, number> {
  if (!globalThis.__proxyHealthConsecutiveFailures) {
    globalThis.__proxyHealthConsecutiveFailures = new Map();
  }
  return globalThis.__proxyHealthConsecutiveFailures;
}

/**
 * PURE: one-line anonymous egress-sharing summary for the sweep log (#10677).
 * Counts only by default; raw shared IPs only when PROXY_LOG_INCLUDE_IPS=true
 * (the redaction decision from #10348 — never leak IPs or account labels).
 */
export function formatEgressSharingSummaryLine(
  summary: EgressSharingSummary,
  warnings: EgressSharingWarning[],
  includeDetails: boolean
): string {
  const base =
    `${LOG_PREFIX} egress: ${summary.sharingByRotationGroup.length} rotation group(s) share an ` +
    `egress IP (max ${summary.maxAccountsSharingOneIp} accounts)`;
  if (!includeDetails) return base;
  const detail = warnings
    .map((w) => `${w.rotationGroup}: ${w.egressIp} (${w.connections.length} accounts)`)
    .join(", ");
  return detail ? `${base} — ${detail}` : base;
}

function isEnabled(): boolean {
  return process.env.PROXY_HEALTH_ENABLED !== "false";
}

function getIntervalMs(): number {
  const raw = parseInt(process.env.PROXY_HEALTH_INTERVAL_MS ?? "", 10);
  return Number.isFinite(raw) && raw >= 60_000 ? raw : DEFAULT_INTERVAL_MS;
}

function isAutoRemoveEnabled(): boolean {
  return process.env.PROXY_AUTO_REMOVE === "true";
}

function isAutoDisableEnabled(): boolean {
  return process.env.PROXY_AUTO_DISABLE === "true";
}

function getRemoveAfter(): number {
  const raw = parseInt(process.env.PROXY_AUTO_REMOVE_AFTER ?? "", 10);
  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_REMOVE_AFTER;
}

function isBuildProcess(): boolean {
  return typeof process !== "undefined" && process.env.NEXT_PHASE === "phase-production-build";
}

function isBackgroundServicesDisabled(): boolean {
  const raw = process.env.OMNIROUTE_DISABLE_BACKGROUND_SERVICES;
  if (!raw) return false;
  return ["1", "true", "yes", "on"].includes(raw.trim().toLowerCase());
}

/**
 * Reachability probe for one proxy, classified so the pure
 * decision layer can apply the #6246 policy:
 *   - "ok"           — the proxy relayed and the target served the request.
 *   - "blocked"      — the proxy relayed, but the TARGET refused this egress IP
 *                      (401/403/429). Neutral like "inconclusive" by default: the
 *                      proxy is not at fault, yet it is not serving that
 *                      destination. With PROXY_HEALTH_BLOCKED_RESETS_STREAK on it
 *                      also resets the consecutive-failure streak (never a status).
 *   - "inconclusive" — NOT the proxy's fault: our own timeout/abort, or the probe
 *                      TARGET returned a 5xx (the proxy connected fine). Never
 *                      penalizes the proxy.
 *   - "fail"         — a proxy-level connection error (refused/unreachable/TLS).
 *   - "hang"         — the host resolved but the handshake never completed before
 *                      our own deadline. Counted exactly like "fail" in the
 *                      decision layer, but observed separately in the sweep log.
 */
export interface ProxyProbeResult {
  outcome: ProxyProbeOutcome;
  /** HTTP status when the target answered; null on connection-level errors. */
  status: number | null;
  /** URL actually probed (generic or provider-resolved); null when config invalid. */
  target: string | null;
}

// Error codes that prove DNS never resolved the proxy host: not a hang (we
// never reached the host), a frank failure of name resolution.
const UNRESOLVED_DNS_CODES = new Set(["ENOTFOUND", "EAI_AGAIN"]);

function probeErrorCode(error: unknown): string | null {
  if (error && typeof error === "object" && "code" in error) {
    const code = (error as { code?: unknown }).code;
    return typeof code === "string" ? code : null;
  }
  return null;
}

async function testOneProxy(proxy: {
  id: string;
  type: string;
  host: string;
  port: number;
  username?: string;
  password?: string;
  family?: string;
}): Promise<ProxyProbeResult> {
  let proxyUrl: string | null;
  try {
    proxyUrl = proxyConfigToUrl(proxy);
  } catch {
    proxyUrl = null;
  }
  if (!proxyUrl) return { outcome: "fail", status: null, target: null };
  // A provider's models endpoint is a real GET-only API surface, unlike httpbin.org/ip: many
  // reject HEAD outright. HEAD stays the default for the generic target — this changes nothing
  // for a proxy with no eligible provider assignment.
  const providerTarget = await resolveProviderProbeTarget(proxy.id);
  const target = providerTarget ?? TEST_URL;
  const method = providerTarget ? "GET" : "HEAD";
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
  try {
    const dispatcher = createProxyDispatcher(proxyUrl);
    const resp = await undiciFetch(target, {
      method,
      signal: controller.signal,
      dispatcher,
      headers: { "User-Agent": "OmniRoute/1.0" },
    });
    return { outcome: classifyProbeStatus(resp.status), status: resp.status, target };
  } catch (error) {
    // Our own deadline elapsed → hang when we had resolved the host but the
    // handshake never completed; otherwise inconclusive (slow, not dead).
    if (controller.signal.aborted) {
      const code = probeErrorCode(error);
      const verdict = classifyProbeError({
        aborted: true,
        resolved: code === null || !UNRESOLVED_DNS_CODES.has(code),
        tlsNegotiated: false,
      });
      // A provider-resolved target's connection health is not proven the way the
      // operator-configured generic target is — keep today's inconclusive there.
      if (providerTarget) return { outcome: "inconclusive", status: null, target };
      return { outcome: verdict, status: null, target };
    }
    // A provider-resolved target's connection health is not proven the way the
    // operator-configured generic target is: a registry baseUrl can be a placeholder that
    // never resolves for anyone (e.g. databricks's default azuredatabricks.net host is
    // literally 16 zeros). A connection failure there says nothing about this proxy —
    // same principle as the 5xx case above, extended to connection-level errors.
    return providerTarget
      ? { outcome: "inconclusive", status: null, target }
      : { outcome: "fail", status: null, target };
  } finally {
    clearTimeout(timeout);
  }
}

/**
 * Passive skip proof from production traffic, grouped by proxy endpoint.
 * Filled from the persisted request log once per sweep: an endpoint whose
 * window shows recent single-provider success and no attributed failure
 * skips the redundant live probe. A recent failure never skips: it keeps
 * the probe that alone can confirm it.
 */
interface EndpointVerdict {
  skip: boolean;
  /** Distinct providers observed for the endpoint; skips need exactly one. */
  providers: string[];
}

function readPassiveVerdicts(
  proxies: Array<{ host: string; port: number }>
): Map<string, EndpointVerdict> {
  const verdicts = new Map<string, EndpointVerdict>();
  const windowMs = resolvePassiveWindowMs();
  const now = Date.now();
  const since = new Date(now - windowMs).toISOString();
  let rows: Record<string, unknown>[] = [];
  try {
    // Bounded by the short window via idx_pl_timestamp; never let a DB
    // hiccup fail the sweep — fall back to live probes (no skip anywhere).
    rows = exportProxyLogsSince(since);
  } catch (error) {
    console.error(`${LOG_PREFIX} Passive verdicts skipped:`, error);
    return verdicts;
  }
  const endpoints = new Set(proxies.map((p) => passiveVerdictKey(p.host, p.port)));
  const byEndpoint = new Map<string, PassiveLogRow[]>();
  for (const row of rows) {
    const host = typeof row.proxy_host === "string" ? row.proxy_host : null;
    const port = typeof row.proxy_port === "number" ? row.proxy_port : null;
    if (!host || !port || !endpoints.has(passiveVerdictKey(host, port))) continue;
    const key = passiveVerdictKey(host, port);
    const list = byEndpoint.get(key) ?? [];
    list.push(row as PassiveLogRow);
    byEndpoint.set(key, list);
  }
  for (const [key, keyedRows] of byEndpoint) {
    const providers = [...new Set(keyedRows.map((r) => r.provider).filter((p) => p))];
    const skip = hasProvenSuccessWithoutAttributedFailure(keyedRows) && providers.length === 1;
    verdicts.set(key, { skip, providers: providers as string[] });
  }
  return verdicts;
}

/** Test-only: skip lookup seam for sweep-level tests. */
export type PassiveVerdictReader = (proxyId: string) => { skip: boolean; providers: string[] };

let passiveReader: PassiveVerdictReader | null = null;

/** Test-only: override the skip lookup. */
export function __setPassiveVerdictReaderForTesting(fn: PassiveVerdictReader | null): void {
  passiveReader = fn;
}

/**
 * Passive skip check for one proxy: recent production traffic already proves
 * this endpoint, so the live probe is redundant — but only when the window
 * shows a single provider (a success for one provider never skips the probe
 * another provider would need). A recent failure never skips: the live probe
 * stays the one that can confirm it and feed auto-disable/auto-remove.
 */
function resolvePassiveSkip(
  proxy: { id: string; host: string; port: number },
  passiveVerdicts: Map<string, EndpointVerdict>
): boolean {
  const endpointVerdict = passiveReader
    ? passiveReader(proxy.id)
    : (passiveVerdicts.get(`${proxy.host}:${proxy.port}`) ?? {
        skip: false,
        providers: [] as string[],
      });
  return endpointVerdict.skip && isSingleProviderSkip(endpointVerdict);
}

/** A proven endpoint skips the probe only with exactly one observed provider. */
function isSingleProviderSkip(endpointVerdict: EndpointVerdict): boolean {
  return endpointVerdict.providers.length === 1;
}

async function sweep(): Promise<void> {
  // #10677: anonymous egress-sharing signal from persisted proxy_logs (no live
  // probes). Logged only when sharing exists — the sweep line is a warning
  // signal, not a heartbeat. Runs before the empty-registry early return so
  // sharing from direct connections is still reported when no proxies are
  // configured. Never let a DB hiccup suppress the completion line or fail the
  // sweep itself.
  try {
    const { summary, warnings } = await getRecentEgressSharingSummary();
    if (summary.sharingByRotationGroup.length > 0) {
      console.log(formatEgressSharingSummaryLine(summary, warnings, isProxyLogIncludeIps()));
    }
  } catch (error) {
    console.error(`${LOG_PREFIX} Egress summary skipped:`, error);
  }

  const { items: proxies } = await listProxies({ includeSecrets: true });
  // Before the empty-registry return, so deleting the last proxy prunes too.
  pruneBlockedHistory(proxies.map((proxy) => proxy.id));
  if (proxies.length === 0) return;

  // Passive verdicts from production traffic (opt-in): an endpoint already
  // judged by recent traffic skips the live probe. Scheduling only — skips
  // are partitioned before the probe phase, so they never touch the failure
  // streak, the cross-proxy evidence, the promotion tally, or the sweep
  // verdict display.
  const passiveEnabled = isPassiveSweepSkipEnabled();
  const passiveVerdicts = passiveEnabled ? readPassiveVerdicts(proxies) : new Map();
  const failureMap = getFailureMap();
  const removeAfter = getRemoveAfter();
  const autoRemove = isAutoRemoveEnabled();
  const autoDisable = isAutoDisableEnabled();
  const blockedResetsStreak = isProxyHealthBlockedResetsStreakEnabled();

  // Phase 1 — collect raw probe results across all batches WITHOUT deciding
  // (cross-proxy evidence requires every response of the target first).
  // Passive skips are partitioned before collecting: they never enter the
  // batching, the failure streak, the cross-proxy evidence, the promotion
  // tally, or the sweep verdict display — they are only counted apart below.
  const skippedIds = new Set<string>();
  if (passiveEnabled) {
    for (const proxy of proxies) {
      if (resolvePassiveSkip(proxy, passiveVerdicts)) skippedIds.add(proxy.id);
    }
  }
  // #14984: edge-relay rows are probed on their own long cadence, not every sweep.
  const relaySkipped = selectRelayIdsToSkip(proxies);
  const toProbe = proxies.filter(
    (proxy) => !skippedIds.has(proxy.id) && !relaySkipped.has(proxy.id)
  );
  const passiveSkipped = skippedIds.size;
  const collected = await collectProbeResults(toProbe, async (proxy) => {
    const { outcome, status, target } = await testOneProxy(proxy);
    // Ledger: only a sweep-observed 429 with a usable key is recorded.
    // No memory write happens on fail/hang here, except a promoted
    // fail in phase 2 (`proxy_unreachable` under the opt-in flag).
    if (outcome === "blocked" && status === 429) {
      noteSweepRefusal(proxyEgressKey(proxy), status);
    }
    return { id: proxy.id, proxy, outcome, status, target };
  });

  // Only live probes feed the decision phase and the evidence: skipped
  // endpoints contribute nothing. Previous-generation evidence is replaced
  // wholesale (never merged): the current sweep's answered targets become
  // generation N-1 for the next sweep.
  const { tested, alive, inconclusive, blocked, hangs, removed, disabled, promoted } =
    await decideCollectedResults(collected, {
      failureMap,
      removeAfter,
      autoRemove,
      autoDisable,
      blockedResetsStreak,
    });
  refreshTargetEvidence(collected);

  console.log(
    `${LOG_PREFIX} Sweep complete: ${tested} tested, ${alive} alive, ` +
      `${blocked} refused by target, ${inconclusive} inconclusive, ${promoted} promoted, ` +
      `${removed} auto-removed, ${disabled} auto-disabled` +
      `${passiveSkipped > 0 ? `, ${passiveSkipped} passive-skipped (recent success)` : ""}`
  );
  if (hangs > 0) {
    console.debug(`${LOG_PREFIX} stalled handshakes observed: ${hangs}`);
  }
}

/**
 * Targeted recovery pass: re-test set-aside proxies before their
 * period expires, off the hot path. A success ends the set-aside now (the
 * streak is kept, so a repeat still doubles); a refusal re-records it, and
 * the existing doubling provides the longer repeat period. Never writes a
 * proxy status, never redirects traffic — only the set-aside memory changes.
 */
export async function runRecoveryPass(
  probe?: RecoveryProbe,
  candidates?: RecoveryCandidate[],
  limits?: RecoveryProbeLimits,
  env?: NodeJS.ProcessEnv
): Promise<{ planned: number; recovered: number; refused: number }> {
  const empty = { planned: 0, recovered: 0, refused: 0 };
  if (!isRecoveryEnabled(env)) return empty;
  const active = limits ?? resolveRecoveryLimits(env);
  const entries =
    candidates ??
    [...recoveryLedger.entries()].map(([key, { setAsideAt }]) => ({
      key,
      kind: "ip_quota_429" as const,
      setAsideAt,
    }));
  const planned = planRecoveryProbes(entries, active);
  if (planned.length === 0) return empty;
  const concurrency = Math.max(
    1,
    Math.min(active.concurrency, MAX_PROBE_CONCURRENCY, planned.length)
  );
  const deadline = Date.now() + active.durationMs;
  let recovered = 0;
  let refused = 0;
  const runProbe = probe ?? probeLedgerKey;
  for (let i = 0; i < planned.length; i += concurrency) {
    if (Date.now() >= deadline) break;
    const batch = planned.slice(i, i + concurrency);
    const results = await Promise.allSettled(
      batch.map(async (candidate, indexInBatch) => {
        await waitForProbeSlot(indexInBatch, STAGGER_MS);
        const outcome = await runProbe(candidate.key as string);
        return { candidate, outcome };
      })
    );
    for (const result of results) {
      if (result.status !== "fulfilled") continue;
      const { candidate, outcome } = result.value;
      const key = candidate.key as string;
      recoveryLedger.delete(key);
      if (outcome === "ok") {
        noteProxyRecovered(key, "ip_quota_429");
        recovered++;
      } else if (outcome === "blocked" || outcome === "fail" || outcome === "hang") {
        noteProxyRefusal(key, "ip_quota_429");
        refused++;
      }
    }
  }
  return { planned: planned.length, recovered, refused };
}

/**
 * Default recovery probe for a ledger key: resolve the registry entry behind
 * the egress key and run the same reachability probe as the sweep. Keys with
 * no registry entry resolve to `inconclusive` (opaque entry, never tested).
 */
export async function probeLedgerKey(key: string): Promise<ProxyProbeOutcome> {
  try {
    const { items } = await listProxies({ includeSecrets: true });
    const entry = items.find((p) => proxyEgressKey(p) === key);
    if (!entry) return "inconclusive";
    const { outcome } = await testOneProxy(entry);
    return outcome;
  } catch {
    return "inconclusive";
  }
}

/** Test-only: clear the owned sweep-refusal ledger. */
export function __resetRecoveryLedgerForTesting(): void {
  recoveryLedger.clear();
}

/** Test-only: number of ledger entries held. */
export function __recoveryLedgerSizeForTesting(): number {
  return recoveryLedger.size;
}

function scheduleSweep(): void {
  const interval = getIntervalMs();
  globalThis.__proxyHealthInterval = setInterval(() => {
    void sweep().catch((err) => {
      console.error(`${LOG_PREFIX} Sweep error:`, err);
    });
  }, interval);
}

function getRecoveryIntervalMs(): number {
  const raw = parseInt(process.env.PROXY_HEALTH_RECOVERY_INTERVAL_MS ?? "", 10);
  return Number.isFinite(raw) && raw >= 60_000 ? raw : DEFAULT_INTERVAL_MS;
}

function scheduleRecoveryPass(): void {
  const interval = getRecoveryIntervalMs();
  globalThis.__proxyHealthRecoveryInterval = setInterval(() => {
    void runRecoveryPass().catch((err) => {
      console.error(`${LOG_PREFIX} Recovery error:`, err);
    });
  }, interval);
}

export function initProxyHealthCheck(): void {
  if (!isEnabled() || isBuildProcess() || isBackgroundServicesDisabled()) return;
  if (globalThis.__proxyHealthInterval) return;

  setTimeout(() => {
    console.log(`${LOG_PREFIX} Starting proxy health scheduler (interval: ${getIntervalMs()}ms)`);
    void sweep().catch(() => {});
    scheduleSweep();
    // The recovery pass runs on its own timer, off by default. The gate
    // lives inside runRecoveryPass, so the timer is cheap when disabled.
    if (!globalThis.__proxyHealthRecoveryInterval) scheduleRecoveryPass();
  }, INITIAL_DELAY_MS);
}

export function stopProxyHealthCheck(): void {
  if (globalThis.__proxyHealthInterval) {
    clearInterval(globalThis.__proxyHealthInterval);
    globalThis.__proxyHealthInterval = undefined;
  }
  if (globalThis.__proxyHealthRecoveryInterval) {
    clearInterval(globalThis.__proxyHealthRecoveryInterval);
    globalThis.__proxyHealthRecoveryInterval = undefined;
  }
}

export async function forceProxyHealthSweep(): Promise<void> {
  await sweep();
}

// Auto-initialize on first import
initProxyHealthCheck();