Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

30.4 KB
Raw Blame
/**
 * opencodeEgressThrottle.ts — per-egress pacing plus fleet-wide backoff for the
 * opencode multi-account rotation.
 *
 * Pure store, same pattern as proxyRefusalMemory: no flag reads at the hot path
 * (the caller resolves config once and injects it), injectable clock (`nowMs`)
 * and RNG (`rand`) so unit tests drive time deterministically, test-only
 * reset/size helpers. Zero business imports — no breaker, no cooldown, no
 * lockout, no featureFlags (activation is an env var local to this module,
 * default off, same truthy motif as the opencodeHeaders falsy guard
 * (opencodeHeaders.ts:61, negative form `/^(0|false|no|off)$/`).
 *
 * Fail-open everywhere: a saturated queue, an elapsed wait budget, a disabled
 * module or an aborted wait resolves `null` — the caller proceeds WITHOUT a
 * slot rather than ever rejecting a dispatchable request.
 */

import { sleepAbortable } from "./opencodeTransientFailure.ts";
import { classifyUpstream429, type RateLimit429Verdict } from "./opencodeRateLimited.ts";
import { proxyKeyOf } from "./opencodeGeoBlock.ts";
import { pickAccount, type RotatableAccount } from "./accountRotation.ts";
import {
  hasSlowOverrunEvidence,
  noteProxyRefusal,
  proxyEgressKey,
  recordSlowOverrun,
  recordSlowServe,
  transportDestinationKey,
  type ProxyRefusalKind,
} from "../utils/proxyRefusalMemory.ts";
import { stripIpv6Brackets } from "../utils/proxyFamily.ts";

export const DIRECT_EGRESS_SENTINEL = "direct";

/** Upper bound on tracked egress keys (active egress keys stay far below refused-pair counts). */
export const MAX_EGRESS_KEYS = 512;

export interface EgressThrottleConfig {
  enabled: boolean;
  cap: number;
  waitMinMs: number;
  waitMaxMs: number;
  waitBudgetMs: number;
  fleetWindowMs: number;
  fleetThreshold: number;
  suspectMinMs: number;
  suspectMaxMs: number;
  suspectSlots: number;
}

export const EGRESS_THROTTLE_DEFAULTS: EgressThrottleConfig = {
  enabled: false,
  cap: 2,
  waitMinMs: 5000,
  waitMaxMs: 15000,
  waitBudgetMs: 30000,
  fleetWindowMs: 60000,
  fleetThreshold: 10,
  suspectMinMs: 60000,
  suspectMaxMs: 120000,
  suspectSlots: 2,
};

function envFlagEnabled(raw: string | undefined): boolean {
  return raw != null && /^(1|true|yes|on)$/i.test(raw.trim());
}

function envNum(raw: string | undefined, fallback: number, min: number, max: number): number {
  if (raw == null || raw.trim() === "") return fallback;
  const n = Number(raw);
  if (!Number.isFinite(n) || n < min || n > max) return fallback;
  return n;
}

/**
 * Resolve config from the environment (never throws: unreadable values fall
 * back to defaults, off). Pass an explicit `env` in tests; defaults to
 * `process.env`.
 */
export function resolveEgressThrottleConfig(
  env: NodeJS.ProcessEnv = process.env
): EgressThrottleConfig {
  const d = EGRESS_THROTTLE_DEFAULTS;
  const waitMinMs = envNum(env.OPENCODE_EGRESS_THROTTLE_WAIT_MIN_MS, d.waitMinMs, 0, 120_000);
  return {
    enabled: envFlagEnabled(env.OPENCODE_EGRESS_THROTTLE_ENABLED),
    cap: Math.floor(envNum(env.OPENCODE_EGRESS_THROTTLE_CAP, d.cap, 1, 64)),
    waitMinMs,
    waitMaxMs: Math.max(
      waitMinMs,
      envNum(env.OPENCODE_EGRESS_THROTTLE_WAIT_MAX_MS, d.waitMaxMs, 0, 120_000)
    ),
    waitBudgetMs: envNum(env.OPENCODE_EGRESS_THROTTLE_WAIT_BUDGET_MS, d.waitBudgetMs, 0, 300_000),
    fleetWindowMs: envNum(
      env.OPENCODE_EGRESS_THROTTLE_FLEET_WINDOW_MS,
      d.fleetWindowMs,
      1_000,
      600_000
    ),
    fleetThreshold: Math.floor(
      envNum(env.OPENCODE_EGRESS_THROTTLE_FLEET_THRESHOLD, d.fleetThreshold, 1, 10_000)
    ),
    suspectMinMs: envNum(env.OPENCODE_EGRESS_THROTTLE_SUSPECT_MIN_MS, d.suspectMinMs, 0, 600_000),
    suspectMaxMs: Math.max(
      envNum(env.OPENCODE_EGRESS_THROTTLE_SUSPECT_MIN_MS, d.suspectMinMs, 0, 600_000),
      envNum(env.OPENCODE_EGRESS_THROTTLE_SUSPECT_MAX_MS, d.suspectMaxMs, 0, 600_000)
    ),
    suspectSlots: Math.floor(
      envNum(env.OPENCODE_EGRESS_THROTTLE_SUSPECT_SLOTS, d.suspectSlots, 1, 64)
    ),
  };
}

/** Egress key for an account proxy; `direct` accounts share one sentinel (shared egress). */
export function egressKeyOf(proxy: { host: string; port: number } | null): string {
  return proxyEgressKey(proxy) ?? DIRECT_EGRESS_SENTINEL;
}

/** Port written in the authority, which `new URL()` drops at the scheme default. */
function explicitEgressPortOf(url: string): string | null {
  const start = url.indexOf("://");
  if (start === -1) return null;
  const rest = url.slice(start + 3);
  const slash = rest.indexOf("/");
  const authority = slash === -1 ? rest : rest.slice(0, slash);
  const colon = authority.lastIndexOf(":");
  if (colon === -1 || colon < authority.lastIndexOf("@") || colon < authority.lastIndexOf("]")) {
    return null;
  }
  const port = authority.slice(colon + 1);
  if (!/^\d+$/.test(port)) return null;
  const n = Number(port);
  return n >= 1 && n <= 65535 ? String(n) : null;
}

/**
 * Log label for the egress really applied to one attempt: `(proxy <host>:<port>)`
 * with host and port only, `(proxy direct)` when no proxy applies. Parses the
 * applied key directly (it already holds `scheme://`); only the extracted
 * hostname and port are ever printed, so member keys carrying user info cannot
 * leak. Fail-open: any unusable key reports direct, never throws.
 */
export function egressLabel(
  account: AppliedEgressAccount,
  readApplied?: AppliedEgressReader | null
): string {
  let key: string;
  try {
    key = resolveAppliedEgressKey(account, readApplied);
  } catch {
    return "(proxy direct)";
  }
  if (!key || key === DIRECT_EGRESS_SENTINEL) return "(proxy direct)";
  try {
    const bare = key.replace(/\?family=(ipv4|ipv6)$/, "");
    const parsed = new URL(bare);
    const port = explicitEgressPortOf(bare) || parsed.port || null;
    const host = stripIpv6Brackets(parsed.hostname).toLowerCase();
    if (!host || !port || !/^\d+$/.test(port)) return "(proxy direct)";
    return `(proxy ${host}:${port})`;
  } catch {
    return "(proxy direct)";
  }
}

// ---------------------------------------------------------------------------
// Applied egress key: pool-served accounts without their own proxy share the
// `direct` sentinel above, although the request actually leaves through the
// ambient pool proxy. The reader below resolves the key really applied to one
// attempt; the default (no reader) keeps the historical behavior byte-identical.
// ---------------------------------------------------------------------------

/** Minimal account shape the applied-key seam needs (proxy + optional print). */
export interface AppliedEgressAccount {
  proxy: { host: string; port: number } | null;
  fingerprint?: string;
}

/** Returns the normalized egress key applied to this account, or null. */
export type AppliedEgressReader = (account: AppliedEgressAccount) => string | null;

/**
 * Key really applied to one attempt: dedicated proxies take the fast path
 * (the reader is never called), proxyless accounts consult the reader and
 * fall back to the shared sentinel (fail-open: absent/error/empty/throw).
 */
export function resolveAppliedEgressKey(
  account: AppliedEgressAccount,
  readApplied?: AppliedEgressReader | null
): string {
  if (account.proxy !== null) return egressKeyOf(account.proxy);
  try {
    const applied = readApplied?.(account);
    if (applied && applied !== DIRECT_EGRESS_SENTINEL) return applied;
  } catch {
    /* best-effort: never break the request path */
  }
  return DIRECT_EGRESS_SENTINEL;
}

// ---------------------------------------------------------------------------
// Attempt-scoped applied-key tracker: pool-served accounts without their own
// proxy share one ambient context, so the per-fingerprint history below is the
// only structure telling member A from member B apart.
// ---------------------------------------------------------------------------

/** Resolver shape of `resolveProxyForRequest` (injected — no import cycle). */
export type AppliedProxyResolver = (targetUrl: string) => unknown;

/** Attempt-scoped reader + member key + per-iteration memo control. */
export interface AppliedEgressTracker {
  readAppliedKey: AppliedEgressReader;
  keyOfMember: (account: AppliedEgressAccount) => string | null;
  destinationOfRequest: () => string | null;
  resetAttempt: () => void;
  rememberServed: (account: AppliedEgressAccount) => void;
  noteRefused: (
    account: AppliedEgressAccount,
    skipRecentlyFailed: boolean,
    kind?: ProxyRefusalKind
  ) => number | null;
  noteServed: <T>(account: AppliedEgressAccount, result: T, waitMs: number, enabled: boolean) => T;
}

/**
 * Track the egress key really applied to each attempt of one request. The
 * ambient memo resets at the top of every iteration; served keys persist per
 * fingerprint. Non-sentinel keys only — the sentinel means "no pool context".
 */
export function createAppliedEgressTracker(
  dispatchUrl: string,
  resolveProxy: AppliedProxyResolver
): AppliedEgressTracker {
  const byFingerprint = new Map<string, string>();
  let attemptAmbientKey: string | null | undefined;
  const resolveAmbientKey = (): string | null => {
    let resolved: unknown;
    try {
      resolved = resolveProxy(dispatchUrl);
    } catch {
      return null;
    }
    if (!resolved || typeof resolved !== "object") return null;
    const { source, proxyUrl } = resolved as { source?: unknown; proxyUrl?: unknown };
    if (source !== "context" || typeof proxyUrl !== "string") return null;
    return proxyEgressKey(proxyUrl);
  };
  const readAppliedKey: AppliedEgressReader = (a) => {
    if (a.proxy !== null) return null;
    if (typeof a.fingerprint === "string") {
      const known = byFingerprint.get(a.fingerprint);
      if (known) return known;
    }
    if (attemptAmbientKey === undefined) attemptAmbientKey = resolveAmbientKey();
    return attemptAmbientKey;
  };
  const noteRefused = (
    a: AppliedEgressAccount,
    skipRecentlyFailed: boolean,
    kind: ProxyRefusalKind = "ip_quota_429"
  ) => noteRefusedMember(a.proxy, skipRecentlyFailed, readAppliedKey, a.fingerprint, kind);
  const destinationOfRequest = (): string | null => transportDestinationKey(dispatchUrl);
  const noteServed = <T>(
    a: AppliedEgressAccount,
    result: T,
    waitMs: number,
    enabled: boolean
  ): T => {
    try {
      if (!enabled || !(waitMs > 0)) return result;
      const key = resolveAppliedEgressKey(a, readAppliedKey);
      if (key === DIRECT_EGRESS_SENTINEL) return result;
      const destination = destinationOfRequest();
      if (destination === null) return result;
      recordSlowServe(key, destination);
    } catch {
      /* evidence is best-effort; never break the request path */
    }
    return result;
  };
  return {
    readAppliedKey,
    keyOfMember: (a) => (a.proxy !== null ? proxyEgressKey(a.proxy) : readAppliedKey(a)),
    destinationOfRequest,
    resetAttempt: () => {
      attemptAmbientKey = undefined;
    },
    rememberServed: (a) => {
      if (a.proxy === null && typeof a.fingerprint === "string") {
        const key = readAppliedKey(a);
        if (key && key !== DIRECT_EGRESS_SENTINEL) byFingerprint.set(a.fingerprint, key);
      }
    },
    noteRefused,
    noteServed,
  };
}

// ---------------------------------------------------------------------------
// Per-egress semaphore
// ---------------------------------------------------------------------------

interface Waiter {
  startWait: (slot: () => void) => void;
  abort: () => void;
  enqueuedAt: number;
}

interface EgressGate {
  running: number;
  queue: Waiter[];
  lastUsed: number;
}

const gates = new Map<string, EgressGate>();

function getGate(key: string, nowMs: number): EgressGate {
  let gate = gates.get(key);
  if (!gate) {
    gate = { running: 0, queue: [], lastUsed: nowMs };
    gates.set(key, gate);
    if (gates.size > MAX_EGRESS_KEYS) {
      // Evict the stalest idle gate (never one with running slots or waiters).
      let oldestKey: string | undefined;
      let oldestUsed = Infinity;
      for (const [k, g] of gates) {
        if (k !== key && g.running === 0 && g.queue.length === 0 && g.lastUsed < oldestUsed) {
          oldestUsed = g.lastUsed;
          oldestKey = k;
        }
      }
      if (oldestKey !== undefined) gates.delete(oldestKey);
      else gates.delete(gates.keys().next().value as string);
    }
  }
  gate.lastUsed = nowMs;
  return gate;
}

function createReleaseFn(key: string): () => void {
  let released = false;
  return () => {
    if (released) return;
    released = true;
    const gate = gates.get(key);
    if (!gate || gate.running <= 0) return;
    gate.running--;
    gate.lastUsed = Date.now();
    // Drain FIFO while slots are free.
    while (gate.queue.length > 0 && gate.running < currentCap(key)) {
      const next = gate.queue.shift();
      if (!next) break;
      gate.running++;
      next.startWait(createReleaseFn(key));
    }
    if (gate.running === 0 && gate.queue.length === 0) {
      gates.delete(key);
      caps.delete(key); // m-d: caps follows gates so idle keys leave no residue
    }
  };
}

// Cap recorded per key at acquire time (config is per-process; last writer wins,
// same lazy pattern as rateLimitSemaphore.getGate).
const caps = new Map<string, number>();
function currentCap(key: string): number {
  return caps.get(key) ?? EGRESS_THROTTLE_DEFAULTS.cap;
}

export interface AcquireEgressSlotOptions {
  signal?: AbortSignal | null;
  rand?: () => number;
  nowMs?: number;
}

/**
 * Acquire one of `cfg.cap` concurrent slots for `key`. Resolves a release
 * callback (call exactly once, in `finally`), or `null` when the caller must
 * proceed WITHOUT a slot: module disabled, empty key, queue wait elapsed
 * (5-15 s jittered, bounded by the per-call wait budget), or client abort.
 */
export async function acquireEgressSlot(
  key: string,
  cfg: EgressThrottleConfig,
  opts: AcquireEgressSlotOptions = {}
): Promise<(() => void) | null> {
  const nowMs = opts.nowMs ?? Date.now();
  if (!cfg.enabled || !key) return null;
  caps.set(key, cfg.cap);
  const gate = getGate(key, nowMs);
  if (gate.running < cfg.cap) {
    gate.running++;
    return createReleaseFn(key);
  }
  const rand = opts.rand ?? Math.random;
  const waitMs = Math.min(
    cfg.waitMinMs + rand() * Math.max(0, cfg.waitMaxMs - cfg.waitMinMs),
    Math.max(0, cfg.waitBudgetMs)
  );
  if (waitMs <= 0) return null;
  return new Promise<(() => void) | null>((resolve) => {
    let settled = false;
    const waiter: Waiter = {
      enqueuedAt: nowMs,
      startWait: (slot) => {
        if (settled) return slot();
        settled = true;
        resolve(slot);
      },
      abort: () => {
        if (settled) return;
        settled = true;
        const idx = gate.queue.indexOf(waiter);
        if (idx !== -1) gate.queue.splice(idx, 1);
        resolve(null);
      },
    };
    gate.queue.push(waiter);
    void sleepAbortable(waitMs, opts.signal ?? null).then((elapsed) => {
      if (!elapsed) waiter.abort();
      else {
        // Wait elapsed without a slot: fail open, leave the queue.
        const idx = gate.queue.indexOf(waiter);
        if (idx !== -1) gate.queue.splice(idx, 1);
        if (!settled) {
          settled = true;
          resolve(null);
        }
      }
    });
    opts.signal?.addEventListener("abort", waiter.abort, { once: true });
  });
}

// Fleet-wide backoff state (burst 429s only; rate-limit verdicts stay on the
// early-stop path and are never recorded here)
// ---------------------------------------------------------------------------

let burst429At: number[] = [];
let suspectUntil = 0;
let suspectSeedRand: (() => number) | null = null;

function pruneBursts(nowMs: number, windowMs: number): void {
  const cutoff = nowMs - windowMs;
  burst429At = burst429At.filter((t) => t > cutoff);
}

/**
 * Record one burst (non-`rate_limited`) 429. Callers only invoke this once the
 * early-stop path has ruled out a rate-limit verdict, so rate-limit verdicts
 * stay on the early-stop path and never reach this counter. Time is
 * injectable; the window is read lazily from the last resolved fleet window.
 */
export function noteEgress429(nowMs: number = Date.now()): void {
  if (!lastEnabled) return;
  burst429At.push(nowMs);
  pruneBursts(nowMs, lastFleetWindowMs);
  if (burst429At.length >= lastFleetThreshold && nowMs >= suspectUntil) {
    const rand = suspectSeedRand ?? Math.random;
    suspectUntil =
      nowMs + lastSuspectMinMs + rand() * Math.max(0, lastSuspectMaxMs - lastSuspectMinMs);
  }
}

// Last resolved fleet params (set by configureFleetForTest / seam wiring;
// defaults otherwise). Kept module-local so noteEgress429 stays sync and cheap.
let lastFleetWindowMs = EGRESS_THROTTLE_DEFAULTS.fleetWindowMs;
let lastFleetThreshold = EGRESS_THROTTLE_DEFAULTS.fleetThreshold;
let lastSuspectMinMs = EGRESS_THROTTLE_DEFAULTS.suspectMinMs;
let lastSuspectMaxMs = EGRESS_THROTTLE_DEFAULTS.suspectMaxMs;
let lastEnabled = false;

/** Wire the fleet params from resolved config (called once per request path setup). */
export function configureFleetFromConfig(cfg: EgressThrottleConfig): void {
  lastEnabled = cfg.enabled;
  lastFleetWindowMs = cfg.fleetWindowMs;
  lastFleetThreshold = cfg.fleetThreshold;
  lastSuspectMinMs = cfg.suspectMinMs;
  lastSuspectMaxMs = cfg.suspectMaxMs;
}

/** The first success clears everything: burst history + suspect. */
export function noteEgressSuccess(): void {
  burst429At = [];
  suspectUntil = 0;
}

/** Lazy read: an expired suspect is invisible without any timer (breaker/cooldown motif). */
export function isFleetSuspect(nowMs: number = Date.now()): boolean {
  return nowMs < suspectUntil;
}

// ---------------------------------------------------------------------------
// Request wiring: keeps the executor seam small (init + one call per arm)
// ---------------------------------------------------------------------------

/** Per-request pacing state. Null budget = fleet-wide backoff inactive. */
export interface EgressPacing {
  config: EgressThrottleConfig;
  slotBudget: number | null;
  slotsUsed: number;
}

/**
 * Resolve config once per request and snapshot the fleet-wide backoff budget.
 * Off by default: rotation unchanged.
 */
export function initEgressPacingForRequest(env: NodeJS.ProcessEnv = process.env): EgressPacing {
  const config = resolveEgressThrottleConfig(env);
  configureFleetFromConfig(config);
  return {
    config,
    slotBudget: config.enabled && isFleetSuspect() ? config.suspectSlots : null,
    slotsUsed: 0,
  };
}

/**
 * Pace one dispatch through the per-egress semaphore. Resolves a release
 * callback, or null when the caller proceeds without a slot (fail-open:
 * module off, elapsed wait budget, saturated queue, aborted wait).
 */
export function acquirePacingSlot(
  pacing: EgressPacing,
  proxy: { host: string; port: number } | null,
  signal: AbortSignal | null | undefined,
  readApplied?: AppliedEgressReader | null,
  fingerprint?: string
): Promise<(() => void) | null> {
  if (!pacing.config.enabled) return Promise.resolve(null);
  return acquireEgressSlot(
    resolveAppliedEgressKey({ proxy, fingerprint }, readApplied),
    pacing.config,
    {
      signal: signal ?? null,
    }
  );
}

/**
 * Start one paced dispatch: acquire the per-egress slot (fail-open null when
 * the module is off or the wait elapses) and re-validate the pick — the
 * account may have cooled down meanwhile, so re-pick once instead of serving
 * a dead account. Returns the release and the account to use.
 */
export async function startPacedDispatch<
  A extends { proxy: { host: string; port: number } | null; fingerprint?: string },
>(
  pacing: EgressPacing,
  account: A,
  isCandidate: (a: A) => boolean,
  repick: () => A,
  signal: AbortSignal | null | undefined,
  readApplied?: AppliedEgressReader | null
): Promise<{ release: (() => void) | null; account: A }> {
  const release = await acquirePacingSlot(
    pacing,
    account.proxy,
    signal,
    readApplied,
    account.fingerprint
  );
  if (release !== null && !isCandidate(account)) {
    release();
    return { release: null, account: repick() };
  }
  return { release, account };
}

/**
 * Set-aside note for one refused member: hands the refusal to the proxy
 * memory when the opt-in is on. Returns the set-aside duration for the log.
 */
export function noteRefusedMember(
  proxy: { host: string; port: number } | null,
  skipRecentlyFailed: boolean,
  readApplied?: AppliedEgressReader | null,
  fingerprint?: string,
  kind: ProxyRefusalKind = "ip_quota_429"
): number | null {
  if (!skipRecentlyFailed) return null;
  const key =
    proxy !== null
      ? proxyEgressKey(proxy)
      : resolveAppliedEgressKey({ proxy, fingerprint }, readApplied);
  if (key === null || key === DIRECT_EGRESS_SENTINEL) return null;
  return noteProxyRefusal(key, kind);
}

/**
 * Set-aside note for one settled headers-wait overrun: records the overrun,
 * then hands the refusal to the proxy memory only on repetition (three settled
 * overruns through the same egress key inside five minutes) and only when the
 * opt-in is on. A lone slow wait is the upstream queue, not the member.
 * The key resolves through the same seam as the 429 note: the effective egress
 * really applied to the attempt (dedicated proxy fast path, ambient reader for
 * proxyless accounts, direct sentinel as no-op). Returns the set-aside
 * duration for the log, or null when nothing was set aside.
 */
export function noteSlowOverrun(
  account: AppliedEgressAccount,
  skipRecentlyFailed: boolean,
  readApplied?: AppliedEgressReader | null,
  nowMs: number = Date.now(),
  opts?: { destination?: string | null }
): number | null {
  const destination = opts?.destination ?? null;
  if (!skipRecentlyFailed) return null;
  if (account.proxy !== null) {
    const key = proxyEgressKey(account.proxy);
    if (key === null) return null;
    recordSlowOverrun(key, nowMs);
    if (!hasSlowOverrunEvidence(key, destination, nowMs)) return null;
    return noteProxyRefusal(key, "slow", nowMs);
  }
  const key = resolveAppliedEgressKey(account, readApplied);
  if (key === DIRECT_EGRESS_SENTINEL) return null;
  recordSlowOverrun(key, nowMs);
  if (!hasSlowOverrunEvidence(key, destination, nowMs)) return null;
  return noteProxyRefusal(key, "slow", nowMs);
}

/**
 * Resolve the 429 verdict for one refused dispatch. When the rate-limited
 * early-stop is on, the classifier decides; a rate-limit verdict stays on the
 * early-stop path and never reaches the burst counter. When the early-stop is
 * off no body is read and the observed 429 counts as a burst by default.
 */
export async function resolveBurstVerdict(
  response: Response,
  earlyStopEnabled: boolean
): Promise<RateLimit429Verdict> {
  if (earlyStopEnabled && (await classifyUpstream429(response)) === "rate_limited") {
    return "rate_limited";
  }
  return "burst";
}

/**
 * Record one burst 429 (a 429 the early-stop did NOT classify as rate-limited)
 * and decide whether this request parks: under fleet-wide backoff the request
 * stops after its slot budget and the caller surfaces the last
 * upstream answer as-is.
 */
export function observeBurst429(pacing: EgressPacing): "rotate" | "park" {
  if (pacing.config.enabled) noteEgress429();
  if (pacing.slotBudget === null) return "rotate";
  pacing.slotsUsed++;
  return pacing.slotsUsed >= pacing.slotBudget ? "park" : "rotate";
}

/** Clear fleet-wide backoff on the first success (any response with ok true). */
export function observePacingSuccess(pacing: EgressPacing, ok: boolean): void {
  if (ok && pacing.config.enabled) noteEgressSuccess();
}

/**
 * Release the pacing slot held for one loop iteration. Every loop exit after
 * the acquire (rotate, park, return, throw) funnels through this single site,
 * so no path can leak a slot. Idempotent: safe to call twice.
 */
export function releasePacingSlot(release: (() => void) | null): void {
  release?.();
}

/**
 * Log one refused-member outcome on the request logger. The wording lives
 * here so each arm holds one call; the loop control stays at the seam.
 */
export function logRefusedOutcome(
  log: { warn?: (tag: string, message: string) => void } | undefined,
  cid: string,
  masked: string,
  setAsideMs: number | null,
  refusal: "geo-blocked" | "burst 429",
  egress = ""
): void {
  // Egress label position is pinned by tests/unit/opencode-egress-label.test.ts:
  // after the account for a burst 429, at the very end for a geo-block.
  const burst = refusal === "burst 429";
  log?.warn?.(
    "OPENCODE",
    `${cid}${refusal} on account ${masked}` +
      (burst && egress ? ` ${egress}` : "") +
      (setAsideMs ? `, member set aside for ${Math.round(setAsideMs / 1000)}s` : "") +
      ", rotating" +
      (burst ? " to next" : "") +
      "…" +
      (!burst && egress ? ` ${egress}` : "")
  );
}
/**
 * Log one 429 outcome on the request logger. The stop/park/rotate wording
 * lives here so the arm holds one call; the loop control stays at the seam.
 */
export function log429Outcome(
  log: { warn?: (tag: string, message: string) => void } | undefined,
  cid: string,
  arm: "stop" | "park" | "rotate",
  masked: string,
  setAsideMs: number | null,
  label: string
): void {
  if (arm === "stop") {
    log?.warn?.(
      "OPENCODE",
      `${cid}rate-limited 429 on account ${masked} ${label}, stopping the wave`
    );
  } else if (arm === "park") {
    log?.warn?.(
      "OPENCODE",
      `${cid}fleet backing off ${label}: slot budget used, parking (returning last answer)`
    );
  } else {
    logRefusedOutcome(log, cid, masked, setAsideMs, "burst 429", label);
  }
}
/**
 * Settle the 429 arm: resolve the verdict, release the slot exactly once, and
 * record a burst. Returns "stop" (early-stop: caller returns result),
 * "park" (slot budget spent: caller breaks), or "rotate" (caller continues).
 * The release happens inside, so the arm holds no release call at all.
 */
export async function settle429Arm(
  release: (() => void) | null,
  pacing: EgressPacing,
  response: Response,
  isEarlyStopEnabled: () => boolean
): Promise<"stop" | "park" | "rotate"> {
  const verdict = await resolveBurstVerdict(response, isEarlyStopEnabled());
  releasePacingSlot(release);
  if (verdict === "rate_limited") return "stop";
  return observeBurst429(pacing) === "park" ? "park" : "rotate";
}

/**
 * Rethrow a dispatch error after releasing the pacing slot. Single site for
 * every `throw err` after the acquire, so no throw path can leak a slot.
 */
export function throwPacedError(release: (() => void) | null, err: unknown): never {
  releasePacingSlot(release);
  throw err;
}

/**
 * Rotation-loop wiring for the stall arm: the tried-set plus a mutable stall
 * counter, bundled so the arm holds one call. The loop owns `stalled` and the
 * helper reads-then-bumps it. The optional slow note records one settled
 * headers-wait overrun per call; the stall arm never passes it.
 */
export interface StallLoopWiring {
  tried: Set<string>;
  stalled: { attempts: number };
  cooldown: (account: { proxy: { host: string; port: number } | null }) => void;
  markDirect: () => void;
  slow?: { account: AppliedEgressAccount; enabled: boolean; egress: AppliedEgressTracker };
}

/**
 * Settle a stalled dispatch: release the slot, cool the account down, join the
 * tried-set, and bump the stall counter. Returns true on the first stall
 * (caller rotates), false afterwards (caller rethrows). The log line stays at
 * the call site (it needs the request logger).
 */
export function settleStalledDispatch(
  release: (() => void) | null,
  account: { proxy: { host: string; port: number } | null },
  loop: StallLoopWiring
): boolean {
  releasePacingSlot(release);
  loop.cooldown(account);
  const key = proxyKeyOf(account.proxy);
  if (key !== null) loop.tried.add(key);
  else loop.markDirect();
  const first = loop.stalled.attempts === 0;
  loop.stalled.attempts++;
  const slow = loop.slow;
  if (slow)
    noteSlowOverrun(slow.account, slow.enabled, slow.egress.readAppliedKey, Date.now(), {
      destination: slow.egress.destinationOfRequest(),
    });
  return first;
}

// ---------------------------------------------------------------------------
// Test helpers (never call in production code)
// ---------------------------------------------------------------------------

/** Clear all throttle state. Tests only. */
export function _clearEgressThrottleForTest(): void {
  for (const [, gate] of gates) {
    for (const w of gate.queue) w.abort();
  }
  gates.clear();
  caps.clear();
  burst429At = [];
  suspectUntil = 0;
  suspectSeedRand = null;
  lastEnabled = false;
  lastFleetWindowMs = EGRESS_THROTTLE_DEFAULTS.fleetWindowMs;
  lastFleetThreshold = EGRESS_THROTTLE_DEFAULTS.fleetThreshold;
  lastSuspectMinMs = EGRESS_THROTTLE_DEFAULTS.suspectMinMs;
  lastSuspectMaxMs = EGRESS_THROTTLE_DEFAULTS.suspectMaxMs;
}

/** Current gate count. Tests only. */
export function _egressThrottleSizeForTest(): number {
  return gates.size;
}

/** Insert an idle gate key (GC test). Tests only. */
export function _touchEgressKeyForTest(key: string, nowMs: number = Date.now()): void {
  getGate(key, nowMs);
}

/** Inject RNG for the suspect duration (deterministic tests). Tests only. */
export function _setSuspectRandForTest(rand: (() => number) | null): void {
  suspectSeedRand = rand;
}

/**
 * One real call after a 429 when no account is a candidate. The rotation guard
 * skips a non-candidate without calling it, so members excluded only by state
 * left by earlier requests (proxy set aside, account cooling down) were never
 * tried and the request served the 429. `take` returns, once per request, an
 * account whose proxy this request has not refused yet, or null; the caller
 * lets that returned account through the guard.
 *
 * Proxy-less accounts are never handed out: a 429 on one records no key, so
 * the request cannot tell whether the shared direct egress already refused it,
 * and calling it could replay that 429. A rejected pick leaves the rotation
 * cursor where it was.
 */
export function lastResort429<A extends RotatableAccount>(
  accounts: A[],
  cursor: { nextAccountIdx: number; lastHealthyFingerprint?: string },
  ...refusedHere: Set<string>[]
) {
  let spent = false;
  const isOpen = (account: A): boolean => {
    const key = proxyKeyOf(account.proxy);
    return key !== null && !refusedHere.some((keys) => keys.has(key));
  };
  return {
    take(lastStatus: number | null, account: A, isCandidate: (a: A) => boolean): A | null {
      if (spent || lastStatus !== 429 || isCandidate(account)) return null;
      const saved = { ...cursor };
      const spare = pickAccount(accounts, cursor, isOpen);
      if (!isOpen(spare)) {
        cursor.nextAccountIdx = saved.nextAccountIdx;
        cursor.lastHealthyFingerprint = saved.lastHealthyFingerprint;
        return null;
      }
      spent = true;
      return spare;
    },
  };
}