Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
/**
* Short-lived, per-process memory of proxies that just failed, shared by both places that
* pick a proxy: registry pools (#6365) and the per-account rotation of noauth executors.
* A failed proxy is set aside for a period that doubles on each repeat, up to a cap, then
* comes back. Nothing is persisted and no proxy status is written: only the order in which
* candidates are tried changes. Keys are entry points (scheme, username, host, port),
* never passwords.
*
* This module is a pure store: it never reads the PROXY_SKIP_RECENTLY_FAILED feature flag
* (callers gate writes and decisions on it) and it stays free of the proxy dispatcher, so
* the DB layer can consult it without loading undici or the SOCKS connector.
*/
import { notifyProxyTransition, getSharedRefusalStore } from "./proxyTransitionListeners.ts";
import type {
RefusalState,
SharedRefusalStore,
SlowOverrun,
SlowServe,
TransportFailure,
TransportSuccess,
} from "./proxyTransitionListeners.ts";
import { stripIpv6Brackets } from "./proxyFamily.ts";
// Field trends show a refused egress rarely recovers within minutes, so the
// first set-aside lasts five minutes and doubles from there; the cap lets a
// recovered member return within the quarter-hour.
const DEFAULT_QUOTA_429_BASE_MS = 300_000;
const DEFAULT_QUOTA_429_MAX_MS = 900_000;
const MIN_QUOTA_429_MS = 1_000;
const MAX_QUOTA_429_MS = 3_600_000;
// Number() idiom (cf. readTimeoutMs): accepts hex/exponents/whitespace, then floored and bounded.
function readBoundedMs(name: string, def: number, min: number, max: number): number {
const raw = process.env[name];
if (raw == null || raw.trim() === "") return def;
const parsed = Math.floor(Number(raw));
if (!Number.isFinite(parsed) || parsed < min || parsed > max) {
console.warn(`[ProxyRefusalMemory] Invalid ${name}="${raw}". Using default ${def}ms.`);
return def;
}
return parsed;
}
type RefusalPolicy = { baseMs: number; maxMs: number };
const quota429BaseMs = readBoundedMs(
"PROXY_QUOTA_429_BASE_MS",
DEFAULT_QUOTA_429_BASE_MS,
MIN_QUOTA_429_MS,
MAX_QUOTA_429_MS
);
let quota429MaxMs = readBoundedMs(
"PROXY_QUOTA_429_MAX_MS",
DEFAULT_QUOTA_429_MAX_MS,
MIN_QUOTA_429_MS,
MAX_QUOTA_429_MS
);
// When max < base the curve would start at max, so fall back to the default cap instead.
if (quota429MaxMs < quota429BaseMs) {
console.warn(
`[ProxyRefusalMemory] Invalid PROXY_QUOTA_429_MAX_MS="${process.env.PROXY_QUOTA_429_MAX_MS}": below PROXY_QUOTA_429_BASE_MS. Using default ${DEFAULT_QUOTA_429_MAX_MS}ms.`
);
quota429MaxMs = DEFAULT_QUOTA_429_MAX_MS;
}
export const REFUSAL_POLICIES: {
proxy_unreachable: { baseMs: 60_000; maxMs: 600_000 };
ip_quota_429: RefusalPolicy;
transport: { baseMs: 60_000; maxMs: 600_000 };
slow: { baseMs: 60_000; maxMs: 600_000 };
geo_blocked: { baseMs: 60_000; maxMs: 600_000 };
} = {
/** The TCP probe could not open a connection to the proxy. */
proxy_unreachable: { baseMs: 60_000, maxMs: 600_000 },
/** The provider refused through this proxy; the member is set aside for a cooldown. */
ip_quota_429: { baseMs: quota429BaseMs, maxMs: quota429MaxMs },
/**
* Repeated tagged transport failures through this egress with cross-evidence:
* the same destination answers through a different egress, so the member —
* not the destination — is at fault. Same curve as a refused probe.
*/
transport: { baseMs: 60_000, maxMs: 600_000 },
/**
* Repeated settled waits for upstream response headers through this egress:
* one slow wait is the upstream queue, not the member. Same short curve as
* a refused probe, kept apart from the quota curve.
*/
slow: { baseMs: 60_000, maxMs: 600_000 },
/** The provider refused this region through this member; short set-aside. */
geo_blocked: { baseMs: 60_000, maxMs: 600_000 },
};
export type ProxyRefusalKind = keyof typeof REFUSAL_POLICIES;
// Shared across duplicated server module copies (see getSharedRefusalStore):
// rebind on each module evaluation so HMR keeps the same object.
const store: SharedRefusalStore = getSharedRefusalStore();
const memory: Map<string, RefusalState> = store.memory;
const MAX_ENTRIES = 1000;
const REFUSAL_KINDS = Object.keys(REFUSAL_POLICIES) as ProxyRefusalKind[];
// Same protocol set and default ports as proxyConfigToUrl() in proxyDispatcher.ts.
const DEFAULT_PORTS: Record<string, string> = { http: "8080", https: "443", socks5: "1080" };
const RELAY_TYPES = new Set(["vercel", "deno", "cloudflare"]);
const FAMILY_MARKER = /\?family=(ipv4|ipv6)$/;
/**
* Credential-free label for one egress key (`scheme://user@host:port` as
* proxyEgressKey writes it): `scheme://host:port`. String surgery only, no
* `new URL` (empty userinfo and bare IPv6 break URL parsing). Never receives
* anything but egress keys, which never carry passwords or observed addresses.
*/
export function describeEgressForLog(key: string): string {
const bare = key.replace(FAMILY_MARKER, "");
const schemeEnd = bare.indexOf("://");
const scheme = (schemeEnd === -1 ? "http" : bare.slice(0, schemeEnd)).toLowerCase();
const rest = schemeEnd === -1 ? bare : bare.slice(schemeEnd + 3);
const at = rest.lastIndexOf("@");
const hostPort = at === -1 ? rest : rest.slice(at + 1);
const slash = hostPort.indexOf("/");
const authority = slash === -1 ? hostPort : hostPort.slice(0, slash);
const colon = authority.lastIndexOf(":");
const rawHost = colon === -1 ? authority : authority.slice(0, colon);
const port = colon === -1 ? "" : authority.slice(colon + 1);
const host = stripIpv6Brackets(rawHost).toLowerCase();
const bracketed = host.includes(":") && !host.startsWith("[") ? `[${host}]` : host;
return port ? `${scheme}://${bracketed}:${port}` : `${scheme}://${bracketed}`;
}
// Instance proof: fixed at module load so two readers seeing different values
// spot a duplicated store at a glance. Random hex, not the pid: two copies in
// one process share the pid but not this value.
const storeInstanceId = Math.floor(Math.random() * 0xffffffff)
.toString(16)
.padStart(8, "0");
/** Opaque id of this store instance, set once at module load. */
export function getRefusalStoreInstance(): string {
return storeInstanceId;
}
// Duplicate-load alarm: counts module loads on shared globalThis keys (plain
// Symbol.for scalars, independent of any shared-store object) and warns once.
const LOADS_KEY = Symbol.for("omniroute.proxyRefusalMemory.loads");
const WARNED_KEY = Symbol.for("omniroute.proxyRefusalMemory.warned");
{
const holder = globalThis as unknown as { [key: symbol]: unknown };
const loads = typeof holder[LOADS_KEY] === "number" ? (holder[LOADS_KEY] as number) : 0;
holder[LOADS_KEY] = loads + 1;
if (loads + 1 > 1 && !holder[WARNED_KEY]) {
holder[WARNED_KEY] = true;
console.warn("[ProxyRefusalMemory] module loaded more than once (duplicated bundle copy)");
}
}
const textField = (value: unknown): string => (typeof value === "string" ? value : "");
// The port as proxyConfigToUrl() normalizes it: the scheme default when unset, null if invalid.
function configPort(port: unknown, type: string): string | null {
if (!port) return DEFAULT_PORTS[type] ?? "";
const parsed = Number(port);
return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65535 ? String(parsed) : null;
}
// A config object as the URL proxyConfigToUrl() would build from it; null when unusable.
function configObjectToUrl(proxy: Record<string, unknown>): string | null {
const host = textField(proxy.host);
const type = String(proxy.type || "http").toLowerCase();
const port = configPort(proxy.port, type);
if (!host || RELAY_TYPES.has(type) || port === null) return null;
const bracketed = host.includes(":") && !host.startsWith("[") ? `[${host}]` : host;
const username = textField(proxy.username);
const password = textField(proxy.password);
const auth =
username || password ? `${encodeURIComponent(username)}:${encodeURIComponent(password)}@` : "";
return `${type}://${auth}${bracketed}:${port}`;
}
// The port written in the authority, which `new URL()` drops when it is the scheme default.
function explicitPortOf(url: string): string | null {
const start = url.indexOf("://");
if (start === -1) return null;
const rest = url.slice(start + 3);
const slash = rest.indexOf("/");
const authority = slash === -1 ? rest : rest.slice(0, slash);
const colon = authority.lastIndexOf(":");
if (colon === -1 || colon < authority.lastIndexOf("@") || colon < authority.lastIndexOf("]")) {
return null;
}
const port = Number(authority.slice(colon + 1));
return /^\d+$/.test(authority.slice(colon + 1)) && port >= 1 && port <= 65535
? String(port)
: null;
}
/**
* One key per proxy entry point, whether the proxy comes as a config object, a URL or a
* legacy string: scheme, decoded username, lower-case host without IPv6 brackets, port as
* normalization writes it. Password and ?family= are ignored. Anything unusable, and edge
* relays, give null, which never sets anything aside.
*/
export function proxyEgressKey(proxy: unknown): string | null {
try {
let url: string | null = null;
if (typeof proxy === "string") url = proxy.trim();
else if (proxy && typeof proxy === "object" && !Array.isArray(proxy)) {
url = configObjectToUrl(proxy as Record<string, unknown>);
}
if (!url) return null;
url = url.replace(FAMILY_MARKER, "");
const parsed = new URL(url);
const scheme = parsed.protocol.replace(/:$/, "").toLowerCase();
const defaultPort = DEFAULT_PORTS[scheme];
if (!defaultPort || !parsed.hostname) return null;
const port = explicitPortOf(url) || parsed.port || defaultPort;
const user = parsed.username ? decodeURIComponent(parsed.username) : "";
return `${scheme}://${user}@${stripIpv6Brackets(parsed.hostname).toLowerCase()}:${port}`;
} catch {
return null;
}
}
function entryId(key: string, kind: ProxyRefusalKind): string {
return `${kind} ${key}`;
}
/**
* Composite key for one (entry, selector member) pair. JSON-array encoding is
* unambiguous by construction: member names carrying spaces, brackets, quotes
* or unicode cannot alias each other or an entry-level key. Entry-level keys
* stay plain `proxyEgressKey` strings so pool/rotation reads are untouched.
*/
export function keyForEntryMember(entryKey: string, member: string): string {
return JSON.stringify([entryKey, member]);
}
// Read one composite (entry, member, kind) state, same TTL discipline as entry
// states: dropped once its period ended more than 2 x maxMs ago.
function readMemberState(
entryKey: string,
member: string,
kind: ProxyRefusalKind,
nowMs: number
): RefusalState | undefined {
const id = entryId(keyForEntryMember(entryKey, member), kind);
const state = memory.get(id);
if (state && nowMs - state.until >= 2 * REFUSAL_POLICIES[kind].maxMs) {
memory.delete(id);
return undefined;
}
return state;
}
// Shared insert: same curve, same seq, same oldest-first eviction as entries.
function insertState(key: string, kind: ProxyRefusalKind, nowMs: number): number {
const state = readState(key, kind, nowMs);
if (state && state.until > nowMs) return -1;
const policy = REFUSAL_POLICIES[kind];
const streak = (state?.streak ?? 0) + 1;
const periodMs = Math.min(policy.baseMs * 2 ** (streak - 1), policy.maxMs);
const id = entryId(key, kind);
memory.delete(id);
memory.set(id, { streak, until: nowMs + periodMs, seq: ++store.seq.value });
if (memory.size > MAX_ENTRIES) {
const oldest = memory.keys().next().value;
if (oldest !== undefined) memory.delete(oldest);
}
return periodMs;
}
// Read one (key, kind) state, dropping it once its period ended more than 2 x maxMs ago.
function readState(key: string, kind: ProxyRefusalKind, nowMs: number): RefusalState | undefined {
const id = entryId(key, kind);
const state = memory.get(id);
if (state && nowMs - state.until >= 2 * REFUSAL_POLICIES[kind].maxMs) {
memory.delete(id);
return undefined;
}
return state;
}
/** Set a proxy aside for `kind`. Returns the new period in ms, or null if nothing changed. */
export function noteProxyRefusal(
key: string | null,
kind: ProxyRefusalKind,
nowMs: number = Date.now()
): number | null {
if (key === null) return null;
const before = readState(key, kind, nowMs);
const periodMs = insertState(key, kind, nowMs);
if (periodMs < 0) return null;
const streak = (before?.streak ?? 0) + 1;
notifyProxyTransition({ key, kind, periodMs, until: nowMs + periodMs });
console.warn(
`[ProxyRefusalMemory] set aside ${describeEgressForLog(key)} kind=${kind} periodMs=${periodMs} streak=${streak}`
);
return periodMs;
}
/**
* Set one selector member aside for `kind`, keyed by (entry, member). Reuses
* the same policy curve, streak discipline and memory bound as entry writes:
* at equal streak the period equals the entry-level one. The entry-level write
* stays the caller's job (kept separate so pool/rotation avoidance on the
* entry key never regresses). Null entry or member writes nothing.
*/
export function noteProxyMemberRefusal(
entryKey: string | null,
member: string | null,
kind: ProxyRefusalKind,
nowMs: number = Date.now()
): number | null {
if (entryKey === null || member === null) return null;
const before = readMemberState(entryKey, member, kind, nowMs);
const periodMs = insertState(keyForEntryMember(entryKey, member), kind, nowMs);
if (periodMs < 0) return null;
console.warn(
`[ProxyRefusalMemory] set aside ${describeEgressForLog(entryKey)} member=${member} kind=${kind} periodMs=${periodMs} streak=${(before?.streak ?? 0) + 1}`
);
return periodMs;
}
/** The proxy answered again: end its period now, keep the streak so a repeat doubles. */
export function noteProxyRecovered(
key: string | null,
kind: ProxyRefusalKind,
nowMs: number = Date.now()
): void {
if (key === null) return;
const state = readState(key, kind, nowMs);
if (!state || state.until <= nowMs) return;
const policy = REFUSAL_POLICIES[kind];
const periodMs = Math.min(policy.baseMs * 2 ** (state.streak - 1), policy.maxMs);
const setAsideAt = state.until - periodMs;
console.warn(
`[ProxyRefusalMemory] recovered ${describeEgressForLog(key)} kind=${kind} setAsideMs=${nowMs - setAsideAt} streak=${state.streak}`
);
state.until = nowMs;
}
/** A response came back through this proxy: forget every refusal kind for it. */
export function noteProxyServed(key: string | null): void {
if (key === null) return;
for (const kind of REFUSAL_KINDS) memory.delete(entryId(key, kind));
}
export function isProxyAvoided(key: string | null, nowMs: number = Date.now()): boolean {
return proxySetAsideSeq(key, nowMs) !== null;
}
/**
* Sequence number of the most recent set-aside event still in force for this proxy, or
* null when it is not set aside. Compare with getProxyRefusalSeq() captured earlier to
* know whether the event happened after that point.
*/
export function proxySetAsideSeq(key: string | null, nowMs: number = Date.now()): number | null {
if (key === null || memory.size === 0) return null;
let latest: number | null = null;
for (const kind of REFUSAL_KINDS) {
const state = readState(key, kind, nowMs);
if (state && state.until > nowMs && (latest === null || state.seq > latest)) {
latest = state.seq;
}
}
return latest;
}
/**
* Read-only snapshot of the set-aside state still in force for a proxy egress
* key, or null when the proxy is not set aside. Exposes the refusal motive,
* the set-aside start, the expected end and the repeat count so a read-only
* status screen can explain why a pool member is currently deprioritized.
* Never mutates the memory (expired entries are dropped by readState as usual).
*/
export interface ProxySetAsideSnapshot {
kind: ProxyRefusalKind;
setAsideAt: number;
endsAt: number;
streak: number;
}
export function snapshotProxySetAside(
key: string | null,
nowMs: number = Date.now()
): ProxySetAsideSnapshot | null {
if (key === null || memory.size === 0) return null;
let latest: (ProxySetAsideSnapshot & { seq: number }) | null = null;
for (const kind of REFUSAL_KINDS) {
const state = readState(key, kind, nowMs);
if (!state || state.until <= nowMs || (latest !== null && state.seq <= latest.seq)) continue;
// Recompute the period from the stored streak with the same doubling curve as
// noteProxyRefusal so setAsideAt = until - periodMs (read-only, no state change).
const policy = REFUSAL_POLICIES[kind];
const periodMs = Math.min(policy.baseMs * 2 ** (state.streak - 1), policy.maxMs);
latest = {
kind,
setAsideAt: state.until - periodMs,
endsAt: state.until,
streak: state.streak,
seq: state.seq,
};
}
if (!latest) return null;
const { seq: _seq, ...snapshot } = latest;
return snapshot;
}
/**
* Read-only set-aside state across every refusal motive still in force for
* one proxy egress key. Unlike the latest-event snapshot above, quotaOnly is
* true only when every motive in force is a quota refusal, and endsAt is the
* end of the real set-aside (the latest motive end), so one stale slow or
* transport record cannot pass the quota filter by recency.
*/
export interface ProxySetAsideStatus {
setAside: boolean;
quotaOnly: boolean;
endsAt: number | null;
}
export function proxySetAsideStatus(
key: string | null,
nowMs: number = Date.now()
): ProxySetAsideStatus {
if (key === null || memory.size === 0) return { setAside: false, quotaOnly: false, endsAt: null };
let found = false;
let quotaOnly = true;
let endsAt = 0;
for (const kind of REFUSAL_KINDS) {
const state = readState(key, kind, nowMs);
if (!state || state.until <= nowMs) continue;
found = true;
if (kind !== "ip_quota_429") quotaOnly = false;
if (state.until > endsAt) endsAt = state.until;
}
if (!found) return { setAside: false, quotaOnly: false, endsAt: null };
return { setAside: true, quotaOnly, endsAt };
}
/**
* Read-only snapshot of the set-aside state still in force for one
* (entry, selector member) pair, or null when the member is not set aside.
* Same shape as snapshotProxySetAside so the pool visibility route can reuse
* one view. Never mutates the memory (expired entries are dropped by
* readMemberState as usual).
*/
export function snapshotMemberSetAside(
entryKey: string | null,
member: string | null,
nowMs: number = Date.now()
): ProxySetAsideSnapshot | null {
if (entryKey === null || member === null || memory.size === 0) return null;
let latest: (ProxySetAsideSnapshot & { seq: number }) | null = null;
for (const kind of REFUSAL_KINDS) {
const state = readMemberState(entryKey, member, kind, nowMs);
if (!state || state.until <= nowMs || (latest !== null && state.seq <= latest.seq)) continue;
const policy = REFUSAL_POLICIES[kind];
const periodMs = Math.min(policy.baseMs * 2 ** (state.streak - 1), policy.maxMs);
latest = {
kind,
setAsideAt: state.until - periodMs,
endsAt: state.until,
streak: state.streak,
seq: state.seq,
};
}
if (!latest) return null;
const { seq: _seq, ...snapshot } = latest;
return snapshot;
}
/**
* Distinct selector member names recorded for one entry key, in insertion
* order. Memory-only enumeration: parses composite keys back (strip the
* `${kind} ` prefix BEFORE JSON.parse, skip entry-level keys that do not
* parse as [entryKey, member]). Never throws; order is not stable across
* restarts or post-purge reinsertion.
*/
export function listEntryMembers(entryKey: string | null): string[] {
if (entryKey === null || memory.size === 0) return [];
const seen = new Set<string>();
for (const kind of REFUSAL_KINDS) {
const prefix = `${kind} `;
for (const id of memory.keys()) {
if (!id.startsWith(prefix)) continue;
let parsed: unknown;
try {
parsed = JSON.parse(id.slice(prefix.length));
} catch {
continue;
}
if (!Array.isArray(parsed) || parsed[0] !== entryKey || typeof parsed[1] !== "string")
continue;
seen.add(parsed[1]);
}
}
return [...seen];
}
/** Sequence number of the last set-aside event recorded in this process (0 = none yet). */
export function getProxyRefusalSeq(): number {
return store.seq.value;
}
/**
* True while the (entry, member) pair is set aside for any refusal kind.
* Complements `isProxyAvoided` (entry-level, used by pool rotation) — it never
* replaces it.
*/
export function isSelectorMemberAvoided(
entryKey: string | null,
member: string | null,
nowMs: number = Date.now()
): boolean {
return selectorMemberSetAsideSeq(entryKey, member, nowMs) !== null;
}
/**
* Sequence number of the most recent set-aside event still in force for this
* (entry, member) pair, or null when it is not set aside. Powers the
* least-recently-set-aside fallback below: among set-aside members the
* smallest seq is the oldest event.
*/
export function selectorMemberSetAsideSeq(
entryKey: string | null,
member: string | null,
nowMs: number = Date.now()
): number | null {
if (entryKey === null || member === null || memory.size === 0) return null;
let latest: number | null = null;
for (const kind of REFUSAL_KINDS) {
const state = readMemberState(entryKey, member, kind, nowMs);
if (state && state.until > nowMs && (latest === null || state.seq > latest)) {
latest = state.seq;
}
}
return latest;
}
/**
* Oldest set-aside event among `members` still in force for this entry, or
* null when none is set aside. Lets a switch fall back to the least recently
* set-aside member instead of refusing to move. Reads `nowMs` once per caller
* (pass a frozen timestamp for the whole switch).
*/
export function leastRecentlySetAside(
entryKey: string | null,
members: string[],
nowMs: number = Date.now()
): string | null {
if (entryKey === null || memory.size === 0) return null;
let oldest: { member: string; seq: number } | null = null;
for (const member of members) {
const seq = selectorMemberSetAsideSeq(entryKey, member, nowMs);
if (seq !== null && (oldest === null || seq < oldest.seq)) {
oldest = { member, seq };
}
}
return oldest?.member ?? null;
}
/** True when anything is held at all: lets hot paths skip key computation and flag reads. */
export function hasProxyRefusals(): boolean {
return memory.size > 0;
}
/** Test-only: forget everything. */
export function __resetProxyRefusalMemoryForTesting(): void {
memory.clear();
}
/**
* The destination a transport outcome is attributed to: the lower-cased host of
* the requested target URL, without IPv6 brackets. Shared by the failure and
* success hooks so cross-evidence always matches on the same normalization.
* Null when the target is not a parseable URL.
*/
export function transportDestinationKey(targetUrl: string): string | null {
try {
const host = new URL(targetUrl).hostname;
if (!host) return null;
return stripIpv6Brackets(host).toLowerCase();
} catch {
return null;
}
}
/**
* Cross-evidence for the `transport` refusal kind. A tagged transport failure
* alone never sets a member aside: it is only evidence, kept per (egress key,
* destination host). A later success to the same destination through a
* *different* egress proves the destination answers and the failing member —
* not the destination — is at fault.
*/
export const TRANSPORT_EVIDENCE_WINDOW_MS = 300_000;
export const TRANSPORT_EVIDENCE_THRESHOLD = 3;
const MAX_TRANSPORT_EVIDENCE = 1000;
const transportFailures: TransportFailure[] = store.transportFailures;
const transportSuccesses: TransportSuccess[] = store.transportSuccesses;
// Lazy purge mirrors readState: entries older than the evidence window plus
// twice the transport cap can no longer contribute, so drop them on read.
function purgeTransportEvidence(nowMs: number): void {
const cutoff = nowMs - TRANSPORT_EVIDENCE_WINDOW_MS - 2 * REFUSAL_POLICIES.transport.maxMs;
while (transportFailures.length > 0 && transportFailures[0].at < cutoff) {
transportFailures.shift();
}
while (transportSuccesses.length > 0 && transportSuccesses[0].at < cutoff) {
transportSuccesses.shift();
}
}
/** Record one final tagged transport failure. Never throws, never writes refusal memory. */
export function recordTransportFailure(
key: string | null,
destination: string | null,
nowMs: number = Date.now()
): void {
if (key === null || destination === null || destination === "") return;
purgeTransportEvidence(nowMs);
transportFailures.push({ key, destination, at: nowMs });
while (transportFailures.length > MAX_TRANSPORT_EVIDENCE) transportFailures.shift();
}
/** Record one success to a destination through an egress. Never throws. */
export function recordTransportSuccess(
destination: string | null,
key: string | null,
nowMs: number = Date.now()
): void {
if (key === null || destination === null || destination === "") return;
purgeTransportEvidence(nowMs);
transportSuccesses.push({ destination, key, at: nowMs });
while (transportSuccesses.length > MAX_TRANSPORT_EVIDENCE) transportSuccesses.shift();
}
/**
* True when the evidence condemns this egress for this destination: at least
* TRANSPORT_EVIDENCE_THRESHOLD tagged failures through it inside the window
* AND at least one success to the same destination through a different egress
* inside the window. A success through the egress itself is not evidence.
*/
export function hasTransportCrossEvidence(
key: string | null,
destination: string | null,
nowMs: number = Date.now()
): boolean {
if (key === null || destination === null || destination === "") return false;
purgeTransportEvidence(nowMs);
const from = nowMs - TRANSPORT_EVIDENCE_WINDOW_MS;
let failures = 0;
for (const f of transportFailures) {
if (f.key === key && f.destination === destination && f.at >= from) {
failures++;
if (failures >= TRANSPORT_EVIDENCE_THRESHOLD) break;
}
}
if (failures < TRANSPORT_EVIDENCE_THRESHOLD) return false;
return transportSuccesses.some(
(s) => s.destination === destination && s.key !== key && s.at >= from
);
}
/** Test-only: forget transport evidence (refusal memory is separate). */
export function __resetTransportEvidenceForTesting(): void {
transportFailures.length = 0;
transportSuccesses.length = 0;
}
/**
* Read-only transport proof for one entry key: tagged failures through this
* egress plus successes to the same destinations through a different egress,
* both inside TRANSPORT_EVIDENCE_WINDOW_MS. Runs the existing lazy purge
* first (bounded and pre-existing, never a refusal write) so counts never
* grow stale.
*/
export function countTransportEvidenceFor(
key: string | null,
nowMs: number = Date.now()
): { failures: number; crossSuccesses: number } {
if (key === null || (transportFailures.length === 0 && transportSuccesses.length === 0)) {
return { failures: 0, crossSuccesses: 0 };
}
purgeTransportEvidence(nowMs);
const from = nowMs - TRANSPORT_EVIDENCE_WINDOW_MS;
let failures = 0;
let destinations: string[] | null = null;
for (const f of transportFailures) {
if (f.key === key && f.at >= from) {
failures++;
(destinations ??= []).push(f.destination);
}
}
if (destinations === null) return { failures: 0, crossSuccesses: 0 };
let crossSuccesses = 0;
for (const s of transportSuccesses) {
if (s.key !== key && s.at >= from && destinations.includes(s.destination)) crossSuccesses++;
}
return { failures, crossSuccesses };
}
/** Test-only: current evidence store sizes. */
export function __transportEvidenceSizeForTesting(): { failures: number; successes: number } {
return { failures: transportFailures.length, successes: transportSuccesses.length };
}
/**
* Settled waits for upstream response headers, per egress key. One slow wait
* is the upstream queue, not the member — only repetition condemns it, so the
* store keeps bare timestamps and the gated helper decides at k=3 / 5 min.
*/
export const SLOW_OVERRUN_WINDOW_MS = 300_000;
export const SLOW_OVERRUN_THRESHOLD = 3;
const MAX_SLOW_OVERRUNS = 1000;
const slowOverruns: SlowOverrun[] = store.slowOverruns;
const slowServes: SlowServe[] = store.slowServes;
// Lazy purge mirrors readState: entries older than the evidence window plus
// twice the slow cap can no longer contribute, so drop them on record. Slow
// serves share the same window, so one purge covers both stores.
function purgeSlowOverruns(nowMs: number): void {
const cutoff = nowMs - SLOW_OVERRUN_WINDOW_MS - 2 * REFUSAL_POLICIES.slow.maxMs;
while (slowOverruns.length > 0 && slowOverruns[0].at < cutoff) {
slowOverruns.shift();
}
while (slowServes.length > 0 && slowServes[0].at < cutoff) {
slowServes.shift();
}
}
/** Record one settled headers-wait overrun. Never throws, never writes refusal memory. */
export function recordSlowOverrun(key: string | null, nowMs: number = Date.now()): void {
if (key === null || key === "") return;
purgeSlowOverruns(nowMs);
slowOverruns.push({ key, at: nowMs });
while (slowOverruns.length > MAX_SLOW_OVERRUNS) slowOverruns.shift();
}
/**
* True when this egress waited out the headers window at least
* SLOW_OVERRUN_THRESHOLD times inside SLOW_OVERRUN_WINDOW_MS, with
* cross-egress proof: another egress served the same destination inside the
* window. Without any other egress active in the window the repetition alone
* decides (a lone upstream queue still condemns a lone member). A null
* destination keeps the repetition rule (callers without a destination).
*/
export function hasSlowOverrunEvidence(
key: string | null,
destinationOrNowMs: string | null | number = Date.now(),
nowMs: number = Date.now()
): boolean {
const destination = typeof destinationOrNowMs === "number" ? null : (destinationOrNowMs ?? null);
const at = typeof destinationOrNowMs === "number" ? destinationOrNowMs : nowMs;
if (key === null || key === "") return false;
if (!hasSlowOverrunRepetition(key, at)) return false;
if (destination === null || destination === "") return true;
if (hasSlowServeCrossEvidence(key, destination, at)) return true;
return !hasOtherSlowEgressActivity(key, at);
}
// Three settled waits through the same key inside the window: repetition alone.
function hasSlowOverrunRepetition(key: string, at: number): boolean {
purgeSlowOverruns(at);
const from = at - SLOW_OVERRUN_WINDOW_MS;
let overruns = 0;
for (const o of slowOverruns) {
if (o.key === key && o.at >= from) {
overruns++;
if (overruns >= SLOW_OVERRUN_THRESHOLD) return true;
}
}
return false;
}
// Any overrun or serve by another live egress inside the window. The direct
// sentinel never counts: it can never be accused, so it can never witness.
function hasOtherSlowEgressActivity(key: string, at: number): boolean {
const from = at - SLOW_OVERRUN_WINDOW_MS;
for (const o of slowOverruns) {
if (o.key !== key && o.key !== "direct" && o.at >= from) return true;
}
for (const s of slowServes) {
if (s.key !== key && s.key !== "direct" && s.at >= from) return true;
}
return false;
}
/**
* True when another egress served the same destination inside the slow
* window: a headers wait answered in time there, so the destination answers
* and the accused member — not the destination — is at fault. The egress
* itself is never its own witness.
*/
export function hasSlowServeCrossEvidence(
key: string | null,
destination: string | null,
nowMs: number = Date.now()
): boolean {
if (key === null || destination === null || destination === "") return false;
purgeSlowOverruns(nowMs);
const from = nowMs - SLOW_OVERRUN_WINDOW_MS;
return slowServes.some((s) => s.destination === destination && s.key !== key && s.at >= from);
}
/**
* Record one headers wait answered in time through this egress. The writers
* (never this store) keep the sentinel out: `direct` is never recorded, so
* readers need no sentinel filter. Never throws, never writes refusal memory.
*/
export function recordSlowServe(
key: string | null,
destination: string | null,
nowMs: number = Date.now()
): void {
if (key === null || key === "" || destination === null || destination === "") return;
purgeSlowOverruns(nowMs);
slowServes.push({ key, destination, at: nowMs });
while (slowServes.length > MAX_SLOW_OVERRUNS) slowServes.shift();
}
/** Test-only: forget slow overruns and slow serves (refusal memory is separate). */
export function __resetSlowOverrunsForTesting(): void {
slowOverruns.length = 0;
slowServes.length = 0;
}
/** Test-only: current slow-overrun store size. */
export function __slowOverrunSizeForTesting(): number {
return slowOverruns.length;
}
/** Test-only: current slow-serve store size. */
export function __slowServeSizeForTesting(): number {
return slowServes.length;
}
/** Test-only: number of (key, kind) entries held. */
export function __proxyRefusalMemorySizeForTesting(): number {
return memory.size;
}