Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
import {
DEFAULT_API_LIMITS,
PROVIDER_PROFILES,
STREAM_THROUGHPUT_WATCHDOG,
} from "@omniroute/open-sse/config/constants";
import type { JsonRecord, ResilienceSettings, ResilienceSettingsPatch } from "./settings/types";
import {
asRecord,
toInteger,
resolveStreamRecoveryDefaults,
normalizeLegacyConnectionCooldownProfile,
normalizeRequestQueueSettings,
normalizeConnectionCooldownProfile,
normalizeProviderBreakerProfile,
normalizeTokenRefreshBreakerSettings,
normalizeWaitForCooldownSettings,
normalizeComboCooldownWaitSettings,
normalizeQuotaShareConcurrencyLimitSettings,
normalizeStreamStallCooldownSettings,
normalizeProviderCooldownSettings,
normalizeQuotaPreflightSettings,
normalizeStreamRecoverySettings,
normalizeProviderQuotaOverrides,
normalizeCredentialHealthCheckSettings,
pruneResilienceSettingsToProvided,
} from "./settings/normalize";
export { pruneResilienceSettingsToProvided };
// Re-export the settings shape (moved to ./settings/types) so this module's
// public API is unchanged.
export type {
RequestQueueSettings,
ConnectionCooldownProfileSettings,
ProviderBreakerProfileSettings,
TokenRefreshBreakerScope,
TokenRefreshBreakerSettings,
WaitForCooldownSettings,
ComboCooldownWaitSettings,
QuotaShareConcurrencyLimitSettings,
ProviderCooldownSettings,
QuotaPreflightSettings,
StreamRecoverySettings,
StreamThroughputWatchdogSettings,
ProviderQuotaOverrideSettings,
CredentialHealthCheckSettings,
ResilienceSettings,
ResilienceSettingsPatch,
} from "./settings/types";
export const DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS = (() => {
const parsed = Number(process.env.RATE_LIMIT_MAX_WAIT_MS || "30000");
return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : 30000;
})();
// Limiter-managed execution backstop (Bottleneck `expiration`). Deliberately
// separate from the queue-wait budget: non-incremental gateways (Console Go /
// Command Code) buffer whole generations before first bytes, so legitimate
// executions run minutes. Default 10 min; the backstop only catches executors
// without their own upstream timeout.
export const DEFAULT_REQUEST_QUEUE_EXECUTION_MAX_WAIT_MS = (() => {
const parsed = Number(process.env.RATE_LIMIT_EXECUTION_MAX_WAIT_MS || "600000");
return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : 600000;
})();
// Issue #6593: opt-in admission cap on the local rate-limit queue depth.
// Default 0 = disabled (unbounded queue, today's behavior unchanged).
export const DEFAULT_REQUEST_QUEUE_MAX_DEPTH = (() => {
const parsed = Number(process.env.RATE_LIMIT_MAX_QUEUE_DEPTH || "0");
return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : 0;
})();
export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = {
requestQueue: {
autoEnableApiKeyProviders: true,
requestsPerMinute: DEFAULT_API_LIMITS.requestsPerMinute,
minTimeBetweenRequestsMs: DEFAULT_API_LIMITS.minTimeBetweenRequests,
concurrentRequests: DEFAULT_API_LIMITS.concurrentRequests,
globalConcurrentRequests: 0,
maxWaitMs: DEFAULT_REQUEST_QUEUE_MAX_WAIT_MS,
executionMaxWaitMs: DEFAULT_REQUEST_QUEUE_EXECUTION_MAX_WAIT_MS,
maxQueueDepth: DEFAULT_REQUEST_QUEUE_MAX_DEPTH,
},
connectionCooldown: {
oauth: {
baseCooldownMs: PROVIDER_PROFILES.oauth.transientCooldown,
useUpstreamRetryHints: PROVIDER_PROFILES.oauth.rateLimitCooldown === 0,
maxBackoffSteps: PROVIDER_PROFILES.oauth.maxBackoffLevel,
},
apikey: {
baseCooldownMs: PROVIDER_PROFILES.apikey.transientCooldown,
useUpstreamRetryHints: PROVIDER_PROFILES.apikey.rateLimitCooldown === 0,
maxBackoffSteps: PROVIDER_PROFILES.apikey.maxBackoffLevel,
},
},
providerBreaker: {
oauth: {
failureThreshold: PROVIDER_PROFILES.oauth.circuitBreakerThreshold,
degradationThreshold: PROVIDER_PROFILES.oauth.degradationThreshold,
resetTimeoutMs: PROVIDER_PROFILES.oauth.circuitBreakerReset,
},
apikey: {
failureThreshold: PROVIDER_PROFILES.apikey.circuitBreakerThreshold,
degradationThreshold: PROVIDER_PROFILES.apikey.degradationThreshold,
resetTimeoutMs: PROVIDER_PROFILES.apikey.circuitBreakerReset,
},
},
// Token-refresh breaker: provider-wide by default (current behavior), so
// existing installs see no change. Operators can switch to per-connection
// scope to isolate one dead account from healthy ones on the same provider.
tokenRefreshBreaker: {
scope: "provider",
failureThreshold: 5,
cooldownMs: 30 * 60 * 1000,
},
// Wait at most 90s for a single connection cooldown (covers Gemini-class
// TPM/RPM windows, which report ~60s retry-after live), at most 5 retry
// cycles, never more than 300s (5 min) total (#7360 follow-up — raised now
// that withEarlyStreamKeepalive gives streaming clients an immediate
// synthetic keep-alive, eliminating the client-side first-byte-timeout risk
// a long wait used to carry). Applies to direct (non-combo) model requests.
waitForCooldown: {
enabled: true,
maxRetries: 5,
maxRetryWaitSec: 90,
maxRetryWaitMs: 90000,
budgetMs: 300000,
},
// Wait at most 90s for a single short transient cooldown (covers Gemini-class
// TPM/RPM windows, which report ~60s retry-after live — #7360), at most 5
// redispatch cycles, never more than 300s (5 min) total. Active for every
// combo strategy when enabled, and only for transient (non quota_exhausted)
// reasons.
comboCooldownWait: {
enabled: true,
maxWaitMs: 90000,
maxAttempts: 5,
budgetMs: 300000,
},
// FASE 2.1: serialize concurrent quota-share requests per connection when the
// connection sets a max_concurrent cap, so a subscription account is not
// flooded past its concurrency ceiling. Kill-switch only (default on); the cap
// comes from each connection's max_concurrent.
quotaShareConcurrencyLimit: {
enabled: true,
},
// A stream content stall fails one request; it does not cool the account unless
// the operator opts in (see StreamStallCooldownSettings).
streamStallCooldown: {
enabled: false,
},
providerCooldown: {
minRetryCooldownMs: Number(process.env.PROVIDER_COOLDOWN_MIN_MS || "5000"),
maxRetryCooldownMs: Number(process.env.PROVIDER_COOLDOWN_MAX_MS || "300000"),
// Opt-in (default OFF): this global cross-request cooldown overlaps the
// existing Connection Cooldown / Provider Circuit Breaker layers, so it is
// disabled by default and must be explicitly enabled by the operator until
// its interaction with those layers is validated in production.
enabled: ["true", "1", "on"].includes(
(process.env.PROVIDER_COOLDOWN_ENABLED || "").trim().toLowerCase()
),
},
quotaPreflight: {
// Opt-in (default OFF): the auto-routing hard cutoff drops low-quota candidates
// before scoring, overlapping the existing soft quota penalty + connection
// cooldown, so it must be explicitly enabled by the operator until its
// interaction with the scorer is validated in production.
enabled: ["true", "1", "on"].includes(
(process.env.QUOTA_PREFLIGHT_CUTOFF_ENABLED || "").trim().toLowerCase()
),
// Remaining-% semantics. 2 = "stop when only 2% remaining" (= 98% used).
// Uniform across all providers and windows; operators set per-window
// overrides per connection via the Cutoff modal in Dashboard › Limits,
// or per-(provider, window) globally via the providerWindowDefaults map
// below (no factory seeds — keep behavior consistent across providers).
defaultThresholdPercent: 2,
warnThresholdPercent: 20,
providerWindowDefaults: {},
},
streamRecovery: {
// Opt-in (default OFF): the holdback that powers transparent early-retry adds
// up to STREAM_RECOVERY.HOLDBACK_MS of time-to-first-token latency on every
// streaming request, so it must be explicitly enabled by the operator.
enabled: ["true", "1", "on"].includes(
(process.env.STREAM_RECOVERY_ENABLED || "").trim().toLowerCase()
),
// Opt-in (default OFF): mid-stream continuation re-requests after a post-commit cut.
continueMidStream: ["true", "1", "on"].includes(
(process.env.STREAM_RECOVERY_MIDSTREAM_ENABLED || "").trim().toLowerCase()
),
throughputWatchdog: {
enabled: ["true", "1", "on"].includes(
(process.env.STREAM_THROUGHPUT_WATCHDOG_ENABLED || "").trim().toLowerCase()
),
warmupMs: STREAM_THROUGHPUT_WATCHDOG.WARMUP_MS,
windowMs: STREAM_THROUGHPUT_WATCHDOG.WINDOW_MS,
minUsefulBytesPerSecond: STREAM_THROUGHPUT_WATCHDOG.MIN_USEFUL_BYTES_PER_SECOND,
minUsefulBytes: STREAM_THROUGHPUT_WATCHDOG.MIN_USEFUL_BYTES,
},
},
// #6846 Phase 1: empty by default — nvidia (and any future header-less
// provider registered in providerDefaultRateLimit.ts) uses its static
// default until an operator adds an override here.
providerQuotaOverrides: {},
// Global default cadence for the background credential health check sweep.
// 60 minutes: the sweep makes a real upstream probe against EVERY active
// connection, so the previous 5-minute default cost 12 requests/hour per
// connection. 0 disables the sweep entirely. Per-connection overrides win.
credentialHealthCheck: {
intervalMinutes: 60,
},
};
function buildLegacyFallback(settings: JsonRecord): ResilienceSettings {
const profiles = asRecord(settings.providerProfiles);
const defaults = asRecord(settings.rateLimitDefaults);
const streamRecoveryDefaults = resolveStreamRecoveryDefaults();
const oauthLegacy = asRecord(profiles.oauth);
const apikeyLegacy = asRecord(profiles.apikey);
const waitMaxRetrySec = toInteger(
settings.maxRetryIntervalSec,
DEFAULT_RESILIENCE_SETTINGS.waitForCooldown.maxRetryWaitSec,
{ min: 0, max: 300 }
);
const waitMaxRetries = toInteger(
settings.requestRetry,
DEFAULT_RESILIENCE_SETTINGS.waitForCooldown.maxRetries,
{ min: 0, max: 10 }
);
return {
requestQueue: {
autoEnableApiKeyProviders: DEFAULT_RESILIENCE_SETTINGS.requestQueue.autoEnableApiKeyProviders,
requestsPerMinute: toInteger(
defaults.requestsPerMinute,
DEFAULT_RESILIENCE_SETTINGS.requestQueue.requestsPerMinute,
{ min: 1, max: 1_000_000 }
),
minTimeBetweenRequestsMs: toInteger(
defaults.minTimeBetweenRequests,
DEFAULT_RESILIENCE_SETTINGS.requestQueue.minTimeBetweenRequestsMs,
{ min: 0, max: 60 * 60 * 1000 }
),
concurrentRequests: toInteger(
defaults.concurrentRequests,
DEFAULT_RESILIENCE_SETTINGS.requestQueue.concurrentRequests,
{ min: 1, max: 10_000 }
),
globalConcurrentRequests: DEFAULT_RESILIENCE_SETTINGS.requestQueue.globalConcurrentRequests,
maxWaitMs: DEFAULT_RESILIENCE_SETTINGS.requestQueue.maxWaitMs,
executionMaxWaitMs: DEFAULT_RESILIENCE_SETTINGS.requestQueue.executionMaxWaitMs,
maxQueueDepth: DEFAULT_RESILIENCE_SETTINGS.requestQueue.maxQueueDepth,
},
connectionCooldown: {
oauth: normalizeLegacyConnectionCooldownProfile(
oauthLegacy,
DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.oauth
),
apikey: normalizeLegacyConnectionCooldownProfile(
apikeyLegacy,
DEFAULT_RESILIENCE_SETTINGS.connectionCooldown.apikey
),
},
providerBreaker: {
oauth: {
failureThreshold: toInteger(
oauthLegacy.circuitBreakerThreshold,
DEFAULT_RESILIENCE_SETTINGS.providerBreaker.oauth.failureThreshold,
{ min: 1, max: 1000 }
),
degradationThreshold:
DEFAULT_RESILIENCE_SETTINGS.providerBreaker.oauth.degradationThreshold,
resetTimeoutMs: toInteger(
oauthLegacy.circuitBreakerReset,
DEFAULT_RESILIENCE_SETTINGS.providerBreaker.oauth.resetTimeoutMs,
{ min: 1000, max: 24 * 60 * 60 * 1000 }
),
},
apikey: {
failureThreshold: toInteger(
apikeyLegacy.circuitBreakerThreshold,
DEFAULT_RESILIENCE_SETTINGS.providerBreaker.apikey.failureThreshold,
{ min: 1, max: 1000 }
),
degradationThreshold:
DEFAULT_RESILIENCE_SETTINGS.providerBreaker.apikey.degradationThreshold,
resetTimeoutMs: toInteger(
apikeyLegacy.circuitBreakerReset,
DEFAULT_RESILIENCE_SETTINGS.providerBreaker.apikey.resetTimeoutMs,
{ min: 1000, max: 24 * 60 * 60 * 1000 }
),
},
},
waitForCooldown: {
enabled: waitMaxRetries > 0 && waitMaxRetrySec > 0,
maxRetries: waitMaxRetries,
maxRetryWaitSec: waitMaxRetrySec,
maxRetryWaitMs: waitMaxRetrySec * 1000,
budgetMs: Math.max(
waitMaxRetrySec * 1000,
DEFAULT_RESILIENCE_SETTINGS.waitForCooldown.budgetMs
),
},
comboCooldownWait: DEFAULT_RESILIENCE_SETTINGS.comboCooldownWait,
quotaShareConcurrencyLimit: DEFAULT_RESILIENCE_SETTINGS.quotaShareConcurrencyLimit,
streamStallCooldown: DEFAULT_RESILIENCE_SETTINGS.streamStallCooldown,
providerCooldown: DEFAULT_RESILIENCE_SETTINGS.providerCooldown,
tokenRefreshBreaker: DEFAULT_RESILIENCE_SETTINGS.tokenRefreshBreaker,
quotaPreflight: DEFAULT_RESILIENCE_SETTINGS.quotaPreflight,
streamRecovery: streamRecoveryDefaults,
providerQuotaOverrides: DEFAULT_RESILIENCE_SETTINGS.providerQuotaOverrides,
credentialHealthCheck: DEFAULT_RESILIENCE_SETTINGS.credentialHealthCheck,
};
}
/**
* True when the operator has an explicit stream-recovery configuration —
* either a DB/settings override (`resilienceSettings.streamRecovery.enabled`
* present as a boolean) or a non-empty `STREAM_RECOVERY_ENABLED` env var.
*
* `ResilienceSettings` has no `isExplicit`/`source` field to distinguish
* "default off" from "operator explicitly turned it off", so callers that
* need to respect an explicit operator choice (e.g. the agent-goal-policy
* override in chatCore.ts) must check this before layering any heuristic
* on top of `resolveResilienceSettings(...).streamRecovery.enabled`.
*/
export function isStreamRecoveryExplicitlyConfigured(
settings: Record<string, unknown> | null | undefined
): boolean {
const record = asRecord(settings);
const current = asRecord(record.resilienceSettings);
const streamRecoveryRecord = asRecord(current.streamRecovery);
if (typeof streamRecoveryRecord.enabled === "boolean") {
return true;
}
return (process.env.STREAM_RECOVERY_ENABLED || "").trim().length > 0;
}
export function resolveResilienceSettings(
settings: Record<string, unknown> | null | undefined
): ResilienceSettings {
const record = asRecord(settings);
const current = asRecord(record.resilienceSettings);
const fallback = buildLegacyFallback(record);
return {
requestQueue: normalizeRequestQueueSettings(current.requestQueue, fallback.requestQueue),
connectionCooldown: {
oauth: normalizeConnectionCooldownProfile(
asRecord(current.connectionCooldown).oauth,
fallback.connectionCooldown.oauth
),
apikey: normalizeConnectionCooldownProfile(
asRecord(current.connectionCooldown).apikey,
fallback.connectionCooldown.apikey
),
},
providerBreaker: {
oauth: normalizeProviderBreakerProfile(
asRecord(current.providerBreaker).oauth,
fallback.providerBreaker.oauth
),
apikey: normalizeProviderBreakerProfile(
asRecord(current.providerBreaker).apikey,
fallback.providerBreaker.apikey
),
},
tokenRefreshBreaker: normalizeTokenRefreshBreakerSettings(
current.tokenRefreshBreaker,
fallback.tokenRefreshBreaker
),
waitForCooldown: normalizeWaitForCooldownSettings(
current.waitForCooldown,
fallback.waitForCooldown
),
comboCooldownWait: normalizeComboCooldownWaitSettings(
current.comboCooldownWait,
fallback.comboCooldownWait
),
quotaShareConcurrencyLimit: normalizeQuotaShareConcurrencyLimitSettings(
current.quotaShareConcurrencyLimit,
fallback.quotaShareConcurrencyLimit
),
streamStallCooldown: normalizeStreamStallCooldownSettings(
current.streamStallCooldown,
fallback.streamStallCooldown
),
providerCooldown: normalizeProviderCooldownSettings(
current.providerCooldown,
fallback.providerCooldown
),
quotaPreflight: normalizeQuotaPreflightSettings(
current.quotaPreflight,
fallback.quotaPreflight
),
streamRecovery: normalizeStreamRecoverySettings(
current.streamRecovery,
fallback.streamRecovery
),
providerQuotaOverrides: normalizeProviderQuotaOverrides(
current.providerQuotaOverrides,
fallback.providerQuotaOverrides
),
credentialHealthCheck: normalizeCredentialHealthCheckSettings(
current.credentialHealthCheck,
fallback.credentialHealthCheck
),
};
}
export function mergeResilienceSettings(
current: ResilienceSettings,
updates: ResilienceSettingsPatch
): ResilienceSettings {
return {
requestQueue: normalizeRequestQueueSettings(updates.requestQueue, current.requestQueue),
connectionCooldown: {
oauth: normalizeConnectionCooldownProfile(
updates.connectionCooldown?.oauth,
current.connectionCooldown.oauth
),
apikey: normalizeConnectionCooldownProfile(
updates.connectionCooldown?.apikey,
current.connectionCooldown.apikey
),
},
providerBreaker: {
oauth: normalizeProviderBreakerProfile(
updates.providerBreaker?.oauth,
current.providerBreaker.oauth
),
apikey: normalizeProviderBreakerProfile(
updates.providerBreaker?.apikey,
current.providerBreaker.apikey
),
},
tokenRefreshBreaker: normalizeTokenRefreshBreakerSettings(
updates.tokenRefreshBreaker,
current.tokenRefreshBreaker
),
waitForCooldown: normalizeWaitForCooldownSettings(
updates.waitForCooldown,
current.waitForCooldown
),
comboCooldownWait: normalizeComboCooldownWaitSettings(
updates.comboCooldownWait,
current.comboCooldownWait
),
quotaShareConcurrencyLimit: normalizeQuotaShareConcurrencyLimitSettings(
updates.quotaShareConcurrencyLimit,
current.quotaShareConcurrencyLimit
),
streamStallCooldown: normalizeStreamStallCooldownSettings(
updates.streamStallCooldown,
current.streamStallCooldown
),
providerCooldown: normalizeProviderCooldownSettings(
updates.providerCooldown,
current.providerCooldown
),
quotaPreflight: normalizeQuotaPreflightSettings(updates.quotaPreflight, current.quotaPreflight),
streamRecovery: normalizeStreamRecoverySettings(updates.streamRecovery, current.streamRecovery),
providerQuotaOverrides: normalizeProviderQuotaOverrides(
updates.providerQuotaOverrides,
current.providerQuotaOverrides
),
credentialHealthCheck: normalizeCredentialHealthCheckSettings(
updates.credentialHealthCheck,
current.credentialHealthCheck
),
};
}
export function buildLegacyResilienceCompat(settings: ResilienceSettings) {
return {
profiles: {
oauth: {
transientCooldown: settings.connectionCooldown.oauth.baseCooldownMs,
rateLimitCooldown: settings.connectionCooldown.oauth.useUpstreamRetryHints
? 0
: settings.connectionCooldown.oauth.baseCooldownMs,
maxBackoffLevel: settings.connectionCooldown.oauth.maxBackoffSteps,
circuitBreakerThreshold: settings.providerBreaker.oauth.failureThreshold,
degradationThreshold: settings.providerBreaker.oauth.degradationThreshold,
circuitBreakerReset: settings.providerBreaker.oauth.resetTimeoutMs,
},
apikey: {
transientCooldown: settings.connectionCooldown.apikey.baseCooldownMs,
rateLimitCooldown: settings.connectionCooldown.apikey.useUpstreamRetryHints
? 0
: settings.connectionCooldown.apikey.baseCooldownMs,
maxBackoffLevel: settings.connectionCooldown.apikey.maxBackoffSteps,
circuitBreakerThreshold: settings.providerBreaker.apikey.failureThreshold,
degradationThreshold: settings.providerBreaker.apikey.degradationThreshold,
circuitBreakerReset: settings.providerBreaker.apikey.resetTimeoutMs,
},
},
defaults: {
requestsPerMinute: settings.requestQueue.requestsPerMinute,
minTimeBetweenRequests: settings.requestQueue.minTimeBetweenRequestsMs,
concurrentRequests: settings.requestQueue.concurrentRequests,
},
};
}