Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
feat: adaptive keepalive threshold for web-session providers (#3397)
Paijo5b72dc62506c
diff --git a/open-sse/utils/keepaliveThreshold.ts b/open-sse/utils/keepaliveThreshold.tsnew file mode 100644index 0000000000..9dd6d32938--- /dev/null+++ b/open-sse/utils/keepaliveThreshold.ts@@ -0,0 +1,62 @@+/**+ * Adaptive keepalive threshold resolver for streaming routes.+ *+ * Web-session and anonymous-fallback providers are slower to produce the first+ * byte because they route through browser sessions or public rate-limited+ * endpoints. The default 2 s keepalive threshold is too aggressive for these+ * providers — the keepalive stream commits before the upstream has a chance to+ * respond, adding unnecessary SSE framing overhead.+ *+ * `resolveKeepaliveThreshold(model)` inspects the model prefix and returns a+ * longer threshold (15 s) for known-slow providers, or the default (2 s) for+ * everything else.+ */++import { NOAUTH_PROVIDERS } from "@/shared/constants/providers";+import { APIKEY_PROVIDERS } from "@/shared/constants/providers";+import { WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers";+import { WEB_SESSION_CREDENTIAL_REQUIREMENTS } from "@/shared/providers/webSessionCredentials";++const DEFAULT_THRESHOLD_MS = 2_000;+const SLOW_THRESHOLD_MS = 15_000;++const SLOW_PROVIDER_IDS: Set<string> = new Set();++function addSlowProvider(id: string, alias?: string) {+ SLOW_PROVIDER_IDS.add(id);+ if (typeof alias === "string" && alias) SLOW_PROVIDER_IDS.add(alias);+}++for (const [id, def] of Object.entries(NOAUTH_PROVIDERS)) {+ if ((def as Record<string, unknown>).noAuth === true) {+ addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);+ }+}++for (const [id, def] of Object.entries(APIKEY_PROVIDERS)) {+ if ((def as Record<string, unknown>).anonymousFallback === true) {+ addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);+ }+}++for (const [id, def] of Object.entries(WEB_COOKIE_PROVIDERS)) {+ addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);+}++for (const id of Object.keys(WEB_SESSION_CREDENTIAL_REQUIREMENTS)) {+ SLOW_PROVIDER_IDS.add(id);+}++export const SLOW_KEEPALIVE_PROVIDERS: ReadonlySet<string> = SLOW_PROVIDER_IDS;++export function resolveKeepaliveThreshold(model: string | undefined | null): number {+ if (!model || typeof model !== "string") return DEFAULT_THRESHOLD_MS;++ const slashIndex = model.indexOf("/");+ if (slashIndex <= 0) return DEFAULT_THRESHOLD_MS;++ const prefix = model.slice(0, slashIndex);+ if (SLOW_PROVIDER_IDS.has(prefix)) return SLOW_THRESHOLD_MS;++ return DEFAULT_THRESHOLD_MS;+}diff --git a/src/app/api/v1/responses/route.ts b/src/app/api/v1/responses/route.tsindex b737ae01c1..a6c5d8218c 100644--- a/src/app/api/v1/responses/route.ts+++ b/src/app/api/v1/responses/route.ts@@ -3,6 +3,7 @@ import { withEarlyStreamKeepalive } from "@omniroute/open-sse/utils/earlyStreamKimport { resolveResponsesApiModel } from "@/app/api/internal/codex-responses-ws/modelResolution";import { getModelInfo } from "@/sse/services/model";import { getComboByName } from "@/lib/db/combos";+import { resolveKeepaliveThreshold } from "@omniroute/open-sse/utils/keepaliveThreshold";// NOTE: We do NOT call initTranslators() here — the translator registry is// bootstrapped at module level inside open-sse/translator/index.ts when it@@ -68,7 +69,19 @@ export async function POST(request) {const resolved = await withCodexPreferredModel(request);const accept = String(request.headers?.get?.("accept") || "").toLowerCase();if (accept.includes("text/event-stream")) {- return await withEarlyStreamKeepalive(handleChat(resolved), { signal: request.signal });+ // Adaptive threshold: web-session and anonymous-fallback providers are slower+ // to produce the first byte, so use a longer keepalive threshold (15s vs 2s).+ let model;+ try {+ const body = await resolved.clone().json().catch(() => null);+ model = body?.model;+ } catch {+ }+ const thresholdMs = resolveKeepaliveThreshold(model);+ return await withEarlyStreamKeepalive(handleChat(resolved), {+ signal: request.signal,+ thresholdMs,+ });}return await handleChat(resolved);}diff --git a/tests/unit/keepalive-threshold.test.ts b/tests/unit/keepalive-threshold.test.tsnew file mode 100644index 0000000000..a8973d2de9--- /dev/null+++ b/tests/unit/keepalive-threshold.test.ts@@ -0,0 +1,75 @@+/**+ * Adaptive keepalive threshold — Unit Tests (PR5 of issue #3368)+ *+ * Run: node --import tsx/esm --test tests/unit/keepalive-threshold.test.ts+ */++import { describe, it } from "node:test";+import assert from "node:assert/strict";+import {+ resolveKeepaliveThreshold,+ SLOW_KEEPALIVE_PROVIDERS,+} from "../../open-sse/utils/keepaliveThreshold.ts";++describe("resolveKeepaliveThreshold", () => {+ it("returns 2000ms default for undefined model", () => {+ assert.equal(resolveKeepaliveThreshold(undefined), 2000);+ });++ it("returns 2000ms default for null model", () => {+ assert.equal(resolveKeepaliveThreshold(null), 2000);+ });++ it("returns 2000ms default for empty string", () => {+ assert.equal(resolveKeepaliveThreshold(""), 2000);+ });++ it("returns 2000ms default for model without prefix", () => {+ assert.equal(resolveKeepaliveThreshold("gpt-4"), 2000);+ });++ it("returns 2000ms default for normal API-key provider", () => {+ assert.equal(resolveKeepaliveThreshold("openai/gpt-4"), 2000);+ assert.equal(resolveKeepaliveThreshold("anthropic/claude-sonnet-4"), 2000);+ assert.equal(resolveKeepaliveThreshold("deepseek/deepseek-chat"), 2000);+ });++ it("returns 15000ms for anonymous fallback provider (pollinations)", () => {+ assert.equal(resolveKeepaliveThreshold("pollinations/gpt-5"), 15000);+ });++ it("returns 15000ms for anonymous fallback provider alias (pol)", () => {+ assert.equal(resolveKeepaliveThreshold("pol/gpt-5"), 15000);+ });++ it("returns 15000ms for anonymous fallback provider (opencode-zen)", () => {+ assert.equal(resolveKeepaliveThreshold("opencode-zen/gpt-4"), 15000);+ });++ it("returns 15000ms for web-session provider (chatgpt-web)", () => {+ assert.equal(resolveKeepaliveThreshold("chatgpt-web/gpt-5"), 15000);+ });++ it("returns 15000ms for web-session provider (grok-web)", () => {+ assert.equal(resolveKeepaliveThreshold("grok-web/grok-4"), 15000);+ });++ it("returns 15000ms for web-session provider (claude-web)", () => {+ assert.equal(resolveKeepaliveThreshold("claude-web/claude-sonnet-4"), 15000);+ });++ it("SLOW_KEEPALIVE_PROVIDERS set contains expected providers", () => {+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pollinations"));+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pol"));+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("opencode-zen"));+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("chatgpt-web"));+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("grok-web"));+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("claude-web"));+ });++ it("SLOW_KEEPALIVE_PROVIDERS does not contain normal providers", () => {+ assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("openai"));+ assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("anthropic"));+ assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("deepseek"));+ });+});