Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

feat: adaptive keepalive threshold for web-session providers (#3397)

Paijo5b72dc62506c
diff --git a/open-sse/utils/keepaliveThreshold.ts b/open-sse/utils/keepaliveThreshold.ts
new file mode 100644
index 0000000000..9dd6d32938
--- /dev/null
+++ b/open-sse/utils/keepaliveThreshold.ts
@@ -0,0 +1,62 @@
+/**
+ * Adaptive keepalive threshold resolver for streaming routes.
+ *
+ * Web-session and anonymous-fallback providers are slower to produce the first
+ * byte because they route through browser sessions or public rate-limited
+ * endpoints. The default 2 s keepalive threshold is too aggressive for these
+ * providers — the keepalive stream commits before the upstream has a chance to
+ * respond, adding unnecessary SSE framing overhead.
+ *
+ * `resolveKeepaliveThreshold(model)` inspects the model prefix and returns a
+ * longer threshold (15 s) for known-slow providers, or the default (2 s) for
+ * everything else.
+ */
+
+import { NOAUTH_PROVIDERS } from "@/shared/constants/providers";
+import { APIKEY_PROVIDERS } from "@/shared/constants/providers";
+import { WEB_COOKIE_PROVIDERS } from "@/shared/constants/providers";
+import { WEB_SESSION_CREDENTIAL_REQUIREMENTS } from "@/shared/providers/webSessionCredentials";
+
+const DEFAULT_THRESHOLD_MS = 2_000;
+const SLOW_THRESHOLD_MS = 15_000;
+
+const SLOW_PROVIDER_IDS: Set<string> = new Set();
+
+function addSlowProvider(id: string, alias?: string) {
+ SLOW_PROVIDER_IDS.add(id);
+ if (typeof alias === "string" && alias) SLOW_PROVIDER_IDS.add(alias);
+}
+
+for (const [id, def] of Object.entries(NOAUTH_PROVIDERS)) {
+ if ((def as Record<string, unknown>).noAuth === true) {
+ addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);
+ }
+}
+
+for (const [id, def] of Object.entries(APIKEY_PROVIDERS)) {
+ if ((def as Record<string, unknown>).anonymousFallback === true) {
+ addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);
+ }
+}
+
+for (const [id, def] of Object.entries(WEB_COOKIE_PROVIDERS)) {
+ addSlowProvider(id, (def as Record<string, unknown>).alias as string | undefined);
+}
+
+for (const id of Object.keys(WEB_SESSION_CREDENTIAL_REQUIREMENTS)) {
+ SLOW_PROVIDER_IDS.add(id);
+}
+
+export const SLOW_KEEPALIVE_PROVIDERS: ReadonlySet<string> = SLOW_PROVIDER_IDS;
+
+export function resolveKeepaliveThreshold(model: string | undefined | null): number {
+ if (!model || typeof model !== "string") return DEFAULT_THRESHOLD_MS;
+
+ const slashIndex = model.indexOf("/");
+ if (slashIndex <= 0) return DEFAULT_THRESHOLD_MS;
+
+ const prefix = model.slice(0, slashIndex);
+ if (SLOW_PROVIDER_IDS.has(prefix)) return SLOW_THRESHOLD_MS;
+
+ return DEFAULT_THRESHOLD_MS;
+}
diff --git a/src/app/api/v1/responses/route.ts b/src/app/api/v1/responses/route.ts
index b737ae01c1..a6c5d8218c 100644
--- a/src/app/api/v1/responses/route.ts
+++ b/src/app/api/v1/responses/route.ts
@@ -3,6 +3,7 @@ import { withEarlyStreamKeepalive } from "@omniroute/open-sse/utils/earlyStreamK
import { resolveResponsesApiModel } from "@/app/api/internal/codex-responses-ws/modelResolution";
import { getModelInfo } from "@/sse/services/model";
import { getComboByName } from "@/lib/db/combos";
+import { resolveKeepaliveThreshold } from "@omniroute/open-sse/utils/keepaliveThreshold";
// NOTE: We do NOT call initTranslators() here — the translator registry is
// bootstrapped at module level inside open-sse/translator/index.ts when it
@@ -68,7 +69,19 @@ export async function POST(request) {
const resolved = await withCodexPreferredModel(request);
const accept = String(request.headers?.get?.("accept") || "").toLowerCase();
if (accept.includes("text/event-stream")) {
- return await withEarlyStreamKeepalive(handleChat(resolved), { signal: request.signal });
+ // Adaptive threshold: web-session and anonymous-fallback providers are slower
+ // to produce the first byte, so use a longer keepalive threshold (15s vs 2s).
+ let model;
+ try {
+ const body = await resolved.clone().json().catch(() => null);
+ model = body?.model;
+ } catch {
+ }
+ const thresholdMs = resolveKeepaliveThreshold(model);
+ return await withEarlyStreamKeepalive(handleChat(resolved), {
+ signal: request.signal,
+ thresholdMs,
+ });
}
return await handleChat(resolved);
}
diff --git a/tests/unit/keepalive-threshold.test.ts b/tests/unit/keepalive-threshold.test.ts
new file mode 100644
index 0000000000..a8973d2de9
--- /dev/null
+++ b/tests/unit/keepalive-threshold.test.ts
@@ -0,0 +1,75 @@
+/**
+ * Adaptive keepalive threshold — Unit Tests (PR5 of issue #3368)
+ *
+ * Run: node --import tsx/esm --test tests/unit/keepalive-threshold.test.ts
+ */
+
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import {
+ resolveKeepaliveThreshold,
+ SLOW_KEEPALIVE_PROVIDERS,
+} from "../../open-sse/utils/keepaliveThreshold.ts";
+
+describe("resolveKeepaliveThreshold", () => {
+ it("returns 2000ms default for undefined model", () => {
+ assert.equal(resolveKeepaliveThreshold(undefined), 2000);
+ });
+
+ it("returns 2000ms default for null model", () => {
+ assert.equal(resolveKeepaliveThreshold(null), 2000);
+ });
+
+ it("returns 2000ms default for empty string", () => {
+ assert.equal(resolveKeepaliveThreshold(""), 2000);
+ });
+
+ it("returns 2000ms default for model without prefix", () => {
+ assert.equal(resolveKeepaliveThreshold("gpt-4"), 2000);
+ });
+
+ it("returns 2000ms default for normal API-key provider", () => {
+ assert.equal(resolveKeepaliveThreshold("openai/gpt-4"), 2000);
+ assert.equal(resolveKeepaliveThreshold("anthropic/claude-sonnet-4"), 2000);
+ assert.equal(resolveKeepaliveThreshold("deepseek/deepseek-chat"), 2000);
+ });
+
+ it("returns 15000ms for anonymous fallback provider (pollinations)", () => {
+ assert.equal(resolveKeepaliveThreshold("pollinations/gpt-5"), 15000);
+ });
+
+ it("returns 15000ms for anonymous fallback provider alias (pol)", () => {
+ assert.equal(resolveKeepaliveThreshold("pol/gpt-5"), 15000);
+ });
+
+ it("returns 15000ms for anonymous fallback provider (opencode-zen)", () => {
+ assert.equal(resolveKeepaliveThreshold("opencode-zen/gpt-4"), 15000);
+ });
+
+ it("returns 15000ms for web-session provider (chatgpt-web)", () => {
+ assert.equal(resolveKeepaliveThreshold("chatgpt-web/gpt-5"), 15000);
+ });
+
+ it("returns 15000ms for web-session provider (grok-web)", () => {
+ assert.equal(resolveKeepaliveThreshold("grok-web/grok-4"), 15000);
+ });
+
+ it("returns 15000ms for web-session provider (claude-web)", () => {
+ assert.equal(resolveKeepaliveThreshold("claude-web/claude-sonnet-4"), 15000);
+ });
+
+ it("SLOW_KEEPALIVE_PROVIDERS set contains expected providers", () => {
+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pollinations"));
+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("pol"));
+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("opencode-zen"));
+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("chatgpt-web"));
+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("grok-web"));
+ assert.ok(SLOW_KEEPALIVE_PROVIDERS.has("claude-web"));
+ });
+
+ it("SLOW_KEEPALIVE_PROVIDERS does not contain normal providers", () => {
+ assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("openai"));
+ assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("anthropic"));
+ assert.ok(!SLOW_KEEPALIVE_PROVIDERS.has("deepseek"));
+ });
+});