7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(providers): 60s chat timeout for zhipu, agnes, cloudflare (reasoning-model TTFB) (#521)
Tashfeene3fdd3bdbe98
diff --git a/server/src/__tests__/providers/reasoning-timeouts.test.ts b/server/src/__tests__/providers/reasoning-timeouts.test.tsnew file mode 100644index 00000000..b2d7e8f7--- /dev/null+++ b/server/src/__tests__/providers/reasoning-timeouts.test.ts@@ -0,0 +1,56 @@+import { describe, it, expect, vi, afterEach } from 'vitest';+import { resolveProvider } from '../../providers/index.js';+import { CloudflareProvider } from '../../providers/cloudflare.js';++// Regression guard for the 2026-07-11 live-sweep finding: platforms hosting+// hidden-reasoning models (zhipu glm-4.7-flash 41s TTFB, agnes-2.0-flash 20s,+// @cf/zai-org/glm-4.7-flash repeated 15s aborts) need a chat timeout above+// the 15s fetch default or every attempt — streaming included — is aborted+// before the first byte. The value is a private construction detail, so the+// registry entries are asserted via the stored field and Cloudflare via the+// setTimeout the abort rides on.++describe('reasoning-model chat timeouts', () => {+ afterEach(() => {+ vi.restoreAllMocks();+ });++ it.each([+ ['zhipu', 60_000],+ ['agnes', 60_000],+ ['ollama', 120_000], // pre-existing bump; keep it from regressing too+ ] as const)('%s is registered with a %dms chat timeout', (platform, ms) => {+ const provider = resolveProvider(platform);+ expect(provider).toBeDefined();+ expect((provider as unknown as { timeoutMs: number }).timeoutMs).toBe(ms);+ });++ it('cloudflare chat aborts on a 60s timer, not the 15s default', async () => {+ const provider = new CloudflareProvider();+ const delays: number[] = [];+ const origSetTimeout = global.setTimeout;+ vi.spyOn(global, 'setTimeout').mockImplementation(((fn: () => void, ms?: number) => {+ delays.push(ms ?? 0);+ return origSetTimeout(fn, ms);+ }) as typeof setTimeout);+ vi.spyOn(global, 'fetch').mockResolvedValue({+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-cf', object: 'chat.completion', created: 1,+ model: '@cf/zai-org/glm-4.7-flash',+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ headers: new Headers(),+ } as unknown as Response);++ await provider.chatCompletion(+ 'acct:token',+ [{ role: 'user', content: 'hi' }],+ '@cf/zai-org/glm-4.7-flash',+ );++ expect(delays).toContain(60_000);+ expect(delays).not.toContain(15_000);+ });+});diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex 7f940bf0..e5515df3 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -13,6 +13,12 @@ import { recordQuotaObservationsFromResponse, type QuotaObservationContext } fro* API key format expected: "account_id:api_token"* The account_id is extracted from the key to build the URL.*/++// Workers AI hosts reasoning models (@cf/zai-org/glm-4.7-flash) that spend+// well over the 15s fetch default before the first byte (live sweep+// 2026-07-11: repeated 15s aborts). Matches zhipu/agnes/ollama bumps.+const CHAT_TIMEOUT_MS = 60_000;+export class CloudflareProvider extends BaseProvider {readonly platform = 'cloudflare' as const;readonly name = 'Cloudflare Workers AI';@@ -59,7 +65,7 @@ export class CloudflareProvider extends BaseProvider {parallel_tool_calls: options?.parallel_tool_calls,...extendedBodyParams(this.platform, options),}),- });+ }, CHAT_TIMEOUT_MS);recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,@@ -108,7 +114,7 @@ export class CloudflareProvider extends BaseProvider {...extendedBodyParams(this.platform, options),stream: true,}),- });+ }, CHAT_TIMEOUT_MS);recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,diff --git a/server/src/providers/index.ts b/server/src/providers/index.tsindex dfd74698..80bd2a6b 100644--- a/server/src/providers/index.ts+++ b/server/src/providers/index.ts@@ -83,10 +83,17 @@ register(new CohereProvider());register(new CloudflareProvider());// Zhipu (Z.ai / bigmodel.cn) - OpenAI-compatible+//+// glm-4.7-flash is a hidden-reasoning model: it burns through a long+// reasoning_content before the first answer byte (live-probed 41s TTFB on a+// one-word completion, 2026-07-11), and Zhipu buffers that phase even when+// streaming — so the default 15s timeout aborted every attempt. 60s covers+// the observed worst case with headroom.register(new OpenAICompatProvider({platform: 'zhipu',name: 'Zhipu AI',baseUrl: 'https://open.bigmodel.cn/api/paas/v4',+ timeoutMs: 60_000,}));// Hugging Face Inference Providers router — re-added in V13. The V4 removal@@ -200,10 +207,14 @@ register(new OpenAICompatProvider({// before 429s (no documented RPM/RPD). Free key from platform.agnes-ai.com,// no card. Catalog rows live in the catalog (premium → age into free); not// shipped as freeapi model migrations.+// agnes-2.0-flash reasons before answering (live-probed 20s TTFB on a+// one-word completion, 2026-07-11), so the default 15s timeout aborted it;+// 60s matches the other reasoning-hosting platforms.register(new OpenAICompatProvider({platform: 'agnes',name: 'Agnes AI',baseUrl: 'https://apihub.agnes-ai.com/v1',+ timeoutMs: 60_000,}));// Chutes was evaluated for V11 and dropped: probe with a free-tier key