7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(provider): distinguish Cloudflare-truncated 200 from non-OpenAI-compat endpoint (#430) (#439)
Jasnoor Gill7c16a662e93d
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.tsindex 173554dd..3c4b47b1 100644--- a/server/src/__tests__/providers/openai-compat.test.ts+++ b/server/src/__tests__/providers/openai-compat.test.ts@@ -159,6 +159,89 @@ describe('OpenAICompatProvider', () => {).rejects.toThrow(/not OpenAI-compatible/);});+ it('distinguishes a truncated 200 body (CDN keepalive) from a wrong-endpoint 200 body (#430)', async () => {+ // Cloudflare-fronted free-tier upstreams (Kilo routing Nemotron / Poolside Laguna)+ // deliver a partial JSON body when the 600s edge idle-keepalive fires mid-response.+ // Node's JSON.parse surfaces that as "Unexpected end of JSON input" — different from+ // the NDJSON / native-API case above. Operators were chasing a base-URL bug that+ // didn't exist because both paths produced the same error string.+ const headers = new Headers({ 'content-type': 'application/json' });+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({+ ok: true,+ status: 200,+ headers,+ json: () => Promise.reject(new SyntaxError('Unexpected end of JSON input')),+ } as any);++ let caught: Error | null = null;+ try {+ await provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model');+ } catch (err) {+ caught = err as Error;+ }+ expect(caught).not.toBeNull();+ expect(caught!.message).toMatch(/truncated mid-stream/);+ expect(caught!.message).toMatch(/Cloudflare's 600s edge limit/);+ expect(caught!.message).not.toMatch(/not OpenAI-compatible/);+ });++ it('attributes an "after JSON at position" parse error with Content-Type: application/json to CDN truncation (#430)', async () => {+ // Same edge-keepalive case but the body parses far enough to reach a+ // mid-token garbage character before EOF. Content-Type is the only signal+ // that distinguishes this from a real NDJSON upstream (next test).+ const headers = new Headers({ 'content-type': 'application/json' });+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({+ ok: true,+ status: 200,+ headers,+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 4096 (line 1 column 4097)')),+ } as any);++ await expect(+ provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model')+ ).rejects.toThrow(/truncated mid-stream/);+ });++ it('does not flag NDJSON Content-Type as a truncation (#430 regression guard)', async () => {+ // A real Ollama-style upstream announces NDJSON via Content-Type. Even if+ // the parse error message happens to match the "after JSON at position"+ // substring (NDJSON bodies do — the parser eats one object, then trips+ // on the next one), we must not misclassify it as a CDN truncation.+ const headers = new Headers({ 'content-type': 'application/x-ndjson' });+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({+ ok: true,+ status: 200,+ headers,+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 120 (line 3 column 1)')),+ } as any);++ await expect(+ provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model')+ ).rejects.toThrow(/not OpenAI-compatible/);+ });++ it('does not classify NDJSON Content-Type as a truncation — explicit not.toThrow (#430)', async () => {+ // Companion to the previous test: explicitly assert the truncation message+ // does NOT appear, so a future refactor can't silently switch the branch.+ const headers = new Headers({ 'content-type': 'application/x-ndjson' });+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({+ ok: true,+ status: 200,+ headers,+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 120 (line 3 column 1)')),+ } as any);++ let caught: Error | null = null;+ try {+ await provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model');+ } catch (err) {+ caught = err as Error;+ }+ expect(caught).not.toBeNull();+ expect(caught!.message).not.toMatch(/truncated mid-stream/);+ expect(caught!.message).toMatch(/not OpenAI-compatible/);+ });+it('should validate key using models endpoint', async () => {vi.spyOn(global, 'fetch').mockResolvedValueOnce({ ok: true, status: 200 } as any);expect(await provider.validateKey('valid')).toBe(true);diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex d6e7ade7..e5d42e59 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -147,13 +147,51 @@ export class OpenAICompatProvider extends BaseProvider {}let data: ChatCompletionResponse;+ let parseErr: unknown;try {data = await res.json() as ChatCompletionResponse;- } catch {- // A 200 whose body isn't a single JSON document — typically a base URL- // pointing at a non-OpenAI-compatible API (e.g. Ollama's native NDJSON- // /api endpoints instead of /v1, #189). Surface what's wrong instead of- // the raw JSON.parse position error.+ } catch (err) {+ parseErr = err;+ data = undefined as unknown as ChatCompletionResponse;+ }+ if (!data) {+ // A 200 whose body isn't a single JSON document. Two distinct causes:+ // (1) base URL points at a non-OpenAI-compatible API (Ollama's native+ // NDJSON /api endpoints, llama.cpp's non-/v1 server, etc., #189).+ // Typical signals: Content-Type is application/x-ndjson or+ // text/event-stream; parser sees a SECOND JSON object after the+ // first one ends ("Unexpected non-whitespace character after JSON+ // at position <n> (line <n> column <n>)") where <n> sits inside+ // the whitespace between two valid JSON documents.+ // (2) the upstream connection was cut short mid-response — most often+ // Cloudflare's 600-second edge idle keepalive dropping a slow+ // free-tier queue (Kilo provider, NVIDIA nemotron / Poolside+ // Laguna models on Cloudflare-fronted upstreams). Signals: body+ // ends inside a string or mid-token, parser sees+ // "Unexpected end of JSON input"; Content-Type is application/json+ // (CF proxies it transparently); latency_ms ≈ 600000.+ // Without this split every Cloudflare-truncated request was logged as+ // "endpoint is not OpenAI-compatible", which sent operators chasing a+ // base-URL config bug that doesn't exist (#430).+ const msg = parseErr instanceof Error ? parseErr.message : String(parseErr);+ const contentType = (res.headers?.get?.('content-type') ?? '').toLowerCase();+ const looksLikeNdjson = /ndjson|text\/event-stream|x-ndjson/.test(contentType);+ const looksLikeJson = /application\/json/.test(contentType);+ // Truncation = body parsed cleanly until mid-stream EOF/mid-token garbage.+ // Only attribute it to a CDN keepalive when the upstream claims to be+ // sending JSON (Content-Type: application/json). Without that hint,+ // the parser is more likely choking on NDJSON, native API output, or+ // HTML — all "wrong endpoint" cases. This is the safe default.+ const looksTruncated =+ /Unexpected end of JSON input/.test(msg) ||+ (/Unexpected non-whitespace character after JSON at position/.test(msg) && looksLikeJson && !looksLikeNdjson);+ if (looksTruncated) {+ throw new Error(+ `${this.name} returned 200 but the response body was truncated mid-stream ` ++ `(likely an idle-keepalive timeout at an upstream proxy or CDN, e.g. Cloudflare's ` ++ `600s edge limit). Retry, or switch to a faster model/upstream.`,+ );+ }throw new Error(`${this.name} returned 200 with a non-JSON body — the endpoint is not OpenAI-compatible. ` +`Check the base URL (for Ollama use http://host:11434/v1, for llama.cpp/vLLM/LM Studio the /v1 path).`,