Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(provider): distinguish Cloudflare-truncated 200 from non-OpenAI-compat endpoint (#430) (#439)

Jasnoor Gill7c16a662e93d
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.ts
index 173554dd..3c4b47b1 100644
--- a/server/src/__tests__/providers/openai-compat.test.ts
+++ b/server/src/__tests__/providers/openai-compat.test.ts
@@ -159,6 +159,89 @@ describe('OpenAICompatProvider', () => {
).rejects.toThrow(/not OpenAI-compatible/);
});
+ it('distinguishes a truncated 200 body (CDN keepalive) from a wrong-endpoint 200 body (#430)', async () => {
+ // Cloudflare-fronted free-tier upstreams (Kilo routing Nemotron / Poolside Laguna)
+ // deliver a partial JSON body when the 600s edge idle-keepalive fires mid-response.
+ // Node's JSON.parse surfaces that as "Unexpected end of JSON input" — different from
+ // the NDJSON / native-API case above. Operators were chasing a base-URL bug that
+ // didn't exist because both paths produced the same error string.
+ const headers = new Headers({ 'content-type': 'application/json' });
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ headers,
+ json: () => Promise.reject(new SyntaxError('Unexpected end of JSON input')),
+ } as any);
+
+ let caught: Error | null = null;
+ try {
+ await provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model');
+ } catch (err) {
+ caught = err as Error;
+ }
+ expect(caught).not.toBeNull();
+ expect(caught!.message).toMatch(/truncated mid-stream/);
+ expect(caught!.message).toMatch(/Cloudflare's 600s edge limit/);
+ expect(caught!.message).not.toMatch(/not OpenAI-compatible/);
+ });
+
+ it('attributes an "after JSON at position" parse error with Content-Type: application/json to CDN truncation (#430)', async () => {
+ // Same edge-keepalive case but the body parses far enough to reach a
+ // mid-token garbage character before EOF. Content-Type is the only signal
+ // that distinguishes this from a real NDJSON upstream (next test).
+ const headers = new Headers({ 'content-type': 'application/json' });
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ headers,
+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 4096 (line 1 column 4097)')),
+ } as any);
+
+ await expect(
+ provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model')
+ ).rejects.toThrow(/truncated mid-stream/);
+ });
+
+ it('does not flag NDJSON Content-Type as a truncation (#430 regression guard)', async () => {
+ // A real Ollama-style upstream announces NDJSON via Content-Type. Even if
+ // the parse error message happens to match the "after JSON at position"
+ // substring (NDJSON bodies do — the parser eats one object, then trips
+ // on the next one), we must not misclassify it as a CDN truncation.
+ const headers = new Headers({ 'content-type': 'application/x-ndjson' });
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ headers,
+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 120 (line 3 column 1)')),
+ } as any);
+
+ await expect(
+ provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model')
+ ).rejects.toThrow(/not OpenAI-compatible/);
+ });
+
+ it('does not classify NDJSON Content-Type as a truncation — explicit not.toThrow (#430)', async () => {
+ // Companion to the previous test: explicitly assert the truncation message
+ // does NOT appear, so a future refactor can't silently switch the branch.
+ const headers = new Headers({ 'content-type': 'application/x-ndjson' });
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ headers,
+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 120 (line 3 column 1)')),
+ } as any);
+
+ let caught: Error | null = null;
+ try {
+ await provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model');
+ } catch (err) {
+ caught = err as Error;
+ }
+ expect(caught).not.toBeNull();
+ expect(caught!.message).not.toMatch(/truncated mid-stream/);
+ expect(caught!.message).toMatch(/not OpenAI-compatible/);
+ });
+
it('should validate key using models endpoint', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({ ok: true, status: 200 } as any);
expect(await provider.validateKey('valid')).toBe(true);
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index d6e7ade7..e5d42e59 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -147,13 +147,51 @@ export class OpenAICompatProvider extends BaseProvider {
}
let data: ChatCompletionResponse;
+ let parseErr: unknown;
try {
data = await res.json() as ChatCompletionResponse;
- } catch {
- // A 200 whose body isn't a single JSON document — typically a base URL
- // pointing at a non-OpenAI-compatible API (e.g. Ollama's native NDJSON
- // /api endpoints instead of /v1, #189). Surface what's wrong instead of
- // the raw JSON.parse position error.
+ } catch (err) {
+ parseErr = err;
+ data = undefined as unknown as ChatCompletionResponse;
+ }
+ if (!data) {
+ // A 200 whose body isn't a single JSON document. Two distinct causes:
+ // (1) base URL points at a non-OpenAI-compatible API (Ollama's native
+ // NDJSON /api endpoints, llama.cpp's non-/v1 server, etc., #189).
+ // Typical signals: Content-Type is application/x-ndjson or
+ // text/event-stream; parser sees a SECOND JSON object after the
+ // first one ends ("Unexpected non-whitespace character after JSON
+ // at position <n> (line <n> column <n>)") where <n> sits inside
+ // the whitespace between two valid JSON documents.
+ // (2) the upstream connection was cut short mid-response — most often
+ // Cloudflare's 600-second edge idle keepalive dropping a slow
+ // free-tier queue (Kilo provider, NVIDIA nemotron / Poolside
+ // Laguna models on Cloudflare-fronted upstreams). Signals: body
+ // ends inside a string or mid-token, parser sees
+ // "Unexpected end of JSON input"; Content-Type is application/json
+ // (CF proxies it transparently); latency_ms ≈ 600000.
+ // Without this split every Cloudflare-truncated request was logged as
+ // "endpoint is not OpenAI-compatible", which sent operators chasing a
+ // base-URL config bug that doesn't exist (#430).
+ const msg = parseErr instanceof Error ? parseErr.message : String(parseErr);
+ const contentType = (res.headers?.get?.('content-type') ?? '').toLowerCase();
+ const looksLikeNdjson = /ndjson|text\/event-stream|x-ndjson/.test(contentType);
+ const looksLikeJson = /application\/json/.test(contentType);
+ // Truncation = body parsed cleanly until mid-stream EOF/mid-token garbage.
+ // Only attribute it to a CDN keepalive when the upstream claims to be
+ // sending JSON (Content-Type: application/json). Without that hint,
+ // the parser is more likely choking on NDJSON, native API output, or
+ // HTML — all "wrong endpoint" cases. This is the safe default.
+ const looksTruncated =
+ /Unexpected end of JSON input/.test(msg) ||
+ (/Unexpected non-whitespace character after JSON at position/.test(msg) && looksLikeJson && !looksLikeNdjson);
+ if (looksTruncated) {
+ throw new Error(
+ `${this.name} returned 200 but the response body was truncated mid-stream ` +
+ `(likely an idle-keepalive timeout at an upstream proxy or CDN, e.g. Cloudflare's ` +
+ `600s edge limit). Retry, or switch to a faster model/upstream.`,
+ );
+ }
throw new Error(
`${this.name} returned 200 with a non-JSON body — the endpoint is not OpenAI-compatible. ` +
`Check the base URL (for Ollama use http://host:11434/v1, for llama.cpp/vLLM/LM Studio the /v1 path).`,