7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(fallback): bench a route after consecutive truncated streams (#1218) (#1221)
suantea (asuan_suan)1153a7cc5663
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.tsindex a02a151d..696c57dd 100644--- a/server/src/__tests__/lib/fallback-loop.test.ts+++ b/server/src/__tests__/lib/fallback-loop.test.ts@@ -21,6 +21,7 @@ import {recordRetryableFailure,recordUpstreamSuccess,resetEmptyCompletionStreaks,+ resetTruncationStreaks,exhaustedRetryError,formatAttemptTrail,classifyAttemptError,@@ -29,6 +30,8 @@ import {DEFAULT_FALLBACK_TIME_BUDGET_MS,AUTH_FAILURE_COOLDOWN_MS,EMPTY_COMPLETION_STREAK_LIMIT,+ TRUNCATION_STREAK_LIMIT,+ TRUNCATION_BENCH_MS,type AttemptRecord,type FallbackHooks,type FallbackState,@@ -83,6 +86,7 @@ beforeEach(() => {mockCheckKeyHealth.mockResolvedValue('invalid');getDb().prepare('DELETE FROM rate_limit_cooldowns').run();resetEmptyCompletionStreaks();+ resetTruncationStreaks();});describe('isKeyAuthError (401 = key-fatal, rotate instead of 502)', () => {@@ -278,6 +282,64 @@ describe('empty-completion streak lifts the skipBench exemption (#751)', () => {});});+describe('truncated-stream streak benches a sick route (#1218)', () => {+ const truncErr = (name: string) =>+ new Error(`${name} stream ended unexpectedly (no [DONE], no finish_reason) — connection reset or truncated upstream`);+ const cooldownExpiry = (route: RouteResult): number | undefined => {+ const row = getDb().prepare(+ 'SELECT expires_at_ms FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?',+ ).get('fake', route.keyId) as { expires_at_ms: number } | undefined;+ return row?.expires_at_ms;+ };++ it('a single truncation keeps the ordinary short transient bench', () => {+ const route = fakeRoute();+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ // The retryable ladder already benches briefly; the streak adds nothing yet.+ const expiry = cooldownExpiry(route);+ expect(expiry).toBeDefined();+ expect(expiry! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);+ });++ it(`extends the bench to TRUNCATION_BENCH_MS from the Nth consecutive truncation on the same model+key`, () => {+ const route = fakeRoute();+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ expect(cooldownExpiry(route)! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);+ }+ // Streak limit reached: the bench is now the full truncation window.+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ expect(cooldownExpiry(route)! - Date.now()).toBeGreaterThanOrEqual(TRUNCATION_BENCH_MS - 1000);+ });++ it('a success on the route resets the streak', () => {+ const route = fakeRoute();+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ }+ recordUpstreamSuccess(route, 0);+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ }+ expect(cooldownExpiry(route)! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);+ });++ it('a differently-classified failure on the route breaks the streak', () => {+ const route = fakeRoute();+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ }+ // A 429 takes the cooldown ladder — and breaks the truncation streak.+ recordRetryableFailure(route, Object.assign(new Error('429 Too Many Requests'), { status: 429 }), newFallbackState());+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());+ }+ expect(cooldownExpiry(route)! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);+ });+});+describe('exhaustedRetryError attempt trail + auth exhaustion', () => {it('all-auth attempts produce a distinct 502 provider_error body, not a rate-limit 429', () => {const body = exhaustedRetryError(new Error('Groq API error 401: Invalid API Key'), 20, { attempts: authRecord(3) });diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex c4a26cac..cf645938 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -652,3 +652,28 @@ export function modelRetirementSignal(err: any): ModelRetirementConfidence | nulif (!isModelNotFoundError(err)) return null;return MODEL_GONE_PHRASES.some(phrase => msg.includes(phrase)) ? 'probable' : null;}++// A stream that ended without its terminal marker (`[DONE]` and/or a+// finish_reason): the upstream connection was reset or the response truncated+// mid-generation. Observed live on Kilo Gateway (5 attempts in one session,+// #1218): the gateway answers 200, streams content, then dies without+// `data: [DONE]` — `readSseStream` throws+// "…stream ended unexpectedly (no [DONE], no finish_reason)".+//+// The truncation is UPSTREAM transport, not request shape, so the plain+// retryable path (a fresh attempt on a sibling key, or even the same key+// later) is the right response — but a stream that dies with zero content+// deltas is usually the ROUTE (platform+model+key edge) that is sick, not+// bad luck: three empty-ended truncations in a row on the same route+// reliably precede another one. Callers use this signal to bench the route+// after a short streak instead of re-paying the round trip every request.+export function isStreamTruncatedError(err: any): boolean {+ const msg = (err?.message ?? '').toLowerCase();+ return msg.includes('stream ended unexpectedly')+ || msg.includes('no [done], no finish_reason')+ // undici surfaces an abrupt RST as "terminated" on the body read; with+ // our SSE reader the top-level message is "terminated" and the real+ // cause is buried in err.cause (see isTransportError). Count it too:+ // a terminated mid-stream body is the same dead route either way.+ || msg === 'terminated';+}diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex ba6a1234..6ec28d7f 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -48,6 +48,7 @@ import {isProviderLevelError,isContextTooLargeError,isTimeoutErrorText,+ isStreamTruncatedError,} from './error-classify.js';import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';import { parseProviderReportedSize } from './provider-size-parser.js';@@ -289,6 +290,23 @@ export function cooldownDecisionForError(route: RouteResult, err: any): Cooldown);}+// ── Truncated-stream streak (#1218) ──────────────────────────────────────────+// A stream that answers 200, sends partial SSE, then dies without [DONE] /+// finish_reason ("stream ended unexpectedly") is retried normally — one+// truncation is common on flaky free gateways. But the same route producing+// them back-to-back is a sick edge: bench it briefly so the ladder stops+// re-paying the round trip. Success on the route resets the streak.+export const TRUNCATION_STREAK_LIMIT = 3;+// 5 min vs the ordinary 90s transient bench: the streak must add REAL distance+// over the default or it changes nothing — the escalation ladder already gives+// a single truncation 90s.+export const TRUNCATION_BENCH_MS = 5 * 60 * 1000;+const truncationStreaks = new Map<string, number>(); // "platform:modelId:keyId"++export function resetTruncationStreaks(): void {+ truncationStreaks.clear();+}+// ── Empty-completion streak (issue #751) ─────────────────────────────────────// The skipBench exemption below assumes an empty 'length' completion is// REQUEST behavior (this turn's max_tokens spent on hidden reasoning). A@@ -418,6 +436,27 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (consumeSkipBenchExemption(route, err)) return true;const decision = cooldownDecisionForError(route, err);setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);+ // A truncated stream (200 + partial SSE, no [DONE]/finish_reason) on the+ // SAME route repeatedly is a sick edge, not bad luck — but a single one is+ // common enough on flaky free gateways that benching on first sight would+ // over-fire. Streak-bounded like empty completions (#751): N truncations in+ // a row on this platform+model+key bench the route for one cooldown window;+ // a success resets the streak (recordUpstreamSuccess).+ if (isStreamTruncatedError(err)) {+ const tk = `${route.platform}:${route.modelId}:${route.keyId}`;+ const streak = (truncationStreaks.get(tk) ?? 0) + 1;+ if (streak >= TRUNCATION_STREAK_LIMIT) {+ truncationStreaks.delete(tk);+ setCooldown(route.platform, route.modelId, route.keyId, TRUNCATION_BENCH_MS, 'heuristic');+ console.warn(`[FallbackLoop] ${route.platform} ${route.modelId} key ${route.keyId}: ${streak} consecutive truncated streams — benching the route for ${Math.round(TRUNCATION_BENCH_MS / 1000)}s`);+ } else {+ truncationStreaks.set(tk, streak);+ }+ } else {+ // Any other failure class on this route breaks the truncation streak: the+ // cooldown ladder is already handling whatever that is.+ truncationStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);+ }// A suspended ACCOUNT fails every model behind the key identically. The// per-route cooldown above benches only the model that happened to be// tried, so the next request picks the platform's next model and pays the@@ -508,6 +547,9 @@ export function recordUpstreamSuccess(route: RouteResult, rateLimitTokens: numbe// A served request proves the model+key can complete: the empty-completion// streak (#751) starts over.emptyCompletionStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);+ // A served request is the strongest evidence the route's edge is alive:+ // the truncated-stream streak (#1218) starts over too.+ truncationStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);// A served request is the strongest possible evidence the model works, so// clear any model-level failure streak that could bench it later.clearModelFailure(route);