Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(fallback): bench a route after consecutive truncated streams (#1218) (#1221)

suantea (asuan_suan)1153a7cc5663
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index a02a151d..696c57dd 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -21,6 +21,7 @@ import {
recordRetryableFailure,
recordUpstreamSuccess,
resetEmptyCompletionStreaks,
+ resetTruncationStreaks,
exhaustedRetryError,
formatAttemptTrail,
classifyAttemptError,
@@ -29,6 +30,8 @@ import {
DEFAULT_FALLBACK_TIME_BUDGET_MS,
AUTH_FAILURE_COOLDOWN_MS,
EMPTY_COMPLETION_STREAK_LIMIT,
+ TRUNCATION_STREAK_LIMIT,
+ TRUNCATION_BENCH_MS,
type AttemptRecord,
type FallbackHooks,
type FallbackState,
@@ -83,6 +86,7 @@ beforeEach(() => {
mockCheckKeyHealth.mockResolvedValue('invalid');
getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
resetEmptyCompletionStreaks();
+ resetTruncationStreaks();
});
describe('isKeyAuthError (401 = key-fatal, rotate instead of 502)', () => {
@@ -278,6 +282,64 @@ describe('empty-completion streak lifts the skipBench exemption (#751)', () => {
});
});
+describe('truncated-stream streak benches a sick route (#1218)', () => {
+ const truncErr = (name: string) =>
+ new Error(`${name} stream ended unexpectedly (no [DONE], no finish_reason) — connection reset or truncated upstream`);
+ const cooldownExpiry = (route: RouteResult): number | undefined => {
+ const row = getDb().prepare(
+ 'SELECT expires_at_ms FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?',
+ ).get('fake', route.keyId) as { expires_at_ms: number } | undefined;
+ return row?.expires_at_ms;
+ };
+
+ it('a single truncation keeps the ordinary short transient bench', () => {
+ const route = fakeRoute();
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ // The retryable ladder already benches briefly; the streak adds nothing yet.
+ const expiry = cooldownExpiry(route);
+ expect(expiry).toBeDefined();
+ expect(expiry! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);
+ });
+
+ it(`extends the bench to TRUNCATION_BENCH_MS from the Nth consecutive truncation on the same model+key`, () => {
+ const route = fakeRoute();
+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ expect(cooldownExpiry(route)! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);
+ }
+ // Streak limit reached: the bench is now the full truncation window.
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ expect(cooldownExpiry(route)! - Date.now()).toBeGreaterThanOrEqual(TRUNCATION_BENCH_MS - 1000);
+ });
+
+ it('a success on the route resets the streak', () => {
+ const route = fakeRoute();
+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ }
+ recordUpstreamSuccess(route, 0);
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ }
+ expect(cooldownExpiry(route)! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);
+ });
+
+ it('a differently-classified failure on the route breaks the streak', () => {
+ const route = fakeRoute();
+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ }
+ // A 429 takes the cooldown ladder — and breaks the truncation streak.
+ recordRetryableFailure(route, Object.assign(new Error('429 Too Many Requests'), { status: 429 }), newFallbackState());
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ for (let i = 1; i < TRUNCATION_STREAK_LIMIT; i++) {
+ recordRetryableFailure(route, truncErr(route.displayName), newFallbackState());
+ }
+ expect(cooldownExpiry(route)! - Date.now()).toBeLessThan(TRUNCATION_BENCH_MS);
+ });
+});
+
describe('exhaustedRetryError attempt trail + auth exhaustion', () => {
it('all-auth attempts produce a distinct 502 provider_error body, not a rate-limit 429', () => {
const body = exhaustedRetryError(new Error('Groq API error 401: Invalid API Key'), 20, { attempts: authRecord(3) });
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index c4a26cac..cf645938 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -652,3 +652,28 @@ export function modelRetirementSignal(err: any): ModelRetirementConfidence | nul
if (!isModelNotFoundError(err)) return null;
return MODEL_GONE_PHRASES.some(phrase => msg.includes(phrase)) ? 'probable' : null;
}
+
+// A stream that ended without its terminal marker (`[DONE]` and/or a
+// finish_reason): the upstream connection was reset or the response truncated
+// mid-generation. Observed live on Kilo Gateway (5 attempts in one session,
+// #1218): the gateway answers 200, streams content, then dies without
+// `data: [DONE]` — `readSseStream` throws
+// "…stream ended unexpectedly (no [DONE], no finish_reason)".
+//
+// The truncation is UPSTREAM transport, not request shape, so the plain
+// retryable path (a fresh attempt on a sibling key, or even the same key
+// later) is the right response — but a stream that dies with zero content
+// deltas is usually the ROUTE (platform+model+key edge) that is sick, not
+// bad luck: three empty-ended truncations in a row on the same route
+// reliably precede another one. Callers use this signal to bench the route
+// after a short streak instead of re-paying the round trip every request.
+export function isStreamTruncatedError(err: any): boolean {
+ const msg = (err?.message ?? '').toLowerCase();
+ return msg.includes('stream ended unexpectedly')
+ || msg.includes('no [done], no finish_reason')
+ // undici surfaces an abrupt RST as "terminated" on the body read; with
+ // our SSE reader the top-level message is "terminated" and the real
+ // cause is buried in err.cause (see isTransportError). Count it too:
+ // a terminated mid-stream body is the same dead route either way.
+ || msg === 'terminated';
+}
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index ba6a1234..6ec28d7f 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -48,6 +48,7 @@ import {
isProviderLevelError,
isContextTooLargeError,
isTimeoutErrorText,
+ isStreamTruncatedError,
} from './error-classify.js';
import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';
import { parseProviderReportedSize } from './provider-size-parser.js';
@@ -289,6 +290,23 @@ export function cooldownDecisionForError(route: RouteResult, err: any): Cooldown
);
}
+// ── Truncated-stream streak (#1218) ──────────────────────────────────────────
+// A stream that answers 200, sends partial SSE, then dies without [DONE] /
+// finish_reason ("stream ended unexpectedly") is retried normally — one
+// truncation is common on flaky free gateways. But the same route producing
+// them back-to-back is a sick edge: bench it briefly so the ladder stops
+// re-paying the round trip. Success on the route resets the streak.
+export const TRUNCATION_STREAK_LIMIT = 3;
+// 5 min vs the ordinary 90s transient bench: the streak must add REAL distance
+// over the default or it changes nothing — the escalation ladder already gives
+// a single truncation 90s.
+export const TRUNCATION_BENCH_MS = 5 * 60 * 1000;
+const truncationStreaks = new Map<string, number>(); // "platform:modelId:keyId"
+
+export function resetTruncationStreaks(): void {
+ truncationStreaks.clear();
+}
+
// ── Empty-completion streak (issue #751) ─────────────────────────────────────
// The skipBench exemption below assumes an empty 'length' completion is
// REQUEST behavior (this turn's max_tokens spent on hidden reasoning). A
@@ -418,6 +436,27 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (consumeSkipBenchExemption(route, err)) return true;
const decision = cooldownDecisionForError(route, err);
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
+ // A truncated stream (200 + partial SSE, no [DONE]/finish_reason) on the
+ // SAME route repeatedly is a sick edge, not bad luck — but a single one is
+ // common enough on flaky free gateways that benching on first sight would
+ // over-fire. Streak-bounded like empty completions (#751): N truncations in
+ // a row on this platform+model+key bench the route for one cooldown window;
+ // a success resets the streak (recordUpstreamSuccess).
+ if (isStreamTruncatedError(err)) {
+ const tk = `${route.platform}:${route.modelId}:${route.keyId}`;
+ const streak = (truncationStreaks.get(tk) ?? 0) + 1;
+ if (streak >= TRUNCATION_STREAK_LIMIT) {
+ truncationStreaks.delete(tk);
+ setCooldown(route.platform, route.modelId, route.keyId, TRUNCATION_BENCH_MS, 'heuristic');
+ console.warn(`[FallbackLoop] ${route.platform} ${route.modelId} key ${route.keyId}: ${streak} consecutive truncated streams — benching the route for ${Math.round(TRUNCATION_BENCH_MS / 1000)}s`);
+ } else {
+ truncationStreaks.set(tk, streak);
+ }
+ } else {
+ // Any other failure class on this route breaks the truncation streak: the
+ // cooldown ladder is already handling whatever that is.
+ truncationStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+ }
// A suspended ACCOUNT fails every model behind the key identically. The
// per-route cooldown above benches only the model that happened to be
// tried, so the next request picks the platform's next model and pays the
@@ -508,6 +547,9 @@ export function recordUpstreamSuccess(route: RouteResult, rateLimitTokens: numbe
// A served request proves the model+key can complete: the empty-completion
// streak (#751) starts over.
emptyCompletionStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+ // A served request is the strongest evidence the route's edge is alive:
+ // the truncated-stream streak (#1218) starts over too.
+ truncationStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
// A served request is the strongest possible evidence the model works, so
// clear any model-level failure streak that could bench it later.
clearModelFailure(route);