7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(routing): fail over on a relay's bare "safe"/"unsafe" classification output (#809) (#830)
suantea (asuan_suan)a961d9316dd4
diff --git a/server/src/__tests__/lib/upstream-classification.test.ts b/server/src/__tests__/lib/upstream-classification.test.tsnew file mode 100644index 00000000..b5872fa6--- /dev/null+++ b/server/src/__tests__/lib/upstream-classification.test.ts@@ -0,0 +1,55 @@+import { describe, it, expect } from 'vitest';+import { isUpstreamClassificationOutput } from '../../lib/error-classify.js';++/**+ * #809: kilo's free relay sometimes answers with a bare "safe"/"unsafe" — an+ * upstream content filter's verdict standing in for the model's reply. The+ * surfaces treat that as an empty completion and fail over.+ *+ * The rule is deliberately scoped to that relay. "safe"/"unsafe" is a perfectly+ * valid one-word answer for moderation and guard-model work, so firing it on+ * every platform would discard correct responses and burn the whole fallback+ * chain for anyone doing classification.+ */+describe('isUpstreamClassificationOutput (#809)', () => {+ it('flags a bare classification verdict from the kilo relay', () => {+ expect(isUpstreamClassificationOutput('safe', 'kilo')).toBe(true);+ expect(isUpstreamClassificationOutput('unsafe', 'kilo')).toBe(true);+ });++ it('ignores surrounding whitespace and case', () => {+ expect(isUpstreamClassificationOutput(' Safe \n', 'kilo')).toBe(true);+ expect(isUpstreamClassificationOutput('UNSAFE', 'kilo')).toBe(true);+ });++ it('leaves the same answer alone on every other platform', () => {+ // A moderation/guard workload legitimately answering one word must survive.+ for (const platform of ['groq', 'cerebras', 'openrouter', 'custom', 'gemini']) {+ expect(isUpstreamClassificationOutput('safe', platform)).toBe(false);+ expect(isUpstreamClassificationOutput('unsafe', platform)).toBe(false);+ }+ });++ it('does not fire when the platform is unknown', () => {+ expect(isUpstreamClassificationOutput('safe')).toBe(false);+ expect(isUpstreamClassificationOutput('safe', '')).toBe(false);+ expect(isUpstreamClassificationOutput('safe', undefined)).toBe(false);+ });++ it('matches the platform case-insensitively', () => {+ expect(isUpstreamClassificationOutput('safe', 'Kilo')).toBe(true);+ });++ it('leaves a real answer that merely contains the word alone', () => {+ expect(isUpstreamClassificationOutput('This code is safe to run.', 'kilo')).toBe(false);+ expect(isUpstreamClassificationOutput('safe mode', 'kilo')).toBe(false);+ expect(isUpstreamClassificationOutput('unsafe: buffer overflow on line 12', 'kilo')).toBe(false);+ });++ it('ignores empty and non-string content', () => {+ expect(isUpstreamClassificationOutput('', 'kilo')).toBe(false);+ expect(isUpstreamClassificationOutput(null, 'kilo')).toBe(false);+ expect(isUpstreamClassificationOutput(undefined, 'kilo')).toBe(false);+ expect(isUpstreamClassificationOutput(42, 'kilo')).toBe(false);+ });+});diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 3ea8bd41..72d60090 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -282,6 +282,25 @@ export function isProviderLevelError(err: any): boolean {|| msg.includes('fetch failed'); // undici transport error (DNS/TLS/proxy down)}+// #809: kilo's free relay occasionally answers with a bare classification+// word ("safe" / "unsafe") instead of a real reply — an upstream content+// filter, not the requested model. Such a turn is a dead turn: treat it like+// an empty completion so the fallback loop fails over to the next provider+// instead of surfacing "safe"/"unsafe" as the answer.+//+// Scoped to the relay that actually does this. "safe"/"unsafe" is a LEGITIMATE+// one-word answer for moderation and guard-model workloads, so applying the+// rule everywhere would throw away correct responses and burn the whole+// fallback chain for anyone doing classification. Only platforms observed+// injecting a filter verdict in place of the model's reply belong here.+const CLASSIFICATION_RELAY_PLATFORMS = new Set(['kilo']);++export function isUpstreamClassificationOutput(text: unknown, platform?: string): boolean {+ if (!platform || !CLASSIFICATION_RELAY_PLATFORMS.has(platform.toLowerCase())) return false;+ const t = (typeof text === 'string' ? text : '').trim().toLowerCase();+ return t === 'safe' || t === 'unsafe';+}+// Provider-side 400s are retryable because another provider may accept the same// request shape. If every routed provider rejects it, however, the client should// see an invalid-request error rather than a misleading rate-limit exhaustion.diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.tsindex 50533940..3aa80979 100644--- a/server/src/lib/inbound-chat.ts+++ b/server/src/lib/inbound-chat.ts@@ -41,7 +41,7 @@ import {startsWithDialectMarker,} from './tool-call-rescue.js';import { sanitizeProviderErrorMessage } from './error-redaction.js';-import { newClientAbortError } from './error-classify.js';+import { newClientAbortError, isUpstreamClassificationOutput } from './error-classify.js';import { logRequest } from './request-log.js';import { getStickyModel, setStickyModel } from '../routes/proxy.js';import type { CompletionOptions } from '../providers/base.js';@@ -259,6 +259,15 @@ export async function runInboundChat(result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},);}+ // #809: a bare "safe"/"unsafe" classification word from a relay is an+ // upstream filter, not the requested model — fail over like an empty+ // completion.+ if (isUpstreamClassificationOutput(text, route.platform) && toolCalls.length === 0) {+ throw Object.assign(+ new Error(`empty completion from ${route.displayName} (upstream classification output)`),+ result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},+ );+ }if (wantsTools && text && toolCalls.length === 0) {const rescue = rescueInlineToolCalls(text,@@ -464,6 +473,16 @@ export async function runInboundChat(finishReason === 'length' ? { skipBench: true } : {},);}+ // #809: bare "safe"/"unsafe" classification output from a relay is an+ // upstream filter, not the requested model — fail over like an empty+ // completion.+ if (isUpstreamClassificationOutput(text, route.platform) && toolCalls.length === 0) {+ if (clientGone) return 'committed';+ throw Object.assign(+ new Error(`empty completion from ${route.displayName} (upstream classification output)`),+ finishReason === 'length' ? { skipBench: true } : {},+ );+ }const normalized: InboundChatResult = {route,diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex e3c0df99..9b66aea5 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -17,7 +17,7 @@ import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidaimport { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropic-documents.js';-import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';+import { isClientAbortError, newClientAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';@@ -595,6 +595,15 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},);}+ // #809: bare "safe"/"unsafe" classification output from a relay is an+ // upstream filter, not the requested model — fail over like an empty+ // completion.+ if (isUpstreamClassificationOutput(respText, route.platform) && respToolCalls.length === 0) {+ throw Object.assign(+ new Error(`empty completion from ${route.displayName} (upstream classification output)`),+ result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},+ );+ }// Inline tool-call dialect rescue (#231): a tool-bearing request answered// with the call serialized as TEXT → re-parse it into structured tool_usediff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex da511ee2..6ceb9fab 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -18,7 +18,7 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError } from '../lib/error-classify.js';+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { observeServedModel } from '../lib/served-model.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';@@ -1139,6 +1139,15 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},);}+ // #809: a bare "safe"/"unsafe" classification word from a relay is an+ // upstream filter, not the requested model — fail over like an empty+ // completion.+ if (isUpstreamClassificationOutput(text, route.platform)) {+ throw Object.assign(+ new Error(`empty completion from ${route.displayName} (upstream classification output)`),+ result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},+ );+ }// Usage fallback: providers that omit `usage` used to be logged as 0// tokens, silently undercounting analytics and the rate-limit ledger.@@ -2086,6 +2095,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {upstreamFinish === 'length' ? { skipBench: true } : {},);}+ // #809: a bare "safe"/"unsafe" classification word streamed by a+ // relay is an upstream filter, not the requested model — fail over+ // like an empty completion.+ if (isUpstreamClassificationOutput(heldText, route.platform) && completedCalls.length === 0) {+ throw Object.assign(+ new Error(`empty completion from ${route.displayName} (upstream classification output)`),+ upstreamFinish === 'length' ? { skipBench: true } : {},+ );+ }flushHeaders();if (heldText.length > 0) {@@ -2198,6 +2216,15 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},);}+ // #809: a bare "safe"/"unsafe" classification word from a relay is an+ // upstream filter, not the requested model — fail over like an empty+ // completion.+ if (isUpstreamClassificationOutput(respText, route.platform) && (respMsg?.tool_calls?.length ?? 0) === 0) {+ throw Object.assign(+ new Error(`empty completion from ${route.displayName} (upstream classification output)`),+ result.choices?.[0]?.finish_reason === 'length' ? { skipBench: true } : {},+ );+ }// Inline tool-call dialect rescue (#231 audit): a tool-bearing// request answered with the call serialized as TEXT (a mid-