7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(server): treat NVIDIA NIM 'DEGRADED function' 400 as provider capacity, not an invalid request
Tashaa68fc25862d
diff --git a/server/src/__tests__/lib/degraded-error.test.ts b/server/src/__tests__/lib/degraded-error.test.tsnew file mode 100644index 00000000..aa54503d--- /dev/null+++ b/server/src/__tests__/lib/degraded-error.test.ts@@ -0,0 +1,78 @@+import { describe, it, expect, beforeAll, vi } from 'vitest';++// Issue #522: NVIDIA NIM reports a temporarily-degraded hosted deployment as+// `400 {"detail":"Function id '...': DEGRADED function cannot be invoked"}`.+// That is provider health, not request shape — it must stay retryable, must+// NOT classify as provider_bad_request, and an exhausted chain must render a+// 503, never a client-blaming 400 invalid_request_error.++vi.mock('../../services/health.js', () => ({ checkKeyHealth: vi.fn() }));++import { initDb } from '../../db/index.js';+import {+ isProviderBadRequestError,+ isProviderDegradedError,+ isRetryableError,+} from '../../lib/error-classify.js';+import { classifyAttemptError, exhaustedRetryError } from '../../lib/fallback-loop.js';++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++function nimDegraded400(): Error & { status?: number } {+ const err = new Error(+ "NVIDIA NIM API error 400: Function id 'abc-123': DEGRADED function cannot be invoked",+ ) as Error & { status?: number };+ err.status = 400;+ return err;+}++function plainBadRequest400(): Error & { status?: number } {+ const err = new Error('Groq API error 400: max_tokens must be a positive integer') as Error & { status?: number };+ err.status = 400;+ return err;+}++describe('isProviderDegradedError', () => {+ it('matches the NIM DEGRADED-function 400 shape', () => {+ expect(isProviderDegradedError(nimDegraded400())).toBe(true);+ });++ it('does not match an ordinary provider 400', () => {+ expect(isProviderDegradedError(plainBadRequest400())).toBe(false);+ });+});++describe('degraded 400 classification', () => {+ it('stays retryable so the failover chain continues', () => {+ expect(isRetryableError(nimDegraded400())).toBe(true);+ });++ it('is excluded from provider_bad_request', () => {+ expect(isProviderBadRequestError(nimDegraded400())).toBe(false);+ expect(isProviderBadRequestError(plainBadRequest400())).toBe(true);+ });++ it('shows up as upstream_error in the attempt trail', () => {+ expect(classifyAttemptError(nimDegraded400())).toBe('upstream_error');+ expect(classifyAttemptError(plainBadRequest400())).toBe('provider_bad_request');+ });+});++describe('exhaustedRetryError on a degraded last error', () => {+ it('renders 503 service_unavailable instead of 400 invalid_request_error', () => {+ const body = exhaustedRetryError(nimDegraded400(), 3);+ expect(body.status).toBe(503);+ expect(body.type).toBe('service_unavailable');+ expect(body.kind).toBe('unavailable');+ expect(body.message).toContain('degraded');+ });++ it('still renders 400 invalid_request_error for a genuine whole-chain bad request', () => {+ const body = exhaustedRetryError(plainBadRequest400(), 3);+ expect(body.status).toBe(400);+ expect(body.type).toBe('invalid_request_error');+ });+});diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 710565a9..c37e5010 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -28,6 +28,11 @@ export function isRetryableError(err: any): boolean {|| msg.includes('econnrefused') || msg.includes('econnreset')|| msg.includes('fetch failed') // undici transport error (proxy down, DNS, TLS, etc.)|| msg.includes('503') || msg.includes('unavailable')+ // Provider marks the hosted deployment itself as sick (NVIDIA NIM's+ // "DEGRADED function cannot be invoked" arrives as a 400, #522). The+ // 'api error 400' rule below already catches the NIM shape; this keeps+ // degraded conditions retryable even when a provider words it differently.+ || msg.includes('degraded')|| msg.includes('500') || msg.includes('internal server error')// 413: this model's payload limit is too small for the request, but another// provider in the fallback chain may have a larger limit. Same reasoning as 503.@@ -134,10 +139,22 @@ export function isDailyQuotaExhaustedError(err: any): boolean {return /allocation|quota|limit|exhaust|used up/.test(msg);}+// A provider-side "this hosted model is temporarily degraded" condition dressed+// up as a 400. Observed live on NVIDIA NIM (issue #522): a degraded function+// returns `400 {"detail":"Function id '...': DEGRADED function cannot be+// invoked"}` — the request is fine; the deployment is sick. Must NOT classify+// as a provider bad-request: exhausting on it would render a client-blaming+// 400 invalid_request_error for what is capacity/health, not request shape.+export function isProviderDegradedError(err: any): boolean {+ const msg = (err?.message ?? '').toLowerCase();+ return msg.includes('degraded');+}+// Provider-side 400s are retryable because another provider may accept the same// request shape. If every routed provider rejects it, however, the client should// see an invalid-request error rather than a misleading rate-limit exhaustion.export function isProviderBadRequestError(err: any): boolean {+ if (isProviderDegradedError(err)) return false;const status = typeof err?.status === 'number' ? err.status : 0;const msg = (err?.message ?? '').toLowerCase();if (status === 400) return msg.includes('api error 400');diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex dd23f61c..c0fbc17b 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -35,6 +35,7 @@ import {isModelNotFoundError,isModelAccessForbiddenError,isProviderBadRequestError,+ isProviderDegradedError,} from './error-classify.js';import { sanitizeProviderErrorMessage } from './error-redaction.js';import { checkKeyHealth } from '../services/health.js';@@ -247,6 +248,9 @@ export function classifyAttemptError(err: any): AttemptErrorClass {if (isDailyQuotaExhaustedError(err)) return 'daily_quota_exhausted';if (isModelNotFoundError(err)) return 'model_not_found';if (isModelAccessForbiddenError(err)) return 'forbidden';+ // A DEGRADED-function 400 (NVIDIA NIM, #522) is provider health, not request+ // shape — keep it out of provider_bad_request so the trail reads honestly.+ if (isProviderDegradedError(err)) return 'upstream_error';if (isProviderBadRequestError(err)) return 'provider_bad_request';const msg = (err?.message ?? '').toLowerCase();if (msg.includes('empty completion')) return 'empty_completion';@@ -344,6 +348,19 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E};}+ // A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) is a+ // provider-capacity outage, not a bad request: render 503 so clients retry+ // later instead of "fixing" a request that was never broken.+ if (isProviderDegradedError(lastError)) {+ return {+ kind: 'unavailable',+ status: 503,+ type: 'service_unavailable',+ message: `The routed provider reported the model's hosted deployment as temporarily degraded${budgetNote}. ` ++ `This is a provider-side condition; retry later or route to another provider.${trail} Last error: ${safeLastError}`,+ };+ }+if (isProviderBadRequestError(lastError)) {return {kind: 'bad_request',