Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(server): treat NVIDIA NIM 'DEGRADED function' 400 as provider capacity, not an invalid request

Tashaa68fc25862d
diff --git a/server/src/__tests__/lib/degraded-error.test.ts b/server/src/__tests__/lib/degraded-error.test.ts
new file mode 100644
index 00000000..aa54503d
--- /dev/null
+++ b/server/src/__tests__/lib/degraded-error.test.ts
@@ -0,0 +1,78 @@
+import { describe, it, expect, beforeAll, vi } from 'vitest';
+
+// Issue #522: NVIDIA NIM reports a temporarily-degraded hosted deployment as
+// `400 {"detail":"Function id '...': DEGRADED function cannot be invoked"}`.
+// That is provider health, not request shape — it must stay retryable, must
+// NOT classify as provider_bad_request, and an exhausted chain must render a
+// 503, never a client-blaming 400 invalid_request_error.
+
+vi.mock('../../services/health.js', () => ({ checkKeyHealth: vi.fn() }));
+
+import { initDb } from '../../db/index.js';
+import {
+ isProviderBadRequestError,
+ isProviderDegradedError,
+ isRetryableError,
+} from '../../lib/error-classify.js';
+import { classifyAttemptError, exhaustedRetryError } from '../../lib/fallback-loop.js';
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+function nimDegraded400(): Error & { status?: number } {
+ const err = new Error(
+ "NVIDIA NIM API error 400: Function id 'abc-123': DEGRADED function cannot be invoked",
+ ) as Error & { status?: number };
+ err.status = 400;
+ return err;
+}
+
+function plainBadRequest400(): Error & { status?: number } {
+ const err = new Error('Groq API error 400: max_tokens must be a positive integer') as Error & { status?: number };
+ err.status = 400;
+ return err;
+}
+
+describe('isProviderDegradedError', () => {
+ it('matches the NIM DEGRADED-function 400 shape', () => {
+ expect(isProviderDegradedError(nimDegraded400())).toBe(true);
+ });
+
+ it('does not match an ordinary provider 400', () => {
+ expect(isProviderDegradedError(plainBadRequest400())).toBe(false);
+ });
+});
+
+describe('degraded 400 classification', () => {
+ it('stays retryable so the failover chain continues', () => {
+ expect(isRetryableError(nimDegraded400())).toBe(true);
+ });
+
+ it('is excluded from provider_bad_request', () => {
+ expect(isProviderBadRequestError(nimDegraded400())).toBe(false);
+ expect(isProviderBadRequestError(plainBadRequest400())).toBe(true);
+ });
+
+ it('shows up as upstream_error in the attempt trail', () => {
+ expect(classifyAttemptError(nimDegraded400())).toBe('upstream_error');
+ expect(classifyAttemptError(plainBadRequest400())).toBe('provider_bad_request');
+ });
+});
+
+describe('exhaustedRetryError on a degraded last error', () => {
+ it('renders 503 service_unavailable instead of 400 invalid_request_error', () => {
+ const body = exhaustedRetryError(nimDegraded400(), 3);
+ expect(body.status).toBe(503);
+ expect(body.type).toBe('service_unavailable');
+ expect(body.kind).toBe('unavailable');
+ expect(body.message).toContain('degraded');
+ });
+
+ it('still renders 400 invalid_request_error for a genuine whole-chain bad request', () => {
+ const body = exhaustedRetryError(plainBadRequest400(), 3);
+ expect(body.status).toBe(400);
+ expect(body.type).toBe('invalid_request_error');
+ });
+});
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 710565a9..c37e5010 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -28,6 +28,11 @@ export function isRetryableError(err: any): boolean {
|| msg.includes('econnrefused') || msg.includes('econnreset')
|| msg.includes('fetch failed') // undici transport error (proxy down, DNS, TLS, etc.)
|| msg.includes('503') || msg.includes('unavailable')
+ // Provider marks the hosted deployment itself as sick (NVIDIA NIM's
+ // "DEGRADED function cannot be invoked" arrives as a 400, #522). The
+ // 'api error 400' rule below already catches the NIM shape; this keeps
+ // degraded conditions retryable even when a provider words it differently.
+ || msg.includes('degraded')
|| msg.includes('500') || msg.includes('internal server error')
// 413: this model's payload limit is too small for the request, but another
// provider in the fallback chain may have a larger limit. Same reasoning as 503.
@@ -134,10 +139,22 @@ export function isDailyQuotaExhaustedError(err: any): boolean {
return /allocation|quota|limit|exhaust|used up/.test(msg);
}
+// A provider-side "this hosted model is temporarily degraded" condition dressed
+// up as a 400. Observed live on NVIDIA NIM (issue #522): a degraded function
+// returns `400 {"detail":"Function id '...': DEGRADED function cannot be
+// invoked"}` — the request is fine; the deployment is sick. Must NOT classify
+// as a provider bad-request: exhausting on it would render a client-blaming
+// 400 invalid_request_error for what is capacity/health, not request shape.
+export function isProviderDegradedError(err: any): boolean {
+ const msg = (err?.message ?? '').toLowerCase();
+ return msg.includes('degraded');
+}
+
// Provider-side 400s are retryable because another provider may accept the same
// request shape. If every routed provider rejects it, however, the client should
// see an invalid-request error rather than a misleading rate-limit exhaustion.
export function isProviderBadRequestError(err: any): boolean {
+ if (isProviderDegradedError(err)) return false;
const status = typeof err?.status === 'number' ? err.status : 0;
const msg = (err?.message ?? '').toLowerCase();
if (status === 400) return msg.includes('api error 400');
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index dd23f61c..c0fbc17b 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -35,6 +35,7 @@ import {
isModelNotFoundError,
isModelAccessForbiddenError,
isProviderBadRequestError,
+ isProviderDegradedError,
} from './error-classify.js';
import { sanitizeProviderErrorMessage } from './error-redaction.js';
import { checkKeyHealth } from '../services/health.js';
@@ -247,6 +248,9 @@ export function classifyAttemptError(err: any): AttemptErrorClass {
if (isDailyQuotaExhaustedError(err)) return 'daily_quota_exhausted';
if (isModelNotFoundError(err)) return 'model_not_found';
if (isModelAccessForbiddenError(err)) return 'forbidden';
+ // A DEGRADED-function 400 (NVIDIA NIM, #522) is provider health, not request
+ // shape — keep it out of provider_bad_request so the trail reads honestly.
+ if (isProviderDegradedError(err)) return 'upstream_error';
if (isProviderBadRequestError(err)) return 'provider_bad_request';
const msg = (err?.message ?? '').toLowerCase();
if (msg.includes('empty completion')) return 'empty_completion';
@@ -344,6 +348,19 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
};
}
+ // A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) is a
+ // provider-capacity outage, not a bad request: render 503 so clients retry
+ // later instead of "fixing" a request that was never broken.
+ if (isProviderDegradedError(lastError)) {
+ return {
+ kind: 'unavailable',
+ status: 503,
+ type: 'service_unavailable',
+ message: `The routed provider reported the model's hosted deployment as temporarily degraded${budgetNote}. ` +
+ `This is a provider-side condition; retry later or route to another provider.${trail} Last error: ${safeLastError}`,
+ };
+ }
+
if (isProviderBadRequestError(lastError)) {
return {
kind: 'bad_request',