Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(fusion): provider-moment attribution covers the panel/judge retry path (#1347) (#1576)

tashdroidc3eb3aabb5ab
diff --git a/server/src/__tests__/services/fusion-provider-moment.test.ts b/server/src/__tests__/services/fusion-provider-moment.test.ts
new file mode 100644
index 00000000..544f1a40
--- /dev/null
+++ b/server/src/__tests__/services/fusion-provider-moment.test.ts
@@ -0,0 +1,104 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';
+
+// #1347 follow-up: the provider-moment attribution switch must cover the
+// fusion panel/judge retry path too. That path calls recordRateLimitHit on
+// every retryable failure and logs the request row with no attribution, so
+// with the switch ON a healthy model whose provider threw a 429 inside a
+// fusion call was still penalized and still counted against reliability —
+// exactly what the feature says it stops doing. Key cooldowns are untouched.
+
+const keySeqStart = 950000;
+let keySeq = keySeqStart;
+
+import { initDb, getDb } from '../../db/index.js';
+import { runModelCall } from '../../services/fusion.js';
+import { getAllPenalties, type RouteResult } from '../../services/router.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+function fakeRoute(throwErr: any): { route: RouteResult; calls: () => number } {
+ const n = ++keySeq;
+ let calls = 0;
+ const route: RouteResult = {
+ provider: {
+ chatCompletion: async () => { calls++; throw throwErr; },
+ } as any,
+ modelId: 'fake-fusion-model', modelDbId: n, apiKey: 'test-key',
+ keyId: n, platform: 'fakefusion', displayName: 'Fake Fusion Model',
+ rpdLimit: null, tpdLimit: null,
+ };
+ return { route, calls: () => calls };
+}
+
+const err429 = () => Object.assign(new Error('API error 429: rate limit exceeded'), { status: 429 });
+
+function penaltyFor(modelDbId: number) {
+ return getAllPenalties().find((p) => p.modelDbId === modelDbId);
+}
+
+function lastLogRow(): { status: string; failure_attribution: string | null } | undefined {
+ return getDb().prepare(
+ 'SELECT status, failure_attribution FROM requests WHERE model_id = ? ORDER BY id DESC LIMIT 1',
+ ).get('fake-fusion-model') as any;
+}
+
+const MESSAGES: ChatMessage[] = [{ role: 'user', content: 'hi' }];
+
+async function runSlot(route: RouteResult) {
+ let served = 0;
+ await runModelCall(
+ () => (served++ === 0 ? route : null),
+ MESSAGES,
+ {} as any,
+ 10,
+ 1,
+ );
+}
+
+describe('provider-moment attribution in the fusion retry path (#1347)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ beforeEach(() => {
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ getDb().prepare('DELETE FROM requests').run();
+ delete process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION;
+ });
+
+ afterEach(() => {
+ delete process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION;
+ });
+
+ it('with attribution ON: a 429 does not penalize the model and the row is tagged', async () => {
+ process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION = '1';
+ const { route } = fakeRoute(err429());
+ await runSlot(route);
+
+ expect(penaltyFor(route.modelDbId)).toBeUndefined();
+ // The key cooldown ladder still fires — the failure still steers the key.
+ const cooldowns = getDb().prepare('SELECT COUNT(*) AS n FROM rate_limit_cooldowns WHERE key_id = ?')
+ .get(route.keyId) as { n: number };
+ expect(cooldowns.n).toBe(1);
+ expect(lastLogRow()).toEqual({ status: 'error', failure_attribution: 'provider_moment' });
+ });
+
+ it('with attribution OFF (default): penalty and untagged row, exactly as before', async () => {
+ const { route } = fakeRoute(err429());
+ await runSlot(route);
+
+ expect(penaltyFor(route.modelDbId)).toBeDefined();
+ expect(lastLogRow()).toEqual({ status: 'error', failure_attribution: null });
+ });
+
+ it('with attribution ON: a non-provider-moment failure still penalizes and stays untagged', async () => {
+ process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION = '1';
+ const err = Object.assign(new Error('API error 400: unknown parameter'), { status: 400 });
+ const { route } = fakeRoute(err);
+ await runSlot(route);
+
+ // 400 is not retryable at all, so the slot stops without a penalty —
+ // assert instead that the log row was NOT tagged as a provider moment.
+ expect(lastLogRow()).toEqual({ status: 'error', failure_attribution: null });
+ });
+});
diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.ts
index b3108a79..343711df 100644
--- a/server/src/services/fusion.ts
+++ b/server/src/services/fusion.ts
@@ -3,7 +3,7 @@ import type { ChatMessage, ChatCompletionChoice, ChatCompletionResponse, ChatToo
import {
routePinnedModel, routeRequest, getOrderedFusionChain, resolveFusionCandidate,
pinnedModelFitsRequestSize,
- recordRateLimitHit, recordSuccess, type RouteResult, type FusionCandidate,
+ recordRateLimitHit, recordSuccess, failureAttributionFor, type RouteResult, type FusionCandidate,
} from './router.js';
import {
recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,
@@ -286,7 +286,9 @@ function addUsage(a: TokenUsage, b: TokenUsage | undefined): TokenUsage {
* WHICH model is tried: a pinned-model closure for a panel slot (never
* substitutes), or the auto-router for the judge (falls over across the chain).
*/
-async function runModelCall(
+// Exported for the attribution regression test: the panel/judge retry path is
+// the fusion mirror of lib/fallback-loop.ts's recordRetryableFailure.
+export async function runModelCall(
getRoute: (skipKeys: Set<string>, skipModels: Set<number>) => RouteResult | null,
messages: ChatMessage[],
options: CompletionOptions,
@@ -353,7 +355,13 @@ async function runModelCall(
};
} catch (err: any) {
const safe = sanitizeProviderErrorMessage(err?.message);
- logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http');
+ // #1347: mirror the fallback loop — with provider-moment attribution on,
+ // a 429/5xx/timeout/transport failure on a fusion panel slot is the
+ // provider's bad minute, not model evidence: the key still cools down,
+ // but the model's rate-limit penalty is skipped and the log row is
+ // tagged so the stats cache excludes it.
+ const attribution = failureAttributionFor(err);
+ logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http', attribution);
lastError = safe;
failure = mergeCallFailure(failure, classifyCallFailure(err));
@@ -373,7 +381,7 @@ async function runModelCall(
? { durationMs: getModelForbiddenCooldownMs(), source: 'tier' as const }
: getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs, { quotaSignal: isRateLimitSignal(err) });
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
- recordRateLimitHit(route.modelDbId);
+ if (attribution !== 'provider_moment') recordRateLimitHit(route.modelDbId);
continue;
}
// Non-retryable (auth, validation) — this slot/judge is done.
@@ -454,7 +462,9 @@ async function runJudgeStreaming(
return { ok: true, route, text, usage };
} catch (err: any) {
const safe = sanitizeProviderErrorMessage(err?.message);
- logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http');
+ // #1347: same attribution as the panel path above.
+ const attribution = failureAttributionFor(err);
+ logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http', attribution);
lastError = safe;
// Already streamed bytes — can't fail over without duplicating output.
// Keep whatever the client already received.
@@ -479,7 +489,7 @@ async function runJudgeStreaming(
? { durationMs: getModelForbiddenCooldownMs(), source: 'tier' as const }
: getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs, { quotaSignal: isRateLimitSignal(err) });
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
- recordRateLimitHit(route.modelDbId);
+ if (attribution !== 'provider_moment') recordRateLimitHit(route.modelDbId);
continue;
}
break;