7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(fusion): provider-moment attribution covers the panel/judge retry path (#1347) (#1576)
tashdroidc3eb3aabb5ab
diff --git a/server/src/__tests__/services/fusion-provider-moment.test.ts b/server/src/__tests__/services/fusion-provider-moment.test.tsnew file mode 100644index 00000000..544f1a40--- /dev/null+++ b/server/src/__tests__/services/fusion-provider-moment.test.ts@@ -0,0 +1,104 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest';++// #1347 follow-up: the provider-moment attribution switch must cover the+// fusion panel/judge retry path too. That path calls recordRateLimitHit on+// every retryable failure and logs the request row with no attribution, so+// with the switch ON a healthy model whose provider threw a 429 inside a+// fusion call was still penalized and still counted against reliability —+// exactly what the feature says it stops doing. Key cooldowns are untouched.++const keySeqStart = 950000;+let keySeq = keySeqStart;++import { initDb, getDb } from '../../db/index.js';+import { runModelCall } from '../../services/fusion.js';+import { getAllPenalties, type RouteResult } from '../../services/router.js';+import type { ChatMessage } from '@freellmapi/shared/types.js';++function fakeRoute(throwErr: any): { route: RouteResult; calls: () => number } {+ const n = ++keySeq;+ let calls = 0;+ const route: RouteResult = {+ provider: {+ chatCompletion: async () => { calls++; throw throwErr; },+ } as any,+ modelId: 'fake-fusion-model', modelDbId: n, apiKey: 'test-key',+ keyId: n, platform: 'fakefusion', displayName: 'Fake Fusion Model',+ rpdLimit: null, tpdLimit: null,+ };+ return { route, calls: () => calls };+}++const err429 = () => Object.assign(new Error('API error 429: rate limit exceeded'), { status: 429 });++function penaltyFor(modelDbId: number) {+ return getAllPenalties().find((p) => p.modelDbId === modelDbId);+}++function lastLogRow(): { status: string; failure_attribution: string | null } | undefined {+ return getDb().prepare(+ 'SELECT status, failure_attribution FROM requests WHERE model_id = ? ORDER BY id DESC LIMIT 1',+ ).get('fake-fusion-model') as any;+}++const MESSAGES: ChatMessage[] = [{ role: 'user', content: 'hi' }];++async function runSlot(route: RouteResult) {+ let served = 0;+ await runModelCall(+ () => (served++ === 0 ? route : null),+ MESSAGES,+ {} as any,+ 10,+ 1,+ );+}++describe('provider-moment attribution in the fusion retry path (#1347)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ beforeEach(() => {+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ getDb().prepare('DELETE FROM requests').run();+ delete process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION;+ });++ afterEach(() => {+ delete process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION;+ });++ it('with attribution ON: a 429 does not penalize the model and the row is tagged', async () => {+ process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION = '1';+ const { route } = fakeRoute(err429());+ await runSlot(route);++ expect(penaltyFor(route.modelDbId)).toBeUndefined();+ // The key cooldown ladder still fires — the failure still steers the key.+ const cooldowns = getDb().prepare('SELECT COUNT(*) AS n FROM rate_limit_cooldowns WHERE key_id = ?')+ .get(route.keyId) as { n: number };+ expect(cooldowns.n).toBe(1);+ expect(lastLogRow()).toEqual({ status: 'error', failure_attribution: 'provider_moment' });+ });++ it('with attribution OFF (default): penalty and untagged row, exactly as before', async () => {+ const { route } = fakeRoute(err429());+ await runSlot(route);++ expect(penaltyFor(route.modelDbId)).toBeDefined();+ expect(lastLogRow()).toEqual({ status: 'error', failure_attribution: null });+ });++ it('with attribution ON: a non-provider-moment failure still penalizes and stays untagged', async () => {+ process.env.ROUTER_PROVIDER_MOMENT_ATTRIBUTION = '1';+ const err = Object.assign(new Error('API error 400: unknown parameter'), { status: 400 });+ const { route } = fakeRoute(err);+ await runSlot(route);++ // 400 is not retryable at all, so the slot stops without a penalty —+ // assert instead that the log row was NOT tagged as a provider moment.+ expect(lastLogRow()).toEqual({ status: 'error', failure_attribution: null });+ });+});diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex b3108a79..343711df 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -3,7 +3,7 @@ import type { ChatMessage, ChatCompletionChoice, ChatCompletionResponse, ChatTooimport {routePinnedModel, routeRequest, getOrderedFusionChain, resolveFusionCandidate,pinnedModelFitsRequestSize,- recordRateLimitHit, recordSuccess, type RouteResult, type FusionCandidate,+ recordRateLimitHit, recordSuccess, failureAttributionFor, type RouteResult, type FusionCandidate,} from './router.js';import {recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,@@ -286,7 +286,9 @@ function addUsage(a: TokenUsage, b: TokenUsage | undefined): TokenUsage {* WHICH model is tried: a pinned-model closure for a panel slot (never* substitutes), or the auto-router for the judge (falls over across the chain).*/-async function runModelCall(+// Exported for the attribution regression test: the panel/judge retry path is+// the fusion mirror of lib/fallback-loop.ts's recordRetryableFailure.+export async function runModelCall(getRoute: (skipKeys: Set<string>, skipModels: Set<number>) => RouteResult | null,messages: ChatMessage[],options: CompletionOptions,@@ -353,7 +355,13 @@ async function runModelCall(};} catch (err: any) {const safe = sanitizeProviderErrorMessage(err?.message);- logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http');+ // #1347: mirror the fallback loop — with provider-moment attribution on,+ // a 429/5xx/timeout/transport failure on a fusion panel slot is the+ // provider's bad minute, not model evidence: the key still cools down,+ // but the model's rate-limit penalty is skipped and the log row is+ // tagged so the stats cache excludes it.+ const attribution = failureAttributionFor(err);+ logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http', attribution);lastError = safe;failure = mergeCallFailure(failure, classifyCallFailure(err));@@ -373,7 +381,7 @@ async function runModelCall(? { durationMs: getModelForbiddenCooldownMs(), source: 'tier' as const }: getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs, { quotaSignal: isRateLimitSignal(err) });setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);- recordRateLimitHit(route.modelDbId);+ if (attribution !== 'provider_moment') recordRateLimitHit(route.modelDbId);continue;}// Non-retryable (auth, validation) — this slot/judge is done.@@ -454,7 +462,9 @@ async function runJudgeStreaming(return { ok: true, route, text, usage };} catch (err: any) {const safe = sanitizeProviderErrorMessage(err?.message);- logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http');+ // #1347: same attribution as the panel path above.+ const attribution = failureAttributionFor(err);+ logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http', attribution);lastError = safe;// Already streamed bytes — can't fail over without duplicating output.// Keep whatever the client already received.@@ -479,7 +489,7 @@ async function runJudgeStreaming(? { durationMs: getModelForbiddenCooldownMs(), source: 'tier' as const }: getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs, { quotaSignal: isRateLimitSignal(err) });setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);- recordRateLimitHit(route.modelDbId);+ if (attribution !== 'provider_moment') recordRateLimitHit(route.modelDbId);continue;}break;