7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(routing): scope the decision-pick cache to the min-confidence threshold (#1581)
tashdroid99cfaa92a305
diff --git a/server/src/__tests__/services/routing-decision.test.ts b/server/src/__tests__/services/routing-decision.test.tsindex 92dba88f..21b77cda 100644--- a/server/src/__tests__/services/routing-decision.test.ts+++ b/server/src/__tests__/services/routing-decision.test.ts@@ -129,6 +129,22 @@ describe('decision routing (#1354)', () => {expect((await chooseRoutingPreset(HARD)).strategy).toBe('reliable');});+ it('re-judges when the minimum probability changes (cache is threshold-scoped)', async () => {+ // A low-confidence pick cached as Balanced must not outlive a lowering of+ // the threshold: the point of lowering it is to let the model's own pick+ // through for texts already judged during the cache TTL.+ addKey('cloudflare', 'acct:token');+ const fn = mockFetch(() => cloudflareAnswer('reliable', 0.31));+ expect(await chooseRoutingPreset(HARD)).toMatchObject({ strategy: 'balanced', source: 'low_confidence' });+ setDecisionRoutingSettings({ minProbability: 0.3 });+ const after = await chooseRoutingPreset(HARD);+ expect(after).toMatchObject({ strategy: 'reliable', source: 'decision' });+ expect(fn).toHaveBeenCalledTimes(2);+ // Same threshold again: answered from cache, no third call.+ expect(await chooseRoutingPreset(HARD)).toMatchObject({ strategy: 'reliable', source: 'cache' });+ expect(fn).toHaveBeenCalledTimes(2);+ });+it('asks once per turn: the same text is answered from cache', async () => {addKey('cloudflare', 'acct:token');const fn = mockFetch(() => cloudflareAnswer('fastest', 0.9));diff --git a/server/src/services/routing-decision.ts b/server/src/services/routing-decision.tsindex ac1598ad..a5b88742 100644--- a/server/src/services/routing-decision.ts+++ b/server/src/services/routing-decision.ts@@ -175,7 +175,13 @@ export async function chooseRoutingPreset(messages: readonly ChatMessage[], signif (!state) return { strategy: 'balanced', source: 'unavailable' };const settings = getDecisionRoutingSettings();- const key = createHash('sha256').update(`${settings.model}\0${state}`).digest('hex');+ // minProbability is part of the key: a pick cached under one threshold must+ // not answer a request judged under another (a low_confidence Balanced held+ // for the TTL would silently undo a threshold lowering — the whole point of+ // which is to let the model's own pick through).+ const key = createHash('sha256')+ .update(`${settings.model}\0${settings.minProbability}\0${state}`)+ .digest('hex');const cached = cacheGet(key);if (cached) return { ...cached, source: 'cache' };