Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(routing): scope the decision-pick cache to the min-confidence threshold (#1581)

tashdroid99cfaa92a305
diff --git a/server/src/__tests__/services/routing-decision.test.ts b/server/src/__tests__/services/routing-decision.test.ts
index 92dba88f..21b77cda 100644
--- a/server/src/__tests__/services/routing-decision.test.ts
+++ b/server/src/__tests__/services/routing-decision.test.ts
@@ -129,6 +129,22 @@ describe('decision routing (#1354)', () => {
expect((await chooseRoutingPreset(HARD)).strategy).toBe('reliable');
});
+ it('re-judges when the minimum probability changes (cache is threshold-scoped)', async () => {
+ // A low-confidence pick cached as Balanced must not outlive a lowering of
+ // the threshold: the point of lowering it is to let the model's own pick
+ // through for texts already judged during the cache TTL.
+ addKey('cloudflare', 'acct:token');
+ const fn = mockFetch(() => cloudflareAnswer('reliable', 0.31));
+ expect(await chooseRoutingPreset(HARD)).toMatchObject({ strategy: 'balanced', source: 'low_confidence' });
+ setDecisionRoutingSettings({ minProbability: 0.3 });
+ const after = await chooseRoutingPreset(HARD);
+ expect(after).toMatchObject({ strategy: 'reliable', source: 'decision' });
+ expect(fn).toHaveBeenCalledTimes(2);
+ // Same threshold again: answered from cache, no third call.
+ expect(await chooseRoutingPreset(HARD)).toMatchObject({ strategy: 'reliable', source: 'cache' });
+ expect(fn).toHaveBeenCalledTimes(2);
+ });
+
it('asks once per turn: the same text is answered from cache', async () => {
addKey('cloudflare', 'acct:token');
const fn = mockFetch(() => cloudflareAnswer('fastest', 0.9));
diff --git a/server/src/services/routing-decision.ts b/server/src/services/routing-decision.ts
index ac1598ad..a5b88742 100644
--- a/server/src/services/routing-decision.ts
+++ b/server/src/services/routing-decision.ts
@@ -175,7 +175,13 @@ export async function chooseRoutingPreset(messages: readonly ChatMessage[], sign
if (!state) return { strategy: 'balanced', source: 'unavailable' };
const settings = getDecisionRoutingSettings();
- const key = createHash('sha256').update(`${settings.model}\0${state}`).digest('hex');
+ // minProbability is part of the key: a pick cached under one threshold must
+ // not answer a request judged under another (a low_confidence Balanced held
+ // for the TTL would silently undo a threshold lowering — the whole point of
+ // which is to let the model's own pick through).
+ const key = createHash('sha256')
+ .update(`${settings.model}\0${settings.minProbability}\0${state}`)
+ .digest('hex');
const cached = cacheGet(key);
if (cached) return { ...cached, source: 'cache' };