7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): bind idempotency keys to extended generation options (#1440)
Alex6d63bf125c18
diff --git a/server/src/__tests__/routes/proxy-cache.test.ts b/server/src/__tests__/routes/proxy-cache.test.tsindex 0b9dbf52..bb6aed76 100644--- a/server/src/__tests__/routes/proxy-cache.test.ts+++ b/server/src/__tests__/routes/proxy-cache.test.ts@@ -126,6 +126,26 @@ describe('Response cache (proxy integration)', () => {...extra,}, { ...authHeaders(), ...headers });+ it.each([+ { response_format: { type: 'json_object' } },+ { seed: 42 },+ { stop: 'END' },+ { parallel_tool_calls: false },+ ])('rejects idempotent retries with changed generation options: %j', async (extra) => {+ getDb().prepare('DELETE FROM idempotency_claims').run();+ const counter = mockGroq('plain text');+ const headers = { 'Idempotency-Key': 'generation-options', 'X-FreeLLM-Cache': 'off' };+ const first = await chat({}, headers);+ expect(first.status).toBe(200);+ const replay = await chat({}, headers);+ expect(replay.status).toBe(200);+ expect(replay.headers.get('x-routed-via')).toBe('idempotency');+ const second = await chat(extra, headers);+ expect(second.status).toBe(409);+ expect(second.body.error.message).toBe('idempotency_key_conflict');+ expect(counter.calls).toBe(1);+ });+it('serves an identical second request from cache without calling the provider', async () => {const counter = mockGroq('the answer is four');diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex bc5c04b4..49761731 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -1966,6 +1966,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {max_tokens,tools,tool_choice,+ generationOptions: { stop, parallel_tool_calls, ...samplingParams },}): null;if (idemKey && idemFingerprint) {diff --git a/server/src/services/idempotency.ts b/server/src/services/idempotency.tsindex ea9c552a..e9e5dc03 100644--- a/server/src/services/idempotency.ts+++ b/server/src/services/idempotency.ts@@ -56,6 +56,7 @@ export function computeIdempotencyFingerprint(input: {max_tokens?: number | null;tools?: unknown;tool_choice?: unknown;+ generationOptions?: Record<string, unknown>;}): string {const stable = JSON.stringify({model: input.model ?? null,@@ -65,6 +66,7 @@ export function computeIdempotencyFingerprint(input: {max_tokens: input.max_tokens ?? null,tools: input.tools ?? null,tool_choice: input.tool_choice ?? null,+ generationOptions: input.generationOptions,});return crypto.createHash('sha256').update(stable).digest('hex');}