Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(proxy): bind idempotency keys to extended generation options (#1440)

Alex6d63bf125c18
diff --git a/server/src/__tests__/routes/proxy-cache.test.ts b/server/src/__tests__/routes/proxy-cache.test.ts
index 0b9dbf52..bb6aed76 100644
--- a/server/src/__tests__/routes/proxy-cache.test.ts
+++ b/server/src/__tests__/routes/proxy-cache.test.ts
@@ -126,6 +126,26 @@ describe('Response cache (proxy integration)', () => {
...extra,
}, { ...authHeaders(), ...headers });
+ it.each([
+ { response_format: { type: 'json_object' } },
+ { seed: 42 },
+ { stop: 'END' },
+ { parallel_tool_calls: false },
+ ])('rejects idempotent retries with changed generation options: %j', async (extra) => {
+ getDb().prepare('DELETE FROM idempotency_claims').run();
+ const counter = mockGroq('plain text');
+ const headers = { 'Idempotency-Key': 'generation-options', 'X-FreeLLM-Cache': 'off' };
+ const first = await chat({}, headers);
+ expect(first.status).toBe(200);
+ const replay = await chat({}, headers);
+ expect(replay.status).toBe(200);
+ expect(replay.headers.get('x-routed-via')).toBe('idempotency');
+ const second = await chat(extra, headers);
+ expect(second.status).toBe(409);
+ expect(second.body.error.message).toBe('idempotency_key_conflict');
+ expect(counter.calls).toBe(1);
+ });
+
it('serves an identical second request from cache without calling the provider', async () => {
const counter = mockGroq('the answer is four');
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index bc5c04b4..49761731 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -1966,6 +1966,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
max_tokens,
tools,
tool_choice,
+ generationOptions: { stop, parallel_tool_calls, ...samplingParams },
})
: null;
if (idemKey && idemFingerprint) {
diff --git a/server/src/services/idempotency.ts b/server/src/services/idempotency.ts
index ea9c552a..e9e5dc03 100644
--- a/server/src/services/idempotency.ts
+++ b/server/src/services/idempotency.ts
@@ -56,6 +56,7 @@ export function computeIdempotencyFingerprint(input: {
max_tokens?: number | null;
tools?: unknown;
tool_choice?: unknown;
+ generationOptions?: Record<string, unknown>;
}): string {
const stable = JSON.stringify({
model: input.model ?? null,
@@ -65,6 +66,7 @@ export function computeIdempotencyFingerprint(input: {
max_tokens: input.max_tokens ?? null,
tools: input.tools ?? null,
tool_choice: input.tool_choice ?? null,
+ generationOptions: input.generationOptions,
});
return crypto.createHash('sha256').update(stable).digest('hex');
}