Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(proxy): Idempotency-Key support — stop client retries from double-spending free-tier quota (#1095)

suantea (asuan_suan)36b877d82066
diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.ts
index 715248ad..63a493fd 100644
--- a/server/src/__tests__/db/migrate/roundtrip.test.ts
+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts
@@ -32,6 +32,7 @@ const SERVER_LOGS_FILENAME = '20260823_000001_server_logs.ts';
const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';
const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';
const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';
+const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';
interface SchemaRow {
type: string;
@@ -110,6 +111,7 @@ describe('migration round trip', () => {
BACKUPS_TABLE_FILENAME,
ATTEMPT_KEY_LABEL_FILENAME,
PROFILE_AUTO_INCLUDE_FILENAME,
+ IDEMPOTENCY_CLAIMS_FILENAME,
]);
} finally {
db.close();
diff --git a/server/src/__tests__/services/idempotency.test.ts b/server/src/__tests__/services/idempotency.test.ts
new file mode 100644
index 00000000..bb852495
--- /dev/null
+++ b/server/src/__tests__/services/idempotency.test.ts
@@ -0,0 +1,126 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { initDb } from '../../db/index.js';
+import {
+ hashIdempotencyKey,
+ computeIdempotencyFingerprint,
+ lookupIdempotencyReplay,
+ storeIdempotencyResult,
+ clearIdempotencyResult,
+ normalizeIdempotencyKey,
+ idempotencyTtlMs,
+} from '../../services/idempotency.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+const IDEM_ENV = ['IDEMPOTENCY_TTL_MS'] as const;
+
+function msg(role: ChatMessage['role'], content: string): ChatMessage {
+ return { role, content } as ChatMessage;
+}
+
+const fingerprintFor = (model: string, content: string) =>
+ computeIdempotencyFingerprint({
+ model,
+ messages: [msg('user', content)],
+ });
+
+const sampleBody = (text: string) => ({
+ id: 'chatcmpl-test',
+ object: 'chat.completion',
+ choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+});
+
+describe('idempotency', () => {
+ const saved: Record<string, string | undefined> = {};
+
+ beforeEach(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ for (const k of IDEM_ENV) saved[k] = process.env[k];
+ initDb(':memory:');
+ });
+
+ afterEach(() => {
+ for (const k of IDEM_ENV) {
+ if (saved[k] === undefined) delete process.env[k];
+ else process.env[k] = saved[k];
+ }
+ });
+
+ it('miss on an unknown key hash', () => {
+ const r = lookupIdempotencyReplay(hashIdempotencyKey('never-stored'), fingerprintFor('m', 'hi'));
+ expect(r.kind).toBe('miss');
+ });
+
+ it('replays a stored response with the same key + fingerprint', () => {
+ const keyHash = hashIdempotencyKey('retry-me');
+ const fp = fingerprintFor('groq/llama', 'same question');
+ storeIdempotencyResult(keyHash, fp, 200, sampleBody('first answer'), 'exec-1');
+ const r = lookupIdempotencyReplay(keyHash, fp);
+ expect(r.kind).toBe('replay');
+ if (r.kind === 'replay') {
+ expect(r.status).toBe(200);
+ expect(r.body).toEqual(sampleBody('first answer'));
+ }
+ });
+
+ it('reports conflict when the same key is reused with different content', () => {
+ const keyHash = hashIdempotencyKey('shared-key');
+ storeIdempotencyResult(
+ keyHash,
+ fingerprintFor('m', 'question A'),
+ 200,
+ sampleBody('answer A'),
+ );
+ const r = lookupIdempotencyReplay(keyHash, fingerprintFor('m', 'question B'));
+ expect(r.kind).toBe('conflict');
+ });
+
+ it('treats an expired claim as a miss', () => {
+ const keyHash = hashIdempotencyKey('expired');
+ const fp = fingerprintFor('m', 'q');
+ const now = 1_000_000;
+ storeIdempotencyResult(keyHash, fp, 200, sampleBody('old'), undefined, now);
+ const r = lookupIdempotencyReplay(keyHash, fp, now + idempotencyTtlMs() + 1);
+ expect(r.kind).toBe('miss');
+ });
+
+ it('replaces the previous claim when the same key is reused after completion', () => {
+ const keyHash = hashIdempotencyKey('replaced');
+ const fpA = fingerprintFor('m', 'first');
+ storeIdempotencyResult(keyHash, fpA, 200, sampleBody('answer A'));
+ const fpB = fingerprintFor('m', 'second');
+ storeIdempotencyResult(keyHash, fpB, 200, sampleBody('answer B'));
+ // After a new successful completion with a DIFFERENT fingerprint, the old
+ // claim is replaced: the new fingerprint replays, and the old fingerprint
+ // (same key, different content) is a conflict — not a silent wrong answer.
+ expect(lookupIdempotencyReplay(keyHash, fpB).kind).toBe('replay');
+ expect(lookupIdempotencyReplay(keyHash, fpA).kind).toBe('conflict');
+ });
+
+ it('clear removes the claim', () => {
+ const keyHash = hashIdempotencyKey('cleared');
+ const fp = fingerprintFor('m', 'q');
+ storeIdempotencyResult(keyHash, fp, 200, sampleBody('x'));
+ clearIdempotencyResult(keyHash);
+ expect(lookupIdempotencyReplay(keyHash, fp).kind).toBe('miss');
+ });
+
+ it('normalizeIdempotencyKey trims, rejects empties and >255-byte values', () => {
+ expect(normalizeIdempotencyKey(' abc ')).toBe('abc');
+ expect(normalizeIdempotencyKey(' ')).toBeNull();
+ expect(normalizeIdempotencyKey('')).toBeNull();
+ expect(normalizeIdempotencyKey(undefined)).toBeNull();
+ expect(normalizeIdempotencyKey('a'.repeat(256))).toBeNull();
+ expect(normalizeIdempotencyKey('a'.repeat(255))).toBe('a'.repeat(255));
+ // Repeated header (Express lower-cases and arrays multi-value) → first wins.
+ expect(normalizeIdempotencyKey(['first', 'second'])).toBe('first');
+ });
+
+ it('different content produces different fingerprints; order-insensitive JSON does not collide wrongly', () => {
+ const a = fingerprintFor('m', 'hello');
+ const b = fingerprintFor('m', 'world');
+ expect(a).not.toBe(b);
+ // Same model + content → same fingerprint.
+ expect(fingerprintFor('m', 'hello')).toBe(a);
+ });
+});
diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.ts
index 1fc49664..6c4edbf7 100644
--- a/server/src/db/migrate/defaults.ts
+++ b/server/src/db/migrate/defaults.ts
@@ -27,6 +27,7 @@ import * as serverLogs from '../migrations/20260823_000001_server_logs.js';
import * as backupsTable from '../migrations/20260823_000002_backups_table.js';
import * as attemptKeyLabel from '../migrations/20260823_000003_attempt_key_label.js';
import * as profileAutoInclude from '../migrations/20260823_000004_profile_auto_include.js';
+import * as idempotencyClaims from '../migrations/20260901_000001_idempotency_claims.js';
export interface MigrationModule {
up(db: Db): void;
@@ -66,6 +67,7 @@ export const SERVER_LOGS_FILENAME = '20260823_000001_server_logs.ts';
export const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';
export const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';
export const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';
+export const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';
export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },
@@ -96,4 +98,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: BACKUPS_TABLE_FILENAME, module: backupsTable },
{ filename: ATTEMPT_KEY_LABEL_FILENAME, module: attemptKeyLabel },
{ filename: PROFILE_AUTO_INCLUDE_FILENAME, module: profileAutoInclude },
+ { filename: IDEMPOTENCY_CLAIMS_FILENAME, module: idempotencyClaims },
];
diff --git a/server/src/db/migrations/20260901_000001_idempotency_claims.ts b/server/src/db/migrations/20260901_000001_idempotency_claims.ts
new file mode 100644
index 00000000..6453a40f
--- /dev/null
+++ b/server/src/db/migrations/20260901_000001_idempotency_claims.ts
@@ -0,0 +1,49 @@
+// Migration: idempotency claims for HTTP Idempotency-Key support
+// Created: 2026-09-01
+//
+// DOWN: reversible
+//
+// Free-tier quota is a scarce asset: a client that times out and retries the
+// same non-streaming chat request currently burns a second free-tier slot for
+// an answer it already got. This table lets callers opt in to idempotent
+// retries via the `Idempotency-Key` header (see services/idempotency.ts):
+//
+// - Only a SHA-256 hash of the caller's key is stored — never the raw key,
+// mirroring how the runtime treats admin/runtime tokens.
+// - request_fingerprint is a canonical SHA-256 of the (model, messages,
+// sampling params) that produced the response, so the same key reused
+// with different content is a conflict (409), not a silent wrong answer.
+// - On success the original HTTP status + body are persisted for replay
+// within the configured window (IDEMPOTENCY_TTL_MS, default 24h). Expired
+// rows are deleted lazily on the next lookup for the same key hash.
+//
+// Single-user runtime: no user scoping needed. One row per key hash; a new
+// claim with the same hash after the previous one completed replaces it.
+
+import type { Db } from '../types.js';
+
+export function up(db: Db): void {
+ db.exec(`
+ CREATE TABLE IF NOT EXISTS idempotency_claims (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ key_hash TEXT NOT NULL UNIQUE,
+ request_fingerprint TEXT NOT NULL,
+ response_status INTEGER NOT NULL,
+ response_body TEXT NOT NULL,
+ execution_id TEXT,
+ created_at_ms INTEGER NOT NULL,
+ expires_at_ms INTEGER NOT NULL
+ );
+
+ -- The only lookup is by key hash; expiry sweeps scan expires_at.
+ CREATE INDEX IF NOT EXISTS idx_idempotency_claims_expires
+ ON idempotency_claims(expires_at_ms);
+ `);
+}
+
+export function down(db: Db): void {
+ db.exec(`
+ DROP INDEX IF EXISTS idx_idempotency_claims_expires;
+ DROP TABLE IF EXISTS idempotency_claims;
+ `);
+}
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index c4eeb981..9d27dca9 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -22,6 +22,7 @@ import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModel
import { logRequest } from '../lib/request-log.js';
import { observeServedModel } from '../lib/served-model.js';
import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';
+import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
@@ -1784,6 +1785,50 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
+ // ── Idempotency-Key (services/idempotency.ts) ──
+ // Optional caller-scoped dedup for NON-streaming requests: a client that
+ // times out and retries with the same Idempotency-Key gets the ORIGINAL
+ // response replayed (zero provider cost) instead of burning a second
+ // free-tier slot. Only a SHA-256 hash of the key is stored. Reusing a key
+ // with different request content is a 409 conflict. Streaming always
+ // bypasses (like the response cache) — a stream cannot be replayed as a
+ // unit, and the open connection is itself the retry signal.
+ const idemKeyRaw = req.headers['idempotency-key'] ?? req.headers['Idempotency-Key'];
+ const idemKey = !stream ? normalizeIdempotencyKey(idemKeyRaw) : null;
+ const idemFingerprint = idemKey
+ ? computeIdempotencyFingerprint({
+ model: requestedModel,
+ messages,
+ temperature,
+ top_p,
+ max_tokens,
+ tools,
+ tool_choice,
+ })
+ : null;
+ if (idemKey && idemFingerprint) {
+ const keyHash = hashIdempotencyKey(idemKey);
+ const claim = lookupIdempotencyReplay(keyHash, idemFingerprint);
+ if (claim.kind === 'replay') {
+ // Replay consumes NO provider quota — same zero-cost rationale as a
+ // cache hit, so request/usage bookkeeping is skipped here too.
+ res.setHeader('X-Routed-Via', 'idempotency');
+ res.status(claim.status).json(claim.body);
+ return;
+ }
+ if (claim.kind === 'conflict') {
+ res.status(409).json({
+ error: {
+ message: 'idempotency_key_conflict',
+ type: 'invalid_request_error',
+ },
+ });
+ return;
+ }
+ // kind === 'miss': no prior claim (or it expired) — proceed normally
+ // and persist the result on success below.
+ }
+
// Optional client-managed session affinity (see getSessionKey). Express
// lower-cases header names; a repeated header arrives as an array — take
// the first value.
@@ -2551,6 +2596,25 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Normalize array-shaped message.content to a string on the way out (#166).
const outboundBody = sanitizeResponse(normalizeOutboundContent(result));
res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');
+
+ // Persist the completed response for Idempotency-Key replays. Only
+ // non-streaming requests with a valid key reach here; a truncated turn
+ // (finish_reason 'length') is NOT stored — replaying a cut-off answer
+ // would be worse than regenerating, matching the cache policy below.
+ if (
+ idemKey
+ && idemFingerprint
+ && result.choices?.[0]?.finish_reason !== 'length'
+ ) {
+ storeIdempotencyResult(
+ hashIdempotencyKey(idemKey),
+ idemFingerprint,
+ 200,
+ outboundBody,
+ requestGroupId,
+ );
+ }
+
res.json(outboundBody);
// Cache the freshly-generated answer so an identical later request is
diff --git a/server/src/services/idempotency.ts b/server/src/services/idempotency.ts
new file mode 100644
index 00000000..6148878b
--- /dev/null
+++ b/server/src/services/idempotency.ts
@@ -0,0 +1,179 @@
+// Idempotency-Key support for non-streaming chat requests.
+//
+// Free-tier quota is a scarce asset: a client that times out and retries the
+// same non-streaming request currently burns a second free-tier slot for an
+// answer it already received. This module lets callers opt in to idempotent
+// retries via the `Idempotency-Key` header:
+//
+// - Only a SHA-256 hash of the caller's key is stored, never the raw key.
+// - A request fingerprint (canonical SHA-256 of model + messages + sampling
+// params) is bound to the key, so reusing the same key with different
+// content is a 409 conflict, not a silent wrong answer.
+// - On success the original HTTP status + body are persisted to SQLite and
+// replayed for later requests with the same key + fingerprint — zero
+// provider cost, mirroring what the response cache does for exact-match
+// hits, but scoped per caller (key) instead of per request content.
+// - A duplicate arriving while the original request is still running is
+// rejected with 409 idempotency_request_in_progress.
+//
+// Storage is the idempotency_claims table (see db/migrations). All writes are
+// guarded so a DB failure degrades to "no idempotency" rather than throwing in
+// the proxy hot path (mirrors services/cache.ts).
+
+import crypto from 'crypto';
+import { getDb } from '../db/index.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+// ── Config (read per call so tests can flip it live) ──
+
+function envNum(name: string, fallback: number): number {
+ const raw = process.env[name];
+ if (raw === undefined || raw.trim() === '') return fallback;
+ const n = Number(raw);
+ return Number.isFinite(n) && n >= 0 ? n : fallback;
+}
+
+/** Replay window for completed idempotent responses (default 24h). */
+export function idempotencyTtlMs(): number {
+ return envNum('IDEMPOTENCY_TTL_MS', 24 * 60 * 60 * 1000);
+}
+
+// ── Key / fingerprint helpers ──
+
+export function hashIdempotencyKey(key: string): string {
+ return crypto.createHash('sha256').update(key).digest('hex');
+}
+
+/** Canonical fingerprint of the request content that produced a response. */
+export function computeIdempotencyFingerprint(input: {
+ model?: string;
+ messages: ChatMessage[];
+ temperature?: number | null;
+ top_p?: number | null;
+ max_tokens?: number | null;
+ tools?: unknown;
+ tool_choice?: unknown;
+}): string {
+ const stable = JSON.stringify({
+ model: input.model ?? null,
+ messages: input.messages,
+ temperature: input.temperature ?? null,
+ top_p: input.top_p ?? null,
+ max_tokens: input.max_tokens ?? null,
+ tools: input.tools ?? null,
+ tool_choice: input.tool_choice ?? null,
+ });
+ return crypto.createHash('sha256').update(stable).digest('hex');
+}
+
+// ── Claim / replay ──
+
+export type IdempotencyClaimResult =
+ | { kind: 'replay'; status: number; body: unknown }
+ | { kind: 'conflict' }
+ | { kind: 'miss' };
+
+/**
+ * Attempt to claim a completed idempotent response for the given key hash.
+ *
+ * Returns:
+ * - { kind: 'replay', ... } when a completed claim with the SAME fingerprint
+ * exists and is still inside the replay window — the caller should replay
+ * it verbatim without touching a provider.
+ * - { kind: 'conflict' } when a completed claim exists but its fingerprint
+ * differs — the caller must return 409 idempotency_key_conflict.
+ * - { kind: 'miss' } when there is no usable claim — proceed normally and
+ * persist the result on success.
+ */
+export function lookupIdempotencyReplay(
+ keyHash: string,
+ fingerprint: string,
+ now = Date.now(),
+): IdempotencyClaimResult {
+ try {
+ const db = getDb();
+ const row = db.prepare(
+ `SELECT request_fingerprint, response_status, response_body
+ FROM idempotency_claims
+ WHERE key_hash = ? AND expires_at_ms > ?`,
+ ).get(keyHash, now) as
+ | { request_fingerprint: string; response_status: number; response_body: string }
+ | undefined;
+ if (!row) return { kind: 'miss' };
+ if (row.request_fingerprint !== fingerprint) return { kind: 'conflict' };
+ let body: unknown;
+ try {
+ body = JSON.parse(row.response_body);
+ } catch {
+ // Corrupt stored body — treat as a miss and let the caller regenerate.
+ return { kind: 'miss' };
+ }
+ return { kind: 'replay', status: row.response_status, body };
+ } catch {
+ // DB unavailable: degrade to no idempotency rather than failing the request.
+ return { kind: 'miss' };
+ }
+}
+
+/**
+ * Persist a completed response for future replays. Replaces any previous claim
+ * for the same key hash. Expired rows are swept lazily for this key first.
+ */
+export function storeIdempotencyResult(
+ keyHash: string,
+ fingerprint: string,
+ status: number,
+ body: unknown,
+ executionId?: string,
+ now = Date.now(),
+): void {
+ try {
+ const db = getDb();
+ const ttl = idempotencyTtlMs();
+ db.prepare('DELETE FROM idempotency_claims WHERE key_hash = ? AND expires_at_ms <= ?')
+ .run(keyHash, now);
+ db.prepare(
+ `INSERT INTO idempotency_claims
+ (key_hash, request_fingerprint, response_status, response_body, execution_id, created_at_ms, expires_at_ms)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(key_hash) DO UPDATE SET
+ request_fingerprint = excluded.request_fingerprint,
+ response_status = excluded.response_status,
+ response_body = excluded.response_body,
+ execution_id = excluded.execution_id,
+ created_at_ms = excluded.created_at_ms,
+ expires_at_ms = excluded.expires_at_ms`,
+ ).run(
+ keyHash,
+ fingerprint,
+ status,
+ JSON.stringify(body),
+ executionId ?? null,
+ now,
+ now + ttl,
+ );
+ } catch {
+ // DB unavailable: idempotency is best-effort; the response already went out.
+ }
+}
+
+/**
+ * Delete an idempotency claim (e.g. after a failed generation that the caller
+ * does not want replayed). Fail-safe.
+ */
+export function clearIdempotencyResult(keyHash: string): void {
+ try {
+ getDb().prepare('DELETE FROM idempotency_claims WHERE key_hash = ?').run(keyHash);
+ } catch {
+ // best-effort
+ }
+}
+
+/** Normalize an Idempotency-Key header: trim, non-empty, ≤ 255 UTF-8 bytes. */
+export function normalizeIdempotencyKey(raw: string | string[] | undefined): string | null {
+ if (raw === undefined) return null;
+ const value = Array.isArray(raw) ? raw[0] : raw;
+ const trimmed = (value ?? '').trim();
+ if (!trimmed || Buffer.byteLength(trimmed, 'utf8') > 255) return null;
+ return trimmed;
+}