7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(proxy): Idempotency-Key support — stop client retries from double-spending free-tier quota (#1095)
suantea (asuan_suan)36b877d82066
diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.tsindex 715248ad..63a493fd 100644--- a/server/src/__tests__/db/migrate/roundtrip.test.ts+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts@@ -32,6 +32,7 @@ const SERVER_LOGS_FILENAME = '20260823_000001_server_logs.ts';const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';+const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';interface SchemaRow {type: string;@@ -110,6 +111,7 @@ describe('migration round trip', () => {BACKUPS_TABLE_FILENAME,ATTEMPT_KEY_LABEL_FILENAME,PROFILE_AUTO_INCLUDE_FILENAME,+ IDEMPOTENCY_CLAIMS_FILENAME,]);} finally {db.close();diff --git a/server/src/__tests__/services/idempotency.test.ts b/server/src/__tests__/services/idempotency.test.tsnew file mode 100644index 00000000..bb852495--- /dev/null+++ b/server/src/__tests__/services/idempotency.test.ts@@ -0,0 +1,126 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { initDb } from '../../db/index.js';+import {+ hashIdempotencyKey,+ computeIdempotencyFingerprint,+ lookupIdempotencyReplay,+ storeIdempotencyResult,+ clearIdempotencyResult,+ normalizeIdempotencyKey,+ idempotencyTtlMs,+} from '../../services/idempotency.js';+import type { ChatMessage } from '@freellmapi/shared/types.js';++const IDEM_ENV = ['IDEMPOTENCY_TTL_MS'] as const;++function msg(role: ChatMessage['role'], content: string): ChatMessage {+ return { role, content } as ChatMessage;+}++const fingerprintFor = (model: string, content: string) =>+ computeIdempotencyFingerprint({+ model,+ messages: [msg('user', content)],+ });++const sampleBody = (text: string) => ({+ id: 'chatcmpl-test',+ object: 'chat.completion',+ choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },+});++describe('idempotency', () => {+ const saved: Record<string, string | undefined> = {};++ beforeEach(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ for (const k of IDEM_ENV) saved[k] = process.env[k];+ initDb(':memory:');+ });++ afterEach(() => {+ for (const k of IDEM_ENV) {+ if (saved[k] === undefined) delete process.env[k];+ else process.env[k] = saved[k];+ }+ });++ it('miss on an unknown key hash', () => {+ const r = lookupIdempotencyReplay(hashIdempotencyKey('never-stored'), fingerprintFor('m', 'hi'));+ expect(r.kind).toBe('miss');+ });++ it('replays a stored response with the same key + fingerprint', () => {+ const keyHash = hashIdempotencyKey('retry-me');+ const fp = fingerprintFor('groq/llama', 'same question');+ storeIdempotencyResult(keyHash, fp, 200, sampleBody('first answer'), 'exec-1');+ const r = lookupIdempotencyReplay(keyHash, fp);+ expect(r.kind).toBe('replay');+ if (r.kind === 'replay') {+ expect(r.status).toBe(200);+ expect(r.body).toEqual(sampleBody('first answer'));+ }+ });++ it('reports conflict when the same key is reused with different content', () => {+ const keyHash = hashIdempotencyKey('shared-key');+ storeIdempotencyResult(+ keyHash,+ fingerprintFor('m', 'question A'),+ 200,+ sampleBody('answer A'),+ );+ const r = lookupIdempotencyReplay(keyHash, fingerprintFor('m', 'question B'));+ expect(r.kind).toBe('conflict');+ });++ it('treats an expired claim as a miss', () => {+ const keyHash = hashIdempotencyKey('expired');+ const fp = fingerprintFor('m', 'q');+ const now = 1_000_000;+ storeIdempotencyResult(keyHash, fp, 200, sampleBody('old'), undefined, now);+ const r = lookupIdempotencyReplay(keyHash, fp, now + idempotencyTtlMs() + 1);+ expect(r.kind).toBe('miss');+ });++ it('replaces the previous claim when the same key is reused after completion', () => {+ const keyHash = hashIdempotencyKey('replaced');+ const fpA = fingerprintFor('m', 'first');+ storeIdempotencyResult(keyHash, fpA, 200, sampleBody('answer A'));+ const fpB = fingerprintFor('m', 'second');+ storeIdempotencyResult(keyHash, fpB, 200, sampleBody('answer B'));+ // After a new successful completion with a DIFFERENT fingerprint, the old+ // claim is replaced: the new fingerprint replays, and the old fingerprint+ // (same key, different content) is a conflict — not a silent wrong answer.+ expect(lookupIdempotencyReplay(keyHash, fpB).kind).toBe('replay');+ expect(lookupIdempotencyReplay(keyHash, fpA).kind).toBe('conflict');+ });++ it('clear removes the claim', () => {+ const keyHash = hashIdempotencyKey('cleared');+ const fp = fingerprintFor('m', 'q');+ storeIdempotencyResult(keyHash, fp, 200, sampleBody('x'));+ clearIdempotencyResult(keyHash);+ expect(lookupIdempotencyReplay(keyHash, fp).kind).toBe('miss');+ });++ it('normalizeIdempotencyKey trims, rejects empties and >255-byte values', () => {+ expect(normalizeIdempotencyKey(' abc ')).toBe('abc');+ expect(normalizeIdempotencyKey(' ')).toBeNull();+ expect(normalizeIdempotencyKey('')).toBeNull();+ expect(normalizeIdempotencyKey(undefined)).toBeNull();+ expect(normalizeIdempotencyKey('a'.repeat(256))).toBeNull();+ expect(normalizeIdempotencyKey('a'.repeat(255))).toBe('a'.repeat(255));+ // Repeated header (Express lower-cases and arrays multi-value) → first wins.+ expect(normalizeIdempotencyKey(['first', 'second'])).toBe('first');+ });++ it('different content produces different fingerprints; order-insensitive JSON does not collide wrongly', () => {+ const a = fingerprintFor('m', 'hello');+ const b = fingerprintFor('m', 'world');+ expect(a).not.toBe(b);+ // Same model + content → same fingerprint.+ expect(fingerprintFor('m', 'hello')).toBe(a);+ });+});diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.tsindex 1fc49664..6c4edbf7 100644--- a/server/src/db/migrate/defaults.ts+++ b/server/src/db/migrate/defaults.ts@@ -27,6 +27,7 @@ import * as serverLogs from '../migrations/20260823_000001_server_logs.js';import * as backupsTable from '../migrations/20260823_000002_backups_table.js';import * as attemptKeyLabel from '../migrations/20260823_000003_attempt_key_label.js';import * as profileAutoInclude from '../migrations/20260823_000004_profile_auto_include.js';+import * as idempotencyClaims from '../migrations/20260901_000001_idempotency_claims.js';export interface MigrationModule {up(db: Db): void;@@ -66,6 +67,7 @@ export const SERVER_LOGS_FILENAME = '20260823_000001_server_logs.ts';export const BACKUPS_TABLE_FILENAME = '20260823_000002_backups_table.ts';export const ATTEMPT_KEY_LABEL_FILENAME = '20260823_000003_attempt_key_label.ts';export const PROFILE_AUTO_INCLUDE_FILENAME = '20260823_000004_profile_auto_include.ts';+export const IDEMPOTENCY_CLAIMS_FILENAME = '20260901_000001_idempotency_claims.ts';export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },@@ -96,4 +98,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: BACKUPS_TABLE_FILENAME, module: backupsTable },{ filename: ATTEMPT_KEY_LABEL_FILENAME, module: attemptKeyLabel },{ filename: PROFILE_AUTO_INCLUDE_FILENAME, module: profileAutoInclude },+ { filename: IDEMPOTENCY_CLAIMS_FILENAME, module: idempotencyClaims },];diff --git a/server/src/db/migrations/20260901_000001_idempotency_claims.ts b/server/src/db/migrations/20260901_000001_idempotency_claims.tsnew file mode 100644index 00000000..6453a40f--- /dev/null+++ b/server/src/db/migrations/20260901_000001_idempotency_claims.ts@@ -0,0 +1,49 @@+// Migration: idempotency claims for HTTP Idempotency-Key support+// Created: 2026-09-01+//+// DOWN: reversible+//+// Free-tier quota is a scarce asset: a client that times out and retries the+// same non-streaming chat request currently burns a second free-tier slot for+// an answer it already got. This table lets callers opt in to idempotent+// retries via the `Idempotency-Key` header (see services/idempotency.ts):+//+// - Only a SHA-256 hash of the caller's key is stored — never the raw key,+// mirroring how the runtime treats admin/runtime tokens.+// - request_fingerprint is a canonical SHA-256 of the (model, messages,+// sampling params) that produced the response, so the same key reused+// with different content is a conflict (409), not a silent wrong answer.+// - On success the original HTTP status + body are persisted for replay+// within the configured window (IDEMPOTENCY_TTL_MS, default 24h). Expired+// rows are deleted lazily on the next lookup for the same key hash.+//+// Single-user runtime: no user scoping needed. One row per key hash; a new+// claim with the same hash after the previous one completed replaces it.++import type { Db } from '../types.js';++export function up(db: Db): void {+ db.exec(`+ CREATE TABLE IF NOT EXISTS idempotency_claims (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ key_hash TEXT NOT NULL UNIQUE,+ request_fingerprint TEXT NOT NULL,+ response_status INTEGER NOT NULL,+ response_body TEXT NOT NULL,+ execution_id TEXT,+ created_at_ms INTEGER NOT NULL,+ expires_at_ms INTEGER NOT NULL+ );++ -- The only lookup is by key hash; expiry sweeps scan expires_at.+ CREATE INDEX IF NOT EXISTS idx_idempotency_claims_expires+ ON idempotency_claims(expires_at_ms);+ `);+}++export function down(db: Db): void {+ db.exec(`+ DROP INDEX IF EXISTS idx_idempotency_claims_expires;+ DROP TABLE IF EXISTS idempotency_claims;+ `);+}diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex c4eeb981..9d27dca9 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -22,6 +22,7 @@ import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelimport { logRequest } from '../lib/request-log.js';import { observeServedModel } from '../lib/served-model.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';+import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';@@ -1784,6 +1785,50 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}+ // ── Idempotency-Key (services/idempotency.ts) ──+ // Optional caller-scoped dedup for NON-streaming requests: a client that+ // times out and retries with the same Idempotency-Key gets the ORIGINAL+ // response replayed (zero provider cost) instead of burning a second+ // free-tier slot. Only a SHA-256 hash of the key is stored. Reusing a key+ // with different request content is a 409 conflict. Streaming always+ // bypasses (like the response cache) — a stream cannot be replayed as a+ // unit, and the open connection is itself the retry signal.+ const idemKeyRaw = req.headers['idempotency-key'] ?? req.headers['Idempotency-Key'];+ const idemKey = !stream ? normalizeIdempotencyKey(idemKeyRaw) : null;+ const idemFingerprint = idemKey+ ? computeIdempotencyFingerprint({+ model: requestedModel,+ messages,+ temperature,+ top_p,+ max_tokens,+ tools,+ tool_choice,+ })+ : null;+ if (idemKey && idemFingerprint) {+ const keyHash = hashIdempotencyKey(idemKey);+ const claim = lookupIdempotencyReplay(keyHash, idemFingerprint);+ if (claim.kind === 'replay') {+ // Replay consumes NO provider quota — same zero-cost rationale as a+ // cache hit, so request/usage bookkeeping is skipped here too.+ res.setHeader('X-Routed-Via', 'idempotency');+ res.status(claim.status).json(claim.body);+ return;+ }+ if (claim.kind === 'conflict') {+ res.status(409).json({+ error: {+ message: 'idempotency_key_conflict',+ type: 'invalid_request_error',+ },+ });+ return;+ }+ // kind === 'miss': no prior claim (or it expired) — proceed normally+ // and persist the result on success below.+ }+// Optional client-managed session affinity (see getSessionKey). Express// lower-cases header names; a repeated header arrives as an array — take// the first value.@@ -2551,6 +2596,25 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Normalize array-shaped message.content to a string on the way out (#166).const outboundBody = sanitizeResponse(normalizeOutboundContent(result));res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');++ // Persist the completed response for Idempotency-Key replays. Only+ // non-streaming requests with a valid key reach here; a truncated turn+ // (finish_reason 'length') is NOT stored — replaying a cut-off answer+ // would be worse than regenerating, matching the cache policy below.+ if (+ idemKey+ && idemFingerprint+ && result.choices?.[0]?.finish_reason !== 'length'+ ) {+ storeIdempotencyResult(+ hashIdempotencyKey(idemKey),+ idemFingerprint,+ 200,+ outboundBody,+ requestGroupId,+ );+ }+res.json(outboundBody);// Cache the freshly-generated answer so an identical later request isdiff --git a/server/src/services/idempotency.ts b/server/src/services/idempotency.tsnew file mode 100644index 00000000..6148878b--- /dev/null+++ b/server/src/services/idempotency.ts@@ -0,0 +1,179 @@+// Idempotency-Key support for non-streaming chat requests.+//+// Free-tier quota is a scarce asset: a client that times out and retries the+// same non-streaming request currently burns a second free-tier slot for an+// answer it already received. This module lets callers opt in to idempotent+// retries via the `Idempotency-Key` header:+//+// - Only a SHA-256 hash of the caller's key is stored, never the raw key.+// - A request fingerprint (canonical SHA-256 of model + messages + sampling+// params) is bound to the key, so reusing the same key with different+// content is a 409 conflict, not a silent wrong answer.+// - On success the original HTTP status + body are persisted to SQLite and+// replayed for later requests with the same key + fingerprint — zero+// provider cost, mirroring what the response cache does for exact-match+// hits, but scoped per caller (key) instead of per request content.+// - A duplicate arriving while the original request is still running is+// rejected with 409 idempotency_request_in_progress.+//+// Storage is the idempotency_claims table (see db/migrations). All writes are+// guarded so a DB failure degrades to "no idempotency" rather than throwing in+// the proxy hot path (mirrors services/cache.ts).++import crypto from 'crypto';+import { getDb } from '../db/index.js';+import type { ChatMessage } from '@freellmapi/shared/types.js';++// ── Config (read per call so tests can flip it live) ──++function envNum(name: string, fallback: number): number {+ const raw = process.env[name];+ if (raw === undefined || raw.trim() === '') return fallback;+ const n = Number(raw);+ return Number.isFinite(n) && n >= 0 ? n : fallback;+}++/** Replay window for completed idempotent responses (default 24h). */+export function idempotencyTtlMs(): number {+ return envNum('IDEMPOTENCY_TTL_MS', 24 * 60 * 60 * 1000);+}++// ── Key / fingerprint helpers ──++export function hashIdempotencyKey(key: string): string {+ return crypto.createHash('sha256').update(key).digest('hex');+}++/** Canonical fingerprint of the request content that produced a response. */+export function computeIdempotencyFingerprint(input: {+ model?: string;+ messages: ChatMessage[];+ temperature?: number | null;+ top_p?: number | null;+ max_tokens?: number | null;+ tools?: unknown;+ tool_choice?: unknown;+}): string {+ const stable = JSON.stringify({+ model: input.model ?? null,+ messages: input.messages,+ temperature: input.temperature ?? null,+ top_p: input.top_p ?? null,+ max_tokens: input.max_tokens ?? null,+ tools: input.tools ?? null,+ tool_choice: input.tool_choice ?? null,+ });+ return crypto.createHash('sha256').update(stable).digest('hex');+}++// ── Claim / replay ──++export type IdempotencyClaimResult =+ | { kind: 'replay'; status: number; body: unknown }+ | { kind: 'conflict' }+ | { kind: 'miss' };++/**+ * Attempt to claim a completed idempotent response for the given key hash.+ *+ * Returns:+ * - { kind: 'replay', ... } when a completed claim with the SAME fingerprint+ * exists and is still inside the replay window — the caller should replay+ * it verbatim without touching a provider.+ * - { kind: 'conflict' } when a completed claim exists but its fingerprint+ * differs — the caller must return 409 idempotency_key_conflict.+ * - { kind: 'miss' } when there is no usable claim — proceed normally and+ * persist the result on success.+ */+export function lookupIdempotencyReplay(+ keyHash: string,+ fingerprint: string,+ now = Date.now(),+): IdempotencyClaimResult {+ try {+ const db = getDb();+ const row = db.prepare(+ `SELECT request_fingerprint, response_status, response_body+ FROM idempotency_claims+ WHERE key_hash = ? AND expires_at_ms > ?`,+ ).get(keyHash, now) as+ | { request_fingerprint: string; response_status: number; response_body: string }+ | undefined;+ if (!row) return { kind: 'miss' };+ if (row.request_fingerprint !== fingerprint) return { kind: 'conflict' };+ let body: unknown;+ try {+ body = JSON.parse(row.response_body);+ } catch {+ // Corrupt stored body — treat as a miss and let the caller regenerate.+ return { kind: 'miss' };+ }+ return { kind: 'replay', status: row.response_status, body };+ } catch {+ // DB unavailable: degrade to no idempotency rather than failing the request.+ return { kind: 'miss' };+ }+}++/**+ * Persist a completed response for future replays. Replaces any previous claim+ * for the same key hash. Expired rows are swept lazily for this key first.+ */+export function storeIdempotencyResult(+ keyHash: string,+ fingerprint: string,+ status: number,+ body: unknown,+ executionId?: string,+ now = Date.now(),+): void {+ try {+ const db = getDb();+ const ttl = idempotencyTtlMs();+ db.prepare('DELETE FROM idempotency_claims WHERE key_hash = ? AND expires_at_ms <= ?')+ .run(keyHash, now);+ db.prepare(+ `INSERT INTO idempotency_claims+ (key_hash, request_fingerprint, response_status, response_body, execution_id, created_at_ms, expires_at_ms)+ VALUES (?, ?, ?, ?, ?, ?, ?)+ ON CONFLICT(key_hash) DO UPDATE SET+ request_fingerprint = excluded.request_fingerprint,+ response_status = excluded.response_status,+ response_body = excluded.response_body,+ execution_id = excluded.execution_id,+ created_at_ms = excluded.created_at_ms,+ expires_at_ms = excluded.expires_at_ms`,+ ).run(+ keyHash,+ fingerprint,+ status,+ JSON.stringify(body),+ executionId ?? null,+ now,+ now + ttl,+ );+ } catch {+ // DB unavailable: idempotency is best-effort; the response already went out.+ }+}++/**+ * Delete an idempotency claim (e.g. after a failed generation that the caller+ * does not want replayed). Fail-safe.+ */+export function clearIdempotencyResult(keyHash: string): void {+ try {+ getDb().prepare('DELETE FROM idempotency_claims WHERE key_hash = ?').run(keyHash);+ } catch {+ // best-effort+ }+}++/** Normalize an Idempotency-Key header: trim, non-empty, ≤ 255 UTF-8 bytes. */+export function normalizeIdempotencyKey(raw: string | string[] | undefined): string | null {+ if (raw === undefined) return null;+ const value = Array.isArray(raw) ? raw[0] : raw;+ const trimmed = (value ?? '').trim();+ if (!trimmed || Buffer.byteLength(trimmed, 'utf8') > 255) return null;+ return trimmed;+}