Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
/**
* Semantic Cache Configuration
*
* Configurable options for OmniRoute's dual-layer semantic caching system.
* Supports environment variable overrides via OMNIROUTE_SEMANTIC_CACHE_*.
*
* @module config/semanticCacheConfig
*/
export type SemanticCacheBackend = "memory" | "redis";
export type SemanticCacheType = "direct" | "semantic" | "both";
export interface SemanticCacheConfig {
/**
* Master toggle for the dual-layer (in-memory/Redis vector) manager. OFF by default:
* the legacy SQLite exact-match cache (`semanticCacheEnabled`) keeps working on its
* own; this layer is opt-in via the `semanticCacheVectorEnabled` setting or
* `OMNIROUTE_SEMANTIC_CACHE_ENABLED=true`, because enabling it calls an embedding
* endpoint on every cacheable request (#14159 re-land of #12630).
*/
enabled: boolean;
/** Storage and vector backend. Defaults to "memory". */
backend: SemanticCacheBackend;
/** Cosine similarity threshold (0.0 - 1.0) for semantic hits. Defaults to 0.8. */
similarityThreshold: number;
/** Time-to-live for cached entries in milliseconds. Defaults to 30 minutes (1800000 ms). */
ttlMs: number;
/** Maximum number of entries stored in memory. Defaults to 1000. */
maxEntries: number;
/** Embedding provider to use for semantic embeddings. Defaults to "openai". */
embeddingProvider: string;
/** Embedding model name. Defaults to "text-embedding-3-small". */
embeddingModel: string;
/** Vector dimension of the embedding model (e.g. 1536). Optional/auto-detected. */
embeddingDimension?: number;
/** Maximum milliseconds to wait for embedding generation before failing open. Defaults to 3000 ms. */
embeddingTimeoutMs: number;
/** If true, cache entries are isolated per model name. Defaults to true. */
cacheByModel: boolean;
/** If true, cache entries are isolated per provider name. Defaults to true. */
cacheByProvider: boolean;
/** Number of recent conversation turns embedded and checked. Defaults to 3. */
conversationHistoryDepth: number;
/** Maximum message turns in conversation allowed before bypassing caching. Defaults to 50. */
conversationHistoryThreshold: number;
/** If true, system prompt messages are excluded from embedding and cache keys. Defaults to false. */
excludeSystemPrompt: boolean;
/** Optional custom base URL for embedding provider (e.g. http://192.168.31.147:13305/v1/embeddings). */
embeddingBaseUrl?: string;
/** Optional custom API key / Bearer token for embedding provider. */
embeddingApiKey?: string;
/** Redis connection URL when backend is "redis". */
redisUrl?: string;
/** Key prefix for Redis cache keys. Defaults to "omniroute:semcache:". */
redisPrefix: string;
/** If true, only temperature=0 requests are cacheable. Defaults to true for strict determinism. */
requireZeroTemperature: boolean;
}
export const DEFAULT_SEMANTIC_CACHE_CONFIG: SemanticCacheConfig = {
enabled: false,
backend: "memory",
similarityThreshold: 0.8,
ttlMs: 1800000, // 30 minutes
maxEntries: 1000,
embeddingProvider: "openai",
embeddingModel: "text-embedding-3-small",
embeddingDimension: 1536,
embeddingTimeoutMs: 3000,
cacheByModel: true,
cacheByProvider: true,
conversationHistoryDepth: 3,
conversationHistoryThreshold: 50,
excludeSystemPrompt: false,
redisUrl: undefined,
redisPrefix: "omniroute:semcache:",
requireZeroTemperature: true,
};
function parseBoolean(val: string | undefined, fallback: boolean): boolean {
if (val === undefined || val === "") return fallback;
const lower = val.toLowerCase().trim();
if (lower === "true" || lower === "1" || lower === "yes") return true;
if (lower === "false" || lower === "0" || lower === "no") return false;
return fallback;
}
function parseNumber(val: string | undefined, fallback: number): number {
if (val === undefined || val === "") return fallback;
const parsed = Number(val);
return Number.isFinite(parsed) ? parsed : fallback;
}
type DynamicConfigResolver = () => Partial<SemanticCacheConfig> | null | undefined;
let dynamicResolver: DynamicConfigResolver | null = null;
export function registerSemanticCacheConfigResolver(resolver: DynamicConfigResolver): void {
dynamicResolver = resolver;
}
/**
* Resolves semantic cache configuration from environment variables, merged
* with optional dynamic database settings and explicit overrides.
*/
export function resolveSemanticCacheConfig(
overrides?: Partial<SemanticCacheConfig>
): SemanticCacheConfig {
const dynamic = dynamicResolver ? dynamicResolver() : null;
const env = process.env;
const backendEnv = (env.OMNIROUTE_SEMANTIC_CACHE_BACKEND || "").toLowerCase().trim();
const backend: SemanticCacheBackend =
backendEnv === "redis"
? "redis"
: backendEnv === "memory"
? "memory"
: (dynamic?.backend ?? DEFAULT_SEMANTIC_CACHE_CONFIG.backend);
const resolved: SemanticCacheConfig = {
enabled:
env.OMNIROUTE_SEMANTIC_CACHE_ENABLED !== undefined
? parseBoolean(env.OMNIROUTE_SEMANTIC_CACHE_ENABLED, DEFAULT_SEMANTIC_CACHE_CONFIG.enabled)
: (dynamic?.enabled ?? DEFAULT_SEMANTIC_CACHE_CONFIG.enabled),
backend,
similarityThreshold:
env.OMNIROUTE_SEMANTIC_CACHE_THRESHOLD !== undefined
? parseNumber(
env.OMNIROUTE_SEMANTIC_CACHE_THRESHOLD,
DEFAULT_SEMANTIC_CACHE_CONFIG.similarityThreshold
)
: (dynamic?.similarityThreshold ?? DEFAULT_SEMANTIC_CACHE_CONFIG.similarityThreshold),
ttlMs:
env.OMNIROUTE_SEMANTIC_CACHE_TTL_MS !== undefined
? parseNumber(env.OMNIROUTE_SEMANTIC_CACHE_TTL_MS, DEFAULT_SEMANTIC_CACHE_CONFIG.ttlMs)
: (dynamic?.ttlMs ?? DEFAULT_SEMANTIC_CACHE_CONFIG.ttlMs),
maxEntries:
env.OMNIROUTE_SEMANTIC_CACHE_MAX_ENTRIES !== undefined
? parseNumber(
env.OMNIROUTE_SEMANTIC_CACHE_MAX_ENTRIES,
DEFAULT_SEMANTIC_CACHE_CONFIG.maxEntries
)
: (dynamic?.maxEntries ?? DEFAULT_SEMANTIC_CACHE_CONFIG.maxEntries),
embeddingProvider:
env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_PROVIDER?.trim() ||
dynamic?.embeddingProvider ||
DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingProvider,
embeddingModel:
env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_MODEL?.trim() ||
dynamic?.embeddingModel ||
DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingModel,
embeddingDimension: env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_DIMENSION
? parseNumber(env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_DIMENSION, 1536)
: (dynamic?.embeddingDimension ?? DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingDimension),
embeddingTimeoutMs: parseNumber(
env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_TIMEOUT_MS,
dynamic?.embeddingTimeoutMs ?? DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingTimeoutMs
),
cacheByModel: parseBoolean(
env.OMNIROUTE_SEMANTIC_CACHE_BY_MODEL,
dynamic?.cacheByModel ?? DEFAULT_SEMANTIC_CACHE_CONFIG.cacheByModel
),
cacheByProvider: parseBoolean(
env.OMNIROUTE_SEMANTIC_CACHE_BY_PROVIDER,
dynamic?.cacheByProvider ?? DEFAULT_SEMANTIC_CACHE_CONFIG.cacheByProvider
),
conversationHistoryDepth: parseNumber(
env.OMNIROUTE_SEMANTIC_CACHE_HISTORY_DEPTH,
dynamic?.conversationHistoryDepth ?? DEFAULT_SEMANTIC_CACHE_CONFIG.conversationHistoryDepth
),
conversationHistoryThreshold: parseNumber(
env.OMNIROUTE_SEMANTIC_CACHE_HISTORY_THRESHOLD,
dynamic?.conversationHistoryThreshold ??
DEFAULT_SEMANTIC_CACHE_CONFIG.conversationHistoryThreshold
),
excludeSystemPrompt: parseBoolean(
env.OMNIROUTE_SEMANTIC_CACHE_EXCLUDE_SYSTEM,
dynamic?.excludeSystemPrompt ?? DEFAULT_SEMANTIC_CACHE_CONFIG.excludeSystemPrompt
),
embeddingBaseUrl:
env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_BASE_URL?.trim() ||
dynamic?.embeddingBaseUrl ||
overrides?.embeddingBaseUrl ||
undefined,
embeddingApiKey:
env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_API_KEY?.trim() ||
dynamic?.embeddingApiKey ||
overrides?.embeddingApiKey ||
undefined,
redisUrl:
env.OMNIROUTE_SEMANTIC_CACHE_REDIS_URL || env.REDIS_URL || dynamic?.redisUrl || undefined,
redisPrefix:
env.OMNIROUTE_SEMANTIC_CACHE_REDIS_PREFIX?.trim() ||
dynamic?.redisPrefix ||
DEFAULT_SEMANTIC_CACHE_CONFIG.redisPrefix,
requireZeroTemperature:
env.OMNIROUTE_SEMANTIC_CACHE_REQUIRE_ZERO_TEMP !== undefined
? parseBoolean(
env.OMNIROUTE_SEMANTIC_CACHE_REQUIRE_ZERO_TEMP,
DEFAULT_SEMANTIC_CACHE_CONFIG.requireZeroTemperature
)
: (dynamic?.requireZeroTemperature ?? DEFAULT_SEMANTIC_CACHE_CONFIG.requireZeroTemperature),
...overrides,
};
// Clamp similarity threshold to [0.0, 1.0]
if (resolved.similarityThreshold < 0) resolved.similarityThreshold = 0;
if (resolved.similarityThreshold > 1) resolved.similarityThreshold = 1;
return resolved;
}