Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

13.4 KB
Raw Blame
/**
 * Memory Injection — prepend retrieved memories into the request message list.
 *
 * Injection strategy:
 *   1. If the provider supports system messages (most providers), inject as a
 *      leading system message so it takes effect without disrupting user turns.
 *   2. Otherwise (fallback for providers that reject system role), inject as the
 *      first user message prefixed with the memory context label.
 *
 * Format: "Memory context: <content>"
 */

import { Memory } from "./types";
import { logger } from "../../../open-sse/utils/logger.ts";
import {
  isAnthropicCompatibleProvider,
  isClaudeCodeCompatibleProvider,
} from "../../shared/constants/providers";

const log = logger("MEMORY_INJECTION");

export interface ChatMessage {
  role: "system" | "user" | "assistant";
  content: string;
  name?: string;
}

export interface ChatRequest {
  model: string;
  messages: ChatMessage[];
  /** Anthropic-shaped bodies carry the system prompt here, as a string or text blocks (#13425). */
  system?: string | Array<{ type: string; text?: string; [key: string]: unknown }>;
  temperature?: number;
  max_tokens?: number;
  stream?: boolean;
  [key: string]: unknown;
}

/**
 * Providers known NOT to support a top-level system-role message.
 * These receive memories injected as the first user message instead.
 */
const PROVIDERS_WITHOUT_SYSTEM_MESSAGE = new Set([
  "o1",
  "o1-mini",
  "o1-preview",
  "glm", // GLM/ZhipuAI rejects system role (#1701)
  "glmt", // GLM Thinking variant
  "glm-cn", // GLM China variant
  "zai", // Z.AI uses same GLM backend
  "qianfan", // Baidu ERNIE rejects system role
]);

/**
 * Returns true when the given provider accepts a system-role message.
 * Falls back to true for unknown/null providers (safe default).
 */
export function providerSupportsSystemMessage(provider: string | null | undefined): boolean {
  if (!provider) return true;
  const normalized = provider.toLowerCase().trim();
  return !PROVIDERS_WITHOUT_SYSTEM_MESSAGE.has(normalized);
}

/**
 * Providers that accept a system-role message ONLY at index 0 (a `system`
 * message at any later position is rejected with HTTP 400). For these, the
 * cache-safe mid-array splice (which inserts memory just before the last user
 * turn) is unsafe in multi-turn conversations, so memory must be merged into /
 * prepended as the leading system message instead. See #6135.
 *
 * Populated with the Xiaomi MiMo endpoint (provider id `xiaomi-mimo`, registry
 * alias `mimo`, serving mimo-v2.5) confirmed live to 400 on a non-first system
 * message, and the TokenRouter gateway (provider id `tokenrouter`), confirmed
 * live on 2026-08-22 to reject mid-array system messages — including the
 * compression notice spliced by purifyHistory() before that splice was fixed to
 * merge into the leading system message. Add other providers here only when
 * they are documented as strict.
 *
 * Self-hosted deployments can extend this list without a source change via
 * OMNIROUTE_STRICT_SYSTEM_PROVIDERS (comma-separated provider ids,
 * case-insensitive) — e.g. a custom OpenAI-compatible connection in front of a
 * self-hosted Qwen3.5+/3.6 model, whose chat template enforces the same
 * single-leading-system-message constraint as xiaomi-mimo.
 */
const BUILTIN_PROVIDERS_SYSTEM_MUST_BE_FIRST = new Set(["xiaomi-mimo", "mimo", "tokenrouter"]);

/**
 * Parses OMNIROUTE_STRICT_SYSTEM_PROVIDERS into a normalized id list.
 * Exported for tests; not expected to be called directly by other modules.
 */
export function parseStrictSystemProvidersEnv(env: NodeJS.ProcessEnv = process.env): string[] {
  const raw = env.OMNIROUTE_STRICT_SYSTEM_PROVIDERS ?? "";
  return raw
    .split(",")
    .map((id) => id.trim().toLowerCase())
    .filter(Boolean);
}

function resolveProvidersSystemMustBeFirst(env: NodeJS.ProcessEnv = process.env): Set<string> {
  const extra = parseStrictSystemProvidersEnv(env);
  if (extra.length === 0) return BUILTIN_PROVIDERS_SYSTEM_MUST_BE_FIRST;
  return new Set([...BUILTIN_PROVIDERS_SYSTEM_MUST_BE_FIRST, ...extra]);
}

/**
 * Returns true when the given provider requires the system message to be first.
 * Falls back to false for unknown/null providers (preserves current behavior).
 * Honors OMNIROUTE_STRICT_SYSTEM_PROVIDERS for self-hosted additions (see above).
 */
export function systemMessageMustBeFirst(
  provider: string | null | undefined,
  env: NodeJS.ProcessEnv = process.env
): boolean {
  if (!provider) return false;
  const normalized = provider.toLowerCase().trim();
  return resolveProvidersSystemMustBeFirst(env).has(normalized);
}

/**
 * Format memories into a single labeled context string.
 * Format: "Memory context: <content1>\n<content2>..."
 */
export function formatMemoryContext(memories: Memory[]): string {
  if (!memories || memories.length === 0) return "";

  const content = memories
    .map((m) => m.content.trim())
    .filter(Boolean)
    .join("\n");

  return content ? `Memory context: ${content}` : "";
}

/**
 * Inject retrieved memories into the request message array.
 *
 * @param request  - The chat completion request body
 * @param memories - Memories retrieved for the current API key / session
 * @param provider - Provider identifier used to choose injection strategy
 * @returns A new request body with memories prepended to messages
 */
export interface InjectMemoryOptions {
  /**
   * #3890: when the request uses prompt caching (cache_control breakpoints), prepending
   * the memory message at index 0 shifts the entire cacheable prefix — and since the
   * retrieved memories vary per user query, every cache breakpoint then misses on each
   * turn (observed as a sustained cache-miss / cost spike). When `cacheSafe` is set, the
   * memory message is inserted immediately before the LAST user message instead, so the
   * cacheable prefix (system prompt + prior turns) stays byte-stable while memory still
   * contextualizes the current turn.
   */
  cacheSafe?: boolean;
}

/**
 * #6135: place the memory as a leading system message for providers that reject
 * a non-first system role — merging into an existing index-0 system message when
 * present, else prepending. Split out of injectMemory to keep it flat.
 */
function injectSystemFirst(
  request: ChatRequest,
  messages: ChatMessage[],
  memoryText: string,
  count: number
): ChatRequest {
  log.info("memory.injection.injected", { count, strategy: "system-first", model: request.model });
  const first = messages[0];
  if (first && first.role === "system") {
    const merged: ChatMessage = { ...first, content: `${memoryText}\n${first.content}` };
    return { ...request, messages: [merged, ...messages.slice(1)] };
  }
  // #13425: Anthropic-shaped bodies carry the system prompt in the top-level
  // `system` field, not in messages[0]. Unshifting a `{role:"system"}` at
  // messages[0] triggers a 400 ("use the top-level 'system' parameter").
  // Merge the memory text into the top-level field instead.
  if (typeof request.system === "string") {
    return { ...request, system: `${memoryText}\n${request.system}` };
  }
  if (Array.isArray(request.system)) {
    return { ...request, system: [{ type: "text", text: memoryText }, ...request.system] };
  }
  const memorySystemMessage: ChatMessage = { role: "system", content: memoryText };
  return { ...request, messages: [memorySystemMessage, ...messages] };
}

/**
 * #11290: providers in the Claude family (direct Anthropic, and any
 * anthropic-compatible / Claude-Code-compatible passthrough connection) — the
 * ones affected by the stricter Opus 5 message-ordering validation described
 * below. Deliberately narrower than `systemMessageMustBeFirst()`'s strict-set:
 * this only gates the cache-safe mid-array splice, not the leading-system-message
 * requirement, so non-Claude providers keep the #3890 cache-hit optimization
 * unconditionally.
 */
function isClaudeFamilyProvider(provider: string | null | undefined): boolean {
  if (!provider) return false;
  const normalized = provider.toLowerCase().trim();
  return (
    normalized === "claude" ||
    normalized === "anthropic" ||
    isClaudeCodeCompatibleProvider(provider) ||
    isAnthropicCompatibleProvider(provider)
  );
}

/**
 * True when an assistant message's content ends in a server-side tool result
 * block (e.g. `web_search_tool_result`, `code_execution_tool_result`,
 * `mcp_tool_result` — any Anthropic content block whose type ends in
 * `_tool_result`, produced by a server-executed tool rather than a
 * client-executed one). `content` is typed as `string` on `ChatMessage` for
 * the common case, but the Claude-native wire shape carries an array of
 * content blocks — this only recognizes that richer shape.
 */
function endsWithServerToolResult(message: ChatMessage | undefined): boolean {
  if (!message || message.role !== "assistant") return false;
  const content = message.content as unknown;
  if (!Array.isArray(content) || content.length === 0) return false;
  const lastBlock = content[content.length - 1] as { type?: unknown } | null | undefined;
  return typeof lastBlock?.type === "string" && lastBlock.type.endsWith("_tool_result");
}

/**
 * Place a memory message at the #3890 cache-safe anchor (just before the last
 * user turn) when one exists, else prepend it. Shared by the system and user
 * injection strategies to keep injectMemory flat.
 */
function placeMessage(
  request: ChatRequest,
  messages: ChatMessage[],
  msg: ChatMessage,
  cacheSafeIndex: number
): ChatRequest {
  if (cacheSafeIndex >= 0) {
    const next = [...messages];
    next.splice(cacheSafeIndex, 0, msg);
    return { ...request, messages: next };
  }
  return { ...request, messages: [msg, ...messages] };
}

export function injectMemory(
  request: ChatRequest,
  memories: Memory[],
  provider: string | null | undefined,
  options: InjectMemoryOptions = {}
): ChatRequest {
  if (!memories || memories.length === 0) {
    log.info("memory.injection.skipped", { reason: "no_memories", model: request.model });
    return request;
  }

  const memoryText = formatMemoryContext(memories);
  if (!memoryText) {
    log.info("memory.injection.skipped", { reason: "empty_context", model: request.model });
    return request;
  }

  const messages: ChatMessage[] = Array.isArray(request.messages) ? [...request.messages] : [];

  // #3890: in a caching context, anchor the injection just before the LAST user message so
  // the cacheable prefix (system prompt + prior turns) is preserved byte-for-byte. Falls
  // back to a leading message when caching is off or there is no user turn to anchor on.
  const cacheSafeIndex = options.cacheSafe ? messages.findLastIndex((m) => m.role === "user") : -1;

  const supportsSystem = providerSupportsSystemMessage(provider);

  // #6135: strict providers reject a system message at a non-zero index. Never
  // apply the cache-safe mid-array splice for these — keep the system message
  // first (extracted to injectSystemFirst to keep this function flat).
  if (supportsSystem && systemMessageMustBeFirst(provider)) {
    return injectSystemFirst(request, messages, memoryText, memories.length);
  }

  // #11290: Claude Opus 5 tightened server-side validation of the cache-safe
  // mid-array splice — a system message spliced right after a plain-text assistant
  // turn is rejected with HTTP 400 (the immediately preceding message must end in a
  // server-side tool result for a following system message to be accepted). Rather
  // than adding "claude"/"anthropic" outright to `systemMessageMustBeFirst()` (which
  // would revert the #3890 cache-hit optimization for every Claude request, including
  // the ones that work fine today), only fall back to the leading-system-message
  // placement for the specific requests where the turn right before the splice point
  // isn't a server tool result.
  if (
    supportsSystem &&
    cacheSafeIndex >= 0 &&
    isClaudeFamilyProvider(provider) &&
    !endsWithServerToolResult(messages[cacheSafeIndex - 1])
  ) {
    return injectSystemFirst(request, messages, memoryText, memories.length);
  }

  // #13425: Anthropic-shaped bodies carry the system prompt in the top-level
  // `system` field, not in messages[0]. If supportsSystem is true and the
  // request already has a top-level `system` field, merge memory there instead
  // of prepending a `{role:"system"}` at messages[0] — Anthropic rejects that.
  if (supportsSystem && (typeof request.system === "string" || Array.isArray(request.system))) {
    if (typeof request.system === "string") {
      return { ...request, system: `${memoryText}\n${request.system}` };
    }
    return { ...request, system: [{ type: "text", text: memoryText }, ...request.system] };
  }

  // Strategy 1 (system): prepend before existing system messages, preserving the
  // caller's own instructions. Strategy 2 (user, e.g. o1-mini): inject as a user
  // message. Both honor the #3890 cache-safe anchor via placeMessage.
  const role: ChatMessage["role"] = supportsSystem ? "system" : "user";
  const memoryMessage: ChatMessage = { role, content: memoryText };
  const base = supportsSystem ? "system" : "user";
  log.info("memory.injection.injected", {
    count: memories.length,
    strategy: cacheSafeIndex >= 0 ? `${base}-cache-safe` : base,
    model: request.model,
  });
  return placeMessage(request, messages, memoryMessage, cacheSafeIndex);
}

/**
 * Returns true when memory injection should be attempted for this request.
 */
export function shouldInjectMemory(request: ChatRequest, config?: { enabled?: boolean }): boolean {
  if (config?.enabled === false) return false;
  return Array.isArray(request.messages) && request.messages.length > 0;
}