The Open Framework for Building Agents
feat(eve): send prompt cache keys for direct OpenAI calls (#4583)
Ben Pankow416e63eba220
diff --git a/.changeset/openai-prompt-cache-key.md b/.changeset/openai-prompt-cache-key.mdnew file mode 100644index 000000000..cd6bb236d--- /dev/null+++ b/.changeset/openai-prompt-cache-key.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Direct OpenAI and `chatgpt()` model calls now send a per-session `promptCacheKey`, which helps OpenAI route a session's calls to the cache holding its prompt prefix. `chatgpt()` calls also send it as the `session-id` header the Codex backend routes on. An authored `providerOptions.openai.promptCacheKey` still takes precedence.diff --git a/docs/agent-config.md b/docs/agent-config.mdindex 43d2c4d99..7c11eea60 100644--- a/docs/agent-config.md+++ b/docs/agent-config.md@@ -67,6 +67,14 @@ Gateway, share that ID so their Gateway generations can be found together.Direct-provider calls do not receive the option. An authored `gateway.sessionId`takes precedence; AI Gateway hashes IDs longer than 256 characters.+For direct OpenAI and `chatgpt()` model calls, eve sets+`providerOptions.openai.promptCacheKey` to a SHA-256 hash of the session ID,+which helps OpenAI route a session's calls to the cache that holds its prompt+prefix. Each subagent session gets its own key. An authored+`openai.promptCacheKey` takes precedence. `chatgpt()` calls also send the key+in the `session-id` header, which the Codex backend uses for cache routing.+Gateway-routed calls do not receive the option.+### Choose the model dynamicallyTo select a model from the incoming prompt with an AI SDK decision model, usediff --git a/packages/eve/src/harness/compaction/step.ts b/packages/eve/src/harness/compaction/step.tsindex c4a547377..0767ff327 100644--- a/packages/eve/src/harness/compaction/step.ts+++ b/packages/eve/src/harness/compaction/step.ts@@ -49,7 +49,6 @@ import {import { getRequestEnvelopeTokens } from "#harness/request-envelope.js";import { idle } from "#harness/session-machine/transitions.js";import { resolveCallProviderOptions } from "#harness/provider-safety.js";-import { resolveConversationId } from "#shared/conversation-identity.js";const log = createLogger("harness.tool-loop");/**@@ -190,9 +189,9 @@ export async function maybeCompact(input: {const profile = resolveModelProfile(model);const providerOptions = resolveCallProviderOptions({auth: input.auth,- conversationId: resolveConversationId(session.rootSessionId ?? session.sessionId),profile,providerOptions: compactionModelReference.providerOptions,+ session,}) as Parameters<typeof streamText>[0]["providerOptions"];const call = {abortSignal: input.abortSignal,diff --git a/packages/eve/src/harness/provider-safety.test.ts b/packages/eve/src/harness/provider-safety.test.tsindex 14b2fb2bc..8c4a57e2c 100644--- a/packages/eve/src/harness/provider-safety.test.ts+++ b/packages/eve/src/harness/provider-safety.test.ts@@ -1,7 +1,11 @@import { describe, expect, it } from "vitest";import type { SessionAuthContext } from "#channel/types.js";-import { mergeProviderSafetyIdentifier } from "#harness/provider-safety.js";+import type { ModelProfile } from "#harness/model-profile.js";+import {+ mergeProviderSafetyIdentifier,+ resolveCallProviderOptions,+} from "#harness/provider-safety.js";import { invocationOwnerKey } from "#internal/invocation/metadata.js";const auth: SessionAuthContext = {@@ -94,3 +98,48 @@ describe("mergeProviderSafetyIdentifier", () => {expect(mergeProviderSafetyIdentifier("anthropic", providerOptions, null)).toBe(providerOptions);});});++describe("resolveCallProviderOptions", () => {+ const openai: ModelProfile = {+ anthropicCache: false,+ filesOutsideToolResults: false,+ gateway: false,+ googleSearchDropsTools: false,+ provider: "openai",+ };+ const resolve = (+ profile: ModelProfile,+ providerOptions?: Record<string, unknown>,+ sessionId = "session-1",+ ) =>+ resolveCallProviderOptions({+ auth: null,+ profile,+ providerOptions,+ session: { rootSessionId: "conversation-1", sessionId },+ });++ it("keys a direct OpenAI call's prompt cache to its session", () => {+ const key = (resolve(openai) as { openai: { promptCacheKey: string } }).openai.promptCacheKey;++ expect(key).toMatch(/^[\w-]{43}$/);+ expect(resolve(openai)).toEqual({ openai: { promptCacheKey: key } });+ expect(resolve({ ...openai, provider: "codex" })).toEqual({ openai: { promptCacheKey: key } });+ expect(resolve(openai, undefined, "session-2")).not.toEqual({+ openai: { promptCacheKey: key },+ });+ });++ it("preserves an authored OpenAI prompt cache key", () => {+ const providerOptions = { openai: { promptCacheKey: "authored", store: false } };++ expect(resolve(openai, providerOptions)).toEqual(providerOptions);+ });++ it("leaves Gateway and other providers to their own cache routing", () => {+ expect(resolve({ ...openai, gateway: true })).toEqual({+ gateway: { sessionId: "conversation-1" },+ });+ expect(resolve({ ...openai, provider: "anthropic" })).toBeUndefined();+ });+});diff --git a/packages/eve/src/harness/provider-safety.ts b/packages/eve/src/harness/provider-safety.tsindex f188a7a06..aa5bf174c 100644--- a/packages/eve/src/harness/provider-safety.ts+++ b/packages/eve/src/harness/provider-safety.ts@@ -1,6 +1,10 @@+import { createHash } from "node:crypto";+import type { SessionAuthContext } from "#channel/types.js";import type { ModelProfile } from "#harness/model-profile.js";+import type { HarnessSession } from "#harness/types.js";import { invocationOwnerKey } from "#internal/invocation/metadata.js";+import { resolveConversationId } from "#shared/conversation-identity.js";import { mergeObjects } from "#shared/objects.js";/**@@ -30,20 +34,46 @@ export function mergeProviderSafetyIdentifier(/** Composes the per-call defaults shared by model steps and compaction. */export function resolveCallProviderOptions(input: {readonly auth: SessionAuthContext | null;- readonly conversationId: string;readonly profile: ModelProfile;readonly providerOptions: Readonly<Record<string, unknown>> | undefined;+ readonly session: Pick<HarnessSession, "rootSessionId" | "sessionId">;}): Record<string, unknown> | undefined {+ const { session } = input;const providerOptions = mergeProviderSafetyIdentifier(input.profile.provider,input.providerOptions,input.auth,);- return input.profile.gateway- ? mergeGatewaySessionId(providerOptions, input.conversationId)+ if (input.profile.gateway) {+ return mergeGatewaySessionId(+ providerOptions,+ resolveConversationId(session.rootSessionId ?? session.sessionId),+ );+ }+ return PROMPT_CACHE_KEY_PROVIDERS.has(input.profile.provider)+ ? mergeOpenAIPromptCacheKey(providerOptions, session.sessionId): providerOptions;}+/**+ * Direct OpenAI, and the ChatGPT subscription's Codex backend, which the Codex CLI itself sends a+ * per-session `prompt_cache_key`.+ */+const PROMPT_CACHE_KEY_PROVIDERS: ReadonlySet<string> = new Set(["openai", "codex"]);++/**+ * OpenAI uses the cache key to route requests that share a prompt prefix toward the same cache,+ * and matches less reliably without one. Each session keeps its own prefix, so its id makes the key. The hash+ * keeps the key short and opaque whatever the session id looks like.+ */+function mergeOpenAIPromptCacheKey(+ providerOptions: Readonly<Record<string, unknown>> | undefined,+ sessionId: string,+): Record<string, unknown> {+ const promptCacheKey = createHash("sha256").update(sessionId, "utf8").digest("base64url");+ return mergeObjects({ openai: { promptCacheKey } }, providerOptions);+}+/** Groups Gateway generations under the same identity used by eve's agent spans. */function mergeGatewaySessionId(providerOptions: Readonly<Record<string, unknown>> | undefined,diff --git a/packages/eve/src/harness/step-hooks.ts b/packages/eve/src/harness/step-hooks.tsindex 79c7df790..8d8e34e53 100644--- a/packages/eve/src/harness/step-hooks.ts+++ b/packages/eve/src/harness/step-hooks.ts@@ -45,7 +45,6 @@ import { contextStorage } from "#context/container.js";import { isAuthorizationSignal, isPendingAuthorizationToolOutput } from "#harness/authorization.js";import { readToolInterrupt } from "#harness/tool-interrupts.js";import { AuthKey } from "#context/keys.js";-import { resolveConversationId } from "#shared/conversation-identity.js";// ---------------------------------------------------------------------------// Step result type@@ -171,9 +170,9 @@ export function buildStepHooks(input: StepHooksInput): StepHooks {const modelReference = requireSessionModelReference(session);const providerOptions = resolveCallProviderOptions({auth: input.auth ?? contextStorage.getStore()?.get(AuthKey) ?? null,- conversationId: resolveConversationId(session.rootSessionId ?? session.sessionId),profile,providerOptions: modelReference.providerOptions,+ session,});if (profile.gateway) {stepResult.providerOptions = mergeGatewayAutoCaching(providerOptions) as NonNullable<diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 14d61a351..0e38ea1c2 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -1380,7 +1380,9 @@ describe("createToolLoopHarness", () => {stepNumber: 0,steps: [],});- expect(prepared.providerOptions).toEqual({ openai: { parallelToolCalls: false } });+ expect(prepared.providerOptions).toEqual({+ openai: { parallelToolCalls: false, promptCacheKey: expect.any(String) },+ });expect(dispatchDynamicModelEvent).toHaveBeenCalledTimes(1);expect(resolveModel).not.toHaveBeenCalled();expect(result.session.agent.modelReference).toEqual({@@ -7919,6 +7921,7 @@ describe("createToolLoopHarness", () => {});expect(summaryCall()?.providerOptions).toEqual({openai: {+ promptCacheKey: expect.any(String),safetyIdentifier: invocationOwnerKey(auth),store: false,},@@ -8314,7 +8317,9 @@ describe("createToolLoopHarness", () => {);expect(call?.[1].threshold).toBeLessThan(100_000);expect(call?.[1].threshold).toBeGreaterThan(99_000);- expect(summaryCall()?.providerOptions).toBeUndefined();+ expect(summaryCall()?.providerOptions).toEqual({+ openai: { promptCacheKey: expect.any(String) },+ });});it("emits reasoning.completed when reasoning text is available", async () => {@@ -9042,7 +9047,7 @@ describe("createToolLoopHarness", () => {});});- it("none path: direct OpenAI instance gets no caching changes", async () => {+ it("none path: direct OpenAI instance gets only a prompt cache key", async () => {setupStopResult();const config: ToolLoopHarnessConfig = {resolveModel: vi.fn().mockResolvedValue({@@ -9077,7 +9082,9 @@ describe("createToolLoopHarness", () => {model: agentCall?.model,context: undefined,});- expect(stepResult.providerOptions).toBeUndefined();+ expect(stepResult.providerOptions).toEqual({+ openai: { promptCacheKey: expect.any(String) },+ });expect(agentCall?.providerOptions).toBeUndefined();const toolsPassed = agentCall?.tools as Record<string,diff --git a/packages/eve/src/public/models/openai/chatgpt/model.test.ts b/packages/eve/src/public/models/openai/chatgpt/model.test.tsindex e6487b594..5f7bb6b90 100644--- a/packages/eve/src/public/models/openai/chatgpt/model.test.ts+++ b/packages/eve/src/public/models/openai/chatgpt/model.test.ts@@ -143,6 +143,21 @@ describe("Codex model", () => {expect(JSON.parse(requests[0]?.body ?? "{}").service_tier).toBe("priority");});+ it("sends the prompt cache key to the Codex backend", async () => {+ const requests: RecordedRequest[] = [];+ const model = createCodexSubscriptionModel(+ { model: "gpt-6.1-sol" },+ { broker: fakeBroker(), fetch: createRecordingFetch(requests) },+ );++ await model.doGenerate({+ prompt: [{ role: "user", content: [{ type: "text", text: "hello" }] }],+ providerOptions: { openai: { promptCacheKey: "session-key" } },+ });++ expect(JSON.parse(requests[0]?.body ?? "{}").prompt_cache_key).toBe("session-key");+ });+it("groups summaries by reasoning item and preserves encrypted-only items", async () => {const requests: RecordedRequest[] = [];const model = createCodexSubscriptionModel(diff --git a/packages/eve/src/public/models/openai/chatgpt/transport.test.ts b/packages/eve/src/public/models/openai/chatgpt/transport.test.tsindex 21fa6ff6a..5f673dc35 100644--- a/packages/eve/src/public/models/openai/chatgpt/transport.test.ts+++ b/packages/eve/src/public/models/openai/chatgpt/transport.test.ts@@ -29,6 +29,30 @@ describe("Codex direct transport", () => {expect(requests[0]?.headers.get("originator")).toBe("eve");});+ it("mirrors the prompt cache key into the session-id header Codex routes cache affinity on", async () => {+ const requests: RecordedRequest[] = [];+ const codexFetch = createCodexFetch({+ broker: fakeBroker([{ token: "access-token" }, { token: "access-token" }]),+ codexApiEndpoint: CODEX_ENDPOINT,+ fetch: createRecordingFetch(requests),+ });++ await codexFetch("https://api.openai.com/v1/responses", {+ body: JSON.stringify({ prompt_cache_key: "session-key", stream: true }),+ method: "POST",+ });+ await codexFetch("https://api.openai.com/v1/responses", {+ body: JSON.stringify({ prompt_cache_key: "session-key", stream: true }),+ headers: { "session-id": "authored" },+ method: "POST",+ });++ expect(requests.map((request) => request.headers.get("session-id"))).toEqual([+ "session-key",+ "authored",+ ]);+ });+it("refreshes through the broker and replays one string-body request after 401", async () => {const requests: RecordedRequest[] = [];const broker = fakeBroker([{ token: "stale-token" }, { token: "fresh-token" }]);diff --git a/packages/eve/src/public/models/openai/chatgpt/transport.ts b/packages/eve/src/public/models/openai/chatgpt/transport.tsindex f0a1264dc..6b4a09212 100644--- a/packages/eve/src/public/models/openai/chatgpt/transport.ts+++ b/packages/eve/src/public/models/openai/chatgpt/transport.ts@@ -32,28 +32,36 @@ export function createCodexFetch(options: CodexTransportOptions = {}): Fetch {return async (input: FetchInput, init?: RequestInit): Promise<Response> => {const url = rewriteCodexEndpoint(requestUrl(input), codexApiEndpoint);- const requestInit = stripInputItemIds(init);+ const request = prepareCodexBody(init);const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined);const token = await resolveToken(undefined, signal);- const first = await httpFetch(url, authenticatedInit(input, requestInit, token));+ const first = await httpFetch(url, authenticatedInit(input, request, token));if (first.status !== 401 || !isReplayable(input, init)) return first;await first.body?.cancel();const refreshed = await resolveToken(token.token, signal);- return httpFetch(url, authenticatedInit(input, requestInit, refreshed));+ return httpFetch(url, authenticatedInit(input, request, refreshed));};}-function stripInputItemIds(init: RequestInit | undefined): RequestInit | undefined {- if (typeof init?.body !== "string") return init;+interface CodexRequest {+ readonly init: RequestInit | undefined;+ readonly promptCacheKey?: string;+}++function prepareCodexBody(init: RequestInit | undefined): CodexRequest {+ if (typeof init?.body !== "string") return { init };let body: unknown;try {body = JSON.parse(init.body);} catch {- return init;+ return { init };}- if (!isObject(body) || !Array.isArray(body.input)) return init;+ if (!isObject(body)) return { init };+ const promptCacheKey =+ typeof body.prompt_cache_key === "string" ? body.prompt_cache_key : undefined;+ if (!Array.isArray(body.input)) return { init, promptCacheKey };// Only response item IDs are forbidden; tool call IDs and IDs inside tool// inputs or outputs belong to the conversation and must survive replay.@@ -64,7 +72,7 @@ function stripInputItemIds(init: RequestInit | undefined): RequestInit | undefinchanged = true;}}- return changed ? { ...init, body: JSON.stringify(body) } : init;+ return { init: changed ? { ...init, body: JSON.stringify(body) } : init, promptCacheKey };}export function rewriteCodexEndpoint(input: string, codexApiEndpoint = CODEX_API_ENDPOINT): string {@@ -77,12 +85,17 @@ export function rewriteCodexEndpoint(input: string, codexApiEndpoint = CODEX_APIfunction authenticatedInit(input: FetchInput,- init: RequestInit | undefined,+ { init, promptCacheKey }: CodexRequest,token: { readonly accountId?: string; readonly token: string },): RequestInit {const headers = cloneHeaders(init?.headers ?? (input instanceof Request ? input.headers : undefined),);+ // The Codex backend routes prompt-cache affinity on `session-id`, not on the body's+ // `prompt_cache_key`; the Codex CLI sends the same value in both.+ if (promptCacheKey !== undefined && !headers.has("session-id")) {+ headers.set("session-id", promptCacheKey);+ }headers.delete("authorization");headers.delete("Authorization");headers.set("authorization", `Bearer ${token.token}`);