Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): send prompt cache keys for direct OpenAI calls (#4583)

Ben Pankow416e63eba220
diff --git a/.changeset/openai-prompt-cache-key.md b/.changeset/openai-prompt-cache-key.md
new file mode 100644
index 000000000..cd6bb236d
--- /dev/null
+++ b/.changeset/openai-prompt-cache-key.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Direct OpenAI and `chatgpt()` model calls now send a per-session `promptCacheKey`, which helps OpenAI route a session's calls to the cache holding its prompt prefix. `chatgpt()` calls also send it as the `session-id` header the Codex backend routes on. An authored `providerOptions.openai.promptCacheKey` still takes precedence.
diff --git a/docs/agent-config.md b/docs/agent-config.md
index 43d2c4d99..7c11eea60 100644
--- a/docs/agent-config.md
+++ b/docs/agent-config.md
@@ -67,6 +67,14 @@ Gateway, share that ID so their Gateway generations can be found together.
Direct-provider calls do not receive the option. An authored `gateway.sessionId`
takes precedence; AI Gateway hashes IDs longer than 256 characters.
+For direct OpenAI and `chatgpt()` model calls, eve sets
+`providerOptions.openai.promptCacheKey` to a SHA-256 hash of the session ID,
+which helps OpenAI route a session's calls to the cache that holds its prompt
+prefix. Each subagent session gets its own key. An authored
+`openai.promptCacheKey` takes precedence. `chatgpt()` calls also send the key
+in the `session-id` header, which the Codex backend uses for cache routing.
+Gateway-routed calls do not receive the option.
+
### Choose the model dynamically
To select a model from the incoming prompt with an AI SDK decision model, use
diff --git a/packages/eve/src/harness/compaction/step.ts b/packages/eve/src/harness/compaction/step.ts
index c4a547377..0767ff327 100644
--- a/packages/eve/src/harness/compaction/step.ts
+++ b/packages/eve/src/harness/compaction/step.ts
@@ -49,7 +49,6 @@ import {
import { getRequestEnvelopeTokens } from "#harness/request-envelope.js";
import { idle } from "#harness/session-machine/transitions.js";
import { resolveCallProviderOptions } from "#harness/provider-safety.js";
-import { resolveConversationId } from "#shared/conversation-identity.js";
const log = createLogger("harness.tool-loop");
/**
@@ -190,9 +189,9 @@ export async function maybeCompact(input: {
const profile = resolveModelProfile(model);
const providerOptions = resolveCallProviderOptions({
auth: input.auth,
- conversationId: resolveConversationId(session.rootSessionId ?? session.sessionId),
profile,
providerOptions: compactionModelReference.providerOptions,
+ session,
}) as Parameters<typeof streamText>[0]["providerOptions"];
const call = {
abortSignal: input.abortSignal,
diff --git a/packages/eve/src/harness/provider-safety.test.ts b/packages/eve/src/harness/provider-safety.test.ts
index 14b2fb2bc..8c4a57e2c 100644
--- a/packages/eve/src/harness/provider-safety.test.ts
+++ b/packages/eve/src/harness/provider-safety.test.ts
@@ -1,7 +1,11 @@
import { describe, expect, it } from "vitest";
import type { SessionAuthContext } from "#channel/types.js";
-import { mergeProviderSafetyIdentifier } from "#harness/provider-safety.js";
+import type { ModelProfile } from "#harness/model-profile.js";
+import {
+ mergeProviderSafetyIdentifier,
+ resolveCallProviderOptions,
+} from "#harness/provider-safety.js";
import { invocationOwnerKey } from "#internal/invocation/metadata.js";
const auth: SessionAuthContext = {
@@ -94,3 +98,48 @@ describe("mergeProviderSafetyIdentifier", () => {
expect(mergeProviderSafetyIdentifier("anthropic", providerOptions, null)).toBe(providerOptions);
});
});
+
+describe("resolveCallProviderOptions", () => {
+ const openai: ModelProfile = {
+ anthropicCache: false,
+ filesOutsideToolResults: false,
+ gateway: false,
+ googleSearchDropsTools: false,
+ provider: "openai",
+ };
+ const resolve = (
+ profile: ModelProfile,
+ providerOptions?: Record<string, unknown>,
+ sessionId = "session-1",
+ ) =>
+ resolveCallProviderOptions({
+ auth: null,
+ profile,
+ providerOptions,
+ session: { rootSessionId: "conversation-1", sessionId },
+ });
+
+ it("keys a direct OpenAI call's prompt cache to its session", () => {
+ const key = (resolve(openai) as { openai: { promptCacheKey: string } }).openai.promptCacheKey;
+
+ expect(key).toMatch(/^[\w-]{43}$/);
+ expect(resolve(openai)).toEqual({ openai: { promptCacheKey: key } });
+ expect(resolve({ ...openai, provider: "codex" })).toEqual({ openai: { promptCacheKey: key } });
+ expect(resolve(openai, undefined, "session-2")).not.toEqual({
+ openai: { promptCacheKey: key },
+ });
+ });
+
+ it("preserves an authored OpenAI prompt cache key", () => {
+ const providerOptions = { openai: { promptCacheKey: "authored", store: false } };
+
+ expect(resolve(openai, providerOptions)).toEqual(providerOptions);
+ });
+
+ it("leaves Gateway and other providers to their own cache routing", () => {
+ expect(resolve({ ...openai, gateway: true })).toEqual({
+ gateway: { sessionId: "conversation-1" },
+ });
+ expect(resolve({ ...openai, provider: "anthropic" })).toBeUndefined();
+ });
+});
diff --git a/packages/eve/src/harness/provider-safety.ts b/packages/eve/src/harness/provider-safety.ts
index f188a7a06..aa5bf174c 100644
--- a/packages/eve/src/harness/provider-safety.ts
+++ b/packages/eve/src/harness/provider-safety.ts
@@ -1,6 +1,10 @@
+import { createHash } from "node:crypto";
+
import type { SessionAuthContext } from "#channel/types.js";
import type { ModelProfile } from "#harness/model-profile.js";
+import type { HarnessSession } from "#harness/types.js";
import { invocationOwnerKey } from "#internal/invocation/metadata.js";
+import { resolveConversationId } from "#shared/conversation-identity.js";
import { mergeObjects } from "#shared/objects.js";
/**
@@ -30,20 +34,46 @@ export function mergeProviderSafetyIdentifier(
/** Composes the per-call defaults shared by model steps and compaction. */
export function resolveCallProviderOptions(input: {
readonly auth: SessionAuthContext | null;
- readonly conversationId: string;
readonly profile: ModelProfile;
readonly providerOptions: Readonly<Record<string, unknown>> | undefined;
+ readonly session: Pick<HarnessSession, "rootSessionId" | "sessionId">;
}): Record<string, unknown> | undefined {
+ const { session } = input;
const providerOptions = mergeProviderSafetyIdentifier(
input.profile.provider,
input.providerOptions,
input.auth,
);
- return input.profile.gateway
- ? mergeGatewaySessionId(providerOptions, input.conversationId)
+ if (input.profile.gateway) {
+ return mergeGatewaySessionId(
+ providerOptions,
+ resolveConversationId(session.rootSessionId ?? session.sessionId),
+ );
+ }
+ return PROMPT_CACHE_KEY_PROVIDERS.has(input.profile.provider)
+ ? mergeOpenAIPromptCacheKey(providerOptions, session.sessionId)
: providerOptions;
}
+/**
+ * Direct OpenAI, and the ChatGPT subscription's Codex backend, which the Codex CLI itself sends a
+ * per-session `prompt_cache_key`.
+ */
+const PROMPT_CACHE_KEY_PROVIDERS: ReadonlySet<string> = new Set(["openai", "codex"]);
+
+/**
+ * OpenAI uses the cache key to route requests that share a prompt prefix toward the same cache,
+ * and matches less reliably without one. Each session keeps its own prefix, so its id makes the key. The hash
+ * keeps the key short and opaque whatever the session id looks like.
+ */
+function mergeOpenAIPromptCacheKey(
+ providerOptions: Readonly<Record<string, unknown>> | undefined,
+ sessionId: string,
+): Record<string, unknown> {
+ const promptCacheKey = createHash("sha256").update(sessionId, "utf8").digest("base64url");
+ return mergeObjects({ openai: { promptCacheKey } }, providerOptions);
+}
+
/** Groups Gateway generations under the same identity used by eve's agent spans. */
function mergeGatewaySessionId(
providerOptions: Readonly<Record<string, unknown>> | undefined,
diff --git a/packages/eve/src/harness/step-hooks.ts b/packages/eve/src/harness/step-hooks.ts
index 79c7df790..8d8e34e53 100644
--- a/packages/eve/src/harness/step-hooks.ts
+++ b/packages/eve/src/harness/step-hooks.ts
@@ -45,7 +45,6 @@ import { contextStorage } from "#context/container.js";
import { isAuthorizationSignal, isPendingAuthorizationToolOutput } from "#harness/authorization.js";
import { readToolInterrupt } from "#harness/tool-interrupts.js";
import { AuthKey } from "#context/keys.js";
-import { resolveConversationId } from "#shared/conversation-identity.js";
// ---------------------------------------------------------------------------
// Step result type
@@ -171,9 +170,9 @@ export function buildStepHooks(input: StepHooksInput): StepHooks {
const modelReference = requireSessionModelReference(session);
const providerOptions = resolveCallProviderOptions({
auth: input.auth ?? contextStorage.getStore()?.get(AuthKey) ?? null,
- conversationId: resolveConversationId(session.rootSessionId ?? session.sessionId),
profile,
providerOptions: modelReference.providerOptions,
+ session,
});
if (profile.gateway) {
stepResult.providerOptions = mergeGatewayAutoCaching(providerOptions) as NonNullable<
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 14d61a351..0e38ea1c2 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -1380,7 +1380,9 @@ describe("createToolLoopHarness", () => {
stepNumber: 0,
steps: [],
});
- expect(prepared.providerOptions).toEqual({ openai: { parallelToolCalls: false } });
+ expect(prepared.providerOptions).toEqual({
+ openai: { parallelToolCalls: false, promptCacheKey: expect.any(String) },
+ });
expect(dispatchDynamicModelEvent).toHaveBeenCalledTimes(1);
expect(resolveModel).not.toHaveBeenCalled();
expect(result.session.agent.modelReference).toEqual({
@@ -7919,6 +7921,7 @@ describe("createToolLoopHarness", () => {
});
expect(summaryCall()?.providerOptions).toEqual({
openai: {
+ promptCacheKey: expect.any(String),
safetyIdentifier: invocationOwnerKey(auth),
store: false,
},
@@ -8314,7 +8317,9 @@ describe("createToolLoopHarness", () => {
);
expect(call?.[1].threshold).toBeLessThan(100_000);
expect(call?.[1].threshold).toBeGreaterThan(99_000);
- expect(summaryCall()?.providerOptions).toBeUndefined();
+ expect(summaryCall()?.providerOptions).toEqual({
+ openai: { promptCacheKey: expect.any(String) },
+ });
});
it("emits reasoning.completed when reasoning text is available", async () => {
@@ -9042,7 +9047,7 @@ describe("createToolLoopHarness", () => {
});
});
- it("none path: direct OpenAI instance gets no caching changes", async () => {
+ it("none path: direct OpenAI instance gets only a prompt cache key", async () => {
setupStopResult();
const config: ToolLoopHarnessConfig = {
resolveModel: vi.fn().mockResolvedValue({
@@ -9077,7 +9082,9 @@ describe("createToolLoopHarness", () => {
model: agentCall?.model,
context: undefined,
});
- expect(stepResult.providerOptions).toBeUndefined();
+ expect(stepResult.providerOptions).toEqual({
+ openai: { promptCacheKey: expect.any(String) },
+ });
expect(agentCall?.providerOptions).toBeUndefined();
const toolsPassed = agentCall?.tools as Record<
string,
diff --git a/packages/eve/src/public/models/openai/chatgpt/model.test.ts b/packages/eve/src/public/models/openai/chatgpt/model.test.ts
index e6487b594..5f7bb6b90 100644
--- a/packages/eve/src/public/models/openai/chatgpt/model.test.ts
+++ b/packages/eve/src/public/models/openai/chatgpt/model.test.ts
@@ -143,6 +143,21 @@ describe("Codex model", () => {
expect(JSON.parse(requests[0]?.body ?? "{}").service_tier).toBe("priority");
});
+ it("sends the prompt cache key to the Codex backend", async () => {
+ const requests: RecordedRequest[] = [];
+ const model = createCodexSubscriptionModel(
+ { model: "gpt-6.1-sol" },
+ { broker: fakeBroker(), fetch: createRecordingFetch(requests) },
+ );
+
+ await model.doGenerate({
+ prompt: [{ role: "user", content: [{ type: "text", text: "hello" }] }],
+ providerOptions: { openai: { promptCacheKey: "session-key" } },
+ });
+
+ expect(JSON.parse(requests[0]?.body ?? "{}").prompt_cache_key).toBe("session-key");
+ });
+
it("groups summaries by reasoning item and preserves encrypted-only items", async () => {
const requests: RecordedRequest[] = [];
const model = createCodexSubscriptionModel(
diff --git a/packages/eve/src/public/models/openai/chatgpt/transport.test.ts b/packages/eve/src/public/models/openai/chatgpt/transport.test.ts
index 21fa6ff6a..5f673dc35 100644
--- a/packages/eve/src/public/models/openai/chatgpt/transport.test.ts
+++ b/packages/eve/src/public/models/openai/chatgpt/transport.test.ts
@@ -29,6 +29,30 @@ describe("Codex direct transport", () => {
expect(requests[0]?.headers.get("originator")).toBe("eve");
});
+ it("mirrors the prompt cache key into the session-id header Codex routes cache affinity on", async () => {
+ const requests: RecordedRequest[] = [];
+ const codexFetch = createCodexFetch({
+ broker: fakeBroker([{ token: "access-token" }, { token: "access-token" }]),
+ codexApiEndpoint: CODEX_ENDPOINT,
+ fetch: createRecordingFetch(requests),
+ });
+
+ await codexFetch("https://api.openai.com/v1/responses", {
+ body: JSON.stringify({ prompt_cache_key: "session-key", stream: true }),
+ method: "POST",
+ });
+ await codexFetch("https://api.openai.com/v1/responses", {
+ body: JSON.stringify({ prompt_cache_key: "session-key", stream: true }),
+ headers: { "session-id": "authored" },
+ method: "POST",
+ });
+
+ expect(requests.map((request) => request.headers.get("session-id"))).toEqual([
+ "session-key",
+ "authored",
+ ]);
+ });
+
it("refreshes through the broker and replays one string-body request after 401", async () => {
const requests: RecordedRequest[] = [];
const broker = fakeBroker([{ token: "stale-token" }, { token: "fresh-token" }]);
diff --git a/packages/eve/src/public/models/openai/chatgpt/transport.ts b/packages/eve/src/public/models/openai/chatgpt/transport.ts
index f0a1264dc..6b4a09212 100644
--- a/packages/eve/src/public/models/openai/chatgpt/transport.ts
+++ b/packages/eve/src/public/models/openai/chatgpt/transport.ts
@@ -32,28 +32,36 @@ export function createCodexFetch(options: CodexTransportOptions = {}): Fetch {
return async (input: FetchInput, init?: RequestInit): Promise<Response> => {
const url = rewriteCodexEndpoint(requestUrl(input), codexApiEndpoint);
- const requestInit = stripInputItemIds(init);
+ const request = prepareCodexBody(init);
const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
const token = await resolveToken(undefined, signal);
- const first = await httpFetch(url, authenticatedInit(input, requestInit, token));
+ const first = await httpFetch(url, authenticatedInit(input, request, token));
if (first.status !== 401 || !isReplayable(input, init)) return first;
await first.body?.cancel();
const refreshed = await resolveToken(token.token, signal);
- return httpFetch(url, authenticatedInit(input, requestInit, refreshed));
+ return httpFetch(url, authenticatedInit(input, request, refreshed));
};
}
-function stripInputItemIds(init: RequestInit | undefined): RequestInit | undefined {
- if (typeof init?.body !== "string") return init;
+interface CodexRequest {
+ readonly init: RequestInit | undefined;
+ readonly promptCacheKey?: string;
+}
+
+function prepareCodexBody(init: RequestInit | undefined): CodexRequest {
+ if (typeof init?.body !== "string") return { init };
let body: unknown;
try {
body = JSON.parse(init.body);
} catch {
- return init;
+ return { init };
}
- if (!isObject(body) || !Array.isArray(body.input)) return init;
+ if (!isObject(body)) return { init };
+ const promptCacheKey =
+ typeof body.prompt_cache_key === "string" ? body.prompt_cache_key : undefined;
+ if (!Array.isArray(body.input)) return { init, promptCacheKey };
// Only response item IDs are forbidden; tool call IDs and IDs inside tool
// inputs or outputs belong to the conversation and must survive replay.
@@ -64,7 +72,7 @@ function stripInputItemIds(init: RequestInit | undefined): RequestInit | undefin
changed = true;
}
}
- return changed ? { ...init, body: JSON.stringify(body) } : init;
+ return { init: changed ? { ...init, body: JSON.stringify(body) } : init, promptCacheKey };
}
export function rewriteCodexEndpoint(input: string, codexApiEndpoint = CODEX_API_ENDPOINT): string {
@@ -77,12 +85,17 @@ export function rewriteCodexEndpoint(input: string, codexApiEndpoint = CODEX_API
function authenticatedInit(
input: FetchInput,
- init: RequestInit | undefined,
+ { init, promptCacheKey }: CodexRequest,
token: { readonly accountId?: string; readonly token: string },
): RequestInit {
const headers = cloneHeaders(
init?.headers ?? (input instanceof Request ? input.headers : undefined),
);
+ // The Codex backend routes prompt-cache affinity on `session-id`, not on the body's
+ // `prompt_cache_key`; the Codex CLI sends the same value in both.
+ if (promptCacheKey !== undefined && !headers.has("session-id")) {
+ headers.set("session-id", promptCacheKey);
+ }
headers.delete("authorization");
headers.delete("Authorization");
headers.set("authorization", `Bearer ${token.token}`);