Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): count only new input toward the session input-token limit (#5011)

Rui91b3183f3f63
diff --git a/.changeset/budget-excludes-cache-reads.md b/.changeset/budget-excludes-cache-reads.md
new file mode 100644
index 000000000..15e43cfe8
--- /dev/null
+++ b/.changeset/budget-excludes-cache-reads.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+The session input-token limit now counts only new input (uncached input plus cache writes). Cache reads no longer count toward the limit, the grant after Continue, or the quota shared with subagents, so long cache-heavy sessions stop hitting continuation prompts for replaying their own history. Usage telemetry still reports raw input and cache totals. Providers that report no cache details count exactly as before.
diff --git a/docs/agent-config.md b/docs/agent-config.md
index b1aa0d4d6..62e7b5fb5 100644
--- a/docs/agent-config.md
+++ b/docs/agent-config.md
@@ -294,6 +294,10 @@ disable the timeout. Session expiration follows
stored payloads when it ends.
Input tokens, output tokens, and model token cost are checked independently.
+The input limit counts only new input: uncached input plus cache writes. Cache
+reads, the history a provider replays from its prompt cache on every step, do
+not count, so long tool-heavy sessions are not stopped for re-reading their own
+context. When a provider reports no cache details, all input counts.
The model call that crosses a limit is allowed to finish because exact usage
arrives after the call completes. Before the next model call, eve pauses the
turn (`turn.waiting`) and sends a deterministic continuation prompt with two options:
@@ -323,7 +327,7 @@ no inherited quota also fails instead of raising a continuation prompt that
could only grant another zero-value window.
When `maxInputTokensPerSession` is omitted, root sessions apply a default
-input budget of `40_000_000` provider-reported input tokens.
+input budget of `40_000_000` new input tokens.
`maxOutputTokensPerSession` and `maxTokenCostUsdPerSession` are unset by
default. `maxTokenCostUsdPerSession` is a US-dollar limit on model token cost,
not tool or infrastructure spend. It uses the cost reported with each model
diff --git a/packages/eve/src/harness/hitl/budget.ts b/packages/eve/src/harness/hitl/budget.ts
index 06cb46f84..a91078b79 100644
--- a/packages/eve/src/harness/hitl/budget.ts
+++ b/packages/eve/src/harness/hitl/budget.ts
@@ -69,6 +69,7 @@ export function checkSessionUsageLimit(input: {
usedCostUsd: violation.usedCostUsd,
}
: {
+ cacheReadTokens: usage.cacheReadTokens,
inputTokens: usage.inputTokens,
kind: violation.kind,
limit: violation.limit,
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 48cc57d99..fad0c3a2a 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -1986,8 +1986,10 @@ describe("createToolLoopHarness", () => {
it.each([
{
+ // The limit counts new input (14 minus 2 cache reads); details keep the raw totals.
details: {
- inputTokens: 12,
+ cacheReadTokens: 2,
+ inputTokens: 14,
kind: "input",
limit: 12,
outputTokens: 3,
@@ -2001,7 +2003,7 @@ describe("createToolLoopHarness", () => {
cacheReadTokens: 2,
cacheWriteTokens: 1,
costUsd: 0,
- inputTokens: 12,
+ inputTokens: 14,
outputTokens: 3,
sawCost: false,
},
diff --git a/packages/eve/src/harness/turn-tag-state.test.ts b/packages/eve/src/harness/turn-tag-state.test.ts
index 285177be0..243418e6f 100644
--- a/packages/eve/src/harness/turn-tag-state.test.ts
+++ b/packages/eve/src/harness/turn-tag-state.test.ts
@@ -1,8 +1,11 @@
import { describe, expect, it } from "vitest";
+import { extractTokenUsageDelta } from "#harness/model-call/usage.js";
+import type { HarnessStepResult } from "#harness/step-hooks.js";
import {
accumulateSessionUsage,
accumulateTurnUsage,
+ addTurnUsage,
bumpSessionRuntimeUsageLimits,
getSessionRemainingUsageQuota,
getSessionRuntimeUsageLimits,
@@ -429,6 +432,57 @@ describe("session token limits", () => {
outputTokens: false,
});
});
+
+ describe("counts only new input", () => {
+ // One model step that replayed 90 cached tokens, wrote 4 to the cache and sent 6 uncached.
+ function sessionAfterStep(usage: Partial<NonNullable<HarnessStepResult["usage"]>>) {
+ return {
+ ...addTurnUsage(
+ makeSession(),
+ "turn_0",
+ extractTokenUsageDelta({
+ costUsd: undefined,
+ usage: usage as NonNullable<HarnessStepResult["usage"]>,
+ }),
+ ),
+ limits: { maxInputTokensPerSession: 30 },
+ };
+ }
+
+ it("excludes cache reads from the limit, the grant and the remaining quota", () => {
+ const session = sessionAfterStep({
+ inputTokens: 100,
+ inputTokenDetails: { cacheReadTokens: 90, cacheWriteTokens: 4, noCacheTokens: 6 },
+ outputTokens: 1,
+ });
+
+ // Telemetry keeps the raw totals.
+ expect(getSessionTokenUsage(session)).toMatchObject({
+ cacheReadTokens: 90,
+ cacheWriteTokens: 4,
+ inputTokens: 100,
+ });
+ expect(getSessionUsageLimitViolation(session)).toBeNull();
+ expect(getSessionRemainingUsageQuota(session)).toMatchObject({ inputTokens: 20 });
+ expect(getSessionRuntimeUsageLimits(bumpSessionRuntimeUsageLimits(session))).toEqual({
+ inputTokens: 40,
+ });
+ });
+
+ it("counts the full input when the provider reports no cache details", () => {
+ const session = sessionAfterStep({ inputTokens: 100, outputTokens: 1 });
+
+ expect(getSessionUsageLimitViolation(session)).toEqual({
+ kind: "input",
+ limit: 30,
+ usedTokens: 100,
+ });
+ expect(getSessionRemainingUsageQuota(session)).toMatchObject({ inputTokens: 0 });
+ expect(getSessionRuntimeUsageLimits(bumpSessionRuntimeUsageLimits(session))).toEqual({
+ inputTokens: 130,
+ });
+ });
+ });
});
describe("accumulateSessionUsage", () => {
diff --git a/packages/eve/src/harness/turn-tag-state.ts b/packages/eve/src/harness/turn-tag-state.ts
index 15d402971..a1412681b 100644
--- a/packages/eve/src/harness/turn-tag-state.ts
+++ b/packages/eve/src/harness/turn-tag-state.ts
@@ -115,15 +115,14 @@ export function getSessionRuntimeUsageLimits(
/**
* Bumps the runtime usage limits after the user grants a continuation. Each
- * capped axis is re-anchored to `current usage + configured limit`, so one
+ * capped axis is re-anchored to `current budget usage + configured limit`, so one
* approval buys one full configured window even after an overshoot. All axes
* bump together to avoid back-to-back prompts. Configured limits never change.
*/
export function bumpSessionRuntimeUsageLimits(session: HarnessSession): HarnessSession {
- const usage = getSessionTokenUsage(session);
const bumped = grantRuntimeUsageLimits({
configured: configuredSessionUsageLimits(session),
- usage,
+ usage: getSessionBudgetUsage(session),
});
return {
...session,
@@ -144,23 +143,31 @@ export function getSessionRemainingUsageQuota(session: Pick<HarnessSession, "lim
inputTokens: number | false;
outputTokens: number | false;
} {
- const usage = getSessionTokenUsage(session);
const runtime = getSessionRuntimeUsageLimits(session);
- return remainingRuntimeUsageQuota({ runtime, usage });
+ return remainingRuntimeUsageQuota({ runtime, usage: getSessionBudgetUsage(session) });
}
export function getSessionUsageLimitViolation(
session: Pick<HarnessSession, "limits" | "state">,
): SessionUsageLimitViolation | null {
- const usage = getSessionTokenUsage(session);
const runtime = getSessionRuntimeUsageLimits(session);
return findRuntimeUsageLimitViolation({
configured: configuredSessionUsageLimits(session),
runtime,
- usage,
+ usage: getSessionBudgetUsage(session),
});
}
+/**
+ * The session's usage as the limits count it. Cache reads are replayed history, not new work,
+ * so the input axis counts only new input: uncached input plus cache writes. A provider that
+ * reports no cache details counts its full input, as before.
+ */
+function getSessionBudgetUsage(session: Pick<HarnessSession, "state">): TokenUsageTotals {
+ const usage = getSessionTokenUsage(session);
+ return { ...usage, inputTokens: Math.max(0, usage.inputTokens - usage.cacheReadTokens) };
+}
+
function configuredSessionUsageLimits(
session: Pick<HarnessSession, "limits">,
): SessionRuntimeUsageLimits {
diff --git a/packages/eve/src/harness/types.ts b/packages/eve/src/harness/types.ts
index 714ab3b5e..dd26de6ca 100644
--- a/packages/eve/src/harness/types.ts
+++ b/packages/eve/src/harness/types.ts
@@ -119,8 +119,8 @@ export function requireSessionModelReference(session: HarnessSession): RuntimeMo
*/
export interface SessionLimits {
/**
- * Maximum provider-reported input tokens this durable session may spend
- * before eve refuses to start another model call. Absent when the session
+ * Maximum new input tokens (input minus cache reads) this durable session
+ * may spend before eve refuses to start another model call. Absent when the session
* is uncapped. Root sessions default to 40M unless authored otherwise;
* delegated subagent sessions receive the parent's remaining quota at
* dispatch time.
diff --git a/packages/eve/src/shared/agent-definition.ts b/packages/eve/src/shared/agent-definition.ts
index 15793dd79..74a3f4c74 100644
--- a/packages/eve/src/shared/agent-definition.ts
+++ b/packages/eve/src/shared/agent-definition.ts
@@ -184,7 +184,9 @@ export interface AgentLimitsDefinition {
*/
readonly sessionTimeoutMs?: number | false;
/**
- * Maximum provider-reported input tokens accumulated by one durable session.
+ * Maximum new input tokens accumulated by one durable session: uncached
+ * input plus cache writes. Cache reads don't count. When the provider
+ * reports no cache details, all input counts.
*
* eve checks this before starting each model call. The model call that crosses
* the limit is allowed to finish because providers only report exact usage