Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): give each turn's budget prompt its own request id (#4361)

Ben Pankow8ecfd4fc8bed
diff --git a/.changeset/budget-prompt-id-per-turn.md b/.changeset/budget-prompt-id-per-turn.md
new file mode 100644
index 000000000..78bbc9ae0
--- /dev/null
+++ b/.changeset/budget-prompt-id-per-turn.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Show the budget prompt again when someone sends a message after pressing Stop on it. The re-raised prompt reused the earlier prompt's request id, so `eve/client`, the Web Chat template, and the dev TUI treated it as already answered and left it unanswerable.
diff --git a/e2e/fixtures/agent-subagents/evals/delegated-usage-limit.shared.ts b/e2e/fixtures/agent-subagents/evals/delegated-usage-limit.shared.ts
index f7d1814be..8352973a2 100644
--- a/e2e/fixtures/agent-subagents/evals/delegated-usage-limit.shared.ts
+++ b/e2e/fixtures/agent-subagents/evals/delegated-usage-limit.shared.ts
@@ -20,13 +20,12 @@ export async function expectSurveyCountedAgainstParent(t: EveEvalContext, messag
optionIds: ["continue", "stop"],
toolName: "session_limit_continuation",
});
- const parentLimitPrefix = `${session.sessionId ?? ""}:limit:input:`;
+ const parentLimitId = new RegExp(`^${session.sessionId ?? ""}:\\d+:limit:input:(\\d+)$`, "u");
await t.require(
request.requestId,
satisfies(
(requestId: string) =>
- requestId.startsWith(parentLimitPrefix) &&
- Number(requestId.slice(parentLimitPrefix.length)) >= SURVEY_WORKER_INPUT_TOKENS,
+ Number(parentLimitId.exec(requestId)?.[1] ?? -1) >= SURVEY_WORKER_INPUT_TOKENS,
"the parent's input limit prompt counts the worker's tokens",
),
);
diff --git a/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts b/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts
index 594038573..a8816c6df 100644
--- a/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts
@@ -35,7 +35,7 @@ export default defineEval({
await t.require(
continueRequest.requestId,
satisfies(
- (requestId: string) => !requestId.startsWith(`${rootSessionId}:limit:`),
+ (requestId: string) => !new RegExp(`^${rootSessionId}:\\d+:limit:`, "u").test(requestId),
"continuation request belongs to a descendant session",
),
);
diff --git a/packages/eve/src/harness/input-request-class.test.ts b/packages/eve/src/harness/input-request-class.test.ts
index 4216e7bd7..d76b897f2 100644
--- a/packages/eve/src/harness/input-request-class.test.ts
+++ b/packages/eve/src/harness/input-request-class.test.ts
@@ -40,6 +40,7 @@ describe("isApprovalRequest", () => {
isApprovalRequest(
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit: 12, usedTokens: 12 },
}),
),
diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.ts
index b206cdc7b..df74bfffb 100644
--- a/packages/eve/src/harness/input-requests.test.ts
+++ b/packages/eve/src/harness/input-requests.test.ts
@@ -1088,6 +1088,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
requests: [
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit: 12, usedTokens: 12 },
}),
],
@@ -1101,6 +1102,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
requests: [
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit: 12, usedTokens: 12 },
}),
approvalRequest(),
@@ -1118,7 +1120,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
const result = resolvePendingInput({
session: createLimitBatchSession(),
stepInput: {
- inputResponses: [{ optionId: "continue", requestId: "sess-test:limit:input:12" }],
+ inputResponses: [{ optionId: "continue", requestId: "sess-test:0:limit:input:12" }],
},
});
@@ -1133,7 +1135,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
const result = resolvePendingInput({
session: createLimitBatchSession(),
stepInput: {
- inputResponses: [{ optionId: "stop", requestId: "sess-test:limit:input:12" }],
+ inputResponses: [{ optionId: "stop", requestId: "sess-test:0:limit:input:12" }],
},
});
@@ -1167,6 +1169,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
requests: [
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit: 12, usedTokens: 12 },
}),
],
@@ -1193,6 +1196,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
requests: [
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit: 12, usedTokens: 12 },
}),
],
@@ -1207,7 +1211,7 @@ describe("resolvePendingInput with a session-limit continuation batch", () => {
expect(result.outcome).toBe("unresolved");
expect(getPendingInputRequestIds(result.session.state)).toEqual(
- new Set(["approval-1", "sess-test:limit:input:12"]),
+ new Set(["approval-1", "sess-test:0:limit:input:12"]),
);
const deferred = consumeDeferredStepInput({ session: result.session });
expect(deferred.input).toEqual({
@@ -1222,6 +1226,7 @@ describe("clearPendingSessionLimitPrompt", () => {
requests: [
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit: 12, usedTokens: 12 },
}),
],
diff --git a/packages/eve/src/harness/session-limit-continuation.test.ts b/packages/eve/src/harness/session-limit-continuation.test.ts
index d41623d36..8e6ec9c24 100644
--- a/packages/eve/src/harness/session-limit-continuation.test.ts
+++ b/packages/eve/src/harness/session-limit-continuation.test.ts
@@ -12,6 +12,7 @@ const VIOLATION = { kind: "input", limit: 40_000_000, usedTokens: 40_120_500 } a
function createTestRequest() {
return createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: VIOLATION,
});
}
@@ -24,7 +25,7 @@ describe("createSessionLimitContinuationRequest", () => {
expect(first).toEqual(second);
expect(first).toEqual({
action: {
- callId: "sess-test:limit:input:40120500",
+ callId: "sess-test:0:limit:input:40120500",
input: { kind: "input", limit: 40_000_000, usedTokens: 40_120_500 },
kind: "tool-call",
toolName: "session_limit_continuation",
@@ -50,26 +51,27 @@ describe("createSessionLimitContinuationRequest", () => {
"This session has hit the input-token limit (40M) per session. This is a guardrail " +
"against defective long-running sessions. If session activity looks fine, just " +
"approve to keep going.",
- requestId: "sess-test:limit:input:40120500",
+ requestId: "sess-test:0:limit:input:40120500",
});
});
it("creates a token-cost continuation prompt", () => {
const request = createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "token-cost", limitUsd: 1.5, usedCostUsd: 1.5123 },
});
expect(request).toMatchObject({
action: {
- callId: "sess-test:limit:token-cost:1.5123",
+ callId: "sess-test:0:limit:token-cost:1.5123",
input: { kind: "token-cost", limitUsd: 1.5, usedCostUsd: 1.5123 },
},
prompt:
"This session has hit the $1.5 model token-cost limit per session. This is a guardrail " +
"against defective long-running sessions. If session activity looks fine, just " +
"approve to keep going.",
- requestId: "sess-test:limit:token-cost:1.5123",
+ requestId: "sess-test:0:limit:token-cost:1.5123",
});
});
@@ -77,6 +79,7 @@ describe("createSessionLimitContinuationRequest", () => {
const promptFor = (limit: number): string =>
createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "input", limit, usedTokens: limit + 1 },
}).prompt;
@@ -88,6 +91,7 @@ describe("createSessionLimitContinuationRequest", () => {
const tinyCost = createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { kind: "token-cost", limitUsd: 5e-7, usedCostUsd: 6e-7 },
});
expect(tinyCost.prompt).toContain("$5e-7 model token-cost limit");
@@ -99,6 +103,7 @@ describe("createSessionLimitContinuationRequest", () => {
// response to an earlier prompt never resolves a later one.
const later = createSessionLimitContinuationRequest({
sessionId: "sess-test",
+ turnSequence: 0,
violation: { ...VIOLATION, usedTokens: 80_500_000 },
});
@@ -111,10 +116,10 @@ describe("createSessionLimitContinuationRequest", () => {
it("mints ids recognized by isSessionLimitContinuationRequestId", () => {
expect(isSessionLimitContinuationRequestId(createTestRequest().requestId)).toBe(true);
- expect(isSessionLimitContinuationRequestId("sess-test:limit:output:12")).toBe(true);
- expect(isSessionLimitContinuationRequestId("sess-test:limit:token-cost:1.5123")).toBe(true);
+ expect(isSessionLimitContinuationRequestId("sess-test:0:limit:output:12")).toBe(true);
+ expect(isSessionLimitContinuationRequestId("sess-test:0:limit:token-cost:1.5123")).toBe(true);
expect(isSessionLimitContinuationRequestId("approval-1")).toBe(false);
- expect(isSessionLimitContinuationRequestId("sess-test:limit:input:")).toBe(false);
+ expect(isSessionLimitContinuationRequestId("sess-test:0:limit:input:")).toBe(false);
});
});
diff --git a/packages/eve/src/harness/session-limit-continuation.ts b/packages/eve/src/harness/session-limit-continuation.ts
index 8a786d5ae..53cbab539 100644
--- a/packages/eve/src/harness/session-limit-continuation.ts
+++ b/packages/eve/src/harness/session-limit-continuation.ts
@@ -3,7 +3,7 @@
*
* When a durable session reaches its configured token or token-cost budget, the harness
* parks on a harness-authored input request instead of failing the session.
- * The request is derived only from the session identity and violation, so
+ * The request is derived only from the session, turn, and violation, so
* identical session state always produces an identical prompt — no model call
* is involved.
*/
@@ -26,14 +26,18 @@ export const SESSION_LIMIT_STOP_OPTION_ID = "stop";
*/
export function createSessionLimitContinuationRequest(input: {
readonly sessionId: string;
+ /** The turn's `sequence`, which tells its prompt apart from another turn's at the same usage. */
+ readonly turnSequence: number;
readonly violation: SessionUsageLimitViolation;
}): InputRequest {
- const { sessionId, violation } = input;
+ const { sessionId, turnSequence, violation } = input;
const used = violation.kind === "token-cost" ? violation.usedCostUsd : violation.usedTokens;
- // The absolute session usage is strictly increasing across violations, so
- // each prompt gets a deterministic id and stale controls cannot resolve a
- // later prompt.
- const requestId = `${sessionId}:limit:${violation.kind}:${String(used)}`;
+ // Re-raising an unanswered prompt within a turn keeps its id. Usage alone
+ // doesn't separate prompts across turns: after Stop the next turn is over
+ // budget at the same usage, and clients drop request ids they've seen. The
+ // sequence rather than the turn id keeps the id short enough for Discord's
+ // 100-character `custom_id`.
+ const requestId = `${sessionId}:${String(turnSequence)}:limit:${violation.kind}:${String(used)}`;
const actionInput: JsonObject =
violation.kind === "token-cost"
? {
diff --git a/packages/eve/src/harness/session-limit-enforcement.ts b/packages/eve/src/harness/session-limit-enforcement.ts
index 83a458fc9..84fd35c8e 100644
--- a/packages/eve/src/harness/session-limit-enforcement.ts
+++ b/packages/eve/src/harness/session-limit-enforcement.ts
@@ -120,6 +120,7 @@ async function parkOnSessionUsageLimit(input: {
}): Promise<StepResult> {
const request = createSessionLimitContinuationRequest({
sessionId: input.session.sessionId,
+ turnSequence: input.emissionState.sequence,
violation: input.violation,
});
let emissionState = input.emissionState;
diff --git a/packages/eve/src/harness/stale-input-responses.test.ts b/packages/eve/src/harness/stale-input-responses.test.ts
index 946d16bb9..0a07e3185 100644
--- a/packages/eve/src/harness/stale-input-responses.test.ts
+++ b/packages/eve/src/harness/stale-input-responses.test.ts
@@ -161,7 +161,7 @@ it("drops a stale session-limit continuation answer and keeps the rest of the in
const stepInput = dropStaleSessionLimitContinuationResponses({
pendingRequestIds: new Set(),
stepInput: {
- inputResponses: [{ optionId: "stop", requestId: "sess-1:limit:input:12" }],
+ inputResponses: [{ optionId: "stop", requestId: "sess-1:0:limit:input:12" }],
message: "also do this",
},
});
@@ -174,12 +174,12 @@ it("drops a stale session-limit continuation answer and keeps the rest of the in
it("keeps pending and non-continuation responses while dropping stale continuation answers", () => {
const stepInput = dropStaleSessionLimitContinuationResponses({
- pendingRequestIds: new Set(["question-1", "sess-1:limit:input:20"]),
+ pendingRequestIds: new Set(["question-1", "sess-1:0:limit:input:20"]),
stepInput: {
inputResponses: [
{ optionId: "candidate", requestId: "question-1" },
- { optionId: "continue", requestId: "sess-1:limit:input:12" },
- { optionId: "stop", requestId: "sess-1:limit:input:20" },
+ { optionId: "continue", requestId: "sess-1:0:limit:input:12" },
+ { optionId: "stop", requestId: "sess-1:0:limit:input:20" },
],
},
});
@@ -188,7 +188,7 @@ it("keeps pending and non-continuation responses while dropping stale continuati
// to the currently pending continuation prompt pass through.
expect(stepInput?.inputResponses).toEqual([
{ optionId: "candidate", requestId: "question-1" },
- { optionId: "stop", requestId: "sess-1:limit:input:20" },
+ { optionId: "stop", requestId: "sess-1:0:limit:input:20" },
]);
});
@@ -212,7 +212,7 @@ it("converts remaining stale responses after the drop pass", () => {
stepInput: {
inputResponses: [
{ optionId: "candidate", requestId: "question-1" },
- { optionId: "stop", requestId: "sess-1:limit:input:12" },
+ { optionId: "stop", requestId: "sess-1:0:limit:input:12" },
],
},
});
diff --git a/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts b/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts
index 20032484a..3c1682e94 100644
--- a/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts
+++ b/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts
@@ -753,13 +753,11 @@ describe("tool loop generate approval resume (real AI SDK)", () => {
});
expect(execute).not.toHaveBeenCalled();
+ const [limitBatch] = limited.session.state?.["eve.runtime.pendingInputBatches"] as readonly {
+ readonly requests: readonly { readonly requestId: string }[];
+ }[];
const resumed = await runStep(limited.session, {
- inputResponses: [
- {
- optionId: "continue",
- requestId: `${session.sessionId}:limit:input:12`,
- },
- ],
+ inputResponses: [{ optionId: "continue", requestId: limitBatch!.requests[0]!.requestId }],
});
expect(execute).toHaveBeenCalledExactlyOnceWith(
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 2a78becfd..56be252b0 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -1925,7 +1925,7 @@ describe("createToolLoopHarness", () => {
});
}
- const LIMIT_REQUEST_ID = "test-session:limit:input:12";
+ const LIMIT_REQUEST_ID = "test-session:0:limit:input:12";
it("parks on a deterministic continuation prompt when the session reaches its token limit", async () => {
const { emit, events } = createEventCollector();
@@ -1992,7 +1992,7 @@ describe("createToolLoopHarness", () => {
input: { kind: "token-cost", limitUsd: 1.5, usedCostUsd: 1.51 },
},
prompt: expect.stringContaining("$1.5 model token-cost limit"),
- requestId: "test-session:limit:token-cost:1.51",
+ requestId: "test-session:0:limit:token-cost:1.51",
},
],
});
@@ -2005,7 +2005,7 @@ describe("createToolLoopHarness", () => {
toolResults: [],
});
const resumed = await runStep(parked.session, {
- inputResponses: [{ optionId: "continue", requestId: "test-session:limit:token-cost:1.51" }],
+ inputResponses: [{ optionId: "continue", requestId: "test-session:0:limit:token-cost:1.51" }],
});
expect(vi.mocked(ToolLoopAgent)).toHaveBeenCalledTimes(1);
diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
index a2ffb984c..7455ad46a 100644
--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
@@ -59,22 +59,22 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
| running out of budget opens budget prompt | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |
| pressing approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| reply of approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| pressing stop on budget prompt halts work, next message asks again | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
-| reply of stop on budget prompt halts work, next message asks again | ❌<sup>[11](#note-11)</sup> | —<sup>[11](#note-11)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| pressing stop on budget prompt halts work, next message asks again | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| reply of stop on budget prompt halts work, next message asks again | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| query sent during budget prompt answered after budget approval | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
## Sign-ins
| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |
| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
-| a sign-in names the service and shows its sign-in link | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
-| a sign-in shows its confirmation code | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
-| only the person signing in sees the sign-in link and code | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ❌<sup>[16](#note-16)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[17](#note-17)</sup> | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> |
-| a sign-in without a link shows its instructions | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
+| a sign-in names the service and shows its sign-in link | ✅ | ✅ | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
+| a sign-in shows its confirmation code | ✅ | ✅ | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
+| only the person signing in sees the sign-in link and code | —<sup>[14](#note-14)</sup> | —<sup>[14](#note-14)</sup> | ✅ | —<sup>[14](#note-14)</sup> | —<sup>[14](#note-14)</sup> | ✅ | —<sup>[14](#note-14)</sup> | ✅ | —<sup>[14](#note-14)</sup> | ✅ | —<sup>[14](#note-14)</sup> | ❌<sup>[15](#note-15)</sup> | —<sup>[14](#note-14)</sup> | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[16](#note-16)</sup> | —<sup>[14](#note-14)</sup> | —<sup>[14](#note-14)</sup> |
+| a sign-in without a link shows its instructions | ✅ | ✅ | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ |
| after signing in, the agent carries on with the request | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |
-| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
+| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ |
| message after ignored sign-in gets an answer, signing in late doesn't run | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
+| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ |
## Notes
@@ -88,10 +88,9 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
8. <a id="note-8"></a>one person answers in their own browser tab; there's nobody else to tell
9. <a id="note-9"></a>a resolved prompt doesn't say who answered; input.resolved carries no responder
10. <a id="note-10"></a>the card loses its buttons after a typed approval but doesn't say who approved
-11. <a id="note-11"></a>a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen
-12. <a id="note-12"></a>outside a DM the bot says to continue in a direct message but never sends one
-13. <a id="note-13"></a>the channel has no default sign-in renderer
-14. <a id="note-14"></a>the rule applies only where the conversation is shared or private, and this one is public
-15. <a id="note-15"></a>the rule applies only where the conversation is public or shared, and this one is private
-16. <a id="note-16"></a>the sign-in prompt, link included, is posted to the whole thread
-17. <a id="note-17"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified
+11. <a id="note-11"></a>outside a DM the bot says to continue in a direct message but never sends one
+12. <a id="note-12"></a>the channel has no default sign-in renderer
+13. <a id="note-13"></a>the rule applies only where the conversation is shared or private, and this one is public
+14. <a id="note-14"></a>the rule applies only where the conversation is public or shared, and this one is private
+15. <a id="note-15"></a>the sign-in prompt, link included, is posted to the whole thread
+16. <a id="note-16"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified
diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
index bd1e605dd..0f1c8f41d 100644
--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
@@ -111,19 +111,6 @@ function noSignInRenderer(...rules: HitlRule[]): Partial<Record<HitlRule, Broken
);
}
-/** Budget prompts in `rules` never show; see each caller for why. */
-function budgetPromptNotShown(
- reason: string,
- ...rules: HitlRule[]
-): Partial<Record<HitlRule, BrokenCell>> {
- return Object.fromEntries(
- rules.map((rule) => [
- rule,
- { reason, symptom: /Timed out waiting for one of the questions \["This session has hit/u },
- ]),
- );
-}
-
/** Chat SDK's default sign-in, outside a DM, points the person at a DM it never sends. */
const SIGN_IN_ONLY_IN_DMS = Object.fromEntries(
[
@@ -255,13 +242,6 @@ const hitlConformance = {
tui: [
{
driver: tuiDriver,
- broken: {
- ...budgetPromptNotShown(
- "a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",
- "pressing stop on budget prompt halts work, next message asks again",
- "reply of stop on budget prompt halts work, next message asks again",
- ),
- },
unsupported: {
"pressing an option of an answered question sends it to the agent as new input":
"an answered question's drawer closes, so nothing is left to press",
@@ -283,18 +263,9 @@ const hitlConformance = {
"web-chat": [
{
driver: webChatDriver,
- broken: {
- ...budgetPromptNotShown(
- "a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",
- "pressing stop on budget prompt halts work, next message asks again",
- ),
- },
unsupported: {
"pressing an option of an answered question sends it to the agent as new input":
"an answered question disables its options, so nothing is left to press",
- // Skipped rather than broken: whether the re-raised prompt shows depends on event timing.
- "reply of stop on budget prompt halts work, next message asks again":
- "a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",
"pressing Approve names who approved on the approval": WEB_CHAT_SINGLE_PERSON,
"approving by text names who approved on the approval": WEB_CHAT_SINGLE_PERSON,
"pressing an option names who answered on the question": WEB_CHAT_SINGLE_PERSON,
diff --git a/packages/eve/src/public/channels/discord/hitl.ts b/packages/eve/src/public/channels/discord/hitl.ts
index adc30f64d..22d8c07d2 100644
--- a/packages/eve/src/public/channels/discord/hitl.ts
+++ b/packages/eve/src/public/channels/discord/hitl.ts
@@ -208,7 +208,7 @@ export function deriveModalInputResponses(
/**
* Encodes the payload as a raw JSON array, which leaves room for long request
- * ids such as a session-limit prompt's `<sessionId>:limit:input:<tokens>`.
+ * ids such as a session-limit prompt's `<sessionId>:<turn>:limit:input:<tokens>`.
*/
function encodeHitlCustomId(prefix: string, payload: HitlCustomIdPayload): string {
const fields =
diff --git a/packages/eve/src/subagents/event-proxy-step.integration.test.ts b/packages/eve/src/subagents/event-proxy-step.integration.test.ts
index 2a14f9ede..0b5a77067 100644
--- a/packages/eve/src/subagents/event-proxy-step.integration.test.ts
+++ b/packages/eve/src/subagents/event-proxy-step.integration.test.ts
@@ -87,6 +87,7 @@ function fixture() {
};
const request = createSessionLimitContinuationRequest({
sessionId: "child-session",
+ turnSequence: 0,
violation: { kind: "input", limit: 100, usedTokens: 101 },
});
const hookPayload: SubagentInputRequestHookPayload = {
diff --git a/packages/eve/test/scenarios/turn-cancellation.scenario.test.ts b/packages/eve/test/scenarios/turn-cancellation.scenario.test.ts
index 096863979..ef18710eb 100644
--- a/packages/eve/test/scenarios/turn-cancellation.scenario.test.ts
+++ b/packages/eve/test/scenarios/turn-cancellation.scenario.test.ts
@@ -301,7 +301,9 @@ describe("turn cancellation descendant cascade", () => {
);
expect(tasks).toHaveLength(1);
expect(requests).toHaveLength(1);
- expect(requests[0]?.requestId.startsWith(`${response.sessionId}:limit:`)).toBe(true);
+ expect(requests[0]?.requestId).toMatch(
+ new RegExp(`^${response.sessionId}:\\d+:limit:`, "u"),
+ );
const requestId = requests[0]?.requestId;
if (requestId === undefined) throw new Error("Root limit prompt has no request id.");