Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Emit rejected action.result for denied tool calls (#7)

John Phamous28e8ecc98724
diff --git a/.changeset/rejected-tool-calls.md b/.changeset/rejected-tool-calls.md
new file mode 100644
index 000000000..a32b594e3
--- /dev/null
+++ b/.changeset/rejected-tool-calls.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Emit a `rejected` `action.result` stream event when a tool call is denied at a HITL approval gate. Denied calls previously left no trace in the session stream (the denial lived only in model history), so consumers like observability never saw the tool call resolve. The `action.result` status union now includes `rejected`, and the message stream version is bumped to `16`.
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/shared.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/shared.ts
index 7280c0b3e..052847844 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/shared.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/shared.ts
@@ -6,13 +6,14 @@ export const GUARDED_ECHO_TOKEN = "guarded-echo-ok-T4Q9";
* Approval results resolve in the turn AFTER the request was raised, so
* per-turn `toolCalls` cannot pair them; scan the whole session stream for
* executed `guarded-echo` action results instead. A denied call executes
- * nothing and emits no `action.result` at all — the execution-denied output
- * returns to the model as message context only.
+ * nothing: it surfaces as a `rejected` `action.result` (skipped here) while
+ * the execution-denied output returns to the model as message context.
*/
export function guardedEchoResults(events: readonly HandleMessageStreamEvent[]): string[] {
const results: string[] = [];
for (const event of events) {
if (event.type !== "action.result") continue;
+ if (event.data.status === "rejected") continue;
const result = event.data.result;
if (result.kind !== "tool-result" || result.toolName !== "guarded-echo") continue;
results.push(JSON.stringify(result.output ?? ""));
diff --git a/packages/eve/src/harness/input-requests.test.ts b/packages/eve/src/harness/input-requests.test.ts
index e58b81b26..d1716a6f9 100644
--- a/packages/eve/src/harness/input-requests.test.ts
+++ b/packages/eve/src/harness/input-requests.test.ts
@@ -467,6 +467,138 @@ describe("resolvePendingInput", () => {
});
});
+ it("returns a rejected action for an explicitly denied approval", () => {
+ const session = setPendingInputBatch({
+ event: { sequence: 5, stepIndex: 1, turnId: "turn_0" },
+ requests: [
+ {
+ action: {
+ callId: "approval-call",
+ input: { command: "pwd" },
+ kind: "tool-call",
+ toolName: "bash",
+ },
+ allowFreeform: false,
+ display: "confirmation",
+ options: [
+ { id: "approve", label: "Yes" },
+ { id: "deny", label: "No" },
+ ],
+ prompt: "Approve tool call: bash",
+ requestId: "approval-1",
+ } satisfies InputRequest,
+ ],
+ responseMessages: [],
+ session: createHarnessSession(),
+ });
+
+ const result = resolvePendingInput({
+ stepInput: {
+ inputResponses: [{ requestId: "approval-1", optionId: "deny" }],
+ },
+ session,
+ });
+
+ expect(result.outcome).toBe("resolved");
+ expect(result.rejectedActions).toEqual({
+ event: { sequence: 5, stepIndex: 1, turnId: "turn_0" },
+ results: [
+ {
+ callId: "approval-call",
+ isError: true,
+ kind: "tool-result",
+ output: {
+ code: "TOOL_EXECUTION_DENIED",
+ message: "Tool execution was denied.",
+ },
+ toolName: "bash",
+ },
+ ],
+ });
+ });
+
+ it("does not return a rejected action when an approval is granted", () => {
+ const session = setPendingInputBatch({
+ event: { sequence: 5, stepIndex: 1, turnId: "turn_0" },
+ requests: [
+ {
+ action: {
+ callId: "approval-call",
+ input: { command: "pwd" },
+ kind: "tool-call",
+ toolName: "bash",
+ },
+ allowFreeform: false,
+ display: "confirmation",
+ options: [
+ { id: "approve", label: "Yes" },
+ { id: "deny", label: "No" },
+ ],
+ prompt: "Approve tool call: bash",
+ requestId: "approval-1",
+ } satisfies InputRequest,
+ ],
+ responseMessages: [],
+ session: createHarnessSession(),
+ });
+
+ const result = resolvePendingInput({
+ stepInput: {
+ inputResponses: [{ requestId: "approval-1", optionId: "approve" }],
+ },
+ session,
+ });
+
+ expect(result.outcome).toBe("resolved");
+ expect(result.rejectedActions).toBeUndefined();
+ });
+
+ it("returns a rejected action when a pending approval is auto-denied by a follow-up message", () => {
+ const session = setPendingInputBatch({
+ event: { sequence: 7, stepIndex: 2, turnId: "turn_1" },
+ requests: [
+ {
+ action: {
+ callId: "approval-call",
+ input: { command: "pwd" },
+ kind: "tool-call",
+ toolName: "bash",
+ },
+ allowFreeform: false,
+ display: "confirmation",
+ options: [
+ { id: "approve", label: "Yes" },
+ { id: "deny", label: "No" },
+ ],
+ prompt: "Approve tool call: bash",
+ requestId: "approval-1",
+ } satisfies InputRequest,
+ ],
+ responseMessages: [],
+ session: createHarnessSession(),
+ });
+
+ const result = resolvePendingInput({
+ stepInput: { message: "Never mind, do something else." },
+ session,
+ });
+
+ expect(result.outcome).toBe("resolved");
+ expect(result.rejectedActions?.event).toEqual({ sequence: 7, stepIndex: 2, turnId: "turn_1" });
+ expect(result.rejectedActions?.results).toEqual([
+ {
+ callId: "approval-call",
+ isError: true,
+ kind: "tool-result",
+ output: {
+ code: "TOOL_EXECUTION_DENIED",
+ message: "Ignored because the user continued without responding.",
+ },
+ toolName: "bash",
+ },
+ ]);
+ });
+
it("falls back to tool name when no approvalKey is provided", () => {
const session = setPendingInputBatch({
requests: [
diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.ts
index 2f4edd108..dc8bff59f 100644
--- a/packages/eve/src/harness/input-requests.ts
+++ b/packages/eve/src/harness/input-requests.ts
@@ -1,6 +1,9 @@
import type { ModelMessage, ToolSet, TypedToolCall } from "ai";
-import type { RuntimeToolCallActionRequest } from "#runtime/actions/types.js";
+import type {
+ RuntimeToolCallActionRequest,
+ RuntimeToolResultActionResult,
+} from "#runtime/actions/types.js";
import type { InputRequest, InputResponse } from "#runtime/input/types.js";
import { parseJsonObject } from "#shared/json.js";
import { coalesceTurnInputs } from "#harness/messages.js";
@@ -12,16 +15,39 @@ const DEFERRED_STEP_INPUT_KEY = "eve.runtime.deferredStepInput";
const IGNORED_INPUT_REASON = "Ignored because the user continued without responding.";
+const TOOL_EXECUTION_DENIED_CODE = "TOOL_EXECUTION_DENIED";
+const TOOL_EXECUTION_DENIED_MESSAGE = "Tool execution was denied.";
+
type ToolResponsePart = Extract<ModelMessage, { role: "tool" }>["content"][number];
+/**
+ * Stream-emit coordinates carried so a parked batch's resolution can attribute
+ * its events to the turn and step that requested the input.
+ */
+interface PendingInputBatchEvent {
+ readonly sequence: number;
+ readonly stepIndex: number;
+ readonly turnId: string;
+}
+
/**
* Serializable pending input batch stored on the session state.
*/
interface PendingInputBatch {
+ readonly event?: PendingInputBatchEvent;
readonly requests: readonly InputRequest[];
readonly responseMessages: readonly ModelMessage[];
}
+/**
+ * Denied tool-call approvals from one resolved batch, ready for the caller to
+ * emit as `rejected` `action.result` events against the originating turn.
+ */
+export interface RejectedActionBatch {
+ readonly event: PendingInputBatchEvent;
+ readonly results: readonly RuntimeToolResultActionResult[];
+}
+
/**
* Returns true when the step input carries user-facing turn input.
*/
@@ -129,16 +155,17 @@ export function resolvePendingInput(input: {
messages.push({ content: toolParts, role: "tool" });
}
+ const rejectedActions = buildRejectedActionBatch(pendingBatch, []);
session = clearPendingInputBatch(session);
if (pendingBatch.requests.some((request) => isApprovalRequest(request))) {
session = queueDeferredStepInput(session, {
message: stepInput.message,
});
- return { deferredMessage: true, outcome: "resolved", messages, session };
+ return { deferredMessage: true, outcome: "resolved", messages, rejectedActions, session };
}
- return { outcome: "resolved", messages, session };
+ return { outcome: "resolved", messages, rejectedActions, session };
}
// Record approved tools before clearing the batch.
@@ -157,6 +184,7 @@ export function resolvePendingInput(input: {
messages.push({ content: toolParts, role: "tool" });
}
+ const rejectedActions = buildRejectedActionBatch(pendingBatch, responses);
session = clearPendingInputBatch(session);
// AI SDK cannot process tool-approval responses and a new user message
@@ -170,16 +198,17 @@ export function resolvePendingInput(input: {
message: stepInput.message,
});
- return { deferredMessage: true, outcome: "resolved", messages, session };
+ return { deferredMessage: true, outcome: "resolved", messages, rejectedActions, session };
}
- return { outcome: "resolved", messages, session };
+ return { outcome: "resolved", messages, rejectedActions, session };
}
type ResolvePendingInputResult = {
readonly deferredMessage?: boolean;
readonly outcome: "resolved" | "continue" | "unresolved";
readonly messages: ModelMessage[];
+ readonly rejectedActions?: RejectedActionBatch;
readonly session: HarnessSession;
};
@@ -215,12 +244,14 @@ function getPendingInputBatch(state: SessionStateMap | undefined): PendingInputB
* Stores one pending HITL batch on the session until the user responds.
*/
export function setPendingInputBatch(input: {
+ readonly event?: PendingInputBatchEvent;
readonly requests: readonly InputRequest[];
readonly responseMessages: readonly ModelMessage[];
readonly session: HarnessSession;
}): HarnessSession {
const state = { ...input.session.state };
state[PENDING_INPUT_BATCH_KEY] = {
+ event: input.event,
requests: [...input.requests],
responseMessages: [...input.responseMessages],
} satisfies PendingInputBatch;
@@ -326,6 +357,59 @@ function recordApprovedTools(input: {
// Tool response building
// ---------------------------------------------------------------------------
+/**
+ * Resolves whether an approval request was granted and, when auto-denied
+ * because the user continued without responding, the reason to record.
+ */
+function resolveApprovalOutcome(response: InputResponse | undefined): {
+ readonly approved: boolean;
+ readonly reason: string | undefined;
+} {
+ return {
+ approved: response?.optionId === "approve",
+ reason: response === undefined ? IGNORED_INPUT_REASON : undefined,
+ };
+}
+
+/**
+ * Builds one rejected `action.result` payload per denied tool-call approval so
+ * the stream records denials that otherwise live only in model history.
+ */
+function buildRejectedActionBatch(
+ batch: PendingInputBatch,
+ responses: readonly InputResponse[],
+): RejectedActionBatch | undefined {
+ if (batch.event === undefined) {
+ return undefined;
+ }
+
+ const responseMap = new Map(responses.map((r) => [r.requestId, r]));
+ const results: RuntimeToolResultActionResult[] = [];
+ for (const request of batch.requests) {
+ if (!isApprovalRequest(request)) {
+ continue;
+ }
+
+ const { approved, reason } = resolveApprovalOutcome(responseMap.get(request.requestId));
+ if (approved) {
+ continue;
+ }
+
+ results.push({
+ callId: request.action.callId,
+ isError: true,
+ kind: "tool-result",
+ output: {
+ code: TOOL_EXECUTION_DENIED_CODE,
+ message: reason ?? TOOL_EXECUTION_DENIED_MESSAGE,
+ },
+ toolName: request.action.toolName,
+ });
+ }
+
+ return results.length > 0 ? { event: batch.event, results } : undefined;
+}
+
function buildToolResponseParts(
batch: PendingInputBatch,
responses: readonly InputResponse[],
@@ -344,8 +428,7 @@ function buildToolResponsePartsForRequest(
response: InputResponse | undefined,
): ToolResponsePart[] {
if (isApprovalRequest(request)) {
- const approved = response?.optionId === "approve";
- const reason = response === undefined ? IGNORED_INPUT_REASON : undefined;
+ const { approved, reason } = resolveApprovalOutcome(response);
const parts: ToolResponsePart[] = [
{
approvalId: request.requestId,
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index cbb008815..28393bdf3 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -31,6 +31,7 @@ import { buildDynamicTools } from "#context/build-dynamic-tools.js";
import { PendingSkillAnnouncementKey } from "#context/dynamic-skill-lifecycle.js";
import { toErrorMessage } from "#shared/errors.js";
import {
+ createActionResultEvent,
createCompactionCompletedEvent,
createCompactionRequestedEvent,
createInputRequestedEvent,
@@ -459,6 +460,24 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
return { next: null, session: pending.session };
}
+ // Surface denied tool-call approvals as rejected `action.result` events.
+ // The denial otherwise lives only in model history, so consumers (e.g.
+ // observability) never see the tool call resolve. Attributed to the turn
+ // that requested approval via the parked batch's emit coordinates.
+ if (emit && pending.rejectedActions) {
+ for (const result of pending.rejectedActions.results) {
+ await emit(
+ createActionResultEvent({
+ rejected: true,
+ result,
+ sequence: pending.rejectedActions.event.sequence,
+ stepIndex: pending.rejectedActions.event.stepIndex,
+ turnId: pending.rejectedActions.event.turnId,
+ }),
+ );
+ }
+ }
+
// --- Turn preamble ------------------------------------------------------
if (emit && hasStepInput(input)) {
@@ -1470,6 +1489,11 @@ async function handleStepResult(input: {
if (inputRequests.length > 0) {
let parkedSession = setPendingInputBatch({
+ event: {
+ sequence: emissionState.sequence,
+ stepIndex: emissionState.stepIndex,
+ turnId: emissionState.turnId,
+ },
requests: inputRequests,
responseMessages,
session: { ...baseSession, history: [...promptMessages] },
@@ -1969,6 +1993,11 @@ async function parkOnCodeModeInterrupt(input: {
});
let parkedSession = setPendingInputBatch({
+ event: {
+ sequence: input.emissionState.sequence,
+ stepIndex: input.emissionState.stepIndex,
+ turnId: input.emissionState.turnId,
+ },
requests: approvalRequests,
responseMessages: approvalMessages,
session: setPendingCodeModeInterrupt({
diff --git a/packages/eve/src/protocol/message.test.ts b/packages/eve/src/protocol/message.test.ts
index 0f0d91800..95f1a633f 100644
--- a/packages/eve/src/protocol/message.test.ts
+++ b/packages/eve/src/protocol/message.test.ts
@@ -14,7 +14,7 @@ import { createEveConnectionCallbackRoutePath } from "#protocol/routes.js";
describe("message stream protocol", () => {
it("pins the stream version for timed session events", () => {
- expect(EVE_MESSAGE_STREAM_VERSION).toBe("15");
+ expect(EVE_MESSAGE_STREAM_VERSION).toBe("16");
});
it("creates result.completed events", () => {
@@ -170,4 +170,26 @@ describe("message stream protocol", () => {
turnId: "turn_0",
});
});
+
+ it("marks denied action results as rejected", () => {
+ const event = createActionResultEvent({
+ rejected: true,
+ result: {
+ callId: "approval-call",
+ isError: true,
+ kind: "tool-result",
+ output: { code: "TOOL_EXECUTION_DENIED", message: "Tool execution was denied." },
+ toolName: "bash",
+ },
+ sequence: 2,
+ stepIndex: 0,
+ turnId: "turn_0",
+ });
+
+ expect(event.data.status).toBe("rejected");
+ expect(event.data.error).toEqual({
+ code: "TOOL_EXECUTION_DENIED",
+ message: "Tool execution was denied.",
+ });
+ });
});
diff --git a/packages/eve/src/protocol/message.ts b/packages/eve/src/protocol/message.ts
index 6064f3046..17215bc7d 100644
--- a/packages/eve/src/protocol/message.ts
+++ b/packages/eve/src/protocol/message.ts
@@ -10,7 +10,7 @@ export const EVE_STREAM_FORMAT_HEADER = "x-eve-stream-format";
export const EVE_STREAM_VERSION_HEADER = "x-eve-stream-version";
export const EVE_MESSAGE_STREAM_CONTENT_TYPE = "application/x-ndjson; charset=utf-8";
export const EVE_MESSAGE_STREAM_FORMAT = "ndjson";
-export const EVE_MESSAGE_STREAM_VERSION = "15";
+export const EVE_MESSAGE_STREAM_VERSION = "16";
/**
* Eve-owned finish reason for one completed assistant step.
@@ -39,8 +39,12 @@ export interface HandleMessageStreamEventMeta {
/**
* Normalized completion status for one emitted runtime action result.
+ *
+ * `rejected` marks a tool call the user (or a policy) denied at a HITL
+ * approval gate: it never executed, so it is neither a success nor a
+ * runtime failure.
*/
-export type ActionResultStatus = "completed" | "failed";
+export type ActionResultStatus = "completed" | "failed" | "rejected";
/**
* Stable failure payload projected onto `action.result`.
@@ -785,14 +789,22 @@ export function createInputRequestedEvent(input: {
/**
* Creates the `action.result` event for one runtime action result.
+ *
+ * Pass `rejected: true` for a tool call denied at a HITL approval gate. The
+ * call never executed, so the outcome is forced to `rejected` rather than
+ * derived from the synthesized denial output.
*/
export function createActionResultEvent(input: {
+ readonly rejected?: boolean;
readonly result: RuntimeActionResult;
readonly sequence: number;
readonly stepIndex: number;
readonly turnId: string;
}): ActionResultStreamEvent {
- const outcome = normalizeActionResultOutcome(input.result);
+ const outcome =
+ input.rejected === true
+ ? { error: buildActionResultError(input.result), status: "rejected" as const }
+ : normalizeActionResultOutcome(input.result);
return {
data: {