Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

feat(omp): carry the decided tool's arguments onto approval events

Claudea6ef5801d71e
diff --git a/cli/beacon-hooks/cmd/omp_approval_test.go b/cli/beacon-hooks/cmd/omp_approval_test.go
index 8fc436b1..b6cb71fa 100644
--- a/cli/beacon-hooks/cmd/omp_approval_test.go
+++ b/cli/beacon-hooks/cmd/omp_approval_test.go
@@ -130,6 +130,67 @@ func TestOmpApprovalJoinsToTheToolCallItDecided(t *testing.T) {
}
}
+// The extension carries the decided call's arguments across from its tool_call, and the mapper has
+// to turn them into the fields a detection reads. Every approval rule Beacon ships matches on
+// `command.command` or `file.path`, never on a tool name, so an approval that resolved to neither
+// would be telemetry no rule could act on.
+func TestOmpApprovalResolvesTheDecidedCommand(t *testing.T) {
+ logPath := ompTestLog(t)
+
+ runHookWithInput(t, runOmpEvent, map[string]interface{}{
+ "type": "tool_approval_resolved", "sessionId": "sess-1",
+ "toolName": "bash", "toolCallId": "call-1", "approved": false,
+ "input": map[string]interface{}{"command": "rm -rf /var/data"},
+ })
+
+ event := ompEventWithAction(t, logPath, "approval.denied")
+ if command := nested(t, event, "command"); command["command"] != "rm -rf /var/data" {
+ t.Fatalf("command.command = %v, want the command the operator denied -- every approval "+
+ "rule matches on this field", command["command"])
+ }
+ if tool := nested(t, event, "tool"); tool["name"] != "bash" || tool["command"] != "rm -rf /var/data" {
+ t.Fatalf("tool = %v, want the bash tool and its command", tool)
+ }
+}
+
+func TestOmpApprovalResolvesTheDecidedFile(t *testing.T) {
+ logPath := ompTestLog(t)
+
+ runHookWithInput(t, runOmpEvent, map[string]interface{}{
+ "type": "tool_approval_requested", "sessionId": "sess-1",
+ "toolName": "write", "toolCallId": "call-1", "approvalMode": "always-ask",
+ "input": map[string]interface{}{"path": "/repo/.github/workflows/release.yml"},
+ })
+
+ event := ompEventWithAction(t, logPath, "approval.requested")
+ file := nested(t, event, "file")
+ if file["path"] != "/repo/.github/workflows/release.yml" || file["operation"] != "create" {
+ t.Fatalf("file = %v, want the path the operator was asked about", file)
+ }
+}
+
+// An approval for a call the extension never saw proposed still records the decision. Losing the
+// arguments is bad; losing the record that an operator was asked at all is worse.
+func TestOmpApprovalWithoutArgumentsStillRecordsTheDecision(t *testing.T) {
+ logPath := ompTestLog(t)
+
+ runHookWithInput(t, runOmpEvent, map[string]interface{}{
+ "type": "tool_approval_resolved", "sessionId": "sess-1",
+ "toolName": "bash", "toolCallId": "call-1", "approved": true,
+ })
+
+ event := ompEventWithAction(t, logPath, "approval.allowed")
+ if approval := nested(t, event, "approval"); approval["decision"] != "approve" {
+ t.Fatalf("approval.decision = %v, want approve", approval["decision"])
+ }
+ if tool := nested(t, event, "tool"); tool["name"] != "bash" {
+ t.Fatalf("tool.name = %v, want bash", tool["name"])
+ }
+ if _, ok := event["command"]; ok {
+ t.Fatalf("an approval with no arguments invented a command block: %v", event["command"])
+ }
+}
+
// Pi must not gain approvals by sharing the mapper. It never sends these events, so the cases are
// unreachable for it -- a stronger guarantee than a platform check, which could be forgotten.
func TestOmpOnlyEventTypesAreNotInPis(t *testing.T) {
diff --git a/cli/beacon-hooks/cmd/pi_family.go b/cli/beacon-hooks/cmd/pi_family.go
index cda6e088..4f03aa6f 100644
--- a/cli/beacon-hooks/cmd/pi_family.go
+++ b/cli/beacon-hooks/cmd/pi_family.go
@@ -169,14 +169,22 @@ func (f piFamily) endpointEvents(input map[string]interface{}, sessionID string)
// a human made. Here the runtime reports a real prompt and a real answer, so the event is
// `observed` rather than `inferred` and carries the operator's decision verbatim.
//
-// The approval carries the tool's name and the runtime's `toolCallId` but not its arguments -- the
-// runtime does not put them on these events. The call id is the join back to the tool.invoked that
-// does carry them, which is why it is promoted here as carefully as on the tool events themselves.
+// The runtime does not put the tool's arguments on these events, so the extension carries them: it
+// remembers the `tool_call` that proposed the call and attaches its `input` to the approval under
+// the same key. That is what makes the approval readable by a detection. Every approval rule Beacon
+// ships matches on `command.command` or `file.path` rather than on a tool name, so an approval that
+// said only "the operator denied bash" would be telemetry no rule could act on.
+//
+// When the arguments are absent -- an approval for a call the extension never saw proposed -- the
+// event still records the decision, the tool name and the call id. The call id is the join back to
+// the tool.invoked that does carry the arguments, which is why it is promoted here as carefully as
+// on the tool events themselves.
func (f piFamily) approvalEvents(input, fields map[string]interface{}, action, decision, messageSuffix string) []normalizedEvent {
toolName := piToolName(input)
if toolName != "" {
- fields["tool"] = mergeNested(fields["tool"], map[string]interface{}{"name": toolName})
- f.applyMCPAttribution(fields, toolName)
+ // toolFields resolves the command, file and MCP blocks from the decided call's arguments
+ // when the extension attached them, and yields just the tool name when it did not.
+ mergeMap(fields, f.toolFields(input, false))
}
approval := map[string]interface{}{"required": true, "decision": decision}
diff --git a/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts b/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts
index 4a2f3fad..b7ad5c8d 100644
--- a/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts
+++ b/cli/beacon/internal/endpoint/hooks/assets/omp/beacon.ts
@@ -75,6 +75,57 @@ const subscribedEvents = [
"message_end",
] as const
+// How many in-flight tool calls the approval enrichment below will remember at once.
+//
+// Bounded because the map is cleared by `tool_result`, and a call that never produces one -- an
+// aborted run, a session torn down mid-tool -- would otherwise leave an entry behind for the life
+// of the process. Oh My Pi runs a handful of tools concurrently at most, so this is far above the
+// working set and small enough that the worst case is a few kilobytes.
+const maxPendingToolCalls = 64
+
+// Arguments of tool calls that have been proposed but not yet finished, keyed by the runtime's own
+// tool call id.
+//
+// This exists for one reason: Oh My Pi's approval events name the tool and the call, but carry none
+// of its arguments. An approval that says "the operator denied `bash`" is far less than one that
+// says "the operator denied `rm -rf /`", and every approval detection Beacon ships matches on the
+// command or file path rather than on the tool name -- so without this, approval telemetry is
+// recorded but no detection can read it.
+//
+// The `tool_call` event carries those arguments and fires before the approval gate, in this same
+// process, so remembering it until `tool_result` is exact: the join is the runtime's own call id,
+// not a timestamp or a guess.
+const pendingToolCalls = new Map<string, unknown>()
+
+function rememberToolCall(event: OmpEvent) {
+ const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
+ if (!id) return
+ // Re-inserting moves the entry to the end of the Map's insertion order, which is what makes the
+ // eviction below drop the genuinely oldest call rather than the least recently seen one.
+ pendingToolCalls.delete(id)
+ pendingToolCalls.set(id, event.input)
+ while (pendingToolCalls.size > maxPendingToolCalls) {
+ const oldest = pendingToolCalls.keys().next()
+ if (oldest.done) break
+ pendingToolCalls.delete(oldest.value)
+ }
+}
+
+function forgetToolCall(event: OmpEvent) {
+ const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
+ if (id) pendingToolCalls.delete(id)
+}
+
+// decidedToolInput returns the arguments of the tool call an approval event is deciding on.
+//
+// Absent rather than guessed when the call is unknown: an approval enriched with some other call's
+// arguments would be worse than one with none, because it would read as evidence.
+function decidedToolInput(event: OmpEvent): unknown {
+ const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
+ if (!id) return undefined
+ return pendingToolCalls.get(id)
+}
+
function debugLog(message: string, extra?: unknown) {
if (!debugEnabled) return
try {
@@ -256,7 +307,26 @@ export function createBeaconExtension() {
// The event is spread last so a field it carries itself wins over the context's. `user_bash`
// and `user_python` both carry their own cwd, and the approval events carry their own
// sessionId -- in each case the event's is the one that describes this action.
- await sendToBeacon({ ...identity(ctx), ...event })
+ const envelope: Record<string, unknown> = { ...identity(ctx), ...event }
+
+ if (event.type === "tool_call") {
+ rememberToolCall(event)
+ } else if (event.type === "tool_result") {
+ forgetToolCall(event)
+ } else if (event.type === "session_start" || event.type === "session_shutdown") {
+ // `/new`, a resume, a fork and a shutdown all end the run these calls belonged to. Clearing
+ // keeps a call abandoned mid-flight from being joined to an approval in a later session.
+ // Done here rather than in a second `pi.on` registration so the whole cache lifecycle is
+ // visible in one place.
+ pendingToolCalls.clear()
+ } else if (event.type === "tool_approval_requested" || event.type === "tool_approval_resolved") {
+ // Attached under `input`, the same key `tool_call` uses, so the mapper reads one shape for
+ // both and an approval resolves to the same command or file path its tool call did.
+ const decided = decidedToolInput(event)
+ if (decided !== undefined) envelope.input = decided
+ }
+
+ await sendToBeacon(envelope)
} catch (err) {
// Unreachable in practice: sendToBeacon swallows its own failures. Kept because a handler
// that rejects is a handler that can break the run it was observing.
diff --git a/plugins/omp-beacon/src/beacon.test.ts b/plugins/omp-beacon/src/beacon.test.ts
index 7dbd302a..a0b6e3bd 100644
--- a/plugins/omp-beacon/src/beacon.test.ts
+++ b/plugins/omp-beacon/src/beacon.test.ts
@@ -300,6 +300,121 @@ describe("beacon oh my pi extension", () => {
}
})
+ // Oh My Pi's approval events name the tool and the call but carry none of its arguments. Every
+ // approval detection Beacon ships matches on the command or the file path rather than on a tool
+ // name, so an approval that said only "the operator denied bash" would be telemetry no rule could
+ // act on. The extension carries the arguments across from the tool_call that proposed the call.
+ test("carries the decided tool's arguments onto an approval", async () => {
+ const sent = captureSends()
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+
+ await omp.fire(
+ { type: "tool_call", toolName: "bash", toolCallId: "call-1", input: { command: "rm -rf /var/data" } },
+ context(),
+ )
+ await omp.fire(
+ { type: "tool_approval_requested", toolName: "bash", toolCallId: "call-1", approvalMode: "always-ask" },
+ context(),
+ )
+ await omp.fire(
+ { type: "tool_approval_resolved", toolName: "bash", toolCallId: "call-1", approved: false },
+ context(),
+ )
+
+ expect(sent[1].input).toEqual({ command: "rm -rf /var/data" })
+ expect(sent[2].input).toEqual({ command: "rm -rf /var/data" })
+ })
+
+ // The join is the runtime's own call id, never a timestamp or a guess. An approval enriched with
+ // some other call's arguments would be worse than one with none, because it would read as
+ // evidence of a decision that was never made about it.
+ test("never attaches another call's arguments to an approval", async () => {
+ const sent = captureSends()
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+
+ await omp.fire(
+ { type: "tool_call", toolName: "bash", toolCallId: "call-1", input: { command: "ls" } },
+ context(),
+ )
+ await omp.fire(
+ { type: "tool_approval_requested", toolName: "write", toolCallId: "call-2" },
+ context(),
+ )
+
+ expect(sent[1].input).toBeUndefined()
+ })
+
+ test("an approval with no call id is not enriched", async () => {
+ const sent = captureSends()
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+
+ await omp.fire(
+ { type: "tool_call", toolName: "bash", toolCallId: "call-1", input: { command: "ls" } },
+ context(),
+ )
+ await omp.fire({ type: "tool_approval_requested", toolName: "bash" }, context())
+
+ expect(sent[1].input).toBeUndefined()
+ })
+
+ // A finished call is forgotten, so a later approval reusing an id cannot inherit stale arguments.
+ test("forgets a tool call once it has finished", async () => {
+ const sent = captureSends()
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+
+ await omp.fire(
+ { type: "tool_call", toolName: "bash", toolCallId: "call-1", input: { command: "ls" } },
+ context(),
+ )
+ await omp.fire({ type: "tool_result", toolName: "bash", toolCallId: "call-1", isError: false }, context())
+ await omp.fire({ type: "tool_approval_requested", toolName: "bash", toolCallId: "call-1" }, context())
+
+ expect(sent[2].input).toBeUndefined()
+ })
+
+ // `/new`, a resume, a fork and a shutdown all end the run a pending call belonged to.
+ test("forgets pending calls when the session ends or restarts", async () => {
+ for (const boundary of ["session_start", "session_shutdown"]) {
+ const sent = captureSends()
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+
+ await omp.fire(
+ { type: "tool_call", toolName: "bash", toolCallId: "call-1", input: { command: "ls" } },
+ context(),
+ )
+ await omp.fire({ type: boundary }, context())
+ await omp.fire({ type: "tool_approval_requested", toolName: "bash", toolCallId: "call-1" }, context())
+
+ expect(sent[2].input).toBeUndefined()
+ }
+ })
+
+ // The cache is bounded, so a run whose tool calls never finish -- an aborted session, a tool torn
+ // down mid-flight -- cannot grow it for the life of the process.
+ test("bounds how many in-flight tool calls it remembers", async () => {
+ const sent = captureSends()
+ const omp = fakeOmp()
+ createBeaconExtension().register(omp.api)
+
+ for (let i = 0; i < 200; i++) {
+ await omp.fire(
+ { type: "tool_call", toolName: "bash", toolCallId: `call-${i}`, input: { command: `echo ${i}` } },
+ context(),
+ )
+ }
+ // The oldest is evicted; the newest is still there.
+ await omp.fire({ type: "tool_approval_requested", toolName: "bash", toolCallId: "call-0" }, context())
+ await omp.fire({ type: "tool_approval_requested", toolName: "bash", toolCallId: "call-199" }, context())
+
+ expect(sent[200].input).toBeUndefined()
+ expect(sent[201].input).toEqual({ command: "echo 199" })
+ })
+
// Event fields win over the lifted identity fields on a key collision, so a value Oh My Pi
// reported is never overwritten by one Beacon derived. user_bash and user_python carry their own
// cwd; the approval events carry their own sessionId.
diff --git a/plugins/omp-beacon/src/beacon.ts b/plugins/omp-beacon/src/beacon.ts
index 4a2f3fad..b7ad5c8d 100644
--- a/plugins/omp-beacon/src/beacon.ts
+++ b/plugins/omp-beacon/src/beacon.ts
@@ -75,6 +75,57 @@ const subscribedEvents = [
"message_end",
] as const
+// How many in-flight tool calls the approval enrichment below will remember at once.
+//
+// Bounded because the map is cleared by `tool_result`, and a call that never produces one -- an
+// aborted run, a session torn down mid-tool -- would otherwise leave an entry behind for the life
+// of the process. Oh My Pi runs a handful of tools concurrently at most, so this is far above the
+// working set and small enough that the worst case is a few kilobytes.
+const maxPendingToolCalls = 64
+
+// Arguments of tool calls that have been proposed but not yet finished, keyed by the runtime's own
+// tool call id.
+//
+// This exists for one reason: Oh My Pi's approval events name the tool and the call, but carry none
+// of its arguments. An approval that says "the operator denied `bash`" is far less than one that
+// says "the operator denied `rm -rf /`", and every approval detection Beacon ships matches on the
+// command or file path rather than on the tool name -- so without this, approval telemetry is
+// recorded but no detection can read it.
+//
+// The `tool_call` event carries those arguments and fires before the approval gate, in this same
+// process, so remembering it until `tool_result` is exact: the join is the runtime's own call id,
+// not a timestamp or a guess.
+const pendingToolCalls = new Map<string, unknown>()
+
+function rememberToolCall(event: OmpEvent) {
+ const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
+ if (!id) return
+ // Re-inserting moves the entry to the end of the Map's insertion order, which is what makes the
+ // eviction below drop the genuinely oldest call rather than the least recently seen one.
+ pendingToolCalls.delete(id)
+ pendingToolCalls.set(id, event.input)
+ while (pendingToolCalls.size > maxPendingToolCalls) {
+ const oldest = pendingToolCalls.keys().next()
+ if (oldest.done) break
+ pendingToolCalls.delete(oldest.value)
+ }
+}
+
+function forgetToolCall(event: OmpEvent) {
+ const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
+ if (id) pendingToolCalls.delete(id)
+}
+
+// decidedToolInput returns the arguments of the tool call an approval event is deciding on.
+//
+// Absent rather than guessed when the call is unknown: an approval enriched with some other call's
+// arguments would be worse than one with none, because it would read as evidence.
+function decidedToolInput(event: OmpEvent): unknown {
+ const id = typeof event.toolCallId === "string" ? event.toolCallId : ""
+ if (!id) return undefined
+ return pendingToolCalls.get(id)
+}
+
function debugLog(message: string, extra?: unknown) {
if (!debugEnabled) return
try {
@@ -256,7 +307,26 @@ export function createBeaconExtension() {
// The event is spread last so a field it carries itself wins over the context's. `user_bash`
// and `user_python` both carry their own cwd, and the approval events carry their own
// sessionId -- in each case the event's is the one that describes this action.
- await sendToBeacon({ ...identity(ctx), ...event })
+ const envelope: Record<string, unknown> = { ...identity(ctx), ...event }
+
+ if (event.type === "tool_call") {
+ rememberToolCall(event)
+ } else if (event.type === "tool_result") {
+ forgetToolCall(event)
+ } else if (event.type === "session_start" || event.type === "session_shutdown") {
+ // `/new`, a resume, a fork and a shutdown all end the run these calls belonged to. Clearing
+ // keeps a call abandoned mid-flight from being joined to an approval in a later session.
+ // Done here rather than in a second `pi.on` registration so the whole cache lifecycle is
+ // visible in one place.
+ pendingToolCalls.clear()
+ } else if (event.type === "tool_approval_requested" || event.type === "tool_approval_resolved") {
+ // Attached under `input`, the same key `tool_call` uses, so the mapper reads one shape for
+ // both and an approval resolves to the same command or file path its tool call did.
+ const decided = decidedToolInput(event)
+ if (decided !== undefined) envelope.input = decided
+ }
+
+ await sendToBeacon(envelope)
} catch (err) {
// Unreachable in practice: sendToBeacon swallows its own failures. Kept because a handler
// that rejects is a handler that can break the run it was observing.