The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package cmd
import "testing"
// The capability Oh My Pi has and Pi does not, and the reason it gets its own mapping rather than
// being folded into the tool events: these are decisions an operator was actually asked to make.
//
// Beacon has always refused to synthesize an approval from a tool call, because a `tool_call`
// handler that blocks is an extension deciding and recording that as an approval would be
// indistinguishable from a human's answer. Here the runtime reports the prompt and the answer, so
// the events are real and are marked `observed`.
func TestOmpApprovalRequestedIsRecorded(t *testing.T) {
logPath := ompTestLog(t)
runHookWithInput(t, runOmpEvent, map[string]interface{}{
"type": "tool_approval_requested", "sessionId": "sess-1",
"toolName": "bash", "toolCallId": "call-1",
"reason": "writes outside the workspace", "approvalMode": "always-ask",
})
event := ompEventWithAction(t, logPath, "approval.requested")
approval := nested(t, event, "approval")
if approval["required"] != true || approval["decision"] != "requested" {
t.Fatalf("approval = %v, want a required, still-pending decision", approval)
}
// The runtime's own words for why, not a Beacon-authored sentence.
if approval["reason"] != "writes outside the workspace" {
t.Fatalf("approval.reason = %v, want the runtime's reason", approval["reason"])
}
if meta := nested(t, event, "event"); meta["category"] != "approval" {
t.Fatalf("category = %v, want approval", meta["category"])
}
// A decision the operator was genuinely asked for is observed, never inferred.
if fidelity := nested(t, event, "event")["fidelity"]; fidelity != "observed" {
t.Fatalf("event.fidelity = %v, want observed", fidelity)
}
// Whether the prompts were on at all: "yolo" means nobody was asked.
if raw := nested(t, event, "raw"); raw["omp_approval_mode"] != "always-ask" {
t.Fatalf("raw.omp_approval_mode = %v, want always-ask", raw["omp_approval_mode"])
}
}
func TestOmpApprovalResolvedRecordsTheDecision(t *testing.T) {
for _, tc := range []struct {
approved bool
action string
decision string
}{
{true, "approval.allowed", "approve"},
{false, "approval.denied", "deny"},
} {
t.Run(tc.action, func(t *testing.T) {
logPath := ompTestLog(t)
runHookWithInput(t, runOmpEvent, map[string]interface{}{
"type": "tool_approval_resolved", "sessionId": "sess-1",
"toolName": "bash", "toolCallId": "call-1", "approved": tc.approved,
})
event := ompEventWithAction(t, logPath, tc.action)
if approval := nested(t, event, "approval"); approval["decision"] != tc.decision {
t.Fatalf("approval.decision = %v, want %q", approval["decision"], tc.decision)
}
if tool := nested(t, event, "tool"); tool["name"] != "bash" {
t.Fatalf("tool.name = %v, want bash", tool["name"])
}
// No arguments were sent: the decision is still recorded, and no command is invented.
if _, ok := event["command"]; ok {
t.Fatalf("an approval with no arguments invented a command block: %v", event["command"])
}
})
}
}
// A resolved approval that carries no `approved` field is a denial, not an allow.
//
// The runtime states the outcome as a boolean, so a missing or non-boolean value means Beacon
// cannot see an approval -- and the safe reading of "no evidence the operator said yes" is not
// "the operator said yes". Defaulting the other way would turn a malformed payload into a clean
// record of consent.
func TestOmpApprovalWithoutAnAffirmativeIsNotAnAllow(t *testing.T) {
for _, payload := range []map[string]interface{}{
{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "c1"},
{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "c1", "approved": "yes"},
{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "c1", "approved": nil},
} {
events := ompRuntime.endpointEvents(payload, "sess-1")
if len(events) != 1 {
t.Fatalf("payload %v produced %d events, want 1", payload, len(events))
}
if events[0].action != "approval.denied" {
t.Fatalf("payload %v produced %q; an unreadable outcome must not be recorded as consent",
payload, events[0].action)
}
}
}
// An approval carries no tool arguments, because the runtime does not put them on these events. The
// call id is the join back to the tool.invoked that does carry them -- so an investigation can go
// from "this was approved" to "this is what was approved". Without it the approval names a tool and
// nothing else.
func TestOmpApprovalJoinsToTheToolCallItDecided(t *testing.T) {
for _, payload := range []map[string]interface{}{
{"type": "tool_approval_requested", "toolName": "bash", "toolCallId": "call-77"},
{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "call-77", "approved": true},
} {
t.Run(payload["type"].(string), func(t *testing.T) {
events := ompRuntime.endpointEvents(payload, "sess-1")
if len(events) != 1 {
t.Fatalf("produced %d events, want 1", len(events))
}
if got := piFamilyToolCallID(t, events[0]); got != "call-77" {
t.Fatalf("gen_ai.tool.call.id = %q, want call-77 -- an approval that cannot be "+
"joined to its tool call names a tool and nothing else", got)
}
})
}
}
// The extension carries the decided call's arguments across from its tool_call, and the mapper has
// to turn them into the fields a detection reads. Every approval rule Beacon ships matches on
// `command.command` or `file.path`, never on a tool name, so an approval that resolved to neither
// would be telemetry no rule could act on.
func TestOmpApprovalResolvesTheDecidedCommand(t *testing.T) {
logPath := ompTestLog(t)
runHookWithInput(t, runOmpEvent, map[string]interface{}{
"type": "tool_approval_resolved", "sessionId": "sess-1",
"toolName": "bash", "toolCallId": "call-1", "approved": false,
"input": map[string]interface{}{"command": "rm -rf /var/data"},
})
event := ompEventWithAction(t, logPath, "approval.denied")
if command := nested(t, event, "command"); command["command"] != "rm -rf /var/data" {
t.Fatalf("command.command = %v, want the command the operator denied -- every approval "+
"rule matches on this field", command["command"])
}
if tool := nested(t, event, "tool"); tool["name"] != "bash" || tool["command"] != "rm -rf /var/data" {
t.Fatalf("tool = %v, want the bash tool and its command", tool)
}
}
func TestOmpApprovalResolvesTheDecidedFile(t *testing.T) {
logPath := ompTestLog(t)
runHookWithInput(t, runOmpEvent, map[string]interface{}{
"type": "tool_approval_requested", "sessionId": "sess-1",
"toolName": "write", "toolCallId": "call-1", "approvalMode": "always-ask",
"input": map[string]interface{}{"path": "/repo/.github/workflows/release.yml"},
})
event := ompEventWithAction(t, logPath, "approval.requested")
file := nested(t, event, "file")
if file["path"] != "/repo/.github/workflows/release.yml" || file["operation"] != "create" {
t.Fatalf("file = %v, want the path the operator was asked about", file)
}
}
// Pi must not gain approvals by sharing the mapper. It never sends these events, so the cases are
// unreachable for it -- a stronger guarantee than a platform check, which could be forgotten.
func TestOmpOnlyEventTypesAreNotInPis(t *testing.T) {
pi := map[string]bool{}
for _, name := range supportedPiEventTypes() {
pi[name] = true
}
for _, name := range []string{"tool_approval_requested", "tool_approval_resolved", "user_python", "context"} {
if pi[name] {
t.Fatalf("Pi's extension subscribes to %q; Pi exposes no such event, and subscribing "+
"to one would put a decision nobody made into the log", name)
}
}
}
// The `$` prefix runs Python in the runtime's own REPL. It is the operator's code, executed as
// literally as a shell command -- os.system("rm -rf /") is a shell command wearing a Python hat --
// so it lands in the command category where the risky-command rules can see it, marked as the
// operator's and as Python rather than passed off as a shell command.
func TestOmpUserPythonIsRecordedAsAnOperatorCommand(t *testing.T) {
logPath := ompTestLog(t)
code := "import os; os.system('id')"
runHookWithInput(t, runOmpEvent, map[string]interface{}{
"type": "user_python", "code": code, "excludeFromContext": false,
"cwd": "/repo", "sessionId": "sess-1",
})
event := ompEventWithAction(t, logPath, "command.executed")
if command := nested(t, event, "command"); command["command"] != code {
t.Fatalf("command.command = %v, want the Python source", command["command"])
}
if tool := nested(t, event, "tool"); tool["name"] != "user_python" {
t.Fatalf("tool.name = %v, want user_python -- it must be distinguishable from a shell "+
"command that happens to have been run by the operator", tool["name"])
}
raw := nested(t, event, "raw")
if raw["omp_user_initiated"] != true || raw["omp_user_python"] != true {
t.Fatalf("raw = %v, want it marked operator-initiated and Python", raw)
}
}
func TestOmpEmptyUserPythonProducesNothing(t *testing.T) {
events := ompRuntime.endpointEvents(map[string]interface{}{
"type": "user_python", "cwd": "/repo",
}, "sess-1")
if len(events) != 0 {
t.Fatalf("empty user_python produced %d events, want none", len(events))
}
}
// MCP attribution. Without it an MCP call lands in the log as a tool named
// `mcp__beacon_managed_beacon_lookup` and nothing else, so the two questions actually asked about
// MCP activity -- which server did this agent reach, and what did it call there -- have no field to
// answer them. The server is the one the runtime names in the result's details: a split of the
// minted name reads this one as server `beacon`, which is Beacon's local stdio server, not the
// hosted `beacon-managed` one the call actually reached.
func TestOmpMCPToolIsAttributedToTheServerTheRuntimeNames(t *testing.T) {
for _, isError := range []bool{false, true} {
logPath := ompTestLog(t)
runHookWithInput(t, runOmpEvent, map[string]interface{}{
"type": "tool_result", "toolName": "mcp__beacon_managed_beacon_lookup", "toolCallId": "call-1",
"input": map[string]interface{}{"kind": "harness"}, "sessionId": "sess-1", "isError": isError,
"details": map[string]interface{}{"serverName": "beacon-managed", "mcpToolName": "beacon_lookup"},
})
action, category := "mcp.tool_invoked", "mcp"
if isError {
action, category = "tool.failed", "tool"
}
event := ompEventWithAction(t, logPath, action)
if mcp := nested(t, event, "mcp"); mcp["server"] != "beacon-managed" || mcp["tool"] != "beacon_lookup" {
t.Fatalf("isError=%v: mcp = %v, want the beacon-managed server and its beacon_lookup tool", isError, mcp)
}
if meta := nested(t, event, "event"); meta["category"] != category {
t.Fatalf("isError=%v: category = %v, want %s", isError, meta["category"], category)
}
}
}
func TestPiMCPServerToolReadsOnlyWhatIsStated(t *testing.T) {
runtime := map[string]interface{}{"serverName": "beacon-managed", "mcpToolName": "beacon_lookup"}
for _, tc := range []struct {
name string
call piToolCall
server, tool string
isMCP bool
}{
{"runtime details", piToolCall{name: "mcp__beacon_managed_beacon_lookup", details: runtime}, "beacon-managed", "beacon_lookup", true},
{"double-underscore spelling", piToolCall{name: "mcp__github__create_issue"}, "github", "create_issue", true},
// Oh My Pi's minted spelling cannot be split back into its halves, so it names no server --
// but it is still MCP activity, which is what the `mcp__` namespace means to the runtime.
{"minted spelling alone", piToolCall{name: "mcp__beacon_managed_beacon_lookup"}, "", "", true},
{"minted spelling, simple server", piToolCall{name: "mcp__github_create_issue"}, "", "", true},
{"bare prefix", piToolCall{name: "mcp__"}, "", "", false},
{"bash", piToolCall{name: "bash"}, "", "", false},
{"custom tool", piToolCall{name: "my_custom_tool"}, "", "", false},
} {
t.Run(tc.name, func(t *testing.T) {
if server, tool := piMCPServerTool(tc.call); server != tc.server || tool != tc.tool {
t.Fatalf("piMCPServerTool = (%q, %q), want (%q, %q)", server, tool, tc.server, tc.tool)
}
if got := piIsMCPTool(tc.call); got != tc.isMCP {
t.Fatalf("piIsMCPTool = %v, want %v", got, tc.isMCP)
}
})
}
}
// The tool_call and the approval for an MCP call carry the minted name and nothing else. They must
// not guess a server from it: a wrong mcp.server files the call under a server it never reached,
// where an absent one leaves the call's result -- joined by gen_ai.tool.call.id -- to say.
func TestOmpMCPNameAloneNamesNoServer(t *testing.T) {
for _, payload := range []map[string]interface{}{
{"type": "tool_call", "toolName": "mcp__beacon_managed_beacon_lookup", "toolCallId": "call-1",
"input": map[string]interface{}{"kind": "harness"}},
{"type": "tool_approval_resolved", "toolName": "mcp__beacon_managed_beacon_lookup", "toolCallId": "call-1",
"approved": true},
} {
events := ompRuntime.endpointEvents(payload, "sess-1")
if len(events) != 1 {
t.Fatalf("%s produced %d events, want 1", payload["type"], len(events))
}
if mcp, ok := events[0].fields["mcp"]; ok {
t.Fatalf("%s guessed an mcp block from the tool name: %v", payload["type"], mcp)
}
if tool, _ := events[0].fields["tool"].(map[string]interface{}); tool["name"] != "mcp__beacon_managed_beacon_lookup" {
t.Fatalf("%s tool = %v, want the tool name kept", payload["type"], tool)
}
}
}
// A non-MCP tool must not grow an mcp block. A row with an mcp block is one every MCP-scoped query
// matches, and a bash call is not MCP activity.
func TestOmpNonMCPToolsHaveNoMCPBlock(t *testing.T) {
for _, tool := range []string{"bash", "read"} {
t.Run(tool, func(t *testing.T) {
events := ompRuntime.endpointEvents(map[string]interface{}{
"type": "tool_result", "toolName": tool,
"input": map[string]interface{}{"command": "ls", "path": "/repo/a.go"},
}, "sess-1")
if len(events) != 1 {
t.Fatalf("produced %d events, want 1", len(events))
}
if _, ok := events[0].fields["mcp"]; ok {
t.Fatalf("%s grew an mcp block: %v", tool, events[0].fields["mcp"])
}
if events[0].action == "mcp.tool_invoked" {
t.Fatalf("%s was recorded as MCP activity", tool)
}
})
}
}