Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

14.0 KB
Raw Blame
package cmd

import "testing"

// The capability Oh My Pi has and Pi does not, and the reason it gets its own mapping rather than
// being folded into the tool events: these are decisions an operator was actually asked to make.
//
// Beacon has always refused to synthesize an approval from a tool call, because a `tool_call`
// handler that blocks is an extension deciding and recording that as an approval would be
// indistinguishable from a human's answer. Here the runtime reports the prompt and the answer, so
// the events are real and are marked `observed`.
func TestOmpApprovalRequestedIsRecorded(t *testing.T) {
	logPath := ompTestLog(t)

	runHookWithInput(t, runOmpEvent, map[string]interface{}{
		"type": "tool_approval_requested", "sessionId": "sess-1",
		"toolName": "bash", "toolCallId": "call-1",
		"reason": "writes outside the workspace", "approvalMode": "always-ask",
	})

	event := ompEventWithAction(t, logPath, "approval.requested")
	approval := nested(t, event, "approval")
	if approval["required"] != true || approval["decision"] != "requested" {
		t.Fatalf("approval = %v, want a required, still-pending decision", approval)
	}
	// The runtime's own words for why, not a Beacon-authored sentence.
	if approval["reason"] != "writes outside the workspace" {
		t.Fatalf("approval.reason = %v, want the runtime's reason", approval["reason"])
	}
	if meta := nested(t, event, "event"); meta["category"] != "approval" {
		t.Fatalf("category = %v, want approval", meta["category"])
	}
	// A decision the operator was genuinely asked for is observed, never inferred.
	if fidelity := nested(t, event, "event")["fidelity"]; fidelity != "observed" {
		t.Fatalf("event.fidelity = %v, want observed", fidelity)
	}
	// Whether the prompts were on at all: "yolo" means nobody was asked.
	if raw := nested(t, event, "raw"); raw["omp_approval_mode"] != "always-ask" {
		t.Fatalf("raw.omp_approval_mode = %v, want always-ask", raw["omp_approval_mode"])
	}
}

func TestOmpApprovalResolvedRecordsTheDecision(t *testing.T) {
	for _, tc := range []struct {
		approved bool
		action   string
		decision string
	}{
		{true, "approval.allowed", "approve"},
		{false, "approval.denied", "deny"},
	} {
		t.Run(tc.action, func(t *testing.T) {
			logPath := ompTestLog(t)

			runHookWithInput(t, runOmpEvent, map[string]interface{}{
				"type": "tool_approval_resolved", "sessionId": "sess-1",
				"toolName": "bash", "toolCallId": "call-1", "approved": tc.approved,
			})

			event := ompEventWithAction(t, logPath, tc.action)
			if approval := nested(t, event, "approval"); approval["decision"] != tc.decision {
				t.Fatalf("approval.decision = %v, want %q", approval["decision"], tc.decision)
			}
			if tool := nested(t, event, "tool"); tool["name"] != "bash" {
				t.Fatalf("tool.name = %v, want bash", tool["name"])
			}
			// No arguments were sent: the decision is still recorded, and no command is invented.
			if _, ok := event["command"]; ok {
				t.Fatalf("an approval with no arguments invented a command block: %v", event["command"])
			}
		})
	}
}

// A resolved approval that carries no `approved` field is a denial, not an allow.
//
// The runtime states the outcome as a boolean, so a missing or non-boolean value means Beacon
// cannot see an approval -- and the safe reading of "no evidence the operator said yes" is not
// "the operator said yes". Defaulting the other way would turn a malformed payload into a clean
// record of consent.
func TestOmpApprovalWithoutAnAffirmativeIsNotAnAllow(t *testing.T) {
	for _, payload := range []map[string]interface{}{
		{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "c1"},
		{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "c1", "approved": "yes"},
		{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "c1", "approved": nil},
	} {
		events := ompRuntime.endpointEvents(payload, "sess-1")
		if len(events) != 1 {
			t.Fatalf("payload %v produced %d events, want 1", payload, len(events))
		}
		if events[0].action != "approval.denied" {
			t.Fatalf("payload %v produced %q; an unreadable outcome must not be recorded as consent",
				payload, events[0].action)
		}
	}
}

// An approval carries no tool arguments, because the runtime does not put them on these events. The
// call id is the join back to the tool.invoked that does carry them -- so an investigation can go
// from "this was approved" to "this is what was approved". Without it the approval names a tool and
// nothing else.
func TestOmpApprovalJoinsToTheToolCallItDecided(t *testing.T) {
	for _, payload := range []map[string]interface{}{
		{"type": "tool_approval_requested", "toolName": "bash", "toolCallId": "call-77"},
		{"type": "tool_approval_resolved", "toolName": "bash", "toolCallId": "call-77", "approved": true},
	} {
		t.Run(payload["type"].(string), func(t *testing.T) {
			events := ompRuntime.endpointEvents(payload, "sess-1")
			if len(events) != 1 {
				t.Fatalf("produced %d events, want 1", len(events))
			}
			if got := piFamilyToolCallID(t, events[0]); got != "call-77" {
				t.Fatalf("gen_ai.tool.call.id = %q, want call-77 -- an approval that cannot be "+
					"joined to its tool call names a tool and nothing else", got)
			}
		})
	}
}

// The extension carries the decided call's arguments across from its tool_call, and the mapper has
// to turn them into the fields a detection reads. Every approval rule Beacon ships matches on
// `command.command` or `file.path`, never on a tool name, so an approval that resolved to neither
// would be telemetry no rule could act on.
func TestOmpApprovalResolvesTheDecidedCommand(t *testing.T) {
	logPath := ompTestLog(t)

	runHookWithInput(t, runOmpEvent, map[string]interface{}{
		"type": "tool_approval_resolved", "sessionId": "sess-1",
		"toolName": "bash", "toolCallId": "call-1", "approved": false,
		"input": map[string]interface{}{"command": "rm -rf /var/data"},
	})

	event := ompEventWithAction(t, logPath, "approval.denied")
	if command := nested(t, event, "command"); command["command"] != "rm -rf /var/data" {
		t.Fatalf("command.command = %v, want the command the operator denied -- every approval "+
			"rule matches on this field", command["command"])
	}
	if tool := nested(t, event, "tool"); tool["name"] != "bash" || tool["command"] != "rm -rf /var/data" {
		t.Fatalf("tool = %v, want the bash tool and its command", tool)
	}
}

func TestOmpApprovalResolvesTheDecidedFile(t *testing.T) {
	logPath := ompTestLog(t)

	runHookWithInput(t, runOmpEvent, map[string]interface{}{
		"type": "tool_approval_requested", "sessionId": "sess-1",
		"toolName": "write", "toolCallId": "call-1", "approvalMode": "always-ask",
		"input": map[string]interface{}{"path": "/repo/.github/workflows/release.yml"},
	})

	event := ompEventWithAction(t, logPath, "approval.requested")
	file := nested(t, event, "file")
	if file["path"] != "/repo/.github/workflows/release.yml" || file["operation"] != "create" {
		t.Fatalf("file = %v, want the path the operator was asked about", file)
	}
}

// Pi must not gain approvals by sharing the mapper. It never sends these events, so the cases are
// unreachable for it -- a stronger guarantee than a platform check, which could be forgotten.
func TestOmpOnlyEventTypesAreNotInPis(t *testing.T) {
	pi := map[string]bool{}
	for _, name := range supportedPiEventTypes() {
		pi[name] = true
	}
	for _, name := range []string{"tool_approval_requested", "tool_approval_resolved", "user_python", "context"} {
		if pi[name] {
			t.Fatalf("Pi's extension subscribes to %q; Pi exposes no such event, and subscribing "+
				"to one would put a decision nobody made into the log", name)
		}
	}
}

// The `$` prefix runs Python in the runtime's own REPL. It is the operator's code, executed as
// literally as a shell command -- os.system("rm -rf /") is a shell command wearing a Python hat --
// so it lands in the command category where the risky-command rules can see it, marked as the
// operator's and as Python rather than passed off as a shell command.
func TestOmpUserPythonIsRecordedAsAnOperatorCommand(t *testing.T) {
	logPath := ompTestLog(t)
	code := "import os; os.system('id')"

	runHookWithInput(t, runOmpEvent, map[string]interface{}{
		"type": "user_python", "code": code, "excludeFromContext": false,
		"cwd": "/repo", "sessionId": "sess-1",
	})

	event := ompEventWithAction(t, logPath, "command.executed")
	if command := nested(t, event, "command"); command["command"] != code {
		t.Fatalf("command.command = %v, want the Python source", command["command"])
	}
	if tool := nested(t, event, "tool"); tool["name"] != "user_python" {
		t.Fatalf("tool.name = %v, want user_python -- it must be distinguishable from a shell "+
			"command that happens to have been run by the operator", tool["name"])
	}
	raw := nested(t, event, "raw")
	if raw["omp_user_initiated"] != true || raw["omp_user_python"] != true {
		t.Fatalf("raw = %v, want it marked operator-initiated and Python", raw)
	}
}

func TestOmpEmptyUserPythonProducesNothing(t *testing.T) {
	events := ompRuntime.endpointEvents(map[string]interface{}{
		"type": "user_python", "cwd": "/repo",
	}, "sess-1")
	if len(events) != 0 {
		t.Fatalf("empty user_python produced %d events, want none", len(events))
	}
}

// MCP attribution. Without it an MCP call lands in the log as a tool named
// `mcp__beacon_managed_beacon_lookup` and nothing else, so the two questions actually asked about
// MCP activity -- which server did this agent reach, and what did it call there -- have no field to
// answer them. The server is the one the runtime names in the result's details: a split of the
// minted name reads this one as server `beacon`, which is Beacon's local stdio server, not the
// hosted `beacon-managed` one the call actually reached.
func TestOmpMCPToolIsAttributedToTheServerTheRuntimeNames(t *testing.T) {
	for _, isError := range []bool{false, true} {
		logPath := ompTestLog(t)

		runHookWithInput(t, runOmpEvent, map[string]interface{}{
			"type": "tool_result", "toolName": "mcp__beacon_managed_beacon_lookup", "toolCallId": "call-1",
			"input": map[string]interface{}{"kind": "harness"}, "sessionId": "sess-1", "isError": isError,
			"details": map[string]interface{}{"serverName": "beacon-managed", "mcpToolName": "beacon_lookup"},
		})

		action, category := "mcp.tool_invoked", "mcp"
		if isError {
			action, category = "tool.failed", "tool"
		}
		event := ompEventWithAction(t, logPath, action)
		if mcp := nested(t, event, "mcp"); mcp["server"] != "beacon-managed" || mcp["tool"] != "beacon_lookup" {
			t.Fatalf("isError=%v: mcp = %v, want the beacon-managed server and its beacon_lookup tool", isError, mcp)
		}
		if meta := nested(t, event, "event"); meta["category"] != category {
			t.Fatalf("isError=%v: category = %v, want %s", isError, meta["category"], category)
		}
	}
}

func TestPiMCPServerToolReadsOnlyWhatIsStated(t *testing.T) {
	runtime := map[string]interface{}{"serverName": "beacon-managed", "mcpToolName": "beacon_lookup"}
	for _, tc := range []struct {
		name         string
		call         piToolCall
		server, tool string
		isMCP        bool
	}{
		{"runtime details", piToolCall{name: "mcp__beacon_managed_beacon_lookup", details: runtime}, "beacon-managed", "beacon_lookup", true},
		{"double-underscore spelling", piToolCall{name: "mcp__github__create_issue"}, "github", "create_issue", true},
		// Oh My Pi's minted spelling cannot be split back into its halves, so it names no server --
		// but it is still MCP activity, which is what the `mcp__` namespace means to the runtime.
		{"minted spelling alone", piToolCall{name: "mcp__beacon_managed_beacon_lookup"}, "", "", true},
		{"minted spelling, simple server", piToolCall{name: "mcp__github_create_issue"}, "", "", true},
		{"bare prefix", piToolCall{name: "mcp__"}, "", "", false},
		{"bash", piToolCall{name: "bash"}, "", "", false},
		{"custom tool", piToolCall{name: "my_custom_tool"}, "", "", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			if server, tool := piMCPServerTool(tc.call); server != tc.server || tool != tc.tool {
				t.Fatalf("piMCPServerTool = (%q, %q), want (%q, %q)", server, tool, tc.server, tc.tool)
			}
			if got := piIsMCPTool(tc.call); got != tc.isMCP {
				t.Fatalf("piIsMCPTool = %v, want %v", got, tc.isMCP)
			}
		})
	}
}

// The tool_call and the approval for an MCP call carry the minted name and nothing else. They must
// not guess a server from it: a wrong mcp.server files the call under a server it never reached,
// where an absent one leaves the call's result -- joined by gen_ai.tool.call.id -- to say.
func TestOmpMCPNameAloneNamesNoServer(t *testing.T) {
	for _, payload := range []map[string]interface{}{
		{"type": "tool_call", "toolName": "mcp__beacon_managed_beacon_lookup", "toolCallId": "call-1",
			"input": map[string]interface{}{"kind": "harness"}},
		{"type": "tool_approval_resolved", "toolName": "mcp__beacon_managed_beacon_lookup", "toolCallId": "call-1",
			"approved": true},
	} {
		events := ompRuntime.endpointEvents(payload, "sess-1")
		if len(events) != 1 {
			t.Fatalf("%s produced %d events, want 1", payload["type"], len(events))
		}
		if mcp, ok := events[0].fields["mcp"]; ok {
			t.Fatalf("%s guessed an mcp block from the tool name: %v", payload["type"], mcp)
		}
		if tool, _ := events[0].fields["tool"].(map[string]interface{}); tool["name"] != "mcp__beacon_managed_beacon_lookup" {
			t.Fatalf("%s tool = %v, want the tool name kept", payload["type"], tool)
		}
	}
}

// A non-MCP tool must not grow an mcp block. A row with an mcp block is one every MCP-scoped query
// matches, and a bash call is not MCP activity.
func TestOmpNonMCPToolsHaveNoMCPBlock(t *testing.T) {
	for _, tool := range []string{"bash", "read"} {
		t.Run(tool, func(t *testing.T) {
			events := ompRuntime.endpointEvents(map[string]interface{}{
				"type": "tool_result", "toolName": tool,
				"input": map[string]interface{}{"command": "ls", "path": "/repo/a.go"},
			}, "sess-1")
			if len(events) != 1 {
				t.Fatalf("produced %d events, want 1", len(events))
			}
			if _, ok := events[0].fields["mcp"]; ok {
				t.Fatalf("%s grew an mcp block: %v", tool, events[0].fields["mcp"])
			}
			if events[0].action == "mcp.tool_invoked" {
				t.Fatalf("%s was recorded as MCP activity", tool)
			}
		})
	}
}