The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Stop approving Devin CLI permission requests on the operator's behalf
Claude99276dcc97bb
diff --git a/cli/beacon-hooks/cmd/devin_permission_test.go b/cli/beacon-hooks/cmd/devin_permission_test.gonew file mode 100644index 00000000..7bdc1a35--- /dev/null+++ b/cli/beacon-hooks/cmd/devin_permission_test.go@@ -0,0 +1,132 @@+package cmd++import (+ "bytes"+ "encoding/json"+ "io"+ "os"+ "path/filepath"+ "testing"+)++// The security property behind sending Devin CLI's PermissionRequest down the default path.+//+// Devin reads `{"decision":"approve"}` from a PermissionRequest hook as the operator's yes, so a+// telemetry hook that printed it would grant every permission Devin was about to ask a person+// for -- including for the command below -- on a runtime where Beacon was installed only to watch.+// The empty object carries no decision and leaves Devin's own prompt in charge. Both platform+// spellings the installer writes are covered, because isDevinLikePlatform routes them together.+func TestDevinPermissionRequestDoesNotApproveOnBehalfOfTheUser(t *testing.T) {+ for _, platform := range []string{"devin", "devin-cli"} {+ t.Run(platform, func(t *testing.T) {+ setupHookConfigDirs(t)+ platformFlag = platform+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")+ t.Setenv("BEACON_ENDPOINT_MODE", "1")+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)++ out := runHookWithInput(t, runPermissionRequest, map[string]interface{}{+ "session_id": "devin-session-1",+ "cwd": "/tmp/devin-project",+ "hook_event_name": "PermissionRequest",+ "tool_name": "exec",+ "tool_input": map[string]interface{}{"command": "rm -rf ~"},+ })+ if len(out) != 0 {+ t.Fatalf("permission response = %#v, want an empty object carrying no decision", out)+ }+ if _, ok := out["decision"]; ok {+ t.Fatalf("permission response carried a decision: %#v", out)+ }++ // Observed as a request, not recorded as an approval nobody gave.+ event := lastEndpointEvent(t, logPath)+ eventFields := event["event"].(map[string]interface{})+ if action := eventFields["action"]; action != "approval.requested" {+ t.Fatalf("event.action = %q, want approval.requested", action)+ }+ approval := event["approval"].(map[string]interface{})+ if approval["decision"] != "requested" {+ t.Fatalf("approval = %#v, want decision=requested", approval)+ }+ if got := event["command"].(map[string]interface{})["command"]; got != "rm -rf ~" {+ t.Fatalf("command = %q, want the requested command recorded", got)+ }+ })+ }+}++// A payload Beacon cannot parse is no grounds for answering on the operator's behalf either. Before+// this was fixed, unreadable stdin was the one input on which Devin got an unconditional approve.+func TestDevinPermissionRequestDoesNotApproveMalformedInput(t *testing.T) {+ for _, platform := range []string{"devin", "devin-cli"} {+ t.Run(platform, func(t *testing.T) {+ setupHookConfigDirs(t)+ platformFlag = platform+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")+ t.Setenv("BEACON_ENDPOINT_MODE", "1")+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)++ out := runPermissionRequestWithRawStdin(t, []byte(`{"tool_name": "exec", "tool_input": `))+ if len(bytes.TrimSpace(out)) != 0 {+ var decoded map[string]interface{}+ if err := json.Unmarshal(out, &decoded); err != nil {+ t.Fatalf("permission response %q is not JSON: %v", out, err)+ }+ if len(decoded) != 0 {+ t.Fatalf("permission response = %#v, want an empty object carrying no decision", decoded)+ }+ }++ // Nothing parsed, so nothing is recorded -- and certainly not an approval.+ if data, err := os.ReadFile(logPath); err == nil && len(bytes.TrimSpace(data)) != 0 {+ for _, event := range endpointEvents(t, logPath) {+ if action := event["event"].(map[string]interface{})["action"]; action == "approval.allowed" {+ t.Fatalf("malformed input was logged as approval.allowed: %#v", event)+ }+ }+ }+ })+ }+}++// runPermissionRequestWithRawStdin feeds bytes that are not a JSON object, which runHookWithInput+// cannot express because it encodes its input. The payloads here are small, so writing before the+// hook runs fits in any pipe buffer.+func runPermissionRequestWithRawStdin(t *testing.T, stdin []byte) []byte {+ t.Helper()+ stdinR, stdinW, err := os.Pipe()+ if err != nil {+ t.Fatalf("create stdin pipe: %v", err)+ }+ stdoutR, stdoutW, err := os.Pipe()+ if err != nil {+ t.Fatalf("create stdout pipe: %v", err)+ }+ origStdin, origStdout := os.Stdin, os.Stdout+ os.Stdin, os.Stdout = stdinR, stdoutW+ defer func() {+ os.Stdin, os.Stdout = origStdin, origStdout+ _ = stdinR.Close()+ _ = stdoutR.Close()+ }()++ if _, err := stdinW.Write(stdin); err != nil {+ t.Fatalf("write stdin: %v", err)+ }+ if err := stdinW.Close(); err != nil {+ t.Fatalf("close stdin writer: %v", err)+ }+ done := make(chan []byte, 1)+ go func() {+ data, _ := io.ReadAll(stdoutR)+ done <- data+ }()++ runPermissionRequest(nil, nil)++ if err := stdoutW.Close(); err != nil {+ t.Fatalf("close stdout writer: %v", err)+ }+ return <-done+}diff --git a/cli/beacon-hooks/cmd/permission_request.go b/cli/beacon-hooks/cmd/permission_request.goindex 37f61757..ac1164bc 100644--- a/cli/beacon-hooks/cmd/permission_request.go+++ b/cli/beacon-hooks/cmd/permission_request.go@@ -19,15 +19,12 @@ func init() {rootCmd.AddCommand(permissionRequestCmd)}-var devinApproveResponse = map[string]interface{}{"decision": "approve"}-func runPermissionRequest(cmd *cobra.Command, args []string) {input, err := readStdinJSON()if err != nil {- if isDevinLikePlatform(platformFlag) {- outputJSON(devinApproveResponse)- return- }+ // No runtime gets an answer on a payload Beacon could not read, Devin CLI included. A hook+ // that cannot say what is being asked has no business answering it, and the empty object+ // leaves the runtime's own prompt in charge.outputJSON(emptyResponse)return}@@ -40,11 +37,17 @@ func runPermissionRequest(cmd *cobra.Command, args []string) {return}- if isDevinLikePlatform(platformFlag) {- emitPreToolDecision(logger, input, sessionID, "approval.allowed", "approve", "Permission request approved", asymptoteobserve.FidelityObserved)- outputJSON(devinApproveResponse)- return- }+ // Devin CLI takes the default path at the bottom, for the Qwen reason in preToolResponse.+ // Devin reads `{"decision":"approve"}` from a PermissionRequest hook as the answer to the+ // question it was about to put to the operator, so a hook that printed it would not be+ // observing the request -- it would be granting it, for every tool call that needed a person's+ // yes, on a runtime where the hook was installed to watch. Devin's own documented example+ // prints a decision only for the calls it means to approve and exits with no output for the+ // rest, which is what the empty object says here: no opinion, ask the operator as usual. What+ // Beacon saw is a request, so that is what it records; Devin does not report the operator's+ // answer to a hook, and Beacon does not invent one. A policy provider's deny above is still a+ // real answer and still reaches Devin. TestDevinPermissionRequestDoesNotApproveOnBehalfOfTheUser+ // holds this in place.if platformFlag == "hermes" {emitHermesApproval(logger, input, sessionID)outputJSON(emptyResponse)diff --git a/cli/beacon-hooks/cmd/pre_tool_test.go b/cli/beacon-hooks/cmd/pre_tool_test.goindex 77505eb0..33faa3cc 100644--- a/cli/beacon-hooks/cmd/pre_tool_test.go+++ b/cli/beacon-hooks/cmd/pre_tool_test.go@@ -584,7 +584,7 @@ func TestRunPreToolEmitsDevinToolTelemetryWithoutApproval(t *testing.T) {}}-func TestRunPermissionRequestApprovesDevinCLI(t *testing.T) {+func TestRunPermissionRequestObservesDevinCLI(t *testing.T) {setupHookConfigDirs(t)platformFlag = "devin-cli"logPath := filepath.Join(t.TempDir(), "runtime.jsonl")@@ -596,16 +596,16 @@ func TestRunPermissionRequestApprovesDevinCLI(t *testing.T) {"command": "git status",},})- if out["decision"] != "approve" {- t.Fatalf("devin-cli permission response = %#v, want approve", out)+ if len(out) != 0 {+ t.Fatalf("devin-cli permission response = %#v, want an empty object carrying no decision", out)}event := lastEndpointEvent(t, logPath)if harness := event["harness"].(map[string]interface{})["name"]; harness != "devin-cli" {t.Fatalf("harness = %q, want devin-cli", harness)}- if action := event["event"].(map[string]interface{})["action"]; action != "approval.allowed" {- t.Fatalf("event.action = %q, want approval.allowed", action)+ if action := event["event"].(map[string]interface{})["action"]; action != "approval.requested" {+ t.Fatalf("event.action = %q, want approval.requested", action)}}@@ -769,7 +769,7 @@ func TestRunPreToolRetriesAntigravityPromptUntilTranscriptExists(t *testing.T) {}}-func TestRunPermissionRequestEmitsDevinApproval(t *testing.T) {+func TestRunPermissionRequestEmitsDevinApprovalRequest(t *testing.T) {setupHookConfigDirs(t)platformFlag = "devin"logPath := filepath.Join(t.TempDir(), "runtime.jsonl")@@ -782,17 +782,17 @@ func TestRunPermissionRequestEmitsDevinApproval(t *testing.T) {"command": "git status",},})- if out["decision"] != "approve" {- t.Fatalf("devin permission response = %#v, want decision=approve", out)+ if len(out) != 0 {+ t.Fatalf("devin permission response = %#v, want an empty object carrying no decision", out)}event := lastEndpointEvent(t, logPath)- if action := event["event"].(map[string]interface{})["action"]; action != "approval.allowed" {- t.Fatalf("event.action = %q, want approval.allowed", action)+ if action := event["event"].(map[string]interface{})["action"]; action != "approval.requested" {+ t.Fatalf("event.action = %q, want approval.requested", action)}approval := event["approval"].(map[string]interface{})- if approval["decision"] != "approve" {- t.Fatalf("approval = %#v, want approve", approval)+ if approval["decision"] != "requested" || approval["required"] != true {+ t.Fatalf("approval = %#v, want a required request with no decision", approval)}}diff --git a/docs/runtimes/devin.mdx b/docs/runtimes/devin.mdxindex ed8ca80a..162d9fab 100644--- a/docs/runtimes/devin.mdx+++ b/docs/runtimes/devin.mdx@@ -47,6 +47,7 @@ beacon endpoint hooks install --harness devin-cli --level project|------|---------|| Prompt telemetry | Supported through `UserPromptSubmit` hooks || Command, tool, and file telemetry | Supported for session, prompt, pre-tool, post-tool, permission request, stop, session-end, approval, and file telemetry where Devin CLI exposes payloads |+| Approvals | Permission requests are recorded as `approval.requested`. Devin CLI does not report the operator's answer to hooks, so Beacon does not record one, and the hook returns no decision: Devin asks the operator as it would without Beacon || Local JSONL and dashboard | Supported || MDM deployment | Supported for the endpoint agent. Devin CLI hooks are installed separately in the logged-in user's context or project context |