Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Stop approving Devin CLI permission requests on the operator's behalf

Claude99276dcc97bb
diff --git a/cli/beacon-hooks/cmd/devin_permission_test.go b/cli/beacon-hooks/cmd/devin_permission_test.go
new file mode 100644
index 00000000..7bdc1a35
--- /dev/null
+++ b/cli/beacon-hooks/cmd/devin_permission_test.go
@@ -0,0 +1,132 @@
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// The security property behind sending Devin CLI's PermissionRequest down the default path.
+//
+// Devin reads `{"decision":"approve"}` from a PermissionRequest hook as the operator's yes, so a
+// telemetry hook that printed it would grant every permission Devin was about to ask a person
+// for -- including for the command below -- on a runtime where Beacon was installed only to watch.
+// The empty object carries no decision and leaves Devin's own prompt in charge. Both platform
+// spellings the installer writes are covered, because isDevinLikePlatform routes them together.
+func TestDevinPermissionRequestDoesNotApproveOnBehalfOfTheUser(t *testing.T) {
+ for _, platform := range []string{"devin", "devin-cli"} {
+ t.Run(platform, func(t *testing.T) {
+ setupHookConfigDirs(t)
+ platformFlag = platform
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ t.Setenv("BEACON_ENDPOINT_MODE", "1")
+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)
+
+ out := runHookWithInput(t, runPermissionRequest, map[string]interface{}{
+ "session_id": "devin-session-1",
+ "cwd": "/tmp/devin-project",
+ "hook_event_name": "PermissionRequest",
+ "tool_name": "exec",
+ "tool_input": map[string]interface{}{"command": "rm -rf ~"},
+ })
+ if len(out) != 0 {
+ t.Fatalf("permission response = %#v, want an empty object carrying no decision", out)
+ }
+ if _, ok := out["decision"]; ok {
+ t.Fatalf("permission response carried a decision: %#v", out)
+ }
+
+ // Observed as a request, not recorded as an approval nobody gave.
+ event := lastEndpointEvent(t, logPath)
+ eventFields := event["event"].(map[string]interface{})
+ if action := eventFields["action"]; action != "approval.requested" {
+ t.Fatalf("event.action = %q, want approval.requested", action)
+ }
+ approval := event["approval"].(map[string]interface{})
+ if approval["decision"] != "requested" {
+ t.Fatalf("approval = %#v, want decision=requested", approval)
+ }
+ if got := event["command"].(map[string]interface{})["command"]; got != "rm -rf ~" {
+ t.Fatalf("command = %q, want the requested command recorded", got)
+ }
+ })
+ }
+}
+
+// A payload Beacon cannot parse is no grounds for answering on the operator's behalf either. Before
+// this was fixed, unreadable stdin was the one input on which Devin got an unconditional approve.
+func TestDevinPermissionRequestDoesNotApproveMalformedInput(t *testing.T) {
+ for _, platform := range []string{"devin", "devin-cli"} {
+ t.Run(platform, func(t *testing.T) {
+ setupHookConfigDirs(t)
+ platformFlag = platform
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ t.Setenv("BEACON_ENDPOINT_MODE", "1")
+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)
+
+ out := runPermissionRequestWithRawStdin(t, []byte(`{"tool_name": "exec", "tool_input": `))
+ if len(bytes.TrimSpace(out)) != 0 {
+ var decoded map[string]interface{}
+ if err := json.Unmarshal(out, &decoded); err != nil {
+ t.Fatalf("permission response %q is not JSON: %v", out, err)
+ }
+ if len(decoded) != 0 {
+ t.Fatalf("permission response = %#v, want an empty object carrying no decision", decoded)
+ }
+ }
+
+ // Nothing parsed, so nothing is recorded -- and certainly not an approval.
+ if data, err := os.ReadFile(logPath); err == nil && len(bytes.TrimSpace(data)) != 0 {
+ for _, event := range endpointEvents(t, logPath) {
+ if action := event["event"].(map[string]interface{})["action"]; action == "approval.allowed" {
+ t.Fatalf("malformed input was logged as approval.allowed: %#v", event)
+ }
+ }
+ }
+ })
+ }
+}
+
+// runPermissionRequestWithRawStdin feeds bytes that are not a JSON object, which runHookWithInput
+// cannot express because it encodes its input. The payloads here are small, so writing before the
+// hook runs fits in any pipe buffer.
+func runPermissionRequestWithRawStdin(t *testing.T, stdin []byte) []byte {
+ t.Helper()
+ stdinR, stdinW, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("create stdin pipe: %v", err)
+ }
+ stdoutR, stdoutW, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("create stdout pipe: %v", err)
+ }
+ origStdin, origStdout := os.Stdin, os.Stdout
+ os.Stdin, os.Stdout = stdinR, stdoutW
+ defer func() {
+ os.Stdin, os.Stdout = origStdin, origStdout
+ _ = stdinR.Close()
+ _ = stdoutR.Close()
+ }()
+
+ if _, err := stdinW.Write(stdin); err != nil {
+ t.Fatalf("write stdin: %v", err)
+ }
+ if err := stdinW.Close(); err != nil {
+ t.Fatalf("close stdin writer: %v", err)
+ }
+ done := make(chan []byte, 1)
+ go func() {
+ data, _ := io.ReadAll(stdoutR)
+ done <- data
+ }()
+
+ runPermissionRequest(nil, nil)
+
+ if err := stdoutW.Close(); err != nil {
+ t.Fatalf("close stdout writer: %v", err)
+ }
+ return <-done
+}
diff --git a/cli/beacon-hooks/cmd/permission_request.go b/cli/beacon-hooks/cmd/permission_request.go
index 37f61757..ac1164bc 100644
--- a/cli/beacon-hooks/cmd/permission_request.go
+++ b/cli/beacon-hooks/cmd/permission_request.go
@@ -19,15 +19,12 @@ func init() {
rootCmd.AddCommand(permissionRequestCmd)
}
-var devinApproveResponse = map[string]interface{}{"decision": "approve"}
-
func runPermissionRequest(cmd *cobra.Command, args []string) {
input, err := readStdinJSON()
if err != nil {
- if isDevinLikePlatform(platformFlag) {
- outputJSON(devinApproveResponse)
- return
- }
+ // No runtime gets an answer on a payload Beacon could not read, Devin CLI included. A hook
+ // that cannot say what is being asked has no business answering it, and the empty object
+ // leaves the runtime's own prompt in charge.
outputJSON(emptyResponse)
return
}
@@ -40,11 +37,17 @@ func runPermissionRequest(cmd *cobra.Command, args []string) {
return
}
- if isDevinLikePlatform(platformFlag) {
- emitPreToolDecision(logger, input, sessionID, "approval.allowed", "approve", "Permission request approved", asymptoteobserve.FidelityObserved)
- outputJSON(devinApproveResponse)
- return
- }
+ // Devin CLI takes the default path at the bottom, for the Qwen reason in preToolResponse.
+ // Devin reads `{"decision":"approve"}` from a PermissionRequest hook as the answer to the
+ // question it was about to put to the operator, so a hook that printed it would not be
+ // observing the request -- it would be granting it, for every tool call that needed a person's
+ // yes, on a runtime where the hook was installed to watch. Devin's own documented example
+ // prints a decision only for the calls it means to approve and exits with no output for the
+ // rest, which is what the empty object says here: no opinion, ask the operator as usual. What
+ // Beacon saw is a request, so that is what it records; Devin does not report the operator's
+ // answer to a hook, and Beacon does not invent one. A policy provider's deny above is still a
+ // real answer and still reaches Devin. TestDevinPermissionRequestDoesNotApproveOnBehalfOfTheUser
+ // holds this in place.
if platformFlag == "hermes" {
emitHermesApproval(logger, input, sessionID)
outputJSON(emptyResponse)
diff --git a/cli/beacon-hooks/cmd/pre_tool_test.go b/cli/beacon-hooks/cmd/pre_tool_test.go
index 77505eb0..33faa3cc 100644
--- a/cli/beacon-hooks/cmd/pre_tool_test.go
+++ b/cli/beacon-hooks/cmd/pre_tool_test.go
@@ -584,7 +584,7 @@ func TestRunPreToolEmitsDevinToolTelemetryWithoutApproval(t *testing.T) {
}
}
-func TestRunPermissionRequestApprovesDevinCLI(t *testing.T) {
+func TestRunPermissionRequestObservesDevinCLI(t *testing.T) {
setupHookConfigDirs(t)
platformFlag = "devin-cli"
logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
@@ -596,16 +596,16 @@ func TestRunPermissionRequestApprovesDevinCLI(t *testing.T) {
"command": "git status",
},
})
- if out["decision"] != "approve" {
- t.Fatalf("devin-cli permission response = %#v, want approve", out)
+ if len(out) != 0 {
+ t.Fatalf("devin-cli permission response = %#v, want an empty object carrying no decision", out)
}
event := lastEndpointEvent(t, logPath)
if harness := event["harness"].(map[string]interface{})["name"]; harness != "devin-cli" {
t.Fatalf("harness = %q, want devin-cli", harness)
}
- if action := event["event"].(map[string]interface{})["action"]; action != "approval.allowed" {
- t.Fatalf("event.action = %q, want approval.allowed", action)
+ if action := event["event"].(map[string]interface{})["action"]; action != "approval.requested" {
+ t.Fatalf("event.action = %q, want approval.requested", action)
}
}
@@ -769,7 +769,7 @@ func TestRunPreToolRetriesAntigravityPromptUntilTranscriptExists(t *testing.T) {
}
}
-func TestRunPermissionRequestEmitsDevinApproval(t *testing.T) {
+func TestRunPermissionRequestEmitsDevinApprovalRequest(t *testing.T) {
setupHookConfigDirs(t)
platformFlag = "devin"
logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
@@ -782,17 +782,17 @@ func TestRunPermissionRequestEmitsDevinApproval(t *testing.T) {
"command": "git status",
},
})
- if out["decision"] != "approve" {
- t.Fatalf("devin permission response = %#v, want decision=approve", out)
+ if len(out) != 0 {
+ t.Fatalf("devin permission response = %#v, want an empty object carrying no decision", out)
}
event := lastEndpointEvent(t, logPath)
- if action := event["event"].(map[string]interface{})["action"]; action != "approval.allowed" {
- t.Fatalf("event.action = %q, want approval.allowed", action)
+ if action := event["event"].(map[string]interface{})["action"]; action != "approval.requested" {
+ t.Fatalf("event.action = %q, want approval.requested", action)
}
approval := event["approval"].(map[string]interface{})
- if approval["decision"] != "approve" {
- t.Fatalf("approval = %#v, want approve", approval)
+ if approval["decision"] != "requested" || approval["required"] != true {
+ t.Fatalf("approval = %#v, want a required request with no decision", approval)
}
}
diff --git a/docs/runtimes/devin.mdx b/docs/runtimes/devin.mdx
index ed8ca80a..162d9fab 100644
--- a/docs/runtimes/devin.mdx
+++ b/docs/runtimes/devin.mdx
@@ -47,6 +47,7 @@ beacon endpoint hooks install --harness devin-cli --level project
|------|---------|
| Prompt telemetry | Supported through `UserPromptSubmit` hooks |
| Command, tool, and file telemetry | Supported for session, prompt, pre-tool, post-tool, permission request, stop, session-end, approval, and file telemetry where Devin CLI exposes payloads |
+| Approvals | Permission requests are recorded as `approval.requested`. Devin CLI does not report the operator's answer to hooks, so Beacon does not record one, and the hook returns no decision: Devin asks the operator as it would without Beacon |
| Local JSONL and dashboard | Supported |
| MDM deployment | Supported for the endpoint agent. Devin CLI hooks are installed separately in the logged-in user's context or project context |