Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

14.7 KB
Raw Blame
package cmd

import (
	"path/filepath"
	"reflect"
	"testing"
)

// piTestLog puts a Pi extension run in a temp endpoint log and returns its path.
func piTestLog(t *testing.T) string {
	t.Helper()
	setupHookConfigDirs(t)
	platformFlag = "pi"
	logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
	t.Setenv("BEACON_ENDPOINT_MODE", "1")
	t.Setenv("BEACON_ENDPOINT_LOG", logPath)
	t.Setenv("BEACON_CONTENT_RETENTION", "full")
	return logPath
}

func piEventActions(t *testing.T, logPath string) []string {
	t.Helper()
	var actions []string
	for _, event := range endpointEvents(t, logPath) {
		meta, _ := event["event"].(map[string]interface{})
		actions = append(actions, meta["action"].(string))
	}
	return actions
}

// The tool results are mapped by the code the whole Pi family shares, so Pi's bash output and the
// file it read reach the same fields Oh My Pi's do, under the path as Pi was given it.
func TestPiToolResultsAreRecordedWhereRulesReadThem(t *testing.T) {
	logPath := piTestLog(t)
	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "tool_result", "toolName": "bash", "toolCallId": "call-1", "sessionId": "sess-1",
		"input": map[string]interface{}{"command": "cat notes.md"}, "isError": false,
		"content": ompTextBlocks("Pi shell output marker"),
	})
	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "tool_result", "toolName": "read", "toolCallId": "call-2", "sessionId": "sess-1",
		"input": map[string]interface{}{"path": "notes.md"}, "isError": false,
		"content": ompTextBlocks("Pi file content marker"),
	})

	if got := nested(t, piEventWithAction(t, logPath, "command.executed"), "command")["output"]; got != "Pi shell output marker" {
		t.Fatalf("command.output = %v, want the bash tool's output", got)
	}
	read := piEventWithAction(t, logPath, "file.read")
	if got := nested(t, read, "file")["path"]; got != "notes.md" {
		t.Fatalf("file.path = %v, want the path as Pi was given it", got)
	}
	if !reflect.DeepEqual(fieldsToolCallResult(read), map[string]interface{}{"content": ompTextBlocks("Pi file content marker")}) {
		t.Fatalf("gen_ai.tool.call.result = %v, want the file the model read", fieldsToolCallResult(read))
	}
}

func piEventWithAction(t *testing.T, logPath, action string) map[string]interface{} {
	t.Helper()
	for _, event := range endpointEvents(t, logPath) {
		meta, _ := event["event"].(map[string]interface{})
		if meta["action"] == action {
			return event
		}
	}
	t.Fatalf("no %s event in log; got %v", action, piEventActions(t, logPath))
	return nil
}

// An unrecognized type is silent rather than generic. Pi publishes far more events than the
// extension subscribes to, and a future one becoming an undifferentiated row would fill the log
// with records no query asks for.
func TestPiEventUnknownTypeProducesNothing(t *testing.T) {
	for _, name := range []string{"message_update", "before_provider_request", "turn_start", "", "totally_new"} {
		events := piRuntime.endpointEvents(map[string]interface{}{"type": name}, "sess-1")
		if len(events) != 0 {
			t.Fatalf("Pi event type %q produced %d events, want none", name, len(events))
		}
	}
}

func TestPiEventSessionStartAndShutdown(t *testing.T) {
	logPath := piTestLog(t)

	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "session_start", "reason": "resume", "sessionId": "sess-1", "cwd": "/repo",
	})
	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "session_shutdown", "reason": "quit", "sessionId": "sess-1",
	})

	if got := piEventActions(t, logPath); len(got) != 2 || got[0] != "session.started" || got[1] != "session.ended" {
		t.Fatalf("actions = %v, want [session.started session.ended]", got)
	}
	started := piEventWithAction(t, logPath, "session.started")
	session, _ := started["session"].(map[string]interface{})
	if session["id"] != "sess-1" {
		t.Fatalf("session.id = %v, want sess-1", session["id"])
	}
	// The reason distinguishes a fresh session from one that already has history behind it, which a
	// reader counting sessions needs.
	raw := nested(t, started, "raw")
	if raw["pi_session_reason"] != "resume" {
		t.Fatalf("raw.pi_session_reason = %v, want resume", raw["pi_session_reason"])
	}
}

func TestPiEventInputRecordsPrompt(t *testing.T) {
	logPath := piTestLog(t)

	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "input", "text": "refactor the parser", "source": "interactive",
		"sessionId": "sess-1", "cwd": "/repo",
	})

	event := piEventWithAction(t, logPath, "prompt.submitted")
	prompt := nested(t, event, "prompt")
	if prompt["text"] != "refactor the parser" {
		t.Fatalf("prompt.text = %v, want the submitted text", prompt["text"])
	}
	content := nested(t, event, "content")
	if content["included"] != true || content["hash"] == "" {
		t.Fatalf("content = %v, want retained content with a hash", content)
	}
	// "A human typed this" and "a script sent this" are different facts about the same prompt.
	if raw := nested(t, event, "raw"); raw["pi_input_source"] != "interactive" {
		t.Fatalf("raw.pi_input_source = %v, want interactive", raw["pi_input_source"])
	}
}

// An input event with no text is Pi announcing an empty submission. Recording a prompt event with
// no prompt would put a row in the log that every prompt query matches and none can explain.
func TestPiEventEmptyInputProducesNothing(t *testing.T) {
	events := piRuntime.endpointEvents(map[string]interface{}{"type": "input", "source": "interactive"}, "sess-1")
	if len(events) != 0 {
		t.Fatalf("empty input produced %d events, want none", len(events))
	}
}

// The decision this mapper must not make. Pi's tool_call handler can block a call, but that is an
// extension deciding rather than an operator being asked -- Pi exposes no approval decision through
// the extension API. An approval event here would put a decision nobody made into the log, and
// would be indistinguishable from Claude Code's PermissionRequest, which is a real one.
func TestPiEventNeverSynthesizesApprovals(t *testing.T) {
	logPath := piTestLog(t)

	for _, payload := range []map[string]interface{}{
		{"type": "tool_call", "toolName": "bash", "input": map[string]interface{}{"command": "rm -rf /tmp/x"}, "sessionId": "sess-1"},
		{"type": "tool_result", "toolName": "bash", "input": map[string]interface{}{"command": "rm -rf /tmp/x"}, "sessionId": "sess-1"},
		{"type": "user_bash", "command": "sudo -s", "sessionId": "sess-1"},
	} {
		runHookWithInput(t, runPiEvent, payload)
	}

	for _, event := range endpointEvents(t, logPath) {
		meta, _ := event["event"].(map[string]interface{})
		action, _ := meta["action"].(string)
		if len(action) >= 9 && action[:9] == "approval." {
			t.Fatalf("Pi produced an approval event (%s); Pi exposes no approval decision to observe", action)
		}
		if _, ok := event["approval"]; ok {
			t.Fatalf("Pi event carries an approval block: %v", event)
		}
	}
}

func TestPiEventToolResultActionsPerTool(t *testing.T) {
	for _, tc := range []struct {
		tool   string
		args   map[string]interface{}
		action string
	}{
		{"bash", map[string]interface{}{"command": "ls"}, "command.executed"},
		{"read", map[string]interface{}{"path": "/repo/main.go"}, "file.read"},
		{"edit", map[string]interface{}{"path": "/repo/main.go"}, "file.modified"},
		{"write", map[string]interface{}{"path": "/repo/new.go"}, "file.created"},
		// grep takes a pattern, find takes a glob, ls takes a directory. None is a file operation,
		// and a custom tool's arguments mean whatever its author decided.
		{"grep", map[string]interface{}{"pattern": "TODO"}, "tool.completed"},
		{"glob", map[string]interface{}{"pattern": "**/*.go"}, "tool.completed"},
		{"my_custom_tool", map[string]interface{}{"anything": true}, "tool.completed"},
		// A file action with no file, or a command action with no command, is not that action: it
		// would be a row every file- or command-scoped rule matches and none can explain.
		{"read", map[string]interface{}{}, "tool.completed"},
		{"bash", map[string]interface{}{}, "tool.completed"},
	} {
		t.Run(tc.tool, func(t *testing.T) {
			events := piRuntime.endpointEvents(map[string]interface{}{
				"type": "tool_result", "toolName": tc.tool, "input": tc.args,
			}, "sess-1")
			if len(events) != 1 {
				t.Fatalf("%s produced %d events, want 1", tc.tool, len(events))
			}
			if events[0].action != tc.action {
				t.Fatalf("%s action = %q, want %q", tc.tool, events[0].action, tc.action)
			}
		})
	}
}

func TestPiEventMessageEndRecordsUsageAndReasoning(t *testing.T) {
	logPath := piTestLog(t)

	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "message_end", "sessionId": "sess-1",
		"message": map[string]interface{}{
			"role":  "assistant",
			"model": "claude-opus-5",
			"content": []interface{}{
				map[string]interface{}{"type": "thinking", "thinking": "weighing two approaches"},
				map[string]interface{}{"type": "text", "text": "here is the answer"},
			},
			"usage": map[string]interface{}{
				"input": float64(120), "output": float64(40),
				"cacheRead": float64(90), "cacheWrite": float64(10), "reasoning": float64(25),
				"totalTokens": float64(160),
				"cost":        map[string]interface{}{"total": 0.0042},
			},
		},
	})

	reasoning := piEventWithAction(t, logPath, "agent.reasoning")
	// Only the thinking parts. The assistant's visible answer is not reasoning, and recording it as
	// such would put the model's output where a reader looking for its deliberation expects it.
	parts := nested(t, reasoning, "gen_ai", "output")
	messages, ok := parts["messages"].([]interface{})
	if !ok || len(messages) != 1 {
		t.Fatalf("gen_ai.output.messages = %v, want one reasoning message", parts["messages"])
	}
	first, _ := messages[0].(map[string]interface{})
	firstParts, _ := first["parts"].([]interface{})
	part, _ := firstParts[0].(map[string]interface{})
	if part["type"] != "reasoning" || part["content"] != "weighing two approaches" {
		t.Fatalf("reasoning part = %v, want only the thinking text", part)
	}

	usageEvent := piEventWithAction(t, logPath, "token.usage")
	usage := nested(t, usageEvent, "gen_ai", "usage")
	if usage["input_tokens"] != float64(120) || usage["output_tokens"] != float64(40) {
		t.Fatalf("usage = %v, want the semconv token names", usage)
	}
	// Cache and reasoning are nested objects in the canonical shape, not scalars.
	if got := nested(t, usageEvent, "gen_ai", "usage", "cache_read"); got["input_tokens"] != float64(90) {
		t.Fatalf("cache_read = %v, want input_tokens 90", got)
	}
	if got := nested(t, usageEvent, "gen_ai", "usage", "cache_creation"); got["input_tokens"] != float64(10) {
		t.Fatalf("cache_creation = %v, want input_tokens 10", got)
	}
	if got := nested(t, usageEvent, "gen_ai", "usage", "reasoning"); got["output_tokens"] != float64(25) {
		t.Fatalf("reasoning = %v, want output_tokens 25", got)
	}
	if usage["cost_usd"] != 0.0042 {
		t.Fatalf("cost_usd = %v, want the runtime-reported 0.0042", usage["cost_usd"])
	}
	// Pi reports a totalTokens that can disagree with its own parts, and Beacon's usage shape has no
	// total. Carrying it would add a field nothing reads and something could contradict.
	if _, ok := usage["total_tokens"]; ok {
		t.Fatalf("usage carries total_tokens, which is not part of the canonical shape: %v", usage)
	}
	if _, ok := usage["totalTokens"]; ok {
		t.Fatalf("usage carries Pi's own totalTokens spelling: %v", usage)
	}
}

// Pi's Usage carries cacheWrite1h, the subset of cacheWrite written with one-hour retention, when
// the provider is Anthropic. It is recorded inside cache_creation as the subset it is, clamped to
// cacheWrite, and left absent when the provider does not report it.
func TestPiEventMessageEndRecordsTheOneHourCacheWriteSplit(t *testing.T) {
	cases := []struct {
		name    string
		usage   map[string]interface{}
		oneHour interface{} // nil: absent
	}{
		{"anthropic", map[string]interface{}{"input": float64(3), "output": float64(40), "cacheRead": float64(110000),
			"cacheWrite": float64(20000), "cacheWrite1h": float64(16000), "totalTokens": float64(130043)}, float64(16000)},
		{"clamped", map[string]interface{}{"input": float64(3), "output": float64(40),
			"cacheWrite": float64(500), "cacheWrite1h": float64(900), "totalTokens": float64(543)}, float64(500)},
		{"not reported", map[string]interface{}{"input": float64(3), "output": float64(40),
			"cacheWrite": float64(500), "totalTokens": float64(543)}, nil},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			logPath := piTestLog(t)
			runHookWithInput(t, runPiEvent, map[string]interface{}{
				"type": "message_end", "sessionId": "sess-1",
				"message": map[string]interface{}{"role": "assistant", "model": "claude-opus-4-6", "usage": tc.usage},
			})
			got := nested(t, piEventWithAction(t, logPath, "token.usage"), "gen_ai", "usage", "cache_creation")
			if got["input_tokens"] != tc.usage["cacheWrite"] {
				t.Fatalf("cache_creation = %v, want input_tokens %v", got, tc.usage["cacheWrite"])
			}
			oneHour, present := got["ephemeral_1h_input_tokens"]
			if tc.oneHour == nil {
				if present {
					t.Fatalf("cache_creation = %v, want no one-hour split", got)
				}
				return
			}
			if oneHour != tc.oneHour {
				t.Fatalf("cache_creation = %v, want ephemeral_1h_input_tokens %v", got, tc.oneHour)
			}
		})
	}
}

// The extension lifts Pi's cwd onto the envelope, and an event that carried its own wins. Without
// this the working directory, repository, and branch are missing from every event of a run.
func TestPiEventResolvesWorkspaceFromTheEnvelope(t *testing.T) {
	logPath := piTestLog(t)
	repo := t.TempDir()

	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "input", "text": "hello", "sessionId": "sess-1", "cwd": repo,
	})

	event := piEventWithAction(t, logPath, "prompt.submitted")
	session := nested(t, event, "session")
	if session["working_directory"] != repo {
		t.Fatalf("session.working_directory = %v, want %q", session["working_directory"], repo)
	}
}

// The harness name is normalized at write time, so a Pi session is not split across two spellings
// in any query that groups by harness.name.
func TestPiEventNormalizesTheHarnessName(t *testing.T) {
	logPath := piTestLog(t)

	runHookWithInput(t, runPiEvent, map[string]interface{}{
		"type": "session_start", "reason": "startup", "sessionId": "sess-1",
	})

	event := piEventWithAction(t, logPath, "session.started")
	harness := nested(t, event, "harness")
	if harness["name"] != "pi_cli" {
		t.Fatalf("harness.name = %v, want pi_cli", harness["name"])
	}
}

// A payload with no session id still produces telemetry. Losing the grouping key is bad; losing the
// event is worse, and a run whose accessor failed once should not go dark.
func TestPiEventSurvivesAMissingSessionID(t *testing.T) {
	logPath := piTestLog(t)

	runHookWithInput(t, runPiEvent, map[string]interface{}{"type": "input", "text": "hello"})

	if got := piEventActions(t, logPath); len(got) != 1 || got[0] != "prompt.submitted" {
		t.Fatalf("actions = %v, want [prompt.submitted]", got)
	}
}