The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(fx): recover per-turn tokens and the command output handle
Claudece3151b7dcfe
diff --git a/cli/beacon/internal/fxsession/mapper.go b/cli/beacon/internal/fxsession/mapper.goindex c12bb233..65aecd01 100644--- a/cli/beacon/internal/fxsession/mapper.go+++ b/cli/beacon/internal/fxsession/mapper.go@@ -94,6 +94,13 @@ type mapper struct {// lastUsage is the previous cumulative usage snapshot, kept so a checkpoint can be emitted as// the difference from the one before it. See usageEvent.lastUsage *UsageSnapshot+ // lastTotals is the session's cumulative token count as of the previous committed turn, used+ // to derive a turn's own usage on fx builds that write no turn summary. See emitTurnUsage.+ lastTotals struct {+ input int64+ output int64+ seen bool+ }// turnIndex counts committed turns, so a dedup id can name which turn an event came from even// though fx numbers turns only within its own history.turnIndex int@@ -599,36 +606,75 @@ func unifiedDiff(file *CommittedFile) string {// Only input and output tokens come from here. Cost and cache tokens exist only in fx's cumulative// usage snapshots and are emitted from those as deltas, in fields this event leaves unset -- so the// two sources add up rather than overlapping.+//+// Two sources for the per-turn counts, because fx has shipped both shapes. A build that writes a+// turn summary is read from it, flags and durations included. A build that writes none -- fx 0.0.7+// omits the summary entirely, which is how this was found: a real session through the released+// binary produced no token usage at all -- falls back to the difference between this turn's+// cumulative session totals and the previous turn's. Same discipline as the checkpoint deltas, and+// the same reason: the alternative is either no usage or a session's whole history counted once per+// turn. raw.fx.token_source says which was used, because an exact per-turn count and a difference+// of two running totals are not equally trustworthy.func (m *mapper) emitTurnUsage(event *Event, turn Turn) {- if turn.Execution == nil || turn.Execution.TurnSummary == nil {- return+ usage := &schema.GenAIUsageInfo{}+ raw := map[string]interface{}{}++ summary := (*TurnSummary)(nil)+ if turn.Execution != nil {+ summary = turn.Execution.TurnSummary}- progress := turn.Execution.TurnSummary.TokenProgress- if progress.InputTokens <= 0 && progress.OutputTokens <= 0 {+ switch {+ case summary != nil && (summary.TokenProgress.InputTokens > 0 || summary.TokenProgress.OutputTokens > 0):+ progress := summary.TokenProgress+ if progress.InputTokens > 0 {+ input := progress.InputTokens+ usage.InputTokens = &input+ }+ if progress.OutputTokens > 0 {+ output := progress.OutputTokens+ usage.OutputTokens = &output+ }+ // fx says when a count is its own estimate rather than the provider's number. Carrying the+ // flags is what keeps a cost report from presenting an estimate as a measurement.+ raw["token_source"] = "turn_summary"+ raw["input_exact"] = progress.InputExact+ raw["output_exact"] = progress.OutputExact+ raw["turn_duration_ms"] = summary.TurnDurationMS+ raw["thinking_duration_ms"] = summary.ThinkingDurationMS+ default:+ input := delta(int64(event.TurnCommitted.TotalInputTokens), m.lastTotals.input)+ output := delta(int64(event.TurnCommitted.TotalOutputTokens), m.lastTotals.output)+ if input > 0 {+ usage.InputTokens = &input+ }+ if output > 0 {+ usage.OutputTokens = &output+ }+ raw["token_source"] = "session_totals_delta"+ }+ m.recordTotals(event.TurnCommitted)++ if usage.InputTokens == nil && usage.OutputTokens == nil {return}ev := m.base(event, "token.usage", "metric", schema.SeverityInfo, schema.FidelityObserved, "fx token usage")- usage := &schema.GenAIUsageInfo{}- if progress.InputTokens > 0 {- input := progress.InputTokens- usage.InputTokens = &input- }- if progress.OutputTokens > 0 {- output := progress.OutputTokens- usage.OutputTokens = &output- }ev.GenAI = m.withGenAI(ev.GenAI, func(genAI *schema.GenAIInfo) { genAI.Usage = usage })- // fx says when a count is its own estimate rather than the provider's number. Carrying the- // flags is what keeps a cost report from presenting an estimate as a measurement.- ev.Raw = mergeRaw(ev.Raw, map[string]interface{}{- "input_exact": progress.InputExact,- "output_exact": progress.OutputExact,- "turn_duration_ms": turn.Execution.TurnSummary.TurnDurationMS,- "thinking_duration_ms": turn.Execution.TurnSummary.ThinkingDurationMS,- })+ ev.Raw = mergeRaw(ev.Raw, raw)m.append(event, fmt.Sprintf("turn.%d.usage", m.turnIndex), ev)}+// recordTotals advances the cumulative baseline the fallback above measures against. It runs for+// every committed turn, including ones this sweep is not emitting, so a resumed sweep measures the+// first turn it emits against the turn before it rather than against zero.+func (m *mapper) recordTotals(committed *TurnCommitted) {+ if committed == nil {+ return+ }+ m.lastTotals.input = int64(committed.TotalInputTokens)+ m.lastTotals.output = int64(committed.TotalOutputTokens)+ m.lastTotals.seen = true+}+// emitUsage reports a usage checkpoint as the difference from the previous one.//// fx's snapshots are cumulative replacements: each one restates the session's whole usage. Emitting@@ -738,6 +784,13 @@ func toolResultRaw(result ToolResult) map[string]interface{} {"truncated": result.Truncated,"provider_native": result.ProviderNative,}+ if result.CommandOutputReplay != nil && result.CommandOutputReplay.Handle != "" {+ // The full captured output of a command is not in the record: fx keeps it in a framed+ // replay file beside the session and puts the handle here. Beacon cannot read that framing,+ // so it carries the handle rather than pretending the stored text is the whole output.+ raw["command_output_handle"] = result.CommandOutputReplay.Handle.String()+ raw["command_output_framed_bytes"] = result.CommandOutputReplay.FramedBytes+ }if result.CommandProcess != nil {raw["process_outcome"] = result.CommandProcess.Kindif result.CommandProcess.Value != nil {diff --git a/cli/beacon/internal/fxsession/realsession_test.go b/cli/beacon/internal/fxsession/realsession_test.gonew file mode 100644index 00000000..40357fef--- /dev/null+++ b/cli/beacon/internal/fxsession/realsession_test.go@@ -0,0 +1,177 @@+package fxsession++import (+ "path/filepath"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+)++// This test runs against a session record fx actually wrote.+//+// testdata/fx-0.0.7-session holds the events.jsonl and session.json left behind by the released fx+// 0.0.7 binary driving a real turn -- a command that exits 3, a file read, a file write -- against a+// local fake gateway. The only edit is the workspace path, rewritten to /home/dev/api so the fixture+// does not carry the machine it was captured on (and the manifest's byte watermark adjusted to+// match, since the rewrite changed the file's length).+//+// It exists because hand-transcribing another project's format is exactly the kind of work that+// looks right and is wrong, and because this capture found two real defects the transcribed+// fixtures could not: fx 0.0.7 writes no turn summary at all, so every per-turn token count was+// being dropped, and a command's full output lives in a framed replay file whose handle the mapper+// was discarding.+func TestRealFxSessionRecordCollectsEndToEnd(t *testing.T) {+ dir := filepath.Join("testdata", "fx-0.0.7-session")+ store := &Store{Dir: dir}+ refs, err := store.List()+ if err != nil {+ t.Fatalf("List: %v", err)+ }+ if len(refs) != 1 {+ t.Fatalf("expected one recorded session, got %d", len(refs))+ }+ ref := refs[0]+ if ref.Manifest == nil {+ t.Fatal("the recorded session's manifest was not readable")+ }++ events, stats, err := store.Read(ref)+ if err != nil {+ t.Fatalf("Read: %v", err)+ }+ if stats.Malformed != 0 {+ t.Fatalf("%d malformed lines in a log fx wrote: %v", stats.Malformed, stats.FirstError)+ }+ // fx writes far more storage bookkeeping than agent activity: recovery checkpoints and usage+ // snapshots outnumber the one committed turn roughly twenty to one. Skipping them by kind+ // rather than emitting them is what keeps the runtime log readable.+ if stats.Skipped == 0 {+ t.Error("no storage-only records were skipped, which is not what an fx log looks like")+ }++ mapped := MapSession(ref, events, MapOptions{})+ got := make([]string, 0, len(mapped))+ for _, item := range mapped {+ got = append(got, item.Event.Event.Action)+ }+ want := []string{+ "session.started",+ "prompt.submitted",+ "command.executed",+ "file.read",+ "file.created",+ "agent.message",+ "token.usage",+ }+ if strings.Join(got, ",") != strings.Join(want, ",") {+ t.Fatalf("actions = %v, want %v", got, want)+ }+}++// The command in the recorded session exited 3 through fx's `terminal` tool, which is the tool the+// released binary actually runs commands with -- `run_command` is rejected as unsupported there.+// Both names are classified as commands for that reason.+func TestRealFxCommandCarriesItsExitCodeAndOutputHandle(t *testing.T) {+ mapped := mapRealSession(t)+ command := findMapped(t, mapped, "command.executed")++ if command.Tool == nil || command.Tool.Name != "terminal" {+ t.Fatalf("tool = %+v, want fx's terminal tool", command.Tool)+ }+ if command.Command == nil || command.Command.Command != "echo beacon-verify && exit 3" {+ t.Fatalf("command = %+v", command.Command)+ }+ if command.Command.ExitCode == nil || *command.Command.ExitCode != 3 {+ t.Fatalf("exit code = %+v, want 3", command.Command.ExitCode)+ }+ // fx stores the full captured output in a framed file beside the session and leaves a handle+ // in the record. Beacon carries the handle rather than presenting the stored excerpt as the+ // whole output.+ raw := rawFx(t, command)+ if handle, _ := raw["command_output_handle"].(string); !strings.HasPrefix(handle, "fx-command-replay-") {+ t.Errorf("raw.fx.command_output_handle = %v, want fx's replay handle", raw["command_output_handle"])+ }+}++// fx 0.0.7 writes no turn summary, so the per-turn token counts have to come from the difference in+// the session's cumulative totals. Without the fallback this session reported no token usage at+// all, which is how the gap was found.+func TestRealFxSessionReportsTokenUsageWithoutATurnSummary(t *testing.T) {+ mapped := mapRealSession(t)+ usage := findMapped(t, mapped, "token.usage")++ if usage.GenAI == nil || usage.GenAI.Usage == nil {+ t.Fatal("no gen_ai.usage on the usage event")+ }+ if usage.GenAI.Usage.InputTokens == nil || *usage.GenAI.Usage.InputTokens != 1234 {+ t.Errorf("input tokens = %+v, want the 1234 the provider reported", usage.GenAI.Usage.InputTokens)+ }+ if usage.GenAI.Usage.OutputTokens == nil || *usage.GenAI.Usage.OutputTokens != 56 {+ t.Errorf("output tokens = %+v, want 56", usage.GenAI.Usage.OutputTokens)+ }+ // Which source the numbers came from is on the event, because a difference of two running+ // totals and an exact per-turn count are not equally trustworthy.+ if source := rawFx(t, usage)["token_source"]; source != "session_totals_delta" {+ t.Errorf("raw.fx.token_source = %v, want session_totals_delta", source)+ }+}++// The write in the recorded session produced a real committed diff, which is the field a reviewer+// and a sensitive-edit rule both read.+func TestRealFxWriteCarriesItsDiff(t *testing.T) {+ mapped := mapRealSession(t)+ created := findMapped(t, mapped, "file.created")++ if created.File == nil || !strings.HasSuffix(created.File.Path, "/health.ts") {+ t.Fatalf("file = %+v", created.File)+ }+ if !strings.Contains(created.File.Diff, "+export function health()") {+ t.Fatalf("diff does not carry the added line:\n%s", created.File.Diff)+ }+ if created.File.Operation != "create" {+ t.Errorf("operation = %q, want create", created.File.Operation)+ }+ if created.File.Language != "ts" {+ t.Errorf("language = %q, want ts", created.File.Language)+ }+}++// fx resolves the model from its own catalog rather than from what the request asked for, and the+// session record is where that resolution is visible. Every event from the session carries it.+func TestRealFxSessionCarriesTheResolvedModelAndProvider(t *testing.T) {+ mapped := mapRealSession(t)+ for _, item := range mapped {+ if item.Event.Model == "" {+ t.Errorf("%s carries no model", item.Event.Event.Action)+ }+ if item.Event.GenAI == nil || item.Event.GenAI.Provider == nil || item.Event.GenAI.Provider.Name != "gateway" {+ t.Errorf("%s provider = %+v, want gateway", item.Event.Event.Action, item.Event.GenAI)+ }+ }+}++func mapRealSession(t *testing.T) []MappedEvent {+ t.Helper()+ store := &Store{Dir: filepath.Join("testdata", "fx-0.0.7-session")}+ refs, err := store.List()+ if err != nil || len(refs) != 1 {+ t.Fatalf("List: %v (%d sessions)", err, len(refs))+ }+ events, _, err := store.Read(refs[0])+ if err != nil {+ t.Fatalf("Read: %v", err)+ }+ return MapSession(refs[0], events, MapOptions{})+}++func findMapped(t *testing.T, mapped []MappedEvent, action string) schema.Event {+ t.Helper()+ for _, item := range mapped {+ if item.Event.Event.Action == action {+ return item.Event+ }+ }+ t.Fatalf("no %q event in the recorded session", action)+ return schema.Event{}+}diff --git a/cli/beacon/internal/fxsession/testdata/fx-0.0.7-session/1788051418424-1788051418424884928-a177316124162631/session.json b/cli/beacon/internal/fxsession/testdata/fx-0.0.7-session/1788051418424-1788051418424884928-a177316124162631/session.jsonnew file mode 100644index 00000000..a3ef1247--- /dev/null+++ b/cli/beacon/internal/fxsession/testdata/fx-0.0.7-session/1788051418424-1788051418424884928-a177316124162631/session.json@@ -0,0 +1,28 @@+{+ "schema_version": 3,+ "storage_format": "event_log_v1",+ "id": "1788051418424-1788051418424884928-a177316124162631",+ "authority_id": "08f69f0df4ac45e82079810bd03449a4",+ "log_generation": "3b4c72027563ff60190f0299461ee01c",+ "created_at_ms": 1788051418424,+ "updated_at_ms": 1788051418741,+ "origin_workspace_root": "/home/dev/api",+ "workspace_root": "/home/dev/api",+ "conversation_language": "und-Latn",+ "history_len": 1,+ "total_input_tokens": 1234,+ "total_output_tokens": 56,+ "last_event_seq": 26,+ "event_log_bytes": 34379,+ "event_log_stat_fingerprint": "23fcca61e5a55cc64ad35e6c0760e9f5638555dc5a806970998f44d7ed794e61",+ "generation_base_seq": 1,+ "generation_base_bytes": 1054,+ "checkpoint_seq": 2,+ "checkpoint_sha256": "9575eb1719ae531296bc95b1b16e8e40ec684e31ad4478a4327e1ee7602e0250",+ "preferences": {+ "model": "moonshotai/kimi-k3",+ "effort": "auto",+ "fast_mode": false,+ "provider": "gateway"+ }+}diff --git a/cli/beacon/internal/fxsession/types.go b/cli/beacon/internal/fxsession/types.goindex 975c83a4..24cd71a3 100644--- a/cli/beacon/internal/fxsession/types.go+++ b/cli/beacon/internal/fxsession/types.go@@ -231,21 +231,31 @@ type ToolCall struct {// three are carried through to Beacon rather than collapsed, because a reader who cannot tell a// short output from a truncated one cannot tell what the agent actually saw.type ToolResult struct {- ToolCallID DurableString `json:"tool_call_id"`- ToolName DurableString `json:"tool_name"`- Status string `json:"status"`- Output DurableString `json:"output"`- OutputHandle *string `json:"output_handle"`- Preview *string `json:"preview"`- OutputBytes int64 `json:"output_bytes"`- StoredOutputBytes int64 `json:"stored_output_bytes"`- Truncated bool `json:"truncated"`- ProviderNative bool `json:"provider_native"`- CreatedAtMS int64 `json:"created_at_ms"`- PermissionFeedback []DurableString `json:"permission_feedback"`- CommittedFile *CommittedFile `json:"committed_file_presentation"`- CommandProcess *ProcessOutcome `json:"command_process_presentation"`- TerminalAction *TerminalOutcome `json:"terminal_action_presentation"`+ ToolCallID DurableString `json:"tool_call_id"`+ ToolName DurableString `json:"tool_name"`+ Status string `json:"status"`+ Output DurableString `json:"output"`+ OutputHandle *string `json:"output_handle"`+ Preview *string `json:"preview"`+ OutputBytes int64 `json:"output_bytes"`+ StoredOutputBytes int64 `json:"stored_output_bytes"`+ Truncated bool `json:"truncated"`+ ProviderNative bool `json:"provider_native"`+ CreatedAtMS int64 `json:"created_at_ms"`+ PermissionFeedback []DurableString `json:"permission_feedback"`+ CommittedFile *CommittedFile `json:"committed_file_presentation"`+ CommandOutputReplay *CommandReplay `json:"command_output_replay"`+ CommandProcess *ProcessOutcome `json:"command_process_presentation"`+ TerminalAction *TerminalOutcome `json:"terminal_action_presentation"`+}++// CommandReplay is fx's pointer to a command's full captured output, which it stores in a framed+// file beside the session rather than in the record. Kind is "available" with a handle, or+// "unavailable". Beacon records the handle and does not try to read the frame format.+type CommandReplay struct {+ Kind string `json:"kind"`+ Handle DurableString `json:"handle"`+ FramedBytes int64 `json:"framed_bytes"`}// CommittedFile is the diff fx committed for a write or edit: the path, whether the file was added