Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(fx): recover per-turn tokens and the command output handle

Claudece3151b7dcfe
diff --git a/cli/beacon/internal/fxsession/mapper.go b/cli/beacon/internal/fxsession/mapper.go
index c12bb233..65aecd01 100644
--- a/cli/beacon/internal/fxsession/mapper.go
+++ b/cli/beacon/internal/fxsession/mapper.go
@@ -94,6 +94,13 @@ type mapper struct {
// lastUsage is the previous cumulative usage snapshot, kept so a checkpoint can be emitted as
// the difference from the one before it. See usageEvent.
lastUsage *UsageSnapshot
+ // lastTotals is the session's cumulative token count as of the previous committed turn, used
+ // to derive a turn's own usage on fx builds that write no turn summary. See emitTurnUsage.
+ lastTotals struct {
+ input int64
+ output int64
+ seen bool
+ }
// turnIndex counts committed turns, so a dedup id can name which turn an event came from even
// though fx numbers turns only within its own history.
turnIndex int
@@ -599,36 +606,75 @@ func unifiedDiff(file *CommittedFile) string {
// Only input and output tokens come from here. Cost and cache tokens exist only in fx's cumulative
// usage snapshots and are emitted from those as deltas, in fields this event leaves unset -- so the
// two sources add up rather than overlapping.
+//
+// Two sources for the per-turn counts, because fx has shipped both shapes. A build that writes a
+// turn summary is read from it, flags and durations included. A build that writes none -- fx 0.0.7
+// omits the summary entirely, which is how this was found: a real session through the released
+// binary produced no token usage at all -- falls back to the difference between this turn's
+// cumulative session totals and the previous turn's. Same discipline as the checkpoint deltas, and
+// the same reason: the alternative is either no usage or a session's whole history counted once per
+// turn. raw.fx.token_source says which was used, because an exact per-turn count and a difference
+// of two running totals are not equally trustworthy.
func (m *mapper) emitTurnUsage(event *Event, turn Turn) {
- if turn.Execution == nil || turn.Execution.TurnSummary == nil {
- return
+ usage := &schema.GenAIUsageInfo{}
+ raw := map[string]interface{}{}
+
+ summary := (*TurnSummary)(nil)
+ if turn.Execution != nil {
+ summary = turn.Execution.TurnSummary
}
- progress := turn.Execution.TurnSummary.TokenProgress
- if progress.InputTokens <= 0 && progress.OutputTokens <= 0 {
+ switch {
+ case summary != nil && (summary.TokenProgress.InputTokens > 0 || summary.TokenProgress.OutputTokens > 0):
+ progress := summary.TokenProgress
+ if progress.InputTokens > 0 {
+ input := progress.InputTokens
+ usage.InputTokens = &input
+ }
+ if progress.OutputTokens > 0 {
+ output := progress.OutputTokens
+ usage.OutputTokens = &output
+ }
+ // fx says when a count is its own estimate rather than the provider's number. Carrying the
+ // flags is what keeps a cost report from presenting an estimate as a measurement.
+ raw["token_source"] = "turn_summary"
+ raw["input_exact"] = progress.InputExact
+ raw["output_exact"] = progress.OutputExact
+ raw["turn_duration_ms"] = summary.TurnDurationMS
+ raw["thinking_duration_ms"] = summary.ThinkingDurationMS
+ default:
+ input := delta(int64(event.TurnCommitted.TotalInputTokens), m.lastTotals.input)
+ output := delta(int64(event.TurnCommitted.TotalOutputTokens), m.lastTotals.output)
+ if input > 0 {
+ usage.InputTokens = &input
+ }
+ if output > 0 {
+ usage.OutputTokens = &output
+ }
+ raw["token_source"] = "session_totals_delta"
+ }
+ m.recordTotals(event.TurnCommitted)
+
+ if usage.InputTokens == nil && usage.OutputTokens == nil {
return
}
ev := m.base(event, "token.usage", "metric", schema.SeverityInfo, schema.FidelityObserved, "fx token usage")
- usage := &schema.GenAIUsageInfo{}
- if progress.InputTokens > 0 {
- input := progress.InputTokens
- usage.InputTokens = &input
- }
- if progress.OutputTokens > 0 {
- output := progress.OutputTokens
- usage.OutputTokens = &output
- }
ev.GenAI = m.withGenAI(ev.GenAI, func(genAI *schema.GenAIInfo) { genAI.Usage = usage })
- // fx says when a count is its own estimate rather than the provider's number. Carrying the
- // flags is what keeps a cost report from presenting an estimate as a measurement.
- ev.Raw = mergeRaw(ev.Raw, map[string]interface{}{
- "input_exact": progress.InputExact,
- "output_exact": progress.OutputExact,
- "turn_duration_ms": turn.Execution.TurnSummary.TurnDurationMS,
- "thinking_duration_ms": turn.Execution.TurnSummary.ThinkingDurationMS,
- })
+ ev.Raw = mergeRaw(ev.Raw, raw)
m.append(event, fmt.Sprintf("turn.%d.usage", m.turnIndex), ev)
}
+// recordTotals advances the cumulative baseline the fallback above measures against. It runs for
+// every committed turn, including ones this sweep is not emitting, so a resumed sweep measures the
+// first turn it emits against the turn before it rather than against zero.
+func (m *mapper) recordTotals(committed *TurnCommitted) {
+ if committed == nil {
+ return
+ }
+ m.lastTotals.input = int64(committed.TotalInputTokens)
+ m.lastTotals.output = int64(committed.TotalOutputTokens)
+ m.lastTotals.seen = true
+}
+
// emitUsage reports a usage checkpoint as the difference from the previous one.
//
// fx's snapshots are cumulative replacements: each one restates the session's whole usage. Emitting
@@ -738,6 +784,13 @@ func toolResultRaw(result ToolResult) map[string]interface{} {
"truncated": result.Truncated,
"provider_native": result.ProviderNative,
}
+ if result.CommandOutputReplay != nil && result.CommandOutputReplay.Handle != "" {
+ // The full captured output of a command is not in the record: fx keeps it in a framed
+ // replay file beside the session and puts the handle here. Beacon cannot read that framing,
+ // so it carries the handle rather than pretending the stored text is the whole output.
+ raw["command_output_handle"] = result.CommandOutputReplay.Handle.String()
+ raw["command_output_framed_bytes"] = result.CommandOutputReplay.FramedBytes
+ }
if result.CommandProcess != nil {
raw["process_outcome"] = result.CommandProcess.Kind
if result.CommandProcess.Value != nil {
diff --git a/cli/beacon/internal/fxsession/realsession_test.go b/cli/beacon/internal/fxsession/realsession_test.go
new file mode 100644
index 00000000..40357fef
--- /dev/null
+++ b/cli/beacon/internal/fxsession/realsession_test.go
@@ -0,0 +1,177 @@
+package fxsession
+
+import (
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+)
+
+// This test runs against a session record fx actually wrote.
+//
+// testdata/fx-0.0.7-session holds the events.jsonl and session.json left behind by the released fx
+// 0.0.7 binary driving a real turn -- a command that exits 3, a file read, a file write -- against a
+// local fake gateway. The only edit is the workspace path, rewritten to /home/dev/api so the fixture
+// does not carry the machine it was captured on (and the manifest's byte watermark adjusted to
+// match, since the rewrite changed the file's length).
+//
+// It exists because hand-transcribing another project's format is exactly the kind of work that
+// looks right and is wrong, and because this capture found two real defects the transcribed
+// fixtures could not: fx 0.0.7 writes no turn summary at all, so every per-turn token count was
+// being dropped, and a command's full output lives in a framed replay file whose handle the mapper
+// was discarding.
+func TestRealFxSessionRecordCollectsEndToEnd(t *testing.T) {
+ dir := filepath.Join("testdata", "fx-0.0.7-session")
+ store := &Store{Dir: dir}
+ refs, err := store.List()
+ if err != nil {
+ t.Fatalf("List: %v", err)
+ }
+ if len(refs) != 1 {
+ t.Fatalf("expected one recorded session, got %d", len(refs))
+ }
+ ref := refs[0]
+ if ref.Manifest == nil {
+ t.Fatal("the recorded session's manifest was not readable")
+ }
+
+ events, stats, err := store.Read(ref)
+ if err != nil {
+ t.Fatalf("Read: %v", err)
+ }
+ if stats.Malformed != 0 {
+ t.Fatalf("%d malformed lines in a log fx wrote: %v", stats.Malformed, stats.FirstError)
+ }
+ // fx writes far more storage bookkeeping than agent activity: recovery checkpoints and usage
+ // snapshots outnumber the one committed turn roughly twenty to one. Skipping them by kind
+ // rather than emitting them is what keeps the runtime log readable.
+ if stats.Skipped == 0 {
+ t.Error("no storage-only records were skipped, which is not what an fx log looks like")
+ }
+
+ mapped := MapSession(ref, events, MapOptions{})
+ got := make([]string, 0, len(mapped))
+ for _, item := range mapped {
+ got = append(got, item.Event.Event.Action)
+ }
+ want := []string{
+ "session.started",
+ "prompt.submitted",
+ "command.executed",
+ "file.read",
+ "file.created",
+ "agent.message",
+ "token.usage",
+ }
+ if strings.Join(got, ",") != strings.Join(want, ",") {
+ t.Fatalf("actions = %v, want %v", got, want)
+ }
+}
+
+// The command in the recorded session exited 3 through fx's `terminal` tool, which is the tool the
+// released binary actually runs commands with -- `run_command` is rejected as unsupported there.
+// Both names are classified as commands for that reason.
+func TestRealFxCommandCarriesItsExitCodeAndOutputHandle(t *testing.T) {
+ mapped := mapRealSession(t)
+ command := findMapped(t, mapped, "command.executed")
+
+ if command.Tool == nil || command.Tool.Name != "terminal" {
+ t.Fatalf("tool = %+v, want fx's terminal tool", command.Tool)
+ }
+ if command.Command == nil || command.Command.Command != "echo beacon-verify && exit 3" {
+ t.Fatalf("command = %+v", command.Command)
+ }
+ if command.Command.ExitCode == nil || *command.Command.ExitCode != 3 {
+ t.Fatalf("exit code = %+v, want 3", command.Command.ExitCode)
+ }
+ // fx stores the full captured output in a framed file beside the session and leaves a handle
+ // in the record. Beacon carries the handle rather than presenting the stored excerpt as the
+ // whole output.
+ raw := rawFx(t, command)
+ if handle, _ := raw["command_output_handle"].(string); !strings.HasPrefix(handle, "fx-command-replay-") {
+ t.Errorf("raw.fx.command_output_handle = %v, want fx's replay handle", raw["command_output_handle"])
+ }
+}
+
+// fx 0.0.7 writes no turn summary, so the per-turn token counts have to come from the difference in
+// the session's cumulative totals. Without the fallback this session reported no token usage at
+// all, which is how the gap was found.
+func TestRealFxSessionReportsTokenUsageWithoutATurnSummary(t *testing.T) {
+ mapped := mapRealSession(t)
+ usage := findMapped(t, mapped, "token.usage")
+
+ if usage.GenAI == nil || usage.GenAI.Usage == nil {
+ t.Fatal("no gen_ai.usage on the usage event")
+ }
+ if usage.GenAI.Usage.InputTokens == nil || *usage.GenAI.Usage.InputTokens != 1234 {
+ t.Errorf("input tokens = %+v, want the 1234 the provider reported", usage.GenAI.Usage.InputTokens)
+ }
+ if usage.GenAI.Usage.OutputTokens == nil || *usage.GenAI.Usage.OutputTokens != 56 {
+ t.Errorf("output tokens = %+v, want 56", usage.GenAI.Usage.OutputTokens)
+ }
+ // Which source the numbers came from is on the event, because a difference of two running
+ // totals and an exact per-turn count are not equally trustworthy.
+ if source := rawFx(t, usage)["token_source"]; source != "session_totals_delta" {
+ t.Errorf("raw.fx.token_source = %v, want session_totals_delta", source)
+ }
+}
+
+// The write in the recorded session produced a real committed diff, which is the field a reviewer
+// and a sensitive-edit rule both read.
+func TestRealFxWriteCarriesItsDiff(t *testing.T) {
+ mapped := mapRealSession(t)
+ created := findMapped(t, mapped, "file.created")
+
+ if created.File == nil || !strings.HasSuffix(created.File.Path, "/health.ts") {
+ t.Fatalf("file = %+v", created.File)
+ }
+ if !strings.Contains(created.File.Diff, "+export function health()") {
+ t.Fatalf("diff does not carry the added line:\n%s", created.File.Diff)
+ }
+ if created.File.Operation != "create" {
+ t.Errorf("operation = %q, want create", created.File.Operation)
+ }
+ if created.File.Language != "ts" {
+ t.Errorf("language = %q, want ts", created.File.Language)
+ }
+}
+
+// fx resolves the model from its own catalog rather than from what the request asked for, and the
+// session record is where that resolution is visible. Every event from the session carries it.
+func TestRealFxSessionCarriesTheResolvedModelAndProvider(t *testing.T) {
+ mapped := mapRealSession(t)
+ for _, item := range mapped {
+ if item.Event.Model == "" {
+ t.Errorf("%s carries no model", item.Event.Event.Action)
+ }
+ if item.Event.GenAI == nil || item.Event.GenAI.Provider == nil || item.Event.GenAI.Provider.Name != "gateway" {
+ t.Errorf("%s provider = %+v, want gateway", item.Event.Event.Action, item.Event.GenAI)
+ }
+ }
+}
+
+func mapRealSession(t *testing.T) []MappedEvent {
+ t.Helper()
+ store := &Store{Dir: filepath.Join("testdata", "fx-0.0.7-session")}
+ refs, err := store.List()
+ if err != nil || len(refs) != 1 {
+ t.Fatalf("List: %v (%d sessions)", err, len(refs))
+ }
+ events, _, err := store.Read(refs[0])
+ if err != nil {
+ t.Fatalf("Read: %v", err)
+ }
+ return MapSession(refs[0], events, MapOptions{})
+}
+
+func findMapped(t *testing.T, mapped []MappedEvent, action string) schema.Event {
+ t.Helper()
+ for _, item := range mapped {
+ if item.Event.Event.Action == action {
+ return item.Event
+ }
+ }
+ t.Fatalf("no %q event in the recorded session", action)
+ return schema.Event{}
+}
diff --git a/cli/beacon/internal/fxsession/testdata/fx-0.0.7-session/1788051418424-1788051418424884928-a177316124162631/session.json b/cli/beacon/internal/fxsession/testdata/fx-0.0.7-session/1788051418424-1788051418424884928-a177316124162631/session.json
new file mode 100644
index 00000000..a3ef1247
--- /dev/null
+++ b/cli/beacon/internal/fxsession/testdata/fx-0.0.7-session/1788051418424-1788051418424884928-a177316124162631/session.json
@@ -0,0 +1,28 @@
+{
+ "schema_version": 3,
+ "storage_format": "event_log_v1",
+ "id": "1788051418424-1788051418424884928-a177316124162631",
+ "authority_id": "08f69f0df4ac45e82079810bd03449a4",
+ "log_generation": "3b4c72027563ff60190f0299461ee01c",
+ "created_at_ms": 1788051418424,
+ "updated_at_ms": 1788051418741,
+ "origin_workspace_root": "/home/dev/api",
+ "workspace_root": "/home/dev/api",
+ "conversation_language": "und-Latn",
+ "history_len": 1,
+ "total_input_tokens": 1234,
+ "total_output_tokens": 56,
+ "last_event_seq": 26,
+ "event_log_bytes": 34379,
+ "event_log_stat_fingerprint": "23fcca61e5a55cc64ad35e6c0760e9f5638555dc5a806970998f44d7ed794e61",
+ "generation_base_seq": 1,
+ "generation_base_bytes": 1054,
+ "checkpoint_seq": 2,
+ "checkpoint_sha256": "9575eb1719ae531296bc95b1b16e8e40ec684e31ad4478a4327e1ee7602e0250",
+ "preferences": {
+ "model": "moonshotai/kimi-k3",
+ "effort": "auto",
+ "fast_mode": false,
+ "provider": "gateway"
+ }
+}
diff --git a/cli/beacon/internal/fxsession/types.go b/cli/beacon/internal/fxsession/types.go
index 975c83a4..24cd71a3 100644
--- a/cli/beacon/internal/fxsession/types.go
+++ b/cli/beacon/internal/fxsession/types.go
@@ -231,21 +231,31 @@ type ToolCall struct {
// three are carried through to Beacon rather than collapsed, because a reader who cannot tell a
// short output from a truncated one cannot tell what the agent actually saw.
type ToolResult struct {
- ToolCallID DurableString `json:"tool_call_id"`
- ToolName DurableString `json:"tool_name"`
- Status string `json:"status"`
- Output DurableString `json:"output"`
- OutputHandle *string `json:"output_handle"`
- Preview *string `json:"preview"`
- OutputBytes int64 `json:"output_bytes"`
- StoredOutputBytes int64 `json:"stored_output_bytes"`
- Truncated bool `json:"truncated"`
- ProviderNative bool `json:"provider_native"`
- CreatedAtMS int64 `json:"created_at_ms"`
- PermissionFeedback []DurableString `json:"permission_feedback"`
- CommittedFile *CommittedFile `json:"committed_file_presentation"`
- CommandProcess *ProcessOutcome `json:"command_process_presentation"`
- TerminalAction *TerminalOutcome `json:"terminal_action_presentation"`
+ ToolCallID DurableString `json:"tool_call_id"`
+ ToolName DurableString `json:"tool_name"`
+ Status string `json:"status"`
+ Output DurableString `json:"output"`
+ OutputHandle *string `json:"output_handle"`
+ Preview *string `json:"preview"`
+ OutputBytes int64 `json:"output_bytes"`
+ StoredOutputBytes int64 `json:"stored_output_bytes"`
+ Truncated bool `json:"truncated"`
+ ProviderNative bool `json:"provider_native"`
+ CreatedAtMS int64 `json:"created_at_ms"`
+ PermissionFeedback []DurableString `json:"permission_feedback"`
+ CommittedFile *CommittedFile `json:"committed_file_presentation"`
+ CommandOutputReplay *CommandReplay `json:"command_output_replay"`
+ CommandProcess *ProcessOutcome `json:"command_process_presentation"`
+ TerminalAction *TerminalOutcome `json:"terminal_action_presentation"`
+}
+
+// CommandReplay is fx's pointer to a command's full captured output, which it stores in a framed
+// file beside the session rather than in the record. Kind is "available" with a handle, or
+// "unavailable". Beacon records the handle and does not try to read the frame format.
+type CommandReplay struct {
+ Kind string `json:"kind"`
+ Handle DurableString `json:"handle"`
+ FramedBytes int64 `json:"framed_bytes"`
}
// CommittedFile is the diff fx committed for a write or edit: the path, whether the file was added