The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Add Codex token usage attribution
shukieshahfdf6233353f3
diff --git a/cli/beacon/internal/endpoint/harness/harness.go b/cli/beacon/internal/endpoint/harness/harness.goindex 9c7203fa..58d2a9ba 100644--- a/cli/beacon/internal/endpoint/harness/harness.go+++ b/cli/beacon/internal/endpoint/harness/harness.go@@ -437,6 +437,12 @@ func codexOTELBlock(endpoint string, logUserPrompt bool) []string {"","[otel.exporter.\"otlp-grpc\"]",fmt.Sprintf("endpoint = %q", endpoint),+ "",+ // metrics_exporter is a separate Codex setting from the log/event+ // exporter above; without it Codex exports logs but no metrics, so the+ // codex.turn.token_usage histogram never reaches the collector.+ "[otel.metrics_exporter.\"otlp-grpc\"]",+ fmt.Sprintf("endpoint = %q", endpoint),}}diff --git a/cli/beacon/internal/endpoint/harness/harness_test.go b/cli/beacon/internal/endpoint/harness/harness_test.goindex b5bf83e2..4aaa82c5 100644--- a/cli/beacon/internal/endpoint/harness/harness_test.go+++ b/cli/beacon/internal/endpoint/harness/harness_test.go@@ -176,14 +176,18 @@ func TestConfigureCodexWritesTelemetryBlockAndBackup(t *testing.T) {"log_user_prompt = true","[otel.exporter.\"otlp-grpc\"]","endpoint = \"http://127.0.0.1:4317\"",+ // metrics_exporter is required for the codex.turn.token_usage metric that+ // backs Codex token/cost attribution.+ "[otel.metrics_exporter.\"otlp-grpc\"]",} {if !strings.Contains(text, want) {t.Fatalf("codex config missing %q:\n%s", want, text)}}+ // Traces stay disabled: Codex spans are high-volume and opt-in via+ // --include-codex-spans, and are not needed for token usage.for _, noisy := range []string{"[otel.trace_exporter.\"otlp-grpc\"]",- "[otel.metrics_exporter.\"otlp-grpc\"]",} {if strings.Contains(text, noisy) {t.Fatalf("codex config should not enable noisy exporter %q:\n%s", noisy, text)diff --git a/collector-builder/exporter/beaconjsonexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/exporter_test.goindex a83bcbcb..9ded5945 100644--- a/collector-builder/exporter/beaconjsonexporter/exporter_test.go+++ b/collector-builder/exporter/beaconjsonexporter/exporter_test.go@@ -767,9 +767,10 @@ func TestConsumeMetricsDropsCodexMetricsByDefault(t *testing.T) {rm := metrics.ResourceMetrics().AppendEmpty()rm.Resource().Attributes().PutStr("service.name", "codex-cli")sm := rm.ScopeMetrics().AppendEmpty()+ // codex.turn.token_usage is intentionally excluded: it is the one codex.*+ // metric kept for gen_ai.usage normalization (see the beaconevent tests).for _, name := range []string{"codex.turn.memory",- "codex.turn.token_usage","codex.websocket.event.duration_ms","codex.remote_models.load_cache.duration_ms","codex.tool.call.duration_ms",diff --git a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.goindex afb7a8f2..21e615d2 100644--- a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go+++ b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go@@ -192,6 +192,12 @@ func shouldDropCodexMetric(resourceAttrs map[string]interface{}, name string) boreturn false}normalized := strings.ToLower(strings.TrimSpace(name))+ if normalized == "codex.turn.token_usage" {+ // Codex reports per-turn token usage only on this metric, so keep it for+ // gen_ai.usage normalization. Other codex.* metrics (including the memory+ // *.token_usage phases) stay dropped as high-volume runtime noise.+ return false+ }return strings.HasPrefix(normalized, "codex.")}@@ -415,7 +421,9 @@ func (c Converter) EventFromMetric(resourceAttrs map[string]interface{}, metric// tight so unrelated metrics keep the generic metric.observed conversion.func IsTokenUsageMetric(name string) bool {normalized := strings.ToLower(strings.TrimSpace(name))- return normalized == "gen_ai.client.token.usage" || strings.HasSuffix(normalized, ".token.usage")+ return normalized == "gen_ai.client.token.usage" ||+ normalized == "codex.turn.token_usage" ||+ strings.HasSuffix(normalized, ".token.usage")}// IsCostUsageMetric reports whether a metric carries runtime-reported cost as@@ -439,6 +447,23 @@ func (c Converter) EventsFromMetric(resourceAttrs map[string]interface{}, metricfunc (c Converter) eventsFromUsageMetric(resourceAttrs map[string]interface{}, metric pmetric.Metric) []Event {var events []Event+ // Codex reports input tokens inclusive of the cached_input subset (input =+ // uncached + cached, and total = input + output). Beacon's gen_ai.usage keeps+ // input_tokens and cache_read disjoint like Claude Code, so reduce input by+ // the per-turn cached_input before normalizing; otherwise totals double-count+ // cached prompt tokens. Returns nil (no-op) for every other usage metric.+ cachedInputByTurn := codexCachedInputByTimestamp(metric)+ adjustValue := func(dpAttrs pcommon.Map, ts pcommon.Timestamp, value float64) float64 {+ if cachedInputByTurn == nil {+ return value+ }+ if tt := FirstString(AttrsToMap(dpAttrs), "token_type"); strings.EqualFold(tt, "input") {+ if value -= cachedInputByTurn[ts.AsTime().UnixNano()]; value < 0 {+ value = 0+ }+ }+ return value+ }switch metric.Type() {case pmetric.MetricTypeSum:sum := metric.Sum()@@ -449,14 +474,14 @@ func (c Converter) eventsFromUsageMetric(resourceAttrs map[string]interface{}, m}for i := 0; i < sum.DataPoints().Len(); i++ {dp := sum.DataPoints().At(i)- events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp), extra))+ events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), adjustValue(dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp)), extra))}case pmetric.MetricTypeGauge:gauge := metric.Gauge()extra := map[string]interface{}{"metric_type": metric.Type().String()}for i := 0; i < gauge.DataPoints().Len(); i++ {dp := gauge.DataPoints().At(i)- events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp), extra))+ events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), adjustValue(dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp)), extra))}case pmetric.MetricTypeHistogram:histogram := metric.Histogram()@@ -470,12 +495,38 @@ func (c Converter) eventsFromUsageMetric(resourceAttrs map[string]interface{}, m"metric_temporality": histogram.AggregationTemporality().String(),"metric_count": int64(dp.Count()),}- events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), dp.Sum(), extra))+ events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), adjustValue(dp.Attributes(), dp.Timestamp(), dp.Sum()), extra))}}return events}+// codexCachedInputByTimestamp sums each turn's cached_input datapoints from the+// codex.turn.token_usage histogram, keyed by datapoint timestamp, so the input+// datapoint can be reduced to its uncached portion (Codex's input is inclusive+// of cached_input). Returns nil for any other metric so the generic usage path+// is unaffected.+func codexCachedInputByTimestamp(metric pmetric.Metric) map[int64]float64 {+ if !strings.EqualFold(strings.TrimSpace(metric.Name()), "codex.turn.token_usage") {+ return nil+ }+ if metric.Type() != pmetric.MetricTypeHistogram {+ return nil+ }+ out := map[int64]float64{}+ dps := metric.Histogram().DataPoints()+ for i := 0; i < dps.Len(); i++ {+ dp := dps.At(i)+ if !dp.HasSum() {+ continue+ }+ if tt := FirstString(AttrsToMap(dp.Attributes()), "token_type"); strings.EqualFold(tt, "cached_input") {+ out[dp.Timestamp().AsTime().UnixNano()] += dp.Sum()+ }+ }+ return out+}+func numberDataPointValue(dp pmetric.NumberDataPoint) float64 {if dp.ValueType() == pmetric.NumberDataPointValueTypeInt {return float64(dp.IntValue())@@ -509,7 +560,7 @@ func (c Converter) usageEventFromDataPoint(resourceAttrs map[string]interface{},}event.GenAI.Usage.CostUSD = &cost} else {- ApplyTokenUsage(&event, FirstString(attrs, "type", "gen_ai.token.type"), int64(math.Round(value)))+ ApplyTokenUsage(&event, FirstString(attrs, "type", "token_type", "gen_ai.token.type"), int64(math.Round(value)))}rawExtra := map[string]interface{}{"otel_signal": "metrics",@@ -526,9 +577,11 @@ func (c Converter) usageEventFromDataPoint(resourceAttrs map[string]interface{},}// ApplyTokenUsage merges a typed token count into the event's canonical-// gen_ai.usage struct. Claude Code's cacheRead/cacheCreation token types-// extend the semconv input/output enum. Unknown types only record the type-// so the raw value stays inspectable without polluting usage totals.+// gen_ai.usage struct. Claude Code's cacheRead/cacheCreation and Codex's+// cachedInput/reasoningOutput token types extend the semconv input/output enum.+// Codex's "total" rollup type is left unmapped so it never sums on top of the+// per-type counts. Unknown types only record the type so the raw value stays+// inspectable without polluting usage totals.func ApplyTokenUsage(event *Event, tokenType string, value int64) {if event.GenAI == nil {event.GenAI = &GenAIInfo{}@@ -542,11 +595,11 @@ func ApplyTokenUsage(event *Event, tokenType string, value int64) {usage.InputTokens = &valuecase "output", "completion":usage.OutputTokens = &value- case "cacheread":+ case "cacheread", "cachedinput":usage.CacheRead = &GenAIUsageCacheReadInfo{InputTokens: &value}case "cachecreation":usage.CacheCreation = &GenAIUsageCacheCreationInfo{InputTokens: &value}- case "reasoning":+ case "reasoning", "reasoningoutput":usage.Reasoning = &GenAIUsageReasoningInfo{OutputTokens: &value}default:if event.GenAI.Token == nil && tokenType != "" {diff --git a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.goindex 305ac7cd..782c7809 100644--- a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go+++ b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go@@ -429,6 +429,69 @@ func TestEventsFromMetricsCapturesTokenUsageHistogram(t *testing.T) {}}+func TestEventsFromMetricsNormalizesCodexTurnTokenUsage(t *testing.T) {+ // Codex reports per-turn usage only on the codex.turn.token_usage histogram,+ // split by a token_type dimension. It must survive the codex.* metric drop+ // and normalize into gen_ai.usage; the "total" rollup must not add usage.+ metrics := pmetric.NewMetrics()+ resourceMetrics := metrics.ResourceMetrics().AppendEmpty()+ resourceMetrics.Resource().Attributes().PutStr("beacon.harness.name", "codex_cli")+ metric := resourceMetrics.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty()+ metric.SetName("codex.turn.token_usage")+ histogram := metric.SetEmptyHistogram()+ histogram.SetAggregationTemporality(pmetric.AggregationTemporalityDelta)+ // Codex's input (120) is inclusive of cached_input (90), and total = input ++ // output. The normalized input_tokens must drop to the uncached portion (30)+ // so input_tokens + cache_read == input and totals don't double-count cache.+ tokenTypes := map[string]float64{+ "input": 120,+ "cached_input": 90,+ "output": 45,+ "reasoning_output": 30,+ "total": 165,+ }+ for tokenType, value := range tokenTypes {+ dp := histogram.DataPoints().AppendEmpty()+ dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Unix(1700000400, 0).UTC()))+ dp.SetCount(1)+ dp.SetSum(value)+ dp.Attributes().PutStr("token_type", tokenType)+ }++ events := NewConverter(Options{}).EventsFromMetrics(metrics)+ if len(events) != len(tokenTypes) {+ t.Fatalf("expected %d events (one per token_type), got %d", len(tokenTypes), len(events))+ }+ var input, cacheRead, output, reasoning int64+ for _, event := range events {+ if event.Event.Action != "token.usage" || event.Harness.Name != "codex_cli" {+ t.Fatalf("event = %#v, want codex token.usage", event.Event)+ }+ usage := event.GenAI.Usage+ if usage == nil {+ continue+ }+ if usage.InputTokens != nil {+ input += *usage.InputTokens+ }+ if usage.OutputTokens != nil {+ output += *usage.OutputTokens+ }+ if usage.CacheRead != nil && usage.CacheRead.InputTokens != nil {+ cacheRead += *usage.CacheRead.InputTokens+ }+ if usage.Reasoning != nil && usage.Reasoning.OutputTokens != nil {+ reasoning += *usage.Reasoning.OutputTokens+ }+ }+ if input != 30 || cacheRead != 90 || output != 45 || reasoning != 30 {+ t.Fatalf("normalized usage = input %d, cache_read %d, output %d, reasoning %d (want 30/90/45/30)", input, cacheRead, output, reasoning)+ }+ if input+cacheRead != 120 {+ t.Fatalf("input_tokens + cache_read = %d, want 120 (= Codex input, no double-count)", input+cacheRead)+ }+}+func TestEventsFromMetricsUnknownTokenTypeKeepsRawValueOnly(t *testing.T) {metrics := pmetric.NewMetrics()resourceMetrics := metrics.ResourceMetrics().AppendEmpty()