Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Add Codex token usage attribution

shukieshahfdf6233353f3
diff --git a/cli/beacon/internal/endpoint/harness/harness.go b/cli/beacon/internal/endpoint/harness/harness.go
index 9c7203fa..58d2a9ba 100644
--- a/cli/beacon/internal/endpoint/harness/harness.go
+++ b/cli/beacon/internal/endpoint/harness/harness.go
@@ -437,6 +437,12 @@ func codexOTELBlock(endpoint string, logUserPrompt bool) []string {
"",
"[otel.exporter.\"otlp-grpc\"]",
fmt.Sprintf("endpoint = %q", endpoint),
+ "",
+ // metrics_exporter is a separate Codex setting from the log/event
+ // exporter above; without it Codex exports logs but no metrics, so the
+ // codex.turn.token_usage histogram never reaches the collector.
+ "[otel.metrics_exporter.\"otlp-grpc\"]",
+ fmt.Sprintf("endpoint = %q", endpoint),
}
}
diff --git a/cli/beacon/internal/endpoint/harness/harness_test.go b/cli/beacon/internal/endpoint/harness/harness_test.go
index b5bf83e2..4aaa82c5 100644
--- a/cli/beacon/internal/endpoint/harness/harness_test.go
+++ b/cli/beacon/internal/endpoint/harness/harness_test.go
@@ -176,14 +176,18 @@ func TestConfigureCodexWritesTelemetryBlockAndBackup(t *testing.T) {
"log_user_prompt = true",
"[otel.exporter.\"otlp-grpc\"]",
"endpoint = \"http://127.0.0.1:4317\"",
+ // metrics_exporter is required for the codex.turn.token_usage metric that
+ // backs Codex token/cost attribution.
+ "[otel.metrics_exporter.\"otlp-grpc\"]",
} {
if !strings.Contains(text, want) {
t.Fatalf("codex config missing %q:\n%s", want, text)
}
}
+ // Traces stay disabled: Codex spans are high-volume and opt-in via
+ // --include-codex-spans, and are not needed for token usage.
for _, noisy := range []string{
"[otel.trace_exporter.\"otlp-grpc\"]",
- "[otel.metrics_exporter.\"otlp-grpc\"]",
} {
if strings.Contains(text, noisy) {
t.Fatalf("codex config should not enable noisy exporter %q:\n%s", noisy, text)
diff --git a/collector-builder/exporter/beaconjsonexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/exporter_test.go
index a83bcbcb..9ded5945 100644
--- a/collector-builder/exporter/beaconjsonexporter/exporter_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/exporter_test.go
@@ -767,9 +767,10 @@ func TestConsumeMetricsDropsCodexMetricsByDefault(t *testing.T) {
rm := metrics.ResourceMetrics().AppendEmpty()
rm.Resource().Attributes().PutStr("service.name", "codex-cli")
sm := rm.ScopeMetrics().AppendEmpty()
+ // codex.turn.token_usage is intentionally excluded: it is the one codex.*
+ // metric kept for gen_ai.usage normalization (see the beaconevent tests).
for _, name := range []string{
"codex.turn.memory",
- "codex.turn.token_usage",
"codex.websocket.event.duration_ms",
"codex.remote_models.load_cache.duration_ms",
"codex.tool.call.duration_ms",
diff --git a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go
index afb7a8f2..21e615d2 100644
--- a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go
+++ b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter.go
@@ -192,6 +192,12 @@ func shouldDropCodexMetric(resourceAttrs map[string]interface{}, name string) bo
return false
}
normalized := strings.ToLower(strings.TrimSpace(name))
+ if normalized == "codex.turn.token_usage" {
+ // Codex reports per-turn token usage only on this metric, so keep it for
+ // gen_ai.usage normalization. Other codex.* metrics (including the memory
+ // *.token_usage phases) stay dropped as high-volume runtime noise.
+ return false
+ }
return strings.HasPrefix(normalized, "codex.")
}
@@ -415,7 +421,9 @@ func (c Converter) EventFromMetric(resourceAttrs map[string]interface{}, metric
// tight so unrelated metrics keep the generic metric.observed conversion.
func IsTokenUsageMetric(name string) bool {
normalized := strings.ToLower(strings.TrimSpace(name))
- return normalized == "gen_ai.client.token.usage" || strings.HasSuffix(normalized, ".token.usage")
+ return normalized == "gen_ai.client.token.usage" ||
+ normalized == "codex.turn.token_usage" ||
+ strings.HasSuffix(normalized, ".token.usage")
}
// IsCostUsageMetric reports whether a metric carries runtime-reported cost as
@@ -439,6 +447,23 @@ func (c Converter) EventsFromMetric(resourceAttrs map[string]interface{}, metric
func (c Converter) eventsFromUsageMetric(resourceAttrs map[string]interface{}, metric pmetric.Metric) []Event {
var events []Event
+ // Codex reports input tokens inclusive of the cached_input subset (input =
+ // uncached + cached, and total = input + output). Beacon's gen_ai.usage keeps
+ // input_tokens and cache_read disjoint like Claude Code, so reduce input by
+ // the per-turn cached_input before normalizing; otherwise totals double-count
+ // cached prompt tokens. Returns nil (no-op) for every other usage metric.
+ cachedInputByTurn := codexCachedInputByTimestamp(metric)
+ adjustValue := func(dpAttrs pcommon.Map, ts pcommon.Timestamp, value float64) float64 {
+ if cachedInputByTurn == nil {
+ return value
+ }
+ if tt := FirstString(AttrsToMap(dpAttrs), "token_type"); strings.EqualFold(tt, "input") {
+ if value -= cachedInputByTurn[ts.AsTime().UnixNano()]; value < 0 {
+ value = 0
+ }
+ }
+ return value
+ }
switch metric.Type() {
case pmetric.MetricTypeSum:
sum := metric.Sum()
@@ -449,14 +474,14 @@ func (c Converter) eventsFromUsageMetric(resourceAttrs map[string]interface{}, m
}
for i := 0; i < sum.DataPoints().Len(); i++ {
dp := sum.DataPoints().At(i)
- events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp), extra))
+ events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), adjustValue(dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp)), extra))
}
case pmetric.MetricTypeGauge:
gauge := metric.Gauge()
extra := map[string]interface{}{"metric_type": metric.Type().String()}
for i := 0; i < gauge.DataPoints().Len(); i++ {
dp := gauge.DataPoints().At(i)
- events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp), extra))
+ events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), adjustValue(dp.Attributes(), dp.Timestamp(), numberDataPointValue(dp)), extra))
}
case pmetric.MetricTypeHistogram:
histogram := metric.Histogram()
@@ -470,12 +495,38 @@ func (c Converter) eventsFromUsageMetric(resourceAttrs map[string]interface{}, m
"metric_temporality": histogram.AggregationTemporality().String(),
"metric_count": int64(dp.Count()),
}
- events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), dp.Sum(), extra))
+ events = append(events, c.usageEventFromDataPoint(resourceAttrs, metric, dp.Attributes(), dp.Timestamp(), adjustValue(dp.Attributes(), dp.Timestamp(), dp.Sum()), extra))
}
}
return events
}
+// codexCachedInputByTimestamp sums each turn's cached_input datapoints from the
+// codex.turn.token_usage histogram, keyed by datapoint timestamp, so the input
+// datapoint can be reduced to its uncached portion (Codex's input is inclusive
+// of cached_input). Returns nil for any other metric so the generic usage path
+// is unaffected.
+func codexCachedInputByTimestamp(metric pmetric.Metric) map[int64]float64 {
+ if !strings.EqualFold(strings.TrimSpace(metric.Name()), "codex.turn.token_usage") {
+ return nil
+ }
+ if metric.Type() != pmetric.MetricTypeHistogram {
+ return nil
+ }
+ out := map[int64]float64{}
+ dps := metric.Histogram().DataPoints()
+ for i := 0; i < dps.Len(); i++ {
+ dp := dps.At(i)
+ if !dp.HasSum() {
+ continue
+ }
+ if tt := FirstString(AttrsToMap(dp.Attributes()), "token_type"); strings.EqualFold(tt, "cached_input") {
+ out[dp.Timestamp().AsTime().UnixNano()] += dp.Sum()
+ }
+ }
+ return out
+}
+
func numberDataPointValue(dp pmetric.NumberDataPoint) float64 {
if dp.ValueType() == pmetric.NumberDataPointValueTypeInt {
return float64(dp.IntValue())
@@ -509,7 +560,7 @@ func (c Converter) usageEventFromDataPoint(resourceAttrs map[string]interface{},
}
event.GenAI.Usage.CostUSD = &cost
} else {
- ApplyTokenUsage(&event, FirstString(attrs, "type", "gen_ai.token.type"), int64(math.Round(value)))
+ ApplyTokenUsage(&event, FirstString(attrs, "type", "token_type", "gen_ai.token.type"), int64(math.Round(value)))
}
rawExtra := map[string]interface{}{
"otel_signal": "metrics",
@@ -526,9 +577,11 @@ func (c Converter) usageEventFromDataPoint(resourceAttrs map[string]interface{},
}
// ApplyTokenUsage merges a typed token count into the event's canonical
-// gen_ai.usage struct. Claude Code's cacheRead/cacheCreation token types
-// extend the semconv input/output enum. Unknown types only record the type
-// so the raw value stays inspectable without polluting usage totals.
+// gen_ai.usage struct. Claude Code's cacheRead/cacheCreation and Codex's
+// cachedInput/reasoningOutput token types extend the semconv input/output enum.
+// Codex's "total" rollup type is left unmapped so it never sums on top of the
+// per-type counts. Unknown types only record the type so the raw value stays
+// inspectable without polluting usage totals.
func ApplyTokenUsage(event *Event, tokenType string, value int64) {
if event.GenAI == nil {
event.GenAI = &GenAIInfo{}
@@ -542,11 +595,11 @@ func ApplyTokenUsage(event *Event, tokenType string, value int64) {
usage.InputTokens = &value
case "output", "completion":
usage.OutputTokens = &value
- case "cacheread":
+ case "cacheread", "cachedinput":
usage.CacheRead = &GenAIUsageCacheReadInfo{InputTokens: &value}
case "cachecreation":
usage.CacheCreation = &GenAIUsageCacheCreationInfo{InputTokens: &value}
- case "reasoning":
+ case "reasoning", "reasoningoutput":
usage.Reasoning = &GenAIUsageReasoningInfo{OutputTokens: &value}
default:
if event.GenAI.Token == nil && tokenType != "" {
diff --git a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go
index 305ac7cd..782c7809 100644
--- a/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/internal/beaconevent/converter_test.go
@@ -429,6 +429,69 @@ func TestEventsFromMetricsCapturesTokenUsageHistogram(t *testing.T) {
}
}
+func TestEventsFromMetricsNormalizesCodexTurnTokenUsage(t *testing.T) {
+ // Codex reports per-turn usage only on the codex.turn.token_usage histogram,
+ // split by a token_type dimension. It must survive the codex.* metric drop
+ // and normalize into gen_ai.usage; the "total" rollup must not add usage.
+ metrics := pmetric.NewMetrics()
+ resourceMetrics := metrics.ResourceMetrics().AppendEmpty()
+ resourceMetrics.Resource().Attributes().PutStr("beacon.harness.name", "codex_cli")
+ metric := resourceMetrics.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty()
+ metric.SetName("codex.turn.token_usage")
+ histogram := metric.SetEmptyHistogram()
+ histogram.SetAggregationTemporality(pmetric.AggregationTemporalityDelta)
+ // Codex's input (120) is inclusive of cached_input (90), and total = input +
+ // output. The normalized input_tokens must drop to the uncached portion (30)
+ // so input_tokens + cache_read == input and totals don't double-count cache.
+ tokenTypes := map[string]float64{
+ "input": 120,
+ "cached_input": 90,
+ "output": 45,
+ "reasoning_output": 30,
+ "total": 165,
+ }
+ for tokenType, value := range tokenTypes {
+ dp := histogram.DataPoints().AppendEmpty()
+ dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Unix(1700000400, 0).UTC()))
+ dp.SetCount(1)
+ dp.SetSum(value)
+ dp.Attributes().PutStr("token_type", tokenType)
+ }
+
+ events := NewConverter(Options{}).EventsFromMetrics(metrics)
+ if len(events) != len(tokenTypes) {
+ t.Fatalf("expected %d events (one per token_type), got %d", len(tokenTypes), len(events))
+ }
+ var input, cacheRead, output, reasoning int64
+ for _, event := range events {
+ if event.Event.Action != "token.usage" || event.Harness.Name != "codex_cli" {
+ t.Fatalf("event = %#v, want codex token.usage", event.Event)
+ }
+ usage := event.GenAI.Usage
+ if usage == nil {
+ continue
+ }
+ if usage.InputTokens != nil {
+ input += *usage.InputTokens
+ }
+ if usage.OutputTokens != nil {
+ output += *usage.OutputTokens
+ }
+ if usage.CacheRead != nil && usage.CacheRead.InputTokens != nil {
+ cacheRead += *usage.CacheRead.InputTokens
+ }
+ if usage.Reasoning != nil && usage.Reasoning.OutputTokens != nil {
+ reasoning += *usage.Reasoning.OutputTokens
+ }
+ }
+ if input != 30 || cacheRead != 90 || output != 45 || reasoning != 30 {
+ t.Fatalf("normalized usage = input %d, cache_read %d, output %d, reasoning %d (want 30/90/45/30)", input, cacheRead, output, reasoning)
+ }
+ if input+cacheRead != 120 {
+ t.Fatalf("input_tokens + cache_read = %d, want 120 (= Codex input, no double-count)", input+cacheRead)
+ }
+}
+
func TestEventsFromMetricsUnknownTokenTypeKeepsRawValueOnly(t *testing.T) {
metrics := pmetric.NewMetrics()
resourceMetrics := metrics.ResourceMetrics().AppendEmpty()