Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Flatten token usage for S3 forwarding

shukieshah7d17e6f7de21
diff --git a/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl
index 9275e2a5..2b12e54c 100644
--- a/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl
+++ b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl
@@ -21,6 +21,42 @@ type = "remap"
inputs = ["beacon_runtime"]
source = '''
. = parse_json!(.message)
+if exists(.gen_ai.usage.input_tokens) {
+ value, err = to_int(.gen_ai.usage.input_tokens)
+ if err == null {
+ .input_tokens = value
+ }
+}
+if exists(.gen_ai.usage.output_tokens) {
+ value, err = to_int(.gen_ai.usage.output_tokens)
+ if err == null {
+ .output_tokens = value
+ }
+}
+if exists(.gen_ai.usage.cache_read.input_tokens) {
+ value, err = to_int(.gen_ai.usage.cache_read.input_tokens)
+ if err == null {
+ .cache_read_input_tokens = value
+ }
+}
+if exists(.gen_ai.usage.cache_creation.input_tokens) {
+ value, err = to_int(.gen_ai.usage.cache_creation.input_tokens)
+ if err == null {
+ .cache_creation_input_tokens = value
+ }
+}
+if exists(.gen_ai.usage.reasoning.output_tokens) {
+ value, err = to_int(.gen_ai.usage.reasoning.output_tokens)
+ if err == null {
+ .reasoning_output_tokens = value
+ }
+}
+if exists(.gen_ai.usage.cost_usd) {
+ value, err = to_float(.gen_ai.usage.cost_usd)
+ if err == null {
+ .cost_usd = value
+ }
+}
'''
[transforms.beacon_inventory_json]
diff --git a/cli/beacon/internal/endpoint/s3/s3_test.go b/cli/beacon/internal/endpoint/s3/s3_test.go
index ff26c800..cd2889cf 100644
--- a/cli/beacon/internal/endpoint/s3/s3_test.go
+++ b/cli/beacon/internal/endpoint/s3/s3_test.go
@@ -92,6 +92,10 @@ func TestVectorConfigUsesAWSS3SinkAndPreservesJSONShape(t *testing.T) {
`read_from = "end"`,
`read_from = "beginning"`,
`. = parse_json!(.message)`,
+ `value, err = to_int(.gen_ai.usage.input_tokens)`,
+ `.input_tokens = value`,
+ `value, err = to_float(.gen_ai.usage.cost_usd)`,
+ `.cost_usd = value`,
`type = "aws_s3"`,
`bucket = "${BEACON_S3_BUCKET}"`,
`region = "${AWS_REGION:-us-east-1}"`,