The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Flatten token usage for S3 forwarding
shukieshah7d17e6f7de21
diff --git a/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmplindex 9275e2a5..2b12e54c 100644--- a/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl+++ b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl@@ -21,6 +21,42 @@ type = "remap"inputs = ["beacon_runtime"]source = '''. = parse_json!(.message)+if exists(.gen_ai.usage.input_tokens) {+ value, err = to_int(.gen_ai.usage.input_tokens)+ if err == null {+ .input_tokens = value+ }+}+if exists(.gen_ai.usage.output_tokens) {+ value, err = to_int(.gen_ai.usage.output_tokens)+ if err == null {+ .output_tokens = value+ }+}+if exists(.gen_ai.usage.cache_read.input_tokens) {+ value, err = to_int(.gen_ai.usage.cache_read.input_tokens)+ if err == null {+ .cache_read_input_tokens = value+ }+}+if exists(.gen_ai.usage.cache_creation.input_tokens) {+ value, err = to_int(.gen_ai.usage.cache_creation.input_tokens)+ if err == null {+ .cache_creation_input_tokens = value+ }+}+if exists(.gen_ai.usage.reasoning.output_tokens) {+ value, err = to_int(.gen_ai.usage.reasoning.output_tokens)+ if err == null {+ .reasoning_output_tokens = value+ }+}+if exists(.gen_ai.usage.cost_usd) {+ value, err = to_float(.gen_ai.usage.cost_usd)+ if err == null {+ .cost_usd = value+ }+}'''[transforms.beacon_inventory_json]diff --git a/cli/beacon/internal/endpoint/s3/s3_test.go b/cli/beacon/internal/endpoint/s3/s3_test.goindex ff26c800..cd2889cf 100644--- a/cli/beacon/internal/endpoint/s3/s3_test.go+++ b/cli/beacon/internal/endpoint/s3/s3_test.go@@ -92,6 +92,10 @@ func TestVectorConfigUsesAWSS3SinkAndPreservesJSONShape(t *testing.T) {`read_from = "end"`,`read_from = "beginning"`,`. = parse_json!(.message)`,+ `value, err = to_int(.gen_ai.usage.input_tokens)`,+ `.input_tokens = value`,+ `value, err = to_float(.gen_ai.usage.cost_usd)`,+ `.cost_usd = value`,`type = "aws_s3"`,`bucket = "${BEACON_S3_BUCKET}"`,`region = "${AWS_REGION:-us-east-1}"`,