Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(collector): redact telemetry before direct HEC fanout (#959)

tetsu400746df8939
diff --git a/cli/beacon/internal/endpoint/collector/collector.go b/cli/beacon/internal/endpoint/collector/collector.go
index f983adb8..e63d6696 100644
--- a/cli/beacon/internal/endpoint/collector/collector.go
+++ b/cli/beacon/internal/endpoint/collector/collector.go
@@ -140,6 +140,13 @@ func ConfigYAML(cfg endpointconfig.Config) string {
exporterNames = append(exporterNames, "falcon_hec")
}
exporters := "[" + strings.Join(exporterNames, ", ") + "]"
+ // The direct HEC exporters send what they receive, so beacon_privacy redacts before the
+ // fanout. beaconjson redacts in its own writer; with no HEC exporter the processor would only
+ // repeat that on every record.
+ privacyYAML, privacyStep := "", ""
+ if len(exporterNames) > 1 {
+ privacyYAML, privacyStep = " beacon_privacy: {}\n", "beacon_privacy, "
+ }
runtimeMetricsYAML := ""
if cfg.Collector.IncludeRuntimeMetrics {
runtimeMetricsYAML = " include_runtime_metrics: true\n"
@@ -177,7 +184,7 @@ processors:
send_batch_size: 128
claude_api_body:
%s
-exporters:
+%sexporters:
beaconjson:
path: %q
max_event_bytes: 65536
@@ -197,17 +204,17 @@ service:
pipelines:
logs:
receivers: [otlp]
- processors: [memory_limiter, claude_api_body, batch]
+ processors: [memory_limiter, claude_api_body, %sbatch]
exporters: %s
traces:
receivers: [otlp]
- processors: [memory_limiter, batch]
+ processors: [memory_limiter, %sbatch]
exporters: %s
metrics:
receivers: [otlp]
- processors: [memory_limiter, batch]
+ processors: [memory_limiter, %sbatch]
exporters: %s
-`, cfg.Collector.GRPCPort, grpcLimitYAML, cfg.Collector.HTTPPort, modelContextYAML, cfg.LogPath, runtimeMetricsYAML+codexSpansYAML+modelContextYAML+splunkExporter+falconExporter, endpointconfig.HealthCheckPort(cfg.Collector), exporters, exporters, exporters)
+`, cfg.Collector.GRPCPort, grpcLimitYAML, cfg.Collector.HTTPPort, modelContextYAML, privacyYAML, cfg.LogPath, runtimeMetricsYAML+codexSpansYAML+modelContextYAML+splunkExporter+falconExporter, endpointconfig.HealthCheckPort(cfg.Collector), privacyStep, exporters, privacyStep, exporters, privacyStep, exporters)
}
func splunkHECYAML(cfg endpointconfig.Config) string {
diff --git a/cli/beacon/internal/endpoint/collector/collector_test.go b/cli/beacon/internal/endpoint/collector/collector_test.go
index 000edd7c..ef7c166e 100644
--- a/cli/beacon/internal/endpoint/collector/collector_test.go
+++ b/cli/beacon/internal/endpoint/collector/collector_test.go
@@ -14,6 +14,8 @@ import (
"testing"
"time"
+ "gopkg.in/yaml.v3"
+
endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
)
@@ -32,6 +34,56 @@ func testConfig(t *testing.T) endpointconfig.Config {
}
}
+// The direct HEC exporters send what they receive, so privacy has to run before the fanout. The
+// local writer redacts on its own, so a config without them keeps its pipelines unchanged.
+func TestPrivacyRunsBeforeHECFanoutForEverySignal(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ destinations *endpointconfig.Destinations
+ privacy string
+ exporters string
+ }{
+ {"hec", &endpointconfig.Destinations{
+ SplunkHEC: &endpointconfig.SplunkHEC{Endpoint: "https://splunk.example/services/collector", Token: "test-token"},
+ FalconHEC: &endpointconfig.FalconHEC{Endpoint: "https://falcon.example/api/v1/ingest/hec", Token: "test-token"},
+ }, "beacon_privacy,", "beaconjson,splunk_hec,falcon_hec"},
+ {"local only", nil, "", "beaconjson"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ cfg := testConfig(t)
+ cfg.Destinations = tc.destinations
+ var rendered struct {
+ Processors map[string]any `yaml:"processors"`
+ Service struct {
+ Pipelines map[string]struct {
+ Processors []string `yaml:"processors"`
+ Exporters []string `yaml:"exporters"`
+ } `yaml:"pipelines"`
+ } `yaml:"service"`
+ }
+ if err := yaml.Unmarshal([]byte(ConfigYAML(cfg)), &rendered); err != nil {
+ t.Fatal(err)
+ }
+ if _, exists := rendered.Processors["beacon_privacy"]; exists != (tc.privacy != "") {
+ t.Fatalf("beacon_privacy declared = %v, want %v", exists, tc.privacy != "")
+ }
+ for _, signal := range []string{"logs", "traces", "metrics"} {
+ pipeline := rendered.Service.Pipelines[signal]
+ want := "memory_limiter," + tc.privacy + "batch"
+ if signal == "logs" {
+ want = "memory_limiter,claude_api_body," + tc.privacy + "batch"
+ }
+ if strings.Join(pipeline.Processors, ",") != want {
+ t.Fatalf("%s processor order = %v, want %s", signal, pipeline.Processors, want)
+ }
+ if strings.Join(pipeline.Exporters, ",") != tc.exporters {
+ t.Fatalf("%s fanout = %v", signal, pipeline.Exporters)
+ }
+ }
+ })
+ }
+}
+
func TestConfigYAMLIncludesReleaseContractFields(t *testing.T) {
cfg := testConfig(t)
diff --git a/cli/beacon/internal/endpoint/collector/components.go b/cli/beacon/internal/endpoint/collector/components.go
index 95c4cb0a..043112eb 100644
--- a/cli/beacon/internal/endpoint/collector/components.go
+++ b/cli/beacon/internal/endpoint/collector/components.go
@@ -12,7 +12,7 @@ import (
// requiredProcessors are the processors ConfigYAML puts in every pipeline that a collector older
// than this CLI does not have. A collector refuses a whole config that names a component it does
// not know, so one of these missing stops capture for every runtime, not just the one it serves.
-var requiredProcessors = []string{"claude_api_body"}
+var requiredProcessors = []string{"claude_api_body", "beacon_privacy"}
// listComponents runs `beacon-otelcol components`, which prints what the binary was built with.
var listComponents = func(ctx context.Context, binary string) ([]byte, error) {
diff --git a/cli/beacon/internal/endpoint/collector/components_test.go b/cli/beacon/internal/endpoint/collector/components_test.go
index 3730680e..fe2dc33a 100644
--- a/cli/beacon/internal/endpoint/collector/components_test.go
+++ b/cli/beacon/internal/endpoint/collector/components_test.go
@@ -35,12 +35,19 @@ func TestCheckComponentsRefusesACollectorWithoutTheBodyProcessor(t *testing.T) {
}
func TestCheckComponentsAcceptsACollectorWithTheBodyProcessor(t *testing.T) {
- stubComponents(t, releaseComponents+" - name: claude_api_body\n module: github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter v0.0.0\n", nil)
+ stubComponents(t, releaseComponents+" - name: claude_api_body\n - name: beacon_privacy\n", nil)
if err := CheckComponents("beacon-otelcol"); err != nil {
t.Fatalf("CheckComponents = %v, want a collector that has claude_api_body accepted", err)
}
}
+func TestCheckComponentsRefusesACollectorWithoutPrivacy(t *testing.T) {
+ stubComponents(t, releaseComponents+" - name: claude_api_body\n", nil)
+ if err := CheckComponents("beacon-otelcol"); err == nil || !strings.Contains(err.Error(), `"beacon_privacy"`) {
+ t.Fatalf("CheckComponents = %v, want missing privacy processor", err)
+ }
+}
+
// A stand-in that cannot list its components is not judged on them.
func TestCheckComponentsAcceptsABinaryThatCannotListComponents(t *testing.T) {
for name, stub := range map[string]struct {
diff --git a/collector-builder/builder.yaml b/collector-builder/builder.yaml
index 0e39a09b..42b3acfb 100644
--- a/collector-builder/builder.yaml
+++ b/collector-builder/builder.yaml
@@ -12,6 +12,9 @@ processors:
- gomod: github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter v0.0.0
import: github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter/claudeapibodyprocessor
path: ./exporter/beaconjsonexporter
+ - gomod: github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter v0.0.0
+ import: github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter/privacyprocessor
+ path: ./exporter/beaconjsonexporter
exporters:
- gomod: github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter v0.0.0
diff --git a/collector-builder/exporter/beaconjsonexporter/falconhecexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/falconhecexporter/exporter_test.go
index fbdc0a51..1e32eb2e 100644
--- a/collector-builder/exporter/beaconjsonexporter/falconhecexporter/exporter_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/falconhecexporter/exporter_test.go
@@ -9,14 +9,60 @@ import (
"testing"
"time"
+ "go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/config/configretry"
+ "go.opentelemetry.io/collector/consumer"
"go.opentelemetry.io/collector/exporter"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/plog"
+ "go.opentelemetry.io/collector/processor/processortest"
"github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter/internal/beaconevent"
+ "github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter/privacyprocessor"
)
+func TestPrivacyPipelineRedactsBeforeFalconHTTP(t *testing.T) {
+ const secret = "synthetic-forwarding-secret"
+ var received string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var payload hecPayload
+ if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+ t.Error(err)
+ }
+ data, _ := json.Marshal(payload.Event)
+ received = string(data)
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+ cfg := createDefaultConfig()
+ cfg.Endpoint, cfg.Token = server.URL, "test-ingest-token"
+ exp, err := newExporter(cfg, exporter.Settings{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ next, err := consumer.NewLogs(exp.consumeLogs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ proc, err := privacyprocessor.NewFactory().CreateLogs(context.Background(), processortest.NewNopSettings(component.MustNewType("beacon_privacy")), &privacyprocessor.Config{}, next)
+ if err != nil {
+ t.Fatal(err)
+ }
+ logs := plog.NewLogs()
+ record := logs.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty()
+ record.Body().SetStr("prompt submitted")
+ record.Attributes().PutStr("beacon.event.action", "prompt.submitted")
+ record.Attributes().PutStr("service.name", "claude-code")
+ record.Attributes().PutStr("gen_ai.prompt", "keep ordinary text token="+secret)
+ record.Attributes().PutStr("password", secret)
+ if err := proc.ConsumeLogs(context.Background(), logs); err != nil {
+ t.Fatal(err)
+ }
+ if received == "" || strings.Contains(received, secret) || !strings.Contains(received, "[REDACTED]") || !strings.Contains(received, "keep ordinary text") {
+ t.Fatalf("HEC payload did not preserve redacted prompt: %s", received)
+ }
+}
+
func TestConfigValidateRequiresEndpointAndToken(t *testing.T) {
cfg := createDefaultConfig()
if err := cfg.Validate(); err == nil {
diff --git a/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor.go b/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor.go
new file mode 100644
index 00000000..269854c0
--- /dev/null
+++ b/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor.go
@@ -0,0 +1,208 @@
+// Package privacyprocessor redacts secrets before telemetry fans out to exporters.
+// Exporter-local sanitization cannot protect a sibling exporter such as splunk_hec.
+package privacyprocessor
+
+import (
+ "context"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+ "go.opentelemetry.io/collector/component"
+ "go.opentelemetry.io/collector/consumer"
+ "go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/plog"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+ "go.opentelemetry.io/collector/pdata/ptrace"
+ "go.opentelemetry.io/collector/processor"
+ "go.opentelemetry.io/collector/processor/processorhelper"
+)
+
+var componentType = component.MustNewType("beacon_privacy")
+
+type Config struct{}
+
+func NewFactory() processor.Factory {
+ return processor.NewFactory(componentType, func() component.Config { return &Config{} },
+ processor.WithLogs(createLogs, component.StabilityLevelBeta),
+ processor.WithTraces(createTraces, component.StabilityLevelBeta),
+ processor.WithMetrics(createMetrics, component.StabilityLevelBeta))
+}
+
+// Keep encoded message lists intact until the converter parses them. The JSONL
+// writer still applies its normal per-field limits after conversion. This bound
+// is above the generated receiver's largest accepted message (32 MiB).
+const preConversionStringLimit = 64 << 20
+
+var privacy = asymptoteobserve.PrivacyOptions{RedactSecrets: true, StringLimit: preConversionStringLimit}
+
+func redactMap(attrs pcommon.Map) error {
+ return attrs.FromRaw(asymptoteobserve.SanitizeMap(attrs.AsRaw(), privacy))
+}
+
+func redactValue(value pcommon.Value) error {
+ switch value.Type() {
+ case pcommon.ValueTypeStr:
+ value.SetStr(asymptoteobserve.RedactString(value.Str()))
+ case pcommon.ValueTypeMap:
+ return redactMap(value.Map())
+ case pcommon.ValueTypeSlice:
+ return value.FromRaw(asymptoteobserve.SanitizeSlice(value.Slice().AsRaw(), privacy))
+ }
+ return nil
+}
+
+func createLogs(ctx context.Context, set processor.Settings, cfg component.Config, next consumer.Logs) (processor.Logs, error) {
+ return processorhelper.NewLogs(ctx, set, cfg, next, processLogs,
+ processorhelper.WithCapabilities(consumer.Capabilities{MutatesData: true}))
+}
+
+func processLogs(_ context.Context, logs plog.Logs) (plog.Logs, error) {
+ for resourceIndex := 0; resourceIndex < logs.ResourceLogs().Len(); resourceIndex++ {
+ resource := logs.ResourceLogs().At(resourceIndex)
+ if err := redactMap(resource.Resource().Attributes()); err != nil {
+ return logs, err
+ }
+ for scopeIndex := 0; scopeIndex < resource.ScopeLogs().Len(); scopeIndex++ {
+ scope := resource.ScopeLogs().At(scopeIndex)
+ if err := redactMap(scope.Scope().Attributes()); err != nil {
+ return logs, err
+ }
+ for recordIndex := 0; recordIndex < scope.LogRecords().Len(); recordIndex++ {
+ record := scope.LogRecords().At(recordIndex)
+ if err := redactMap(record.Attributes()); err != nil {
+ return logs, err
+ }
+ if err := redactValue(record.Body()); err != nil {
+ return logs, err
+ }
+ }
+ }
+ }
+ return logs, nil
+}
+
+func createTraces(ctx context.Context, set processor.Settings, cfg component.Config, next consumer.Traces) (processor.Traces, error) {
+ return processorhelper.NewTraces(ctx, set, cfg, next, processTraces,
+ processorhelper.WithCapabilities(consumer.Capabilities{MutatesData: true}))
+}
+
+func processTraces(_ context.Context, traces ptrace.Traces) (ptrace.Traces, error) {
+ for resourceIndex := 0; resourceIndex < traces.ResourceSpans().Len(); resourceIndex++ {
+ resource := traces.ResourceSpans().At(resourceIndex)
+ if err := redactMap(resource.Resource().Attributes()); err != nil {
+ return traces, err
+ }
+ for scopeIndex := 0; scopeIndex < resource.ScopeSpans().Len(); scopeIndex++ {
+ scope := resource.ScopeSpans().At(scopeIndex)
+ if err := redactMap(scope.Scope().Attributes()); err != nil {
+ return traces, err
+ }
+ for spanIndex := 0; spanIndex < scope.Spans().Len(); spanIndex++ {
+ span := scope.Spans().At(spanIndex)
+ span.SetName(asymptoteobserve.RedactString(span.Name()))
+ span.Status().SetMessage(asymptoteobserve.RedactString(span.Status().Message()))
+ if err := redactMap(span.Attributes()); err != nil {
+ return traces, err
+ }
+ for eventIndex := 0; eventIndex < span.Events().Len(); eventIndex++ {
+ event := span.Events().At(eventIndex)
+ event.SetName(asymptoteobserve.RedactString(event.Name()))
+ if err := redactMap(event.Attributes()); err != nil {
+ return traces, err
+ }
+ }
+ for linkIndex := 0; linkIndex < span.Links().Len(); linkIndex++ {
+ link := span.Links().At(linkIndex)
+ if err := redactMap(link.Attributes()); err != nil {
+ return traces, err
+ }
+ }
+ }
+ }
+ }
+ return traces, nil
+}
+
+func createMetrics(ctx context.Context, set processor.Settings, cfg component.Config, next consumer.Metrics) (processor.Metrics, error) {
+ return processorhelper.NewMetrics(ctx, set, cfg, next, processMetrics,
+ processorhelper.WithCapabilities(consumer.Capabilities{MutatesData: true}))
+}
+
+func redactExemplars(exemplars pmetric.ExemplarSlice) error {
+ for exemplarIndex := 0; exemplarIndex < exemplars.Len(); exemplarIndex++ {
+ exemplar := exemplars.At(exemplarIndex)
+ if err := redactMap(exemplar.FilteredAttributes()); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func redactNumberPoints(points pmetric.NumberDataPointSlice) error {
+ for pointIndex := 0; pointIndex < points.Len(); pointIndex++ {
+ point := points.At(pointIndex)
+ if err := redactMap(point.Attributes()); err != nil {
+ return err
+ }
+ if err := redactExemplars(point.Exemplars()); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func processMetrics(_ context.Context, metrics pmetric.Metrics) (pmetric.Metrics, error) {
+ for resourceIndex := 0; resourceIndex < metrics.ResourceMetrics().Len(); resourceIndex++ {
+ resource := metrics.ResourceMetrics().At(resourceIndex)
+ if err := redactMap(resource.Resource().Attributes()); err != nil {
+ return metrics, err
+ }
+ for scopeIndex := 0; scopeIndex < resource.ScopeMetrics().Len(); scopeIndex++ {
+ scope := resource.ScopeMetrics().At(scopeIndex)
+ if err := redactMap(scope.Scope().Attributes()); err != nil {
+ return metrics, err
+ }
+ for metricIndex := 0; metricIndex < scope.Metrics().Len(); metricIndex++ {
+ metric := scope.Metrics().At(metricIndex)
+ metric.SetDescription(asymptoteobserve.RedactString(metric.Description()))
+ switch metric.Type() {
+ case pmetric.MetricTypeGauge:
+ if err := redactNumberPoints(metric.Gauge().DataPoints()); err != nil {
+ return metrics, err
+ }
+ case pmetric.MetricTypeSum:
+ if err := redactNumberPoints(metric.Sum().DataPoints()); err != nil {
+ return metrics, err
+ }
+ case pmetric.MetricTypeHistogram:
+ for pointIndex := 0; pointIndex < metric.Histogram().DataPoints().Len(); pointIndex++ {
+ point := metric.Histogram().DataPoints().At(pointIndex)
+ if err := redactMap(point.Attributes()); err != nil {
+ return metrics, err
+ }
+ if err := redactExemplars(point.Exemplars()); err != nil {
+ return metrics, err
+ }
+ }
+ case pmetric.MetricTypeExponentialHistogram:
+ for pointIndex := 0; pointIndex < metric.ExponentialHistogram().DataPoints().Len(); pointIndex++ {
+ point := metric.ExponentialHistogram().DataPoints().At(pointIndex)
+ if err := redactMap(point.Attributes()); err != nil {
+ return metrics, err
+ }
+ if err := redactExemplars(point.Exemplars()); err != nil {
+ return metrics, err
+ }
+ }
+ case pmetric.MetricTypeSummary:
+ for pointIndex := 0; pointIndex < metric.Summary().DataPoints().Len(); pointIndex++ {
+ point := metric.Summary().DataPoints().At(pointIndex)
+ if err := redactMap(point.Attributes()); err != nil {
+ return metrics, err
+ }
+ }
+ }
+ }
+ }
+ }
+ return metrics, nil
+}
diff --git a/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go b/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go
new file mode 100644
index 00000000..e2a06f07
--- /dev/null
+++ b/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go
@@ -0,0 +1,198 @@
+package privacyprocessor
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "go.opentelemetry.io/collector/consumer/consumertest"
+ "go.opentelemetry.io/collector/pdata/pcommon"
+ "go.opentelemetry.io/collector/pdata/plog"
+ "go.opentelemetry.io/collector/pdata/pmetric"
+ "go.opentelemetry.io/collector/pdata/ptrace"
+ "go.opentelemetry.io/collector/processor/processortest"
+)
+
+const credential = "synthetic-private-value"
+
+func secretAttrs(t *testing.T, attrs pcommon.Map) {
+ t.Helper()
+ if err := attrs.FromRaw(map[string]any{
+ "password": credential,
+ "nested": map[string]any{"Cookie": []any{credential}, "text": "token=" + credential},
+ "gen_ai.usage.input_tokens": int64(812),
+ "model": "ordinary-model",
+ }); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func assertRedacted(t *testing.T, attrs pcommon.Map) {
+ t.Helper()
+ data, err := json.Marshal(attrs.AsRaw())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(data), credential) || !strings.Contains(string(data), "[REDACTED]") {
+ t.Fatalf("secret reached downstream consumer: %s", data)
+ }
+ if count, ok := attrs.Get("gen_ai.usage.input_tokens"); !ok || count.Int() != 812 {
+ t.Fatal("numeric token usage changed")
+ }
+ if model, ok := attrs.Get("model"); !ok || model.Str() != "ordinary-model" {
+ t.Fatal("non-sensitive metadata changed")
+ }
+}
+
+func TestLogsRedactedBeforeDownstreamConsumer(t *testing.T) {
+ sink := new(consumertest.LogsSink)
+ proc, err := NewFactory().CreateLogs(context.Background(), processortest.NewNopSettings(componentType), &Config{}, sink)
+ if err != nil {
+ t.Fatal(err)
+ }
+ logs := plog.NewLogs()
+ resource := logs.ResourceLogs().AppendEmpty()
+ secretAttrs(t, resource.Resource().Attributes())
+ scope := resource.ScopeLogs().AppendEmpty()
+ secretAttrs(t, scope.Scope().Attributes())
+ for _, kind := range []string{"string", "map", "slice"} {
+ record := scope.LogRecords().AppendEmpty()
+ secretAttrs(t, record.Attributes())
+ switch kind {
+ case "string":
+ record.Body().SetStr("Authorization: Bearer " + credential)
+ case "map":
+ secretAttrs(t, record.Body().SetEmptyMap())
+ case "slice":
+ if err := record.Body().FromRaw([]any{"token=" + credential, map[string]any{"password": credential}}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ // A message-list attribute must still be parseable by the event converter;
+ // applying the JSONL field cap at this boundary would corrupt its JSON.
+ longMessages := `[{"role":"user","content":"` + strings.Repeat("x", 8192) + ` token=` + credential + `"}]`
+ scope.LogRecords().At(0).Attributes().PutStr("gen_ai.input.messages", longMessages)
+ if err := proc.ConsumeLogs(context.Background(), logs); err != nil {
+ t.Fatal(err)
+ }
+ got := sink.AllLogs()[0].ResourceLogs().At(0)
+ assertRedacted(t, got.Resource().Attributes())
+ assertRedacted(t, got.ScopeLogs().At(0).Scope().Attributes())
+ for recordIndex := 0; recordIndex < got.ScopeLogs().At(0).LogRecords().Len(); recordIndex++ {
+ record := got.ScopeLogs().At(0).LogRecords().At(recordIndex)
+ assertRedacted(t, record.Attributes())
+ body, _ := json.Marshal(record.Body().AsRaw())
+ if strings.Contains(string(body), credential) {
+ t.Fatalf("body leaked: %s", body)
+ }
+ }
+ encoded, _ := got.ScopeLogs().At(0).LogRecords().At(0).Attributes().Get("gen_ai.input.messages")
+ if !json.Valid([]byte(encoded.Str())) || len(encoded.Str()) < 8192 {
+ t.Fatal("encoded conversation was truncated before conversion")
+ }
+}
+
+func TestTracesRedactedWithoutChangingIdentifiers(t *testing.T) {
+ sink := new(consumertest.TracesSink)
+ proc, err := NewFactory().CreateTraces(context.Background(), processortest.NewNopSettings(componentType), &Config{}, sink)
+ if err != nil {
+ t.Fatal(err)
+ }
+ traces := ptrace.NewTraces()
+ resource := traces.ResourceSpans().AppendEmpty()
+ secretAttrs(t, resource.Resource().Attributes())
+ scope := resource.ScopeSpans().AppendEmpty()
+ secretAttrs(t, scope.Scope().Attributes())
+ span := scope.Spans().AppendEmpty()
+ span.SetTraceID(pcommon.TraceID{1, 2, 3})
+ span.SetSpanID(pcommon.SpanID{4, 5, 6})
+ span.SetStartTimestamp(1234)
+ span.SetName("token=" + credential)
+ span.Status().SetMessage("token=" + credential)
+ secretAttrs(t, span.Attributes())
+ event := span.Events().AppendEmpty()
+ event.SetName("token=" + credential)
+ secretAttrs(t, event.Attributes())
+ secretAttrs(t, span.Links().AppendEmpty().Attributes())
+ if err := proc.ConsumeTraces(context.Background(), traces); err != nil {
+ t.Fatal(err)
+ }
+ got := sink.AllTraces()[0].ResourceSpans().At(0)
+ assertRedacted(t, got.Resource().Attributes())
+ assertRedacted(t, got.ScopeSpans().At(0).Scope().Attributes())
+ s := got.ScopeSpans().At(0).Spans().At(0)
+ assertRedacted(t, s.Attributes())
+ assertRedacted(t, s.Events().At(0).Attributes())
+ assertRedacted(t, s.Links().At(0).Attributes())
+ if s.TraceID() != (pcommon.TraceID{1, 2, 3}) || s.SpanID() != (pcommon.SpanID{4, 5, 6}) || s.StartTimestamp() != 1234 {
+ t.Fatal("trace identity or timestamp changed")
+ }
+ for _, value := range []string{s.Name(), s.Status().Message(), s.Events().At(0).Name()} {
+ if strings.Contains(value, credential) {
+ t.Fatal("trace text leaked")
+ }
+ }
+}
+
+func TestEveryMetricKindRedactsLabelsAndExemplars(t *testing.T) {
+ sink := new(consumertest.MetricsSink)
+ proc, err := NewFactory().CreateMetrics(context.Background(), processortest.NewNopSettings(componentType), &Config{}, sink)
+ if err != nil {
+ t.Fatal(err)
+ }
+ metrics := pmetric.NewMetrics()
+ resource := metrics.ResourceMetrics().AppendEmpty()
+ secretAttrs(t, resource.Resource().Attributes())
+ scope := resource.ScopeMetrics().AppendEmpty()
+ secretAttrs(t, scope.Scope().Attributes())
+ for _, kind := range []string{"gauge", "sum", "histogram", "exponential", "summary"} {
+ metric := scope.Metrics().AppendEmpty()
+ metric.SetName(kind)
+ metric.SetDescription("token=" + credential)
+ var attrs pcommon.Map
+ var exemplars pmetric.ExemplarSlice
+ switch kind {
+ case "gauge":
+ point := metric.SetEmptyGauge().DataPoints().AppendEmpty()
+ point.SetIntValue(42)
+ attrs, exemplars = point.Attributes(), point.Exemplars()
+ case "sum":
+ point := metric.SetEmptySum().DataPoints().AppendEmpty()
+ point.SetIntValue(42)
+ attrs, exemplars = point.Attributes(), point.Exemplars()
+ case "histogram":
+ point := metric.SetEmptyHistogram().DataPoints().AppendEmpty()
+ point.SetCount(42)
+ attrs, exemplars = point.Attributes(), point.Exemplars()
+ case "exponential":
+ point := metric.SetEmptyExponentialHistogram().DataPoints().AppendEmpty()
+ point.SetCount(42)
+ attrs, exemplars = point.Attributes(), point.Exemplars()
+ case "summary":
+ point := metric.SetEmptySummary().DataPoints().AppendEmpty()
+ point.SetCount(42)
+ attrs = point.Attributes()
+ }
+ secretAttrs(t, attrs)
+ if kind != "summary" {
+ secretAttrs(t, exemplars.AppendEmpty().FilteredAttributes())
+ }
+ }
+ if err := proc.ConsumeMetrics(context.Background(), metrics); err != nil {
+ t.Fatal(err)
+ }
+ got := sink.AllMetrics()[0]
+ data, err := (&pmetric.JSONMarshaler{}).MarshalMetrics(got)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(data), credential) {
+ t.Fatal("metric secret reached downstream consumer")
+ }
+ ms := got.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics()
+ if ms.Len() != 5 || ms.At(0).Gauge().DataPoints().At(0).IntValue() != 42 || ms.At(1).Sum().DataPoints().At(0).IntValue() != 42 || ms.At(2).Histogram().DataPoints().At(0).Count() != 42 || ms.At(3).ExponentialHistogram().DataPoints().At(0).Count() != 42 || ms.At(4).Summary().DataPoints().At(0).Count() != 42 {
+ t.Fatal("metric type, point count or numeric values changed")
+ }
+}
diff --git a/docs/changelog/cli.mdx b/docs/changelog/cli.mdx
index cc0baed4..9afd6e5d 100644
--- a/docs/changelog/cli.mdx
+++ b/docs/changelog/cli.mdx
@@ -22,6 +22,7 @@ Stay up to date with the latest changes to the Agent Beacon CLI.
- **Failed Grok writes are recorded as failures** · When a Grok `write_file` or edit failed, the hook recorded it as a completed `file.modified` with a diff built from content that was never written, and the threat scan read that content as a change. The failed attempt is now a high-severity `tool.failed` with its tool, path and call ID and no diff. Successful writes are unchanged.
- **Vector forwarding configs work with Windows log paths** · Beacon wrote log paths into the Vector configs for Beacon Cloud and the SIEM packs without escaping them, so on Windows a path's backslashes made `vector.toml` fail to parse, or quietly point Vector at a different file. Paths are now escaped and Vector reads exactly the configured runtime, inventory and backfill logs.
- **Memory candidate and evaluation search finds older matches** · `beacon memory candidates list` and `beacon memory evaluations list` with `--query` paged before matching, so a page could come back empty or skip matches whenever newer records that didn't match filled it. Searches now page over matching records, as `beacon memory list` already did.
+- **Splunk and Falcon HEC forwarding redacts secrets** · With direct Splunk HEC or Falcon LogScale HEC forwarding configured, credentials in prompts, tool output and other telemetry reached the SIEM unredacted, while the local runtime log redacted them. The collector now redacts secrets before telemetry fans out to those exporters, with the same rules as the local log. Installs without HEC forwarding are unchanged.
- **Dashboard output tokens count reasoning once** · The local dashboard's token view added reasoning tokens to output, but `gen_ai.usage.output_tokens` already includes them, so models that report reasoning had their output and total tokens overstated. Output and the total now count reasoning once, the split bar no longer shows reasoning as a separate segment, and a note below it gives the reasoning count included in output. `beacon token-usage` already counted it this way.
---
diff --git a/docs/log-forwarding/falcon.mdx b/docs/log-forwarding/falcon.mdx
index e430c67f..a56133f9 100644
--- a/docs/log-forwarding/falcon.mdx
+++ b/docs/log-forwarding/falcon.mdx
@@ -303,7 +303,7 @@ sudo launchctl print system/com.beacon.endpoint.collector
## Content handling
-Beacon applies redaction, sanitization, truncation, and event-size limits before events are written locally or forwarded through Falcon HEC. Review repository access, parser behavior, retention, and downstream consumers so retained telemetry matches your approved collection policy.
+Beacon redacts secrets before events are written locally or forwarded through Falcon HEC. Truncation and event-size limits apply to the local runtime log only, so forwarded events can carry longer content than the log does. Review repository access, parser behavior, retention, and downstream consumers so retained telemetry matches your approved collection policy.
## Related
diff --git a/docs/log-forwarding/splunk.mdx b/docs/log-forwarding/splunk.mdx
index 38096eaf..d24765f8 100644
--- a/docs/log-forwarding/splunk.mdx
+++ b/docs/log-forwarding/splunk.mdx
@@ -92,9 +92,9 @@ When Splunk HEC is configured, Beacon writes a collector pipeline with both expo
exporters: [beaconjson, splunk_hec]
```
-The `beaconjson` exporter preserves the local runtime log. The `splunk_hec` exporter sends OTLP logs, traces, and metrics to Splunk HEC, with their attributes as received.
+The `beaconjson` exporter preserves the local runtime log. The `splunk_hec` exporter sends OTLP logs, traces, and metrics to Splunk HEC. Before they reach it, the `beacon_privacy` processor redacts secrets with the same rules as the local runtime log, leaving identifiers and usage numbers unchanged. Redaction is pattern-based, so treat forwarded content as sensitive.
-Before log records reach either exporter, the logs pipeline runs Beacon's `claude_api_body` processor (`processors: [memory_limiter, claude_api_body, batch]`). It drops Claude Code API request and response bodies, which can contain the system prompt, conversation history, and tool results. When Beacon was installed with `--claude-capture-model-context`, two kinds of request are forwarded with their body replaced by what Beacon keeps, the same copy the runtime log stores. The WebFetch summarizer request carries the fetched page text in `beacon.web_fetch.input`, with secrets redacted and capped at 2048 characters. A request that advertises MCP tools carries `beacon.mcp.tools`, the name and redacted description of each tool the session has not been sent before. See [Claude Code: model context from API bodies](/runtimes/claude-code#model-context-from-api-bodies).
+Before log records reach either exporter, the logs pipeline runs Beacon's `claude_api_body` processor, followed by `beacon_privacy` (`processors: [memory_limiter, claude_api_body, beacon_privacy, batch]`). The body processor drops Claude Code API request and response bodies, which can contain the system prompt, conversation history, and tool results. When Beacon was installed with `--claude-capture-model-context`, two kinds of request are forwarded with their body replaced by what Beacon keeps, the same copy the runtime log stores. The WebFetch summarizer request carries the fetched page text in `beacon.web_fetch.input`, with secrets redacted and capped at 2048 characters. A request that advertises MCP tools carries `beacon.mcp.tools`, the name and redacted description of each tool the session has not been sent before. See [Claude Code: model context from API bodies](/runtimes/claude-code#model-context-from-api-bodies).
## Validate forwarding