Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(privacy): sanitize values of every type in SanitizeMap (#1151)

kyle-elliott-asymptoteade6f5a4734a
diff --git a/cli/beacon/cmd/endpoint_diagnostics.go b/cli/beacon/cmd/endpoint_diagnostics.go
index 6b070f9d..1a8e385a 100644
--- a/cli/beacon/cmd/endpoint_diagnostics.go
+++ b/cli/beacon/cmd/endpoint_diagnostics.go
@@ -34,6 +34,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionfs"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/version"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
type doctorResult struct {
@@ -669,6 +670,14 @@ func writeInventorySnapshotEvents(cfg endpointconfig.Config, settings endpointco
Harness: schema.HarnessInfo{Name: "endpoint"},
Message: "Agent configuration inventory snapshot observed",
})
+ // The scanner has already redacted what it captures and cut file contents to max_content_bytes;
+ // an MCP server definition, whose keys come from the user's config, had the field-name and flag
+ // rules applied (inventory.redactStructured). CleanValue redacts every string in them again and
+ // leaves our own field names such as env_keys, a list of variable names, as they are. Its limit
+ // is max_content_bytes, so captured contents are not cut again, but never below the raw-field
+ // limit the event writer applies to other raw strings, so a small max_content_bytes does not cut
+ // paths, names and hashes.
+ contentLimit := max(endpointinventory.ContentLimit(settings.MaxContentBytes), asymptoteobserve.DefaultRawStringLimit)
snapshot.Raw = map[string]interface{}{
"inventory": map[string]interface{}{
"generated_at": result.GeneratedAt,
@@ -678,9 +687,9 @@ func writeInventorySnapshotEvents(cfg endpointconfig.Config, settings endpointco
"previous_snapshot_digest": previousDigest,
"counts": counts,
"user_scope": result.UserScope,
- "configs": existingInventoryConfigs(result.Configs),
- "mcp_servers": result.MCPServers,
- "skills": existingInventorySkills(result.Skills),
+ "configs": asymptoteobserve.CleanValue(existingInventoryConfigs(result.Configs), contentLimit),
+ "mcp_servers": asymptoteobserve.CleanValue(result.MCPServers, contentLimit),
+ "skills": asymptoteobserve.CleanValue(existingInventorySkills(result.Skills), contentLimit),
},
}
if _, err := writer.AppendEvent(snapshot, writer.Options{Path: inventoryLogPath, UserMode: cfg.UserMode}); err != nil {
diff --git a/cli/beacon/cmd/endpoint_test.go b/cli/beacon/cmd/endpoint_test.go
index 00eb1366..d73dad5d 100644
--- a/cli/beacon/cmd/endpoint_test.go
+++ b/cli/beacon/cmd/endpoint_test.go
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
+ "math"
"os"
"path/filepath"
"runtime"
@@ -1180,6 +1181,160 @@ func TestRunEndpointInventoryWriteEventHonorsConfigContentOptIn(t *testing.T) {
}
}
+// inventorySnapshot runs `beacon endpoint inventory --json` with event writing and content capture
+// on, and returns the JSON it printed and the inventory.snapshot line it wrote.
+func inventorySnapshot(t *testing.T, runtimes []string, maxContentBytes int) (inventoryResult, string) {
+ t.Helper()
+ old := endpointOpts
+ t.Cleanup(func() { endpointOpts = old })
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ include := true
+ cfg := endpointconfig.Default(true, logPath)
+ cfg.Inventory = &endpointconfig.Inventory{Runtimes: runtimes, IncludeContents: &include, MaxContentBytes: maxContentBytes}
+ if _, err := endpointconfig.Save(cfg); err != nil {
+ t.Fatal(err)
+ }
+ endpointOpts.userMode = true
+ endpointOpts.logPath = logPath
+ endpointOpts.jsonOutput = true
+ endpointOpts.writeInventoryEvent = true
+ endpointOpts.inventoryContents = false
+ output, err := captureStdout(t, func() error { return runEndpointInventory(endpointInventoryCmd, nil) })
+ if err != nil {
+ t.Fatalf("runEndpointInventory returned error: %v", err)
+ }
+ var result inventoryResult
+ if err := json.Unmarshal([]byte(output), &result); err != nil {
+ t.Fatalf("inventory JSON did not decode: %v output=%s", err, output)
+ }
+ data, err := os.ReadFile(endpointinventory.LogPath(logPath, true))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, line := range strings.Split(string(data), "\n") {
+ if strings.Contains(line, `"inventory.snapshot"`) {
+ return result, line
+ }
+ }
+ t.Fatalf("no inventory.snapshot in %s", data)
+ return result, ""
+}
+
+// The snapshot keeps opted-in content to max_content_bytes rather than the raw-field limit the event
+// writer applies to other raw strings, and keeps env_keys, a list of variable names, as names. A
+// definition's values held under keys named for a credential are redacted even when nothing about
+// the value itself looks like one.
+func TestInventorySnapshotKeepsContentToItsLimit(t *testing.T) {
+ home := t.TempDir()
+ testenv.SetHome(t, home)
+ description := strings.Repeat("a long server description ", 200)
+ writeTestFile(t, filepath.Join(home, ".cursor", "mcp.json"), `{"mcpServers":{"one":{"command":"npx","description":"`+description+
+ `","env":{"NODE_ENV":"production","API_KEY":"hunter2"},"headers":{"Authorization":"plain"}}}}`)
+ _, snapshot := inventorySnapshot(t, []string{"cursor"}, 16384)
+ for _, secret := range []string{"hunter2", `plain"`, `plain\"`} {
+ if strings.Contains(snapshot, secret) {
+ t.Fatalf("snapshot leaked %s: %s", secret, snapshot)
+ }
+ }
+ for _, want := range []string{`"API_KEY":"[REDACTED]"`, `"Authorization":"[REDACTED]"`} {
+ if !strings.Contains(snapshot, want) {
+ t.Fatalf("snapshot definition missing %s: %.400s", want, snapshot)
+ }
+ }
+ if !strings.Contains(snapshot, description) {
+ t.Fatalf("snapshot cut the captured content short of max_content_bytes: %.400s", snapshot)
+ }
+ if !strings.Contains(snapshot, `"env_keys":["NODE_ENV"]`) {
+ t.Fatalf("snapshot did not keep env_keys as names: %.400s", snapshot)
+ }
+}
+
+// A small max_content_bytes limits captured contents only: paths, server names and hashes in the
+// snapshot stay whole.
+func TestInventorySnapshotKeepsMetadataWholeUnderASmallContentLimit(t *testing.T) {
+ home := filepath.Join(t.TempDir(), strings.Repeat("deep-directory-name-", 4))
+ testenv.SetHome(t, home)
+ name := strings.Repeat("server-with-a-long-name-", 4)
+ writeTestFile(t, filepath.Join(home, ".cursor", "mcp.json"), `{"mcpServers":{"`+name+`":{"command":"npx"}}}`)
+ result, snapshot := inventorySnapshot(t, []string{"cursor"}, 64)
+ var kept []string
+ for _, config := range result.Configs {
+ if config.Exists {
+ kept = append(kept, config.Path, config.PathHash)
+ }
+ }
+ for _, server := range result.MCPServers {
+ kept = append(kept, server.ServerName, server.ServerNameHash, server.DefinitionHash)
+ }
+ long := 0
+ for _, value := range kept {
+ if len(value) <= 64 {
+ continue
+ }
+ long++
+ quoted, _ := json.Marshal(value)
+ if !strings.Contains(snapshot, string(quoted)) {
+ t.Errorf("snapshot cut %s: %.600s", quoted, snapshot)
+ }
+ }
+ if long == 0 {
+ t.Fatalf("no path, name or hash over 64 bytes to check: %q", kept)
+ }
+}
+
+// max_content_bytes set to the largest int, as a configuration may for "unlimited", keeps captured
+// contents whole and redacted rather than panicking where the redaction window is worked out.
+func TestInventorySnapshotTakesAnUnlimitedContentLimit(t *testing.T) {
+ home := t.TempDir()
+ testenv.SetHome(t, home)
+ description := strings.Repeat("a long server description ", 400)
+ writeTestFile(t, filepath.Join(home, ".cursor", "mcp.json"), `{"mcpServers":{"one":{"command":"npx","description":"`+description+
+ `","args":["--token","ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"],"env":{"API_KEY":"hunter2"}}}}`)
+ _, snapshot := inventorySnapshot(t, []string{"cursor"}, math.MaxInt)
+ for _, secret := range []string{"hunter2", "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"} {
+ if strings.Contains(snapshot, secret) {
+ t.Fatalf("snapshot leaked %s: %.400s", secret, snapshot)
+ }
+ }
+ if strings.Count(snapshot, description) < 2 {
+ t.Fatalf("snapshot did not keep the content and the definition whole: %.400s", snapshot)
+ }
+}
+
+// One MCP server whose definition JSON cannot represent (a TOML nan) leaves the other servers, and
+// the rest of its own definition, in the scheduled snapshot; only the value itself is null.
+func TestInventorySnapshotKeepsServersBesideAnUnrepresentableValue(t *testing.T) {
+ home := t.TempDir()
+ testenv.SetHome(t, home)
+ t.Setenv("CODEX_HOME", filepath.Join(home, ".codex"))
+ writeTestFile(t, filepath.Join(home, ".codex", "config.toml"), "[mcp_servers.first]\ncommand = \"npx\"\n\n"+
+ "[mcp_servers.broken]\ncommand = \"uvx\"\nstartup_timeout_sec = nan\n\n[mcp_servers.last]\ncommand = \"node\"\n")
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ cfg := endpointconfig.Default(true, logPath)
+ settings := endpointconfig.InventoryConfig(cfg)
+ settings.Runtimes = []string{"codex_cli"}
+ settings.IncludeContents = true
+ result, err := writeInventoryHeartbeat(cfg, settings, false, inventoryTriggerScheduled, heartbeatScope{})
+ if err != nil || !result.SnapshotWritten {
+ t.Fatalf("heartbeat = %#v, %v; want a snapshot written", result, err)
+ }
+ data, err := os.ReadFile(endpointinventory.LogPath(logPath, true))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var snapshot string
+ for _, line := range strings.Split(string(data), "\n") {
+ if strings.Contains(line, `"inventory.snapshot"`) {
+ snapshot = line
+ }
+ }
+ for _, want := range []string{`"server_name":"first"`, `"server_name":"broken"`, `"server_name":"last"`, `"command":"uvx"`, `"startup_timeout_sec":null`} {
+ if !strings.Contains(snapshot, want) {
+ t.Errorf("snapshot missing %s: %.3000s", want, snapshot)
+ }
+ }
+}
+
func TestConfigAutoUpdateModeDoesNotFallBackOnSystemLoadError(t *testing.T) {
systemErr := fmt.Errorf("invalid system config")
got := configAutoUpdateModeFrom(
diff --git a/cli/beacon/internal/endpoint/inventory/content.go b/cli/beacon/internal/endpoint/inventory/content.go
index 203e5deb..1f7ecfe1 100644
--- a/cli/beacon/internal/endpoint/inventory/content.go
+++ b/cli/beacon/internal/endpoint/inventory/content.go
@@ -61,6 +61,11 @@ func (c contentOptions) limit() int {
return DefaultMaxContentBytes
}
+// ContentLimit is how many bytes of a captured file a scan given MaxContentBytes maxBytes keeps.
+func ContentLimit(maxBytes int) int {
+ return contentOptions{maxBytes: maxBytes}.limit()
+}
+
// captureContent redacts secrets in the raw file body and truncates the result
// to maxBytes on a UTF-8 rune boundary. Redaction runs before truncation so a
// secret can never be partially exposed by the cut.
diff --git a/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go b/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go
index e2a06f07..79f782e5 100644
--- a/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/privacyprocessor/processor_test.go
@@ -1,6 +1,7 @@
package privacyprocessor
import (
+ "bytes"
"context"
"encoding/json"
"strings"
@@ -23,6 +24,8 @@ func secretAttrs(t *testing.T, attrs pcommon.Map) {
"nested": map[string]any{"Cookie": []any{credential}, "text": "token=" + credential},
"gen_ai.usage.input_tokens": int64(812),
"model": "ordinary-model",
+ "payload": []byte("token=" + credential),
+ "blob": []byte{0xff, 0x00, 0x01},
}); err != nil {
t.Fatal(err)
}
@@ -43,6 +46,14 @@ func assertRedacted(t *testing.T, attrs pcommon.Map) {
if model, ok := attrs.Get("model"); !ok || model.Str() != "ordinary-model" {
t.Fatal("non-sensitive metadata changed")
}
+ // A bytes attribute stays bytes: text in it is redacted, binary data is kept.
+ if payload, ok := attrs.Get("payload"); !ok || payload.Type() != pcommon.ValueTypeBytes ||
+ strings.Contains(string(payload.Bytes().AsRaw()), credential) {
+ t.Fatalf("bytes attribute not kept as redacted bytes: %#v", attrs.AsRaw()["payload"])
+ }
+ if blob, ok := attrs.Get("blob"); !ok || blob.Type() != pcommon.ValueTypeBytes || !bytes.Equal(blob.Bytes().AsRaw(), []byte{0xff, 0x00, 0x01}) {
+ t.Fatalf("binary bytes attribute changed: %#v", attrs.AsRaw()["blob"])
+ }
}
func TestLogsRedactedBeforeDownstreamConsumer(t *testing.T) {
diff --git a/pkg/asymptoteobserve/privacy.go b/pkg/asymptoteobserve/privacy.go
index 522c3637..dedc01d1 100644
--- a/pkg/asymptoteobserve/privacy.go
+++ b/pkg/asymptoteobserve/privacy.go
@@ -3,6 +3,7 @@ package asymptoteobserve
import (
"encoding/json"
"maps"
+ "math"
"regexp"
"slices"
"strings"
@@ -761,22 +762,44 @@ func CleanString(value string, limit int, redactSecrets bool) string {
if !redactSecrets {
return TruncateString(value, limit)
}
- if limit <= 0 {
- return TruncateString(RedactString(value), limit)
+ return TruncateString(redactBeforeCut(value, limit), limit)
+}
+
+// redactBeforeCut redacts value as far as a cut at limit needs: the whole of it, or a window past
+// the limit that still holds redactWindow bytes there once redacted. Everything up to the cut is
+// then redacted text.
+func redactBeforeCut(value string, limit int) string {
+ // Compared without adding to limit, which may be as large as math.MaxInt: a whole string within
+ // redactWindow of the limit, or shorter, is redacted whole.
+ if limit <= 0 || len(value)-limit <= redactWindow {
+ return RedactString(value)
}
// Redaction shortens what it matches, so a window that shrank pulls text from near its own cut
// inside the limit, where an unrecognizable fragment of a secret would survive. Widen the window
// until a full redactWindow of redacted text is left past the limit, or the whole string fits.
+ // Here limit+redactWindow < len(value), and the window doubles only while twice it is still
+ // shorter than value, so nothing overflows.
for window := limit + redactWindow; ; window *= 2 {
- if len(value) <= window {
- return TruncateString(RedactString(value), limit)
- }
if redacted := RedactString(value[:window]); len(redacted) >= limit+redactWindow {
- return TruncateString(redacted, limit)
+ return redacted
+ }
+ if len(value)-window <= window {
+ return RedactString(value)
}
}
}
+// SanitizeMap returns a sanitized copy of input: every string redacted (when opts.RedactSecrets is
+// set) and cut to opts.StringLimit, a value under a field named for a credential redacted whole,
+// at any depth.
+//
+// A value of a type JSON decoding does not produce ([]string, map[string]string, a struct, a
+// pointer, a named string type, json.RawMessage) is first converted into what encoding/json
+// writes for it, decoded back into an interface{}, then sanitized the same way (jsonValue); the
+// parts of it JSON cannot represent (a NaN, a channel, a function, a cycle) become nil. Booleans
+// and Go's number types are kept as they are, so an integer stays an integer. A []byte stays a
+// []byte (an OTLP bytes attribute keeps its type) and is redacted and cut (cleanBytes). A
+// CleanedValue is kept as CleanValue left it. The result always marshals.
func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]interface{} {
limit := opts.StringLimit
if limit <= 0 {
@@ -784,55 +807,224 @@ func SanitizeMap(input map[string]interface{}, opts PrivacyOptions) map[string]i
}
out := make(map[string]interface{}, len(input))
for key, value := range input {
- switch typed := value.(type) {
- case string:
- // A value whose field is named for a credential is one, whatever it looks like:
- // `{"password": "hunter2"}` has no shape a pattern could recognize.
- if opts.RedactSecrets && typed != "" && IsSecretField(key, typed) {
- out[key] = redactedValue
- continue
- }
- out[key] = CleanString(typed, limit, opts.RedactSecrets)
- case map[string]interface{}:
- out[key] = SanitizeMap(typed, opts)
- case []interface{}:
- // A multi-valued header (`{"Cookie": ["sid=1"]}`, OTel's http.request.header.*) is as
- // much a credential as a single value.
- if opts.RedactSecrets && IsSecretKeyName(key) {
- out[key] = redactAllStrings(typed, opts)
- continue
- }
- out[key] = SanitizeSlice(typed, opts)
- default:
- out[key] = typed
- }
+ out[key] = sanitizeField(key, value, opts, limit)
}
return out
}
+// sanitizeField is SanitizeMap's result for a value held under key.
+func sanitizeField(key string, value interface{}, opts PrivacyOptions, limit int) interface{} {
+ switch typed := value.(type) {
+ case string:
+ // A value whose field is named for a credential is one, whatever it looks like:
+ // `{"password": "hunter2"}` has no shape a pattern could recognize.
+ if opts.RedactSecrets && typed != "" && IsSecretField(key, typed) {
+ return redactedValue
+ }
+ return CleanString(typed, limit, opts.RedactSecrets)
+ case map[string]interface{}:
+ return SanitizeMap(typed, opts)
+ case []interface{}:
+ // A multi-valued header (`{"Cookie": ["sid=1"]}`, OTel's http.request.header.*) is as
+ // much a credential as a single value.
+ if opts.RedactSecrets && IsSecretKeyName(key) {
+ return redactAllStrings(typed, opts)
+ }
+ return SanitizeSlice(typed, opts)
+ case []byte:
+ if opts.RedactSecrets && len(typed) > 0 && IsSecretField(key, string(typed)) {
+ return []byte(redactedValue)
+ }
+ return cleanBytes(typed, limit, opts.RedactSecrets)
+ case CleanedValue:
+ // The field's name still redacts it.
+ if opts.RedactSecrets && IsSecretKeyName(key) {
+ return sanitizeField(key, typed.value, opts, limit)
+ }
+ return typed
+ }
+ if scalar, ok := jsonScalar(value); ok {
+ return scalar
+ }
+ return sanitizeField(key, jsonValue(value), opts, limit)
+}
+
+// cleanBytes is a []byte sanitized as a string when it is valid UTF-8. Otherwise it is still
+// redacted, since bytes that are not valid UTF-8 as a whole can hold readable text (Latin-1, a
+// multi-byte character cut in two), and then cut to limit bytes. Redaction replaces only what its
+// rules match and leaves every other byte exactly as it was.
+func cleanBytes(value []byte, limit int, redactSecrets bool) []byte {
+ if value == nil {
+ return nil
+ }
+ if utf8.Valid(value) {
+ return []byte(CleanString(string(value), limit, redactSecrets))
+ }
+ text := string(value)
+ if redactSecrets {
+ text = redactBeforeCut(text, limit)
+ }
+ return []byte(text[:min(len(text), max(limit, 0))])
+}
+
+// CleanedValue is a value CleanValue sanitized. SanitizeMap and SanitizeSlice keep it as it is,
+// unless it is held under a field named for a credential, so its strings keep the limit CleanValue
+// was given rather than the caller's. Only CleanValue makes one.
+type CleanedValue struct {
+ value interface{}
+}
+
+// CleanValue returns v sanitized for a producer that sets its own string limit: v is converted as
+// SanitizeMap converts a value, and every string in it is redacted as RedactString redacts it and
+// cut to limit as CleanString cuts it. Field names are not judged, so a list of environment
+// variable names under `env_keys` keeps them; the producer must already have applied IsSecretField
+// and IsSecretFlag to every key and argument list that comes from data rather than from its own
+// types, as the endpoint inventory does for the MCP server definitions it captures. limit <= 0
+// uses DefaultStringLimit.
+func CleanValue(v interface{}, limit int) CleanedValue {
+ if limit <= 0 {
+ limit = DefaultStringLimit
+ }
+ return CleanedValue{value: cleanAll(v, limit)}
+}
+
+func cleanAll(value interface{}, limit int) interface{} {
+ switch typed := value.(type) {
+ case string:
+ return CleanString(typed, limit, true)
+ case map[string]interface{}:
+ out := make(map[string]interface{}, len(typed))
+ for key, item := range typed {
+ out[key] = cleanAll(item, limit)
+ }
+ return out
+ case []interface{}:
+ out := make([]interface{}, len(typed))
+ for index, item := range typed {
+ out[index] = cleanAll(item, limit)
+ }
+ return out
+ case []byte:
+ return cleanBytes(typed, limit, true)
+ case CleanedValue:
+ return cleanAll(typed.value, limit)
+ }
+ if scalar, ok := jsonScalar(value); ok {
+ return scalar
+ }
+ return cleanAll(jsonValue(value), limit)
+}
+
+// MarshalJSON writes the sanitized value.
+func (c CleanedValue) MarshalJSON() ([]byte, error) {
+ return json.Marshal(c.value)
+}
+
+// jsonScalar returns a value that holds no text, and true: nil, a boolean, or a number of one of
+// Go's number types, or a json.Number, kept as it is. A number JSON cannot write (a NaN, an
+// infinity, a json.Number that does not spell one) is nil.
+func jsonScalar(value interface{}) (interface{}, bool) {
+ switch typed := value.(type) {
+ case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
+ return value, true
+ case float64:
+ if math.IsNaN(typed) || math.IsInf(typed, 0) {
+ return nil, true
+ }
+ return value, true
+ case float32:
+ if math.IsNaN(float64(typed)) || math.IsInf(float64(typed), 0) {
+ return nil, true
+ }
+ return value, true
+ case json.Number:
+ // The encoder writes an empty json.Number as 0.
+ if typed != "" && !isJSONNumber(string(typed)) {
+ return nil, true
+ }
+ return value, true
+ }
+ return nil, false
+}
+
+// isJSONNumber reports whether s is a number as JSON spells it, which is what encoding/json
+// requires of a json.Number it writes.
+func isJSONNumber(s string) bool {
+ if s != "" && s[0] == '-' {
+ s = s[1:]
+ }
+ digits := func() bool {
+ n := 0
+ for n < len(s) && s[n] >= '0' && s[n] <= '9' {
+ n++
+ }
+ s = s[n:]
+ return n > 0
+ }
+ switch {
+ case s == "":
+ return false
+ case s[0] == '0':
+ s = s[1:]
+ case !digits():
+ return false
+ }
+ if s != "" && s[0] == '.' {
+ s = s[1:]
+ if !digits() {
+ return false
+ }
+ }
+ if s != "" && (s[0] == 'e' || s[0] == 'E') {
+ s = s[1:]
+ if s != "" && (s[0] == '+' || s[0] == '-') {
+ s = s[1:]
+ }
+ if !digits() {
+ return false
+ }
+ }
+ return s == ""
+}
+
// redactAllStrings replaces every non-empty string in a slice held under a secret-named field,
// at any depth of nested slices; maps inside it are sanitized by their own field names.
func redactAllStrings(input []interface{}, opts PrivacyOptions) []interface{} {
out := make([]interface{}, len(input))
for index, value := range input {
- switch typed := value.(type) {
- case string:
- if typed == "" {
- out[index] = typed
- } else {
- out[index] = redactedValue
- }
- case []interface{}:
- out[index] = redactAllStrings(typed, opts)
- case map[string]interface{}:
- out[index] = SanitizeMap(typed, opts)
- default:
- out[index] = typed
- }
+ out[index] = redactedItem(value, opts)
}
return out
}
+// redactedItem is redactAllStrings' result for one item.
+func redactedItem(value interface{}, opts PrivacyOptions) interface{} {
+ switch typed := value.(type) {
+ case string:
+ if typed == "" {
+ return typed
+ }
+ return redactedValue
+ case []interface{}:
+ return redactAllStrings(typed, opts)
+ case map[string]interface{}:
+ return SanitizeMap(typed, opts)
+ case []byte:
+ if len(typed) == 0 {
+ return typed
+ }
+ return []byte(redactedValue)
+ case CleanedValue:
+ return redactedItem(typed.value, opts)
+ }
+ if scalar, ok := jsonScalar(value); ok {
+ return scalar
+ }
+ return redactedItem(jsonValue(value), opts)
+}
+
+// SanitizeSlice returns a sanitized copy of input, each item sanitized as SanitizeMap sanitizes a
+// value, and a string after a flag named for a secret (`["--api-key", "x"]`) redacted.
func SanitizeSlice(input []interface{}, opts PrivacyOptions) []interface{} {
limit := opts.StringLimit
if limit <= 0 {
@@ -840,28 +1032,67 @@ func SanitizeSlice(input []interface{}, opts PrivacyOptions) []interface{} {
}
out := make([]interface{}, len(input))
for index, value := range input {
- switch typed := value.(type) {
- case string:
- // A flag named for a secret has its value in the next item: `["--api-key", "x"]`.
- if opts.RedactSecrets && typed != "" && index > 0 && followsSecretFlag(input[index-1]) {
- out[index] = redactedValue
- continue
+ out[index] = sanitizeItem(input, index, value, opts, limit)
+ }
+ return out
+}
+
+// sanitizeItem is SanitizeSlice's result for value, the item of input at index.
+func sanitizeItem(input []interface{}, index int, value interface{}, opts PrivacyOptions, limit int) interface{} {
+ switch typed := value.(type) {
+ case string:
+ // A flag named for a secret has its value in the next item: `["--api-key", "x"]`.
+ if opts.RedactSecrets && typed != "" && index > 0 && followsSecretFlag(input[index-1]) {
+ return redactedValue
+ }
+ return CleanString(typed, limit, opts.RedactSecrets)
+ case map[string]interface{}:
+ return SanitizeMap(typed, opts)
+ case []interface{}:
+ return SanitizeSlice(typed, opts)
+ case []byte:
+ if opts.RedactSecrets && len(typed) > 0 && index > 0 && followsSecretFlag(input[index-1]) {
+ return []byte(redactedValue)
+ }
+ return cleanBytes(typed, limit, opts.RedactSecrets)
+ case CleanedValue:
+ if opts.RedactSecrets && index > 0 && followsSecretFlag(input[index-1]) {
+ switch text := typed.value.(type) {
+ case string:
+ if text != "" {
+ return redactedValue
+ }
+ case []byte:
+ if len(text) > 0 {
+ return []byte(redactedValue)
+ }
}
- out[index] = CleanString(typed, limit, opts.RedactSecrets)
- case map[string]interface{}:
- out[index] = SanitizeMap(typed, opts)
- case []interface{}:
- out[index] = SanitizeSlice(typed, opts)
- default:
- out[index] = typed
}
+ return typed
}
- return out
+ if scalar, ok := jsonScalar(value); ok {
+ return scalar
+ }
+ return sanitizeItem(input, index, jsonValue(value), opts, limit)
}
+// followsSecretFlag reports whether the item before a string is a flag named for a secret, read
+// as JSON writes it, so a named string type holding `--api-key` is one.
func followsSecretFlag(previous interface{}) bool {
- flag, ok := previous.(string)
- return ok && IsSecretFlag(flag)
+ switch typed := previous.(type) {
+ case string:
+ return IsSecretFlag(typed)
+ case map[string]interface{}, []interface{}:
+ return false
+ case []byte:
+ return IsSecretFlag(string(typed))
+ case CleanedValue:
+ return followsSecretFlag(typed.value)
+ }
+ if _, scalar := jsonScalar(previous); scalar {
+ return false
+ }
+ return followsSecretFlag(jsonValue(previous))
}
func SanitizeEvent(event Event, maxBytes int) Event {
diff --git a/pkg/asymptoteobserve/privacy_bench_test.go b/pkg/asymptoteobserve/privacy_bench_test.go
index 56d58e34..d2fec914 100644
--- a/pkg/asymptoteobserve/privacy_bench_test.go
+++ b/pkg/asymptoteobserve/privacy_bench_test.go
@@ -1,6 +1,7 @@
package asymptoteobserve_test
import (
+ "encoding/json"
"fmt"
"strings"
"testing"
@@ -131,6 +132,28 @@ func BenchmarkSanitizeEvent(b *testing.B) {
}
}
+// benchHookPayload is a decoded PostToolUse hook payload: the shapes SanitizeMap sees on the hook
+// path, mostly short strings, with numbers, booleans and nulls among them.
+const benchHookPayload = `{"session_id":"abc-123","transcript_path":"/home/user/.claude/projects/p/abc-123.jsonl",
+"cwd":"/home/user/project","permission_mode":"default","hook_event_name":"PostToolUse","tool_name":"Bash",
+"tool_input":{"command":"go test ./internal/cache/ -run TestLRUEviction -count=20","description":"Run the test","timeout":120000,"run_in_background":false},
+"tool_response":{"stdout":"ok \tgithub.com/example/project/internal/cache\t0.412s","stderr":"","interrupted":false,"isImage":false,"exit_code":0,"duration_ms":412.5,"sandbox":null},
+"usage":{"input_tokens":812,"output_tokens":95,"cache_read_input_tokens":40211,"service_tier":"standard"},
+"args":["-y","@upstash/context7-mcp","--max-tokens","4096",1,2.5,true,null],
+"headers":{"Accept":["application/json"],"X-Request-Id":["req_01"]}}`
+
+// BenchmarkSanitizeMap sanitizes a decoded hook payload.
+func BenchmarkSanitizeMap(b *testing.B) {
+ var payload map[string]interface{}
+ if err := json.Unmarshal([]byte(benchHookPayload), &payload); err != nil {
+ b.Fatal(err)
+ }
+ b.ReportAllocs()
+ for b.Loop() {
+ obs.SanitizeMap(payload, obs.PrivacyOptions{RedactSecrets: true, StringLimit: obs.DefaultRawStringLimit})
+ }
+}
+
// BenchmarkToolResultText derives gen_ai.tool.call.result_text from the sanitized file read.
func BenchmarkToolResultText(b *testing.B) {
e := obs.SanitizeEvent(benchEvents()[0].event, 65536)
diff --git a/pkg/asymptoteobserve/sanitize_map_types_test.go b/pkg/asymptoteobserve/sanitize_map_types_test.go
new file mode 100644
index 00000000..9a236367
--- /dev/null
+++ b/pkg/asymptoteobserve/sanitize_map_types_test.go
@@ -0,0 +1,321 @@
+package asymptoteobserve
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "math"
+ "reflect"
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+// The tests in this file check that SanitizeMap and SanitizeSlice sanitize values of every Go type
+// in-process callers put in a map, not only the shapes JSON decoding produces, and that what they
+// return always marshals.
+
+const typesTestToken = "ghp_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"
+
+// typesTestText holds a credential RedactString recognizes and runs past DefaultRawStringLimit.
+var typesTestText = "clone with " + typesTestToken + " " + strings.Repeat("x", 3000)
+
+type typesTestLabel string
+
+type typesTestRecord struct {
+ Name string `json:"name"`
+ Password string `json:"password"`
+ Notes []string `json:"notes"`
+}
+
+type typesTestMarshaler struct{ text string }
+
+func (m typesTestMarshaler) MarshalText() ([]byte, error) { return []byte(m.text), nil }
+
+// typesTestValues are values of the types SanitizeMap kept as they were, each holding
+// typesTestText, and "hunter2" under a field named for a credential where the type has fields.
+func typesTestValues() map[string]interface{} {
+ text := typesTestText
+ record := typesTestRecord{Name: text, Password: "hunter2", Notes: []string{text}}
+ return map[string]interface{}{
+ "[]string": []string{text},
+ "map[string]string": map[string]string{"note": text, "password": "hunter2"},
+ "[]map[string]interface{}": []map[string]interface{}{{"note": text, "password": "hunter2"}},
+ "map[string][]string": map[string][]string{"note": {text}, "Cookie": {"hunter2"}},
+ "struct": record,
+ "pointer to struct": &record,
+ "*string": &text,
+ "named string type": typesTestLabel(text),
+ "json.RawMessage": json.RawMessage(`{"note":"` + text + `","password":"hunter2"}`),
+ "[]byte": []byte(text),
+ "encoding.TextMarshaler": typesTestMarshaler{text},
+ "[]interface{} of []string": []interface{}{[]string{text}},
+ "map[string]interface{} of T": map[string]interface{}{"record": record},
+ }
+}
+
+// checkSanitized fails unless v holds only the shapes JSON decoding produces, Go's scalar types
+// and []byte, and every string in it (and every []byte that is text) is free of the credential and
+// within limit.
+func checkSanitized(t *testing.T, path string, v interface{}, limit int) {
+ t.Helper()
+ checkText := func(s string) {
+ t.Helper()
+ if strings.Contains(s, typesTestToken) || strings.Contains(s, "hunter2") {
+ t.Errorf("%s: credential left in %.80q", path, s)
+ }
+ if len(s) > limit {
+ t.Errorf("%s: %d bytes, limit %d", path, len(s), limit)
+ }
+ }
+ switch x := v.(type) {
+ case nil, bool, float64, float32, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, json.Number:
+ case string:
+ checkText(x)
+ case []byte:
+ checkText(string(x))
+ case map[string]interface{}:
+ for k, item := range x {
+ checkSanitized(t, path+"."+k, item, limit)
+ }
+ case []interface{}:
+ for i, item := range x {
+ checkSanitized(t, fmt.Sprintf("%s[%d]", path, i), item, limit)
+ }
+ default:
+ t.Errorf("%s: %T returned as it was given", path, v)
+ }
+}
+
+func TestSanitizeMapSanitizesEveryType(t *testing.T) {
+ opts := PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit}
+ for name, value := range typesTestValues() {
+ t.Run(name, func(t *testing.T) {
+ checkSanitized(t, "map", SanitizeMap(map[string]interface{}{"value": value}, opts), DefaultRawStringLimit)
+ checkSanitized(t, "slice", SanitizeSlice([]interface{}{value}, opts), DefaultRawStringLimit)
+ checkSanitized(t, "secret list", SanitizeMap(map[string]interface{}{"Cookie": []interface{}{value}}, opts), DefaultRawStringLimit)
+ })
+ }
+ // A field named for a credential, and an argument after a flag named for one, are redacted
+ // whatever type holds them.
+ named := SanitizeMap(map[string]interface{}{
+ "password": typesTestLabel("hunter2"),
+ "Cookie": []string{"hunter2"},
+ "token": []byte("hunter2"),
+ "args": []typesTestLabel{"--api-key", "hunter2"},
+ "argv": []interface{}{"--api-key", []byte("hunter2")},
+ "flags": []interface{}{typesTestLabel("--api-key"), "hunter2"},
+ }, opts)
+ checkSanitized(t, "named", named, DefaultRawStringLimit)
+ if data, _ := json.Marshal(named["args"]); string(data) != `["--api-key","[REDACTED]"]` {
+ t.Errorf("args = %s, want the value after --api-key redacted", data)
+ }
+}
+
+// Go's number types, booleans and []byte keep their types: the beacon_privacy processor hands
+// SanitizeMap an OTLP attribute map and builds the attributes back from what it returns.
+func TestSanitizeMapKeepsScalarAndBytesTypes(t *testing.T) {
+ binary := append([]byte{0xff, 0xfe}, bytes.Repeat([]byte{0x80}, 3000)...)
+ input := map[string]interface{}{
+ "count": int64(812), "port": uint16(443), "ratio": float32(1.5), "cost": 0.25,
+ "n": json.Number("12"), "ok": true, "text": []byte(typesTestText), "binary": binary, "none": []byte(nil),
+ }
+ got := SanitizeMap(input, PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit})
+ for _, key := range []string{"count", "port", "ratio", "cost", "n", "ok", "none"} {
+ if !reflect.DeepEqual(got[key], input[key]) {
+ t.Errorf("%s = %#v, want %#v", key, got[key], input[key])
+ }
+ }
+ text, ok := got["text"].([]byte)
+ if !ok || !utf8.Valid(text) || len(text) > DefaultRawStringLimit || bytes.Contains(text, []byte(typesTestToken)) {
+ t.Errorf("text = %T %.80q, want []byte redacted and cut to the limit", got["text"], got["text"])
+ }
+ if b, ok := got["binary"].([]byte); !ok || !bytes.Equal(b, binary[:DefaultRawStringLimit]) {
+ t.Errorf("binary = %T of %d bytes, want the first %d bytes", got["binary"], len(b), DefaultRawStringLimit)
+ }
+ // Values JSON cannot write are nil.
+ for name, value := range map[string]interface{}{
+ "NaN": math.NaN(), "infinity": float32(math.Inf(1)), "json.Number": json.Number(typesTestToken),
+ } {
+ if got := SanitizeMap(map[string]interface{}{"v": value}, PrivacyOptions{RedactSecrets: true})["v"]; got != nil {
+ t.Errorf("%s = %#v, want nil", name, got)
+ }
+ }
+}
+
+// Bytes that are not valid UTF-8 as a whole can still hold readable text, Latin-1 here: a
+// credential in them is redacted, and every other byte is kept exactly as it was.
+func TestSanitizeMapRedactsBytesThatAreNotUTF8(t *testing.T) {
+ latin1 := []byte("caf\xe9 clone with " + typesTestToken + " na\xefve")
+ opts := PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit}
+ want := []byte("caf\xe9 clone with ghp_[REDACTED] na\xefve")
+ if got := SanitizeMap(map[string]interface{}{"payload": latin1}, opts)["payload"]; !reflect.DeepEqual(got, want) {
+ t.Errorf("payload = %q, want %q", got, want)
+ }
+ if got := SanitizeSlice([]interface{}{latin1}, opts)[0]; !reflect.DeepEqual(got, want) {
+ t.Errorf("item = %q, want %q", got, want)
+ }
+ // A long one is cut to the limit, every byte before the cut kept.
+ long := append(bytes.Repeat([]byte("\xe9t\xe9 "), 600), latin1...)
+ got, ok := SanitizeMap(map[string]interface{}{"payload": long}, opts)["payload"].([]byte)
+ if !ok || !bytes.Equal(got, long[:DefaultRawStringLimit]) {
+ t.Errorf("long payload = %d bytes, want the first %d", len(got), DefaultRawStringLimit)
+ }
+ // A credential straddling the cut is redacted before it is cut, so no part of it is left (its
+ // body would start 8 bytes before the cut).
+ cut := append(bytes.Repeat([]byte{'\xe9'}, DefaultRawStringLimit-28), latin1...)
+ if got := SanitizeMap(map[string]interface{}{"payload": cut}, opts)["payload"].([]byte); bytes.Contains(got, []byte(typesTestToken[4:8])) {
+ t.Errorf("credential straddling the cut left a prefix: %q", got[len(got)-40:])
+ }
+}
+
+type typesTestMeta struct {
+ Runtime string `json:"runtime"`
+}
+
+type typesTestServer struct {
+ typesTestMeta
+ Name string `json:"name"`
+ Definition map[string]interface{} `json:"definition,omitempty"`
+ Note string `json:"note,omitempty"`
+ Internal string `json:"-"`
+}
+
+// A value JSON cannot represent turns only itself into nil, not the list or the struct holding it.
+func TestSanitizeMapKeepsPartsBesideAnUnrepresentableValue(t *testing.T) {
+ servers := []typesTestServer{
+ {typesTestMeta{"cursor"}, "first", map[string]interface{}{"command": "npx"}, "", "hidden"},
+ {typesTestMeta{"codex"}, "broken", map[string]interface{}{"command": "uvx", "timeout": math.NaN(), "note": typesTestText}, "n", ""},
+ {typesTestMeta{"cursor"}, "last", nil, "", ""},
+ }
+ opts := PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit}
+ want := []interface{}{
+ map[string]interface{}{"runtime": "cursor", "name": "first", "definition": map[string]interface{}{"command": "npx"}},
+ map[string]interface{}{"runtime": "codex", "name": "broken", "note": "n", "definition": map[string]interface{}{
+ "command": "uvx", "timeout": nil, "note": CleanString(typesTestText, DefaultRawStringLimit, true),
+ }},
+ map[string]interface{}{"runtime": "cursor", "name": "last"},
+ }
+ if got := SanitizeMap(map[string]interface{}{"servers": servers}, opts)["servers"]; !reflect.DeepEqual(got, want) {
+ t.Errorf("servers = %#v\nwant %#v", got, want)
+ }
+ if got := SanitizeSlice([]interface{}{servers}, opts)[0]; !reflect.DeepEqual(got, want) {
+ t.Errorf("servers in a slice = %#v", got)
+ }
+ data, err := json.Marshal(CleanValue(servers, DefaultRawStringLimit))
+ if wantData, _ := json.Marshal(want); err != nil || !bytes.Equal(data, wantData) {
+ t.Errorf("CleanValue = %s, %v\nwant %s", data, err, wantData)
+ }
+}
+
+type typesTestNode struct {
+ Name string `json:"name"`
+ Next *typesTestNode `json:"next"`
+}
+
+// Whatever SanitizeMap and SanitizeSlice are given, what they return marshals, and so does a
+// CleanedValue.
+func TestSanitizeMapOutputAlwaysMarshals(t *testing.T) {
+ cycle := &typesTestNode{Name: typesTestText}
+ cycle.Next = cycle
+ values := []interface{}{
+ math.NaN(), math.Inf(-1), float32(math.NaN()), make(chan int), func() {}, cycle, complex(1, 2),
+ map[bool]string{true: typesTestText}, json.Number("not a number"), json.RawMessage(`{"open":`),
+ struct{ F float64 }{math.NaN()}, []float64{1, math.Inf(1)}, map[string]interface{}{"c": make(chan int)},
+ []interface{}{func() {}, typesTestText}, nil, (*typesTestRecord)(nil), []string(nil),
+ }
+ for _, value := range typesTestValues() {
+ values = append(values, value)
+ }
+ // Nest each value in the others' containers.
+ for _, value := range append([]interface{}(nil), values...) {
+ values = append(values,
+ map[string]interface{}{"nested": []interface{}{value}},
+ []map[string]interface{}{{"nested": value}},
+ struct{ V interface{} }{value},
+ )
+ }
+ opts := PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit}
+ for i, value := range values {
+ for what, out := range map[string]interface{}{
+ "SanitizeMap": SanitizeMap(map[string]interface{}{"v": value, "token": value}, opts),
+ "SanitizeSlice": SanitizeSlice([]interface{}{"--token", value, value}, opts),
+ "CleanValue": CleanValue(value, DefaultRawStringLimit),
+ } {
+ if _, err := json.Marshal(out); err != nil {
+ t.Errorf("%s of value %d (%T): %v", what, i, value, err)
+ }
+ if what != "CleanValue" {
+ checkSanitized(t, fmt.Sprintf("%s of value %d (%T)", what, i, value), out, DefaultRawStringLimit)
+ }
+ }
+ }
+}
+
+// A limit up to math.MaxInt, which a configuration may use for "unlimited", keeps every string
+// whole and redacted: the window past the limit is worked out without adding to the limit.
+func TestCleanStringTakesAnyLimit(t *testing.T) {
+ text := "dump " + typesTestToken + " " + strings.Repeat("y", 10000)
+ want := RedactString(text)
+ for _, limit := range []int{math.MaxInt, math.MaxInt - 1, math.MaxInt - redactWindow, math.MaxInt - redactWindow + 1, math.MaxInt / 2} {
+ if got := CleanString(text, limit, true); got != want {
+ t.Errorf("CleanString at limit %d = %.80q, want the whole text redacted", limit, got)
+ }
+ }
+}
+
+func TestSanitizeMapAndCleanValueTakeAnyLimit(t *testing.T) {
+ text := "dump " + typesTestToken + " " + strings.Repeat("y", 10000)
+ want := RedactString(text)
+ for _, limit := range []int{math.MaxInt, math.MaxInt - 1} {
+ opts := PrivacyOptions{RedactSecrets: true, StringLimit: limit}
+ got := SanitizeMap(map[string]interface{}{"text": text, "bytes": []byte("\xe9" + text), "list": []string{text}}, opts)
+ if got["text"] != want || !bytes.Equal(got["bytes"].([]byte), []byte("\xe9"+want)) || !reflect.DeepEqual(got["list"], []interface{}{want}) {
+ t.Errorf("SanitizeMap at limit %d did not keep every string whole and redacted", limit)
+ }
+ data, err := json.Marshal(CleanValue(map[string]interface{}{"text": text}, limit))
+ if wantData, _ := json.Marshal(map[string]string{"text": want}); err != nil || !bytes.Equal(data, wantData) {
+ t.Errorf("CleanValue at limit %d = %.80s, %v", limit, data, err)
+ }
+ }
+}
+
+// A CleanedValue keeps the limit CleanValue was given, through SanitizeMap and SanitizeEvent,
+// with every string in it redacted and its field names left alone; under a field named for a
+// credential, or after a flag named for one, it is still redacted.
+func TestCleanValueKeepsItsLimit(t *testing.T) {
+ content := "dump " + typesTestToken + " " + strings.Repeat("y", 10000)
+ cleaned := CleanValue([]map[string]interface{}{{"text": content, "env_keys": []string{"API_TOKEN"}}}, 64<<10)
+ opts := PrivacyOptions{RedactSecrets: true, StringLimit: DefaultRawStringLimit}
+
+ event := NewEvent(NewEventOptions{Action: "inventory.snapshot", Harness: HarnessInfo{Name: "test"}})
+ event.Raw = SanitizeMap(map[string]interface{}{"configs": cleaned}, opts)
+ event = SanitizeEvent(event, 1<<20)
+ data, err := json.Marshal(event)
+ if err != nil {
+ t.Fatal(err)
+ }
+ text := string(data)
+ if strings.Contains(text, typesTestToken) {
+ t.Fatalf("CleanValue left the credential: %.200s", text)
+ }
+ if !strings.Contains(text, strings.Repeat("y", 10000)) || !strings.Contains(text, `"env_keys":["API_TOKEN"]`) {
+ t.Fatalf("CleanValue's text was cut to the raw limit or its names judged: %.200s", text)
+ }
+
+ if data, _ := json.Marshal(CleanValue(strings.Repeat("z", 500), 100)); len(data) > 102 {
+ t.Errorf("CleanValue ignored its limit: %d bytes", len(data))
+ }
+ for name, out := range map[string]interface{}{
+ "secret field": SanitizeMap(map[string]interface{}{"password": CleanValue("hunter2", 100)}, opts),
+ "secret list": SanitizeMap(map[string]interface{}{"Cookie": CleanValue([]string{"hunter2"}, 100)}, opts),
+ "after a flag": SanitizeSlice([]interface{}{"--token", CleanValue("hunter2", 100)}, opts),
+ } {
+ if data, _ := json.Marshal(out); strings.Contains(string(data), "hunter2") {
+ t.Errorf("%s: %s", name, data)
+ }
+ }
+ if got := SanitizeSlice([]interface{}{"--token", CleanValue([]byte("hunter2"), 100)}, opts)[1]; !reflect.DeepEqual(got, []byte(redactedValue)) {
+ t.Errorf("bytes after a flag = %#v, want redacted", got)
+ }
+}
diff --git a/pkg/asymptoteobserve/sanitize_typed.go b/pkg/asymptoteobserve/sanitize_typed.go
index 4dee3ef2..5f62ce40 100644
--- a/pkg/asymptoteobserve/sanitize_typed.go
+++ b/pkg/asymptoteobserve/sanitize_typed.go
@@ -295,6 +295,172 @@ func roundTrip(v any) any {
return out
}
+// jsonValue is a value of a type JSON decoding does not produce, as encoding/json marshals it and
+// decodes it back: a string, a []interface{}, a map[string]interface{}, a float64, a bool, or nil.
+// SanitizeMap, SanitizeSlice and CleanValue convert such values with it.
+//
+// A value encoding/json cannot write whole (a NaN somewhere in it, a channel, a map whose keys are
+// not strings, a cycle) is converted part by part instead: the items of a slice or an array, the
+// values of a map, the fields of a struct, what a pointer or an interface holds. Only the parts it
+// cannot represent become nil, so one bad value does not take the rest of a list with it. Fields
+// are named as encoding/json names them (a tag's name, `-`, omitempty and omitzero, the fields of
+// an embedded struct promoted); a `,string` option is not applied on this path. A part that has
+// its own marshaler and fails is nil.
+func jsonValue(value any) any {
+ w := jsonWalk{seen: map[uintptr]bool{}}
+ return w.value(value, 0)
+}
+
+type jsonWalk struct {
+ // seen holds the pointers, maps and slices on the path being converted, so a cycle ends in nil.
+ seen map[uintptr]bool
+}
+
+func (w jsonWalk) value(value any, depth int) any {
+ switch value.(type) {
+ case []string, map[string]string:
+ return decoded(value, 0)
+ }
+ data, err := json.Marshal(value)
+ if err == nil {
+ var out any
+ if json.Unmarshal(data, &out) == nil {
+ return out
+ }
+ }
+ if depth >= maxGenericDepth {
+ return nil
+ }
+ return w.parts(reflect.ValueOf(value), depth+1)
+}
+
+func (w jsonWalk) parts(v reflect.Value, depth int) any {
+ if !v.IsValid() {
+ return nil
+ }
+ if t := v.Type(); t.Implements(jsonMarshalerType) || t.Implements(textMarshalerType) ||
+ reflect.PointerTo(t).Implements(jsonMarshalerType) || reflect.PointerTo(t).Implements(textMarshalerType) {
+ return nil
+ }
+ switch v.Kind() {
+ case reflect.Pointer, reflect.Map, reflect.Slice:
+ if v.IsNil() {
+ return nil
+ }
+ if w.seen[v.Pointer()] {
+ return nil
+ }
+ w.seen[v.Pointer()] = true
+ defer delete(w.seen, v.Pointer())
+ }
+ switch v.Kind() {
+ case reflect.Pointer, reflect.Interface:
+ if v.IsNil() {
+ return nil
+ }
+ return w.value(v.Elem().Interface(), depth)
+ case reflect.Slice, reflect.Array:
+ out := make([]any, v.Len())
+ for i := range out {
+ out[i] = w.value(v.Index(i).Interface(), depth)
+ }
+ return out
+ case reflect.Map:
+ out := make(map[string]any, v.Len())
+ for iter := v.MapRange(); iter.Next(); {
+ key, ok := jsonKey(iter.Key())
+ if !ok {
+ return nil
+ }
+ out[key] = w.value(iter.Value().Interface(), depth)
+ }
+ return out
+ case reflect.Struct:
+ out := map[string]any{}
+ w.fields(out, v, depth)
+ return out
+ }
+ return nil
+}
+
+// jsonKey is a map key as encoding/json writes it, and false for a key it cannot write.
+func jsonKey(k reflect.Value) (string, bool) {
+ switch k.Kind() {
+ case reflect.String:
+ return k.String(), true
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ return strconv.FormatInt(k.Int(), 10), true
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+ return strconv.FormatUint(k.Uint(), 10), true
+ }
+ if m, ok := k.Interface().(encoding.TextMarshaler); ok {
+ text, err := m.MarshalText()
+ return string(text), err == nil
+ }
+ return "", false
+}
+
+// fields adds the fields of struct v to out, under the names encoding/json gives them.
+func (w jsonWalk) fields(out map[string]any, v reflect.Value, depth int) {
+ t := v.Type()
+ for i := range t.NumField() {
+ f, fv := t.Field(i), v.Field(i)
+ tag := f.Tag.Get("json")
+ if tag == "-" {
+ continue
+ }
+ name, opts, _ := strings.Cut(tag, ",")
+ if f.Anonymous && name == "" {
+ if fv.Kind() == reflect.Pointer {
+ if fv.IsNil() {
+ continue
+ }
+ fv = fv.Elem()
+ }
+ if fv.Kind() == reflect.Struct {
+ w.fields(out, fv, depth)
+ continue
+ }
+ }
+ if !f.IsExported() || !fv.CanInterface() {
+ continue
+ }
+ if name == "" {
+ name = f.Name
+ }
+ if hasJSONOption(opts, "omitempty") && isEmptyJSONValue(fv) || hasJSONOption(opts, "omitzero") && fv.IsZero() {
+ continue
+ }
+ if _, taken := out[name]; !taken {
+ out[name] = w.value(fv.Interface(), depth)
+ }
+ }
+}
+
+func hasJSONOption(opts, option string) bool {
+ for opts != "" {
+ var next string
+ next, opts, _ = strings.Cut(opts, ",")
+ if next == option {
+ return true
+ }
+ }
+ return false
+}
+
+// isEmptyJSONValue is encoding/json's test for omitempty.
+func isEmptyJSONValue(v reflect.Value) bool {
+ switch v.Kind() {
+ case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+ return v.Len() == 0
+ case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+ reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
+ reflect.Float32, reflect.Float64, reflect.Interface, reflect.Pointer:
+ return v.IsZero()
+ }
+ return false
+}
+
// SanitizeMCP returns a sanitized copy of an MCP block: every string converted as the JSON encoder
// writes it and cleaned with opts, as SanitizeEvent cleans it.
func SanitizeMCP(in *MCPInfo, opts PrivacyOptions) *MCPInfo {