The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(privacy): hash stored content after redaction (#1159)
kyle-elliott-asymptotebd672bd6f2ba
diff --git a/cli/beacon-hooks/cmd/agent_thought.go b/cli/beacon-hooks/cmd/agent_thought.goindex 5627d804..c095cf35 100644--- a/cli/beacon-hooks/cmd/agent_thought.go+++ b/cli/beacon-hooks/cmd/agent_thought.go@@ -57,8 +57,9 @@ func reasoningOutputMessages(text string) []interface{} {}// retainedContentFields builds the content marker for an event that retains raw-// text, computed against the original text so hash and byte count stay stable-// even after the sink redacts or truncates the stored copy.+// text: the hash of the whole redacted text and the original byte count, so both+// stay stable when the sink truncates the stored copy, and the hash confirms+// nothing redaction removed.//// The limit passed is the one this writer applies: the hook logger sanitizes the// whole event map at DefaultStringLimit, so that is what "truncated" has to mean@@ -67,6 +68,18 @@ func retainedContentFields(text string) map[string]interface{} {return asymptoteobserve.RetainedContentFields(text, asymptoteobserve.DefaultStringLimit)}+// retainedDiffFields puts diff, its diff_hash and its diff_bytes into file and+// returns the content marker for it, all from one redaction of diff at the+// limit retainedContentFields uses, so diff_hash and diff_bytes always equal the+// marker's hash and bytes.+func retainedDiffFields(file map[string]interface{}, diff string) map[string]interface{} {+ content, diffHash, diffBytes := asymptoteobserve.RetainedDiffFields(diff, asymptoteobserve.DefaultStringLimit)+ file["diff"] = diff+ file["diff_hash"] = diffHash+ file["diff_bytes"] = diffBytes+ return content+}+// thoughtMetadataFields extracts the non-content afterAgentThought payload// metadata worth retaining; the reasoning text itself is excluded because it// already lives in gen_ai.output.messages.diff --git a/cli/beacon-hooks/cmd/content_test.go b/cli/beacon-hooks/cmd/content_test.goindex 183fe4e4..674987b6 100644--- a/cli/beacon-hooks/cmd/content_test.go+++ b/cli/beacon-hooks/cmd/content_test.go@@ -81,6 +81,49 @@ func TestPromptSubmitRecordsRetainedPromptContent(t *testing.T) {}}+// content.hash and file.diff_hash leave the machine where the text does not (Metadata-only keeps+// them), so each is the hash of what the hook writer stores after redaction: a hash of the text+// before it lets anyone holding the event confirm a guessed secret offline.+func TestHookHashesAreOfTheStoredRedactedText(t *testing.T) {+ const secret = "deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2"+ const stored = "deploy with ghp_[REDACTED] and password=[REDACTED]"+ hexHash := func(text string) string {+ sum := sha256.Sum256([]byte(text))+ return hex.EncodeToString(sum[:])+ }+ setupHookConfigDirs(t)+ platformFlag = "claude"+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)++ runHookWithInput(t, runPromptSubmit, map[string]interface{}{"session_id": "redacted-hash-session", "prompt": secret})+ prompt := lastEndpointEvent(t, logPath)+ if got := prompt["prompt"].(map[string]interface{})["text"]; got != stored {+ t.Fatalf("prompt.text = %q, want %q", got, stored)+ }+ content := prompt["content"].(map[string]interface{})+ if content["hash"] != hexHash(stored) || content["bytes"] != float64(len(secret)) || content["redacted"] != true {+ t.Fatalf("content = %#v, want the hash of %q and the original size", content, stored)+ }++ runHookWithInput(t, runPostTool, map[string]interface{}{+ "session_id": "redacted-hash-session",+ "cwd": "/repo",+ "tool_name": "Edit",+ "tool_input": map[string]interface{}{"file_path": "/repo/main.go", "old_string": "old", "new_string": secret},+ })+ edit := lastEndpointEvent(t, logPath)+ file := edit["file"].(map[string]interface{})+ diff, _ := file["diff"].(string)+ if !strings.Contains(diff, stored) || strings.Contains(diff, "hunter2") {+ t.Fatalf("file.diff = %q, want the redacted secret", diff)+ }+ marker := edit["content"].(map[string]interface{})+ if file["diff_hash"] != hexHash(diff) || marker["hash"] != file["diff_hash"] || marker["bytes"] != file["diff_bytes"] {+ t.Fatalf("file = %#v content = %#v, want diff_hash of the stored diff, matching the marker", file, marker)+ }+}+func TestPromptSubmitWithoutAPromptWritesNoContentMarker(t *testing.T) {setupHookConfigDirs(t)platformFlag = "claude"diff --git a/cli/beacon-hooks/cmd/endpoint_events.go b/cli/beacon-hooks/cmd/endpoint_events.goindex 90653673..809c62c0 100644--- a/cli/beacon-hooks/cmd/endpoint_events.go+++ b/cli/beacon-hooks/cmd/endpoint_events.go@@ -2,8 +2,6 @@ package cmdimport ("context"- "crypto/sha256"- "encoding/hex""fmt""os""path/filepath"@@ -791,14 +789,10 @@ func diffFields(filePath, diffStr string) map[string]interface{} {}fields := map[string]interface{}{"file": file}if diffStr != "" {- sum := sha256.Sum256([]byte(diffStr))- file["diff_hash"] = hex.EncodeToString(sum[:])- file["diff_bytes"] = len(diffStr)- file["diff"] = diffStr- // diff_hash and diff_bytes already describe the original diff; the marker+ // diff_hash (of the whole redacted diff) and diff_bytes (its original size) describe the diff; the marker// adds what they cannot say -- whether the stored copy is the whole diff,// and whether redaction rewrote part of it.- fields["content"] = retainedContentFields(diffStr)+ fields["content"] = retainedDiffFields(file, diffStr)}return fields}diff --git a/cli/beacon-hooks/cmd/openclaw_event.go b/cli/beacon-hooks/cmd/openclaw_event.goindex d5efc0ff..1d816df9 100644--- a/cli/beacon-hooks/cmd/openclaw_event.go+++ b/cli/beacon-hooks/cmd/openclaw_event.go@@ -1,8 +1,6 @@package cmdimport (- "crypto/sha256"- "encoding/hex""path/filepath""strings"@@ -632,16 +630,11 @@ func openClawToolFields(event map[string]interface{}, result map[string]interfacfile["language"] = language}if diff := openClawFileDiff(name, path, params); diff != "" {- // The diff text itself, alongside the hash and byte count taken before the writer- // redacts or truncates the stored copy. The `content` marker beside it says- // whether what landed in the log is the whole diff; the hash and size say what the- // original was. diffFields builds the same three keys for the shared hook path, so- // a Beacon consumer reads one shape whichever runtime produced the row.- sum := sha256.Sum256([]byte(diff))- file["diff"] = diff- file["diff_hash"] = hex.EncodeToString(sum[:])- file["diff_bytes"] = len(diff)- fields["content"] = retainedContentFields(diff)+ // The diff text itself, alongside the hash of the whole redacted diff and the original's+ // byte count. The `content` marker beside it says whether what landed in the log is the+ // whole diff. diffFields builds the same three keys for the shared hook path, so a+ // Beacon consumer reads one shape whichever runtime produced the row.+ fields["content"] = retainedDiffFields(file, diff)}fields["file"] = file}diff --git a/cli/beacon/internal/copilotsession/mapper.go b/cli/beacon/internal/copilotsession/mapper.goindex ff5f2fe1..8b57f5d0 100644--- a/cli/beacon/internal/copilotsession/mapper.go+++ b/cli/beacon/internal/copilotsession/mapper.go@@ -2,7 +2,6 @@ package copilotsessionimport ("crypto/sha1"- "crypto/sha256""encoding/hex""encoding/json""fmt"@@ -589,10 +588,7 @@ func applyFile(ev *schema.Event, call toolCall, result toolResult, action string}if diff := diffFromCall(call, path); diff != "" && action != "file.read" {info.Diff = diff- info.DiffBytes = len(diff)- sum := sha256.Sum256([]byte(diff))- info.DiffHash = hex.EncodeToString(sum[:])- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)} else if result.Text != "" {ev.Content = asymptoteobserve.RetainedContent(result.Text, asymptoteobserve.DefaultStringLimit)}diff --git a/cli/beacon/internal/cursorsession/mapper.go b/cli/beacon/internal/cursorsession/mapper.goindex 4f3dbace..6bb677bf 100644--- a/cli/beacon/internal/cursorsession/mapper.go+++ b/cli/beacon/internal/cursorsession/mapper.go@@ -2,7 +2,6 @@ package cursorsessionimport ("crypto/sha1"- "crypto/sha256""encoding/hex""encoding/json""fmt"@@ -277,9 +276,7 @@ func applyFile(ev *schema.Event, record Record) {}if diff := diffFromArgs(record.Args); diff != "" {info.Diff = diff- info.DiffBytes = len(diff)- info.DiffHash = sha256Hex(diff)- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}ev.File = info}@@ -345,11 +342,6 @@ func withGenAI(genAI *schema.GenAIInfo, edit func(*schema.GenAIInfo)) *schema.Gereturn genAI}-func sha256Hex(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}-var cursorIDNamespace = [16]byte{0x50, 0x73, 0x15, 0x64, 0xc6, 0x90, 0x44, 0x0f,0x91, 0x4a, 0x42, 0x31, 0x77, 0xe1, 0x82, 0x9c,diff --git a/cli/beacon/internal/cursorsession/redacted_hash_test.go b/cli/beacon/internal/cursorsession/redacted_hash_test.gonew file mode 100644index 00000000..51f5f416--- /dev/null+++ b/cli/beacon/internal/cursorsession/redacted_hash_test.go@@ -0,0 +1,57 @@+package cursorsession++import (+ "crypto/sha256"+ "encoding/hex"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++func storedTextHash(text string) string {+ sum := sha256.Sum256([]byte(text))+ return hex.EncodeToString(sum[:])+}++// content.hash and file.diff_hash leave the machine where the text does not (Metadata-only keeps+// them), so each must be the hash of the text the writer stores after redaction: a hash of the+// text before it lets anyone holding the event confirm a guessed secret.+func TestMapTraceHashesWhatTheWriterStores(t *testing.T) {+ const secret = "deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2"+ ref := TraceRef{ID: "s-1", Kind: SourceTranscript, SourcePath: "/tmp/s-1.jsonl", Workspace: "/work"}+ records := []Record{+ {Order: 1, NativeID: "u1", Type: "user_message", Content: secret},+ {Order: 2, NativeID: "t1", Type: "tool_result", ToolName: "edit_file", Args: map[string]interface{}{"path": "main.go", "old_string": "x", "new_string": secret}},+ {Order: 3, NativeID: "u2", Type: "user_message", Content: "the quick brown fox"},+ }+ mapped := MapTrace(ref, records, MapOptions{})+ if len(mapped) != 3 {+ t.Fatalf("mapped = %d events, want 3: %+v", len(mapped), mapped)+ }++ prompt := asymptoteobserve.SanitizeEvent(mapped[0].Event, asymptoteobserve.DefaultMaxEventBytes)+ if prompt.Prompt.Text != "deploy with ghp_[REDACTED] and password=[REDACTED]" {+ t.Fatalf("stored prompt = %q", prompt.Prompt.Text)+ }+ if prompt.Content.Hash != storedTextHash(prompt.Prompt.Text) || prompt.Content.Bytes != len(secret) {+ t.Fatalf("prompt marker = %+v, want the hash of the stored text %q and the original size", prompt.Content, prompt.Prompt.Text)+ }++ originalDiffBytes := len(mapped[1].Event.File.Diff) // SanitizeEvent rewrites it through the pointer+ edit := asymptoteobserve.SanitizeEvent(mapped[1].Event, asymptoteobserve.DefaultMaxEventBytes)+ if edit.File == nil || !strings.Contains(edit.File.Diff, "+deploy with ghp_[REDACTED] and password=[REDACTED]") {+ t.Fatalf("edit file = %+v", edit.File)+ }+ if edit.File.DiffHash != storedTextHash(edit.File.Diff) || edit.File.DiffBytes != originalDiffBytes || originalDiffBytes == len(edit.File.Diff) {+ t.Fatalf("diff_hash %s over %d bytes, want the hash of the stored diff %q and the original size", edit.File.DiffHash, edit.File.DiffBytes, edit.File.Diff)+ }+ if edit.Content.Hash != edit.File.DiffHash || !edit.Content.Redacted {+ t.Fatalf("edit marker = %+v, want the diff_hash and redacted", edit.Content)+ }++ // Text with nothing to redact keeps the hash it always had.+ if plain := mapped[2].Event.Content; plain.Hash != "9ecb36561341d18eb65484e833efea61edc74b84cf5e6ae1b81c63533e25fc8f" || plain.Bytes != 19 {+ t.Fatalf("plain marker = %+v", plain)+ }+}diff --git a/cli/beacon/internal/dshsession/mapper.go b/cli/beacon/internal/dshsession/mapper.goindex 0e1c1b75..d92cffbd 100644--- a/cli/beacon/internal/dshsession/mapper.go+++ b/cli/beacon/internal/dshsession/mapper.go@@ -2,7 +2,6 @@ package dshsessionimport ("crypto/sha1"- "crypto/sha256""encoding/hex""encoding/json""fmt"@@ -427,10 +426,7 @@ func applyFile(ev *schema.Event, call toolCall, result toolResult) {}if diff := diffFromCall(call); diff != "" && !result.IsError {info.Diff = diff- info.DiffBytes = len(diff)- sum := sha256.Sum256([]byte(diff))- info.DiffHash = hex.EncodeToString(sum[:])- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}ev.File = info}diff --git a/cli/beacon/internal/fxsession/mapper.go b/cli/beacon/internal/fxsession/mapper.goindex 80a8c471..d4da8553 100644--- a/cli/beacon/internal/fxsession/mapper.go+++ b/cli/beacon/internal/fxsession/mapper.go@@ -2,7 +2,6 @@ package fxsessionimport ("crypto/sha1"- "crypto/sha256""encoding/hex""encoding/json""fmt"@@ -553,9 +552,7 @@ func (m *mapper) applyFile(ev *schema.Event, result ToolResult, evidence *FileRe}if diff := unifiedDiff(result.CommittedFile); diff != "" {info.Diff = diff- info.DiffBytes = len(diff)- info.DiffHash = sha256Hex(diff)- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}}if info.Path == "" && ev.Tool != nil {@@ -916,11 +913,6 @@ func firstNonEmpty(values ...string) string {return ""}-func sha256Hex(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}-// fxIDNamespace is the fixed UUID namespace for fx dedup-coordinate event IDs.// It is distinct from the content-based namespace in asymptoteobserve so the two// derivations can never collide.diff --git a/cli/beacon/internal/harness2session/mapper.go b/cli/beacon/internal/harness2session/mapper.goindex 78705ffb..2e651228 100644--- a/cli/beacon/internal/harness2session/mapper.go+++ b/cli/beacon/internal/harness2session/mapper.go@@ -320,8 +320,3 @@ func outputText(value interface{}) string {return string(data)}}--func sha256Hex(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}diff --git a/cli/beacon/internal/harness2session/opencode.go b/cli/beacon/internal/harness2session/opencode.goindex 397d5991..00c5aeeb 100644--- a/cli/beacon/internal/harness2session/opencode.go+++ b/cli/beacon/internal/harness2session/opencode.go@@ -81,7 +81,7 @@ func (m *mapper) emitOpenCodeTool(line Line, ts int64, part map[string]interfacecase "file":ev.File = openCodeFile(action, ev.Tool.Path, args, object(state, "metadata"))if ev.File.Diff != "" {- ev.Content = asymptoteobserve.RetainedContent(ev.File.Diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, ev.File.DiffHash, ev.File.DiffBytes = asymptoteobserve.RetainedDiff(ev.File.Diff, asymptoteobserve.DefaultStringLimit)}case "mcp":ev.MCP = &schema.MCPInfo{Tool: name}@@ -131,12 +131,7 @@ func openCodeFile(action, path string, args, metadata map[string]interface{}) *sfile.Language = strings.TrimPrefix(filepath.Ext(path), ".")}if operation != "read" {- diff := firstNonEmpty(str(metadata, "diff"), argString(args, "diff", "patch", "patchText"), argString(args, "content", "newString", "new_string"))- if diff != "" {- file.Diff = diff- file.DiffBytes = len(diff)- file.DiffHash = sha256Hex(diff)- }+ file.Diff = firstNonEmpty(str(metadata, "diff"), argString(args, "diff", "patch", "patchText"), argString(args, "content", "newString", "new_string"))}return file}diff --git a/cli/beacon/internal/hermessession/collect.go b/cli/beacon/internal/hermessession/collect.goindex b0774955..d547e3f3 100644--- a/cli/beacon/internal/hermessession/collect.go+++ b/cli/beacon/internal/hermessession/collect.go@@ -809,9 +809,7 @@ func (m *mapper) mapToolResult(message Message, call toolCall) {}if diff := firstString(call.Arguments, "diff", "patch"); diff != "" {ev.File.Diff = diff- ev.File.DiffBytes = len(diff)- ev.File.DiffHash = sha256String(diff)- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, ev.File.DiffHash, ev.File.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}case "mcp.tool_invoked":ev.MCP = &schema.MCPInfo{Tool: call.Name}@@ -1101,11 +1099,6 @@ func sessionCost(session Session) float64 {return session.EstimatedCostUSD}-func sha256String(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}-var hermesIDNamespace = [16]byte{0x58, 0x24, 0x3d, 0x29, 0x5e, 0xa1, 0x4b, 0xbe,0x85, 0xa7, 0x49, 0x11, 0xa7, 0x18, 0xdb, 0x0c,diff --git a/cli/beacon/internal/openclawsession/mapper.go b/cli/beacon/internal/openclawsession/mapper.goindex 58134834..dc7ecd28 100644--- a/cli/beacon/internal/openclawsession/mapper.go+++ b/cli/beacon/internal/openclawsession/mapper.go@@ -2,7 +2,6 @@ package openclawsessionimport ("crypto/sha1"- "crypto/sha256""encoding/hex""encoding/json""fmt"@@ -295,9 +294,7 @@ func applyFile(ev *schema.Event, name string, record Record) {}if diff := diffText(name, record); diff != "" {file.Diff = diff- file.DiffBytes = len(diff)- file.DiffHash = sha256Hex(diff)- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, file.DiffHash, file.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}ev.File = file}@@ -438,8 +435,3 @@ func firstNonZero(values ...int64) int64 {}return 0}--func sha256Hex(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}diff --git a/cli/beacon/internal/opencodesession/helpers.go b/cli/beacon/internal/opencodesession/helpers.goindex 26fa8fa6..08ff0d9b 100644--- a/cli/beacon/internal/opencodesession/helpers.go+++ b/cli/beacon/internal/opencodesession/helpers.go@@ -1,8 +1,6 @@package opencodesessionimport (- "crypto/sha256"- "encoding/hex""encoding/json""fmt""math"@@ -176,8 +174,3 @@ func opencodeModel(raw map[string]interface{}) string {}return name}--func sha256Hex(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}diff --git a/cli/beacon/internal/opencodesession/mapper.go b/cli/beacon/internal/opencodesession/mapper.goindex 6ad83870..d318be2d 100644--- a/cli/beacon/internal/opencodesession/mapper.go+++ b/cli/beacon/internal/opencodesession/mapper.go@@ -325,9 +325,7 @@ func applyFile(ev *schema.Event, name string, record Record) {}if diff := diffText(name, record); diff != "" {file.Diff = diff- file.DiffBytes = len(diff)- file.DiffHash = sha256Hex(diff)- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, file.DiffHash, file.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}ev.File = file}diff --git a/cli/beacon/internal/pisession/mapper.go b/cli/beacon/internal/pisession/mapper.goindex 5d13606f..c080a306 100644--- a/cli/beacon/internal/pisession/mapper.go+++ b/cli/beacon/internal/pisession/mapper.go@@ -2,7 +2,6 @@ package pisessionimport ("crypto/sha1"- "crypto/sha256""encoding/hex""encoding/json""fmt"@@ -413,9 +412,7 @@ func (m *mapper) applyFileResult(ev *schema.Event, name string, message map[stridiff := firstNonEmpty(stringValue(details["patch"]), stringValue(details["diff"]))if diff != "" {ev.File.Diff = diff- ev.File.DiffBytes = len(diff)- ev.File.DiffHash = sha256Hex(diff)- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)+ ev.Content, ev.File.DiffHash, ev.File.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)}}@@ -798,11 +795,6 @@ func refTimestamp(ref SessionRef) time.Time {return time.Now().UTC()}-func sha256Hex(value string) string {- sum := sha256.Sum256([]byte(value))- return hex.EncodeToString(sum[:])-}-var piIDNamespace = [16]byte{0x30, 0xa5, 0xe5, 0x5d, 0x8e, 0x20, 0x49, 0x24,0x9f, 0x81, 0x15, 0xb7, 0x59, 0x2a, 0x4f, 0x0d,diff --git a/collector-builder/exporter/beaconjsonexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/exporter_test.goindex 240b939b..ca447a22 100644--- a/collector-builder/exporter/beaconjsonexporter/exporter_test.go+++ b/collector-builder/exporter/beaconjsonexporter/exporter_test.go@@ -2,6 +2,8 @@ package beaconjsonexporterimport ("context"+ "crypto/sha256"+ "encoding/hex""encoding/json""fmt""math"@@ -965,6 +967,43 @@ func TestConsumeLogsMapsGeminiPrompt(t *testing.T) {}}+// content.hash leaves the machine where the prompt does not (Metadata-only keeps it), so it is the+// hash of the prompt the exporter stores after redaction, never of the text before it.+func TestConsumeLogsHashesTheStoredRedactedPrompt(t *testing.T) {+ hexHash := func(text string) string {+ sum := sha256.Sum256([]byte(text))+ return hex.EncodeToString(sum[:])+ }+ for _, tc := range []struct{ prompt, stored string }{+ {"deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2", "deploy with ghp_[REDACTED] and password=[REDACTED]"},+ // Nothing to redact: the hash is the one it always was.+ {"the quick brown fox", "the quick brown fox"},+ } {+ exp, path := newTestExporter(t)+ logs := plog.NewLogs()+ rl := logs.ResourceLogs().AppendEmpty()+ rl.Resource().Attributes().PutStr("service.name", "claude-code")+ rec := rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty()+ rec.Body().SetStr("claude_code.user_prompt")+ rec.Attributes().PutStr("event.name", "user_prompt")+ rec.Attributes().PutStr("session.id", "claude-session")+ rec.Attributes().PutStr("prompt", tc.prompt)+ if err := exp.consumeLogs(context.Background(), logs); err != nil {+ t.Fatalf("consumeLogs returned error: %v", err)+ }+ event := readEvent(t, path)+ if event.Prompt == nil || event.Prompt.Text != tc.stored {+ t.Fatalf("prompt = %#v, want %q", event.Prompt, tc.stored)+ }+ if event.Content == nil || event.Content.Hash != hexHash(tc.stored) || event.Content.Bytes != len(tc.prompt) {+ t.Fatalf("content = %#v, want hash %s and the original %d bytes", event.Content, hexHash(tc.stored), len(tc.prompt))+ }+ if event.Content.Redacted != (tc.prompt != tc.stored) {+ t.Fatalf("content.redacted = %t for %q", event.Content.Redacted, tc.prompt)+ }+ }+}+func TestConsumeLogsMapsGeminiMCPTool(t *testing.T) {exp, path := newTestExporter(t)diff --git a/collector-builder/exporter/beaconjsonexporter/writer.go b/collector-builder/exporter/beaconjsonexporter/writer.goindex 57675b1b..f744132b 100644--- a/collector-builder/exporter/beaconjsonexporter/writer.go+++ b/collector-builder/exporter/beaconjsonexporter/writer.go@@ -73,7 +73,7 @@ func (w jsonlWriter) append(event beaconEvent) error {// compactRetainedContent drops an event's retained text and records that it did,// so a reader can still tell what the event was about and that its content was// held back rather than never captured. The hash and byte count in the content-// marker describe the original text and stay put.+// marker describe the whole redacted text and stay put.func compactRetainedContent(event *beaconEvent) {if event.File != nil {event.File.Diff = ""diff --git a/pkg/asymptoteobserve/content.go b/pkg/asymptoteobserve/content.goindex 2ecd9dae..b3feac85 100644--- a/pkg/asymptoteobserve/content.go+++ b/pkg/asymptoteobserve/content.go@@ -159,12 +159,19 @@ func ReasoningOutputMessages(text string) []interface{} {// RetainedContent builds the content marker for an event that keeps text// verbatim -- a prompt, an assistant response, command output, a diff.//-// Hash and Bytes are computed over the original text, before the writer's-// redaction and truncation touch the stored copy, so an event keeps a stable-// identifier and a true size even when what is stored is shorter. That is the-// point of the marker: without it a reader cannot tell a short response from a-// truncated long one, and cannot tell that two truncated copies of the same-// response are the same response.+// Hash is computed over the whole text after secret redaction (RedactString)+// and before the writer's truncation; Bytes is the size of the original text.+// So an event keeps a stable identifier and a true size even when what is+// stored is shorter. That is the point of the marker: without it a reader+// cannot tell a short response from a truncated long one, and cannot tell that+// two truncated copies of the same response are the same response.+//+// The hash is of the redacted text because it leaves the machine wherever the+// text does not (a Metadata-only forward keeps it and drops the text). A hash of+// the text before redaction would let anyone holding the event confirm a guess+// at the secret the stored copy shows as [REDACTED]; a hash of the redacted text+// says nothing the stored text does not. Text with nothing to redact hashes+// exactly as it is.//// storeLimit is the per-string truncation limit the writer emitting this event// will apply to the text, and it is a parameter rather than a constant because@@ -180,7 +187,8 @@ func RetainedContent(text string, storeLimit int) *ContentInfo {if text == "" {return nil}- sum := sha256.Sum256([]byte(text))+ redacted := RedactString(text)+ sum := sha256.Sum256([]byte(redacted))info := &ContentInfo{Retention: ContentRetentionFull,Included: true,@@ -190,12 +198,31 @@ func RetainedContent(text string, storeLimit int) *ContentInfo {if storeLimit > 0 && len(text) > storeLimit {info.Truncated = true}- if RedactString(text) != text {+ if redacted != text {info.Redacted = true}return info}+// RetainedDiff is RetainedContent for a retained diff, together with the+// file.diff_hash and file.diff_bytes that sit beside it. All three come from one+// redaction of diff, so diff_hash is the marker's hash of the redacted diff,+// diff_bytes its original size, and every capture path computes them the same+// way. An empty diff returns nil, "" and 0.+func RetainedDiff(diff string, storeLimit int) (content *ContentInfo, diffHash string, diffBytes int) {+ content = RetainedContent(diff, storeLimit)+ if content == nil {+ return nil, "", 0+ }+ return content, content.Hash, content.Bytes+}++// RetainedDiffFields is RetainedDiff in the map form used by the hook adapter.+func RetainedDiffFields(diff string, storeLimit int) (content map[string]interface{}, diffHash string, diffBytes int) {+ info, diffHash, diffBytes := RetainedDiff(diff, storeLimit)+ return contentFields(info), diffHash, diffBytes+}+// RetainedContentFields is RetainedContent in the map form used by the hook// adapter, which assembles events as maps rather than as the Event struct.//diff --git a/pkg/asymptoteobserve/content_test.go b/pkg/asymptoteobserve/content_test.goindex 4fd12a50..cdaa5683 100644--- a/pkg/asymptoteobserve/content_test.go+++ b/pkg/asymptoteobserve/content_test.go@@ -34,6 +34,60 @@ func TestRetainedContentDescribesTheOriginalText(t *testing.T) {}}+// secretBearingText holds a GitHub token and an assigned password, both of which+// RedactString replaces; secretBearingStored is what every writer stores for it.+const (+ secretBearingText = "deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2"+ secretBearingStored = "deploy with ghp_[REDACTED] and password=[REDACTED]"+)++func sha256Hex(text string) string {+ sum := sha256.Sum256([]byte(text))+ return hex.EncodeToString(sum[:])+}++func TestRetainedContentHashesTheRedactedText(t *testing.T) {+ // The hash leaves the machine where the text does not (Metadata-only keeps it),+ // so a hash of the text before redaction confirms a guessed secret offline.+ if stored := CleanString(secretBearingText, DefaultStringLimit, true); stored != secretBearingStored {+ t.Fatalf("stored text = %q, want %q", stored, secretBearingStored)+ }+ info := RetainedContent(secretBearingText, DefaultStringLimit)+ if info.Hash != sha256Hex(secretBearingStored) || info.Bytes != len(secretBearingText) {+ t.Fatalf("marker = %#v, want the hash of the stored redacted text %s and the original size %d", info, sha256Hex(secretBearingStored), len(secretBearingText))+ }+ if info.Hash == sha256Hex(secretBearingText) {+ t.Fatal("hash is of the text before redaction; it confirms the secret")+ }+ if !info.Redacted || info.Truncated {+ t.Fatalf("marker = %#v, want redacted and not truncated", info)+ }++ content, diffHash, diffBytes := RetainedDiff(secretBearingText, DefaultStringLimit)+ if *content != *info || diffHash != info.Hash || diffBytes != info.Bytes {+ t.Fatalf("RetainedDiff = %#v %q %d, want the RetainedContent marker and its hash and size", content, diffHash, diffBytes)+ }+ fields, fieldsHash, fieldsBytes := RetainedDiffFields(secretBearingText, DefaultStringLimit)+ if fields["hash"] != info.Hash || fieldsHash != info.Hash || fieldsBytes != info.Bytes {+ t.Fatalf("RetainedDiffFields = %#v %q %d, want the RetainedContent hash and size", fields, fieldsHash, fieldsBytes)+ }+ if content, diffHash, diffBytes := RetainedDiff("", DefaultStringLimit); content != nil || diffHash != "" || diffBytes != 0 {+ t.Fatalf("RetainedDiff(\"\") = %#v %q %d, want nothing", content, diffHash, diffBytes)+ }+}++func TestRetainedContentHashOfTextWithoutSecretsIsUnchanged(t *testing.T) {+ // Pinned from before hashes were taken after redaction: ordinary events keep+ // the content.hash and file.diff_hash they always had.+ const want = "9ecb36561341d18eb65484e833efea61edc74b84cf5e6ae1b81c63533e25fc8f"+ if info := RetainedContent("the quick brown fox", DefaultStringLimit); info.Hash != want || info.Bytes != 19 || info.Redacted {+ t.Fatalf("marker = %#v, want hash %s over 19 bytes", info, want)+ }+ if _, diffHash, diffBytes := RetainedDiff("the quick brown fox", DefaultStringLimit); diffHash != want || diffBytes != 19 {+ t.Fatalf("diff = %q %d, want %s over 19 bytes", diffHash, diffBytes, want)+ }+}+func TestRetainedContentTruncationFollowsTheWritersLimit(t *testing.T) {// The same text is truncated by one writer and not the other, so the marker// has to answer against the limit it is given rather than a constant.diff --git a/pkg/asymptoteobserve/event.go b/pkg/asymptoteobserve/event.goindex 789cfe80..509aa64f 100644--- a/pkg/asymptoteobserve/event.go+++ b/pkg/asymptoteobserve/event.go@@ -164,9 +164,11 @@ type FileInfo struct {Language string `json:"language,omitempty"`DiffHash string `json:"diff_hash,omitempty"`DiffBytes int `json:"diff_bytes,omitempty"`- // Diff is the retained unified diff for the edit. DiffHash and DiffBytes- // describe the original diff, so they stay stable when the stored text is- // redacted or truncated; the accompanying Content marker says which happened.+ // Diff is the retained unified diff for the edit. DiffHash is the hash of the+ // whole diff after secret redaction and DiffBytes the original diff's size+ // (RetainedDiff), so both stay stable when the stored copy is truncated and+ // the hash never confirms a guess at a secret the stored diff no longer+ // shows; the accompanying Content marker says whether either happened.//// Declared here because the hook adapter has always written it. It assembles// events as maps, so `file.diff` reached the log while this struct -- what the@@ -308,9 +310,11 @@ const (SystemContextSourceTranscript = "transcript")-// ContentInfo marks how an event's retained raw content was handled. Hash and-// Bytes describe the original content before redaction/truncation, so events-// keep a stable identifier and size even when the stored text was cut down.+// ContentInfo marks how an event's retained raw content was handled. Hash is of+// the whole content after secret redaction and Bytes the original content's+// size, both before truncation, so events keep a stable identifier and size+// even when the stored text was cut down, and the hash confirms nothing+// redaction removed.type ContentInfo struct {Retention string `json:"retention"`Included bool `json:"included"`