Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(privacy): hash stored content after redaction (#1159)

kyle-elliott-asymptotebd672bd6f2ba
diff --git a/cli/beacon-hooks/cmd/agent_thought.go b/cli/beacon-hooks/cmd/agent_thought.go
index 5627d804..c095cf35 100644
--- a/cli/beacon-hooks/cmd/agent_thought.go
+++ b/cli/beacon-hooks/cmd/agent_thought.go
@@ -57,8 +57,9 @@ func reasoningOutputMessages(text string) []interface{} {
}
// retainedContentFields builds the content marker for an event that retains raw
-// text, computed against the original text so hash and byte count stay stable
-// even after the sink redacts or truncates the stored copy.
+// text: the hash of the whole redacted text and the original byte count, so both
+// stay stable when the sink truncates the stored copy, and the hash confirms
+// nothing redaction removed.
//
// The limit passed is the one this writer applies: the hook logger sanitizes the
// whole event map at DefaultStringLimit, so that is what "truncated" has to mean
@@ -67,6 +68,18 @@ func retainedContentFields(text string) map[string]interface{} {
return asymptoteobserve.RetainedContentFields(text, asymptoteobserve.DefaultStringLimit)
}
+// retainedDiffFields puts diff, its diff_hash and its diff_bytes into file and
+// returns the content marker for it, all from one redaction of diff at the
+// limit retainedContentFields uses, so diff_hash and diff_bytes always equal the
+// marker's hash and bytes.
+func retainedDiffFields(file map[string]interface{}, diff string) map[string]interface{} {
+ content, diffHash, diffBytes := asymptoteobserve.RetainedDiffFields(diff, asymptoteobserve.DefaultStringLimit)
+ file["diff"] = diff
+ file["diff_hash"] = diffHash
+ file["diff_bytes"] = diffBytes
+ return content
+}
+
// thoughtMetadataFields extracts the non-content afterAgentThought payload
// metadata worth retaining; the reasoning text itself is excluded because it
// already lives in gen_ai.output.messages.
diff --git a/cli/beacon-hooks/cmd/content_test.go b/cli/beacon-hooks/cmd/content_test.go
index 183fe4e4..674987b6 100644
--- a/cli/beacon-hooks/cmd/content_test.go
+++ b/cli/beacon-hooks/cmd/content_test.go
@@ -81,6 +81,49 @@ func TestPromptSubmitRecordsRetainedPromptContent(t *testing.T) {
}
}
+// content.hash and file.diff_hash leave the machine where the text does not (Metadata-only keeps
+// them), so each is the hash of what the hook writer stores after redaction: a hash of the text
+// before it lets anyone holding the event confirm a guessed secret offline.
+func TestHookHashesAreOfTheStoredRedactedText(t *testing.T) {
+ const secret = "deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2"
+ const stored = "deploy with ghp_[REDACTED] and password=[REDACTED]"
+ hexHash := func(text string) string {
+ sum := sha256.Sum256([]byte(text))
+ return hex.EncodeToString(sum[:])
+ }
+ setupHookConfigDirs(t)
+ platformFlag = "claude"
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)
+
+ runHookWithInput(t, runPromptSubmit, map[string]interface{}{"session_id": "redacted-hash-session", "prompt": secret})
+ prompt := lastEndpointEvent(t, logPath)
+ if got := prompt["prompt"].(map[string]interface{})["text"]; got != stored {
+ t.Fatalf("prompt.text = %q, want %q", got, stored)
+ }
+ content := prompt["content"].(map[string]interface{})
+ if content["hash"] != hexHash(stored) || content["bytes"] != float64(len(secret)) || content["redacted"] != true {
+ t.Fatalf("content = %#v, want the hash of %q and the original size", content, stored)
+ }
+
+ runHookWithInput(t, runPostTool, map[string]interface{}{
+ "session_id": "redacted-hash-session",
+ "cwd": "/repo",
+ "tool_name": "Edit",
+ "tool_input": map[string]interface{}{"file_path": "/repo/main.go", "old_string": "old", "new_string": secret},
+ })
+ edit := lastEndpointEvent(t, logPath)
+ file := edit["file"].(map[string]interface{})
+ diff, _ := file["diff"].(string)
+ if !strings.Contains(diff, stored) || strings.Contains(diff, "hunter2") {
+ t.Fatalf("file.diff = %q, want the redacted secret", diff)
+ }
+ marker := edit["content"].(map[string]interface{})
+ if file["diff_hash"] != hexHash(diff) || marker["hash"] != file["diff_hash"] || marker["bytes"] != file["diff_bytes"] {
+ t.Fatalf("file = %#v content = %#v, want diff_hash of the stored diff, matching the marker", file, marker)
+ }
+}
+
func TestPromptSubmitWithoutAPromptWritesNoContentMarker(t *testing.T) {
setupHookConfigDirs(t)
platformFlag = "claude"
diff --git a/cli/beacon-hooks/cmd/endpoint_events.go b/cli/beacon-hooks/cmd/endpoint_events.go
index 90653673..809c62c0 100644
--- a/cli/beacon-hooks/cmd/endpoint_events.go
+++ b/cli/beacon-hooks/cmd/endpoint_events.go
@@ -2,8 +2,6 @@ package cmd
import (
"context"
- "crypto/sha256"
- "encoding/hex"
"fmt"
"os"
"path/filepath"
@@ -791,14 +789,10 @@ func diffFields(filePath, diffStr string) map[string]interface{} {
}
fields := map[string]interface{}{"file": file}
if diffStr != "" {
- sum := sha256.Sum256([]byte(diffStr))
- file["diff_hash"] = hex.EncodeToString(sum[:])
- file["diff_bytes"] = len(diffStr)
- file["diff"] = diffStr
- // diff_hash and diff_bytes already describe the original diff; the marker
+ // diff_hash (of the whole redacted diff) and diff_bytes (its original size) describe the diff; the marker
// adds what they cannot say -- whether the stored copy is the whole diff,
// and whether redaction rewrote part of it.
- fields["content"] = retainedContentFields(diffStr)
+ fields["content"] = retainedDiffFields(file, diffStr)
}
return fields
}
diff --git a/cli/beacon-hooks/cmd/openclaw_event.go b/cli/beacon-hooks/cmd/openclaw_event.go
index d5efc0ff..1d816df9 100644
--- a/cli/beacon-hooks/cmd/openclaw_event.go
+++ b/cli/beacon-hooks/cmd/openclaw_event.go
@@ -1,8 +1,6 @@
package cmd
import (
- "crypto/sha256"
- "encoding/hex"
"path/filepath"
"strings"
@@ -632,16 +630,11 @@ func openClawToolFields(event map[string]interface{}, result map[string]interfac
file["language"] = language
}
if diff := openClawFileDiff(name, path, params); diff != "" {
- // The diff text itself, alongside the hash and byte count taken before the writer
- // redacts or truncates the stored copy. The `content` marker beside it says
- // whether what landed in the log is the whole diff; the hash and size say what the
- // original was. diffFields builds the same three keys for the shared hook path, so
- // a Beacon consumer reads one shape whichever runtime produced the row.
- sum := sha256.Sum256([]byte(diff))
- file["diff"] = diff
- file["diff_hash"] = hex.EncodeToString(sum[:])
- file["diff_bytes"] = len(diff)
- fields["content"] = retainedContentFields(diff)
+ // The diff text itself, alongside the hash of the whole redacted diff and the original's
+ // byte count. The `content` marker beside it says whether what landed in the log is the
+ // whole diff. diffFields builds the same three keys for the shared hook path, so a
+ // Beacon consumer reads one shape whichever runtime produced the row.
+ fields["content"] = retainedDiffFields(file, diff)
}
fields["file"] = file
}
diff --git a/cli/beacon/internal/copilotsession/mapper.go b/cli/beacon/internal/copilotsession/mapper.go
index ff5f2fe1..8b57f5d0 100644
--- a/cli/beacon/internal/copilotsession/mapper.go
+++ b/cli/beacon/internal/copilotsession/mapper.go
@@ -2,7 +2,6 @@ package copilotsession
import (
"crypto/sha1"
- "crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
@@ -589,10 +588,7 @@ func applyFile(ev *schema.Event, call toolCall, result toolResult, action string
}
if diff := diffFromCall(call, path); diff != "" && action != "file.read" {
info.Diff = diff
- info.DiffBytes = len(diff)
- sum := sha256.Sum256([]byte(diff))
- info.DiffHash = hex.EncodeToString(sum[:])
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
} else if result.Text != "" {
ev.Content = asymptoteobserve.RetainedContent(result.Text, asymptoteobserve.DefaultStringLimit)
}
diff --git a/cli/beacon/internal/cursorsession/mapper.go b/cli/beacon/internal/cursorsession/mapper.go
index 4f3dbace..6bb677bf 100644
--- a/cli/beacon/internal/cursorsession/mapper.go
+++ b/cli/beacon/internal/cursorsession/mapper.go
@@ -2,7 +2,6 @@ package cursorsession
import (
"crypto/sha1"
- "crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
@@ -277,9 +276,7 @@ func applyFile(ev *schema.Event, record Record) {
}
if diff := diffFromArgs(record.Args); diff != "" {
info.Diff = diff
- info.DiffBytes = len(diff)
- info.DiffHash = sha256Hex(diff)
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
ev.File = info
}
@@ -345,11 +342,6 @@ func withGenAI(genAI *schema.GenAIInfo, edit func(*schema.GenAIInfo)) *schema.Ge
return genAI
}
-func sha256Hex(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
-
var cursorIDNamespace = [16]byte{
0x50, 0x73, 0x15, 0x64, 0xc6, 0x90, 0x44, 0x0f,
0x91, 0x4a, 0x42, 0x31, 0x77, 0xe1, 0x82, 0x9c,
diff --git a/cli/beacon/internal/cursorsession/redacted_hash_test.go b/cli/beacon/internal/cursorsession/redacted_hash_test.go
new file mode 100644
index 00000000..51f5f416
--- /dev/null
+++ b/cli/beacon/internal/cursorsession/redacted_hash_test.go
@@ -0,0 +1,57 @@
+package cursorsession
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+func storedTextHash(text string) string {
+ sum := sha256.Sum256([]byte(text))
+ return hex.EncodeToString(sum[:])
+}
+
+// content.hash and file.diff_hash leave the machine where the text does not (Metadata-only keeps
+// them), so each must be the hash of the text the writer stores after redaction: a hash of the
+// text before it lets anyone holding the event confirm a guessed secret.
+func TestMapTraceHashesWhatTheWriterStores(t *testing.T) {
+ const secret = "deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2"
+ ref := TraceRef{ID: "s-1", Kind: SourceTranscript, SourcePath: "/tmp/s-1.jsonl", Workspace: "/work"}
+ records := []Record{
+ {Order: 1, NativeID: "u1", Type: "user_message", Content: secret},
+ {Order: 2, NativeID: "t1", Type: "tool_result", ToolName: "edit_file", Args: map[string]interface{}{"path": "main.go", "old_string": "x", "new_string": secret}},
+ {Order: 3, NativeID: "u2", Type: "user_message", Content: "the quick brown fox"},
+ }
+ mapped := MapTrace(ref, records, MapOptions{})
+ if len(mapped) != 3 {
+ t.Fatalf("mapped = %d events, want 3: %+v", len(mapped), mapped)
+ }
+
+ prompt := asymptoteobserve.SanitizeEvent(mapped[0].Event, asymptoteobserve.DefaultMaxEventBytes)
+ if prompt.Prompt.Text != "deploy with ghp_[REDACTED] and password=[REDACTED]" {
+ t.Fatalf("stored prompt = %q", prompt.Prompt.Text)
+ }
+ if prompt.Content.Hash != storedTextHash(prompt.Prompt.Text) || prompt.Content.Bytes != len(secret) {
+ t.Fatalf("prompt marker = %+v, want the hash of the stored text %q and the original size", prompt.Content, prompt.Prompt.Text)
+ }
+
+ originalDiffBytes := len(mapped[1].Event.File.Diff) // SanitizeEvent rewrites it through the pointer
+ edit := asymptoteobserve.SanitizeEvent(mapped[1].Event, asymptoteobserve.DefaultMaxEventBytes)
+ if edit.File == nil || !strings.Contains(edit.File.Diff, "+deploy with ghp_[REDACTED] and password=[REDACTED]") {
+ t.Fatalf("edit file = %+v", edit.File)
+ }
+ if edit.File.DiffHash != storedTextHash(edit.File.Diff) || edit.File.DiffBytes != originalDiffBytes || originalDiffBytes == len(edit.File.Diff) {
+ t.Fatalf("diff_hash %s over %d bytes, want the hash of the stored diff %q and the original size", edit.File.DiffHash, edit.File.DiffBytes, edit.File.Diff)
+ }
+ if edit.Content.Hash != edit.File.DiffHash || !edit.Content.Redacted {
+ t.Fatalf("edit marker = %+v, want the diff_hash and redacted", edit.Content)
+ }
+
+ // Text with nothing to redact keeps the hash it always had.
+ if plain := mapped[2].Event.Content; plain.Hash != "9ecb36561341d18eb65484e833efea61edc74b84cf5e6ae1b81c63533e25fc8f" || plain.Bytes != 19 {
+ t.Fatalf("plain marker = %+v", plain)
+ }
+}
diff --git a/cli/beacon/internal/dshsession/mapper.go b/cli/beacon/internal/dshsession/mapper.go
index 0e1c1b75..d92cffbd 100644
--- a/cli/beacon/internal/dshsession/mapper.go
+++ b/cli/beacon/internal/dshsession/mapper.go
@@ -2,7 +2,6 @@ package dshsession
import (
"crypto/sha1"
- "crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
@@ -427,10 +426,7 @@ func applyFile(ev *schema.Event, call toolCall, result toolResult) {
}
if diff := diffFromCall(call); diff != "" && !result.IsError {
info.Diff = diff
- info.DiffBytes = len(diff)
- sum := sha256.Sum256([]byte(diff))
- info.DiffHash = hex.EncodeToString(sum[:])
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
ev.File = info
}
diff --git a/cli/beacon/internal/fxsession/mapper.go b/cli/beacon/internal/fxsession/mapper.go
index 80a8c471..d4da8553 100644
--- a/cli/beacon/internal/fxsession/mapper.go
+++ b/cli/beacon/internal/fxsession/mapper.go
@@ -2,7 +2,6 @@ package fxsession
import (
"crypto/sha1"
- "crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
@@ -553,9 +552,7 @@ func (m *mapper) applyFile(ev *schema.Event, result ToolResult, evidence *FileRe
}
if diff := unifiedDiff(result.CommittedFile); diff != "" {
info.Diff = diff
- info.DiffBytes = len(diff)
- info.DiffHash = sha256Hex(diff)
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, info.DiffHash, info.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
}
if info.Path == "" && ev.Tool != nil {
@@ -916,11 +913,6 @@ func firstNonEmpty(values ...string) string {
return ""
}
-func sha256Hex(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
-
// fxIDNamespace is the fixed UUID namespace for fx dedup-coordinate event IDs.
// It is distinct from the content-based namespace in asymptoteobserve so the two
// derivations can never collide.
diff --git a/cli/beacon/internal/harness2session/mapper.go b/cli/beacon/internal/harness2session/mapper.go
index 78705ffb..2e651228 100644
--- a/cli/beacon/internal/harness2session/mapper.go
+++ b/cli/beacon/internal/harness2session/mapper.go
@@ -320,8 +320,3 @@ func outputText(value interface{}) string {
return string(data)
}
}
-
-func sha256Hex(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
diff --git a/cli/beacon/internal/harness2session/opencode.go b/cli/beacon/internal/harness2session/opencode.go
index 397d5991..00c5aeeb 100644
--- a/cli/beacon/internal/harness2session/opencode.go
+++ b/cli/beacon/internal/harness2session/opencode.go
@@ -81,7 +81,7 @@ func (m *mapper) emitOpenCodeTool(line Line, ts int64, part map[string]interface
case "file":
ev.File = openCodeFile(action, ev.Tool.Path, args, object(state, "metadata"))
if ev.File.Diff != "" {
- ev.Content = asymptoteobserve.RetainedContent(ev.File.Diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, ev.File.DiffHash, ev.File.DiffBytes = asymptoteobserve.RetainedDiff(ev.File.Diff, asymptoteobserve.DefaultStringLimit)
}
case "mcp":
ev.MCP = &schema.MCPInfo{Tool: name}
@@ -131,12 +131,7 @@ func openCodeFile(action, path string, args, metadata map[string]interface{}) *s
file.Language = strings.TrimPrefix(filepath.Ext(path), ".")
}
if operation != "read" {
- diff := firstNonEmpty(str(metadata, "diff"), argString(args, "diff", "patch", "patchText"), argString(args, "content", "newString", "new_string"))
- if diff != "" {
- file.Diff = diff
- file.DiffBytes = len(diff)
- file.DiffHash = sha256Hex(diff)
- }
+ file.Diff = firstNonEmpty(str(metadata, "diff"), argString(args, "diff", "patch", "patchText"), argString(args, "content", "newString", "new_string"))
}
return file
}
diff --git a/cli/beacon/internal/hermessession/collect.go b/cli/beacon/internal/hermessession/collect.go
index b0774955..d547e3f3 100644
--- a/cli/beacon/internal/hermessession/collect.go
+++ b/cli/beacon/internal/hermessession/collect.go
@@ -809,9 +809,7 @@ func (m *mapper) mapToolResult(message Message, call toolCall) {
}
if diff := firstString(call.Arguments, "diff", "patch"); diff != "" {
ev.File.Diff = diff
- ev.File.DiffBytes = len(diff)
- ev.File.DiffHash = sha256String(diff)
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, ev.File.DiffHash, ev.File.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
case "mcp.tool_invoked":
ev.MCP = &schema.MCPInfo{Tool: call.Name}
@@ -1101,11 +1099,6 @@ func sessionCost(session Session) float64 {
return session.EstimatedCostUSD
}
-func sha256String(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
-
var hermesIDNamespace = [16]byte{
0x58, 0x24, 0x3d, 0x29, 0x5e, 0xa1, 0x4b, 0xbe,
0x85, 0xa7, 0x49, 0x11, 0xa7, 0x18, 0xdb, 0x0c,
diff --git a/cli/beacon/internal/openclawsession/mapper.go b/cli/beacon/internal/openclawsession/mapper.go
index 58134834..dc7ecd28 100644
--- a/cli/beacon/internal/openclawsession/mapper.go
+++ b/cli/beacon/internal/openclawsession/mapper.go
@@ -2,7 +2,6 @@ package openclawsession
import (
"crypto/sha1"
- "crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
@@ -295,9 +294,7 @@ func applyFile(ev *schema.Event, name string, record Record) {
}
if diff := diffText(name, record); diff != "" {
file.Diff = diff
- file.DiffBytes = len(diff)
- file.DiffHash = sha256Hex(diff)
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, file.DiffHash, file.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
ev.File = file
}
@@ -438,8 +435,3 @@ func firstNonZero(values ...int64) int64 {
}
return 0
}
-
-func sha256Hex(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
diff --git a/cli/beacon/internal/opencodesession/helpers.go b/cli/beacon/internal/opencodesession/helpers.go
index 26fa8fa6..08ff0d9b 100644
--- a/cli/beacon/internal/opencodesession/helpers.go
+++ b/cli/beacon/internal/opencodesession/helpers.go
@@ -1,8 +1,6 @@
package opencodesession
import (
- "crypto/sha256"
- "encoding/hex"
"encoding/json"
"fmt"
"math"
@@ -176,8 +174,3 @@ func opencodeModel(raw map[string]interface{}) string {
}
return name
}
-
-func sha256Hex(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
diff --git a/cli/beacon/internal/opencodesession/mapper.go b/cli/beacon/internal/opencodesession/mapper.go
index 6ad83870..d318be2d 100644
--- a/cli/beacon/internal/opencodesession/mapper.go
+++ b/cli/beacon/internal/opencodesession/mapper.go
@@ -325,9 +325,7 @@ func applyFile(ev *schema.Event, name string, record Record) {
}
if diff := diffText(name, record); diff != "" {
file.Diff = diff
- file.DiffBytes = len(diff)
- file.DiffHash = sha256Hex(diff)
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, file.DiffHash, file.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
ev.File = file
}
diff --git a/cli/beacon/internal/pisession/mapper.go b/cli/beacon/internal/pisession/mapper.go
index 5d13606f..c080a306 100644
--- a/cli/beacon/internal/pisession/mapper.go
+++ b/cli/beacon/internal/pisession/mapper.go
@@ -2,7 +2,6 @@ package pisession
import (
"crypto/sha1"
- "crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
@@ -413,9 +412,7 @@ func (m *mapper) applyFileResult(ev *schema.Event, name string, message map[stri
diff := firstNonEmpty(stringValue(details["patch"]), stringValue(details["diff"]))
if diff != "" {
ev.File.Diff = diff
- ev.File.DiffBytes = len(diff)
- ev.File.DiffHash = sha256Hex(diff)
- ev.Content = asymptoteobserve.RetainedContent(diff, asymptoteobserve.DefaultStringLimit)
+ ev.Content, ev.File.DiffHash, ev.File.DiffBytes = asymptoteobserve.RetainedDiff(diff, asymptoteobserve.DefaultStringLimit)
}
}
@@ -798,11 +795,6 @@ func refTimestamp(ref SessionRef) time.Time {
return time.Now().UTC()
}
-func sha256Hex(value string) string {
- sum := sha256.Sum256([]byte(value))
- return hex.EncodeToString(sum[:])
-}
-
var piIDNamespace = [16]byte{
0x30, 0xa5, 0xe5, 0x5d, 0x8e, 0x20, 0x49, 0x24,
0x9f, 0x81, 0x15, 0xb7, 0x59, 0x2a, 0x4f, 0x0d,
diff --git a/collector-builder/exporter/beaconjsonexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/exporter_test.go
index 240b939b..ca447a22 100644
--- a/collector-builder/exporter/beaconjsonexporter/exporter_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/exporter_test.go
@@ -2,6 +2,8 @@ package beaconjsonexporter
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
"encoding/json"
"fmt"
"math"
@@ -965,6 +967,43 @@ func TestConsumeLogsMapsGeminiPrompt(t *testing.T) {
}
}
+// content.hash leaves the machine where the prompt does not (Metadata-only keeps it), so it is the
+// hash of the prompt the exporter stores after redaction, never of the text before it.
+func TestConsumeLogsHashesTheStoredRedactedPrompt(t *testing.T) {
+ hexHash := func(text string) string {
+ sum := sha256.Sum256([]byte(text))
+ return hex.EncodeToString(sum[:])
+ }
+ for _, tc := range []struct{ prompt, stored string }{
+ {"deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2", "deploy with ghp_[REDACTED] and password=[REDACTED]"},
+ // Nothing to redact: the hash is the one it always was.
+ {"the quick brown fox", "the quick brown fox"},
+ } {
+ exp, path := newTestExporter(t)
+ logs := plog.NewLogs()
+ rl := logs.ResourceLogs().AppendEmpty()
+ rl.Resource().Attributes().PutStr("service.name", "claude-code")
+ rec := rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty()
+ rec.Body().SetStr("claude_code.user_prompt")
+ rec.Attributes().PutStr("event.name", "user_prompt")
+ rec.Attributes().PutStr("session.id", "claude-session")
+ rec.Attributes().PutStr("prompt", tc.prompt)
+ if err := exp.consumeLogs(context.Background(), logs); err != nil {
+ t.Fatalf("consumeLogs returned error: %v", err)
+ }
+ event := readEvent(t, path)
+ if event.Prompt == nil || event.Prompt.Text != tc.stored {
+ t.Fatalf("prompt = %#v, want %q", event.Prompt, tc.stored)
+ }
+ if event.Content == nil || event.Content.Hash != hexHash(tc.stored) || event.Content.Bytes != len(tc.prompt) {
+ t.Fatalf("content = %#v, want hash %s and the original %d bytes", event.Content, hexHash(tc.stored), len(tc.prompt))
+ }
+ if event.Content.Redacted != (tc.prompt != tc.stored) {
+ t.Fatalf("content.redacted = %t for %q", event.Content.Redacted, tc.prompt)
+ }
+ }
+}
+
func TestConsumeLogsMapsGeminiMCPTool(t *testing.T) {
exp, path := newTestExporter(t)
diff --git a/collector-builder/exporter/beaconjsonexporter/writer.go b/collector-builder/exporter/beaconjsonexporter/writer.go
index 57675b1b..f744132b 100644
--- a/collector-builder/exporter/beaconjsonexporter/writer.go
+++ b/collector-builder/exporter/beaconjsonexporter/writer.go
@@ -73,7 +73,7 @@ func (w jsonlWriter) append(event beaconEvent) error {
// compactRetainedContent drops an event's retained text and records that it did,
// so a reader can still tell what the event was about and that its content was
// held back rather than never captured. The hash and byte count in the content
-// marker describe the original text and stay put.
+// marker describe the whole redacted text and stay put.
func compactRetainedContent(event *beaconEvent) {
if event.File != nil {
event.File.Diff = ""
diff --git a/pkg/asymptoteobserve/content.go b/pkg/asymptoteobserve/content.go
index 2ecd9dae..b3feac85 100644
--- a/pkg/asymptoteobserve/content.go
+++ b/pkg/asymptoteobserve/content.go
@@ -159,12 +159,19 @@ func ReasoningOutputMessages(text string) []interface{} {
// RetainedContent builds the content marker for an event that keeps text
// verbatim -- a prompt, an assistant response, command output, a diff.
//
-// Hash and Bytes are computed over the original text, before the writer's
-// redaction and truncation touch the stored copy, so an event keeps a stable
-// identifier and a true size even when what is stored is shorter. That is the
-// point of the marker: without it a reader cannot tell a short response from a
-// truncated long one, and cannot tell that two truncated copies of the same
-// response are the same response.
+// Hash is computed over the whole text after secret redaction (RedactString)
+// and before the writer's truncation; Bytes is the size of the original text.
+// So an event keeps a stable identifier and a true size even when what is
+// stored is shorter. That is the point of the marker: without it a reader
+// cannot tell a short response from a truncated long one, and cannot tell that
+// two truncated copies of the same response are the same response.
+//
+// The hash is of the redacted text because it leaves the machine wherever the
+// text does not (a Metadata-only forward keeps it and drops the text). A hash of
+// the text before redaction would let anyone holding the event confirm a guess
+// at the secret the stored copy shows as [REDACTED]; a hash of the redacted text
+// says nothing the stored text does not. Text with nothing to redact hashes
+// exactly as it is.
//
// storeLimit is the per-string truncation limit the writer emitting this event
// will apply to the text, and it is a parameter rather than a constant because
@@ -180,7 +187,8 @@ func RetainedContent(text string, storeLimit int) *ContentInfo {
if text == "" {
return nil
}
- sum := sha256.Sum256([]byte(text))
+ redacted := RedactString(text)
+ sum := sha256.Sum256([]byte(redacted))
info := &ContentInfo{
Retention: ContentRetentionFull,
Included: true,
@@ -190,12 +198,31 @@ func RetainedContent(text string, storeLimit int) *ContentInfo {
if storeLimit > 0 && len(text) > storeLimit {
info.Truncated = true
}
- if RedactString(text) != text {
+ if redacted != text {
info.Redacted = true
}
return info
}
+// RetainedDiff is RetainedContent for a retained diff, together with the
+// file.diff_hash and file.diff_bytes that sit beside it. All three come from one
+// redaction of diff, so diff_hash is the marker's hash of the redacted diff,
+// diff_bytes its original size, and every capture path computes them the same
+// way. An empty diff returns nil, "" and 0.
+func RetainedDiff(diff string, storeLimit int) (content *ContentInfo, diffHash string, diffBytes int) {
+ content = RetainedContent(diff, storeLimit)
+ if content == nil {
+ return nil, "", 0
+ }
+ return content, content.Hash, content.Bytes
+}
+
+// RetainedDiffFields is RetainedDiff in the map form used by the hook adapter.
+func RetainedDiffFields(diff string, storeLimit int) (content map[string]interface{}, diffHash string, diffBytes int) {
+ info, diffHash, diffBytes := RetainedDiff(diff, storeLimit)
+ return contentFields(info), diffHash, diffBytes
+}
+
// RetainedContentFields is RetainedContent in the map form used by the hook
// adapter, which assembles events as maps rather than as the Event struct.
//
diff --git a/pkg/asymptoteobserve/content_test.go b/pkg/asymptoteobserve/content_test.go
index 4fd12a50..cdaa5683 100644
--- a/pkg/asymptoteobserve/content_test.go
+++ b/pkg/asymptoteobserve/content_test.go
@@ -34,6 +34,60 @@ func TestRetainedContentDescribesTheOriginalText(t *testing.T) {
}
}
+// secretBearingText holds a GitHub token and an assigned password, both of which
+// RedactString replaces; secretBearingStored is what every writer stores for it.
+const (
+ secretBearingText = "deploy with ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8 and password=hunter2"
+ secretBearingStored = "deploy with ghp_[REDACTED] and password=[REDACTED]"
+)
+
+func sha256Hex(text string) string {
+ sum := sha256.Sum256([]byte(text))
+ return hex.EncodeToString(sum[:])
+}
+
+func TestRetainedContentHashesTheRedactedText(t *testing.T) {
+ // The hash leaves the machine where the text does not (Metadata-only keeps it),
+ // so a hash of the text before redaction confirms a guessed secret offline.
+ if stored := CleanString(secretBearingText, DefaultStringLimit, true); stored != secretBearingStored {
+ t.Fatalf("stored text = %q, want %q", stored, secretBearingStored)
+ }
+ info := RetainedContent(secretBearingText, DefaultStringLimit)
+ if info.Hash != sha256Hex(secretBearingStored) || info.Bytes != len(secretBearingText) {
+ t.Fatalf("marker = %#v, want the hash of the stored redacted text %s and the original size %d", info, sha256Hex(secretBearingStored), len(secretBearingText))
+ }
+ if info.Hash == sha256Hex(secretBearingText) {
+ t.Fatal("hash is of the text before redaction; it confirms the secret")
+ }
+ if !info.Redacted || info.Truncated {
+ t.Fatalf("marker = %#v, want redacted and not truncated", info)
+ }
+
+ content, diffHash, diffBytes := RetainedDiff(secretBearingText, DefaultStringLimit)
+ if *content != *info || diffHash != info.Hash || diffBytes != info.Bytes {
+ t.Fatalf("RetainedDiff = %#v %q %d, want the RetainedContent marker and its hash and size", content, diffHash, diffBytes)
+ }
+ fields, fieldsHash, fieldsBytes := RetainedDiffFields(secretBearingText, DefaultStringLimit)
+ if fields["hash"] != info.Hash || fieldsHash != info.Hash || fieldsBytes != info.Bytes {
+ t.Fatalf("RetainedDiffFields = %#v %q %d, want the RetainedContent hash and size", fields, fieldsHash, fieldsBytes)
+ }
+ if content, diffHash, diffBytes := RetainedDiff("", DefaultStringLimit); content != nil || diffHash != "" || diffBytes != 0 {
+ t.Fatalf("RetainedDiff(\"\") = %#v %q %d, want nothing", content, diffHash, diffBytes)
+ }
+}
+
+func TestRetainedContentHashOfTextWithoutSecretsIsUnchanged(t *testing.T) {
+ // Pinned from before hashes were taken after redaction: ordinary events keep
+ // the content.hash and file.diff_hash they always had.
+ const want = "9ecb36561341d18eb65484e833efea61edc74b84cf5e6ae1b81c63533e25fc8f"
+ if info := RetainedContent("the quick brown fox", DefaultStringLimit); info.Hash != want || info.Bytes != 19 || info.Redacted {
+ t.Fatalf("marker = %#v, want hash %s over 19 bytes", info, want)
+ }
+ if _, diffHash, diffBytes := RetainedDiff("the quick brown fox", DefaultStringLimit); diffHash != want || diffBytes != 19 {
+ t.Fatalf("diff = %q %d, want %s over 19 bytes", diffHash, diffBytes, want)
+ }
+}
+
func TestRetainedContentTruncationFollowsTheWritersLimit(t *testing.T) {
// The same text is truncated by one writer and not the other, so the marker
// has to answer against the limit it is given rather than a constant.
diff --git a/pkg/asymptoteobserve/event.go b/pkg/asymptoteobserve/event.go
index 789cfe80..509aa64f 100644
--- a/pkg/asymptoteobserve/event.go
+++ b/pkg/asymptoteobserve/event.go
@@ -164,9 +164,11 @@ type FileInfo struct {
Language string `json:"language,omitempty"`
DiffHash string `json:"diff_hash,omitempty"`
DiffBytes int `json:"diff_bytes,omitempty"`
- // Diff is the retained unified diff for the edit. DiffHash and DiffBytes
- // describe the original diff, so they stay stable when the stored text is
- // redacted or truncated; the accompanying Content marker says which happened.
+ // Diff is the retained unified diff for the edit. DiffHash is the hash of the
+ // whole diff after secret redaction and DiffBytes the original diff's size
+ // (RetainedDiff), so both stay stable when the stored copy is truncated and
+ // the hash never confirms a guess at a secret the stored diff no longer
+ // shows; the accompanying Content marker says whether either happened.
//
// Declared here because the hook adapter has always written it. It assembles
// events as maps, so `file.diff` reached the log while this struct -- what the
@@ -308,9 +310,11 @@ const (
SystemContextSourceTranscript = "transcript"
)
-// ContentInfo marks how an event's retained raw content was handled. Hash and
-// Bytes describe the original content before redaction/truncation, so events
-// keep a stable identifier and size even when the stored text was cut down.
+// ContentInfo marks how an event's retained raw content was handled. Hash is of
+// the whole content after secret redaction and Bytes the original content's
+// size, both before truncation, so events keep a stable identifier and size
+// even when the stored text was cut down, and the hash confirms nothing
+// redaction removed.
type ContentInfo struct {
Retention string `json:"retention"`
Included bool `json:"included"`