Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Thread original chunk data through engine pipeline (#4780)
Dustin Decker648aca62d543
diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.goindex b8574bc10..074373f96 100644--- a/pkg/decoders/escaped_unicode.go+++ b/pkg/decoders/escaped_unicode.go@@ -105,6 +105,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {DecoderType: d.Type(),Chunk: &sources.Chunk{Data: chunkData,+ OriginalData: chunk.OriginalData,SourceName: chunk.SourceName,SourceID: chunk.SourceID,JobID: chunk.JobID,diff --git a/pkg/detectors/copy_metadata_test.go b/pkg/detectors/copy_metadata_test.gonew file mode 100644index 000000000..2eb9f6626--- /dev/null+++ b/pkg/detectors/copy_metadata_test.go@@ -0,0 +1,40 @@+package detectors++import (+ "testing"++ "github.com/stretchr/testify/assert"++ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++func TestCopyMetadata_ChunkDataFromOriginalData(t *testing.T) {+ chunk := &sources.Chunk{+ Data: []byte("decoded-data"),+ OriginalData: []byte("original-source-data"),+ SourceName: "test-source",+ }+ result := Result{+ DetectorType: 1,+ Raw: []byte("secret"),+ }++ rwm := CopyMetadata(chunk, result)++ assert.Equal(t, "original-source-data", string(rwm.ChunkData))+}++func TestCopyMetadata_ChunkDataFallsBackToData(t *testing.T) {+ chunk := &sources.Chunk{+ Data: []byte("only-data"),+ SourceName: "test-source",+ }+ result := Result{+ DetectorType: 1,+ Raw: []byte("secret"),+ }++ rwm := CopyMetadata(chunk, result)++ assert.Equal(t, "only-data", string(rwm.ChunkData))+}diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex cf1ff4649..38f1ad446 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -217,10 +217,19 @@ type ResultWithMetadata struct {DetectorDescription string// DecoderType is the type of decoder that was used to generate this result's data.DecoderType detectorspb.DecoderType+ // ChunkData holds the original pre-decode source chunk data, preserved+ // for secret storage encryption in the dispatcher.+ ChunkData []byte}// CopyMetadata returns a detector result with included metadata from the source chunk.func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {+ // OriginalData may be nil when CopyMetadata is called outside the engine+ // pipeline (e.g., in tests or external consumers that construct chunks directly).+ chunkData := chunk.OriginalData+ if chunkData == nil {+ chunkData = chunk.Data+ }return ResultWithMetadata{SourceMetadata: chunk.SourceMetadata,SourceID: chunk.SourceID,@@ -229,6 +238,7 @@ func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {SourceType: chunk.SourceType,SourceName: chunk.SourceName,Result: result,+ ChunkData: chunkData,}}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 5e64ba63e..25f8270aa 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -854,6 +854,7 @@ func (e *Engine) scannerWorker(ctx context.Context) {startTime := time.Now()sourceVerify := chunk.SourceVerify+ chunk.OriginalData = chunk.Datadecoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth)for _, d := range decoded {diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex 761b7b0ac..6a8c2b1d5 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -21,14 +21,13 @@ type ()// Chunk contains data to be decoded and scanned along with context on where it came from.-//-// **Important:** The order of the fields in this struct is specifically designed to optimize-// struct alignment and minimize memory usage. Do not change the field order without carefully considering-// the potential impact on memory consumption.-// Ex: https://go.dev/play/p/Azf4a7O-DhCtype Chunk struct {// Data is the data to decode and scan.Data []byte+ // OriginalData holds the pre-decode source data, preserved for secret+ // storage. Set before iterative decoding so it retains the original+ // content even after Data is replaced with decoded forms.+ OriginalData []byte// SourceName is the name of the Source that produced the chunk.SourceName stringdiff --git a/pkg/sources/sources_test.go b/pkg/sources/sources_test.goindex 49fd853fb..6db3fccbc 100644--- a/pkg/sources/sources_test.go+++ b/pkg/sources/sources_test.go@@ -7,8 +7,10 @@ import ("github.com/stretchr/testify/assert")-// TestChunkSize ensures that the Chunk struct does not exceed 80 bytes.+// TestChunkSize ensures that the Chunk struct does not exceed 104 bytes.+// Size increased from 80 to 104 with the addition of OriginalData []byte+// (24-byte slice header) for secret storage chunk threading.func TestChunkSize(t *testing.T) {t.Parallel()- assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(80), "Chunk struct size exceeds 80 bytes")+ assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(104), "Chunk struct size exceeds 104 bytes")}