Skip to content

Find, verify, and analyze leaked credentials

Thread original chunk data through engine pipeline (#4780)

Dustin Decker648aca62d543
diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.go
index b8574bc10..074373f96 100644
--- a/pkg/decoders/escaped_unicode.go
+++ b/pkg/decoders/escaped_unicode.go
@@ -105,6 +105,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
DecoderType: d.Type(),
Chunk: &sources.Chunk{
Data: chunkData,
+ OriginalData: chunk.OriginalData,
SourceName: chunk.SourceName,
SourceID: chunk.SourceID,
JobID: chunk.JobID,
diff --git a/pkg/detectors/copy_metadata_test.go b/pkg/detectors/copy_metadata_test.go
new file mode 100644
index 000000000..2eb9f6626
--- /dev/null
+++ b/pkg/detectors/copy_metadata_test.go
@@ -0,0 +1,40 @@
+package detectors
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestCopyMetadata_ChunkDataFromOriginalData(t *testing.T) {
+ chunk := &sources.Chunk{
+ Data: []byte("decoded-data"),
+ OriginalData: []byte("original-source-data"),
+ SourceName: "test-source",
+ }
+ result := Result{
+ DetectorType: 1,
+ Raw: []byte("secret"),
+ }
+
+ rwm := CopyMetadata(chunk, result)
+
+ assert.Equal(t, "original-source-data", string(rwm.ChunkData))
+}
+
+func TestCopyMetadata_ChunkDataFallsBackToData(t *testing.T) {
+ chunk := &sources.Chunk{
+ Data: []byte("only-data"),
+ SourceName: "test-source",
+ }
+ result := Result{
+ DetectorType: 1,
+ Raw: []byte("secret"),
+ }
+
+ rwm := CopyMetadata(chunk, result)
+
+ assert.Equal(t, "only-data", string(rwm.ChunkData))
+}
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go
index cf1ff4649..38f1ad446 100644
--- a/pkg/detectors/detectors.go
+++ b/pkg/detectors/detectors.go
@@ -217,10 +217,19 @@ type ResultWithMetadata struct {
DetectorDescription string
// DecoderType is the type of decoder that was used to generate this result's data.
DecoderType detectorspb.DecoderType
+ // ChunkData holds the original pre-decode source chunk data, preserved
+ // for secret storage encryption in the dispatcher.
+ ChunkData []byte
}
// CopyMetadata returns a detector result with included metadata from the source chunk.
func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {
+ // OriginalData may be nil when CopyMetadata is called outside the engine
+ // pipeline (e.g., in tests or external consumers that construct chunks directly).
+ chunkData := chunk.OriginalData
+ if chunkData == nil {
+ chunkData = chunk.Data
+ }
return ResultWithMetadata{
SourceMetadata: chunk.SourceMetadata,
SourceID: chunk.SourceID,
@@ -229,6 +238,7 @@ func CopyMetadata(chunk *sources.Chunk, result Result) ResultWithMetadata {
SourceType: chunk.SourceType,
SourceName: chunk.SourceName,
Result: result,
+ ChunkData: chunkData,
}
}
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 5e64ba63e..25f8270aa 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -854,6 +854,7 @@ func (e *Engine) scannerWorker(ctx context.Context) {
startTime := time.Now()
sourceVerify := chunk.SourceVerify
+ chunk.OriginalData = chunk.Data
decoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth)
for _, d := range decoded {
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 761b7b0ac..6a8c2b1d5 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -21,14 +21,13 @@ type (
)
// Chunk contains data to be decoded and scanned along with context on where it came from.
-//
-// **Important:** The order of the fields in this struct is specifically designed to optimize
-// struct alignment and minimize memory usage. Do not change the field order without carefully considering
-// the potential impact on memory consumption.
-// Ex: https://go.dev/play/p/Azf4a7O-DhC
type Chunk struct {
// Data is the data to decode and scan.
Data []byte
+ // OriginalData holds the pre-decode source data, preserved for secret
+ // storage. Set before iterative decoding so it retains the original
+ // content even after Data is replaced with decoded forms.
+ OriginalData []byte
// SourceName is the name of the Source that produced the chunk.
SourceName string
diff --git a/pkg/sources/sources_test.go b/pkg/sources/sources_test.go
index 49fd853fb..6db3fccbc 100644
--- a/pkg/sources/sources_test.go
+++ b/pkg/sources/sources_test.go
@@ -7,8 +7,10 @@ import (
"github.com/stretchr/testify/assert"
)
-// TestChunkSize ensures that the Chunk struct does not exceed 80 bytes.
+// TestChunkSize ensures that the Chunk struct does not exceed 104 bytes.
+// Size increased from 80 to 104 with the addition of OriginalData []byte
+// (24-byte slice header) for secret storage chunk threading.
func TestChunkSize(t *testing.T) {
t.Parallel()
- assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(80), "Chunk struct size exceeds 80 bytes")
+ assert.Equal(t, unsafe.Sizeof(Chunk{}), uintptr(104), "Chunk struct size exceeds 104 bytes")
}