Skip to content

Find, verify, and analyze leaked credentials

Utf16 bom support (#4326)

joeleonjrc319bb811e15
diff --git a/pkg/decoders/utf16_test.go b/pkg/decoders/utf16_test.go
index 9a05e0113..1d06b3764 100644
--- a/pkg/decoders/utf16_test.go
+++ b/pkg/decoders/utf16_test.go
@@ -27,6 +27,18 @@ func TestUTF16Decoder(t *testing.T) {
expected: []byte("Hello World"),
expectNil: false,
},
+ {
+ name: "Valid UTF-16LE input with BOM (FF FE)",
+ input: []byte{255, 254, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100, 0},
+ expected: []byte("Hello World"),
+ expectNil: false,
+ },
+ {
+ name: "Valid UTF-16BE input with BOM (FE FF)",
+ input: []byte{254, 255, 0, 72, 0, 101, 0, 108, 0, 108, 0, 111, 0, 32, 0, 87, 0, 111, 0, 114, 0, 108, 0, 100},
+ expected: []byte("Hello World"),
+ expectNil: false,
+ },
{
name: "Invalid UTF-16 input (it's UTF-8)",
input: []byte("Hello World!"),
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 287a19188..f5de6cb3b 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -795,7 +795,9 @@ func (e *Engine) scannerWorker(ctx context.Context) {
sourceVerify := chunk.Verify
for _, decoder := range e.decoders {
decodeStart := time.Now()
- decoded := decoder.FromChunk(chunk)
+ // This copy is needed to preserve the original chunk.Data across multiple decoders.
+ chunkCopy := *chunk
+ decoded := decoder.FromChunk(&chunkCopy)
decodeTime := time.Since(decodeStart).Microseconds()
decodeLatency.WithLabelValues(decoder.Type().String(), chunk.SourceName).Observe(float64(decodeTime))