Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[fix] - Improve UTF8 decoder's handling of non-printable characters (#3588)
ahravc6abe859ff1d
diff --git a/pkg/decoders/utf16.go b/pkg/decoders/utf16.goindex 5949196e4..f3da9c370 100644--- a/pkg/decoders/utf16.go+++ b/pkg/decoders/utf16.go@@ -37,12 +37,12 @@ func utf16ToUTF8(b []byte) ([]byte, error) {var bufBE, bufLE bytes.Bufferfor i := 0; i < len(b)-1; i += 2 {if r := rune(binary.BigEndian.Uint16(b[i:])); b[i] == 0 && utf8.ValidRune(r) {- if isValidByte(byte(r)) {+ if isPrintableByte(byte(r)) {bufBE.WriteRune(r)}}if r := rune(binary.LittleEndian.Uint16(b[i:])); b[i+1] == 0 && utf8.ValidRune(r) {- if isValidByte(byte(r)) {+ if isPrintableByte(byte(r)) {bufLE.WriteRune(r)}}diff --git a/pkg/decoders/utf8.go b/pkg/decoders/utf8.goindex 49cfea451..f8dd847a0 100644--- a/pkg/decoders/utf8.go+++ b/pkg/decoders/utf8.go@@ -1,7 +1,6 @@package decodersimport (- "bytes""unicode/utf8""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -29,35 +28,58 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {return decodableChunk}-// extractSubstrings performs similarly to the strings binutil,-// extacting contigous portions of printable characters that we care-// about from some bytes-func extractSubstrings(b []byte) []byte {+// utf8ReplacementBytes holds the UTF-8 encoded form of the Unicode replacement character (U+FFFD).+// This is pre-computed since it's used frequently when replacing invalid UTF-8 sequences+// and control characters.+var utf8ReplacementBytes = []byte(string(utf8.RuneError))- field := make([]byte, len(b))- fieldLen := 0- buf := &bytes.Buffer{}- for i, c := range b {- if isValidByte(c) {- field[fieldLen] = c- fieldLen++- } else {- if fieldLen > 5 {- buf.Write(field[:fieldLen])+// extractSubstrings sanitizes byte sequences to ensure consistent handling of malformed input+// while maintaining readable content. It handles ASCII and UTF-8 data as follows:+//+// For ASCII range (0-127): preserves printable characters (32-126) while replacing+// control characters with the UTF-8 replacement character.+// https://cs.opensource.google/go/go/+/refs/tags/go1.23.3:src/unicode/utf8/utf8.go;l=16+//+// For multi-byte sequences: preserves valid UTF-8 as-is, while invalid sequences+// are replaced with a single UTF-8 replacement character.+func extractSubstrings(b []byte) []byte {+ dataLen := len(b)+ buf := make([]byte, 0, dataLen)+ for idx := 0; idx < dataLen; {+ // If it's ASCII, handle separately.+ // This is faster than decoding for common cases.+ if b[idx] < utf8.RuneSelf {+ if isPrintableByte(b[idx]) {+ buf = append(buf, b[idx])+ } else {+ buf = append(buf, utf8ReplacementBytes...)}- fieldLen = 0+ idx+++ continue}- if i == len(b)-1 && fieldLen > 5 {- buf.Write(field[:fieldLen])+ r, size := utf8.DecodeRune(b[idx:])+ if r == utf8.RuneError {+ // Collapse any malformed sequence into a single replacement character+ // rather than replacing each byte individually.+ buf = append(buf, utf8ReplacementBytes...)+ idx+++ } else {+ // Keep valid multi-byte UTF-8 sequences intact to preserve unicode characters.+ buf = append(buf, b[idx:idx+size]...)+ idx += size}}- return buf.Bytes()+ return buf}-func isValidByte(c byte) bool {- // https://www.rapidtables.com/code/text/ascii-table.html- // split on anything that is not ascii space through tilde- return c > 31 && c < 127-}+// isPrintableByte reports whether a byte represents a printable ASCII character+// using a fast byte-range check. This avoids the overhead of utf8.DecodeRune+// for the common case of ASCII characters (0-127), since we know any byte < 128+// represents a complete ASCII character and doesn't need UTF-8 decoding.+// This includes letters, digits, punctuation, and symbols, but excludes control characters.+// The upper bound is 127 (not 128) because 127 is the DEL control character.+//+// https://www.rapidtables.com/code/text/ascii-table.html+func isPrintableByte(c byte) bool { return c > 31 && c < 127 }diff --git a/pkg/decoders/utf8_test.go b/pkg/decoders/utf8_test.goindex 18205d327..7d1c0c86a 100644--- a/pkg/decoders/utf8_test.go+++ b/pkg/decoders/utf8_test.go@@ -1,6 +1,7 @@package decodersimport (+ "strings""testing""github.com/kylelemons/godebug/pretty"@@ -8,7 +9,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources")-func TestUTF8_FromChunk(t *testing.T) {+func TestUTF8_FromChunk_ValidUTF8(t *testing.T) {type args struct {chunk *sources.Chunk}@@ -29,35 +30,327 @@ func TestUTF8_FromChunk(t *testing.T) {wantErr: false,},{- name: "successful binary decode",+ name: "empty chunk",d: &UTF8{},args: args{- chunk: &sources.Chunk{Data: []byte("\xf0\x28\x8c\x28 not-entirely utf8 chunk that should decode successfully")},+ chunk: nil,},- want: &sources.Chunk{Data: []byte("( not-entirely utf8 chunk that should decode successfully")},+ want: nil,wantErr: false,},{- name: "unsuccessful decode",+ name: "valid UTF8 with control characters",d: &UTF8{},args: args{- chunk: nil,+ chunk: &sources.Chunk{Data: []byte("FIRST_KEY_123456\x00SECOND_KEY_789012")},},- want: nil,+ want: &sources.Chunk{Data: []byte("FIRST_KEY_123456\x00SECOND_KEY_789012")},+ wantErr: false,+ },+ {+ name: "valid UTF8 with all ASCII control characters",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 'S', 'T', 'A', 'R', 'T',+ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,+ 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14,+ 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,+ 'E', 'N', 'D',+ }},+ },+ want: &sources.Chunk{Data: []byte("START\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1FEND")},+ wantErr: false,+ },+ {+ name: "aws key in binary data - valid utf8",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("AWS_ACCESS_KEY_ID\x00\x00\x00AKIAEXAMPLEKEY123\x00")},+ },+ want: &sources.Chunk{Data: []byte("AWS_ACCESS_KEY_ID\x00\x00\x00AKIAEXAMPLEKEY123\x00")},+ wantErr: false,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ d := &UTF8{}+ got := d.FromChunk(tt.args.chunk)+ if got != nil && tt.want != nil {+ if diff := pretty.Compare(string(got.Data), string(tt.want.Data)); diff != "" {+ t.Errorf("%s: UTF8.FromChunk() diff: (-got +want)\n%s", tt.name, diff)+ }+ } else {+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("%s: UTF8.FromChunk() diff: (-got +want)\n%s", tt.name, diff)+ }+ }+ })+ }+}++func TestUTF8_FromChunk_InvalidUTF8(t *testing.T) {+ type args struct {+ chunk *sources.Chunk+ }+ tests := []struct {+ name string+ d *UTF8+ args args+ want *sources.Chunk+ wantErr bool+ }{+ {+ name: "basic invalid utf8",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("\xF0\x28\x8C\x28")},+ },+ want: &sources.Chunk{Data: []byte("�(�(")},+ wantErr: false,+ },+ {+ name: "invalid utf8 between words",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("START\xF0\x28\x8C\x28MIDDLE\xC0\x80END")},+ },+ want: &sources.Chunk{Data: []byte("START�(�(MIDDLE��END")},+ wantErr: false,+ },+ {+ name: "binary data with embedded text",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0xF0, 'S', 'E', 'C', 'R', 'E', 'T', // Invalid UTF-8 before text+ 0xC0, 0x80, // Invalid UTF-8 sequence+ 'V', 'A', 'L', 'U', 'E',+ 0xFF, 0x8C, // More invalid UTF-8+ }},+ },+ want: &sources.Chunk{Data: []byte("�SECRET��VALUE��")},+ wantErr: false,+ },+ {+ name: "binary protocol with length fields",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0x02, // frame type+ 0x00, 0x00, 0x00, 0x0A, // length field+ 'P', 'A', 'S', 'S', 'W', 'O', 'R', 'D', '1', '2',+ 0xFE, 0xFF, // checksum+ }},+ },+ want: &sources.Chunk{Data: []byte("�����PASSWORD12��")},+ wantErr: false,+ },+ {+ name: "truncated utf8 sequence",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("PREFIX\xF0\x28SUFFIX")},+ },+ want: &sources.Chunk{Data: []byte("PREFIX�(SUFFIX")},+ wantErr: false,+ },+ {+ name: "multiple invalid sequences",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0xF0, 'A', // Invalid + ASCII+ 0xC0, 0x80, // Invalid sequence+ 'B',+ 0xFF, // Single invalid byte+ 'C',+ 0xF0, 0x28, 0x8C, 0x28, // Invalid sequence+ 'D',+ }},+ },+ want: &sources.Chunk{Data: []byte("�A��B�C�(�(D")},+ wantErr: false,+ },+ {+ name: "invalid utf8 header with embedded secret",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0xF0, 0x28, 0x8C, // Invalid UTF-8 sequence+ 'S', 'E', 'C', 'R', 'E', 'T', '=',+ 0xC0, 0x80, // Another invalid UTF-8 sequence+ 'A', 'K', 'I', 'A', '1', '2', '3', '4', '5', '6',+ 0xF8, 0x88, // More invalid UTF-8+ }},+ },+ want: &sources.Chunk{Data: []byte("�(�SECRET=��AKIA123456��")},+ wantErr: false,+ },+ {+ name: "key value pairs with length prefixes",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0x00, 0x01, // header+ 'A', 'P', 'I', '_', 'K', 'E', 'Y', '=',+ 0x00, 0x00, 0x00, 0x05, // length+ 'A', 'K', 'I', 'A', '5',+ 0xFF, // separator+ 'S', 'E', 'C', 'R', 'E', 'T', '=',+ 0x00, 0x00, 0x00, 0x06,+ 'S', 'E', 'C', 'R', 'E', 'T',+ }},+ },+ want: &sources.Chunk{Data: []byte("��API_KEY=����AKIA5�SECRET=����SECRET")},+ wantErr: false,+ },+ {+ name: "mixed binary and invalid utf8",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0x00, 0x01, // valid binary+ 0xF0, 0x28, // invalid UTF-8+ 'K', 'E', 'Y', '=',+ 0xC0, 0x80, // more invalid UTF-8+ 'V', 'A', 'L', 'U', 'E',+ }},+ },+ want: &sources.Chunk{Data: []byte("���(KEY=��VALUE")},+ wantErr: false,+ },+ {+ name: "very large utf8 sequence",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte(strings.Repeat("世界", 1000))},+ },+ want: &sources.Chunk{Data: []byte(strings.Repeat("世界", 1000))},+ wantErr: false,+ },+ {+ name: "single byte chunk",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{0x41}}, // Single 'A'+ },+ want: &sources.Chunk{Data: []byte("A")},+ wantErr: false,+ },+ {+ name: "chunk with zero bytes between valid utf8",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("hello\x00world\x00!")},+ },+ want: &sources.Chunk{Data: []byte("hello\x00world\x00!")},+ wantErr: false,+ },+ {+ name: "multi-byte unicode characters",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("🌍🌎🌏")},+ },+ want: &sources.Chunk{Data: []byte("🌍🌎🌏")},+ wantErr: false,+ },+ {+ name: "mixed ascii and multi-byte unicode with invalid sequences",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("Hello 世界\xF0\x28\x8C\x28Testing🌍")},+ },+ want: &sources.Chunk{Data: []byte("Hello 世界�(�(Testing🌍")},+ wantErr: false,+ },+ {+ name: "chunk ending with partial utf8 sequence",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("Hello\xE2\x80")}, // Incomplete UTF-8 sequence+ },+ want: &sources.Chunk{Data: []byte("Hello��")},+ wantErr: false,+ },+ {+ name: "chunk with all printable ascii chars",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte(" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~")},+ },+ want: &sources.Chunk{Data: []byte(" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~")},+ wantErr: false,+ },+ {+ name: "alternating valid and invalid utf8",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte("A\xF0B\xF0C\xF0D")},+ },+ want: &sources.Chunk{Data: []byte("A�B�C�D")},+ wantErr: false,+ },+ {+ name: "overlong utf8 encoding",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{0xF0, 0x82, 0x82, 0xAC}}, // Overlong encoding of €+ },+ want: &sources.Chunk{Data: []byte("����")},+ wantErr: false,+ },+ {+ name: "utf8 boundary conditions",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{+ 0xFF, // Invalid single byte -> �+ 0xC2, 0x80, // Minimum valid 2-byte UTF-8 sequence (U+0080) -> \u0080+ 0xDF, 0xBF, // Maximum valid 2-byte UTF-8 sequence (U+07FF) -> ߿+ 0xE0, 0x80, 0x80, // Invalid 3-byte (overlong encoding) -> �+ 0xEF, 0xBF, 0xBF, // Valid 3-byte sequence for U+FFFF -> \uffff+ 0xF0, 0x28, 0x8C, 0x28, // Invalid UTF-8 mixed with ASCII -> �(�(+ 0xF4, 0x8F, 0xBF, 0xBF, // Valid 4-byte sequence for U+10FFFF -> \U0010ffff+ }},+ },+ want: &sources.Chunk{Data: []byte("�\u0080߿���\uffff�(�(\U0010ffff")},+ wantErr: false,+ },+ {+ name: "chunk with byte order mark (BOM)",+ d: &UTF8{},+ args: args{+ chunk: &sources.Chunk{Data: []byte{0xEF, 0xBB, 0xBF, 'h', 'e', 'l', 'l', 'o'}},+ },+ want: &sources.Chunk{Data: []byte("\uFEFFhello")},+ wantErr: false,+ },+ {+ name: "chunk with surrogate pairs",+ d: &UTF8{},+ args: args{+ // Invalid UTF-8 encoding of surrogate pairs+ chunk: &sources.Chunk{Data: []byte{0xED, 0xA0, 0x80, 0xED, 0xB0, 0x80}},+ },+ want: &sources.Chunk{Data: []byte("������")},wantErr: false,},}+for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {d := &UTF8{}got := d.FromChunk(tt.args.chunk)if got != nil && tt.want != nil {if diff := pretty.Compare(string(got.Data), string(tt.want.Data)); diff != "" {- t.Errorf("%s: Plain.FromChunk() diff: (-got +want)\n%s", tt.name, diff)+ t.Errorf("%s: UTF8.FromChunk() diff: (-got +want)\n%s", tt.name, diff)}} else {if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("%s: Plain.FromChunk() diff: (-got +want)\n%s", tt.name, diff)+ t.Errorf("%s: UTF8.FromChunk() diff: (-got +want)\n%s", tt.name, diff)}}})