Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[PERF] Remove \b anchor from escaped-unicode codePointPat (#5320)
Ahsan-Sarbazed325715c2ec
diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.goindex be042c3d1..347936eab 100644--- a/pkg/decoders/escaped_unicode.go+++ b/pkg/decoders/escaped_unicode.go@@ -19,7 +19,14 @@ var _ Decoder = (*EscapedUnicode)(nil)var (// Standard Unicode notation.//https://unicode.org/standard/principles.html- codePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)+ //+ // The word boundary that belongs at the front of this pattern is applied by+ // codePointMatches instead of being written here. A leading empty-width+ // assertion stops regexp from extracting a literal prefix, and without a+ // prefix every offset in the chunk has to be tried rather than skipping+ // straight to the next "U+". That one difference made this pattern cost+ // more than every other pattern in this file combined.+ codePointPat = regexp.MustCompile(`U\+([a-fA-F0-9]{4}).?`)// Common escape sequence used in programming languages.escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`)@@ -52,6 +59,53 @@ var (// hexEscapePat = regexp.MustCompile(`0x([a-fA-F0-9]{1,6})(?:\s|$)`))+// Every escape form this decoder understands starts with a fixed literal: a+// backslash for the \uXXXX family and CSS escapes, "U+" for standard code+// point notation, "&#x" for HTML and "%u" for percent encoding. Looking for+// that literal is far cheaper than running the pattern, and a chunk without it+// cannot match, so each pattern is only run once its anchor is present.+var (+ codePointAnchor = []byte("U+")+ htmlAnchor = []byte("&#x")+ percentAnchor = []byte("%u")+)++// isWordByte reports whether c is a word character as \b defines it: ASCII+// letters, digits and underscore.+func isWordByte(c byte) bool {+ return c == '_' ||+ ('0' <= c && c <= '9') ||+ ('a' <= c && c <= 'z') ||+ ('A' <= c && c <= 'Z')+}++// codePointMatches returns the submatch indices of every U+XXXX sequence in+// input that starts on a word boundary.+//+// This is the check the leading \b used to perform. "U" is itself a word+// character, so the boundary holds exactly when the sequence starts the input+// or follows a byte that is not a word character.+//+// Dropping a match here can never hide another one. A second "U+" cannot begin+// inside the four hex digits, since "+" is not a hex digit, so the only overlap+// possible is the trailing optional byte covering a following "U". That "U"+// is preceded by a hex digit, which is a word character, so the boundary would+// have rejected it as well.+func codePointMatches(input []byte) [][]int {+ if !bytes.Contains(input, codePointAnchor) {+ return nil+ }+ all := codePointPat.FindAllSubmatchIndex(input, -1)+ kept := all[:0]+ for _, m := range all {+ if m[0] > 0 && isWordByte(input[m[0]-1]) {+ continue+ }+ kept = append(kept, m)+ }+ return kept+}+func (d *EscapedUnicode) Type() detectorspb.DecoderType {return detectorspb.DecoderType_ESCAPED_UNICODE}@@ -70,29 +124,31 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {// Process patterns in priority order - more specific patterns first// This prevents conflicts where multiple patterns match the same input+ hasBackslash := bytes.IndexByte(chunkData, '\\') >= 0+// Long escape format (8 hex digits) - highest priority- if longEscapePat.Match(chunkData) {+ if hasBackslash && longEscapePat.Match(chunkData) {matched = truechunkData = decodeLongEscape(chunkData)- } else if braceEscapePat.Match(chunkData) {+ } else if hasBackslash && braceEscapePat.Match(chunkData) {matched = truechunkData = decodeBraceEscape(chunkData)- } else if perlEscapePat.Match(chunkData) {+ } else if hasBackslash && perlEscapePat.Match(chunkData) {matched = truechunkData = decodePerlEscape(chunkData)- } else if htmlEscapePat.Match(chunkData) {+ } else if bytes.Contains(chunkData, htmlAnchor) && htmlEscapePat.Match(chunkData) {matched = truechunkData = decodeHtmlEscape(chunkData)- } else if percentEscapePat.Match(chunkData) {+ } else if bytes.Contains(chunkData, percentAnchor) && percentEscapePat.Match(chunkData) {matched = truechunkData = decodePercentEscape(chunkData)- } else if escapePat.Match(chunkData) {+ } else if hasBackslash && escapePat.Match(chunkData) {matched = truechunkData = decodeEscaped(chunkData)- } else if codePointPat.Match(chunkData) {+ } else if codePointIdx := codePointMatches(chunkData); len(codePointIdx) > 0 {matched = true- chunkData = decodeCodePoint(chunkData)- } else if cssEscapePat.Match(chunkData) {+ chunkData = decodeCodePointAt(chunkData, codePointIdx)+ } else if hasBackslash && cssEscapePat.Match(chunkData) {matched = truechunkData = decodeCssEscape(chunkData)// } else if hexEscapePat.Match(chunkData) {@@ -154,8 +210,13 @@ func decodeWithPattern(input []byte, re *regexp.Regexp) []byte {func decodeCodePoint(input []byte) []byte {// Find all Unicode escape sequences in the input byte slice- indices := codePointPat.FindAllSubmatchIndex(input, -1)+ return decodeCodePointAt(input, codePointMatches(input))+}+// decodeCodePointAt is decodeCodePoint driven by indices the caller already+// has. FromChunk needs the match list to decide whether this pattern applies,+// so it passes that list straight through rather than scanning a second time.+func decodeCodePointAt(input []byte, indices [][]int) []byte {// Iterate over found indices in reverse order to avoid modifying the slice lengthutf8Bytes := make([]byte, maxBytesPerRune)for i := len(indices) - 1; i >= 0; i-- {diff --git a/pkg/decoders/escaped_unicode_equivalence_test.go b/pkg/decoders/escaped_unicode_equivalence_test.gonew file mode 100644index 000000000..167624a26--- /dev/null+++ b/pkg/decoders/escaped_unicode_equivalence_test.go@@ -0,0 +1,181 @@+package decoders++import (+ "bytes"+ "math/rand"+ "strconv"+ "testing"+ "unicode/utf8"++ regexp "github.com/wasilibs/go-re2"++ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++// legacyCodePointPat is main's pattern, word boundary and all.+var legacyCodePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)++// legacyDecodeCodePoint is main's decodeCodePoint verbatim, driven by the+// boundary-carrying pattern.+func legacyDecodeCodePoint(input []byte) []byte {+ indices := legacyCodePointPat.FindAllSubmatchIndex(input, -1)+ utf8Bytes := make([]byte, maxBytesPerRune)+ for i := len(indices) - 1; i >= 0; i-- {+ matches := indices[i]+ startIndex, endIndex := matches[0], matches[1]+ hexStartIndex, hexEndIndex := matches[2], matches[3]+ if endIndex != hexEndIndex && input[endIndex-1] != spaceChar {+ endIndex = endIndex - 1+ }+ unicodeInt, err := strconv.ParseInt(string(input[hexStartIndex:hexEndIndex]), 16, 32)+ if err != nil {+ continue+ }+ utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))+ input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)+ }+ return input+}++// compareIndices checks that the boundary filter selects exactly the matches+// the \b pattern would have found.+func compareIndices(t *testing.T, data []byte) {+ t.Helper()+ want := legacyCodePointPat.FindAllSubmatchIndex(data, -1)+ got := codePointMatches(bytes.Clone(data))+ if len(want) != len(got) {+ t.Fatalf("input %q: legacy found %d matches %v, filtered found %d %v", data, len(want), want, len(got), got)+ }+ for i := range want {+ for j := range want[i] {+ if want[i][j] != got[i][j] {+ t.Fatalf("input %q: match %d differs: legacy %v filtered %v", data, i, want[i], got[i])+ }+ }+ }+}++// compareDecode checks the full decode output agrees.+func compareDecode(t *testing.T, data []byte) {+ t.Helper()+ want := legacyDecodeCodePoint(bytes.Clone(data))+ got := decodeCodePoint(bytes.Clone(data))+ if !bytes.Equal(want, got) {+ t.Fatalf("input %q: legacy decoded to %q, new decoded to %q", data, want, got)+ }+}++// compareMatched checks the decoder's decode/skip decision agrees.+func compareMatched(t *testing.T, data []byte) {+ t.Helper()+ wantMatched := legacyCodePointPat.Match(data)+ gotMatched := len(codePointMatches(bytes.Clone(data))) > 0+ if wantMatched != gotMatched {+ t.Fatalf("input %q: legacy matched=%v, new matched=%v", data, wantMatched, gotMatched)+ }+}++var codePointSeeds = []string{+ "", "U+0041", "U+0041 ", "U+0041x", "U+0041U+0042", "U+0041 U+0042",+ "AU+0041", "_U+0041", "9U+0041", " U+0041", "\tU+0041", "\nU+0041",+ "AU+0041 U+0042", "U+0041\nU+0042", "U+0041\tU+0042", "U+00",+ "U+ZZZZ", "u+0041", "UU+0041", "U+0041U", "U+0041+", "xU+0041U+0042 U+0043",+ "U+D800", "U+FFFF", "U+0000", "café U+00E9", "\xff\xfeU+0041",+ "U+0041U+0042U+0043", "-U+0041-U+0042-", "U+0041.U+0042", "1U+0041 2U+0042",+}++func TestCodePointBoundaryEquivalence(t *testing.T) {+ for _, s := range codePointSeeds {+ compareIndices(t, []byte(s))+ compareDecode(t, []byte(s))+ compareMatched(t, []byte(s))+ }++ // Alphabet weighted towards everything the pattern and boundary care about.+ alphabet := []byte("UU++0123456789abcdefABCDEFxyz_ \t\n.-\xff")+ rnd := rand.New(rand.NewSource(7))+ for i := 0; i < 300000; i++ {+ b := make([]byte, 1+rnd.Intn(40))+ for j := range b {+ b[j] = alphabet[rnd.Intn(len(alphabet))]+ }+ compareIndices(t, b)+ compareDecode(t, b)+ compareMatched(t, b)+ }+}++// legacyFromChunk is main's whole FromChunk, unanchored and with \b.+func legacyFromChunk(chunk *sources.Chunk) []byte {+ chunkData := bytes.Clone(chunk.Data)+ matched := false+ if longEscapePat.Match(chunkData) {+ matched, chunkData = true, decodeLongEscape(chunkData)+ } else if braceEscapePat.Match(chunkData) {+ matched, chunkData = true, decodeBraceEscape(chunkData)+ } else if perlEscapePat.Match(chunkData) {+ matched, chunkData = true, decodePerlEscape(chunkData)+ } else if htmlEscapePat.Match(chunkData) {+ matched, chunkData = true, decodeHtmlEscape(chunkData)+ } else if percentEscapePat.Match(chunkData) {+ matched, chunkData = true, decodePercentEscape(chunkData)+ } else if escapePat.Match(chunkData) {+ matched, chunkData = true, decodeEscaped(chunkData)+ } else if legacyCodePointPat.Match(chunkData) {+ matched, chunkData = true, legacyDecodeCodePoint(chunkData)+ } else if cssEscapePat.Match(chunkData) {+ matched, chunkData = true, decodeCssEscape(chunkData)+ }+ if !matched {+ return nil+ }+ return chunkData+}++func compareFromChunk(t *testing.T, data []byte) {+ t.Helper()+ want := legacyFromChunk(&sources.Chunk{Data: bytes.Clone(data)})+ got := (&EscapedUnicode{}).FromChunk(&sources.Chunk{Data: bytes.Clone(data)})+ switch {+ case want == nil && got != nil:+ t.Fatalf("new decoded %q to %q, legacy skipped it", data, got.Data)+ case want != nil && got == nil:+ t.Fatalf("new skipped %q, legacy decoded it to %q", data, want)+ case want != nil && !bytes.Equal(want, got.Data):+ t.Fatalf("input %q: legacy %q, new %q", data, want, got.Data)+ }+}++func TestFromChunkEquivalence(t *testing.T) {+ for _, s := range append(codePointSeeds,+ "\\u0041", "\\\\u0041", "\\U0001F600", "\\u{1F600}", "\\x{263A}",+ "A", "%u0041", "\\41 ", "\\41\\", "plain text", "a+b&c%d\\e",+ ) {+ compareFromChunk(t, []byte(s))+ }+ alphabet := []byte(`UU++\\\uUxX{}&#%;0123456789abcdefABCDEF _.-` + "\t\n")+ rnd := rand.New(rand.NewSource(11))+ for i := 0; i < 300000; i++ {+ b := make([]byte, 1+rnd.Intn(48))+ for j := range b {+ b[j] = alphabet[rnd.Intn(len(alphabet))]+ }+ compareFromChunk(t, b)+ }+}++func FuzzCodePointBoundary(f *testing.F) {+ for _, s := range codePointSeeds {+ f.Add([]byte(s))+ }+ f.Add([]byte("\\u0041"))+ f.Add([]byte("A"))+ f.Fuzz(func(t *testing.T, data []byte) {+ if len(data) == 0 {+ return+ }+ compareIndices(t, data)+ compareDecode(t, data)+ compareFromChunk(t, data)+ })+}