Skip to content

Find, verify, and analyze leaked credentials

[PERF] Remove \b anchor from escaped-unicode codePointPat (#5320)

Ahsan-Sarbazed325715c2ec
diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.go
index be042c3d1..347936eab 100644
--- a/pkg/decoders/escaped_unicode.go
+++ b/pkg/decoders/escaped_unicode.go
@@ -19,7 +19,14 @@ var _ Decoder = (*EscapedUnicode)(nil)
var (
// Standard Unicode notation.
//https://unicode.org/standard/principles.html
- codePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)
+ //
+ // The word boundary that belongs at the front of this pattern is applied by
+ // codePointMatches instead of being written here. A leading empty-width
+ // assertion stops regexp from extracting a literal prefix, and without a
+ // prefix every offset in the chunk has to be tried rather than skipping
+ // straight to the next "U+". That one difference made this pattern cost
+ // more than every other pattern in this file combined.
+ codePointPat = regexp.MustCompile(`U\+([a-fA-F0-9]{4}).?`)
// Common escape sequence used in programming languages.
escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`)
@@ -52,6 +59,53 @@ var (
// hexEscapePat = regexp.MustCompile(`0x([a-fA-F0-9]{1,6})(?:\s|$)`)
)
+// Every escape form this decoder understands starts with a fixed literal: a
+// backslash for the \uXXXX family and CSS escapes, "U+" for standard code
+// point notation, "&#x" for HTML and "%u" for percent encoding. Looking for
+// that literal is far cheaper than running the pattern, and a chunk without it
+// cannot match, so each pattern is only run once its anchor is present.
+var (
+ codePointAnchor = []byte("U+")
+ htmlAnchor = []byte("&#x")
+ percentAnchor = []byte("%u")
+)
+
+// isWordByte reports whether c is a word character as \b defines it: ASCII
+// letters, digits and underscore.
+func isWordByte(c byte) bool {
+ return c == '_' ||
+ ('0' <= c && c <= '9') ||
+ ('a' <= c && c <= 'z') ||
+ ('A' <= c && c <= 'Z')
+}
+
+// codePointMatches returns the submatch indices of every U+XXXX sequence in
+// input that starts on a word boundary.
+//
+// This is the check the leading \b used to perform. "U" is itself a word
+// character, so the boundary holds exactly when the sequence starts the input
+// or follows a byte that is not a word character.
+//
+// Dropping a match here can never hide another one. A second "U+" cannot begin
+// inside the four hex digits, since "+" is not a hex digit, so the only overlap
+// possible is the trailing optional byte covering a following "U". That "U"
+// is preceded by a hex digit, which is a word character, so the boundary would
+// have rejected it as well.
+func codePointMatches(input []byte) [][]int {
+ if !bytes.Contains(input, codePointAnchor) {
+ return nil
+ }
+ all := codePointPat.FindAllSubmatchIndex(input, -1)
+ kept := all[:0]
+ for _, m := range all {
+ if m[0] > 0 && isWordByte(input[m[0]-1]) {
+ continue
+ }
+ kept = append(kept, m)
+ }
+ return kept
+}
+
func (d *EscapedUnicode) Type() detectorspb.DecoderType {
return detectorspb.DecoderType_ESCAPED_UNICODE
}
@@ -70,29 +124,31 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
// Process patterns in priority order - more specific patterns first
// This prevents conflicts where multiple patterns match the same input
+ hasBackslash := bytes.IndexByte(chunkData, '\\') >= 0
+
// Long escape format (8 hex digits) - highest priority
- if longEscapePat.Match(chunkData) {
+ if hasBackslash && longEscapePat.Match(chunkData) {
matched = true
chunkData = decodeLongEscape(chunkData)
- } else if braceEscapePat.Match(chunkData) {
+ } else if hasBackslash && braceEscapePat.Match(chunkData) {
matched = true
chunkData = decodeBraceEscape(chunkData)
- } else if perlEscapePat.Match(chunkData) {
+ } else if hasBackslash && perlEscapePat.Match(chunkData) {
matched = true
chunkData = decodePerlEscape(chunkData)
- } else if htmlEscapePat.Match(chunkData) {
+ } else if bytes.Contains(chunkData, htmlAnchor) && htmlEscapePat.Match(chunkData) {
matched = true
chunkData = decodeHtmlEscape(chunkData)
- } else if percentEscapePat.Match(chunkData) {
+ } else if bytes.Contains(chunkData, percentAnchor) && percentEscapePat.Match(chunkData) {
matched = true
chunkData = decodePercentEscape(chunkData)
- } else if escapePat.Match(chunkData) {
+ } else if hasBackslash && escapePat.Match(chunkData) {
matched = true
chunkData = decodeEscaped(chunkData)
- } else if codePointPat.Match(chunkData) {
+ } else if codePointIdx := codePointMatches(chunkData); len(codePointIdx) > 0 {
matched = true
- chunkData = decodeCodePoint(chunkData)
- } else if cssEscapePat.Match(chunkData) {
+ chunkData = decodeCodePointAt(chunkData, codePointIdx)
+ } else if hasBackslash && cssEscapePat.Match(chunkData) {
matched = true
chunkData = decodeCssEscape(chunkData)
// } else if hexEscapePat.Match(chunkData) {
@@ -154,8 +210,13 @@ func decodeWithPattern(input []byte, re *regexp.Regexp) []byte {
func decodeCodePoint(input []byte) []byte {
// Find all Unicode escape sequences in the input byte slice
- indices := codePointPat.FindAllSubmatchIndex(input, -1)
+ return decodeCodePointAt(input, codePointMatches(input))
+}
+// decodeCodePointAt is decodeCodePoint driven by indices the caller already
+// has. FromChunk needs the match list to decide whether this pattern applies,
+// so it passes that list straight through rather than scanning a second time.
+func decodeCodePointAt(input []byte, indices [][]int) []byte {
// Iterate over found indices in reverse order to avoid modifying the slice length
utf8Bytes := make([]byte, maxBytesPerRune)
for i := len(indices) - 1; i >= 0; i-- {
diff --git a/pkg/decoders/escaped_unicode_equivalence_test.go b/pkg/decoders/escaped_unicode_equivalence_test.go
new file mode 100644
index 000000000..167624a26
--- /dev/null
+++ b/pkg/decoders/escaped_unicode_equivalence_test.go
@@ -0,0 +1,181 @@
+package decoders
+
+import (
+ "bytes"
+ "math/rand"
+ "strconv"
+ "testing"
+ "unicode/utf8"
+
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+// legacyCodePointPat is main's pattern, word boundary and all.
+var legacyCodePointPat = regexp.MustCompile(`\bU\+([a-fA-F0-9]{4}).?`)
+
+// legacyDecodeCodePoint is main's decodeCodePoint verbatim, driven by the
+// boundary-carrying pattern.
+func legacyDecodeCodePoint(input []byte) []byte {
+ indices := legacyCodePointPat.FindAllSubmatchIndex(input, -1)
+ utf8Bytes := make([]byte, maxBytesPerRune)
+ for i := len(indices) - 1; i >= 0; i-- {
+ matches := indices[i]
+ startIndex, endIndex := matches[0], matches[1]
+ hexStartIndex, hexEndIndex := matches[2], matches[3]
+ if endIndex != hexEndIndex && input[endIndex-1] != spaceChar {
+ endIndex = endIndex - 1
+ }
+ unicodeInt, err := strconv.ParseInt(string(input[hexStartIndex:hexEndIndex]), 16, 32)
+ if err != nil {
+ continue
+ }
+ utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
+ input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
+ }
+ return input
+}
+
+// compareIndices checks that the boundary filter selects exactly the matches
+// the \b pattern would have found.
+func compareIndices(t *testing.T, data []byte) {
+ t.Helper()
+ want := legacyCodePointPat.FindAllSubmatchIndex(data, -1)
+ got := codePointMatches(bytes.Clone(data))
+ if len(want) != len(got) {
+ t.Fatalf("input %q: legacy found %d matches %v, filtered found %d %v", data, len(want), want, len(got), got)
+ }
+ for i := range want {
+ for j := range want[i] {
+ if want[i][j] != got[i][j] {
+ t.Fatalf("input %q: match %d differs: legacy %v filtered %v", data, i, want[i], got[i])
+ }
+ }
+ }
+}
+
+// compareDecode checks the full decode output agrees.
+func compareDecode(t *testing.T, data []byte) {
+ t.Helper()
+ want := legacyDecodeCodePoint(bytes.Clone(data))
+ got := decodeCodePoint(bytes.Clone(data))
+ if !bytes.Equal(want, got) {
+ t.Fatalf("input %q: legacy decoded to %q, new decoded to %q", data, want, got)
+ }
+}
+
+// compareMatched checks the decoder's decode/skip decision agrees.
+func compareMatched(t *testing.T, data []byte) {
+ t.Helper()
+ wantMatched := legacyCodePointPat.Match(data)
+ gotMatched := len(codePointMatches(bytes.Clone(data))) > 0
+ if wantMatched != gotMatched {
+ t.Fatalf("input %q: legacy matched=%v, new matched=%v", data, wantMatched, gotMatched)
+ }
+}
+
+var codePointSeeds = []string{
+ "", "U+0041", "U+0041 ", "U+0041x", "U+0041U+0042", "U+0041 U+0042",
+ "AU+0041", "_U+0041", "9U+0041", " U+0041", "\tU+0041", "\nU+0041",
+ "AU+0041 U+0042", "U+0041\nU+0042", "U+0041\tU+0042", "U+00",
+ "U+ZZZZ", "u+0041", "UU+0041", "U+0041U", "U+0041+", "xU+0041U+0042 U+0043",
+ "U+D800", "U+FFFF", "U+0000", "café U+00E9", "\xff\xfeU+0041",
+ "U+0041U+0042U+0043", "-U+0041-U+0042-", "U+0041.U+0042", "1U+0041 2U+0042",
+}
+
+func TestCodePointBoundaryEquivalence(t *testing.T) {
+ for _, s := range codePointSeeds {
+ compareIndices(t, []byte(s))
+ compareDecode(t, []byte(s))
+ compareMatched(t, []byte(s))
+ }
+
+ // Alphabet weighted towards everything the pattern and boundary care about.
+ alphabet := []byte("UU++0123456789abcdefABCDEFxyz_ \t\n.-\xff")
+ rnd := rand.New(rand.NewSource(7))
+ for i := 0; i < 300000; i++ {
+ b := make([]byte, 1+rnd.Intn(40))
+ for j := range b {
+ b[j] = alphabet[rnd.Intn(len(alphabet))]
+ }
+ compareIndices(t, b)
+ compareDecode(t, b)
+ compareMatched(t, b)
+ }
+}
+
+// legacyFromChunk is main's whole FromChunk, unanchored and with \b.
+func legacyFromChunk(chunk *sources.Chunk) []byte {
+ chunkData := bytes.Clone(chunk.Data)
+ matched := false
+ if longEscapePat.Match(chunkData) {
+ matched, chunkData = true, decodeLongEscape(chunkData)
+ } else if braceEscapePat.Match(chunkData) {
+ matched, chunkData = true, decodeBraceEscape(chunkData)
+ } else if perlEscapePat.Match(chunkData) {
+ matched, chunkData = true, decodePerlEscape(chunkData)
+ } else if htmlEscapePat.Match(chunkData) {
+ matched, chunkData = true, decodeHtmlEscape(chunkData)
+ } else if percentEscapePat.Match(chunkData) {
+ matched, chunkData = true, decodePercentEscape(chunkData)
+ } else if escapePat.Match(chunkData) {
+ matched, chunkData = true, decodeEscaped(chunkData)
+ } else if legacyCodePointPat.Match(chunkData) {
+ matched, chunkData = true, legacyDecodeCodePoint(chunkData)
+ } else if cssEscapePat.Match(chunkData) {
+ matched, chunkData = true, decodeCssEscape(chunkData)
+ }
+ if !matched {
+ return nil
+ }
+ return chunkData
+}
+
+func compareFromChunk(t *testing.T, data []byte) {
+ t.Helper()
+ want := legacyFromChunk(&sources.Chunk{Data: bytes.Clone(data)})
+ got := (&EscapedUnicode{}).FromChunk(&sources.Chunk{Data: bytes.Clone(data)})
+ switch {
+ case want == nil && got != nil:
+ t.Fatalf("new decoded %q to %q, legacy skipped it", data, got.Data)
+ case want != nil && got == nil:
+ t.Fatalf("new skipped %q, legacy decoded it to %q", data, want)
+ case want != nil && !bytes.Equal(want, got.Data):
+ t.Fatalf("input %q: legacy %q, new %q", data, want, got.Data)
+ }
+}
+
+func TestFromChunkEquivalence(t *testing.T) {
+ for _, s := range append(codePointSeeds,
+ "\\u0041", "\\\\u0041", "\\U0001F600", "\\u{1F600}", "\\x{263A}",
+ "&#x41;", "%u0041", "\\41 ", "\\41\\", "plain text", "a+b&c%d\\e",
+ ) {
+ compareFromChunk(t, []byte(s))
+ }
+ alphabet := []byte(`UU++\\\uUxX{}&#%;0123456789abcdefABCDEF _.-` + "\t\n")
+ rnd := rand.New(rand.NewSource(11))
+ for i := 0; i < 300000; i++ {
+ b := make([]byte, 1+rnd.Intn(48))
+ for j := range b {
+ b[j] = alphabet[rnd.Intn(len(alphabet))]
+ }
+ compareFromChunk(t, b)
+ }
+}
+
+func FuzzCodePointBoundary(f *testing.F) {
+ for _, s := range codePointSeeds {
+ f.Add([]byte(s))
+ }
+ f.Add([]byte("\\u0041"))
+ f.Add([]byte("&#x41;"))
+ f.Fuzz(func(t *testing.T, data []byte) {
+ if len(data) == 0 {
+ return
+ }
+ compareIndices(t, data)
+ compareDecode(t, data)
+ compareFromChunk(t, data)
+ })
+}