Skip to content

Find, verify, and analyze leaked credentials

11.4 KB
Raw Blame
package decoders

import (
	"bytes"
	regexp "github.com/wasilibs/go-re2"
	"strconv"
	"unicode/utf8"

	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)

type EscapedUnicode struct{}

var _ Decoder = (*EscapedUnicode)(nil)

// It might be advantageous to limit these to a subset of acceptable characters, similar to base64.
// https://dencode.com/en/string/unicode-escape
var (
	// Standard Unicode notation.
	//https://unicode.org/standard/principles.html
	//
	// The word boundary that belongs at the front of this pattern is applied by
	// codePointMatches instead of being written here. A leading empty-width
	// assertion stops regexp from extracting a literal prefix, and without a
	// prefix every offset in the chunk has to be tried rather than skipping
	// straight to the next "U+". That one difference made this pattern cost
	// more than every other pattern in this file combined.
	codePointPat = regexp.MustCompile(`U\+([a-fA-F0-9]{4}).?`)

	// Common escape sequence used in programming languages.
	escapePat = regexp.MustCompile(`(?i:\\{1,2}u)([a-fA-F0-9]{4})`)

	// Additional Unicode escape formats from dencode.com

	// \u{X} format - Rust, Swift, some JS, etc. (variable length hex in braces)
	braceEscapePat = regexp.MustCompile(`\\u\{([a-fA-F0-9]{1,6})\}`)

	// \U00XXXXXX format - Python, etc. (8-digit format for non-BMP characters)
	longEscapePat = regexp.MustCompile(`\\U([a-fA-F0-9]{8})`)

	// \x{X} format - Perl (variable length hex in braces)
	perlEscapePat = regexp.MustCompile(`\\x\{([a-fA-F0-9]{1,6})\}`)

	// \X format - CSS (hex without padding). Go's regexp (RE2) has no look-ahead, so we
	// include the delimiter (whitespace, another backslash, or end-of-string) in the
	// match using a non-capturing group. The delimiter is later re-inserted by the
	// decoder when necessary.
	cssEscapePat = regexp.MustCompile(`\\([a-fA-F0-9]{1,6})(?:\s|\\|$)`)

	// &#xX; format - HTML/XML (hex with semicolon)
	htmlEscapePat = regexp.MustCompile(`&#x([a-fA-F0-9]{1,6});`)

	// %uXXXX format - Percent-encoding (non-standard)
	percentEscapePat = regexp.MustCompile(`%u([a-fA-F0-9]{4})`)

	// // 0xX format - Hexadecimal notation with space separation
	// Note: Commenting out for now due to high memory overhead. Review ways to handle this.
	// hexEscapePat = regexp.MustCompile(`0x([a-fA-F0-9]{1,6})(?:\s|$)`)
)

// Every escape form this decoder understands starts with a fixed literal: a
// backslash for the \uXXXX family and CSS escapes, "U+" for standard code
// point notation, "&#x" for HTML and "%u" for percent encoding. Looking for
// that literal is far cheaper than running the pattern, and a chunk without it
// cannot match, so each pattern is only run once its anchor is present.
var (
	codePointAnchor = []byte("U+")
	htmlAnchor      = []byte("&#x")
	percentAnchor   = []byte("%u")
)

// isWordByte reports whether c is a word character as \b defines it: ASCII
// letters, digits and underscore.
func isWordByte(c byte) bool {
	return c == '_' ||
		('0' <= c && c <= '9') ||
		('a' <= c && c <= 'z') ||
		('A' <= c && c <= 'Z')
}

// codePointMatches returns the submatch indices of every U+XXXX sequence in
// input that starts on a word boundary.
//
// This is the check the leading \b used to perform. "U" is itself a word
// character, so the boundary holds exactly when the sequence starts the input
// or follows a byte that is not a word character.
//
// Dropping a match here can never hide another one. A second "U+" cannot begin
// inside the four hex digits, since "+" is not a hex digit, so the only overlap
// possible is the trailing optional byte covering a following "U". That "U"
// is preceded by a hex digit, which is a word character, so the boundary would
// have rejected it as well.
func codePointMatches(input []byte) [][]int {
	if !bytes.Contains(input, codePointAnchor) {
		return nil
	}
	all := codePointPat.FindAllSubmatchIndex(input, -1)
	kept := all[:0]
	for _, m := range all {
		if m[0] > 0 && isWordByte(input[m[0]-1]) {
			continue
		}
		kept = append(kept, m)
	}
	return kept
}

func (d *EscapedUnicode) Type() detectorspb.DecoderType {
	return detectorspb.DecoderType_ESCAPED_UNICODE
}

func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
	if chunk == nil || len(chunk.Data) == 0 {
		return nil
	}

	var (
		// Necessary to avoid data races.
		chunkData = bytes.Clone(chunk.Data)
		matched   = false
	)

	// Process patterns in priority order - more specific patterns first
	// This prevents conflicts where multiple patterns match the same input

	hasBackslash := bytes.IndexByte(chunkData, '\\') >= 0

	// Long escape format (8 hex digits) - highest priority
	if hasBackslash && longEscapePat.Match(chunkData) {
		matched = true
		chunkData = decodeLongEscape(chunkData)
	} else if hasBackslash && braceEscapePat.Match(chunkData) {
		matched = true
		chunkData = decodeBraceEscape(chunkData)
	} else if hasBackslash && perlEscapePat.Match(chunkData) {
		matched = true
		chunkData = decodePerlEscape(chunkData)
	} else if bytes.Contains(chunkData, htmlAnchor) && htmlEscapePat.Match(chunkData) {
		matched = true
		chunkData = decodeHtmlEscape(chunkData)
	} else if bytes.Contains(chunkData, percentAnchor) && percentEscapePat.Match(chunkData) {
		matched = true
		chunkData = decodePercentEscape(chunkData)
	} else if hasBackslash && escapePat.Match(chunkData) {
		matched = true
		chunkData = decodeEscaped(chunkData)
	} else if codePointIdx := codePointMatches(chunkData); len(codePointIdx) > 0 {
		matched = true
		chunkData = decodeCodePointAt(chunkData, codePointIdx)
	} else if hasBackslash && cssEscapePat.Match(chunkData) {
		matched = true
		chunkData = decodeCssEscape(chunkData)
		// } else if hexEscapePat.Match(chunkData) {
		// 	matched = true
		// 	chunkData = decodeHexEscape(chunkData)
	}

	if matched {
		return &DecodableChunk{
			DecoderType: d.Type(),
			Chunk: &sources.Chunk{
				Data:           chunkData,
				OriginalData:   chunk.OriginalData,
				SourceName:     chunk.SourceName,
				SourceID:       chunk.SourceID,
				JobID:          chunk.JobID,
				SecretID:       chunk.SecretID,
				SourceMetadata: chunk.SourceMetadata,
				SourceType:     chunk.SourceType,
				SourceVerify:   chunk.SourceVerify,
			},
		}
	} else {
		return nil
	}
}

// Unicode characters are encoded as 1 to 4 bytes per rune.
const maxBytesPerRune = 4
const spaceChar = byte(' ')

// decodeWithPattern replaces escape sequences matched by re with their UTF-8
// equivalents. The regex *must* have the first capturing group contain the
// hexadecimal code-point digits. Any invalid value (> 0x10FFFF or parse error)
// is skipped. The replacement walks matches in reverse order to avoid index
// shifts.
func decodeWithPattern(input []byte, re *regexp.Regexp) []byte {
	indices := re.FindAllSubmatchIndex(input, -1)
	if len(indices) == 0 {
		return input
	}

	utf8Bytes := make([]byte, maxBytesPerRune)
	for i := len(indices) - 1; i >= 0; i-- {
		m := indices[i]
		start, end := m[0], m[1]
		hexStart, hexEnd := m[2], m[3]

		cp, err := strconv.ParseUint(string(input[hexStart:hexEnd]), 16, 32)
		if err != nil || cp > 0x10FFFF {
			continue
		}

		utf8Len := utf8.EncodeRune(utf8Bytes, rune(cp))
		input = append(input[:start], append(utf8Bytes[:utf8Len], input[end:]...)...)
	}
	return input
}

func decodeCodePoint(input []byte) []byte {
	// Find all Unicode escape sequences in the input byte slice
	return decodeCodePointAt(input, codePointMatches(input))
}

// decodeCodePointAt is decodeCodePoint driven by indices the caller already
// has. FromChunk needs the match list to decide whether this pattern applies,
// so it passes that list straight through rather than scanning a second time.
func decodeCodePointAt(input []byte, indices [][]int) []byte {
	// Iterate over found indices in reverse order to avoid modifying the slice length
	utf8Bytes := make([]byte, maxBytesPerRune)
	for i := len(indices) - 1; i >= 0; i-- {
		matches := indices[i]

		startIndex := matches[0]
		endIndex := matches[1]
		hexStartIndex := matches[2]
		hexEndIndex := matches[3]

		// If the input is like `U+1234 U+5678` we should replace `U+1234 `.
		// Otherwise, we should only replace `U+1234`.
		if endIndex != hexEndIndex && input[endIndex-1] != spaceChar {
			endIndex = endIndex - 1
		}

		// Extract the hexadecimal value from the escape sequence
		hexValue := string(input[hexStartIndex:hexEndIndex])

		// Parse the hexadecimal value to an integer
		unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
		if err != nil {
			// If there's an error, continue to the next escape sequence
			continue
		}

		// Convert the Unicode code point to a UTF-8 representation
		utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))

		// Replace the escape sequence with the UTF-8 representation
		input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
	}

	return input
}

func decodeEscaped(input []byte) []byte {
	return decodeWithPattern(input, escapePat)
}

// decodeBraceEscape handles \u{X} format - Rust, Swift, some JS, etc.
func decodeBraceEscape(input []byte) []byte {
	return decodeWithPattern(input, braceEscapePat)
}

// decodeLongEscape handles \U00XXXXXX format - Python, etc.
func decodeLongEscape(input []byte) []byte {
	return decodeWithPattern(input, longEscapePat)
}

// decodePerlEscape handles \x{X} format - Perl
func decodePerlEscape(input []byte) []byte {
	return decodeWithPattern(input, perlEscapePat)
}

// decodeCssEscape handles \X format - CSS (hex without padding, with space delimiter or end of string or next hex sequence)
func decodeCssEscape(input []byte) []byte {
	return decodeWithPattern(input, cssEscapePat)
}

// decodeHtmlEscape handles &#xX; format - HTML/XML
func decodeHtmlEscape(input []byte) []byte {
	return decodeWithPattern(input, htmlEscapePat)
}

// decodePercentEscape handles %uXXXX format - Percent-encoding (non-standard)
func decodePercentEscape(input []byte) []byte {
	return decodeWithPattern(input, percentEscapePat)
}

// decodeHexEscape handles 0xX format - Hexadecimal notation with space separation
// func decodeHexEscape(input []byte) []byte {
// 	// This format requires consecutive 0xNN sequences to be considered for decoding
// 	// We'll look for patterns of multiple consecutive hex values
// 	hexPattern := regexp.MustCompile(`(?:0x[a-fA-F0-9]{1,2}(?:\s+|$))+`)

// 	matches := hexPattern.FindAll(input, -1)
// 	if len(matches) == 0 {
// 		return input
// 	}

// 	result := input
// 	for _, match := range matches {
// 		// Extract individual hex values
// 		individualHex := regexp.MustCompile(`0x([a-fA-F0-9]{1,2})`)
// 		hexMatches := individualHex.FindAllSubmatch(match, -1)

// 		// Only decode if we have multiple consecutive hex values (likely to be a Unicode string)
// 		if len(hexMatches) < 3 {
// 			continue
// 		}

// 		var decoded []byte
// 		for _, hexMatch := range hexMatches {
// 			hexValue := string(hexMatch[1])
// 			if len(hexValue) == 1 {
// 				hexValue = "0" + hexValue // Pad single digit hex values
// 			}

// 			unicodeInt, err := strconv.ParseUint(hexValue, 16, 32)
// 			if err != nil || unicodeInt > 0x10FFFF {
// 				break
// 			}

// 			if unicodeInt <= 0x7F {
// 				// ASCII character
// 				decoded = append(decoded, byte(unicodeInt))
// 			} else {
// 				// Unicode character
// 				utf8Bytes := make([]byte, maxBytesPerRune)
// 				utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
// 				decoded = append(decoded, utf8Bytes[:utf8Len]...)
// 			}
// 		}

// 		// Replace the original sequence with decoded bytes
// 		result = bytes.Replace(result, match, decoded, 1)
// 	}

// 	return result
// }