Skip to content

Find, verify, and analyze leaked credentials

optimize utf-8 decoder (#1275)

ahravabdff53d5dd5
diff --git a/pkg/decoders/utf8.go b/pkg/decoders/utf8.go
index 6a494a201..63a8ea437 100644
--- a/pkg/decoders/utf8.go
+++ b/pkg/decoders/utf8.go
@@ -26,19 +26,30 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
// extacting contigous portions of printable characters that we care
// about from some bytes
func extractSubstrings(b []byte) []byte {
- fields := bytes.FieldsFunc(b, func(r rune) bool {
+ isValidByte := func(c byte) bool {
// https://www.rapidtables.com/code/text/ascii-table.html
// split on anything that is not ascii space through tilde
- return !(r > 31 && r < 127)
- })
-
- keep := [][]byte{}
- for _, field := range fields {
- // Remove fields shorter than 6 characters.
- if bts := bytes.TrimSpace(field); len(bts) > 5 {
- keep = append(keep, bts)
+ return c > 31 && c < 127
+ }
+
+ field := make([]byte, len(b))
+ fieldLen := 0
+ buf := &bytes.Buffer{}
+ for i, c := range b {
+ if isValidByte(c) {
+ field[fieldLen] = c
+ fieldLen++
+ } else {
+ if fieldLen > 5 {
+ buf.Write(field[:fieldLen])
+ }
+ fieldLen = 0
+ }
+
+ if i == len(b)-1 && fieldLen > 5 {
+ buf.Write(field[:fieldLen])
}
}
- return bytes.Join(keep, []byte("\n"))
+ return buf.Bytes()
}