Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Capture decoding time metric (#3209)
Cody Rose9718ec6a5147
diff --git a/pkg/decoders/base64.go b/pkg/decoders/base64.goindex 499ddb91c..8d19e7eb5 100644--- a/pkg/decoders/base64.go+++ b/pkg/decoders/base64.go@@ -28,7 +28,7 @@ func init() {}func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {- decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: d.Type()}encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)decodedSubstrings := make(map[string][]byte)@@ -67,6 +67,10 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {return nil}+func (d *Base64) Type() detectorspb.DecoderType {+ return detectorspb.DecoderType_BASE64+}+func isASCII(b []byte) bool {for i := 0; i < len(b); i++ {if b[i] > unicode.MaxASCII {diff --git a/pkg/decoders/decoders.go b/pkg/decoders/decoders.goindex 87491a67e..c49eee403 100644--- a/pkg/decoders/decoders.go+++ b/pkg/decoders/decoders.go@@ -24,6 +24,7 @@ type DecodableChunk struct {type Decoder interface {FromChunk(chunk *sources.Chunk) *DecodableChunk+ Type() detectorspb.DecoderType}// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.goindex 9f4acda79..b47d9383f 100644--- a/pkg/decoders/escaped_unicode.go+++ b/pkg/decoders/escaped_unicode.go@@ -41,7 +41,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {if matched {decodableChunk := &DecodableChunk{- DecoderType: detectorspb.DecoderType_ESCAPED_UNICODE,+ DecoderType: d.Type(),Chunk: chunk,}return decodableChunk@@ -94,6 +94,10 @@ func decodeCodePoint(input []byte) []byte {return input}+func (d *EscapedUnicode) Type() detectorspb.DecoderType {+ return detectorspb.DecoderType_ESCAPED_UNICODE+}+func decodeEscaped(input []byte) []byte {// Find all Unicode escape sequences in the input byte sliceindices := escapePat.FindAllSubmatchIndex(input, -1)diff --git a/pkg/decoders/utf16.go b/pkg/decoders/utf16.goindex c534b5812..7f0616d25 100644--- a/pkg/decoders/utf16.go+++ b/pkg/decoders/utf16.go@@ -16,7 +16,7 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {return nil}- decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: d.Type()}if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {if len(utf16Data) == 0 {return nil@@ -28,6 +28,10 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {return nil}+func (d *UTF16) Type() detectorspb.DecoderType {+ return detectorspb.DecoderType_UTF16+}+// utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.func utf16ToUTF8(b []byte) ([]byte, error) {var bufBE, bufLE bytes.Bufferdiff --git a/pkg/decoders/utf8.go b/pkg/decoders/utf8.goindex be4c22dfa..6672aade3 100644--- a/pkg/decoders/utf8.go+++ b/pkg/decoders/utf8.go@@ -15,7 +15,7 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {return nil}- decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: d.Type()}if !utf8.Valid(chunk.Data) {chunk.Data = extractSubstrings(chunk.Data)@@ -25,6 +25,10 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {return decodableChunk}+func (d *UTF8) Type() detectorspb.DecoderType {+ return detectorspb.DecoderType_PLAIN+}+// extractSubstrings performs similarly to the strings binutil,// extacting contigous portions of printable characters that we care// about from some bytesdiff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex d1c163852..dd051c596 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -733,7 +733,11 @@ func (e *Engine) scannerWorker(ctx context.Context) {startTime := time.Now()sourceVerify := chunk.Verifyfor _, decoder := range e.decoders {+ decodeStart := time.Now()decoded := decoder.FromChunk(chunk)+ decodeTime := time.Since(decodeStart).Microseconds()+ decodeLatency.WithLabelValues(decoder.Type().String(), chunk.SourceName).Observe(float64(decodeTime))+if decoded == nil {ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)continuediff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.goindex e7399a7b2..353497795 100644--- a/pkg/engine/metrics.go+++ b/pkg/engine/metrics.go@@ -8,6 +8,17 @@ import ()var (+ decodeLatency = promauto.NewHistogramVec(+ prometheus.HistogramOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "decode_latency",+ Help: "Time spent decoding a chunk in microseconds",+ Buckets: prometheus.ExponentialBuckets(50, 2, 20),+ },+ []string{"decoder_type", "source_name"},+ )+// Detector metrics.detectorExecutionCount = promauto.NewCounterVec(prometheus.CounterOpts{