Skip to content

Find, verify, and analyze leaked credentials

Capture decoding time metric (#3209)

Cody Rose9718ec6a5147
diff --git a/pkg/decoders/base64.go b/pkg/decoders/base64.go
index 499ddb91c..8d19e7eb5 100644
--- a/pkg/decoders/base64.go
+++ b/pkg/decoders/base64.go
@@ -28,7 +28,7 @@ func init() {
}
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
- decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: d.Type()}
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
decodedSubstrings := make(map[string][]byte)
@@ -67,6 +67,10 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
return nil
}
+func (d *Base64) Type() detectorspb.DecoderType {
+ return detectorspb.DecoderType_BASE64
+}
+
func isASCII(b []byte) bool {
for i := 0; i < len(b); i++ {
if b[i] > unicode.MaxASCII {
diff --git a/pkg/decoders/decoders.go b/pkg/decoders/decoders.go
index 87491a67e..c49eee403 100644
--- a/pkg/decoders/decoders.go
+++ b/pkg/decoders/decoders.go
@@ -24,6 +24,7 @@ type DecodableChunk struct {
type Decoder interface {
FromChunk(chunk *sources.Chunk) *DecodableChunk
+ Type() detectorspb.DecoderType
}
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.go
index 9f4acda79..b47d9383f 100644
--- a/pkg/decoders/escaped_unicode.go
+++ b/pkg/decoders/escaped_unicode.go
@@ -41,7 +41,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {
if matched {
decodableChunk := &DecodableChunk{
- DecoderType: detectorspb.DecoderType_ESCAPED_UNICODE,
+ DecoderType: d.Type(),
Chunk: chunk,
}
return decodableChunk
@@ -94,6 +94,10 @@ func decodeCodePoint(input []byte) []byte {
return input
}
+func (d *EscapedUnicode) Type() detectorspb.DecoderType {
+ return detectorspb.DecoderType_ESCAPED_UNICODE
+}
+
func decodeEscaped(input []byte) []byte {
// Find all Unicode escape sequences in the input byte slice
indices := escapePat.FindAllSubmatchIndex(input, -1)
diff --git a/pkg/decoders/utf16.go b/pkg/decoders/utf16.go
index c534b5812..7f0616d25 100644
--- a/pkg/decoders/utf16.go
+++ b/pkg/decoders/utf16.go
@@ -16,7 +16,7 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
return nil
}
- decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: d.Type()}
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
if len(utf16Data) == 0 {
return nil
@@ -28,6 +28,10 @@ func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
return nil
}
+func (d *UTF16) Type() detectorspb.DecoderType {
+ return detectorspb.DecoderType_UTF16
+}
+
// utf16ToUTF8 converts a byte slice containing UTF-16 encoded data to a UTF-8 encoded byte slice.
func utf16ToUTF8(b []byte) ([]byte, error) {
var bufBE, bufLE bytes.Buffer
diff --git a/pkg/decoders/utf8.go b/pkg/decoders/utf8.go
index be4c22dfa..6672aade3 100644
--- a/pkg/decoders/utf8.go
+++ b/pkg/decoders/utf8.go
@@ -15,7 +15,7 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
return nil
}
- decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: d.Type()}
if !utf8.Valid(chunk.Data) {
chunk.Data = extractSubstrings(chunk.Data)
@@ -25,6 +25,10 @@ func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
return decodableChunk
}
+func (d *UTF8) Type() detectorspb.DecoderType {
+ return detectorspb.DecoderType_PLAIN
+}
+
// extractSubstrings performs similarly to the strings binutil,
// extacting contigous portions of printable characters that we care
// about from some bytes
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index d1c163852..dd051c596 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -733,7 +733,11 @@ func (e *Engine) scannerWorker(ctx context.Context) {
startTime := time.Now()
sourceVerify := chunk.Verify
for _, decoder := range e.decoders {
+ decodeStart := time.Now()
decoded := decoder.FromChunk(chunk)
+ decodeTime := time.Since(decodeStart).Microseconds()
+ decodeLatency.WithLabelValues(decoder.Type().String(), chunk.SourceName).Observe(float64(decodeTime))
+
if decoded == nil {
ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)
continue
diff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.go
index e7399a7b2..353497795 100644
--- a/pkg/engine/metrics.go
+++ b/pkg/engine/metrics.go
@@ -8,6 +8,17 @@ import (
)
var (
+ decodeLatency = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Namespace: common.MetricsNamespace,
+ Subsystem: common.MetricsSubsystem,
+ Name: "decode_latency",
+ Help: "Time spent decoding a chunk in microseconds",
+ Buckets: prometheus.ExponentialBuckets(50, 2, 20),
+ },
+ []string{"decoder_type", "source_name"},
+ )
+
// Detector metrics.
detectorExecutionCount = promauto.NewCounterVec(
prometheus.CounterOpts{