Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[refactor] - s3 metrics (#3760)
ahrav3525c6f45ce1
diff --git a/pkg/sources/s3/metrics.go b/pkg/sources/s3/metrics.goindex 3eb8b7c7b..534241477 100644--- a/pkg/sources/s3/metrics.go+++ b/pkg/sources/s3/metrics.go@@ -11,8 +11,8 @@ import (type metricsCollector interface {// Object metrics.- RecordObjectScanned(bucket string)- RecordObjectSkipped(bucket, reason string)+ RecordObjectScanned(bucket string, sizeBytes float64)+ RecordObjectSkipped(bucket, reason string, sizeBytes float64)RecordObjectError(bucket string)// Role metrics.@@ -22,8 +22,8 @@ type metricsCollector interface {}type collector struct {- objectsScanned *prometheus.CounterVec- objectsSkipped *prometheus.CounterVec+ objectsScanned *prometheus.HistogramVec+ objectsSkipped *prometheus.HistogramVecobjectsErrors *prometheus.CounterVecrolesScanned *prometheus.GaugeVecbucketsPerRole *prometheus.GaugeVec@@ -33,18 +33,22 @@ var metricsInstance metricsCollectorfunc init() {metricsInstance = &collector{- objectsScanned: promauto.NewCounterVec(prometheus.CounterOpts{+ objectsScanned: promauto.NewHistogramVec(prometheus.HistogramOpts{Namespace: common.MetricsNamespace,Subsystem: common.MetricsSubsystem,- Name: "objects_scanned_total",- Help: "Total number of S3 objects successfully scanned",+ Name: "objects_scanned_bytes",+ Help: "Size distribution of successfully scanned S3 objects in bytes",+ // 64B, 512B, 4KB, 32KB, 256KB, 2MB, 16MB, 128MB, 1GB.+ Buckets: prometheus.ExponentialBuckets(64, 8, 9),}, []string{"bucket"}),- objectsSkipped: promauto.NewCounterVec(prometheus.CounterOpts{+ objectsSkipped: promauto.NewHistogramVec(prometheus.HistogramOpts{Namespace: common.MetricsNamespace,Subsystem: common.MetricsSubsystem,- Name: "objects_skipped_total",- Help: "Total number of S3 objects skipped during scan",+ Name: "objects_skipped_bytes",+ Help: "Size distribution of skipped S3 objects in bytes",+ // 64B, 512B, 4KB, 32KB, 256KB, 2MB, 16MB, 128MB, 1GB.+ Buckets: prometheus.ExponentialBuckets(64, 8, 9),}, []string{"bucket", "reason"}),objectsErrors: promauto.NewCounterVec(prometheus.CounterOpts{@@ -70,12 +74,12 @@ func init() {}}-func (c *collector) RecordObjectScanned(bucket string) {- c.objectsScanned.WithLabelValues(bucket).Inc()+func (c *collector) RecordObjectScanned(bucket string, sizeBytes float64) {+ c.objectsScanned.WithLabelValues(bucket).Observe(sizeBytes)}-func (c *collector) RecordObjectSkipped(bucket, reason string) {- c.objectsSkipped.WithLabelValues(bucket, reason).Inc()+func (c *collector) RecordObjectSkipped(bucket, reason string, sizeBytes float64) {+ c.objectsSkipped.WithLabelValues(bucket, reason).Observe(sizeBytes)}func (c *collector) RecordObjectError(bucket string) {diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex 959a0bbfa..bed0b9ddc 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -432,7 +432,7 @@ func (s *Source) pageChunker(for objIdx, obj := range metadata.page.Contents {if obj == nil {- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "nil_object")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "nil_object", 0)if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for nil object")}@@ -448,7 +448,7 @@ func (s *Source) pageChunker(// Skip GLACIER and GLACIER_IR objects.if obj.StorageClass == nil || strings.Contains(*obj.StorageClass, "GLACIER") {ctx.Logger().V(5).Info("Skipping object in storage class", "storage_class", *obj.StorageClass)- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "storage_class")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "storage_class", float64(*obj.Size))if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for glacier object")}@@ -458,7 +458,7 @@ func (s *Source) pageChunker(// Ignore large files.if *obj.Size > s.maxObjectSize {ctx.Logger().V(5).Info("Skipping %d byte file (over maxObjectSize limit)")- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "size_limit")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "size_limit", float64(*obj.Size))if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for large file")}@@ -468,7 +468,7 @@ func (s *Source) pageChunker(// File empty file.if *obj.Size == 0 {ctx.Logger().V(5).Info("Skipping empty file")- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "empty_file")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "empty_file", 0)if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for empty file")}@@ -478,7 +478,7 @@ func (s *Source) pageChunker(// Skip incompatible extensions.if common.SkipFile(*obj.Key) {ctx.Logger().V(5).Info("Skipping file with incompatible extension")- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "incompatible_extension")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "incompatible_extension", float64(*obj.Size))if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for incompatible file")}@@ -493,7 +493,7 @@ func (s *Source) pageChunker(if strings.HasSuffix(*obj.Key, "/") {ctx.Logger().V(5).Info("Skipping directory")- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "directory")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "directory", float64(*obj.Size))return nil}@@ -521,7 +521,7 @@ func (s *Source) pageChunker(if err != nil {if strings.Contains(err.Error(), "AccessDenied") {ctx.Logger().Error(err, "could not get S3 object; access denied")- s.metricsCollector.RecordObjectSkipped(metadata.bucket, "access_denied")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "access_denied", float64(*obj.Size))} else {ctx.Logger().Error(err, "could not get S3 object")s.metricsCollector.RecordObjectError(metadata.bucket)@@ -596,7 +596,7 @@ func (s *Source) pageChunker(if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for scanned object")}- s.metricsCollector.RecordObjectScanned(metadata.bucket)+ s.metricsCollector.RecordObjectScanned(metadata.bucket, float64(*obj.Size))return nil})