Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[feat] - S3 metrics (#3577)
ahrav7b3d98d1332d
diff --git a/pkg/sources/s3/metrics.go b/pkg/sources/s3/metrics.gonew file mode 100644index 000000000..3eb8b7c7b--- /dev/null+++ b/pkg/sources/s3/metrics.go@@ -0,0 +1,99 @@+package s3++import (+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+)++// metricsCollector defines the interface for recording S3 scan metrics.+type metricsCollector interface {+ // Object metrics.++ RecordObjectScanned(bucket string)+ RecordObjectSkipped(bucket, reason string)+ RecordObjectError(bucket string)++ // Role metrics.++ RecordRoleScanned(roleArn string)+ RecordBucketForRole(roleArn string)+}++type collector struct {+ objectsScanned *prometheus.CounterVec+ objectsSkipped *prometheus.CounterVec+ objectsErrors *prometheus.CounterVec+ rolesScanned *prometheus.GaugeVec+ bucketsPerRole *prometheus.GaugeVec+}++var metricsInstance metricsCollector++func init() {+ metricsInstance = &collector{+ objectsScanned: promauto.NewCounterVec(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "objects_scanned_total",+ Help: "Total number of S3 objects successfully scanned",+ }, []string{"bucket"}),++ objectsSkipped: promauto.NewCounterVec(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "objects_skipped_total",+ Help: "Total number of S3 objects skipped during scan",+ }, []string{"bucket", "reason"}),++ objectsErrors: promauto.NewCounterVec(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "objects_errors_total",+ Help: "Total number of errors encountered during S3 scan",+ }, []string{"bucket"}),++ rolesScanned: promauto.NewGaugeVec(prometheus.GaugeOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "roles_scanned",+ Help: "Number of AWS roles being scanned",+ }, []string{"role_arn"}),++ bucketsPerRole: promauto.NewGaugeVec(prometheus.GaugeOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "buckets_per_role",+ Help: "Number of buckets accessible per AWS role",+ }, []string{"role_arn"}),+ }+}++func (c *collector) RecordObjectScanned(bucket string) {+ c.objectsScanned.WithLabelValues(bucket).Inc()+}++func (c *collector) RecordObjectSkipped(bucket, reason string) {+ c.objectsSkipped.WithLabelValues(bucket, reason).Inc()+}++func (c *collector) RecordObjectError(bucket string) {+ c.objectsErrors.WithLabelValues(bucket).Inc()+}++const defaultRoleARN = "default"++func (c *collector) RecordRoleScanned(roleArn string) {+ if roleArn == "" {+ roleArn = defaultRoleARN+ }+ c.rolesScanned.WithLabelValues(roleArn).Set(1)+}++func (c *collector) RecordBucketForRole(roleArn string) {+ if roleArn == "" {+ roleArn = defaultRoleARN+ }+ c.bucketsPerRole.WithLabelValues(roleArn).Inc()+}diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex 91970e9fd..959a0bbfa 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -48,6 +48,7 @@ type Source struct {checkpointer *Checkpointersources.Progress+ metricsCollector metricsCollectorerrorCount *sync.MapjobPool *errgroup.Group@@ -94,6 +95,7 @@ func (s *Source) Init(s.conn = &conns.checkpointer = NewCheckpointer(ctx, conn.GetEnableResumption(), &s.Progress)+ s.metricsCollector = metricsInstances.setMaxObjectSize(conn.GetMaxObjectSize())@@ -106,11 +108,12 @@ func (s *Source) Init(func (s *Source) Validate(ctx context.Context) []error {var errs []error- visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {+ visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) error {roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)if len(roleErrs) > 0 {errs = append(errs, roleErrs...)}+ return nil}if err := s.visitRoles(ctx, visitor); err != nil {@@ -307,6 +310,7 @@ func (s *Source) scanBuckets(bucketsToScanCount := len(bucketsToScan)for bucketIdx := pos.index; bucketIdx < bucketsToScanCount; bucketIdx++ {+ s.metricsCollector.RecordBucketForRole(role)bucket := bucketsToScan[bucketIdx]ctx := context.WithValue(ctx, "bucket", bucket)@@ -385,8 +389,9 @@ func (s *Source) scanBuckets(// Chunks emits chunks of bytes over a channel.func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {- visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {+ visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) error {s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)+ return nil}return s.visitRoles(ctx, visitor)@@ -427,6 +432,7 @@ func (s *Source) pageChunker(for objIdx, obj := range metadata.page.Contents {if obj == nil {+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "nil_object")if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for nil object")}@@ -442,6 +448,7 @@ func (s *Source) pageChunker(// Skip GLACIER and GLACIER_IR objects.if obj.StorageClass == nil || strings.Contains(*obj.StorageClass, "GLACIER") {ctx.Logger().V(5).Info("Skipping object in storage class", "storage_class", *obj.StorageClass)+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "storage_class")if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for glacier object")}@@ -451,6 +458,7 @@ func (s *Source) pageChunker(// Ignore large files.if *obj.Size > s.maxObjectSize {ctx.Logger().V(5).Info("Skipping %d byte file (over maxObjectSize limit)")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "size_limit")if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for large file")}@@ -460,6 +468,7 @@ func (s *Source) pageChunker(// File empty file.if *obj.Size == 0 {ctx.Logger().V(5).Info("Skipping empty file")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "empty_file")if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for empty file")}@@ -469,6 +478,7 @@ func (s *Source) pageChunker(// Skip incompatible extensions.if common.SkipFile(*obj.Key) {ctx.Logger().V(5).Info("Skipping file with incompatible extension")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "incompatible_extension")if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for incompatible file")}@@ -483,6 +493,7 @@ func (s *Source) pageChunker(if strings.HasSuffix(*obj.Key, "/") {ctx.Logger().V(5).Info("Skipping directory")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "directory")return nil}@@ -508,8 +519,12 @@ func (s *Source) pageChunker(Key: obj.Key,})if err != nil {- if !strings.Contains(err.Error(), "AccessDenied") {+ if strings.Contains(err.Error(), "AccessDenied") {+ ctx.Logger().Error(err, "could not get S3 object; access denied")+ s.metricsCollector.RecordObjectSkipped(metadata.bucket, "access_denied")+ } else {ctx.Logger().Error(err, "could not get S3 object")+ s.metricsCollector.RecordObjectError(metadata.bucket)}// According to the documentation for GetObjectWithContext,// the response can be non-nil even if there was an error.@@ -563,6 +578,7 @@ func (s *Source) pageChunker(if err := handlers.HandleFile(ctx, res.Body, chunkSkel, sources.ChanReporter{Ch: chunksChan}); err != nil {ctx.Logger().Error(err, "error handling file")+ s.metricsCollector.RecordObjectError(metadata.bucket)return nil}@@ -580,6 +596,7 @@ func (s *Source) pageChunker(if err := s.checkpointer.UpdateObjectCompletion(ctx, objIdx, metadata.bucket, metadata.page.Contents); err != nil {ctx.Logger().Error(err, "could not update progress for scanned object")}+ s.metricsCollector.RecordObjectScanned(metadata.bucket)return nil})@@ -633,7 +650,7 @@ func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleAr// If no roles are configured, it will call the function with an empty role ARN.func (s *Source) visitRoles(ctx context.Context,- f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string),+ f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) error,) error {roles := s.conn.GetRoles()if len(roles) == 0 {@@ -641,6 +658,8 @@ func (s *Source) visitRoles(}for _, role := range roles {+ s.metricsCollector.RecordRoleScanned(role)+client, err := s.newClient(defaultAWSRegion, role)if err != nil {return fmt.Errorf("could not create s3 client: %w", err)@@ -651,7 +670,9 @@ func (s *Source) visitRoles(return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)}- f(ctx, client, role, bucketsToScan)+ if err := f(ctx, client, role, bucketsToScan); err != nil {+ return err+ }}return nildiff --git a/pkg/sources/s3/s3_integration_test.go b/pkg/sources/s3/s3_integration_test.goindex 1801e29e2..3276e40fd 100644--- a/pkg/sources/s3/s3_integration_test.go+++ b/pkg/sources/s3/s3_integration_test.go@@ -82,6 +82,37 @@ func TestSource_ChunksLarge(t *testing.T) {assert.Equal(t, got, wantChunkCount)}+func TestSourceChunksNoResumption(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)+ defer cancel()++ s := Source{}+ connection := &sourcespb.S3{+ Credential: &sourcespb.S3_Unauthenticated{},+ Buckets: []string{"trufflesec-ahrav-test-2"},+ }+ conn, err := anypb.New(connection)+ if err != nil {+ t.Fatal(err)+ }++ err = s.Init(ctx, "test name", 0, 0, false, conn, 1)+ chunksCh := make(chan *sources.Chunk)+ go func() {+ defer close(chunksCh)+ err = s.Chunks(ctx, chunksCh)+ assert.Nil(t, err)+ }()++ wantChunkCount := 19787+ got := 0++ for range chunksCh {+ got+++ }+ assert.Equal(t, got, wantChunkCount)+}+func TestSource_Validate(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)defer cancel()