Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
S3 scanner improvements (#938)
Dustin Decker28dd25beebf5
diff --git a/go.mod b/go.modindex 74dbbf161..334a430ce 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,6 @@ require (github.com/google/go-cmp v0.5.9github.com/google/go-github/v42 v42.0.0github.com/gorilla/mux v1.8.0- github.com/h2non/filetype v1.1.3github.com/hashicorp/go-retryablehttp v0.7.1github.com/jlaffaye/ftp v0.1.0github.com/joho/godotenv v1.4.0@@ -46,6 +45,7 @@ require (github.com/mholt/archiver/v4 v4.0.0-alpha.7github.com/paulbellamy/ratecounter v0.2.0github.com/pkg/errors v0.9.1+ github.com/rabbitmq/amqp091-go v1.5.0github.com/sergi/go-diff v1.2.0github.com/sirupsen/logrus v1.9.0github.com/stretchr/testify v1.8.1@@ -122,7 +122,6 @@ require (github.com/pierrec/lz4/v4 v4.1.14 // indirectgithub.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirectgithub.com/pmezard/go-difflib v1.0.0 // indirect- github.com/rabbitmq/amqp091-go v1.5.0 // indirectgithub.com/therootcompany/xz v1.0.1 // indirectgithub.com/ulikunitz/xz v0.5.10 // indirectgithub.com/xanzy/ssh-agent v0.3.0 // indirectdiff --git a/go.sum b/go.sumindex 451133881..fd5b57ad5 100644--- a/go.sum+++ b/go.sum@@ -270,8 +270,6 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=-github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=-github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=@@ -459,7 +457,6 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=diff --git a/pkg/common/vars.go b/pkg/common/vars.goindex d611b96e7..c89c325e2 100644--- a/pkg/common/vars.go+++ b/pkg/common/vars.go@@ -2,22 +2,14 @@ package commonimport ("path/filepath"-- "github.com/h2non/filetype")var (KB, MB, GB, TB, PB = 1e3, 1e6, 1e9, 1e12, 1e15- IGNORED_EXTENSIONS = []string{"pdf", "mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "tif", "tiff", "jpg", "jpeg", "png", "gif", "zip", "webp"}+ IGNORED_EXTENSIONS = []string{"mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "webp"})-func SkipFile(filename string, data []byte) bool {- if filepath.Ext(filename) == "" {- //no sepcified extension, check mimetype- if filetype.IsArchive(data[:256]) {- return true- }- }+func SkipFile(filename string) bool {for _, ext := range IGNORED_EXTENSIONS {if filepath.Ext(filename) == ext {return truediff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex a50cdc0b2..07b88e2ae 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -14,7 +14,7 @@ import ("github.com/aws/aws-sdk-go/service/s3/s3manager"diskbufferreader "github.com/bill-rich/disk-buffer-reader""github.com/go-errors/errors"- log "github.com/sirupsen/logrus"+ "github.com/go-logr/logr""golang.org/x/sync/semaphore""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -35,7 +35,7 @@ type Source struct {verify boolconcurrency intaCtx context.Context- log *log.Entry+ log logr.Loggersources.ProgresserrorCount *sync.Mapconn *sourcespb.S3@@ -59,7 +59,7 @@ func (s *Source) JobID() int64 {// Init returns an initialized AWS sourcefunc (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {- s.log = log.WithField("source", s.Type()).WithField("name", name)+ s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()s.aCtx = aCtxs.name = name@@ -120,7 +120,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errif len(s.conn.Buckets) == 0 {res, err := client.ListBuckets(&s3.ListBucketsInput{})if err != nil {- s.log.Errorf("could not list s3 buckets: %s", err)+ s.log.Error(err, "could not list s3 buckets")return errors.WrapPrefix(err, "could not list s3 buckets", 0)}buckets := res.Buckets@@ -143,23 +143,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errs.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")- s.log.Debugf("Scanning bucket: %s", bucket)+ s.log.Info("Scanning bucket", "bucket", bucket)region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)if err != nil {- s.log.WithError(err).Errorf("could not get s3 region for bucket: %s", bucket)+ s.log.Error(err, "could not get s3 region for bucket", "bucket", bucket)continue}var regionalClient *s3.S3if region != "us-east-1" {regionalClient, err = s.newClient(region)if err != nil {- s.log.WithError(err).Error("could not make regional s3 client")+ s.log.Error(err, "could not make regional s3 client")}} else {regionalClient = client}- // Forced prefix for testing- // pf := "public"+errorCount := sync.Map{}err = regionalClient.ListObjectsV2PagesWithContext(@@ -170,7 +169,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err})if err != nil {- s.log.WithError(err).Errorf("could not list objects in s3 bucket: %s", bucket)+ s.log.Error(err, "could not list objects in s3 bucket", "bucket", bucket)return errors.WrapPrefix(err, fmt.Sprintf("could not list objects in s3 bucket: %s", bucket), 0)}}@@ -188,9 +187,37 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chanreturn}+ if obj == nil {+ continue+ }++ // skip GLACIER and GLACIER_IR objects+ if obj.StorageClass == nil || strings.Contains(*obj.StorageClass, "GLACIER") {+ s.log.V(5).Info("Skipping object in storage class", "storage_class", *obj.StorageClass, "object", *obj.Key)+ continue+ }++ // ignore large files+ if *obj.Size > int64(250*common.MB) {+ s.log.V(3).Info("Skipping %d byte file (over 250MB limit)", "object", *obj.Key)+ return+ }++ // file empty file+ if *obj.Size == 0 {+ s.log.V(5).Info("Skipping 0 byte file", "object", *obj.Key)+ return+ }++ // skip incompatible extensions+ if common.SkipFile(*obj.Key) {+ s.log.V(5).Info("Skipping file with incompatible extension", "object", *obj.Key)+ return+ }+err := sem.Acquire(ctx, 1)if err != nil {- log.WithError(err).Error("could not acquire semaphore")+ s.log.Error(err, "could not acquire semaphore")continue}wg.Add(1)@@ -198,12 +225,10 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chandefer common.RecoverWithExit(ctx)defer sem.Release(1)defer wg.Done()- // defer log.Debugf("DONE - %s", *obj.Key)if (*obj.Key)[len(*obj.Key)-1:] == "/" {return}- // log.Debugf("Object: %s", *obj.Key)path := strings.Split(*obj.Key, "/")prefix := strings.Join(path[:len(path)-1], "/")@@ -213,17 +238,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan channErr = 0}if nErr.(int) > 3 {- log.Debugf("Skipped: %s", *obj.Key)- return- }-- // ignore large files- if *obj.Size > int64(10*common.MB) {- return- }-- // file is 0 bytes - likely no permissions - skipping- if *obj.Size == 0 {+ s.log.V(2).Info("Skipped due to excessive errors", "object", *obj.Key)return}@@ -237,7 +252,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan})if err != nil {if !strings.Contains(err.Error(), "AccessDenied") {- s.log.WithError(err).Errorf("could not get S3 object: %s", *obj.Key)+ s.log.Error(err, "could not get S3 object", "object", *obj.Key)}nErr, ok := errorCount.Load(prefix)@@ -245,23 +260,22 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan channErr = 0}if nErr.(int) > 3 {- log.Debugf("Skipped: %s", *obj.Key)+ s.log.V(3).Info("Skipped due to excessive errors", "object", *obj.Key)return}nErr = nErr.(int) + 1errorCount.Store(prefix, nErr)// too many consective errors on this pageif nErr.(int) > 3 {- s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)+ s.log.V(2).Info("Too many consecutive errors, excluding prefix", "prefix", prefix)}- log.Debugf("Error Counts: %s:%d", prefix, nErr)return}defer res.Body.Close()reader, err := diskbufferreader.New(res.Body)if err != nil {- log.WithError(err).Error("Could not create reader.")+ s.log.Error(err, "Could not create reader.")return}@@ -292,14 +306,14 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan}if err := reader.Reset(); err != nil {- log.WithError(err).Error("Error resetting reader to start.")+ s.log.Error(err, "Error resetting reader to start.")}reader.Stop()chunk := *chunkSkelchunkData, err := io.ReadAll(reader)if err != nil {- log.WithError(err).Error("Could not read file data.")+ s.log.Error(err, "Could not read file data.")return}chunk.Data = chunkData