Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add s3 support to CLI (#76)
Bill Rich6486c185657c
diff --git a/go.mod b/go.modindex 5fa61d3e0..343a78146 100644--- a/go.mod+++ b/go.mod@@ -6,6 +6,7 @@ require (cloud.google.com/go/secretmanager v1.3.0github.com/Azure/go-autorest/autorest/azure/auth v0.5.11github.com/aliyun/alibaba-cloud-sdk-go v1.61.1465+ github.com/aws/aws-sdk-go v1.43.18github.com/aws/aws-sdk-go-v2/credentials v1.10.0github.com/aws/aws-sdk-go-v2/service/sts v1.16.0github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fbdiff --git a/go.sum b/go.sumindex b0fcd60c2..8f248f8aa 100644--- a/go.sum+++ b/go.sum@@ -93,6 +93,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUgithub.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=+github.com/aws/aws-sdk-go v1.43.18 h1:nwLaIz2m1f7YBEMNyEc6bBB276AIEaGaIQrc2G9h4zY=+github.com/aws/aws-sdk-go v1.43.18/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=github.com/aws/aws-sdk-go-v2 v1.15.0 h1:f9kWLNfyCzCB43eupDAk3/XgJ2EpgktiySD6leqs0js=github.com/aws/aws-sdk-go-v2 v1.15.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI=github.com/aws/aws-sdk-go-v2/credentials v1.10.0 h1:M/FFpf2w31F7xqJqJLgiM0mFpLOtBvwZggORr6QCpo8=diff --git a/main.go b/main.goindex 456fd2b84..1772f4860 100644--- a/main.go+++ b/main.go@@ -67,8 +67,11 @@ func main() {// filesystemScanIncludePaths := filesystemScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()// filesystemScanExcludePaths := filesystemScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()- s3Scan := cli.Command("s3", "Coming soon. Find credentials in an S3 bucket.")-+ s3Scan := cli.Command("s3", "Find credentials in S3 buckets.")+ s3ScanKey := s3Scan.Flag("key", "S3 key used to authenticate.").String()+ s3ScanSecret := s3Scan.Flag("secret", "S3 secret used to authenticate.").String()+ s3ScanCloudEnv := s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()+ s3ScanBuckets := s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))// When setting a base commit, chunks must be scanned in order.@@ -143,7 +146,10 @@ func main() {logrus.WithError(err).Fatal("Failed to scan filesystem")}case s3Scan.FullCommand():- log.Fatal("S3 not implemented. Coming soon.")+ err := e.ScanS3(ctx, *s3ScanKey, *s3ScanSecret, *s3ScanCloudEnv, *s3ScanBuckets)+ if err != nil {+ logrus.WithError(err).Fatal("Failed to scan S3.")+ }}if !*jsonLegacy && !*jsonOut {diff --git a/pkg/engine/s3.go b/pkg/engine/s3.gonew file mode 100644index 000000000..16bf42d13--- /dev/null+++ b/pkg/engine/s3.go@@ -0,0 +1,57 @@+package engine++import (+ "context"+ "fmt"+ "github.com/go-errors/errors"+ "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb"+ "runtime"+)++func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool, buckets []string) error {+ connection := &sourcespb.S3{+ Credential: &sourcespb.S3_Unauthenticated{},+ }+ if cloudCred {+ if len(key) > 0 || len(secret) > 0 {+ return fmt.Errorf("cannot use cloud credentials and basic auth together")+ }+ connection.Credential = &sourcespb.S3_CloudEnvironment{}+ }+ if len(key) > 0 && len(secret) > 0 {+ connection.Credential = &sourcespb.S3_AccessKey{+ AccessKey: &credentialspb.KeySecret{+ Key: key,+ Secret: secret,+ },+ }+ }+ if len(buckets) > 0 {+ connection.Buckets = buckets+ }+ var conn anypb.Any+ err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})+ if err != nil {+ logrus.WithError(err).Error("failed to marshal github connection")+ return err+ }++ s3Source := s3.Source{}+ err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())+ if err != nil {+ return errors.WrapPrefix(err, "failed to init S3 source", 0)+ }+ go func() {+ err := s3Source.Chunks(ctx, e.ChunksChan())+ if err != nil {+ logrus.WithError(err).Error("error scanning s3")+ }+ close(e.ChunksChan())+ }()+ return nil+}diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.gonew file mode 100644index 000000000..8e5b78bdb--- /dev/null+++ b/pkg/sources/s3/s3.go@@ -0,0 +1,327 @@+package s3++import (+ "context"+ "fmt"+ "io/ioutil"+ "strings"+ "sync"+ "time"++ "github.com/aws/aws-sdk-go/aws"+ "github.com/aws/aws-sdk-go/aws/credentials"+ "github.com/aws/aws-sdk-go/aws/session"+ "github.com/aws/aws-sdk-go/service/s3"+ "github.com/aws/aws-sdk-go/service/s3/s3manager"+ "github.com/go-errors/errors"+ log "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+ "golang.org/x/sync/semaphore"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb"+)++type Source struct {+ name string+ sourceId int64+ jobId int64+ verify bool+ concurrency int+ aCtx context.Context+ log *log.Entry+ sources.Progress+ errorCount *sync.Map+ conn *sourcespb.S3+}++// Ensure the Source satisfies the interface at compile time+var _ sources.Source = (*Source)(nil)++// Type returns the type of source+func (s *Source) Type() sourcespb.SourceType {+ return sourcespb.SourceType_SOURCE_TYPE_S3+}++func (s *Source) SourceID() int64 {+ return s.sourceId+}++func (s *Source) JobID() int64 {+ return s.jobId+}++// Init returns an initialized AWS source+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {+ s.log = log.WithField("source", s.Type()).WithField("name", name)++ s.aCtx = aCtx+ s.name = name+ s.sourceId = sourceId+ s.jobId = jobId+ s.verify = verify+ s.concurrency = concurrency+ s.errorCount = &sync.Map{}++ var conn sourcespb.S3+ err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})+ if err != nil {+ return errors.WrapPrefix(err, "error unmarshalling connection", 0)+ }+ s.conn = &conn++ return nil+}++func (s *Source) newClient(region string) (*s3.S3, error) {+ cfg := aws.NewConfig()+ cfg.CredentialsChainVerboseErrors = aws.Bool(true)+ cfg.Region = aws.String(region)++ switch cred := s.conn.GetCredential().(type) {+ case *sourcespb.S3_AccessKey:+ cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")+ case *sourcespb.S3_Unauthenticated:+ cfg.Credentials = credentials.AnonymousCredentials+ case *sourcespb.S3_CloudEnvironment:+ // Nothing needs to be done!+ default:+ return nil, errors.Errorf("invalid configuration given for %s source", s.name)+ }++ sess, err := session.NewSessionWithOptions(session.Options{+ SharedConfigState: session.SharedConfigEnable,+ Config: *cfg,+ })+ if err != nil {+ return nil, err+ }++ return s3.New(sess), nil+}++// Chunks emits chunks of bytes over a channel.+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+ client, err := s.newClient("us-east-1")+ if err != nil {+ return errors.WrapPrefix(err, "could not create s3 client", 0)+ }++ bucketsToScan := []string{}++ switch s.conn.GetCredential().(type) {+ case *sourcespb.S3_AccessKey, *sourcespb.S3_CloudEnvironment:+ if len(s.conn.Buckets) == 0 {+ res, err := client.ListBuckets(&s3.ListBucketsInput{})+ if err != nil {+ s.log.Errorf("could not list s3 buckets: %s", err)+ return errors.WrapPrefix(err, "could not list s3 buckets", 0)+ }+ buckets := res.Buckets+ for _, bucket := range buckets {+ bucketsToScan = append(bucketsToScan, *bucket.Name)+ }+ } else {+ bucketsToScan = s.conn.Buckets+ }+ case *sourcespb.S3_Unauthenticated:+ bucketsToScan = s.conn.Buckets+ default:+ return errors.Errorf("invalid configuration given for %s source", s.name)+ }++ for i, bucket := range bucketsToScan {+ if common.IsDone(ctx) {+ return nil+ }++ s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket))++ s.log.Debugf("Scanning bucket: %s", bucket)+ region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)+ if err != nil {+ s.log.WithError(err).Errorf("could not get s3 region for bucket: %s", bucket)+ continue+ }+ var regionalClient *s3.S3+ if region != "us-east-1" {+ regionalClient, err = s.newClient(region)+ if err != nil {+ s.log.WithError(err).Error("could not make regional s3 client")+ }+ } else {+ regionalClient = client+ }+ //Forced prefix for testing+ //pf := "public"+ errorCount := sync.Map{}++ err = regionalClient.ListObjectsV2PagesWithContext(+ ctx, &s3.ListObjectsV2Input{Bucket: &bucket},+ func(page *s3.ListObjectsV2Output, last bool) bool {+ s.pageChunker(ctx, regionalClient, chunksChan, bucket, page, &errorCount)+ return true+ })++ if err != nil {+ s.log.WithError(err).Errorf("could not list objects in s3 bucket: %s", bucket)+ return errors.WrapPrefix(err, fmt.Sprintf("could not list objects in s3 bucket: %s", bucket), 0)+ }++ }++ return nil+}++// pageChunker emits chunks onto the given channel from a page+func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan *sources.Chunk, bucket string, page *s3.ListObjectsV2Output, errorCount *sync.Map) {+ sem := semaphore.NewWeighted(int64(s.concurrency))+ var wg sync.WaitGroup+ for _, obj := range page.Contents {+ if common.IsDone(ctx) {+ return+ }++ err := sem.Acquire(ctx, 1)+ if err != nil {+ log.WithError(err).Error("could not acquire semaphore")+ continue+ }+ wg.Add(1)+ go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {+ defer sem.Release(1)+ defer wg.Done()+ //defer log.Debugf("DONE - %s", *obj.Key)++ if (*obj.Key)[len(*obj.Key)-1:] == "/" {+ return+ }+ //log.Debugf("Object: %s", *obj.Key)++ path := strings.Split(*obj.Key, "/")+ prefix := strings.Join(path[:len(path)-1], "/")++ nErr, ok := errorCount.Load(prefix)+ if !ok {+ nErr = 0+ }+ if nErr.(int) > 3 {+ log.Debugf("Skipped: %s", *obj.Key)+ return+ }++ // ignore large files+ if *obj.Size > int64(10*common.MB) {+ return+ }++ //file is 0 bytes - likely no permissions - skipping+ if *obj.Size == 0 {+ return+ }++ //files break with spaces, must replace with ++ //objKey := strings.ReplaceAll(*obj.Key, " ", "+")+ ctx, cancel := context.WithTimeout(ctx, time.Second*5)+ defer cancel()+ res, err := client.GetObjectWithContext(ctx, &s3.GetObjectInput{+ Bucket: &bucket,+ Key: obj.Key,+ })+ if err != nil {+ if !strings.Contains(err.Error(), "AccessDenied") {+ s.log.WithError(err).Errorf("could not get S3 object: %s", *obj.Key)+ }++ nErr, ok := errorCount.Load(prefix)+ if !ok {+ nErr = 0+ }+ if nErr.(int) > 3 {+ log.Debugf("Skipped: %s", *obj.Key)+ return+ }+ nErr = nErr.(int) + 1+ errorCount.Store(prefix, nErr)+ //too many consective errors on this page+ if nErr.(int) > 3 {+ s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)+ }+ log.Debugf("Error Counts: %s:%s", prefix, nErr)+ return+ }+ body, err := ioutil.ReadAll(res.Body)+ if err != nil {+ s.log.WithError(err).Error("could not read S3 object body")+ nErr, ok := errorCount.Load(prefix)+ if !ok {+ nErr = 0+ }+ //too many consective errors on this page+ if nErr.(int) > 3 {+ log.Debugf("Skipped: %s", *obj.Key)+ return+ }+ nErr = nErr.(int) + 1+ errorCount.Store(prefix, nErr)++ if nErr.(int) > 3 {+ s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)+ }+ return+ }++ // ignore files that don't have secrets+ if common.SkipFile(*obj.Key, body) {+ return+ }++ email := "Unknown"+ if obj.Owner != nil {+ email = *obj.Owner.DisplayName+ }+ modified := obj.LastModified.String()+ chunk := sources.Chunk{+ SourceType: s.Type(),+ SourceName: s.name,+ SourceID: s.SourceID(),+ Data: body,+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_S3{+ S3: &source_metadatapb.S3{+ Bucket: bucket,+ File: sanitizer.UTF8(*obj.Key),+ Link: sanitizer.UTF8(makeS3Link(bucket, *client.Config.Region, *obj.Key)),+ Email: sanitizer.UTF8(email),+ Timestamp: sanitizer.UTF8(modified),+ },+ },+ },+ Verify: s.verify,+ }+ nErr, ok = errorCount.Load(prefix)+ if !ok {+ nErr = 0+ }+ if nErr.(int) > 0 {+ errorCount.Store(prefix, 0)+ }+ chunksChan <- &chunk+ }(ctx, &wg, sem, obj)+ }+ wg.Wait()+}++// S3 links currently have the general format of:+// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]+func makeS3Link(bucket, region, key string) string {+ if region == "us-east-1" {+ region = ""+ } else {+ region = "." + region+ }+ return fmt.Sprintf("https://%s.s3%s.amazonaws.com/%s", bucket, region, key)+}diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.gonew file mode 100644index 000000000..993ebfce9--- /dev/null+++ b/pkg/sources/s3/s3_test.go@@ -0,0 +1,97 @@+package s3++import (+ "context"+ "encoding/base64"+ "fmt"+ "os"+ "sync"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ log "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+ "google.golang.org/protobuf/types/known/anypb"+)++func TestSource_Chunks(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)+ defer cancel()++ secret, err := common.GetTestSecret(ctx)+ if err != nil {+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ }++ s3key := secret.MustGetField("AWS_S3_KEY")+ s3secret := secret.MustGetField("AWS_S3_SECRET")++ type init struct {+ name string+ verify bool+ connection *sourcespb.S3+ }+ tests := []struct {+ name string+ init init+ wantErr bool+ wantChunkData string+ }{+ {+ name: "gets chunks",+ init: init{+ connection: &sourcespb.S3{+ Credential: &sourcespb.S3_AccessKey{+ AccessKey: &credentialspb.KeySecret{+ Key: s3key,+ Secret: s3secret,+ },+ },+ Buckets: []string{"thog-tmp-test"},+ },+ },+ wantErr: false,+ wantChunkData: `W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkID0gQUtJQTM1T0hYMkRTT1pHNjQ3TkgKYXdzX3NlY3JldF9hY2Nlc3Nfa2V5ID0gUXk5OVMrWkIvQ1dsRk50eFBBaWQ3Z0d6dnNyWGhCQjd1ckFDQUxwWgpvdXRwdXQgPSBqc29uCnJlZ2lvbiA9IHVzLWVhc3QtMg==`,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ log.SetFormatter(&log.TextFormatter{ForceColors: true})+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)+ var cancelOnce sync.Once+ defer cancelOnce.Do(cancel)++ s := Source{}+ log.SetLevel(log.DebugLevel)++ conn, err := anypb.New(tt.init.connection)+ if err != nil {+ t.Fatal(err)+ }++ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 10)+ if (err != nil) != tt.wantErr {+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ chunksCh := make(chan *sources.Chunk, 100)+ go func() {+ err = s.Chunks(ctx, chunksCh)+ if (err != nil) != tt.wantErr {+ t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)+ os.Exit(1)+ }+ }()+ gotChunk := <-chunksCh+ wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData)++ if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" {+ t.Errorf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}