Skip to content

Find, verify, and analyze leaked credentials

Add s3 support to CLI (#76)

Bill Rich6486c185657c
diff --git a/go.mod b/go.mod
index 5fa61d3e0..343a78146 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
cloud.google.com/go/secretmanager v1.3.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1465
+ github.com/aws/aws-sdk-go v1.43.18
github.com/aws/aws-sdk-go-v2/credentials v1.10.0
github.com/aws/aws-sdk-go-v2/service/sts v1.16.0
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
diff --git a/go.sum b/go.sum
index b0fcd60c2..8f248f8aa 100644
--- a/go.sum
+++ b/go.sum
@@ -93,6 +93,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/aws/aws-sdk-go v1.43.18 h1:nwLaIz2m1f7YBEMNyEc6bBB276AIEaGaIQrc2G9h4zY=
+github.com/aws/aws-sdk-go v1.43.18/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.15.0 h1:f9kWLNfyCzCB43eupDAk3/XgJ2EpgktiySD6leqs0js=
github.com/aws/aws-sdk-go-v2 v1.15.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI=
github.com/aws/aws-sdk-go-v2/credentials v1.10.0 h1:M/FFpf2w31F7xqJqJLgiM0mFpLOtBvwZggORr6QCpo8=
diff --git a/main.go b/main.go
index 456fd2b84..1772f4860 100644
--- a/main.go
+++ b/main.go
@@ -67,8 +67,11 @@ func main() {
// filesystemScanIncludePaths := filesystemScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
// filesystemScanExcludePaths := filesystemScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
- s3Scan := cli.Command("s3", "Coming soon. Find credentials in an S3 bucket.")
-
+ s3Scan := cli.Command("s3", "Find credentials in S3 buckets.")
+ s3ScanKey := s3Scan.Flag("key", "S3 key used to authenticate.").String()
+ s3ScanSecret := s3Scan.Flag("secret", "S3 secret used to authenticate.").String()
+ s3ScanCloudEnv := s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
+ s3ScanBuckets := s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))
// When setting a base commit, chunks must be scanned in order.
@@ -143,7 +146,10 @@ func main() {
logrus.WithError(err).Fatal("Failed to scan filesystem")
}
case s3Scan.FullCommand():
- log.Fatal("S3 not implemented. Coming soon.")
+ err := e.ScanS3(ctx, *s3ScanKey, *s3ScanSecret, *s3ScanCloudEnv, *s3ScanBuckets)
+ if err != nil {
+ logrus.WithError(err).Fatal("Failed to scan S3.")
+ }
}
if !*jsonLegacy && !*jsonOut {
diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go
new file mode 100644
index 000000000..16bf42d13
--- /dev/null
+++ b/pkg/engine/s3.go
@@ -0,0 +1,57 @@
+package engine
+
+import (
+ "context"
+ "fmt"
+ "github.com/go-errors/errors"
+ "github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
+ "runtime"
+)
+
+func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool, buckets []string) error {
+ connection := &sourcespb.S3{
+ Credential: &sourcespb.S3_Unauthenticated{},
+ }
+ if cloudCred {
+ if len(key) > 0 || len(secret) > 0 {
+ return fmt.Errorf("cannot use cloud credentials and basic auth together")
+ }
+ connection.Credential = &sourcespb.S3_CloudEnvironment{}
+ }
+ if len(key) > 0 && len(secret) > 0 {
+ connection.Credential = &sourcespb.S3_AccessKey{
+ AccessKey: &credentialspb.KeySecret{
+ Key: key,
+ Secret: secret,
+ },
+ }
+ }
+ if len(buckets) > 0 {
+ connection.Buckets = buckets
+ }
+ var conn anypb.Any
+ err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+ if err != nil {
+ logrus.WithError(err).Error("failed to marshal github connection")
+ return err
+ }
+
+ s3Source := s3.Source{}
+ err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
+ if err != nil {
+ return errors.WrapPrefix(err, "failed to init S3 source", 0)
+ }
+ go func() {
+ err := s3Source.Chunks(ctx, e.ChunksChan())
+ if err != nil {
+ logrus.WithError(err).Error("error scanning s3")
+ }
+ close(e.ChunksChan())
+ }()
+ return nil
+}
diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go
new file mode 100644
index 000000000..8e5b78bdb
--- /dev/null
+++ b/pkg/sources/s3/s3.go
@@ -0,0 +1,327 @@
+package s3
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+ "github.com/go-errors/errors"
+ log "github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+ "golang.org/x/sync/semaphore"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
+)
+
+type Source struct {
+ name string
+ sourceId int64
+ jobId int64
+ verify bool
+ concurrency int
+ aCtx context.Context
+ log *log.Entry
+ sources.Progress
+ errorCount *sync.Map
+ conn *sourcespb.S3
+}
+
+// Ensure the Source satisfies the interface at compile time
+var _ sources.Source = (*Source)(nil)
+
+// Type returns the type of source
+func (s *Source) Type() sourcespb.SourceType {
+ return sourcespb.SourceType_SOURCE_TYPE_S3
+}
+
+func (s *Source) SourceID() int64 {
+ return s.sourceId
+}
+
+func (s *Source) JobID() int64 {
+ return s.jobId
+}
+
+// Init returns an initialized AWS source
+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+ s.log = log.WithField("source", s.Type()).WithField("name", name)
+
+ s.aCtx = aCtx
+ s.name = name
+ s.sourceId = sourceId
+ s.jobId = jobId
+ s.verify = verify
+ s.concurrency = concurrency
+ s.errorCount = &sync.Map{}
+
+ var conn sourcespb.S3
+ err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
+ if err != nil {
+ return errors.WrapPrefix(err, "error unmarshalling connection", 0)
+ }
+ s.conn = &conn
+
+ return nil
+}
+
+func (s *Source) newClient(region string) (*s3.S3, error) {
+ cfg := aws.NewConfig()
+ cfg.CredentialsChainVerboseErrors = aws.Bool(true)
+ cfg.Region = aws.String(region)
+
+ switch cred := s.conn.GetCredential().(type) {
+ case *sourcespb.S3_AccessKey:
+ cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
+ case *sourcespb.S3_Unauthenticated:
+ cfg.Credentials = credentials.AnonymousCredentials
+ case *sourcespb.S3_CloudEnvironment:
+ // Nothing needs to be done!
+ default:
+ return nil, errors.Errorf("invalid configuration given for %s source", s.name)
+ }
+
+ sess, err := session.NewSessionWithOptions(session.Options{
+ SharedConfigState: session.SharedConfigEnable,
+ Config: *cfg,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return s3.New(sess), nil
+}
+
+// Chunks emits chunks of bytes over a channel.
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+ client, err := s.newClient("us-east-1")
+ if err != nil {
+ return errors.WrapPrefix(err, "could not create s3 client", 0)
+ }
+
+ bucketsToScan := []string{}
+
+ switch s.conn.GetCredential().(type) {
+ case *sourcespb.S3_AccessKey, *sourcespb.S3_CloudEnvironment:
+ if len(s.conn.Buckets) == 0 {
+ res, err := client.ListBuckets(&s3.ListBucketsInput{})
+ if err != nil {
+ s.log.Errorf("could not list s3 buckets: %s", err)
+ return errors.WrapPrefix(err, "could not list s3 buckets", 0)
+ }
+ buckets := res.Buckets
+ for _, bucket := range buckets {
+ bucketsToScan = append(bucketsToScan, *bucket.Name)
+ }
+ } else {
+ bucketsToScan = s.conn.Buckets
+ }
+ case *sourcespb.S3_Unauthenticated:
+ bucketsToScan = s.conn.Buckets
+ default:
+ return errors.Errorf("invalid configuration given for %s source", s.name)
+ }
+
+ for i, bucket := range bucketsToScan {
+ if common.IsDone(ctx) {
+ return nil
+ }
+
+ s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket))
+
+ s.log.Debugf("Scanning bucket: %s", bucket)
+ region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)
+ if err != nil {
+ s.log.WithError(err).Errorf("could not get s3 region for bucket: %s", bucket)
+ continue
+ }
+ var regionalClient *s3.S3
+ if region != "us-east-1" {
+ regionalClient, err = s.newClient(region)
+ if err != nil {
+ s.log.WithError(err).Error("could not make regional s3 client")
+ }
+ } else {
+ regionalClient = client
+ }
+ //Forced prefix for testing
+ //pf := "public"
+ errorCount := sync.Map{}
+
+ err = regionalClient.ListObjectsV2PagesWithContext(
+ ctx, &s3.ListObjectsV2Input{Bucket: &bucket},
+ func(page *s3.ListObjectsV2Output, last bool) bool {
+ s.pageChunker(ctx, regionalClient, chunksChan, bucket, page, &errorCount)
+ return true
+ })
+
+ if err != nil {
+ s.log.WithError(err).Errorf("could not list objects in s3 bucket: %s", bucket)
+ return errors.WrapPrefix(err, fmt.Sprintf("could not list objects in s3 bucket: %s", bucket), 0)
+ }
+
+ }
+
+ return nil
+}
+
+// pageChunker emits chunks onto the given channel from a page
+func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan *sources.Chunk, bucket string, page *s3.ListObjectsV2Output, errorCount *sync.Map) {
+ sem := semaphore.NewWeighted(int64(s.concurrency))
+ var wg sync.WaitGroup
+ for _, obj := range page.Contents {
+ if common.IsDone(ctx) {
+ return
+ }
+
+ err := sem.Acquire(ctx, 1)
+ if err != nil {
+ log.WithError(err).Error("could not acquire semaphore")
+ continue
+ }
+ wg.Add(1)
+ go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {
+ defer sem.Release(1)
+ defer wg.Done()
+ //defer log.Debugf("DONE - %s", *obj.Key)
+
+ if (*obj.Key)[len(*obj.Key)-1:] == "/" {
+ return
+ }
+ //log.Debugf("Object: %s", *obj.Key)
+
+ path := strings.Split(*obj.Key, "/")
+ prefix := strings.Join(path[:len(path)-1], "/")
+
+ nErr, ok := errorCount.Load(prefix)
+ if !ok {
+ nErr = 0
+ }
+ if nErr.(int) > 3 {
+ log.Debugf("Skipped: %s", *obj.Key)
+ return
+ }
+
+ // ignore large files
+ if *obj.Size > int64(10*common.MB) {
+ return
+ }
+
+ //file is 0 bytes - likely no permissions - skipping
+ if *obj.Size == 0 {
+ return
+ }
+
+ //files break with spaces, must replace with +
+ //objKey := strings.ReplaceAll(*obj.Key, " ", "+")
+ ctx, cancel := context.WithTimeout(ctx, time.Second*5)
+ defer cancel()
+ res, err := client.GetObjectWithContext(ctx, &s3.GetObjectInput{
+ Bucket: &bucket,
+ Key: obj.Key,
+ })
+ if err != nil {
+ if !strings.Contains(err.Error(), "AccessDenied") {
+ s.log.WithError(err).Errorf("could not get S3 object: %s", *obj.Key)
+ }
+
+ nErr, ok := errorCount.Load(prefix)
+ if !ok {
+ nErr = 0
+ }
+ if nErr.(int) > 3 {
+ log.Debugf("Skipped: %s", *obj.Key)
+ return
+ }
+ nErr = nErr.(int) + 1
+ errorCount.Store(prefix, nErr)
+ //too many consective errors on this page
+ if nErr.(int) > 3 {
+ s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
+ }
+ log.Debugf("Error Counts: %s:%s", prefix, nErr)
+ return
+ }
+ body, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ s.log.WithError(err).Error("could not read S3 object body")
+ nErr, ok := errorCount.Load(prefix)
+ if !ok {
+ nErr = 0
+ }
+ //too many consective errors on this page
+ if nErr.(int) > 3 {
+ log.Debugf("Skipped: %s", *obj.Key)
+ return
+ }
+ nErr = nErr.(int) + 1
+ errorCount.Store(prefix, nErr)
+
+ if nErr.(int) > 3 {
+ s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
+ }
+ return
+ }
+
+ // ignore files that don't have secrets
+ if common.SkipFile(*obj.Key, body) {
+ return
+ }
+
+ email := "Unknown"
+ if obj.Owner != nil {
+ email = *obj.Owner.DisplayName
+ }
+ modified := obj.LastModified.String()
+ chunk := sources.Chunk{
+ SourceType: s.Type(),
+ SourceName: s.name,
+ SourceID: s.SourceID(),
+ Data: body,
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_S3{
+ S3: &source_metadatapb.S3{
+ Bucket: bucket,
+ File: sanitizer.UTF8(*obj.Key),
+ Link: sanitizer.UTF8(makeS3Link(bucket, *client.Config.Region, *obj.Key)),
+ Email: sanitizer.UTF8(email),
+ Timestamp: sanitizer.UTF8(modified),
+ },
+ },
+ },
+ Verify: s.verify,
+ }
+ nErr, ok = errorCount.Load(prefix)
+ if !ok {
+ nErr = 0
+ }
+ if nErr.(int) > 0 {
+ errorCount.Store(prefix, 0)
+ }
+ chunksChan <- &chunk
+ }(ctx, &wg, sem, obj)
+ }
+ wg.Wait()
+}
+
+// S3 links currently have the general format of:
+// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
+func makeS3Link(bucket, region, key string) string {
+ if region == "us-east-1" {
+ region = ""
+ } else {
+ region = "." + region
+ }
+ return fmt.Sprintf("https://%s.s3%s.amazonaws.com/%s", bucket, region, key)
+}
diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go
new file mode 100644
index 000000000..993ebfce9
--- /dev/null
+++ b/pkg/sources/s3/s3_test.go
@@ -0,0 +1,97 @@
+package s3
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "os"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ log "github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+ "google.golang.org/protobuf/types/known/anypb"
+)
+
+func TestSource_Chunks(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+ defer cancel()
+
+ secret, err := common.GetTestSecret(ctx)
+ if err != nil {
+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+ }
+
+ s3key := secret.MustGetField("AWS_S3_KEY")
+ s3secret := secret.MustGetField("AWS_S3_SECRET")
+
+ type init struct {
+ name string
+ verify bool
+ connection *sourcespb.S3
+ }
+ tests := []struct {
+ name string
+ init init
+ wantErr bool
+ wantChunkData string
+ }{
+ {
+ name: "gets chunks",
+ init: init{
+ connection: &sourcespb.S3{
+ Credential: &sourcespb.S3_AccessKey{
+ AccessKey: &credentialspb.KeySecret{
+ Key: s3key,
+ Secret: s3secret,
+ },
+ },
+ Buckets: []string{"thog-tmp-test"},
+ },
+ },
+ wantErr: false,
+ wantChunkData: `W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkID0gQUtJQTM1T0hYMkRTT1pHNjQ3TkgKYXdzX3NlY3JldF9hY2Nlc3Nfa2V5ID0gUXk5OVMrWkIvQ1dsRk50eFBBaWQ3Z0d6dnNyWGhCQjd1ckFDQUxwWgpvdXRwdXQgPSBqc29uCnJlZ2lvbiA9IHVzLWVhc3QtMg==`,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ log.SetFormatter(&log.TextFormatter{ForceColors: true})
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+ var cancelOnce sync.Once
+ defer cancelOnce.Do(cancel)
+
+ s := Source{}
+ log.SetLevel(log.DebugLevel)
+
+ conn, err := anypb.New(tt.init.connection)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 10)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ chunksCh := make(chan *sources.Chunk, 100)
+ go func() {
+ err = s.Chunks(ctx, chunksCh)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
+ os.Exit(1)
+ }
+ }()
+ gotChunk := <-chunksCh
+ wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData)
+
+ if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" {
+ t.Errorf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}