Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add progress bar to CFOR (#3151)
Dustin Decker05e4635824a2
diff --git a/go.mod b/go.modindex 6183bf731..790541f51 100644--- a/go.mod+++ b/go.mod@@ -64,6 +64,7 @@ require (github.com/jlaffaye/ftp v0.2.0github.com/joho/godotenv v1.5.1github.com/jpillora/overseer v1.1.6+ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213github.com/kylelemons/godebug v1.1.0github.com/lib/pq v1.10.9github.com/lrstanley/bubblezone v0.0.0-20240125042004-b7bafc493195@@ -79,6 +80,7 @@ require (github.com/prometheus/client_golang v1.19.1github.com/rabbitmq/amqp091-go v1.10.0github.com/sassoftware/go-rpmutils v0.4.0+ github.com/schollz/progressbar/v3 v3.14.5github.com/sendgrid/sendgrid-go v3.14.0+incompatiblegithub.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3github.com/shuheiktgw/go-travis v0.3.1@@ -235,6 +237,7 @@ require (github.com/mattn/go-localereader v0.0.1 // indirectgithub.com/mattn/go-runewidth v0.0.15 // indirectgithub.com/microcosm-cc/bluemonday v1.0.25 // indirect+ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirectgithub.com/mitchellh/go-homedir v1.1.0 // indirectgithub.com/mitchellh/mapstructure v1.5.0 // indirectgithub.com/moby/docker-image-spec v1.3.1 // indirectdiff --git a/go.sum b/go.sumindex aa2c62a8e..88db0c2e1 100644--- a/go.sum+++ b/go.sum@@ -491,6 +491,8 @@ github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=@@ -550,6 +552,8 @@ github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJgithub.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE=github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=@@ -655,6 +659,8 @@ github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f h1:MvTmaQdww/z0Q4wrgithub.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg=github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=+github.com/schollz/progressbar/v3 v3.14.5 h1:97RrSxbBASxQuZN9yemnyGrFZ/swnG6IrEe2R0BseX8=+github.com/schollz/progressbar/v3 v3.14.5/go.mod h1:Nrzpuw3Nl0srLY0VlTvC4V6RL50pcEymjy6qyJAaLa0=github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fcif2i7P7wzISv1sU6DUA=diff --git a/pkg/sources/github_experimental/github_experimental.go b/pkg/sources/github_experimental/github_experimental.goindex 1f5fbb222..fecf5c273 100644--- a/pkg/sources/github_experimental/github_experimental.go+++ b/pkg/sources/github_experimental/github_experimental.go@@ -7,7 +7,6 @@ import ("github.com/go-logr/logr""github.com/google/go-github/v63/github"- "golang.org/x/sync/errgroup""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -23,40 +22,21 @@ import (const (SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL-- // unauthGithubOrgRateLimt = 30- // defaultPagination = 100- // membersAppPagination = 500)type Source struct {- name string-- // Protects the user and token.- //userMu sync.Mutex- //githubUser string- //githubToken string-- sourceID sources.SourceID- jobID sources.JobID- verify bool- //orgsCache cache.Cache[string]- //memberCache map[string]struct{}- //repos []string- //filteredRepoCache *filteredRepoCache- repoInfoCache repoInfoCache- //totalRepoSize int // total size of all repos in kb-+ name string+ sourceID sources.SourceID+ jobID sources.JobID+ verify bool+ repoInfoCache repoInfoCacheuseCustomContentWriter boolgit *git.Git-- scanOptions *git.ScanOptions-- httpClient *http.Client- log logr.Logger- conn *sourcespb.GitHubExperimental- jobPool *errgroup.Group- apiClient *github.Client+ scanOptions *git.ScanOptions+ httpClient *http.Client+ log logr.Logger+ conn *sourcespb.GitHubExperimental+ apiClient *github.Clientsources.Progresssources.CommonSourceUnitUnmarshaller@@ -100,8 +80,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sos.sourceID = sourceIDs.jobID = jobIDs.verify = verify- s.jobPool = &errgroup.Group{}- s.jobPool.SetLimit(concurrency)s.httpClient = common.RetryableHTTPClientTimeout(60)s.apiClient = github.NewClient(s.httpClient)diff --git a/pkg/sources/github_experimental/object_discovery.go b/pkg/sources/github_experimental/object_discovery.goindex 9af113358..7651ad2fb 100644--- a/pkg/sources/github_experimental/object_discovery.go+++ b/pkg/sources/github_experimental/object_discovery.go@@ -12,6 +12,8 @@ import ("time""github.com/google/go-github/v63/github"+ "github.com/k0kubun/go-ansi"+ "github.com/schollz/progressbar/v3""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"@@ -194,8 +196,6 @@ func getShortShaLen(knownKeySet int) int {shortShaLen := startingCharLenkeySpace := 1 << (shortShaLen * 4)collisions := estimateCollisions(keySpace, knownKeySet)- fmt.Println("Collisions: ", collisions)- fmt.Println("Collision Threshold: ", collisionThreshold)for collisions > collisionThreshold {if shortShaLen >= maxCharLen {break@@ -326,6 +326,14 @@ func processCommits(ctx context.Context, needsProcessing []string, owner, repo,startingSize := float64(len(needsProcessing))queryChunkSize := newBackoff(initialChunkSize, 10, 10, 1)++ // Initialize the progress bar for commit processing+ bar := progressbar.NewOptions(int(startingSize),+ progressbar.OptionSetDescription("[green]Processing commits[reset]"),+ progressbar.OptionSetWriter(ansi.NewAnsiStderr()),+ progressbar.OptionEnableColorCodes(true),+ )+for len(needsProcessing) > 0 {if len(needsProcessing) < queryChunkSize.getValue() {queryChunkSize.value = float64(len(needsProcessing))@@ -359,7 +367,13 @@ func processCommits(ctx context.Context, needsProcessing []string, owner, repo,if err != nil {repoCtx.Logger().V(2).Info("Failed to write invalid commits to disk", "error", err)}++ // Update the progress bar+ _ = bar.Add(chunkSize)}++ // Finish the progress bar+ _ = bar.Finish()}type commitData struct {