Skip to content

Find, verify, and analyze leaked credentials

Add progress bar to CFOR (#3151)

Dustin Decker05e4635824a2
diff --git a/go.mod b/go.mod
index 6183bf731..790541f51 100644
--- a/go.mod
+++ b/go.mod
@@ -64,6 +64,7 @@ require (
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
github.com/jpillora/overseer v1.1.6
+ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
github.com/kylelemons/godebug v1.1.0
github.com/lib/pq v1.10.9
github.com/lrstanley/bubblezone v0.0.0-20240125042004-b7bafc493195
@@ -79,6 +80,7 @@ require (
github.com/prometheus/client_golang v1.19.1
github.com/rabbitmq/amqp091-go v1.10.0
github.com/sassoftware/go-rpmutils v0.4.0
+ github.com/schollz/progressbar/v3 v3.14.5
github.com/sendgrid/sendgrid-go v3.14.0+incompatible
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3
github.com/shuheiktgw/go-travis v0.3.1
@@ -235,6 +237,7 @@ require (
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/microcosm-cc/bluemonday v1.0.25 // indirect
+ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
diff --git a/go.sum b/go.sum
index aa2c62a8e..88db0c2e1 100644
--- a/go.sum
+++ b/go.sum
@@ -491,6 +491,8 @@ github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -550,6 +552,8 @@ github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJ
github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE=
github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
@@ -655,6 +659,8 @@ github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f h1:MvTmaQdww/z0Q4wr
github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg=
github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
+github.com/schollz/progressbar/v3 v3.14.5 h1:97RrSxbBASxQuZN9yemnyGrFZ/swnG6IrEe2R0BseX8=
+github.com/schollz/progressbar/v3 v3.14.5/go.mod h1:Nrzpuw3Nl0srLY0VlTvC4V6RL50pcEymjy6qyJAaLa0=
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE=
github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fcif2i7P7wzISv1sU6DUA=
diff --git a/pkg/sources/github_experimental/github_experimental.go b/pkg/sources/github_experimental/github_experimental.go
index 1f5fbb222..fecf5c273 100644
--- a/pkg/sources/github_experimental/github_experimental.go
+++ b/pkg/sources/github_experimental/github_experimental.go
@@ -7,7 +7,6 @@ import (
"github.com/go-logr/logr"
"github.com/google/go-github/v63/github"
- "golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -23,40 +22,21 @@ import (
const (
SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL
-
- // unauthGithubOrgRateLimt = 30
- // defaultPagination = 100
- // membersAppPagination = 500
)
type Source struct {
- name string
-
- // Protects the user and token.
- //userMu sync.Mutex
- //githubUser string
- //githubToken string
-
- sourceID sources.SourceID
- jobID sources.JobID
- verify bool
- //orgsCache cache.Cache[string]
- //memberCache map[string]struct{}
- //repos []string
- //filteredRepoCache *filteredRepoCache
- repoInfoCache repoInfoCache
- //totalRepoSize int // total size of all repos in kb
-
+ name string
+ sourceID sources.SourceID
+ jobID sources.JobID
+ verify bool
+ repoInfoCache repoInfoCache
useCustomContentWriter bool
git *git.Git
-
- scanOptions *git.ScanOptions
-
- httpClient *http.Client
- log logr.Logger
- conn *sourcespb.GitHubExperimental
- jobPool *errgroup.Group
- apiClient *github.Client
+ scanOptions *git.ScanOptions
+ httpClient *http.Client
+ log logr.Logger
+ conn *sourcespb.GitHubExperimental
+ apiClient *github.Client
sources.Progress
sources.CommonSourceUnitUnmarshaller
@@ -100,8 +80,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
s.sourceID = sourceID
s.jobID = jobID
s.verify = verify
- s.jobPool = &errgroup.Group{}
- s.jobPool.SetLimit(concurrency)
s.httpClient = common.RetryableHTTPClientTimeout(60)
s.apiClient = github.NewClient(s.httpClient)
diff --git a/pkg/sources/github_experimental/object_discovery.go b/pkg/sources/github_experimental/object_discovery.go
index 9af113358..7651ad2fb 100644
--- a/pkg/sources/github_experimental/object_discovery.go
+++ b/pkg/sources/github_experimental/object_discovery.go
@@ -12,6 +12,8 @@ import (
"time"
"github.com/google/go-github/v63/github"
+ "github.com/k0kubun/go-ansi"
+ "github.com/schollz/progressbar/v3"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -194,8 +196,6 @@ func getShortShaLen(knownKeySet int) int {
shortShaLen := startingCharLen
keySpace := 1 << (shortShaLen * 4)
collisions := estimateCollisions(keySpace, knownKeySet)
- fmt.Println("Collisions: ", collisions)
- fmt.Println("Collision Threshold: ", collisionThreshold)
for collisions > collisionThreshold {
if shortShaLen >= maxCharLen {
break
@@ -326,6 +326,14 @@ func processCommits(ctx context.Context, needsProcessing []string, owner, repo,
startingSize := float64(len(needsProcessing))
queryChunkSize := newBackoff(initialChunkSize, 10, 10, 1)
+
+ // Initialize the progress bar for commit processing
+ bar := progressbar.NewOptions(int(startingSize),
+ progressbar.OptionSetDescription("[green]Processing commits[reset]"),
+ progressbar.OptionSetWriter(ansi.NewAnsiStderr()),
+ progressbar.OptionEnableColorCodes(true),
+ )
+
for len(needsProcessing) > 0 {
if len(needsProcessing) < queryChunkSize.getValue() {
queryChunkSize.value = float64(len(needsProcessing))
@@ -359,7 +367,13 @@ func processCommits(ctx context.Context, needsProcessing []string, owner, repo,
if err != nil {
repoCtx.Logger().V(2).Info("Failed to write invalid commits to disk", "error", err)
}
+
+ // Update the progress bar
+ _ = bar.Add(chunkSize)
}
+
+ // Finish the progress bar
+ _ = bar.Finish()
}
type commitData struct {