Skip to content

Find, verify, and analyze leaked credentials

helpful logging

Dustin Deckere15fa3a5be9d
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 37ea974a0..a602e0d3d 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -68,6 +68,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
logrus.Warn("No concurrency specified, defaulting to ", numCPU)
e.concurrency = numCPU
}
+ logrus.Debugf("running with up to %d workers", e.concurrency)
var workerWg sync.WaitGroup
for i := 0; i < e.concurrency; i++ {
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index d2d996360..61f9a032b 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -17,6 +17,7 @@ import (
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v41/github"
+ "github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"golang.org/x/sync/semaphore"
@@ -300,6 +301,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
rand.Shuffle(len(s.repos), func(i, j int) { s.repos[i], s.repos[j] = s.repos[j], s.repos[i] })
}
+ scanned := 0
+
log.Debugf("Found %v total repos to scan", len(s.repos))
wg := sync.WaitGroup{}
errChan := make(chan error)
@@ -309,6 +312,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
go func(ctx context.Context, errCh chan error, repoURL string, i int) {
defer s.jobSem.Release(1)
defer wg.Done()
+
s.SetProgressComplete(i, len(s.repos), fmt.Sprintf("Repo: %s", repoURL))
if !strings.HasSuffix(repoURL, ".git") {
@@ -317,7 +321,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if strings.Contains(repoURL, "DefinitelyTyped") {
return
}
- s.log.WithField("repo", repoURL).Debug("attempting to clone repo")
+ s.log.WithField("repo", repoURL).Debugf("attempting to clone repo %d/%d", i+1, len(s.repos))
var path string
var repo *gogit.Repository
var err error
@@ -344,6 +348,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if err != nil {
log.WithError(err).Errorf("unable to scan repo, continuing")
}
+ scanned += 1
+ logrus.Debugf("scanned %d/%d repos", scanned, len(s.repos))
}(ctx, errChan, repoURL, i)
}