Skip to content

Find, verify, and analyze leaked credentials

Improve GitHub scan logging (#2220)

Richard Gomez38eb5d08e71d
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 1035c34d3..d44b5e6ab 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -737,7 +737,7 @@ func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Cl
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
- var scanned uint64
+ var scannedCount uint64
s.log.V(2).Info("Found repos to scan", "count", len(s.repos))
@@ -773,10 +773,12 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}
logger := s.log.WithValues("repo", repoURL)
- logger.V(2).Info(fmt.Sprintf("attempting to clone repo %d/%d", i+1, len(s.repos)))
- var path string
- var repo *gogit.Repository
- var err error
+ logger.V(2).Info("attempting to clone repo")
+ var (
+ path string
+ repo *gogit.Repository
+ err error
+ )
path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
if err != nil {
@@ -793,34 +795,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
s.setScanOptions(s.conn.Base, s.conn.Head)
repoSize := s.repoSizes.getRepo(repoURL)
- logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)), "repo_size_kb", repoSize)
+ logger.V(2).Info("scanning repo", "repo_size_kb", repoSize)
now := time.Now()
- defer func(start time.Time) {
- logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
- }(now)
+ scanFailed := false
+ defer func(start time.Time, failed *bool) {
+ if *failed {
+ return
+ }
+ logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scannedCount, len(s.repos)), "duration_seconds", time.Since(start).Seconds())
+ }(now, &scanFailed)
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, sources.ChanReporter{Ch: chunksChan}); err != nil {
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
+ scanFailed = true
return nil
}
- githubReposScanned.WithLabelValues(s.name).Inc()
-
if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))
+ scanFailed = true
return nil
}
- atomic.AddUint64(&scanned, 1)
-
+ githubReposScanned.WithLabelValues(s.name).Inc()
+ atomic.AddUint64(&scannedCount, 1)
return nil
})
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
- s.log.V(2).Info("Errors encountered while scanning", "error-count", scanErrs.Count(), "errors", scanErrs)
+ s.log.V(0).Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs)
}
s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")