Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improve GitHub scan logging (#2220)
Richard Gomez38eb5d08e71d
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 1035c34d3..d44b5e6ab 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -737,7 +737,7 @@ func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Cl}func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {- var scanned uint64+ var scannedCount uint64s.log.V(2).Info("Found repos to scan", "count", len(s.repos))@@ -773,10 +773,12 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch}logger := s.log.WithValues("repo", repoURL)- logger.V(2).Info(fmt.Sprintf("attempting to clone repo %d/%d", i+1, len(s.repos)))- var path string- var repo *gogit.Repository- var err error+ logger.V(2).Info("attempting to clone repo")+ var (+ path string+ repo *gogit.Repository+ err error+ )path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)if err != nil {@@ -793,34 +795,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chs.setScanOptions(s.conn.Base, s.conn.Head)repoSize := s.repoSizes.getRepo(repoURL)- logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)), "repo_size_kb", repoSize)+ logger.V(2).Info("scanning repo", "repo_size_kb", repoSize)now := time.Now()- defer func(start time.Time) {- logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())- }(now)+ scanFailed := false+ defer func(start time.Time, failed *bool) {+ if *failed {+ return+ }+ logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scannedCount, len(s.repos)), "duration_seconds", time.Since(start).Seconds())+ }(now, &scanFailed)if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, sources.ChanReporter{Ch: chunksChan}); err != nil {scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))+ scanFailed = truereturn nil}- githubReposScanned.WithLabelValues(s.name).Inc()-if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))+ scanFailed = truereturn nil}- atomic.AddUint64(&scanned, 1)-+ githubReposScanned.WithLabelValues(s.name).Inc()+ atomic.AddUint64(&scannedCount, 1)return nil})}_ = s.jobPool.Wait()if scanErrs.Count() > 0 {- s.log.V(2).Info("Errors encountered while scanning", "error-count", scanErrs.Count(), "errors", scanErrs)+ s.log.V(0).Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs)}s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")