Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
add additional logging (#1298)
ahrava2266b4e2866
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 32d8eb044..446590d33 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -560,7 +560,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strinif err := normalizeConfig(scanOptions, repo); err != nil {return err}- start := time.Now().UnixNano()+ start := time.Now().Unix()+if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {return err}@@ -576,8 +577,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strinrepoUrl = getSafeRemoteURL(repo, remotes[0].Config().Name)}- scanTime := time.Now().UnixNano() - start- ctx.Logger().V(1).Info("scanning git repo complete", "Repo", repoUrl, "path", repoPath, "time", scanTime)+ scanTime := time.Now().Unix() - start+ ctx.Logger().V(1).Info("scanning git repo complete", "Repo", repoUrl, "path", repoPath, "time (seconds)", scanTime)return nil}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex c0c6fe6c0..882f31f03 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -320,6 +320,8 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token strighUser *github.Userresp *github.Response)++ ctx.Logger().V(1).Info("Enumerating with token", "endpoint", apiEndpoint)for {ghUser, resp, err = s.apiClient.Users.Get(context.TODO(), "")if handled := s.handleRateLimit(err, resp); handled {@@ -373,16 +375,10 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token strilogger.Error(err, "error fetching repos for user", "user", ghUser.GetLogin())}- // TODO: Test it actually works to list org gists like this.- if err := s.addGistsByUser(ctx, org); err != nil {- logger.Error(err, "error fetching gists by org")- }-if s.conn.ScanUsers {err := s.addMembersByOrg(ctx, org)if err != nil {logger.Error(err, "Unable to add members by org for org")- continue}}}@@ -392,6 +388,9 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token striif err := s.addGistsByUser(ctx, ghUser.GetLogin()); err != nil {s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())}++ s.log.Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", len(s.orgs), "num_members", len(s.members))+ return nil}if s.conn.ScanUsers {@@ -503,14 +502,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err// We must sort the repos so we can resume later if necessary.sort.Strings(s.repos)- for _, err := range s.scan(ctx, installationClient, chunksChan) {- s.log.Error(err, "error scanning repository")- }-- return nil+ return s.scan(ctx, installationClient, chunksChan)}-func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) []error {+func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {var scanned uint64s.log.V(2).Info("Found repos to scan", "count", len(s.repos))@@ -524,7 +519,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chs.scanOptions = &git.ScanOptions{}}- var scanErrs []error+ scanErrs := sources.NewScanErrors()for i, repoURL := range s.repos {i, repoURL := i, repoURLs.jobPool.Go(func() error {@@ -542,7 +537,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch}(s, repoURL)if !strings.HasSuffix(repoURL, ".git") {- scanErrs = append(scanErrs, fmt.Errorf("repo %s does not end in .git", repoURL))+ scanErrs.Add(fmt.Errorf("repo %s does not end in .git", repoURL))return nil}@@ -553,20 +548,18 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chvar err errorpath, repo, err = s.cloneRepo(ctx, repoURL, installationClient)- if err != nil {- scanErrs = append(scanErrs, err)- }-defer os.RemoveAll(path)if err != nil {+ scanErrs.Add(fmt.Errorf("error cloning repo %s: %w", repoURL, err))return nil}s.scanOptions.BaseHash = s.conn.Bases.scanOptions.HeadHash = s.conn.Head+ logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)))if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {- logger.Error(err, "unable to scan repo, continuing")+ scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))return nil}atomic.AddUint64(&scanned, 1)@@ -577,11 +570,12 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch}_ = s.jobPool.Wait()- if len(scanErrs) == 0 {- s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")+ if scanErrs.Count() > 0 {+ s.log.V(2).Info("Errors encountered while scanning", "error-count", scanErrs.Count(), "errors", scanErrs)}+ s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")- return scanErrs+ return nil}func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClient *github.Client) (string, *gogit.Repository, error) {diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 9e292baea..afbeb2c0d 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -498,9 +498,10 @@ func Test_scan_SetProgressComplete(t *testing.T) {wantComplete: true,},{- name: "one valid repo",- repos: []string{"a"},- wantErr: true,+ name: "one valid repo",+ repos: []string{"a"},+ wantComplete: true,+ wantErr: true,},}