Skip to content

Find, verify, and analyze leaked credentials

add additional logging (#1298)

ahrava2266b4e2866
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 32d8eb044..446590d33 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -560,7 +560,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
if err := normalizeConfig(scanOptions, repo); err != nil {
return err
}
- start := time.Now().UnixNano()
+ start := time.Now().Unix()
+
if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
return err
}
@@ -576,8 +577,8 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
repoUrl = getSafeRemoteURL(repo, remotes[0].Config().Name)
}
- scanTime := time.Now().UnixNano() - start
- ctx.Logger().V(1).Info("scanning git repo complete", "Repo", repoUrl, "path", repoPath, "time", scanTime)
+ scanTime := time.Now().Unix() - start
+ ctx.Logger().V(1).Info("scanning git repo complete", "Repo", repoUrl, "path", repoPath, "time (seconds)", scanTime)
return nil
}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index c0c6fe6c0..882f31f03 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -320,6 +320,8 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
ghUser *github.User
resp *github.Response
)
+
+ ctx.Logger().V(1).Info("Enumerating with token", "endpoint", apiEndpoint)
for {
ghUser, resp, err = s.apiClient.Users.Get(context.TODO(), "")
if handled := s.handleRateLimit(err, resp); handled {
@@ -373,16 +375,10 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
logger.Error(err, "error fetching repos for user", "user", ghUser.GetLogin())
}
- // TODO: Test it actually works to list org gists like this.
- if err := s.addGistsByUser(ctx, org); err != nil {
- logger.Error(err, "error fetching gists by org")
- }
-
if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org for org")
- continue
}
}
}
@@ -392,6 +388,9 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
if err := s.addGistsByUser(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())
}
+
+ s.log.Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", len(s.orgs), "num_members", len(s.members))
+ return nil
}
if s.conn.ScanUsers {
@@ -503,14 +502,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
// We must sort the repos so we can resume later if necessary.
sort.Strings(s.repos)
- for _, err := range s.scan(ctx, installationClient, chunksChan) {
- s.log.Error(err, "error scanning repository")
- }
-
- return nil
+ return s.scan(ctx, installationClient, chunksChan)
}
-func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) []error {
+func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
var scanned uint64
s.log.V(2).Info("Found repos to scan", "count", len(s.repos))
@@ -524,7 +519,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
s.scanOptions = &git.ScanOptions{}
}
- var scanErrs []error
+ scanErrs := sources.NewScanErrors()
for i, repoURL := range s.repos {
i, repoURL := i, repoURL
s.jobPool.Go(func() error {
@@ -542,7 +537,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}(s, repoURL)
if !strings.HasSuffix(repoURL, ".git") {
- scanErrs = append(scanErrs, fmt.Errorf("repo %s does not end in .git", repoURL))
+ scanErrs.Add(fmt.Errorf("repo %s does not end in .git", repoURL))
return nil
}
@@ -553,20 +548,18 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
var err error
path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
- if err != nil {
- scanErrs = append(scanErrs, err)
- }
-
defer os.RemoveAll(path)
if err != nil {
+ scanErrs.Add(fmt.Errorf("error cloning repo %s: %w", repoURL, err))
return nil
}
s.scanOptions.BaseHash = s.conn.Base
s.scanOptions.HeadHash = s.conn.Head
+ logger.V(2).Info(fmt.Sprintf("scanning repo %d/%d", i, len(s.repos)))
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
- logger.Error(err, "unable to scan repo, continuing")
+ scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
return nil
}
atomic.AddUint64(&scanned, 1)
@@ -577,11 +570,12 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}
_ = s.jobPool.Wait()
- if len(scanErrs) == 0 {
- s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")
+ if scanErrs.Count() > 0 {
+ s.log.V(2).Info("Errors encountered while scanning", "error-count", scanErrs.Count(), "errors", scanErrs)
}
+ s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Github scan", "")
- return scanErrs
+ return nil
}
func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClient *github.Client) (string, *gogit.Repository, error) {
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 9e292baea..afbeb2c0d 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -498,9 +498,10 @@ func Test_scan_SetProgressComplete(t *testing.T) {
wantComplete: true,
},
{
- name: "one valid repo",
- repos: []string{"a"},
- wantErr: true,
+ name: "one valid repo",
+ repos: []string{"a"},
+ wantComplete: true,
+ wantErr: true,
},
}