Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Implement SourceUnitEnumChunker for GitHub (#3298)
Miccah2f3a410e382a
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex c98031627..44335d6e3 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -84,20 +84,20 @@ var _ sources.SourceUnit = (*RepoUnit)(nil)var _ sources.SourceUnit = (*GistUnit)(nil)type RepoUnit struct {- name string- url string+ Name string `json:"name"`+ URL string `json:"url"`}-func (r RepoUnit) SourceUnitID() (string, sources.SourceUnitKind) { return r.url, "repo" }-func (r RepoUnit) Display() string { return r.name }+func (r RepoUnit) SourceUnitID() (string, sources.SourceUnitKind) { return r.URL, "repo" }+func (r RepoUnit) Display() string { return r.Name }type GistUnit struct {- name string- url string+ Name string `json:"name"`+ URL string `json:"url"`}-func (g GistUnit) SourceUnitID() (string, sources.SourceUnitKind) { return g.url, "gist" }-func (g GistUnit) Display() string { return g.name }+func (g GistUnit) SourceUnitID() (string, sources.SourceUnitKind) { return g.URL, "gist" }+func (g GistUnit) Display() string { return g.Name }// --------------------------------------------------------------------------------@@ -118,6 +118,7 @@ func (s *Source) setScanOptions(base, head string) {// Ensure the Source satisfies the interfaces at compile timevar _ sources.Source = (*Source)(nil)var _ sources.SourceUnitUnmarshaller = (*Source)(nil)+var _ sources.SourceUnitEnumChunker = (*Source)(nil)var endsWithGithub = regexp.MustCompile(`github\.com/?$`)@@ -212,6 +213,11 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sos.jobPool = &errgroup.Group{}s.jobPool.SetLimit(concurrency)+ // Setup scan options if it wasn't provided.+ if s.scanOptions == nil {+ s.scanOptions = &git.ScanOptions{}+ }+var conn sourcespb.GitHuberr = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})if err != nil {@@ -338,7 +344,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tarreturn nil},}- err := s.enumerate(ctx, noopReporter)+ err := s.Enumerate(ctx, noopReporter)if err != nil {return fmt.Errorf("error enumerating: %w", err)}@@ -346,11 +352,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tarreturn s.scan(ctx, chunksReporter)}-// enumerate enumerates the GitHub source based on authentication method and+// Enumerate enumerates the GitHub source based on authentication method and// user configuration. It populates s.filteredRepoCache, s.repoInfoCache,// s.memberCache, s.totalRepoSize, s.orgsCache, and s.repos. Additionally,// repositories and gists are reported to the provided UnitReporter.-func (s *Source) enumerate(ctx context.Context, reporter sources.UnitReporter) error {+func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {seenUnits := make(map[sources.SourceUnit]struct{})// Wrapper reporter to deduplicate and filter found units.dedupeReporter := sources.VisitorReporter{@@ -370,9 +376,19 @@ func (s *Source) enumerate(ctx context.Context, reporter sources.UnitReporter) eVisitErr: reporter.UnitErr,}// Report any values that were already configured.+ // This compensates for differences in enumeration logic between `--org` and `--repo`.+ // See: https://github.com/trufflesecurity/trufflehog/pull/2379#discussion_r1487454788for _, name := range s.filteredRepoCache.Keys() {url, _ := s.filteredRepoCache.Get(name)- _ = dedupeReporter.UnitOk(ctx, RepoUnit{name: name, url: url})+ url, err := s.ensureRepoInfoCache(ctx, url)+ if err != nil {+ if err := dedupeReporter.UnitErr(ctx, err); err != nil {+ return err+ }+ }+ if err := dedupeReporter.UnitOk(ctx, RepoUnit{Name: name, URL: url}); err != nil {+ return err+ }}// I'm not wild about switching on the connector type here (as opposed to dispatching to the connector itself) but@@ -395,55 +411,14 @@ func (s *Source) enumerate(ctx context.Context, reporter sources.UnitReporter) e}s.repos = make([]string, 0, s.filteredRepoCache.Count())-RepoLoop:+ // Double make sure that all enumerated repositories in the+ // filteredRepoCache have an entry in the repoInfoCache.for _, repo := range s.filteredRepoCache.Values() {- repoCtx := context.WithValue(ctx, "repo", repo)-- // Ensure that |s.repoInfoCache| contains an entry for |repo|.- // This compensates for differences in enumeration logic between `--org` and `--repo`.- // See: https://github.com/trufflesecurity/trufflehog/pull/2379#discussion_r1487454788- if _, ok := s.repoInfoCache.get(repo); !ok {- repoCtx.Logger().V(2).Info("Caching repository info")+ ctx := context.WithValue(ctx, "repo", repo)- _, urlParts, err := getRepoURLParts(repo)- if err != nil {- repoCtx.Logger().Error(err, "Failed to parse repository URL")- continue- }-- if isGistUrl(urlParts) {- // Cache gist info.- for {- gistID := extractGistID(urlParts)- gist, _, err := s.connector.APIClient().Gists.Get(repoCtx, gistID)- // Normalize the URL to the Gist's pull URL.- // See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937- repo = gist.GetGitPullURL()- if s.handleRateLimit(repoCtx, err) {- continue- }- if err != nil {- repoCtx.Logger().Error(err, "Failed to fetch gist")- continue RepoLoop- }- s.cacheGistInfo(gist)- break- }- } else {- // Cache repository info.- for {- ghRepo, _, err := s.connector.APIClient().Repositories.Get(repoCtx, urlParts[1], urlParts[2])- if s.handleRateLimit(repoCtx, err) {- continue- }- if err != nil {- repoCtx.Logger().Error(err, "Failed to fetch repository")- continue RepoLoop- }- s.cacheRepoInfo(ghRepo)- break- }- }+ repo, err := s.ensureRepoInfoCache(ctx, repo)+ if err != nil {+ ctx.Logger().Error(err, "error caching repo info")}s.repos = append(s.repos, repo)}@@ -454,6 +429,55 @@ RepoLoop:return nil}+// ensureRepoInfoCache checks that s.repoInfoCache has an entry for the+// provided repository URL. If not, it fetches and stores the metadata for the+// repository. In some cases, the gist URL needs to be normalized, which is+// returned by this function.+func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string) (string, error) {+ if _, ok := s.repoInfoCache.get(repo); ok {+ return repo, nil+ }+ ctx.Logger().V(2).Info("Caching repository info")++ _, urlParts, err := getRepoURLParts(repo)+ if err != nil {+ return repo, fmt.Errorf("failed to parse repository URL: %w", err)+ }++ if isGistUrl(urlParts) {+ // Cache gist info.+ for {+ gistID := extractGistID(urlParts)+ gist, _, err := s.connector.APIClient().Gists.Get(ctx, gistID)+ // Normalize the URL to the Gist's pull URL.+ // See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937+ repo = gist.GetGitPullURL()+ if s.handleRateLimit(ctx, err) {+ continue+ }+ if err != nil {+ return repo, fmt.Errorf("failed to fetch gist")+ }+ s.cacheGistInfo(gist)+ break+ }+ } else {+ // Cache repository info.+ for {+ ghRepo, _, err := s.connector.APIClient().Repositories.Get(ctx, urlParts[1], urlParts[2])+ if s.handleRateLimit(ctx, err) {+ continue+ }+ if err != nil {+ return repo, fmt.Errorf("failed to fetch repository")+ }+ s.cacheRepoInfo(ghRepo)+ break+ }+ }+ return repo, nil+}+func (s *Source) enumerateBasicAuth(ctx context.Context, reporter sources.UnitReporter) error {for _, org := range s.orgsCache.Keys() {orgCtx := context.WithValue(ctx, "account", org)@@ -603,18 +627,13 @@ func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) errorreposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)s.repos = reposToScan- scanErrs := sources.NewScanErrors()- // Setup scan options if it wasn't provided.- if s.scanOptions == nil {- s.scanOptions = &git.ScanOptions{}- }-for i, repoURL := range s.repos {i, repoURL := i, repoURLs.jobPool.Go(func() error {if common.IsDone(ctx) {return nil}+ ctx := context.WithValue(ctx, "repo", repoURL)// TODO: set progress complete is being called concurrently with is.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)@@ -625,64 +644,72 @@ func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) errors.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)}(s, repoURL)- if !strings.HasSuffix(repoURL, ".git") {- scanErrs.Add(fmt.Errorf("repo %s does not end in .git", repoURL))+ if err := s.scanRepo(ctx, repoURL, reporter); err != nil {+ ctx.Logger().Error(err, "error scanning repo")return nil}- // Scan the repository- repoInfo, ok := s.repoInfoCache.get(repoURL)- if !ok {- // This should never happen.- err := fmt.Errorf("no repoInfo for URL: %s", repoURL)- ctx.Logger().Error(err, "failed to scan repository")- return nil- }- repoCtx := context.WithValues(ctx, "repo", repoURL)- duration, err := s.cloneAndScanRepo(repoCtx, repoURL, repoInfo, reporter)- if err != nil {- scanErrs.Add(err)- return nil- }+ atomic.AddUint64(&scannedCount, 1)+ return nil+ })+ }- // Scan the wiki, if enabled, and the repo has one.- if s.conn.IncludeWikis && repoInfo.hasWiki && s.wikiIsReachable(ctx, repoURL) {- wikiURL := strings.TrimSuffix(repoURL, ".git") + ".wiki.git"- wikiCtx := context.WithValue(ctx, "repo", wikiURL)+ _ = s.jobPool.Wait()+ s.SetProgressComplete(len(s.repos), len(s.repos), "Completed GitHub scan", "")- _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)- if err != nil {- // Ignore "Repository not found" errors.- // It's common for GitHub's API to say a repo has a wiki when it doesn't.- if !strings.Contains(err.Error(), "not found") {- scanErrs.Add(fmt.Errorf("error scanning wiki: %w", err))- }+ return nil+}- // Don't return, it still might be possible to scan comments.- }- }+// scanRepo attempts to scan the provided URL and any associated wiki and+// comments if configured. An error is returned if we could not find necessary+// repository metadata or clone the repo, otherwise all errors are reported to+// the ChunkReporter.+func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.ChunkReporter) error {+ if !strings.HasSuffix(repoURL, ".git") {+ return fmt.Errorf("repo does not end in .git")+ }+ // Scan the repository+ repoInfo, ok := s.repoInfoCache.get(repoURL)+ if !ok {+ // This should never happen.+ return fmt.Errorf("no repoInfo for URL: %s", repoURL)+ }+ duration, err := s.cloneAndScanRepo(ctx, repoURL, repoInfo, reporter)+ if err != nil {+ return err+ }++ // Scan the wiki, if enabled, and the repo has one.+ if s.conn.IncludeWikis && repoInfo.hasWiki && s.wikiIsReachable(ctx, repoURL) {+ wikiURL := strings.TrimSuffix(repoURL, ".git") + ".wiki.git"+ wikiCtx := context.WithValue(ctx, "repo", wikiURL)- // Scan comments, if enabled.- if s.includeGistComments || s.includeIssueComments || s.includePRComments {- if err = s.scanComments(repoCtx, repoURL, repoInfo, reporter); err != nil {- scanErrs.Add(fmt.Errorf("error scanning comments in repo %s: %w", repoURL, err))- return nil+ _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)+ if err != nil {+ // Ignore "Repository not found" errors.+ // It's common for GitHub's API to say a repo has a wiki when it doesn't.+ if !strings.Contains(err.Error(), "not found") {+ if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil {+ return err}}- repoCtx.Logger().V(2).Info(fmt.Sprintf("scanned %d/%d repos", scannedCount, len(s.repos)), "duration_seconds", duration)- githubReposScanned.WithLabelValues(s.name).Inc()- atomic.AddUint64(&scannedCount, 1)- return nil- })+ // Don't return, it still might be possible to scan comments.+ }}- _ = s.jobPool.Wait()- if scanErrs.Count() > 0 {- ctx.Logger().Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs.String())+ // Scan comments, if enabled.+ if s.includeGistComments || s.includeIssueComments || s.includePRComments {+ if err := s.scanComments(ctx, repoURL, repoInfo, reporter); err != nil {+ err := fmt.Errorf("error scanning comments: %w", err)+ if err := reporter.ChunkErr(ctx, err); err != nil {+ return err+ }+ }}- s.SetProgressComplete(len(s.repos), len(s.repos), "Completed GitHub scan", "")+ ctx.Logger().V(2).Info("finished scanning repo", "duration_seconds", duration)+ githubReposScanned.WithLabelValues(s.name).Inc()return nil}@@ -815,7 +842,7 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporterfor _, gist := range gists {s.filteredRepoCache.Set(gist.GetID(), gist.GetGitPullURL())s.cacheGistInfo(gist)- if err := reporter.UnitOk(ctx, GistUnit{name: gist.GetID(), url: gist.GetGitPullURL()}); err != nil {+ if err := reporter.UnitOk(ctx, GistUnit{Name: gist.GetID(), URL: gist.GetGitPullURL()}); err != nil {return err}}@@ -1478,3 +1505,15 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget,Verify: s.verify}return handlers.HandleFile(ctx, readCloser, &chunkSkel, reporter)}++func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {+ repoURL, _ := unit.SourceUnitID()+ ctx = context.WithValue(ctx, "repo", repoURL)+ // ChunkUnit is not guaranteed to be called from Enumerate, so we must+ // check and fetch the repoInfoCache for this repo.+ repoURL, err := s.ensureRepoInfoCache(ctx, repoURL)+ if err != nil {+ return err+ }+ return s.scanRepo(ctx, repoURL, reporter)+}diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.goindex 35069aee5..f028af797 100644--- a/pkg/sources/github/github_integration_test.go+++ b/pkg/sources/github/github_integration_test.go@@ -843,3 +843,38 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {})}}++func TestChunkUnit(t *testing.T) {+ ctx := context.Background()+ conn, _ := anypb.New(&sourcespb.GitHub{+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},+ Credential: &sourcespb.GitHub_Unauthenticated{},+ })+ s := Source{}+ if err := s.Init(ctx, "github integration test source", 0, 0, false, conn, 1); err != nil {+ t.Errorf("Init() failed: %v", err)+ }++ unit := RepoUnit{Name: "driftwood", URL: "https://github.com/trufflesecurity/driftwood.git"}+ reporter := &countChunkReporter{}+ if err := s.ChunkUnit(ctx, unit, reporter); err != nil {+ t.Errorf("ChunkUnit() failed: %v", err)+ }+ assert.GreaterOrEqual(t, reporter.chunkCount, 65)+ assert.Equal(t, 0, reporter.errCount)+}++type countChunkReporter struct {+ chunkCount int+ errCount int+}++func (m *countChunkReporter) ChunkOk(ctx context.Context, chunk sources.Chunk) error {+ m.chunkCount+++ return nil+}++func (m *countChunkReporter) ChunkErr(ctx context.Context, err error) error {+ m.errCount+++ return nil+}diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex d659ca014..bf722fa21 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -577,7 +577,7 @@ func TestEnumerate(t *testing.T) {}// Act- err := s.enumerate(context.Background(), reporter)+ err := s.Enumerate(context.Background(), reporter)slices.Sort(reportedRepos)// Assert@@ -653,7 +653,7 @@ func BenchmarkEnumerate(b *testing.B) {setupMocks(b)b.StartTimer()- _ = s.enumerate(context.Background(), noopReporter())+ _ = s.Enumerate(context.Background(), noopReporter())}}diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.goindex 168f39c77..1f01291ea 100644--- a/pkg/sources/github/repo.go+++ b/pkg/sources/github/repo.go@@ -217,7 +217,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourcs.totalRepoSize += r.GetSize()s.filteredRepoCache.Set(repoName, repoURL)s.cacheRepoInfo(r)- if err := reporter.UnitOk(ctx, RepoUnit{name: repoName, url: repoURL}); err != nil {+ if err := reporter.UnitOk(ctx, RepoUnit{Name: repoName, URL: repoURL}); err != nil {return err}logger.V(3).Info("repo attributes", "name", repoName, "kb_size", r.GetSize(), "repo_url", repoURL)