Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix(github): scan user repos (#2814)
Richard Gomez1441289d410e
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 782f296f8..4571224a0 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -15,12 +15,12 @@ import ("time""golang.org/x/exp/rand"+ "golang.org/x/oauth2""github.com/bradleyfalzon/ghinstallation/v2""github.com/go-logr/logr""github.com/gobwas/glob""github.com/google/go-github/v62/github"- "golang.org/x/oauth2""golang.org/x/sync/errgroup""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -480,8 +480,17 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bass.apiClient = ghClientfor _, org := range s.orgsCache.Keys() {- if err := s.getReposByOrg(ctx, org); err != nil {- s.log.Error(err, "error fetching repos for org or user")+ orgCtx := context.WithValue(ctx, "account", org)+ userType, err := s.getReposByOrgOrUser(ctx, org)+ if err != nil {+ orgCtx.Logger().Error(err, "error fetching repos for org or user")+ continue+ }++ if userType == organization && s.conn.ScanUsers {+ if err := s.addMembersByOrg(ctx, org); err != nil {+ orgCtx.Logger().Error(err, "Unable to add members by org")+ }}}@@ -499,17 +508,15 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin}for _, org := range s.orgsCache.Keys() {- if err := s.getReposByOrg(ctx, org); err != nil {- s.log.Error(err, "error fetching repos for org")- }-- // We probably don't need to do this, since getting repos by org makes more sense?- if err := s.getReposByUser(ctx, org); err != nil {- s.log.Error(err, "error fetching repos for user")+ orgCtx := context.WithValue(ctx, "account", org)+ userType, err := s.getReposByOrgOrUser(ctx, org)+ if err != nil {+ orgCtx.Logger().Error(err, "error fetching repos for org or user")+ continue}- if s.conn.ScanUsers {- s.log.Info("Enumerating unauthenticated does not support scanning organization members")+ if userType == organization && s.conn.ScanUsers {+ orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)")}}}@@ -562,16 +569,16 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token striif s.orgsCache.Count() > 0 {specificScope = truefor _, org := range s.orgsCache.Keys() {- logger := s.log.WithValues("org", org)- if err := s.getReposByOrg(ctx, org); err != nil {- logger.Error(err, "error fetching repos for org")+ orgCtx := context.WithValue(ctx, "account", org)+ userType, err := s.getReposByOrgOrUser(ctx, org)+ if err != nil {+ orgCtx.Logger().Error(err, "error fetching repos for org or user")+ continue}- if s.conn.ScanUsers {- err := s.addMembersByOrg(ctx, org)- if err != nil {- logger.Error(err, "Unable to add members by org")- continue+ if userType == organization && s.conn.ScanUsers {+ if err := s.addMembersByOrg(ctx, org); err != nil {+ orgCtx.Logger().Error(err, "Unable to add members by org")}}}@@ -593,27 +600,28 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri}for _, org := range s.orgsCache.Keys() {- logger := s.log.WithValues("org", org)- if err := s.getReposByOrg(ctx, org); err != nil {- logger.Error(err, "error fetching repos by org")- }-- if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {- logger.Error(err, "error fetching repos by user")+ orgCtx := context.WithValue(ctx, "account", org)+ userType, err := s.getReposByOrgOrUser(ctx, org)+ if err != nil {+ orgCtx.Logger().Error(err, "error fetching repos for org or user")+ continue}- if s.conn.ScanUsers {- err := s.addMembersByOrg(ctx, org)- if err != nil {- logger.Error(err, "Unable to add members by org for org")+ if userType == organization && s.conn.ScanUsers {+ if err := s.addMembersByOrg(ctx, org); err != nil {+ orgCtx.Logger().Error(err, "Unable to add members by org for org")}}}+ if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {+ s.log.Error(err, "error fetching repos for the current user", "user", ghUser.GetLogin())+ }+// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.// So if we don't have ScanUsers enabled, add the user gists as normal.if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil {- s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())+ s.log.Error(err, "error fetching gists for the current user", "user", ghUser.GetLogin())}return nil@@ -693,7 +701,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *s.log.Info("Scanning repos", "org_members", len(s.memberCache))for member := range s.memberCache {logger := s.log.WithValues("member", member)- if err := s.getReposByUser(ctx, member); err != nil {+ if err := s.addUserGistsToCache(ctx, member); err != nil {logger.Error(err, "error fetching gists by user")}if err := s.getReposByUser(ctx, member); err != nil {diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.goindex 35a1f42b2..f1eb88ce5 100644--- a/pkg/sources/github/repo.go+++ b/pkg/sources/github/repo.go@@ -1,6 +1,7 @@package githubimport (+ "errors""fmt""io""net/http"@@ -169,14 +170,6 @@ func (a *appListOptions) getListOptions() *github.ListOptions {return &a.ListOptions}-func (s *Source) getReposByApp(ctx context.Context) error {- return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{- ListOptions: github.ListOptions{- PerPage: defaultPagination,- },- })-}-func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())if someRepos != nil {@@ -185,6 +178,14 @@ func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoLisreturn nil, res, err}+func (s *Source) getReposByApp(ctx context.Context) error {+ return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{+ ListOptions: github.ListOptions{+ PerPage: defaultPagination,+ },+ })+}+type userListOptions struct {github.RepositoryListByUserOptions}@@ -193,6 +194,10 @@ func (u *userListOptions) getListOptions() *github.ListOptions {return &u.ListOptions}+func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {+ return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)+}+func (s *Source) getReposByUser(ctx context.Context, user string) error {return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{RepositoryListByUserOptions: github.RepositoryListByUserOptions{@@ -203,10 +208,6 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {})}-func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {- return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)-}-type orgListOptions struct {github.RepositoryListByOrgOptions}@@ -215,6 +216,10 @@ func (o *orgListOptions) getListOptions() *github.ListOptions {return &o.ListOptions}+func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {+ return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)+}+func (s *Source) getReposByOrg(ctx context.Context, org string) error {return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{@@ -225,8 +230,55 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {})}-func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {- return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)+// userType indicates whether an account belongs to a person or organization.+//+// See:+// - https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts+// - https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user+type userType int++const (+ // Default invalid state.+ unknown userType = iota+ // The account is a person (https://docs.github.com/en/rest/users/users).+ user+ // The account is an organization (https://docs.github.com/en/rest/orgs/orgs).+ organization+)++func (s *Source) getReposByOrgOrUser(ctx context.Context, name string) (userType, error) {+ var err error++ // List repositories for the organization |name|.+ err = s.getReposByOrg(ctx, name)+ if err == nil {+ return organization, nil+ } else if !isGitHub404Error(err) {+ return unknown, err+ }++ // List repositories for the user |name|.+ err = s.getReposByUser(ctx, name)+ if err == nil {+ if err := s.addUserGistsToCache(ctx, name); err != nil {+ ctx.Logger().Error(err, "Unable to add user to cache")+ }+ return user, nil+ } else if !isGitHub404Error(err) {+ return unknown, err+ }++ return unknown, fmt.Errorf("account '%s' not found", name)+}++// isGitHub404Error returns true if |err| is a `github.ErrorResponse` and has the status code `404`.+func isGitHub404Error(err error) bool {+ var ghErr *github.ErrorResponse+ if !errors.As(err, &ghErr) {+ return false+ }++ return ghErr.Response.StatusCode == http.StatusNotFound}func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {