Skip to content

Find, verify, and analyze leaked credentials

fix(github): scan user repos (#2814)

Richard Gomez1441289d410e
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 782f296f8..4571224a0 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -15,12 +15,12 @@ import (
"time"
"golang.org/x/exp/rand"
+ "golang.org/x/oauth2"
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-logr/logr"
"github.com/gobwas/glob"
"github.com/google/go-github/v62/github"
- "golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -480,8 +480,17 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
s.apiClient = ghClient
for _, org := range s.orgsCache.Keys() {
- if err := s.getReposByOrg(ctx, org); err != nil {
- s.log.Error(err, "error fetching repos for org or user")
+ orgCtx := context.WithValue(ctx, "account", org)
+ userType, err := s.getReposByOrgOrUser(ctx, org)
+ if err != nil {
+ orgCtx.Logger().Error(err, "error fetching repos for org or user")
+ continue
+ }
+
+ if userType == organization && s.conn.ScanUsers {
+ if err := s.addMembersByOrg(ctx, org); err != nil {
+ orgCtx.Logger().Error(err, "Unable to add members by org")
+ }
}
}
@@ -499,17 +508,15 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
}
for _, org := range s.orgsCache.Keys() {
- if err := s.getReposByOrg(ctx, org); err != nil {
- s.log.Error(err, "error fetching repos for org")
- }
-
- // We probably don't need to do this, since getting repos by org makes more sense?
- if err := s.getReposByUser(ctx, org); err != nil {
- s.log.Error(err, "error fetching repos for user")
+ orgCtx := context.WithValue(ctx, "account", org)
+ userType, err := s.getReposByOrgOrUser(ctx, org)
+ if err != nil {
+ orgCtx.Logger().Error(err, "error fetching repos for org or user")
+ continue
}
- if s.conn.ScanUsers {
- s.log.Info("Enumerating unauthenticated does not support scanning organization members")
+ if userType == organization && s.conn.ScanUsers {
+ orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)")
}
}
}
@@ -562,16 +569,16 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
if s.orgsCache.Count() > 0 {
specificScope = true
for _, org := range s.orgsCache.Keys() {
- logger := s.log.WithValues("org", org)
- if err := s.getReposByOrg(ctx, org); err != nil {
- logger.Error(err, "error fetching repos for org")
+ orgCtx := context.WithValue(ctx, "account", org)
+ userType, err := s.getReposByOrgOrUser(ctx, org)
+ if err != nil {
+ orgCtx.Logger().Error(err, "error fetching repos for org or user")
+ continue
}
- if s.conn.ScanUsers {
- err := s.addMembersByOrg(ctx, org)
- if err != nil {
- logger.Error(err, "Unable to add members by org")
- continue
+ if userType == organization && s.conn.ScanUsers {
+ if err := s.addMembersByOrg(ctx, org); err != nil {
+ orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}
@@ -593,27 +600,28 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
}
for _, org := range s.orgsCache.Keys() {
- logger := s.log.WithValues("org", org)
- if err := s.getReposByOrg(ctx, org); err != nil {
- logger.Error(err, "error fetching repos by org")
- }
-
- if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
- logger.Error(err, "error fetching repos by user")
+ orgCtx := context.WithValue(ctx, "account", org)
+ userType, err := s.getReposByOrgOrUser(ctx, org)
+ if err != nil {
+ orgCtx.Logger().Error(err, "error fetching repos for org or user")
+ continue
}
- if s.conn.ScanUsers {
- err := s.addMembersByOrg(ctx, org)
- if err != nil {
- logger.Error(err, "Unable to add members by org for org")
+ if userType == organization && s.conn.ScanUsers {
+ if err := s.addMembersByOrg(ctx, org); err != nil {
+ orgCtx.Logger().Error(err, "Unable to add members by org for org")
}
}
}
+ if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
+ s.log.Error(err, "error fetching repos for the current user", "user", ghUser.GetLogin())
+ }
+
// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
// So if we don't have ScanUsers enabled, add the user gists as normal.
if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil {
- s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())
+ s.log.Error(err, "error fetching gists for the current user", "user", ghUser.GetLogin())
}
return nil
@@ -693,7 +701,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
s.log.Info("Scanning repos", "org_members", len(s.memberCache))
for member := range s.memberCache {
logger := s.log.WithValues("member", member)
- if err := s.getReposByUser(ctx, member); err != nil {
+ if err := s.addUserGistsToCache(ctx, member); err != nil {
logger.Error(err, "error fetching gists by user")
}
if err := s.getReposByUser(ctx, member); err != nil {
diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go
index 35a1f42b2..f1eb88ce5 100644
--- a/pkg/sources/github/repo.go
+++ b/pkg/sources/github/repo.go
@@ -1,6 +1,7 @@
package github
import (
+ "errors"
"fmt"
"io"
"net/http"
@@ -169,14 +170,6 @@ func (a *appListOptions) getListOptions() *github.ListOptions {
return &a.ListOptions
}
-func (s *Source) getReposByApp(ctx context.Context) error {
- return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
- ListOptions: github.ListOptions{
- PerPage: defaultPagination,
- },
- })
-}
-
func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())
if someRepos != nil {
@@ -185,6 +178,14 @@ func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoLis
return nil, res, err
}
+func (s *Source) getReposByApp(ctx context.Context) error {
+ return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
+ ListOptions: github.ListOptions{
+ PerPage: defaultPagination,
+ },
+ })
+}
+
type userListOptions struct {
github.RepositoryListByUserOptions
}
@@ -193,6 +194,10 @@ func (u *userListOptions) getListOptions() *github.ListOptions {
return &u.ListOptions
}
+func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
+ return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
+}
+
func (s *Source) getReposByUser(ctx context.Context, user string) error {
return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{
RepositoryListByUserOptions: github.RepositoryListByUserOptions{
@@ -203,10 +208,6 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {
})
}
-func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
- return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
-}
-
type orgListOptions struct {
github.RepositoryListByOrgOptions
}
@@ -215,6 +216,10 @@ func (o *orgListOptions) getListOptions() *github.ListOptions {
return &o.ListOptions
}
+func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
+ return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
+}
+
func (s *Source) getReposByOrg(ctx context.Context, org string) error {
return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{
RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{
@@ -225,8 +230,55 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {
})
}
-func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
- return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
+// userType indicates whether an account belongs to a person or organization.
+//
+// See:
+// - https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts
+// - https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user
+type userType int
+
+const (
+ // Default invalid state.
+ unknown userType = iota
+ // The account is a person (https://docs.github.com/en/rest/users/users).
+ user
+ // The account is an organization (https://docs.github.com/en/rest/orgs/orgs).
+ organization
+)
+
+func (s *Source) getReposByOrgOrUser(ctx context.Context, name string) (userType, error) {
+ var err error
+
+ // List repositories for the organization |name|.
+ err = s.getReposByOrg(ctx, name)
+ if err == nil {
+ return organization, nil
+ } else if !isGitHub404Error(err) {
+ return unknown, err
+ }
+
+ // List repositories for the user |name|.
+ err = s.getReposByUser(ctx, name)
+ if err == nil {
+ if err := s.addUserGistsToCache(ctx, name); err != nil {
+ ctx.Logger().Error(err, "Unable to add user to cache")
+ }
+ return user, nil
+ } else if !isGitHub404Error(err) {
+ return unknown, err
+ }
+
+ return unknown, fmt.Errorf("account '%s' not found", name)
+}
+
+// isGitHub404Error returns true if |err| is a `github.ErrorResponse` and has the status code `404`.
+func isGitHub404Error(err error) bool {
+ var ghErr *github.ErrorResponse
+ if !errors.As(err, &ghErr) {
+ return false
+ }
+
+ return ghErr.Response.StatusCode == http.StatusNotFound
}
func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {