Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Move GitHub integration tests behind a build flag and add unit tests (#527)
Miccahedaf1e1fd3d2
diff --git a/go.mod b/go.modindex a3ceb04fe..28d1b1a49 100644--- a/go.mod+++ b/go.mod@@ -31,13 +31,13 @@ require (github.com/joho/godotenv v1.4.0github.com/jpillora/overseer v1.1.6github.com/kylelemons/godebug v1.1.0- github.com/mattn/go-colorable v0.1.12github.com/paulbellamy/ratecounter v0.2.0github.com/pkg/errors v0.9.1github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2github.com/rs/zerolog v1.26.1github.com/sergi/go-diff v1.2.0github.com/sirupsen/logrus v1.8.1+ github.com/stretchr/testify v1.7.0github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502github.com/xanzy/go-gitlab v0.64.0github.com/zricethezav/gitleaks/v8 v8.5.2@@ -48,6 +48,7 @@ require (google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bfgoogle.golang.org/protobuf v1.28.0gopkg.in/alecthomas/kingpin.v2 v2.2.6+ gopkg.in/h2non/gock.v1 v1.1.2)require (@@ -71,6 +72,7 @@ require (github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirectgithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirectgithub.com/aws/smithy-go v1.11.2 // indirect+ github.com/davecgh/go-spew v1.1.1 // indirectgithub.com/dimchansky/utfbom v1.1.1 // indirectgithub.com/emirpasic/gods v1.12.0 // indirectgithub.com/go-git/gcfg v1.5.0 // indirect@@ -84,6 +86,7 @@ require (github.com/google/go-querystring v1.1.0 // indirectgithub.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirectgithub.com/googleapis/gax-go/v2 v2.2.0 // indirect+ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirectgithub.com/hashicorp/go-cleanhttp v0.5.1 // indirectgithub.com/imdario/mergo v0.3.12 // indirectgithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect@@ -91,11 +94,13 @@ require (github.com/jpillora/s3 v1.1.4 // indirectgithub.com/json-iterator/go v1.1.11 // indirectgithub.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect+ github.com/mattn/go-colorable v0.1.12 // indirectgithub.com/mattn/go-isatty v0.0.14 // indirectgithub.com/mitchellh/go-homedir v1.1.0 // indirectgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirectgithub.com/modern-go/reflect2 v1.0.1 // indirectgithub.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect+ github.com/pmezard/go-difflib v1.0.0 // indirectgithub.com/xanzy/ssh-agent v0.3.0 // indirectgithub.com/yusufpapurcu/wmi v1.2.2 // indirectgo.opencensus.io v0.23.0 // indirect@@ -110,4 +115,5 @@ require (google.golang.org/grpc v1.45.0 // indirectgopkg.in/ini.v1 v1.66.2 // indirectgopkg.in/warnings.v0 v0.1.2 // indirect+ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect)diff --git a/go.sum b/go.sumindex 3e6fa7ee4..88770615c 100644--- a/go.sum+++ b/go.sum@@ -296,6 +296,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adgithub.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=@@ -391,6 +393,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJgithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=@@ -930,6 +934,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=+gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=+gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 8c20e8a63..6e7055947 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -11,13 +11,13 @@ import ("strconv""strings""sync"+ "sync/atomic""time""github.com/bradleyfalzon/ghinstallation/v2""github.com/go-errors/errors"gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v42/github"- "github.com/sirupsen/logrus"log "github.com/sirupsen/logrus""golang.org/x/oauth2""golang.org/x/sync/semaphore"@@ -37,8 +37,8 @@ import (type Source struct {name string- sourceId int64- jobId int64+ sourceID int64+ jobID int64verify boolrepos []stringorgs []string@@ -64,11 +64,11 @@ func (s *Source) Type() sourcespb.SourceType {}func (s *Source) SourceID() int64 {- return s.sourceId+ return s.sourceID}func (s *Source) JobID() int64 {- return s.jobId+ return s.jobID}func (s *Source) Token(ctx context.Context, installationClient *github.Client) (string, error) {@@ -94,13 +94,13 @@ func (s *Source) Token(ctx context.Context, installationClient *github.Client) (}// Init returns an initialized GitHub source.-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {+func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {s.log = log.WithField("source", s.Type()).WithField("name", name)s.aCtx = aCtxs.name = name- s.sourceId = sourceId- s.jobId = jobId+ s.sourceID = sourceID+ s.jobID = jobIDs.verify = verifys.jobSem = semaphore.NewWeighted(int64(concurrency))@@ -333,16 +333,27 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {- scanned := 0+ var scanned uint64log.Debugf("Found %v total repos to scan", len(s.repos))wg := sync.WaitGroup{}- errs := []error{}- var errsMut sync.Mutex+ errs := make(chan error, 1)+ reportErr := func(err error) {+ // save the error if there's room, otherwise log and drop it+ select {+ case errs <- err:+ default:+ log.WithError(err).Warn("dropping error")+ }+ }+for i, repoURL := range s.repos {if err := s.jobSem.Acquire(ctx, 1); err != nil {+ // Acquire blocks until it can acquire the semaphore or returns an+ // error if the context is finishedlog.WithError(err).Debug("could not acquire semaphore")- continue+ reportErr(err)+ break}wg.Add(1)go func(ctx context.Context, repoURL string, i int) {@@ -367,10 +378,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chvar token stringtoken, err = s.Token(ctx, installationClient)if err != nil {- // TODO: maybe we can use a channel here- errsMut.Lock()- errs = append(errs, err)- errsMut.Unlock()+ reportErr(err)return}path, repo, err = git.CloneRepoUsingToken(token, repoURL, "clone")@@ -391,20 +399,20 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chif err != nil {log.WithError(err).Errorf("unable to scan repo, continuing")}- // TODO: use atomic library- scanned++- logrus.Debugf("scanned %d/%d repos", scanned, len(s.repos))+ atomic.AddUint64(&scanned, 1)+ log.Debugf("scanned %d/%d repos", scanned, len(s.repos))}(ctx, repoURL, i)}wg.Wait()// This only returns first error which is what we did prior to concurrency- if len(errs) > 0 {- return errs[0]+ select {+ case err := <-errs:+ return err+ default:+ return nil}-- return nil}// handleRateLimit returns true if a rate limit was handled@@ -443,7 +451,8 @@ func handleRateLimit(errIn error, res *github.Response) bool {return true}-func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error {+func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, org string) ([]string, error) {+ repos := []string{}opts := &github.RepositoryListByOrgOptions{ListOptions: github.ListOptions{PerPage: 100,@@ -459,7 +468,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, orcontinue}if err != nil {- return fmt.Errorf("could not list repos for org %s: %w", org, err)+ return nil, fmt.Errorf("could not list repos for org %s: %w", org, err)}if len(someRepos) == 0 {break@@ -472,7 +481,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, orcontinue}}- common.AddStringSliceItem(r.GetCloneURL(), &s.repos)+ repos = append(repos, r.GetCloneURL())}if res.NextPage == 0 {break@@ -480,10 +489,23 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, oropts.Page = res.NextPage}log.WithField("org", org).Debugf("Found %d repos (%d forks)", numRepos, numForks)+ return repos, nil+}++func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error {+ repos, err := s.getReposByOrg(ctx, apiClient, org)+ if err != nil {+ return err+ }+ // add the repos to the set of repos+ for _, repo := range repos {+ common.AddStringSliceItem(repo, &s.repos)+ }return nil}-func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error {+func (s *Source) getReposByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) {+ repos := []string{}opts := &github.RepositoryListOptions{ListOptions: github.ListOptions{PerPage: 50,@@ -498,23 +520,36 @@ func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, ucontinue}if err != nil {- return fmt.Errorf("could not list repos for user %s: %w", user, err)+ return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)}for _, r := range someRepos {if r.GetFork() && !s.conn.IncludeForks {continue}- common.AddStringSliceItem(r.GetCloneURL(), &s.repos)+ repos = append(repos, r.GetCloneURL())}if res.NextPage == 0 {break}opts.Page = res.NextPage}+ return repos, nil+}++func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error {+ repos, err := s.getReposByUser(ctx, apiClient, user)+ if err != nil {+ return err+ }+ // add the repos to the set of repos+ for _, repo := range repos {+ common.AddStringSliceItem(repo, &s.repos)+ }return nil}-func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) {+func (s *Source) getGistsByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) {+ gistURLs := []string{}gistOpts := &github.GistListOptions{}for {gists, resp, err := apiClient.Gists.List(ctx, user, gistOpts)@@ -525,21 +560,33 @@ func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, ucontinue}if err != nil {- log.WithError(err).Warnf("Could not get gists for user %s", user)+ log.WithError(err).Warnf("could not list repos for user %s", user)+ return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)}for _, gist := range gists {- common.AddStringSliceItem(gist.GetGitPullURL(), &s.repos)+ gistURLs = append(gistURLs, gist.GetGitPullURL())}if resp == nil || resp.NextPage == 0 {break}gistOpts.Page = resp.NextPage}- return+ return gistURLs, nil}-func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error {+func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) error {+ gists, err := s.getGistsByUser(ctx, apiClient, user)+ if err != nil {+ return err+ }+ // add the gists to the set of repos+ for _, gist := range gists {+ common.AddStringSliceItem(gist, &s.repos)+ }+ return nil+}+func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error {opts := &github.ListOptions{PerPage: 500,}@@ -650,24 +697,34 @@ func (s *Source) addOrgsByUser(ctx context.Context, apiClient *github.Client, usfunc (s *Source) normalizeRepos(ctx context.Context, apiClient *github.Client) {// TODO: Add check/fix for repos that are missing scheme- repoIter := make([]string, len(s.repos))- copy(repoIter, s.repos)- for _, repo := range repoIter {- if parts := strings.Split(repo, "/"); len(parts) == 1 {- origSources := len(s.repos)- s.addGistsByUser(ctx, apiClient, repo)- if err := s.addReposByUser(ctx, apiClient, repo); err != nil {- log.WithError(err).Error("error fetching repos by user")- }- if origSources != len(s.repos) {- common.RemoveStringSliceItem(repo, &s.repos)+ normalizedRepos := map[string]struct{}{}+ for _, repo := range s.repos {+ // if there's a '/', assume it's a URL and try to normalize it+ if strings.ContainsRune(repo, '/') {+ repoNormalized, err := giturl.NormalizeGithubRepo(repo)+ if err != nil {+ log.WithError(err).Warnf("Repo not in expected format: %s", repo)continue}+ normalizedRepos[repoNormalized] = struct{}{}+ continue}- repoNormalized, err := giturl.NormalizeGithubRepo(repo)- if err != nil {- log.WithError(err).Warnf("Repo not in expected format: %s", repo)+ // otherwise, assume it's a user and enumerate repositories and gists+ if repos, err := s.getReposByUser(ctx, apiClient, repo); err == nil {+ for _, repo := range repos {+ normalizedRepos[repo] = struct{}{}+ }+ }+ if gists, err := s.getGistsByUser(ctx, apiClient, repo); err == nil {+ for _, gist := range gists {+ normalizedRepos[gist] = struct{}{}+ }}- common.AddStringSliceItem(repoNormalized, &s.repos)+ }++ // replace s.repos+ s.repos = s.repos[:0]+ for key := range normalizedRepos {+ s.repos = append(s.repos, key)}}diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.gonew file mode 100644index 000000000..78477c627--- /dev/null+++ b/pkg/sources/github/github_integration_test.go@@ -0,0 +1,587 @@+//go:build integration+// +build integration++package github++import (+ "context"+ "encoding/base64"+ "fmt"+ "os"+ "testing"+ "time"++ "github.com/google/go-github/v42/github"++ "github.com/kylelemons/godebug/pretty"+ "github.com/mattn/go-colorable"+ log "github.com/sirupsen/logrus"+ "google.golang.org/protobuf/types/known/anypb"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++func TestSource_Scan(t *testing.T) {+ os.Setenv("DO_NOT_RANDOMIZE", "true")++ ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)+ defer cancel()++ secret, err := common.GetTestSecret(ctx)+ if err != nil {+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ }++ // For the personal access token test+ githubToken := secret.MustGetField("GITHUB_TOKEN")++ //For the NEW github app test (+Member enum)+ githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")+ githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)+ if err != nil {+ t.Fatal(err)+ }+ githubPrivateKeyNew := string(githubPrivateKeyBytesNew)+ githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")+ githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")++ //OLD app for breaking app change tests+ // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")+ // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)+ // if err != nil {+ // t.Fatal(err)+ // }+ // githubPrivateKey := string(githubPrivateKeyBytes)+ // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")+ // githubAppID := secret.MustGetField("GITHUB_APP_ID")++ type init struct {+ name string+ verify bool+ connection *sourcespb.GitHub+ }+ tests := []struct {+ name string+ init init+ wantChunk *sources.Chunk+ wantErr bool+ minRepo int+ minOrg int+ }{+ {+ name: "token authenticated, single repo",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: &sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://github.com/dustin-decker/secretsandstuff.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ },+ {+ name: "token authenticated, single repo, no .git",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"},+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: &sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://github.com/dustin-decker/secretsandstuff.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ },+ {+ name: "token authenticated, single org",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Organizations: []string{"trufflesecurity"},+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 0,+ },+ {+ name: "token authenticated, username in org",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Organizations: []string{"dustin-decker"},+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 0,+ },+ {+ name: "token authenticated, username in repo",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"dustin-decker"},+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 0,+ },+ {+ name: "token authenticated, org in repo",+ // I do not think that this is a supported case, but adding the test to specify there is no requirement.+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"trufflesecurity"},+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 0,+ minOrg: 0,+ },+ /*+ {+ name: "token authenticated, no org or user (enum)",+ // This configuration currently will only find gists from the user. No repos or orgs will be scanned.+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Credential: &sourcespb.GitHub_Token{+ Token: githubToken,+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 0,+ minOrg: 0,+ },+ {+ name: "app authenticated (old), no repo or org (enum)",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ ScanUsers: false,+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKey,+ InstallationId: githubInstallationID,+ AppId: githubAppID,+ },+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 0,+ },+ */+ {+ name: "unauthenticated, single org",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Organizations: []string{"trufflesecurity"},+ Credential: &sourcespb.GitHub_Unauthenticated{},+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 1,+ },+ {+ name: "unauthenticated, single repo",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},+ Credential: &sourcespb.GitHub_Unauthenticated{},+ },+ },+ wantChunk: &sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://github.com/trufflesecurity/driftwood.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ },+ /*+ {+ name: "app authenticated, no repo or org",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ ScanUsers: true,+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 0,+ },+ */+ {+ name: "app authenticated, single repo",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ },+ },+ wantChunk: &sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://github.com/trufflesecurity/driftwood.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ minRepo: 1,+ minOrg: 0,+ },+ {+ name: "app authenticated, single org",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Organizations: []string{"trufflesecurity"},+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ minRepo: 3,+ minOrg: 1,+ },+ }++ for i, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ log.Debugf("Beginning test %d: %s", i, tt.name)+ s := Source{}++ log.SetLevel(log.DebugLevel)+ //uncomment for windows Testing+ log.SetFormatter(&log.TextFormatter{ForceColors: true})+ log.SetOutput(colorable.NewColorableStdout())++ conn, err := anypb.New(tt.init.connection)+ if err != nil {+ t.Fatal(err)+ }++ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)+ if (err != nil) != tt.wantErr {+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ chunksCh := make(chan *sources.Chunk, 5)+ go func() {+ err = s.Chunks(ctx, chunksCh)+ if (err != nil) != tt.wantErr {+ if ctx.Err() != nil {+ return+ }+ t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ }()+ if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {+ t.Error(err)+ }+ })+ }+}++func TestSource_paginateGists(t *testing.T) {++ os.Setenv("DO_NOT_RANDOMIZE", "true")++ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)+ defer cancel()++ secret, err := common.GetTestSecret(ctx)+ if err != nil {+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ }+ //For the NEW github app test (+Member enum)+ githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")+ githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)+ if err != nil {+ t.Fatal(err)+ }+ githubPrivateKeyNew := string(githubPrivateKeyBytesNew)+ githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")+ githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")+ type init struct {+ name string+ verify bool+ connection *sourcespb.GitHub+ }+ tests := []struct {+ name string+ init init+ wantChunk *sources.Chunk+ wantErr bool+ user string+ minRepos int+ }{+ {+ name: "get gist secret",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ },+ },+ wantChunk: &sources.Chunk{+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ user: "dustin-decker",+ minRepos: 1,+ },+ {+ name: "get multiple pages of gists",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ },+ },+ wantChunk: nil,+ wantErr: false,+ user: "andrew",+ minRepos: 101,+ },+ /* {+ name: "get multiple pages of gists",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ },+ },+ wantChunk: &sources.Chunk{+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ user: "andrew",+ },+ */+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Source{}++ log.SetLevel(log.DebugLevel)+ //uncomment for windows Testing+ log.SetFormatter(&log.TextFormatter{ForceColors: true})+ log.SetOutput(colorable.NewColorableStdout())++ conn, err := anypb.New(tt.init.connection)+ if err != nil {+ t.Fatal(err)+ }++ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)+ if (err != nil) != tt.wantErr {+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ chunksCh := make(chan *sources.Chunk, 5)+ go func() {+ s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user)+ chunksCh <- &sources.Chunk{}+ }()+ var wantedRepo string+ if tt.wantChunk != nil {+ wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository+ }+ if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {+ t.Error(err)+ }+ })+ }+}++func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {+ return func(chunk *sources.Chunk) error {+ if minRepos != 0 && minRepos > len(s.repos) {+ return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))+ }+ if expected != "" {+ for _, repo := range s.repos {+ if repo == expected {+ return nil+ }+ }+ return fmt.Errorf("expected repo not included: %s", expected)+ }+ return nil+ }+}++func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {+ return func(chunk *sources.Chunk) error {+ if minOrg != 0 && minOrg > len(s.orgs) {+ return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))+ }+ if minRepo != 0 && minRepo > len(s.repos) {+ return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))+ }+ if wantChunk != nil {+ if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {+ return nil+ }+ return common.MatchError+ }+ return nil+ }+}++// func TestSource_paginateRepos(t *testing.T) {+// type args struct {+// ctx context.Context+// apiClient *github.Client+// }+// tests := []struct {+// name string+// org string+// args args+// }{+// {+// org: "fakeNetflix",+// args: args{+// ctx: context.Background(),+// apiClient: github.NewClient(common.SaneHttpClient()),+// },+// },+// }+// for _, tt := range tests {+// t.Run(tt.name, func(t *testing.T) {+// s := &Source{httpClient: common.SaneHttpClient()}+// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)+// if len(s.repos) < 101 {+// t.Errorf("expected > 100 repos, got %d", len(s.repos))+// }+// })+// }+// }diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 45207d90e..d4899cf57 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -1,584 +1,339 @@package githubimport (+ "bytes""context"- "encoding/base64"+ "crypto/rand"+ "crypto/rsa"+ "crypto/x509"+ "encoding/pem""fmt"- "os"+ "net/http"+ "sort"+ "strconv""testing""time""github.com/google/go-github/v42/github"-- "github.com/kylelemons/godebug/pretty"- "github.com/mattn/go-colorable"- log "github.com/sirupsen/logrus"+ "github.com/stretchr/testify/assert""google.golang.org/protobuf/types/known/anypb"+ "gopkg.in/h2non/gock.v1"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"- "github.com/trufflesecurity/trufflehog/v3/pkg/sources")-func TestSource_Scan(t *testing.T) {- os.Setenv("DO_NOT_RANDOMIZE", "true")-- ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)- defer cancel()-- secret, err := common.GetTestSecret(ctx)+func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {+ s := &Source{}+ conn, err := anypb.New(src)if err != nil {- t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ panic(err)}+ return s, conn+}- // For the personal access token test- githubToken := secret.MustGetField("GITHUB_TOKEN")-- //For the NEW github app test (+Member enum)- githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")- githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)- if err != nil {- t.Fatal(err)- }- githubPrivateKeyNew := string(githubPrivateKeyBytesNew)- githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")- githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")-- //OLD app for breaking app change tests- // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")- // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)- // if err != nil {- // t.Fatal(err)- // }- // githubPrivateKey := string(githubPrivateKeyBytes)- // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")- // githubAppID := secret.MustGetField("GITHUB_APP_ID")-- type init struct {- name string- verify bool- connection *sourcespb.GitHub+func initTestSource(src *sourcespb.GitHub) *Source {+ s, conn := createTestSource(src)+ if err := s.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1); err != nil {+ panic(err)}- tests := []struct {- name string- init init- wantChunk *sources.Chunk- wantErr bool- minRepo int- minOrg int- }{- {- name: "token authenticated, single repo",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,- SourceName: "test source",- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Github{- Github: &source_metadatapb.Github{- Repository: "https://github.com/dustin-decker/secretsandstuff.git",- },- },- },- Verify: false,- },- wantErr: false,- },- {- name: "token authenticated, single repo, no .git",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"},- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,- SourceName: "test source",- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Github{- Github: &source_metadatapb.Github{- Repository: "https://github.com/dustin-decker/secretsandstuff.git",- },- },- },- Verify: false,- },- wantErr: false,- },- {- name: "token authenticated, single org",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Organizations: []string{"trufflesecurity"},- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 0,- },- {- name: "token authenticated, username in org",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Organizations: []string{"dustin-decker"},- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 0,- },- {- name: "token authenticated, username in repo",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Repositories: []string{"dustin-decker"},- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 0,- },- {- name: "token authenticated, org in repo",- // I do not think that this is a supported case, but adding the test to specify there is no requirement.- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Repositories: []string{"trufflesecurity"},- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 0,- minOrg: 0,+ return s+}++func TestInit(t *testing.T) {+ source, conn := createTestSource(&sourcespb.GitHub{+ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},+ Credential: &sourcespb.GitHub_Token{+ Token: "super secret token",},- /*- {- name: "token authenticated, no org or user (enum)",- // This configuration currently will only find gists from the user. No repos or orgs will be scanned.- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Credential: &sourcespb.GitHub_Token{- Token: githubToken,- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 0,- minOrg: 0,- },- {- name: "app authenticated (old), no repo or org (enum)",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- ScanUsers: false,- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKey,- InstallationId: githubInstallationID,- AppId: githubAppID,- },- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 0,+ })++ err := source.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1)+ assert.Nil(t, err)++ // TODO: test error case+}++func TestAddReposByOrg(t *testing.T) {+ defer gock.Off()++ gock.New("https://api.github.com").+ Get("/orgs/super-secret-org/repos").+ Reply(200).+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})++ s := initTestSource(nil)+ // gock works here because github.NewClient is using the default HTTP Transport+ err := s.addReposByOrg(context.TODO(), github.NewClient(nil), "super-secret-org")+ assert.Nil(t, err)+ assert.Equal(t, 1, len(s.repos))+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)+ assert.True(t, gock.IsDone())+}++func TestAddReposByUser(t *testing.T) {+ defer gock.Off()++ gock.New("https://api.github.com").+ Get("/users/super-secret-user/repos").+ Reply(200).+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})++ s := initTestSource(nil)+ err := s.addReposByUser(context.TODO(), github.NewClient(nil), "super-secret-user")+ assert.Nil(t, err)+ assert.Equal(t, 1, len(s.repos))+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)+ assert.True(t, gock.IsDone())+}++func TestAddGistsByUser(t *testing.T) {+ defer gock.Off()++ gock.New("https://api.github.com").+ Get("/users/super-secret-user/gists").+ Reply(200).+ JSON([]map[string]string{{"git_pull_url": "super-secret-gist"}})++ s := initTestSource(nil)+ err := s.addGistsByUser(context.TODO(), github.NewClient(nil), "super-secret-user")+ assert.Nil(t, err)+ assert.Equal(t, 1, len(s.repos))+ assert.Equal(t, []string{"super-secret-gist"}, s.repos)+ assert.True(t, gock.IsDone())+}++func TestAddMembersByApp(t *testing.T) {+ defer gock.Off()++ gock.New("https://api.github.com").+ Get("/app/installations").+ Reply(200).+ JSON([]map[string]interface{}{+ {"account": map[string]string{"login": "super-secret-org"}},+ })+ gock.New("https://api.github.com").+ Get("/orgs/super-secret-org/members").+ Reply(200).+ JSON([]map[string]interface{}{+ {"login": "ssm1"},+ {"login": "ssm2"},+ {"login": "ssm3"},+ })++ s := initTestSource(nil)+ err := s.addMembersByApp(context.TODO(), github.NewClient(nil), github.NewClient(nil))+ assert.Nil(t, err)+ assert.Equal(t, 3, len(s.members))+ assert.Equal(t, []string{"ssm1", "ssm2", "ssm3"}, s.members)+ assert.True(t, gock.IsDone())+}++func TestAddReposByApp(t *testing.T) {+ defer gock.Off()++ gock.New("https://api.github.com").+ Get("/installation/repositories").+ Reply(200).+ JSON(map[string]interface{}{+ "repositories": []map[string]string{+ {"clone_url": "ssr1"},+ {"clone_url": "ssr2"},},- */+ })++ s := initTestSource(nil)+ err := s.addReposByApp(context.TODO(), github.NewClient(nil))+ assert.Nil(t, err)+ assert.Equal(t, 2, len(s.repos))+ assert.Equal(t, []string{"ssr1", "ssr2"}, s.repos)+ assert.True(t, gock.IsDone())+}++func TestAddOrgsByUser(t *testing.T) {+ defer gock.Off()++ // NOTE: addOrgsByUser calls /user/orgs to get the orgs of the+ // authenticated user+ gock.New("https://api.github.com").+ Get("/user/orgs").+ Reply(200).+ JSON([]map[string]interface{}{+ {"name": "sso1"},+ {"login": "sso2"},+ })++ s := initTestSource(nil)+ s.addOrgsByUser(context.TODO(), github.NewClient(nil), "super-secret-user")+ assert.Equal(t, 2, len(s.orgs))+ assert.Equal(t, []string{"sso1", "sso2"}, s.orgs)+ assert.True(t, gock.IsDone())+}++func TestNormalizeRepos(t *testing.T) {+ defer gock.Off()++ tests := []struct {+ name string+ setup func()+ repos []string+ expected []string+ }{{- name: "unauthenticated, single org",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Organizations: []string{"trufflesecurity"},- Credential: &sourcespb.GitHub_Unauthenticated{},- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 1,+ name: "repo url",+ setup: func() {},+ repos: []string{"https://github.com/super-secret-user/super-secret-repo"},+ expected: []string{"https://github.com/super-secret-user/super-secret-repo.git"},},{- name: "unauthenticated, single repo",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},- Credential: &sourcespb.GitHub_Unauthenticated{},- },+ name: "username with gists",+ setup: func() {+ gock.New("https://api.github.com").+ Get("/users/super-secret-user/gists").+ Reply(200).+ JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-user/super-secret-gist.git"}})+ gock.New("https://api.github.com").+ Get("/users/super-secret-user/repos").+ Reply(200).+ JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git"}})},- wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,- SourceName: "test source",- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Github{- Github: &source_metadatapb.Github{- Repository: "https://github.com/trufflesecurity/driftwood.git",- },- },- },- Verify: false,+ repos: []string{"super-secret-user"},+ expected: []string{+ "https://github.com/super-secret-user/super-secret-repo.git",+ "https://github.com/super-secret-user/super-secret-gist.git",},- wantErr: false,},- /*- {- name: "app authenticated, no repo or org",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- ScanUsers: true,- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKeyNew,- InstallationId: githubInstallationIDNew,- AppId: githubAppIDNew,- },- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 0,- },- */{- name: "app authenticated, single repo",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKeyNew,- InstallationId: githubInstallationIDNew,- AppId: githubAppIDNew,- },- },- },- },- wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,- SourceName: "test source",- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Github{- Github: &source_metadatapb.Github{- Repository: "https://github.com/trufflesecurity/driftwood.git",- },- },- },- Verify: false,+ name: "not found",+ setup: func() {+ gock.New("https://api.github.com").+ Get("/users/not-found/gists").+ Reply(404)+ gock.New("https://api.github.com").+ Get("/users/not-found/repos").+ Reply(404)},- wantErr: false,- minRepo: 1,- minOrg: 0,+ repos: []string{"not-found"},+ expected: []string{},},{- name: "app authenticated, single org",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Organizations: []string{"trufflesecurity"},- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKeyNew,- InstallationId: githubInstallationIDNew,- AppId: githubAppIDNew,- },- },- },- },- wantChunk: nil,- wantErr: false,- minRepo: 3,- minOrg: 1,+ name: "unexpected format",+ setup: func() {},+ repos: []string{"/foo/"},+ expected: []string{},},}- for i, tt := range tests {+ for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- log.Debugf("Beginning test %d: %s", i, tt.name)- s := Source{}-- log.SetLevel(log.DebugLevel)- //uncomment for windows Testing- log.SetFormatter(&log.TextFormatter{ForceColors: true})- log.SetOutput(colorable.NewColorableStdout())-- conn, err := anypb.New(tt.init.connection)- if err != nil {- t.Fatal(err)- }-- err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)- if (err != nil) != tt.wantErr {- t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)- return- }- chunksCh := make(chan *sources.Chunk, 5)- go func() {- err = s.Chunks(ctx, chunksCh)- if (err != nil) != tt.wantErr {- if ctx.Err() != nil {- return- }- t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)- return- }- }()- if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {- t.Error(err)- }+ defer gock.Off()+ tt.setup()+ s := initTestSource(nil)+ s.repos = tt.repos++ s.normalizeRepos(context.TODO(), github.NewClient(nil))+ assert.Equal(t, len(tt.expected), len(s.repos))+ // sort and compare+ sort.Slice(tt.expected, func(i, j int) bool { return tt.expected[i] < tt.expected[j] })+ sort.Slice(s.repos, func(i, j int) bool { return s.repos[i] < s.repos[j] })+ assert.Equal(t, tt.expected, s.repos)++ assert.True(t, gock.IsDone())})}}-func TestSource_paginateGists(t *testing.T) {+func TestHandleRateLimit(t *testing.T) {+ assert.False(t, handleRateLimit(nil, nil))- os.Setenv("DO_NOT_RANDOMIZE", "true")+ err := &github.RateLimitError{}+ res := &github.Response{Response: &http.Response{Header: make(http.Header)}}+ res.Header.Set("x-ratelimit-remaining", "0")+ res.Header.Set("x-ratelimit-reset", strconv.FormatInt(time.Now().Unix()+1, 10))+ assert.True(t, handleRateLimit(err, res))+}- ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)- defer cancel()+func TestEnumerateUnauthenticated(t *testing.T) {+ defer gock.Off()- secret, err := common.GetTestSecret(ctx)- if err != nil {- t.Fatal(fmt.Errorf("failed to access secret: %v", err))- }- //For the NEW github app test (+Member enum)- githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")- githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)- if err != nil {- t.Fatal(err)- }- githubPrivateKeyNew := string(githubPrivateKeyBytesNew)- githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")- githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")- type init struct {- name string- verify bool- connection *sourcespb.GitHub- }- tests := []struct {- name string- init init- wantChunk *sources.Chunk- wantErr bool- user string- minRepos int- }{- {- name: "get gist secret",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKeyNew,- InstallationId: githubInstallationIDNew,- AppId: githubAppIDNew,- },- },- },- },- wantChunk: &sources.Chunk{- SourceName: "test source",- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Github{- Github: &source_metadatapb.Github{- Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git",- },- },- },- Verify: false,- },- wantErr: false,- user: "dustin-decker",- minRepos: 1,- },- {- name: "get multiple pages of gists",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKeyNew,- InstallationId: githubInstallationIDNew,- AppId: githubAppIDNew,- },- },- },- },- wantChunk: nil,- wantErr: false,- user: "andrew",- minRepos: 101,- },- /* {- name: "get multiple pages of gists",- init: init{- name: "test source",- connection: &sourcespb.GitHub{- Credential: &sourcespb.GitHub_GithubApp{- GithubApp: &credentialspb.GitHubApp{- PrivateKey: githubPrivateKeyNew,- InstallationId: githubInstallationIDNew,- AppId: githubAppIDNew,- },- },- },- },- wantChunk: &sources.Chunk{- SourceName: "test source",- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Github{- Github: &source_metadatapb.Github{- Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",- },- },- },- Verify: false,- },- wantErr: false,- user: "andrew",- },- */- }+ gock.New("https://api.github.com").+ Get("/orgs/super-secret-org/repos").+ Reply(200).+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Source{}-- log.SetLevel(log.DebugLevel)- //uncomment for windows Testing- log.SetFormatter(&log.TextFormatter{ForceColors: true})- log.SetOutput(colorable.NewColorableStdout())-- conn, err := anypb.New(tt.init.connection)- if err != nil {- t.Fatal(err)- }-- err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)- if (err != nil) != tt.wantErr {- t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)- return- }- chunksCh := make(chan *sources.Chunk, 5)- go func() {- s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user)- chunksCh <- &sources.Chunk{}- }()- var wantedRepo string- if tt.wantChunk != nil {- wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository- }- if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {- t.Error(err)- }- })- }+ s := initTestSource(nil)+ s.orgs = []string{"super-secret-org"}+ _ = s.enumerateUnauthenticated(context.TODO())+ assert.Equal(t, 1, len(s.repos))+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)+ assert.True(t, gock.IsDone())}-func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {- return func(chunk *sources.Chunk) error {- if minRepos != 0 && minRepos > len(s.repos) {- return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))- }- if expected != "" {- for _, repo := range s.repos {- if repo == expected {- return nil- }- }- return fmt.Errorf("expected repo not included: %s", expected)- }- return nil- }+func TestEnumerateWithToken(t *testing.T) {+ defer gock.Off()++ gock.New("https://api.github.com").+ Get("/user").+ Reply(200).+ JSON(map[string]string{"login": "super-secret-user"})++ gock.New("https://api.github.com").+ Get("/users/super-secret-user/repos").+ Reply(200).+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})++ s := initTestSource(nil)+ _, err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")+ assert.Nil(t, err)+ assert.Equal(t, 1, len(s.repos))+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)+ assert.True(t, gock.IsDone())}-func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {- return func(chunk *sources.Chunk) error {- if minOrg != 0 && minOrg > len(s.orgs) {- return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))- }- if minRepo != 0 && minRepo > len(s.repos) {- return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))+func TestEnumerateWithApp(t *testing.T) {+ defer gock.Off()++ // generate a private key (it just needs to be in the right format)+ privateKey := func() string {+ key, err := rsa.GenerateKey(rand.Reader, 2048)+ if err != nil {+ panic(err)}- if wantChunk != nil {- if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {- return nil- }- return common.MatchError++ data := x509.MarshalPKCS1PrivateKey(key)+ var pemKey bytes.Buffer+ if err := pem.Encode(&pemKey, &pem.Block{+ Type: "RSA PRIVATE KEY",+ Bytes: data,+ }); err != nil {+ panic(err)}- return nil- }-}+ return pemKey.String()+ }()++ gock.New("https://api.github.com").+ Post("/app/installations/1337/access_tokens").+ Reply(200).+ JSON(map[string]string{"token": "dontlook"})++ gock.New("https://api.github.com").+ Get("/installation/repositories").+ Reply(200).+ JSON(map[string]string{})-// func TestSource_paginateRepos(t *testing.T) {-// type args struct {-// ctx context.Context-// apiClient *github.Client-// }-// tests := []struct {-// name string-// org string-// args args-// }{-// {-// org: "fakeNetflix",-// args: args{-// ctx: context.Background(),-// apiClient: github.NewClient(common.SaneHttpClient()),-// },-// },-// }-// for _, tt := range tests {-// t.Run(tt.name, func(t *testing.T) {-// s := &Source{httpClient: common.SaneHttpClient()}-// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)-// if len(s.repos) < 101 {-// t.Errorf("expected > 100 repos, got %d", len(s.repos))-// }-// })-// }-// }+ s := initTestSource(nil)+ _, _, err := s.enumerateWithApp(+ context.TODO(),+ "https://api.github.com",+ &credentialspb.GitHubApp{+ InstallationId: "1337",+ AppId: "4141",+ PrivateKey: privateKey,+ },+ )+ fmt.Println(err)+ assert.Nil(t, err)+ assert.Equal(t, 0, len(s.repos))++ assert.True(t, gock.IsDone())+}