Skip to content

Find, verify, and analyze leaked credentials

Move GitHub integration tests behind a build flag and add unit tests (#527)

Miccahedaf1e1fd3d2
diff --git a/go.mod b/go.mod
index a3ceb04fe..28d1b1a49 100644
--- a/go.mod
+++ b/go.mod
@@ -31,13 +31,13 @@ require (
github.com/joho/godotenv v1.4.0
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
- github.com/mattn/go-colorable v0.1.12
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1
github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
github.com/rs/zerolog v1.26.1
github.com/sergi/go-diff v1.2.0
github.com/sirupsen/logrus v1.8.1
+ github.com/stretchr/testify v1.7.0
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.64.0
github.com/zricethezav/gitleaks/v8 v8.5.2
@@ -48,6 +48,7 @@ require (
google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf
google.golang.org/protobuf v1.28.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
+ gopkg.in/h2non/gock.v1 v1.1.2
)
require (
@@ -71,6 +72,7 @@ require (
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
@@ -84,6 +86,7 @@ require (
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/googleapis/gax-go/v2 v2.2.0 // indirect
+ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -91,11 +94,13 @@ require (
github.com/jpillora/s3 v1.1.4 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+ github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.23.0 // indirect
@@ -110,4 +115,5 @@ require (
google.golang.org/grpc v1.45.0 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
+ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
diff --git a/go.sum b/go.sum
index 3e6fa7ee4..88770615c 100644
--- a/go.sum
+++ b/go.sum
@@ -296,6 +296,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -391,6 +393,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -930,6 +934,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
+gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 8c20e8a63..6e7055947 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -11,13 +11,13 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v42/github"
- "github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"golang.org/x/sync/semaphore"
@@ -37,8 +37,8 @@ import (
type Source struct {
name string
- sourceId int64
- jobId int64
+ sourceID int64
+ jobID int64
verify bool
repos []string
orgs []string
@@ -64,11 +64,11 @@ func (s *Source) Type() sourcespb.SourceType {
}
func (s *Source) SourceID() int64 {
- return s.sourceId
+ return s.sourceID
}
func (s *Source) JobID() int64 {
- return s.jobId
+ return s.jobID
}
func (s *Source) Token(ctx context.Context, installationClient *github.Client) (string, error) {
@@ -94,13 +94,13 @@ func (s *Source) Token(ctx context.Context, installationClient *github.Client) (
}
// Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
s.log = log.WithField("source", s.Type()).WithField("name", name)
s.aCtx = aCtx
s.name = name
- s.sourceId = sourceId
- s.jobId = jobId
+ s.sourceID = sourceID
+ s.jobID = jobID
s.verify = verify
s.jobSem = semaphore.NewWeighted(int64(concurrency))
@@ -333,16 +333,27 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error {
- scanned := 0
+ var scanned uint64
log.Debugf("Found %v total repos to scan", len(s.repos))
wg := sync.WaitGroup{}
- errs := []error{}
- var errsMut sync.Mutex
+ errs := make(chan error, 1)
+ reportErr := func(err error) {
+ // save the error if there's room, otherwise log and drop it
+ select {
+ case errs <- err:
+ default:
+ log.WithError(err).Warn("dropping error")
+ }
+ }
+
for i, repoURL := range s.repos {
if err := s.jobSem.Acquire(ctx, 1); err != nil {
+ // Acquire blocks until it can acquire the semaphore or returns an
+ // error if the context is finished
log.WithError(err).Debug("could not acquire semaphore")
- continue
+ reportErr(err)
+ break
}
wg.Add(1)
go func(ctx context.Context, repoURL string, i int) {
@@ -367,10 +378,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
var token string
token, err = s.Token(ctx, installationClient)
if err != nil {
- // TODO: maybe we can use a channel here
- errsMut.Lock()
- errs = append(errs, err)
- errsMut.Unlock()
+ reportErr(err)
return
}
path, repo, err = git.CloneRepoUsingToken(token, repoURL, "clone")
@@ -391,20 +399,20 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
if err != nil {
log.WithError(err).Errorf("unable to scan repo, continuing")
}
- // TODO: use atomic library
- scanned++
- logrus.Debugf("scanned %d/%d repos", scanned, len(s.repos))
+ atomic.AddUint64(&scanned, 1)
+ log.Debugf("scanned %d/%d repos", scanned, len(s.repos))
}(ctx, repoURL, i)
}
wg.Wait()
// This only returns first error which is what we did prior to concurrency
- if len(errs) > 0 {
- return errs[0]
+ select {
+ case err := <-errs:
+ return err
+ default:
+ return nil
}
-
- return nil
}
// handleRateLimit returns true if a rate limit was handled
@@ -443,7 +451,8 @@ func handleRateLimit(errIn error, res *github.Response) bool {
return true
}
-func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error {
+func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, org string) ([]string, error) {
+ repos := []string{}
opts := &github.RepositoryListByOrgOptions{
ListOptions: github.ListOptions{
PerPage: 100,
@@ -459,7 +468,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or
continue
}
if err != nil {
- return fmt.Errorf("could not list repos for org %s: %w", org, err)
+ return nil, fmt.Errorf("could not list repos for org %s: %w", org, err)
}
if len(someRepos) == 0 {
break
@@ -472,7 +481,7 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or
continue
}
}
- common.AddStringSliceItem(r.GetCloneURL(), &s.repos)
+ repos = append(repos, r.GetCloneURL())
}
if res.NextPage == 0 {
break
@@ -480,10 +489,23 @@ func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, or
opts.Page = res.NextPage
}
log.WithField("org", org).Debugf("Found %d repos (%d forks)", numRepos, numForks)
+ return repos, nil
+}
+
+func (s *Source) addReposByOrg(ctx context.Context, apiClient *github.Client, org string) error {
+ repos, err := s.getReposByOrg(ctx, apiClient, org)
+ if err != nil {
+ return err
+ }
+ // add the repos to the set of repos
+ for _, repo := range repos {
+ common.AddStringSliceItem(repo, &s.repos)
+ }
return nil
}
-func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error {
+func (s *Source) getReposByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) {
+ repos := []string{}
opts := &github.RepositoryListOptions{
ListOptions: github.ListOptions{
PerPage: 50,
@@ -498,23 +520,36 @@ func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, u
continue
}
if err != nil {
- return fmt.Errorf("could not list repos for user %s: %w", user, err)
+ return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)
}
for _, r := range someRepos {
if r.GetFork() && !s.conn.IncludeForks {
continue
}
- common.AddStringSliceItem(r.GetCloneURL(), &s.repos)
+ repos = append(repos, r.GetCloneURL())
}
if res.NextPage == 0 {
break
}
opts.Page = res.NextPage
}
+ return repos, nil
+}
+
+func (s *Source) addReposByUser(ctx context.Context, apiClient *github.Client, user string) error {
+ repos, err := s.getReposByUser(ctx, apiClient, user)
+ if err != nil {
+ return err
+ }
+ // add the repos to the set of repos
+ for _, repo := range repos {
+ common.AddStringSliceItem(repo, &s.repos)
+ }
return nil
}
-func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) {
+func (s *Source) getGistsByUser(ctx context.Context, apiClient *github.Client, user string) ([]string, error) {
+ gistURLs := []string{}
gistOpts := &github.GistListOptions{}
for {
gists, resp, err := apiClient.Gists.List(ctx, user, gistOpts)
@@ -525,21 +560,33 @@ func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, u
continue
}
if err != nil {
- log.WithError(err).Warnf("Could not get gists for user %s", user)
+ log.WithError(err).Warnf("could not list repos for user %s", user)
+ return nil, fmt.Errorf("could not list repos for user %s: %w", user, err)
}
for _, gist := range gists {
- common.AddStringSliceItem(gist.GetGitPullURL(), &s.repos)
+ gistURLs = append(gistURLs, gist.GetGitPullURL())
}
if resp == nil || resp.NextPage == 0 {
break
}
gistOpts.Page = resp.NextPage
}
- return
+ return gistURLs, nil
}
-func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error {
+func (s *Source) addGistsByUser(ctx context.Context, apiClient *github.Client, user string) error {
+ gists, err := s.getGistsByUser(ctx, apiClient, user)
+ if err != nil {
+ return err
+ }
+ // add the gists to the set of repos
+ for _, gist := range gists {
+ common.AddStringSliceItem(gist, &s.repos)
+ }
+ return nil
+}
+func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client, apiClient *github.Client) error {
opts := &github.ListOptions{
PerPage: 500,
}
@@ -650,24 +697,34 @@ func (s *Source) addOrgsByUser(ctx context.Context, apiClient *github.Client, us
func (s *Source) normalizeRepos(ctx context.Context, apiClient *github.Client) {
// TODO: Add check/fix for repos that are missing scheme
- repoIter := make([]string, len(s.repos))
- copy(repoIter, s.repos)
- for _, repo := range repoIter {
- if parts := strings.Split(repo, "/"); len(parts) == 1 {
- origSources := len(s.repos)
- s.addGistsByUser(ctx, apiClient, repo)
- if err := s.addReposByUser(ctx, apiClient, repo); err != nil {
- log.WithError(err).Error("error fetching repos by user")
- }
- if origSources != len(s.repos) {
- common.RemoveStringSliceItem(repo, &s.repos)
+ normalizedRepos := map[string]struct{}{}
+ for _, repo := range s.repos {
+ // if there's a '/', assume it's a URL and try to normalize it
+ if strings.ContainsRune(repo, '/') {
+ repoNormalized, err := giturl.NormalizeGithubRepo(repo)
+ if err != nil {
+ log.WithError(err).Warnf("Repo not in expected format: %s", repo)
continue
}
+ normalizedRepos[repoNormalized] = struct{}{}
+ continue
}
- repoNormalized, err := giturl.NormalizeGithubRepo(repo)
- if err != nil {
- log.WithError(err).Warnf("Repo not in expected format: %s", repo)
+ // otherwise, assume it's a user and enumerate repositories and gists
+ if repos, err := s.getReposByUser(ctx, apiClient, repo); err == nil {
+ for _, repo := range repos {
+ normalizedRepos[repo] = struct{}{}
+ }
+ }
+ if gists, err := s.getGistsByUser(ctx, apiClient, repo); err == nil {
+ for _, gist := range gists {
+ normalizedRepos[gist] = struct{}{}
+ }
}
- common.AddStringSliceItem(repoNormalized, &s.repos)
+ }
+
+ // replace s.repos
+ s.repos = s.repos[:0]
+ for key := range normalizedRepos {
+ s.repos = append(s.repos, key)
}
}
diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go
new file mode 100644
index 000000000..78477c627
--- /dev/null
+++ b/pkg/sources/github/github_integration_test.go
@@ -0,0 +1,587 @@
+//go:build integration
+// +build integration
+
+package github
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/google/go-github/v42/github"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/mattn/go-colorable"
+ log "github.com/sirupsen/logrus"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestSource_Scan(t *testing.T) {
+ os.Setenv("DO_NOT_RANDOMIZE", "true")
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+ defer cancel()
+
+ secret, err := common.GetTestSecret(ctx)
+ if err != nil {
+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+ }
+
+ // For the personal access token test
+ githubToken := secret.MustGetField("GITHUB_TOKEN")
+
+ //For the NEW github app test (+Member enum)
+ githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
+ githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
+ if err != nil {
+ t.Fatal(err)
+ }
+ githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
+ githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
+ githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
+
+ //OLD app for breaking app change tests
+ // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")
+ // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)
+ // if err != nil {
+ // t.Fatal(err)
+ // }
+ // githubPrivateKey := string(githubPrivateKeyBytes)
+ // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")
+ // githubAppID := secret.MustGetField("GITHUB_APP_ID")
+
+ type init struct {
+ name string
+ verify bool
+ connection *sourcespb.GitHub
+ }
+ tests := []struct {
+ name string
+ init init
+ wantChunk *sources.Chunk
+ wantErr bool
+ minRepo int
+ minOrg int
+ }{
+ {
+ name: "token authenticated, single repo",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://github.com/dustin-decker/secretsandstuff.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ },
+ {
+ name: "token authenticated, single repo, no .git",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://github.com/dustin-decker/secretsandstuff.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ },
+ {
+ name: "token authenticated, single org",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Organizations: []string{"trufflesecurity"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 0,
+ },
+ {
+ name: "token authenticated, username in org",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Organizations: []string{"dustin-decker"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 0,
+ },
+ {
+ name: "token authenticated, username in repo",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"dustin-decker"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 0,
+ },
+ {
+ name: "token authenticated, org in repo",
+ // I do not think that this is a supported case, but adding the test to specify there is no requirement.
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"trufflesecurity"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 0,
+ minOrg: 0,
+ },
+ /*
+ {
+ name: "token authenticated, no org or user (enum)",
+ // This configuration currently will only find gists from the user. No repos or orgs will be scanned.
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Credential: &sourcespb.GitHub_Token{
+ Token: githubToken,
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 0,
+ minOrg: 0,
+ },
+ {
+ name: "app authenticated (old), no repo or org (enum)",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ ScanUsers: false,
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKey,
+ InstallationId: githubInstallationID,
+ AppId: githubAppID,
+ },
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 0,
+ },
+ */
+ {
+ name: "unauthenticated, single org",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Organizations: []string{"trufflesecurity"},
+ Credential: &sourcespb.GitHub_Unauthenticated{},
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 1,
+ },
+ {
+ name: "unauthenticated, single repo",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
+ Credential: &sourcespb.GitHub_Unauthenticated{},
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://github.com/trufflesecurity/driftwood.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ },
+ /*
+ {
+ name: "app authenticated, no repo or org",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ ScanUsers: true,
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 0,
+ },
+ */
+ {
+ name: "app authenticated, single repo",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://github.com/trufflesecurity/driftwood.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ minRepo: 1,
+ minOrg: 0,
+ },
+ {
+ name: "app authenticated, single org",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Organizations: []string{"trufflesecurity"},
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ minRepo: 3,
+ minOrg: 1,
+ },
+ }
+
+ for i, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ log.Debugf("Beginning test %d: %s", i, tt.name)
+ s := Source{}
+
+ log.SetLevel(log.DebugLevel)
+ //uncomment for windows Testing
+ log.SetFormatter(&log.TextFormatter{ForceColors: true})
+ log.SetOutput(colorable.NewColorableStdout())
+
+ conn, err := anypb.New(tt.init.connection)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ chunksCh := make(chan *sources.Chunk, 5)
+ go func() {
+ err = s.Chunks(ctx, chunksCh)
+ if (err != nil) != tt.wantErr {
+ if ctx.Err() != nil {
+ return
+ }
+ t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ }()
+ if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {
+ t.Error(err)
+ }
+ })
+ }
+}
+
+func TestSource_paginateGists(t *testing.T) {
+
+ os.Setenv("DO_NOT_RANDOMIZE", "true")
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+ defer cancel()
+
+ secret, err := common.GetTestSecret(ctx)
+ if err != nil {
+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+ }
+ //For the NEW github app test (+Member enum)
+ githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
+ githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
+ if err != nil {
+ t.Fatal(err)
+ }
+ githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
+ githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
+ githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
+ type init struct {
+ name string
+ verify bool
+ connection *sourcespb.GitHub
+ }
+ tests := []struct {
+ name string
+ init init
+ wantChunk *sources.Chunk
+ wantErr bool
+ user string
+ minRepos int
+ }{
+ {
+ name: "get gist secret",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ user: "dustin-decker",
+ minRepos: 1,
+ },
+ {
+ name: "get multiple pages of gists",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ },
+ },
+ wantChunk: nil,
+ wantErr: false,
+ user: "andrew",
+ minRepos: 101,
+ },
+ /* {
+ name: "get multiple pages of gists",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ user: "andrew",
+ },
+ */
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Source{}
+
+ log.SetLevel(log.DebugLevel)
+ //uncomment for windows Testing
+ log.SetFormatter(&log.TextFormatter{ForceColors: true})
+ log.SetOutput(colorable.NewColorableStdout())
+
+ conn, err := anypb.New(tt.init.connection)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ chunksCh := make(chan *sources.Chunk, 5)
+ go func() {
+ s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user)
+ chunksCh <- &sources.Chunk{}
+ }()
+ var wantedRepo string
+ if tt.wantChunk != nil {
+ wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository
+ }
+ if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {
+ t.Error(err)
+ }
+ })
+ }
+}
+
+func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {
+ return func(chunk *sources.Chunk) error {
+ if minRepos != 0 && minRepos > len(s.repos) {
+ return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))
+ }
+ if expected != "" {
+ for _, repo := range s.repos {
+ if repo == expected {
+ return nil
+ }
+ }
+ return fmt.Errorf("expected repo not included: %s", expected)
+ }
+ return nil
+ }
+}
+
+func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {
+ return func(chunk *sources.Chunk) error {
+ if minOrg != 0 && minOrg > len(s.orgs) {
+ return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))
+ }
+ if minRepo != 0 && minRepo > len(s.repos) {
+ return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))
+ }
+ if wantChunk != nil {
+ if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {
+ return nil
+ }
+ return common.MatchError
+ }
+ return nil
+ }
+}
+
+// func TestSource_paginateRepos(t *testing.T) {
+// type args struct {
+// ctx context.Context
+// apiClient *github.Client
+// }
+// tests := []struct {
+// name string
+// org string
+// args args
+// }{
+// {
+// org: "fakeNetflix",
+// args: args{
+// ctx: context.Background(),
+// apiClient: github.NewClient(common.SaneHttpClient()),
+// },
+// },
+// }
+// for _, tt := range tests {
+// t.Run(tt.name, func(t *testing.T) {
+// s := &Source{httpClient: common.SaneHttpClient()}
+// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)
+// if len(s.repos) < 101 {
+// t.Errorf("expected > 100 repos, got %d", len(s.repos))
+// }
+// })
+// }
+// }
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 45207d90e..d4899cf57 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -1,584 +1,339 @@
package github
import (
+ "bytes"
"context"
- "encoding/base64"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/pem"
"fmt"
- "os"
+ "net/http"
+ "sort"
+ "strconv"
"testing"
"time"
"github.com/google/go-github/v42/github"
-
- "github.com/kylelemons/godebug/pretty"
- "github.com/mattn/go-colorable"
- log "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
+ "gopkg.in/h2non/gock.v1"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
- "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
-func TestSource_Scan(t *testing.T) {
- os.Setenv("DO_NOT_RANDOMIZE", "true")
-
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
- defer cancel()
-
- secret, err := common.GetTestSecret(ctx)
+func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {
+ s := &Source{}
+ conn, err := anypb.New(src)
if err != nil {
- t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+ panic(err)
}
+ return s, conn
+}
- // For the personal access token test
- githubToken := secret.MustGetField("GITHUB_TOKEN")
-
- //For the NEW github app test (+Member enum)
- githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
- githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
- if err != nil {
- t.Fatal(err)
- }
- githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
- githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
- githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
-
- //OLD app for breaking app change tests
- // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY")
- // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64)
- // if err != nil {
- // t.Fatal(err)
- // }
- // githubPrivateKey := string(githubPrivateKeyBytes)
- // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID")
- // githubAppID := secret.MustGetField("GITHUB_APP_ID")
-
- type init struct {
- name string
- verify bool
- connection *sourcespb.GitHub
+func initTestSource(src *sourcespb.GitHub) *Source {
+ s, conn := createTestSource(src)
+ if err := s.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1); err != nil {
+ panic(err)
}
- tests := []struct {
- name string
- init init
- wantChunk *sources.Chunk
- wantErr bool
- minRepo int
- minOrg int
- }{
- {
- name: "token authenticated, single repo",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: &sources.Chunk{
- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
- SourceName: "test source",
- SourceMetadata: &source_metadatapb.MetaData{
- Data: &source_metadatapb.MetaData_Github{
- Github: &source_metadatapb.Github{
- Repository: "https://github.com/dustin-decker/secretsandstuff.git",
- },
- },
- },
- Verify: false,
- },
- wantErr: false,
- },
- {
- name: "token authenticated, single repo, no .git",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Repositories: []string{"https://github.com/dustin-decker/secretsandstuff"},
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: &sources.Chunk{
- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
- SourceName: "test source",
- SourceMetadata: &source_metadatapb.MetaData{
- Data: &source_metadatapb.MetaData_Github{
- Github: &source_metadatapb.Github{
- Repository: "https://github.com/dustin-decker/secretsandstuff.git",
- },
- },
- },
- Verify: false,
- },
- wantErr: false,
- },
- {
- name: "token authenticated, single org",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Organizations: []string{"trufflesecurity"},
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 0,
- },
- {
- name: "token authenticated, username in org",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Organizations: []string{"dustin-decker"},
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 0,
- },
- {
- name: "token authenticated, username in repo",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Repositories: []string{"dustin-decker"},
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 0,
- },
- {
- name: "token authenticated, org in repo",
- // I do not think that this is a supported case, but adding the test to specify there is no requirement.
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Repositories: []string{"trufflesecurity"},
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 0,
- minOrg: 0,
+ return s
+}
+
+func TestInit(t *testing.T) {
+ source, conn := createTestSource(&sourcespb.GitHub{
+ Repositories: []string{"https://github.com/dustin-decker/secretsandstuff.git"},
+ Credential: &sourcespb.GitHub_Token{
+ Token: "super secret token",
},
- /*
- {
- name: "token authenticated, no org or user (enum)",
- // This configuration currently will only find gists from the user. No repos or orgs will be scanned.
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Credential: &sourcespb.GitHub_Token{
- Token: githubToken,
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 0,
- minOrg: 0,
- },
- {
- name: "app authenticated (old), no repo or org (enum)",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- ScanUsers: false,
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKey,
- InstallationId: githubInstallationID,
- AppId: githubAppID,
- },
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 0,
+ })
+
+ err := source.Init(context.TODO(), "test - github", 0, 1337, false, conn, 1)
+ assert.Nil(t, err)
+
+ // TODO: test error case
+}
+
+func TestAddReposByOrg(t *testing.T) {
+ defer gock.Off()
+
+ gock.New("https://api.github.com").
+ Get("/orgs/super-secret-org/repos").
+ Reply(200).
+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
+
+ s := initTestSource(nil)
+ // gock works here because github.NewClient is using the default HTTP Transport
+ err := s.addReposByOrg(context.TODO(), github.NewClient(nil), "super-secret-org")
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(s.repos))
+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)
+ assert.True(t, gock.IsDone())
+}
+
+func TestAddReposByUser(t *testing.T) {
+ defer gock.Off()
+
+ gock.New("https://api.github.com").
+ Get("/users/super-secret-user/repos").
+ Reply(200).
+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
+
+ s := initTestSource(nil)
+ err := s.addReposByUser(context.TODO(), github.NewClient(nil), "super-secret-user")
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(s.repos))
+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)
+ assert.True(t, gock.IsDone())
+}
+
+func TestAddGistsByUser(t *testing.T) {
+ defer gock.Off()
+
+ gock.New("https://api.github.com").
+ Get("/users/super-secret-user/gists").
+ Reply(200).
+ JSON([]map[string]string{{"git_pull_url": "super-secret-gist"}})
+
+ s := initTestSource(nil)
+ err := s.addGistsByUser(context.TODO(), github.NewClient(nil), "super-secret-user")
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(s.repos))
+ assert.Equal(t, []string{"super-secret-gist"}, s.repos)
+ assert.True(t, gock.IsDone())
+}
+
+func TestAddMembersByApp(t *testing.T) {
+ defer gock.Off()
+
+ gock.New("https://api.github.com").
+ Get("/app/installations").
+ Reply(200).
+ JSON([]map[string]interface{}{
+ {"account": map[string]string{"login": "super-secret-org"}},
+ })
+ gock.New("https://api.github.com").
+ Get("/orgs/super-secret-org/members").
+ Reply(200).
+ JSON([]map[string]interface{}{
+ {"login": "ssm1"},
+ {"login": "ssm2"},
+ {"login": "ssm3"},
+ })
+
+ s := initTestSource(nil)
+ err := s.addMembersByApp(context.TODO(), github.NewClient(nil), github.NewClient(nil))
+ assert.Nil(t, err)
+ assert.Equal(t, 3, len(s.members))
+ assert.Equal(t, []string{"ssm1", "ssm2", "ssm3"}, s.members)
+ assert.True(t, gock.IsDone())
+}
+
+func TestAddReposByApp(t *testing.T) {
+ defer gock.Off()
+
+ gock.New("https://api.github.com").
+ Get("/installation/repositories").
+ Reply(200).
+ JSON(map[string]interface{}{
+ "repositories": []map[string]string{
+ {"clone_url": "ssr1"},
+ {"clone_url": "ssr2"},
},
- */
+ })
+
+ s := initTestSource(nil)
+ err := s.addReposByApp(context.TODO(), github.NewClient(nil))
+ assert.Nil(t, err)
+ assert.Equal(t, 2, len(s.repos))
+ assert.Equal(t, []string{"ssr1", "ssr2"}, s.repos)
+ assert.True(t, gock.IsDone())
+}
+
+func TestAddOrgsByUser(t *testing.T) {
+ defer gock.Off()
+
+ // NOTE: addOrgsByUser calls /user/orgs to get the orgs of the
+ // authenticated user
+ gock.New("https://api.github.com").
+ Get("/user/orgs").
+ Reply(200).
+ JSON([]map[string]interface{}{
+ {"name": "sso1"},
+ {"login": "sso2"},
+ })
+
+ s := initTestSource(nil)
+ s.addOrgsByUser(context.TODO(), github.NewClient(nil), "super-secret-user")
+ assert.Equal(t, 2, len(s.orgs))
+ assert.Equal(t, []string{"sso1", "sso2"}, s.orgs)
+ assert.True(t, gock.IsDone())
+}
+
+func TestNormalizeRepos(t *testing.T) {
+ defer gock.Off()
+
+ tests := []struct {
+ name string
+ setup func()
+ repos []string
+ expected []string
+ }{
{
- name: "unauthenticated, single org",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Organizations: []string{"trufflesecurity"},
- Credential: &sourcespb.GitHub_Unauthenticated{},
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 1,
+ name: "repo url",
+ setup: func() {},
+ repos: []string{"https://github.com/super-secret-user/super-secret-repo"},
+ expected: []string{"https://github.com/super-secret-user/super-secret-repo.git"},
},
{
- name: "unauthenticated, single repo",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
- Credential: &sourcespb.GitHub_Unauthenticated{},
- },
+ name: "username with gists",
+ setup: func() {
+ gock.New("https://api.github.com").
+ Get("/users/super-secret-user/gists").
+ Reply(200).
+ JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-user/super-secret-gist.git"}})
+ gock.New("https://api.github.com").
+ Get("/users/super-secret-user/repos").
+ Reply(200).
+ JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git"}})
},
- wantChunk: &sources.Chunk{
- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
- SourceName: "test source",
- SourceMetadata: &source_metadatapb.MetaData{
- Data: &source_metadatapb.MetaData_Github{
- Github: &source_metadatapb.Github{
- Repository: "https://github.com/trufflesecurity/driftwood.git",
- },
- },
- },
- Verify: false,
+ repos: []string{"super-secret-user"},
+ expected: []string{
+ "https://github.com/super-secret-user/super-secret-repo.git",
+ "https://github.com/super-secret-user/super-secret-gist.git",
},
- wantErr: false,
},
- /*
- {
- name: "app authenticated, no repo or org",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- ScanUsers: true,
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKeyNew,
- InstallationId: githubInstallationIDNew,
- AppId: githubAppIDNew,
- },
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 0,
- },
- */
{
- name: "app authenticated, single repo",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKeyNew,
- InstallationId: githubInstallationIDNew,
- AppId: githubAppIDNew,
- },
- },
- },
- },
- wantChunk: &sources.Chunk{
- SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
- SourceName: "test source",
- SourceMetadata: &source_metadatapb.MetaData{
- Data: &source_metadatapb.MetaData_Github{
- Github: &source_metadatapb.Github{
- Repository: "https://github.com/trufflesecurity/driftwood.git",
- },
- },
- },
- Verify: false,
+ name: "not found",
+ setup: func() {
+ gock.New("https://api.github.com").
+ Get("/users/not-found/gists").
+ Reply(404)
+ gock.New("https://api.github.com").
+ Get("/users/not-found/repos").
+ Reply(404)
},
- wantErr: false,
- minRepo: 1,
- minOrg: 0,
+ repos: []string{"not-found"},
+ expected: []string{},
},
{
- name: "app authenticated, single org",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Organizations: []string{"trufflesecurity"},
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKeyNew,
- InstallationId: githubInstallationIDNew,
- AppId: githubAppIDNew,
- },
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- minRepo: 3,
- minOrg: 1,
+ name: "unexpected format",
+ setup: func() {},
+ repos: []string{"/foo/"},
+ expected: []string{},
},
}
- for i, tt := range tests {
+ for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- log.Debugf("Beginning test %d: %s", i, tt.name)
- s := Source{}
-
- log.SetLevel(log.DebugLevel)
- //uncomment for windows Testing
- log.SetFormatter(&log.TextFormatter{ForceColors: true})
- log.SetOutput(colorable.NewColorableStdout())
-
- conn, err := anypb.New(tt.init.connection)
- if err != nil {
- t.Fatal(err)
- }
-
- err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
- if (err != nil) != tt.wantErr {
- t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- chunksCh := make(chan *sources.Chunk, 5)
- go func() {
- err = s.Chunks(ctx, chunksCh)
- if (err != nil) != tt.wantErr {
- if ctx.Err() != nil {
- return
- }
- t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- }()
- if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {
- t.Error(err)
- }
+ defer gock.Off()
+ tt.setup()
+ s := initTestSource(nil)
+ s.repos = tt.repos
+
+ s.normalizeRepos(context.TODO(), github.NewClient(nil))
+ assert.Equal(t, len(tt.expected), len(s.repos))
+ // sort and compare
+ sort.Slice(tt.expected, func(i, j int) bool { return tt.expected[i] < tt.expected[j] })
+ sort.Slice(s.repos, func(i, j int) bool { return s.repos[i] < s.repos[j] })
+ assert.Equal(t, tt.expected, s.repos)
+
+ assert.True(t, gock.IsDone())
})
}
}
-func TestSource_paginateGists(t *testing.T) {
+func TestHandleRateLimit(t *testing.T) {
+ assert.False(t, handleRateLimit(nil, nil))
- os.Setenv("DO_NOT_RANDOMIZE", "true")
+ err := &github.RateLimitError{}
+ res := &github.Response{Response: &http.Response{Header: make(http.Header)}}
+ res.Header.Set("x-ratelimit-remaining", "0")
+ res.Header.Set("x-ratelimit-reset", strconv.FormatInt(time.Now().Unix()+1, 10))
+ assert.True(t, handleRateLimit(err, res))
+}
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- defer cancel()
+func TestEnumerateUnauthenticated(t *testing.T) {
+ defer gock.Off()
- secret, err := common.GetTestSecret(ctx)
- if err != nil {
- t.Fatal(fmt.Errorf("failed to access secret: %v", err))
- }
- //For the NEW github app test (+Member enum)
- githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
- githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
- if err != nil {
- t.Fatal(err)
- }
- githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
- githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
- githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
- type init struct {
- name string
- verify bool
- connection *sourcespb.GitHub
- }
- tests := []struct {
- name string
- init init
- wantChunk *sources.Chunk
- wantErr bool
- user string
- minRepos int
- }{
- {
- name: "get gist secret",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKeyNew,
- InstallationId: githubInstallationIDNew,
- AppId: githubAppIDNew,
- },
- },
- },
- },
- wantChunk: &sources.Chunk{
- SourceName: "test source",
- SourceMetadata: &source_metadatapb.MetaData{
- Data: &source_metadatapb.MetaData_Github{
- Github: &source_metadatapb.Github{
- Repository: "https://gist.github.com/be45ad1ebabe98482d9c0bb80c07c619.git",
- },
- },
- },
- Verify: false,
- },
- wantErr: false,
- user: "dustin-decker",
- minRepos: 1,
- },
- {
- name: "get multiple pages of gists",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKeyNew,
- InstallationId: githubInstallationIDNew,
- AppId: githubAppIDNew,
- },
- },
- },
- },
- wantChunk: nil,
- wantErr: false,
- user: "andrew",
- minRepos: 101,
- },
- /* {
- name: "get multiple pages of gists",
- init: init{
- name: "test source",
- connection: &sourcespb.GitHub{
- Credential: &sourcespb.GitHub_GithubApp{
- GithubApp: &credentialspb.GitHubApp{
- PrivateKey: githubPrivateKeyNew,
- InstallationId: githubInstallationIDNew,
- AppId: githubAppIDNew,
- },
- },
- },
- },
- wantChunk: &sources.Chunk{
- SourceName: "test source",
- SourceMetadata: &source_metadatapb.MetaData{
- Data: &source_metadatapb.MetaData_Github{
- Github: &source_metadatapb.Github{
- Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git",
- },
- },
- },
- Verify: false,
- },
- wantErr: false,
- user: "andrew",
- },
- */
- }
+ gock.New("https://api.github.com").
+ Get("/orgs/super-secret-org/repos").
+ Reply(200).
+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Source{}
-
- log.SetLevel(log.DebugLevel)
- //uncomment for windows Testing
- log.SetFormatter(&log.TextFormatter{ForceColors: true})
- log.SetOutput(colorable.NewColorableStdout())
-
- conn, err := anypb.New(tt.init.connection)
- if err != nil {
- t.Fatal(err)
- }
-
- err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
- if (err != nil) != tt.wantErr {
- t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- chunksCh := make(chan *sources.Chunk, 5)
- go func() {
- s.addGistsByUser(ctx, github.NewClient(s.httpClient), tt.user)
- chunksCh <- &sources.Chunk{}
- }()
- var wantedRepo string
- if tt.wantChunk != nil {
- wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository
- }
- if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {
- t.Error(err)
- }
- })
- }
+ s := initTestSource(nil)
+ s.orgs = []string{"super-secret-org"}
+ _ = s.enumerateUnauthenticated(context.TODO())
+ assert.Equal(t, 1, len(s.repos))
+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)
+ assert.True(t, gock.IsDone())
}
-func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {
- return func(chunk *sources.Chunk) error {
- if minRepos != 0 && minRepos > len(s.repos) {
- return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))
- }
- if expected != "" {
- for _, repo := range s.repos {
- if repo == expected {
- return nil
- }
- }
- return fmt.Errorf("expected repo not included: %s", expected)
- }
- return nil
- }
+func TestEnumerateWithToken(t *testing.T) {
+ defer gock.Off()
+
+ gock.New("https://api.github.com").
+ Get("/user").
+ Reply(200).
+ JSON(map[string]string{"login": "super-secret-user"})
+
+ gock.New("https://api.github.com").
+ Get("/users/super-secret-user/repos").
+ Reply(200).
+ JSON([]map[string]string{{"clone_url": "super-secret-repo"}})
+
+ s := initTestSource(nil)
+ _, err := s.enumerateWithToken(context.TODO(), "https://api.github.com", "token")
+ assert.Nil(t, err)
+ assert.Equal(t, 1, len(s.repos))
+ assert.Equal(t, []string{"super-secret-repo"}, s.repos)
+ assert.True(t, gock.IsDone())
}
-func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {
- return func(chunk *sources.Chunk) error {
- if minOrg != 0 && minOrg > len(s.orgs) {
- return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))
- }
- if minRepo != 0 && minRepo > len(s.repos) {
- return fmt.Errorf("incorrect number of repos. expected at least: %d, got %d", minRepo, len(s.repos))
+func TestEnumerateWithApp(t *testing.T) {
+ defer gock.Off()
+
+ // generate a private key (it just needs to be in the right format)
+ privateKey := func() string {
+ key, err := rsa.GenerateKey(rand.Reader, 2048)
+ if err != nil {
+ panic(err)
}
- if wantChunk != nil {
- if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {
- return nil
- }
- return common.MatchError
+
+ data := x509.MarshalPKCS1PrivateKey(key)
+ var pemKey bytes.Buffer
+ if err := pem.Encode(&pemKey, &pem.Block{
+ Type: "RSA PRIVATE KEY",
+ Bytes: data,
+ }); err != nil {
+ panic(err)
}
- return nil
- }
-}
+ return pemKey.String()
+ }()
+
+ gock.New("https://api.github.com").
+ Post("/app/installations/1337/access_tokens").
+ Reply(200).
+ JSON(map[string]string{"token": "dontlook"})
+
+ gock.New("https://api.github.com").
+ Get("/installation/repositories").
+ Reply(200).
+ JSON(map[string]string{})
-// func TestSource_paginateRepos(t *testing.T) {
-// type args struct {
-// ctx context.Context
-// apiClient *github.Client
-// }
-// tests := []struct {
-// name string
-// org string
-// args args
-// }{
-// {
-// org: "fakeNetflix",
-// args: args{
-// ctx: context.Background(),
-// apiClient: github.NewClient(common.SaneHttpClient()),
-// },
-// },
-// }
-// for _, tt := range tests {
-// t.Run(tt.name, func(t *testing.T) {
-// s := &Source{httpClient: common.SaneHttpClient()}
-// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org)
-// if len(s.repos) < 101 {
-// t.Errorf("expected > 100 repos, got %d", len(s.repos))
-// }
-// })
-// }
-// }
+ s := initTestSource(nil)
+ _, _, err := s.enumerateWithApp(
+ context.TODO(),
+ "https://api.github.com",
+ &credentialspb.GitHubApp{
+ InstallationId: "1337",
+ AppId: "4141",
+ PrivateKey: privateKey,
+ },
+ )
+ fmt.Println(err)
+ assert.Nil(t, err)
+ assert.Equal(t, 0, len(s.repos))
+
+ assert.True(t, gock.IsDone())
+}