Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Provide correct username for app cloning and add integration test (#786)
Dustin Deckerecfdb0105b4c
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex f27e3ce4f..eb34f9983 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -364,12 +364,13 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chvar scanErrs []errorfor i, repoURL := range s.repos {- repoURL := repoURL+ i, repoURL := i, repoURLs.jobPool.Go(func() error {if common.IsDone(ctx) {return nil}+ // TODO: set progress complete is being called concurrently with is.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)// Ensure the repo is removed from the resume info after being scanned.defer func(s *Source, repoURL string) {@@ -388,22 +389,9 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chvar repo *gogit.Repositoryvar err error- switch s.conn.GetCredential().(type) {- case *sourcespb.GitHub_Unauthenticated:- path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)- if err != nil {- scanErrs = append(scanErrs, fmt.Errorf("error cloning repo %s: %w", repoURL, err))- }- default:- var token string- token, err = s.Token(ctx, installationClient)- if err != nil {- scanErrs = append(scanErrs, fmt.Errorf("error getting token for repo %s: %w", repoURL, err))- }- path, repo, err = git.CloneRepoUsingToken(token, repoURL, "")- if err != nil {- scanErrs = append(scanErrs, fmt.Errorf("error cloning repo %s: %w", repoURL, err))- }+ path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)+ if err != nil {+ scanErrs = append(scanErrs, err)}defer os.RemoveAll(path)@@ -435,6 +423,35 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chreturn scanErrs}+func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClient *github.Client) (string, *gogit.Repository, error) {+ var path string+ var repo *gogit.Repository+ var err error++ switch cred := s.conn.GetCredential().(type) {+ case *sourcespb.GitHub_Unauthenticated:+ path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)+ if err != nil {+ return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)+ }+ default:+ var token string+ token, err := s.Token(ctx, installationClient)+ if err != nil {+ return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)+ }+ user := ""+ if _, ok := cred.(*sourcespb.GitHub_GithubApp); ok {+ user = "x-access-token"+ }+ path, repo, err = git.CloneRepoUsingToken(token, repoURL, user)+ if err != nil {+ return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)+ }+ }+ return path, repo, nil+}+// handleRateLimit returns true if a rate limit was handled// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.// This will likely only be exhausted if many users/orgs are scanned without auth@@ -633,7 +650,7 @@ func (s *Source) addMembersByApp(ctx context.Context, installationClient *githubif res == nil {break}- s.log.Debugf("Listed members for org %s page %d/%d", org, opts.Page, res.LastPage)+ s.log.Debugf("Listed members for org %s page %d/%d", *org.Account.Login, opts.Page, res.LastPage)for _, m := range members {usr := m.Loginif usr == nil || *usr == "" {diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.goindex a58d4341a..8eea60bd8 100644--- a/pkg/sources/github/github_integration_test.go+++ b/pkg/sources/github/github_integration_test.go@@ -11,10 +11,10 @@ import ("time""github.com/google/go-github/v42/github"-"github.com/kylelemons/godebug/pretty""github.com/mattn/go-colorable"log "github.com/sirupsen/logrus"+ "github.com/stretchr/testify/assert""google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"@@ -23,8 +23,59 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git")+func TestSource_Token(t *testing.T) {+ log.SetLevel(log.DebugLevel)+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)+ defer cancel()++ secret, err := common.GetTestSecret(ctx)+ if err != nil {+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ }++ githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")+ githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)+ if err != nil {+ t.Fatal(err)+ }+ githubPrivateKeyNew := string(githubPrivateKeyBytesNew)+ githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")+ githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")++ conn := &sourcespb.GitHub{+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: githubPrivateKeyNew,+ InstallationId: githubInstallationIDNew,+ AppId: githubAppIDNew,+ },+ },+ }++ s := Source{+ conn: conn,+ httpClient: common.SaneHttpClient(),+ log: log.WithField("source", "github"),+ }++ _, installationClient, err := s.enumerateWithApp(ctx, "https://api.github.com", conn.GetGithubApp())+ assert.NoError(t, err)++ token, err := s.Token(ctx, installationClient)+ assert.NotEmpty(t, token)+ assert.NoError(t, err)++ _, _, err = git.CloneRepoUsingToken(token, "https://github.com/trufflesecurity/trufflehog-updater.git", "")+ assert.Error(t, err)++ _, _, err = s.cloneRepo(ctx, "https://github.com/trufflesecurity/trufflehog-updater.git", installationClient)+ assert.NoError(t, err)+}+func TestSource_Scan(t *testing.T) {os.Setenv("DO_NOT_RANDOMIZE", "true")diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 7e52f3ed7..bd8b21d08 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -17,13 +17,12 @@ import ("github.com/google/go-github/v42/github""github.com/sirupsen/logrus""github.com/stretchr/testify/assert"- "golang.org/x/sync/errgroup"- "google.golang.org/protobuf/types/known/anypb"- "gopkg.in/h2non/gock.v1"-"github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "golang.org/x/sync/errgroup"+ "google.golang.org/protobuf/types/known/anypb"+ "gopkg.in/h2non/gock.v1")func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {