Skip to content

Find, verify, and analyze leaked credentials

Provide correct username for app cloning and add integration test (#786)

Dustin Deckerecfdb0105b4c
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index f27e3ce4f..eb34f9983 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -364,12 +364,13 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
var scanErrs []error
for i, repoURL := range s.repos {
- repoURL := repoURL
+ i, repoURL := i, repoURL
s.jobPool.Go(func() error {
if common.IsDone(ctx) {
return nil
}
+ // TODO: set progress complete is being called concurrently with i
s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
// Ensure the repo is removed from the resume info after being scanned.
defer func(s *Source, repoURL string) {
@@ -388,22 +389,9 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
var repo *gogit.Repository
var err error
- switch s.conn.GetCredential().(type) {
- case *sourcespb.GitHub_Unauthenticated:
- path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)
- if err != nil {
- scanErrs = append(scanErrs, fmt.Errorf("error cloning repo %s: %w", repoURL, err))
- }
- default:
- var token string
- token, err = s.Token(ctx, installationClient)
- if err != nil {
- scanErrs = append(scanErrs, fmt.Errorf("error getting token for repo %s: %w", repoURL, err))
- }
- path, repo, err = git.CloneRepoUsingToken(token, repoURL, "")
- if err != nil {
- scanErrs = append(scanErrs, fmt.Errorf("error cloning repo %s: %w", repoURL, err))
- }
+ path, repo, err = s.cloneRepo(ctx, repoURL, installationClient)
+ if err != nil {
+ scanErrs = append(scanErrs, err)
}
defer os.RemoveAll(path)
@@ -435,6 +423,35 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
return scanErrs
}
+func (s *Source) cloneRepo(ctx context.Context, repoURL string, installationClient *github.Client) (string, *gogit.Repository, error) {
+ var path string
+ var repo *gogit.Repository
+ var err error
+
+ switch cred := s.conn.GetCredential().(type) {
+ case *sourcespb.GitHub_Unauthenticated:
+ path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)
+ if err != nil {
+ return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
+ }
+ default:
+ var token string
+ token, err := s.Token(ctx, installationClient)
+ if err != nil {
+ return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err)
+ }
+ user := ""
+ if _, ok := cred.(*sourcespb.GitHub_GithubApp); ok {
+ user = "x-access-token"
+ }
+ path, repo, err = git.CloneRepoUsingToken(token, repoURL, user)
+ if err != nil {
+ return "", nil, fmt.Errorf("error cloning repo %s: %w", repoURL, err)
+ }
+ }
+ return path, repo, nil
+}
+
// handleRateLimit returns true if a rate limit was handled
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
// This will likely only be exhausted if many users/orgs are scanned without auth
@@ -633,7 +650,7 @@ func (s *Source) addMembersByApp(ctx context.Context, installationClient *github
if res == nil {
break
}
- s.log.Debugf("Listed members for org %s page %d/%d", org, opts.Page, res.LastPage)
+ s.log.Debugf("Listed members for org %s page %d/%d", *org.Account.Login, opts.Page, res.LastPage)
for _, m := range members {
usr := m.Login
if usr == nil || *usr == "" {
diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go
index a58d4341a..8eea60bd8 100644
--- a/pkg/sources/github/github_integration_test.go
+++ b/pkg/sources/github/github_integration_test.go
@@ -11,10 +11,10 @@ import (
"time"
"github.com/google/go-github/v42/github"
-
"github.com/kylelemons/godebug/pretty"
"github.com/mattn/go-colorable"
log "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -23,8 +23,59 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
+func TestSource_Token(t *testing.T) {
+ log.SetLevel(log.DebugLevel)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
+ defer cancel()
+
+ secret, err := common.GetTestSecret(ctx)
+ if err != nil {
+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+ }
+
+ githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW")
+ githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New)
+ if err != nil {
+ t.Fatal(err)
+ }
+ githubPrivateKeyNew := string(githubPrivateKeyBytesNew)
+ githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW")
+ githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW")
+
+ conn := &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: githubPrivateKeyNew,
+ InstallationId: githubInstallationIDNew,
+ AppId: githubAppIDNew,
+ },
+ },
+ }
+
+ s := Source{
+ conn: conn,
+ httpClient: common.SaneHttpClient(),
+ log: log.WithField("source", "github"),
+ }
+
+ _, installationClient, err := s.enumerateWithApp(ctx, "https://api.github.com", conn.GetGithubApp())
+ assert.NoError(t, err)
+
+ token, err := s.Token(ctx, installationClient)
+ assert.NotEmpty(t, token)
+ assert.NoError(t, err)
+
+ _, _, err = git.CloneRepoUsingToken(token, "https://github.com/trufflesecurity/trufflehog-updater.git", "")
+ assert.Error(t, err)
+
+ _, _, err = s.cloneRepo(ctx, "https://github.com/trufflesecurity/trufflehog-updater.git", installationClient)
+ assert.NoError(t, err)
+}
+
func TestSource_Scan(t *testing.T) {
os.Setenv("DO_NOT_RANDOMIZE", "true")
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 7e52f3ed7..bd8b21d08 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -17,13 +17,12 @@ import (
"github.com/google/go-github/v42/github"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
- "golang.org/x/sync/errgroup"
- "google.golang.org/protobuf/types/known/anypb"
- "gopkg.in/h2non/gock.v1"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/protobuf/types/known/anypb"
+ "gopkg.in/h2non/gock.v1"
)
func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) {