Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fast git scanning (#40)
Bill Rich2d8756938d3b
diff --git a/pkg/common/test_helpers.go b/pkg/common/test_helpers.gonew file mode 100644index 000000000..1ccc07884--- /dev/null+++ b/pkg/common/test_helpers.go@@ -0,0 +1,30 @@+package common++import (+ "errors"+ "fmt"+ "github.com/trufflesecurity/trufflehog/pkg/sources"+ "time"+)++type ChunkFunc func(chunk *sources.Chunk) error++var MatchError error = errors.New("chunk doesn't match")++func HandleTestChannel(chunksCh chan *sources.Chunk, cf ChunkFunc) error {+ for {+ select {+ case gotChunk := <-chunksCh:+ err := cf(gotChunk)+ if err != nil {+ if errors.Is(err, MatchError) {+ continue+ }+ return err+ }+ return nil+ case <-time.After(10 * time.Second):+ return fmt.Errorf("no new chunks recieved after 10 seconds")+ }+ }+}diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 4b662a582..8d65964cd 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -10,8 +10,8 @@ import ("os""path/filepath""runtime"+ "sort""strings"- "sync""time""github.com/go-errors/errors"@@ -19,7 +19,6 @@ import ("github.com/go-git/go-git/v5/plumbing""github.com/go-git/go-git/v5/plumbing/format/diff""github.com/go-git/go-git/v5/plumbing/object"- "github.com/go-git/go-git/v5/plumbing/storer""github.com/go-git/go-git/v5/plumbing/transport/http"log "github.com/sirupsen/logrus""golang.org/x/sync/semaphore"@@ -256,72 +255,102 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Reporeturn}-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {- wg := &sync.WaitGroup{}+func (s *Git) ScanCommits(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+ blobsSeen := map[string]bool{}+ commits := map[int64][]*object.Commit{}- remote, err := repo.Remote("origin")+ logIter, err := repo.Log(scanOptions.LogOptions)if err != nil {- return errors.New(err)+ log.Fatal(err)}- safeRepo, err := stripPassword(remote.Config().URLs[0])++ keys := []int64{}++ sinceTime := int64(0)++ logIter.ForEach(func(commit *object.Commit) error {+ key := commit.Committer.When.Unix()+ if scanOptions.SinceCommit != nil && scanOptions.SinceCommit.Hash.String() == commit.Hash.String() {+ sinceTime = key+ }+ if existing, ok := commits[key]; ok {+ commits[key] = append(existing, commit)+ } else {+ commits[key] = []*object.Commit{commit}+ keys = append(keys, key)+ }+ return nil+ })++ sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })++ remote, err := repo.Remote("origin")if err != nil {return errors.New(err)}-- commitIter, err := repo.Log(scanOptions.LogOptions)+ safeRepo, err := stripPassword(remote.Config().URLs[0])if err != nil {return errors.New(err)}- scanOneCommit := false- if scanOptions.SinceCommit != nil && scanOptions.LogOptions.From.String() == scanOptions.SinceCommit.Hash.String() {- // Our head and base commits are the same, so scan the one commit- scanOneCommit = true+ start := int64(0)+ if scanOptions.MaxDepth > 0 && scanOptions.MaxDepth < int64(len(keys)) {+ start = int64(len(keys)) - scanOptions.MaxDepth}- breakIteration := false-- depth := int64(0)- err = commitIter.ForEach(func(commit *object.Commit) error {- if breakIteration {- return storer.ErrStop- }- if scanOneCommit {- breakIteration = true- }- if FilterCommitByFiles(commit, scanOptions.Filter) {- return nil- }-- // TODO: Clean up this conditional mess.- if (scanOptions.SinceCommit != nil && commit.Hash == scanOptions.SinceCommit.Hash && !scanOneCommit) ||- (scanOptions.MaxDepth >= 0 && depth >= scanOptions.MaxDepth) {- return storer.ErrStop+ scannedCommits := []string{}+ for _, key := range keys[start:] {+ if sinceTime > key {+ continue}- depth++-- err = s.scanCommitPatches(ctx, repo, commit, chunksChan, scanOptions.Filter)- if err != nil {- switch e := err.Error(); {- case strings.Contains(e, "operation canceled"):- return nil- case strings.Contains(e, "packfile not found"):- log.WithError(err).WithField("repo", safeRepo).Warn("invalid commit reference while scanning commit")+ for _, commit := range commits[key] {+ scannedCommits = append(scannedCommits, commit.Hash.String())+ fileIter, err := commit.Files()+ if err != nil {+ return err+ }+ err = fileIter.ForEach(func(file *object.File) error {+ if _, scanned := blobsSeen[file.Hash.String()]; scanned || !scanOptions.Filter.Pass(file.Name) {+ return nil+ }+ blobsSeen[file.Hash.String()] = true+ metadata := s.sourceMetadataFunc(file.Name, commit.Author.Email, commit.Hash.String(), safeRepo)+ reader, err := file.Reader()+ if err != nil {+ return nil+ }+ defer reader.Close()+ buffer := new(bytes.Buffer)+ buffer.ReadFrom(reader)+ chunksChan <- &sources.Chunk{+ SourceType: s.sourceType,+ SourceName: s.sourceName,+ SourceID: s.sourceID,+ Data: buffer.Bytes(),+ SourceMetadata: metadata,+ Verify: s.verify,+ }return nil- default:- log.WithError(err).WithField("repo", safeRepo).Error("unhandled error scanning commit")+ })+ if err != nil {return err}}- return nil- })+ }+ if err != nil && !strings.Contains(err.Error(), "max_depth exceeded") && !strings.Contains(err.Error(), "reached since_commit") {+ return err+ }+ return nil+}++func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+ remote, err := repo.Remote("origin")if err != nil {- // https://github.com/src-d/go-git/issues/879- if strings.Contains(err.Error(), "object not found") {- log.WithError(err).Error("known issue: probably caused by a dangling reference in the repo")- } else {- return errors.New(err)- }+ return errors.New(err)+ }+ safeRepo, err := stripPassword(remote.Config().URLs[0])+ if err != nil {+ return errors.New(err)}// Also scan any unstaged changes in the working tree of the repo@@ -366,8 +395,22 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *S}}}+ return nil+}- wg.Wait()+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+ if err := s.ScanCommits(repo, scanOptions, chunksChan); err != nil {+ return err+ }+ if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil {+ // https://github.com/src-d/go-git/issues/879+ if strings.Contains(err.Error(), "object not found") {+ log.WithError(err).Error("known issue: probably caused by a dangling reference in the repo")+ } else {+ return errors.New(err)+ }+ return err+ }return nil}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 089e4b203..1df5b83e9 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -172,7 +172,6 @@ func TestSource_Chunks_Integration(t *testing.T) {"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte(" s \noops \nf\n")},"73ab4713057944753f1bdeb80e757380e64c6b5b-bump": {B: []byte(" s \noops \n")},"2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte(" \noops \n")},- "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("blah blaj\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n")},"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("\noops \n")},"735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("\noops\n")},"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 0d5189a42..85d6f9536 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -3,7 +3,6 @@ package githubimport ("context""encoding/base64"- "errors""fmt""github.com/google/go-github/v42/github""os"@@ -22,12 +21,7 @@ import ("github.com/trufflesecurity/trufflehog/pkg/sources")-type chunkFunc func(chunk *sources.Chunk) error--var MatchError error = errors.New("chunk doesn't match")-func TestSource_Scan(t *testing.T) {-os.Setenv("DO_NOT_RANDOMIZE", "true")ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)@@ -341,7 +335,7 @@ func TestSource_Scan(t *testing.T) {return}}()- if err = handleChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {+ if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {t.Error(err)}})@@ -488,14 +482,14 @@ func TestSource_paginateGists(t *testing.T) {if tt.wantChunk != nil {wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository}- if err = handleChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {+ if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {t.Error(err)}})}}-func gistsCheckFunc(expected string, minRepos int, s *Source) chunkFunc {+func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {return func(chunk *sources.Chunk) error {if minRepos != 0 && minRepos > len(s.repos) {return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))@@ -512,7 +506,7 @@ func gistsCheckFunc(expected string, minRepos int, s *Source) chunkFunc {}}-func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) chunkFunc {+func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {return func(chunk *sources.Chunk) error {if minOrg != 0 && minOrg > len(s.orgs) {return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))@@ -524,30 +518,12 @@ func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) chif diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {return nil}- return MatchError+ return common.MatchError}return nil}}-func handleChannel(chunksCh chan *sources.Chunk, cf chunkFunc) error {- for {- select {- case gotChunk := <-chunksCh:- err := cf(gotChunk)- if err != nil {- if errors.Is(err, MatchError) {- continue- }- return err- }- return nil- case <-time.After(10 * time.Second):- return fmt.Errorf("no new chunks recieved after 10 seconds")- }- }-}-// func TestSource_paginateRepos(t *testing.T) {// type args struct {// ctx context.Context