Skip to content

Find, verify, and analyze leaked credentials

Fast git scanning (#40)

Bill Rich2d8756938d3b
diff --git a/pkg/common/test_helpers.go b/pkg/common/test_helpers.go
new file mode 100644
index 000000000..1ccc07884
--- /dev/null
+++ b/pkg/common/test_helpers.go
@@ -0,0 +1,30 @@
+package common
+
+import (
+ "errors"
+ "fmt"
+ "github.com/trufflesecurity/trufflehog/pkg/sources"
+ "time"
+)
+
+type ChunkFunc func(chunk *sources.Chunk) error
+
+var MatchError error = errors.New("chunk doesn't match")
+
+func HandleTestChannel(chunksCh chan *sources.Chunk, cf ChunkFunc) error {
+ for {
+ select {
+ case gotChunk := <-chunksCh:
+ err := cf(gotChunk)
+ if err != nil {
+ if errors.Is(err, MatchError) {
+ continue
+ }
+ return err
+ }
+ return nil
+ case <-time.After(10 * time.Second):
+ return fmt.Errorf("no new chunks recieved after 10 seconds")
+ }
+ }
+}
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 4b662a582..8d65964cd 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -10,8 +10,8 @@ import (
"os"
"path/filepath"
"runtime"
+ "sort"
"strings"
- "sync"
"time"
"github.com/go-errors/errors"
@@ -19,7 +19,6 @@ import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/format/diff"
"github.com/go-git/go-git/v5/plumbing/object"
- "github.com/go-git/go-git/v5/plumbing/storer"
"github.com/go-git/go-git/v5/plumbing/transport/http"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
@@ -256,72 +255,102 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Repo
return
}
-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
- wg := &sync.WaitGroup{}
+func (s *Git) ScanCommits(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+ blobsSeen := map[string]bool{}
+ commits := map[int64][]*object.Commit{}
- remote, err := repo.Remote("origin")
+ logIter, err := repo.Log(scanOptions.LogOptions)
if err != nil {
- return errors.New(err)
+ log.Fatal(err)
}
- safeRepo, err := stripPassword(remote.Config().URLs[0])
+
+ keys := []int64{}
+
+ sinceTime := int64(0)
+
+ logIter.ForEach(func(commit *object.Commit) error {
+ key := commit.Committer.When.Unix()
+ if scanOptions.SinceCommit != nil && scanOptions.SinceCommit.Hash.String() == commit.Hash.String() {
+ sinceTime = key
+ }
+ if existing, ok := commits[key]; ok {
+ commits[key] = append(existing, commit)
+ } else {
+ commits[key] = []*object.Commit{commit}
+ keys = append(keys, key)
+ }
+ return nil
+ })
+
+ sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+
+ remote, err := repo.Remote("origin")
if err != nil {
return errors.New(err)
}
-
- commitIter, err := repo.Log(scanOptions.LogOptions)
+ safeRepo, err := stripPassword(remote.Config().URLs[0])
if err != nil {
return errors.New(err)
}
- scanOneCommit := false
- if scanOptions.SinceCommit != nil && scanOptions.LogOptions.From.String() == scanOptions.SinceCommit.Hash.String() {
- // Our head and base commits are the same, so scan the one commit
- scanOneCommit = true
+ start := int64(0)
+ if scanOptions.MaxDepth > 0 && scanOptions.MaxDepth < int64(len(keys)) {
+ start = int64(len(keys)) - scanOptions.MaxDepth
}
- breakIteration := false
-
- depth := int64(0)
- err = commitIter.ForEach(func(commit *object.Commit) error {
- if breakIteration {
- return storer.ErrStop
- }
- if scanOneCommit {
- breakIteration = true
- }
- if FilterCommitByFiles(commit, scanOptions.Filter) {
- return nil
- }
-
- // TODO: Clean up this conditional mess.
- if (scanOptions.SinceCommit != nil && commit.Hash == scanOptions.SinceCommit.Hash && !scanOneCommit) ||
- (scanOptions.MaxDepth >= 0 && depth >= scanOptions.MaxDepth) {
- return storer.ErrStop
+ scannedCommits := []string{}
+ for _, key := range keys[start:] {
+ if sinceTime > key {
+ continue
}
- depth++
-
- err = s.scanCommitPatches(ctx, repo, commit, chunksChan, scanOptions.Filter)
- if err != nil {
- switch e := err.Error(); {
- case strings.Contains(e, "operation canceled"):
- return nil
- case strings.Contains(e, "packfile not found"):
- log.WithError(err).WithField("repo", safeRepo).Warn("invalid commit reference while scanning commit")
+ for _, commit := range commits[key] {
+ scannedCommits = append(scannedCommits, commit.Hash.String())
+ fileIter, err := commit.Files()
+ if err != nil {
+ return err
+ }
+ err = fileIter.ForEach(func(file *object.File) error {
+ if _, scanned := blobsSeen[file.Hash.String()]; scanned || !scanOptions.Filter.Pass(file.Name) {
+ return nil
+ }
+ blobsSeen[file.Hash.String()] = true
+ metadata := s.sourceMetadataFunc(file.Name, commit.Author.Email, commit.Hash.String(), safeRepo)
+ reader, err := file.Reader()
+ if err != nil {
+ return nil
+ }
+ defer reader.Close()
+ buffer := new(bytes.Buffer)
+ buffer.ReadFrom(reader)
+ chunksChan <- &sources.Chunk{
+ SourceType: s.sourceType,
+ SourceName: s.sourceName,
+ SourceID: s.sourceID,
+ Data: buffer.Bytes(),
+ SourceMetadata: metadata,
+ Verify: s.verify,
+ }
return nil
- default:
- log.WithError(err).WithField("repo", safeRepo).Error("unhandled error scanning commit")
+ })
+ if err != nil {
return err
}
}
- return nil
- })
+ }
+ if err != nil && !strings.Contains(err.Error(), "max_depth exceeded") && !strings.Contains(err.Error(), "reached since_commit") {
+ return err
+ }
+ return nil
+}
+
+func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+ remote, err := repo.Remote("origin")
if err != nil {
- // https://github.com/src-d/go-git/issues/879
- if strings.Contains(err.Error(), "object not found") {
- log.WithError(err).Error("known issue: probably caused by a dangling reference in the repo")
- } else {
- return errors.New(err)
- }
+ return errors.New(err)
+ }
+ safeRepo, err := stripPassword(remote.Config().URLs[0])
+ if err != nil {
+ return errors.New(err)
}
// Also scan any unstaged changes in the working tree of the repo
@@ -366,8 +395,22 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *S
}
}
}
+ return nil
+}
- wg.Wait()
+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+ if err := s.ScanCommits(repo, scanOptions, chunksChan); err != nil {
+ return err
+ }
+ if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil {
+ // https://github.com/src-d/go-git/issues/879
+ if strings.Contains(err.Error(), "object not found") {
+ log.WithError(err).Error("known issue: probably caused by a dangling reference in the repo")
+ } else {
+ return errors.New(err)
+ }
+ return err
+ }
return nil
}
diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go
index 089e4b203..1df5b83e9 100644
--- a/pkg/sources/git/git_test.go
+++ b/pkg/sources/git/git_test.go
@@ -172,7 +172,6 @@ func TestSource_Chunks_Integration(t *testing.T) {
"a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte(" s \noops \nf\n")},
"73ab4713057944753f1bdeb80e757380e64c6b5b-bump": {B: []byte(" s \noops \n")},
"2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte(" \noops \n")},
- "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("blah blaj\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n")},
"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("\noops \n")},
"735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("\noops\n")},
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 0d5189a42..85d6f9536 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -3,7 +3,6 @@ package github
import (
"context"
"encoding/base64"
- "errors"
"fmt"
"github.com/google/go-github/v42/github"
"os"
@@ -22,12 +21,7 @@ import (
"github.com/trufflesecurity/trufflehog/pkg/sources"
)
-type chunkFunc func(chunk *sources.Chunk) error
-
-var MatchError error = errors.New("chunk doesn't match")
-
func TestSource_Scan(t *testing.T) {
-
os.Setenv("DO_NOT_RANDOMIZE", "true")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*300)
@@ -341,7 +335,7 @@ func TestSource_Scan(t *testing.T) {
return
}
}()
- if err = handleChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {
+ if err = common.HandleTestChannel(chunksCh, basicCheckFunc(tt.minOrg, tt.minRepo, tt.wantChunk, &s)); err != nil {
t.Error(err)
}
})
@@ -488,14 +482,14 @@ func TestSource_paginateGists(t *testing.T) {
if tt.wantChunk != nil {
wantedRepo = tt.wantChunk.SourceMetadata.GetGithub().Repository
}
- if err = handleChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {
+ if err = common.HandleTestChannel(chunksCh, gistsCheckFunc(wantedRepo, tt.minRepos, &s)); err != nil {
t.Error(err)
}
})
}
}
-func gistsCheckFunc(expected string, minRepos int, s *Source) chunkFunc {
+func gistsCheckFunc(expected string, minRepos int, s *Source) common.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minRepos != 0 && minRepos > len(s.repos) {
return fmt.Errorf("didn't find enough repos. expected: %d, got :%d", minRepos, len(s.repos))
@@ -512,7 +506,7 @@ func gistsCheckFunc(expected string, minRepos int, s *Source) chunkFunc {
}
}
-func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) chunkFunc {
+func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) common.ChunkFunc {
return func(chunk *sources.Chunk) error {
if minOrg != 0 && minOrg > len(s.orgs) {
return fmt.Errorf("incorrect number of orgs. expected at least: %d, got %d", minOrg, len(s.orgs))
@@ -524,30 +518,12 @@ func basicCheckFunc(minOrg, minRepo int, wantChunk *sources.Chunk, s *Source) ch
if diff := pretty.Compare(chunk.SourceMetadata.GetGithub().Repository, wantChunk.SourceMetadata.GetGithub().Repository); diff == "" {
return nil
}
- return MatchError
+ return common.MatchError
}
return nil
}
}
-func handleChannel(chunksCh chan *sources.Chunk, cf chunkFunc) error {
- for {
- select {
- case gotChunk := <-chunksCh:
- err := cf(gotChunk)
- if err != nil {
- if errors.Is(err, MatchError) {
- continue
- }
- return err
- }
- return nil
- case <-time.After(10 * time.Second):
- return fmt.Errorf("no new chunks recieved after 10 seconds")
- }
- }
-}
-
// func TestSource_paginateRepos(t *testing.T) {
// type args struct {
// ctx context.Context