Skip to content

Find, verify, and analyze leaked credentials

Complete support for existing git scan flags (#13)

Bill Rich28ed0c3b7caa
diff --git a/main.go b/main.go
index 6417d986c..e53c60f70 100644
--- a/main.go
+++ b/main.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "github.com/go-git/go-git/v5/plumbing"
"log"
"os"
"runtime"
@@ -32,9 +33,9 @@ func main() {
gitScanURI := gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
- // gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()
+ gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()
gitScanBranch := gitScan.Flag("branch", "Branch to scan.").String()
- // gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()
+ gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()
gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
@@ -95,7 +96,8 @@ func main() {
if remote {
defer os.RemoveAll(repoPath)
}
- err = e.ScanGit(ctx, repoPath, *gitScanBranch, "HEAD", filter)
+ sinceHash := plumbing.NewHash(*gitScanSinceCommit)
+ err = e.ScanGit(ctx, repoPath, *gitScanBranch, "HEAD", &sinceHash, *gitScanMaxDepth, filter)
if err != nil {
logrus.WithError(err).Fatal("Failed to scan git.")
}
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index 81e38adfb..1bd9e4f6f 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -3,6 +3,7 @@ package engine
import (
"context"
"fmt"
+ "github.com/go-git/go-git/v5/plumbing/object"
"runtime"
gogit "github.com/go-git/go-git/v5"
@@ -15,39 +16,33 @@ import (
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
)
-func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef string, filter *common.Filter) error {
+func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef string, sinceHash *plumbing.Hash, maxDepth int, filter *common.Filter) error {
repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
if err != nil {
return fmt.Errorf("could open repo: %s: %w", repoPath, err)
}
- scanOptions := &gogit.LogOptions{
+ logOptions := &gogit.LogOptions{
All: true,
Order: gogit.LogOrderCommitterTime,
PathFilter: func(s string) bool { return filter.Pass(s) },
}
- if gitScanBranch != "" {
- baseHash, err := git.TryAdditionalBaseRefs(repo, gitScanBranch)
+ var sinceCommit, headCommit *object.Commit
+ if !sinceHash.IsZero() {
+ sinceCommit, err = repo.CommitObject(*sinceHash)
if err != nil {
- return fmt.Errorf("could not parse base revision: %q: %w", gitScanBranch, err)
+ return fmt.Errorf("unable to resolve commit %s: %s", sinceHash.String(), err)
}
+ }
- headHash, err := repo.ResolveRevision(plumbing.Revision(headRef))
+ if gitScanBranch != "" {
+ headHash, err := git.TryAdditionalBaseRefs(repo, gitScanBranch)
if err != nil {
return fmt.Errorf("could not parse revision: %q: %w", headRef, err)
}
- baseCommit, err := repo.CommitObject(*baseHash)
- if err != nil {
- return fmt.Errorf("could not find commit: %q: %w", headRef, err)
- }
-
- logrus.WithFields(logrus.Fields{
- "commit": baseCommit.Hash.String(),
- }).Debug("resolved base reference")
-
- headCommit, err := repo.CommitObject(*headHash)
+ headCommit, err = repo.CommitObject(*headHash)
if err != nil {
return fmt.Errorf("could not find commit: %q: %w", headRef, err)
}
@@ -56,24 +51,13 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
"commit": headCommit.Hash.String(),
}).Debug("resolved head reference")
- mergeBase, err := baseCommit.MergeBase(headCommit)
- if err != nil {
- return fmt.Errorf("could not find common base between the given references: %q: %w", headRef, err)
- }
-
- if len(mergeBase) == 0 {
- return fmt.Errorf("no common mergeable base between the given references: %q: %w", headRef, err)
- }
-
- logrus.WithFields(logrus.Fields{
- "commit": mergeBase[0].Hash.String(),
- }).Debug("resolved common merge base between references")
+ logOptions.From = headCommit.Hash
+ logOptions.All = false
+ }
- scanOptions = &gogit.LogOptions{
- All: true,
- From: *headHash,
- Order: gogit.LogOrderCommitterTime,
- PathFilter: func(s string) bool { return filter.Pass(s) },
+ if sinceCommit != nil && headCommit != nil {
+ if ok, _ := sinceCommit.IsAncestor(headCommit); !ok {
+ return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", sinceCommit.Hash.String(), headCommit.Hash.String())
}
}
@@ -91,8 +75,21 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
}
})
+ opts := []git.ScanOption{
+ git.ScanOptionFilter(filter),
+ git.ScanOptionLogOptions(logOptions),
+ }
+ // TODO: Add kingpin type that can differentiate between `not set` and `0` for int.
+ if maxDepth != 0 {
+ opts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))
+ }
+ if sinceCommit != nil {
+ opts = append(opts, git.ScanOptionSinceCommit(sinceCommit))
+ }
+ scanOptions := git.NewScanOptions(opts...)
+
go func() {
- err := gitSource.ScanRepo(ctx, repo, scanOptions, nil, filter, e.ChunksChan())
+ err := gitSource.ScanRepo(ctx, repo, scanOptions, e.ChunksChan())
if err != nil {
logrus.WithError(err).Fatal("could not scan repo")
}
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 5bc32934c..8365b27c9 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -137,7 +137,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if err != nil {
return err
}
- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan)
if err != nil {
return err
}
@@ -153,7 +153,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if err != nil {
return err
}
- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan)
if err != nil {
return err
}
@@ -178,7 +178,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
defer os.RemoveAll(u)
}
- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan)
if err != nil {
return err
@@ -254,7 +254,7 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Repo
return
}
-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOptions, untilCommit *object.Commit, filter *common.Filter, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
wg := &sync.WaitGroup{}
remote, err := repo.Remote("origin")
@@ -266,18 +266,19 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO
return errors.New(err)
}
- commitIter, err := repo.Log(opts)
+ commitIter, err := repo.Log(scanOptions.LogOptions)
if err != nil {
return errors.New(err)
}
scanOneCommit := false
- if untilCommit != nil && opts.From.String() == untilCommit.Hash.String() {
+ if scanOptions.SinceCommit != nil && scanOptions.LogOptions.From.String() == scanOptions.SinceCommit.Hash.String() {
// Our head and base commits are the same, so scan the one commit
scanOneCommit = true
}
breakIteration := false
+ depth := int64(0)
err = commitIter.ForEach(func(commit *object.Commit) error {
if breakIteration {
return storer.ErrStop
@@ -286,19 +287,14 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO
breakIteration = true
}
- if untilCommit != nil && commit.Hash == untilCommit.Hash && !scanOneCommit {
+ // TODO: Clean up this conditional mess.
+ if (scanOptions.SinceCommit != nil && commit.Hash == scanOptions.SinceCommit.Hash && !scanOneCommit) ||
+ (scanOptions.MaxDepth >= 0 && depth >= scanOptions.MaxDepth) {
return storer.ErrStop
}
+ depth++
- // err := s.sem.Acquire(ctx, 1)
- // if err != nil {
- // return err
- // }
- // wg.Add(1)
- // go func(wg *sync.WaitGroup) {
- // defer wg.Done()
- // defer s.sem.Release(1)
- err = s.scanCommitPatches(ctx, repo, commit, chunksChan, filter)
+ err = s.scanCommitPatches(ctx, repo, commit, chunksChan, scanOptions.Filter)
if err != nil {
switch e := err.Error(); {
case strings.Contains(e, "operation canceled"):
@@ -315,7 +311,6 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO
return err
}
}
- // }(wg)
return nil
})
if err != nil {
@@ -342,7 +337,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO
return err
}
for fh := range status {
- if !filter.Pass(fh) {
+ if !scanOptions.Filter.Pass(fh) {
continue
}
metadata := s.sourceMetadataFunc(
diff --git a/pkg/sources/git/scan_options.go b/pkg/sources/git/scan_options.go
new file mode 100644
index 000000000..2d0155b04
--- /dev/null
+++ b/pkg/sources/git/scan_options.go
@@ -0,0 +1,56 @@
+package git
+
+import (
+ "github.com/go-git/go-git/v5"
+ "github.com/go-git/go-git/v5/plumbing/object"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type ScanOptions struct {
+ Filter *common.Filter
+ SinceCommit *object.Commit // When scanning a git.Log, this is the oldest/first commit.
+ MaxDepth int64
+ LogOptions *git.LogOptions
+}
+
+type ScanOption func(*ScanOptions)
+
+func ScanOptionFilter(filter *common.Filter) ScanOption {
+ return func(scanOptions *ScanOptions) {
+ scanOptions.Filter = filter
+ }
+}
+
+func ScanOptionSinceCommit(commit *object.Commit) ScanOption {
+ return func(scanOptions *ScanOptions) {
+ scanOptions.SinceCommit = commit
+ }
+}
+
+func ScanOptionMaxDepth(maxDepth int64) ScanOption {
+ return func(scanOptions *ScanOptions) {
+ scanOptions.MaxDepth = maxDepth
+ }
+}
+
+func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {
+ return func(scanOptions *ScanOptions) {
+ scanOptions.LogOptions = logOptions
+ }
+}
+
+func NewScanOptions(options ...ScanOption) *ScanOptions {
+ scanOptions := &ScanOptions{
+ Filter: common.FilterEmpty(),
+ SinceCommit: nil,
+ MaxDepth: -1,
+ LogOptions: &git.LogOptions{
+ All: true,
+ Order: git.LogOrderCommitterTime,
+ },
+ }
+ for _, option := range options {
+ option(scanOptions)
+ }
+ return scanOptions
+}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index f6f8967ab..5e39652a1 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -328,7 +328,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
log.WithError(err).Warnf("unable to clone repo, continuing")
continue
}
- err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)
if err != nil {
log.WithError(err).Warnf("unable to scan repo")
}
@@ -422,7 +422,7 @@ func (s *Source) paginateGists(ctx context.Context, user string, chunksChan chan
s.log.WithField("repo", *gist.HTMLURL).Debugf("attempting to clone gist from user %s", user)
scanCtx := context.Background()
- err = s.git.ScanRepo(scanCtx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(scanCtx, repo, git.NewScanOptions(), chunksChan)
if err != nil {
log.WithError(err).Warnf("Could not scan after clone: %s", *gist.HTMLURL)
continue
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 4d40162c5..6fa14f4a8 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -8,20 +8,16 @@ import (
"runtime"
"github.com/go-errors/errors"
- gogit "github.com/go-git/go-git/v5"
log "github.com/sirupsen/logrus"
- "github.com/xanzy/go-gitlab"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/types/known/anypb"
-
- "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/giturl"
"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
-
- "github.com/trufflesecurity/trufflehog/pkg/giturl"
"github.com/trufflesecurity/trufflehog/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/pkg/sources"
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
+ "github.com/xanzy/go-gitlab"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
)
type Source struct {
@@ -216,7 +212,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
errors = append(errors, err)
continue
}
- err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)
if err != nil {
errors = append(errors, err)
continue
@@ -237,7 +233,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
errors = append(errors, err)
continue
}
- err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)
if err != nil {
errors = append(errors, err)
continue