Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Complete support for existing git scan flags (#13)
Bill Rich28ed0c3b7caa
diff --git a/main.go b/main.goindex 6417d986c..e53c60f70 100644--- a/main.go+++ b/main.go@@ -4,6 +4,7 @@ import ("context""encoding/json""fmt"+ "github.com/go-git/go-git/v5/plumbing""log""os""runtime"@@ -32,9 +33,9 @@ func main() {gitScanURI := gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()- // gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()+ gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()gitScanBranch := gitScan.Flag("branch", "Branch to scan.").String()- // gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()+ gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()@@ -95,7 +96,8 @@ func main() {if remote {defer os.RemoveAll(repoPath)}- err = e.ScanGit(ctx, repoPath, *gitScanBranch, "HEAD", filter)+ sinceHash := plumbing.NewHash(*gitScanSinceCommit)+ err = e.ScanGit(ctx, repoPath, *gitScanBranch, "HEAD", &sinceHash, *gitScanMaxDepth, filter)if err != nil {logrus.WithError(err).Fatal("Failed to scan git.")}diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex 81e38adfb..1bd9e4f6f 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -3,6 +3,7 @@ package engineimport ("context""fmt"+ "github.com/go-git/go-git/v5/plumbing/object""runtime"gogit "github.com/go-git/go-git/v5"@@ -15,39 +16,33 @@ import ("github.com/trufflesecurity/trufflehog/pkg/sources/git")-func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef string, filter *common.Filter) error {+func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef string, sinceHash *plumbing.Hash, maxDepth int, filter *common.Filter) error {repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})if err != nil {return fmt.Errorf("could open repo: %s: %w", repoPath, err)}- scanOptions := &gogit.LogOptions{+ logOptions := &gogit.LogOptions{All: true,Order: gogit.LogOrderCommitterTime,PathFilter: func(s string) bool { return filter.Pass(s) },}- if gitScanBranch != "" {- baseHash, err := git.TryAdditionalBaseRefs(repo, gitScanBranch)+ var sinceCommit, headCommit *object.Commit+ if !sinceHash.IsZero() {+ sinceCommit, err = repo.CommitObject(*sinceHash)if err != nil {- return fmt.Errorf("could not parse base revision: %q: %w", gitScanBranch, err)+ return fmt.Errorf("unable to resolve commit %s: %s", sinceHash.String(), err)}+ }- headHash, err := repo.ResolveRevision(plumbing.Revision(headRef))+ if gitScanBranch != "" {+ headHash, err := git.TryAdditionalBaseRefs(repo, gitScanBranch)if err != nil {return fmt.Errorf("could not parse revision: %q: %w", headRef, err)}- baseCommit, err := repo.CommitObject(*baseHash)- if err != nil {- return fmt.Errorf("could not find commit: %q: %w", headRef, err)- }-- logrus.WithFields(logrus.Fields{- "commit": baseCommit.Hash.String(),- }).Debug("resolved base reference")-- headCommit, err := repo.CommitObject(*headHash)+ headCommit, err = repo.CommitObject(*headHash)if err != nil {return fmt.Errorf("could not find commit: %q: %w", headRef, err)}@@ -56,24 +51,13 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s"commit": headCommit.Hash.String(),}).Debug("resolved head reference")- mergeBase, err := baseCommit.MergeBase(headCommit)- if err != nil {- return fmt.Errorf("could not find common base between the given references: %q: %w", headRef, err)- }-- if len(mergeBase) == 0 {- return fmt.Errorf("no common mergeable base between the given references: %q: %w", headRef, err)- }-- logrus.WithFields(logrus.Fields{- "commit": mergeBase[0].Hash.String(),- }).Debug("resolved common merge base between references")+ logOptions.From = headCommit.Hash+ logOptions.All = false+ }- scanOptions = &gogit.LogOptions{- All: true,- From: *headHash,- Order: gogit.LogOrderCommitterTime,- PathFilter: func(s string) bool { return filter.Pass(s) },+ if sinceCommit != nil && headCommit != nil {+ if ok, _ := sinceCommit.IsAncestor(headCommit); !ok {+ return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", sinceCommit.Hash.String(), headCommit.Hash.String())}}@@ -91,8 +75,21 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s}})+ opts := []git.ScanOption{+ git.ScanOptionFilter(filter),+ git.ScanOptionLogOptions(logOptions),+ }+ // TODO: Add kingpin type that can differentiate between `not set` and `0` for int.+ if maxDepth != 0 {+ opts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))+ }+ if sinceCommit != nil {+ opts = append(opts, git.ScanOptionSinceCommit(sinceCommit))+ }+ scanOptions := git.NewScanOptions(opts...)+go func() {- err := gitSource.ScanRepo(ctx, repo, scanOptions, nil, filter, e.ChunksChan())+ err := gitSource.ScanRepo(ctx, repo, scanOptions, e.ChunksChan())if err != nil {logrus.WithError(err).Fatal("could not scan repo")}diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 5bc32934c..8365b27c9 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -137,7 +137,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errif err != nil {return err}- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan)if err != nil {return err}@@ -153,7 +153,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errif err != nil {return err}- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan)if err != nil {return err}@@ -178,7 +178,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errdefer os.RemoveAll(u)}- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, NewScanOptions(), chunksChan)if err != nil {return err@@ -254,7 +254,7 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Reporeturn}-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOptions, untilCommit *object.Commit, filter *common.Filter, chunksChan chan *sources.Chunk) error {+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {wg := &sync.WaitGroup{}remote, err := repo.Remote("origin")@@ -266,18 +266,19 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOreturn errors.New(err)}- commitIter, err := repo.Log(opts)+ commitIter, err := repo.Log(scanOptions.LogOptions)if err != nil {return errors.New(err)}scanOneCommit := false- if untilCommit != nil && opts.From.String() == untilCommit.Hash.String() {+ if scanOptions.SinceCommit != nil && scanOptions.LogOptions.From.String() == scanOptions.SinceCommit.Hash.String() {// Our head and base commits are the same, so scan the one commitscanOneCommit = true}breakIteration := false+ depth := int64(0)err = commitIter.ForEach(func(commit *object.Commit) error {if breakIteration {return storer.ErrStop@@ -286,19 +287,14 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogObreakIteration = true}- if untilCommit != nil && commit.Hash == untilCommit.Hash && !scanOneCommit {+ // TODO: Clean up this conditional mess.+ if (scanOptions.SinceCommit != nil && commit.Hash == scanOptions.SinceCommit.Hash && !scanOneCommit) ||+ (scanOptions.MaxDepth >= 0 && depth >= scanOptions.MaxDepth) {return storer.ErrStop}+ depth++- // err := s.sem.Acquire(ctx, 1)- // if err != nil {- // return err- // }- // wg.Add(1)- // go func(wg *sync.WaitGroup) {- // defer wg.Done()- // defer s.sem.Release(1)- err = s.scanCommitPatches(ctx, repo, commit, chunksChan, filter)+ err = s.scanCommitPatches(ctx, repo, commit, chunksChan, scanOptions.Filter)if err != nil {switch e := err.Error(); {case strings.Contains(e, "operation canceled"):@@ -315,7 +311,6 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOreturn err}}- // }(wg)return nil})if err != nil {@@ -342,7 +337,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOreturn err}for fh := range status {- if !filter.Pass(fh) {+ if !scanOptions.Filter.Pass(fh) {continue}metadata := s.sourceMetadataFunc(diff --git a/pkg/sources/git/scan_options.go b/pkg/sources/git/scan_options.gonew file mode 100644index 000000000..2d0155b04--- /dev/null+++ b/pkg/sources/git/scan_options.go@@ -0,0 +1,56 @@+package git++import (+ "github.com/go-git/go-git/v5"+ "github.com/go-git/go-git/v5/plumbing/object"+ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type ScanOptions struct {+ Filter *common.Filter+ SinceCommit *object.Commit // When scanning a git.Log, this is the oldest/first commit.+ MaxDepth int64+ LogOptions *git.LogOptions+}++type ScanOption func(*ScanOptions)++func ScanOptionFilter(filter *common.Filter) ScanOption {+ return func(scanOptions *ScanOptions) {+ scanOptions.Filter = filter+ }+}++func ScanOptionSinceCommit(commit *object.Commit) ScanOption {+ return func(scanOptions *ScanOptions) {+ scanOptions.SinceCommit = commit+ }+}++func ScanOptionMaxDepth(maxDepth int64) ScanOption {+ return func(scanOptions *ScanOptions) {+ scanOptions.MaxDepth = maxDepth+ }+}++func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {+ return func(scanOptions *ScanOptions) {+ scanOptions.LogOptions = logOptions+ }+}++func NewScanOptions(options ...ScanOption) *ScanOptions {+ scanOptions := &ScanOptions{+ Filter: common.FilterEmpty(),+ SinceCommit: nil,+ MaxDepth: -1,+ LogOptions: &git.LogOptions{+ All: true,+ Order: git.LogOrderCommitterTime,+ },+ }+ for _, option := range options {+ option(scanOptions)+ }+ return scanOptions+}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex f6f8967ab..5e39652a1 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -328,7 +328,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errlog.WithError(err).Warnf("unable to clone repo, continuing")continue}- err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)if err != nil {log.WithError(err).Warnf("unable to scan repo")}@@ -422,7 +422,7 @@ func (s *Source) paginateGists(ctx context.Context, user string, chunksChan chans.log.WithField("repo", *gist.HTMLURL).Debugf("attempting to clone gist from user %s", user)scanCtx := context.Background()- err = s.git.ScanRepo(scanCtx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(scanCtx, repo, git.NewScanOptions(), chunksChan)if err != nil {log.WithError(err).Warnf("Could not scan after clone: %s", *gist.HTMLURL)continuediff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 4d40162c5..6fa14f4a8 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -8,20 +8,16 @@ import ("runtime""github.com/go-errors/errors"- gogit "github.com/go-git/go-git/v5"log "github.com/sirupsen/logrus"- "github.com/xanzy/go-gitlab"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/types/known/anypb"-- "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/giturl""github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"-- "github.com/trufflesecurity/trufflehog/pkg/giturl""github.com/trufflesecurity/trufflehog/pkg/sanitizer""github.com/trufflesecurity/trufflehog/pkg/sources""github.com/trufflesecurity/trufflehog/pkg/sources/git"+ "github.com/xanzy/go-gitlab"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb")type Source struct {@@ -216,7 +212,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,errors = append(errors, err)continue}- err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)if err != nil {errors = append(errors, err)continue@@ -237,7 +233,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,errors = append(errors, err)continue}- err = s.git.ScanRepo(ctx, repo, &gogit.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, git.NewScanOptions(), chunksChan)if err != nil {errors = append(errors, err)continue