Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Check commit order, dedupe results, and support using a head commit. (#44)
Bill Rich56dc1b109c6f
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 0438c5673..d71fe5deb 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -1,7 +1,10 @@package engineimport (+ "bytes""context"+ "crypto/sha256"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""runtime""strings""sync"@@ -23,10 +26,16 @@ type Engine struct {detectors map[bool][]detectors.DetectorchunksScanned uint64detectorAvgTime map[string][]time.Duration+ detectedSecret secretTracker}type EngineOption func(*Engine)+type secretTracker struct {+ secret map[[32]byte]bool+ sync sync.Mutex+}+func WithConcurrency(concurrency int) EngineOption {return func(e *Engine) {e.concurrency = concurrency@@ -57,6 +66,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {chunks: make(chan *sources.Chunk),results: make(chan detectors.ResultWithMetadata),detectorAvgTime: map[string][]time.Duration{},+ detectedSecret: secretTracker{+ secret: map[[32]byte]bool{},+ sync: sync.Mutex{},+ },}for _, option := range options {@@ -158,6 +171,18 @@ func (e *Engine) detectorWorker(ctx context.Context) {continue}for _, result := range results {+ if isGitSource(chunk.SourceType) {+ data := bytes.Join([][]byte{result.Raw, []byte(chunk.SourceMetadata.GetGit().Repository), []byte(chunk.SourceMetadata.GetGit().File)}, []byte{})+ sid := sha256.Sum256(data)+ _, exists := e.detectedSecret.secret[sid]+ if exists {+ logrus.Debugf("skipping duplicate result for %s in commit %s", result.Raw, chunk.SourceMetadata.GetGit().Commit)+ continue+ }+ e.detectedSecret.sync.Lock()+ e.detectedSecret.secret[sid] = true+ e.detectedSecret.sync.Unlock()+ }e.results <- detectors.CopyMetadata(chunk, result)}if len(results) > 0 {@@ -171,3 +196,26 @@ func (e *Engine) detectorWorker(ctx context.Context) {atomic.AddUint64(&e.chunksScanned, 1)}}++// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not+// supported.+func gitSources() []sourcespb.SourceType {+ return []sourcespb.SourceType{+ sourcespb.SourceType_SOURCE_TYPE_GIT,+ sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ sourcespb.SourceType_SOURCE_TYPE_GITLAB,+ sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,+ sourcespb.SourceType_SOURCE_TYPE_GERRIT,+ sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,+ sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,+ }+}++func isGitSource(sourceType sourcespb.SourceType) bool {+ for _, i := range gitSources() {+ if i == sourceType {+ return true+ }+ }+ return false+}diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex 9ed93c404..7d733741e 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -5,12 +5,10 @@ import ("fmt""runtime"- "github.com/go-git/go-git/v5/plumbing/object"-gogit "github.com/go-git/go-git/v5""github.com/go-git/go-git/v5/plumbing"+ "github.com/go-git/go-git/v5/plumbing/object""github.com/sirupsen/logrus"-"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"@@ -38,12 +36,12 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef sif gitScanBranch != "" {headHash, err := git.TryAdditionalBaseRefs(repo, gitScanBranch)if err != nil {- return fmt.Errorf("could not parse revision: %q: %w", headRef, err)+ return fmt.Errorf("could not parse revision: %q: %w", gitScanBranch, err)}headCommit, err = repo.CommitObject(*headHash)if err != nil {- return fmt.Errorf("could not find commit: %q: %w", headRef, err)+ return fmt.Errorf("could not find commit: %q: %w", gitScanBranch, err)}logrus.WithFields(logrus.Fields{@@ -54,12 +52,6 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef slogOptions.All = false}- if sinceCommit != nil && headCommit != nil {- if ok, _ := sinceCommit.IsAncestor(headCommit); !ok {- return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", sinceCommit.Hash.String(), headCommit.Hash.String())- }- }-gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(),func(file, email, commit, repository string) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{@@ -83,7 +75,10 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef sopts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))}if sinceCommit != nil {- opts = append(opts, git.ScanOptionSinceCommit(sinceCommit))+ opts = append(opts, git.ScanOptionBaseCommit(sinceCommit))+ }+ if headCommit != nil {+ opts = append(opts, git.ScanOptionHeadCommit(headCommit))}scanOptions := git.NewScanOptions(opts...)diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.gonew file mode 100644index 000000000..2317d1dd4--- /dev/null+++ b/pkg/engine/git_test.go@@ -0,0 +1,65 @@+package engine++import (+ "context"+ "os"+ "testing"++ "github.com/go-git/go-git/v5/plumbing"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/decoders"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"+)++func TestGitEngine(t *testing.T) {+ repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"+ path, _, err := git.PrepareRepo(repoUrl)+ if err != nil {+ t.Error(err)+ }+ defer os.RemoveAll(path)++ ctx, cancel := context.WithCancel(context.TODO())+ defer cancel()+ type testProfile struct {+ expected map[string]string+ branch string+ base string+ maxDepth int+ filter *common.Filter+ }+ for tName, tTest := range map[string]testProfile{+ "all_secrets": {+ expected: map[string]string{+ "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",+ "84e9c75e388ae3e866e121087ea2dd45a71068f2": "AKIAILE3JG6KMS3HZGCA",+ "8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0",+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2",+ },+ filter: common.FilterEmpty(),+ },+ } {+ e := Start(ctx,+ WithConcurrency(1),+ WithDecoders(decoders.DefaultDecoders()...),+ WithDetectors(false, DefaultDetectors()...),+ )+ base := plumbing.NewHash(tTest.base)+ e.ScanGit(ctx, path, tTest.branch, "HEAD", &base, tTest.maxDepth, tTest.filter)+ resultCount := 0+ for result := range e.ResultsChan() {+ switch meta := result.SourceMetadata.GetData().(type) {+ case *source_metadatapb.MetaData_Git:+ if tTest.expected[meta.Git.Commit] != string(result.Raw) {+ t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit])+ }+ }+ resultCount++++ }+ if resultCount != len(tTest.expected) {+ t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))+ }+ }+}diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex cb3c3e052..a2850828a 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -21,15 +21,14 @@ import ("github.com/go-git/go-git/v5/plumbing/object""github.com/go-git/go-git/v5/plumbing/transport/http"log "github.com/sirupsen/logrus"- "golang.org/x/sync/semaphore"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/types/known/anypb"-"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer""github.com/trufflesecurity/trufflehog/v3/pkg/sources"+ "golang.org/x/sync/semaphore"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb")type Source struct {@@ -256,91 +255,129 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Repo}func (s *Git) ScanCommits(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {- blobsSeen := map[string]bool{}- commits := map[int64][]*object.Commit{}-- logIter, err := repo.Log(scanOptions.LogOptions)+ commitIter, err := repo.Log(scanOptions.LogOptions)if err != nil {- log.Fatal(err)+ return err}+ commits := map[int64][]*object.Commit{}- keys := []int64{}-- sinceTime := int64(0)-- logIter.ForEach(func(commit *object.Commit) error {- key := commit.Committer.When.Unix()- if scanOptions.SinceCommit != nil && scanOptions.SinceCommit.Hash.String() == commit.Hash.String() {- sinceTime = key+ depth := int64(0)+ reachedBase := false+ // Create a map of timestamps to commits.+ commitIter.ForEach(func(commit *object.Commit) error {+ if scanOptions.BaseCommit != nil && !reachedBase {+ if commit.Hash == scanOptions.BaseCommit.Hash {+ reachedBase = true+ } else {+ return nil+ }}- if existing, ok := commits[key]; ok {- commits[key] = append(existing, commit)- } else {- commits[key] = []*object.Commit{commit}- keys = append(keys, key)+ time := commit.Committer.When.Unix()+ if _, ok := commits[time]; !ok {+ commits[time] = []*object.Commit{}+ }+ commits[time] = append(commits[time], commit)+ depth+++ if scanOptions.MaxDepth > int64(0) && depth >= scanOptions.MaxDepth {+ return errors.New("reached max depth")}return nil})+ // Make a slice of all the timestamp keys so it can be sorted.+ keys := make([]int64, len(commits))+ i := 0+ for key, _ := range commits {+ keys[i] = key+ i+++ }++ seenMap := map[plumbing.Hash]bool{}++ // Sort the timestampssort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })+ for _, commitTime := range keys {+ commitSlice := commits[commitTime]+ for _, commit := range commitSlice {+ // Check to make sure a commit's parents are scanned before the commit. This is done by checking each+ // commit's timestamp to the later of it's two parents.+ laterParent := int64(0)+ for _, parentHash := range commit.ParentHashes {+ parent, err := repo.CommitObject(parentHash)+ if err != nil {+ continue+ }+ parentTime := parent.Committer.When.Unix()+ if commitTime > parentTime {+ continue+ }+ log.WithField("parentTime", parentTime).WithField("commitTime", commitTime).Debugf("commit %s is out of order", commit.Hash)+ if parentTime > laterParent {+ laterParent = parentTime+ }+ }+ // If the commit has an earlier timestamp than it's parents, something has gone wrong. Moving the commit+ // to the end of the timestamp the parent is in will ensure the parents are scanned first.+ if laterParent > 0 {+ log.Debugf("commit (%s) is out of order moving after its parent", commit.Hash)+ commits[laterParent] = append(commits[laterParent], commit)+ }++ s.scanCommit(repo, commit, &seenMap, scanOptions, chunksChan)+ }+ }+ return nil+}+func (s *Git) scanCommit(repo *git.Repository, commit *object.Commit, seenMap *map[plumbing.Hash]bool, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) {remote, err := repo.Remote("origin")if err != nil {- return errors.New(err)+ log.Errorf("error getting repo name: %s", err)+ return}safeRepo, err := stripPassword(remote.Config().URLs[0])if err != nil {- return errors.New(err)+ log.WithError(err).Errorf("couldn't get repo name")+ return}-- start := int64(0)- if scanOptions.MaxDepth > 0 && scanOptions.MaxDepth < int64(len(keys)) {- start = int64(len(keys)) - scanOptions.MaxDepth+ fileIter, err := commit.Files()+ if err != nil {+ log.WithError(err).WithField("commit", commit.Hash.String()).Errorf("unable to read files")+ return}+ fileIter.ForEach(func(file *object.File) error {+ if _, ok := (*seenMap)[file.Hash]; ok {+ return nil+ }- scannedCommits := []string{}- for _, key := range keys[start:] {- if sinceTime > key {- continue+ if !scanOptions.Filter.Pass(file.Name) {+ return nil}- for _, commit := range commits[key] {- scannedCommits = append(scannedCommits, commit.Hash.String())- fileIter, err := commit.Files()- if err != nil {- return err- }- err = fileIter.ForEach(func(file *object.File) error {- if _, scanned := blobsSeen[file.Hash.String()]; scanned || !scanOptions.Filter.Pass(file.Name) {- return nil- }- blobsSeen[file.Hash.String()] = true- metadata := s.sourceMetadataFunc(file.Name, commit.Author.Email, commit.Hash.String(), safeRepo)- reader, err := file.Reader()- if err != nil {- return nil- }- defer reader.Close()- buffer := new(bytes.Buffer)- buffer.ReadFrom(reader)- chunksChan <- &sources.Chunk{- SourceType: s.sourceType,- SourceName: s.sourceName,- SourceID: s.sourceID,- Data: buffer.Bytes(),- SourceMetadata: metadata,- Verify: s.verify,- }- return nil- })- if err != nil {- return err- }++ reader, err := file.Reader()+ if err != nil {+ log.WithError(err).WithField("file", file.Name).Debugf("failed to get reader from file")+ return nil}- }- if err != nil && !strings.Contains(err.Error(), "max_depth exceeded") && !strings.Contains(err.Error(), "reached since_commit") {- return err- }- return nil+ defer reader.Close()+ bytes, err := ioutil.ReadAll(reader)+ if err != nil {+ log.WithError(err).WithField("file", file.Name).Debugf("failed to read from file")+ return nil+ }++ metadata := s.sourceMetadataFunc(file.Name, commit.Committer.Email, commit.Hash.String(), safeRepo)+ chunksChan <- &sources.Chunk{+ SourceName: s.sourceName,+ SourceID: s.sourceID,+ SourceType: s.sourceType,+ SourceMetadata: metadata,+ Data: bytes,+ Verify: s.verify,+ }+ (*seenMap)[file.Hash] = true+ return nil+ })}func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {@@ -399,6 +436,11 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk}func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+ if err := verifyOptions(scanOptions); err != nil {+ return err+ }++ start := time.Now().UnixNano()if err := s.ScanCommits(repo, scanOptions, chunksChan); err != nil {return err}@@ -411,6 +453,19 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *S}return err}+ scanTime := time.Now().UnixNano() - start+ log.Debugf("Scanning complete. Scan time: %f", time.Duration(scanTime).Seconds())+ return nil+}++func verifyOptions(scanOptions *ScanOptions) error {+ base := scanOptions.BaseCommit+ head := scanOptions.HeadCommit+ if base != nil && head != nil {+ if ok, _ := base.IsAncestor(head); !ok {+ return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", base, head)+ }+ }return nil}diff --git a/pkg/sources/git/scan_options.go b/pkg/sources/git/scan_options.goindex 35fa16237..56399f86b 100644--- a/pkg/sources/git/scan_options.go+++ b/pkg/sources/git/scan_options.go@@ -7,10 +7,11 @@ import ()type ScanOptions struct {- Filter *common.Filter- SinceCommit *object.Commit // When scanning a git.Log, this is the oldest/first commit.- MaxDepth int64- LogOptions *git.LogOptions+ Filter *common.Filter+ BaseCommit *object.Commit // When scanning a git.Log, this is the oldest/first commit.+ HeadCommit *object.Commit+ MaxDepth int64+ LogOptions *git.LogOptions}type ScanOption func(*ScanOptions)@@ -21,9 +22,15 @@ func ScanOptionFilter(filter *common.Filter) ScanOption {}}-func ScanOptionSinceCommit(commit *object.Commit) ScanOption {+func ScanOptionBaseCommit(commit *object.Commit) ScanOption {return func(scanOptions *ScanOptions) {- scanOptions.SinceCommit = commit+ scanOptions.BaseCommit = commit+ }+}++func ScanOptionHeadCommit(commit *object.Commit) ScanOption {+ return func(scanOptions *ScanOptions) {+ scanOptions.HeadCommit = commit}}@@ -41,9 +48,9 @@ func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {func NewScanOptions(options ...ScanOption) *ScanOptions {scanOptions := &ScanOptions{- Filter: common.FilterEmpty(),- SinceCommit: nil,- MaxDepth: -1,+ Filter: common.FilterEmpty(),+ BaseCommit: nil,+ MaxDepth: -1,LogOptions: &git.LogOptions{All: true,},