Skip to content

Find, verify, and analyze leaked credentials

Check commit order, dedupe results, and support using a head commit. (#44)

Bill Rich56dc1b109c6f
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 0438c5673..d71fe5deb 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -1,7 +1,10 @@
package engine
import (
+ "bytes"
"context"
+ "crypto/sha256"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"runtime"
"strings"
"sync"
@@ -23,10 +26,16 @@ type Engine struct {
detectors map[bool][]detectors.Detector
chunksScanned uint64
detectorAvgTime map[string][]time.Duration
+ detectedSecret secretTracker
}
type EngineOption func(*Engine)
+type secretTracker struct {
+ secret map[[32]byte]bool
+ sync sync.Mutex
+}
+
func WithConcurrency(concurrency int) EngineOption {
return func(e *Engine) {
e.concurrency = concurrency
@@ -57,6 +66,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
chunks: make(chan *sources.Chunk),
results: make(chan detectors.ResultWithMetadata),
detectorAvgTime: map[string][]time.Duration{},
+ detectedSecret: secretTracker{
+ secret: map[[32]byte]bool{},
+ sync: sync.Mutex{},
+ },
}
for _, option := range options {
@@ -158,6 +171,18 @@ func (e *Engine) detectorWorker(ctx context.Context) {
continue
}
for _, result := range results {
+ if isGitSource(chunk.SourceType) {
+ data := bytes.Join([][]byte{result.Raw, []byte(chunk.SourceMetadata.GetGit().Repository), []byte(chunk.SourceMetadata.GetGit().File)}, []byte{})
+ sid := sha256.Sum256(data)
+ _, exists := e.detectedSecret.secret[sid]
+ if exists {
+ logrus.Debugf("skipping duplicate result for %s in commit %s", result.Raw, chunk.SourceMetadata.GetGit().Commit)
+ continue
+ }
+ e.detectedSecret.sync.Lock()
+ e.detectedSecret.secret[sid] = true
+ e.detectedSecret.sync.Unlock()
+ }
e.results <- detectors.CopyMetadata(chunk, result)
}
if len(results) > 0 {
@@ -171,3 +196,26 @@ func (e *Engine) detectorWorker(ctx context.Context) {
atomic.AddUint64(&e.chunksScanned, 1)
}
}
+
+// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not
+// supported.
+func gitSources() []sourcespb.SourceType {
+ return []sourcespb.SourceType{
+ sourcespb.SourceType_SOURCE_TYPE_GIT,
+ sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ sourcespb.SourceType_SOURCE_TYPE_GITLAB,
+ sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
+ sourcespb.SourceType_SOURCE_TYPE_GERRIT,
+ sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
+ sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
+ }
+}
+
+func isGitSource(sourceType sourcespb.SourceType) bool {
+ for _, i := range gitSources() {
+ if i == sourceType {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index 9ed93c404..7d733741e 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -5,12 +5,10 @@ import (
"fmt"
"runtime"
- "github.com/go-git/go-git/v5/plumbing/object"
-
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/plumbing/object"
"github.com/sirupsen/logrus"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -38,12 +36,12 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
if gitScanBranch != "" {
headHash, err := git.TryAdditionalBaseRefs(repo, gitScanBranch)
if err != nil {
- return fmt.Errorf("could not parse revision: %q: %w", headRef, err)
+ return fmt.Errorf("could not parse revision: %q: %w", gitScanBranch, err)
}
headCommit, err = repo.CommitObject(*headHash)
if err != nil {
- return fmt.Errorf("could not find commit: %q: %w", headRef, err)
+ return fmt.Errorf("could not find commit: %q: %w", gitScanBranch, err)
}
logrus.WithFields(logrus.Fields{
@@ -54,12 +52,6 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
logOptions.All = false
}
- if sinceCommit != nil && headCommit != nil {
- if ok, _ := sinceCommit.IsAncestor(headCommit); !ok {
- return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", sinceCommit.Hash.String(), headCommit.Hash.String())
- }
- }
-
gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(),
func(file, email, commit, repository string) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
@@ -83,7 +75,10 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, gitScanBranch, headRef s
opts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))
}
if sinceCommit != nil {
- opts = append(opts, git.ScanOptionSinceCommit(sinceCommit))
+ opts = append(opts, git.ScanOptionBaseCommit(sinceCommit))
+ }
+ if headCommit != nil {
+ opts = append(opts, git.ScanOptionHeadCommit(headCommit))
}
scanOptions := git.NewScanOptions(opts...)
diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go
new file mode 100644
index 000000000..2317d1dd4
--- /dev/null
+++ b/pkg/engine/git_test.go
@@ -0,0 +1,65 @@
+package engine
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/go-git/go-git/v5/plumbing"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
+)
+
+func TestGitEngine(t *testing.T) {
+ repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
+ path, _, err := git.PrepareRepo(repoUrl)
+ if err != nil {
+ t.Error(err)
+ }
+ defer os.RemoveAll(path)
+
+ ctx, cancel := context.WithCancel(context.TODO())
+ defer cancel()
+ type testProfile struct {
+ expected map[string]string
+ branch string
+ base string
+ maxDepth int
+ filter *common.Filter
+ }
+ for tName, tTest := range map[string]testProfile{
+ "all_secrets": {
+ expected: map[string]string{
+ "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",
+ "84e9c75e388ae3e866e121087ea2dd45a71068f2": "AKIAILE3JG6KMS3HZGCA",
+ "8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0",
+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2",
+ },
+ filter: common.FilterEmpty(),
+ },
+ } {
+ e := Start(ctx,
+ WithConcurrency(1),
+ WithDecoders(decoders.DefaultDecoders()...),
+ WithDetectors(false, DefaultDetectors()...),
+ )
+ base := plumbing.NewHash(tTest.base)
+ e.ScanGit(ctx, path, tTest.branch, "HEAD", &base, tTest.maxDepth, tTest.filter)
+ resultCount := 0
+ for result := range e.ResultsChan() {
+ switch meta := result.SourceMetadata.GetData().(type) {
+ case *source_metadatapb.MetaData_Git:
+ if tTest.expected[meta.Git.Commit] != string(result.Raw) {
+ t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit])
+ }
+ }
+ resultCount++
+
+ }
+ if resultCount != len(tTest.expected) {
+ t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
+ }
+ }
+}
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index cb3c3e052..a2850828a 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -21,15 +21,14 @@ import (
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport/http"
log "github.com/sirupsen/logrus"
- "golang.org/x/sync/semaphore"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/types/known/anypb"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+ "golang.org/x/sync/semaphore"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
)
type Source struct {
@@ -256,91 +255,129 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Repo
}
func (s *Git) ScanCommits(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
- blobsSeen := map[string]bool{}
- commits := map[int64][]*object.Commit{}
-
- logIter, err := repo.Log(scanOptions.LogOptions)
+ commitIter, err := repo.Log(scanOptions.LogOptions)
if err != nil {
- log.Fatal(err)
+ return err
}
+ commits := map[int64][]*object.Commit{}
- keys := []int64{}
-
- sinceTime := int64(0)
-
- logIter.ForEach(func(commit *object.Commit) error {
- key := commit.Committer.When.Unix()
- if scanOptions.SinceCommit != nil && scanOptions.SinceCommit.Hash.String() == commit.Hash.String() {
- sinceTime = key
+ depth := int64(0)
+ reachedBase := false
+ // Create a map of timestamps to commits.
+ commitIter.ForEach(func(commit *object.Commit) error {
+ if scanOptions.BaseCommit != nil && !reachedBase {
+ if commit.Hash == scanOptions.BaseCommit.Hash {
+ reachedBase = true
+ } else {
+ return nil
+ }
}
- if existing, ok := commits[key]; ok {
- commits[key] = append(existing, commit)
- } else {
- commits[key] = []*object.Commit{commit}
- keys = append(keys, key)
+ time := commit.Committer.When.Unix()
+ if _, ok := commits[time]; !ok {
+ commits[time] = []*object.Commit{}
+ }
+ commits[time] = append(commits[time], commit)
+ depth++
+ if scanOptions.MaxDepth > int64(0) && depth >= scanOptions.MaxDepth {
+ return errors.New("reached max depth")
}
return nil
})
+ // Make a slice of all the timestamp keys so it can be sorted.
+ keys := make([]int64, len(commits))
+ i := 0
+ for key, _ := range commits {
+ keys[i] = key
+ i++
+ }
+
+ seenMap := map[plumbing.Hash]bool{}
+
+ // Sort the timestamps
sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+ for _, commitTime := range keys {
+ commitSlice := commits[commitTime]
+ for _, commit := range commitSlice {
+ // Check to make sure a commit's parents are scanned before the commit. This is done by checking each
+ // commit's timestamp to the later of it's two parents.
+ laterParent := int64(0)
+ for _, parentHash := range commit.ParentHashes {
+ parent, err := repo.CommitObject(parentHash)
+ if err != nil {
+ continue
+ }
+ parentTime := parent.Committer.When.Unix()
+ if commitTime > parentTime {
+ continue
+ }
+ log.WithField("parentTime", parentTime).WithField("commitTime", commitTime).Debugf("commit %s is out of order", commit.Hash)
+ if parentTime > laterParent {
+ laterParent = parentTime
+ }
+ }
+ // If the commit has an earlier timestamp than it's parents, something has gone wrong. Moving the commit
+ // to the end of the timestamp the parent is in will ensure the parents are scanned first.
+ if laterParent > 0 {
+ log.Debugf("commit (%s) is out of order moving after its parent", commit.Hash)
+ commits[laterParent] = append(commits[laterParent], commit)
+ }
+
+ s.scanCommit(repo, commit, &seenMap, scanOptions, chunksChan)
+ }
+ }
+ return nil
+}
+func (s *Git) scanCommit(repo *git.Repository, commit *object.Commit, seenMap *map[plumbing.Hash]bool, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) {
remote, err := repo.Remote("origin")
if err != nil {
- return errors.New(err)
+ log.Errorf("error getting repo name: %s", err)
+ return
}
safeRepo, err := stripPassword(remote.Config().URLs[0])
if err != nil {
- return errors.New(err)
+ log.WithError(err).Errorf("couldn't get repo name")
+ return
}
-
- start := int64(0)
- if scanOptions.MaxDepth > 0 && scanOptions.MaxDepth < int64(len(keys)) {
- start = int64(len(keys)) - scanOptions.MaxDepth
+ fileIter, err := commit.Files()
+ if err != nil {
+ log.WithError(err).WithField("commit", commit.Hash.String()).Errorf("unable to read files")
+ return
}
+ fileIter.ForEach(func(file *object.File) error {
+ if _, ok := (*seenMap)[file.Hash]; ok {
+ return nil
+ }
- scannedCommits := []string{}
- for _, key := range keys[start:] {
- if sinceTime > key {
- continue
+ if !scanOptions.Filter.Pass(file.Name) {
+ return nil
}
- for _, commit := range commits[key] {
- scannedCommits = append(scannedCommits, commit.Hash.String())
- fileIter, err := commit.Files()
- if err != nil {
- return err
- }
- err = fileIter.ForEach(func(file *object.File) error {
- if _, scanned := blobsSeen[file.Hash.String()]; scanned || !scanOptions.Filter.Pass(file.Name) {
- return nil
- }
- blobsSeen[file.Hash.String()] = true
- metadata := s.sourceMetadataFunc(file.Name, commit.Author.Email, commit.Hash.String(), safeRepo)
- reader, err := file.Reader()
- if err != nil {
- return nil
- }
- defer reader.Close()
- buffer := new(bytes.Buffer)
- buffer.ReadFrom(reader)
- chunksChan <- &sources.Chunk{
- SourceType: s.sourceType,
- SourceName: s.sourceName,
- SourceID: s.sourceID,
- Data: buffer.Bytes(),
- SourceMetadata: metadata,
- Verify: s.verify,
- }
- return nil
- })
- if err != nil {
- return err
- }
+
+ reader, err := file.Reader()
+ if err != nil {
+ log.WithError(err).WithField("file", file.Name).Debugf("failed to get reader from file")
+ return nil
}
- }
- if err != nil && !strings.Contains(err.Error(), "max_depth exceeded") && !strings.Contains(err.Error(), "reached since_commit") {
- return err
- }
- return nil
+ defer reader.Close()
+ bytes, err := ioutil.ReadAll(reader)
+ if err != nil {
+ log.WithError(err).WithField("file", file.Name).Debugf("failed to read from file")
+ return nil
+ }
+
+ metadata := s.sourceMetadataFunc(file.Name, commit.Committer.Email, commit.Hash.String(), safeRepo)
+ chunksChan <- &sources.Chunk{
+ SourceName: s.sourceName,
+ SourceID: s.sourceID,
+ SourceType: s.sourceType,
+ SourceMetadata: metadata,
+ Data: bytes,
+ Verify: s.verify,
+ }
+ (*seenMap)[file.Hash] = true
+ return nil
+ })
}
func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
@@ -399,6 +436,11 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
}
func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+ if err := verifyOptions(scanOptions); err != nil {
+ return err
+ }
+
+ start := time.Now().UnixNano()
if err := s.ScanCommits(repo, scanOptions, chunksChan); err != nil {
return err
}
@@ -411,6 +453,19 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, scanOptions *S
}
return err
}
+ scanTime := time.Now().UnixNano() - start
+ log.Debugf("Scanning complete. Scan time: %f", time.Duration(scanTime).Seconds())
+ return nil
+}
+
+func verifyOptions(scanOptions *ScanOptions) error {
+ base := scanOptions.BaseCommit
+ head := scanOptions.HeadCommit
+ if base != nil && head != nil {
+ if ok, _ := base.IsAncestor(head); !ok {
+ return fmt.Errorf("unable to scan from requested head to end commit. %s is not an ancestor of %s", base, head)
+ }
+ }
return nil
}
diff --git a/pkg/sources/git/scan_options.go b/pkg/sources/git/scan_options.go
index 35fa16237..56399f86b 100644
--- a/pkg/sources/git/scan_options.go
+++ b/pkg/sources/git/scan_options.go
@@ -7,10 +7,11 @@ import (
)
type ScanOptions struct {
- Filter *common.Filter
- SinceCommit *object.Commit // When scanning a git.Log, this is the oldest/first commit.
- MaxDepth int64
- LogOptions *git.LogOptions
+ Filter *common.Filter
+ BaseCommit *object.Commit // When scanning a git.Log, this is the oldest/first commit.
+ HeadCommit *object.Commit
+ MaxDepth int64
+ LogOptions *git.LogOptions
}
type ScanOption func(*ScanOptions)
@@ -21,9 +22,15 @@ func ScanOptionFilter(filter *common.Filter) ScanOption {
}
}
-func ScanOptionSinceCommit(commit *object.Commit) ScanOption {
+func ScanOptionBaseCommit(commit *object.Commit) ScanOption {
return func(scanOptions *ScanOptions) {
- scanOptions.SinceCommit = commit
+ scanOptions.BaseCommit = commit
+ }
+}
+
+func ScanOptionHeadCommit(commit *object.Commit) ScanOption {
+ return func(scanOptions *ScanOptions) {
+ scanOptions.HeadCommit = commit
}
}
@@ -41,9 +48,9 @@ func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption {
func NewScanOptions(options ...ScanOption) *ScanOptions {
scanOptions := &ScanOptions{
- Filter: common.FilterEmpty(),
- SinceCommit: nil,
- MaxDepth: -1,
+ Filter: common.FilterEmpty(),
+ BaseCommit: nil,
+ MaxDepth: -1,
LogOptions: &git.LogOptions{
All: true,
},