Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Refactor Engine to wait for workers in a Finish method (#581)
Miccah6fa2171a221b
diff --git a/main.go b/main.goindex 17ef12462..118590961 100644--- a/main.go+++ b/main.go@@ -2,7 +2,6 @@ package mainimport ("context"- "encoding/json""fmt""log""net/http"@@ -26,9 +25,6 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/engine""github.com/trufflesecurity/trufflehog/v3/pkg/output"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git")@@ -220,11 +216,15 @@ func run(state overseer.State) {logrus.WithError(err).Fatal("Failed to scan syslog.")}}+ // asynchronously wait for scanning to finish and cleanup+ go e.Finish()if !*jsonLegacy && !*jsonOut {fmt.Fprintf(os.Stderr, "π·ππ· TruffleHog. Unearth your secrets. π·ππ·\n\n")}+ // NOTE: this loop will terminate when the results channel is closed in+ // e.Finish()foundResults := falsefor r := range e.ResultsChan() {if *onlyVerified && !r.Verified {@@ -234,60 +234,9 @@ func run(state overseer.State) {switch {case *jsonLegacy:- repoPath, remote, err = git.PrepareRepo(r.SourceMetadata.GetGithub().Repository)- if err != nil || repoPath == "" {- logrus.WithError(err).Fatal("error preparing git repo for scanning")- }- legacy := output.ConvertToLegacyJSON(&r, repoPath)- out, err := json.Marshal(legacy)- if err != nil {- logrus.WithError(err).Fatal("could not marshal result")- }- fmt.Println(string(out))-- if remote {- os.RemoveAll(repoPath)- }+ output.PrintLegacyJSON(&r)case *jsonOut:- v := &struct {- // SourceMetadata contains source-specific contextual information.- SourceMetadata *source_metadatapb.MetaData- // SourceID is the ID of the source that the API uses to map secrets to specific sources.- SourceID int64- // SourceType is the type of Source.- SourceType sourcespb.SourceType- // SourceName is the name of the Source.- SourceName string- // DetectorType is the type of Detector.- DetectorType detectorspb.DetectorType- // DetectorName is the string name of the DetectorType.- DetectorName string- Verified bool- // Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.- Raw []byte- // Redacted contains the redacted version of the raw secret identification data for display purposes.- // A secret ID should be used if available.- Redacted string- ExtraData map[string]string- StructuredData *detectorspb.StructuredData- }{- SourceMetadata: r.SourceMetadata,- SourceID: r.SourceID,- SourceType: r.SourceType,- SourceName: r.SourceName,- DetectorType: r.DetectorType,- DetectorName: r.DetectorType.String(),- Verified: r.Verified,- Raw: r.Raw,- Redacted: r.Redacted,- ExtraData: r.ExtraData,- StructuredData: r.StructuredData,- }- out, err := json.Marshal(v)- if err != nil {- logrus.WithError(err).Fatal("could not marshal result")- }- fmt.Println(string(out))+ output.PrintJSON(&r)default:output.PrintPlainOutput(&r)}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex fd68d1dee..57d4e8a9f 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -26,6 +26,8 @@ type Engine struct {detectors map[bool][]detectors.DetectorchunksScanned uint64detectorAvgTime sync.Map+ sourcesWg sync.WaitGroup+ workersWg sync.WaitGroup}type EngineOption func(*Engine)@@ -75,15 +77,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {}logrus.Debugf("running with up to %d workers", e.concurrency)- var workerWg sync.WaitGroup- for i := 0; i < e.concurrency; i++ {- workerWg.Add(1)- go func() {- e.detectorWorker(ctx)- workerWg.Done()- }()- }-if len(e.decoders) == 0 {e.decoders = decoders.DefaultDecoders()}@@ -101,18 +94,36 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {len(e.detectors[false]))// start the workers- go func() {- // close results chan when all workers are done- workerWg.Wait()- // not entirely sure why results don't get processed without this pause- // since we've put all results on the channel at this point.- time.Sleep(time.Second)- close(e.ResultsChan())- }()+ for i := 0; i < e.concurrency; i++ {+ e.workersWg.Add(1)+ go func() {+ defer e.workersWg.Done()+ e.detectorWorker(ctx)+ }()+ }return e}+// Finish waits for running sources to complete and workers to finish scanning+// chunks before closing their respective channels. Once Finish is called, no+// more sources may be scanned by the engine.+func (e *Engine) Finish() {+ // wait for the sources to finish putting chunks onto the chunks channel+ e.sourcesWg.Wait()+ close(e.chunks)+ // wait for the workers to finish processing all of the chunks and putting+ // results onto the results channel+ e.workersWg.Wait()++ // TODO: re-evaluate whether this is needed and investigate why if so+ //+ // not entirely sure why results don't get processed without this pause+ // since we've put all results on the channel at this point.+ time.Sleep(time.Second)+ close(e.results)+}+func (e *Engine) ChunksChan() chan *sources.Chunk {return e.chunks}diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.goindex 060a1c487..e245183c1 100644--- a/pkg/engine/filesystem.go+++ b/pkg/engine/filesystem.go@@ -2,13 +2,14 @@ package engineimport ("context"+ "runtime"+"github.com/go-errors/errors""github.com/sirupsen/logrus""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"- "runtime")func (e *Engine) ScanFileSystem(ctx context.Context, directories []string) error {@@ -27,12 +28,13 @@ func (e *Engine) ScanFileSystem(ctx context.Context, directories []string) errorif err != nil {return errors.WrapPrefix(err, "could not init filesystem source", 0)}+ e.sourcesWg.Add(1)go func() {+ defer e.sourcesWg.Done()err := fileSystemSource.Chunks(ctx, e.ChunksChan())if err != nil {logrus.WithError(err).Error("error scanning filesystem")}- close(e.ChunksChan())}()return nil}diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex bc2f869ad..6cf16552a 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -3,9 +3,10 @@ package engineimport ("context""fmt"+ "runtime"+"github.com/go-errors/errors""github.com/go-git/go-git/v5/plumbing/object"- "runtime"gogit "github.com/go-git/go-git/v5""github.com/go-git/go-git/v5/plumbing"@@ -102,12 +103,13 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,}})+ e.sourcesWg.Add(1)go func() {+ defer e.sourcesWg.Done()err := gitSource.ScanRepo(ctx, repo, repoPath, scanOptions, e.ChunksChan())if err != nil {logrus.WithError(err).Fatal("could not scan repo")}- close(e.ChunksChan())}()return nil}diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.goindex 268976ff0..6b25b8a96 100644--- a/pkg/engine/git_test.go+++ b/pkg/engine/git_test.go@@ -57,6 +57,7 @@ func TestGitEngine(t *testing.T) {WithDetectors(false, DefaultDetectors()...),)e.ScanGit(ctx, path, tTest.branch, tTest.base, tTest.maxDepth, tTest.filter)+ go e.Finish()resultCount := 0for result := range e.ResultsChan() {switch meta := result.SourceMetadata.GetData().(type) {@@ -93,11 +94,17 @@ func BenchmarkGitEngine(b *testing.B) {WithDecoders(decoders.DefaultDecoders()...),WithDetectors(false, DefaultDetectors()...),)- for i := 0; i < b.N; i++ {- e.ScanGit(ctx, path, "", "", 0, common.FilterEmpty())+ go func() {resultCount := 0- for _ = range e.ResultsChan() {+ for range e.ResultsChan() {resultCount++}+ }()++ for i := 0; i < b.N; i++ {+ // TODO: this is measuring the time it takes to initialize the source+ // and not to do the full scan+ e.ScanGit(ctx, path, "", "", 0, common.FilterEmpty())}+ e.Finish()}diff --git a/pkg/engine/github.go b/pkg/engine/github.goindex 438c26b89..c3ebe9b5f 100644--- a/pkg/engine/github.go+++ b/pkg/engine/github.go@@ -40,12 +40,13 @@ func (e *Engine) ScanGitHub(ctx context.Context, endpoint string, repos, orgs []return err}+ e.sourcesWg.Add(1)go func() {+ defer e.sourcesWg.Done()err := source.Chunks(ctx, e.ChunksChan())if err != nil {logrus.WithError(err).Fatal("could not scan github")}- close(e.ChunksChan())}()return nil}diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.goindex 9dcb65af0..ea89be94d 100644--- a/pkg/engine/gitlab.go+++ b/pkg/engine/gitlab.go@@ -2,6 +2,8 @@ package engineimport ("fmt"+ "runtime"+"github.com/go-errors/errors""github.com/sirupsen/logrus""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"@@ -9,7 +11,6 @@ import ("golang.org/x/net/context""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"- "runtime")func (e *Engine) ScanGitLab(ctx context.Context, endpoint, token string, repositories []string) error {@@ -44,12 +45,14 @@ func (e *Engine) ScanGitLab(ctx context.Context, endpoint, token string, repositif err != nil {return errors.WrapPrefix(err, "could not init GitLab source", 0)}++ e.sourcesWg.Add(1)go func() {+ defer e.sourcesWg.Done()err := gitlabSource.Chunks(ctx, e.ChunksChan())if err != nil {logrus.WithError(err).Error("error scanning GitLab")}- close(e.ChunksChan())}()return nil}diff --git a/pkg/engine/s3.go b/pkg/engine/s3.goindex 16bf42d13..84d31bf5c 100644--- a/pkg/engine/s3.go+++ b/pkg/engine/s3.go@@ -3,6 +3,8 @@ package engineimport ("context""fmt"+ "runtime"+"github.com/go-errors/errors""github.com/sirupsen/logrus""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"@@ -10,7 +12,6 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"- "runtime")func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool, buckets []string) error {@@ -46,12 +47,14 @@ func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool,if err != nil {return errors.WrapPrefix(err, "failed to init S3 source", 0)}++ e.sourcesWg.Add(1)go func() {+ defer e.sourcesWg.Done()err := s3Source.Chunks(ctx, e.ChunksChan())if err != nil {logrus.WithError(err).Error("error scanning s3")}- close(e.ChunksChan())}()return nil}diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.goindex 555089ca5..92b011ae4 100644--- a/pkg/engine/syslog.go+++ b/pkg/engine/syslog.go@@ -2,11 +2,12 @@ package engineimport ("context"+ "os"+"github.com/go-errors/errors""github.com/sirupsen/logrus""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"- "os""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog"@@ -46,12 +47,13 @@ func (e *Engine) ScanSyslog(ctx context.Context, address, protocol, certPath, kereturn err}+ e.sourcesWg.Add(1)go func() {+ defer e.sourcesWg.Done()err := source.Chunks(ctx, e.ChunksChan())if err != nil {logrus.WithError(err).Fatal("could not scan syslog")}- close(e.ChunksChan())}()return nil}diff --git a/pkg/output/json.go b/pkg/output/json.gonew file mode 100644index 000000000..ecf101676--- /dev/null+++ b/pkg/output/json.go@@ -0,0 +1,54 @@+package output++import (+ "encoding/json"+ "fmt"++ "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+)++func PrintJSON(r *detectors.ResultWithMetadata) {+ v := &struct {+ // SourceMetadata contains source-specific contextual information.+ SourceMetadata *source_metadatapb.MetaData+ // SourceID is the ID of the source that the API uses to map secrets to specific sources.+ SourceID int64+ // SourceType is the type of Source.+ SourceType sourcespb.SourceType+ // SourceName is the name of the Source.+ SourceName string+ // DetectorType is the type of Detector.+ DetectorType detectorspb.DetectorType+ // DetectorName is the string name of the DetectorType.+ DetectorName string+ Verified bool+ // Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.+ Raw []byte+ // Redacted contains the redacted version of the raw secret identification data for display purposes.+ // A secret ID should be used if available.+ Redacted string+ ExtraData map[string]string+ StructuredData *detectorspb.StructuredData+ }{+ SourceMetadata: r.SourceMetadata,+ SourceID: r.SourceID,+ SourceType: r.SourceType,+ SourceName: r.SourceName,+ DetectorType: r.DetectorType,+ DetectorName: r.DetectorType.String(),+ Verified: r.Verified,+ Raw: r.Raw,+ Redacted: r.Redacted,+ ExtraData: r.ExtraData,+ StructuredData: r.StructuredData,+ }+ out, err := json.Marshal(v)+ if err != nil {+ logrus.WithError(err).Fatal("could not marshal result")+ }+ fmt.Println(string(out))+}diff --git a/pkg/output/legacy_json.go b/pkg/output/legacy_json.goindex 08a7d906a..76ad418b5 100644--- a/pkg/output/legacy_json.go+++ b/pkg/output/legacy_json.go@@ -1,9 +1,11 @@package outputimport (+ "encoding/json""fmt""log""net/url"+ "os""strings"gogit "github.com/go-git/go-git/v5"@@ -13,8 +15,25 @@ import ("github.com/sirupsen/logrus""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git")+func PrintLegacyJSON(r *detectors.ResultWithMetadata) {+ repoPath, remote, err := git.PrepareRepo(r.SourceMetadata.GetGithub().Repository)+ if err != nil || repoPath == "" {+ logrus.WithError(err).Fatal("error preparing git repo for scanning")+ }+ if remote {+ defer os.RemoveAll(repoPath)+ }+ legacy := ConvertToLegacyJSON(r, repoPath)+ out, err := json.Marshal(legacy)+ if err != nil {+ logrus.WithError(err).Fatal("could not marshal result")+ }+ fmt.Println(string(out))+}+func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) *LegacyJSONOutput {var source LegacyJSONCompatibleSourceswitch r.SourceType {