Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add common sentry recover library and add into goroutines (#738)
Dustin Deckerfa9479100ecb
diff --git a/go.mod b/go.modindex 5dbd09dd0..9efbf65b7 100644--- a/go.mod+++ b/go.mod@@ -47,7 +47,6 @@ require (github.com/xanzy/go-gitlab v0.73.1go.uber.org/zap v1.22.0golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e- golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959cgolang.org/x/sync v0.0.0-20220601150217-0de741cfad7fgoogle.golang.org/genproto v0.0.0-20220617124728-180714bec0ad@@ -114,6 +113,7 @@ require (go.uber.org/atomic v1.7.0 // indirectgo.uber.org/multierr v1.6.0 // indirectgolang.org/x/mod v0.5.0 // indirect+ golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirectgolang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirectgolang.org/x/text v0.3.7 // indirectgolang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirectdiff --git a/hack/snifftest/main.go b/hack/snifftest/main.goindex 2de774af3..5f4082157 100644--- a/hack/snifftest/main.go+++ b/hack/snifftest/main.go@@ -1,7 +1,6 @@package mainimport (- "context""fmt""os""reflect"@@ -16,6 +15,7 @@ import ("golang.org/x/sync/semaphore""gopkg.in/alecthomas/kingpin.v2"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine"diff --git a/main.go b/main.goindex 4b7eeaba1..e8742c9ba 100644--- a/main.go+++ b/main.go@@ -1,7 +1,6 @@package mainimport (- "context""fmt""log""net/http"@@ -19,15 +18,15 @@ import ("github.com/sirupsen/logrus""gopkg.in/alecthomas/kingpin.v2"- "github.com/trufflesecurity/trufflehog/v3/pkg/sources"- "github.com/trufflesecurity/trufflehog/v3/pkg/updater"- "github.com/trufflesecurity/trufflehog/v3/pkg/version"-"github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/engine""github.com/trufflesecurity/trufflehog/v3/pkg/output"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"+ "github.com/trufflesecurity/trufflehog/v3/pkg/updater"+ "github.com/trufflesecurity/trufflehog/v3/pkg/version")var (@@ -264,7 +263,7 @@ func run(state overseer.State) {}}// asynchronously wait for scanning to finish and cleanup- go e.Finish()+ go e.Finish(ctx)if !*jsonLegacy && !*jsonOut {fmt.Fprintf(os.Stderr, "π·ππ· TruffleHog. Unearth your secrets. π·ππ·\n\n")diff --git a/pkg/common/recover.go b/pkg/common/recover.gonew file mode 100644index 000000000..56888dd0f--- /dev/null+++ b/pkg/common/recover.go@@ -0,0 +1,27 @@+package common++import (+ "fmt"+ "os"+ "runtime/debug"+ "time"++ "github.com/getsentry/sentry-go"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++// Recover handles panics and reports to Sentry before exiting.+func Recover(ctx context.Context) {+ if err := recover(); err != nil {+ panicStack := string(debug.Stack())+ if eventID := sentry.CurrentHub().Recover(err); eventID != nil {+ ctx.Logger().Info("panic captured", "event_id", *eventID)+ }+ fmt.Fprint(os.Stderr, panicStack)+ if !sentry.Flush(time.Second * 5) {+ ctx.Logger().Info("sentry flush failed")+ }+ os.Exit(1)+ }+}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 57d4e8a9f..4b579a9e1 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -2,7 +2,6 @@ package engineimport ("bytes"- "context""runtime""strings""sync"@@ -11,6 +10,8 @@ import ("github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"@@ -97,6 +98,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {for i := 0; i < e.concurrency; i++ {e.workersWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.workersWg.Done()e.detectorWorker(ctx)}()@@ -108,7 +110,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {// Finish waits for running sources to complete and workers to finish scanning// chunks before closing their respective channels. Once Finish is called, no// more sources may be scanned by the engine.-func (e *Engine) Finish() {+func (e *Engine) Finish(ctx context.Context) {+ defer common.Recover(ctx)// wait for the sources to finish putting chunks onto the chunks channele.sourcesWg.Wait()close(e.chunks)diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.goindex 22eb45452..7823303cc 100644--- a/pkg/engine/filesystem.go+++ b/pkg/engine/filesystem.go@@ -1,7 +1,6 @@package engineimport (- "context""runtime""github.com/go-errors/errors"@@ -9,6 +8,8 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"@@ -33,6 +34,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {}e.sourcesWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.sourcesWg.Done()err := fileSystemSource.Chunks(ctx, e.ChunksChan())if err != nil {diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex d4fed1e68..1638d4ff2 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -1,7 +1,6 @@package engineimport (- "context""fmt""runtime"@@ -11,6 +10,8 @@ import ("github.com/go-git/go-git/v5/plumbing/object""github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"@@ -106,6 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {e.sourcesWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.sourcesWg.Done()err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())if err != nil {diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.goindex 2d841f894..9803066c0 100644--- a/pkg/engine/git_test.go+++ b/pkg/engine/git_test.go@@ -1,11 +1,11 @@package engineimport (- "context""os""testing""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"@@ -67,7 +67,7 @@ func TestGitEngine(t *testing.T) {if err := e.ScanGit(ctx, cfg); err != nil {return}- go e.Finish()+ go e.Finish(ctx)resultCount := 0for result := range e.ResultsChan() {switch meta := result.SourceMetadata.GetData().(type) {@@ -122,5 +122,5 @@ func BenchmarkGitEngine(b *testing.B) {return}}- e.Finish()+ e.Finish(ctx)}diff --git a/pkg/engine/github.go b/pkg/engine/github.goindex f8f8d8480..0c8f9d157 100644--- a/pkg/engine/github.go+++ b/pkg/engine/github.go@@ -1,12 +1,12 @@package engineimport (- "context"-"github.com/sirupsen/logrus""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"@@ -43,6 +43,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {e.sourcesWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.sourcesWg.Done()err := source.Chunks(ctx, e.ChunksChan())if err != nil {diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.goindex 1ae0f6272..cf3fbfcfc 100644--- a/pkg/engine/gitlab.go+++ b/pkg/engine/gitlab.go@@ -6,10 +6,11 @@ import ("github.com/go-errors/errors""github.com/sirupsen/logrus"- "golang.org/x/net/context""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"@@ -51,6 +52,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {e.sourcesWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.sourcesWg.Done()err := gitlabSource.Chunks(ctx, e.ChunksChan())if err != nil {diff --git a/pkg/engine/s3.go b/pkg/engine/s3.goindex 5295e2748..7c1df9f36 100644--- a/pkg/engine/s3.go+++ b/pkg/engine/s3.go@@ -1,7 +1,6 @@package engineimport (- "context""fmt""runtime"@@ -10,6 +9,8 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"@@ -53,6 +54,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {e.sourcesWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.sourcesWg.Done()err := s3Source.Chunks(ctx, e.ChunksChan())if err != nil {diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.goindex 3b0ffbd5b..ff1754ab2 100644--- a/pkg/engine/syslog.go+++ b/pkg/engine/syslog.go@@ -1,7 +1,6 @@package engineimport (- "context""os""github.com/go-errors/errors"@@ -9,6 +8,8 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog"@@ -51,6 +52,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {e.sourcesWg.Add(1)go func() {+ defer common.Recover(ctx)defer e.sourcesWg.Done()err := source.Chunks(ctx, e.ChunksChan())if err != nil {diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.goindex 925da99ce..09ae120fc 100644--- a/pkg/gitparse/gitparse.go+++ b/pkg/gitparse/gitparse.go@@ -11,6 +11,9 @@ import ("time"log "github.com/sirupsen/logrus"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context")// DateFormat is the standard date format for git.@@ -34,7 +37,7 @@ type Diff struct {}// RepoPath parses the output of the `git log` command for the `source` path.-func RepoPath(source string, head string) (chan Commit, error) {+func RepoPath(ctx context.Context, source string, head string) (chan Commit, error) {commitChan := make(chan Commit)args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}@@ -77,6 +80,7 @@ func RepoPath(source string, head string) (chan Commit, error) {}()go func() {+ defer common.Recover(ctx)for {line, err := outReader.ReadBytes([]byte("\n")[0])if err != nil && len(line) == 0 {diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 22fbc81e3..c2030b030 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -1,7 +1,6 @@package filesystemimport (- "context""fmt""io""io/fs"@@ -15,6 +14,7 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.goindex 21c39427b..a3e07525c 100644--- a/pkg/sources/filesystem/filesystem_test.go+++ b/pkg/sources/filesystem/filesystem_test.go@@ -1,7 +1,6 @@package filesystemimport (- "context""testing""time"@@ -9,6 +8,7 @@ import (log "github.com/sirupsen/logrus""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 8b5d5bbfb..894eb61b4 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -2,7 +2,6 @@ package gitimport ("bytes"- "context""fmt""io""io/ioutil"@@ -27,6 +26,7 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/gitparse""github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"@@ -279,7 +279,7 @@ func GitCmdCheck() error {return nil}-func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {if err := GitCmdCheck(); err != nil {return err}@@ -287,7 +287,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOpzerolog.SetGlobalLevel(zerolog.Disabled)}- commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash)+ commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash)if err != nil {return err}@@ -412,9 +412,9 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunkreturn nil}-func (s *Git) ScanRepo(_ context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {start := time.Now().UnixNano()- if err := s.ScanCommits(repo, repoPath, scanOptions, chunksChan); err != nil {+ if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {return err}if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil {@@ -591,9 +591,9 @@ func PrepareRepo(uriString string) (string, bool, error) {remotePath := uri.String()remote = truepath, _, err = CloneRepoUsingSSH(remotePath)- if err != nil {- return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)- }+ if err != nil {+ return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)+ }default:return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString)}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 1169e209e..9b544bd97 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -2,19 +2,20 @@ package gitimport ("bytes"- "context""fmt""strings""testing""github.com/kylelemons/godebug/pretty"log "github.com/sirupsen/logrus"+ "google.golang.org/protobuf/types/known/anypb"+"github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"- "google.golang.org/protobuf/types/known/anypb")func TestSource_Scan(t *testing.T) {diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 457ba0915..77ce7336b 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -1,7 +1,6 @@package githubimport (- "context""fmt""net/http""os"@@ -25,6 +24,7 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.goindex a5ae67f69..a58d4341a 100644--- a/pkg/sources/github/github_integration_test.go+++ b/pkg/sources/github/github_integration_test.go@@ -4,7 +4,6 @@package githubimport (- "context""encoding/base64""fmt""os"@@ -19,6 +18,7 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 0524f0d7e..b74d9b722 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -2,7 +2,6 @@ package githubimport ("bytes"- "context""crypto/rand""crypto/rsa""crypto/x509"@@ -21,6 +20,7 @@ import ("google.golang.org/protobuf/types/known/anypb""gopkg.in/h2non/gock.v1"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb")diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex ef8878fb0..86355c73e 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -1,7 +1,6 @@package gitlabimport (- "context""fmt""net/url""os"@@ -11,6 +10,7 @@ import ("sync""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex 8aa373f27..fdec4cb1a 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -1,7 +1,6 @@package gitlabimport (- "context""fmt""io""reflect"@@ -12,6 +11,7 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex c2b949a19..49ba0a354 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -1,7 +1,6 @@package s3import (- "context""fmt""strings""sync"@@ -15,15 +14,17 @@ import (diskbufferreader "github.com/bill-rich/disk-buffer-reader""github.com/go-errors/errors"log "github.com/sirupsen/logrus"+ "golang.org/x/sync/semaphore"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb"+"github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer""github.com/trufflesecurity/trufflehog/v3/pkg/sources"- "golang.org/x/sync/semaphore"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/types/known/anypb")type Source struct {@@ -193,6 +194,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan}wg.Add(1)go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {+ defer common.Recover(ctx)defer sem.Release(1)defer wg.Done()//defer log.Debugf("DONE - %s", *obj.Key)diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.goindex 5ce7cd2d3..6aa667ef9 100644--- a/pkg/sources/s3/s3_test.go+++ b/pkg/sources/s3/s3_test.go@@ -1,7 +1,6 @@package s3import (- "context""encoding/base64""fmt""os"@@ -14,6 +13,7 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex ea0d36c71..2cdf54063 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -1,12 +1,12 @@package sourcesimport (- "context""sync""google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb")diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.goindex 2731879a5..11b4784ba 100644--- a/pkg/sources/syslog/syslog.go+++ b/pkg/sources/syslog/syslog.go@@ -1,10 +1,8 @@package syslogimport (- "context""crypto/tls""fmt"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""io""net""runtime"@@ -19,6 +17,8 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"@@ -210,6 +210,7 @@ func (s *Source) parseSyslogMetadata(input []byte, remote string) (*source_metad}func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksChan chan *sources.Chunk) {+ defer common.Recover(ctx)for {if common.IsDone(ctx) {return