Skip to content

Find, verify, and analyze leaked credentials

Add common sentry recover library and add into goroutines (#738)

Dustin Deckerfa9479100ecb
diff --git a/go.mod b/go.mod
index 5dbd09dd0..9efbf65b7 100644
--- a/go.mod
+++ b/go.mod
@@ -47,7 +47,6 @@ require (
github.com/xanzy/go-gitlab v0.73.1
go.uber.org/zap v1.22.0
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
- golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad
@@ -114,6 +113,7 @@ require (
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/mod v0.5.0 // indirect
+ golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go
index 2de774af3..5f4082157 100644
--- a/hack/snifftest/main.go
+++ b/hack/snifftest/main.go
@@ -1,7 +1,6 @@
package main
import (
- "context"
"fmt"
"os"
"reflect"
@@ -16,6 +15,7 @@ import (
"golang.org/x/sync/semaphore"
"gopkg.in/alecthomas/kingpin.v2"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
diff --git a/main.go b/main.go
index 4b7eeaba1..e8742c9ba 100644
--- a/main.go
+++ b/main.go
@@ -1,7 +1,6 @@
package main
import (
- "context"
"fmt"
"log"
"net/http"
@@ -19,15 +18,15 @@ import (
"github.com/sirupsen/logrus"
"gopkg.in/alecthomas/kingpin.v2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
- "github.com/trufflesecurity/trufflehog/v3/pkg/updater"
- "github.com/trufflesecurity/trufflehog/v3/pkg/version"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/updater"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/version"
)
var (
@@ -264,7 +263,7 @@ func run(state overseer.State) {
}
}
// asynchronously wait for scanning to finish and cleanup
- go e.Finish()
+ go e.Finish(ctx)
if !*jsonLegacy && !*jsonOut {
fmt.Fprintf(os.Stderr, "πŸ·πŸ”‘πŸ· TruffleHog. Unearth your secrets. πŸ·πŸ”‘πŸ·\n\n")
diff --git a/pkg/common/recover.go b/pkg/common/recover.go
new file mode 100644
index 000000000..56888dd0f
--- /dev/null
+++ b/pkg/common/recover.go
@@ -0,0 +1,27 @@
+package common
+
+import (
+ "fmt"
+ "os"
+ "runtime/debug"
+ "time"
+
+ "github.com/getsentry/sentry-go"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+// Recover handles panics and reports to Sentry before exiting.
+func Recover(ctx context.Context) {
+ if err := recover(); err != nil {
+ panicStack := string(debug.Stack())
+ if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
+ ctx.Logger().Info("panic captured", "event_id", *eventID)
+ }
+ fmt.Fprint(os.Stderr, panicStack)
+ if !sentry.Flush(time.Second * 5) {
+ ctx.Logger().Info("sentry flush failed")
+ }
+ os.Exit(1)
+ }
+}
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 57d4e8a9f..4b579a9e1 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -2,7 +2,6 @@ package engine
import (
"bytes"
- "context"
"runtime"
"strings"
"sync"
@@ -11,6 +10,8 @@ import (
"github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -97,6 +98,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
for i := 0; i < e.concurrency; i++ {
e.workersWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.workersWg.Done()
e.detectorWorker(ctx)
}()
@@ -108,7 +110,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
// Finish waits for running sources to complete and workers to finish scanning
// chunks before closing their respective channels. Once Finish is called, no
// more sources may be scanned by the engine.
-func (e *Engine) Finish() {
+func (e *Engine) Finish(ctx context.Context) {
+ defer common.Recover(ctx)
// wait for the sources to finish putting chunks onto the chunks channel
e.sourcesWg.Wait()
close(e.chunks)
diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go
index 22eb45452..7823303cc 100644
--- a/pkg/engine/filesystem.go
+++ b/pkg/engine/filesystem.go
@@ -1,7 +1,6 @@
package engine
import (
- "context"
"runtime"
"github.com/go-errors/errors"
@@ -9,6 +8,8 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
@@ -33,6 +34,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
}
e.sourcesWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.sourcesWg.Done()
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
if err != nil {
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index d4fed1e68..1638d4ff2 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -1,7 +1,6 @@
package engine
import (
- "context"
"fmt"
"runtime"
@@ -11,6 +10,8 @@ import (
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -106,6 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
e.sourcesWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.sourcesWg.Done()
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
if err != nil {
diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go
index 2d841f894..9803066c0 100644
--- a/pkg/engine/git_test.go
+++ b/pkg/engine/git_test.go
@@ -1,11 +1,11 @@
package engine
import (
- "context"
"os"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -67,7 +67,7 @@ func TestGitEngine(t *testing.T) {
if err := e.ScanGit(ctx, cfg); err != nil {
return
}
- go e.Finish()
+ go e.Finish(ctx)
resultCount := 0
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
@@ -122,5 +122,5 @@ func BenchmarkGitEngine(b *testing.B) {
return
}
}
- e.Finish()
+ e.Finish(ctx)
}
diff --git a/pkg/engine/github.go b/pkg/engine/github.go
index f8f8d8480..0c8f9d157 100644
--- a/pkg/engine/github.go
+++ b/pkg/engine/github.go
@@ -1,12 +1,12 @@
package engine
import (
- "context"
-
"github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
@@ -43,6 +43,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
e.sourcesWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.sourcesWg.Done()
err := source.Chunks(ctx, e.ChunksChan())
if err != nil {
diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
index 1ae0f6272..cf3fbfcfc 100644
--- a/pkg/engine/gitlab.go
+++ b/pkg/engine/gitlab.go
@@ -6,10 +6,11 @@ import (
"github.com/go-errors/errors"
"github.com/sirupsen/logrus"
- "golang.org/x/net/context"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
@@ -51,6 +52,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
e.sourcesWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.sourcesWg.Done()
err := gitlabSource.Chunks(ctx, e.ChunksChan())
if err != nil {
diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go
index 5295e2748..7c1df9f36 100644
--- a/pkg/engine/s3.go
+++ b/pkg/engine/s3.go
@@ -1,7 +1,6 @@
package engine
import (
- "context"
"fmt"
"runtime"
@@ -10,6 +9,8 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -53,6 +54,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
e.sourcesWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.sourcesWg.Done()
err := s3Source.Chunks(ctx, e.ChunksChan())
if err != nil {
diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go
index 3b0ffbd5b..ff1754ab2 100644
--- a/pkg/engine/syslog.go
+++ b/pkg/engine/syslog.go
@@ -1,7 +1,6 @@
package engine
import (
- "context"
"os"
"github.com/go-errors/errors"
@@ -9,6 +8,8 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog"
@@ -51,6 +52,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
e.sourcesWg.Add(1)
go func() {
+ defer common.Recover(ctx)
defer e.sourcesWg.Done()
err := source.Chunks(ctx, e.ChunksChan())
if err != nil {
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go
index 925da99ce..09ae120fc 100644
--- a/pkg/gitparse/gitparse.go
+++ b/pkg/gitparse/gitparse.go
@@ -11,6 +11,9 @@ import (
"time"
log "github.com/sirupsen/logrus"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
// DateFormat is the standard date format for git.
@@ -34,7 +37,7 @@ type Diff struct {
}
// RepoPath parses the output of the `git log` command for the `source` path.
-func RepoPath(source string, head string) (chan Commit, error) {
+func RepoPath(ctx context.Context, source string, head string) (chan Commit, error) {
commitChan := make(chan Commit)
args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}
@@ -77,6 +80,7 @@ func RepoPath(source string, head string) (chan Commit, error) {
}()
go func() {
+ defer common.Recover(ctx)
for {
line, err := outReader.ReadBytes([]byte("\n")[0])
if err != nil && len(line) == 0 {
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 22fbc81e3..c2030b030 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -1,7 +1,6 @@
package filesystem
import (
- "context"
"fmt"
"io"
"io/fs"
@@ -15,6 +14,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go
index 21c39427b..a3e07525c 100644
--- a/pkg/sources/filesystem/filesystem_test.go
+++ b/pkg/sources/filesystem/filesystem_test.go
@@ -1,7 +1,6 @@
package filesystem
import (
- "context"
"testing"
"time"
@@ -9,6 +8,7 @@ import (
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 8b5d5bbfb..894eb61b4 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -2,7 +2,6 @@ package git
import (
"bytes"
- "context"
"fmt"
"io"
"io/ioutil"
@@ -27,6 +26,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -279,7 +279,7 @@ func GitCmdCheck() error {
return nil
}
-func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
if err := GitCmdCheck(); err != nil {
return err
}
@@ -287,7 +287,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
zerolog.SetGlobalLevel(zerolog.Disabled)
}
- commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash)
+ commitChan, err := gitparse.RepoPath(ctx, path, scanOptions.HeadHash)
if err != nil {
return err
}
@@ -412,9 +412,9 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
return nil
}
-func (s *Git) ScanRepo(_ context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
start := time.Now().UnixNano()
- if err := s.ScanCommits(repo, repoPath, scanOptions, chunksChan); err != nil {
+ if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
return err
}
if err := s.ScanUnstaged(repo, scanOptions, chunksChan); err != nil {
@@ -591,9 +591,9 @@ func PrepareRepo(uriString string) (string, bool, error) {
remotePath := uri.String()
remote = true
path, _, err = CloneRepoUsingSSH(remotePath)
- if err != nil {
- return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
- }
+ if err != nil {
+ return path, remote, fmt.Errorf("failed to clone unauthenticated Git repo (%s): %s", remotePath, err)
+ }
default:
return "", remote, fmt.Errorf("unsupported Git URI: %s", uriString)
}
diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go
index 1169e209e..9b544bd97 100644
--- a/pkg/sources/git/git_test.go
+++ b/pkg/sources/git/git_test.go
@@ -2,19 +2,20 @@ package git
import (
"bytes"
- "context"
"fmt"
"strings"
"testing"
"github.com/kylelemons/godebug/pretty"
log "github.com/sirupsen/logrus"
+ "google.golang.org/protobuf/types/known/anypb"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
- "google.golang.org/protobuf/types/known/anypb"
)
func TestSource_Scan(t *testing.T) {
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 457ba0915..77ce7336b 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -1,7 +1,6 @@
package github
import (
- "context"
"fmt"
"net/http"
"os"
@@ -25,6 +24,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go
index a5ae67f69..a58d4341a 100644
--- a/pkg/sources/github/github_integration_test.go
+++ b/pkg/sources/github/github_integration_test.go
@@ -4,7 +4,6 @@
package github
import (
- "context"
"encoding/base64"
"fmt"
"os"
@@ -19,6 +18,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 0524f0d7e..b74d9b722 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -2,7 +2,6 @@ package github
import (
"bytes"
- "context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
@@ -21,6 +20,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"gopkg.in/h2non/gock.v1"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
)
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index ef8878fb0..86355c73e 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -1,7 +1,6 @@
package gitlab
import (
- "context"
"fmt"
"net/url"
"os"
@@ -11,6 +10,7 @@ import (
"sync"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go
index 8aa373f27..fdec4cb1a 100644
--- a/pkg/sources/gitlab/gitlab_test.go
+++ b/pkg/sources/gitlab/gitlab_test.go
@@ -1,7 +1,6 @@
package gitlab
import (
- "context"
"fmt"
"io"
"reflect"
@@ -12,6 +11,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go
index c2b949a19..49ba0a354 100644
--- a/pkg/sources/s3/s3.go
+++ b/pkg/sources/s3/s3.go
@@ -1,7 +1,6 @@
package s3
import (
- "context"
"fmt"
"strings"
"sync"
@@ -15,15 +14,17 @@ import (
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
log "github.com/sirupsen/logrus"
+ "golang.org/x/sync/semaphore"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
- "golang.org/x/sync/semaphore"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/types/known/anypb"
)
type Source struct {
@@ -193,6 +194,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
}
wg.Add(1)
go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {
+ defer common.Recover(ctx)
defer sem.Release(1)
defer wg.Done()
//defer log.Debugf("DONE - %s", *obj.Key)
diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go
index 5ce7cd2d3..6aa667ef9 100644
--- a/pkg/sources/s3/s3_test.go
+++ b/pkg/sources/s3/s3_test.go
@@ -1,7 +1,6 @@
package s3
import (
- "context"
"encoding/base64"
"fmt"
"os"
@@ -14,6 +13,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index ea0d36c71..2cdf54063 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -1,12 +1,12 @@
package sources
import (
- "context"
"sync"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
)
diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.go
index 2731879a5..11b4784ba 100644
--- a/pkg/sources/syslog/syslog.go
+++ b/pkg/sources/syslog/syslog.go
@@ -1,10 +1,8 @@
package syslog
import (
- "context"
"crypto/tls"
"fmt"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"io"
"net"
"runtime"
@@ -19,6 +17,8 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -210,6 +210,7 @@ func (s *Source) parseSyslogMetadata(input []byte, remote string) (*source_metad
}
func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksChan chan *sources.Chunk) {
+ defer common.Recover(ctx)
for {
if common.IsDone(ctx) {
return