Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[chore] Log non-fatal errors encountered during a scan (#3612)
Miccah6a77b73f5133
diff --git a/main.go b/main.goindex abd635f07..31db0e3b7 100644--- a/main.go+++ b/main.go@@ -633,6 +633,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,}}()+ var ref sources.JobProgressRefswitch cmd {case gitScan.FullCommand():gitCfg := sources.GitConfig{@@ -645,7 +646,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,Bare: *gitScanBare,ExcludeGlobs: *gitScanExcludeGlobs,}- if err = eng.ScanGit(ctx, gitCfg); err != nil {+ if ref, err = eng.ScanGit(ctx, gitCfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)}case githubScan.FullCommand():@@ -674,7 +675,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,CommentsTimeframeDays: *githubCommentsTimeframeDays,Filter: filter,}- if err := eng.ScanGitHub(ctx, cfg); err != nil {+ if ref, err = eng.ScanGitHub(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)}case githubExperimentalScan.FullCommand():@@ -685,7 +686,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,CollisionThreshold: *githubExperimentalCollisionThreshold,DeleteCachedData: *githubExperimentalDeleteCache,}- if err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {+ if ref, err = eng.ScanGitHubExperimental(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)}case gitlabScan.FullCommand():@@ -702,7 +703,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,ExcludeRepos: *gitlabScanExcludeRepos,Filter: filter,}- if err := eng.ScanGitLab(ctx, cfg); err != nil {+ if ref, err = eng.ScanGitLab(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)}case filesystemScan.FullCommand():@@ -717,7 +718,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,IncludePathsFile: *filesystemScanIncludePaths,ExcludePathsFile: *filesystemScanExcludePaths,}- if err = eng.ScanFileSystem(ctx, cfg); err != nil {+ if ref, err = eng.ScanFileSystem(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)}case s3Scan.FullCommand():@@ -731,7 +732,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,CloudCred: *s3ScanCloudEnv,MaxObjectSize: int64(*s3ScanMaxObjectSize),}- if err := eng.ScanS3(ctx, cfg); err != nil {+ if ref, err = eng.ScanS3(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)}case syslogScan.FullCommand():@@ -743,15 +744,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,KeyPath: *syslogTLSKey,Concurrency: *concurrency,}- if err := eng.ScanSyslog(ctx, cfg); err != nil {+ if ref, err = eng.ScanSyslog(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)}case circleCiScan.FullCommand():- if err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {+ if ref, err = eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)}case travisCiScan.FullCommand():- if err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {+ if ref, err = eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)}case gcsScan.FullCommand():@@ -768,7 +769,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,Concurrency: *concurrency,MaxObjectSize: int64(*gcsMaxObjectSize),}- if err := eng.ScanGCS(ctx, cfg); err != nil {+ if ref, err = eng.ScanGCS(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)}case dockerScan.FullCommand():@@ -777,7 +778,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,Images: *dockerScanImages,UseDockerKeychain: *dockerScanToken == "",}- if err := eng.ScanDocker(ctx, cfg); err != nil {+ if ref, err = eng.ScanDocker(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)}case postmanScan.FullCommand():@@ -814,7 +815,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,WorkspacePaths: *postmanWorkspacePaths,EnvironmentPaths: *postmanEnvironmentPaths,}- if err := eng.ScanPostman(ctx, cfg); err != nil {+ if ref, err = eng.ScanPostman(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)}case elasticsearchScan.FullCommand():@@ -830,7 +831,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,SinceTimestamp: *elasticsearchSinceTimestamp,BestEffortScan: *elasticsearchBestEffortScan,}- if err := eng.ScanElasticsearch(ctx, cfg); err != nil {+ if ref, err = eng.ScanElasticsearch(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)}case jenkinsScan.FullCommand():@@ -840,7 +841,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,Username: *jenkinsUsername,Password: *jenkinsPassword,}- if err := eng.ScanJenkins(ctx, cfg); err != nil {+ if ref, err = eng.ScanJenkins(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)}case huggingfaceScan.FullCommand():@@ -873,7 +874,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,IncludePrs: *huggingfaceIncludePrs,Concurrency: *concurrency,}- if err := eng.ScanHuggingface(ctx, cfg); err != nil {+ if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)}default:@@ -885,6 +886,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)}+ // Print any errors reported during the scan.+ if errs := ref.Snapshot().Errors; len(errs) > 0 {+ errMsgs := make([]string, len(errs))+ for i := 0; i < len(errs); i++ {+ errMsgs[i] = errs[i].Error()+ }+ ctx.Logger().Error(nil, "encountered errors during scan", "errors", errMsgs)+ }+if *printAvgDetectorTime {printAverageDetectorTime(eng)}diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.goindex 2d588842b..97627dd77 100644--- a/pkg/engine/circleci.go+++ b/pkg/engine/circleci.go@@ -8,11 +8,12 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci")// ScanCircleCI scans CircleCI logs.-func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {+func (e *Engine) ScanCircleCI(ctx context.Context, token string) (sources.JobProgressRef, error) {connection := &sourcespb.CircleCI{Credential: &sourcespb.CircleCI_Token{Token: token,@@ -23,7 +24,7 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal Circle CI connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - Circle CI"@@ -31,8 +32,7 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {circleSource := &circleci.Source{}if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, circleSource)- return err+ return e.sourceManager.Run(ctx, sourceName, circleSource)}diff --git a/pkg/engine/docker.go b/pkg/engine/docker.goindex 49e8af3ca..07f5376eb 100644--- a/pkg/engine/docker.go+++ b/pkg/engine/docker.go@@ -13,7 +13,7 @@ import ()// ScanDocker scans a given docker connection.-func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {+func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (sources.JobProgressRef, error) {connection := &sourcespb.Docker{Images: c.Images}switch {@@ -29,7 +29,7 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal gitlab connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - docker"@@ -37,8 +37,7 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {dockerSource := &docker.Source{}if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, dockerSource)- return err+ return e.sourceManager.Run(ctx, sourceName, dockerSource)}diff --git a/pkg/engine/elasticsearch.go b/pkg/engine/elasticsearch.goindex 3a8c814ba..bc6c49488 100644--- a/pkg/engine/elasticsearch.go+++ b/pkg/engine/elasticsearch.go@@ -13,7 +13,7 @@ import ()// ScanElasticsearch scans a Elasticsearch installation.-func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) error {+func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) (sources.JobProgressRef, error) {connection := &sourcespb.Elasticsearch{Nodes: c.Nodes,Username: c.Username,@@ -31,7 +31,7 @@ func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchCerr := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal Elasticsearch connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - Elasticsearch"@@ -39,8 +39,7 @@ func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchCelasticsearchSource := &elasticsearch.Source{}if err := elasticsearchSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, elasticsearchSource)- return err+ return e.sourceManager.Run(ctx, sourceName, elasticsearchSource)}diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex 2d735c906..9cda13599 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -271,7 +271,7 @@ func TestEngine_DuplicateSecrets(t *testing.T) {e.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{absPath}}- if err := e.ScanFileSystem(ctx, cfg); err != nil {+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {return}@@ -372,7 +372,7 @@ even more`,eng.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}- err = eng.ScanFileSystem(ctx, cfg)+ _, err = eng.ScanFileSystem(ctx, cfg)assert.NoError(t, err)assert.NoError(t, eng.Finish(ctx))@@ -420,7 +420,7 @@ func TestEngine_VersionedDetectorsVerifiedSecrets(t *testing.T) {e.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}- if err := e.ScanFileSystem(ctx, cfg); err != nil {+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {return}@@ -490,7 +490,7 @@ func TestEngine_CustomDetectorsDetectorsVerifiedSecrets(t *testing.T) {e.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}- if err := e.ScanFileSystem(ctx, cfg); err != nil {+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {return}@@ -540,7 +540,7 @@ func TestVerificationOverlapChunk(t *testing.T) {e.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{absPath}}- if err := e.ScanFileSystem(ctx, cfg); err != nil {+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {return}@@ -630,7 +630,7 @@ func TestVerificationOverlapChunkFalsePositive(t *testing.T) {e.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{absPath}}- err = e.ScanFileSystem(ctx, cfg)+ _, err = e.ScanFileSystem(ctx, cfg)assert.NoError(t, err)// Wait for all the chunks to be processed.@@ -678,7 +678,7 @@ func TestRetainFalsePositives(t *testing.T) {e.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{absPath}}- err = e.ScanFileSystem(ctx, cfg)+ _, err = e.ScanFileSystem(ctx, cfg)assert.NoError(t, err)// Wait for all the chunks to be processed.@@ -1259,7 +1259,7 @@ def test_something():eng.Start(ctx)cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}- err = eng.ScanFileSystem(ctx, cfg)+ _, err = eng.ScanFileSystem(ctx, cfg)assert.NoError(t, err)assert.NoError(t, eng.Finish(ctx))diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.goindex 169e52c31..fe61d4948 100644--- a/pkg/engine/filesystem.go+++ b/pkg/engine/filesystem.go@@ -13,7 +13,7 @@ import ()// ScanFileSystem scans a given file system.-func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {+func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) (sources.JobProgressRef, error) {connection := &sourcespb.Filesystem{Paths: c.Paths,IncludePathsFile: c.IncludePathsFile,@@ -23,7 +23,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal filesystem connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - filesystem"@@ -31,8 +31,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)fileSystemSource := &filesystem.Source{}if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)- return err+ return e.sourceManager.Run(ctx, sourceName, fileSystemSource)}diff --git a/pkg/engine/filesystem_integration_test.go b/pkg/engine/filesystem_integration_test.goindex b95b31191..f1dd2625d 100644--- a/pkg/engine/filesystem_integration_test.go+++ b/pkg/engine/filesystem_integration_test.go@@ -68,7 +68,7 @@ func TestFilesystem(t *testing.T) {})assert.NoError(t, err)e.Start(ctx)- err = e.ScanFileSystem(ctx, sources.FilesystemConfig{+ _, err = e.ScanFileSystem(ctx, sources.FilesystemConfig{Paths: []string{rootDir},ExcludePathsFile: filepath.Join(configDir, "exclude"),})diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.goindex f2bf82c98..2284c5c69 100644--- a/pkg/engine/gcs.go+++ b/pkg/engine/gcs.go@@ -13,10 +13,10 @@ import ()// ScanGCS with the provided options.-func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {+func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) (sources.JobProgressRef, error) {// Project ID is required if using any authenticated access.if c.ProjectID == "" && !c.WithoutAuth {- return fmt.Errorf("project ID is required")+ return sources.JobProgressRef{}, fmt.Errorf("project ID is required")}// If using unauthenticated access, the project ID is not used.@@ -35,13 +35,13 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {// Make sure only one auth method is selected.if !isAuthValid(ctx, c, connection) {- return fmt.Errorf("multiple auth methods selected, please select only one")+ return sources.JobProgressRef{}, fmt.Errorf("multiple auth methods selected, please select only one")}var conn anypb.Anyerr := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {- return fmt.Errorf("failed to marshal GCS connection: %w", err)+ return sources.JobProgressRef{}, fmt.Errorf("failed to marshal GCS connection: %w", err)}sourceName := "trufflehog - gcs"@@ -49,10 +49,9 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {gcsSource := &gcs.Source{}if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, gcsSource)- return err+ return e.sourceManager.Run(ctx, sourceName, gcsSource)}func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {diff --git a/pkg/engine/gcs_test.go b/pkg/engine/gcs_test.goindex d73b0bf5e..c903e00ee 100644--- a/pkg/engine/gcs_test.go+++ b/pkg/engine/gcs_test.go@@ -91,7 +91,7 @@ func TestScanGCS(t *testing.T) {}}()- err = e.ScanGCS(ctx, test.gcsConfig)+ _, err = e.ScanGCS(ctx, test.gcsConfig)if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {t.Errorf("ScanGCS() got: %v, want: %v", err, nil)returndiff --git a/pkg/engine/git.go b/pkg/engine/git.goindex 411b2689e..de487591f 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -13,7 +13,7 @@ import ()// ScanGit scans any git source.-func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {+func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) (sources.JobProgressRef, error) {connection := &sourcespb.Git{Head: c.HeadRef,Base: c.BaseRef,@@ -28,7 +28,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {var conn anypb.Anyif err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil {ctx.Logger().Error(err, "failed to marshal git connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - git"@@ -36,9 +36,8 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {gitSource := &git.Source{}if err := gitSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err := e.sourceManager.Run(ctx, sourceName, gitSource)- return err+ return e.sourceManager.Run(ctx, sourceName, gitSource)}diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.goindex 6bac1af99..14a48e0ef 100644--- a/pkg/engine/git_test.go+++ b/pkg/engine/git_test.go@@ -91,7 +91,7 @@ func TestGitEngine(t *testing.T) {BaseRef: tTest.base,MaxDepth: tTest.maxDepth,}- if err := e.ScanGit(ctx, cfg); err != nil {+ if _, err := e.ScanGit(ctx, cfg); err != nil {return}@@ -155,7 +155,7 @@ func BenchmarkGitEngine(b *testing.B) {cfg := sources.GitConfig{URI: repoUrl}b.ResetTimer()for i := 0; i < b.N; i++ {- if err := e.ScanGit(ctx, cfg); err != nil {+ if _, err := e.ScanGit(ctx, cfg); err != nil {return}}diff --git a/pkg/engine/github.go b/pkg/engine/github.goindex a166e34af..b3908afc5 100644--- a/pkg/engine/github.go+++ b/pkg/engine/github.go@@ -13,7 +13,7 @@ import ()// ScanGitHub scans GitHub with the provided options.-func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {+func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) (sources.JobProgressRef, error) {connection := sourcespb.GitHub{Endpoint: c.Endpoint,Organizations: c.Orgs,@@ -41,7 +41,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal github connection")- return err+ return sources.JobProgressRef{}, err}logOptions := &gogit.LogOptions{}@@ -56,9 +56,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {githubSource := &github.Source{}if err := githubSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {- return err+ return sources.JobProgressRef{}, err}githubSource.WithScanOptions(scanOptions)- _, err = e.sourceManager.Run(ctx, sourceName, githubSource)- return err+ return e.sourceManager.Run(ctx, sourceName, githubSource)}diff --git a/pkg/engine/github_experimental.go b/pkg/engine/github_experimental.goindex b10483689..e88fa3f1f 100644--- a/pkg/engine/github_experimental.go+++ b/pkg/engine/github_experimental.go@@ -17,7 +17,7 @@ import ()// ScanGitHubExperimental scans GitHub using an experimental feature. Consider all functionality to be in an alpha release here.-func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExperimentalConfig) error {+func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExperimentalConfig) (sources.JobProgressRef, error) {connection := sourcespb.GitHubExperimental{Repository: c.Repository,ObjectDiscovery: c.ObjectDiscovery,@@ -28,7 +28,7 @@ func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExp// Check at least one experimental sub-module is being used.// Add to this list as more experimental sub-modules are added.if !c.ObjectDiscovery {- return fmt.Errorf("at least one experimental submodule must be enabled")+ return sources.JobProgressRef{}, fmt.Errorf("at least one experimental submodule must be enabled")}if len(c.Token) > 0 {@@ -36,14 +36,14 @@ func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExpToken: c.Token,}} else {- return fmt.Errorf("token is required for github experimental")+ return sources.JobProgressRef{}, fmt.Errorf("token is required for github experimental")}var conn anypb.Anyerr := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal github experimental connection")- return err+ return sources.JobProgressRef{}, err}logOptions := &gogit.LogOptions{}@@ -57,9 +57,8 @@ func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExpgithubExperimentalSource := &github_experimental.Source{}if err := githubExperimentalSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}githubExperimentalSource.WithScanOptions(scanOptions)- _, err = e.sourceManager.Run(ctx, sourceName, githubExperimentalSource)- return err+ return e.sourceManager.Run(ctx, sourceName, githubExperimentalSource)}diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.goindex 699a7584c..9baa07b2f 100644--- a/pkg/engine/gitlab.go+++ b/pkg/engine/gitlab.go@@ -16,7 +16,7 @@ import ()// ScanGitLab scans GitLab with the provided configuration.-func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {+func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (sources.JobProgressRef, error) {logOptions := &gogit.LogOptions{}opts := []git.ScanOption{git.ScanOptionFilter(c.Filter),@@ -32,7 +32,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {Token: c.Token,}default:- return fmt.Errorf("must provide token")+ return sources.JobProgressRef{}, fmt.Errorf("must provide token")}if len(c.Endpoint) > 0 {@@ -55,7 +55,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal gitlab connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - gitlab"@@ -63,9 +63,8 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {gitlabSource := &gitlab.Source{}if err := gitlabSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}gitlabSource.WithScanOptions(scanOptions)- _, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)- return err+ return e.sourceManager.Run(ctx, sourceName, gitlabSource)}diff --git a/pkg/engine/gitlab_integration_test.go b/pkg/engine/gitlab_integration_test.goindex b2b026edd..5cc33cd5f 100644--- a/pkg/engine/gitlab_integration_test.go+++ b/pkg/engine/gitlab_integration_test.go@@ -28,7 +28,7 @@ func TestGitLab(t *testing.T) {if err != nil {t.Fatal(fmt.Errorf("failed to access secret: %v", err))}- err = e.ScanGitLab(ctx, sources.GitlabConfig{+ _, err = e.ScanGitLab(ctx, sources.GitlabConfig{Token: secret.MustGetField("GITLAB_TOKEN"),})assert.NoError(t, err)diff --git a/pkg/engine/huggingface.go b/pkg/engine/huggingface.goindex 0ec8c48d5..f11fc851b 100644--- a/pkg/engine/huggingface.go+++ b/pkg/engine/huggingface.go@@ -6,6 +6,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/huggingface")@@ -33,7 +34,7 @@ type HuggingfaceConfig struct {}// ScanGitHub scans HuggingFace with the provided options.-func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) error {+func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) (sources.JobProgressRef, error) {connection := sourcespb.Huggingface{Endpoint: c.Endpoint,Models: c.Models,@@ -65,7 +66,7 @@ func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) errorerr := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal huggingface connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - huggingface"@@ -73,8 +74,7 @@ func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) errorhuggingfaceSource := &huggingface.Source{}if err := huggingfaceSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, huggingfaceSource)- return err+ return e.sourceManager.Run(ctx, sourceName, huggingfaceSource)}diff --git a/pkg/engine/jenkins.go b/pkg/engine/jenkins.goindex d645c2e8c..363e657a3 100644--- a/pkg/engine/jenkins.go+++ b/pkg/engine/jenkins.go@@ -11,6 +11,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/jenkins")@@ -23,7 +24,7 @@ type JenkinsConfig struct {}// ScanJenkins scans Jenkins logs.-func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) error {+func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) (sources.JobProgressRef, error) {var connection *sourcespb.Jenkinsswitch {case jenkinsConfig.Username != "" && jenkinsConfig.Password != "":@@ -38,7 +39,7 @@ func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) ecase jenkinsConfig.Header != "":splits := strings.Split(jenkinsConfig.Header, ":")if len(splits) != 2 {- return errors.New("invalid header format, expected key: value")+ return sources.JobProgressRef{}, errors.New("invalid header format, expected key: value")}key := splits[0]value := splits[1]@@ -66,7 +67,7 @@ func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) eerr := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal Jenkins connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - Jenkins"@@ -74,8 +75,7 @@ func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) ejenkinsSource := &jenkins.Source{}if err := jenkinsSource.Init(ctx, "trufflehog - Jenkins", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, jenkinsSource)- return err+ return e.sourceManager.Run(ctx, sourceName, jenkinsSource)}diff --git a/pkg/engine/postman.go b/pkg/engine/postman.goindex b8a2fd47d..ad71e07ac 100644--- a/pkg/engine/postman.go+++ b/pkg/engine/postman.go@@ -13,7 +13,7 @@ import ()// ScanPostman scans Postman with the provided options.-func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) error {+func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) (sources.JobProgressRef, error) {connection := sourcespb.Postman{Workspaces: c.Workspaces,Collections: c.Collections,@@ -36,7 +36,7 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) error} else if len(c.WorkspacePaths) > 0 || len(c.CollectionPaths) > 0 || len(c.EnvironmentPaths) > 0 {connection.Credential = &sourcespb.Postman_Unauthenticated{}} else {- return errors.New("no path to locally exported data or API token provided")+ return sources.JobProgressRef{}, errors.New("no path to locally exported data or API token provided")}// Turn AhoCorasick keywordsToDetectors into a map of keywords@@ -49,7 +49,7 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) errorerr := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal Postman connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - postman"@@ -59,8 +59,7 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) errorDetectorKeywords: keywords,}if err := postmanSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, postmanSource)- return err+ return e.sourceManager.Run(ctx, sourceName, postmanSource)}diff --git a/pkg/engine/postman_test.go b/pkg/engine/postman_test.goindex 448e59b6a..081002f7c 100644--- a/pkg/engine/postman_test.go+++ b/pkg/engine/postman_test.go@@ -65,7 +65,7 @@ func TestPostmanEngine(t *testing.T) {e, err := NewEngine(ctx, &conf)assert.NoError(t, err)e.Start(ctx)- err = e.ScanPostman(ctx, test.postmanConfig)+ _, err = e.ScanPostman(ctx, test.postmanConfig)if err != nil && !test.wantErr {t.Errorf("ScanPostman() got: %v, want: %v", err, nil)returndiff --git a/pkg/engine/s3.go b/pkg/engine/s3.goindex 7935999e3..be74e0ec3 100644--- a/pkg/engine/s3.go+++ b/pkg/engine/s3.go@@ -15,13 +15,13 @@ import ()// ScanS3 scans S3 buckets.-func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {+func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) (sources.JobProgressRef, error) {connection := &sourcespb.S3{Credential: &sourcespb.S3_Unauthenticated{},}if c.CloudCred {if len(c.Key) > 0 || len(c.Secret) > 0 || len(c.SessionToken) > 0 {- return fmt.Errorf("cannot use cloud environment and static credentials together")+ return sources.JobProgressRef{}, fmt.Errorf("cannot use cloud environment and static credentials together")}connection.Credential = &sourcespb.S3_CloudEnvironment{}}@@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal S3 connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - s3"@@ -66,8 +66,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {s3Source := &s3.Source{}if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, s3Source)- return err+ return e.sourceManager.Run(ctx, sourceName, s3Source)}diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.goindex 654318bbc..6c8518175 100644--- a/pkg/engine/syslog.go+++ b/pkg/engine/syslog.go@@ -14,7 +14,7 @@ import ()// ScanSyslog is a source that scans syslog files.-func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {+func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) (sources.JobProgressRef, error) {connection := &sourcespb.Syslog{Protocol: c.Protocol,ListenAddress: c.Address,@@ -24,13 +24,13 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {if c.CertPath != "" && c.KeyPath != "" {cert, err := os.ReadFile(c.CertPath)if err != nil {- return errors.WrapPrefix(err, "could not open TLS cert file", 0)+ return sources.JobProgressRef{}, errors.WrapPrefix(err, "could not open TLS cert file", 0)}connection.TlsCert = string(cert)key, err := os.ReadFile(c.KeyPath)if err != nil {- return errors.WrapPrefix(err, "could not open TLS key file", 0)+ return sources.JobProgressRef{}, errors.WrapPrefix(err, "could not open TLS key file", 0)}connection.TlsKey = string(key)}@@ -38,17 +38,16 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {var conn anypb.Anyerr := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {- return errors.WrapPrefix(err, "error unmarshalling connection", 0)+ return sources.JobProgressRef{}, errors.WrapPrefix(err, "error unmarshalling connection", 0)}sourceName := "trufflehog - syslog"sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, syslog.SourceType)syslogSource := &syslog.Source{}if err := syslogSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {- return err+ return sources.JobProgressRef{}, err}syslogSource.InjectConnection(connection)- _, err = e.sourceManager.Run(ctx, sourceName, syslogSource)- return err+ return e.sourceManager.Run(ctx, sourceName, syslogSource)}diff --git a/pkg/engine/travisci.go b/pkg/engine/travisci.goindex f1a002e95..987ff1342 100644--- a/pkg/engine/travisci.go+++ b/pkg/engine/travisci.go@@ -8,11 +8,12 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci")// ScanTravisCI scans TravisCI logs.-func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {+func (e *Engine) ScanTravisCI(ctx context.Context, token string) (sources.JobProgressRef, error) {connection := &sourcespb.TravisCI{Credential: &sourcespb.TravisCI_Token{Token: token,@@ -23,7 +24,7 @@ func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {ctx.Logger().Error(err, "failed to marshal Travis CI connection")- return err+ return sources.JobProgressRef{}, err}sourceName := "trufflehog - Travis CI"@@ -31,8 +32,7 @@ func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {travisSource := &travisci.Source{}if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return err+ return sources.JobProgressRef{}, err}- _, err = e.sourceManager.Run(ctx, sourceName, travisSource)- return err+ return e.sourceManager.Run(ctx, sourceName, travisSource)}