Skip to content

Find, verify, and analyze leaked credentials

[chore] Log non-fatal errors encountered during a scan (#3612)

Miccah6a77b73f5133
diff --git a/main.go b/main.go
index abd635f07..31db0e3b7 100644
--- a/main.go
+++ b/main.go
@@ -633,6 +633,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
}
}()
+ var ref sources.JobProgressRef
switch cmd {
case gitScan.FullCommand():
gitCfg := sources.GitConfig{
@@ -645,7 +646,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Bare: *gitScanBare,
ExcludeGlobs: *gitScanExcludeGlobs,
}
- if err = eng.ScanGit(ctx, gitCfg); err != nil {
+ if ref, err = eng.ScanGit(ctx, gitCfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
}
case githubScan.FullCommand():
@@ -674,7 +675,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
CommentsTimeframeDays: *githubCommentsTimeframeDays,
Filter: filter,
}
- if err := eng.ScanGitHub(ctx, cfg); err != nil {
+ if ref, err = eng.ScanGitHub(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
}
case githubExperimentalScan.FullCommand():
@@ -685,7 +686,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
CollisionThreshold: *githubExperimentalCollisionThreshold,
DeleteCachedData: *githubExperimentalDeleteCache,
}
- if err := eng.ScanGitHubExperimental(ctx, cfg); err != nil {
+ if ref, err = eng.ScanGitHubExperimental(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
}
case gitlabScan.FullCommand():
@@ -702,7 +703,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
ExcludeRepos: *gitlabScanExcludeRepos,
Filter: filter,
}
- if err := eng.ScanGitLab(ctx, cfg); err != nil {
+ if ref, err = eng.ScanGitLab(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
}
case filesystemScan.FullCommand():
@@ -717,7 +718,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
IncludePathsFile: *filesystemScanIncludePaths,
ExcludePathsFile: *filesystemScanExcludePaths,
}
- if err = eng.ScanFileSystem(ctx, cfg); err != nil {
+ if ref, err = eng.ScanFileSystem(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
}
case s3Scan.FullCommand():
@@ -731,7 +732,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
CloudCred: *s3ScanCloudEnv,
MaxObjectSize: int64(*s3ScanMaxObjectSize),
}
- if err := eng.ScanS3(ctx, cfg); err != nil {
+ if ref, err = eng.ScanS3(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
}
case syslogScan.FullCommand():
@@ -743,15 +744,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
KeyPath: *syslogTLSKey,
Concurrency: *concurrency,
}
- if err := eng.ScanSyslog(ctx, cfg); err != nil {
+ if ref, err = eng.ScanSyslog(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
}
case circleCiScan.FullCommand():
- if err := eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
+ if ref, err = eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
}
case travisCiScan.FullCommand():
- if err := eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
+ if ref, err = eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
}
case gcsScan.FullCommand():
@@ -768,7 +769,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Concurrency: *concurrency,
MaxObjectSize: int64(*gcsMaxObjectSize),
}
- if err := eng.ScanGCS(ctx, cfg); err != nil {
+ if ref, err = eng.ScanGCS(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
}
case dockerScan.FullCommand():
@@ -777,7 +778,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Images: *dockerScanImages,
UseDockerKeychain: *dockerScanToken == "",
}
- if err := eng.ScanDocker(ctx, cfg); err != nil {
+ if ref, err = eng.ScanDocker(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
}
case postmanScan.FullCommand():
@@ -814,7 +815,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
WorkspacePaths: *postmanWorkspacePaths,
EnvironmentPaths: *postmanEnvironmentPaths,
}
- if err := eng.ScanPostman(ctx, cfg); err != nil {
+ if ref, err = eng.ScanPostman(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
}
case elasticsearchScan.FullCommand():
@@ -830,7 +831,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
SinceTimestamp: *elasticsearchSinceTimestamp,
BestEffortScan: *elasticsearchBestEffortScan,
}
- if err := eng.ScanElasticsearch(ctx, cfg); err != nil {
+ if ref, err = eng.ScanElasticsearch(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
}
case jenkinsScan.FullCommand():
@@ -840,7 +841,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
Username: *jenkinsUsername,
Password: *jenkinsPassword,
}
- if err := eng.ScanJenkins(ctx, cfg); err != nil {
+ if ref, err = eng.ScanJenkins(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
}
case huggingfaceScan.FullCommand():
@@ -873,7 +874,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
IncludePrs: *huggingfaceIncludePrs,
Concurrency: *concurrency,
}
- if err := eng.ScanHuggingface(ctx, cfg); err != nil {
+ if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
}
default:
@@ -885,6 +886,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
return scanMetrics, fmt.Errorf("engine failed to finish execution: %v", err)
}
+ // Print any errors reported during the scan.
+ if errs := ref.Snapshot().Errors; len(errs) > 0 {
+ errMsgs := make([]string, len(errs))
+ for i := 0; i < len(errs); i++ {
+ errMsgs[i] = errs[i].Error()
+ }
+ ctx.Logger().Error(nil, "encountered errors during scan", "errors", errMsgs)
+ }
+
if *printAvgDetectorTime {
printAverageDetectorTime(eng)
}
diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go
index 2d588842b..97627dd77 100644
--- a/pkg/engine/circleci.go
+++ b/pkg/engine/circleci.go
@@ -8,11 +8,12 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
)
// ScanCircleCI scans CircleCI logs.
-func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
+func (e *Engine) ScanCircleCI(ctx context.Context, token string) (sources.JobProgressRef, error) {
connection := &sourcespb.CircleCI{
Credential: &sourcespb.CircleCI_Token{
Token: token,
@@ -23,7 +24,7 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Circle CI connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - Circle CI"
@@ -31,8 +32,7 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
circleSource := &circleci.Source{}
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, circleSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, circleSource)
}
diff --git a/pkg/engine/docker.go b/pkg/engine/docker.go
index 49e8af3ca..07f5376eb 100644
--- a/pkg/engine/docker.go
+++ b/pkg/engine/docker.go
@@ -13,7 +13,7 @@ import (
)
// ScanDocker scans a given docker connection.
-func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {
+func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Docker{Images: c.Images}
switch {
@@ -29,7 +29,7 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal gitlab connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - docker"
@@ -37,8 +37,7 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) error {
dockerSource := &docker.Source{}
if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, dockerSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, dockerSource)
}
diff --git a/pkg/engine/elasticsearch.go b/pkg/engine/elasticsearch.go
index 3a8c814ba..bc6c49488 100644
--- a/pkg/engine/elasticsearch.go
+++ b/pkg/engine/elasticsearch.go
@@ -13,7 +13,7 @@ import (
)
// ScanElasticsearch scans a Elasticsearch installation.
-func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) error {
+func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Elasticsearch{
Nodes: c.Nodes,
Username: c.Username,
@@ -31,7 +31,7 @@ func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchC
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Elasticsearch connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - Elasticsearch"
@@ -39,8 +39,7 @@ func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchC
elasticsearchSource := &elasticsearch.Source{}
if err := elasticsearchSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, elasticsearchSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, elasticsearchSource)
}
diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go
index 2d735c906..9cda13599 100644
--- a/pkg/engine/engine_test.go
+++ b/pkg/engine/engine_test.go
@@ -271,7 +271,7 @@ func TestEngine_DuplicateSecrets(t *testing.T) {
e.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
- if err := e.ScanFileSystem(ctx, cfg); err != nil {
+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}
@@ -372,7 +372,7 @@ even more`,
eng.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
- err = eng.ScanFileSystem(ctx, cfg)
+ _, err = eng.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)
assert.NoError(t, eng.Finish(ctx))
@@ -420,7 +420,7 @@ func TestEngine_VersionedDetectorsVerifiedSecrets(t *testing.T) {
e.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
- if err := e.ScanFileSystem(ctx, cfg); err != nil {
+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}
@@ -490,7 +490,7 @@ func TestEngine_CustomDetectorsDetectorsVerifiedSecrets(t *testing.T) {
e.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
- if err := e.ScanFileSystem(ctx, cfg); err != nil {
+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}
@@ -540,7 +540,7 @@ func TestVerificationOverlapChunk(t *testing.T) {
e.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
- if err := e.ScanFileSystem(ctx, cfg); err != nil {
+ if _, err := e.ScanFileSystem(ctx, cfg); err != nil {
return
}
@@ -630,7 +630,7 @@ func TestVerificationOverlapChunkFalsePositive(t *testing.T) {
e.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
- err = e.ScanFileSystem(ctx, cfg)
+ _, err = e.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)
// Wait for all the chunks to be processed.
@@ -678,7 +678,7 @@ func TestRetainFalsePositives(t *testing.T) {
e.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
- err = e.ScanFileSystem(ctx, cfg)
+ _, err = e.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)
// Wait for all the chunks to be processed.
@@ -1259,7 +1259,7 @@ def test_something():
eng.Start(ctx)
cfg := sources.FilesystemConfig{Paths: []string{tmpFile.Name()}}
- err = eng.ScanFileSystem(ctx, cfg)
+ _, err = eng.ScanFileSystem(ctx, cfg)
assert.NoError(t, err)
assert.NoError(t, eng.Finish(ctx))
diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go
index 169e52c31..fe61d4948 100644
--- a/pkg/engine/filesystem.go
+++ b/pkg/engine/filesystem.go
@@ -13,7 +13,7 @@ import (
)
// ScanFileSystem scans a given file system.
-func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
+func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Filesystem{
Paths: c.Paths,
IncludePathsFile: c.IncludePathsFile,
@@ -23,7 +23,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal filesystem connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - filesystem"
@@ -31,8 +31,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
fileSystemSource := &filesystem.Source{}
if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, fileSystemSource)
}
diff --git a/pkg/engine/filesystem_integration_test.go b/pkg/engine/filesystem_integration_test.go
index b95b31191..f1dd2625d 100644
--- a/pkg/engine/filesystem_integration_test.go
+++ b/pkg/engine/filesystem_integration_test.go
@@ -68,7 +68,7 @@ func TestFilesystem(t *testing.T) {
})
assert.NoError(t, err)
e.Start(ctx)
- err = e.ScanFileSystem(ctx, sources.FilesystemConfig{
+ _, err = e.ScanFileSystem(ctx, sources.FilesystemConfig{
Paths: []string{rootDir},
ExcludePathsFile: filepath.Join(configDir, "exclude"),
})
diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.go
index f2bf82c98..2284c5c69 100644
--- a/pkg/engine/gcs.go
+++ b/pkg/engine/gcs.go
@@ -13,10 +13,10 @@ import (
)
// ScanGCS with the provided options.
-func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
+func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) (sources.JobProgressRef, error) {
// Project ID is required if using any authenticated access.
if c.ProjectID == "" && !c.WithoutAuth {
- return fmt.Errorf("project ID is required")
+ return sources.JobProgressRef{}, fmt.Errorf("project ID is required")
}
// If using unauthenticated access, the project ID is not used.
@@ -35,13 +35,13 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
// Make sure only one auth method is selected.
if !isAuthValid(ctx, c, connection) {
- return fmt.Errorf("multiple auth methods selected, please select only one")
+ return sources.JobProgressRef{}, fmt.Errorf("multiple auth methods selected, please select only one")
}
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
- return fmt.Errorf("failed to marshal GCS connection: %w", err)
+ return sources.JobProgressRef{}, fmt.Errorf("failed to marshal GCS connection: %w", err)
}
sourceName := "trufflehog - gcs"
@@ -49,10 +49,9 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
gcsSource := &gcs.Source{}
if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, gcsSource)
}
func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool {
diff --git a/pkg/engine/gcs_test.go b/pkg/engine/gcs_test.go
index d73b0bf5e..c903e00ee 100644
--- a/pkg/engine/gcs_test.go
+++ b/pkg/engine/gcs_test.go
@@ -91,7 +91,7 @@ func TestScanGCS(t *testing.T) {
}
}()
- err = e.ScanGCS(ctx, test.gcsConfig)
+ _, err = e.ScanGCS(ctx, test.gcsConfig)
if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
return
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index 411b2689e..de487591f 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -13,7 +13,7 @@ import (
)
// ScanGit scans any git source.
-func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
+func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Git{
Head: c.HeadRef,
Base: c.BaseRef,
@@ -28,7 +28,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
var conn anypb.Any
if err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil {
ctx.Logger().Error(err, "failed to marshal git connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - git"
@@ -36,9 +36,8 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
gitSource := &git.Source{}
if err := gitSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err := e.sourceManager.Run(ctx, sourceName, gitSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, gitSource)
}
diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go
index 6bac1af99..14a48e0ef 100644
--- a/pkg/engine/git_test.go
+++ b/pkg/engine/git_test.go
@@ -91,7 +91,7 @@ func TestGitEngine(t *testing.T) {
BaseRef: tTest.base,
MaxDepth: tTest.maxDepth,
}
- if err := e.ScanGit(ctx, cfg); err != nil {
+ if _, err := e.ScanGit(ctx, cfg); err != nil {
return
}
@@ -155,7 +155,7 @@ func BenchmarkGitEngine(b *testing.B) {
cfg := sources.GitConfig{URI: repoUrl}
b.ResetTimer()
for i := 0; i < b.N; i++ {
- if err := e.ScanGit(ctx, cfg); err != nil {
+ if _, err := e.ScanGit(ctx, cfg); err != nil {
return
}
}
diff --git a/pkg/engine/github.go b/pkg/engine/github.go
index a166e34af..b3908afc5 100644
--- a/pkg/engine/github.go
+++ b/pkg/engine/github.go
@@ -13,7 +13,7 @@ import (
)
// ScanGitHub scans GitHub with the provided options.
-func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
+func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) (sources.JobProgressRef, error) {
connection := sourcespb.GitHub{
Endpoint: c.Endpoint,
Organizations: c.Orgs,
@@ -41,7 +41,7 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal github connection")
- return err
+ return sources.JobProgressRef{}, err
}
logOptions := &gogit.LogOptions{}
@@ -56,9 +56,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
githubSource := &github.Source{}
if err := githubSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
githubSource.WithScanOptions(scanOptions)
- _, err = e.sourceManager.Run(ctx, sourceName, githubSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, githubSource)
}
diff --git a/pkg/engine/github_experimental.go b/pkg/engine/github_experimental.go
index b10483689..e88fa3f1f 100644
--- a/pkg/engine/github_experimental.go
+++ b/pkg/engine/github_experimental.go
@@ -17,7 +17,7 @@ import (
)
// ScanGitHubExperimental scans GitHub using an experimental feature. Consider all functionality to be in an alpha release here.
-func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExperimentalConfig) error {
+func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExperimentalConfig) (sources.JobProgressRef, error) {
connection := sourcespb.GitHubExperimental{
Repository: c.Repository,
ObjectDiscovery: c.ObjectDiscovery,
@@ -28,7 +28,7 @@ func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExp
// Check at least one experimental sub-module is being used.
// Add to this list as more experimental sub-modules are added.
if !c.ObjectDiscovery {
- return fmt.Errorf("at least one experimental submodule must be enabled")
+ return sources.JobProgressRef{}, fmt.Errorf("at least one experimental submodule must be enabled")
}
if len(c.Token) > 0 {
@@ -36,14 +36,14 @@ func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExp
Token: c.Token,
}
} else {
- return fmt.Errorf("token is required for github experimental")
+ return sources.JobProgressRef{}, fmt.Errorf("token is required for github experimental")
}
var conn anypb.Any
err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal github experimental connection")
- return err
+ return sources.JobProgressRef{}, err
}
logOptions := &gogit.LogOptions{}
@@ -57,9 +57,8 @@ func (e *Engine) ScanGitHubExperimental(ctx context.Context, c sources.GitHubExp
githubExperimentalSource := &github_experimental.Source{}
if err := githubExperimentalSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
githubExperimentalSource.WithScanOptions(scanOptions)
- _, err = e.sourceManager.Run(ctx, sourceName, githubExperimentalSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, githubExperimentalSource)
}
diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
index 699a7584c..9baa07b2f 100644
--- a/pkg/engine/gitlab.go
+++ b/pkg/engine/gitlab.go
@@ -16,7 +16,7 @@ import (
)
// ScanGitLab scans GitLab with the provided configuration.
-func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
+func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (sources.JobProgressRef, error) {
logOptions := &gogit.LogOptions{}
opts := []git.ScanOption{
git.ScanOptionFilter(c.Filter),
@@ -32,7 +32,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
Token: c.Token,
}
default:
- return fmt.Errorf("must provide token")
+ return sources.JobProgressRef{}, fmt.Errorf("must provide token")
}
if len(c.Endpoint) > 0 {
@@ -55,7 +55,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal gitlab connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - gitlab"
@@ -63,9 +63,8 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
gitlabSource := &gitlab.Source{}
if err := gitlabSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
gitlabSource.WithScanOptions(scanOptions)
- _, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, gitlabSource)
}
diff --git a/pkg/engine/gitlab_integration_test.go b/pkg/engine/gitlab_integration_test.go
index b2b026edd..5cc33cd5f 100644
--- a/pkg/engine/gitlab_integration_test.go
+++ b/pkg/engine/gitlab_integration_test.go
@@ -28,7 +28,7 @@ func TestGitLab(t *testing.T) {
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
- err = e.ScanGitLab(ctx, sources.GitlabConfig{
+ _, err = e.ScanGitLab(ctx, sources.GitlabConfig{
Token: secret.MustGetField("GITLAB_TOKEN"),
})
assert.NoError(t, err)
diff --git a/pkg/engine/huggingface.go b/pkg/engine/huggingface.go
index 0ec8c48d5..f11fc851b 100644
--- a/pkg/engine/huggingface.go
+++ b/pkg/engine/huggingface.go
@@ -6,6 +6,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/huggingface"
)
@@ -33,7 +34,7 @@ type HuggingfaceConfig struct {
}
// ScanGitHub scans HuggingFace with the provided options.
-func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) error {
+func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) (sources.JobProgressRef, error) {
connection := sourcespb.Huggingface{
Endpoint: c.Endpoint,
Models: c.Models,
@@ -65,7 +66,7 @@ func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) error
err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal huggingface connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - huggingface"
@@ -73,8 +74,7 @@ func (e *Engine) ScanHuggingface(ctx context.Context, c HuggingfaceConfig) error
huggingfaceSource := &huggingface.Source{}
if err := huggingfaceSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, huggingfaceSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, huggingfaceSource)
}
diff --git a/pkg/engine/jenkins.go b/pkg/engine/jenkins.go
index d645c2e8c..363e657a3 100644
--- a/pkg/engine/jenkins.go
+++ b/pkg/engine/jenkins.go
@@ -11,6 +11,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/jenkins"
)
@@ -23,7 +24,7 @@ type JenkinsConfig struct {
}
// ScanJenkins scans Jenkins logs.
-func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) error {
+func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) (sources.JobProgressRef, error) {
var connection *sourcespb.Jenkins
switch {
case jenkinsConfig.Username != "" && jenkinsConfig.Password != "":
@@ -38,7 +39,7 @@ func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) e
case jenkinsConfig.Header != "":
splits := strings.Split(jenkinsConfig.Header, ":")
if len(splits) != 2 {
- return errors.New("invalid header format, expected key: value")
+ return sources.JobProgressRef{}, errors.New("invalid header format, expected key: value")
}
key := splits[0]
value := splits[1]
@@ -66,7 +67,7 @@ func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) e
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Jenkins connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - Jenkins"
@@ -74,8 +75,7 @@ func (e *Engine) ScanJenkins(ctx context.Context, jenkinsConfig JenkinsConfig) e
jenkinsSource := &jenkins.Source{}
if err := jenkinsSource.Init(ctx, "trufflehog - Jenkins", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, jenkinsSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, jenkinsSource)
}
diff --git a/pkg/engine/postman.go b/pkg/engine/postman.go
index b8a2fd47d..ad71e07ac 100644
--- a/pkg/engine/postman.go
+++ b/pkg/engine/postman.go
@@ -13,7 +13,7 @@ import (
)
// ScanPostman scans Postman with the provided options.
-func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) error {
+func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) (sources.JobProgressRef, error) {
connection := sourcespb.Postman{
Workspaces: c.Workspaces,
Collections: c.Collections,
@@ -36,7 +36,7 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) error
} else if len(c.WorkspacePaths) > 0 || len(c.CollectionPaths) > 0 || len(c.EnvironmentPaths) > 0 {
connection.Credential = &sourcespb.Postman_Unauthenticated{}
} else {
- return errors.New("no path to locally exported data or API token provided")
+ return sources.JobProgressRef{}, errors.New("no path to locally exported data or API token provided")
}
// Turn AhoCorasick keywordsToDetectors into a map of keywords
@@ -49,7 +49,7 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) error
err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Postman connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - postman"
@@ -59,8 +59,7 @@ func (e *Engine) ScanPostman(ctx context.Context, c sources.PostmanConfig) error
DetectorKeywords: keywords,
}
if err := postmanSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, postmanSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, postmanSource)
}
diff --git a/pkg/engine/postman_test.go b/pkg/engine/postman_test.go
index 448e59b6a..081002f7c 100644
--- a/pkg/engine/postman_test.go
+++ b/pkg/engine/postman_test.go
@@ -65,7 +65,7 @@ func TestPostmanEngine(t *testing.T) {
e, err := NewEngine(ctx, &conf)
assert.NoError(t, err)
e.Start(ctx)
- err = e.ScanPostman(ctx, test.postmanConfig)
+ _, err = e.ScanPostman(ctx, test.postmanConfig)
if err != nil && !test.wantErr {
t.Errorf("ScanPostman() got: %v, want: %v", err, nil)
return
diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go
index 7935999e3..be74e0ec3 100644
--- a/pkg/engine/s3.go
+++ b/pkg/engine/s3.go
@@ -15,13 +15,13 @@ import (
)
// ScanS3 scans S3 buckets.
-func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
+func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) (sources.JobProgressRef, error) {
connection := &sourcespb.S3{
Credential: &sourcespb.S3_Unauthenticated{},
}
if c.CloudCred {
if len(c.Key) > 0 || len(c.Secret) > 0 || len(c.SessionToken) > 0 {
- return fmt.Errorf("cannot use cloud environment and static credentials together")
+ return sources.JobProgressRef{}, fmt.Errorf("cannot use cloud environment and static credentials together")
}
connection.Credential = &sourcespb.S3_CloudEnvironment{}
}
@@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal S3 connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - s3"
@@ -66,8 +66,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
s3Source := &s3.Source{}
if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, s3Source)
- return err
+ return e.sourceManager.Run(ctx, sourceName, s3Source)
}
diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go
index 654318bbc..6c8518175 100644
--- a/pkg/engine/syslog.go
+++ b/pkg/engine/syslog.go
@@ -14,7 +14,7 @@ import (
)
// ScanSyslog is a source that scans syslog files.
-func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
+func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) (sources.JobProgressRef, error) {
connection := &sourcespb.Syslog{
Protocol: c.Protocol,
ListenAddress: c.Address,
@@ -24,13 +24,13 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
if c.CertPath != "" && c.KeyPath != "" {
cert, err := os.ReadFile(c.CertPath)
if err != nil {
- return errors.WrapPrefix(err, "could not open TLS cert file", 0)
+ return sources.JobProgressRef{}, errors.WrapPrefix(err, "could not open TLS cert file", 0)
}
connection.TlsCert = string(cert)
key, err := os.ReadFile(c.KeyPath)
if err != nil {
- return errors.WrapPrefix(err, "could not open TLS key file", 0)
+ return sources.JobProgressRef{}, errors.WrapPrefix(err, "could not open TLS key file", 0)
}
connection.TlsKey = string(key)
}
@@ -38,17 +38,16 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
- return errors.WrapPrefix(err, "error unmarshalling connection", 0)
+ return sources.JobProgressRef{}, errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
sourceName := "trufflehog - syslog"
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, syslog.SourceType)
syslogSource := &syslog.Source{}
if err := syslogSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
syslogSource.InjectConnection(connection)
- _, err = e.sourceManager.Run(ctx, sourceName, syslogSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, syslogSource)
}
diff --git a/pkg/engine/travisci.go b/pkg/engine/travisci.go
index f1a002e95..987ff1342 100644
--- a/pkg/engine/travisci.go
+++ b/pkg/engine/travisci.go
@@ -8,11 +8,12 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/travisci"
)
// ScanTravisCI scans TravisCI logs.
-func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
+func (e *Engine) ScanTravisCI(ctx context.Context, token string) (sources.JobProgressRef, error) {
connection := &sourcespb.TravisCI{
Credential: &sourcespb.TravisCI_Token{
Token: token,
@@ -23,7 +24,7 @@ func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
ctx.Logger().Error(err, "failed to marshal Travis CI connection")
- return err
+ return sources.JobProgressRef{}, err
}
sourceName := "trufflehog - Travis CI"
@@ -31,8 +32,7 @@ func (e *Engine) ScanTravisCI(ctx context.Context, token string) error {
travisSource := &travisci.Source{}
if err := travisSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return err
+ return sources.JobProgressRef{}, err
}
- _, err = e.sourceManager.Run(ctx, sourceName, travisSource)
- return err
+ return e.sourceManager.Run(ctx, sourceName, travisSource)
}