Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add flag to write job reports to disk (#2298)
Miccah9642d4c8fd16
diff --git a/main.go b/main.goindex b34f9083d..b9e7ba4ef 100644--- a/main.go+++ b/main.go@@ -2,6 +2,7 @@ package mainimport ("fmt"+ "io""net/http"_ "net/http/pprof""os"@@ -64,6 +65,7 @@ var (archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()+ jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)gitScan = cli.Command("git", "Find credentials in git repositories.")gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()@@ -398,6 +400,10 @@ func run(state overseer.State) {fmt.Fprintf(os.Stderr, "π·ππ· TruffleHog. Unearth your secrets. π·ππ·\n\n")}+ var jobReportWriter io.WriteCloser+ if *jobReportFile != nil {+ jobReportWriter = *jobReportFile+ }e, err := engine.Start(ctx,engine.WithConcurrency(uint8(*concurrency)),engine.WithDecoders(decoders.DefaultDecoders()...),@@ -413,6 +419,7 @@ func run(state overseer.State) {engine.WithPrinter(printer),engine.WithFilterEntropy(*filterEntropy),engine.WithVerificationOverlap(*allowVerificationOverlap),+ engine.WithJobReportWriter(jobReportWriter),)if err != nil {logFatal(err, "error initializing engine")diff --git a/pkg/common/export_error.go b/pkg/common/export_error.gonew file mode 100644index 000000000..68baab3e5--- /dev/null+++ b/pkg/common/export_error.go@@ -0,0 +1,16 @@+package common++// ExportError is an implementation of error that can be JSON marshalled. It+// must be a public exported type for this reason.+type ExportError string++func (e ExportError) Error() string { return string(e) }++// ExportErrors converts a list of errors into []ExportError.+func ExportErrors(errs ...error) []error {+ output := make([]error, 0, len(errs))+ for _, err := range errs {+ output = append(output, ExportError(err.Error()))+ }+ return output+}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 3d822e4bb..31d813545 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -2,8 +2,10 @@ package engineimport ("bytes"+ "encoding/json""errors""fmt"+ "io""runtime""sync""sync/atomic"@@ -58,9 +60,10 @@ type Printer interface {type Engine struct {// CLI flags.- concurrency uint8- decoders []decoders.Decoder- detectors []detectors.Detector+ concurrency uint8+ decoders []decoders.Decoder+ detectors []detectors.Detector+ jobReportWriter io.WriteCloser// filterUnverified is used to reduce the number of unverified results.// If there are multiple unverified results for the same chunk for the same detector,// only the first one will be kept.@@ -119,6 +122,12 @@ func (r *verificationOverlapTracker) increment() {// Option is used to configure the engine during initialization using functional options.type Option func(*Engine)+func WithJobReportWriter(w io.WriteCloser) Option {+ return func(e *Engine) {+ e.jobReportWriter = w+ }+}+func WithConcurrency(concurrency uint8) Option {return func(e *Engine) {e.concurrency = concurrency@@ -317,6 +326,7 @@ func Start(ctx context.Context, options ...Option) (*Engine, error) {if err := e.initialize(ctx, options...); err != nil {return nil, err}+ e.initSourceManager(ctx)e.setDefaults(ctx)e.sanityChecks(ctx)e.startWorkers(ctx)@@ -373,6 +383,47 @@ func (e *Engine) initialize(ctx context.Context, options ...Option) error {return nil}+func (e *Engine) initSourceManager(ctx context.Context) {+ opts := []func(*sources.SourceManager){+ sources.WithConcurrentSources(int(e.concurrency)),+ sources.WithConcurrentUnits(int(e.concurrency)),+ sources.WithSourceUnits(),+ sources.WithBufferedOutput(defaultChannelBuffer),+ }+ if e.jobReportWriter != nil {+ unitHook, finishedMetrics := sources.NewUnitHook(ctx)+ opts = append(opts, sources.WithReportHook(unitHook))+ e.wgDetectorWorkers.Add(1)+ go func() {+ defer e.wgDetectorWorkers.Done()+ defer func() {+ e.jobReportWriter.Close()+ // Add a bit of extra information if it's a *os.File.+ if namer, ok := e.jobReportWriter.(interface{ Name() string }); ok {+ ctx.Logger().Info("report written", "path", namer.Name())+ } else {+ ctx.Logger().Info("report written")+ }+ }()+ for metrics := range finishedMetrics {+ metrics.Errors = common.ExportErrors(metrics.Errors...)+ details, err := json.Marshal(map[string]any{+ "version": 1,+ "data": metrics,+ })+ if err != nil {+ ctx.Logger().Error(err, "error marshalling job details")+ continue+ }+ if _, err := e.jobReportWriter.Write(append(details, '\n')); err != nil {+ ctx.Logger().Error(err, "error writing to file")+ }+ }+ }()+ }+ e.sourceManager = sources.NewManager(opts...)+}+// setDefaults ensures that if specific engine properties aren't provided,// they're set to reasonable default values. It makes the engine robust to// incomplete configuration.@@ -384,13 +435,6 @@ func (e *Engine) setDefaults(ctx context.Context) {}ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)- e.sourceManager = sources.NewManager(- sources.WithConcurrentSources(int(e.concurrency)),- sources.WithConcurrentUnits(int(e.concurrency)),- sources.WithSourceUnits(),- sources.WithBufferedOutput(defaultChannelBuffer),- )-// Default decoders handle common encoding formats.if len(e.decoders) == 0 {e.decoders = decoders.DefaultDecoders()