Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Initial implementation of JobReport with SourceManager usage (#1557)
Miccahe391e89f3e13
diff --git a/pkg/sources/job_report.go b/pkg/sources/job_report.gonew file mode 100644index 000000000..01a7734c3--- /dev/null+++ b/pkg/sources/job_report.go@@ -0,0 +1,37 @@+package sources++import (+ "errors"+ "sync"+ "time"+)++// JobReport aggregates information about a run of a Source.+type JobReport struct {+ SourceID int64+ JobID int64+ StartTime time.Time+ EndTime time.Time+ TotalChunks uint64+ errors []error+ errorsLock sync.Mutex+}++// AddError adds a non-nil error to the aggregate of errors encountered during+// scanning.+func (jr *JobReport) AddError(err error) {+ if err == nil {+ return+ }+ jr.errorsLock.Lock()+ defer jr.errorsLock.Unlock()+ jr.errors = append(jr.errors, err)+}++// Errors joins all aggregated errors into one. If there were no errors, nil is+// returned. errors.Is can be used to check for specific errors.+func (jr *JobReport) Errors() error {+ jr.errorsLock.Lock()+ defer jr.errorsLock.Unlock()+ return errors.Join(jr.errors...)+}diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.goindex ec4813897..f061c66b6 100644--- a/pkg/sources/source_manager.go+++ b/pkg/sources/source_manager.go@@ -1,9 +1,11 @@package sourcesimport (+ "errors""fmt""sync""sync/atomic"+ "time""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -25,12 +27,20 @@ type SourceManager struct {// Map of handle to source initializer.handles map[handle]SourceInitFunchandlesLock sync.Mutex+ // Map of handle to job reports.+ // TODO: Manage culling and flushing to the API.+ report map[handle]*JobReport+ reportLock sync.Mutex// Pool limiting the amount of concurrent sources running.- pool errgroup.Group+ pool errgroup.Group+ concurrentUnits int+ // Run the sources using source unit enumeration / chunking if available.+ useSourceUnits bool// Downstream chunks channel to be scanned.outputChunks chan *Chunk- // Set to true when Wait() returns.- done bool+ // Set when Wait() returns.+ done bool+ doneErr error}// apiClient is an interface for optionally communicating with an external API.@@ -56,12 +66,25 @@ func WithBufferedOutput(size int) func(*SourceManager) {return func(mgr *SourceManager) { mgr.outputChunks = make(chan *Chunk, size) }}+// WithSourceUnits enables using source unit enumeration and chunking if the+// source supports it.+func WithSourceUnits() func(*SourceManager) {+ return func(mgr *SourceManager) { mgr.useSourceUnits = true }+}++// WithConcurrentUnits limits the number of units to be scanned concurrently.+// The default is unlimited.+func WithConcurrentUnits(n int) func(*SourceManager) {+ return func(mgr *SourceManager) { mgr.concurrentUnits = n }+}+// NewManager creates a new manager with the provided options.func NewManager(opts ...func(*SourceManager)) *SourceManager {mgr := SourceManager{// Default to the headless API. Can be overwritten by the WithAPI option.api: &headlessAPI{},handles: make(map[handle]SourceInitFunc),+ report: make(map[handle]*JobReport),outputChunks: make(chan *Chunk),}for _, opt := range opts {@@ -100,15 +123,24 @@ func (s *SourceManager) Run(ctx context.Context, handle handle) error {ch := make(chan error)s.pool.Go(func() error {defer common.Recover(ctx)- // TODO: The manager should record these errors.- ch <- s.run(ctx, handle)+ report, err := s.run(ctx, handle)+ if report != nil {+ s.reportLock.Lock()+ s.report[handle] = report+ s.reportLock.Unlock()+ }+ if err != nil {+ ch <- err+ return nil+ }+ ch <- report.Errors()return nil})return <-ch}// ScheduleRun blocks until a resource is available to run the source, then-// asynchronously runs it. Error information is lost in this case.+// asynchronously runs it. Error information is stored and returned by Wait().func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {// Do preflight checks before waiting on the pool.if err := s.preflightChecks(ctx, handle); err != nil {@@ -116,8 +148,14 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {}s.pool.Go(func() error {defer common.Recover(ctx)- // TODO: The manager should record these errors.- _ = s.run(ctx, handle)+ // The error is already saved in the report, so we can ignore+ // it here.+ report, _ := s.run(ctx, handle)+ if report != nil {+ s.reportLock.Lock()+ s.report[handle] = report+ s.reportLock.Unlock()+ }return nil})// TODO: Maybe wait for a signal here that initialization was successful?@@ -136,12 +174,34 @@ func (s *SourceManager) Chunks() <-chan *Chunk {func (s *SourceManager) Wait() error {// Check if the manager has been Waited.if s.done {- return nil+ return s.doneErr}- // TODO: Aggregate all errors from all sources.defer close(s.outputChunks)defer func() { s.done = true }()- return s.pool.Wait()++ // We are only using the errgroup for limiting concurrency.+ // TODO: Maybe switch to using a semaphore.Weighted.+ _ = s.pool.Wait()++ // Aggregate all errors from all job reports.+ // TODO: This should probably only be the fatal errors. We'll also need+ // to rewrite this for when the reports start getting culled.+ s.reportLock.Lock()+ defer s.reportLock.Unlock()+ errs := make([]error, 0, len(s.report))+ for _, report := range s.report {+ errs = append(errs, report.Errors())+ }+ s.doneErr = errors.Join(errs...)+ return s.doneErr+}++// Report retrieves a scan report for a given handle. If no report exists or+// the Source has not finished, nil will be returned.+func (s *SourceManager) Report(handle handle) *JobReport {+ s.reportLock.Lock()+ defer s.reportLock.Unlock()+ return s.report[handle]}// preflightChecks is a helper method to check the Manager or the context isn't@@ -159,23 +219,127 @@ func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) erro}// run is a helper method to sychronously run the source. It does not check for-// acquired resources.-func (s *SourceManager) run(ctx context.Context, handle handle) error {+// acquired resources. Possible return values are:+//+// - *JobReport, nil+// Successfully ran the source, but the report could have errors.+//+// - *JobReport, error+// There was an error calling Init or Chunks. This sort of error indicates+// a fatal error and is also recorded in the report.+//+// - nil, error:+// There was an error from the API or the handle is invalid. The latter of+// which should never happen due to the preflightChecks.+func (s *SourceManager) run(ctx context.Context, handle handle) (*JobReport, error) {jobID, err := s.api.GetJobID(ctx, int64(handle))if err != nil {- return err+ return nil, err}initFunc, ok := s.getInitFunc(handle)if !ok {- return fmt.Errorf("unrecognized handle")+ return nil, fmt.Errorf("unrecognized handle")+ }+ // Create a report for this run.+ report := &JobReport{+ SourceID: int64(handle),+ JobID: jobID,+ StartTime: time.Now(),}+ defer func() { report.EndTime = time.Now() }()++ // Initialize the source.source, err := initFunc(ctx, jobID, int64(handle))if err != nil {- return err+ report.AddError(err)+ return report, err+ }+ // Check for the preferred method of tracking source units.+ if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {+ return s.runWithUnits(ctx, handle, enumChunker, report)+ }+ return s.runWithoutUnits(ctx, handle, source, report)+}++// runWithoutUnits is a helper method to run a Source. It has coarse-grained+// job reporting.+func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobReport) (*JobReport, error) {+ // Introspect on the chunks we get from the Chunks method.+ ch := make(chan *Chunk)+ var wg sync.WaitGroup+ // Consume chunks and export chunks.+ wg.Add(1)+ go func() {+ defer wg.Done()+ for chunk := range ch {+ atomic.AddUint64(&report.TotalChunks, 1)+ _ = common.CancellableWrite(ctx, s.outputChunks, chunk)+ }+ }()+ // Don't return from this function until the goroutine has finished+ // outputting chunks to the downstream channel. Closing the channel+ // will stop the goroutine, so that needs to happen first in the defer+ // stack.+ defer wg.Wait()+ defer close(ch)+ if err := source.Chunks(ctx, ch); err != nil {+ report.AddError(err)+ return report, err}- // TODO: Support UnitChunker and SourceUnitEnumerator.- // TODO: This is where we can introspect on the chunks collected.- return source.Chunks(ctx, s.outputChunks)+ return report, nil+}++// runWithUnits is a helper method to run a Source that is also a+// SourceUnitEnumChunker. This allows better introspection of what is getting+// scanned and any errors encountered.+func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobReport) (*JobReport, error) {+ reporter := &mgrUnitReporter{+ unitCh: make(chan SourceUnit),+ }+ // Produce units.+ go func() {+ // TODO: Catch panics and add to report.+ defer close(reporter.unitCh)+ if err := source.Enumerate(ctx, reporter); err != nil {+ report.AddError(err)+ }+ }()+ var wg sync.WaitGroup+ // TODO: Maybe switch to using a semaphore.Weighted.+ var unitPool errgroup.Group+ if s.concurrentUnits != 0 {+ // Negative values indicated no limit.+ unitPool.SetLimit(s.concurrentUnits)+ }+ for unit := range reporter.unitCh {+ reporter := &mgrChunkReporter{+ unitID: unit.SourceUnitID(),+ chunkCh: make(chan *Chunk),+ }+ unit := unit+ // Consume units and produce chunks.+ unitPool.Go(func() error {+ // TODO: Catch panics and add to report.+ defer close(reporter.chunkCh)+ if err := source.ChunkUnit(ctx, unit, reporter); err != nil {+ report.AddError(err)+ }+ return nil+ })+ // Consume chunks and export chunks.+ wg.Add(1)+ go func() {+ defer wg.Done()+ for chunk := range reporter.chunkCh {+ // TODO: Introspect on the chunks we got from this unit.+ atomic.AddUint64(&report.TotalChunks, 1)+ _ = common.CancellableWrite(ctx, s.outputChunks, chunk)+ }+ }()+ }+ wg.Wait()+ // TODO: Return fatal errors.+ return report, nil}// getInitFunc is a helper method for safe concurrent access to the@@ -201,3 +365,40 @@ func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sofunc (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {return atomic.AddInt64(&api.jobIDCounter, 1), nil}++// mgrUnitReporter implements the UnitReporter interface.+type mgrUnitReporter struct {+ unitCh chan SourceUnit+ unitErrs []error+ unitErrsLock sync.Mutex+}++func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error {+ return common.CancellableWrite(ctx, s.unitCh, unit)+}++func (s *mgrUnitReporter) UnitErr(ctx context.Context, err error) error {+ s.unitErrsLock.Lock()+ defer s.unitErrsLock.Unlock()+ s.unitErrs = append(s.unitErrs, err)+ return nil+}++// mgrChunkReporter implements the ChunkReporter interface.+type mgrChunkReporter struct {+ unitID string+ chunkCh chan *Chunk+ chunkErrs []error+ chunkErrsLock sync.Mutex+}++func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error {+ return common.CancellableWrite(ctx, s.chunkCh, &chunk)+}++func (s *mgrChunkReporter) ChunkErr(ctx context.Context, err error) error {+ s.chunkErrsLock.Lock()+ defer s.chunkErrsLock.Unlock()+ s.chunkErrs = append(s.chunkErrs, err)+ return nil+}diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.goindex 2aa0354e8..a06a05866 100644--- a/pkg/sources/source_manager_test.go+++ b/pkg/sources/source_manager_test.go@@ -29,6 +29,8 @@ func (d *DummySource) GetProgress() *Progress { return nil }// Interface to easily test different chunking methods.type chunker interface {Chunks(context.Context, chan *Chunk) error+ ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error+ Enumerate(ctx context.Context, reporter UnitReporter) error}// Chunk method that writes count bytes to the channel before returning.@@ -45,6 +47,31 @@ func (c *counterChunker) Chunks(_ context.Context, ch chan *Chunk) error {return nil}+// countChunk implements SourceUnit.+type countChunk byte++func (c countChunk) SourceUnitID() string { return fmt.Sprintf("countChunk(%d)", c) }++func (c *counterChunker) Enumerate(ctx context.Context, reporter UnitReporter) error {+ for i := 0; i < c.count; i++ {+ if err := reporter.UnitOk(ctx, countChunk(byte(i))); err != nil {+ return err+ }+ }+ return nil+}++func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error {+ return reporter.ChunkOk(ctx, Chunk{Data: []byte{byte(unit.(countChunk))}})+}++// Chunk method that always returns an error.+type errorChunker struct{ error }++func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }+func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }+// enrollDummy is a helper function to enroll a DummySource with a SourceManager.func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {return mgr.Enroll(context.Background(), "dummy", 1337,@@ -117,3 +144,51 @@ func TestSourceManagerWait(t *testing.T) {t.Fatalf("expected scheduling run to fail")}}++func TestSourceManagerError(t *testing.T) {+ mgr := NewManager()+ handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})+ if err != nil {+ t.Fatalf("unexpected error enrolling source: %v", err)+ }+ // A synchronous run should fail.+ if err := mgr.Run(context.Background(), handle); err == nil {+ t.Fatalf("expected run to fail")+ }+ // Scheduling a run should not fail, but the error should surface in+ // Wait().+ if err := mgr.ScheduleRun(context.Background(), handle); err != nil {+ t.Fatalf("unexpected error scheduling run: %v", err)+ }+ if err := mgr.Wait(); err == nil {+ t.Fatalf("expected wait to fail")+ }+}++func TestSourceManagerReport(t *testing.T) {+ for _, opts := range [][]func(*SourceManager){+ {WithBufferedOutput(8)},+ {WithBufferedOutput(8), WithSourceUnits()},+ {WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},+ } {+ mgr := NewManager(opts...)+ handle, err := enrollDummy(mgr, &counterChunker{count: 4})+ if err != nil {+ t.Fatalf("unexpected error enrolling source: %v", err)+ }+ // Synchronously run the source.+ if err := mgr.Run(context.Background(), handle); err != nil {+ t.Fatalf("unexpected error running source: %v", err)+ }+ report := mgr.Report(handle)+ if report == nil {+ t.Fatalf("expected a report")+ }+ if err := report.Errors(); err != nil {+ t.Fatalf("unexpected error in report: %v", err)+ }+ if report.TotalChunks != 4 {+ t.Fatalf("expected report to have 4 chunks, got: %d", report.TotalChunks)+ }+ }+}diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex 9d8c85eda..a09975f41 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -44,6 +44,13 @@ type Source interface {GetProgress() *Progress}+// SourceUnitEnumChunker are the two required interfaces to support enumerating+// and chunking of units.+type SourceUnitEnumChunker interface {+ SourceUnitEnumerator+ SourceUnitChunker+}+// SourceUnitUnmarshaller defines an optional interface a Source can implement// to support units coming from an external source.type SourceUnitUnmarshaller interface {