Skip to content

Find, verify, and analyze leaked credentials

Implement SourceManager basics (#1515)

Miccah91c5472876e8
diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go
index 7ad70626a..98293e017 100644
--- a/pkg/engine/filesystem.go
+++ b/pkg/engine/filesystem.go
@@ -28,6 +28,22 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
}
fileSystemSource := filesystem.Source{}
+ fileSystemSource.WithFilter(c.Filter)
+
+ // TODO: This is how it would be used; everything after these comments would be removed.
+ // handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
+ // func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+ // fileSystemSource := filesystem.Source{}
+ // fileSystemSource.WithFilter(c.Filter)
+ // if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+ // return nil, err
+ // }
+ // return &fileSystemSource, nil
+ // })
+ // if err != nil {
+ // return err
+ // }
+ // return e.sourceManager.ScheduleRun(ctx, handle)
ctx = context.WithValues(ctx,
"source_type", fileSystemSource.Type().String(),
@@ -37,7 +53,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
if err != nil {
return errors.WrapPrefix(err, "could not init filesystem source", 0)
}
- fileSystemSource.WithFilter(c.Filter)
e.sourcesWg.Go(func() error {
defer common.RecoverWithExit(ctx)
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.go
new file mode 100644
index 000000000..213151846
--- /dev/null
+++ b/pkg/sources/source_manager.go
@@ -0,0 +1,164 @@
+package sources
+
+import (
+ "fmt"
+ "sync"
+ "sync/atomic"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "golang.org/x/sync/errgroup"
+)
+
+// handle uniquely identifies a Source given to the manager to manage. If the
+// SourceManager is connected to the API, it will be equivalent to the unique
+// source ID, otherwise it behaves as a counter.
+type handle int64
+
+// SourceInitFunc is a function that takes a source and job ID and returns an
+// initialized Source.
+type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
+
+type SourceManager struct {
+ api apiClient
+ // Map of handle to source initializer.
+ handles map[handle]SourceInitFunc
+ handlesLock sync.Mutex
+ // Pool limiting the amount of concurrent sources running.
+ pool errgroup.Group
+ // Downstream chunks channel to be scanned.
+ outputChunks chan *Chunk
+}
+
+// apiClient is an interface for optionally communicating with an external API.
+type apiClient interface {
+ // RegisterSource lets the API know we have a source and it returns a unique source ID for it.
+ RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
+ // GetJobID queries the API for an existing job or new job ID.
+ GetJobID(ctx context.Context, id int64) (int64, error)
+}
+
+// WithAPI adds an API client to the manager for tracking jobs and progress.
+func WithAPI(api apiClient) func(*SourceManager) {
+ return func(man *SourceManager) { man.api = api }
+}
+
+// WithConcurrency limits the concurrent number of sources a manager can run.
+func WithConcurrency(concurrency int) func(*SourceManager) {
+ return func(man *SourceManager) { man.pool.SetLimit(concurrency) }
+}
+
+// NewManager creates a new manager with the provided options.
+func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
+ man := SourceManager{
+ // Default to the headless API. Can be overwritten by the WithAPI option.
+ api: &headlessAPI{},
+ handles: make(map[handle]SourceInitFunc),
+ outputChunks: outputChunks,
+ }
+ for _, opt := range opts {
+ opt(&man)
+ }
+ return &man
+}
+
+// Enroll informs the SourceManager to track and manage a Source.
+func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
+ id, err := s.api.RegisterSource(ctx, name, kind)
+ if err != nil {
+ return 0, err
+ }
+ handleID := handle(id)
+ s.handlesLock.Lock()
+ defer s.handlesLock.Unlock()
+ if _, ok := s.handles[handleID]; ok {
+ // TODO: smartly handle this?
+ return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
+ }
+ s.handles[handleID] = f
+ return 0, nil
+}
+
+// Run blocks until a resource is available to run the source, then synchronously runs it.
+func (s *SourceManager) Run(ctx context.Context, handle handle) error {
+ // Check the handle is valid before waiting on the pool.
+ if _, ok := s.getInitFunc(handle); !ok {
+ return fmt.Errorf("unrecognized handle")
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ ch := make(chan error)
+ s.pool.Go(func() error {
+ defer common.Recover(ctx)
+ // TODO: The manager should record these errors.
+ ch <- s.run(ctx, handle)
+ return nil
+ })
+ return <-ch
+}
+
+// ScheduleRun blocks until a resource is available to run the source, then
+// asynchronously runs it. Error information is lost in this case.
+func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
+ // Check the handle is valid before waiting on the pool.
+ if _, ok := s.getInitFunc(handle); !ok {
+ return fmt.Errorf("unrecognized handle")
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ s.pool.Go(func() error {
+ defer common.Recover(ctx)
+ // TODO: The manager should record these errors.
+ _ = s.run(ctx, handle)
+ return nil
+ })
+ // TODO: Maybe wait for a signal here that initialization was successful?
+ return nil
+}
+
+// run is a helper method to sychronously run the source. It does not check for
+// acquired resources.
+func (s *SourceManager) run(ctx context.Context, handle handle) error {
+ jobID, err := s.api.GetJobID(ctx, int64(handle))
+ if err != nil {
+ return err
+ }
+ initFunc, ok := s.getInitFunc(handle)
+ if !ok {
+ return fmt.Errorf("unrecognized handle")
+ }
+ source, err := initFunc(jobID, int64(handle))
+ if err != nil {
+ return err
+ }
+ // TODO: Support UnitChunker and SourceUnitEnumerator.
+ // TODO: This is where we can introspect on the chunks collected.
+ return source.Chunks(ctx, s.outputChunks)
+}
+
+// getInitFunc is a helper method for safe concurrent access to the
+// map[handle]SourceInitFunc map.
+func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
+ s.handlesLock.Lock()
+ defer s.handlesLock.Unlock()
+ f, ok := s.handles[handle]
+ return f, ok
+}
+
+// headlessAPI implements the apiClient interface locally.
+type headlessAPI struct {
+ // Counters for assigning handle and job IDs.
+ sourceIDCounter int64
+ jobIDCounter int64
+}
+
+func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
+ return atomic.AddInt64(&api.sourceIDCounter, 1), nil
+}
+
+func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
+ return atomic.AddInt64(&api.jobIDCounter, 1), nil
+}