Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Refactor SourceManager to remove Enrollment (#1740)
Miccahbe4d0bcb4116
diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.goindex e1113b8e7..4998411e9 100644--- a/pkg/engine/circleci.go+++ b/pkg/engine/circleci.go@@ -8,7 +8,6 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"- "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci")@@ -27,17 +26,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {return err}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- circleSource := circleci.Source{}- if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return nil, err- }- return &circleSource, nil- })- if err != nil {+ sourceName := "trufflehog - Circle CI"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(circleci.Source).Type())++ circleSource := &circleci.Source{}+ if err := circleSource.Init(ctx, "trufflehog - Circle CI", int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ _, err = e.sourceManager.Run(ctx, sourceName, circleSource)return err}diff --git a/pkg/engine/docker.go b/pkg/engine/docker.goindex 97840f983..a17a97ce7 100644--- a/pkg/engine/docker.go+++ b/pkg/engine/docker.go@@ -6,23 +6,18 @@ import ("google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/context"- "github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker")// ScanDocker scans a given docker connection.func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- dockerSource := docker.Source{}- if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {- return nil, err- }- return &dockerSource, nil- })- if err != nil {+ sourceName := "trufflehog - docker"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(docker.Source).Type())++ dockerSource := &docker.Source{}+ if err := dockerSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, conn, runtime.NumCPU()); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ _, err := e.sourceManager.Run(ctx, sourceName, dockerSource)return err}diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.goindex 9e26c75b1..7915e229f 100644--- a/pkg/engine/filesystem.go+++ b/pkg/engine/filesystem.go@@ -24,18 +24,14 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)return err}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- fileSystemSource := filesystem.Source{}- fileSystemSource.WithFilter(c.Filter)- if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return nil, err- }- return &fileSystemSource, nil- })- if err != nil {+ sourceName := "trufflehog - filesystem"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(filesystem.Source).Type())++ fileSystemSource := &filesystem.Source{}+ fileSystemSource.WithFilter(c.Filter)+ if err := fileSystemSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ _, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)return err}diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.goindex 32d977f74..dcd9dd530 100644--- a/pkg/engine/gcs.go+++ b/pkg/engine/gcs.go@@ -44,18 +44,14 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {return fmt.Errorf("failed to marshal GCS connection: %w", err)}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- gcsSource := gcs.Source{}- if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {- return nil, err- }- return &gcsSource, nil- })- if err != nil {+ sourceName := "trufflehog - gcs"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(gcs.Source).Type())++ gcsSource := &gcs.Source{}+ if err := gcsSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, int(c.Concurrency)); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ _, err = e.sourceManager.Run(ctx, sourceName, gcsSource)return err}diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex d7785494e..73b54a694 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -51,21 +51,18 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {return err}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- gitSource := git.Source{}- if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return nil, err- }- gitSource.WithScanOptions(scanOptions)- // Don't try to clean up the provided directory. That's handled by the- // caller of ScanGit.- gitSource.WithPreserveTempDirs(true)- return &gitSource, nil- })- if err != nil {+ sourceName := "trufflehog - git"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(git.Source).Type())++ gitSource := &git.Source{}+ if err := gitSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ gitSource.WithScanOptions(scanOptions)+ // Don't try to clean up the provided directory. That's handled by the+ // caller of ScanGit.+ gitSource.WithPreserveTempDirs(true)++ _, err := e.sourceManager.Run(ctx, sourceName, gitSource)return err}diff --git a/pkg/engine/github.go b/pkg/engine/github.goindex 4e83acbdd..d04d7bebc 100644--- a/pkg/engine/github.go+++ b/pkg/engine/github.go@@ -47,18 +47,14 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {}scanOptions := git.NewScanOptions(opts...)- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- githubSource := github.Source{}- if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {- return nil, err- }- githubSource.WithScanOptions(scanOptions)- return &githubSource, nil- })- if err != nil {+ sourceName := "trufflehog - github"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(github.Source).Type())++ githubSource := &github.Source{}+ if err := githubSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, c.Concurrency); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ githubSource.WithScanOptions(scanOptions)+ _, err = e.sourceManager.Run(ctx, sourceName, githubSource)return err}diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.goindex 7179b238c..0b0149676 100644--- a/pkg/engine/gitlab.go+++ b/pkg/engine/gitlab.go@@ -50,18 +50,14 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {return err}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- gitlabSource := gitlab.Source{}- if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return nil, err- }- gitlabSource.WithScanOptions(scanOptions)- return &gitlabSource, nil- })- if err != nil {+ sourceName := "trufflehog - gitlab"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(gitlab.Source).Type())++ gitlabSource := &gitlab.Source{}+ if err := gitlabSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ gitlabSource.WithScanOptions(scanOptions)+ _, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)return err}diff --git a/pkg/engine/s3.go b/pkg/engine/s3.goindex db9b75be4..04e88897d 100644--- a/pkg/engine/s3.go+++ b/pkg/engine/s3.go@@ -58,17 +58,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {return err}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- s3Source := s3.Source{}- if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {- return nil, err- }- return &s3Source, nil- })- if err != nil {+ sourceName := "trufflehog - s3"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(s3.Source).Type())++ s3Source := &s3.Source{}+ if err := s3Source.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ _, err = e.sourceManager.Run(ctx, sourceName, s3Source)return err}diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.goindex 7bf413ff4..beabe1bef 100644--- a/pkg/engine/syslog.go+++ b/pkg/engine/syslog.go@@ -41,18 +41,14 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {return errors.WrapPrefix(err, "error unmarshalling connection", 0)}- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {- syslogSource := syslog.Source{}- if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {- return nil, err- }- syslogSource.InjectConnection(connection)- return &syslogSource, nil- })- if err != nil {+ sourceName := "trufflehog - syslog"+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(syslog.Source).Type())+ syslogSource := &syslog.Source{}+ if err := syslogSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, c.Concurrency); err != nil {return err}- _, err = e.sourceManager.ScheduleRun(ctx, handle)+ syslogSource.InjectConnection(connection)++ _, err = e.sourceManager.Run(ctx, sourceName, syslogSource)return err}diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.goindex 383b8feb5..addfd4296 100644--- a/pkg/sources/source_manager.go+++ b/pkg/sources/source_manager.go@@ -13,28 +13,9 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb")-// handle uniquely identifies a Source given to the manager to manage. If the-// SourceManager is connected to the API, it will be equivalent to the unique-// source ID, otherwise it behaves as a counter.-type handle int64--// SourceInitFunc is a function that takes a source and job ID and returns an-// initialized Source.-type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, error)--// sourceInfo is an aggregate struct to store source information provided on-// initialization.-type sourceInfo struct {- initFunc SourceInitFunc- name string-}-type SourceManager struct {api apiClienthooks []JobProgressHook- // Map of handle to source initializer.- handles map[handle]sourceInfo- handlesLock sync.Mutex// Pool limiting the amount of concurrent sources running.pool errgroup.GrouppoolLimit int@@ -49,12 +30,16 @@ type SourceManager struct {done bool}+type (+ SourceID int64+ JobID int64+)+// apiClient is an interface for optionally communicating with an external API.type apiClient interface {- // RegisterSource lets the API know we have a source and it returns a unique source ID for it.- RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)- // GetJobID queries the API for an existing job or new job ID.- GetJobID(ctx context.Context, id int64) (int64, error)+ // GetIDs informs the API of the source that's about to run and returns+ // two identifiers used during source initialization.+ GetIDs(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, JobID, error)}// WithAPI adds an API client to the manager for tracking jobs and progress. If@@ -99,7 +84,6 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {mgr := SourceManager{// Default to the headless API. Can be overwritten by the WithAPI option.api: &headlessAPI{},- handles: make(map[handle]sourceInfo),outputChunks: make(chan *Chunk),}for _, opt := range opts {@@ -108,70 +92,33 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {return &mgr}-// Enroll informs the SourceManager to track and manage a Source.-func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {- if s.done {- return 0, fmt.Errorf("manager is done")- }- id, err := s.api.RegisterSource(ctx, name, kind)- if err != nil {- return 0, err- }- handleID := handle(id)- s.handlesLock.Lock()- defer s.handlesLock.Unlock()- if _, ok := s.handles[handleID]; ok {- // TODO: smartly handle this?- return 0, fmt.Errorf("handle ID '%d' already in use", handleID)- }- s.handles[handleID] = sourceInfo{- initFunc: f,- name: name,- }- return handleID, nil+func (s *SourceManager) GetIDs(ctx context.Context, sourceName string, kind sourcespb.SourceType) (SourceID, JobID, error) {+ return s.api.GetIDs(ctx, sourceName, kind)}// Run blocks until a resource is available to run the source, then-// synchronously runs it. The first fatal error, if any, will be returned.-func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {- progress, err := s.asyncRun(ctx, handle)- if err != nil {- return progress, err- }- <-progress.Done()- return progress, progress.Snapshot().FatalError()-}--// ScheduleRun blocks until a resource is available to run the source, then// asynchronously runs it. Error information is stored and accessible via the// JobProgressRef as it becomes available.-func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {- return s.asyncRun(ctx, handle)+func (s *SourceManager) Run(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {+ return s.asyncRun(ctx, sourceName, source)}// asyncRun is a helper method to asynchronously run the Source. It calls out// to the API to get a job ID for this run, creates a JobProgress object, then// waits for an available goroutine to asynchronously run it.-func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {+func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {+ sourceID, jobID := source.SourceID(), source.JobID()// Do preflight checks before waiting on the pool.- if err := s.preflightChecks(ctx, handle); err != nil {- return JobProgressRef{}, err- }- // Get the name. Should never fail due to preflight checks.- sourceInfo, ok := s.getSourceInfo(handle)- if !ok {- return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")- }- sourceName := sourceInfo.name- // Get a Job ID.- ctx = context.WithValue(ctx, "source_id", int64(handle))- jobID, err := s.api.GetJobID(ctx, int64(handle))- if err != nil {- return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err+ if err := s.preflightChecks(ctx); err != nil {+ return JobProgressRef{+ SourceName: sourceName,+ SourceID: sourceID,+ JobID: jobID,+ }, err}// Create a JobProgress object for tracking progress.ctx, cancel := context.WithCancelCause(ctx)- progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))+ progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))s.pool.Go(func() error {atomic.AddInt32(&s.currentRunningCount, 1)defer atomic.AddInt32(&s.currentRunningCount, -1)@@ -181,7 +128,7 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres)defer common.Recover(ctx)defer cancel(nil)- return s.run(ctx, handle, jobID, progress)+ return s.run(ctx, source, progress)})return progress.Ref(), nil}@@ -226,23 +173,19 @@ func (s *SourceManager) AvailableCapacity() int {}// preflightChecks is a helper method to check the Manager or the context isn't-// done and that the handle is valid.-func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {+// done.+func (s *SourceManager) preflightChecks(ctx context.Context) error {// Check if the manager has been Waited.if s.done {return fmt.Errorf("manager is done")}- // Check the handle is valid.- if _, ok := s.getSourceInfo(handle); !ok {- return fmt.Errorf("unrecognized handle")- }return ctx.Err()}// run is a helper method to sychronously run the source. It does not check for// acquired resources. An error is returned if there was a fatal error during// the run. This information is also recorded in the JobProgress.-func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {+func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgress) error {defer report.Finish()report.Start(time.Now())defer func() { report.End(time.Now()) }()@@ -253,34 +196,21 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep}}()- // Initialize the source.- sourceInfo, ok := s.getSourceInfo(handle)- if !ok {- // Shouldn't happen due to preflight checks.- err := fmt.Errorf("unrecognized handle")- report.ReportError(Fatal{err})- return Fatal{err}- }- source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))- if err != nil {- report.ReportError(Fatal{err})- return Fatal{err}- }report.TrackProgress(source.GetProgress())ctx = context.WithValues(ctx,"source_type", source.Type().String(),- "source_name", sourceInfo.name,+ "source_name", report.SourceName,)// Check for the preferred method of tracking source units.if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {- return s.runWithUnits(ctx, handle, enumChunker, report)+ return s.runWithUnits(ctx, enumChunker, report)}- return s.runWithoutUnits(ctx, handle, source, report)+ return s.runWithoutUnits(ctx, source, report)}// runWithoutUnits is a helper method to run a Source. It has coarse-grained// job reporting.-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {+func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {// Introspect on the chunks we get from the Chunks method.ch := make(chan *Chunk, 1)var wg sync.WaitGroup@@ -310,7 +240,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour// runWithUnits is a helper method to run a Source that is also a// SourceUnitEnumChunker. This allows better introspection of what is getting// scanned and any errors encountered.-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {+func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {unitReporter := &mgrUnitReporter{unitCh: make(chan SourceUnit, 1),report: report,@@ -385,28 +315,15 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source}}-// getSourceInfo is a helper method for safe concurrent access to the-// map[handle]SourceInitFunc map.-func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {- s.handlesLock.Lock()- defer s.handlesLock.Unlock()- f, ok := s.handles[handle]- return f, ok-}-// headlessAPI implements the apiClient interface locally.type headlessAPI struct {- // Counters for assigning handle and job IDs.+ // Counters for assigning source and job IDs.sourceIDCounter int64jobIDCounter int64}-func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {- return atomic.AddInt64(&api.sourceIDCounter, 1), nil-}--func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {- return atomic.AddInt64(&api.jobIDCounter, 1), nil+func (api *headlessAPI) GetIDs(context.Context, string, sourcespb.SourceType) (SourceID, JobID, error) {+ return SourceID(atomic.AddInt64(&api.sourceIDCounter, 1)), JobID(atomic.AddInt64(&api.jobIDCounter, 1)), nil}// mgrUnitReporter implements the UnitReporter interface.diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.goindex 31378eb02..38a60b581 100644--- a/pkg/sources/source_manager_test.go+++ b/pkg/sources/source_manager_test.go@@ -80,16 +80,13 @@ func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) erfunc (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }-// enrollDummy is a helper function to enroll a DummySource with a SourceManager.-func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {- return mgr.Enroll(context.Background(), "dummy", 1337,- func(ctx context.Context, jobID, sourceID int64) (Source, error) {- source := &DummySource{chunker: chunkMethod}- if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {- return nil, err- }- return source, nil- })+// buildDummy is a helper function to enroll a DummySource with a SourceManager.+func buildDummy(chunkMethod chunker) (Source, error) {+ source := &DummySource{chunker: chunkMethod}+ if err := source.Init(context.Background(), "dummy", 123, 456, true, nil, 42); err != nil {+ return nil, err+ }+ return source, nil}// tryRead is a helper function that will try to read from a channel and return@@ -105,11 +102,13 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {func TestSourceManagerRun(t *testing.T) {mgr := NewManager(WithBufferedOutput(8))- handle, err := enrollDummy(mgr, &counterChunker{count: 1})+ source, err := buildDummy(&counterChunker{count: 1})assert.NoError(t, err)for i := 0; i < 3; i++ {- _, err = mgr.Run(context.Background(), handle)+ ref, err := mgr.Run(context.Background(), "dummy", source)+ <-ref.Done()assert.NoError(t, err)+ assert.NoError(t, ref.Snapshot().FatalError())chunk, err := tryRead(mgr.Chunks())assert.NoError(t, err)assert.Equal(t, []byte{byte(i)}, chunk.Data)@@ -121,35 +120,31 @@ func TestSourceManagerRun(t *testing.T) {func TestSourceManagerWait(t *testing.T) {mgr := NewManager()- handle, err := enrollDummy(mgr, &counterChunker{count: 1})+ source, err := buildDummy(&counterChunker{count: 1})assert.NoError(t, err)// Asynchronously run the source.- _, err = mgr.ScheduleRun(context.Background(), handle)+ _, err = mgr.Run(context.Background(), "dummy", source)assert.NoError(t, err)// Read the 1 chunk we're expecting so Waiting completes.<-mgr.Chunks()// Wait for all resources to complete.assert.NoError(t, mgr.Wait())- // Enroll and run should return an error now.- _, err = enrollDummy(mgr, &counterChunker{count: 1})- assert.Error(t, err)- _, err = mgr.ScheduleRun(context.Background(), handle)+ // Run should return an error now.+ _, err = buildDummy(&counterChunker{count: 1})+ assert.NoError(t, err)+ _, err = mgr.Run(context.Background(), "dummy", source)assert.Error(t, err)}func TestSourceManagerError(t *testing.T) {mgr := NewManager()- handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})+ source, err := buildDummy(errorChunker{fmt.Errorf("oops")})assert.NoError(t, err)- // A synchronous run should fail.- _, err = mgr.Run(context.Background(), handle)- assert.Error(t, err)- // Scheduling a run should not fail, but the error should surface in- // Wait().- ref, err := mgr.ScheduleRun(context.Background(), handle)+ ref, err := mgr.Run(context.Background(), "dummy", source)assert.NoError(t, err)- assert.Error(t, mgr.Wait())+ <-ref.Done()assert.Error(t, ref.Snapshot().FatalError())+ assert.Error(t, mgr.Wait())}func TestSourceManagerReport(t *testing.T) {@@ -159,11 +154,11 @@ func TestSourceManagerReport(t *testing.T) {{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},} {mgr := NewManager(opts...)- handle, err := enrollDummy(mgr, &counterChunker{count: 4})+ source, err := buildDummy(&counterChunker{count: 4})assert.NoError(t, err)- // Synchronously run the source.- ref, err := mgr.Run(context.Background(), handle)+ ref, err := mgr.Run(context.Background(), "dummy", source)assert.NoError(t, err)+ <-ref.Done()assert.Equal(t, 0, len(ref.Snapshot().Errors))assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)}@@ -221,10 +216,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {{unit: "three", err: "not again"},}mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())- handle, err := enrollDummy(mgr, &unitChunker{input})+ source, err := buildDummy(&unitChunker{input})assert.NoError(t, err)- ref, err := mgr.Run(context.Background(), handle)+ ref, err := mgr.Run(context.Background(), "dummy", source)assert.NoError(t, err)+ <-ref.Done()report := ref.Snapshot()assert.Equal(t, len(input), int(report.TotalUnits))assert.Equal(t, len(input), int(report.FinishedUnits))@@ -235,11 +231,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {func TestSourceManagerContextCancelled(t *testing.T) {mgr := NewManager(WithBufferedOutput(8))- handle, err := enrollDummy(mgr, &counterChunker{count: 100})+ source, err := buildDummy(&counterChunker{count: 100})assert.NoError(t, err)ctx, cancel := context.WithCancel(context.Background())- ref, err := mgr.ScheduleRun(ctx, handle)+ ref, err := mgr.Run(ctx, "dummy", source)assert.NoError(t, err)cancel()@@ -249,47 +245,18 @@ func TestSourceManagerContextCancelled(t *testing.T) {}type DummyAPI struct {- registerSource func(context.Context, string, sourcespb.SourceType) (int64, error)- getJobID func(context.Context, int64) (int64, error)+ registerSource func(context.Context, string, sourcespb.SourceType) (SourceID, error)+ getJobID func(context.Context, SourceID) (JobID, error)}-func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {+func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, error) {return api.registerSource(ctx, name, kind)}-func (api DummyAPI) GetJobID(ctx context.Context, id int64) (int64, error) {+func (api DummyAPI) GetJobID(ctx context.Context, id SourceID) (JobID, error) {return api.getJobID(ctx, id)}-func TestSourceManagerJobAndSourceIDs(t *testing.T) {- mgr := NewManager(WithAPI(DummyAPI{- registerSource: func(context.Context, string, sourcespb.SourceType) (int64, error) {- return 1337, nil- },- getJobID: func(context.Context, int64) (int64, error) {- return 9001, nil- },- }))- var (- initializedJobID int64- initializedSourceID int64- )- handle, err := mgr.Enroll(context.Background(), "dummy", 1337,- func(ctx context.Context, jobID, sourceID int64) (Source, error) {- initializedJobID = jobID- initializedSourceID = sourceID- return nil, fmt.Errorf("ignore")- })- assert.NoError(t, err)-- ref, _ := mgr.Run(context.Background(), handle)- assert.Equal(t, int64(1337), initializedSourceID)- assert.Equal(t, int64(1337), ref.SourceID)- assert.Equal(t, int64(9001), initializedJobID)- assert.Equal(t, int64(9001), ref.JobID)- assert.Equal(t, "dummy", ref.SourceName)-}-// Chunk method that has a custom callback for the Chunks method.type callbackChunker struct {cb func(context.Context, chan *Chunk) error@@ -304,7 +271,7 @@ func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) efunc TestSourceManagerCancelRun(t *testing.T) {mgr := NewManager(WithBufferedOutput(8))var returnedErr error- handle, err := enrollDummy(mgr, callbackChunker{func(ctx context.Context, _ chan *Chunk) error {+ source, err := buildDummy(callbackChunker{func(ctx context.Context, _ chan *Chunk) error {// The context passed to Chunks should get cancelled when ref.CancelRun() is called.<-ctx.Done()returnedErr = fmt.Errorf("oh no: %w", ctx.Err())@@ -312,7 +279,7 @@ func TestSourceManagerCancelRun(t *testing.T) {}})assert.NoError(t, err)- ref, err := mgr.ScheduleRun(context.Background(), handle)+ ref, err := mgr.Run(context.Background(), "dummy", source)assert.NoError(t, err)cancelErr := fmt.Errorf("abort! abort!")@@ -326,7 +293,7 @@ func TestSourceManagerCancelRun(t *testing.T) {func TestSourceManagerAvailableCapacity(t *testing.T) {mgr := NewManager(WithConcurrentSources(1337))start, end := make(chan struct{}), make(chan struct{})- handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {+ source, err := buildDummy(callbackChunker{func(context.Context, chan *Chunk) error {start <- struct{}{} // Send start signal.<-end // Wait for end signal.return nil@@ -334,7 +301,7 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {assert.NoError(t, err)assert.Equal(t, 1337, mgr.AvailableCapacity())- ref, err := mgr.ScheduleRun(context.Background(), handle)+ ref, err := mgr.Run(context.Background(), "dummy", source)assert.NoError(t, err)<-start // Wait for start signal.