Skip to content

Find, verify, and analyze leaked credentials

Refactor SourceManager to remove Enrollment (#1740)

Miccahbe4d0bcb4116
diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go
index e1113b8e7..4998411e9 100644
--- a/pkg/engine/circleci.go
+++ b/pkg/engine/circleci.go
@@ -8,7 +8,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
- "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
)
@@ -27,17 +26,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
return err
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- circleSource := circleci.Source{}
- if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return nil, err
- }
- return &circleSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - Circle CI"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(circleci.Source).Type())
+
+ circleSource := &circleci.Source{}
+ if err := circleSource.Init(ctx, "trufflehog - Circle CI", int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ _, err = e.sourceManager.Run(ctx, sourceName, circleSource)
return err
}
diff --git a/pkg/engine/docker.go b/pkg/engine/docker.go
index 97840f983..a17a97ce7 100644
--- a/pkg/engine/docker.go
+++ b/pkg/engine/docker.go
@@ -6,23 +6,18 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
- "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
)
// ScanDocker scans a given docker connection.
func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- dockerSource := docker.Source{}
- if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
- return nil, err
- }
- return &dockerSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - docker"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(docker.Source).Type())
+
+ dockerSource := &docker.Source{}
+ if err := dockerSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, conn, runtime.NumCPU()); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ _, err := e.sourceManager.Run(ctx, sourceName, dockerSource)
return err
}
diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go
index 9e26c75b1..7915e229f 100644
--- a/pkg/engine/filesystem.go
+++ b/pkg/engine/filesystem.go
@@ -24,18 +24,14 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
return err
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- fileSystemSource := filesystem.Source{}
- fileSystemSource.WithFilter(c.Filter)
- if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return nil, err
- }
- return &fileSystemSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - filesystem"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(filesystem.Source).Type())
+
+ fileSystemSource := &filesystem.Source{}
+ fileSystemSource.WithFilter(c.Filter)
+ if err := fileSystemSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ _, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
return err
}
diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.go
index 32d977f74..dcd9dd530 100644
--- a/pkg/engine/gcs.go
+++ b/pkg/engine/gcs.go
@@ -44,18 +44,14 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
return fmt.Errorf("failed to marshal GCS connection: %w", err)
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- gcsSource := gcs.Source{}
- if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
- return nil, err
- }
- return &gcsSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - gcs"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(gcs.Source).Type())
+
+ gcsSource := &gcs.Source{}
+ if err := gcsSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, int(c.Concurrency)); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ _, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
return err
}
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index d7785494e..73b54a694 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -51,21 +51,18 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
return err
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- gitSource := git.Source{}
- if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return nil, err
- }
- gitSource.WithScanOptions(scanOptions)
- // Don't try to clean up the provided directory. That's handled by the
- // caller of ScanGit.
- gitSource.WithPreserveTempDirs(true)
- return &gitSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - git"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(git.Source).Type())
+
+ gitSource := &git.Source{}
+ if err := gitSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ gitSource.WithScanOptions(scanOptions)
+ // Don't try to clean up the provided directory. That's handled by the
+ // caller of ScanGit.
+ gitSource.WithPreserveTempDirs(true)
+
+ _, err := e.sourceManager.Run(ctx, sourceName, gitSource)
return err
}
diff --git a/pkg/engine/github.go b/pkg/engine/github.go
index 4e83acbdd..d04d7bebc 100644
--- a/pkg/engine/github.go
+++ b/pkg/engine/github.go
@@ -47,18 +47,14 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
}
scanOptions := git.NewScanOptions(opts...)
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- githubSource := github.Source{}
- if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
- return nil, err
- }
- githubSource.WithScanOptions(scanOptions)
- return &githubSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - github"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(github.Source).Type())
+
+ githubSource := &github.Source{}
+ if err := githubSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, c.Concurrency); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ githubSource.WithScanOptions(scanOptions)
+ _, err = e.sourceManager.Run(ctx, sourceName, githubSource)
return err
}
diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
index 7179b238c..0b0149676 100644
--- a/pkg/engine/gitlab.go
+++ b/pkg/engine/gitlab.go
@@ -50,18 +50,14 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
return err
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- gitlabSource := gitlab.Source{}
- if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return nil, err
- }
- gitlabSource.WithScanOptions(scanOptions)
- return &gitlabSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - gitlab"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(gitlab.Source).Type())
+
+ gitlabSource := &gitlab.Source{}
+ if err := gitlabSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ gitlabSource.WithScanOptions(scanOptions)
+ _, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)
return err
}
diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go
index db9b75be4..04e88897d 100644
--- a/pkg/engine/s3.go
+++ b/pkg/engine/s3.go
@@ -58,17 +58,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
return err
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- s3Source := s3.Source{}
- if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
- return nil, err
- }
- return &s3Source, nil
- })
- if err != nil {
+ sourceName := "trufflehog - s3"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(s3.Source).Type())
+
+ s3Source := &s3.Source{}
+ if err := s3Source.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, runtime.NumCPU()); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ _, err = e.sourceManager.Run(ctx, sourceName, s3Source)
return err
}
diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go
index 7bf413ff4..beabe1bef 100644
--- a/pkg/engine/syslog.go
+++ b/pkg/engine/syslog.go
@@ -41,18 +41,14 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
- handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
- func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
- syslogSource := syslog.Source{}
- if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
- return nil, err
- }
- syslogSource.InjectConnection(connection)
- return &syslogSource, nil
- })
- if err != nil {
+ sourceName := "trufflehog - syslog"
+ sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, new(syslog.Source).Type())
+ syslogSource := &syslog.Source{}
+ if err := syslogSource.Init(ctx, sourceName, int64(jobID), int64(sourceID), true, &conn, c.Concurrency); err != nil {
return err
}
- _, err = e.sourceManager.ScheduleRun(ctx, handle)
+ syslogSource.InjectConnection(connection)
+
+ _, err = e.sourceManager.Run(ctx, sourceName, syslogSource)
return err
}
diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.go
index 383b8feb5..addfd4296 100644
--- a/pkg/sources/source_manager.go
+++ b/pkg/sources/source_manager.go
@@ -13,28 +13,9 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
)
-// handle uniquely identifies a Source given to the manager to manage. If the
-// SourceManager is connected to the API, it will be equivalent to the unique
-// source ID, otherwise it behaves as a counter.
-type handle int64
-
-// SourceInitFunc is a function that takes a source and job ID and returns an
-// initialized Source.
-type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, error)
-
-// sourceInfo is an aggregate struct to store source information provided on
-// initialization.
-type sourceInfo struct {
- initFunc SourceInitFunc
- name string
-}
-
type SourceManager struct {
api apiClient
hooks []JobProgressHook
- // Map of handle to source initializer.
- handles map[handle]sourceInfo
- handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
poolLimit int
@@ -49,12 +30,16 @@ type SourceManager struct {
done bool
}
+type (
+ SourceID int64
+ JobID int64
+)
+
// apiClient is an interface for optionally communicating with an external API.
type apiClient interface {
- // RegisterSource lets the API know we have a source and it returns a unique source ID for it.
- RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
- // GetJobID queries the API for an existing job or new job ID.
- GetJobID(ctx context.Context, id int64) (int64, error)
+ // GetIDs informs the API of the source that's about to run and returns
+ // two identifiers used during source initialization.
+ GetIDs(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, JobID, error)
}
// WithAPI adds an API client to the manager for tracking jobs and progress. If
@@ -99,7 +84,6 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
mgr := SourceManager{
// Default to the headless API. Can be overwritten by the WithAPI option.
api: &headlessAPI{},
- handles: make(map[handle]sourceInfo),
outputChunks: make(chan *Chunk),
}
for _, opt := range opts {
@@ -108,70 +92,33 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
return &mgr
}
-// Enroll informs the SourceManager to track and manage a Source.
-func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
- if s.done {
- return 0, fmt.Errorf("manager is done")
- }
- id, err := s.api.RegisterSource(ctx, name, kind)
- if err != nil {
- return 0, err
- }
- handleID := handle(id)
- s.handlesLock.Lock()
- defer s.handlesLock.Unlock()
- if _, ok := s.handles[handleID]; ok {
- // TODO: smartly handle this?
- return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
- }
- s.handles[handleID] = sourceInfo{
- initFunc: f,
- name: name,
- }
- return handleID, nil
+func (s *SourceManager) GetIDs(ctx context.Context, sourceName string, kind sourcespb.SourceType) (SourceID, JobID, error) {
+ return s.api.GetIDs(ctx, sourceName, kind)
}
// Run blocks until a resource is available to run the source, then
-// synchronously runs it. The first fatal error, if any, will be returned.
-func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
- progress, err := s.asyncRun(ctx, handle)
- if err != nil {
- return progress, err
- }
- <-progress.Done()
- return progress, progress.Snapshot().FatalError()
-}
-
-// ScheduleRun blocks until a resource is available to run the source, then
// asynchronously runs it. Error information is stored and accessible via the
// JobProgressRef as it becomes available.
-func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
- return s.asyncRun(ctx, handle)
+func (s *SourceManager) Run(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
+ return s.asyncRun(ctx, sourceName, source)
}
// asyncRun is a helper method to asynchronously run the Source. It calls out
// to the API to get a job ID for this run, creates a JobProgress object, then
// waits for an available goroutine to asynchronously run it.
-func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
+func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
+ sourceID, jobID := source.SourceID(), source.JobID()
// Do preflight checks before waiting on the pool.
- if err := s.preflightChecks(ctx, handle); err != nil {
- return JobProgressRef{}, err
- }
- // Get the name. Should never fail due to preflight checks.
- sourceInfo, ok := s.getSourceInfo(handle)
- if !ok {
- return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
- }
- sourceName := sourceInfo.name
- // Get a Job ID.
- ctx = context.WithValue(ctx, "source_id", int64(handle))
- jobID, err := s.api.GetJobID(ctx, int64(handle))
- if err != nil {
- return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
+ if err := s.preflightChecks(ctx); err != nil {
+ return JobProgressRef{
+ SourceName: sourceName,
+ SourceID: sourceID,
+ JobID: jobID,
+ }, err
}
// Create a JobProgress object for tracking progress.
ctx, cancel := context.WithCancelCause(ctx)
- progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
+ progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
s.pool.Go(func() error {
atomic.AddInt32(&s.currentRunningCount, 1)
defer atomic.AddInt32(&s.currentRunningCount, -1)
@@ -181,7 +128,7 @@ func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgres
)
defer common.Recover(ctx)
defer cancel(nil)
- return s.run(ctx, handle, jobID, progress)
+ return s.run(ctx, source, progress)
})
return progress.Ref(), nil
}
@@ -226,23 +173,19 @@ func (s *SourceManager) AvailableCapacity() int {
}
// preflightChecks is a helper method to check the Manager or the context isn't
-// done and that the handle is valid.
-func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
+// done.
+func (s *SourceManager) preflightChecks(ctx context.Context) error {
// Check if the manager has been Waited.
if s.done {
return fmt.Errorf("manager is done")
}
- // Check the handle is valid.
- if _, ok := s.getSourceInfo(handle); !ok {
- return fmt.Errorf("unrecognized handle")
- }
return ctx.Err()
}
// run is a helper method to sychronously run the source. It does not check for
// acquired resources. An error is returned if there was a fatal error during
// the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
+func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgress) error {
defer report.Finish()
report.Start(time.Now())
defer func() { report.End(time.Now()) }()
@@ -253,34 +196,21 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
}
}()
- // Initialize the source.
- sourceInfo, ok := s.getSourceInfo(handle)
- if !ok {
- // Shouldn't happen due to preflight checks.
- err := fmt.Errorf("unrecognized handle")
- report.ReportError(Fatal{err})
- return Fatal{err}
- }
- source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
- if err != nil {
- report.ReportError(Fatal{err})
- return Fatal{err}
- }
report.TrackProgress(source.GetProgress())
ctx = context.WithValues(ctx,
"source_type", source.Type().String(),
- "source_name", sourceInfo.name,
+ "source_name", report.SourceName,
)
// Check for the preferred method of tracking source units.
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
- return s.runWithUnits(ctx, handle, enumChunker, report)
+ return s.runWithUnits(ctx, enumChunker, report)
}
- return s.runWithoutUnits(ctx, handle, source, report)
+ return s.runWithoutUnits(ctx, source, report)
}
// runWithoutUnits is a helper method to run a Source. It has coarse-grained
// job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
// Introspect on the chunks we get from the Chunks method.
ch := make(chan *Chunk, 1)
var wg sync.WaitGroup
@@ -310,7 +240,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
// runWithUnits is a helper method to run a Source that is also a
// SourceUnitEnumChunker. This allows better introspection of what is getting
// scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
+func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
unitReporter := &mgrUnitReporter{
unitCh: make(chan SourceUnit, 1),
report: report,
@@ -385,28 +315,15 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
}
}
-// getSourceInfo is a helper method for safe concurrent access to the
-// map[handle]SourceInitFunc map.
-func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
- s.handlesLock.Lock()
- defer s.handlesLock.Unlock()
- f, ok := s.handles[handle]
- return f, ok
-}
-
// headlessAPI implements the apiClient interface locally.
type headlessAPI struct {
- // Counters for assigning handle and job IDs.
+ // Counters for assigning source and job IDs.
sourceIDCounter int64
jobIDCounter int64
}
-func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
- return atomic.AddInt64(&api.sourceIDCounter, 1), nil
-}
-
-func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
- return atomic.AddInt64(&api.jobIDCounter, 1), nil
+func (api *headlessAPI) GetIDs(context.Context, string, sourcespb.SourceType) (SourceID, JobID, error) {
+ return SourceID(atomic.AddInt64(&api.sourceIDCounter, 1)), JobID(atomic.AddInt64(&api.jobIDCounter, 1)), nil
}
// mgrUnitReporter implements the UnitReporter interface.
diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.go
index 31378eb02..38a60b581 100644
--- a/pkg/sources/source_manager_test.go
+++ b/pkg/sources/source_manager_test.go
@@ -80,16 +80,13 @@ func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) er
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
-// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
-func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
- return mgr.Enroll(context.Background(), "dummy", 1337,
- func(ctx context.Context, jobID, sourceID int64) (Source, error) {
- source := &DummySource{chunker: chunkMethod}
- if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
- return nil, err
- }
- return source, nil
- })
+// buildDummy is a helper function to enroll a DummySource with a SourceManager.
+func buildDummy(chunkMethod chunker) (Source, error) {
+ source := &DummySource{chunker: chunkMethod}
+ if err := source.Init(context.Background(), "dummy", 123, 456, true, nil, 42); err != nil {
+ return nil, err
+ }
+ return source, nil
}
// tryRead is a helper function that will try to read from a channel and return
@@ -105,11 +102,13 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {
func TestSourceManagerRun(t *testing.T) {
mgr := NewManager(WithBufferedOutput(8))
- handle, err := enrollDummy(mgr, &counterChunker{count: 1})
+ source, err := buildDummy(&counterChunker{count: 1})
assert.NoError(t, err)
for i := 0; i < 3; i++ {
- _, err = mgr.Run(context.Background(), handle)
+ ref, err := mgr.Run(context.Background(), "dummy", source)
+ <-ref.Done()
assert.NoError(t, err)
+ assert.NoError(t, ref.Snapshot().FatalError())
chunk, err := tryRead(mgr.Chunks())
assert.NoError(t, err)
assert.Equal(t, []byte{byte(i)}, chunk.Data)
@@ -121,35 +120,31 @@ func TestSourceManagerRun(t *testing.T) {
func TestSourceManagerWait(t *testing.T) {
mgr := NewManager()
- handle, err := enrollDummy(mgr, &counterChunker{count: 1})
+ source, err := buildDummy(&counterChunker{count: 1})
assert.NoError(t, err)
// Asynchronously run the source.
- _, err = mgr.ScheduleRun(context.Background(), handle)
+ _, err = mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
// Read the 1 chunk we're expecting so Waiting completes.
<-mgr.Chunks()
// Wait for all resources to complete.
assert.NoError(t, mgr.Wait())
- // Enroll and run should return an error now.
- _, err = enrollDummy(mgr, &counterChunker{count: 1})
- assert.Error(t, err)
- _, err = mgr.ScheduleRun(context.Background(), handle)
+ // Run should return an error now.
+ _, err = buildDummy(&counterChunker{count: 1})
+ assert.NoError(t, err)
+ _, err = mgr.Run(context.Background(), "dummy", source)
assert.Error(t, err)
}
func TestSourceManagerError(t *testing.T) {
mgr := NewManager()
- handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
+ source, err := buildDummy(errorChunker{fmt.Errorf("oops")})
assert.NoError(t, err)
- // A synchronous run should fail.
- _, err = mgr.Run(context.Background(), handle)
- assert.Error(t, err)
- // Scheduling a run should not fail, but the error should surface in
- // Wait().
- ref, err := mgr.ScheduleRun(context.Background(), handle)
+ ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
- assert.Error(t, mgr.Wait())
+ <-ref.Done()
assert.Error(t, ref.Snapshot().FatalError())
+ assert.Error(t, mgr.Wait())
}
func TestSourceManagerReport(t *testing.T) {
@@ -159,11 +154,11 @@ func TestSourceManagerReport(t *testing.T) {
{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
} {
mgr := NewManager(opts...)
- handle, err := enrollDummy(mgr, &counterChunker{count: 4})
+ source, err := buildDummy(&counterChunker{count: 4})
assert.NoError(t, err)
- // Synchronously run the source.
- ref, err := mgr.Run(context.Background(), handle)
+ ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
+ <-ref.Done()
assert.Equal(t, 0, len(ref.Snapshot().Errors))
assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)
}
@@ -221,10 +216,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
{unit: "three", err: "not again"},
}
mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())
- handle, err := enrollDummy(mgr, &unitChunker{input})
+ source, err := buildDummy(&unitChunker{input})
assert.NoError(t, err)
- ref, err := mgr.Run(context.Background(), handle)
+ ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
+ <-ref.Done()
report := ref.Snapshot()
assert.Equal(t, len(input), int(report.TotalUnits))
assert.Equal(t, len(input), int(report.FinishedUnits))
@@ -235,11 +231,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
func TestSourceManagerContextCancelled(t *testing.T) {
mgr := NewManager(WithBufferedOutput(8))
- handle, err := enrollDummy(mgr, &counterChunker{count: 100})
+ source, err := buildDummy(&counterChunker{count: 100})
assert.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
- ref, err := mgr.ScheduleRun(ctx, handle)
+ ref, err := mgr.Run(ctx, "dummy", source)
assert.NoError(t, err)
cancel()
@@ -249,47 +245,18 @@ func TestSourceManagerContextCancelled(t *testing.T) {
}
type DummyAPI struct {
- registerSource func(context.Context, string, sourcespb.SourceType) (int64, error)
- getJobID func(context.Context, int64) (int64, error)
+ registerSource func(context.Context, string, sourcespb.SourceType) (SourceID, error)
+ getJobID func(context.Context, SourceID) (JobID, error)
}
-func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
+func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, error) {
return api.registerSource(ctx, name, kind)
}
-func (api DummyAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
+func (api DummyAPI) GetJobID(ctx context.Context, id SourceID) (JobID, error) {
return api.getJobID(ctx, id)
}
-func TestSourceManagerJobAndSourceIDs(t *testing.T) {
- mgr := NewManager(WithAPI(DummyAPI{
- registerSource: func(context.Context, string, sourcespb.SourceType) (int64, error) {
- return 1337, nil
- },
- getJobID: func(context.Context, int64) (int64, error) {
- return 9001, nil
- },
- }))
- var (
- initializedJobID int64
- initializedSourceID int64
- )
- handle, err := mgr.Enroll(context.Background(), "dummy", 1337,
- func(ctx context.Context, jobID, sourceID int64) (Source, error) {
- initializedJobID = jobID
- initializedSourceID = sourceID
- return nil, fmt.Errorf("ignore")
- })
- assert.NoError(t, err)
-
- ref, _ := mgr.Run(context.Background(), handle)
- assert.Equal(t, int64(1337), initializedSourceID)
- assert.Equal(t, int64(1337), ref.SourceID)
- assert.Equal(t, int64(9001), initializedJobID)
- assert.Equal(t, int64(9001), ref.JobID)
- assert.Equal(t, "dummy", ref.SourceName)
-}
-
// Chunk method that has a custom callback for the Chunks method.
type callbackChunker struct {
cb func(context.Context, chan *Chunk) error
@@ -304,7 +271,7 @@ func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) e
func TestSourceManagerCancelRun(t *testing.T) {
mgr := NewManager(WithBufferedOutput(8))
var returnedErr error
- handle, err := enrollDummy(mgr, callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
+ source, err := buildDummy(callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
// The context passed to Chunks should get cancelled when ref.CancelRun() is called.
<-ctx.Done()
returnedErr = fmt.Errorf("oh no: %w", ctx.Err())
@@ -312,7 +279,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
}})
assert.NoError(t, err)
- ref, err := mgr.ScheduleRun(context.Background(), handle)
+ ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
cancelErr := fmt.Errorf("abort! abort!")
@@ -326,7 +293,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
func TestSourceManagerAvailableCapacity(t *testing.T) {
mgr := NewManager(WithConcurrentSources(1337))
start, end := make(chan struct{}), make(chan struct{})
- handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
+ source, err := buildDummy(callbackChunker{func(context.Context, chan *Chunk) error {
start <- struct{}{} // Send start signal.
<-end // Wait for end signal.
return nil
@@ -334,7 +301,7 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1337, mgr.AvailableCapacity())
- ref, err := mgr.ScheduleRun(context.Background(), handle)
+ ref, err := mgr.Run(context.Background(), "dummy", source)
assert.NoError(t, err)
<-start // Wait for start signal.