Skip to content

Find, verify, and analyze leaked credentials

[SCAN-162] Add Err() to JobProgress and JobProgressRef (#5346)

Dipto Chaudhurica9d3b3ea0fa
diff --git a/pkg/sources/job_progress.go b/pkg/sources/job_progress.go
index e3965e4e6..a175987bf 100644
--- a/pkg/sources/job_progress.go
+++ b/pkg/sources/job_progress.go
@@ -39,6 +39,13 @@ type JobProgressHook interface {
Finish(JobProgressRef)
}
+var (
+ // ErrNoJob is returned when there is no JobProgress associated with the reference.
+ ErrNoJob = errors.New("no job")
+ // ErrJobDone is returned when the job finishes without being cancelled.
+ ErrJobDone = errors.New("job done")
+)
+
// JobProgressRef is a wrapper of a JobProgress for read-only access to its state.
// If the job supports it, the reference can also be used to cancel running via
// CancelRun.
@@ -57,6 +64,16 @@ func (r *JobProgressRef) Snapshot() JobProgressMetrics {
return r.jobProgress.Snapshot()
}
+// Err reports why the job ended, or nil if it is still running. See
+// JobProgress.Err for the possible values. If the ref does not reference a
+// JobProgress, it does not track any job and ErrNoJob is returned.
+func (r *JobProgressRef) Err() error {
+ if r.jobProgress == nil {
+ return ErrNoJob
+ }
+ return r.jobProgress.Err()
+}
+
// Done returns a channel that will block until the job has completed.
func (r *JobProgressRef) Done() <-chan struct{} {
if r.jobProgress == nil {
@@ -75,7 +92,15 @@ func (r *JobProgressRef) CancelRun(cause error) {
if r.jobProgress == nil || r.jobProgress.jobCancel == nil {
return
}
- r.jobProgress.jobCancel(cause)
+ if cause == nil {
+ cause = context.Canceled
+ }
+
+ r.jobProgress.cancelCauseOnce.Do(func() {
+ r.jobProgress.cancelCause = cause
+ })
+ // cancel with the first recorded cause.
+ r.jobProgress.jobCancel(r.jobProgress.cancelCause)
}
// Fatal is a wrapper around error to differentiate non-fatal errors from fatal
@@ -108,11 +133,17 @@ type JobProgress struct {
JobID JobID
SourceID SourceID
SourceName string
- // Tracks whether the job is finished or not.
+ // ctx represents the job's lifetime. It stays open while the job runs and is
+ // closed by Finish, at which point Done unblocks and Err returns the reason
+ // the job ended (the CancelRun cause or ErrJobDone).
ctx context.Context
- cancel context.CancelFunc
- // Requests to cancel the job.
+ cancel context.CancelCauseFunc
+ // jobCancel stops the source's run context. It requests shutdown without
+ // marking the job complete. Finish remains responsible for closing Done.
jobCancel context.CancelCauseFunc
+ // Tracks the cause of the cancellation. First cause is used.
+ cancelCauseOnce sync.Once
+ cancelCause error
// Metrics.
metrics JobProgressMetrics
metricsLock sync.Mutex
@@ -160,7 +191,7 @@ func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
// NewJobProgress creates a new job report for the given source and job ID.
func NewJobProgress(jobID JobID, sourceID SourceID, sourceName string, opts ...func(*JobProgress)) *JobProgress {
- ctx, cancel := context.WithCancel(context.Background())
+ ctx, cancel := context.WithCancelCause(context.Background())
jp := &JobProgress{
JobID: jobID,
SourceID: sourceID,
@@ -210,10 +241,31 @@ func (jp *JobProgress) End(end time.Time) {
jp.executeHooks(func(hook JobProgressHook) { hook.End(jp.Ref(), end) })
}
+
+// finishWithCause finishes the job with the given cause. Just calling Finish
+// would use ErrJobDone. This would be inaccurate in some cases, for example, if
+// a job is rejected by the source manager.
+func (jp *JobProgress) finishWithCause(cause error) {
+ jp.cancelCauseOnce.Do(func() {
+ jp.cancelCause = cause
+ })
+ jp.Finish()
+}
+
func (jp *JobProgress) Finish() {
- jp.cancel()
+ jp.cancelCauseOnce.Do(func() {
+ jp.cancelCause = ErrJobDone
+ })
+ jp.cancel(jp.cancelCause)
jp.executeHooks(func(hook JobProgressHook) { hook.Finish(jp.Ref()) })
}
+
+// Err mirrors context.Err for the job. It returns:
+// - nil: The job is still running. This includes the period after CancelRun
+// is called but before the job finishes.
+// - the cause passed to CancelRun: the job finished after being cancelled.
+// - ErrJobDone: the job finished without being cancelled.
+func (jp *JobProgress) Err() error { return context.Cause(jp.ctx) }
func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() }
func (jp *JobProgress) ReportUnit(unit SourceUnit) {
jp.metricsLock.Lock()
diff --git a/pkg/sources/job_progress_test.go b/pkg/sources/job_progress_test.go
index 20ba06785..c56c85de3 100644
--- a/pkg/sources/job_progress_test.go
+++ b/pkg/sources/job_progress_test.go
@@ -1,6 +1,7 @@
package sources
import (
+ "context"
"errors"
"fmt"
"testing"
@@ -107,6 +108,77 @@ func TestJobProgressHook(t *testing.T) {
jp.Finish()
}
+func TestJobProgressErr(t *testing.T) {
+ t.Run("ref with no job", func(t *testing.T) {
+ ref := JobProgressRef{}
+ assert.ErrorIs(t, ref.Err(), ErrNoJob)
+ assert.NotErrorIs(t, ref.Err(), ErrJobDone)
+ })
+
+ // function to create a cancellable job
+ newCancellableJob := func() (context.Context, *JobProgress, JobProgressRef) {
+ runCtx, cancel := context.WithCancelCause(context.Background())
+ jp := NewJobProgress(123, 456, "source name", WithCancel(cancel))
+ return runCtx, jp, jp.Ref()
+ }
+ t.Run("finished without cancel", func(t *testing.T) {
+ _, jp, ref := newCancellableJob()
+ // Still running: Err is nil on both the job and the ref.
+ assert.NoError(t, jp.Err())
+ assert.NoError(t, ref.Err())
+ jp.Finish()
+ <-ref.Done()
+ assert.ErrorIs(t, jp.Err(), ErrJobDone)
+ assert.ErrorIs(t, ref.Err(), ErrJobDone)
+ })
+ t.Run("cancelled via CancelRun", func(t *testing.T) {
+ _, jp, ref := newCancellableJob()
+ cause := fmt.Errorf("abort! abort!")
+ ref.CancelRun(cause)
+ // CancelRun only requests a stop. Until Finish runs, Done is still
+ // open and Err must still report the job as running.
+ select {
+ case <-ref.Done():
+ assert.FailNow(t, "job should not be done before Finish")
+ default:
+ }
+ assert.NoError(t, jp.Err())
+ assert.NoError(t, ref.Err())
+ jp.Finish()
+ <-ref.Done()
+ assert.ErrorIs(t, jp.Err(), cause)
+ assert.ErrorIs(t, ref.Err(), cause)
+ })
+ t.Run("cancel after finish does not change Err", func(t *testing.T) {
+ _, jp, ref := newCancellableJob()
+ jp.Finish()
+ ref.CancelRun(fmt.Errorf("too late"))
+ assert.ErrorIs(t, ref.Err(), ErrJobDone)
+ })
+
+ t.Run("nil cause is reported as cancelled", func(t *testing.T) {
+ _, jp, ref := newCancellableJob()
+ ref.CancelRun(nil)
+ jp.Finish()
+ assert.ErrorIs(t, ref.Err(), context.Canceled)
+ assert.NotErrorIs(t, ref.Err(), ErrJobDone)
+ })
+
+ t.Run("first cause wins", func(t *testing.T) {
+ runCtx, jp, ref := newCancellableJob()
+ first := fmt.Errorf("first cause")
+ second := fmt.Errorf("second cause")
+ ref.CancelRun(first)
+ ref.CancelRun(second)
+ // The source's run context sees the first cause, not the second.
+ assert.ErrorIs(t, context.Cause(runCtx), first)
+ jp.Finish()
+ <-ref.Done()
+ assert.ErrorIs(t, ref.Err(), first)
+ assert.NotErrorIs(t, ref.Err(), second)
+ })
+}
+
func TestJobProgressDone(t *testing.T) {
ref := JobProgressRef{}
select {
diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.go
index 283c57327..c9aadf7e4 100644
--- a/pkg/sources/source_manager.go
+++ b/pkg/sources/source_manager.go
@@ -145,8 +145,10 @@ func (s *SourceManager) EnumerateAndScan(ctx context.Context, sourceName string,
progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
if err := sem.Acquire(ctx, 1); err != nil {
// Context cancelled.
- progress.ReportError(Fatal{err})
- return progress.Ref(), Fatal{err}
+ fatalErr := Fatal{err}
+ progress.ReportError(fatalErr)
+ progress.finishWithCause(fatalErr)
+ return progress.Ref(), fatalErr
}
s.wg.Add(1)
go func() {
@@ -239,8 +241,10 @@ func (s *SourceManager) Scan(ctx context.Context, sourceName string, source Sour
progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
if err := s.sem.Acquire(ctx, 1); err != nil {
// Context cancelled.
- progress.ReportError(Fatal{err})
- return progress.Ref(), Fatal{err}
+ fatalErr := Fatal{err}
+ progress.ReportError(fatalErr)
+ progress.finishWithCause(fatalErr)
+ return progress.Ref(), fatalErr
}
s.wg.Add(1)
go func() {
diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.go
index c01b6803e..d0c3575c6 100644
--- a/pkg/sources/source_manager_test.go
+++ b/pkg/sources/source_manager_test.go
@@ -1,11 +1,11 @@
package sources
import (
- "errors"
"fmt"
"sort"
"sync"
"testing"
+ "testing/synctest"
"time"
"github.com/stretchr/testify/assert"
@@ -125,6 +125,7 @@ func TestSourceManagerRun(t *testing.T) {
// The Chunks channel should be empty now.
_, err = tryRead(mgr.Chunks())
assert.Error(t, err)
+ assert.ErrorIs(t, ref.Err(), ErrJobDone)
}
}
@@ -362,8 +363,9 @@ func TestSourceManagerCancelRun(t *testing.T) {
ref.CancelRun(cancelErr)
<-ref.Done()
assert.Error(t, ref.Snapshot().FatalError())
- assert.True(t, errors.Is(ref.Snapshot().FatalError(), returnedErr))
- assert.True(t, errors.Is(ref.Snapshot().FatalErrors(), cancelErr))
+ assert.ErrorIs(t, ref.Snapshot().FatalError(), returnedErr)
+ assert.ErrorIs(t, ref.Snapshot().FatalErrors(), cancelErr)
+ assert.ErrorIs(t, ref.Err(), cancelErr)
}
func TestSourceManagerAvailableCapacity(t *testing.T) {
@@ -542,3 +544,57 @@ func TestUnitHookCloseConcurrentWithFinish(t *testing.T) {
wg.Wait()
<-drained
}
+
+// TestSourceManagerSemaphoreAcquireError tests what happens when acquiring the
+// semaphore tracking concurrent sources fails.
+func TestSourceManagerSemaphoreAcquireError(t *testing.T) {
+ assertFinishedAfterAcquireError := func(t *testing.T, ref JobProgressRef, err error) {
+ t.Helper()
+ assert.IsType(t, Fatal{}, err)
+
+ // The acquire should fail, so the job should be finished
+ select {
+ case <-ref.Done():
+ default:
+ assert.FailNow(t, "job should have been finished")
+ }
+
+ // Check that the ref has the same error as what the manager returns
+ assert.ErrorIs(t, ref.Err(), err)
+ assert.NotErrorIs(t, ref.Err(), ErrJobDone, "job should not be marked as done on failure to acquire semaphore")
+ }
+
+ // Timeout for the context in these tests.
+ // The duration does not matter since synctest uses a fake clock.
+ const timeoutDuration = time.Hour
+
+ t.Run("EnumerateAndScan", func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ mgr := NewManager(WithConcurrentSources(0))
+ source, err := buildDummy(&counterChunker{count: 1})
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration)
+ defer cancel()
+ ref, err := mgr.EnumerateAndScan(ctx, "dummy", source)
+ assertFinishedAfterAcquireError(t, ref, err)
+ // No job was started, so Wait should not block or error
+ assert.NoError(t, mgr.Wait())
+ })
+ })
+
+ t.Run("Scan", func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ mgr := NewManager(WithConcurrentSources(0))
+ source, err := buildDummy(&counterChunker{count: 1})
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration)
+ defer cancel()
+ ref, err := mgr.Scan(ctx, "dummy", source, countChunk(123))
+ assertFinishedAfterAcquireError(t, ref, err)
+ // No job was started, so Wait should not block or error
+ assert.NoError(t, mgr.Wait())
+ })
+ })
+}