Skip to content

Find, verify, and analyze leaked credentials

Define SourceUnit enumeration interface (#1428)

Miccah5c0ffda61823
diff --git a/pkg/common/context.go b/pkg/common/context.go
index d473532a8..5a6d005b2 100644
--- a/pkg/common/context.go
+++ b/pkg/common/context.go
@@ -10,3 +10,15 @@ func IsDone(ctx context.Context) bool {
return false
}
}
+
+// CancellableWrite blocks on writing the item to the channel but can be
+// cancelled by the context. If both the context is cancelled and the channel
+// write would succeed, either operation will be performed randomly.
+func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error {
+ select {
+ case ch <- item:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index eb6dfcf33..efd4d8b89 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -45,6 +45,7 @@ type Source struct {
// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
+var _ sources.SourceUnitEnumerator = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -215,3 +216,16 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
}
return nil
}
+
+// Enumerate implements SourceUnitEnumerator interface. This implementation simply
+// passes the configured paths as the source unit, whether it be a single
+// filepath or a directory.
+func (s *Source) Enumerate(ctx context.Context, units chan<- sources.EnumerationResult) error {
+ for _, path := range s.paths {
+ item := sources.CommonEnumerationOk(path)
+ if err := common.CancellableWrite(ctx, units, item); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 3d02acf8a..8f14b025d 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -50,6 +50,26 @@ type SourceUnitUnmarshaller interface {
UnmarshalSourceUnit(data []byte) (SourceUnit, error)
}
+// SourceUnitEnumerator defines an optional interface a Source can implement to
+// support enumerating an initialized Source into SourceUnits.
+type SourceUnitEnumerator interface {
+ // Enumerate enumerates the initialized Source, outputting units. This
+ // method is synchronous but can be called in a goroutine to support
+ // concurrent enumeration and chunking. An error should only be
+ // returned from this method in the case of context cancellation. All
+ // other errors related to unit enumeration are tracked in the
+ // EnumerationResult.
+ Enumerate(ctx context.Context, units chan<- EnumerationResult) error
+}
+
+// EnumerationResult is the result of an enumeration, containing the unit and
+// error if any. Unit and Error are mutually exclusive (only one will be
+// non-nil).
+type EnumerationResult struct {
+ Unit SourceUnit
+ Error error
+}
+
// SourceUnit is an object that represents a Source's unit of work. This is
// used as the output of enumeration, progress reporting, and job distribution.
type SourceUnit interface {
@@ -223,3 +243,16 @@ func (p *Progress) GetProgress() *Progress {
defer p.mut.Unlock()
return p
}
+
+// CommonEnumerationOk is a helper function to construct an EnumerationResult
+// using a CommonSourceUnit.
+func CommonEnumerationOk(id string) EnumerationResult {
+ unit := CommonSourceUnit{ID: id}
+ return EnumerationResult{Unit: unit}
+}
+
+// EnumerationErr is a helper function to construct an EnumerationResult from
+// an error.
+func EnumerationErr(err error) EnumerationResult {
+ return EnumerationResult{Error: err}
+}