Skip to content

Find, verify, and analyze leaked credentials

Define SourceUnit chunking interface (#1484)

Miccah4e774d1f01ed
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 2774e2a2f..41673a4e2 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -46,6 +46,7 @@ type Source struct {
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumerator = (*Source)(nil)
+var _ sources.SourceUnitChunker = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -229,3 +230,44 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
}
return nil
}
+
+// ChunkUnit implements SourceUnitChunker interface.
+func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
+ path := unit.SourceUnitID()
+ logger := ctx.Logger().WithValues("path", path)
+
+ cleanPath := filepath.Clean(path)
+ fileInfo, err := os.Stat(cleanPath)
+ if err != nil {
+ return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
+ }
+
+ ch := make(chan *sources.Chunk)
+ var scanErr error
+ go func() {
+ defer close(ch)
+ if fileInfo.IsDir() {
+ // TODO: Finer grain error tracking of individual chunks.
+ scanErr = s.scanDir(ctx, cleanPath, ch)
+ } else {
+ // TODO: Finer grain error tracking of individual
+ // chunks (in the case of archives).
+ scanErr = s.scanFile(ctx, cleanPath, ch)
+ }
+ }()
+
+ for chunk := range ch {
+ if chunk == nil {
+ continue
+ }
+ if err := reporter.ChunkOk(ctx, *chunk); err != nil {
+ return err
+ }
+ }
+
+ if scanErr != nil && scanErr != io.EOF {
+ logger.Info("error scanning filesystem", "error", scanErr)
+ return reporter.ChunkErr(ctx, scanErr)
+ }
+ return nil
+}
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 088799ff8..9d8c85eda 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -71,6 +71,25 @@ type UnitReporter interface {
UnitErr(ctx context.Context, err error) error
}
+// SourceUnitChunker defines an optional interface a Source can implement to
+// support chunking a single SourceUnit.
+type SourceUnitChunker interface {
+ // ChunkUnit creates 0 or more chunks from a unit, reporting them or
+ // any errors to the ChunkReporter. An error should only be returned
+ // from this method in the case of context cancellation, fatal source
+ // errors, or errors returned by the reporter. All other errors related
+ // to unit chunking are tracked by the ChunkReporter.
+ ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
+}
+
+// ChunkReporter defines the interface a source will use to report whether a
+// chunk was found during unit chunking. Either method may be called any number
+// of times. Implementors of this interface should allow for concurrent calls.
+type ChunkReporter interface {
+ ChunkOk(ctx context.Context, chunk Chunk) error
+ ChunkErr(ctx context.Context, err error) error
+}
+
// SourceUnit is an object that represents a Source's unit of work. This is
// used as the output of enumeration, progress reporting, and job distribution.
type SourceUnit interface {