Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Define SourceUnit chunking interface (#1484)
Miccah4e774d1f01ed
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 2774e2a2f..41673a4e2 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -46,6 +46,7 @@ type Source struct {var _ sources.Source = (*Source)(nil)var _ sources.SourceUnitUnmarshaller = (*Source)(nil)var _ sources.SourceUnitEnumerator = (*Source)(nil)+var _ sources.SourceUnitChunker = (*Source)(nil)// Type returns the type of source.// It is used for matching source types in configuration and job input.@@ -229,3 +230,44 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e}return nil}++// ChunkUnit implements SourceUnitChunker interface.+func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {+ path := unit.SourceUnitID()+ logger := ctx.Logger().WithValues("path", path)++ cleanPath := filepath.Clean(path)+ fileInfo, err := os.Stat(cleanPath)+ if err != nil {+ return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))+ }++ ch := make(chan *sources.Chunk)+ var scanErr error+ go func() {+ defer close(ch)+ if fileInfo.IsDir() {+ // TODO: Finer grain error tracking of individual chunks.+ scanErr = s.scanDir(ctx, cleanPath, ch)+ } else {+ // TODO: Finer grain error tracking of individual+ // chunks (in the case of archives).+ scanErr = s.scanFile(ctx, cleanPath, ch)+ }+ }()++ for chunk := range ch {+ if chunk == nil {+ continue+ }+ if err := reporter.ChunkOk(ctx, *chunk); err != nil {+ return err+ }+ }++ if scanErr != nil && scanErr != io.EOF {+ logger.Info("error scanning filesystem", "error", scanErr)+ return reporter.ChunkErr(ctx, scanErr)+ }+ return nil+}diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex 088799ff8..9d8c85eda 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -71,6 +71,25 @@ type UnitReporter interface {UnitErr(ctx context.Context, err error) error}+// SourceUnitChunker defines an optional interface a Source can implement to+// support chunking a single SourceUnit.+type SourceUnitChunker interface {+ // ChunkUnit creates 0 or more chunks from a unit, reporting them or+ // any errors to the ChunkReporter. An error should only be returned+ // from this method in the case of context cancellation, fatal source+ // errors, or errors returned by the reporter. All other errors related+ // to unit chunking are tracked by the ChunkReporter.+ ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error+}++// ChunkReporter defines the interface a source will use to report whether a+// chunk was found during unit chunking. Either method may be called any number+// of times. Implementors of this interface should allow for concurrent calls.+type ChunkReporter interface {+ ChunkOk(ctx context.Context, chunk Chunk) error+ ChunkErr(ctx context.Context, err error) error+}+// SourceUnit is an object that represents a Source's unit of work. This is// used as the output of enumeration, progress reporting, and job distribution.type SourceUnit interface {