Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improve the `json-enumerator` source (#5265)
Brad Larsenf5370819b555
diff --git a/go.mod b/go.modindex fee1845ea..5b6ba4755 100644--- a/go.mod+++ b/go.mod@@ -64,6 +64,7 @@ require (github.com/joho/godotenv v1.5.1github.com/jpillora/overseer v1.1.6github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213+ github.com/klauspost/compress v1.18.6github.com/klauspost/pgzip v1.2.6github.com/kylelemons/godebug v1.1.0github.com/lestrrat-go/jwx/v3 v3.0.12@@ -232,7 +233,6 @@ require (github.com/jpillora/s3 v1.1.4 // indirectgithub.com/kevinburke/ssh_config v1.2.0 // indirectgithub.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect- github.com/klauspost/compress v1.18.6 // indirectgithub.com/klauspost/cpuid/v2 v2.3.0 // indirectgithub.com/lestrrat-go/blackmagic v1.0.4 // indirectgithub.com/lestrrat-go/httpcc v1.0.1 // indirectdiff --git a/pkg/config/config.go b/pkg/config/config.goindex 47fe3868c..ef6b2d5a8 100644--- a/pkg/config/config.go+++ b/pkg/config/config.go@@ -17,6 +17,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources/github""github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab""github.com/trufflesecurity/trufflehog/v3/pkg/sources/jenkins"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/json_enumerator""github.com/trufflesecurity/trufflehog/v3/pkg/sources/postman""github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3")@@ -97,6 +98,8 @@ func instantiateSourceFromType(sourceType string) (sources.Source, error) {source = new(filesystem.Source)case sourcespb.SourceType_SOURCE_TYPE_JENKINS.String():source = new(jenkins.Source)+ case sourcespb.SourceType_SOURCE_TYPE_JSON_ENUMERATOR.String():+ source = new(json_enumerator.Source)case sourcespb.SourceType_SOURCE_TYPE_GCS.String():source = new(gcs.Source)case sourcespb.SourceType_SOURCE_TYPE_GCS_UNAUTHED.String():diff --git a/pkg/sources/json_enumerator/json_enumerator.go b/pkg/sources/json_enumerator/json_enumerator.goindex 991d9fb52..54e98dd51 100644--- a/pkg/sources/json_enumerator/json_enumerator.go+++ b/pkg/sources/json_enumerator/json_enumerator.go@@ -7,8 +7,10 @@ import ("fmt""io""os"+ "strings""unicode/utf8"+ "github.com/klauspost/compress/zstd""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -35,6 +37,7 @@ type Source struct {// Ensure the Source satisfies the interfaces at compile timevar _ sources.Source = (*Source)(nil)var _ sources.SourceUnitUnmarshaller = (*Source)(nil)+var _ sources.SourceUnitEnumChunker = (*Source)(nil)func (s *Source) Type() sourcespb.SourceType { return SourceType }func (s *Source) SourceID() sources.SourceID { return s.sourceId }@@ -73,7 +76,7 @@ func (s *Source) Chunks(return nil}s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")- if err := s.chunkJSONEnumerator(ctx, path, chunksChan); err != nil {+ if err := s.chunkJSONEnumerator(ctx, path, sources.ChanReporter{Ch: chunksChan}); err != nil {ctx.Logger().Error(err, "error scanning JSON enumerator", "path", path)}}@@ -81,6 +84,34 @@ func (s *Source) Chunks(return nil}+// Enumerate implements the SourceUnitEnumerator interface, reporting each+// configured path as its own unit so paths can be chunked concurrently.+func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {+ for _, path := range s.paths {+ f, err := os.Open(path)+ if err != nil {+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }+ } else {+ _ = f.Close()+ if err := reporter.UnitOk(ctx, sources.CommonSourceUnit{ID: path}); err != nil {+ return err+ }+ }+ }+ return nil+}++// ChunkUnit implements the SourceUnitChunker interface.+func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {+ path, _ := unit.SourceUnitID()+ if err := s.chunkJSONEnumerator(ctx, path, reporter); err != nil {+ return reporter.ChunkErr(ctx, err)+ }+ return nil+}+type jsonEntry struct {Metadata json.RawMessageData []byte@@ -142,13 +173,11 @@ func (e *jsonEntry) UnmarshalJSON(data []byte) error {func (s *Source) chunkJSONEnumeratorReader(ctx context.Context,input io.Reader,- chunksChan chan *sources.Chunk,+ reporter sources.ChunkReporter,) error {decoder := json.NewDecoder(input)var entry jsonEntry- reporter := sources.ChanReporter{Ch: chunksChan}-for {if err := decoder.Decode(&entry); err != nil {if errors.Is(err, io.EOF) {@@ -190,7 +219,7 @@ func (s *Source) chunkJSONEnumeratorReader(func (s *Source) chunkJSONEnumerator(ctx context.Context,path string,- chunksChan chan *sources.Chunk,+ reporter sources.ChunkReporter,) error {ctx.Logger().V(3).Info("chunking JSON enumerator", "path", path)@@ -200,5 +229,15 @@ func (s *Source) chunkJSONEnumerator(}defer func() { _ = enumeratorFile.Close() }()- return s.chunkJSONEnumeratorReader(ctx, enumeratorFile, chunksChan)+ if strings.HasSuffix(path, ".zst") || strings.HasSuffix(path, ".zstd") {+ r, err := zstd.NewReader(enumeratorFile)+ if err != nil {+ return fmt.Errorf("unable to open zstd reader: %w", err)+ }+ defer r.Close()+ return s.chunkJSONEnumeratorReader(ctx, r, reporter)+ } else {+ return s.chunkJSONEnumeratorReader(ctx, enumeratorFile, reporter)+ }+}diff --git a/pkg/sources/json_enumerator/json_enumerator_test.go b/pkg/sources/json_enumerator/json_enumerator_test.goindex fd215adf2..11d8f6ad9 100644--- a/pkg/sources/json_enumerator/json_enumerator_test.go+++ b/pkg/sources/json_enumerator/json_enumerator_test.go@@ -105,7 +105,7 @@ func TestScanEnumerator(t *testing.T) {defer close(chunksChan)ctx := context.WithLogger(t.Context(), logr.Discard())source := &Source{}- workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, chunksChan)+ workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, sources.ChanReporter{Ch: chunksChan})}()enc := json.NewEncoder(writeJSON)@@ -190,7 +190,7 @@ func TestScanEnumeratorAPKContentRouting(t *testing.T) {defer close(chunksChan)ctx := context.WithLogger(t.Context(), logr.Discard())source := &Source{}- workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, chunksChan)+ workerError = source.chunkJSONEnumeratorReader(ctx, readJSON, sources.ChanReporter{Ch: chunksChan})}()// No filename is set on the record; routing is content-based only.