Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Support scanning binary files in git sources (#684)
Bill Rich4a93e49eea53
diff --git a/go.mod b/go.modindex fa5aff4be..206ba9673 100644--- a/go.mod+++ b/go.mod@@ -4,7 +4,7 @@ go 1.18replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5-replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom9+replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2diff --git a/go.sum b/go.sumindex 3cb81a077..1eb0a831b 100644--- a/go.sum+++ b/go.sum@@ -400,8 +400,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPgithub.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom9 h1:OvS9aj6Fasot5FaTpSyCV4WNq/8SMov9/bNUMoZFwEI=-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom9/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10 h1:QuGZ5bJcQpVz+3sfvKKPDkQwdUueiBg0V+2eMHzkryo=+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2 h1:srCJzbE3b44+ZIPcgJSfvinHCOQlkMwVghtKf23un6o=github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 2f5288b93..90348dea8 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -14,6 +14,7 @@ import ("strings""time"+ diskbufferreader "github.com/bill-rich/disk-buffer-reader""github.com/gitleaks/go-gitdiff/gitdiff""github.com/go-errors/errors""github.com/go-git/go-git/v5"@@ -27,6 +28,8 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"@@ -340,6 +343,23 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOpwhen = file.PatchHeader.AuthorDate.String()}+ // Handle binary files by reading the entire file rather than using the diff.+ if file.IsBinary {+ commitHash := plumbing.NewHash(hash)+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)+ chunkSkel := &sources.Chunk{+ SourceName: s.sourceName,+ SourceID: s.sourceID,+ SourceType: s.sourceType,+ SourceMetadata: metadata,+ Verify: s.verify,+ }+ if err = handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {+ log.WithError(err).Error("Error handling binary file")+ }+ continue+ }+for _, frag := range file.TextFragments {var sb strings.BuildernewLineNumber := frag.NewPosition@@ -615,3 +635,45 @@ func getSafeRemoteURL(repo *git.Repository, preferred string) string {}return safeURL}++func handleBinary(repo *git.Repository, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error {+ log.WithField("path", path).Trace("Binary file found in repository.")+ commit, err := repo.CommitObject(commitHash)+ if err != nil {+ return err+ }++ file, err := commit.File(path)+ if err != nil {+ return err+ }++ fileReader, err := file.Reader()+ if err != nil {+ return err+ }+ defer fileReader.Close()++ reader, err := diskbufferreader.New(fileReader)+ if err != nil {+ return err+ }++ if handlers.HandleFile(reader, chunkSkel, chunksChan) {+ return nil+ }++ log.WithField("path", path).Trace("Binary file is not recognized by file handlers. Chunking raw.")+ if err := reader.Reset(); err != nil {+ return err+ }+ reader.Stop()++ for chunkData := range common.ChunkReader(reader) {+ chunk := *chunkSkel+ chunk.Data = chunkData+ chunksChan <- &chunk+ }++ return nil+}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 6bdf4d13a..34ee94fc5 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -229,7 +229,7 @@ func TestSource_Chunks_Integration(t *testing.T) {"2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte(" \n")},"e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("oops \n")},"735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("oops\n")},- //"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},+ "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},// Normally we might expect to see this commit, and we may in the future.// But at the moment we're ignoring any commit unless it contains at least one non-space character."27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!")},