Skip to content

Find, verify, and analyze leaked credentials

Use trufflesec git parser (#729)

Bill Richa0d44a39f1ec
diff --git a/go.mod b/go.mod
index 393ffa577..e424d5354 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/TheZeroSlave/zapsentry v1.11.0
github.com/aws/aws-sdk-go v1.44.61
- github.com/bill-rich/disk-buffer-reader v0.1.2
+ github.com/bill-rich/disk-buffer-reader v0.1.3
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
@@ -22,7 +22,6 @@ require (
github.com/fatih/color v1.13.0
github.com/felixge/fgprof v0.9.2
github.com/getsentry/sentry-go v0.13.0
- github.com/gitleaks/go-gitdiff v0.7.4
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.4.2
github.com/go-logr/logr v1.2.3
@@ -46,7 +45,6 @@ require (
github.com/stretchr/testify v1.8.0
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.69.0
- github.com/zricethezav/gitleaks/v8 v8.5.2
go.uber.org/zap v1.22.0
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
@@ -125,5 +123,6 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/grpc v1.47.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index ddb859621..78e7d0211 100644
--- a/go.sum
+++ b/go.sum
@@ -104,8 +104,8 @@ github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRc
github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/bill-rich/disk-buffer-reader v0.1.2 h1:pN9K5JoacTcNWp2SCd3n7mPouSwMP9ouTS66Qa+5IPY=
-github.com/bill-rich/disk-buffer-reader v0.1.2/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
+github.com/bill-rich/disk-buffer-reader v0.1.3 h1:8RUWjBrLB52kwJUjVDdrU0S/7z07DrMshgudfiy6QFs=
+github.com/bill-rich/disk-buffer-reader v0.1.3/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
@@ -415,10 +415,6 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10 h1:QuGZ5bJcQpVz+3sfvKKPDkQwdUueiBg0V+2eMHzkryo=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
-github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2 h1:srCJzbE3b44+ZIPcgJSfvinHCOQlkMwVghtKf23un6o=
-github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
@@ -932,6 +928,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go
new file mode 100644
index 000000000..42e67c3a7
--- /dev/null
+++ b/pkg/gitparse/gitparse.go
@@ -0,0 +1,284 @@
+package gitparse
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ log "github.com/sirupsen/logrus"
+)
+
+// DateFormat is the standard date format for git.
+const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
+
+// Commit contains commit header info and diffs.
+type Commit struct {
+ Hash string
+ Author string
+ Date time.Time
+ Message strings.Builder
+ Diffs []Diff
+}
+
+// Diff contains the info about a file diff in a commit.
+type Diff struct {
+ PathB string
+ LineStart int
+ Content bytes.Buffer
+ IsBinary bool
+}
+
+// RepoPath parses the output of the `git log` command for the `source` path.
+func RepoPath(source string, head string) (chan Commit, error) {
+ commitChan := make(chan Commit)
+
+ args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}
+ if head != "" {
+ args = append(args, head)
+ } else {
+ args = append(args, "--all")
+ }
+
+ cmd := exec.Command("git", args...)
+
+ absPath, err := filepath.Abs(source)
+ if err == nil {
+ cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
+ }
+
+ stdOut, err := cmd.StdoutPipe()
+ if err != nil {
+ return commitChan, err
+ }
+ stdErr, err := cmd.StderrPipe()
+ if err != nil {
+ return commitChan, err
+ }
+
+ err = cmd.Start()
+ if err != nil {
+ return commitChan, err
+ }
+
+ outReader := bufio.NewReader(stdOut)
+ var currentCommit *Commit
+ var currentDiff *Diff
+
+ go func() {
+ scanner := bufio.NewScanner(stdErr)
+ for scanner.Scan() {
+ log.Debug(scanner.Text())
+ }
+ }()
+
+ go func() {
+ for {
+ line, err := outReader.ReadBytes([]byte("\n")[0])
+ if err != nil && len(line) == 0 {
+ break
+ }
+ switch {
+ case isCommitLine(line):
+ // If there is a currentDiff, add it to currentCommit.
+ if currentDiff != nil && currentDiff.Content.Len() > 0 {
+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+ }
+ // If there is a currentCommit, send it to the channel.
+ if currentCommit != nil {
+ commitChan <- *currentCommit
+ }
+ // Create a new currentDiff and currentCommit
+ currentDiff = &Diff{}
+ currentCommit = &Commit{
+ Message: strings.Builder{},
+ }
+ // Check that the commit line contains a hash and set it.
+ if len(line) >= 47 {
+ currentCommit.Hash = string(line[7:47])
+ }
+ case isAuthorLine(line):
+ currentCommit.Author = string(line[8:])
+ case isDateLine(line):
+ date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
+ if err != nil {
+ log.WithError(err).Debug("Could not parse date from git stream.")
+ }
+ currentCommit.Date = date
+ case isDiffLine(line):
+ // This should never be nil, but check in case the stdin stream is messed up.
+ if currentDiff != nil && currentDiff.Content.Len() > 0 {
+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+ }
+ currentDiff = &Diff{}
+ case isModeLine(line):
+ // NoOp
+ case isIndexLine(line):
+ // NoOp
+ case isPlusFileLine(line):
+ currentDiff.PathB = strings.TrimRight(string(line[6:]), "\n")
+ case isMinusFileLine(line):
+ // NoOp
+ case isPlusDiffLine(line):
+ currentDiff.Content.Write(line[1:])
+ case isMinusDiffLine(line):
+ // NoOp. We only care about additions.
+ case isMessageLine(line):
+ currentCommit.Message.Write(line[4:])
+ case isBinaryLine(line):
+ currentDiff.IsBinary = true
+ currentDiff.PathB = pathFromBinaryLine(line)
+ case isLineNumberDiffLine(line):
+ if currentDiff != nil && currentDiff.Content.Len() > 0 {
+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+ }
+ newDiff := &Diff{
+ PathB: currentDiff.PathB,
+ }
+
+ currentDiff = newDiff
+
+ words := bytes.Split(line, []byte(" "))
+ if len(words) >= 3 {
+ startSlice := bytes.Split(words[2], []byte(","))
+ lineStart, err := strconv.Atoi(string(startSlice[0]))
+ if err == nil {
+ currentDiff.LineStart = lineStart
+ }
+ }
+
+ }
+
+ }
+ if currentDiff != nil && currentDiff.Content.Len() > 0 {
+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+ }
+ if currentCommit != nil {
+ commitChan <- *currentCommit
+ }
+ if err := cmd.Wait(); err != nil {
+ log.WithError(err).Debugf("Error waiting for git command to complete.")
+ }
+ close(commitChan)
+ }()
+ return commitChan, nil
+}
+
+// Date: Tue Aug 10 15:20:40 2021 +0100
+func isDateLine(line []byte) bool {
+ if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
+ return true
+ }
+ return false
+}
+
+// Author: Bill Rich <[email protected]>
+func isAuthorLine(line []byte) bool {
+ if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
+ return true
+ }
+ return false
+}
+
+// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05
+func isCommitLine(line []byte) bool {
+ if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
+ return true
+ }
+ return false
+}
+
+// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go
+func isDiffLine(line []byte) bool {
+ if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
+ return true
+ }
+ return false
+}
+
+// index 1ed6fbee1..aea1e643a 100644
+func isIndexLine(line []byte) bool {
+ if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
+ return true
+ }
+ return false
+}
+
+// new file mode 100644
+func isModeLine(line []byte) bool {
+ if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {
+ return true
+ }
+ return false
+}
+
+// --- a/internal/addrs/move_endpoint_module.go
+func isMinusFileLine(line []byte) bool {
+ if len(line) > 3 && bytes.Equal(line[:3], []byte("---")) {
+ return true
+ }
+ return false
+}
+
+// +++ b/internal/addrs/move_endpoint_module.go
+func isPlusFileLine(line []byte) bool {
+ if len(line) > 3 && bytes.Equal(line[:3], []byte("+++")) {
+ return true
+ }
+ return false
+}
+
+// +fmt.Println("ok")
+func isPlusDiffLine(line []byte) bool {
+ if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {
+ return true
+ }
+ return false
+}
+
+// -fmt.Println("ok")
+func isMinusDiffLine(line []byte) bool {
+ if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {
+ return true
+ }
+ return false
+}
+
+// Line that starts with 4 spaces
+func isMessageLine(line []byte) bool {
+ if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {
+ return true
+ }
+ return false
+}
+
+// Binary files /dev/null and b/plugin.sig differ
+func isBinaryLine(line []byte) bool {
+ if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
+ return true
+ }
+ return false
+}
+
+// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)
+func isLineNumberDiffLine(line []byte) bool {
+ if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {
+ return true
+ }
+ return false
+}
+
+// Get the b/ file path.
+func pathFromBinaryLine(line []byte) string {
+ sbytes := bytes.Split(line, []byte(" and "))
+ if len(sbytes) != 2 {
+ log.Debugf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
+ return ""
+ }
+ bRaw := sbytes[1]
+ return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
+}
diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.go
new file mode 100644
index 000000000..ab81d45e1
--- /dev/null
+++ b/pkg/gitparse/gitparse_test.go
@@ -0,0 +1,79 @@
+package gitparse
+
+import (
+ "testing"
+)
+
+type testCase struct {
+ pass []byte
+ fail []byte
+ function func([]byte) bool
+}
+
+func TestIsIndexLine(t *testing.T) {
+ tests := map[string]testCase{
+ "indexLine": {
+ pass: []byte("index 1ed6fbee1..aea1e643a 100644"),
+ fail: []byte("notcorrect"),
+ function: isIndexLine,
+ },
+ "modeLine": {
+ pass: []byte("new file mode 100644"),
+ fail: []byte("notcorrect"),
+ function: isModeLine,
+ },
+ "minusFileLine": {
+ pass: []byte("--- a/internal/addrs/move_endpoint_module.go"),
+ fail: []byte("notcorrect"),
+ function: isMinusFileLine,
+ },
+ "plusFileLine": {
+ pass: []byte("+++ b/internal/addrs/move_endpoint_module.go"),
+ fail: []byte("notcorrect"),
+ function: isPlusFileLine,
+ },
+ "plusDiffLine": {
+ pass: []byte("+fmt.Println"),
+ fail: []byte("notcorrect"),
+ function: isPlusDiffLine,
+ },
+ "minusDiffLine": {
+ pass: []byte("-fmt.Println"),
+ fail: []byte("notcorrect"),
+ function: isMinusDiffLine,
+ },
+ "messageLine": {
+ pass: []byte(" committed"),
+ fail: []byte("notcorrect"),
+ function: isMessageLine,
+ },
+ "binaryLine": {
+ pass: []byte("Binary files /dev/null and b/plugin.sig differ"),
+ fail: []byte("notcorrect"),
+ function: isBinaryLine,
+ },
+ "lineNumberLine": {
+ pass: []byte("@@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)"),
+ fail: []byte("notcorrect"),
+ function: isLineNumberDiffLine,
+ },
+ }
+
+ for name, test := range tests {
+ if !test.function(test.pass) {
+ t.Errorf("%s: Parser did not recognize correct line.", name)
+ }
+ if test.function(test.fail) {
+ t.Errorf("%s: Parser matched an incorrect line.", name)
+ }
+ }
+}
+
+func TestBinaryPathParse(t *testing.T) {
+ filename := pathFromBinaryLine([]byte("Binary files /dev/null and b/plugin.sig differ"))
+ expected := "plugin.sig"
+ if filename != expected {
+ t.Errorf("Expected: %s, Got: %s", expected, filename)
+ }
+
+}
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 90348dea8..e796855d0 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -15,20 +15,19 @@ import (
"time"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
- "github.com/gitleaks/go-gitdiff/gitdiff"
"github.com/go-errors/errors"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/google/go-github/v42/github"
"github.com/rs/zerolog"
log "github.com/sirupsen/logrus"
- glgo "github.com/zricethezav/gitleaks/v8/detect/git"
"golang.org/x/oauth2"
"golang.org/x/sync/semaphore"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/gitparse"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -282,23 +281,12 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
zerolog.SetGlobalLevel(zerolog.Disabled)
}
- // Errors returned on errChan aren't blocking, so just ignore them.
- errChan := make(chan error)
- var gitLogArgs []string
- if scanOptions.HeadHash != "" {
- gitLogArgs = append(gitLogArgs, scanOptions.HeadHash)
- }
- logOpts := glgo.LogOpts{
- Args: gitLogArgs,
- DisableSafeDir: true,
- }
- fileChan, err := glgo.GitLog(path, logOpts, errChan)
+ commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash)
if err != nil {
- return errors.WrapPrefix(err, "could not open repo path", 0)
+ return err
}
- // parser can return nil chan and nil error
- if fileChan == nil {
- return errors.New("nothing to scan")
+ if commitChan == nil {
+ return nil
}
// get the URL metadata for reporting (may be empty)
@@ -306,75 +294,62 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
var depth int64
var reachedBase = false
- for file := range fileChan {
- if file == nil || file.PatchHeader == nil {
- log.Debugf("file missing patch header, skipping")
- continue
- }
- log.WithField("commit", file.PatchHeader.SHA).WithField("file", file.NewName).Trace("Scanning file from git")
- if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
- log.Debugf("reached max depth")
- break
- }
- depth++
- if reachedBase && file.PatchHeader.SHA != scanOptions.BaseHash {
- break
- }
- if len(scanOptions.BaseHash) > 0 {
- if file.PatchHeader.SHA == scanOptions.BaseHash {
- log.Debugf("Reached base commit. Finishing scanning files.")
- reachedBase = true
+ log.Debugf("Scanning repo")
+ for commit := range commitChan {
+ for _, diff := range commit.Diffs {
+ log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git")
+ if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
+ log.Debugf("reached max depth")
+ break
}
- }
- if !scanOptions.Filter.Pass(file.NewName) {
- continue
- }
-
- fileName := file.NewName
- if fileName == "" {
- continue
- }
- var email, hash, when string
- if file.PatchHeader != nil {
- if file.PatchHeader.Author != nil {
- email = file.PatchHeader.Author.Email
+ depth++
+ if reachedBase && commit.Hash != scanOptions.BaseHash {
+ break
}
- hash = file.PatchHeader.SHA
- when = file.PatchHeader.AuthorDate.String()
- }
-
- // Handle binary files by reading the entire file rather than using the diff.
- if file.IsBinary {
- commitHash := plumbing.NewHash(hash)
- metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)
- chunkSkel := &sources.Chunk{
- SourceName: s.sourceName,
- SourceID: s.sourceID,
- SourceType: s.sourceType,
- SourceMetadata: metadata,
- Verify: s.verify,
+ if len(scanOptions.BaseHash) > 0 {
+ if commit.Hash == scanOptions.BaseHash {
+ log.Debugf("Reached base commit. Finishing scanning files.")
+ reachedBase = true
+ }
}
- if err = handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
- log.WithError(err).Error("Error handling binary file")
+
+ if !scanOptions.Filter.Pass(diff.PathB) {
+ continue
}
- continue
- }
- for _, frag := range file.TextFragments {
- var sb strings.Builder
- newLineNumber := frag.NewPosition
- for _, line := range frag.Lines {
- if line.Op == gitdiff.OpAdd {
- sb.WriteString(line.Line)
+ fileName := diff.PathB
+ if fileName == "" {
+ continue
+ }
+ var email, hash, when string
+ email = commit.Author
+ hash = commit.Hash
+ when = commit.Date.String()
+
+ // Handle binary files by reading the entire file rather than using the diff.
+ if diff.IsBinary {
+ commitHash := plumbing.NewHash(hash)
+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)
+ chunkSkel := &sources.Chunk{
+ SourceName: s.sourceName,
+ SourceID: s.sourceID,
+ SourceType: s.sourceType,
+ SourceMetadata: metadata,
+ Verify: s.verify,
}
+ if err := handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {
+ log.WithError(err).WithField("file", fileName).Debug("Error handling binary file")
+ }
+ continue
}
- metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, newLineNumber)
+
+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))
chunksChan <- &sources.Chunk{
SourceName: s.sourceName,
SourceID: s.sourceID,
SourceType: s.sourceType,
SourceMetadata: metadata,
- Data: []byte(sb.String()),
+ Data: diff.Content.Bytes(),
Verify: s.verify,
}
}
diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go
index 34ee94fc5..1169e209e 100644
--- a/pkg/sources/git/git_test.go
+++ b/pkg/sources/git/git_test.go
@@ -232,9 +232,9 @@ func TestSource_Chunks_Integration(t *testing.T) {
"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},
// Normally we might expect to see this commit, and we may in the future.
// But at the moment we're ignoring any commit unless it contains at least one non-space character.
- "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!")},
- "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that"), Multi: true},
- "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye"), Multi: true},
+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!\n")},
+ "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that\n"), Multi: true},
+ "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye\n"), Multi: true},
"84e9c75e388ae3e866e121087ea2dd45a71068f2-aws": {B: []byte("this is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n"), Multi: true},
},
},
@@ -262,11 +262,10 @@ func TestSource_Chunks_Integration(t *testing.T) {
}()
for chunk := range chunksCh {
-
key := ""
switch meta := chunk.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
- key = meta.Git.Commit + "-" + meta.Git.File
+ key = strings.TrimRight(meta.Git.Commit+"-"+meta.Git.File, "\n")
}
if expectedData, exists := tt.expectedChunkData[key]; !exists {