Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Use trufflesec git parser (#729)
Bill Richa0d44a39f1ec
diff --git a/go.mod b/go.modindex 393ffa577..e424d5354 100644--- a/go.mod+++ b/go.mod@@ -13,7 +13,7 @@ require (github.com/Azure/go-autorest/autorest/azure/auth v0.5.11github.com/TheZeroSlave/zapsentry v1.11.0github.com/aws/aws-sdk-go v1.44.61- github.com/bill-rich/disk-buffer-reader v0.1.2+ github.com/bill-rich/disk-buffer-reader v0.1.3github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574cgithub.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fbgithub.com/bradleyfalzon/ghinstallation/v2 v2.1.0@@ -22,7 +22,6 @@ require (github.com/fatih/color v1.13.0github.com/felixge/fgprof v0.9.2github.com/getsentry/sentry-go v0.13.0- github.com/gitleaks/go-gitdiff v0.7.4github.com/go-errors/errors v1.4.2github.com/go-git/go-git/v5 v5.4.2github.com/go-logr/logr v1.2.3@@ -46,7 +45,6 @@ require (github.com/stretchr/testify v1.8.0github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502github.com/xanzy/go-gitlab v0.69.0- github.com/zricethezav/gitleaks/v8 v8.5.2go.uber.org/zap v1.22.0golang.org/x/crypto v0.0.0-20211215165025-cf75a172585egolang.org/x/net v0.0.0-20220617184016-355a448f1bc9@@ -125,5 +123,6 @@ require (google.golang.org/appengine v1.6.7 // indirectgoogle.golang.org/grpc v1.47.0 // indirectgopkg.in/warnings.v0 v0.1.2 // indirect+ gopkg.in/yaml.v2 v2.4.0 // indirectgopkg.in/yaml.v3 v3.0.1 // indirect)diff --git a/go.sum b/go.sumindex ddb859621..78e7d0211 100644--- a/go.sum+++ b/go.sum@@ -104,8 +104,8 @@ github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRcgithub.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=-github.com/bill-rich/disk-buffer-reader v0.1.2 h1:pN9K5JoacTcNWp2SCd3n7mPouSwMP9ouTS66Qa+5IPY=-github.com/bill-rich/disk-buffer-reader v0.1.2/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=+github.com/bill-rich/disk-buffer-reader v0.1.3 h1:8RUWjBrLB52kwJUjVDdrU0S/7z07DrMshgudfiy6QFs=+github.com/bill-rich/disk-buffer-reader v0.1.3/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=@@ -415,10 +415,6 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPgithub.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10 h1:QuGZ5bJcQpVz+3sfvKKPDkQwdUueiBg0V+2eMHzkryo=-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=-github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2 h1:srCJzbE3b44+ZIPcgJSfvinHCOQlkMwVghtKf23un6o=-github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=@@ -932,6 +928,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.gonew file mode 100644index 000000000..42e67c3a7--- /dev/null+++ b/pkg/gitparse/gitparse.go@@ -0,0 +1,284 @@+package gitparse++import (+ "bufio"+ "bytes"+ "fmt"+ "os/exec"+ "path/filepath"+ "strconv"+ "strings"+ "time"++ log "github.com/sirupsen/logrus"+)++// DateFormat is the standard date format for git.+const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"++// Commit contains commit header info and diffs.+type Commit struct {+ Hash string+ Author string+ Date time.Time+ Message strings.Builder+ Diffs []Diff+}++// Diff contains the info about a file diff in a commit.+type Diff struct {+ PathB string+ LineStart int+ Content bytes.Buffer+ IsBinary bool+}++// RepoPath parses the output of the `git log` command for the `source` path.+func RepoPath(source string, head string) (chan Commit, error) {+ commitChan := make(chan Commit)++ args := []string{"-C", source, "log", "-p", "-U0", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}+ if head != "" {+ args = append(args, head)+ } else {+ args = append(args, "--all")+ }++ cmd := exec.Command("git", args...)++ absPath, err := filepath.Abs(source)+ if err == nil {+ cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))+ }++ stdOut, err := cmd.StdoutPipe()+ if err != nil {+ return commitChan, err+ }+ stdErr, err := cmd.StderrPipe()+ if err != nil {+ return commitChan, err+ }++ err = cmd.Start()+ if err != nil {+ return commitChan, err+ }++ outReader := bufio.NewReader(stdOut)+ var currentCommit *Commit+ var currentDiff *Diff++ go func() {+ scanner := bufio.NewScanner(stdErr)+ for scanner.Scan() {+ log.Debug(scanner.Text())+ }+ }()++ go func() {+ for {+ line, err := outReader.ReadBytes([]byte("\n")[0])+ if err != nil && len(line) == 0 {+ break+ }+ switch {+ case isCommitLine(line):+ // If there is a currentDiff, add it to currentCommit.+ if currentDiff != nil && currentDiff.Content.Len() > 0 {+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)+ }+ // If there is a currentCommit, send it to the channel.+ if currentCommit != nil {+ commitChan <- *currentCommit+ }+ // Create a new currentDiff and currentCommit+ currentDiff = &Diff{}+ currentCommit = &Commit{+ Message: strings.Builder{},+ }+ // Check that the commit line contains a hash and set it.+ if len(line) >= 47 {+ currentCommit.Hash = string(line[7:47])+ }+ case isAuthorLine(line):+ currentCommit.Author = string(line[8:])+ case isDateLine(line):+ date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))+ if err != nil {+ log.WithError(err).Debug("Could not parse date from git stream.")+ }+ currentCommit.Date = date+ case isDiffLine(line):+ // This should never be nil, but check in case the stdin stream is messed up.+ if currentDiff != nil && currentDiff.Content.Len() > 0 {+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)+ }+ currentDiff = &Diff{}+ case isModeLine(line):+ // NoOp+ case isIndexLine(line):+ // NoOp+ case isPlusFileLine(line):+ currentDiff.PathB = strings.TrimRight(string(line[6:]), "\n")+ case isMinusFileLine(line):+ // NoOp+ case isPlusDiffLine(line):+ currentDiff.Content.Write(line[1:])+ case isMinusDiffLine(line):+ // NoOp. We only care about additions.+ case isMessageLine(line):+ currentCommit.Message.Write(line[4:])+ case isBinaryLine(line):+ currentDiff.IsBinary = true+ currentDiff.PathB = pathFromBinaryLine(line)+ case isLineNumberDiffLine(line):+ if currentDiff != nil && currentDiff.Content.Len() > 0 {+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)+ }+ newDiff := &Diff{+ PathB: currentDiff.PathB,+ }++ currentDiff = newDiff++ words := bytes.Split(line, []byte(" "))+ if len(words) >= 3 {+ startSlice := bytes.Split(words[2], []byte(","))+ lineStart, err := strconv.Atoi(string(startSlice[0]))+ if err == nil {+ currentDiff.LineStart = lineStart+ }+ }++ }++ }+ if currentDiff != nil && currentDiff.Content.Len() > 0 {+ currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)+ }+ if currentCommit != nil {+ commitChan <- *currentCommit+ }+ if err := cmd.Wait(); err != nil {+ log.WithError(err).Debugf("Error waiting for git command to complete.")+ }+ close(commitChan)+ }()+ return commitChan, nil+}++// Date: Tue Aug 10 15:20:40 2021 +0100+func isDateLine(line []byte) bool {+ if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {+ return true+ }+ return false+}++// Author: Bill Rich <[email protected]>+func isAuthorLine(line []byte) bool {+ if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {+ return true+ }+ return false+}++// commit 7a95bbf0199e280a0e42dbb1d1a3f56cdd0f6e05+func isCommitLine(line []byte) bool {+ if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {+ return true+ }+ return false+}++// diff --git a/internal/addrs/move_endpoint_module.go b/internal/addrs/move_endpoint_module.go+func isDiffLine(line []byte) bool {+ if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {+ return true+ }+ return false+}++// index 1ed6fbee1..aea1e643a 100644+func isIndexLine(line []byte) bool {+ if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {+ return true+ }+ return false+}++// new file mode 100644+func isModeLine(line []byte) bool {+ if len(line) > 13 && bytes.Equal(line[:13], []byte("new file mode")) {+ return true+ }+ return false+}++// --- a/internal/addrs/move_endpoint_module.go+func isMinusFileLine(line []byte) bool {+ if len(line) > 3 && bytes.Equal(line[:3], []byte("---")) {+ return true+ }+ return false+}++// +++ b/internal/addrs/move_endpoint_module.go+func isPlusFileLine(line []byte) bool {+ if len(line) > 3 && bytes.Equal(line[:3], []byte("+++")) {+ return true+ }+ return false+}++// +fmt.Println("ok")+func isPlusDiffLine(line []byte) bool {+ if len(line) >= 1 && bytes.Equal(line[:1], []byte("+")) {+ return true+ }+ return false+}++// -fmt.Println("ok")+func isMinusDiffLine(line []byte) bool {+ if len(line) >= 1 && bytes.Equal(line[:1], []byte("-")) {+ return true+ }+ return false+}++// Line that starts with 4 spaces+func isMessageLine(line []byte) bool {+ if len(line) > 4 && bytes.Equal(line[:4], []byte(" ")) {+ return true+ }+ return false+}++// Binary files /dev/null and b/plugin.sig differ+func isBinaryLine(line []byte) bool {+ if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {+ return true+ }+ return false+}++// @@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)+func isLineNumberDiffLine(line []byte) bool {+ if len(line) >= 8 && bytes.Equal(line[:2], []byte("@@")) {+ return true+ }+ return false+}++// Get the b/ file path.+func pathFromBinaryLine(line []byte) string {+ sbytes := bytes.Split(line, []byte(" and "))+ if len(sbytes) != 2 {+ log.Debugf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)+ return ""+ }+ bRaw := sbytes[1]+ return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"+}diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.gonew file mode 100644index 000000000..ab81d45e1--- /dev/null+++ b/pkg/gitparse/gitparse_test.go@@ -0,0 +1,79 @@+package gitparse++import (+ "testing"+)++type testCase struct {+ pass []byte+ fail []byte+ function func([]byte) bool+}++func TestIsIndexLine(t *testing.T) {+ tests := map[string]testCase{+ "indexLine": {+ pass: []byte("index 1ed6fbee1..aea1e643a 100644"),+ fail: []byte("notcorrect"),+ function: isIndexLine,+ },+ "modeLine": {+ pass: []byte("new file mode 100644"),+ fail: []byte("notcorrect"),+ function: isModeLine,+ },+ "minusFileLine": {+ pass: []byte("--- a/internal/addrs/move_endpoint_module.go"),+ fail: []byte("notcorrect"),+ function: isMinusFileLine,+ },+ "plusFileLine": {+ pass: []byte("+++ b/internal/addrs/move_endpoint_module.go"),+ fail: []byte("notcorrect"),+ function: isPlusFileLine,+ },+ "plusDiffLine": {+ pass: []byte("+fmt.Println"),+ fail: []byte("notcorrect"),+ function: isPlusDiffLine,+ },+ "minusDiffLine": {+ pass: []byte("-fmt.Println"),+ fail: []byte("notcorrect"),+ function: isMinusDiffLine,+ },+ "messageLine": {+ pass: []byte(" committed"),+ fail: []byte("notcorrect"),+ function: isMessageLine,+ },+ "binaryLine": {+ pass: []byte("Binary files /dev/null and b/plugin.sig differ"),+ fail: []byte("notcorrect"),+ function: isBinaryLine,+ },+ "lineNumberLine": {+ pass: []byte("@@ -298 +298 @@ func maxRetryErrorHandler(resp *http.Response, err error, numTries int)"),+ fail: []byte("notcorrect"),+ function: isLineNumberDiffLine,+ },+ }++ for name, test := range tests {+ if !test.function(test.pass) {+ t.Errorf("%s: Parser did not recognize correct line.", name)+ }+ if test.function(test.fail) {+ t.Errorf("%s: Parser matched an incorrect line.", name)+ }+ }+}++func TestBinaryPathParse(t *testing.T) {+ filename := pathFromBinaryLine([]byte("Binary files /dev/null and b/plugin.sig differ"))+ expected := "plugin.sig"+ if filename != expected {+ t.Errorf("Expected: %s, Got: %s", expected, filename)+ }++}diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 90348dea8..e796855d0 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -15,20 +15,19 @@ import ("time"diskbufferreader "github.com/bill-rich/disk-buffer-reader"- "github.com/gitleaks/go-gitdiff/gitdiff""github.com/go-errors/errors""github.com/go-git/go-git/v5""github.com/go-git/go-git/v5/plumbing""github.com/google/go-github/v42/github""github.com/rs/zerolog"log "github.com/sirupsen/logrus"- glgo "github.com/zricethezav/gitleaks/v8/detect/git""golang.org/x/oauth2""golang.org/x/sync/semaphore""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/gitparse""github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"@@ -282,23 +281,12 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOpzerolog.SetGlobalLevel(zerolog.Disabled)}- // Errors returned on errChan aren't blocking, so just ignore them.- errChan := make(chan error)- var gitLogArgs []string- if scanOptions.HeadHash != "" {- gitLogArgs = append(gitLogArgs, scanOptions.HeadHash)- }- logOpts := glgo.LogOpts{- Args: gitLogArgs,- DisableSafeDir: true,- }- fileChan, err := glgo.GitLog(path, logOpts, errChan)+ commitChan, err := gitparse.RepoPath(path, scanOptions.HeadHash)if err != nil {- return errors.WrapPrefix(err, "could not open repo path", 0)+ return err}- // parser can return nil chan and nil error- if fileChan == nil {- return errors.New("nothing to scan")+ if commitChan == nil {+ return nil}// get the URL metadata for reporting (may be empty)@@ -306,75 +294,62 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOpvar depth int64var reachedBase = false- for file := range fileChan {- if file == nil || file.PatchHeader == nil {- log.Debugf("file missing patch header, skipping")- continue- }- log.WithField("commit", file.PatchHeader.SHA).WithField("file", file.NewName).Trace("Scanning file from git")- if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {- log.Debugf("reached max depth")- break- }- depth++- if reachedBase && file.PatchHeader.SHA != scanOptions.BaseHash {- break- }- if len(scanOptions.BaseHash) > 0 {- if file.PatchHeader.SHA == scanOptions.BaseHash {- log.Debugf("Reached base commit. Finishing scanning files.")- reachedBase = true+ log.Debugf("Scanning repo")+ for commit := range commitChan {+ for _, diff := range commit.Diffs {+ log.WithField("commit", commit.Hash).WithField("file", diff.PathB).Trace("Scanning file from git")+ if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {+ log.Debugf("reached max depth")+ break}- }- if !scanOptions.Filter.Pass(file.NewName) {- continue- }-- fileName := file.NewName- if fileName == "" {- continue- }- var email, hash, when string- if file.PatchHeader != nil {- if file.PatchHeader.Author != nil {- email = file.PatchHeader.Author.Email+ depth+++ if reachedBase && commit.Hash != scanOptions.BaseHash {+ break}- hash = file.PatchHeader.SHA- when = file.PatchHeader.AuthorDate.String()- }-- // Handle binary files by reading the entire file rather than using the diff.- if file.IsBinary {- commitHash := plumbing.NewHash(hash)- metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)- chunkSkel := &sources.Chunk{- SourceName: s.sourceName,- SourceID: s.sourceID,- SourceType: s.sourceType,- SourceMetadata: metadata,- Verify: s.verify,+ if len(scanOptions.BaseHash) > 0 {+ if commit.Hash == scanOptions.BaseHash {+ log.Debugf("Reached base commit. Finishing scanning files.")+ reachedBase = true+ }}- if err = handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {- log.WithError(err).Error("Error handling binary file")++ if !scanOptions.Filter.Pass(diff.PathB) {+ continue}- continue- }- for _, frag := range file.TextFragments {- var sb strings.Builder- newLineNumber := frag.NewPosition- for _, line := range frag.Lines {- if line.Op == gitdiff.OpAdd {- sb.WriteString(line.Line)+ fileName := diff.PathB+ if fileName == "" {+ continue+ }+ var email, hash, when string+ email = commit.Author+ hash = commit.Hash+ when = commit.Date.String()++ // Handle binary files by reading the entire file rather than using the diff.+ if diff.IsBinary {+ commitHash := plumbing.NewHash(hash)+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)+ chunkSkel := &sources.Chunk{+ SourceName: s.sourceName,+ SourceID: s.sourceID,+ SourceType: s.sourceType,+ SourceMetadata: metadata,+ Verify: s.verify,}+ if err := handleBinary(repo, chunksChan, chunkSkel, commitHash, fileName); err != nil {+ log.WithError(err).WithField("file", fileName).Debug("Error handling binary file")+ }+ continue}- metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, newLineNumber)++ metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))chunksChan <- &sources.Chunk{SourceName: s.sourceName,SourceID: s.sourceID,SourceType: s.sourceType,SourceMetadata: metadata,- Data: []byte(sb.String()),+ Data: diff.Content.Bytes(),Verify: s.verify,}}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 34ee94fc5..1169e209e 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -232,9 +232,9 @@ func TestSource_Chunks_Integration(t *testing.T) {"ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("\n")},// Normally we might expect to see this commit, and we may in the future.// But at the moment we're ignoring any commit unless it contains at least one non-space character.- "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!")},- "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that"), Multi: true},- "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye"), Multi: true},+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("yup, just did that\n\ngithub_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n\noh, goodness. there's another one!\n")},+ "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("oops might drop a slack token here\n\ngithub_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n\nyup, just did that\n"), Multi: true},+ "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("blah blaj\n\nthis is the secret: AKIA2E0A8F3B244C9986\n\nokay thank you bye\n"), Multi: true},"84e9c75e388ae3e866e121087ea2dd45a71068f2-aws": {B: []byte("this is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n"), Multi: true},},},@@ -262,11 +262,10 @@ func TestSource_Chunks_Integration(t *testing.T) {}()for chunk := range chunksCh {-key := ""switch meta := chunk.SourceMetadata.GetData().(type) {case *source_metadatapb.MetaData_Git:- key = meta.Git.Commit + "-" + meta.Git.File+ key = strings.TrimRight(meta.Git.Commit+"-"+meta.Git.File, "\n")}if expectedData, exists := tt.expectedChunkData[key]; !exists {