Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix(gitparse): handle fromFileLine edge case (#2206)
Richard Gomez241e153dfbc2
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.goindex f7b912448..dcd544c85 100644--- a/pkg/gitparse/gitparse.go+++ b/pkg/gitparse/gitparse.go@@ -4,6 +4,7 @@ import ("bufio""bytes""fmt"+ "github.com/go-logr/logr""io""os""os/exec"@@ -92,6 +93,7 @@ func (state ParseState) String() string {"BinaryFileLine","HunkLineNumberLine","HunkContentLine",+ "ParseFailure",}[state]}@@ -313,7 +315,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan chcase isMessageLine(isStaged, latestState, line):latestState = MessageLine- currentCommit.Message.Write(line[4:])+ currentCommit.Message.Write(line[4:]) // Messages are indented with 4 spaces.case isMessageEndLine(isStaged, latestState, line):latestState = MessageEndLine// NoOp@@ -425,13 +427,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch// Here be dragons...// Build an informative error message.- var err error+ err := fmt.Errorf(`invalid line "%s" after state "%s"`, line, latestState)+ var logger logr.Loggerif currentCommit != nil && currentCommit.Hash != "" {- err = fmt.Errorf(`failed to parse line "%s" after state "%s" (commit=%s)`, line, latestState, currentCommit.Hash)+ logger = ctx.Logger().WithValues("commit", currentCommit.Hash)} else {- err = fmt.Errorf(`failed to parse line "%s" after state "%s"`, line, latestState)+ logger = ctx.Logger()}- ctx.Logger().V(2).Error(err, "Recovering at the latest commit or diff...\n")+ logger.Error(err, "failed to parse Git input. Recovering at the latest commit or diff...")latestState = ParseFailure}@@ -612,8 +615,9 @@ func pathFromBinaryLine(line []byte) string {}// --- a/internal/addrs/move_endpoint_module.go+// --- /dev/nullfunc isFromFileLine(isStaged bool, latestState ParseState, line []byte) bool {- if latestState != IndexLine {+ if !(latestState == IndexLine || latestState == ModeLine) {return false}if len(line) >= 6 && bytes.Equal(line[:4], []byte("--- ")) {diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.goindex 33d6e4483..f343aead0 100644--- a/pkg/gitparse/gitparse_test.go+++ b/pkg/gitparse/gitparse_test.go@@ -323,6 +323,13 @@ func TestLineChecks(t *testing.T) {IndexLine,[]byte("--- /dev/null"),},+ // New file (https://github.com/trufflesecurity/trufflehog/issues/2109)+ // diff --git a/libs/Unfit-1.0 b/libs/Unfit-1.0+ // new file mode 160000+ {+ ModeLine,+ []byte("--- /dev/null"),+ },// Uncommon but valid prefixes. Will these ever show up?// https://stackoverflow.com/a/2530012// https://git-scm.com/docs/git-config#Documentation/git-config.txt-diffmnemonicPrefix@@ -1148,7 +1155,45 @@ func TestMaxCommitSize(t *testing.T) {}-const commitLog = `commit 4727ffb7ad6dc5130bf4b4dd166e00705abdd018 (HEAD -> master)+const commitLog = `commit fd6e99e7a80199b76a694603be57c5ade1de18e7+Author: Jaliborc <[email protected]>+Date: Mon Apr 25 16:28:06 2011 +0100++ Added Unusable coloring++diff --git a/components/item.lua b/components/item.lua+index fc74534..f8d7d50 100755+--- a/components/item.lua++++ b/components/item.lua+@@ -9,6 +9,7 @@ ItemSlot:Hide()+ Bagnon.ItemSlot = ItemSlot++ local ItemSearch = LibStub('LibItemSearch-1.0')++local Unfit = LibStub('Unfit-1.0')++ local function hasBlizzQuestHighlight()+ return GetContainerItemQuestInfo and true or false+diff --git a/embeds.xml b/embeds.xml+index d3f4e7c..0c2df69 100755+--- a/embeds.xml++++ b/embeds.xml+@@ -6,6 +6,7 @@+ <Include file="libs\AceConsole-3.0\AceConsole-3.0.xml"/>+ <Include file="libs\AceLocale-3.0\AceLocale-3.0.xml"/>+++ <Script file="libs\Unfit-1.0\Unfit-1.0.lua"/>+ <Script file="libs\LibDataBroker-1.1.lua"/>+ <Script file="libs\LibItemSearch-1.0\LibItemSearch-1.0.lua"/>+ </Ui>+\ No newline at end of file+diff --git a/libs/Unfit-1.0 b/libs/Unfit-1.0+new file mode 160000+--- /dev/null++++ b/libs/Unfit-1.0+@@ -0,0 +1 @@++Subproject commit 0000000000000000000000000000000000000000++commit 4727ffb7ad6dc5130bf4b4dd166e00705abdd018 (HEAD -> master)Author: John Smith <[email protected]>Date: Tue Jul 11 22:26:11 2023 -0400@@ -1539,6 +1584,33 @@ index 0000000..5af88a8// This throws a nasty panic if it's a top-level var.func expectedCommits() []Commit {return []Commit{+ // a+ {+ Hash: "fd6e99e7a80199b76a694603be57c5ade1de18e7",+ Author: "Jaliborc <[email protected]>",+ Date: newTime("Mon Apr 25 16:28:06 2011 +0100"),+ Message: newStringBuilderValue("Added Unusable coloring\n"),+ Diffs: []Diff{+ {+ PathB: "components/item.lua",+ LineStart: 9,+ Content: *bytes.NewBuffer([]byte("\n\nlocal Unfit = LibStub('Unfit-1.0')\n\n\n")),+ IsBinary: false,+ },+ {+ PathB: "embeds.xml",+ LineStart: 6,+ Content: *bytes.NewBuffer([]byte("\n\n <Script file=\"libs\\Unfit-1.0\\Unfit-1.0.lua\"/>\n\n\n\n")),+ IsBinary: false,+ },+ {+ PathB: "libs/Unfit-1.0",+ LineStart: 1,+ Content: *bytes.NewBuffer([]byte("Subproject commit 0000000000000000000000000000000000000000\n")),+ IsBinary: false,+ },+ },+ },// Empty commit and message. Lord help us.{Hash: "4727ffb7ad6dc5130bf4b4dd166e00705abdd018",diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 028bb5d7a..1237c904b 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -18,12 +18,13 @@ import ("github.com/go-git/go-git/v5/plumbing""github.com/go-git/go-git/v5/plumbing/object""github.com/google/go-github/v42/github"- diskbufferreader "github.com/trufflesecurity/disk-buffer-reader""golang.org/x/oauth2""golang.org/x/sync/semaphore""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"+"github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -462,7 +463,16 @@ func (s *Git) CommitsScanned() uint64 {const gitDirName = ".git"func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, reporter sources.ChunkReporter) error {- commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare)+ // Get the remote URL for reporting (may be empty)+ remoteURL := getSafeRemoteURL(repo, "origin")+ var repoCtx context.Context+ if remoteURL != "" {+ repoCtx = context.WithValue(ctx, "repo", remoteURL)+ } else {+ repoCtx = context.WithValue(ctx, "repo", path)+ }++ commitChan, err := gitparse.NewParser().RepoPath(repoCtx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare)if err != nil {return err}@@ -470,14 +480,10 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringreturn nil}- // get the URL metadata for reporting (may be empty)- urlMetadata := getSafeRemoteURL(repo, "origin")-var depth int64-gitDir := filepath.Join(path, gitDirName)- logger := ctx.Logger().WithValues("repo", urlMetadata)+ logger := repoCtx.Logger()logger.V(1).Info("scanning repo", "base", scanOptions.BaseHash, "head", scanOptions.HeadHash)for commit := range commitChan {if len(scanOptions.BaseHash) > 0 {@@ -510,7 +516,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string// Handle binary files by reading the entire file rather than using the diff.if diff.IsBinary {commitHash := plumbing.NewHash(hash)- metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, 0)+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, remoteURL, 0)chunkSkel := &sources.Chunk{SourceName: s.sourceName,SourceID: s.sourceID,@@ -526,10 +532,10 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string}if diff.Content.Len() > sources.ChunkSize+sources.PeekSize {- s.gitChunk(ctx, diff, fileName, email, hash, when, urlMetadata, reporter)+ s.gitChunk(ctx, diff, fileName, email, hash, when, remoteURL, reporter)continue}- metadata := s.sourceMetadataFunc(fileName, email, hash, when, urlMetadata, int64(diff.LineStart))+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, remoteURL, int64(diff.LineStart))chunk := sources.Chunk{SourceName: s.sourceName,SourceID: s.sourceID,