Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fix Git diff scans at shallow clone boundaries (#5360)
Jaafer Rahmani240ccdd90619
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.goindex 24e5f6611..c0772503e 100644--- a/pkg/gitparse/gitparse.go+++ b/pkg/gitparse/gitparse.go@@ -546,6 +546,7 @@ func (c *Parser) prepGitArgs(source string, head string, base string, excludedGlabsPath, err := filepath.Abs(source)if err == nil {+ // Keep this restricted environment in sync with the merge-base lookup in git.normalizeConfig.if !isBare {args.env = append(args.env, "GIT_DIR="+filepath.Join(absPath, ".git"))} else {diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 9e29dd256..478772b11 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -1362,7 +1362,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strinif scanOptions == nil {scanOptions = NewScanOptions()}- if err := normalizeConfig(scanOptions, repo); err != nil {+ if err := normalizeConfig(ctx, scanOptions, repo, repoPath); err != nil {return err}start := time.Now().Unix()@@ -1414,7 +1414,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin// other non-hash references. This ensures that both the base and head commits are resolved to actual commit hashes.// If either commit cannot be resolved, it returns early.// If both are resolved, it finds and sets the merge base in scanOptions.-func normalizeConfig(scanOptions *ScanOptions, repo *git.Repository) error {+func normalizeConfig(ctx context.Context, scanOptions *ScanOptions, repo *git.Repository, repoPath string) error {baseCommit, err := resolveAndSetCommit(repo, &scanOptions.BaseHash)if err != nil {return err@@ -1429,16 +1429,50 @@ func normalizeConfig(scanOptions *ScanOptions, repo *git.Repository) error {return nil}- // If baseCommit is an ancestor of headCommit, update c.BaseRef to be the common ancestor.- mergeBase, err := headCommit.MergeBase(baseCommit)+ // Use the same Git history and object database as the parser, including shallow grafts.+ // Keep this environment in sync with gitparse.prepGitArgs.+ absPath, err := filepath.Abs(repoPath)if err != nil {+ return fmt.Errorf("unable to resolve repository path: %w", err)+ }+ gitDir := filepath.Join(absPath, gitDirName)+ isBare := isRepoBare(repoPath)+ if isBare {+ gitDir = absPath+ }+ cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "merge-base", headCommit.Hash.String(), baseCommit.Hash.String())+ // Replace the process environment so GIT_DIR or GIT_TRACE cannot redirect or pollute this lookup.+ cmd.Env = []string{"GIT_DIR=" + gitDir}+ if isBare {+ for _, key := range []string{"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} {+ if value := os.Getenv(key); value != "" {+ cmd.Env = append(cmd.Env, key+"="+value)+ }+ }+ }+ out, err := cmd.Output()+ if err != nil {+ var exitErr *exec.ExitError+ if errors.As(err, &exitErr) {+ if exitErr.ExitCode() == 1 && len(exitErr.Stderr) == 0 {+ message := "unable to resolve merge base: no merge base found"+ shallowCmd := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "--is-shallow-repository")+ shallowCmd.Env = cmd.Env+ if shallow, shallowErr := shallowCmd.Output(); shallowErr == nil && strings.TrimSpace(string(shallow)) == "true" {+ message += " (repository is shallow; fetch more history)"+ }+ return errors.New(message)+ }+ return fmt.Errorf("unable to resolve merge base: %w: %s", err, strings.TrimSpace(string(exitErr.Stderr)))+ }return fmt.Errorf("unable to resolve merge base: %w", err)}- if len(mergeBase) == 0 {- return fmt.Errorf("unable to resolve merge base: no merge base found")+ mergeBase := strings.TrimSpace(string(out))+ if mergeBase == "" {+ return errors.New("unable to resolve merge base: no merge base found")}- scanOptions.BaseHash = mergeBase[0].Hash.String()+ scanOptions.BaseHash = mergeBasereturn nil}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 320dc507c..c27b19cfd 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -1889,10 +1889,9 @@ func TestScanRepo_BaseMergedIntoHead(t *testing.T) {// TestNormalizeConfig_BaseWithoutHead pins what normalizeConfig does and does// not do with a base and no head: it resolves the base to a hash and leaves the// head empty. The parser then walks --all ^base (see gitparse.Parser.RepoPath).-// normalizeConfig must not invent a head, because that would route the-// base-only shape through MergeBase, which go-git cannot compute across the-// graft of a shallow clone, including the --shallow-since clone-// prepareRepoSinceCommit makes for exactly this shape.+// normalizeConfig must not invent a head, because that would narrow the+// base-only scan to one branch. prepareRepoSinceCommit makes a shallow clone+// with the base at its graft boundary for exactly this shape.func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {// main: A; feature: A -> B, checked out.path := setupTestRepo(t, "base-without-head")@@ -1920,9 +1919,8 @@ func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {},{// The pre-commit hook passes HEAD as both ends (main.go). Both- // resolve to the same hash and MergeBase short-circuits without- // walking history, so the empty HEAD..HEAD range is safe on a- // shallow local clone too.+ // resolve to the same hash, so the empty HEAD..HEAD range is safe+ // on a shallow local clone too.name: "base HEAD and head HEAD resolve to the same commit",base: "HEAD", head: "HEAD", wantBase: shaB, wantHead: shaB,},@@ -1940,7 +1938,7 @@ func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {opts := NewScanOptions(ScanOptionBaseHash(tt.base), ScanOptionHeadCommit(tt.head))- assert.NoError(t, normalizeConfig(opts, repo))+ assert.NoError(t, normalizeConfig(context.Background(), opts, repo, path))assert.Equal(t, tt.wantBase, opts.BaseHash, "BaseHash")assert.Equal(t, tt.wantHead, opts.HeadHash, "HeadHash")})@@ -2201,18 +2199,13 @@ func TestScanRepo_BaseNotUsable(t *testing.T) {}}-// TestScanRepo_ShallowClone pins where the graft boundary of a shallow clone-// starts to matter for a diff scan. git itself handles a truncated history-// fine, so the boundary that counts is the one go-git needs to walk in-// normalizeConfig: MergeBase loads a commit's parents before it can recognize-// the commit as the range boundary (bfsCommitIterator.Next), so a base sitting-// on the graft has no resolvable merge base.+// TestScanRepo_ShallowClone verifies that a diff scan can use a base at the+// graft boundary when Git can resolve it, even without the base's parent.//-// That walk only runs when both ends are supplied. With a base alone the-// parser defaults the head to HEAD and git computes the range, so the same-// clone scans cleanly; this is the shape `--since-commit X` without `--branch`-// takes against a GitHub URL, where prepareRepoSinceCommit clones with-// --shallow-since and puts X on the graft. See+// Merge-base lookup only runs when both ends are supplied. With a base alone,+// the parser lets Git compute the range across all refs; this is the shape+// `--since-commit X` without `--branch` takes against a GitHub URL, where+// prepareRepoSinceCommit clones with --shallow-since and puts X on the graft. See// https://github.com/trufflesecurity/trufflehog/issues/4895 for the both-ends// failure surfacing on GitLab.func TestScanRepo_ShallowClone(t *testing.T) {@@ -2259,8 +2252,79 @@ func TestScanRepo_ShallowClone(t *testing.T) {got, _, err := scanRepoRange(ctx, t, shallow, base, head)- assert.ErrorContains(t, err, "unable to resolve merge base")- assert.Empty(t, got, "an unresolvable merge base must fail the scan, not scan an arbitrary range")+ assert.NoError(t, err)+ assert.True(t, got[head], "the one commit since base should have been scanned; got %v", got)+ assert.False(t, got[base], "the base is excluded from its own range")+ })++ t.Run("main advances past the feature fork", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ origin := setupTestRepo(t, "advanced-main-origin")+ addTestFileAndCommit(t, origin, "a.txt", "a\n")+ addTestFileAndCommit(t, origin, "fork.txt", "fork\n")+ runGit(t, origin, "branch", "-M", "main")+ fork := gitRevParse(t, origin, "HEAD")+ runGit(t, origin, "switch", "-q", "-c", "feature")+ addTestFileAndCommit(t, origin, "feature.txt", "feature\n")+ featureHead := gitRevParse(t, origin, "HEAD")++ shallow := filepath.Join(t.TempDir(), "shallow-feature")+ runGit(t, "", "clone", "-q", "--depth", "2", "--no-single-branch", "-b", "feature", "file://"+origin, shallow)+ runGit(t, origin, "switch", "-q", "main")+ addTestFileAndCommit(t, origin, "main-1.txt", "main 1\n")+ addTestFileAndCommit(t, origin, "main-2.txt", "main 2\n")+ // Fetch the advanced main without deepening past the shared fork.+ runGit(t, shallow, "fetch", "-q", "--depth=3", "origin", "main:refs/remotes/origin/main")+ mainHead := gitRevParse(t, shallow, "origin/main")+ assert.False(t, gitHasObject(t, shallow, fork+"^"), "the fork should still be at the shallow boundary")++ got, _, err := scanRepoRange(ctx, t, shallow, mainHead, featureHead)++ assert.NoError(t, err)+ assert.True(t, got[featureHead], "the feature commit should be scanned; got %v", got)+ assert.False(t, got[fork], "the merge base is excluded")+ assert.False(t, got[mainHead], "main-only commits are excluded")+ })++ t.Run("fork is below both shallow boundaries", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ origin := setupTestRepo(t, "missing-fork-origin")+ addTestFileAndCommit(t, origin, "fork.txt", "fork\n")+ runGit(t, origin, "branch", "-M", "main")+ runGit(t, origin, "switch", "-q", "-c", "feature")+ addTestFileAndCommit(t, origin, "feature.txt", "feature\n")+ runGit(t, origin, "switch", "-q", "main")+ addTestFileAndCommit(t, origin, "main.txt", "main\n")++ shallow := filepath.Join(t.TempDir(), "shallow-both-tips")+ runGit(t, "", "clone", "-q", "--depth", "1", "--no-single-branch", "-b", "feature", "file://"+origin, shallow)+ assert.False(t, gitHasObject(t, shallow, gitRevParse(t, origin, "HEAD~1")), "the fork should be absent")++ got, _, err := scanRepoRange(ctx, t, shallow, gitRevParse(t, shallow, "origin/main"), gitRevParse(t, shallow, "HEAD"))++ assert.ErrorContains(t, err, "repository is shallow; fetch more history")+ assert.Empty(t, got, "an unresolved merge base must not scan an arbitrary range")+ })++ t.Run("foreign Git environment cannot redirect the range", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ shallow, head, base := graftAtBase(t)+ foreign := setupTestRepo(t, "foreign")+ addTestFileAndCommit(t, foreign, "foreign.txt", "foreign\n")+ t.Setenv("GIT_DIR", filepath.Join(foreign, ".git"))+ t.Setenv("GIT_TRACE", "1")++ got, _, err := scanRepoRange(ctx, t, shallow, base, head)++ assert.NoError(t, err)+ assert.True(t, got[head], "the head commit should have been scanned; got %v", got)+ assert.False(t, got[base], "the base is excluded from its own range")})t.Run("base is the graft boundary with implicit head", func(t *testing.T) {@@ -2268,9 +2332,8 @@ func TestScanRepo_ShallowClone(t *testing.T) {defer cancel()// Same clone, no head: normalizeConfig must not invent a head- // and run MergeBase, or the --shallow-since path would fail- // exactly where it used to work. git walks --all ^base, and the- // clone's only ref is the tip.+ // and narrow the scan to one ref. Git walks --all ^base, and+ // the clone's only ref is the tip.shallow, head, base := graftAtBase(t)got, _, err := scanRepoRange(ctx, t, shallow, base, "")@@ -2284,9 +2347,8 @@ func TestScanRepo_ShallowClone(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)defer cancel()- // One more commit of depth is all it takes: the base's parent is- // present, so the merge base resolves and git scans the range over- // a history it cannot fully walk.+ // One more commit of depth puts the base above the graft, so+ // Git scans the range over a history it cannot fully walk.shallow := clone(t, newOrigin(t), "3")head, base := gitRevParse(t, shallow, "HEAD"), gitRevParse(t, shallow, "HEAD~1")assert.True(t, gitHasObject(t, shallow, base+"^"), "fixture is wrong: the base's parent is missing")