Skip to content

Find, verify, and analyze leaked credentials

Fix Git diff scans at shallow clone boundaries (#5360)

Jaafer Rahmani240ccdd90619
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go
index 24e5f6611..c0772503e 100644
--- a/pkg/gitparse/gitparse.go
+++ b/pkg/gitparse/gitparse.go
@@ -546,6 +546,7 @@ func (c *Parser) prepGitArgs(source string, head string, base string, excludedGl
absPath, err := filepath.Abs(source)
if err == nil {
+ // Keep this restricted environment in sync with the merge-base lookup in git.normalizeConfig.
if !isBare {
args.env = append(args.env, "GIT_DIR="+filepath.Join(absPath, ".git"))
} else {
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 9e29dd256..478772b11 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -1362,7 +1362,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
if scanOptions == nil {
scanOptions = NewScanOptions()
}
- if err := normalizeConfig(scanOptions, repo); err != nil {
+ if err := normalizeConfig(ctx, scanOptions, repo, repoPath); err != nil {
return err
}
start := time.Now().Unix()
@@ -1414,7 +1414,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
// other non-hash references. This ensures that both the base and head commits are resolved to actual commit hashes.
// If either commit cannot be resolved, it returns early.
// If both are resolved, it finds and sets the merge base in scanOptions.
-func normalizeConfig(scanOptions *ScanOptions, repo *git.Repository) error {
+func normalizeConfig(ctx context.Context, scanOptions *ScanOptions, repo *git.Repository, repoPath string) error {
baseCommit, err := resolveAndSetCommit(repo, &scanOptions.BaseHash)
if err != nil {
return err
@@ -1429,16 +1429,50 @@ func normalizeConfig(scanOptions *ScanOptions, repo *git.Repository) error {
return nil
}
- // If baseCommit is an ancestor of headCommit, update c.BaseRef to be the common ancestor.
- mergeBase, err := headCommit.MergeBase(baseCommit)
+ // Use the same Git history and object database as the parser, including shallow grafts.
+ // Keep this environment in sync with gitparse.prepGitArgs.
+ absPath, err := filepath.Abs(repoPath)
if err != nil {
+ return fmt.Errorf("unable to resolve repository path: %w", err)
+ }
+ gitDir := filepath.Join(absPath, gitDirName)
+ isBare := isRepoBare(repoPath)
+ if isBare {
+ gitDir = absPath
+ }
+ cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "merge-base", headCommit.Hash.String(), baseCommit.Hash.String())
+ // Replace the process environment so GIT_DIR or GIT_TRACE cannot redirect or pollute this lookup.
+ cmd.Env = []string{"GIT_DIR=" + gitDir}
+ if isBare {
+ for _, key := range []string{"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} {
+ if value := os.Getenv(key); value != "" {
+ cmd.Env = append(cmd.Env, key+"="+value)
+ }
+ }
+ }
+ out, err := cmd.Output()
+ if err != nil {
+ var exitErr *exec.ExitError
+ if errors.As(err, &exitErr) {
+ if exitErr.ExitCode() == 1 && len(exitErr.Stderr) == 0 {
+ message := "unable to resolve merge base: no merge base found"
+ shallowCmd := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "--is-shallow-repository")
+ shallowCmd.Env = cmd.Env
+ if shallow, shallowErr := shallowCmd.Output(); shallowErr == nil && strings.TrimSpace(string(shallow)) == "true" {
+ message += " (repository is shallow; fetch more history)"
+ }
+ return errors.New(message)
+ }
+ return fmt.Errorf("unable to resolve merge base: %w: %s", err, strings.TrimSpace(string(exitErr.Stderr)))
+ }
return fmt.Errorf("unable to resolve merge base: %w", err)
}
- if len(mergeBase) == 0 {
- return fmt.Errorf("unable to resolve merge base: no merge base found")
+ mergeBase := strings.TrimSpace(string(out))
+ if mergeBase == "" {
+ return errors.New("unable to resolve merge base: no merge base found")
}
- scanOptions.BaseHash = mergeBase[0].Hash.String()
+ scanOptions.BaseHash = mergeBase
return nil
}
diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go
index 320dc507c..c27b19cfd 100644
--- a/pkg/sources/git/git_test.go
+++ b/pkg/sources/git/git_test.go
@@ -1889,10 +1889,9 @@ func TestScanRepo_BaseMergedIntoHead(t *testing.T) {
// TestNormalizeConfig_BaseWithoutHead pins what normalizeConfig does and does
// not do with a base and no head: it resolves the base to a hash and leaves the
// head empty. The parser then walks --all ^base (see gitparse.Parser.RepoPath).
-// normalizeConfig must not invent a head, because that would route the
-// base-only shape through MergeBase, which go-git cannot compute across the
-// graft of a shallow clone, including the --shallow-since clone
-// prepareRepoSinceCommit makes for exactly this shape.
+// normalizeConfig must not invent a head, because that would narrow the
+// base-only scan to one branch. prepareRepoSinceCommit makes a shallow clone
+// with the base at its graft boundary for exactly this shape.
func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {
// main: A; feature: A -> B, checked out.
path := setupTestRepo(t, "base-without-head")
@@ -1920,9 +1919,8 @@ func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {
},
{
// The pre-commit hook passes HEAD as both ends (main.go). Both
- // resolve to the same hash and MergeBase short-circuits without
- // walking history, so the empty HEAD..HEAD range is safe on a
- // shallow local clone too.
+ // resolve to the same hash, so the empty HEAD..HEAD range is safe
+ // on a shallow local clone too.
name: "base HEAD and head HEAD resolve to the same commit",
base: "HEAD", head: "HEAD", wantBase: shaB, wantHead: shaB,
},
@@ -1940,7 +1938,7 @@ func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
opts := NewScanOptions(ScanOptionBaseHash(tt.base), ScanOptionHeadCommit(tt.head))
- assert.NoError(t, normalizeConfig(opts, repo))
+ assert.NoError(t, normalizeConfig(context.Background(), opts, repo, path))
assert.Equal(t, tt.wantBase, opts.BaseHash, "BaseHash")
assert.Equal(t, tt.wantHead, opts.HeadHash, "HeadHash")
})
@@ -2201,18 +2199,13 @@ func TestScanRepo_BaseNotUsable(t *testing.T) {
}
}
-// TestScanRepo_ShallowClone pins where the graft boundary of a shallow clone
-// starts to matter for a diff scan. git itself handles a truncated history
-// fine, so the boundary that counts is the one go-git needs to walk in
-// normalizeConfig: MergeBase loads a commit's parents before it can recognize
-// the commit as the range boundary (bfsCommitIterator.Next), so a base sitting
-// on the graft has no resolvable merge base.
+// TestScanRepo_ShallowClone verifies that a diff scan can use a base at the
+// graft boundary when Git can resolve it, even without the base's parent.
//
-// That walk only runs when both ends are supplied. With a base alone the
-// parser defaults the head to HEAD and git computes the range, so the same
-// clone scans cleanly; this is the shape `--since-commit X` without `--branch`
-// takes against a GitHub URL, where prepareRepoSinceCommit clones with
-// --shallow-since and puts X on the graft. See
+// Merge-base lookup only runs when both ends are supplied. With a base alone,
+// the parser lets Git compute the range across all refs; this is the shape
+// `--since-commit X` without `--branch` takes against a GitHub URL, where
+// prepareRepoSinceCommit clones with --shallow-since and puts X on the graft. See
// https://github.com/trufflesecurity/trufflehog/issues/4895 for the both-ends
// failure surfacing on GitLab.
func TestScanRepo_ShallowClone(t *testing.T) {
@@ -2259,8 +2252,79 @@ func TestScanRepo_ShallowClone(t *testing.T) {
got, _, err := scanRepoRange(ctx, t, shallow, base, head)
- assert.ErrorContains(t, err, "unable to resolve merge base")
- assert.Empty(t, got, "an unresolvable merge base must fail the scan, not scan an arbitrary range")
+ assert.NoError(t, err)
+ assert.True(t, got[head], "the one commit since base should have been scanned; got %v", got)
+ assert.False(t, got[base], "the base is excluded from its own range")
+ })
+
+ t.Run("main advances past the feature fork", func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ origin := setupTestRepo(t, "advanced-main-origin")
+ addTestFileAndCommit(t, origin, "a.txt", "a\n")
+ addTestFileAndCommit(t, origin, "fork.txt", "fork\n")
+ runGit(t, origin, "branch", "-M", "main")
+ fork := gitRevParse(t, origin, "HEAD")
+ runGit(t, origin, "switch", "-q", "-c", "feature")
+ addTestFileAndCommit(t, origin, "feature.txt", "feature\n")
+ featureHead := gitRevParse(t, origin, "HEAD")
+
+ shallow := filepath.Join(t.TempDir(), "shallow-feature")
+ runGit(t, "", "clone", "-q", "--depth", "2", "--no-single-branch", "-b", "feature", "file://"+origin, shallow)
+ runGit(t, origin, "switch", "-q", "main")
+ addTestFileAndCommit(t, origin, "main-1.txt", "main 1\n")
+ addTestFileAndCommit(t, origin, "main-2.txt", "main 2\n")
+ // Fetch the advanced main without deepening past the shared fork.
+ runGit(t, shallow, "fetch", "-q", "--depth=3", "origin", "main:refs/remotes/origin/main")
+ mainHead := gitRevParse(t, shallow, "origin/main")
+ assert.False(t, gitHasObject(t, shallow, fork+"^"), "the fork should still be at the shallow boundary")
+
+ got, _, err := scanRepoRange(ctx, t, shallow, mainHead, featureHead)
+
+ assert.NoError(t, err)
+ assert.True(t, got[featureHead], "the feature commit should be scanned; got %v", got)
+ assert.False(t, got[fork], "the merge base is excluded")
+ assert.False(t, got[mainHead], "main-only commits are excluded")
+ })
+
+ t.Run("fork is below both shallow boundaries", func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ origin := setupTestRepo(t, "missing-fork-origin")
+ addTestFileAndCommit(t, origin, "fork.txt", "fork\n")
+ runGit(t, origin, "branch", "-M", "main")
+ runGit(t, origin, "switch", "-q", "-c", "feature")
+ addTestFileAndCommit(t, origin, "feature.txt", "feature\n")
+ runGit(t, origin, "switch", "-q", "main")
+ addTestFileAndCommit(t, origin, "main.txt", "main\n")
+
+ shallow := filepath.Join(t.TempDir(), "shallow-both-tips")
+ runGit(t, "", "clone", "-q", "--depth", "1", "--no-single-branch", "-b", "feature", "file://"+origin, shallow)
+ assert.False(t, gitHasObject(t, shallow, gitRevParse(t, origin, "HEAD~1")), "the fork should be absent")
+
+ got, _, err := scanRepoRange(ctx, t, shallow, gitRevParse(t, shallow, "origin/main"), gitRevParse(t, shallow, "HEAD"))
+
+ assert.ErrorContains(t, err, "repository is shallow; fetch more history")
+ assert.Empty(t, got, "an unresolved merge base must not scan an arbitrary range")
+ })
+
+ t.Run("foreign Git environment cannot redirect the range", func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ shallow, head, base := graftAtBase(t)
+ foreign := setupTestRepo(t, "foreign")
+ addTestFileAndCommit(t, foreign, "foreign.txt", "foreign\n")
+ t.Setenv("GIT_DIR", filepath.Join(foreign, ".git"))
+ t.Setenv("GIT_TRACE", "1")
+
+ got, _, err := scanRepoRange(ctx, t, shallow, base, head)
+
+ assert.NoError(t, err)
+ assert.True(t, got[head], "the head commit should have been scanned; got %v", got)
+ assert.False(t, got[base], "the base is excluded from its own range")
})
t.Run("base is the graft boundary with implicit head", func(t *testing.T) {
@@ -2268,9 +2332,8 @@ func TestScanRepo_ShallowClone(t *testing.T) {
defer cancel()
// Same clone, no head: normalizeConfig must not invent a head
- // and run MergeBase, or the --shallow-since path would fail
- // exactly where it used to work. git walks --all ^base, and the
- // clone's only ref is the tip.
+ // and narrow the scan to one ref. Git walks --all ^base, and
+ // the clone's only ref is the tip.
shallow, head, base := graftAtBase(t)
got, _, err := scanRepoRange(ctx, t, shallow, base, "")
@@ -2284,9 +2347,8 @@ func TestScanRepo_ShallowClone(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
- // One more commit of depth is all it takes: the base's parent is
- // present, so the merge base resolves and git scans the range over
- // a history it cannot fully walk.
+ // One more commit of depth puts the base above the graft, so
+ // Git scans the range over a history it cannot fully walk.
shallow := clone(t, newOrigin(t), "3")
head, base := gitRevParse(t, shallow, "HEAD"), gitRevParse(t, shallow, "HEAD~1")
assert.True(t, gitHasObject(t, shallow, base+"^"), "fixture is wrong: the base's parent is missing")