Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
give git the real base..head range for diff scans (#5325)
Kevin Phamdc2edce2b667
diff --git a/main.go b/main.goindex a8b01d261..0527f72a4 100644--- a/main.go+++ b/main.go@@ -899,7 +899,11 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,// Override git configuration for pre-commit hook contextgitCfg.TrustLocalGitConfig = true- gitCfg.BaseRef = "HEAD" // Only scan staged changes+ // Only scan staged changes. Both ends are HEAD so the commit range is+ // HEAD..HEAD, which is empty; a base with no head would instead walk+ // every ref not reachable from HEAD (see gitparse.Parser.RepoPath).+ gitCfg.BaseRef = "HEAD"+ gitCfg.HeadRef = "HEAD"// Override result filters for pre-commit hook context// In hook mode, we only want to show verified secrets and unknown findingsdiff --git a/pkg/gitparse/README.md b/pkg/gitparse/README.mdindex 51eb50e3a..d59575c4e 100644--- a/pkg/gitparse/README.md+++ b/pkg/gitparse/README.md@@ -15,7 +15,7 @@ A `Diff` is one file's added content inside one commit, along with the details ocommit:```go-diffChan, err := parser.RepoPath(ctx, repoPath, "", true, nil, false)+diffChan, err := parser.RepoPath(ctx, repoPath, "", "", nil, false)for diff := range diffChan {diff.Commit.Hash // the commit this change belongs todiff.PathB // the path of the changed filediff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.goindex 7bb712a9d..24e5f6611 100644--- a/pkg/gitparse/gitparse.go+++ b/pkg/gitparse/gitparse.go@@ -3,7 +3,6 @@ package gitparseimport ("bufio""bytes"- "cmp""errors""fmt""io"@@ -268,15 +267,25 @@ type gitArgs struct {// RepoPath parses the output of the `git log` command for the `source` path.// The Diff chan will return diffs in the order they are parsed from the log,// though the diffs are generated using `git show` in groups.+//+// head and base are commit hashes or refs. When base is non-empty the log is+// restricted to the range base..head (commits reachable from head but not from+// base), which is the diff-scan contract behind `--since-commit`. The range is+// computed by git itself so that it is independent of commit dates and merge+// topology. An empty base means a full-history scan of head (or of --all when+// head is also empty). An empty head with a non-empty base means every commit+// reachable from any ref but not from base (`--all ^base`), which is what+// `--since-commit X` without `--branch` has always covered; callers that want+// a single-branch diff pass the head explicitly.func (c *Parser) RepoPath(ctx context.Context,source string,head string,- abbreviatedLog bool,+ base string,excludedGlobs []string,isBare bool,) (chan *Diff, error) {- args := c.prepGitArgs(source, head, abbreviatedLog, excludedGlobs, isBare)+ args := c.prepGitArgs(source, head, base, excludedGlobs, isBare)if c.lowMemoryScan {return c.repoPathLowMemory(ctx, args)@@ -469,7 +478,10 @@ func (c *Parser) enumerateCommits(ctx context.Context, args gitArgs) (chan []strreturn commitGroups, nil}-func (c *Parser) prepGitArgs(source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) gitArgs {+func (c *Parser) prepGitArgs(source string, head string, base string, excludedGlobs []string, isBare bool) gitArgs {+ // Full-history scans skip deletions; a diff scan must report every change in the range.+ abbreviatedLog := base == ""+args := gitArgs{global: []string{"-C", source,@@ -502,11 +514,28 @@ func (c *Parser) prepGitArgs(source string, head string, abbreviatedLog bool, exargs.show = append(args.show, "--diff-filter=AM")}- // Keep head or all last, before the --, not required but sensible+ // The positive end of the walk, kept last before the -- (not required but+ // sensible). With no head every ref is walked, so a base-only scan is+ // `--all ^base`: everything since base on any branch, not just the checked+ // out one. The pre-commit hook wants the empty HEAD..HEAD range and passes+ // HEAD as both ends itself (see the isPreCommitHook override in main.go).// https://git-scm.com/docs/git-log#Documentation/git-log.txt---all- args.log = append(args.log, cmp.Or(head, "--all"))+ switch {+ case head != "":+ args.log = append(args.log, head)+ default:+ args.log = append(args.log, "--all")+ }++ // `head ^base` is base..head. Keeping the two revisions as separate args+ // means head is always the positive end of the range and the base is+ // never spliced into a string git has to parse.+ // https://git-scm.com/docs/gitrevisions#_specifying_ranges+ if base != "" {+ args.log = append(args.log, "^"+base)+ }- // And then potentially add -- to args here+ // Pathspecs live in their own slice so `--` always trails every revision.if len(excludedGlobs) != 0 {args.paths = []string{"--", "."}for _, glob := range excludedGlobs {diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.goindex 52dd304f1..141cf0832 100644--- a/pkg/gitparse/gitparse_test.go+++ b/pkg/gitparse/gitparse_test.go@@ -3,6 +3,7 @@ package gitparseimport ("bytes""path/filepath"+ "slices""strings""testing""time"@@ -25,33 +26,71 @@ func TestPrepGitArgs(t *testing.T) {p := Parser{}- args := p.prepGitArgs(repopath, "", false, nil, false)+ // Full history of everything: no head, no base.+ args := p.prepGitArgs(repopath, "", "", nil, false)assert.Equal(t, []string{"-C", repopath}, args.global)assert.Contains(t, args.log, "--all")+ // A full-history scan skips deletions; a diff scan must not. The filter is a+ // diff option, so it lives in show and never reaches rev-list via args.log.assert.NotContains(t, args.log, "--diff-filter=AM")+ assert.Contains(t, args.show, "--diff-filter=AM")+ assertNoRangeExclusion(t, args.log)assert.Equal(t, []string{"GIT_DIR=" + filepath.Join(repopath, ".git")}, args.env)assert.Empty(t, args.paths)- args = p.prepGitArgs(repopath, "branchname", true, []string{"some/file.txt", "bloated.dat"}, true)+ // Full history of one branch, with excludes, in a bare repo.+ args = p.prepGitArgs(repopath, "branchname", "", []string{"some/file.txt", "bloated.dat"}, true)// headassert.Contains(t, args.log, "branchname")assert.NotContains(t, args.log, "--all")- // abbreviatedLog. Only show carries the diff filter: args.log holds the options- // that choose commits and is what the lower-memory scan gives to `git rev-list`,- // which rejects diff options.+ // abbreviatedLog is implied by the empty base. Only show carries the diff+ // filter: args.log holds the options that choose commits and is what the+ // lower-memory scan gives to `git rev-list`, which rejects diff options.assert.NotContains(t, args.log, "--diff-filter=AM")assert.Contains(t, args.show, "--diff-filter=AM")+ assertNoRangeExclusion(t, args.log)// excludedGlobsassert.Contains(t, args.paths, "--")assert.Contains(t, args.paths, ":(exclude)bloated.dat")// isBareassert.Equal(t, []string{"GIT_DIR=" + repopath}, args.env)+ // Diff scan base..head: git must receive the exclusion itself so the range+ // does not depend on commit-date ordering (INT-1054).+ args = p.prepGitArgs(repopath, "headsha", "basesha", nil, false)+ assert.Contains(t, args.log, "headsha")+ assert.Contains(t, args.log, "^basesha")+ assert.NotContains(t, args.log, "--all")+ assert.NotContains(t, args.log, "--diff-filter=AM")+ // git show is run per explicit hash in the low-memory path and must never+ // be handed a range.+ assert.NotContains(t, args.show, "--diff-filter=AM")+ assert.NotContains(t, args.show, "^basesha")+ // Revision args follow the flags; the exclusion belongs with the revisions.+ assert.Greater(t, indexOf(args.log, "^basesha"), indexOf(args.log, "headsha"))++ // A range with excludes: the pathspec separator must land after both+ // revisions or git reads the exclusion as a path.+ args = p.prepGitArgs(repopath, "headsha", "basesha", []string{"bloated.dat"}, false)+ full := slices.Concat(args.log, args.paths)+ assert.Greater(t, indexOf(full, "--"), indexOf(full, "^basesha"))++ // A base with no head is `--all ^base`: everything since the base on any+ // ref, which is what `--since-commit X` without `--branch` covers. The+ // parser must not invent a head here; narrowing to HEAD would silently+ // drop commits that live only on other branches.+ args = p.prepGitArgs(repopath, "", "basesha", nil, false)+ assert.Contains(t, args.log, "--all")+ assert.Contains(t, args.log, "^basesha")+ assert.NotContains(t, args.log, "HEAD")+ assert.NotContains(t, args.log, "--diff-filter=AM")+ assert.Greater(t, indexOf(args.log, "^basesha"), indexOf(args.log, "--all"))+// test env passthrough used for pre-receivet.Setenv("GIT_OBJECT_DIRECTORY", "foo")t.Setenv("GIT_ALTERNATE_OBJECT_DIRECTORIES", "bar")- args = p.prepGitArgs(repopath, "", false, nil, true)+ args = p.prepGitArgs(repopath, "", "", nil, true)assert.Equal(t, []string{"GIT_DIR=" + repopath,"GIT_OBJECT_DIRECTORY=foo",@@ -59,6 +98,24 @@ func TestPrepGitArgs(t *testing.T) {}, args.env)}+// assertNoRangeExclusion fails if any revision argument excludes commits+// (a leading "^"), which must only happen for diff scans with a base.+func assertNoRangeExclusion(t *testing.T, logArgs []string) {+ t.Helper()+ for _, a := range logArgs {+ assert.False(t, strings.HasPrefix(a, "^"), "unexpected range exclusion %q in %v", a, logArgs)+ }+}++func indexOf(args []string, want string) int {+ for i, a := range args {+ if a == want {+ return i+ }+ }+ return -1+}+type testCaseLine struct {latestState ParseStateline []bytediff --git a/pkg/gitparse/lowmemory_test.go b/pkg/gitparse/lowmemory_test.goindex a4ab2e44d..f4220719c 100644--- a/pkg/gitparse/lowmemory_test.go+++ b/pkg/gitparse/lowmemory_test.go@@ -40,28 +40,28 @@ func collectDiffs(t *testing.T, diffChan chan *Diff) []string {func TestLowMemoryScanMatchesSingleProcess(t *testing.T) {repo := testRepoRoot(t)- // abbreviatedLog is the caller's BaseHash == "", so both values are real- // configurations and they take different paths through git. With it on, git drops- // commits whose diffs are all filtered away; with it off, those commits stay.+ // abbreviatedLog is the caller's BaseHash == "". Only the empty base runs here:+ // the checkout may be shallow, so the real history has no reliable base commit.+ // Diff scans with a base go through the lower-memory path in the pkg/sources/git+ // range tests, which build their own fixtures.//// The group sizes go down to 1 on purpose. Every group ends a stream, and a commit// with no diffs used to be dropped when it landed at the end of one, so a size of 1// puts every commit in that spot at once. Before cleanupParse learned to finish off// the last commit, this repository lost 84 diffs at size 1 and 1 at size 75.for _, tc := range []struct {- name string- abbreviated bool+ name string+ base string}{- {"abbreviated", true},- {"full", false},+ {"abbreviated", ""},} {- abbreviated, name := tc.abbreviated, tc.name+ base, name := tc.base, tc.namet.Run(name, func(t *testing.T) {ctx := context.Background()single := NewParser()- singleChan, err := single.RepoPath(ctx, repo, "", abbreviated, nil, false)+ singleChan, err := single.RepoPath(ctx, repo, "", base, nil, false)if err != nil {t.Fatalf("single-process RepoPath: %v", err)}@@ -76,7 +76,7 @@ func TestLowMemoryScanMatchesSingleProcess(t *testing.T) {low := NewParser(UseLowMemoryScan())low.groupSize = groupSize- lowChan, err := low.RepoPath(ctx, repo, "", abbreviated, nil, false)+ lowChan, err := low.RepoPath(ctx, repo, "", base, nil, false)if err != nil {t.Fatalf("group size %d: RepoPath: %v", groupSize, err)}@@ -105,7 +105,7 @@ func TestLowMemoryScanExcludedGlobs(t *testing.T) {globs := []string{"*.go"}single := NewParser()- singleChan, err := single.RepoPath(ctx, repo, "", true, globs, false)+ singleChan, err := single.RepoPath(ctx, repo, "", "", globs, false)if err != nil {t.Fatalf("single-process RepoPath: %v", err)}@@ -113,7 +113,7 @@ func TestLowMemoryScanExcludedGlobs(t *testing.T) {low := NewParser(UseLowMemoryScan())low.groupSize = 13- lowChan, err := low.RepoPath(ctx, repo, "", true, globs, false)+ lowChan, err := low.RepoPath(ctx, repo, "", "", globs, false)if err != nil {t.Fatalf("low-memory RepoPath: %v", err)}@@ -136,7 +136,7 @@ func TestLowMemoryScanUnknownHead(t *testing.T) {repo := testRepoRoot(t)parser := NewParser(UseLowMemoryScan())- if _, err := parser.RepoPath(context.Background(), repo, "no-such-ref-exists", true, nil, false); err == nil {+ if _, err := parser.RepoPath(context.Background(), repo, "no-such-ref-exists", "", nil, false); err == nil {t.Fatal("expected an error for an unknown head, got nil")}}@@ -149,7 +149,7 @@ func TestLowMemoryScanEmptyRepo(t *testing.T) {runTestGit(t, dir, "init", "-q")parser := NewParser(UseLowMemoryScan())- diffChan, err := parser.RepoPath(context.Background(), dir, "", true, nil, false)+ diffChan, err := parser.RepoPath(context.Background(), dir, "", "", nil, false)if err != nil {t.Fatalf("RepoPath on an empty repo: %v", err)}@@ -169,7 +169,7 @@ func TestLowMemoryScanAbandonedByConsumer(t *testing.T) {parser := NewParser(UseLowMemoryScan())parser.groupSize = 2- diffChan, err := parser.RepoPath(ctx, repo, "", true, nil, false)+ diffChan, err := parser.RepoPath(ctx, repo, "", "", nil, false)if err != nil {t.Fatalf("RepoPath: %v", err)}@@ -229,6 +229,18 @@ func runTestGit(tb testing.TB, dir string, args ...string) {}}+func testGitOutput(tb testing.TB, dir string, args ...string) string {+ tb.Helper()++ cmd := exec.Command("git", args...)+ cmd.Dir = dir+ out, err := cmd.Output()+ if err != nil {+ tb.Fatalf("git %s: %v", strings.Join(args, " "), err)+ }+ return string(out)+}+// TestTrailingCommitWithoutDiffsIsReported pins down the rule that a commit is finished// off at the end of a stream, not only when the next commit line arrives.//@@ -255,11 +267,13 @@ func TestTrailingCommitWithoutDiffsIsReported(t *testing.T) {runTestGit(t, dir, "commit", "-q", "--allow-empty", "-m", "empty commit worth scanning")// Group size 1 puts the empty commit at the end of its own stream, which is exactly- // where it used to be lost.+ // where it used to be lost. A real base keeps the scan unabbreviated; without one+ // the diff filter would drop the empty commit for an unrelated reason.parser := NewParser(UseLowMemoryScan())parser.groupSize = 1+ base := strings.TrimSpace(testGitOutput(t, dir, "rev-parse", "HEAD~1"))- diffChan, err := parser.RepoPath(context.Background(), dir, "", false, nil, false)+ diffChan, err := parser.RepoPath(context.Background(), dir, "", base, nil, false)if err != nil {t.Fatalf("RepoPath: %v", err)}diff --git a/pkg/sources/git/README.md b/pkg/sources/git/README.mdindex 5e1f80a41..aecd3e6db 100644--- a/pkg/sources/git/README.md+++ b/pkg/sources/git/README.md@@ -248,7 +248,7 @@ A clone that fails because of a network problem or what looks like a rate limit### What Doesn't Get Scanned- Files excluded by the include paths, exclude paths, or exclude globs filters-- Commits past `--max-depth`, or older than the commit given to `--since-commit`+- Commits past `--max-depth`, or reachable from the commit given to `--since-commit` (that commit and its ancestors)- Refs other than the one named by `--branch`, when that flag is used- Binary files, when `--force-skip-binaries` is used- Binary files whose extension TruffleHog already skips, such as common image, audio, video, and font typesdiff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 89094f263..326188dcc 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -875,7 +875,8 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringlogValues = append(logValues, "max_depth", scanOptions.MaxDepth)}- diffChan, err := s.parser.RepoPath(repoCtx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, isRepoBare(path))+ // git computes the base..head range, so every commit on diffChan is in scope.+ diffChan, err := s.parser.RepoPath(repoCtx, path, scanOptions.HeadHash, scanOptions.BaseHash, scanOptions.ExcludeGlobs, isRepoBare(path))if err != nil {return err}@@ -899,10 +900,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringcommit := diff.CommitfullHash := commit.Hash- if scanOptions.BaseHash != "" && scanOptions.BaseHash == fullHash {- logger.V(1).Info("reached base commit", "commit", fullHash)- break- }email := commit.Authorwhen := commit.Date.UTC().Format("2006-01-02 15:04:05 -0700")@@ -1044,6 +1041,11 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringreturn err}}++ // empty base..head range exits successfully+ if scanOptions.BaseHash != "" && depth == 0 {+ logger.Info("no commits in range", logValues...)+ }return nil}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex a6aeaa369..07db93a2a 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -1647,3 +1647,680 @@ func TestGitLowMemoryScan(t *testing.T) {t.Errorf("Data: %s", string(gotChunk.Data))}}++// Planted secrets for the base..head fixture. Unverifiable on purpose; the+// scan runs with verification off and we only assert they were chunked.+const (+ fixtureAWSKey = "AKIAXYZDQCEN4B6JSJQI"+ fixtureGitHubToken = "ghp_a1B2c3D4e5F6g7H8i9J0kLmNoPqRsTuVwXyZ"+)++// mergedBaseFixture is a repository whose feature branch merged its base in,+// the shape reported in INT-1054 / CSM-2357:+//+// F: feature work after merge <- head+// M: merge main into feature+// |\+// | C: newer base work <- base (and the merge-base of main/feature)+// | B: base work+// E: more feature work <- GitHub token+// D: feature work <- AWS key+// |/+// A: common ancestor+//+// git log C..F is F M E D. With the customer's dates (D, E older than C) the+// pre-fix scanner stopped at C and never reached E or D.+type mergedBaseFixture struct {+ path string+ sha map[string]string // commit letter -> full hash+}++// buildMergedBaseFixture creates the repository above with pinned committer+// dates so the ordering `git log` produces is deterministic. When+// branchNewerThanBase is true the topology is identical but D and E carry+// dates after C, which is the case the pre-fix code happened to get right;+// the fix must produce the same commit set either way. withUnreachableBase+// adds G on main after the merge, so that a base of G is not an ancestor of+// head, the shape GitHub Actions produce via pull_request.base.sha.+func buildMergedBaseFixture(t *testing.T, branchNewerThanBase, withUnreachableBase bool) mergedBaseFixture {+ t.Helper()+ f := mergedBaseFixture{path: setupTestRepo(t, "merged-base"), sha: map[string]string{}}+ git := func(args ...string) {+ t.Helper()+ out, err := exec.Command("git", append([]string{"-C", f.path}, args...)...).CombinedOutput()+ assert.NoError(t, err, "git %v: %s", args, out)+ }+ appendFile := func(name string, lines ...string) {+ t.Helper()+ fh, err := os.OpenFile(filepath.Join(f.path, name), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)+ assert.NoError(t, err)+ _, err = fh.WriteString(strings.Join(lines, "\n") + "\n")+ assert.NoError(t, err)+ assert.NoError(t, fh.Close())+ git("add", name)+ }+ // commit pins both dates so hashes and log ordering are reproducible.+ commit := func(letter, date, msg string) {+ t.Helper()+ cmd := exec.Command("git", "-C", f.path, "commit", "-q", "-m", msg)+ cmd.Env = append(os.Environ(), "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date)+ out, err := cmd.CombinedOutput()+ assert.NoError(t, err, "commit %s: %s", letter, out)+ sha, err := exec.Command("git", "-C", f.path, "rev-parse", "HEAD").Output()+ assert.NoError(t, err)+ f.sha[letter] = strings.TrimSpace(string(sha))+ }++ // Customer dates: branch work predates the base work it later merges in.+ dates := map[string]string{+ "A": "2026-01-01T00:00:00Z",+ "D": "2026-01-02T00:00:00Z", "E": "2026-01-03T00:00:00Z",+ "B": "2026-01-04T00:00:00Z", "C": "2026-01-05T00:00:00Z",+ "M": "2026-01-06T00:00:00Z", "F": "2026-01-07T00:00:00Z", "G": "2026-01-08T00:00:00Z",+ }+ if branchNewerThanBase {+ dates["B"], dates["C"] = "2026-01-02T00:00:00Z", "2026-01-03T00:00:00Z"+ dates["D"], dates["E"] = "2026-01-04T00:00:00Z", "2026-01-05T00:00:00Z"+ }++ git("switch", "-q", "-c", "main")+ appendFile("README.md", "A")+ commit("A", dates["A"], "A: common ancestor")++ git("switch", "-q", "-c", "feature")+ appendFile("feature.txt", "D",+ "aws_access_key_id = "+fixtureAWSKey,+ "aws_secret_access_key = Tg0pz8Jii8hkLx4+PnUisM8GmKs3a2DK+9qz/lie")+ commit("D", dates["D"], "D: feature work")+ appendFile("feature.txt", "E", "github_token = "+fixtureGitHubToken)+ commit("E", dates["E"], "E: more feature work")++ git("switch", "-q", "main")+ appendFile("base.txt", "B")+ commit("B", dates["B"], "B: base work")+ appendFile("base.txt", "C")+ commit("C", dates["C"], "C: newer base work")++ git("switch", "-q", "feature")+ {+ cmd := exec.Command("git", "-C", f.path, "merge", "-q", "--no-ff", "main", "-m", "M: merge main into feature")+ cmd.Env = append(os.Environ(), "GIT_AUTHOR_DATE="+dates["M"], "GIT_COMMITTER_DATE="+dates["M"])+ out, err := cmd.CombinedOutput()+ assert.NoError(t, err, "merge: %s", out)+ sha, err := exec.Command("git", "-C", f.path, "rev-parse", "HEAD").Output()+ assert.NoError(t, err)+ f.sha["M"] = strings.TrimSpace(string(sha))+ }+ appendFile("feature.txt", "F", "github_token = "+fixtureGitHubToken)+ commit("F", dates["F"], "F: feature work after merge")++ if withUnreachableBase {+ git("switch", "-q", "main")+ appendFile("base.txt", "G")+ commit("G", dates["G"], "G: base work after the merge")+ git("switch", "-q", "feature")+ }+ return f+}++// scanFixtureCommits runs Git.ScanRepo over the fixture with the given+// base/head and returns the set of commit hashes that produced chunks plus the+// concatenated chunk data, so callers can assert both coverage and content.+func scanFixtureCommits(t *testing.T, f mergedBaseFixture, base, head string) (map[string]bool, string) {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ got, data, err := scanRepoRange(ctx, t, f.path, base, head)+ assert.NoError(t, err)+ return got, data+}++// scanRepoRange runs a base..head diff scan over the repository at repoPath and+// reports the commits that produced chunks, the concatenated chunk data, and+// the scan error. Callers that expect a failing scan assert on the error.+func scanRepoRange(ctx context.Context, t *testing.T, repoPath, base, head string) (map[string]bool, string, error) {+ t.Helper()++ // Open through the package's own wrapper so the test takes the same+ // path real callers do (bare detection, .git discovery).+ repo, err := RepoFromPath(repoPath)+ if err != nil {+ return nil, "", err+ }++ g := NewGit(&Config{+ SourceName: "range fixture",+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,+ Concurrency: 1,+ SourceMetadataFunc: func(info SourceMetadataInfo) *source_metadatapb.MetaData {+ return &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Git{Git: &source_metadatapb.Git{Commit: info.Commit, File: info.File}},+ }+ },+ })++ chunksCh := make(chan *sources.Chunk, 64)+ scanErr := make(chan error, 1)+ go func() {+ defer close(chunksCh)+ scanErr <- g.ScanRepo(ctx, repo, repoPath, NewScanOptions(ScanOptionBaseHash(base), ScanOptionHeadCommit(head)), sources.ChanReporter{Ch: chunksCh})+ }()++ got := map[string]bool{}+ var data strings.Builder+ for c := range chunksCh {+ got[c.SourceMetadata.GetGit().GetCommit()] = true+ data.Write(c.Data)+ }+ return got, data.String(), <-scanErr+}++// TestScanRepo_BaseMergedIntoHead is the regression test for INT-1054: a+// diff scan must cover exactly `git log base..head` regardless of commit+// dates or whether base is reachable from head.+func TestScanRepo_BaseMergedIntoHead(t *testing.T) {+ // Every commit on the feature side of the range, i.e. git log C..F.+ wantScanned := []string{"F", "M", "E", "D"}+ // The merged-in base work and the common ancestor must never be scanned.+ wantSkipped := []string{"A", "B", "C"}++ cases := []struct {+ name string+ branchNewerThanBase bool+ unreachableBase bool+ base string // commit letter passed as --since-commit+ }{+ {+ // The customer's reproducer: base is the main tip that was merged in,+ // and the branch commits predate it.+ name: "base is the merged-in main tip and older branch commits are skipped",+ base: "C",+ },+ {+ // GitHub Action pull_request path: base.sha has moved past the merge,+ // so normalizeConfig resolves it to the merge-base C. Must match case 1.+ name: "base is unreachable from head",+ unreachableBase: true,+ base: "G",+ },+ {+ // Same topology, dates flipped: pins that the result is a function of+ // the graph, not of committer dates.+ name: "branch commits newer than the merged-in base tip",+ branchNewerThanBase: true,+ base: "C",+ },+ }++ // Both parser strategies build their `git log` from the same args, so both+ // must agree.+ for _, lowMemory := range []bool{false, true} {+ mode := "default"+ if lowMemory {+ mode = "low-memory"+ }+ t.Run(mode, func(t *testing.T) {+ feature.UseGitLowMemoryScan.Store(lowMemory)+ t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) })++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ f := buildMergedBaseFixture(t, tc.branchNewerThanBase, tc.unreachableBase)+ got, data := scanFixtureCommits(t, f, f.sha[tc.base], f.sha["F"])++ for _, letter := range wantScanned {+ assert.True(t, got[f.sha[letter]], "commit %s (%s) should have been scanned; got %v", letter, f.sha[letter][:7], got)+ }+ for _, letter := range wantSkipped {+ assert.False(t, got[f.sha[letter]], "commit %s (%s) is reachable from base and must not be scanned", letter, f.sha[letter][:7])+ }+ // The secrets live in D and E, the commits the pre-fix code dropped.+ assert.Contains(t, data, fixtureAWSKey, "AWS key from commit D missing from scanned data")+ assert.Contains(t, data, fixtureGitHubToken, "GitHub token from commit E/F missing from scanned data")+ })+ }+ })+ }+}++// TestNormalizeConfig_BaseWithoutHead pins what normalizeConfig does and does+// not do with a base and no head: it resolves the base to a hash and leaves the+// head empty. The parser then walks --all ^base (see gitparse.Parser.RepoPath).+// normalizeConfig must not invent a head, because that would route the+// base-only shape through MergeBase, which go-git cannot compute across the+// graft of a shallow clone, including the --shallow-since clone+// prepareRepoSinceCommit makes for exactly this shape.+func TestNormalizeConfig_BaseWithoutHead(t *testing.T) {+ // main: A; feature: A -> B, checked out.+ path := setupTestRepo(t, "base-without-head")+ addTestFileAndCommit(t, path, "a.txt", "a\n")+ runGit(t, path, "branch", "-M", "main")+ shaA := gitRevParse(t, path, "HEAD")+ runGit(t, path, "switch", "-q", "-c", "feature")+ addTestFileAndCommit(t, path, "b.txt", "b\n")+ shaB := gitRevParse(t, path, "HEAD")++ repo, err := RepoFromPath(path)+ assert.NoError(t, err)++ tests := []struct {+ name string+ base, head string+ wantBase string+ wantHead string+ }{+ {+ // The base is resolved to a hash; the head stays empty and no merge+ // base is computed. The parser turns this into --all ^main.+ name: "base ref and no head leaves head empty",+ base: "main", wantBase: shaA, wantHead: "",+ },+ {+ // The pre-commit hook passes HEAD as both ends (main.go). Both+ // resolve to the same hash and MergeBase short-circuits without+ // walking history, so the empty HEAD..HEAD range is safe on a+ // shallow local clone too.+ name: "base HEAD and head HEAD resolve to the same commit",+ base: "HEAD", head: "HEAD", wantBase: shaB, wantHead: shaB,+ },+ {+ // Full-history scans set neither end and must stay that way; an+ // implicit head here would silently narrow --all to one branch.+ name: "no base leaves head empty",+ base: "", wantBase: "", wantHead: "",+ },+ {+ name: "head without base is resolved but gets no base",+ head: "feature", wantBase: "", wantHead: shaB,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ opts := NewScanOptions(ScanOptionBaseHash(tt.base), ScanOptionHeadCommit(tt.head))+ assert.NoError(t, normalizeConfig(opts, repo))+ assert.Equal(t, tt.wantBase, opts.BaseHash, "BaseHash")+ assert.Equal(t, tt.wantHead, opts.HeadHash, "HeadHash")+ })+ }+}++// TestScanRepo_BaseWithoutHead runs the base-without-head shape end to end.+// `--since-commit X` with no `--branch` covers every commit since X on any+// ref (`git log --all ^X`), not just the checked-out branch: users scanning a+// mirror or a remote URL rely on that to catch secrets on side branches. The+// merged-base fixture gets an extra branch with its own planted secret that is+// not reachable from the checked-out feature branch; a base-only scan must+// find it, and the pre-commit hook, which passes HEAD as both ends, must not.+func TestScanRepo_BaseWithoutHead(t *testing.T) {+ const (+ sideSecret = "ghp_SideBranchTokenThatBaseOnlyMustCatch000"+ stagedSecret = "ghp_StagedTokenThatPreCommitMustStillCatch0"+ )++ // Fixture ends checked out on feature at F. Add the side branch off the+ // common ancestor and come back to feature so HEAD is F.+ build := func(t *testing.T) mergedBaseFixture {+ f := buildMergedBaseFixture(t, false, false)+ runGit(t, f.path, "switch", "-q", "-c", "side", f.sha["A"])+ addTestFileAndCommit(t, f.path, "side.txt", "github_token = "+sideSecret+"\n")+ f.sha["X"] = gitRevParse(t, f.path, "HEAD")+ runGit(t, f.path, "switch", "-q", "feature")+ return f+ }++ // assertAllSinceC checks the --all ^C set: everything on feature after the+ // merge base plus the side branch, never the base's own history.+ assertAllSinceC := func(t *testing.T, f mergedBaseFixture, got map[string]bool, data string) {+ t.Helper()+ for _, letter := range []string{"F", "M", "E", "D", "X"} {+ assert.True(t, got[f.sha[letter]], "commit %s should have been scanned; got %v", letter, got)+ }+ for _, letter := range []string{"A", "B", "C"} {+ assert.False(t, got[f.sha[letter]], "commit %s is reachable from C and must not be scanned", letter)+ }+ assert.Contains(t, data, sideSecret, "a base-only scan must cover branches other than the checked-out one")+ }++ for _, lowMemory := range []bool{false, true} {+ mode := "default"+ if lowMemory {+ mode = "low-memory"+ }+ t.Run(mode, func(t *testing.T) {+ feature.UseGitLowMemoryScan.Store(lowMemory)+ t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) })++ t.Run("base only scans every ref since base", func(t *testing.T) {+ f := build(t)+ got, data := scanFixtureCommits(t, f, f.sha["C"], "")+ assertAllSinceC(t, f, got, data)+ })++ t.Run("explicit head narrows to one branch", func(t *testing.T) {+ f := build(t)+ got, data := scanFixtureCommits(t, f, f.sha["C"], f.sha["F"])++ // git log C..F: the side branch is not reachable from F.+ for _, letter := range []string{"F", "M", "E", "D"} {+ assert.True(t, got[f.sha[letter]], "commit %s should have been scanned; got %v", letter, got)+ }+ for _, letter := range []string{"A", "B", "C", "X"} {+ assert.False(t, got[f.sha[letter]], "commit %s is outside C..F and must not be scanned", letter)+ }+ assert.NotContains(t, data, sideSecret)+ })++ // The history from the PR review, where the old --all walk that+ // stopped at the base happened to cover the side branch:+ //+ // o1 - o2 other+ // /+ // a - b - c main <- HEAD+ //+ // --since-commit a with no --branch must still scan o1 and o2.+ t.Run("commits only on other branches are scanned", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ path := setupTestRepo(t, "cross-ref")+ sha := map[string]string{}+ commit := func(name string) {+ addTestFileAndCommit(t, path, name+".txt", name+"\n")+ sha[name] = gitRevParse(t, path, "HEAD")+ }+ commit("a")+ runGit(t, path, "branch", "-M", "main")+ runGit(t, path, "switch", "-q", "-c", "other")+ commit("o1")+ commit("o2")+ runGit(t, path, "switch", "-q", "main")+ commit("b")+ commit("c")++ got, _, err := scanRepoRange(ctx, t, path, sha["a"], "")+ assert.NoError(t, err)+ for _, name := range []string{"b", "c", "o1", "o2"} {+ assert.True(t, got[sha[name]], "commit %s should have been scanned; got %v", name, got)+ }+ assert.False(t, got[sha["a"]], "the base is excluded from its own range")++ // With --branch main the same base covers main only.+ got, _, err = scanRepoRange(ctx, t, path, sha["a"], "main")+ assert.NoError(t, err)+ for _, name := range []string{"b", "c"} {+ assert.True(t, got[sha[name]], "commit %s should have been scanned; got %v", name, got)+ }+ for _, name := range []string{"a", "o1", "o2"} {+ assert.False(t, got[sha[name]], "commit %s is outside a..main and must not be scanned", name)+ }+ })++ // main.go's pre-commit override sets both ends to HEAD, so the+ // commit range is empty and only staged changes are scanned.+ t.Run("pre-commit shape scans only staged changes", func(t *testing.T) {+ f := build(t)+ assert.NoError(t, os.WriteFile(filepath.Join(f.path, "staged.txt"), []byte("github_token = "+stagedSecret+"\n"), 0o644))+ runGit(t, f.path, "add", "staged.txt")++ got, data := scanFixtureCommits(t, f, "HEAD", "HEAD")++ // HEAD..HEAD is empty, so no commit in the repository may produce+ // chunks. Staged chunks carry no commit hash and are asserted on+ // through the data instead.+ for letter, sha := range f.sha {+ assert.False(t, got[sha], "commit %s was scanned but HEAD..HEAD is empty; got %v", letter, got)+ }+ assert.Contains(t, data, stagedSecret, "staged changes must still be scanned")+ assert.NotContains(t, data, sideSecret, "a commit on another branch leaked into the hook scan")+ })++ // Remote URLs are cloned with --mirror, so production base-only scans+ // run against a bare repository and every ref the mirror carries is+ // in scope.+ t.Run("bare mirror clone scans every ref since base", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ f := build(t)+ mirror := filepath.Join(t.TempDir(), "mirror.git")+ runGit(t, "", "clone", "-q", "--mirror", "file://"+f.path, mirror)+ assert.True(t, isRepoBare(mirror), "fixture is not bare")++ got, data, err := scanRepoRange(ctx, t, mirror, f.sha["C"], "")+ assert.NoError(t, err)+ assertAllSinceC(t, f, got, data)+ })+ })+ }+}++// TestScanRepo_SwappedRangeEnds pins the behavior when the base is a descendant+// of the head, e.g. a CI job that passes its arguments in the wrong order. The+// range base..head is empty, so nothing is scanned and the scan succeeds; it+// must not widen to anything else. ScanCommits logs the empty range at Info so+// the case is visible at the default log level without failing the pre-commit+// hook, whose HEAD..HEAD range is empty by design.+func TestScanRepo_SwappedRangeEnds(t *testing.T) {+ f := buildMergedBaseFixture(t, false, false)++ // base F is a descendant of head C: git log C ^F is empty.+ got, data := scanFixtureCommits(t, f, f.sha["F"], f.sha["C"])++ assert.Empty(t, got, "an empty range must not scan any commit; got %v", got)+ assert.NotContains(t, data, fixtureAWSKey)+ assert.NotContains(t, data, fixtureGitHubToken)+}++// TestScanRepo_BaseNotUsable pins the behavior when a diff scan is given a base+// it cannot turn into a commit in the repository. Every case must fail the scan:+// the tempting alternative, degrading to a full-history scan, reports success+// for a scan that covered a different range than the user asked for, which is+// the class of silent miss INT-1054 is about.+func TestScanRepo_BaseNotUsable(t *testing.T) {+ // A syntactically valid hash that is not in any fixture below.+ const absentHash = "1111111111111111111111111111111111111111"++ tests := []struct {+ name string+ // setup returns the repo path plus the base and head to scan.+ setup func(t *testing.T) (repoPath, base, head string)+ wantErr string+ }{+ {+ // A branch name that does not exist, e.g. a CI job passing a deleted+ // or misspelled base branch.+ name: "base ref does not exist",+ setup: func(t *testing.T) (string, string, string) {+ path := setupTestRepo(t, "unknown-ref")+ addTestFileAndCommit(t, path, "a.txt", "a\n")+ return path, "no-such-branch", gitRevParse(t, path, "HEAD")+ },+ wantErr: "unable to resolve ref",+ },+ {+ // A well-formed hash for an object the repository does not have. This+ // is what an externally shallow checkout looks like when the base is+ // older than the fetch depth.+ name: "base hash is not in the repository",+ setup: func(t *testing.T) (string, string, string) {+ path := setupTestRepo(t, "absent-hash")+ addTestFileAndCommit(t, path, "a.txt", "a\n")+ return path, absentHash, gitRevParse(t, path, "HEAD")+ },+ wantErr: "unable to resolve commit",+ },+ {+ // Two root commits: both refs resolve, but they share no history, so+ // there is no range between them to scan.+ name: "base and head have no common ancestor",+ setup: func(t *testing.T) (string, string, string) {+ path := setupTestRepo(t, "unrelated")+ addTestFileAndCommit(t, path, "a.txt", "a\n")+ base := gitRevParse(t, path, "HEAD")+ runGit(t, path, "switch", "-q", "--orphan", "unrelated")+ addTestFileAndCommit(t, path, "b.txt", "b\n")+ return path, base, gitRevParse(t, path, "HEAD")+ },+ wantErr: "merge base",+ },+ {+ // actions/checkout at its default fetch-depth: the clone holds only+ // the tip, so a base from before the cutoff is simply absent.+ name: "shallow clone does not contain the base",+ setup: func(t *testing.T) (string, string, string) {+ origin := setupTestRepo(t, "shallow-origin")+ addTestFileAndCommit(t, origin, "a.txt", "a\n")+ base := gitRevParse(t, origin, "HEAD")+ addTestFileAndCommit(t, origin, "b.txt", "b\n")++ // --depth needs a file:// URL; git ignores it for plain local paths.+ shallow := filepath.Join(t.TempDir(), "shallow")+ runGit(t, "", "clone", "-q", "--depth", "1", "file://"+origin, shallow)+ assert.False(t, gitHasObject(t, shallow, base), "fixture is not shallow: base is present")++ return shallow, base, gitRevParse(t, shallow, "HEAD")+ },+ wantErr: "unable to resolve commit",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ repoPath, base, head := tt.setup(t)+ got, _, err := scanRepoRange(ctx, t, repoPath, base, head)++ assert.ErrorContains(t, err, tt.wantErr)+ assert.Empty(t, got, "a scan that cannot resolve its base must not fall back to scanning commits")+ })+ }+}++// TestScanRepo_ShallowClone pins where the graft boundary of a shallow clone+// starts to matter for a diff scan. git itself handles a truncated history+// fine, so the boundary that counts is the one go-git needs to walk in+// normalizeConfig: MergeBase loads a commit's parents before it can recognize+// the commit as the range boundary (bfsCommitIterator.Next), so a base sitting+// on the graft has no resolvable merge base.+//+// That walk only runs when both ends are supplied. With a base alone the+// parser defaults the head to HEAD and git computes the range, so the same+// clone scans cleanly; this is the shape `--since-commit X` without `--branch`+// takes against a GitHub URL, where prepareRepoSinceCommit clones with+// --shallow-since and puts X on the graft. See+// https://github.com/trufflesecurity/trufflehog/issues/4895 for the both-ends+// failure surfacing on GitLab.+func TestScanRepo_ShallowClone(t *testing.T) {+ // A fixture deep enough that a clone can keep the base and still cut the+ // history off somewhere above it.+ newOrigin := func(t *testing.T) string {+ origin := setupTestRepo(t, "shallow-origin")+ for _, f := range []string{"a.txt", "b.txt", "c.txt", "d.txt"} {+ addTestFileAndCommit(t, origin, f, f+"\n")+ }+ return origin+ }+ clone := func(t *testing.T, origin, depth string) string {+ // --depth needs a file:// URL; git ignores it for plain local paths.+ path := filepath.Join(t.TempDir(), "shallow")+ runGit(t, "", "clone", "-q", "--depth", depth, "file://"+origin, path)+ return path+ }+ // Depth 2 keeps the tip and the base; the base's parent is cut off, which+ // is the shape --shallow-since produces for its own base commit.+ graftAtBase := func(t *testing.T) (shallow, head, base string) {+ shallow = clone(t, newOrigin(t), "2")+ head, base = gitRevParse(t, shallow, "HEAD"), gitRevParse(t, shallow, "HEAD~1")+ assert.False(t, gitHasObject(t, shallow, base+"^"), "fixture is wrong: the base's parent is present")+ return shallow, head, base+ }++ // Both parser strategies build their `git log` from the same args, so both+ // must agree.+ for _, lowMemory := range []bool{false, true} {+ mode := "default"+ if lowMemory {+ mode = "low-memory"+ }+ t.Run(mode, func(t *testing.T) {+ feature.UseGitLowMemoryScan.Store(lowMemory)+ t.Cleanup(func() { feature.UseGitLowMemoryScan.Store(false) })++ t.Run("base is the graft boundary", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ shallow, head, base := graftAtBase(t)++ got, _, err := scanRepoRange(ctx, t, shallow, base, head)++ assert.ErrorContains(t, err, "unable to resolve merge base")+ assert.Empty(t, got, "an unresolvable merge base must fail the scan, not scan an arbitrary range")+ })++ t.Run("base is the graft boundary with implicit head", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ // Same clone, no head: normalizeConfig must not invent a head+ // and run MergeBase, or the --shallow-since path would fail+ // exactly where it used to work. git walks --all ^base, and the+ // clone's only ref is the tip.+ shallow, head, base := graftAtBase(t)++ got, _, err := scanRepoRange(ctx, t, shallow, base, "")++ assert.NoError(t, err)+ assert.True(t, got[head], "the one commit since base should have been scanned; got %v", got)+ assert.False(t, got[base], "the base is excluded from its own range")+ })++ t.Run("base is above the graft boundary", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ // One more commit of depth is all it takes: the base's parent is+ // present, so the merge base resolves and git scans the range over+ // a history it cannot fully walk.+ shallow := clone(t, newOrigin(t), "3")+ head, base := gitRevParse(t, shallow, "HEAD"), gitRevParse(t, shallow, "HEAD~1")+ assert.True(t, gitHasObject(t, shallow, base+"^"), "fixture is wrong: the base's parent is missing")++ got, _, err := scanRepoRange(ctx, t, shallow, base, head)++ assert.NoError(t, err)+ assert.True(t, got[head], "the one commit in base..head should have been scanned; got %v", got)+ assert.False(t, got[base], "the base is excluded from its own range")+ })+ })+ }+}++// runGit runs a git command, failing the test on error. An empty repoPath runs+// git outside any repository, which clone needs.+func runGit(t *testing.T, repoPath string, args ...string) {+ t.Helper()+ if repoPath != "" {+ args = append([]string{"-C", repoPath}, args...)+ }+ out, err := exec.Command("git", args...).CombinedOutput()+ assert.NoError(t, err, "git %v: %s", args, out)+}++func gitRevParse(t *testing.T, repoPath, rev string) string {+ t.Helper()+ out, err := exec.Command("git", "-C", repoPath, "rev-parse", rev).Output()+ assert.NoError(t, err)+ return strings.TrimSpace(string(out))+}++// gitHasObject reports whether the repository holds the commit named by rev,+// used to confirm where a shallow fixture's graft boundary actually landed+// before asserting on behavior that depends on it.+func gitHasObject(t *testing.T, repoPath, rev string) bool {+ t.Helper()+ return exec.Command("git", "-C", repoPath, "cat-file", "-e", rev+"^{commit}").Run() == nil+}