Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Retry git clone on transient network errors (#5132)
Shahzad Haider38999f88bed8
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 8866ab276..8cfd1cfab 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -38,7 +38,15 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources")-const SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT+const (+ SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT+ // maxCloneAttempts is the total number of times a clone is attempted when+ // each failure is classified as a transient network error.+ maxCloneAttempts = 3+ // cloneRetryBackoff is the base wait between clone attempts; it is+ // multiplied by the number of failed attempts so far.+ cloneRetryBackoff = 5 * time.Second+)type Source struct {name string@@ -474,38 +482,86 @@ type cloneParams struct {// infrastructure, ensuring that any encountered errors trigger a cleanup of resources.// The core cloning logic is delegated to a nested function, which returns errors to the// outer function for centralized error handling and cleanup.+//+// Failures classified as transient network errors (e.g. a connection reset+// mid-transfer) are retried up to maxCloneAttempts times, each attempt+// starting from a fresh clone directory. All other failures, including clone+// timeouts (see feature.GitCloneTimeoutDuration), are returned immediately.func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitURL string, clonePath string, authInUrl bool, args ...string) (string, *git.Repository, error) {+ timeout := time.Duration(feature.GitCloneTimeoutDuration.Load())+var path string+ var repo *git.Repositoryvar err error+ for attempt := 1; attempt <= maxCloneAttempts; attempt++ {+ // Each attempt starts from a fresh clone directory.+ path, err = createClonePath(gitURL, clonePath)+ if err != nil {+ return "", nil, err+ }- // If --clone-path is set, create a subdirectory <clonePath>/trufflehog-<repo-name> with permissions 0755.- if clonePath != "" {- path = filepath.Join(clonePath, "trufflehog-"+strings.TrimSuffix(filepath.Base(gitURL), gitDirName))- if err = os.MkdirAll(path, 0755); err != nil {- return "", nil, fmt.Errorf("failed to create clone path %s: %w", clonePath, err)+ repo, err = executeClone(ctx, cloneParams{userInfo, gitURL, args, path, authInUrl, timeout})+ if err == nil {+ return path, repo, nil}- } else {- // otherwise, create a temporary directory in the system temp path.- path, err = cleantemp.MkdirTemp()- if err != nil {- return "", nil, fmt.Errorf("failed to create temporary clone path: %w", err)++ if attempt >= maxCloneAttempts || !isRetryableCloneError(err) || common.IsDone(ctx) {+ // DO NOT FORGET TO CLEAN UP THE CLONE PATH HERE!!+ // If we don't, we'll end up with a bunch of orphaned directories in the temp dir.+ CleanOnError(&err, path)++ // Note: We don't need to record the clone failure here as it's already+ // recorded in executeClone when the error occurs+ return "", nil, err+ }++ // Discard the partial clone; the next iteration recreates a fresh directory.+ if rmErr := os.RemoveAll(path); rmErr != nil {+ return "", nil, fmt.Errorf("failed to clean clone path for retry: %w (original clone error: %w)", rmErr, err)+ }++ ctx.Logger().Info("git clone interrupted by network error; retrying",+ "attempt", attempt,+ "max_attempts", maxCloneAttempts,+ "error", err.Error(),+ )+ select {+ case <-ctx.Done():+ return "", nil, ctx.Err()+ case <-time.After(cloneRetryBackoff * time.Duration(attempt)):}}- timeout := time.Duration(feature.GitCloneTimeoutDuration.Load())+ return "", nil, err+}- repo, err := executeClone(ctx, cloneParams{userInfo, gitURL, args, path, authInUrl, timeout})- if err != nil {- // DO NOT FORGET TO CLEAN UP THE CLONE PATH HERE!!- // If we don't, we'll end up with a bunch of orphaned directories in the temp dir.- CleanOnError(&err, path)+// isRetryableCloneError reports whether a clone failure looks like a+// transient network interruption (e.g. a connection reset mid-transfer)+// rather than a permanent condition like an auth or permission error.+func isRetryableCloneError(err error) bool {+ return err != nil && ClassifyCloneError(err.Error()) == cloneFailureNetwork+}- // Note: We don't need to record the clone failure here as it's already- // recorded in executeClone when the error occurs- return "", nil, err+// createClonePath creates the directory a repository will be cloned into and+// returns its path. When clonePath is set (the --clone-path flag), the+// directory is <clonePath>/trufflehog-<repo-name> with permissions 0755;+// otherwise a fresh temporary directory (0700, per os.MkdirTemp) is created+// in the system temp path. It is called both before the first clone attempt+// and to replace the directory between retries.+func createClonePath(gitURL, clonePath string) (string, error) {+ if clonePath == "" {+ path, err := cleantemp.MkdirTemp()+ if err != nil {+ return "", fmt.Errorf("failed to create temporary clone path: %w", err)+ }+ return path, nil}- return path, repo, nil+ path := filepath.Join(clonePath, "trufflehog-"+strings.TrimSuffix(filepath.Base(gitURL), gitDirName))+ if err := os.MkdirAll(path, 0755); err != nil {+ return "", fmt.Errorf("failed to create clone path %s: %w", clonePath, err)+ }+ return path, nil}// executeClone prepares the Git URL, constructs, and executes the git clone command using the provideddiff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex ca04ee710..bbc93a721 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -2,6 +2,7 @@ package gitimport ("bytes"+ "errors""fmt""os""os/exec"@@ -61,6 +62,131 @@ func TestClone_Timeout(t *testing.T) {})}+func TestIsRetryableCloneError(t *testing.T) {+ retryable := []string{+ // Connection reset mid-transfer.+ `could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, error: RPC failed; curl 56 Recv failure: Connection reset by peer+error: 7589 bytes of body are still expected+fetch-pack: unexpected disconnect while reading sideband packet+fatal: early EOF+fatal: fetch-pack: invalid index-pack output`,+ // HTTP/2 stream reset by the server.+ `could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, error: RPC failed; curl 92 HTTP/2 stream 7 reset by server (error 0x8 CANCEL)+error: 7457 bytes of body are still expected+fetch-pack: unexpected disconnect while reading sideband packet+fatal: early EOF+fatal: fetch-pack: invalid index-pack output`,+ }+ for _, msg := range retryable {+ assert.True(t, isRetryableCloneError(errors.New(msg)), "expected retryable: %q", msg)+ }++ notRetryable := []string{+ "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, remote: The requested URL returned error: 403",+ "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, The requested URL returned error: 429",+ "git clone timed out (after 1h0m0s)",+ "could not clone repo: https://github.com/org/repo.git, error executing git clone: exit status 128, fatal: repository 'https://github.com/org/repo.git/' not found",+ }+ for _, msg := range notRetryable {+ assert.False(t, isRetryableCloneError(errors.New(msg)), "expected not retryable: %q", msg)+ }+ assert.False(t, isRetryableCloneError(nil))+}++func TestCreateClonePath(t *testing.T) {+ t.Run("temp dir when clonePath is empty", func(t *testing.T) {+ path, err := createClonePath("https://github.com/org/repo.git", "")+ assert.NoError(t, err)+ defer func() { _ = os.RemoveAll(path) }()++ info, err := os.Stat(path)+ assert.NoError(t, err)+ assert.True(t, info.IsDir())+ if runtime.GOOS != "windows" {+ // os.MkdirTemp guarantees 0700 so private repo contents aren't+ // exposed to other users on shared systems.+ assert.Equal(t, os.FileMode(0700), info.Mode().Perm())+ }+ })++ t.Run("temp dirs are unique across calls", func(t *testing.T) {+ // A retry replaces the previous directory; a colliding name would+ // mean cloning into a non-empty directory, which git refuses.+ first, err := createClonePath("https://github.com/org/repo.git", "")+ assert.NoError(t, err)+ defer func() { _ = os.RemoveAll(first) }()++ second, err := createClonePath("https://github.com/org/repo.git", "")+ assert.NoError(t, err)+ defer func() { _ = os.RemoveAll(second) }()++ assert.NotEqual(t, first, second)+ })++ t.Run("clonePath set builds trufflehog-<repo> subdirectory", func(t *testing.T) {+ base := t.TempDir()+ path, err := createClonePath("https://github.com/org/repo.git", base)+ assert.NoError(t, err)+ assert.Equal(t, filepath.Join(base, "trufflehog-repo"), path)++ info, err := os.Stat(path)+ assert.NoError(t, err)+ assert.True(t, info.IsDir())+ if runtime.GOOS != "windows" {+ assert.Equal(t, os.FileMode(0755), info.Mode().Perm())+ }+ })++ t.Run("repo name without .git suffix", func(t *testing.T) {+ base := t.TempDir()+ path, err := createClonePath("https://github.com/org/repo", base)+ assert.NoError(t, err)+ assert.Equal(t, filepath.Join(base, "trufflehog-repo"), path)+ })++ t.Run("trailing slash in repo URL", func(t *testing.T) {+ base := t.TempDir()+ path, err := createClonePath("https://github.com/org/repo.git/", base)+ assert.NoError(t, err)+ assert.Equal(t, filepath.Join(base, "trufflehog-repo"), path)+ })++ t.Run("nonexistent clonePath parents are created", func(t *testing.T) {+ base := filepath.Join(t.TempDir(), "a", "b", "c")+ path, err := createClonePath("https://github.com/org/repo.git", base)+ assert.NoError(t, err)+ assert.Equal(t, filepath.Join(base, "trufflehog-repo"), path)++ info, err := os.Stat(path)+ assert.NoError(t, err)+ assert.True(t, info.IsDir())+ })++ t.Run("second call with same arguments reuses the directory", func(t *testing.T) {+ // The retry path calls this again after RemoveAll; it must also+ // tolerate the directory already existing (MkdirAll semantics).+ base := t.TempDir()+ first, err := createClonePath("https://github.com/org/repo.git", base)+ assert.NoError(t, err)++ second, err := createClonePath("https://github.com/org/repo.git", base)+ assert.NoError(t, err)+ assert.Equal(t, first, second)+ })++ t.Run("error when clonePath location is not writable", func(t *testing.T) {+ base := t.TempDir()+ // Create a *file* where the subdirectory should go so MkdirAll fails.+ blocker := filepath.Join(base, "trufflehog-repo")+ assert.NoError(t, os.WriteFile(blocker, []byte("x"), 0644))++ path, err := createClonePath("https://github.com/org/repo.git", base)+ assert.Error(t, err)+ assert.Empty(t, path)+ assert.Contains(t, err.Error(), "failed to create clone path")+ })+}+func TestSource_Scan(t *testing.T) {ctx, cancel := context.WithCancel(context.Background())defer cancel()