Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
add ability to run github-experimental against private repos (#4508)
joeleonjrfd007c74cc95
diff --git a/pkg/sources/github_experimental/github_experimental.go b/pkg/sources/github_experimental/github_experimental.goindex 5a9ca11d0..58be356f9 100644--- a/pkg/sources/github_experimental/github_experimental.go+++ b/pkg/sources/github_experimental/github_experimental.go@@ -2,26 +2,26 @@ package github_experimentalimport ("fmt"- "net/http""strings""github.com/go-logr/logr"- "github.com/google/go-github/v67/github""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/giturl"+ "github.com/trufflesecurity/trufflehog/v3/pkg/log""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"+ githubsource "github.com/trufflesecurity/trufflehog/v3/pkg/sources/github")const (- SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL+ SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL+ cloudV3Endpoint = "https://api.github.com")type Source struct {@@ -33,10 +33,9 @@ type Source struct {useCustomContentWriter boolgit *git.GitscanOptions *git.ScanOptions- httpClient *http.Clientlog logr.Loggerconn *sourcespb.GitHubExperimental- apiClient *github.Client+ connector githubsource.Connectorsources.Progresssources.CommonSourceUnitUnmarshaller@@ -81,9 +80,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sos.jobID = jobIDs.verify = verify- s.httpClient = common.RetryableHTTPClientTimeout(60)- s.apiClient = github.NewClient(s.httpClient)-var conn sourcespb.GitHubExperimentalerr = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})if err != nil {@@ -95,6 +91,32 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, soreturn fmt.Errorf("error normalizing repo: %w", err)}+ // Get the token from the connection+ token := s.conn.GetToken()+ if token == "" {+ return fmt.Errorf("token is required for GitHub Experimental source")+ }++ // Redact token from logs for security+ log.RedactGlobally(token)++ // Create authenticated connector using the TokenConnector pattern+ connector, err := githubsource.NewTokenConnector(+ aCtx,+ cloudV3Endpoint, // API endpoint+ token, // GitHub token+ "", // clonePath (empty for default)+ true, // authInUrl+ func(ctx context.Context, err error) bool {+ // Simple rate limit handler - can be enhanced later+ return false+ },+ )+ if err != nil {+ return fmt.Errorf("could not create GitHub connector: %w", err)+ }+ s.connector = connector+s.repoInfoCache = newRepoInfoCache()cfg := &git.Config{diff --git a/pkg/sources/github_experimental/object_discovery.go b/pkg/sources/github_experimental/object_discovery.goindex 12b0379dc..a09d8415e 100644--- a/pkg/sources/github_experimental/object_discovery.go+++ b/pkg/sources/github_experimental/object_discovery.go@@ -14,11 +14,9 @@ import ("github.com/google/go-github/v67/github""github.com/k0kubun/go-ansi""github.com/schollz/progressbar/v3"- "golang.org/x/oauth2""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/sources"- "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git")// Assumption: sleeping for 60 seconds is enough to reset the secondary rate limit@@ -100,18 +98,7 @@ func (b *backoff) getValue() int {return int(b.value)}-// Github token-var ghToken = ""--func getForksCount(owner, repoName string) (int, error) {- ctx := context.Background()- ts := oauth2.StaticTokenSource(- &oauth2.Token{AccessToken: ghToken},- )- tc := oauth2.NewClient(ctx, ts)-- client := github.NewClient(tc)-+func getForksCount(ctx context.Context, client *github.Client, owner, repoName string) (int, error) {repo, _, err := client.Repositories.Get(ctx, owner, repoName)if err != nil {return 0, err@@ -120,22 +107,6 @@ func getForksCount(owner, repoName string) (int, error) {return repo.GetForksCount(), nil}-func getGitHubUser() (string, error) {- ctx := context.Background()- ts := oauth2.StaticTokenSource(- &oauth2.Token{AccessToken: ghToken},- )- tc := oauth2.NewClient(ctx, ts)-- client := github.NewClient(tc)-- ghUser, _, err := client.Users.Get(ctx, "")- if err != nil {- return "", err- }- return ghUser.GetLogin(), nil-}-// runGitCommand runs a git commandfunc runGitCommand(args []string) ([]byte, error) {cmd := exec.Command("git", args...)@@ -322,7 +293,7 @@ func removeBySHA(existingCommits, newCommits []string, charLen int) []string {return filteredCommits}-func processCommits(ctx context.Context, needsProcessing []string, owner, repo, path string) {+func processCommits(ctx context.Context, apiClient *github.Client, needsProcessing []string, owner, repo, path string) {repoCtx := context.WithValue(ctx, "repo", repo)startingSize := float64(len(needsProcessing))@@ -343,10 +314,12 @@ func processCommits(ctx context.Context, needsProcessing []string, owner, repo,chunk := needsProcessing[:chunkSize]needsProcessing = needsProcessing[chunkSize:]- commitData, err := checkHashes(owner, repo, chunk)+ commitData, err := checkHashes(repoCtx, apiClient, owner, repo, chunk)if err != nil {repoCtx.Logger().V(2).Info("Temporary error occurred in guessing commits", "error", err)- needsProcessing = append(needsProcessing, chunk...)+ // Prepend the failed chunk to the FRONT of the queue for immediate retry+ // This ensures we retry the same hashes instead of moving to the next batch+ needsProcessing = append(chunk, needsProcessing...)queryChunkSize.errorOccurred()if strings.Contains(err.Error(), "You have exceeded a secondary rate limit") {repoCtx.Logger().V(2).Info("Reached secondary GitHub Rate Limit. Sleeping for 60 seconds.")@@ -391,7 +364,7 @@ type responseData struct {Message string `json:"message"`}-func checkHashes(owner, repo string, hashes []string) (map[string][]string, error) {+func checkHashes(ctx context.Context, client *github.Client, owner, repo string, hashes []string) (map[string][]string, error) {testCases := ""for _, h := range hashes {testCase := fmt.Sprintf(`@@ -413,7 +386,6 @@ func checkHashes(owner, repo string, hashes []string) (map[string][]string, erro`, owner, repo, testCases)headers := map[string]string{- "Authorization": "Bearer " + ghToken,"Content-Type": "application/json","Github-Verified-Fetch": "true","X-Requested-With": "XMLHttpRequest",@@ -426,7 +398,7 @@ func checkHashes(owner, repo string, hashes []string) (map[string][]string, erroreturn nil, fmt.Errorf("failed to marshal request body: %w", err)}- req, err := http.NewRequest("POST", "https://api.github.com/graphql", bytes.NewBuffer(requestBody))+ req, err := http.NewRequestWithContext(ctx, "POST", "https://api.github.com/graphql", bytes.NewBuffer(requestBody))if err != nil {return nil, fmt.Errorf("failed to create request: %w", err)}@@ -435,8 +407,9 @@ func checkHashes(owner, repo string, hashes []string) (map[string][]string, erroreq.Header.Set(key, value)}- client := &http.Client{}- resp, err := client.Do(req)+ // Use the authenticated HTTP client from the GitHub API client+ // This client already has the Bearer token configured via OAuth2 transport+ resp, err := client.Client().Do(req)if err != nil {return nil, fmt.Errorf("python request error: %w", err)}@@ -546,9 +519,6 @@ func downloadPatches(valid_cfor []string, path string) error {// scanHiddenData scans hidden data (and non-hidden data) for secrets in a GitHub repositoryfunc (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan *sources.Chunk) error {- // assign github token to global variable- ghToken = s.conn.GetToken()-// set collision threshold to user inputcollisionThreshold = float64(s.conn.CollisionThreshold)@@ -564,7 +534,7 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan// get repo metadata and store in cacheRepoInforepoCtx := context.WithValue(ctx, "repo", owner+"/"+repoName)- ghRepo, _, err := s.apiClient.Repositories.Get(repoCtx, owner, repoName)+ ghRepo, _, err := s.connector.APIClient().Repositories.Get(repoCtx, owner, repoName)if err != nil {return fmt.Errorf("failed to fetch repository: %w", err)}@@ -582,20 +552,14 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chanreturn fmt.Errorf("failed to create .trufflehog folder in user's home directory: %w", err)}- // Get GitHub User tied to token- ghUser, err := getGitHubUser()- if err != nil {- return fmt.Errorf("failed to get GitHub user details: %w", err)- }-// get the number of forks- forksCount, err := getForksCount(owner, repoName)+ forksCount, err := getForksCount(repoCtx, s.connector.APIClient(), owner, repoName)if err != nil {return fmt.Errorf("failed to get forks count: %w", err)}- // download the repo- path, repo, err := git.CloneRepoUsingToken(ctx, ghToken, repoURL, "", ghUser, true)+ // download the repo using the authenticated connector+ path, repo, err := s.connector.Clone(ctx, repoURL)if err != nil {return fmt.Errorf("failed to clone the repository: %w", err)}@@ -639,7 +603,7 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chanpossibleCommits = removeByShortSHA(invalidCommits, possibleCommits)// Guess all possible commit hashes- processCommits(ctx, possibleCommits, owner, repoName, folderPath)+ processCommits(ctx, s.connector.APIClient(), possibleCommits, owner, repoName, folderPath)// Read in the new commitsvalidHiddenCommits, err = readCommitsFromDisk(validHiddenCommit, folderPath)diff --git a/pkg/sources/github_experimental/repo.go b/pkg/sources/github_experimental/repo.goindex 7c8c42a65..3b4bf6760 100644--- a/pkg/sources/github_experimental/repo.go+++ b/pkg/sources/github_experimental/repo.go@@ -2,6 +2,7 @@ package github_experimentalimport ("fmt"+ "regexp""strings""sync"@@ -62,10 +63,15 @@ func (s *Source) cacheRepoInfo(r *github.Repository) {}func (s *Source) normalizeRepo(repo string) (string, error) {- // If there's a '/', assume it's a URL and try to normalize it.- if strings.ContainsRune(repo, '/') {+ // If it's a full URL (has protocol), normalize it+ if regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {return giturl.NormalizeGithubRepo(repo)}+ // If it's a repository name (contains / but not http), convert to full URL first+ if strings.Contains(repo, "/") && !regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {+ fullURL := "https://github.com/" + repo+ return giturl.NormalizeGithubRepo(fullURL)+ }return "", fmt.Errorf("no repositories found for %s", repo)}