Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improve handling of Gist URLs (#2653)
Richard Gomeze53f5bd5c571
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 4571224a0..332ecb9cd 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -427,11 +427,14 @@ RepoLoop:continue}- if strings.EqualFold(urlParts[0], "gist.github.com") {+ if isGistUrl(urlParts) {// Cache gist info.for {gistID := extractGistID(urlParts)gist, _, err := s.apiClient.Gists.Get(repoCtx, gistID)+ // Normalize the URL to the Gist's pull URL.+ // See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937+ r = gist.GetGitPullURL()if s.handleRateLimit(err) {continue}@@ -1105,7 +1108,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo repreturn err}- if s.includeGistComments && urlParts[0] == "gist.github.com" {+ if s.includeGistComments && isGistUrl(urlParts) {return s.processGistComments(ctx, urlString, urlParts, repoInfo, chunksChan)} else if s.includeIssueComments || s.includePRComments {return s.processRepoComments(ctx, repoInfo, chunksChan)@@ -1120,29 +1123,50 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep// - "https://github.com/trufflesecurity/trufflehog" => ["github.com", "trufflesecurity", "trufflehog"]// - "https://gist.github.com/nat/5fdbb7f945d121f197fb074578e53948" => ["gist.github.com", "nat", "5fdbb7f945d121f197fb074578e53948"]// - "https://gist.github.com/ff0e5e8dc8ec22f7a25ddfc3492d3451.git" => ["gist.github.com", "ff0e5e8dc8ec22f7a25ddfc3492d3451"]-func getRepoURLParts(repoURL string) (string, []string, error) {+// - "https://github.company.org/gist/nat/5fdbb7f945d121f197fb074578e53948.git" => ["github.company.org", "gist", "nat", "5fdbb7f945d121f197fb074578e53948"]+func getRepoURLParts(repoURLString string) (string, []string, error) {// Support ssh and https URLs.- url, err := git.GitURLParse(repoURL)+ repoURL, err := git.GitURLParse(repoURLString)if err != nil {- return "", []string{}, err+ return "", nil, err}// Remove the user information.// e.g., `[email protected]` -> `github.com`- if url.User != nil {- url.User = nil+ if repoURL.User != nil {+ repoURL.User = nil}- urlString := url.String()- trimmedURL := strings.TrimPrefix(urlString, url.Scheme+"://")+ urlString := repoURL.String()+ trimmedURL := strings.TrimPrefix(urlString, repoURL.Scheme+"://")trimmedURL = strings.TrimSuffix(trimmedURL, ".git")- splitURL := strings.Split(trimmedURL, "/")-- if len(splitURL) < 2 || len(splitURL) > 3 {- return "", []string{}, fmt.Errorf("invalid repository or gist URL (%s): length of URL segments should be 2 or 3", urlString)+ urlParts := strings.Split(trimmedURL, "/")++ // Validate+ switch len(urlParts) {+ case 2:+ // gist.github.com/<gist_id>+ if !strings.EqualFold(urlParts[0], "gist.github.com") {+ err = fmt.Errorf("failed to parse repository or gist URL (%s): 2 path segments are only expected if the host is 'gist.github.com' ('gist.github.com', '<gist_id>')", urlString)+ }+ case 3:+ // github.com/<user>/repo>+ // gist.github.com/<user>/<gist_id>+ // github.company.org/<user>/repo>+ // github.company.org/gist/<gist_id>+ case 4:+ // github.company.org/gist/<user/<id>+ if !strings.EqualFold(urlParts[1], "gist") || (strings.EqualFold(urlParts[0], "github.com") && strings.EqualFold(urlParts[1], "gist")) {+ err = fmt.Errorf("failed to parse repository or gist URL (%s): 4 path segments are only expected if the host isn't 'github.com' and the path starts with 'gist' ('github.example.com', 'gist', '<owner>', '<gist_id>')", urlString)+ }+ default:+ err = fmt.Errorf("invalid repository or gist URL (%s): length of URL segments should be between 2 and 4, not %d (%v)", urlString, len(urlParts), urlParts)}- return urlString, splitURL, nil+ if err != nil {+ return "", nil, err+ }+ return urlString, urlParts, nil}const initialPage = 1 // page to start listing from@@ -1182,6 +1206,10 @@ func extractGistID(urlParts []string) string {return urlParts[len(urlParts)-1]}+func isGistUrl(urlParts []string) bool {+ return strings.EqualFold(urlParts[0], "gist.github.com") || (len(urlParts) == 4 && strings.EqualFold(urlParts[1], "gist"))+}+func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo repoInfo, comments []*github.GistComment, chunksChan chan *sources.Chunk) error {for _, comment := range comments {// Create chunk and send it to the channel.diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 029a3974f..e54406ce0 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -777,7 +777,7 @@ func Test_scan_SetProgressComplete(t *testing.T) {}func TestGetRepoURLParts(t *testing.T) {- tests := []string{+ repoURLs := []string{"https://github.com/trufflesecurity/trufflehog.git","git+https://github.com/trufflesecurity/trufflehog.git","ssh://github.com/trufflesecurity/trufflehog.git",@@ -786,9 +786,31 @@ func TestGetRepoURLParts(t *testing.T) {"git://github.com/trufflesecurity/trufflehog.git",}expected := []string{"github.com", "trufflesecurity", "trufflehog"}- for _, tt := range tests {+ for _, tt := range repoURLs {+ _, parts, err := getRepoURLParts(tt)+ if err != nil {+ t.Fatalf("failed: %v", err)+ }+ assert.Equal(t, expected, parts)+ }++ gistURLs := map[string][]string{+ // Gists+ "ssh://github.com/6df198861306313246466d23aa4102aa.git": nil,+ "ssh://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"},+ "https://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"},+ "https://gist.github.com/john-smith/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "john-smith", "6df198861306313246466d23aa4102aa"},+ "ssh://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"},+ "https://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"},+ "https://github.contoso.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "john-smith", "6df198861306313246466d23aa4102aa"},+ "https://github.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": nil,+ }+ for tt, expected := range gistURLs {_, parts, err := getRepoURLParts(tt)if err != nil {+ if expected == nil {+ continue+ }t.Fatalf("failed: %v", err)}assert.Equal(t, expected, parts)