Skip to content

Find, verify, and analyze leaked credentials

Improve handling of Gist URLs (#2653)

Richard Gomeze53f5bd5c571
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 4571224a0..332ecb9cd 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -427,11 +427,14 @@ RepoLoop:
continue
}
- if strings.EqualFold(urlParts[0], "gist.github.com") {
+ if isGistUrl(urlParts) {
// Cache gist info.
for {
gistID := extractGistID(urlParts)
gist, _, err := s.apiClient.Gists.Get(repoCtx, gistID)
+ // Normalize the URL to the Gist's pull URL.
+ // See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937
+ r = gist.GetGitPullURL()
if s.handleRateLimit(err) {
continue
}
@@ -1105,7 +1108,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep
return err
}
- if s.includeGistComments && urlParts[0] == "gist.github.com" {
+ if s.includeGistComments && isGistUrl(urlParts) {
return s.processGistComments(ctx, urlString, urlParts, repoInfo, chunksChan)
} else if s.includeIssueComments || s.includePRComments {
return s.processRepoComments(ctx, repoInfo, chunksChan)
@@ -1120,29 +1123,50 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep
// - "https://github.com/trufflesecurity/trufflehog" => ["github.com", "trufflesecurity", "trufflehog"]
// - "https://gist.github.com/nat/5fdbb7f945d121f197fb074578e53948" => ["gist.github.com", "nat", "5fdbb7f945d121f197fb074578e53948"]
// - "https://gist.github.com/ff0e5e8dc8ec22f7a25ddfc3492d3451.git" => ["gist.github.com", "ff0e5e8dc8ec22f7a25ddfc3492d3451"]
-func getRepoURLParts(repoURL string) (string, []string, error) {
+// - "https://github.company.org/gist/nat/5fdbb7f945d121f197fb074578e53948.git" => ["github.company.org", "gist", "nat", "5fdbb7f945d121f197fb074578e53948"]
+func getRepoURLParts(repoURLString string) (string, []string, error) {
// Support ssh and https URLs.
- url, err := git.GitURLParse(repoURL)
+ repoURL, err := git.GitURLParse(repoURLString)
if err != nil {
- return "", []string{}, err
+ return "", nil, err
}
// Remove the user information.
// e.g., `[email protected]` -> `github.com`
- if url.User != nil {
- url.User = nil
+ if repoURL.User != nil {
+ repoURL.User = nil
}
- urlString := url.String()
- trimmedURL := strings.TrimPrefix(urlString, url.Scheme+"://")
+ urlString := repoURL.String()
+ trimmedURL := strings.TrimPrefix(urlString, repoURL.Scheme+"://")
trimmedURL = strings.TrimSuffix(trimmedURL, ".git")
- splitURL := strings.Split(trimmedURL, "/")
-
- if len(splitURL) < 2 || len(splitURL) > 3 {
- return "", []string{}, fmt.Errorf("invalid repository or gist URL (%s): length of URL segments should be 2 or 3", urlString)
+ urlParts := strings.Split(trimmedURL, "/")
+
+ // Validate
+ switch len(urlParts) {
+ case 2:
+ // gist.github.com/<gist_id>
+ if !strings.EqualFold(urlParts[0], "gist.github.com") {
+ err = fmt.Errorf("failed to parse repository or gist URL (%s): 2 path segments are only expected if the host is 'gist.github.com' ('gist.github.com', '<gist_id>')", urlString)
+ }
+ case 3:
+ // github.com/<user>/repo>
+ // gist.github.com/<user>/<gist_id>
+ // github.company.org/<user>/repo>
+ // github.company.org/gist/<gist_id>
+ case 4:
+ // github.company.org/gist/<user/<id>
+ if !strings.EqualFold(urlParts[1], "gist") || (strings.EqualFold(urlParts[0], "github.com") && strings.EqualFold(urlParts[1], "gist")) {
+ err = fmt.Errorf("failed to parse repository or gist URL (%s): 4 path segments are only expected if the host isn't 'github.com' and the path starts with 'gist' ('github.example.com', 'gist', '<owner>', '<gist_id>')", urlString)
+ }
+ default:
+ err = fmt.Errorf("invalid repository or gist URL (%s): length of URL segments should be between 2 and 4, not %d (%v)", urlString, len(urlParts), urlParts)
}
- return urlString, splitURL, nil
+ if err != nil {
+ return "", nil, err
+ }
+ return urlString, urlParts, nil
}
const initialPage = 1 // page to start listing from
@@ -1182,6 +1206,10 @@ func extractGistID(urlParts []string) string {
return urlParts[len(urlParts)-1]
}
+func isGistUrl(urlParts []string) bool {
+ return strings.EqualFold(urlParts[0], "gist.github.com") || (len(urlParts) == 4 && strings.EqualFold(urlParts[1], "gist"))
+}
+
func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo repoInfo, comments []*github.GistComment, chunksChan chan *sources.Chunk) error {
for _, comment := range comments {
// Create chunk and send it to the channel.
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 029a3974f..e54406ce0 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -777,7 +777,7 @@ func Test_scan_SetProgressComplete(t *testing.T) {
}
func TestGetRepoURLParts(t *testing.T) {
- tests := []string{
+ repoURLs := []string{
"https://github.com/trufflesecurity/trufflehog.git",
"git+https://github.com/trufflesecurity/trufflehog.git",
"ssh://github.com/trufflesecurity/trufflehog.git",
@@ -786,9 +786,31 @@ func TestGetRepoURLParts(t *testing.T) {
"git://github.com/trufflesecurity/trufflehog.git",
}
expected := []string{"github.com", "trufflesecurity", "trufflehog"}
- for _, tt := range tests {
+ for _, tt := range repoURLs {
+ _, parts, err := getRepoURLParts(tt)
+ if err != nil {
+ t.Fatalf("failed: %v", err)
+ }
+ assert.Equal(t, expected, parts)
+ }
+
+ gistURLs := map[string][]string{
+ // Gists
+ "ssh://github.com/6df198861306313246466d23aa4102aa.git": nil,
+ "ssh://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"},
+ "https://gist.github.com/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "6df198861306313246466d23aa4102aa"},
+ "https://gist.github.com/john-smith/6df198861306313246466d23aa4102aa.git": {"gist.github.com", "john-smith", "6df198861306313246466d23aa4102aa"},
+ "ssh://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"},
+ "https://github.contoso.com/gist/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "6df198861306313246466d23aa4102aa"},
+ "https://github.contoso.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": {"github.contoso.com", "gist", "john-smith", "6df198861306313246466d23aa4102aa"},
+ "https://github.com/gist/john-smith/6df198861306313246466d23aa4102aa.git": nil,
+ }
+ for tt, expected := range gistURLs {
_, parts, err := getRepoURLParts(tt)
if err != nil {
+ if expected == nil {
+ continue
+ }
t.Fatalf("failed: %v", err)
}
assert.Equal(t, expected, parts)