Skip to content

Find, verify, and analyze leaked credentials

Changes to fix Enterprise UI filtering of Github Hosted Scanner Repositories to Include (#4430)

jordanTunstilla66d9e75c2cb
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index e6c819b6d..38e8803d1 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -205,6 +205,11 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
return false
}
+// wantRepo returns true if the repository should be included based on include/exclude patterns
+func (c *filteredRepoCache) wantRepo(s string) bool {
+ return !c.ignoreRepo(s) && c.includeRepo(s)
+}
+
// Init returns an initialized GitHub source.
func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
err := git.CmdCheck()
@@ -433,14 +438,29 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
// Double make sure that all enumerated repositories in the
// filteredRepoCache have an entry in the repoInfoCache.
for _, repo := range s.filteredRepoCache.Values() {
- ctx := context.WithValue(ctx, "repo", repo)
+ // Extract the repository name from the URL for filtering
+ repoName := repo
+ if strings.Contains(repo, "/") {
+ // Try to extract org/repo name from URL
+ if strings.Contains(repo, "github.com") {
+ parts := strings.Split(repo, "/")
+ if len(parts) >= 2 {
+ repoName = parts[len(parts)-2] + "/" + strings.TrimSuffix(parts[len(parts)-1], ".git")
+ }
+ }
+ }
- repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
- if err != nil {
- ctx.Logger().Error(err, "error caching repo info")
- _ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
+ // Final filter check - only include repositories that pass the filter
+ if s.filteredRepoCache.wantRepo(repoName) {
+ ctx = context.WithValue(ctx, "repo", repo)
+
+ repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
+ if err != nil {
+ ctx.Logger().Error(err, "error caching repo info")
+ _ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
+ }
+ s.repos = append(s.repos, repo)
}
- s.repos = append(s.repos, repo)
}
githubReposEnumerated.WithLabelValues(s.name).Set(float64(len(s.repos)))
ctx.Logger().Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", s.orgsCache.Count(), "num_members", len(s.memberCache))
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index cc5ed953a..c063d04d6 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -423,6 +423,41 @@ func TestNormalizeRepos(t *testing.T) {
}
}
+func TestNormalizeRepo(t *testing.T) {
+ // Test that normalizeRepo correctly identifies URLs with protocols
+ source := &Source{}
+
+ // Test case 1: HTTP URL
+ result, err := source.normalizeRepo("https://github.com/org/repo.git")
+ assert.NoError(t, err)
+ assert.Contains(t, result, "github.com/org/repo")
+
+ // Test case 2: HTTP URL without .git
+ result, err = source.normalizeRepo("http://github.com/org/repo")
+ assert.NoError(t, err)
+ assert.Contains(t, result, "github.com/org/repo")
+
+ // Test case 3: Git protocol URL
+ result, err = source.normalizeRepo("git://github.com/org/repo.git")
+ assert.NoError(t, err)
+ assert.Contains(t, result, "github.com/org/repo")
+
+ // Test case 4: SSH URL
+ result, err = source.normalizeRepo("ssh://[email protected]/org/repo.git")
+ assert.NoError(t, err)
+ assert.Contains(t, result, "github.com/org/repo")
+
+ // Test case 5: Org/repo format (should convert to full URL)
+ result, err = source.normalizeRepo("org/repo")
+ assert.NoError(t, err)
+ assert.Contains(t, result, "github.com/org/repo")
+
+ // Test case 6: Invalid format (no protocol, no slash)
+ _, err = source.normalizeRepo("invalid")
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "no repositories found")
+}
+
func TestHandleRateLimit(t *testing.T) {
s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}})
ctx := context.Background()
@@ -493,7 +528,7 @@ func TestEnumerateWithToken(t *testing.T) {
JSON(map[string]string{"login": "super-secret-user"})
gock.New("https://api.github.com").
- Get("/users/super-secret-user/repos").
+ Get("/user/repos").
MatchParam("per_page", "100").
Reply(200).
JSON([]map[string]string{{"clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "full_name": "super-secret-user/super-secret-repo"}})
@@ -574,7 +609,7 @@ func TestEnumerate(t *testing.T) {
//
gock.New("https://api.github.com").
- Get("/users/super-secret-user/repos").
+ Get("/user/repos").
Reply(200).
JSON(`[{"name": "super-secret-repo", "full_name": "super-secret-user/super-secret-repo", "owner": {"login": "super-secret-user"}, "clone_url": "https://github.com/super-secret-user/super-secret-repo.git", "has_wiki": false, "size": 1}]`)
@@ -978,6 +1013,39 @@ func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) {
}
}
+func TestRepositoryFiltering(t *testing.T) {
+ // Test that the filteredRepoCache correctly filters repositories
+ source := &Source{}
+
+ // Test case 1: No filters specified (should include everything)
+ cache1 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{})
+ assert.True(t, cache1.wantRepo("org/repo1"))
+ assert.True(t, cache1.wantRepo("org/repo2"))
+ assert.True(t, cache1.wantRepo("org/repo3"))
+
+ // Test case 2: Include filter specified (should only include matching repos)
+ cache2 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1", "org/repo2"}, []string{})
+ assert.True(t, cache2.wantRepo("org/repo1"))
+ assert.True(t, cache2.wantRepo("org/repo2"))
+ assert.False(t, cache2.wantRepo("org/repo3"))
+
+ // Test case 3: Exclude filter specified (should exclude matching repos)
+ cache3 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{}, []string{"org/repo1"})
+ assert.False(t, cache3.wantRepo("org/repo1"))
+ assert.True(t, cache3.wantRepo("org/repo2"))
+ assert.True(t, cache3.wantRepo("org/repo3"))
+
+ // Test case 4: Both include and exclude filters (exclude takes precedence)
+ cache4 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/repo1"}, []string{"org/repo1"})
+ assert.False(t, cache4.wantRepo("org/repo1"))
+
+ // Test case 5: Wildcard patterns
+ cache5 := source.newFilteredRepoCache(context.Background(), simple.NewCache[string](), []string{"org/*"}, []string{})
+ assert.True(t, cache5.wantRepo("org/repo1"))
+ assert.True(t, cache5.wantRepo("org/repo2"))
+ assert.False(t, cache5.wantRepo("other/repo1"))
+}
+
func noopReporter() sources.UnitReporter {
return sources.VisitorReporter{
VisitUnit: func(context.Context, sources.SourceUnit) error {
diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go
index aced5feb2..0d0a5a374 100644
--- a/pkg/sources/github/repo.go
+++ b/pkg/sources/github/repo.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"net/http"
+ "regexp"
"strings"
"sync"
@@ -273,6 +274,12 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
}
repoName, repoURL := r.GetFullName(), r.GetCloneURL()
+
+ // Check if we should process this repository based on the filter
+ if !s.filteredRepoCache.wantRepo(repoName) {
+ continue
+ }
+
s.totalRepoSize += r.GetSize()
s.filteredRepoCache.Set(repoName, repoURL)
s.cacheRepoInfo(r)
@@ -349,10 +356,17 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool {
// normalizeRepo normalizes a GitHub repository URL or name to its canonical form.
func (s *Source) normalizeRepo(repo string) (string, error) {
- // if the string contains a '/', assume it's a GitHub repository.
- if strings.ContainsRune(repo, '/') {
+
+ // If it's a full URL (has protocol), normalize it
+ if regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {
+
return giturl.NormalizeGithubRepo(repo)
}
+ // If it's a repository name (contains / but not http), convert to full URL first
+ if strings.Contains(repo, "/") && !regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {
+ fullURL := "https://github.com/" + repo
+ return giturl.NormalizeGithubRepo(fullURL)
+ }
return "", fmt.Errorf("no repositories found for %s", repo)
}