Skip to content

Find, verify, and analyze leaked credentials

Fix `scan_all_installations` Rejecting Org Member Personal Repos (#5142)

Shahzad Haider4d82f215b9ec
diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go
index a9b0ea638..0b66c4684 100644
--- a/pkg/sources/github/connector_app.go
+++ b/pkg/sources/github/connector_app.go
@@ -178,6 +178,27 @@ func (c *appConnector) setRepoInstallation(repoURL string, installationID int64)
c.setRepoInstallationForWiki(repoURL, installationID, wikiCloneURLForRepo)
}
+// ensureRepoInstallation maps a repo to the default installation unless an
+// installation already claims it. Used for repos discovered outside
+// installation listings (e.g. org members' personal repos), which no
+// installation owns but which the default installation token can still
+// access when they are public. The check and set happen under one lock so a
+// concurrent mapping to a real installation is never overwritten.
+func (c *appConnector) ensureRepoInstallation(repoURL, repoName string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ if _, ok := c.repoInstallationMap[repoURL]; ok {
+ return
+ }
+ c.repoInstallationMap[repoURL] = c.installationID
+ if wikiURL, ok := wikiCloneURLForRepoName(repoURL, repoName); ok {
+ if _, taken := c.repoInstallationMap[wikiURL]; !taken {
+ c.repoInstallationMap[wikiURL] = c.installationID
+ }
+ }
+}
+
func (c *appConnector) setRepoInstallationForRepoName(repoURL, repoName string, installationID int64) {
c.setRepoInstallationForWiki(repoURL, installationID, func(repoURL string) (string, bool) {
return wikiCloneURLForRepoName(repoURL, repoName)
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 6068435e9..b691d24f4 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -927,6 +927,18 @@ func (s *Source) mapReposToInstallations(ctx context.Context, connector *appConn
}
}
+ if len(wantedRepos) > 0 {
+ errs = errors.Join(errs, s.mapRemainingAccessibleRepos(ctx, connector, wantedRepos))
+ if len(wantedRepos) == 0 {
+ // Every requested repo was mapped, so returning errs would fail
+ // repos that succeeded; log partial enumeration errors instead.
+ if errs != nil {
+ ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos")
+ }
+ return nil
+ }
+ }
+
if len(wantedRepos) > 0 {
unmatched := make([]string, 0, len(wantedRepos))
for _, requested := range wantedRepos {
@@ -1031,6 +1043,61 @@ func (s *Source) mapRemainingReposByMetadata(
return errs
}
+// mapRemainingAccessibleRepos handles repos that no installation owns, such
+// as org members' personal repos enumerated via scan_users. Installations
+// only ever contain repos owned by the installing account, but the default
+// installation token can still read repos that are visible to it (public
+// repos in particular), so a repo that the API confirms accessible is mapped
+// to the default installation instead of failing the scan.
+func (s *Source) mapRemainingAccessibleRepos(
+ ctx context.Context,
+ connector *appConnector,
+ wantedRepos map[string]repoMappingRequest,
+) error {
+ var errs error
+ client := connector.APIClient()
+ for _, requested := range wantedRepos {
+ for i, lookupURL := range requested.lookupURLs {
+ _, parts, err := getRepoURLParts(lookupURL)
+ if err != nil {
+ errs = errors.Join(errs,
+ fmt.Errorf("could not parse configured repo %q: %w", requested.original, err))
+ break
+ }
+
+ var repo *github.Repository
+ for {
+ repo, _, err = client.Repositories.Get(ctx, parts[1], parts[2])
+ if s.handleRateLimit(ctx, err) {
+ continue
+ }
+ break
+ }
+ if err != nil {
+ if isGitHub404Error(err) {
+ continue
+ }
+ errs = errors.Join(errs,
+ fmt.Errorf("could not fetch repo %q with default installation: %w", requested.original, err))
+ break
+ }
+ if !sameRepoHost(requested.normalized, repo.GetCloneURL()) {
+ if i == 0 {
+ break
+ }
+ continue
+ }
+
+ ctx.Logger().Info("repo is not owned by any app installation; scanning with the default installation token",
+ "repo", requested.original)
+ deleteRepoMappingRequest(wantedRepos, requested)
+ s.recordRepoInstallation(connector, connector.installationID, repo, requested)
+ break
+ }
+ }
+ return errs
+}
+
func (s *Source) recordRepoInstallation(connector *appConnector, installationID int64, repo *github.Repository, requested repoMappingRequest) {
cloneURL := repo.GetCloneURL()
connector.setRepoInstallationForRepoName(cloneURL, repo.GetName(), installationID)
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 3707aac04..64f48b283 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -1380,7 +1380,12 @@ func TestMapExplicitReposToInstallationsRejectsHostMismatch(t *testing.T) {
assert.Contains(t, err.Error(), "configured repos were not found")
}
-func TestMapExplicitReposToInstallationsRejectsRepoMissingFromInstallationList(t *testing.T) {
+// A repo absent from every installation listing but readable with the
+// default installation token (e.g. a public repo, or one the listing missed)
+// maps to the default installation instead of failing the scan. Repos the
+// default token cannot read are still rejected (see the preceding test and
+// TestScanAllInstallationsInaccessibleRepoStillFailsMapping).
+func TestMapExplicitReposToInstallationsFallsBackToDefaultForReadableRepo(t *testing.T) {
privateKey := createPrivateKey()
const repoURL = "https://github.com/other-org/repo.git"
@@ -1415,8 +1420,12 @@ func TestMapExplicitReposToInstallationsRejectsRepoMissingFromInstallationList(t
},
})
err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)
- require.Error(t, err)
- assert.Contains(t, err.Error(), "configured repos were not found")
+ require.NoError(t, err)
+
+ connector := s.connector.(*appConnector)
+ installationID, mapped := connector.installationIDForRepo(repoURL)
+ require.True(t, mapped)
+ assert.Equal(t, int64(1337), installationID)
}
func TestMapExplicitReposToInstallationsMapsWikiRepoURL(t *testing.T) {
@@ -1681,6 +1690,136 @@ func TestScanAllInstallationsMapsUnitBeforeMetadataFetch(t *testing.T) {
assert.Equal(t, int64(2448), installationID)
}
+// Regression test for member personal repos under scan_all_installations
+// (INT-789): they belong to no app installation, so the mapping must fall
+// back to the default installation when the API confirms the repo is
+// readable, both at chunk time and at enumeration time.
+func TestScanAllInstallationsMemberPersonalRepoFallsBackToDefaultInstallation(t *testing.T) {
+ privateKey := createPrivateKey()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+
+ switch {
+ case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
+ parts := strings.Split(r.URL.Path, "/")
+ installID := parts[len(parts)-2]
+ _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
+ _, _ = w.Write([]byte(`[
+ {"id":1337,"account":{"login":"test-org","type":"Organization"}},
+ {"id":2448,"account":{"login":"other-org","type":"Organization"}}
+ ]`))
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
+ // Installations only ever contain org-owned repos.
+ auth := r.Header.Get("Authorization")
+ switch {
+ case strings.Contains(auth, "token-1337"):
+ _, _ = w.Write([]byte(`{"total_count":1,"repositories":[
+ {"name":"backend","full_name":"test-org/backend","clone_url":"https://github.com/test-org/backend.git","owner":{"login":"test-org","type":"Organization"},"size":1}
+ ]}`))
+ case strings.Contains(auth, "token-2448"):
+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
+ default:
+ http.Error(w, "unexpected installation token", http.StatusUnauthorized)
+ }
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/alice/dns-data"):
+ // The member's personal repo exists and is public: any valid
+ // token can read it, mirroring real GitHub behavior.
+ _, _ = w.Write([]byte(`{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}`))
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/users/alice/repos"):
+ _, _ = w.Write([]byte(`[{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}]`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ s, conn := createTestSource(&sourcespb.GitHub{
+ Endpoint: server.URL,
+ ScanAllInstallations: true,
+ ScanUsers: true,
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: privateKey,
+ InstallationId: "1337",
+ AppId: "4141",
+ },
+ },
+ })
+ require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
+
+ // The org-owned repo maps to its owning installation.
+ connector := s.connector.(*appConnector)
+ err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/test-org/backend.git"})
+ require.NoError(t, err)
+
+ // The member's public personal repo is in no installation, but the API
+ // confirms it is accessible, so it maps to the default installation.
+ err = s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/alice/dns-data.git"})
+ require.NoError(t, err)
+
+ installationID, mapped := connector.installationIDForRepo("https://github.com/alice/dns-data.git")
+ require.True(t, mapped)
+ require.Equal(t, int64(1337), installationID)
+
+ // Enumeration must map member repos up front so same-process ChunkUnit
+ // short-circuits without any mapping API calls.
+ connector.mu.Lock()
+ delete(connector.repoInstallationMap, "https://github.com/alice/dns-data.git")
+ connector.mu.Unlock()
+
+ require.NoError(t, s.getReposByUser(context.Background(), "alice", false, noopReporter()))
+ installationID, mapped = connector.installationIDForRepo("https://github.com/alice/dns-data.git")
+ require.True(t, mapped)
+ require.Equal(t, int64(1337), installationID)
+}
+
+// Repos that no installation owns AND the default installation token cannot
+// read must still fail the mapping: the accessibility fallback must not turn
+// the check fail-open.
+func TestScanAllInstallationsInaccessibleRepoStillFailsMapping(t *testing.T) {
+ privateKey := createPrivateKey()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+
+ switch {
+ case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):
+ parts := strings.Split(r.URL.Path, "/")
+ installID := parts[len(parts)-2]
+ _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):
+ _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"test-org","type":"Organization"}}]`))
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):
+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
+ default:
+ // /repos/... lookups 404: repo is private/nonexistent for
+ // every installation token.
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ s, conn := createTestSource(&sourcespb.GitHub{
+ Endpoint: server.URL,
+ ScanAllInstallations: true,
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: privateKey,
+ InstallationId: "1337",
+ AppId: "4141",
+ },
+ },
+ })
+ require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
+
+ connector := s.connector.(*appConnector)
+ err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/ghost/private-repo.git"})
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation")
+}
+
func TestEnumerateAllInstallationReposReportsInstallationErrors(t *testing.T) {
privateKey := createPrivateKey()
diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go
index 2073a8109..6a169233a 100644
--- a/pkg/sources/github/repo.go
+++ b/pkg/sources/github/repo.go
@@ -300,6 +300,13 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
s.totalRepoSize += r.GetSize()
s.filteredRepoCache.Set(repoName, repoURL)
s.cacheRepoInfo(r)
+ // Repos enumerated outside installation listings (e.g. member
+ // personal repos) belong to no installation; map them to the
+ // default installation so the scan-all mapping in ChunkUnit
+ // doesn't reject repos we just enumerated.
+ if connector, ok := s.connector.(*appConnector); ok && s.conn.GetScanAllInstallations() {
+ connector.ensureRepoInstallation(repoURL, r.GetName())
+ }
if err := reporter.UnitOk(ctx, RepoUnit{Name: repoName, URL: repoURL}); err != nil {
return err
}