Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fix `scan_all_installations` Rejecting Org Member Personal Repos (#5142)
Shahzad Haider4d82f215b9ec
diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.goindex a9b0ea638..0b66c4684 100644--- a/pkg/sources/github/connector_app.go+++ b/pkg/sources/github/connector_app.go@@ -178,6 +178,27 @@ func (c *appConnector) setRepoInstallation(repoURL string, installationID int64)c.setRepoInstallationForWiki(repoURL, installationID, wikiCloneURLForRepo)}+// ensureRepoInstallation maps a repo to the default installation unless an+// installation already claims it. Used for repos discovered outside+// installation listings (e.g. org members' personal repos), which no+// installation owns but which the default installation token can still+// access when they are public. The check and set happen under one lock so a+// concurrent mapping to a real installation is never overwritten.+func (c *appConnector) ensureRepoInstallation(repoURL, repoName string) {+ c.mu.Lock()+ defer c.mu.Unlock()++ if _, ok := c.repoInstallationMap[repoURL]; ok {+ return+ }+ c.repoInstallationMap[repoURL] = c.installationID+ if wikiURL, ok := wikiCloneURLForRepoName(repoURL, repoName); ok {+ if _, taken := c.repoInstallationMap[wikiURL]; !taken {+ c.repoInstallationMap[wikiURL] = c.installationID+ }+ }+}+func (c *appConnector) setRepoInstallationForRepoName(repoURL, repoName string, installationID int64) {c.setRepoInstallationForWiki(repoURL, installationID, func(repoURL string) (string, bool) {return wikiCloneURLForRepoName(repoURL, repoName)diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 6068435e9..b691d24f4 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -927,6 +927,18 @@ func (s *Source) mapReposToInstallations(ctx context.Context, connector *appConn}}+ if len(wantedRepos) > 0 {+ errs = errors.Join(errs, s.mapRemainingAccessibleRepos(ctx, connector, wantedRepos))+ if len(wantedRepos) == 0 {+ // Every requested repo was mapped, so returning errs would fail+ // repos that succeeded; log partial enumeration errors instead.+ if errs != nil {+ ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos")+ }+ return nil+ }+ }+if len(wantedRepos) > 0 {unmatched := make([]string, 0, len(wantedRepos))for _, requested := range wantedRepos {@@ -1031,6 +1043,61 @@ func (s *Source) mapRemainingReposByMetadata(return errs}+// mapRemainingAccessibleRepos handles repos that no installation owns, such+// as org members' personal repos enumerated via scan_users. Installations+// only ever contain repos owned by the installing account, but the default+// installation token can still read repos that are visible to it (public+// repos in particular), so a repo that the API confirms accessible is mapped+// to the default installation instead of failing the scan.+func (s *Source) mapRemainingAccessibleRepos(+ ctx context.Context,+ connector *appConnector,+ wantedRepos map[string]repoMappingRequest,+) error {+ var errs error+ client := connector.APIClient()+ for _, requested := range wantedRepos {+ for i, lookupURL := range requested.lookupURLs {+ _, parts, err := getRepoURLParts(lookupURL)+ if err != nil {+ errs = errors.Join(errs,+ fmt.Errorf("could not parse configured repo %q: %w", requested.original, err))+ break+ }++ var repo *github.Repository+ for {+ repo, _, err = client.Repositories.Get(ctx, parts[1], parts[2])+ if s.handleRateLimit(ctx, err) {+ continue+ }+ break+ }+ if err != nil {+ if isGitHub404Error(err) {+ continue+ }+ errs = errors.Join(errs,+ fmt.Errorf("could not fetch repo %q with default installation: %w", requested.original, err))+ break+ }+ if !sameRepoHost(requested.normalized, repo.GetCloneURL()) {+ if i == 0 {+ break+ }+ continue+ }++ ctx.Logger().Info("repo is not owned by any app installation; scanning with the default installation token",+ "repo", requested.original)+ deleteRepoMappingRequest(wantedRepos, requested)+ s.recordRepoInstallation(connector, connector.installationID, repo, requested)+ break+ }+ }+ return errs+}+func (s *Source) recordRepoInstallation(connector *appConnector, installationID int64, repo *github.Repository, requested repoMappingRequest) {cloneURL := repo.GetCloneURL()connector.setRepoInstallationForRepoName(cloneURL, repo.GetName(), installationID)diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 3707aac04..64f48b283 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -1380,7 +1380,12 @@ func TestMapExplicitReposToInstallationsRejectsHostMismatch(t *testing.T) {assert.Contains(t, err.Error(), "configured repos were not found")}-func TestMapExplicitReposToInstallationsRejectsRepoMissingFromInstallationList(t *testing.T) {+// A repo absent from every installation listing but readable with the+// default installation token (e.g. a public repo, or one the listing missed)+// maps to the default installation instead of failing the scan. Repos the+// default token cannot read are still rejected (see the preceding test and+// TestScanAllInstallationsInaccessibleRepoStillFailsMapping).+func TestMapExplicitReposToInstallationsFallsBackToDefaultForReadableRepo(t *testing.T) {privateKey := createPrivateKey()const repoURL = "https://github.com/other-org/repo.git"@@ -1415,8 +1420,12 @@ func TestMapExplicitReposToInstallationsRejectsRepoMissingFromInstallationList(t},})err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1)- require.Error(t, err)- assert.Contains(t, err.Error(), "configured repos were not found")+ require.NoError(t, err)++ connector := s.connector.(*appConnector)+ installationID, mapped := connector.installationIDForRepo(repoURL)+ require.True(t, mapped)+ assert.Equal(t, int64(1337), installationID)}func TestMapExplicitReposToInstallationsMapsWikiRepoURL(t *testing.T) {@@ -1681,6 +1690,136 @@ func TestScanAllInstallationsMapsUnitBeforeMetadataFetch(t *testing.T) {assert.Equal(t, int64(2448), installationID)}+// Regression test for member personal repos under scan_all_installations+// (INT-789): they belong to no app installation, so the mapping must fall+// back to the default installation when the API confirms the repo is+// readable, both at chunk time and at enumeration time.+func TestScanAllInstallationsMemberPersonalRepoFallsBackToDefaultInstallation(t *testing.T) {+ privateKey := createPrivateKey()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")++ switch {+ case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):+ parts := strings.Split(r.URL.Path, "/")+ installID := parts[len(parts)-2]+ _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):+ _, _ = w.Write([]byte(`[+ {"id":1337,"account":{"login":"test-org","type":"Organization"}},+ {"id":2448,"account":{"login":"other-org","type":"Organization"}}+ ]`))+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):+ // Installations only ever contain org-owned repos.+ auth := r.Header.Get("Authorization")+ switch {+ case strings.Contains(auth, "token-1337"):+ _, _ = w.Write([]byte(`{"total_count":1,"repositories":[+ {"name":"backend","full_name":"test-org/backend","clone_url":"https://github.com/test-org/backend.git","owner":{"login":"test-org","type":"Organization"},"size":1}+ ]}`))+ case strings.Contains(auth, "token-2448"):+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))+ default:+ http.Error(w, "unexpected installation token", http.StatusUnauthorized)+ }+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/repos/alice/dns-data"):+ // The member's personal repo exists and is public: any valid+ // token can read it, mirroring real GitHub behavior.+ _, _ = w.Write([]byte(`{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}`))+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/users/alice/repos"):+ _, _ = w.Write([]byte(`[{"name":"dns-data","full_name":"alice/dns-data","clone_url":"https://github.com/alice/dns-data.git","owner":{"login":"alice","type":"User"},"private":false,"size":1}]`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ s, conn := createTestSource(&sourcespb.GitHub{+ Endpoint: server.URL,+ ScanAllInstallations: true,+ ScanUsers: true,+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: privateKey,+ InstallationId: "1337",+ AppId: "4141",+ },+ },+ })+ require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))++ // The org-owned repo maps to its owning installation.+ connector := s.connector.(*appConnector)+ err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/test-org/backend.git"})+ require.NoError(t, err)++ // The member's public personal repo is in no installation, but the API+ // confirms it is accessible, so it maps to the default installation.+ err = s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/alice/dns-data.git"})+ require.NoError(t, err)++ installationID, mapped := connector.installationIDForRepo("https://github.com/alice/dns-data.git")+ require.True(t, mapped)+ require.Equal(t, int64(1337), installationID)++ // Enumeration must map member repos up front so same-process ChunkUnit+ // short-circuits without any mapping API calls.+ connector.mu.Lock()+ delete(connector.repoInstallationMap, "https://github.com/alice/dns-data.git")+ connector.mu.Unlock()++ require.NoError(t, s.getReposByUser(context.Background(), "alice", false, noopReporter()))+ installationID, mapped = connector.installationIDForRepo("https://github.com/alice/dns-data.git")+ require.True(t, mapped)+ require.Equal(t, int64(1337), installationID)+}++// Repos that no installation owns AND the default installation token cannot+// read must still fail the mapping: the accessibility fallback must not turn+// the check fail-open.+func TestScanAllInstallationsInaccessibleRepoStillFailsMapping(t *testing.T) {+ privateKey := createPrivateKey()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")++ switch {+ case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/app/installations/") && strings.HasSuffix(r.URL.Path, "/access_tokens"):+ parts := strings.Split(r.URL.Path, "/")+ installID := parts[len(parts)-2]+ _, _ = fmt.Fprintf(w, `{"token":"token-%s","expires_at":"2099-01-01T00:00:00Z"}`, installID)+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/app/installations"):+ _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"test-org","type":"Organization"}}]`))+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/installation/repositories"):+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))+ default:+ // /repos/... lookups 404: repo is private/nonexistent for+ // every installation token.+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ s, conn := createTestSource(&sourcespb.GitHub{+ Endpoint: server.URL,+ ScanAllInstallations: true,+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: privateKey,+ InstallationId: "1337",+ AppId: "4141",+ },+ },+ })+ require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))++ connector := s.connector.(*appConnector)+ err := s.mapReposToInstallations(context.Background(), connector, []string{"https://github.com/ghost/private-repo.git"})+ require.Error(t, err)+ require.Contains(t, err.Error(), "configured repos were not found in any GitHub App installation")+}+func TestEnumerateAllInstallationReposReportsInstallationErrors(t *testing.T) {privateKey := createPrivateKey()diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.goindex 2073a8109..6a169233a 100644--- a/pkg/sources/github/repo.go+++ b/pkg/sources/github/repo.go@@ -300,6 +300,13 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourcs.totalRepoSize += r.GetSize()s.filteredRepoCache.Set(repoName, repoURL)s.cacheRepoInfo(r)+ // Repos enumerated outside installation listings (e.g. member+ // personal repos) belong to no installation; map them to the+ // default installation so the scan-all mapping in ChunkUnit+ // doesn't reject repos we just enumerated.+ if connector, ok := s.connector.(*appConnector); ok && s.conn.GetScanAllInstallations() {+ connector.ensureRepoInstallation(repoURL, r.GetName())+ }if err := reporter.UnitOk(ctx, RepoUnit{Name: repoName, URL: repoURL}); err != nil {return err}