Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
github: check App installation grants before listing comments (#5454)
Dave Hubbardc85580b71284
diff --git a/pkg/sources/github/app_client_registry.go b/pkg/sources/github/app_client_registry.goindex 9bc8c58aa..d8f87cad5 100644--- a/pkg/sources/github/app_client_registry.go+++ b/pkg/sources/github/app_client_registry.go@@ -95,6 +95,22 @@ func (cs *AppInstallationClients) AccessToken(ctx context.Context) (string, erroreturn token.GetToken(), nil}+// GrantedPermissions returns the permissions the installation grants, as+// listed on its access token. GitHub sends them with every token it issues,+// so this costs no API call when a token is already cached, which is the+// usual case: a repo is cloned with the same token before its comments are+// scanned. The comment scan reads them to skip, or report as incomplete,+// listings the installation can't fully cover (see installationGrants).+func (cs *AppInstallationClients) GrantedPermissions(ctx context.Context) (*github.InstallationPermissions, error) {+ if _, err := cs.AccessToken(ctx); err != nil {+ return nil, err+ }++ cs.tokenMu.RLock()+ defer cs.tokenMu.RUnlock()+ return cs.token.GetPermissions(), nil+}+func (cs *AppInstallationClients) accessTokenIfValid() *github.InstallationToken {if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) {return nildiff --git a/pkg/sources/github/app_client_registry_test.go b/pkg/sources/github/app_client_registry_test.goindex 295c655ec..66ccfa79a 100644--- a/pkg/sources/github/app_client_registry_test.go+++ b/pkg/sources/github/app_client_registry_test.go@@ -53,6 +53,37 @@ func TestAccessTokenForInstallation(t *testing.T) {require.NoError(t, err)})+ t.Run("granted permissions come from the token", func(t *testing.T) {+ var hits atomic.Int64+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "token": "test-token",+ "expires_at": "2099-01-01T00:00:00Z",+ "permissions": map[string]string{"issues": "read", "contents": "read"},+ })+ })+ defer server.Close()++ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)+ require.NoError(t, err)++ perms, err := cs.GrantedPermissions(t.Context())+ require.NoError(t, err)+ require.Equal(t, "read", perms.GetIssues())+ require.Empty(t, perms.GetPullRequests())++ _, err = cs.AccessToken(t.Context())+ require.NoError(t, err)+ require.Equal(t, int64(1), hits.Load(), "permissions and clone share one token")+ })++ t.Run("granted permissions without a token", func(t *testing.T) {+ cs := &AppInstallationClients{InstallationID: 0}+ _, err := cs.GrantedPermissions(t.Context())+ require.Error(t, err)+ })+t.Run("rate limit", func(t *testing.T) {server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {w.Header().Add("X-RateLimit-Remaining", "0")diff --git a/pkg/sources/github/comments_test.go b/pkg/sources/github/comments_test.goindex c3166b6cb..539213dff 100644--- a/pkg/sources/github/comments_test.go+++ b/pkg/sources/github/comments_test.go@@ -124,10 +124,17 @@ func seedRepo(gh *fakeGitHub) {}}+// scanComments runs the REST comment scan with unknown installation grants,+// as a token-authenticated source would.func scanComments(t *testing.T, s *Source, gh *fakeGitHub, cutoff *time.Time) (*collectingReporter, error) {+ t.Helper()+ return scanCommentsWithGrants(t, s, gh, nil, cutoff)+}++func scanCommentsWithGrants(t *testing.T, s *Source, gh *fakeGitHub, grants *installationGrants, cutoff *time.Time) (*collectingReporter, error) {t.Helper()reporter := &collectingReporter{}- err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, reporter, cutoff)+ err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, grants, reporter, cutoff)return reporter, err}@@ -357,6 +364,158 @@ func TestCommentScan_EachFailedPhaseIsReportedAsItsOwnChunkError(t *testing.T) {assert.Empty(t, reporter.links)}+// reportedGrantErrors runs reportCommentErrors on err and describes each+// chunk error it reports as "<prefix>: skip" or "<prefix>: narrow", where+// prefix names the phase or the content left out. Any reported error that+// isn't a grants-based permissionError fails the test.+func reportedGrantErrors(t *testing.T, err error) []string {+ t.Helper()+ reporter := &collectingReporter{}+ require.NoError(t, reportCommentErrors(context.Background(), reporter, err))+ var got []string+ for _, reported := range reporter.errs {+ var permErr *permissionError+ require.ErrorAs(t, reported, &permErr)+ prefix, _, _ := strings.Cut(strings.TrimPrefix(reported.Error(), "error scanning comments: "), ": missing")+ switch {+ case strings.Contains(permErr.outcome, "skipped"):+ got = append(got, prefix+": skip")+ case strings.Contains(permErr.outcome, "left them out"):+ got = append(got, prefix+": narrow")+ default:+ t.Fatalf("unexpected permission error: %v", reported)+ }+ }+ return got+}++func TestCommentScan_InstallationGrantsShapeTheListings(t *testing.T) {+ type grant struct{ issues, pullRequests bool }+ var (+ neither = grant{}+ prsOnly = grant{pullRequests: true}+ issues = grant{issues: true}+ both = grant{issues: true, pullRequests: true}+ )+ tests := []struct {+ name string+ issueComments bool+ prComments bool+ grant grant+ wantErrs []string+ // wantListed names every endpoint the scan requests; the rest must+ // not be requested.+ wantListed []string+ }{+ {name: "issue comments, nothing granted", issueComments: true, grant: neither,+ wantErrs: []string{"issues: skip", "issue comments: skip"}},+ {name: "issue comments, pull requests granted", issueComments: true, grant: prsOnly,+ wantErrs: []string{"issues: skip", "issue comments: narrow"},+ wantListed: []string{"issue comments"}},+ {name: "issue comments, issues granted", issueComments: true, grant: issues,+ wantListed: []string{"issues", "issue comments"}},+ {name: "issue comments, both granted", issueComments: true, grant: both,+ wantListed: []string{"issues", "issue comments"}},++ {name: "PR comments, nothing granted", prComments: true, grant: neither,+ wantErrs: []string{"pull requests: skip", "pull request comments: skip"}},+ {name: "PR comments, pull requests granted", prComments: true, grant: prsOnly,+ wantListed: []string{"pulls", "pull comments"}},+ {name: "PR comments, issues granted", prComments: true, grant: issues,+ wantErrs: []string{"pull requests: skip", "pull request comments: skip"}},+ {name: "PR comments, both granted", prComments: true, grant: both,+ wantListed: []string{"pulls", "pull comments"}},++ {name: "both settings, nothing granted", issueComments: true, prComments: true, grant: neither,+ wantErrs: []string{"issues and pull requests: skip", "issue comments: skip", "pull request comments: skip"}},+ {name: "both settings, pull requests granted", issueComments: true, prComments: true, grant: prsOnly,+ wantErrs: []string{"issues: skip", "issue comments: narrow"},+ wantListed: []string{"pulls", "issue comments", "pull comments"}},+ {name: "both settings, issues granted", issueComments: true, prComments: true, grant: issues,+ wantErrs: []string{"pull request bodies: narrow", "pull request conversation comments: narrow", "pull request comments: skip"},+ wantListed: []string{"issues", "issue comments"}},+ {name: "both settings, both granted", issueComments: true, prComments: true, grant: both,+ wantListed: []string{"issues", "issue comments", "pull comments"}},++ {name: "comments off, nothing granted", grant: neither},+ {name: "comments off, pull requests granted", grant: prsOnly},+ {name: "comments off, issues granted", grant: issues},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includeIssueComments: tt.issueComments, includePRComments: tt.prComments}+ grants := &installationGrants{installationID: 42, issues: tt.grant.issues, pullRequests: tt.grant.pullRequests}++ _, err := scanCommentsWithGrants(t, s, gh, grants, nil)++ if len(tt.wantErrs) == 0 {+ require.NoError(t, err)+ } else {+ assert.Equal(t, tt.wantErrs, reportedGrantErrors(t, err))+ }+ endpoints := map[string]*fakeListing{+ "issues": gh.issues, "issue comments": gh.issueComments,+ "pulls": gh.pulls, "pull comments": gh.prComments,+ }+ var listed []string+ for name, listing := range endpoints {+ if listing.requestCount() > 0 {+ listed = append(listed, name)+ }+ }+ assert.ElementsMatch(t, tt.wantListed, listed)+ })+ }+}++func TestCommentScan_PullRequestBodiesComeFromPullsWithoutIssuesGrant(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includeIssueComments: true, includePRComments: true}++ reporter, _ := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42, pullRequests: true}, nil)++ assert.Contains(t, reporter.links, prLink(2))+ assert.NotContains(t, reporter.links, "https://github.com/o/r/issues/1")+ assert.Zero(t, gh.issues.requestCount())+}++func TestCommentScan_GrantErrorsNameTheInstallationAndPermissions(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includeIssueComments: true, includePRComments: true}++ _, err := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42, issues: true}, nil)++ reporter := &collectingReporter{}+ require.NoError(t, reportCommentErrors(context.Background(), reporter, err))+ var got []string+ for _, reported := range reporter.errs {+ got = append(got, reported.Error())+ }+ fix := "update the app's permissions and accept the change on the installation"+ assert.Equal(t, []string{+ "error scanning comments: pull request bodies: missing GitHub App permission: Pull requests (read) is needed; " + fix ++ " (not granted to installation 42, so GitHub left them out of the /issues listing)",+ "error scanning comments: pull request conversation comments: missing GitHub App permission: Pull requests (read) is needed; " + fix ++ " (not granted to installation 42, so GitHub left them out of the /issues/comments listing)",+ "error scanning comments: pull request comments: missing GitHub App permission: Pull requests (read) is needed; " + fix ++ " (not granted to installation 42, so the listing was skipped)",+ }, got)+}++func TestCommentScan_SkippedListingNeedingBothPermissionsSaysSo(t *testing.T) {+ gh := newFakeGitHub(t)+ s := &Source{includeIssueComments: true, includePRComments: true}++ _, err := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42}, nil)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "issues and pull requests: missing GitHub App permission: Issues (read) and Pull requests (read) are needed;")+}+func TestReportCommentErrors_UnjoinedErrorIsReportedOnce(t *testing.T) {reporter := &collectingReporter{}@@ -380,7 +539,7 @@ func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(2)))ctx := context.WithLogger(context.Background(), logger)- err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, &collectingReporter{}, nil)+ err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, nil, &collectingReporter{}, nil)require.NoError(t, err)var listings []stringdiff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.goindex 027d967ae..17e0ec0ef 100644--- a/pkg/sources/github/connector_app.go+++ b/pkg/sources/github/connector_app.go@@ -149,6 +149,28 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...stringreturn git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...)}+// grantsForRepo returns what the installation that owns repoURL grants, for+// the permissions the comment scan needs. Installation permissions apply to+// every repo the installation can see, so repos on one installation share+// the answer, and the cached token makes it free after the repo's clone.+func (c *appConnector) grantsForRepo(ctx context.Context, repoURL string) (*installationGrants, error) {+ installID, _ := c.installationIDForRepo(repoURL)+ if installID == 0 {+ return nil, fmt.Errorf("no GitHub App installation resolved for repo %q", repoURL)+ }++ cs, err := c.clientsForInstallation(installID)+ if err != nil {+ return nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)+ }++ perms, err := cs.GrantedPermissions(ctx)+ if err != nil {+ return nil, fmt.Errorf("could not read permissions for installation %d: %w", installID, err)+ }+ return newInstallationGrants(installID, perms), nil+}+// installationIDForRepo returns the mapped installation ID for repoURL. When no// mapping exists, it falls back to the configured installation and returns false.//diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 5192e1abd..bdfa96a6b 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -780,12 +780,16 @@ func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector,ctx.Logger().Info("Skipping org member repos and gists: githubApp.installationId is required as a fallback installation to scan members together with scanAllInstallations", "org_members", len(s.memberCache))return nil}- // TODO: Replace loop below with a call to s.addReposForMembers(ctx, reporter)+ // Members' gists aren't listed, unlike in addReposForMembers: none+ // of GitHub's gist endpoints accept an installation access token,+ // and no app permission changes that, so the listing fails with a+ // 403 for every member. One line here stands in for those+ // failures; there's nothing for the customer to grant.+ if !s.ignoreGists {+ ctx.Logger().Info("Skipping org member gists: GitHub doesn't allow listing gists with a GitHub App installation token", "org_members", len(s.memberCache))+ }for member := range s.memberCache {logger := ctx.Logger().WithValues("member", member)- if err := s.addUserGistsToCache(ctx, member, reporter); err != nil {- logger.Error(err, "error fetching gists by user")- }// TODO: Add authenticated user list repo for app token. It does support as per docs but need to test it before we enable it here.// docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-the-authenticated-userif err := s.getReposByUser(ctx, member, false, reporter); err != nil {@@ -1830,12 +1834,30 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo repreturn s.processRepoIssueandPRsWithCommentsGraphql(ctx, graphqlClient, repoInfo, reporter, cutoffTime)}- return s.processIssueandPRsWithCommentsREST(ctx, apiClient, repoInfo, reporter, cutoffTime)+ return s.processIssueandPRsWithCommentsREST(ctx, apiClient, repoInfo, s.commentGrantsForRepo(ctx, repoPath), reporter, cutoffTime)}return nil}+// commentGrantsForRepo returns what the GitHub App installation behind+// repoURL grants, or nil (unknown) when the source doesn't authenticate as an+// App. A failure to read the grants also gives nil: the listings then run+// without the precheck, as they would for a token, and any 403 is still+// classified from its response.+func (s *Source) commentGrantsForRepo(ctx context.Context, repoURL string) *installationGrants {+ app, ok := s.connector.(*appConnector)+ if !ok {+ return nil+ }+ grants, err := app.grantsForRepo(ctx, repoURL)+ if err != nil {+ ctx.Logger().V(2).Info("could not read installation permissions, scanning comments without the permission precheck", "error", err)+ return nil+ }+ return grants+}+// trimURLAndSplit removes extraneous information from the |url| and splits it into segments.// This is typically 3 segments: host, owner, and name/ID; however, Gists have some edge cases.//@@ -2022,10 +2044,18 @@ const (// commentPhase is one listing in a repo's comment scan. run returns how many// chunks it emitted. startLog, when set, is logged at V(2) before the phase// runs.+//+// skipped and narrowed carry what the installation's grants cost the phase+// (see installationGrants). skipped, when set, is why the listing can't+// succeed; the phase reports it in place of running. narrowed holds one error+// per kind of item GitHub will leave out of a listing that still runs, each+// already prefixed with what's missing, and is reported after the run.type commentPhase struct {name stringstartLog stringrun func() (int, error)+ skipped error+ narrowed []error}// processIssueandPRsWithCommentsREST scans a repo's issues, pull requests, and@@ -2037,13 +2067,20 @@ type commentPhase struct {// so each failed phase is its own ChunkErr (see reportCommentErrors). Chunks// emitted before a failure are kept.//+// grants, when known, shapes the phases. A mismatch between the comment+// settings and what the installation grants is a configuration error the+// customer can fix, so it's always reported: every listing skipped because it+// can't succeed, and every kind of item GitHub will silently leave out of a+// listing that does run, gets exactly one permissionError. Items no setting+// asked for don't count. Skipped listings make no request.+//// The "Scanning issues" and "Scanning pull requests" V(2) lines are the only// log signal for scans on the unit path, whose comment errors go to the scan// details UI instead of the logs, so they keep their wording. The summary// line at the end records how far each phase got.func (s *Source) processIssueandPRsWithCommentsREST(ctx context.Context, apiClient *github.Client, repoInfo repoInfo,- reporter sources.ChunkReporter, cutoffTime *time.Time,+ grants *installationGrants, reporter sources.ChunkReporter, cutoffTime *time.Time,) error {p := s.commentPager(ctx, apiClient, reporter)var since time.Time@@ -2051,47 +2088,102 @@ func (s *Source) processIssueandPRsWithCommentsREST(since = *cutoffTime}+ var installationID int64+ if grants != nil {+ installationID = grants.installationID+ }+ issuesOK, prsOK := grants.issuesGranted(), grants.pullRequestsGranted()+ narrowed := func(missing, listing string, needs permission) error {+ return fmt.Errorf("%s: %w", missing, newNarrowedListingError(installationID, listing, []permission{needs}))+ }+ pullsPhase := func(startLog string) commentPhase {+ return commentPhase{name: "pull requests", startLog: startLog, run: func() (int, error) {+ return s.processPRs(ctx, p, repoInfo, reporter)+ }}+ }+// /issues returns issues and pull requests together. When issue comments// are enabled, that one walk also emits PR bodies, so /pulls is only- // walked for configs that enable PR comments alone.+ // walked for configs that enable PR comments alone, or when the+ // installation can list pull requests but not issues.var phases []commentPhaseif s.includeIssueComments {- bodies := "issues"+ bodies := commentPhase{name: "issues", startLog: "Scanning issues", run: func() (int, error) {+ return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)+ }}+ needs := []permission{issuesRead}if s.includePRComments {- bodies = "issues and pull requests"- }- phases = append(phases,- commentPhase{name: bodies, startLog: "Scanning issues", run: func() (int, error) {- return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)- }},- commentPhase{name: "issue comments", run: func() (int, error) {- return s.processIssueComments(ctx, p, repoInfo, reporter, since)- }},- )+ bodies.name = "issues and pull requests"+ needs = append(needs, pullRequestsRead)+ }+ switch {+ case issuesOK:+ if s.includePRComments && !prsOK {+ bodies.narrowed = append(bodies.narrowed, narrowed("pull request bodies", "/issues", pullRequestsRead))+ }+ phases = append(phases, bodies)+ case s.includePRComments && prsOK:+ // /issues is refused without Issues (read), but the PR bodies it+ // would have carried are still reachable through /pulls.+ phases = append(phases,+ commentPhase{name: "issues", skipped: newSkippedListingError(installationID, []permission{issuesRead})},+ pullsPhase(""))+ default:+ bodies.skipped = newSkippedListingError(installationID, needs)+ phases = append(phases, bodies)+ }++ // /issues/comments carries comments on issues and on pull requests,+ // each kind only when its permission is granted. PR conversation+ // comments are scanned only when PR comments are enabled, so their+ // absence otherwise isn't a mismatch.+ comments := commentPhase{name: "issue comments", run: func() (int, error) {+ return s.processIssueComments(ctx, p, repoInfo, reporter, since)+ }}+ switch {+ case !issuesOK && !prsOK:+ comments.skipped = newSkippedListingError(installationID, needs)+ case !issuesOK:+ comments.narrowed = append(comments.narrowed, narrowed("issue comments", "/issues/comments", issuesRead))+ case s.includePRComments && !prsOK:+ comments.narrowed = append(comments.narrowed, narrowed("pull request conversation comments", "/issues/comments", pullRequestsRead))+ }+ phases = append(phases, comments)}if s.includePRComments {+ var prsSkipped error+ if !prsOK {+ prsSkipped = newSkippedListingError(installationID, []permission{pullRequestsRead})+ }prStartLog := "Scanning pull requests"if !s.includeIssueComments {- phases = append(phases, commentPhase{name: "pull requests", startLog: prStartLog, run: func() (int, error) {- return s.processPRs(ctx, p, repoInfo, reporter)- }})+ pulls := pullsPhase(prStartLog)+ pulls.skipped = prsSkipped+ phases = append(phases, pulls)prStartLog = ""}- phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, run: func() (int, error) {+ phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, skipped: prsSkipped, run: func() (int, error) {return s.processPRComments(ctx, p, repoInfo, reporter, since)}})}var (- errs []error- failed []string- summary []any+ errs []error+ failed []string+ skipped []string+ narrowedOut []string+ summary []any)for _, phase := range phases {if err := ctx.Err(); err != nil {errs = append(errs, err)break}+ if phase.skipped != nil {+ skipped = append(skipped, phase.name)+ errs = append(errs, fmt.Errorf("%s: %w", phase.name, phase.skipped))+ continue+ }if phase.startLog != "" {ctx.Logger().V(2).Info(phase.startLog)}@@ -2101,8 +2193,13 @@ func (s *Source) processIssueandPRsWithCommentsREST(failed = append(failed, phase.name)errs = append(errs, fmt.Errorf("%s: %w", phase.name, err))}+ if len(phase.narrowed) > 0 {+ narrowedOut = append(narrowedOut, phase.name)+ errs = append(errs, phase.narrowed...)+ }}- ctx.Logger().V(2).Info("finished scanning comments", append(summary, "failed_phases", failed)...)+ ctx.Logger().V(2).Info("finished scanning comments", append(summary,+ "failed_phases", failed, "skipped_phases", skipped, "narrowed_phases", narrowedOut)...)return errors.Join(errs...)}diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex dad98fee8..e71c2f289 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -15,6 +15,7 @@ import ("slices""strconv""strings"+ "sync""testing""time"@@ -1071,6 +1072,61 @@ func TestEnumerateWithAppScanAllInstallationsIncludesConfiguredInstallation(t *tassert.True(t, s.filteredRepoCache.Exists("other-org/repo"))}+func TestEnumerateWithAppScanUsersListsMemberReposButNotGists(t *testing.T) {+ privateKey := createPrivateKey()++ var (+ mu sync.Mutex+ paths []string+ )+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ paths = append(paths, r.URL.Path)+ mu.Unlock()+ w.Header().Set("Content-Type", "application/json")++ switch {+ case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/access_tokens"):+ _, _ = w.Write([]byte(`{"token":"token","expires_at":"2099-01-01T00:00:00Z"}`))+ case strings.HasSuffix(r.URL.Path, "/installation/repositories"):+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))+ case strings.HasSuffix(r.URL.Path, "/app/installations"):+ _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}}]`))+ case strings.HasSuffix(r.URL.Path, "/orgs/default-org/members"):+ _, _ = w.Write([]byte(`[{"login":"alice"}]`))+ case strings.HasSuffix(r.URL.Path, "/users/alice/repos"):+ _, _ = w.Write([]byte(`[]`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ s, conn := createTestSource(&sourcespb.GitHub{+ Endpoint: server.URL,+ ScanUsers: true,+ Credential: &sourcespb.GitHub_GithubApp{+ GithubApp: &credentialspb.GitHubApp{+ PrivateKey: privateKey,+ InstallationId: "1337",+ AppId: "4141",+ },+ },+ })+ require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))+ require.False(t, s.ignoreGists)++ err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector), noopReporter())+ require.NoError(t, err)++ mu.Lock()+ defer mu.Unlock()+ assert.Contains(t, paths, "/api/v3/users/alice/repos")+ for _, path := range paths {+ assert.NotContains(t, path, "gists")+ }+}+func TestEnumerateWithAppScanAllInstallationsMapsExplicitReposBeforeMetadataFetch(t *testing.T) {privateKey := createPrivateKey()diff --git a/pkg/sources/github/installation_grants.go b/pkg/sources/github/installation_grants.gonew file mode 100644index 000000000..13ef090a3--- /dev/null+++ b/pkg/sources/github/installation_grants.go@@ -0,0 +1,65 @@+package github++import "github.com/google/go-github/v67/github"++// Permissions the comment listings need, in the form permissionError names+// them.+var (+ issuesRead = permission{name: "issues", access: "read"}+ pullRequestsRead = permission{name: "pull_requests", access: "read"}+)++// installationGrants is what a GitHub App installation grants, for the+// permissions the comment scan needs. A nil *installationGrants means the+// grants aren't known (a personal access token, or an App token whose+// permissions couldn't be read), and every listing runs as if granted; a 403+// is then classified from its response by asPermissionError.+//+// The grants matter beyond outright refusals because GitHub doesn't refuse+// every listing they only partly cover. Measured against a live installation:+// - /issues with Issues (read) but not Pull requests (read) returns the+// issues and leaves out the pull requests.+// - /issues/comments with only one of the two returns only the comments that+// one covers: issue comments with Issues (read), pull request+// conversation comments with Pull requests (read).+//+// Both answer 200 with no error, so the comment scan reads the grants to skip+// listings that can't succeed and to report what the others leave out (see+// processIssueandPRsWithCommentsREST).+type installationGrants struct {+ installationID int64+ issues bool+ pullRequests bool+}++// newInstallationGrants reads the comment-relevant grants from an access+// token's permissions. GitHub lists them on every installation token, so a+// token without any is unexpected; it returns nil (unknown) rather than+// treating the installation as granting nothing.+func newInstallationGrants(installationID int64, perms *github.InstallationPermissions) *installationGrants {+ if perms == nil {+ return nil+ }+ return &installationGrants{+ installationID: installationID,+ issues: grantsRead(perms.GetIssues()),+ pullRequests: grantsRead(perms.GetPullRequests()),+ }+}++// grantsRead reports whether an installation permission level allows+// reading. Write access includes read.+func grantsRead(level string) bool {+ switch level {+ case "read", "write", "admin":+ return true+ default:+ return false+ }+}++// issuesGranted and pullRequestsGranted report whether the permission is+// granted, treating unknown grants as granted so listings run as they would+// without the precheck.+func (g *installationGrants) issuesGranted() bool { return g == nil || g.issues }+func (g *installationGrants) pullRequestsGranted() bool { return g == nil || g.pullRequests }diff --git a/pkg/sources/github/installation_grants_test.go b/pkg/sources/github/installation_grants_test.gonew file mode 100644index 000000000..5730469cf--- /dev/null+++ b/pkg/sources/github/installation_grants_test.go@@ -0,0 +1,89 @@+package github++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/google/go-github/v67/github"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++func TestNewInstallationGrants(t *testing.T) {+ tests := []struct {+ name string+ perms *github.InstallationPermissions+ wantIssues bool+ wantPullRequests bool+ }{+ {name: "read grants", perms: &github.InstallationPermissions{Issues: github.String("read"), PullRequests: github.String("read")},+ wantIssues: true, wantPullRequests: true},+ {name: "write includes read", perms: &github.InstallationPermissions{Issues: github.String("write")},+ wantIssues: true},+ {name: "absent permissions aren't granted", perms: &github.InstallationPermissions{Contents: github.String("read")}},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ g := newInstallationGrants(7, tt.perms)++ require.NotNil(t, g)+ assert.Equal(t, int64(7), g.installationID)+ assert.Equal(t, tt.wantIssues, g.issuesGranted())+ assert.Equal(t, tt.wantPullRequests, g.pullRequestsGranted())+ })+ }+}++func TestNewInstallationGrants_TokenWithoutPermissionsIsUnknown(t *testing.T) {+ g := newInstallationGrants(7, nil)++ assert.Nil(t, g)+ assert.True(t, g.issuesGranted(), "unknown grants let listings run")+ assert.True(t, g.pullRequestsGranted(), "unknown grants let listings run")+}++func TestCommentGrantsForRepo_ReadsTheRepoInstallation(t *testing.T) {+ privKey := generateTestPrivateKey(t)+ var tokenPath string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ tokenPath = r.URL.Path+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]any{+ "token": "test-token",+ "expires_at": "2099-01-01T00:00:00Z",+ "permissions": map[string]string{"pull_requests": "read"},+ })+ }))+ defer server.Close()+ conn := &appConnector{+ apiEndpoint: server.URL, appID: 12345, appPrivateKey: string(privKey), installationID: 1,+ repoInstallationMap: map[string]int64{"https://github.com/o/r.git": 777},+ }+ s := &Source{connector: conn}++ grants := s.commentGrantsForRepo(context.Background(), "https://github.com/o/r.git")++ assert.Equal(t, "/api/v3/app/installations/777/access_tokens", tokenPath)+ assert.Equal(t, &installationGrants{installationID: 777, pullRequests: true}, grants)+}++func TestCommentGrantsForRepo_UnreadableGrantsAreUnknown(t *testing.T) {+ privKey := generateTestPrivateKey(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusNotFound)+ }))+ defer server.Close()+ s := &Source{connector: &appConnector{apiEndpoint: server.URL, appID: 12345, appPrivateKey: string(privKey), installationID: 1}}++ assert.Nil(t, s.commentGrantsForRepo(context.Background(), "https://github.com/o/r.git"))+}++func TestCommentGrantsForRepo_UnknownWithoutAppAuth(t *testing.T) {+ s := &Source{connector: &tokenConnector{}}++ assert.Nil(t, s.commentGrantsForRepo(context.Background(), "https://github.com/o/r.git"))+}diff --git a/pkg/sources/github/permission_error.go b/pkg/sources/github/permission_error.goindex ed0e27206..0672dd650 100644--- a/pkg/sources/github/permission_error.go+++ b/pkg/sources/github/permission_error.go@@ -46,11 +46,18 @@ const (credentialFineGrainedToken)-// permissionError reports a request GitHub refused because the credential-// lacks a permission. accepted holds the permission sets that would satisfy-// the endpoint: any one set is enough, and a set needs every permission in it.-// method and path identify the request, and message is GitHub's own error-// text, kept so the error still reads like the 403 it replaces.+// permissionError reports comment-scan coverage lost because the credential+// lacks a permission. accepted holds the permission sets that would restore+// it: any one set is enough, and a set needs every permission in it.+//+// Most come from a request GitHub refused, built by asPermissionError: method+// and path identify the request, and message is GitHub's own error text, kept+// so the error still reads like the 403 it replaces.+//+// The rest come from a GitHub App installation's grants, read before any+// request (see installationGrants): installationID names the installation,+// and outcome says what the scan did about the missing permission. These+// have no request and wrap no error.type permissionError struct {accepted [][]permissioncredential credentialKind@@ -58,6 +65,35 @@ type permissionError struct {path stringmessage stringerr error++ installationID int64+ outcome string+}++// newSkippedListingError reports a listing the scan skipped without calling+// GitHub, because the installation grants none of the permission sets that+// would let it succeed.+func newSkippedListingError(installationID int64, accepted ...[]permission) *permissionError {+ return &permissionError{+ accepted: accepted,+ credential: credentialApp,+ installationID: installationID,+ outcome: "the listing was skipped",+ }+}++// newNarrowedListingError reports items GitHub leaves out of a listing+// because the installation doesn't grant the permission that covers them.+// The listing itself succeeds and returns everything else, with nothing in+// the response to say items are missing, so this error is the only record of+// the gap.+func newNarrowedListingError(installationID int64, listing string, accepted ...[]permission) *permissionError {+ return &permissionError{+ accepted: accepted,+ credential: credentialApp,+ installationID: installationID,+ outcome: "GitHub left them out of the " + listing + " listing",+ }}func (e *permissionError) Error() string {@@ -74,6 +110,15 @@ func (e *permissionError) Error() string {fix = "update the credential's permissions"}+ if e.outcome != "" {+ verb := "is"+ if len(e.accepted) == 1 && len(e.accepted[0]) > 1 {+ verb = "are"+ }+ return fmt.Sprintf("%s: %s %s needed; %s (not granted to installation %d, so %s)",+ subject, describeAccepted(e.accepted), verb, fix, e.installationID, e.outcome)+ }+request := "request"if e.path != "" {request = e.method + " " + e.path