Skip to content

Find, verify, and analyze leaked credentials

github: check App installation grants before listing comments (#5454)

Dave Hubbardc85580b71284
diff --git a/pkg/sources/github/app_client_registry.go b/pkg/sources/github/app_client_registry.go
index 9bc8c58aa..d8f87cad5 100644
--- a/pkg/sources/github/app_client_registry.go
+++ b/pkg/sources/github/app_client_registry.go
@@ -95,6 +95,22 @@ func (cs *AppInstallationClients) AccessToken(ctx context.Context) (string, erro
return token.GetToken(), nil
}
+// GrantedPermissions returns the permissions the installation grants, as
+// listed on its access token. GitHub sends them with every token it issues,
+// so this costs no API call when a token is already cached, which is the
+// usual case: a repo is cloned with the same token before its comments are
+// scanned. The comment scan reads them to skip, or report as incomplete,
+// listings the installation can't fully cover (see installationGrants).
+func (cs *AppInstallationClients) GrantedPermissions(ctx context.Context) (*github.InstallationPermissions, error) {
+ if _, err := cs.AccessToken(ctx); err != nil {
+ return nil, err
+ }
+
+ cs.tokenMu.RLock()
+ defer cs.tokenMu.RUnlock()
+ return cs.token.GetPermissions(), nil
+}
+
func (cs *AppInstallationClients) accessTokenIfValid() *github.InstallationToken {
if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) {
return nil
diff --git a/pkg/sources/github/app_client_registry_test.go b/pkg/sources/github/app_client_registry_test.go
index 295c655ec..66ccfa79a 100644
--- a/pkg/sources/github/app_client_registry_test.go
+++ b/pkg/sources/github/app_client_registry_test.go
@@ -53,6 +53,37 @@ func TestAccessTokenForInstallation(t *testing.T) {
require.NoError(t, err)
})
+ t.Run("granted permissions come from the token", func(t *testing.T) {
+ var hits atomic.Int64
+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "token": "test-token",
+ "expires_at": "2099-01-01T00:00:00Z",
+ "permissions": map[string]string{"issues": "read", "contents": "read"},
+ })
+ })
+ defer server.Close()
+
+ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)
+ require.NoError(t, err)
+
+ perms, err := cs.GrantedPermissions(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, "read", perms.GetIssues())
+ require.Empty(t, perms.GetPullRequests())
+
+ _, err = cs.AccessToken(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, int64(1), hits.Load(), "permissions and clone share one token")
+ })
+
+ t.Run("granted permissions without a token", func(t *testing.T) {
+ cs := &AppInstallationClients{InstallationID: 0}
+ _, err := cs.GrantedPermissions(t.Context())
+ require.Error(t, err)
+ })
+
t.Run("rate limit", func(t *testing.T) {
server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("X-RateLimit-Remaining", "0")
diff --git a/pkg/sources/github/comments_test.go b/pkg/sources/github/comments_test.go
index c3166b6cb..539213dff 100644
--- a/pkg/sources/github/comments_test.go
+++ b/pkg/sources/github/comments_test.go
@@ -124,10 +124,17 @@ func seedRepo(gh *fakeGitHub) {
}
}
+// scanComments runs the REST comment scan with unknown installation grants,
+// as a token-authenticated source would.
func scanComments(t *testing.T, s *Source, gh *fakeGitHub, cutoff *time.Time) (*collectingReporter, error) {
+ t.Helper()
+ return scanCommentsWithGrants(t, s, gh, nil, cutoff)
+}
+
+func scanCommentsWithGrants(t *testing.T, s *Source, gh *fakeGitHub, grants *installationGrants, cutoff *time.Time) (*collectingReporter, error) {
t.Helper()
reporter := &collectingReporter{}
- err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, reporter, cutoff)
+ err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, grants, reporter, cutoff)
return reporter, err
}
@@ -357,6 +364,158 @@ func TestCommentScan_EachFailedPhaseIsReportedAsItsOwnChunkError(t *testing.T) {
assert.Empty(t, reporter.links)
}
+// reportedGrantErrors runs reportCommentErrors on err and describes each
+// chunk error it reports as "<prefix>: skip" or "<prefix>: narrow", where
+// prefix names the phase or the content left out. Any reported error that
+// isn't a grants-based permissionError fails the test.
+func reportedGrantErrors(t *testing.T, err error) []string {
+ t.Helper()
+ reporter := &collectingReporter{}
+ require.NoError(t, reportCommentErrors(context.Background(), reporter, err))
+ var got []string
+ for _, reported := range reporter.errs {
+ var permErr *permissionError
+ require.ErrorAs(t, reported, &permErr)
+ prefix, _, _ := strings.Cut(strings.TrimPrefix(reported.Error(), "error scanning comments: "), ": missing")
+ switch {
+ case strings.Contains(permErr.outcome, "skipped"):
+ got = append(got, prefix+": skip")
+ case strings.Contains(permErr.outcome, "left them out"):
+ got = append(got, prefix+": narrow")
+ default:
+ t.Fatalf("unexpected permission error: %v", reported)
+ }
+ }
+ return got
+}
+
+func TestCommentScan_InstallationGrantsShapeTheListings(t *testing.T) {
+ type grant struct{ issues, pullRequests bool }
+ var (
+ neither = grant{}
+ prsOnly = grant{pullRequests: true}
+ issues = grant{issues: true}
+ both = grant{issues: true, pullRequests: true}
+ )
+ tests := []struct {
+ name string
+ issueComments bool
+ prComments bool
+ grant grant
+ wantErrs []string
+ // wantListed names every endpoint the scan requests; the rest must
+ // not be requested.
+ wantListed []string
+ }{
+ {name: "issue comments, nothing granted", issueComments: true, grant: neither,
+ wantErrs: []string{"issues: skip", "issue comments: skip"}},
+ {name: "issue comments, pull requests granted", issueComments: true, grant: prsOnly,
+ wantErrs: []string{"issues: skip", "issue comments: narrow"},
+ wantListed: []string{"issue comments"}},
+ {name: "issue comments, issues granted", issueComments: true, grant: issues,
+ wantListed: []string{"issues", "issue comments"}},
+ {name: "issue comments, both granted", issueComments: true, grant: both,
+ wantListed: []string{"issues", "issue comments"}},
+
+ {name: "PR comments, nothing granted", prComments: true, grant: neither,
+ wantErrs: []string{"pull requests: skip", "pull request comments: skip"}},
+ {name: "PR comments, pull requests granted", prComments: true, grant: prsOnly,
+ wantListed: []string{"pulls", "pull comments"}},
+ {name: "PR comments, issues granted", prComments: true, grant: issues,
+ wantErrs: []string{"pull requests: skip", "pull request comments: skip"}},
+ {name: "PR comments, both granted", prComments: true, grant: both,
+ wantListed: []string{"pulls", "pull comments"}},
+
+ {name: "both settings, nothing granted", issueComments: true, prComments: true, grant: neither,
+ wantErrs: []string{"issues and pull requests: skip", "issue comments: skip", "pull request comments: skip"}},
+ {name: "both settings, pull requests granted", issueComments: true, prComments: true, grant: prsOnly,
+ wantErrs: []string{"issues: skip", "issue comments: narrow"},
+ wantListed: []string{"pulls", "issue comments", "pull comments"}},
+ {name: "both settings, issues granted", issueComments: true, prComments: true, grant: issues,
+ wantErrs: []string{"pull request bodies: narrow", "pull request conversation comments: narrow", "pull request comments: skip"},
+ wantListed: []string{"issues", "issue comments"}},
+ {name: "both settings, both granted", issueComments: true, prComments: true, grant: both,
+ wantListed: []string{"issues", "issue comments", "pull comments"}},
+
+ {name: "comments off, nothing granted", grant: neither},
+ {name: "comments off, pull requests granted", grant: prsOnly},
+ {name: "comments off, issues granted", grant: issues},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includeIssueComments: tt.issueComments, includePRComments: tt.prComments}
+ grants := &installationGrants{installationID: 42, issues: tt.grant.issues, pullRequests: tt.grant.pullRequests}
+
+ _, err := scanCommentsWithGrants(t, s, gh, grants, nil)
+
+ if len(tt.wantErrs) == 0 {
+ require.NoError(t, err)
+ } else {
+ assert.Equal(t, tt.wantErrs, reportedGrantErrors(t, err))
+ }
+ endpoints := map[string]*fakeListing{
+ "issues": gh.issues, "issue comments": gh.issueComments,
+ "pulls": gh.pulls, "pull comments": gh.prComments,
+ }
+ var listed []string
+ for name, listing := range endpoints {
+ if listing.requestCount() > 0 {
+ listed = append(listed, name)
+ }
+ }
+ assert.ElementsMatch(t, tt.wantListed, listed)
+ })
+ }
+}
+
+func TestCommentScan_PullRequestBodiesComeFromPullsWithoutIssuesGrant(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includeIssueComments: true, includePRComments: true}
+
+ reporter, _ := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42, pullRequests: true}, nil)
+
+ assert.Contains(t, reporter.links, prLink(2))
+ assert.NotContains(t, reporter.links, "https://github.com/o/r/issues/1")
+ assert.Zero(t, gh.issues.requestCount())
+}
+
+func TestCommentScan_GrantErrorsNameTheInstallationAndPermissions(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includeIssueComments: true, includePRComments: true}
+
+ _, err := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42, issues: true}, nil)
+
+ reporter := &collectingReporter{}
+ require.NoError(t, reportCommentErrors(context.Background(), reporter, err))
+ var got []string
+ for _, reported := range reporter.errs {
+ got = append(got, reported.Error())
+ }
+ fix := "update the app's permissions and accept the change on the installation"
+ assert.Equal(t, []string{
+ "error scanning comments: pull request bodies: missing GitHub App permission: Pull requests (read) is needed; " + fix +
+ " (not granted to installation 42, so GitHub left them out of the /issues listing)",
+ "error scanning comments: pull request conversation comments: missing GitHub App permission: Pull requests (read) is needed; " + fix +
+ " (not granted to installation 42, so GitHub left them out of the /issues/comments listing)",
+ "error scanning comments: pull request comments: missing GitHub App permission: Pull requests (read) is needed; " + fix +
+ " (not granted to installation 42, so the listing was skipped)",
+ }, got)
+}
+
+func TestCommentScan_SkippedListingNeedingBothPermissionsSaysSo(t *testing.T) {
+ gh := newFakeGitHub(t)
+ s := &Source{includeIssueComments: true, includePRComments: true}
+
+ _, err := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42}, nil)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "issues and pull requests: missing GitHub App permission: Issues (read) and Pull requests (read) are needed;")
+}
+
func TestReportCommentErrors_UnjoinedErrorIsReportedOnce(t *testing.T) {
reporter := &collectingReporter{}
@@ -380,7 +539,7 @@ func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {
logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(2)))
ctx := context.WithLogger(context.Background(), logger)
- err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, &collectingReporter{}, nil)
+ err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, nil, &collectingReporter{}, nil)
require.NoError(t, err)
var listings []string
diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go
index 027d967ae..17e0ec0ef 100644
--- a/pkg/sources/github/connector_app.go
+++ b/pkg/sources/github/connector_app.go
@@ -149,6 +149,28 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string
return git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...)
}
+// grantsForRepo returns what the installation that owns repoURL grants, for
+// the permissions the comment scan needs. Installation permissions apply to
+// every repo the installation can see, so repos on one installation share
+// the answer, and the cached token makes it free after the repo's clone.
+func (c *appConnector) grantsForRepo(ctx context.Context, repoURL string) (*installationGrants, error) {
+ installID, _ := c.installationIDForRepo(repoURL)
+ if installID == 0 {
+ return nil, fmt.Errorf("no GitHub App installation resolved for repo %q", repoURL)
+ }
+
+ cs, err := c.clientsForInstallation(installID)
+ if err != nil {
+ return nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)
+ }
+
+ perms, err := cs.GrantedPermissions(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("could not read permissions for installation %d: %w", installID, err)
+ }
+ return newInstallationGrants(installID, perms), nil
+}
+
// installationIDForRepo returns the mapped installation ID for repoURL. When no
// mapping exists, it falls back to the configured installation and returns false.
//
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 5192e1abd..bdfa96a6b 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -780,12 +780,16 @@ func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector,
ctx.Logger().Info("Skipping org member repos and gists: githubApp.installationId is required as a fallback installation to scan members together with scanAllInstallations", "org_members", len(s.memberCache))
return nil
}
- // TODO: Replace loop below with a call to s.addReposForMembers(ctx, reporter)
+ // Members' gists aren't listed, unlike in addReposForMembers: none
+ // of GitHub's gist endpoints accept an installation access token,
+ // and no app permission changes that, so the listing fails with a
+ // 403 for every member. One line here stands in for those
+ // failures; there's nothing for the customer to grant.
+ if !s.ignoreGists {
+ ctx.Logger().Info("Skipping org member gists: GitHub doesn't allow listing gists with a GitHub App installation token", "org_members", len(s.memberCache))
+ }
for member := range s.memberCache {
logger := ctx.Logger().WithValues("member", member)
- if err := s.addUserGistsToCache(ctx, member, reporter); err != nil {
- logger.Error(err, "error fetching gists by user")
- }
// TODO: Add authenticated user list repo for app token. It does support as per docs but need to test it before we enable it here.
// docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-the-authenticated-user
if err := s.getReposByUser(ctx, member, false, reporter); err != nil {
@@ -1830,12 +1834,30 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep
return s.processRepoIssueandPRsWithCommentsGraphql(ctx, graphqlClient, repoInfo, reporter, cutoffTime)
}
- return s.processIssueandPRsWithCommentsREST(ctx, apiClient, repoInfo, reporter, cutoffTime)
+ return s.processIssueandPRsWithCommentsREST(ctx, apiClient, repoInfo, s.commentGrantsForRepo(ctx, repoPath), reporter, cutoffTime)
}
return nil
}
+// commentGrantsForRepo returns what the GitHub App installation behind
+// repoURL grants, or nil (unknown) when the source doesn't authenticate as an
+// App. A failure to read the grants also gives nil: the listings then run
+// without the precheck, as they would for a token, and any 403 is still
+// classified from its response.
+func (s *Source) commentGrantsForRepo(ctx context.Context, repoURL string) *installationGrants {
+ app, ok := s.connector.(*appConnector)
+ if !ok {
+ return nil
+ }
+ grants, err := app.grantsForRepo(ctx, repoURL)
+ if err != nil {
+ ctx.Logger().V(2).Info("could not read installation permissions, scanning comments without the permission precheck", "error", err)
+ return nil
+ }
+ return grants
+}
+
// trimURLAndSplit removes extraneous information from the |url| and splits it into segments.
// This is typically 3 segments: host, owner, and name/ID; however, Gists have some edge cases.
//
@@ -2022,10 +2044,18 @@ const (
// commentPhase is one listing in a repo's comment scan. run returns how many
// chunks it emitted. startLog, when set, is logged at V(2) before the phase
// runs.
+//
+// skipped and narrowed carry what the installation's grants cost the phase
+// (see installationGrants). skipped, when set, is why the listing can't
+// succeed; the phase reports it in place of running. narrowed holds one error
+// per kind of item GitHub will leave out of a listing that still runs, each
+// already prefixed with what's missing, and is reported after the run.
type commentPhase struct {
name string
startLog string
run func() (int, error)
+ skipped error
+ narrowed []error
}
// processIssueandPRsWithCommentsREST scans a repo's issues, pull requests, and
@@ -2037,13 +2067,20 @@ type commentPhase struct {
// so each failed phase is its own ChunkErr (see reportCommentErrors). Chunks
// emitted before a failure are kept.
//
+// grants, when known, shapes the phases. A mismatch between the comment
+// settings and what the installation grants is a configuration error the
+// customer can fix, so it's always reported: every listing skipped because it
+// can't succeed, and every kind of item GitHub will silently leave out of a
+// listing that does run, gets exactly one permissionError. Items no setting
+// asked for don't count. Skipped listings make no request.
+//
// The "Scanning issues" and "Scanning pull requests" V(2) lines are the only
// log signal for scans on the unit path, whose comment errors go to the scan
// details UI instead of the logs, so they keep their wording. The summary
// line at the end records how far each phase got.
func (s *Source) processIssueandPRsWithCommentsREST(
ctx context.Context, apiClient *github.Client, repoInfo repoInfo,
- reporter sources.ChunkReporter, cutoffTime *time.Time,
+ grants *installationGrants, reporter sources.ChunkReporter, cutoffTime *time.Time,
) error {
p := s.commentPager(ctx, apiClient, reporter)
var since time.Time
@@ -2051,47 +2088,102 @@ func (s *Source) processIssueandPRsWithCommentsREST(
since = *cutoffTime
}
+ var installationID int64
+ if grants != nil {
+ installationID = grants.installationID
+ }
+ issuesOK, prsOK := grants.issuesGranted(), grants.pullRequestsGranted()
+ narrowed := func(missing, listing string, needs permission) error {
+ return fmt.Errorf("%s: %w", missing, newNarrowedListingError(installationID, listing, []permission{needs}))
+ }
+ pullsPhase := func(startLog string) commentPhase {
+ return commentPhase{name: "pull requests", startLog: startLog, run: func() (int, error) {
+ return s.processPRs(ctx, p, repoInfo, reporter)
+ }}
+ }
+
// /issues returns issues and pull requests together. When issue comments
// are enabled, that one walk also emits PR bodies, so /pulls is only
- // walked for configs that enable PR comments alone.
+ // walked for configs that enable PR comments alone, or when the
+ // installation can list pull requests but not issues.
var phases []commentPhase
if s.includeIssueComments {
- bodies := "issues"
+ bodies := commentPhase{name: "issues", startLog: "Scanning issues", run: func() (int, error) {
+ return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)
+ }}
+ needs := []permission{issuesRead}
if s.includePRComments {
- bodies = "issues and pull requests"
- }
- phases = append(phases,
- commentPhase{name: bodies, startLog: "Scanning issues", run: func() (int, error) {
- return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)
- }},
- commentPhase{name: "issue comments", run: func() (int, error) {
- return s.processIssueComments(ctx, p, repoInfo, reporter, since)
- }},
- )
+ bodies.name = "issues and pull requests"
+ needs = append(needs, pullRequestsRead)
+ }
+ switch {
+ case issuesOK:
+ if s.includePRComments && !prsOK {
+ bodies.narrowed = append(bodies.narrowed, narrowed("pull request bodies", "/issues", pullRequestsRead))
+ }
+ phases = append(phases, bodies)
+ case s.includePRComments && prsOK:
+ // /issues is refused without Issues (read), but the PR bodies it
+ // would have carried are still reachable through /pulls.
+ phases = append(phases,
+ commentPhase{name: "issues", skipped: newSkippedListingError(installationID, []permission{issuesRead})},
+ pullsPhase(""))
+ default:
+ bodies.skipped = newSkippedListingError(installationID, needs)
+ phases = append(phases, bodies)
+ }
+
+ // /issues/comments carries comments on issues and on pull requests,
+ // each kind only when its permission is granted. PR conversation
+ // comments are scanned only when PR comments are enabled, so their
+ // absence otherwise isn't a mismatch.
+ comments := commentPhase{name: "issue comments", run: func() (int, error) {
+ return s.processIssueComments(ctx, p, repoInfo, reporter, since)
+ }}
+ switch {
+ case !issuesOK && !prsOK:
+ comments.skipped = newSkippedListingError(installationID, needs)
+ case !issuesOK:
+ comments.narrowed = append(comments.narrowed, narrowed("issue comments", "/issues/comments", issuesRead))
+ case s.includePRComments && !prsOK:
+ comments.narrowed = append(comments.narrowed, narrowed("pull request conversation comments", "/issues/comments", pullRequestsRead))
+ }
+ phases = append(phases, comments)
}
if s.includePRComments {
+ var prsSkipped error
+ if !prsOK {
+ prsSkipped = newSkippedListingError(installationID, []permission{pullRequestsRead})
+ }
prStartLog := "Scanning pull requests"
if !s.includeIssueComments {
- phases = append(phases, commentPhase{name: "pull requests", startLog: prStartLog, run: func() (int, error) {
- return s.processPRs(ctx, p, repoInfo, reporter)
- }})
+ pulls := pullsPhase(prStartLog)
+ pulls.skipped = prsSkipped
+ phases = append(phases, pulls)
prStartLog = ""
}
- phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, run: func() (int, error) {
+ phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, skipped: prsSkipped, run: func() (int, error) {
return s.processPRComments(ctx, p, repoInfo, reporter, since)
}})
}
var (
- errs []error
- failed []string
- summary []any
+ errs []error
+ failed []string
+ skipped []string
+ narrowedOut []string
+ summary []any
)
for _, phase := range phases {
if err := ctx.Err(); err != nil {
errs = append(errs, err)
break
}
+ if phase.skipped != nil {
+ skipped = append(skipped, phase.name)
+ errs = append(errs, fmt.Errorf("%s: %w", phase.name, phase.skipped))
+ continue
+ }
if phase.startLog != "" {
ctx.Logger().V(2).Info(phase.startLog)
}
@@ -2101,8 +2193,13 @@ func (s *Source) processIssueandPRsWithCommentsREST(
failed = append(failed, phase.name)
errs = append(errs, fmt.Errorf("%s: %w", phase.name, err))
}
+ if len(phase.narrowed) > 0 {
+ narrowedOut = append(narrowedOut, phase.name)
+ errs = append(errs, phase.narrowed...)
+ }
}
- ctx.Logger().V(2).Info("finished scanning comments", append(summary, "failed_phases", failed)...)
+ ctx.Logger().V(2).Info("finished scanning comments", append(summary,
+ "failed_phases", failed, "skipped_phases", skipped, "narrowed_phases", narrowedOut)...)
return errors.Join(errs...)
}
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index dad98fee8..e71c2f289 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -15,6 +15,7 @@ import (
"slices"
"strconv"
"strings"
+ "sync"
"testing"
"time"
@@ -1071,6 +1072,61 @@ func TestEnumerateWithAppScanAllInstallationsIncludesConfiguredInstallation(t *t
assert.True(t, s.filteredRepoCache.Exists("other-org/repo"))
}
+func TestEnumerateWithAppScanUsersListsMemberReposButNotGists(t *testing.T) {
+ privateKey := createPrivateKey()
+
+ var (
+ mu sync.Mutex
+ paths []string
+ )
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ paths = append(paths, r.URL.Path)
+ mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+
+ switch {
+ case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/access_tokens"):
+ _, _ = w.Write([]byte(`{"token":"token","expires_at":"2099-01-01T00:00:00Z"}`))
+ case strings.HasSuffix(r.URL.Path, "/installation/repositories"):
+ _, _ = w.Write([]byte(`{"total_count":0,"repositories":[]}`))
+ case strings.HasSuffix(r.URL.Path, "/app/installations"):
+ _, _ = w.Write([]byte(`[{"id":1337,"account":{"login":"default-org","type":"Organization"}}]`))
+ case strings.HasSuffix(r.URL.Path, "/orgs/default-org/members"):
+ _, _ = w.Write([]byte(`[{"login":"alice"}]`))
+ case strings.HasSuffix(r.URL.Path, "/users/alice/repos"):
+ _, _ = w.Write([]byte(`[]`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ s, conn := createTestSource(&sourcespb.GitHub{
+ Endpoint: server.URL,
+ ScanUsers: true,
+ Credential: &sourcespb.GitHub_GithubApp{
+ GithubApp: &credentialspb.GitHubApp{
+ PrivateKey: privateKey,
+ InstallationId: "1337",
+ AppId: "4141",
+ },
+ },
+ })
+ require.NoError(t, s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1))
+ require.False(t, s.ignoreGists)
+
+ err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector), noopReporter())
+ require.NoError(t, err)
+
+ mu.Lock()
+ defer mu.Unlock()
+ assert.Contains(t, paths, "/api/v3/users/alice/repos")
+ for _, path := range paths {
+ assert.NotContains(t, path, "gists")
+ }
+}
+
func TestEnumerateWithAppScanAllInstallationsMapsExplicitReposBeforeMetadataFetch(t *testing.T) {
privateKey := createPrivateKey()
diff --git a/pkg/sources/github/installation_grants.go b/pkg/sources/github/installation_grants.go
new file mode 100644
index 000000000..13ef090a3
--- /dev/null
+++ b/pkg/sources/github/installation_grants.go
@@ -0,0 +1,65 @@
+package github
+
+import "github.com/google/go-github/v67/github"
+
+// Permissions the comment listings need, in the form permissionError names
+// them.
+var (
+ issuesRead = permission{name: "issues", access: "read"}
+ pullRequestsRead = permission{name: "pull_requests", access: "read"}
+)
+
+// installationGrants is what a GitHub App installation grants, for the
+// permissions the comment scan needs. A nil *installationGrants means the
+// grants aren't known (a personal access token, or an App token whose
+// permissions couldn't be read), and every listing runs as if granted; a 403
+// is then classified from its response by asPermissionError.
+//
+// The grants matter beyond outright refusals because GitHub doesn't refuse
+// every listing they only partly cover. Measured against a live installation:
+// - /issues with Issues (read) but not Pull requests (read) returns the
+// issues and leaves out the pull requests.
+// - /issues/comments with only one of the two returns only the comments that
+// one covers: issue comments with Issues (read), pull request
+// conversation comments with Pull requests (read).
+//
+// Both answer 200 with no error, so the comment scan reads the grants to skip
+// listings that can't succeed and to report what the others leave out (see
+// processIssueandPRsWithCommentsREST).
+type installationGrants struct {
+ installationID int64
+ issues bool
+ pullRequests bool
+}
+
+// newInstallationGrants reads the comment-relevant grants from an access
+// token's permissions. GitHub lists them on every installation token, so a
+// token without any is unexpected; it returns nil (unknown) rather than
+// treating the installation as granting nothing.
+func newInstallationGrants(installationID int64, perms *github.InstallationPermissions) *installationGrants {
+ if perms == nil {
+ return nil
+ }
+ return &installationGrants{
+ installationID: installationID,
+ issues: grantsRead(perms.GetIssues()),
+ pullRequests: grantsRead(perms.GetPullRequests()),
+ }
+}
+
+// grantsRead reports whether an installation permission level allows
+// reading. Write access includes read.
+func grantsRead(level string) bool {
+ switch level {
+ case "read", "write", "admin":
+ return true
+ default:
+ return false
+ }
+}
+
+// issuesGranted and pullRequestsGranted report whether the permission is
+// granted, treating unknown grants as granted so listings run as they would
+// without the precheck.
+func (g *installationGrants) issuesGranted() bool { return g == nil || g.issues }
+func (g *installationGrants) pullRequestsGranted() bool { return g == nil || g.pullRequests }
diff --git a/pkg/sources/github/installation_grants_test.go b/pkg/sources/github/installation_grants_test.go
new file mode 100644
index 000000000..5730469cf
--- /dev/null
+++ b/pkg/sources/github/installation_grants_test.go
@@ -0,0 +1,89 @@
+package github
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/google/go-github/v67/github"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+func TestNewInstallationGrants(t *testing.T) {
+ tests := []struct {
+ name string
+ perms *github.InstallationPermissions
+ wantIssues bool
+ wantPullRequests bool
+ }{
+ {name: "read grants", perms: &github.InstallationPermissions{Issues: github.String("read"), PullRequests: github.String("read")},
+ wantIssues: true, wantPullRequests: true},
+ {name: "write includes read", perms: &github.InstallationPermissions{Issues: github.String("write")},
+ wantIssues: true},
+ {name: "absent permissions aren't granted", perms: &github.InstallationPermissions{Contents: github.String("read")}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := newInstallationGrants(7, tt.perms)
+
+ require.NotNil(t, g)
+ assert.Equal(t, int64(7), g.installationID)
+ assert.Equal(t, tt.wantIssues, g.issuesGranted())
+ assert.Equal(t, tt.wantPullRequests, g.pullRequestsGranted())
+ })
+ }
+}
+
+func TestNewInstallationGrants_TokenWithoutPermissionsIsUnknown(t *testing.T) {
+ g := newInstallationGrants(7, nil)
+
+ assert.Nil(t, g)
+ assert.True(t, g.issuesGranted(), "unknown grants let listings run")
+ assert.True(t, g.pullRequestsGranted(), "unknown grants let listings run")
+}
+
+func TestCommentGrantsForRepo_ReadsTheRepoInstallation(t *testing.T) {
+ privKey := generateTestPrivateKey(t)
+ var tokenPath string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ tokenPath = r.URL.Path
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "token": "test-token",
+ "expires_at": "2099-01-01T00:00:00Z",
+ "permissions": map[string]string{"pull_requests": "read"},
+ })
+ }))
+ defer server.Close()
+ conn := &appConnector{
+ apiEndpoint: server.URL, appID: 12345, appPrivateKey: string(privKey), installationID: 1,
+ repoInstallationMap: map[string]int64{"https://github.com/o/r.git": 777},
+ }
+ s := &Source{connector: conn}
+
+ grants := s.commentGrantsForRepo(context.Background(), "https://github.com/o/r.git")
+
+ assert.Equal(t, "/api/v3/app/installations/777/access_tokens", tokenPath)
+ assert.Equal(t, &installationGrants{installationID: 777, pullRequests: true}, grants)
+}
+
+func TestCommentGrantsForRepo_UnreadableGrantsAreUnknown(t *testing.T) {
+ privKey := generateTestPrivateKey(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+ }))
+ defer server.Close()
+ s := &Source{connector: &appConnector{apiEndpoint: server.URL, appID: 12345, appPrivateKey: string(privKey), installationID: 1}}
+
+ assert.Nil(t, s.commentGrantsForRepo(context.Background(), "https://github.com/o/r.git"))
+}
+
+func TestCommentGrantsForRepo_UnknownWithoutAppAuth(t *testing.T) {
+ s := &Source{connector: &tokenConnector{}}
+
+ assert.Nil(t, s.commentGrantsForRepo(context.Background(), "https://github.com/o/r.git"))
+}
diff --git a/pkg/sources/github/permission_error.go b/pkg/sources/github/permission_error.go
index ed0e27206..0672dd650 100644
--- a/pkg/sources/github/permission_error.go
+++ b/pkg/sources/github/permission_error.go
@@ -46,11 +46,18 @@ const (
credentialFineGrainedToken
)
-// permissionError reports a request GitHub refused because the credential
-// lacks a permission. accepted holds the permission sets that would satisfy
-// the endpoint: any one set is enough, and a set needs every permission in it.
-// method and path identify the request, and message is GitHub's own error
-// text, kept so the error still reads like the 403 it replaces.
+// permissionError reports comment-scan coverage lost because the credential
+// lacks a permission. accepted holds the permission sets that would restore
+// it: any one set is enough, and a set needs every permission in it.
+//
+// Most come from a request GitHub refused, built by asPermissionError: method
+// and path identify the request, and message is GitHub's own error text, kept
+// so the error still reads like the 403 it replaces.
+//
+// The rest come from a GitHub App installation's grants, read before any
+// request (see installationGrants): installationID names the installation,
+// and outcome says what the scan did about the missing permission. These
+// have no request and wrap no error.
type permissionError struct {
accepted [][]permission
credential credentialKind
@@ -58,6 +65,35 @@ type permissionError struct {
path string
message string
err error
+
+ installationID int64
+ outcome string
+}
+
+// newSkippedListingError reports a listing the scan skipped without calling
+// GitHub, because the installation grants none of the permission sets that
+// would let it succeed.
+func newSkippedListingError(installationID int64, accepted ...[]permission) *permissionError {
+ return &permissionError{
+ accepted: accepted,
+ credential: credentialApp,
+ installationID: installationID,
+ outcome: "the listing was skipped",
+ }
+}
+
+// newNarrowedListingError reports items GitHub leaves out of a listing
+// because the installation doesn't grant the permission that covers them.
+// The listing itself succeeds and returns everything else, with nothing in
+// the response to say items are missing, so this error is the only record of
+// the gap.
+func newNarrowedListingError(installationID int64, listing string, accepted ...[]permission) *permissionError {
+ return &permissionError{
+ accepted: accepted,
+ credential: credentialApp,
+ installationID: installationID,
+ outcome: "GitHub left them out of the " + listing + " listing",
+ }
}
func (e *permissionError) Error() string {
@@ -74,6 +110,15 @@ func (e *permissionError) Error() string {
fix = "update the credential's permissions"
}
+ if e.outcome != "" {
+ verb := "is"
+ if len(e.accepted) == 1 && len(e.accepted[0]) > 1 {
+ verb = "are"
+ }
+ return fmt.Sprintf("%s: %s %s needed; %s (not granted to installation %d, so %s)",
+ subject, describeAccepted(e.accepted), verb, fix, e.installationID, e.outcome)
+ }
+
request := "request"
if e.path != "" {
request = e.method + " " + e.path