Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[INT-1051] Fix GitHub comment scans stopping early on large repositories (#5405)
Dave Hubbardb40e7fe3b767
diff --git a/pkg/sources/github/README.md b/pkg/sources/github/README.mdindex 805d8309e..8fb85e37d 100644--- a/pkg/sources/github/README.md+++ b/pkg/sources/github/README.md@@ -270,6 +270,16 @@ With `--clone-path` set, each repository still gets its own fresh subdirectory c5. **Comment Scanning**: If any comment flags are set, fetches issues, pull requests, or gist comments through the GitHub API and scans their text6. **Progress Tracking**: Keeps track of which repositories have already been scanned, so a scan can be resumed without redoing work+### How Comments Are Fetched++- `--issue-comments` scans issue titles and bodies, and every discussion comment in the repository. GitHub treats pull requests as issues, so this includes the discussion comments on pull requests.+- `--pr-comments` scans pull request titles and bodies, and every review comment (the comments left inline on a diff).+- Listings are paged by following the links GitHub returns, sorted by last update time. Some GitHub endpoints stop returning results after a fixed number of pages (the repository-wide issue comment listing stops at 300 pages); when that happens, the scan picks up from the last update time it reached, so large repositories are scanned in full.+- A request that fails for a transient reason (a GitHub server error, a dropped connection, or a response cut off partway) is retried up to five times with backoff before it counts as a failure. Rate limits are waited out and retried separately, without a limit.+- The comments timeframe is applied by GitHub. Issue and pull request bodies are scanned regardless of the timeframe.+- Each listing is independent. If one fails, the others still run, everything read before the failure is still scanned, and the failure is reported as a scan error for that repository naming the listing, how many items it read, and where it stopped.+- Gist comments can't be filtered or sorted by GitHub, so the timeframe is applied as they are read, and a gist whose comments GitHub stops paging through is reported as a scan error.+### What Gets Scanned- Every commit in every repository matching the scan target and filters@@ -282,7 +292,7 @@ With `--clone-path` set, each repository still gets its own fresh subdirectory c- Archived repositories, when `--exclude-archived` is set- Repositories excluded by `--include-repos` or `--exclude-repos`, for an organization or user scan- Gists, when `--ignore-gists` is set, or when the scan target is a genuine organization rather than a user (gists only surface for user accounts, see the note above)-- Comments older than the configured comments timeframe, when one is set+- Comments last updated before the configured comments timeframe, when one is set (gist comments go by creation time)## Usage Examplesdiff --git a/pkg/sources/github/comments_test.go b/pkg/sources/github/comments_test.gonew file mode 100644index 000000000..77f66d237--- /dev/null+++ b/pkg/sources/github/comments_test.go@@ -0,0 +1,380 @@+package github++import (+ "bytes"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/google/go-github/v67/github"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/log"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++// fakeGitHub serves the REST endpoints the comment scan reads for repo o/r+// and gist g1, each backed by its own fakeListing.+type fakeGitHub struct {+ issues *fakeListing+ issueComments *fakeListing+ pulls *fakeListing+ prComments *fakeListing+ gistComments *fakeListing+ client *github.Client+}++func newFakeGitHub(t *testing.T) *fakeGitHub {+ t.Helper()+ gh := &fakeGitHub{+ issues: &fakeListing{},+ issueComments: &fakeListing{},+ pulls: &fakeListing{},+ prComments: &fakeListing{},+ gistComments: &fakeListing{},+ }+ mux := http.NewServeMux()+ mux.Handle("/repos/o/r/issues", gh.issues)+ mux.Handle("/repos/o/r/issues/comments", gh.issueComments)+ mux.Handle("/repos/o/r/pulls", gh.pulls)+ mux.Handle("/repos/o/r/pulls/comments", gh.prComments)+ mux.Handle("/gists/g1/comments", gh.gistComments)+ srv := httptest.NewServer(mux)+ t.Cleanup(srv.Close)+ gh.client = newTestClient(t, srv)+ return gh+}++var testRepo = repoInfo{owner: "o", name: "r", fullName: "o/r"}++// collectingReporter records the link of every chunk the scan emits.+type collectingReporter struct {+ mu sync.Mutex+ links []string+}++func (r *collectingReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {+ r.mu.Lock()+ defer r.mu.Unlock()+ r.links = append(r.links, chunk.SourceMetadata.GetGithub().GetLink())+ return nil+}++func (r *collectingReporter) ChunkErr(context.Context, error) error { return nil }++func issueRecord(number int, updated time.Time) fakeRecord {+ return fakeRecord{updated: updated, body: map[string]any{+ "id": number, "number": number, "updated_at": updated, "created_at": updated,+ "html_url": fmt.Sprintf("https://github.com/o/r/issues/%d", number),+ "title": "issue", "body": "body",+ }}+}++// prIssueRecord is a pull request as /issues returns it.+func prIssueRecord(number int, updated time.Time) fakeRecord {+ return fakeRecord{updated: updated, body: map[string]any{+ "id": number, "number": number, "updated_at": updated, "created_at": updated,+ "html_url": prLink(number),+ "title": "pr", "body": "body",+ "pull_request": map[string]any{"url": fmt.Sprintf("https://api.github.com/repos/o/r/pulls/%d", number)},+ }}+}++func prRecord(number int, updated time.Time) fakeRecord {+ return fakeRecord{updated: updated, body: map[string]any{+ "id": number, "number": number, "updated_at": updated, "created_at": updated,+ "html_url": prLink(number), "title": "pr", "body": "body",+ }}+}++func prLink(number int) string { return fmt.Sprintf("https://github.com/o/r/pull/%d", number) }++func commentRecord(id int, updated time.Time, htmlURL string) fakeRecord {+ return fakeRecord{updated: updated, body: map[string]any{+ "id": id, "updated_at": updated, "created_at": updated, "html_url": htmlURL, "body": "comment",+ }}+}++// seedRepo gives o/r one issue, one pull request, a discussion comment on+// each, and a review comment on the pull request.+func seedRepo(gh *fakeGitHub) {+ gh.issues.records = []fakeRecord{issueRecord(1, testEpoch), prIssueRecord(2, testEpoch.Add(time.Minute))}+ gh.pulls.records = []fakeRecord{prRecord(2, testEpoch.Add(time.Minute))}+ gh.issueComments.records = []fakeRecord{+ commentRecord(100, testEpoch.Add(2*time.Minute), "https://github.com/o/r/issues/1#issuecomment-100"),+ commentRecord(101, testEpoch.Add(3*time.Minute), "https://github.com/o/r/pull/2#issuecomment-101"),+ }+ gh.prComments.records = []fakeRecord{+ commentRecord(200, testEpoch.Add(4*time.Minute), "https://github.com/o/r/pull/2#discussion_r200"),+ }+}++func scanComments(t *testing.T, s *Source, gh *fakeGitHub, cutoff *time.Time) (*collectingReporter, error) {+ t.Helper()+ reporter := &collectingReporter{}+ err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, reporter, cutoff)+ return reporter, err+}++func TestCommentScan_FlagCombinations(t *testing.T) {+ tests := []struct {+ name string+ issueComments bool+ prComments bool+ wantLinks []string+ // unlisted names the endpoints the scan must not request.+ unlisted []string+ }{+ {+ name: "issue comments only",+ issueComments: true,+ wantLinks: []string{+ "https://github.com/o/r/issues/1",+ "https://github.com/o/r/issues/1#issuecomment-100",+ "https://github.com/o/r/pull/2#issuecomment-101",+ },+ unlisted: []string{"pulls", "pull comments"},+ },+ {+ name: "pull request comments only",+ prComments: true,+ wantLinks: []string{+ prLink(2),+ "https://github.com/o/r/pull/2#discussion_r200",+ },+ unlisted: []string{"issues", "issue comments"},+ },+ {+ name: "both",+ issueComments: true,+ prComments: true,+ wantLinks: []string{+ "https://github.com/o/r/issues/1",+ prLink(2),+ "https://github.com/o/r/issues/1#issuecomment-100",+ "https://github.com/o/r/pull/2#issuecomment-101",+ "https://github.com/o/r/pull/2#discussion_r200",+ },+ unlisted: []string{"pulls"},+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includeIssueComments: tt.issueComments, includePRComments: tt.prComments}++ reporter, err := scanComments(t, s, gh, nil)++ require.NoError(t, err)+ assert.ElementsMatch(t, tt.wantLinks, reporter.links)+ endpoints := map[string]*fakeListing{+ "issues": gh.issues, "issue comments": gh.issueComments,+ "pulls": gh.pulls, "pull comments": gh.prComments,+ }+ for _, name := range tt.unlisted {+ assert.Zero(t, endpoints[name].requestCount(), "%s should not be listed", name)+ }+ })+ }+}++func TestCommentScan_RequestsSortedListingsWithServerSideCutoff(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includeIssueComments: true, includePRComments: true}+ cutoff := testEpoch.Add(-time.Hour)++ _, err := scanComments(t, s, gh, &cutoff)+ require.NoError(t, err)++ issues := gh.issues.requests[0]+ assert.Equal(t, "all", issues.Get("state"))+ assert.Equal(t, "updated", issues.Get("sort"))+ assert.Equal(t, "asc", issues.Get("direction"))+ assert.Equal(t, "100", issues.Get("per_page"))+ assert.Empty(t, issues.Get("since"), "issue and PR bodies are scanned regardless of the comments timeframe")++ for name, listing := range map[string]*fakeListing{"issue comments": gh.issueComments, "pull comments": gh.prComments} {+ q := listing.requests[0]+ assert.Equal(t, "updated", q.Get("sort"), name)+ assert.Equal(t, "asc", q.Get("direction"), name)+ assert.Equal(t, cutoff.Format(time.RFC3339), q.Get("since"), name)+ }+}++func TestCommentScan_OmitsSinceWithoutCutoff(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includeIssueComments: true, includePRComments: true}++ _, err := scanComments(t, s, gh, nil)+ require.NoError(t, err)++ assert.False(t, gh.issueComments.requests[0].Has("since"))+ assert.False(t, gh.prComments.requests[0].Has("since"))+}++func TestCommentScan_PullsListOnlyConfigListsPullsWithAllStates(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ s := &Source{includePRComments: true}++ _, err := scanComments(t, s, gh, nil)+ require.NoError(t, err)++ q := gh.pulls.requests[0]+ assert.Equal(t, "all", q.Get("state"))+ assert.Equal(t, "updated", q.Get("sort"))+ assert.Equal(t, "asc", q.Get("direction"))+}++func TestCommentScan_IssueCommentsContinuePastPageCap(t *testing.T) {+ gh := newFakeGitHub(t)+ for i := range 250 {+ gh.issueComments.records = append(gh.issueComments.records,+ commentRecord(i+1, testEpoch.Add(time.Duration(i)*time.Minute), fmt.Sprintf("https://github.com/o/r/issues/1#issuecomment-%d", i+1)))+ }+ gh.issueComments.maxPages = 1+ s := &Source{includeIssueComments: true}++ reporter, err := scanComments(t, s, gh, nil)++ require.NoError(t, err)+ require.Len(t, reporter.links, 250)+ seen := map[string]bool{}+ for _, link := range reporter.links {+ assert.False(t, seen[link], "comment emitted twice: %s", link)+ seen[link] = true+ }+}++func TestCommentScan_PullsCappedFallsBackToIssueListing(t *testing.T) {+ gh := newFakeGitHub(t)+ const prs = 150+ for n := 1; n <= prs; n++ {+ updated := testEpoch.Add(time.Duration(n) * time.Minute)+ gh.pulls.records = append(gh.pulls.records, prRecord(n, updated))+ gh.issues.records = append(gh.issues.records, prIssueRecord(n, updated))+ if n%10 == 0 {+ gh.issues.records = append(gh.issues.records, issueRecord(1000+n, updated))+ }+ }+ gh.pulls.maxPages = 1+ s := &Source{includePRComments: true}++ reporter, err := scanComments(t, s, gh, nil)++ require.NoError(t, err)+ want := make([]string, 0, prs)+ for n := 1; n <= prs; n++ {+ want = append(want, prLink(n))+ }+ assert.ElementsMatch(t, want, reporter.links, "every PR body exactly once, and no issue bodies")+}++func TestCommentScan_FailedPhaseDoesNotStopTheOthers(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ gh.issueComments.fail = func(_ int, w http.ResponseWriter, _ *http.Request) bool {+ writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Server Error"})+ return true+ }+ var retries int+ s := &Source{+ includeIssueComments: true,+ includePRComments: true,+ commentRetryWait: func(context.Context, time.Duration) bool { retries++; return true },+ }+ cutoff := testEpoch.Add(-time.Hour)++ reporter, err := scanComments(t, s, gh, &cutoff)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "issue comments: walk incomplete after 0 items (since=2023-12-31T23:00:00Z): giving up after 6 attempts")+ assert.Contains(t, err.Error(), "Server Error")+ assert.Equal(t, maxTransientRetries, retries)+ assert.NotContains(t, err.Error(), "pull request comments:")+ var walkErr *walkError+ assert.ErrorAs(t, err, &walkErr)+ assert.ElementsMatch(t, []string{+ "https://github.com/o/r/issues/1",+ prLink(2),+ "https://github.com/o/r/pull/2#discussion_r200",+ }, reporter.links)+}++func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {+ gh := newFakeGitHub(t)+ seedRepo(gh)+ gh.issueComments.fail = failRequest(http.StatusBadGateway, 1)+ gh.prComments.fail = failRequest(http.StatusBadGateway, 1)+ s := &Source{+ includeIssueComments: true,+ includePRComments: true,+ commentRetryWait: func(context.Context, time.Duration) bool { return true },+ }+ var logs bytes.Buffer+ logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(2)))+ ctx := context.WithLogger(context.Background(), logger)++ err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, &collectingReporter{}, nil)+ require.NoError(t, err)++ var listings []string+ for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") {+ var entry map[string]any+ require.NoError(t, json.Unmarshal([]byte(line), &entry))+ if entry["msg"] == "retrying GitHub API request after transient error" {+ listings = append(listings, fmt.Sprint(entry["listing"]))+ }+ }+ assert.Equal(t, []string{"issue comments", "pull request comments"}, listings)+}++func TestProcessGistComments_CappedListingIsAnError(t *testing.T) {+ gh := newFakeGitHub(t)+ for i := range 150 {+ gh.gistComments.records = append(gh.gistComments.records, commentRecord(i+1, testEpoch, ""))+ }+ gh.gistComments.maxPages = 1+ s := &Source{includeGistComments: true}+ reporter := &collectingReporter{}++ err := s.processGistComments(context.Background(), gh.client, "https://gist.github.com/u/g1",+ []string{"gist.github.com", "u", "g1"}, repoInfo{}, reporter, nil)++ require.Error(t, err)+ assert.Contains(t, err.Error(), "gist comments: walk incomplete after 100 items")+ assert.Contains(t, err.Error(), "no since filter")+ assert.Len(t, reporter.links, 100, "comments read before the cap are kept")+}++func TestProcessGistComments_SkipsOldCommentsAndKeepsGoing(t *testing.T) {+ gh := newFakeGitHub(t)+ for i := range 5 {+ created := testEpoch.Add(time.Duration(i) * time.Hour)+ gh.gistComments.records = append(gh.gistComments.records, fakeRecord{updated: created, body: map[string]any{+ "id": i + 1, "created_at": created, "url": fmt.Sprintf("https://api.github.com/gists/g1/comments/%d", i+1),+ }})+ }+ s := &Source{includeGistComments: true}+ reporter := &collectingReporter{}+ cutoff := testEpoch.Add(150 * time.Minute)++ err := s.processGistComments(context.Background(), gh.client, "https://gist.github.com/u/g1",+ []string{"gist.github.com", "u", "g1"}, repoInfo{}, reporter, &cutoff)++ require.NoError(t, err)+ assert.Equal(t, []string{+ "https://api.github.com/gists/g1/comments/4",+ "https://api.github.com/gists/g1/comments/5",+ }, reporter.links)+}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 1ccc72714..dbe3a0fd0 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -80,6 +80,10 @@ type Source struct {ignoreGists boolincludeGistComments boolcommentsTimeframeDays uint32+ // commentRetryWait sleeps between retries of transient comment API+ // failures and returns false if the context ended first. Nil uses+ // sleepWithContext; tests inject a wait that returns immediately.+ commentRetryWait func(context.Context, time.Duration) boolsources.Progress@@ -1358,7 +1362,8 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.// This should never happen.return fmt.Errorf("no repoInfo for URL: %s", repoURL)}- duration, err := s.cloneAndScanRepo(ctx, repoURL, repoInfo, reporter)+ start := time.Now()+ gitDuration, err := s.cloneAndScanRepo(ctx, repoURL, repoInfo, reporter)if err != nil {return err}@@ -1385,8 +1390,12 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.}// Scan comments, if enabled.+ var commentsDuration time.Durationif s.includeGistComments || s.includeIssueComments || s.includePRComments {- if err := s.scanComments(ctx, repoURL, repoInfo, reporter); err != nil {+ commentsStart := time.Now()+ err := s.scanComments(ctx, repoURL, repoInfo, reporter)+ commentsDuration = time.Since(commentsStart)+ if err != nil {err := fmt.Errorf("error scanning comments: %w", err)if err := reporter.ChunkErr(ctx, err); err != nil {return err@@ -1394,7 +1403,13 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.}}- ctx.Logger().V(2).Info("finished scanning repo", "duration_seconds", duration)+ // duration_seconds covers everything above: clone, git history, wiki, and+ // comments. Git history and comments are broken out because on repos with+ // many comments, the comment walks can take far longer than the git scan.+ ctx.Logger().V(2).Info("finished scanning repo",+ "duration_seconds", time.Since(start).Seconds(),+ "git_scan_seconds", gitDuration.Seconds(),+ "comments_seconds", commentsDuration.Seconds())githubReposScanned.WithLabelValues(s.name).Inc()return nil}@@ -1862,35 +1877,54 @@ func getRepoURLParts(repoURLString string) (string, []string, error) {return urlString, urlParts, nil}-const initialPage = 1 // page to start listing from+// commentPager returns the pager every comment walk for one repo shares. Rate+// limits are handled by waiting and retrying the same request, and each wait+// is also reported to the chunk reporter. Transient failures are retried a+// bounded number of times; see fetchWithRetry.+func (s *Source) commentPager(ctx context.Context, apiClient *github.Client, reporter sources.ChunkReporter) pager {+ wait := s.commentRetryWait+ if wait == nil {+ wait = func(ctx context.Context, d time.Duration) bool {+ _, canceled := sleepWithContext(ctx, d)+ return !canceled+ }+ }+ return pager{+ client: apiClient,+ perPage: defaultPagination,+ retry: func(err error) bool {+ return s.handleRateLimitWithChunkReporter(ctx, reporter, err)+ },+ wait: wait,+ }+}+// processGistComments scans a gist's comments. The endpoint has no `since` or+// sort parameters, so the comments timeframe is applied to each page as it+// arrives, and a capped walk has no way to continue: it is reported as an+// error rather than ending silently.func (s *Source) processGistComments(ctx context.Context, apiClient *github.Client, gistURL string, urlParts []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {ctx.Logger().V(2).Info("Scanning GitHub Gist comments")- // GitHub Gist URL.gistID := extractGistID(urlParts)-- options := &github.ListOptions{- PerPage: defaultPagination,- Page: initialPage,+ p := s.commentPager(ctx, apiClient, reporter)+ p.listing = "gist comments"+ outcome, err := walkPages(ctx, p,+ func(ctx context.Context) ([]*github.GistComment, *github.Response, error) {+ return apiClient.Gists.ListComments(ctx, gistID, &github.ListOptions{PerPage: defaultPagination})+ },+ func(comments []*github.GistComment) error {+ return s.chunkGistComments(ctx, gistURL, repoInfo, comments, reporter, cutoffTime)+ },+ )+ if err != nil {+ return fmt.Errorf("gist comments: %w", &walkError{items: outcome.items, err: err})}- for {- comments, _, err := apiClient.Gists.ListComments(ctx, gistID, options)- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {- continue- }- if err != nil {- return err- }-- if err = s.chunkGistComments(ctx, gistURL, repoInfo, comments, reporter, cutoffTime); err != nil {- return err- }-- options.Page++- if len(comments) < options.PerPage {- break- }+ if outcome.capped {+ return fmt.Errorf("gist comments: %w", &walkError{+ items: outcome.items,+ err: errors.New("GitHub stopped paginating and this endpoint has no since filter to continue from"),+ })}return nil}@@ -1923,9 +1957,10 @@ func isGistUrl(urlParts []string) bool {func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo repoInfo, comments []*github.GistComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {for _, comment := range comments {- // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted+ // Gist comments come back oldest first, so an old comment says nothing+ // about the ones after it. Skip it and keep going.if cutoffTime != nil && comment.GetCreatedAt().Before(*cutoffTime) {- break+ continue}// Create chunk and send it to the channel.@@ -1957,46 +1992,100 @@ func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInforeturn nil}-// Note: these can't be consts because the address is needed when using with the GitHub library.-var (- // sortType defines the criteria for sorting comments.- // By setting this to "updated" we can use this to reliably manage the comment timeframe filtering below- sortType = "updated"- // directionType defines the direction of sorting.- // "desc" means comments will be sorted in descending order, showing the latest comments first, which is critical for managing the comment timeframe filtering- directionType = "desc"- // allComments is a placeholder for specifying the comment ID to start listing from.- // A value of 0 means that all comments will be listed.- allComments = 0- // state of "all" for the ListByRepo captures both open and closed issues.- state = "all"+// Repo listings sort by update time, oldest first, so a capped walk can+// continue from the last update time it saw. See walkUpdatedSince.+const (+ sortByUpdated = "updated"+ sortAscending = "asc"+ // stateAll lists open and closed issues and pull requests.+ stateAll = "all")+// commentPhase is one listing in a repo's comment scan. run returns how many+// chunks it emitted. startLog, when set, is logged at V(2) before the phase+// runs.+type commentPhase struct {+ name string+ startLog string+ run func() (int, error)+}++// processIssueandPRsWithCommentsREST scans a repo's issues, pull requests, and+// their comments through the REST API.+//+// The phases are independent: a failure in one is collected and the rest+// still run, so one capped or forbidden endpoint can't hide the others. All+// failures come back joined into one error, which scanRepo reports as a+// single ChunkErr for the repo. Chunks emitted before a failure are kept.+//+// The "Scanning issues" and "Scanning pull requests" V(2) lines are the only+// log signal for scans on the unit path, whose comment errors go to the scan+// details UI instead of the logs, so they keep their wording. The summary+// line at the end records how far each phase got.func (s *Source) processIssueandPRsWithCommentsREST(ctx context.Context, apiClient *github.Client, repoInfo repoInfo,reporter sources.ChunkReporter, cutoffTime *time.Time,) error {- if s.includeIssueComments {- ctx.Logger().V(2).Info("Scanning issues")- if err := s.processIssues(ctx, apiClient, repoInfo, reporter); err != nil {- return err- }- if err := s.processIssueComments(ctx, apiClient, repoInfo, reporter, cutoffTime); err != nil {- return err- }+ p := s.commentPager(ctx, apiClient, reporter)+ var since time.Time+ if cutoffTime != nil {+ since = *cutoffTime}+ // /issues returns issues and pull requests together. When issue comments+ // are enabled, that one walk also emits PR bodies, so /pulls is only+ // walked for configs that enable PR comments alone.+ var phases []commentPhase+ if s.includeIssueComments {+ bodies := "issues"+ if s.includePRComments {+ bodies = "issues and pull requests"+ }+ phases = append(phases,+ commentPhase{name: bodies, startLog: "Scanning issues", run: func() (int, error) {+ return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)+ }},+ commentPhase{name: "issue comments", run: func() (int, error) {+ return s.processIssueComments(ctx, p, repoInfo, reporter, since)+ }},+ )+ }if s.includePRComments {- ctx.Logger().V(2).Info("Scanning pull requests")- if err := s.processPRs(ctx, apiClient, repoInfo, reporter); err != nil {- return err+ prStartLog := "Scanning pull requests"+ if !s.includeIssueComments {+ phases = append(phases, commentPhase{name: "pull requests", startLog: prStartLog, run: func() (int, error) {+ return s.processPRs(ctx, p, repoInfo, reporter)+ }})+ prStartLog = ""+ }+ phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, run: func() (int, error) {+ return s.processPRComments(ctx, p, repoInfo, reporter, since)+ }})+ }++ var (+ errs []error+ failed []string+ summary []any+ )+ for _, phase := range phases {+ if err := ctx.Err(); err != nil {+ errs = append(errs, err)+ break}- if err := s.processPRComments(ctx, apiClient, repoInfo, reporter, cutoffTime); err != nil {- return err+ if phase.startLog != "" {+ ctx.Logger().V(2).Info(phase.startLog)+ }+ emitted, err := phase.run()+ summary = append(summary, strings.ReplaceAll(phase.name, " ", "_"), emitted)+ if err != nil {+ failed = append(failed, phase.name)+ errs = append(errs, fmt.Errorf("%s: %w", phase.name, err))}}+ ctx.Logger().V(2).Info("finished scanning comments", append(summary, "failed_phases", failed)...)- return nil+ return errors.Join(errs...)}func (s *Source) processRepoIssueandPRsWithCommentsGraphql(@@ -2024,44 +2113,52 @@ func (s *Source) processRepoIssueandPRsWithCommentsGraphql(return nil}-func (s *Source) processIssues(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter) error {- bodyTextsOpts := &github.IssueListByRepoOptions{- Sort: sortType,- Direction: directionType,- State: state,- ListOptions: github.ListOptions{- PerPage: defaultPagination,- Page: initialPage,+// processIssues walks /issues and emits issue bodies, plus pull request+// bodies when includePRs is set. The listing always contains both, because+// GitHub treats every pull request as an issue. Bodies are scanned in full+// regardless of the comments timeframe.+func (s *Source) processIssues(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, includePRs bool) (int, error) {+ emitted := 0+ _, err := walkUpdatedSince(ctx, p, sinceWalk[*github.Issue]{+ name: "issues",+ list: listIssuesSince(p, repoInfo),+ key: issueKey,+ visit: func(issues []*github.Issue) error {+ n, err := s.chunkIssues(ctx, repoInfo, issues, reporter, func(issue *github.Issue) bool {+ return includePRs || !issue.IsPullRequest()+ })+ emitted += n+ return err},+ })+ return emitted, err+}++// listIssuesSince returns the /issues listing, issues and pull requests+// together, for walkUpdatedSince.+func listIssuesSince(p pager, repoInfo repoInfo) func(context.Context, time.Time) ([]*github.Issue, *github.Response, error) {+ return func(ctx context.Context, since time.Time) ([]*github.Issue, *github.Response, error) {+ return p.client.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, &github.IssueListByRepoOptions{+ State: stateAll,+ Sort: sortByUpdated,+ Direction: sortAscending,+ Since: since,+ ListOptions: github.ListOptions{PerPage: p.perPage},+ })}+}- for {- issues, _, err := apiClient.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts)- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {- continue- }-- if err != nil {- return err- }-- if err = s.chunkIssues(ctx, repoInfo, issues, reporter); err != nil {- return err- }-- bodyTextsOpts.Page++-- if len(issues) < defaultPagination {- break- }- }- return nil+func issueKey(issue *github.Issue) (int64, time.Time) {+ return issue.GetID(), issue.GetUpdatedAt().Time}-func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, reporter sources.ChunkReporter) error {+// chunkIssues emits a chunk for each issue that keep accepts, and returns how+// many it emitted. Pull requests from /issues produce the same chunk as+// chunkPullRequests: same link, author, timestamp, and title and body.+func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, reporter sources.ChunkReporter, keep func(*github.Issue) bool) (int, error) {+ emitted := 0for _, issue := range issues {- // Skip pull requests since covered by processPRs.- if issue.IsPullRequest() {+ if !keep(issue) {continue}@@ -2088,50 +2185,46 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g}if err := reporter.ChunkOk(ctx, chunk); err != nil {- return err- }- }- return nil-}--func (s *Source) processIssueComments(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {- issueOpts := &github.IssueListCommentsOptions{- Sort: &sortType,- Direction: &directionType,- ListOptions: github.ListOptions{- PerPage: defaultPagination,- Page: initialPage,+ return emitted, err+ }+ emitted+++ }+ return emitted, nil+}++// processIssueComments walks every issue comment in the repo updated at or+// after since. /issues/comments also returns comments on pull requests, since+// GitHub treats every pull request as an issue. The endpoint stops at 300+// pages, so the walk continues past that cap in since windows.+func (s *Source) processIssueComments(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, since time.Time) (int, error) {+ return walkUpdatedSince(ctx, p, sinceWalk[*github.IssueComment]{+ name: "issue comments",+ since: since,+ list: func(ctx context.Context, since time.Time) ([]*github.IssueComment, *github.Response, error) {+ opts := &github.IssueListCommentsOptions{+ Sort: github.String(sortByUpdated),+ Direction: github.String(sortAscending),+ ListOptions: github.ListOptions{PerPage: p.perPage},+ }+ if !since.IsZero() {+ opts.Since = &since+ }+ // Issue number 0 lists comments across the whole repo.+ return p.client.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, 0, opts)},- }-- for {- issueComments, _, err := apiClient.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts)- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {- continue- }- if err != nil {- return err- }-- if err = s.chunkIssueComments(ctx, repoInfo, issueComments, reporter, cutoffTime); err != nil {- return err- }-- issueOpts.Page++- if len(issueComments) < defaultPagination {- break- }- }- return nil+ key: func(comment *github.IssueComment) (int64, time.Time) {+ return comment.GetID(), comment.GetUpdatedAt().Time+ },+ visit: func(comments []*github.IssueComment) error {+ return s.chunkIssueComments(ctx, repoInfo, comments, reporter)+ },+ })}-func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+// chunkIssueComments emits a chunk for each comment. The comments timeframe+// is already applied by the API through `since`.+func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, reporter sources.ChunkReporter) error {for _, comment := range comments {- // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted- if cutoffTime != nil && comment.GetUpdatedAt().Before(*cutoffTime) {- continue- }-// Create chunk and send it to the channel.chunk := sources.Chunk{SourceName: s.name,@@ -2161,69 +2254,87 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, commreturn nil}-func (s *Source) processPRs(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter) error {- prOpts := &github.PullRequestListOptions{- Sort: sortType,- Direction: directionType,- State: state,- ListOptions: github.ListOptions{- PerPage: defaultPagination,- Page: initialPage,+// processPRs emits pull request bodies for configs that enable PR comments+// but not issue comments; otherwise the /issues walk emits them. /pulls has+// no `since` filter, so if GitHub caps it, the walk falls back to a windowed+// /issues walk that emits only pull requests, skipping the ones /pulls+// already emitted. The common case never pages through issues.+func (s *Source) processPRs(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter) (int, error) {+ emitted := 0+ emittedNumbers := map[int]struct{}{}+ p.listing = "pull requests"+ outcome, err := walkPages(ctx, p,+ func(ctx context.Context) ([]*github.PullRequest, *github.Response, error) {+ return p.client.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, &github.PullRequestListOptions{+ State: stateAll,+ Sort: sortByUpdated,+ Direction: sortAscending,+ ListOptions: github.ListOptions{PerPage: p.perPage},+ })},- }-- for {- prs, _, err := apiClient.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts)- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {- continue- }- if err != nil {- return err- }-- if err = s.chunkPullRequests(ctx, repoInfo, prs, reporter); err != nil {- return err- }-- prOpts.Page++-- if len(prs) < defaultPagination {- break- }- }- return nil-}--func (s *Source) processPRComments(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {- prOpts := &github.PullRequestListCommentsOptions{- Sort: sortType,- Direction: directionType,- ListOptions: github.ListOptions{- PerPage: defaultPagination,- Page: initialPage,+ func(prs []*github.PullRequest) error {+ if err := s.chunkPullRequests(ctx, repoInfo, prs, reporter); err != nil {+ return err+ }+ for _, pr := range prs {+ emittedNumbers[pr.GetNumber()] = struct{}{}+ }+ emitted += len(prs)+ return nil},+ )+ if err != nil {+ return emitted, &walkError{items: outcome.items, err: err}+ }+ if !outcome.capped {+ return emitted, nil}- for {- prComments, _, err := apiClient.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts)- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {- continue- }- if err != nil {- return err- }-- if err = s.chunkPullRequestComments(ctx, repoInfo, prComments, reporter, cutoffTime); err != nil {+ ctx.Logger().V(2).Info("pull request listing capped, continuing from the issue listing", "pull_requests_so_far", emitted)+ _, err = walkUpdatedSince(ctx, p, sinceWalk[*github.Issue]{+ name: "pull requests from issues",+ list: listIssuesSince(p, repoInfo),+ key: issueKey,+ visit: func(issues []*github.Issue) error {+ n, err := s.chunkIssues(ctx, repoInfo, issues, reporter, func(issue *github.Issue) bool {+ if !issue.IsPullRequest() {+ return false+ }+ _, done := emittedNumbers[issue.GetNumber()]+ return !done+ })+ emitted += nreturn err- }-- prOpts.Page++-- if len(prComments) < defaultPagination {- break- }- }- return nil+ },+ })+ return emitted, err+}++// processPRComments walks every pull request review comment (the inline+// comments on diffs) in the repo updated at or after since. Discussion+// comments on pull requests come from /issues/comments instead. No cap has+// been observed on this endpoint, but it gets the same since windows as+// /issues/comments in case one exists.+func (s *Source) processPRComments(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, since time.Time) (int, error) {+ return walkUpdatedSince(ctx, p, sinceWalk[*github.PullRequestComment]{+ name: "pull request comments",+ since: since,+ list: func(ctx context.Context, since time.Time) ([]*github.PullRequestComment, *github.Response, error) {+ // Pull request number 0 lists review comments across the whole repo.+ return p.client.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, 0, &github.PullRequestListCommentsOptions{+ Sort: sortByUpdated,+ Direction: sortAscending,+ Since: since,+ ListOptions: github.ListOptions{PerPage: p.perPage},+ })+ },+ key: func(comment *github.PullRequestComment) (int64, time.Time) {+ return comment.GetID(), comment.GetUpdatedAt().Time+ },+ visit: func(comments []*github.PullRequestComment) error {+ return s.chunkPullRequestComments(ctx, repoInfo, comments, reporter)+ },+ })}func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, reporter sources.ChunkReporter) error {@@ -2257,13 +2368,10 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [return nil}-func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+// chunkPullRequestComments emits a chunk for each review comment. The+// comments timeframe is already applied by the API through `since`.+func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, reporter sources.ChunkReporter) error {for _, comment := range comments {- // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted- if cutoffTime != nil && comment.GetUpdatedAt().Before(*cutoffTime) {- continue- }-// Create chunk and send it to the channel.chunk := sources.Chunk{SourceName: s.name,diff --git a/pkg/sources/github/pagination.go b/pkg/sources/github/pagination.gonew file mode 100644index 000000000..d030e4b84--- /dev/null+++ b/pkg/sources/github/pagination.go@@ -0,0 +1,453 @@+package github++import (+ "errors"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/url"+ "strconv"+ "strings"+ "syscall"+ "time"++ "github.com/google/go-github/v67/github"+ regexp "github.com/wasilibs/go-re2"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++// GitHub list endpoints don't all paginate the same way, and the differences+// are mostly undocumented:+//+// - /issues offers cursors (`after`) in its next links and returns a 422+// ("Pagination with the page parameter is not supported") somewhere+// between 1,000 and 10,000 items when driven by page number alone.+// - /issues/comments offers only page numbers and stops at 300 pages: page+// 300 comes back full with no next link, and page 301 returns a 422+// ("pagination is limited for this resource").+// - /pulls ignores `after` and copies it into its next link, so a client+// that sends only a cursor fetches page 1 forever.+//+// The helpers in this file never build page or cursor parameters themselves.+// They follow the Link header's rel="next" URL as-is, which uses cursors+// wherever GitHub provides them and page numbers everywhere else, and they+// treat the ways a walk can end as distinct outcomes rather than assuming a+// short page means "done". Every walk either reaches a confirmed end or+// reports why it couldn't, so a cap can never silently truncate a scan.++// pager carries what every walk needs: the client to send follow-up requests+// with, the page size the walk was started with (used to recognize a full+// final page), the caller's rate-limit handler, which sleeps and returns true+// when a request should be retried, and wait, which sleeps between retries of+// transient failures and returns false if the context ended first. A nil wait+// turns transient retries off. listing names the walk in log lines, for+// example "issue comments"; pager is passed by value, so each walk sets its+// own without affecting the caller's.+type pager struct {+ client *github.Client+ perPage int+ retry func(error) bool+ wait func(context.Context, time.Duration) bool+ listing string+}++// walkOutcome describes how a page walk ended. A walk that isn't capped+// reached a confirmed end: a short final page, or an empty probe page.+type walkOutcome struct {+ capped bool+ items int+}++// walkError reports a walk that could not finish, with enough context for a+// support engineer to see how far it got. since is zero for walks that don't+// use since windows.+type walkError struct {+ items int+ since time.Time+ err error+}++func (e *walkError) Error() string {+ if e.since.IsZero() {+ return fmt.Sprintf("walk incomplete after %d items: %v", e.items, e.err)+ }+ return fmt.Sprintf("walk incomplete after %d items (since=%s): %v", e.items, e.since.UTC().Format(time.RFC3339), e.err)+}++func (e *walkError) Unwrap() error { return e.err }++// walkPages fetches the first page with first (a typed go-github call that+// sets the filters and page size), then follows rel="next" links until the+// walk ends. visit receives each page before the next one is requested, so a+// later failure can never discard pages that were already read.+//+// When there is no next link, the walk ends one of three ways:+// - The final page is short: the walk is complete.+// - The final page is full and its URL carries a cursor (`after` or+// `before`): the walk is capped. GitHub goes by the cursor, so asking for+// the next page number would just return the same page again.+// - The final page is full and its URL uses page numbers (or none, for the+// first page): the next page number is probed. An empty page means the+// results exactly filled the last page, data means the walk continues,+// and a pagination 422 means the endpoint is capped.+//+// A pagination 422 at any point is reported as capped rather than as an+// error so callers can decide whether they have a way past it.+func walkPages[T any](+ ctx context.Context,+ p pager,+ first func(context.Context) ([]T, *github.Response, error),+ visit func([]T) error,+) (walkOutcome, error) {+ var out walkOutcome++ items, resp, err := fetchWithRetry(ctx, p, func() ([]T, *github.Response, error) { return first(ctx) })+ if err != nil {+ if isPaginationCap(err) {+ out.capped = true+ return out, nil+ }+ return out, err+ }++ current := requestURL(resp)+ fetched := map[string]struct{}{}+ if current != nil {+ fetched[current.String()] = struct{}{}+ }++ for {+ if len(items) > 0 {+ if err := visit(items); err != nil {+ return out, err+ }+ out.items += len(items)+ }++ next := nextLink(resp)+ if next == "" {+ if len(items) < p.perPage {+ return out, nil+ }+ if current == nil || hasCursor(current) {+ out.capped = true+ return out, nil+ }+ next = withNextPage(current)+ }++ nextURL, err := p.followable(next)+ if err != nil {+ return out, err+ }+ if _, ok := fetched[nextURL.String()]; ok {+ return out, fmt.Errorf("pagination did not advance: next link repeats %s", redactQuery(nextURL))+ }+ fetched[nextURL.String()] = struct{}{}++ items, resp, err = fetchWithRetry(ctx, p, func() ([]T, *github.Response, error) {+ return fetchPage[T](ctx, p.client, nextURL.String())+ })+ if err != nil {+ if isPaginationCap(err) {+ out.capped = true+ return out, nil+ }+ return out, err+ }+ current = nextURL+ }+}++// sinceWalk describes a walk over an endpoint that accepts `since` and can+// sort by update time: /issues, /issues/comments, and /pulls/comments.+type sinceWalk[T any] struct {+ // name labels log lines, for example "issue comments", and becomes the+ // pager's listing for the walk.+ name string+ // since is the earliest update time to fetch. Zero fetches everything.+ since time.Time+ // list fetches the first page of a window starting at since. It must sort+ // by update time ascending, and must omit `since` when it is zero.+ list func(ctx context.Context, since time.Time) ([]T, *github.Response, error)+ // key returns an item's ID and last update time.+ key func(T) (int64, time.Time)+ // visit receives each page of items not already seen.+ visit func([]T) error+}++// walkUpdatedSince walks an endpoint in update-time order, and gets past+// GitHub's per-query result caps by starting a new window at the last update+// time seen whenever a window ends capped. `since` is inclusive and each+// window gets a fresh cap, so the walk continues where the previous window+// stopped. Items at the boundary timestamp that were already visited are+// dropped by ID.+//+// Ascending order also keeps the walk complete while the repo changes under+// it: an item updated mid-walk moves to the end, where the walk still reaches+// it, instead of shifting earlier pages.+//+// A window that ends complete finishes the walk, even if it produced nothing+// new; that is what an exact fit on a cursor endpoint looks like. A window+// that ends capped without producing anything new is an error, because more+// items share one timestamp than a window can hold and no later window can+// get past them.+func walkUpdatedSince[T any](ctx context.Context, p pager, w sinceWalk[T]) (int, error) {+ p.listing = w.name+ windowStart := w.since+ var (+ total int+ // latest is the newest update time visited, and atLatest holds the IDs+ // visited at exactly that time. When a window starts at latest, the+ // API returns those items again, and atLatest is how they are dropped.+ latest time.Time+ atLatest = map[int64]struct{}{}+ )++ for {+ newInWindow := 0+ outcome, err := walkPages(ctx, p,+ func(ctx context.Context) ([]T, *github.Response, error) { return w.list(ctx, windowStart) },+ func(items []T) error {+ fresh := make([]T, 0, len(items))+ for _, item := range items {+ id, updated := w.key(item)+ if updated.Equal(windowStart) {+ if _, seen := atLatest[id]; seen {+ continue+ }+ }+ fresh = append(fresh, item)+ switch {+ case updated.After(latest):+ latest = updated+ atLatest = map[int64]struct{}{id: {}}+ case updated.Equal(latest):+ atLatest[id] = struct{}{}+ }+ }+ if len(fresh) == 0 {+ return nil+ }+ newInWindow += len(fresh)+ total += len(fresh)+ return w.visit(fresh)+ },+ )+ if err != nil {+ return total, &walkError{items: total, since: windowStart, err: err}+ }+ if !outcome.capped {+ return total, nil+ }+ if newInWindow == 0 {+ return total, &walkError{+ items: total,+ since: windowStart,+ err: fmt.Errorf("listing is capped and a new window made no progress: more items share update time %s than one window can return", windowStart.UTC().Format(time.RFC3339)),+ }+ }++ windowStart = latest+ ctx.Logger().V(2).Info("listing capped, continuing from last update time",+ "listing", w.name, "since", windowStart.UTC().Format(time.RFC3339), "items_so_far", total)+ }+}++// maxTransientRetries bounds how many times one page request is retried after+// a transient failure. With the backoff below, a request gets about a minute+// of retries before the walk reports it as an error.+const maxTransientRetries = 5++// transientBackoff returns how long to wait before the given retry, starting+// at 2 seconds, doubling, and capped at 30 seconds.+func transientBackoff(retry int) time.Duration {+ return min(2*time.Second<<retry, 30*time.Second)+}++// fetchWithRetry runs fetch, retrying in two independent cases. Rate limits+// are retried for as long as the rate-limit handler says to; it sleeps until+// the limit resets and returns false once the context is cancelled. Transient+// failures (see isTransient) are retried up to maxTransientRetries times with+// backoff. Every request here is a GET, so a retry can't repeat a side effect.+//+// Long walks make transient failures likely: a full walk of a large repo's+// review comments takes hours at several seconds per page, and GitHub+// occasionally cancels a slow request outright.+//+// The connectors' HTTP clients (common.RetryableHTTPClientTimeout) already+// retry a failed round trip a few times with short backoff, so this is a+// second layer, not a duplicate. It covers what that layer can't: a response+// whose body fails partway through reading (an HTTP/2 stream reset arrives+// here, from the JSON decoder, after the round trip has succeeded), and a+// failure that outlasts the client's own retries, which surfaces as+// "giving up after N attempt(s)". Each attempt here is bounded by the+// client's 60 second timeout, so a page that keeps failing is reported after+// several minutes rather than retried indefinitely.+func fetchWithRetry[T any](ctx context.Context, p pager, fetch func() ([]T, *github.Response, error)) ([]T, *github.Response, error) {+ transientRetries := 0+ for {+ items, resp, err := fetch()+ if p.retry != nil && p.retry(err) {+ continue+ }+ if err == nil || p.wait == nil || !isTransient(ctx, err) {+ return items, resp, err+ }+ if transientRetries == maxTransientRetries {+ return items, resp, fmt.Errorf("giving up after %d attempts: %w", transientRetries+1, err)+ }+ delay := transientBackoff(transientRetries)+ transientRetries+++ ctx.Logger().V(2).Info("retrying GitHub API request after transient error",+ "listing", p.listing, "retry", transientRetries, "max_retries", maxTransientRetries, "retry_in", delay.String(), "error", err.Error())+ if !p.wait(ctx, delay) {+ return items, resp, err+ }+ }+}++// isTransient reports whether a failed request is worth retrying as-is:+// a 5xx from GitHub, or a transport failure such as a reset connection, a+// timeout, or a response cut off partway. Anything caused by the context+// ending is not transient, and neither is any other API error; rate limits+// have their own handler, and a 4xx (including the pagination 422s) will fail+// the same way again.+func isTransient(ctx context.Context, err error) bool {+ if ctx.Err() != nil {+ return false+ }++ var (+ errResp *github.ErrorResponse+ rateLimit *github.RateLimitError+ abuseLimit *github.AbuseRateLimitError+ )+ switch {+ case errors.As(err, &rateLimit), errors.As(err, &abuseLimit):+ return false+ case errors.As(err, &errResp):+ return errResp.Response != nil && errResp.Response.StatusCode >= http.StatusInternalServerError+ }++ var (+ urlErr *url.Error+ netErr net.Error+ )+ if errors.As(err, &urlErr) || errors.As(err, &netErr) ||+ errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) {+ return true+ }+ // A stream reset that arrives while the body is being read surfaces from+ // the JSON decoder as net/http's bundled HTTP/2 stream error. That type+ // is unexported, so its message is the only way to recognize it.+ return strings.Contains(err.Error(), "stream error:")+}++// fetchPage sends a GET for an absolute next-page URL through the go-github+// client, so authentication, rate-limit errors, and error decoding behave+// exactly as they do for the typed list methods.+func fetchPage[T any](ctx context.Context, client *github.Client, rawURL string) ([]T, *github.Response, error) {+ req, err := client.NewRequest(http.MethodGet, rawURL, nil)+ if err != nil {+ return nil, nil, err+ }+ var page []T+ resp, err := client.Do(ctx, req, &page)+ return page, resp, err+}++// followable parses a next-page URL and refuses one that points at a host+// other than the API's. The connectors attach credentials in the HTTP+// transport, which would send the token to whatever host the URL names.+func (p pager) followable(raw string) (*url.URL, error) {+ u, err := url.Parse(raw)+ if err != nil {+ return nil, fmt.Errorf("invalid pagination link: %w", err)+ }+ if !u.IsAbs() {+ u = p.client.BaseURL.ResolveReference(u)+ }+ if !strings.EqualFold(u.Host, p.client.BaseURL.Host) {+ return nil, fmt.Errorf("pagination link points at %q, not the API host %q", u.Host, p.client.BaseURL.Host)+ }+ return u, nil+}++// isPaginationCap reports whether err is one of GitHub's pagination 422s:+// the /issues offset limit ("Pagination with the page parameter is not+// supported for large datasets") or the /issues/comments page cap+// ("pagination is limited for this resource"). Any other 422 is a real+// error.+func isPaginationCap(err error) bool {+ var errResp *github.ErrorResponse+ if !errors.As(err, &errResp) || errResp.Response == nil {+ return false+ }+ return errResp.Response.StatusCode == http.StatusUnprocessableEntity &&+ strings.Contains(strings.ToLower(errResp.Message), "pagination")+}++// linkPattern matches one `<url>; rel="..."` entry of a Link header. Matching+// the angle brackets, instead of splitting on commas, keeps URLs that+// contain commas intact.+var linkPattern = regexp.MustCompile(`<([^>]*)>\s*;\s*rel="([^"]*)"`)++// nextLink returns the rel="next" URL from a response's Link headers, or ""+// when there is none.+func nextLink(resp *github.Response) string {+ if resp == nil || resp.Response == nil {+ return ""+ }+ for _, header := range resp.Header.Values("Link") {+ for _, match := range linkPattern.FindAllStringSubmatch(header, -1) {+ for _, rel := range strings.Fields(match[2]) {+ if rel == "next" {+ return match[1]+ }+ }+ }+ }+ return ""+}++// requestURL returns the URL a response was fetched from, or nil when the+// response doesn't record it.+func requestURL(resp *github.Response) *url.URL {+ if resp == nil || resp.Response == nil || resp.Request == nil || resp.Request.URL == nil {+ return nil+ }+ u := *resp.Request.URL+ return &u+}++// hasCursor reports whether a page URL is positioned by cursor. GitHub's+// /issues links carry both `page` and `after`, and the cursor is what counts.+func hasCursor(u *url.URL) bool {+ q := u.Query()+ return q.Get("after") != "" || q.Get("before") != ""+}++// withNextPage returns u with its page number incremented. A URL without a+// page number is page 1.+func withNextPage(u *url.URL) string {+ q := u.Query()+ page, err := strconv.Atoi(q.Get("page"))+ if err != nil || page < 1 {+ page = 1+ }+ q.Set("page", strconv.Itoa(page+1))+ next := *u+ next.RawQuery = q.Encode()+ return next.String()+}++// redactQuery drops the query string from a URL for error messages, since+// cursors are long and opaque and add nothing for a reader.+func redactQuery(u *url.URL) string {+ redacted := *u+ redacted.RawQuery = ""+ return redacted.String()+}diff --git a/pkg/sources/github/pagination_test.go b/pkg/sources/github/pagination_test.gonew file mode 100644index 000000000..5cc143224--- /dev/null+++ b/pkg/sources/github/pagination_test.go@@ -0,0 +1,766 @@+package github++import (+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "slices"+ "strconv"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/google/go-github/v67/github"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++// fakeListing serves one GitHub list endpoint with the paging behaviors the+// helpers have to cope with: `since` filtering (inclusive), page numbers or+// cursors in next links, and a page cap that ends with a full page and no+// next link, followed by a pagination 422.+type fakeListing struct {+ mu sync.Mutex+ // records must be sorted by updated, ascending, as the real endpoints+ // return them for sort=updated&direction=asc.+ records []fakeRecord+ // cursor adds `after` to next links, and positions requests by it,+ // the way /issues does.+ cursor bool+ // maxPages caps the listing: the last allowed page has no next link and+ // any later page returns a pagination 422. Zero means no cap.+ maxPages int+ // linkPastCap keeps the next link on the last allowed page, so the 422+ // arrives mid-walk instead of on a probe.+ linkPastCap bool+ // omitNext drops the next link from the given pages even when more+ // records remain.+ omitNext map[int]bool+ // fail, when set, can replace any response. It receives the 1-based+ // request count and the request, and returns true if it wrote a response.+ fail func(n int, w http.ResponseWriter, r *http.Request) bool++ requests []url.Values+}++type fakeRecord struct {+ updated time.Time+ body map[string]any+}++const paginationCapMessage = "In order to keep the API fast for everyone, pagination is limited for this resource."++func (f *fakeListing) ServeHTTP(w http.ResponseWriter, r *http.Request) {+ f.mu.Lock()+ defer f.mu.Unlock()++ q := r.URL.Query()+ f.requests = append(f.requests, q)+ if f.fail != nil && f.fail(len(f.requests), w, r) {+ return+ }++ records := f.records+ if s := q.Get("since"); s != "" {+ since, err := time.Parse(time.RFC3339, s)+ if err != nil {+ writeJSON(w, http.StatusBadRequest, map[string]string{"message": "bad since"})+ return+ }+ filtered := make([]fakeRecord, 0, len(records))+ for _, rec := range records {+ if !rec.updated.Before(since) {+ filtered = append(filtered, rec)+ }+ }+ records = filtered+ }++ perPage := 30+ if v, _ := strconv.Atoi(q.Get("per_page")); v > 0 {+ perPage = v+ }+ page := 1+ if v, _ := strconv.Atoi(q.Get("page")); v > 0 {+ page = v+ }+ if after := q.Get("after"); f.cursor && after != "" {+ page, _ = strconv.Atoi(strings.TrimPrefix(after, "cursor-"))+ }+ if f.maxPages > 0 && page > f.maxPages {+ writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"message": paginationCapMessage})+ return+ }++ start := min((page-1)*perPage, len(records))+ end := min(start+perPage, len(records))+ atCap := f.maxPages > 0 && page == f.maxPages+ if end < len(records) && !f.omitNext[page] && (!atCap || f.linkPastCap) {+ next := url.Values{}+ for k, v := range q {+ next[k] = v+ }+ next.Set("page", strconv.Itoa(page+1))+ if f.cursor {+ next.Set("after", fmt.Sprintf("cursor-%d", page+1))+ }+ w.Header().Set("Link", fmt.Sprintf(`<http://%s%s?%s>; rel="next", <http://%s%s?page=1>; rel="first"`,+ r.Host, r.URL.Path, next.Encode(), r.Host, r.URL.Path))+ }++ body := make([]map[string]any, 0, end-start)+ for _, rec := range records[start:end] {+ body = append(body, rec.body)+ }+ writeJSON(w, http.StatusOK, body)+}++func (f *fakeListing) requestCount() int {+ f.mu.Lock()+ defer f.mu.Unlock()+ return len(f.requests)+}++func (f *fakeListing) requestedPages() []string {+ f.mu.Lock()+ defer f.mu.Unlock()+ pages := make([]string, 0, len(f.requests))+ for _, q := range f.requests {+ page := q.Get("page")+ if page == "" {+ page = "1"+ }+ pages = append(pages, page)+ }+ return pages+}++func writeJSON(w http.ResponseWriter, status int, v any) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(status)+ _ = json.NewEncoder(w).Encode(v)+}++// testItem is the shape the helper tests decode pages into.+type testItem struct {+ ID int64 `json:"id"`+ UpdatedAt time.Time `json:"updated_at"`+}++var testEpoch = time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)++// sequentialRecords returns n records with IDs 1..n updated one minute apart.+func sequentialRecords(n int) []fakeRecord {+ updated := make([]time.Time, n)+ for i := range updated {+ updated[i] = testEpoch.Add(time.Duration(i) * time.Minute)+ }+ return recordsAt(updated...)+}++// recordsAt returns one record per timestamp, with IDs 1..n in order.+func recordsAt(updated ...time.Time) []fakeRecord {+ recs := make([]fakeRecord, len(updated))+ for i, t := range updated {+ recs[i] = fakeRecord{updated: t, body: map[string]any{"id": i + 1, "updated_at": t}}+ }+ return recs+}++// newTestPager serves handler at /items and returns a pager for it with a+// page size of 3 and no retries.+func newTestPager(t *testing.T, handler http.Handler) pager {+ t.Helper()+ mux := http.NewServeMux()+ mux.Handle("/items", handler)+ srv := httptest.NewServer(mux)+ t.Cleanup(srv.Close)+ return pager{client: newTestClient(t, srv), perPage: 3}+}++func newTestClient(t *testing.T, srv *httptest.Server) *github.Client {+ t.Helper()+ client := github.NewClient(srv.Client())+ base, err := url.Parse(srv.URL + "/")+ require.NoError(t, err)+ client.BaseURL = base+ return client+}++// firstItems fetches the first page of /items the way a typed go-github list+// call would, with the pager's page size.+func firstItems(p pager) func(context.Context) ([]testItem, *github.Response, error) {+ return func(ctx context.Context) ([]testItem, *github.Response, error) {+ return fetchPage[testItem](ctx, p.client, fmt.Sprintf("items?per_page=%d", p.perPage))+ }+}++// collect returns a visit func that records item IDs in order.+func collect(ids *[]int64) func([]testItem) error {+ return func(items []testItem) error {+ for _, item := range items {+ *ids = append(*ids, item.ID)+ }+ return nil+ }+}++func idsUpTo(n int) []int64 {+ ids := make([]int64, n)+ for i := range ids {+ ids[i] = int64(i + 1)+ }+ return ids+}++func TestWalkPages_FollowsNextLinksToShortFinalPage(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(7)}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.False(t, outcome.capped)+ assert.Equal(t, 7, outcome.items)+ assert.Equal(t, idsUpTo(7), ids)+ assert.Equal(t, []string{"1", "2", "3"}, listing.requestedPages())+}++func TestWalkPages_FullFinalPageWithEmptyProbeIsComplete(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(6)}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.False(t, outcome.capped)+ assert.Equal(t, idsUpTo(6), ids)+ assert.Equal(t, []string{"1", "2", "3"}, listing.requestedPages(), "page 3 is the probe")+}++func TestWalkPages_ProbeThatFindsDataKeepsWalking(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(8), omitNext: map[int]bool{1: true}}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.False(t, outcome.capped)+ assert.Equal(t, idsUpTo(8), ids)+}++func TestWalkPages_ProbeThatHitsPaginationLimitIsCapped(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(9), maxPages: 2}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.True(t, outcome.capped)+ assert.Equal(t, 6, outcome.items)+ assert.Equal(t, idsUpTo(6), ids)+ assert.Equal(t, []string{"1", "2", "3"}, listing.requestedPages())+}++func TestWalkPages_PaginationLimitMidWalkIsCapped(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(9), maxPages: 2, linkPastCap: true}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.True(t, outcome.capped)+ assert.Equal(t, idsUpTo(6), ids)+}++func TestWalkPages_PaginationLimitOnFirstPageIsCapped(t *testing.T) {+ listing := &fakeListing{fail: func(_ int, w http.ResponseWriter, _ *http.Request) bool {+ writeJSON(w, http.StatusUnprocessableEntity, map[string]string{+ "message": "Pagination with the page parameter is not supported for large datasets, please use cursor based pagination (after/before)",+ })+ return true+ }}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.True(t, outcome.capped)+ assert.Empty(t, ids)+}++func TestWalkPages_FullFinalCursorPageIsCappedWithoutProbe(t *testing.T) {+ // /issues next links carry both page and after; probing page+1 would be+ // ignored in favor of the cursor and return the same page again.+ listing := &fakeListing{records: sequentialRecords(9), cursor: true, maxPages: 2}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.True(t, outcome.capped)+ assert.Equal(t, idsUpTo(6), ids)+ assert.Equal(t, 2, listing.requestCount(), "no probe after a cursor page")+}++func TestWalkPages_FollowsCursorLinksVerbatim(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(8), cursor: true}+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.False(t, outcome.capped)+ assert.Equal(t, idsUpTo(8), ids)+ require.Len(t, listing.requests, 3)+ assert.Equal(t, "cursor-3", listing.requests[2].Get("after"))+}++func TestWalkPages_NonPaginationUnprocessableEntityIsAnError(t *testing.T) {+ listing := &fakeListing{+ records: sequentialRecords(9),+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {+ if n != 2 {+ return false+ }+ writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"message": "Validation Failed"})+ return true+ },+ }+ p := newTestPager(t, listing)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.Error(t, err)+ assert.Contains(t, err.Error(), "Validation Failed")+ assert.Equal(t, 3, outcome.items)+ assert.Equal(t, idsUpTo(3), ids, "pages read before the error are kept")+}++func TestWalkPages_RejectsNextLinkToAnotherHost(t *testing.T) {+ listing := &fakeListing{fail: func(_ int, w http.ResponseWriter, _ *http.Request) bool {+ w.Header().Set("Link", `<https://attacker.example/items?page=2>; rel="next"`)+ writeJSON(w, http.StatusOK, []map[string]any{{"id": 1}, {"id": 2}, {"id": 3}})+ return true+ }}+ p := newTestPager(t, listing)++ var ids []int64+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.Error(t, err)+ assert.Contains(t, err.Error(), "attacker.example")+ assert.Equal(t, idsUpTo(3), ids)+ assert.Equal(t, 1, listing.requestCount())+}++func TestWalkPages_ErrorsWhenNextLinkRepeats(t *testing.T) {+ // /pulls echoes an `after` it ignores, so a cursor-only next link would+ // fetch the same page forever.+ listing := &fakeListing{fail: func(_ int, w http.ResponseWriter, r *http.Request) bool {+ w.Header().Set("Link", fmt.Sprintf(`<http://%s/items?per_page=3&after=abc>; rel="next"`, r.Host))+ writeJSON(w, http.StatusOK, []map[string]any{{"id": 1}, {"id": 2}, {"id": 3}})+ return true+ }}+ p := newTestPager(t, listing)++ var ids []int64+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.Error(t, err)+ assert.Contains(t, err.Error(), "did not advance")+ assert.Equal(t, 2, listing.requestCount())+}++func TestWalkPages_RetriesWhenRateLimitHandlerSaysTo(t *testing.T) {+ listing := &fakeListing{+ records: sequentialRecords(7),+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {+ if n != 2 {+ return false+ }+ writeJSON(w, http.StatusForbidden, map[string]string{"message": "slow down"})+ return true+ },+ }+ p := newTestPager(t, listing)+ retries := 0+ p.retry = func(err error) bool {+ if err == nil || retries > 0 {+ return false+ }+ retries+++ return true+ }++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.False(t, outcome.capped)+ assert.Equal(t, 1, retries)+ assert.Equal(t, idsUpTo(7), ids)+ assert.Equal(t, []string{"1", "2", "2", "3"}, listing.requestedPages())+}++// recordWaits turns on transient retries for p and records each requested+// delay instead of sleeping.+func recordWaits(p *pager) *[]time.Duration {+ var waits []time.Duration+ p.wait = func(_ context.Context, d time.Duration) bool {+ waits = append(waits, d)+ return true+ }+ return &waits+}++// failRequest returns a fail func that writes status for the given request+// numbers.+func failRequest(status int, requests ...int) func(int, http.ResponseWriter, *http.Request) bool {+ return func(n int, w http.ResponseWriter, _ *http.Request) bool {+ if !slices.Contains(requests, n) {+ return false+ }+ writeJSON(w, status, map[string]string{"message": http.StatusText(status)})+ return true+ }+}++func TestWalkPages_RetriesTransientServerErrors(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusBadGateway, 2, 3)}+ p := newTestPager(t, listing)+ waits := recordWaits(&p)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.False(t, outcome.capped)+ assert.Equal(t, idsUpTo(7), ids)+ assert.Equal(t, []time.Duration{2 * time.Second, 4 * time.Second}, *waits)+ assert.Equal(t, []string{"1", "2", "2", "2", "3"}, listing.requestedPages())+}++func TestWalkPages_RetriesResponseCutOffPartway(t *testing.T) {+ listing := &fakeListing{+ records: sequentialRecords(7),+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {+ if n != 2 {+ return false+ }+ // Promise a longer body than is sent, then drop the connection,+ // the way a stream reset mid-response looks to the client.+ conn, buf, err := w.(http.Hijacker).Hijack()+ if err != nil {+ panic(err)+ }+ _, _ = buf.WriteString("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000\r\n\r\n[{\"id\":4")+ _ = buf.Flush()+ _ = conn.Close()+ return true+ },+ }+ p := newTestPager(t, listing)+ waits := recordWaits(&p)++ var ids []int64+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.NoError(t, err)+ assert.Equal(t, idsUpTo(7), ids)+ assert.Len(t, *waits, 1)+}++func TestWalkPages_GivesUpAfterBoundedTransientRetries(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusServiceUnavailable, 2, 3, 4, 5, 6, 7)}+ p := newTestPager(t, listing)+ waits := recordWaits(&p)++ var ids []int64+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.Error(t, err)+ assert.Contains(t, err.Error(), "giving up after 6 attempts")+ assert.Contains(t, err.Error(), "503")+ assert.Equal(t, []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second}, *waits)+ assert.Equal(t, 3, outcome.items)+ assert.Equal(t, idsUpTo(3), ids, "pages read before the failure are kept")+}++func TestWalkPages_DoesNotRetryClientErrors(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusNotFound, 2)}+ p := newTestPager(t, listing)+ waits := recordWaits(&p)++ var ids []int64+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.Error(t, err)+ assert.NotContains(t, err.Error(), "giving up")+ assert.Empty(t, *waits)+ assert.Equal(t, 2, listing.requestCount())+}++func TestWalkPages_StopsRetryingWhenWaitIsInterrupted(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusBadGateway, 2, 3)}+ p := newTestPager(t, listing)+ p.wait = func(context.Context, time.Duration) bool { return false }++ var ids []int64+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))++ require.Error(t, err)+ assert.Equal(t, 2, listing.requestCount())+}++func TestIsTransient(t *testing.T) {+ apiErr := func(status int) error {+ return &github.ErrorResponse{Response: &http.Response{StatusCode: status}, Message: http.StatusText(status)}+ }+ cancelled, cancel := context.WithCancel(context.Background())+ cancel()++ tests := []struct {+ name string+ ctx context.Context+ err error+ want bool+ }{+ {name: "server error", err: apiErr(http.StatusBadGateway), want: true},+ {name: "client error", err: apiErr(http.StatusNotFound)},+ {name: "pagination cap", err: apiErr(http.StatusUnprocessableEntity)},+ {name: "primary rate limit", err: &github.RateLimitError{Response: &http.Response{StatusCode: http.StatusForbidden}}},+ {name: "secondary rate limit", err: &github.AbuseRateLimitError{Response: &http.Response{StatusCode: http.StatusForbidden}}},+ {name: "transport failure", err: &url.Error{Op: "Get", URL: "https://api.github.com/x", Err: errors.New("connection reset by peer")}, want: true},+ {name: "body cut off", err: fmt.Errorf("decoding: %w", io.ErrUnexpectedEOF), want: true},+ {name: "http2 stream reset", err: errors.New("stream error: stream ID 261; CANCEL; received from peer"), want: true},+ {name: "context ended", ctx: cancelled, err: apiErr(http.StatusBadGateway)},+ {name: "other error", err: errors.New("invalid pagination link")},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ ctx := tt.ctx+ if ctx == nil {+ ctx = context.Background()+ }+ assert.Equal(t, tt.want, isTransient(ctx, tt.err))+ })+ }+}++func TestWalkPages_StopsWhenVisitFails(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(9)}+ p := newTestPager(t, listing)+ visitErr := errors.New("reporter closed")++ _, err := walkPages(context.Background(), p, firstItems(p), func([]testItem) error { return visitErr })++ require.ErrorIs(t, err, visitErr)+ assert.Equal(t, 1, listing.requestCount())+}++// sinceItems lists /items through walkUpdatedSince the way the comment+// phases do: sorted by update time, `since` only when set.+func sinceItems(p pager, ids *[]int64) sinceWalk[testItem] {+ return sinceWalk[testItem]{+ name: "items",+ list: func(ctx context.Context, since time.Time) ([]testItem, *github.Response, error) {+ u := fmt.Sprintf("items?per_page=%d&sort=updated&direction=asc", p.perPage)+ if !since.IsZero() {+ u += "&since=" + url.QueryEscape(since.UTC().Format(time.RFC3339))+ }+ return fetchPage[testItem](ctx, p.client, u)+ },+ key: func(item testItem) (int64, time.Time) { return item.ID, item.UpdatedAt },+ visit: collect(ids),+ }+}++func TestWalkUpdatedSince_ContinuesPastCapInNewWindows(t *testing.T) {+ listing := &fakeListing{records: sequentialRecords(20), maxPages: 2}+ p := newTestPager(t, listing)++ var ids []int64+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))++ require.NoError(t, err)+ assert.Equal(t, 20, total)+ assert.Equal(t, idsUpTo(20), ids, "every item exactly once, in order")+}++func TestWalkUpdatedSince_DropsItemsAlreadySeenAtWindowBoundary(t *testing.T) {+ // Items 5-8 share a timestamp that straddles the first window's cap, so+ // the second window starts at that timestamp and gets items 5 and 6+ // again before the new ones.+ shared := testEpoch.Add(time.Hour)+ listing := &fakeListing{+ records: recordsAt(+ testEpoch, testEpoch.Add(time.Minute), testEpoch.Add(2*time.Minute), testEpoch.Add(3*time.Minute),+ shared, shared, shared, shared,+ shared.Add(time.Minute), shared.Add(2*time.Minute),+ ),+ maxPages: 2,+ }+ p := newTestPager(t, listing)++ var ids []int64+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))++ require.NoError(t, err)+ assert.Equal(t, 10, total)+ assert.Equal(t, idsUpTo(10), ids)+}++func TestWalkUpdatedSince_CompleteWindowOfDuplicatesFinishes(t *testing.T) {+ // The first window exactly fills its cap on a cursor endpoint, so it ends+ // capped without a probe. The next window holds only the last item seen,+ // which is the confirmed end of the listing.+ listing := &fakeListing{records: sequentialRecords(6), cursor: true, maxPages: 2}+ p := newTestPager(t, listing)++ var ids []int64+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))++ require.NoError(t, err)+ assert.Equal(t, 6, total)+ assert.Equal(t, idsUpTo(6), ids)+}++func TestWalkUpdatedSince_CappedWindowWithoutProgressIsAnError(t *testing.T) {+ shared := testEpoch.Add(time.Hour)+ listing := &fakeListing{records: recordsAt(shared, shared, shared, shared, shared, shared, shared, shared), maxPages: 2}+ p := newTestPager(t, listing)++ var ids []int64+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))++ require.Error(t, err)+ var walkErr *walkError+ require.ErrorAs(t, err, &walkErr)+ assert.Equal(t, shared, walkErr.since)+ assert.Equal(t, 6, walkErr.items)+ assert.Contains(t, err.Error(), "walk incomplete after 6 items (since=2024-01-01T01:00:00Z)")+ assert.Equal(t, 6, total)+ assert.Equal(t, idsUpTo(6), ids, "items read before the error are kept")+}++func TestWalkUpdatedSince_ReportsWindowStartOnError(t *testing.T) {+ listing := &fakeListing{+ records: sequentialRecords(20),+ maxPages: 2,+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {+ if n != 4 {+ return false+ }+ writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "boom"})+ return true+ },+ }+ p := newTestPager(t, listing)++ var ids []int64+ _, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))++ require.Error(t, err)+ var walkErr *walkError+ require.ErrorAs(t, err, &walkErr)+ assert.Equal(t, testEpoch.Add(5*time.Minute), walkErr.since, "second window starts at item 6")+ assert.Equal(t, 6, walkErr.items)+}++func TestNextLink(t *testing.T) {+ tests := []struct {+ name string+ header []string+ want string+ }{+ {name: "no header"},+ {+ name: "next among other rels",+ header: []string{`<https://api.github.com/x?page=1>; rel="prev", <https://api.github.com/x?page=3>; rel="next"`},+ want: "https://api.github.com/x?page=3",+ },+ {+ name: "url containing a comma",+ header: []string{`<https://api.github.com/x?labels=a,b&page=2>; rel="next"`},+ want: "https://api.github.com/x?labels=a,b&page=2",+ },+ {+ name: "next in a second header",+ header: []string{`<https://api.github.com/x?page=1>; rel="first"`, `<https://api.github.com/x?page=2>; rel="next"`},+ want: "https://api.github.com/x?page=2",+ },+ {+ name: "no next",+ header: []string{`<https://api.github.com/x?page=1>; rel="first"`},+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ h := http.Header{}+ for _, v := range tt.header {+ h.Add("Link", v)+ }+ resp := &github.Response{Response: &http.Response{Header: h}}+ assert.Equal(t, tt.want, nextLink(resp))+ })+ }+}++func TestFollowableResolvesRelativeLinksAgainstAPIHost(t *testing.T) {+ client := github.NewClient(nil)+ base, err := url.Parse("https://ghe.example.com/api/v3/")+ require.NoError(t, err)+ client.BaseURL = base+ p := pager{client: client}++ u, err := p.followable("/api/v3/repos/o/r/issues?page=2")+ require.NoError(t, err)+ assert.Equal(t, "https://ghe.example.com/api/v3/repos/o/r/issues?page=2", u.String())++ u, err = p.followable("https://GHE.example.com/api/v3/repos/o/r/issues?page=3")+ require.NoError(t, err)+ assert.Equal(t, "GHE.example.com", u.Host)++ _, err = p.followable("https://other.example.com/api/v3/repos/o/r/issues?page=3")+ assert.Error(t, err)+}++func TestWithNextPage(t *testing.T) {+ tests := map[string]string{+ "https://api.github.com/x?per_page=100": "https://api.github.com/x?page=2&per_page=100",+ "https://api.github.com/x?page=7&per_page=100": "https://api.github.com/x?page=8&per_page=100",+ }+ for in, want := range tests {+ u, err := url.Parse(in)+ require.NoError(t, err)+ assert.Equal(t, want, withNextPage(u))+ }+}++func TestIsPaginationCap(t *testing.T) {+ errFor := func(status int, msg string) error {+ return &github.ErrorResponse{Response: &http.Response{StatusCode: status}, Message: msg}+ }+ assert.True(t, isPaginationCap(errFor(http.StatusUnprocessableEntity, paginationCapMessage)))+ assert.True(t, isPaginationCap(fmt.Errorf("wrapped: %w", errFor(http.StatusUnprocessableEntity, "Pagination with the page parameter is not supported"))))+ assert.False(t, isPaginationCap(errFor(http.StatusUnprocessableEntity, "Validation Failed")))+ assert.False(t, isPaginationCap(errFor(http.StatusForbidden, paginationCapMessage)))+ assert.False(t, isPaginationCap(errors.New("pagination")))+}