Skip to content

Find, verify, and analyze leaked credentials

[INT-1051] Fix GitHub comment scans stopping early on large repositories (#5405)

Dave Hubbardb40e7fe3b767
diff --git a/pkg/sources/github/README.md b/pkg/sources/github/README.md
index 805d8309e..8fb85e37d 100644
--- a/pkg/sources/github/README.md
+++ b/pkg/sources/github/README.md
@@ -270,6 +270,16 @@ With `--clone-path` set, each repository still gets its own fresh subdirectory c
5. **Comment Scanning**: If any comment flags are set, fetches issues, pull requests, or gist comments through the GitHub API and scans their text
6. **Progress Tracking**: Keeps track of which repositories have already been scanned, so a scan can be resumed without redoing work
+### How Comments Are Fetched
+
+- `--issue-comments` scans issue titles and bodies, and every discussion comment in the repository. GitHub treats pull requests as issues, so this includes the discussion comments on pull requests.
+- `--pr-comments` scans pull request titles and bodies, and every review comment (the comments left inline on a diff).
+- Listings are paged by following the links GitHub returns, sorted by last update time. Some GitHub endpoints stop returning results after a fixed number of pages (the repository-wide issue comment listing stops at 300 pages); when that happens, the scan picks up from the last update time it reached, so large repositories are scanned in full.
+- A request that fails for a transient reason (a GitHub server error, a dropped connection, or a response cut off partway) is retried up to five times with backoff before it counts as a failure. Rate limits are waited out and retried separately, without a limit.
+- The comments timeframe is applied by GitHub. Issue and pull request bodies are scanned regardless of the timeframe.
+- Each listing is independent. If one fails, the others still run, everything read before the failure is still scanned, and the failure is reported as a scan error for that repository naming the listing, how many items it read, and where it stopped.
+- Gist comments can't be filtered or sorted by GitHub, so the timeframe is applied as they are read, and a gist whose comments GitHub stops paging through is reported as a scan error.
+
### What Gets Scanned
- Every commit in every repository matching the scan target and filters
@@ -282,7 +292,7 @@ With `--clone-path` set, each repository still gets its own fresh subdirectory c
- Archived repositories, when `--exclude-archived` is set
- Repositories excluded by `--include-repos` or `--exclude-repos`, for an organization or user scan
- Gists, when `--ignore-gists` is set, or when the scan target is a genuine organization rather than a user (gists only surface for user accounts, see the note above)
-- Comments older than the configured comments timeframe, when one is set
+- Comments last updated before the configured comments timeframe, when one is set (gist comments go by creation time)
## Usage Examples
diff --git a/pkg/sources/github/comments_test.go b/pkg/sources/github/comments_test.go
new file mode 100644
index 000000000..77f66d237
--- /dev/null
+++ b/pkg/sources/github/comments_test.go
@@ -0,0 +1,380 @@
+package github
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/google/go-github/v67/github"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+// fakeGitHub serves the REST endpoints the comment scan reads for repo o/r
+// and gist g1, each backed by its own fakeListing.
+type fakeGitHub struct {
+ issues *fakeListing
+ issueComments *fakeListing
+ pulls *fakeListing
+ prComments *fakeListing
+ gistComments *fakeListing
+ client *github.Client
+}
+
+func newFakeGitHub(t *testing.T) *fakeGitHub {
+ t.Helper()
+ gh := &fakeGitHub{
+ issues: &fakeListing{},
+ issueComments: &fakeListing{},
+ pulls: &fakeListing{},
+ prComments: &fakeListing{},
+ gistComments: &fakeListing{},
+ }
+ mux := http.NewServeMux()
+ mux.Handle("/repos/o/r/issues", gh.issues)
+ mux.Handle("/repos/o/r/issues/comments", gh.issueComments)
+ mux.Handle("/repos/o/r/pulls", gh.pulls)
+ mux.Handle("/repos/o/r/pulls/comments", gh.prComments)
+ mux.Handle("/gists/g1/comments", gh.gistComments)
+ srv := httptest.NewServer(mux)
+ t.Cleanup(srv.Close)
+ gh.client = newTestClient(t, srv)
+ return gh
+}
+
+var testRepo = repoInfo{owner: "o", name: "r", fullName: "o/r"}
+
+// collectingReporter records the link of every chunk the scan emits.
+type collectingReporter struct {
+ mu sync.Mutex
+ links []string
+}
+
+func (r *collectingReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.links = append(r.links, chunk.SourceMetadata.GetGithub().GetLink())
+ return nil
+}
+
+func (r *collectingReporter) ChunkErr(context.Context, error) error { return nil }
+
+func issueRecord(number int, updated time.Time) fakeRecord {
+ return fakeRecord{updated: updated, body: map[string]any{
+ "id": number, "number": number, "updated_at": updated, "created_at": updated,
+ "html_url": fmt.Sprintf("https://github.com/o/r/issues/%d", number),
+ "title": "issue", "body": "body",
+ }}
+}
+
+// prIssueRecord is a pull request as /issues returns it.
+func prIssueRecord(number int, updated time.Time) fakeRecord {
+ return fakeRecord{updated: updated, body: map[string]any{
+ "id": number, "number": number, "updated_at": updated, "created_at": updated,
+ "html_url": prLink(number),
+ "title": "pr", "body": "body",
+ "pull_request": map[string]any{"url": fmt.Sprintf("https://api.github.com/repos/o/r/pulls/%d", number)},
+ }}
+}
+
+func prRecord(number int, updated time.Time) fakeRecord {
+ return fakeRecord{updated: updated, body: map[string]any{
+ "id": number, "number": number, "updated_at": updated, "created_at": updated,
+ "html_url": prLink(number), "title": "pr", "body": "body",
+ }}
+}
+
+func prLink(number int) string { return fmt.Sprintf("https://github.com/o/r/pull/%d", number) }
+
+func commentRecord(id int, updated time.Time, htmlURL string) fakeRecord {
+ return fakeRecord{updated: updated, body: map[string]any{
+ "id": id, "updated_at": updated, "created_at": updated, "html_url": htmlURL, "body": "comment",
+ }}
+}
+
+// seedRepo gives o/r one issue, one pull request, a discussion comment on
+// each, and a review comment on the pull request.
+func seedRepo(gh *fakeGitHub) {
+ gh.issues.records = []fakeRecord{issueRecord(1, testEpoch), prIssueRecord(2, testEpoch.Add(time.Minute))}
+ gh.pulls.records = []fakeRecord{prRecord(2, testEpoch.Add(time.Minute))}
+ gh.issueComments.records = []fakeRecord{
+ commentRecord(100, testEpoch.Add(2*time.Minute), "https://github.com/o/r/issues/1#issuecomment-100"),
+ commentRecord(101, testEpoch.Add(3*time.Minute), "https://github.com/o/r/pull/2#issuecomment-101"),
+ }
+ gh.prComments.records = []fakeRecord{
+ commentRecord(200, testEpoch.Add(4*time.Minute), "https://github.com/o/r/pull/2#discussion_r200"),
+ }
+}
+
+func scanComments(t *testing.T, s *Source, gh *fakeGitHub, cutoff *time.Time) (*collectingReporter, error) {
+ t.Helper()
+ reporter := &collectingReporter{}
+ err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, reporter, cutoff)
+ return reporter, err
+}
+
+func TestCommentScan_FlagCombinations(t *testing.T) {
+ tests := []struct {
+ name string
+ issueComments bool
+ prComments bool
+ wantLinks []string
+ // unlisted names the endpoints the scan must not request.
+ unlisted []string
+ }{
+ {
+ name: "issue comments only",
+ issueComments: true,
+ wantLinks: []string{
+ "https://github.com/o/r/issues/1",
+ "https://github.com/o/r/issues/1#issuecomment-100",
+ "https://github.com/o/r/pull/2#issuecomment-101",
+ },
+ unlisted: []string{"pulls", "pull comments"},
+ },
+ {
+ name: "pull request comments only",
+ prComments: true,
+ wantLinks: []string{
+ prLink(2),
+ "https://github.com/o/r/pull/2#discussion_r200",
+ },
+ unlisted: []string{"issues", "issue comments"},
+ },
+ {
+ name: "both",
+ issueComments: true,
+ prComments: true,
+ wantLinks: []string{
+ "https://github.com/o/r/issues/1",
+ prLink(2),
+ "https://github.com/o/r/issues/1#issuecomment-100",
+ "https://github.com/o/r/pull/2#issuecomment-101",
+ "https://github.com/o/r/pull/2#discussion_r200",
+ },
+ unlisted: []string{"pulls"},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includeIssueComments: tt.issueComments, includePRComments: tt.prComments}
+
+ reporter, err := scanComments(t, s, gh, nil)
+
+ require.NoError(t, err)
+ assert.ElementsMatch(t, tt.wantLinks, reporter.links)
+ endpoints := map[string]*fakeListing{
+ "issues": gh.issues, "issue comments": gh.issueComments,
+ "pulls": gh.pulls, "pull comments": gh.prComments,
+ }
+ for _, name := range tt.unlisted {
+ assert.Zero(t, endpoints[name].requestCount(), "%s should not be listed", name)
+ }
+ })
+ }
+}
+
+func TestCommentScan_RequestsSortedListingsWithServerSideCutoff(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includeIssueComments: true, includePRComments: true}
+ cutoff := testEpoch.Add(-time.Hour)
+
+ _, err := scanComments(t, s, gh, &cutoff)
+ require.NoError(t, err)
+
+ issues := gh.issues.requests[0]
+ assert.Equal(t, "all", issues.Get("state"))
+ assert.Equal(t, "updated", issues.Get("sort"))
+ assert.Equal(t, "asc", issues.Get("direction"))
+ assert.Equal(t, "100", issues.Get("per_page"))
+ assert.Empty(t, issues.Get("since"), "issue and PR bodies are scanned regardless of the comments timeframe")
+
+ for name, listing := range map[string]*fakeListing{"issue comments": gh.issueComments, "pull comments": gh.prComments} {
+ q := listing.requests[0]
+ assert.Equal(t, "updated", q.Get("sort"), name)
+ assert.Equal(t, "asc", q.Get("direction"), name)
+ assert.Equal(t, cutoff.Format(time.RFC3339), q.Get("since"), name)
+ }
+}
+
+func TestCommentScan_OmitsSinceWithoutCutoff(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includeIssueComments: true, includePRComments: true}
+
+ _, err := scanComments(t, s, gh, nil)
+ require.NoError(t, err)
+
+ assert.False(t, gh.issueComments.requests[0].Has("since"))
+ assert.False(t, gh.prComments.requests[0].Has("since"))
+}
+
+func TestCommentScan_PullsListOnlyConfigListsPullsWithAllStates(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ s := &Source{includePRComments: true}
+
+ _, err := scanComments(t, s, gh, nil)
+ require.NoError(t, err)
+
+ q := gh.pulls.requests[0]
+ assert.Equal(t, "all", q.Get("state"))
+ assert.Equal(t, "updated", q.Get("sort"))
+ assert.Equal(t, "asc", q.Get("direction"))
+}
+
+func TestCommentScan_IssueCommentsContinuePastPageCap(t *testing.T) {
+ gh := newFakeGitHub(t)
+ for i := range 250 {
+ gh.issueComments.records = append(gh.issueComments.records,
+ commentRecord(i+1, testEpoch.Add(time.Duration(i)*time.Minute), fmt.Sprintf("https://github.com/o/r/issues/1#issuecomment-%d", i+1)))
+ }
+ gh.issueComments.maxPages = 1
+ s := &Source{includeIssueComments: true}
+
+ reporter, err := scanComments(t, s, gh, nil)
+
+ require.NoError(t, err)
+ require.Len(t, reporter.links, 250)
+ seen := map[string]bool{}
+ for _, link := range reporter.links {
+ assert.False(t, seen[link], "comment emitted twice: %s", link)
+ seen[link] = true
+ }
+}
+
+func TestCommentScan_PullsCappedFallsBackToIssueListing(t *testing.T) {
+ gh := newFakeGitHub(t)
+ const prs = 150
+ for n := 1; n <= prs; n++ {
+ updated := testEpoch.Add(time.Duration(n) * time.Minute)
+ gh.pulls.records = append(gh.pulls.records, prRecord(n, updated))
+ gh.issues.records = append(gh.issues.records, prIssueRecord(n, updated))
+ if n%10 == 0 {
+ gh.issues.records = append(gh.issues.records, issueRecord(1000+n, updated))
+ }
+ }
+ gh.pulls.maxPages = 1
+ s := &Source{includePRComments: true}
+
+ reporter, err := scanComments(t, s, gh, nil)
+
+ require.NoError(t, err)
+ want := make([]string, 0, prs)
+ for n := 1; n <= prs; n++ {
+ want = append(want, prLink(n))
+ }
+ assert.ElementsMatch(t, want, reporter.links, "every PR body exactly once, and no issue bodies")
+}
+
+func TestCommentScan_FailedPhaseDoesNotStopTheOthers(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ gh.issueComments.fail = func(_ int, w http.ResponseWriter, _ *http.Request) bool {
+ writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Server Error"})
+ return true
+ }
+ var retries int
+ s := &Source{
+ includeIssueComments: true,
+ includePRComments: true,
+ commentRetryWait: func(context.Context, time.Duration) bool { retries++; return true },
+ }
+ cutoff := testEpoch.Add(-time.Hour)
+
+ reporter, err := scanComments(t, s, gh, &cutoff)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "issue comments: walk incomplete after 0 items (since=2023-12-31T23:00:00Z): giving up after 6 attempts")
+ assert.Contains(t, err.Error(), "Server Error")
+ assert.Equal(t, maxTransientRetries, retries)
+ assert.NotContains(t, err.Error(), "pull request comments:")
+ var walkErr *walkError
+ assert.ErrorAs(t, err, &walkErr)
+ assert.ElementsMatch(t, []string{
+ "https://github.com/o/r/issues/1",
+ prLink(2),
+ "https://github.com/o/r/pull/2#discussion_r200",
+ }, reporter.links)
+}
+
+func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {
+ gh := newFakeGitHub(t)
+ seedRepo(gh)
+ gh.issueComments.fail = failRequest(http.StatusBadGateway, 1)
+ gh.prComments.fail = failRequest(http.StatusBadGateway, 1)
+ s := &Source{
+ includeIssueComments: true,
+ includePRComments: true,
+ commentRetryWait: func(context.Context, time.Duration) bool { return true },
+ }
+ var logs bytes.Buffer
+ logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(2)))
+ ctx := context.WithLogger(context.Background(), logger)
+
+ err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, &collectingReporter{}, nil)
+ require.NoError(t, err)
+
+ var listings []string
+ for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") {
+ var entry map[string]any
+ require.NoError(t, json.Unmarshal([]byte(line), &entry))
+ if entry["msg"] == "retrying GitHub API request after transient error" {
+ listings = append(listings, fmt.Sprint(entry["listing"]))
+ }
+ }
+ assert.Equal(t, []string{"issue comments", "pull request comments"}, listings)
+}
+
+func TestProcessGistComments_CappedListingIsAnError(t *testing.T) {
+ gh := newFakeGitHub(t)
+ for i := range 150 {
+ gh.gistComments.records = append(gh.gistComments.records, commentRecord(i+1, testEpoch, ""))
+ }
+ gh.gistComments.maxPages = 1
+ s := &Source{includeGistComments: true}
+ reporter := &collectingReporter{}
+
+ err := s.processGistComments(context.Background(), gh.client, "https://gist.github.com/u/g1",
+ []string{"gist.github.com", "u", "g1"}, repoInfo{}, reporter, nil)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "gist comments: walk incomplete after 100 items")
+ assert.Contains(t, err.Error(), "no since filter")
+ assert.Len(t, reporter.links, 100, "comments read before the cap are kept")
+}
+
+func TestProcessGistComments_SkipsOldCommentsAndKeepsGoing(t *testing.T) {
+ gh := newFakeGitHub(t)
+ for i := range 5 {
+ created := testEpoch.Add(time.Duration(i) * time.Hour)
+ gh.gistComments.records = append(gh.gistComments.records, fakeRecord{updated: created, body: map[string]any{
+ "id": i + 1, "created_at": created, "url": fmt.Sprintf("https://api.github.com/gists/g1/comments/%d", i+1),
+ }})
+ }
+ s := &Source{includeGistComments: true}
+ reporter := &collectingReporter{}
+ cutoff := testEpoch.Add(150 * time.Minute)
+
+ err := s.processGistComments(context.Background(), gh.client, "https://gist.github.com/u/g1",
+ []string{"gist.github.com", "u", "g1"}, repoInfo{}, reporter, &cutoff)
+
+ require.NoError(t, err)
+ assert.Equal(t, []string{
+ "https://api.github.com/gists/g1/comments/4",
+ "https://api.github.com/gists/g1/comments/5",
+ }, reporter.links)
+}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 1ccc72714..dbe3a0fd0 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -80,6 +80,10 @@ type Source struct {
ignoreGists bool
includeGistComments bool
commentsTimeframeDays uint32
+ // commentRetryWait sleeps between retries of transient comment API
+ // failures and returns false if the context ended first. Nil uses
+ // sleepWithContext; tests inject a wait that returns immediately.
+ commentRetryWait func(context.Context, time.Duration) bool
sources.Progress
@@ -1358,7 +1362,8 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.
// This should never happen.
return fmt.Errorf("no repoInfo for URL: %s", repoURL)
}
- duration, err := s.cloneAndScanRepo(ctx, repoURL, repoInfo, reporter)
+ start := time.Now()
+ gitDuration, err := s.cloneAndScanRepo(ctx, repoURL, repoInfo, reporter)
if err != nil {
return err
}
@@ -1385,8 +1390,12 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.
}
// Scan comments, if enabled.
+ var commentsDuration time.Duration
if s.includeGistComments || s.includeIssueComments || s.includePRComments {
- if err := s.scanComments(ctx, repoURL, repoInfo, reporter); err != nil {
+ commentsStart := time.Now()
+ err := s.scanComments(ctx, repoURL, repoInfo, reporter)
+ commentsDuration = time.Since(commentsStart)
+ if err != nil {
err := fmt.Errorf("error scanning comments: %w", err)
if err := reporter.ChunkErr(ctx, err); err != nil {
return err
@@ -1394,7 +1403,13 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.
}
}
- ctx.Logger().V(2).Info("finished scanning repo", "duration_seconds", duration)
+ // duration_seconds covers everything above: clone, git history, wiki, and
+ // comments. Git history and comments are broken out because on repos with
+ // many comments, the comment walks can take far longer than the git scan.
+ ctx.Logger().V(2).Info("finished scanning repo",
+ "duration_seconds", time.Since(start).Seconds(),
+ "git_scan_seconds", gitDuration.Seconds(),
+ "comments_seconds", commentsDuration.Seconds())
githubReposScanned.WithLabelValues(s.name).Inc()
return nil
}
@@ -1862,35 +1877,54 @@ func getRepoURLParts(repoURLString string) (string, []string, error) {
return urlString, urlParts, nil
}
-const initialPage = 1 // page to start listing from
+// commentPager returns the pager every comment walk for one repo shares. Rate
+// limits are handled by waiting and retrying the same request, and each wait
+// is also reported to the chunk reporter. Transient failures are retried a
+// bounded number of times; see fetchWithRetry.
+func (s *Source) commentPager(ctx context.Context, apiClient *github.Client, reporter sources.ChunkReporter) pager {
+ wait := s.commentRetryWait
+ if wait == nil {
+ wait = func(ctx context.Context, d time.Duration) bool {
+ _, canceled := sleepWithContext(ctx, d)
+ return !canceled
+ }
+ }
+ return pager{
+ client: apiClient,
+ perPage: defaultPagination,
+ retry: func(err error) bool {
+ return s.handleRateLimitWithChunkReporter(ctx, reporter, err)
+ },
+ wait: wait,
+ }
+}
+// processGistComments scans a gist's comments. The endpoint has no `since` or
+// sort parameters, so the comments timeframe is applied to each page as it
+// arrives, and a capped walk has no way to continue: it is reported as an
+// error rather than ending silently.
func (s *Source) processGistComments(ctx context.Context, apiClient *github.Client, gistURL string, urlParts []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
ctx.Logger().V(2).Info("Scanning GitHub Gist comments")
- // GitHub Gist URL.
gistID := extractGistID(urlParts)
-
- options := &github.ListOptions{
- PerPage: defaultPagination,
- Page: initialPage,
+ p := s.commentPager(ctx, apiClient, reporter)
+ p.listing = "gist comments"
+ outcome, err := walkPages(ctx, p,
+ func(ctx context.Context) ([]*github.GistComment, *github.Response, error) {
+ return apiClient.Gists.ListComments(ctx, gistID, &github.ListOptions{PerPage: defaultPagination})
+ },
+ func(comments []*github.GistComment) error {
+ return s.chunkGistComments(ctx, gistURL, repoInfo, comments, reporter, cutoffTime)
+ },
+ )
+ if err != nil {
+ return fmt.Errorf("gist comments: %w", &walkError{items: outcome.items, err: err})
}
- for {
- comments, _, err := apiClient.Gists.ListComments(ctx, gistID, options)
- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {
- continue
- }
- if err != nil {
- return err
- }
-
- if err = s.chunkGistComments(ctx, gistURL, repoInfo, comments, reporter, cutoffTime); err != nil {
- return err
- }
-
- options.Page++
- if len(comments) < options.PerPage {
- break
- }
+ if outcome.capped {
+ return fmt.Errorf("gist comments: %w", &walkError{
+ items: outcome.items,
+ err: errors.New("GitHub stopped paginating and this endpoint has no since filter to continue from"),
+ })
}
return nil
}
@@ -1923,9 +1957,10 @@ func isGistUrl(urlParts []string) bool {
func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo repoInfo, comments []*github.GistComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
for _, comment := range comments {
- // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted
+ // Gist comments come back oldest first, so an old comment says nothing
+ // about the ones after it. Skip it and keep going.
if cutoffTime != nil && comment.GetCreatedAt().Before(*cutoffTime) {
- break
+ continue
}
// Create chunk and send it to the channel.
@@ -1957,46 +1992,100 @@ func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo
return nil
}
-// Note: these can't be consts because the address is needed when using with the GitHub library.
-var (
- // sortType defines the criteria for sorting comments.
- // By setting this to "updated" we can use this to reliably manage the comment timeframe filtering below
- sortType = "updated"
- // directionType defines the direction of sorting.
- // "desc" means comments will be sorted in descending order, showing the latest comments first, which is critical for managing the comment timeframe filtering
- directionType = "desc"
- // allComments is a placeholder for specifying the comment ID to start listing from.
- // A value of 0 means that all comments will be listed.
- allComments = 0
- // state of "all" for the ListByRepo captures both open and closed issues.
- state = "all"
+// Repo listings sort by update time, oldest first, so a capped walk can
+// continue from the last update time it saw. See walkUpdatedSince.
+const (
+ sortByUpdated = "updated"
+ sortAscending = "asc"
+ // stateAll lists open and closed issues and pull requests.
+ stateAll = "all"
)
+// commentPhase is one listing in a repo's comment scan. run returns how many
+// chunks it emitted. startLog, when set, is logged at V(2) before the phase
+// runs.
+type commentPhase struct {
+ name string
+ startLog string
+ run func() (int, error)
+}
+
+// processIssueandPRsWithCommentsREST scans a repo's issues, pull requests, and
+// their comments through the REST API.
+//
+// The phases are independent: a failure in one is collected and the rest
+// still run, so one capped or forbidden endpoint can't hide the others. All
+// failures come back joined into one error, which scanRepo reports as a
+// single ChunkErr for the repo. Chunks emitted before a failure are kept.
+//
+// The "Scanning issues" and "Scanning pull requests" V(2) lines are the only
+// log signal for scans on the unit path, whose comment errors go to the scan
+// details UI instead of the logs, so they keep their wording. The summary
+// line at the end records how far each phase got.
func (s *Source) processIssueandPRsWithCommentsREST(
ctx context.Context, apiClient *github.Client, repoInfo repoInfo,
reporter sources.ChunkReporter, cutoffTime *time.Time,
) error {
- if s.includeIssueComments {
- ctx.Logger().V(2).Info("Scanning issues")
- if err := s.processIssues(ctx, apiClient, repoInfo, reporter); err != nil {
- return err
- }
- if err := s.processIssueComments(ctx, apiClient, repoInfo, reporter, cutoffTime); err != nil {
- return err
- }
+ p := s.commentPager(ctx, apiClient, reporter)
+ var since time.Time
+ if cutoffTime != nil {
+ since = *cutoffTime
}
+ // /issues returns issues and pull requests together. When issue comments
+ // are enabled, that one walk also emits PR bodies, so /pulls is only
+ // walked for configs that enable PR comments alone.
+ var phases []commentPhase
+ if s.includeIssueComments {
+ bodies := "issues"
+ if s.includePRComments {
+ bodies = "issues and pull requests"
+ }
+ phases = append(phases,
+ commentPhase{name: bodies, startLog: "Scanning issues", run: func() (int, error) {
+ return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)
+ }},
+ commentPhase{name: "issue comments", run: func() (int, error) {
+ return s.processIssueComments(ctx, p, repoInfo, reporter, since)
+ }},
+ )
+ }
if s.includePRComments {
- ctx.Logger().V(2).Info("Scanning pull requests")
- if err := s.processPRs(ctx, apiClient, repoInfo, reporter); err != nil {
- return err
+ prStartLog := "Scanning pull requests"
+ if !s.includeIssueComments {
+ phases = append(phases, commentPhase{name: "pull requests", startLog: prStartLog, run: func() (int, error) {
+ return s.processPRs(ctx, p, repoInfo, reporter)
+ }})
+ prStartLog = ""
+ }
+ phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, run: func() (int, error) {
+ return s.processPRComments(ctx, p, repoInfo, reporter, since)
+ }})
+ }
+
+ var (
+ errs []error
+ failed []string
+ summary []any
+ )
+ for _, phase := range phases {
+ if err := ctx.Err(); err != nil {
+ errs = append(errs, err)
+ break
}
- if err := s.processPRComments(ctx, apiClient, repoInfo, reporter, cutoffTime); err != nil {
- return err
+ if phase.startLog != "" {
+ ctx.Logger().V(2).Info(phase.startLog)
+ }
+ emitted, err := phase.run()
+ summary = append(summary, strings.ReplaceAll(phase.name, " ", "_"), emitted)
+ if err != nil {
+ failed = append(failed, phase.name)
+ errs = append(errs, fmt.Errorf("%s: %w", phase.name, err))
}
}
+ ctx.Logger().V(2).Info("finished scanning comments", append(summary, "failed_phases", failed)...)
- return nil
+ return errors.Join(errs...)
}
func (s *Source) processRepoIssueandPRsWithCommentsGraphql(
@@ -2024,44 +2113,52 @@ func (s *Source) processRepoIssueandPRsWithCommentsGraphql(
return nil
}
-func (s *Source) processIssues(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter) error {
- bodyTextsOpts := &github.IssueListByRepoOptions{
- Sort: sortType,
- Direction: directionType,
- State: state,
- ListOptions: github.ListOptions{
- PerPage: defaultPagination,
- Page: initialPage,
+// processIssues walks /issues and emits issue bodies, plus pull request
+// bodies when includePRs is set. The listing always contains both, because
+// GitHub treats every pull request as an issue. Bodies are scanned in full
+// regardless of the comments timeframe.
+func (s *Source) processIssues(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, includePRs bool) (int, error) {
+ emitted := 0
+ _, err := walkUpdatedSince(ctx, p, sinceWalk[*github.Issue]{
+ name: "issues",
+ list: listIssuesSince(p, repoInfo),
+ key: issueKey,
+ visit: func(issues []*github.Issue) error {
+ n, err := s.chunkIssues(ctx, repoInfo, issues, reporter, func(issue *github.Issue) bool {
+ return includePRs || !issue.IsPullRequest()
+ })
+ emitted += n
+ return err
},
+ })
+ return emitted, err
+}
+
+// listIssuesSince returns the /issues listing, issues and pull requests
+// together, for walkUpdatedSince.
+func listIssuesSince(p pager, repoInfo repoInfo) func(context.Context, time.Time) ([]*github.Issue, *github.Response, error) {
+ return func(ctx context.Context, since time.Time) ([]*github.Issue, *github.Response, error) {
+ return p.client.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, &github.IssueListByRepoOptions{
+ State: stateAll,
+ Sort: sortByUpdated,
+ Direction: sortAscending,
+ Since: since,
+ ListOptions: github.ListOptions{PerPage: p.perPage},
+ })
}
+}
- for {
- issues, _, err := apiClient.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts)
- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {
- continue
- }
-
- if err != nil {
- return err
- }
-
- if err = s.chunkIssues(ctx, repoInfo, issues, reporter); err != nil {
- return err
- }
-
- bodyTextsOpts.Page++
-
- if len(issues) < defaultPagination {
- break
- }
- }
- return nil
+func issueKey(issue *github.Issue) (int64, time.Time) {
+ return issue.GetID(), issue.GetUpdatedAt().Time
}
-func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, reporter sources.ChunkReporter) error {
+// chunkIssues emits a chunk for each issue that keep accepts, and returns how
+// many it emitted. Pull requests from /issues produce the same chunk as
+// chunkPullRequests: same link, author, timestamp, and title and body.
+func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, reporter sources.ChunkReporter, keep func(*github.Issue) bool) (int, error) {
+ emitted := 0
for _, issue := range issues {
- // Skip pull requests since covered by processPRs.
- if issue.IsPullRequest() {
+ if !keep(issue) {
continue
}
@@ -2088,50 +2185,46 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
}
if err := reporter.ChunkOk(ctx, chunk); err != nil {
- return err
- }
- }
- return nil
-}
-
-func (s *Source) processIssueComments(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
- issueOpts := &github.IssueListCommentsOptions{
- Sort: &sortType,
- Direction: &directionType,
- ListOptions: github.ListOptions{
- PerPage: defaultPagination,
- Page: initialPage,
+ return emitted, err
+ }
+ emitted++
+ }
+ return emitted, nil
+}
+
+// processIssueComments walks every issue comment in the repo updated at or
+// after since. /issues/comments also returns comments on pull requests, since
+// GitHub treats every pull request as an issue. The endpoint stops at 300
+// pages, so the walk continues past that cap in since windows.
+func (s *Source) processIssueComments(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, since time.Time) (int, error) {
+ return walkUpdatedSince(ctx, p, sinceWalk[*github.IssueComment]{
+ name: "issue comments",
+ since: since,
+ list: func(ctx context.Context, since time.Time) ([]*github.IssueComment, *github.Response, error) {
+ opts := &github.IssueListCommentsOptions{
+ Sort: github.String(sortByUpdated),
+ Direction: github.String(sortAscending),
+ ListOptions: github.ListOptions{PerPage: p.perPage},
+ }
+ if !since.IsZero() {
+ opts.Since = &since
+ }
+ // Issue number 0 lists comments across the whole repo.
+ return p.client.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, 0, opts)
},
- }
-
- for {
- issueComments, _, err := apiClient.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts)
- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {
- continue
- }
- if err != nil {
- return err
- }
-
- if err = s.chunkIssueComments(ctx, repoInfo, issueComments, reporter, cutoffTime); err != nil {
- return err
- }
-
- issueOpts.Page++
- if len(issueComments) < defaultPagination {
- break
- }
- }
- return nil
+ key: func(comment *github.IssueComment) (int64, time.Time) {
+ return comment.GetID(), comment.GetUpdatedAt().Time
+ },
+ visit: func(comments []*github.IssueComment) error {
+ return s.chunkIssueComments(ctx, repoInfo, comments, reporter)
+ },
+ })
}
-func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+// chunkIssueComments emits a chunk for each comment. The comments timeframe
+// is already applied by the API through `since`.
+func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, reporter sources.ChunkReporter) error {
for _, comment := range comments {
- // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted
- if cutoffTime != nil && comment.GetUpdatedAt().Before(*cutoffTime) {
- continue
- }
-
// Create chunk and send it to the channel.
chunk := sources.Chunk{
SourceName: s.name,
@@ -2161,69 +2254,87 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
return nil
}
-func (s *Source) processPRs(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter) error {
- prOpts := &github.PullRequestListOptions{
- Sort: sortType,
- Direction: directionType,
- State: state,
- ListOptions: github.ListOptions{
- PerPage: defaultPagination,
- Page: initialPage,
+// processPRs emits pull request bodies for configs that enable PR comments
+// but not issue comments; otherwise the /issues walk emits them. /pulls has
+// no `since` filter, so if GitHub caps it, the walk falls back to a windowed
+// /issues walk that emits only pull requests, skipping the ones /pulls
+// already emitted. The common case never pages through issues.
+func (s *Source) processPRs(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter) (int, error) {
+ emitted := 0
+ emittedNumbers := map[int]struct{}{}
+ p.listing = "pull requests"
+ outcome, err := walkPages(ctx, p,
+ func(ctx context.Context) ([]*github.PullRequest, *github.Response, error) {
+ return p.client.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, &github.PullRequestListOptions{
+ State: stateAll,
+ Sort: sortByUpdated,
+ Direction: sortAscending,
+ ListOptions: github.ListOptions{PerPage: p.perPage},
+ })
},
- }
-
- for {
- prs, _, err := apiClient.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts)
- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {
- continue
- }
- if err != nil {
- return err
- }
-
- if err = s.chunkPullRequests(ctx, repoInfo, prs, reporter); err != nil {
- return err
- }
-
- prOpts.Page++
-
- if len(prs) < defaultPagination {
- break
- }
- }
- return nil
-}
-
-func (s *Source) processPRComments(ctx context.Context, apiClient *github.Client, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
- prOpts := &github.PullRequestListCommentsOptions{
- Sort: sortType,
- Direction: directionType,
- ListOptions: github.ListOptions{
- PerPage: defaultPagination,
- Page: initialPage,
+ func(prs []*github.PullRequest) error {
+ if err := s.chunkPullRequests(ctx, repoInfo, prs, reporter); err != nil {
+ return err
+ }
+ for _, pr := range prs {
+ emittedNumbers[pr.GetNumber()] = struct{}{}
+ }
+ emitted += len(prs)
+ return nil
},
+ )
+ if err != nil {
+ return emitted, &walkError{items: outcome.items, err: err}
+ }
+ if !outcome.capped {
+ return emitted, nil
}
- for {
- prComments, _, err := apiClient.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts)
- if s.handleRateLimitWithChunkReporter(ctx, reporter, err) {
- continue
- }
- if err != nil {
- return err
- }
-
- if err = s.chunkPullRequestComments(ctx, repoInfo, prComments, reporter, cutoffTime); err != nil {
+ ctx.Logger().V(2).Info("pull request listing capped, continuing from the issue listing", "pull_requests_so_far", emitted)
+ _, err = walkUpdatedSince(ctx, p, sinceWalk[*github.Issue]{
+ name: "pull requests from issues",
+ list: listIssuesSince(p, repoInfo),
+ key: issueKey,
+ visit: func(issues []*github.Issue) error {
+ n, err := s.chunkIssues(ctx, repoInfo, issues, reporter, func(issue *github.Issue) bool {
+ if !issue.IsPullRequest() {
+ return false
+ }
+ _, done := emittedNumbers[issue.GetNumber()]
+ return !done
+ })
+ emitted += n
return err
- }
-
- prOpts.Page++
-
- if len(prComments) < defaultPagination {
- break
- }
- }
- return nil
+ },
+ })
+ return emitted, err
+}
+
+// processPRComments walks every pull request review comment (the inline
+// comments on diffs) in the repo updated at or after since. Discussion
+// comments on pull requests come from /issues/comments instead. No cap has
+// been observed on this endpoint, but it gets the same since windows as
+// /issues/comments in case one exists.
+func (s *Source) processPRComments(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, since time.Time) (int, error) {
+ return walkUpdatedSince(ctx, p, sinceWalk[*github.PullRequestComment]{
+ name: "pull request comments",
+ since: since,
+ list: func(ctx context.Context, since time.Time) ([]*github.PullRequestComment, *github.Response, error) {
+ // Pull request number 0 lists review comments across the whole repo.
+ return p.client.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, 0, &github.PullRequestListCommentsOptions{
+ Sort: sortByUpdated,
+ Direction: sortAscending,
+ Since: since,
+ ListOptions: github.ListOptions{PerPage: p.perPage},
+ })
+ },
+ key: func(comment *github.PullRequestComment) (int64, time.Time) {
+ return comment.GetID(), comment.GetUpdatedAt().Time
+ },
+ visit: func(comments []*github.PullRequestComment) error {
+ return s.chunkPullRequestComments(ctx, repoInfo, comments, reporter)
+ },
+ })
}
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, reporter sources.ChunkReporter) error {
@@ -2257,13 +2368,10 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [
return nil
}
-func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+// chunkPullRequestComments emits a chunk for each review comment. The
+// comments timeframe is already applied by the API through `since`.
+func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, reporter sources.ChunkReporter) error {
for _, comment := range comments {
- // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted
- if cutoffTime != nil && comment.GetUpdatedAt().Before(*cutoffTime) {
- continue
- }
-
// Create chunk and send it to the channel.
chunk := sources.Chunk{
SourceName: s.name,
diff --git a/pkg/sources/github/pagination.go b/pkg/sources/github/pagination.go
new file mode 100644
index 000000000..d030e4b84
--- /dev/null
+++ b/pkg/sources/github/pagination.go
@@ -0,0 +1,453 @@
+package github
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/google/go-github/v67/github"
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+// GitHub list endpoints don't all paginate the same way, and the differences
+// are mostly undocumented:
+//
+// - /issues offers cursors (`after`) in its next links and returns a 422
+// ("Pagination with the page parameter is not supported") somewhere
+// between 1,000 and 10,000 items when driven by page number alone.
+// - /issues/comments offers only page numbers and stops at 300 pages: page
+// 300 comes back full with no next link, and page 301 returns a 422
+// ("pagination is limited for this resource").
+// - /pulls ignores `after` and copies it into its next link, so a client
+// that sends only a cursor fetches page 1 forever.
+//
+// The helpers in this file never build page or cursor parameters themselves.
+// They follow the Link header's rel="next" URL as-is, which uses cursors
+// wherever GitHub provides them and page numbers everywhere else, and they
+// treat the ways a walk can end as distinct outcomes rather than assuming a
+// short page means "done". Every walk either reaches a confirmed end or
+// reports why it couldn't, so a cap can never silently truncate a scan.
+
+// pager carries what every walk needs: the client to send follow-up requests
+// with, the page size the walk was started with (used to recognize a full
+// final page), the caller's rate-limit handler, which sleeps and returns true
+// when a request should be retried, and wait, which sleeps between retries of
+// transient failures and returns false if the context ended first. A nil wait
+// turns transient retries off. listing names the walk in log lines, for
+// example "issue comments"; pager is passed by value, so each walk sets its
+// own without affecting the caller's.
+type pager struct {
+ client *github.Client
+ perPage int
+ retry func(error) bool
+ wait func(context.Context, time.Duration) bool
+ listing string
+}
+
+// walkOutcome describes how a page walk ended. A walk that isn't capped
+// reached a confirmed end: a short final page, or an empty probe page.
+type walkOutcome struct {
+ capped bool
+ items int
+}
+
+// walkError reports a walk that could not finish, with enough context for a
+// support engineer to see how far it got. since is zero for walks that don't
+// use since windows.
+type walkError struct {
+ items int
+ since time.Time
+ err error
+}
+
+func (e *walkError) Error() string {
+ if e.since.IsZero() {
+ return fmt.Sprintf("walk incomplete after %d items: %v", e.items, e.err)
+ }
+ return fmt.Sprintf("walk incomplete after %d items (since=%s): %v", e.items, e.since.UTC().Format(time.RFC3339), e.err)
+}
+
+func (e *walkError) Unwrap() error { return e.err }
+
+// walkPages fetches the first page with first (a typed go-github call that
+// sets the filters and page size), then follows rel="next" links until the
+// walk ends. visit receives each page before the next one is requested, so a
+// later failure can never discard pages that were already read.
+//
+// When there is no next link, the walk ends one of three ways:
+// - The final page is short: the walk is complete.
+// - The final page is full and its URL carries a cursor (`after` or
+// `before`): the walk is capped. GitHub goes by the cursor, so asking for
+// the next page number would just return the same page again.
+// - The final page is full and its URL uses page numbers (or none, for the
+// first page): the next page number is probed. An empty page means the
+// results exactly filled the last page, data means the walk continues,
+// and a pagination 422 means the endpoint is capped.
+//
+// A pagination 422 at any point is reported as capped rather than as an
+// error so callers can decide whether they have a way past it.
+func walkPages[T any](
+ ctx context.Context,
+ p pager,
+ first func(context.Context) ([]T, *github.Response, error),
+ visit func([]T) error,
+) (walkOutcome, error) {
+ var out walkOutcome
+
+ items, resp, err := fetchWithRetry(ctx, p, func() ([]T, *github.Response, error) { return first(ctx) })
+ if err != nil {
+ if isPaginationCap(err) {
+ out.capped = true
+ return out, nil
+ }
+ return out, err
+ }
+
+ current := requestURL(resp)
+ fetched := map[string]struct{}{}
+ if current != nil {
+ fetched[current.String()] = struct{}{}
+ }
+
+ for {
+ if len(items) > 0 {
+ if err := visit(items); err != nil {
+ return out, err
+ }
+ out.items += len(items)
+ }
+
+ next := nextLink(resp)
+ if next == "" {
+ if len(items) < p.perPage {
+ return out, nil
+ }
+ if current == nil || hasCursor(current) {
+ out.capped = true
+ return out, nil
+ }
+ next = withNextPage(current)
+ }
+
+ nextURL, err := p.followable(next)
+ if err != nil {
+ return out, err
+ }
+ if _, ok := fetched[nextURL.String()]; ok {
+ return out, fmt.Errorf("pagination did not advance: next link repeats %s", redactQuery(nextURL))
+ }
+ fetched[nextURL.String()] = struct{}{}
+
+ items, resp, err = fetchWithRetry(ctx, p, func() ([]T, *github.Response, error) {
+ return fetchPage[T](ctx, p.client, nextURL.String())
+ })
+ if err != nil {
+ if isPaginationCap(err) {
+ out.capped = true
+ return out, nil
+ }
+ return out, err
+ }
+ current = nextURL
+ }
+}
+
+// sinceWalk describes a walk over an endpoint that accepts `since` and can
+// sort by update time: /issues, /issues/comments, and /pulls/comments.
+type sinceWalk[T any] struct {
+ // name labels log lines, for example "issue comments", and becomes the
+ // pager's listing for the walk.
+ name string
+ // since is the earliest update time to fetch. Zero fetches everything.
+ since time.Time
+ // list fetches the first page of a window starting at since. It must sort
+ // by update time ascending, and must omit `since` when it is zero.
+ list func(ctx context.Context, since time.Time) ([]T, *github.Response, error)
+ // key returns an item's ID and last update time.
+ key func(T) (int64, time.Time)
+ // visit receives each page of items not already seen.
+ visit func([]T) error
+}
+
+// walkUpdatedSince walks an endpoint in update-time order, and gets past
+// GitHub's per-query result caps by starting a new window at the last update
+// time seen whenever a window ends capped. `since` is inclusive and each
+// window gets a fresh cap, so the walk continues where the previous window
+// stopped. Items at the boundary timestamp that were already visited are
+// dropped by ID.
+//
+// Ascending order also keeps the walk complete while the repo changes under
+// it: an item updated mid-walk moves to the end, where the walk still reaches
+// it, instead of shifting earlier pages.
+//
+// A window that ends complete finishes the walk, even if it produced nothing
+// new; that is what an exact fit on a cursor endpoint looks like. A window
+// that ends capped without producing anything new is an error, because more
+// items share one timestamp than a window can hold and no later window can
+// get past them.
+func walkUpdatedSince[T any](ctx context.Context, p pager, w sinceWalk[T]) (int, error) {
+ p.listing = w.name
+ windowStart := w.since
+ var (
+ total int
+ // latest is the newest update time visited, and atLatest holds the IDs
+ // visited at exactly that time. When a window starts at latest, the
+ // API returns those items again, and atLatest is how they are dropped.
+ latest time.Time
+ atLatest = map[int64]struct{}{}
+ )
+
+ for {
+ newInWindow := 0
+ outcome, err := walkPages(ctx, p,
+ func(ctx context.Context) ([]T, *github.Response, error) { return w.list(ctx, windowStart) },
+ func(items []T) error {
+ fresh := make([]T, 0, len(items))
+ for _, item := range items {
+ id, updated := w.key(item)
+ if updated.Equal(windowStart) {
+ if _, seen := atLatest[id]; seen {
+ continue
+ }
+ }
+ fresh = append(fresh, item)
+ switch {
+ case updated.After(latest):
+ latest = updated
+ atLatest = map[int64]struct{}{id: {}}
+ case updated.Equal(latest):
+ atLatest[id] = struct{}{}
+ }
+ }
+ if len(fresh) == 0 {
+ return nil
+ }
+ newInWindow += len(fresh)
+ total += len(fresh)
+ return w.visit(fresh)
+ },
+ )
+ if err != nil {
+ return total, &walkError{items: total, since: windowStart, err: err}
+ }
+ if !outcome.capped {
+ return total, nil
+ }
+ if newInWindow == 0 {
+ return total, &walkError{
+ items: total,
+ since: windowStart,
+ err: fmt.Errorf("listing is capped and a new window made no progress: more items share update time %s than one window can return", windowStart.UTC().Format(time.RFC3339)),
+ }
+ }
+
+ windowStart = latest
+ ctx.Logger().V(2).Info("listing capped, continuing from last update time",
+ "listing", w.name, "since", windowStart.UTC().Format(time.RFC3339), "items_so_far", total)
+ }
+}
+
+// maxTransientRetries bounds how many times one page request is retried after
+// a transient failure. With the backoff below, a request gets about a minute
+// of retries before the walk reports it as an error.
+const maxTransientRetries = 5
+
+// transientBackoff returns how long to wait before the given retry, starting
+// at 2 seconds, doubling, and capped at 30 seconds.
+func transientBackoff(retry int) time.Duration {
+ return min(2*time.Second<<retry, 30*time.Second)
+}
+
+// fetchWithRetry runs fetch, retrying in two independent cases. Rate limits
+// are retried for as long as the rate-limit handler says to; it sleeps until
+// the limit resets and returns false once the context is cancelled. Transient
+// failures (see isTransient) are retried up to maxTransientRetries times with
+// backoff. Every request here is a GET, so a retry can't repeat a side effect.
+//
+// Long walks make transient failures likely: a full walk of a large repo's
+// review comments takes hours at several seconds per page, and GitHub
+// occasionally cancels a slow request outright.
+//
+// The connectors' HTTP clients (common.RetryableHTTPClientTimeout) already
+// retry a failed round trip a few times with short backoff, so this is a
+// second layer, not a duplicate. It covers what that layer can't: a response
+// whose body fails partway through reading (an HTTP/2 stream reset arrives
+// here, from the JSON decoder, after the round trip has succeeded), and a
+// failure that outlasts the client's own retries, which surfaces as
+// "giving up after N attempt(s)". Each attempt here is bounded by the
+// client's 60 second timeout, so a page that keeps failing is reported after
+// several minutes rather than retried indefinitely.
+func fetchWithRetry[T any](ctx context.Context, p pager, fetch func() ([]T, *github.Response, error)) ([]T, *github.Response, error) {
+ transientRetries := 0
+ for {
+ items, resp, err := fetch()
+ if p.retry != nil && p.retry(err) {
+ continue
+ }
+ if err == nil || p.wait == nil || !isTransient(ctx, err) {
+ return items, resp, err
+ }
+ if transientRetries == maxTransientRetries {
+ return items, resp, fmt.Errorf("giving up after %d attempts: %w", transientRetries+1, err)
+ }
+ delay := transientBackoff(transientRetries)
+ transientRetries++
+ ctx.Logger().V(2).Info("retrying GitHub API request after transient error",
+ "listing", p.listing, "retry", transientRetries, "max_retries", maxTransientRetries, "retry_in", delay.String(), "error", err.Error())
+ if !p.wait(ctx, delay) {
+ return items, resp, err
+ }
+ }
+}
+
+// isTransient reports whether a failed request is worth retrying as-is:
+// a 5xx from GitHub, or a transport failure such as a reset connection, a
+// timeout, or a response cut off partway. Anything caused by the context
+// ending is not transient, and neither is any other API error; rate limits
+// have their own handler, and a 4xx (including the pagination 422s) will fail
+// the same way again.
+func isTransient(ctx context.Context, err error) bool {
+ if ctx.Err() != nil {
+ return false
+ }
+
+ var (
+ errResp *github.ErrorResponse
+ rateLimit *github.RateLimitError
+ abuseLimit *github.AbuseRateLimitError
+ )
+ switch {
+ case errors.As(err, &rateLimit), errors.As(err, &abuseLimit):
+ return false
+ case errors.As(err, &errResp):
+ return errResp.Response != nil && errResp.Response.StatusCode >= http.StatusInternalServerError
+ }
+
+ var (
+ urlErr *url.Error
+ netErr net.Error
+ )
+ if errors.As(err, &urlErr) || errors.As(err, &netErr) ||
+ errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) {
+ return true
+ }
+ // A stream reset that arrives while the body is being read surfaces from
+ // the JSON decoder as net/http's bundled HTTP/2 stream error. That type
+ // is unexported, so its message is the only way to recognize it.
+ return strings.Contains(err.Error(), "stream error:")
+}
+
+// fetchPage sends a GET for an absolute next-page URL through the go-github
+// client, so authentication, rate-limit errors, and error decoding behave
+// exactly as they do for the typed list methods.
+func fetchPage[T any](ctx context.Context, client *github.Client, rawURL string) ([]T, *github.Response, error) {
+ req, err := client.NewRequest(http.MethodGet, rawURL, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ var page []T
+ resp, err := client.Do(ctx, req, &page)
+ return page, resp, err
+}
+
+// followable parses a next-page URL and refuses one that points at a host
+// other than the API's. The connectors attach credentials in the HTTP
+// transport, which would send the token to whatever host the URL names.
+func (p pager) followable(raw string) (*url.URL, error) {
+ u, err := url.Parse(raw)
+ if err != nil {
+ return nil, fmt.Errorf("invalid pagination link: %w", err)
+ }
+ if !u.IsAbs() {
+ u = p.client.BaseURL.ResolveReference(u)
+ }
+ if !strings.EqualFold(u.Host, p.client.BaseURL.Host) {
+ return nil, fmt.Errorf("pagination link points at %q, not the API host %q", u.Host, p.client.BaseURL.Host)
+ }
+ return u, nil
+}
+
+// isPaginationCap reports whether err is one of GitHub's pagination 422s:
+// the /issues offset limit ("Pagination with the page parameter is not
+// supported for large datasets") or the /issues/comments page cap
+// ("pagination is limited for this resource"). Any other 422 is a real
+// error.
+func isPaginationCap(err error) bool {
+ var errResp *github.ErrorResponse
+ if !errors.As(err, &errResp) || errResp.Response == nil {
+ return false
+ }
+ return errResp.Response.StatusCode == http.StatusUnprocessableEntity &&
+ strings.Contains(strings.ToLower(errResp.Message), "pagination")
+}
+
+// linkPattern matches one `<url>; rel="..."` entry of a Link header. Matching
+// the angle brackets, instead of splitting on commas, keeps URLs that
+// contain commas intact.
+var linkPattern = regexp.MustCompile(`<([^>]*)>\s*;\s*rel="([^"]*)"`)
+
+// nextLink returns the rel="next" URL from a response's Link headers, or ""
+// when there is none.
+func nextLink(resp *github.Response) string {
+ if resp == nil || resp.Response == nil {
+ return ""
+ }
+ for _, header := range resp.Header.Values("Link") {
+ for _, match := range linkPattern.FindAllStringSubmatch(header, -1) {
+ for _, rel := range strings.Fields(match[2]) {
+ if rel == "next" {
+ return match[1]
+ }
+ }
+ }
+ }
+ return ""
+}
+
+// requestURL returns the URL a response was fetched from, or nil when the
+// response doesn't record it.
+func requestURL(resp *github.Response) *url.URL {
+ if resp == nil || resp.Response == nil || resp.Request == nil || resp.Request.URL == nil {
+ return nil
+ }
+ u := *resp.Request.URL
+ return &u
+}
+
+// hasCursor reports whether a page URL is positioned by cursor. GitHub's
+// /issues links carry both `page` and `after`, and the cursor is what counts.
+func hasCursor(u *url.URL) bool {
+ q := u.Query()
+ return q.Get("after") != "" || q.Get("before") != ""
+}
+
+// withNextPage returns u with its page number incremented. A URL without a
+// page number is page 1.
+func withNextPage(u *url.URL) string {
+ q := u.Query()
+ page, err := strconv.Atoi(q.Get("page"))
+ if err != nil || page < 1 {
+ page = 1
+ }
+ q.Set("page", strconv.Itoa(page+1))
+ next := *u
+ next.RawQuery = q.Encode()
+ return next.String()
+}
+
+// redactQuery drops the query string from a URL for error messages, since
+// cursors are long and opaque and add nothing for a reader.
+func redactQuery(u *url.URL) string {
+ redacted := *u
+ redacted.RawQuery = ""
+ return redacted.String()
+}
diff --git a/pkg/sources/github/pagination_test.go b/pkg/sources/github/pagination_test.go
new file mode 100644
index 000000000..5cc143224
--- /dev/null
+++ b/pkg/sources/github/pagination_test.go
@@ -0,0 +1,766 @@
+package github
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/google/go-github/v67/github"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+// fakeListing serves one GitHub list endpoint with the paging behaviors the
+// helpers have to cope with: `since` filtering (inclusive), page numbers or
+// cursors in next links, and a page cap that ends with a full page and no
+// next link, followed by a pagination 422.
+type fakeListing struct {
+ mu sync.Mutex
+ // records must be sorted by updated, ascending, as the real endpoints
+ // return them for sort=updated&direction=asc.
+ records []fakeRecord
+ // cursor adds `after` to next links, and positions requests by it,
+ // the way /issues does.
+ cursor bool
+ // maxPages caps the listing: the last allowed page has no next link and
+ // any later page returns a pagination 422. Zero means no cap.
+ maxPages int
+ // linkPastCap keeps the next link on the last allowed page, so the 422
+ // arrives mid-walk instead of on a probe.
+ linkPastCap bool
+ // omitNext drops the next link from the given pages even when more
+ // records remain.
+ omitNext map[int]bool
+ // fail, when set, can replace any response. It receives the 1-based
+ // request count and the request, and returns true if it wrote a response.
+ fail func(n int, w http.ResponseWriter, r *http.Request) bool
+
+ requests []url.Values
+}
+
+type fakeRecord struct {
+ updated time.Time
+ body map[string]any
+}
+
+const paginationCapMessage = "In order to keep the API fast for everyone, pagination is limited for this resource."
+
+func (f *fakeListing) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+
+ q := r.URL.Query()
+ f.requests = append(f.requests, q)
+ if f.fail != nil && f.fail(len(f.requests), w, r) {
+ return
+ }
+
+ records := f.records
+ if s := q.Get("since"); s != "" {
+ since, err := time.Parse(time.RFC3339, s)
+ if err != nil {
+ writeJSON(w, http.StatusBadRequest, map[string]string{"message": "bad since"})
+ return
+ }
+ filtered := make([]fakeRecord, 0, len(records))
+ for _, rec := range records {
+ if !rec.updated.Before(since) {
+ filtered = append(filtered, rec)
+ }
+ }
+ records = filtered
+ }
+
+ perPage := 30
+ if v, _ := strconv.Atoi(q.Get("per_page")); v > 0 {
+ perPage = v
+ }
+ page := 1
+ if v, _ := strconv.Atoi(q.Get("page")); v > 0 {
+ page = v
+ }
+ if after := q.Get("after"); f.cursor && after != "" {
+ page, _ = strconv.Atoi(strings.TrimPrefix(after, "cursor-"))
+ }
+ if f.maxPages > 0 && page > f.maxPages {
+ writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"message": paginationCapMessage})
+ return
+ }
+
+ start := min((page-1)*perPage, len(records))
+ end := min(start+perPage, len(records))
+ atCap := f.maxPages > 0 && page == f.maxPages
+ if end < len(records) && !f.omitNext[page] && (!atCap || f.linkPastCap) {
+ next := url.Values{}
+ for k, v := range q {
+ next[k] = v
+ }
+ next.Set("page", strconv.Itoa(page+1))
+ if f.cursor {
+ next.Set("after", fmt.Sprintf("cursor-%d", page+1))
+ }
+ w.Header().Set("Link", fmt.Sprintf(`<http://%s%s?%s>; rel="next", <http://%s%s?page=1>; rel="first"`,
+ r.Host, r.URL.Path, next.Encode(), r.Host, r.URL.Path))
+ }
+
+ body := make([]map[string]any, 0, end-start)
+ for _, rec := range records[start:end] {
+ body = append(body, rec.body)
+ }
+ writeJSON(w, http.StatusOK, body)
+}
+
+func (f *fakeListing) requestCount() int {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return len(f.requests)
+}
+
+func (f *fakeListing) requestedPages() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ pages := make([]string, 0, len(f.requests))
+ for _, q := range f.requests {
+ page := q.Get("page")
+ if page == "" {
+ page = "1"
+ }
+ pages = append(pages, page)
+ }
+ return pages
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _ = json.NewEncoder(w).Encode(v)
+}
+
+// testItem is the shape the helper tests decode pages into.
+type testItem struct {
+ ID int64 `json:"id"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
+var testEpoch = time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+
+// sequentialRecords returns n records with IDs 1..n updated one minute apart.
+func sequentialRecords(n int) []fakeRecord {
+ updated := make([]time.Time, n)
+ for i := range updated {
+ updated[i] = testEpoch.Add(time.Duration(i) * time.Minute)
+ }
+ return recordsAt(updated...)
+}
+
+// recordsAt returns one record per timestamp, with IDs 1..n in order.
+func recordsAt(updated ...time.Time) []fakeRecord {
+ recs := make([]fakeRecord, len(updated))
+ for i, t := range updated {
+ recs[i] = fakeRecord{updated: t, body: map[string]any{"id": i + 1, "updated_at": t}}
+ }
+ return recs
+}
+
+// newTestPager serves handler at /items and returns a pager for it with a
+// page size of 3 and no retries.
+func newTestPager(t *testing.T, handler http.Handler) pager {
+ t.Helper()
+ mux := http.NewServeMux()
+ mux.Handle("/items", handler)
+ srv := httptest.NewServer(mux)
+ t.Cleanup(srv.Close)
+ return pager{client: newTestClient(t, srv), perPage: 3}
+}
+
+func newTestClient(t *testing.T, srv *httptest.Server) *github.Client {
+ t.Helper()
+ client := github.NewClient(srv.Client())
+ base, err := url.Parse(srv.URL + "/")
+ require.NoError(t, err)
+ client.BaseURL = base
+ return client
+}
+
+// firstItems fetches the first page of /items the way a typed go-github list
+// call would, with the pager's page size.
+func firstItems(p pager) func(context.Context) ([]testItem, *github.Response, error) {
+ return func(ctx context.Context) ([]testItem, *github.Response, error) {
+ return fetchPage[testItem](ctx, p.client, fmt.Sprintf("items?per_page=%d", p.perPage))
+ }
+}
+
+// collect returns a visit func that records item IDs in order.
+func collect(ids *[]int64) func([]testItem) error {
+ return func(items []testItem) error {
+ for _, item := range items {
+ *ids = append(*ids, item.ID)
+ }
+ return nil
+ }
+}
+
+func idsUpTo(n int) []int64 {
+ ids := make([]int64, n)
+ for i := range ids {
+ ids[i] = int64(i + 1)
+ }
+ return ids
+}
+
+func TestWalkPages_FollowsNextLinksToShortFinalPage(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(7)}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.False(t, outcome.capped)
+ assert.Equal(t, 7, outcome.items)
+ assert.Equal(t, idsUpTo(7), ids)
+ assert.Equal(t, []string{"1", "2", "3"}, listing.requestedPages())
+}
+
+func TestWalkPages_FullFinalPageWithEmptyProbeIsComplete(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(6)}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.False(t, outcome.capped)
+ assert.Equal(t, idsUpTo(6), ids)
+ assert.Equal(t, []string{"1", "2", "3"}, listing.requestedPages(), "page 3 is the probe")
+}
+
+func TestWalkPages_ProbeThatFindsDataKeepsWalking(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(8), omitNext: map[int]bool{1: true}}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.False(t, outcome.capped)
+ assert.Equal(t, idsUpTo(8), ids)
+}
+
+func TestWalkPages_ProbeThatHitsPaginationLimitIsCapped(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(9), maxPages: 2}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.True(t, outcome.capped)
+ assert.Equal(t, 6, outcome.items)
+ assert.Equal(t, idsUpTo(6), ids)
+ assert.Equal(t, []string{"1", "2", "3"}, listing.requestedPages())
+}
+
+func TestWalkPages_PaginationLimitMidWalkIsCapped(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(9), maxPages: 2, linkPastCap: true}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.True(t, outcome.capped)
+ assert.Equal(t, idsUpTo(6), ids)
+}
+
+func TestWalkPages_PaginationLimitOnFirstPageIsCapped(t *testing.T) {
+ listing := &fakeListing{fail: func(_ int, w http.ResponseWriter, _ *http.Request) bool {
+ writeJSON(w, http.StatusUnprocessableEntity, map[string]string{
+ "message": "Pagination with the page parameter is not supported for large datasets, please use cursor based pagination (after/before)",
+ })
+ return true
+ }}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.True(t, outcome.capped)
+ assert.Empty(t, ids)
+}
+
+func TestWalkPages_FullFinalCursorPageIsCappedWithoutProbe(t *testing.T) {
+ // /issues next links carry both page and after; probing page+1 would be
+ // ignored in favor of the cursor and return the same page again.
+ listing := &fakeListing{records: sequentialRecords(9), cursor: true, maxPages: 2}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.True(t, outcome.capped)
+ assert.Equal(t, idsUpTo(6), ids)
+ assert.Equal(t, 2, listing.requestCount(), "no probe after a cursor page")
+}
+
+func TestWalkPages_FollowsCursorLinksVerbatim(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(8), cursor: true}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.False(t, outcome.capped)
+ assert.Equal(t, idsUpTo(8), ids)
+ require.Len(t, listing.requests, 3)
+ assert.Equal(t, "cursor-3", listing.requests[2].Get("after"))
+}
+
+func TestWalkPages_NonPaginationUnprocessableEntityIsAnError(t *testing.T) {
+ listing := &fakeListing{
+ records: sequentialRecords(9),
+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {
+ if n != 2 {
+ return false
+ }
+ writeJSON(w, http.StatusUnprocessableEntity, map[string]string{"message": "Validation Failed"})
+ return true
+ },
+ }
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "Validation Failed")
+ assert.Equal(t, 3, outcome.items)
+ assert.Equal(t, idsUpTo(3), ids, "pages read before the error are kept")
+}
+
+func TestWalkPages_RejectsNextLinkToAnotherHost(t *testing.T) {
+ listing := &fakeListing{fail: func(_ int, w http.ResponseWriter, _ *http.Request) bool {
+ w.Header().Set("Link", `<https://attacker.example/items?page=2>; rel="next"`)
+ writeJSON(w, http.StatusOK, []map[string]any{{"id": 1}, {"id": 2}, {"id": 3}})
+ return true
+ }}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "attacker.example")
+ assert.Equal(t, idsUpTo(3), ids)
+ assert.Equal(t, 1, listing.requestCount())
+}
+
+func TestWalkPages_ErrorsWhenNextLinkRepeats(t *testing.T) {
+ // /pulls echoes an `after` it ignores, so a cursor-only next link would
+ // fetch the same page forever.
+ listing := &fakeListing{fail: func(_ int, w http.ResponseWriter, r *http.Request) bool {
+ w.Header().Set("Link", fmt.Sprintf(`<http://%s/items?per_page=3&after=abc>; rel="next"`, r.Host))
+ writeJSON(w, http.StatusOK, []map[string]any{{"id": 1}, {"id": 2}, {"id": 3}})
+ return true
+ }}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "did not advance")
+ assert.Equal(t, 2, listing.requestCount())
+}
+
+func TestWalkPages_RetriesWhenRateLimitHandlerSaysTo(t *testing.T) {
+ listing := &fakeListing{
+ records: sequentialRecords(7),
+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {
+ if n != 2 {
+ return false
+ }
+ writeJSON(w, http.StatusForbidden, map[string]string{"message": "slow down"})
+ return true
+ },
+ }
+ p := newTestPager(t, listing)
+ retries := 0
+ p.retry = func(err error) bool {
+ if err == nil || retries > 0 {
+ return false
+ }
+ retries++
+ return true
+ }
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.False(t, outcome.capped)
+ assert.Equal(t, 1, retries)
+ assert.Equal(t, idsUpTo(7), ids)
+ assert.Equal(t, []string{"1", "2", "2", "3"}, listing.requestedPages())
+}
+
+// recordWaits turns on transient retries for p and records each requested
+// delay instead of sleeping.
+func recordWaits(p *pager) *[]time.Duration {
+ var waits []time.Duration
+ p.wait = func(_ context.Context, d time.Duration) bool {
+ waits = append(waits, d)
+ return true
+ }
+ return &waits
+}
+
+// failRequest returns a fail func that writes status for the given request
+// numbers.
+func failRequest(status int, requests ...int) func(int, http.ResponseWriter, *http.Request) bool {
+ return func(n int, w http.ResponseWriter, _ *http.Request) bool {
+ if !slices.Contains(requests, n) {
+ return false
+ }
+ writeJSON(w, status, map[string]string{"message": http.StatusText(status)})
+ return true
+ }
+}
+
+func TestWalkPages_RetriesTransientServerErrors(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusBadGateway, 2, 3)}
+ p := newTestPager(t, listing)
+ waits := recordWaits(&p)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.False(t, outcome.capped)
+ assert.Equal(t, idsUpTo(7), ids)
+ assert.Equal(t, []time.Duration{2 * time.Second, 4 * time.Second}, *waits)
+ assert.Equal(t, []string{"1", "2", "2", "2", "3"}, listing.requestedPages())
+}
+
+func TestWalkPages_RetriesResponseCutOffPartway(t *testing.T) {
+ listing := &fakeListing{
+ records: sequentialRecords(7),
+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {
+ if n != 2 {
+ return false
+ }
+ // Promise a longer body than is sent, then drop the connection,
+ // the way a stream reset mid-response looks to the client.
+ conn, buf, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ panic(err)
+ }
+ _, _ = buf.WriteString("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000\r\n\r\n[{\"id\":4")
+ _ = buf.Flush()
+ _ = conn.Close()
+ return true
+ },
+ }
+ p := newTestPager(t, listing)
+ waits := recordWaits(&p)
+
+ var ids []int64
+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.NoError(t, err)
+ assert.Equal(t, idsUpTo(7), ids)
+ assert.Len(t, *waits, 1)
+}
+
+func TestWalkPages_GivesUpAfterBoundedTransientRetries(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusServiceUnavailable, 2, 3, 4, 5, 6, 7)}
+ p := newTestPager(t, listing)
+ waits := recordWaits(&p)
+
+ var ids []int64
+ outcome, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "giving up after 6 attempts")
+ assert.Contains(t, err.Error(), "503")
+ assert.Equal(t, []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, 30 * time.Second}, *waits)
+ assert.Equal(t, 3, outcome.items)
+ assert.Equal(t, idsUpTo(3), ids, "pages read before the failure are kept")
+}
+
+func TestWalkPages_DoesNotRetryClientErrors(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusNotFound, 2)}
+ p := newTestPager(t, listing)
+ waits := recordWaits(&p)
+
+ var ids []int64
+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.Error(t, err)
+ assert.NotContains(t, err.Error(), "giving up")
+ assert.Empty(t, *waits)
+ assert.Equal(t, 2, listing.requestCount())
+}
+
+func TestWalkPages_StopsRetryingWhenWaitIsInterrupted(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(7), fail: failRequest(http.StatusBadGateway, 2, 3)}
+ p := newTestPager(t, listing)
+ p.wait = func(context.Context, time.Duration) bool { return false }
+
+ var ids []int64
+ _, err := walkPages(context.Background(), p, firstItems(p), collect(&ids))
+
+ require.Error(t, err)
+ assert.Equal(t, 2, listing.requestCount())
+}
+
+func TestIsTransient(t *testing.T) {
+ apiErr := func(status int) error {
+ return &github.ErrorResponse{Response: &http.Response{StatusCode: status}, Message: http.StatusText(status)}
+ }
+ cancelled, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ tests := []struct {
+ name string
+ ctx context.Context
+ err error
+ want bool
+ }{
+ {name: "server error", err: apiErr(http.StatusBadGateway), want: true},
+ {name: "client error", err: apiErr(http.StatusNotFound)},
+ {name: "pagination cap", err: apiErr(http.StatusUnprocessableEntity)},
+ {name: "primary rate limit", err: &github.RateLimitError{Response: &http.Response{StatusCode: http.StatusForbidden}}},
+ {name: "secondary rate limit", err: &github.AbuseRateLimitError{Response: &http.Response{StatusCode: http.StatusForbidden}}},
+ {name: "transport failure", err: &url.Error{Op: "Get", URL: "https://api.github.com/x", Err: errors.New("connection reset by peer")}, want: true},
+ {name: "body cut off", err: fmt.Errorf("decoding: %w", io.ErrUnexpectedEOF), want: true},
+ {name: "http2 stream reset", err: errors.New("stream error: stream ID 261; CANCEL; received from peer"), want: true},
+ {name: "context ended", ctx: cancelled, err: apiErr(http.StatusBadGateway)},
+ {name: "other error", err: errors.New("invalid pagination link")},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ ctx := tt.ctx
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ assert.Equal(t, tt.want, isTransient(ctx, tt.err))
+ })
+ }
+}
+
+func TestWalkPages_StopsWhenVisitFails(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(9)}
+ p := newTestPager(t, listing)
+ visitErr := errors.New("reporter closed")
+
+ _, err := walkPages(context.Background(), p, firstItems(p), func([]testItem) error { return visitErr })
+
+ require.ErrorIs(t, err, visitErr)
+ assert.Equal(t, 1, listing.requestCount())
+}
+
+// sinceItems lists /items through walkUpdatedSince the way the comment
+// phases do: sorted by update time, `since` only when set.
+func sinceItems(p pager, ids *[]int64) sinceWalk[testItem] {
+ return sinceWalk[testItem]{
+ name: "items",
+ list: func(ctx context.Context, since time.Time) ([]testItem, *github.Response, error) {
+ u := fmt.Sprintf("items?per_page=%d&sort=updated&direction=asc", p.perPage)
+ if !since.IsZero() {
+ u += "&since=" + url.QueryEscape(since.UTC().Format(time.RFC3339))
+ }
+ return fetchPage[testItem](ctx, p.client, u)
+ },
+ key: func(item testItem) (int64, time.Time) { return item.ID, item.UpdatedAt },
+ visit: collect(ids),
+ }
+}
+
+func TestWalkUpdatedSince_ContinuesPastCapInNewWindows(t *testing.T) {
+ listing := &fakeListing{records: sequentialRecords(20), maxPages: 2}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))
+
+ require.NoError(t, err)
+ assert.Equal(t, 20, total)
+ assert.Equal(t, idsUpTo(20), ids, "every item exactly once, in order")
+}
+
+func TestWalkUpdatedSince_DropsItemsAlreadySeenAtWindowBoundary(t *testing.T) {
+ // Items 5-8 share a timestamp that straddles the first window's cap, so
+ // the second window starts at that timestamp and gets items 5 and 6
+ // again before the new ones.
+ shared := testEpoch.Add(time.Hour)
+ listing := &fakeListing{
+ records: recordsAt(
+ testEpoch, testEpoch.Add(time.Minute), testEpoch.Add(2*time.Minute), testEpoch.Add(3*time.Minute),
+ shared, shared, shared, shared,
+ shared.Add(time.Minute), shared.Add(2*time.Minute),
+ ),
+ maxPages: 2,
+ }
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))
+
+ require.NoError(t, err)
+ assert.Equal(t, 10, total)
+ assert.Equal(t, idsUpTo(10), ids)
+}
+
+func TestWalkUpdatedSince_CompleteWindowOfDuplicatesFinishes(t *testing.T) {
+ // The first window exactly fills its cap on a cursor endpoint, so it ends
+ // capped without a probe. The next window holds only the last item seen,
+ // which is the confirmed end of the listing.
+ listing := &fakeListing{records: sequentialRecords(6), cursor: true, maxPages: 2}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))
+
+ require.NoError(t, err)
+ assert.Equal(t, 6, total)
+ assert.Equal(t, idsUpTo(6), ids)
+}
+
+func TestWalkUpdatedSince_CappedWindowWithoutProgressIsAnError(t *testing.T) {
+ shared := testEpoch.Add(time.Hour)
+ listing := &fakeListing{records: recordsAt(shared, shared, shared, shared, shared, shared, shared, shared), maxPages: 2}
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ total, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))
+
+ require.Error(t, err)
+ var walkErr *walkError
+ require.ErrorAs(t, err, &walkErr)
+ assert.Equal(t, shared, walkErr.since)
+ assert.Equal(t, 6, walkErr.items)
+ assert.Contains(t, err.Error(), "walk incomplete after 6 items (since=2024-01-01T01:00:00Z)")
+ assert.Equal(t, 6, total)
+ assert.Equal(t, idsUpTo(6), ids, "items read before the error are kept")
+}
+
+func TestWalkUpdatedSince_ReportsWindowStartOnError(t *testing.T) {
+ listing := &fakeListing{
+ records: sequentialRecords(20),
+ maxPages: 2,
+ fail: func(n int, w http.ResponseWriter, _ *http.Request) bool {
+ if n != 4 {
+ return false
+ }
+ writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "boom"})
+ return true
+ },
+ }
+ p := newTestPager(t, listing)
+
+ var ids []int64
+ _, err := walkUpdatedSince(context.Background(), p, sinceItems(p, &ids))
+
+ require.Error(t, err)
+ var walkErr *walkError
+ require.ErrorAs(t, err, &walkErr)
+ assert.Equal(t, testEpoch.Add(5*time.Minute), walkErr.since, "second window starts at item 6")
+ assert.Equal(t, 6, walkErr.items)
+}
+
+func TestNextLink(t *testing.T) {
+ tests := []struct {
+ name string
+ header []string
+ want string
+ }{
+ {name: "no header"},
+ {
+ name: "next among other rels",
+ header: []string{`<https://api.github.com/x?page=1>; rel="prev", <https://api.github.com/x?page=3>; rel="next"`},
+ want: "https://api.github.com/x?page=3",
+ },
+ {
+ name: "url containing a comma",
+ header: []string{`<https://api.github.com/x?labels=a,b&page=2>; rel="next"`},
+ want: "https://api.github.com/x?labels=a,b&page=2",
+ },
+ {
+ name: "next in a second header",
+ header: []string{`<https://api.github.com/x?page=1>; rel="first"`, `<https://api.github.com/x?page=2>; rel="next"`},
+ want: "https://api.github.com/x?page=2",
+ },
+ {
+ name: "no next",
+ header: []string{`<https://api.github.com/x?page=1>; rel="first"`},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ h := http.Header{}
+ for _, v := range tt.header {
+ h.Add("Link", v)
+ }
+ resp := &github.Response{Response: &http.Response{Header: h}}
+ assert.Equal(t, tt.want, nextLink(resp))
+ })
+ }
+}
+
+func TestFollowableResolvesRelativeLinksAgainstAPIHost(t *testing.T) {
+ client := github.NewClient(nil)
+ base, err := url.Parse("https://ghe.example.com/api/v3/")
+ require.NoError(t, err)
+ client.BaseURL = base
+ p := pager{client: client}
+
+ u, err := p.followable("/api/v3/repos/o/r/issues?page=2")
+ require.NoError(t, err)
+ assert.Equal(t, "https://ghe.example.com/api/v3/repos/o/r/issues?page=2", u.String())
+
+ u, err = p.followable("https://GHE.example.com/api/v3/repos/o/r/issues?page=3")
+ require.NoError(t, err)
+ assert.Equal(t, "GHE.example.com", u.Host)
+
+ _, err = p.followable("https://other.example.com/api/v3/repos/o/r/issues?page=3")
+ assert.Error(t, err)
+}
+
+func TestWithNextPage(t *testing.T) {
+ tests := map[string]string{
+ "https://api.github.com/x?per_page=100": "https://api.github.com/x?page=2&per_page=100",
+ "https://api.github.com/x?page=7&per_page=100": "https://api.github.com/x?page=8&per_page=100",
+ }
+ for in, want := range tests {
+ u, err := url.Parse(in)
+ require.NoError(t, err)
+ assert.Equal(t, want, withNextPage(u))
+ }
+}
+
+func TestIsPaginationCap(t *testing.T) {
+ errFor := func(status int, msg string) error {
+ return &github.ErrorResponse{Response: &http.Response{StatusCode: status}, Message: msg}
+ }
+ assert.True(t, isPaginationCap(errFor(http.StatusUnprocessableEntity, paginationCapMessage)))
+ assert.True(t, isPaginationCap(fmt.Errorf("wrapped: %w", errFor(http.StatusUnprocessableEntity, "Pagination with the page parameter is not supported"))))
+ assert.False(t, isPaginationCap(errFor(http.StatusUnprocessableEntity, "Validation Failed")))
+ assert.False(t, isPaginationCap(errFor(http.StatusForbidden, paginationCapMessage)))
+ assert.False(t, isPaginationCap(errors.New("pagination")))
+}