Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
package github
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/google/go-github/v67/github"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
// fakeGitHub serves the REST endpoints the comment scan reads for repo o/r
// and gist g1, each backed by its own fakeListing.
type fakeGitHub struct {
issues *fakeListing
issueComments *fakeListing
pulls *fakeListing
prComments *fakeListing
gistComments *fakeListing
client *github.Client
}
func newFakeGitHub(t *testing.T) *fakeGitHub {
t.Helper()
gh := &fakeGitHub{
issues: &fakeListing{},
issueComments: &fakeListing{},
pulls: &fakeListing{},
prComments: &fakeListing{},
gistComments: &fakeListing{},
}
mux := http.NewServeMux()
mux.Handle("/repos/o/r/issues", gh.issues)
mux.Handle("/repos/o/r/issues/comments", gh.issueComments)
mux.Handle("/repos/o/r/pulls", gh.pulls)
mux.Handle("/repos/o/r/pulls/comments", gh.prComments)
mux.Handle("/gists/g1/comments", gh.gistComments)
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
gh.client = newTestClient(t, srv)
return gh
}
var testRepo = repoInfo{owner: "o", name: "r", fullName: "o/r"}
// collectingReporter records the link of every chunk the scan emits, and every
// chunk error it reports.
type collectingReporter struct {
mu sync.Mutex
links []string
errs []error
}
func (r *collectingReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {
r.mu.Lock()
defer r.mu.Unlock()
r.links = append(r.links, chunk.SourceMetadata.GetGithub().GetLink())
return nil
}
func (r *collectingReporter) ChunkErr(_ context.Context, err error) error {
r.mu.Lock()
defer r.mu.Unlock()
r.errs = append(r.errs, err)
return nil
}
func issueRecord(number int, updated time.Time) fakeRecord {
return fakeRecord{updated: updated, body: map[string]any{
"id": number, "number": number, "updated_at": updated, "created_at": updated,
"html_url": fmt.Sprintf("https://github.com/o/r/issues/%d", number),
"title": "issue", "body": "body",
}}
}
// prIssueRecord is a pull request as /issues returns it.
func prIssueRecord(number int, updated time.Time) fakeRecord {
return fakeRecord{updated: updated, body: map[string]any{
"id": number, "number": number, "updated_at": updated, "created_at": updated,
"html_url": prLink(number),
"title": "pr", "body": "body",
"pull_request": map[string]any{"url": fmt.Sprintf("https://api.github.com/repos/o/r/pulls/%d", number)},
}}
}
func prRecord(number int, updated time.Time) fakeRecord {
return fakeRecord{updated: updated, body: map[string]any{
"id": number, "number": number, "updated_at": updated, "created_at": updated,
"html_url": prLink(number), "title": "pr", "body": "body",
}}
}
func prLink(number int) string { return fmt.Sprintf("https://github.com/o/r/pull/%d", number) }
func commentRecord(id int, updated time.Time, htmlURL string) fakeRecord {
return fakeRecord{updated: updated, body: map[string]any{
"id": id, "updated_at": updated, "created_at": updated, "html_url": htmlURL, "body": "comment",
}}
}
// seedRepo gives o/r one issue, one pull request, a discussion comment on
// each, and a review comment on the pull request.
func seedRepo(gh *fakeGitHub) {
gh.issues.records = []fakeRecord{issueRecord(1, testEpoch), prIssueRecord(2, testEpoch.Add(time.Minute))}
gh.pulls.records = []fakeRecord{prRecord(2, testEpoch.Add(time.Minute))}
gh.issueComments.records = []fakeRecord{
commentRecord(100, testEpoch.Add(2*time.Minute), "https://github.com/o/r/issues/1#issuecomment-100"),
commentRecord(101, testEpoch.Add(3*time.Minute), "https://github.com/o/r/pull/2#issuecomment-101"),
}
gh.prComments.records = []fakeRecord{
commentRecord(200, testEpoch.Add(4*time.Minute), "https://github.com/o/r/pull/2#discussion_r200"),
}
}
// scanComments runs the REST comment scan with unknown installation grants,
// as a token-authenticated source would.
func scanComments(t *testing.T, s *Source, gh *fakeGitHub, cutoff *time.Time) (*collectingReporter, error) {
t.Helper()
return scanCommentsWithGrants(t, s, gh, nil, cutoff)
}
func scanCommentsWithGrants(t *testing.T, s *Source, gh *fakeGitHub, grants *installationGrants, cutoff *time.Time) (*collectingReporter, error) {
t.Helper()
reporter := &collectingReporter{}
err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, testRepo, grants, reporter, cutoff)
return reporter, err
}
func TestCommentScan_FlagCombinations(t *testing.T) {
tests := []struct {
name string
issueComments bool
prComments bool
wantLinks []string
// unlisted names the endpoints the scan must not request.
unlisted []string
}{
{
name: "issue comments only",
issueComments: true,
wantLinks: []string{
"https://github.com/o/r/issues/1",
"https://github.com/o/r/issues/1#issuecomment-100",
"https://github.com/o/r/pull/2#issuecomment-101",
},
unlisted: []string{"pulls", "pull comments"},
},
{
name: "pull request comments only",
prComments: true,
wantLinks: []string{
prLink(2),
"https://github.com/o/r/pull/2#discussion_r200",
},
unlisted: []string{"issues", "issue comments"},
},
{
name: "both",
issueComments: true,
prComments: true,
wantLinks: []string{
"https://github.com/o/r/issues/1",
prLink(2),
"https://github.com/o/r/issues/1#issuecomment-100",
"https://github.com/o/r/pull/2#issuecomment-101",
"https://github.com/o/r/pull/2#discussion_r200",
},
unlisted: []string{"pulls"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includeIssueComments: tt.issueComments, includePRComments: tt.prComments}
reporter, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
assert.ElementsMatch(t, tt.wantLinks, reporter.links)
endpoints := map[string]*fakeListing{
"issues": gh.issues, "issue comments": gh.issueComments,
"pulls": gh.pulls, "pull comments": gh.prComments,
}
for _, name := range tt.unlisted {
assert.Zero(t, endpoints[name].requestCount(), "%s should not be listed", name)
}
})
}
}
func TestCommentScan_RequestsSortedListingsWithServerSideCutoff(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includeIssueComments: true, includePRComments: true}
cutoff := testEpoch.Add(-time.Hour)
_, err := scanComments(t, s, gh, &cutoff)
require.NoError(t, err)
issues := gh.issues.requests[0]
assert.Equal(t, "all", issues.Get("state"))
assert.Equal(t, "updated", issues.Get("sort"))
assert.Equal(t, "asc", issues.Get("direction"))
assert.Equal(t, "100", issues.Get("per_page"))
assert.Empty(t, issues.Get("since"), "issue and PR bodies are scanned regardless of the comments timeframe")
for name, listing := range map[string]*fakeListing{"issue comments": gh.issueComments, "pull comments": gh.prComments} {
q := listing.requests[0]
assert.Equal(t, "updated", q.Get("sort"), name)
assert.Equal(t, "asc", q.Get("direction"), name)
assert.Equal(t, cutoff.Format(time.RFC3339), q.Get("since"), name)
}
}
func TestCommentScan_OmitsSinceWithoutCutoff(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includeIssueComments: true, includePRComments: true}
_, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
assert.False(t, gh.issueComments.requests[0].Has("since"))
assert.False(t, gh.prComments.requests[0].Has("since"))
}
func TestCommentScan_PullsListOnlyConfigListsPullsWithAllStates(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includePRComments: true}
_, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
q := gh.pulls.requests[0]
assert.Equal(t, "all", q.Get("state"))
assert.Equal(t, "updated", q.Get("sort"))
assert.Equal(t, "asc", q.Get("direction"))
}
func TestCommentScan_IssueCommentsContinuePastPageCap(t *testing.T) {
gh := newFakeGitHub(t)
for i := range 250 {
gh.issueComments.records = append(gh.issueComments.records,
commentRecord(i+1, testEpoch.Add(time.Duration(i)*time.Minute), fmt.Sprintf("https://github.com/o/r/issues/1#issuecomment-%d", i+1)))
}
gh.issueComments.maxPages = 1
s := &Source{includeIssueComments: true}
reporter, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
require.Len(t, reporter.links, 250)
seen := map[string]bool{}
for _, link := range reporter.links {
assert.False(t, seen[link], "comment emitted twice: %s", link)
seen[link] = true
}
}
func TestCommentScan_PullsCappedFallsBackToIssueListing(t *testing.T) {
gh := newFakeGitHub(t)
const prs = 150
for n := 1; n <= prs; n++ {
updated := testEpoch.Add(time.Duration(n) * time.Minute)
gh.pulls.records = append(gh.pulls.records, prRecord(n, updated))
gh.issues.records = append(gh.issues.records, prIssueRecord(n, updated))
if n%10 == 0 {
gh.issues.records = append(gh.issues.records, issueRecord(1000+n, updated))
}
}
gh.pulls.maxPages = 1
s := &Source{includePRComments: true}
reporter, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
want := make([]string, 0, prs)
for n := 1; n <= prs; n++ {
want = append(want, prLink(n))
}
assert.ElementsMatch(t, want, reporter.links, "every PR body exactly once, and no issue bodies")
}
func TestCommentScan_FailedPhaseDoesNotStopTheOthers(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
gh.issueComments.fail = func(_ int, w http.ResponseWriter, _ *http.Request) bool {
writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "Server Error"})
return true
}
var retries int
s := &Source{
includeIssueComments: true,
includePRComments: true,
commentRetryWait: func(context.Context, time.Duration) bool { retries++; return true },
}
cutoff := testEpoch.Add(-time.Hour)
reporter, err := scanComments(t, s, gh, &cutoff)
require.Error(t, err)
assert.Contains(t, err.Error(), "issue comments: walk incomplete after 0 items (since=2023-12-31T23:00:00Z): giving up after 6 attempts")
assert.Contains(t, err.Error(), "Server Error")
assert.Equal(t, maxTransientRetries, retries)
assert.NotContains(t, err.Error(), "pull request comments:")
var walkErr *walkError
assert.ErrorAs(t, err, &walkErr)
assert.ElementsMatch(t, []string{
"https://github.com/o/r/issues/1",
prLink(2),
"https://github.com/o/r/pull/2#discussion_r200",
}, reporter.links)
}
// deniedWith returns a fail func that refuses every request the way GitHub
// refuses a GitHub App that lacks a permission, naming accepted as what the
// endpoint requires.
func deniedWith(accepted string) func(int, http.ResponseWriter, *http.Request) bool {
return func(_ int, w http.ResponseWriter, _ *http.Request) bool {
w.Header().Set(acceptedPermissionsHeader, accepted)
writeJSON(w, http.StatusForbidden, map[string]string{"message": appDeniedMessage})
return true
}
}
func TestCommentScan_EachFailedPhaseIsReportedAsItsOwnChunkError(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
gh.issues.fail = deniedWith("issues=read")
gh.issueComments.fail = deniedWith("issues=read; pull_requests=read")
gh.prComments.fail = deniedWith("pull_requests=read")
s := &Source{includeIssueComments: true, includePRComments: true}
reporter, err := scanComments(t, s, gh, nil)
require.Error(t, err)
require.NoError(t, reportCommentErrors(context.Background(), reporter, err))
require.Len(t, reporter.errs, 3)
wantPrefixes := []string{
"error scanning comments: issues and pull requests: walk incomplete after 0 items: missing GitHub App permission: the request needs Issues (read);",
"error scanning comments: issue comments: walk incomplete after 0 items: missing GitHub App permission: the request needs Issues (read) or Pull requests (read);",
"error scanning comments: pull request comments: walk incomplete after 0 items: missing GitHub App permission: the request needs Pull requests (read);",
}
for i, reported := range reporter.errs {
assert.True(t, strings.HasPrefix(reported.Error(), wantPrefixes[i]), "got %q", reported.Error())
assert.NotContains(t, reported.Error(), "\n", "each chunk error covers one phase")
var permErr *permissionError
assert.ErrorAs(t, reported, &permErr)
}
assert.Empty(t, reporter.links)
}
// reportedGrantErrors runs reportCommentErrors on err and describes each
// chunk error it reports as "<prefix>: skip" or "<prefix>: narrow", where
// prefix names the phase or the content left out. Any reported error that
// isn't a grants-based permissionError fails the test.
func reportedGrantErrors(t *testing.T, err error) []string {
t.Helper()
reporter := &collectingReporter{}
require.NoError(t, reportCommentErrors(context.Background(), reporter, err))
var got []string
for _, reported := range reporter.errs {
var permErr *permissionError
require.ErrorAs(t, reported, &permErr)
prefix, _, _ := strings.Cut(strings.TrimPrefix(reported.Error(), "error scanning comments: "), ": missing")
switch {
case strings.Contains(permErr.outcome, "skipped"):
got = append(got, prefix+": skip")
case strings.Contains(permErr.outcome, "left them out"):
got = append(got, prefix+": narrow")
default:
t.Fatalf("unexpected permission error: %v", reported)
}
}
return got
}
func TestCommentScan_InstallationGrantsShapeTheListings(t *testing.T) {
type grant struct{ issues, pullRequests bool }
var (
neither = grant{}
prsOnly = grant{pullRequests: true}
issues = grant{issues: true}
both = grant{issues: true, pullRequests: true}
)
tests := []struct {
name string
issueComments bool
prComments bool
grant grant
wantErrs []string
// wantListed names every endpoint the scan requests; the rest must
// not be requested.
wantListed []string
}{
{name: "issue comments, nothing granted", issueComments: true, grant: neither,
wantErrs: []string{"issues: skip", "issue comments: skip"}},
{name: "issue comments, pull requests granted", issueComments: true, grant: prsOnly,
wantErrs: []string{"issues: skip", "issue comments: narrow"},
wantListed: []string{"issue comments"}},
{name: "issue comments, issues granted", issueComments: true, grant: issues,
wantListed: []string{"issues", "issue comments"}},
{name: "issue comments, both granted", issueComments: true, grant: both,
wantListed: []string{"issues", "issue comments"}},
{name: "PR comments, nothing granted", prComments: true, grant: neither,
wantErrs: []string{"pull requests: skip", "pull request comments: skip"}},
{name: "PR comments, pull requests granted", prComments: true, grant: prsOnly,
wantListed: []string{"pulls", "pull comments"}},
{name: "PR comments, issues granted", prComments: true, grant: issues,
wantErrs: []string{"pull requests: skip", "pull request comments: skip"}},
{name: "PR comments, both granted", prComments: true, grant: both,
wantListed: []string{"pulls", "pull comments"}},
{name: "both settings, nothing granted", issueComments: true, prComments: true, grant: neither,
wantErrs: []string{"issues and pull requests: skip", "issue comments: skip", "pull request comments: skip"}},
{name: "both settings, pull requests granted", issueComments: true, prComments: true, grant: prsOnly,
wantErrs: []string{"issues: skip", "issue comments: narrow"},
wantListed: []string{"pulls", "issue comments", "pull comments"}},
{name: "both settings, issues granted", issueComments: true, prComments: true, grant: issues,
wantErrs: []string{"pull request bodies: narrow", "pull request conversation comments: narrow", "pull request comments: skip"},
wantListed: []string{"issues", "issue comments"}},
{name: "both settings, both granted", issueComments: true, prComments: true, grant: both,
wantListed: []string{"issues", "issue comments", "pull comments"}},
{name: "comments off, nothing granted", grant: neither},
{name: "comments off, pull requests granted", grant: prsOnly},
{name: "comments off, issues granted", grant: issues},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includeIssueComments: tt.issueComments, includePRComments: tt.prComments}
grants := &installationGrants{installationID: 42, issues: tt.grant.issues, pullRequests: tt.grant.pullRequests}
_, err := scanCommentsWithGrants(t, s, gh, grants, nil)
if len(tt.wantErrs) == 0 {
require.NoError(t, err)
} else {
assert.Equal(t, tt.wantErrs, reportedGrantErrors(t, err))
}
endpoints := map[string]*fakeListing{
"issues": gh.issues, "issue comments": gh.issueComments,
"pulls": gh.pulls, "pull comments": gh.prComments,
}
var listed []string
for name, listing := range endpoints {
if listing.requestCount() > 0 {
listed = append(listed, name)
}
}
assert.ElementsMatch(t, tt.wantListed, listed)
})
}
}
func TestCommentScan_PullRequestBodiesComeFromPullsWithoutIssuesGrant(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includeIssueComments: true, includePRComments: true}
reporter, _ := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42, pullRequests: true}, nil)
assert.Contains(t, reporter.links, prLink(2))
assert.NotContains(t, reporter.links, "https://github.com/o/r/issues/1")
assert.Zero(t, gh.issues.requestCount())
}
func TestCommentScan_GrantErrorsNameTheInstallationAndPermissions(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
s := &Source{includeIssueComments: true, includePRComments: true}
_, err := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42, issues: true}, nil)
reporter := &collectingReporter{}
require.NoError(t, reportCommentErrors(context.Background(), reporter, err))
var got []string
for _, reported := range reporter.errs {
got = append(got, reported.Error())
}
fix := "update the app's permissions and accept the change on the installation"
assert.Equal(t, []string{
"error scanning comments: pull request bodies: missing GitHub App permission: Pull requests (read) is needed; " + fix +
" (not granted to installation 42, so GitHub left them out of the /issues listing)",
"error scanning comments: pull request conversation comments: missing GitHub App permission: Pull requests (read) is needed; " + fix +
" (not granted to installation 42, so GitHub left them out of the /issues/comments listing)",
"error scanning comments: pull request comments: missing GitHub App permission: Pull requests (read) is needed; " + fix +
" (not granted to installation 42, so the listing was skipped)",
}, got)
}
func TestCommentScan_SkippedListingNeedingBothPermissionsSaysSo(t *testing.T) {
gh := newFakeGitHub(t)
s := &Source{includeIssueComments: true, includePRComments: true}
_, err := scanCommentsWithGrants(t, s, gh, &installationGrants{installationID: 42}, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "issues and pull requests: missing GitHub App permission: Issues (read) and Pull requests (read) are needed;")
}
// respondWith returns a fail func that answers every request with status,
// the way GitHub answers listings for a feature the repo has turned off.
func respondWith(status int) func(int, http.ResponseWriter, *http.Request) bool {
return func(_ int, w http.ResponseWriter, _ *http.Request) bool {
writeJSON(w, status, map[string]string{"message": http.StatusText(status)})
return true
}
}
func TestCommentScan_IssueCommentsDisabledWhenRepoHasNoIssues(t *testing.T) {
for _, status := range []int{http.StatusNotFound, http.StatusGone} {
t.Run(http.StatusText(status), func(t *testing.T) {
// Issues and pull requests are both off: /issues is empty and
// /issues/comments refuses, but old review comments remain.
gh := newFakeGitHub(t)
gh.issueComments.fail = respondWith(status)
gh.prComments.records = []fakeRecord{
commentRecord(200, testEpoch, "https://github.com/o/r/pull/2#discussion_r200"),
}
s := &Source{includeIssueComments: true, includePRComments: true}
repo := testRepo
repo.hasNoIssues = true
reporter := &collectingReporter{}
err := s.processIssueandPRsWithCommentsREST(context.Background(), gh.client, repo, nil, reporter, nil)
require.NoError(t, err)
assert.Equal(t, []string{"https://github.com/o/r/pull/2#discussion_r200"}, reporter.links)
})
}
}
func TestCommentScan_IssueCommentsNotFoundIsReportedWhenIssuesAreOn(t *testing.T) {
gh := newFakeGitHub(t)
gh.issueComments.fail = respondWith(http.StatusNotFound)
s := &Source{includeIssueComments: true}
_, err := scanComments(t, s, gh, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "issue comments:")
assert.Contains(t, err.Error(), "404")
}
func TestCommentScan_PullsDisabledStillListsReviewComments(t *testing.T) {
gh := newFakeGitHub(t)
gh.pulls.fail = respondWith(http.StatusNotFound)
gh.prComments.records = []fakeRecord{
commentRecord(200, testEpoch, "https://github.com/o/r/pull/2#discussion_r200"),
}
s := &Source{includePRComments: true}
reporter, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
assert.Equal(t, []string{"https://github.com/o/r/pull/2#discussion_r200"}, reporter.links)
assert.Equal(t, 1, gh.prComments.requestCount())
}
func TestCommentScan_NoPullRequestsSkipsReviewComments(t *testing.T) {
gh := newFakeGitHub(t)
s := &Source{includePRComments: true}
reporter, err := scanComments(t, s, gh, nil)
require.NoError(t, err)
assert.Empty(t, reporter.links)
assert.Equal(t, 1, gh.pulls.requestCount())
assert.Zero(t, gh.prComments.requestCount())
}
func TestCacheRepoInfo_HasNoIssues(t *testing.T) {
off, on := false, true
tests := []struct {
name string
hasIssues *bool
want bool
}{
{name: "issues off", hasIssues: &off, want: true},
{name: "issues on", hasIssues: &on, want: false},
{name: "field missing", hasIssues: nil, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &Source{repoInfoCache: newRepoInfoCache()}
cloneURL := "https://github.com/o/r.git"
s.cacheRepoInfo(&github.Repository{CloneURL: &cloneURL, HasIssues: tt.hasIssues})
info, ok := s.repoInfoCache.get(cloneURL)
require.True(t, ok)
assert.Equal(t, tt.want, info.hasNoIssues)
})
}
}
func TestReportCommentErrors_UnjoinedErrorIsReportedOnce(t *testing.T) {
reporter := &collectingReporter{}
require.NoError(t, reportCommentErrors(context.Background(), reporter, errors.New("could not get API client")))
require.Len(t, reporter.errs, 1)
assert.EqualError(t, reporter.errs[0], "error scanning comments: could not get API client")
}
func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {
gh := newFakeGitHub(t)
seedRepo(gh)
gh.issueComments.fail = failRequest(http.StatusBadGateway, 1)
gh.prComments.fail = failRequest(http.StatusBadGateway, 1)
s := &Source{
includeIssueComments: true,
includePRComments: true,
commentRetryWait: func(context.Context, time.Duration) bool { return true },
}
// retries are logged at V(4), with other API decisions
var logs bytes.Buffer
logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(4)))
ctx := context.WithLogger(context.Background(), logger)
err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, nil, &collectingReporter{}, nil)
require.NoError(t, err)
var listings []string
for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") {
var entry map[string]any
require.NoError(t, json.Unmarshal([]byte(line), &entry))
if entry["msg"] == "retrying GitHub API request after transient error" {
listings = append(listings, fmt.Sprint(entry["listing"]))
}
}
assert.Equal(t, []string{"issue comments", "pull request comments"}, listings)
}
func TestProcessGistComments_CappedListingIsAnError(t *testing.T) {
gh := newFakeGitHub(t)
for i := range 150 {
gh.gistComments.records = append(gh.gistComments.records, commentRecord(i+1, testEpoch, ""))
}
gh.gistComments.maxPages = 1
s := &Source{includeGistComments: true}
reporter := &collectingReporter{}
err := s.processGistComments(context.Background(), gh.client, "https://gist.github.com/u/g1",
[]string{"gist.github.com", "u", "g1"}, repoInfo{}, reporter, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "gist comments: walk incomplete after 100 items")
assert.Contains(t, err.Error(), "no since filter")
assert.Len(t, reporter.links, 100, "comments read before the cap are kept")
}
func TestProcessGistComments_SkipsOldCommentsAndKeepsGoing(t *testing.T) {
gh := newFakeGitHub(t)
for i := range 5 {
created := testEpoch.Add(time.Duration(i) * time.Hour)
gh.gistComments.records = append(gh.gistComments.records, fakeRecord{updated: created, body: map[string]any{
"id": i + 1, "created_at": created, "url": fmt.Sprintf("https://api.github.com/gists/g1/comments/%d", i+1),
}})
}
s := &Source{includeGistComments: true}
reporter := &collectingReporter{}
cutoff := testEpoch.Add(150 * time.Minute)
err := s.processGistComments(context.Background(), gh.client, "https://gist.github.com/u/g1",
[]string{"gist.github.com", "u", "g1"}, repoInfo{}, reporter, &cutoff)
require.NoError(t, err)
assert.Equal(t, []string{
"https://api.github.com/gists/g1/comments/4",
"https://api.github.com/gists/g1/comments/5",
}, reporter.links)
}