Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Implemented Graphql requests for Github PR's, Issues and comments scanning (#4431)
Kashif Khandaf7e5626bc8
diff --git a/main.go b/main.goindex d212fa7da..992c5b0cb 100644--- a/main.go+++ b/main.go@@ -467,6 +467,9 @@ func run(state overseer.State) {feature.UseSimplifiedGitlabEnumeration.Store(true)feature.GitlabProjectsPerPage.Store(100)+ // OSS Default using github graphql api for issues, pr's and comments+ feature.UseGithubGraphQLAPI.Store(false)+conf := &config.Config{}if *configFilename != "" {var err errordiff --git a/pkg/feature/feature.go b/pkg/feature/feature.goindex 3c030a631..a3642b417 100644--- a/pkg/feature/feature.go+++ b/pkg/feature/feature.go@@ -11,6 +11,7 @@ var (UseSimplifiedGitlabEnumeration atomic.BoolUseGitMirror atomic.BoolGitlabProjectsPerPage atomic.Int64+ UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments)type AtomicString struct {diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex b9699a065..b11740b82 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -27,6 +27,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/feature""github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"@@ -1105,8 +1106,14 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo repif s.includeGistComments && isGistUrl(urlParts) && !s.ignoreGists {return s.processGistComments(ctx, urlString, urlParts, repoInfo, reporter, cutoffTime)} else if s.includeIssueComments || s.includePRComments {- return s.processRepoComments(ctx, repoInfo, reporter, cutoffTime)+ // if we need to use graphql api for repo issues, prs and comments+ if feature.UseGithubGraphQLAPI.Load() {+ return s.processRepoIssueandPRsWithCommentsGraphql(ctx, repoInfo, reporter, cutoffTime)+ }++ return s.processIssueandPRsWithCommentsREST(ctx, repoInfo, reporter, cutoffTime)}+return nil}@@ -1273,7 +1280,10 @@ var (state = "all")-func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+func (s *Source) processIssueandPRsWithCommentsREST(+ ctx context.Context, repoInfo repoInfo,+ reporter sources.ChunkReporter, cutoffTime *time.Time,+) error {if s.includeIssueComments {ctx.Logger().V(2).Info("Scanning issues")if err := s.processIssues(ctx, repoInfo, reporter); err != nil {@@ -1297,6 +1307,31 @@ func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, repreturn nil}+func (s *Source) processRepoIssueandPRsWithCommentsGraphql(+ ctx context.Context, repoInfo repoInfo,+ reporter sources.ChunkReporter, cutoffTime *time.Time,+) error {+ if s.includeIssueComments {+ ctx.Logger().V(2).Info("Scanning issues")+ if err := s.processIssuesWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil {+ return err+ }+ }++ if s.includePRComments {+ ctx.Logger().V(2).Info("Scanning pull requests")+ if err := s.processPRWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil {+ return err+ }++ if err := s.processReviewThreads(ctx, repoInfo, reporter, cutoffTime); err != nil {+ return err+ }+ }++ return nil+}+func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter) error {bodyTextsOpts := &github.IssueListByRepoOptions{Sort: sortType,diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.goindex 42af8f4e2..6e5ce443b 100644--- a/pkg/sources/github/github_integration_test.go+++ b/pkg/sources/github/github_integration_test.go@@ -17,6 +17,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/feature""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"@@ -968,6 +969,64 @@ func TestSource_Validate(t *testing.T) {}}+func TestSource_ScanCommentsWithGraphql(t *testing.T) {+ feature.UseGithubGraphQLAPI.Store(true)++ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()++ source := &sourcespb.GitHub{+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},+ IncludeIssueComments: true,+ IncludePullRequestComments: true,+ Credential: &sourcespb.GitHub_Unauthenticated{},+ }++ wantChunk := sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Link: "https://github.com/trufflesecurity/driftwood.git/issues/1",+ Username: "truffle-sandbox",+ Timestamp: "2023-06-22 23:33:46 +0000 UTC",+ },+ },+ },+ Verify: false,+ }++ s := Source{}++ conn, err := anypb.New(source)+ assert.NoError(t, err)++ err = s.Init(ctx, "test-source", 0, 0, false, conn, 4)+ assert.NoError(t, err)++ chunksCh := make(chan *sources.Chunk, 1)+ go func() {+ // Close the channel+ defer close(chunksCh)+ err = s.Chunks(ctx, chunksCh)+ assert.NoError(t, err)+ }()++ i := 0+ for gotChunk := range chunksCh {+ // Skip chunks that are not comments.+ if gotChunk.SourceMetadata.GetGithub().GetCommit() != "" {+ continue+ }+ i+++ githubCommentCheckFunc(gotChunk, &wantChunk, i, t, "test-source")+ }++ // Confirm all comments were processed.+ assert.Equal(t, i, 5)+}+type countChunkReporter struct {chunkCount interrCount intdiff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.gonew file mode 100644index 000000000..e7e2f08cf--- /dev/null+++ b/pkg/sources/github/graphql.go@@ -0,0 +1,495 @@+package github++import (+ "cmp"+ "fmt"+ "math/rand/v2"+ "strings"+ "time"++ "github.com/shurcooL/githubv4"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++// processIssuesWithComments process github repo issues with comments using graphql API+func (s *Source) processIssuesWithComments(+ ctx context.Context, repoInfo repoInfo,+ reporter sources.ChunkReporter, cutoffTime *time.Time,+) error {+ vars := map[string]any{+ owner: githubv4.String(repoInfo.owner),+ repository: githubv4.String(repoInfo.name),+ issuesPerPage: githubv4.Int(defaultPagination),+ issuesPagination: (*githubv4.String)(nil),+ commentsPerPage: githubv4.Int(defaultPagination),+ commentsPagination: (*githubv4.String)(nil),+ }++ var totalIssues int++ // loop will continue as long as there are issues in the repository+ for {+ var query issuesWithComments+ err := s.connector.GraphQLClient().Query(ctx, &query, vars)+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {+ continue+ }++ if err != nil {+ return fmt.Errorf("error fetching issues: %w", err)+ }++ totalIssues += len(query.GetIssues())++ ctx.Logger().V(5).Info("Scanning Issues",+ "total_issues", len(query.GetIssues()))++ if err := s.chunkGraphqlIssues(ctx, repoInfo, query.Repository.Issues.Nodes, reporter); err != nil {+ return err+ }++ // process each issue comments+ for _, issue := range query.GetIssues() {+ ctx.Logger().V(5).Info("Scanning Issue Comments",+ "issue_id", issue.Number,+ "total_comments", len(issue.GetIssueComments()),+ )++ if err := s.chunkComments(ctx, repoInfo, issue.GetIssueComments(), reporter, cutoffTime); err != nil {+ return err+ }++ // if issue has more than 100 comments, we need to send another request for that specific issue to pull more comments+ for issue.Comments.PageInfo.HasNextPage {+ commentVars := map[string]any{+ owner: githubv4.String(repoInfo.owner),+ repository: githubv4.String(repoInfo.name),+ issueNumber: githubv4.Int(issue.Number),+ commentsPerPage: githubv4.Int(defaultPagination),+ commentsPagination: issue.Comments.PageInfo.EndCursor,+ }++ // request this issue more comments+ var commentsQuery singleIssueComments+ err := s.connector.GraphQLClient().Query(ctx, &commentsQuery, commentVars)+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentsQuery.RateLimit, err) {+ continue+ }++ if err != nil {+ return fmt.Errorf("error fetching issue: %w", err)+ }++ ctx.Logger().V(5).Info("Scanning additional issue comments",+ "issue_id", issue.Number,+ "total_comments", len(commentsQuery.GetIssueComments()))++ if err := s.chunkComments(ctx, repoInfo, commentsQuery.GetIssueComments(), reporter, cutoffTime); err != nil {+ return err+ }++ // update page info for loop+ issue.Comments.PageInfo = commentsQuery.Repository.Issue.Comments.PageInfo+ }+ }++ // paginate issues+ if !query.Repository.Issues.PageInfo.HasNextPage {+ ctx.Logger().V(4).Info("Scanned all repository issues with comments", "total_issues_scanned", totalIssues)+ break+ }++ // update issues pagination to go to next page+ vars[issuesPagination] = githubv4.NewString(query.Repository.Issues.PageInfo.EndCursor)+ }++ return nil+}++// processPRWithComments process github repo pull requests with inline comments+func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+ vars := map[string]any{+ owner: githubv4.String(repoInfo.owner),+ repository: githubv4.String(repoInfo.name),+ pullRequestPerPage: githubv4.Int(defaultPagination),+ pullRequestPagination: (*githubv4.String)(nil),+ commentsPerPage: githubv4.Int(defaultPagination),+ commentsPagination: (*githubv4.String)(nil),+ }++ var totalPRs int++ // continue loop as long as there are pull requests remaining+ for {+ var query pullRequestWithComments+ err := s.connector.GraphQLClient().Query(ctx, &query, vars)+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {+ continue+ }++ if err != nil {+ return fmt.Errorf("error fetching pull requests with comments: %w", err)+ }++ totalPRs += len(query.GetPullRequests())++ ctx.Logger().V(5).Info("Scanning pull requests",+ "total_pull_requests", len(query.GetPullRequests()))++ if err := s.chunkGraphqlPullRequests(ctx, repoInfo, query.GetPullRequests(), reporter); err != nil {+ return err+ }++ // process each pr comments+ for _, pr := range query.GetPullRequests() {+ ctx.Logger().V(5).Info("Scanning pull request comments",+ "pull_request_no", pr.Number,+ "total_comments", len(pr.Comments.Nodes))++ if err := s.chunkComments(ctx, repoInfo, pr.GetPRComments(), reporter, cutoffTime); err != nil {+ return err+ }++ // if a pull request has more than 100 comments - some interns pull request might endup here :)+ for pr.Comments.PageInfo.HasNextPage {+ // request this pull request more comments+ var commentQuery singlePRComments+ singlePRVars := map[string]any{+ owner: githubv4.String(repoInfo.owner),+ repository: githubv4.String(repoInfo.name),+ pullRequestNumber: pr.Number,+ commentsPerPage: githubv4.Int(defaultPagination),+ commentsPagination: pr.Comments.PageInfo.EndCursor,+ }++ err := s.connector.GraphQLClient().Query(ctx, &commentQuery, singlePRVars)+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentQuery.RateLimit, err) {+ continue+ }++ if err != nil {+ return fmt.Errorf("error fetching pull request with comments: %w", err)+ }++ ctx.Logger().V(5).Info("Scanning additional comments",+ "pull_request_no", pr.Number,+ "total_comments", len(commentQuery.GetPRComments()))++ if err := s.chunkComments(ctx, repoInfo, commentQuery.GetPRComments(), reporter, cutoffTime); err != nil {+ return err+ }++ // update pr.Comments.PageInfo so loop condition reflects new state+ pr.Comments.PageInfo = commentQuery.Repository.PullRequest.Comments.PageInfo+ }+ }++ // move to next page of PRs+ if !query.Repository.PullRequests.PageInfo.HasNextPage {+ ctx.Logger().V(4).Info("Scanned all repository pull requests with comments", "total_pullrequests_scanned", totalPRs)+ break+ }++ // update pull request pagination to go to next page+ vars[pullRequestPagination] = githubv4.NewString(query.Repository.PullRequests.PageInfo.EndCursor)+ }++ return nil+}++// processReviewThreads process github repo pull request review threads+func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+ vars := map[string]any{+ owner: githubv4.String(repoInfo.owner),+ repository: githubv4.String(repoInfo.name),+ pullRequestPerPage: githubv4.Int(defaultPagination),+ pullRequestPagination: (*githubv4.String)(nil),+ threadPerPage: githubv4.Int(defaultPagination),+ threadPagination: (*githubv4.String)(nil),+ }++ var threadIDs = make([]string, 0)++ // continue as long as pull requests have threads+ for {+ var query prWithReviewThreadIDs+ err := s.connector.GraphQLClient().Query(ctx, &query, vars)+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {+ continue+ }++ if err != nil {+ return fmt.Errorf("error fetching pr thread reviews: %w", err)+ }++ // collect thread ids+ for _, pr := range query.GetMinimalPullRequests() {+ prThreadIDs := pr.ReviewThreads.GetThreadIDs()+ threadIDs = append(threadIDs, prThreadIDs...)+ }++ if !query.Repository.PullRequests.PageInfo.HasNextPage {+ ctx.Logger().V(4).Info("Pulled all repository PR's threads IDs")+ break+ }++ // update pull request pagination to go to next page+ vars[pullRequestPagination] = githubv4.NewString(query.Repository.PullRequests.PageInfo.EndCursor)+ }++ ctx.Logger().V(4).Info("Pulled all thread IDs", "total_threads", len(threadIDs))++ // if we got more than 0 threads unfortunately :( than we need to pull their comments in batches+ if len(threadIDs) > 0 {+ // process in batches of max 100+ for _, batch := range chunkIDs(threadIDs, 100) {+ ctx.Logger().V(5).Info("Processing Thread comments in Batches", "batch_length", len(batch))+ // fetch comments for the batch of threads+ if err := s.fetchThreadComments(ctx, batch, repoInfo, reporter, cutoffTime); err != nil {+ return fmt.Errorf("error fetching thread review comments: %w", err)+ }+ }+ }++ return nil+}++// fetchThreadComments process github repo pull request threads and their comments+func (s *Source) fetchThreadComments(ctx context.Context, threadIDs []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+ // Process in batches of 100+ var query multiReviewThreadComments+ vars := map[string]any{+ "ids": threadIDs,+ commentsPerPage: githubv4.Int(defaultPagination),+ commentsPagination: (*githubv4.String)(nil),+ }++ if err := s.connector.GraphQLClient().Query(ctx, &query, vars); err != nil {+ return fmt.Errorf("multi-thread query failed: %w", err)+ }++ // process each thread in batch+ for _, thread := range query.GetThreads() {+ if err := s.chunkComments(ctx, repoInfo, thread.GetThreadComments(), reporter, cutoffTime); err != nil {+ return err+ }++ // if a thread has more than 100 comments :)+ for thread.Comments.PageInfo.HasNextPage {+ // request this thread more comments+ var query singleReviewThreadComments+ reviewThreadVars := map[string]any{+ threadID: thread.ID,+ commentsPerPage: githubv4.Int(defaultPagination),+ commentsPagination: (*githubv4.String)(nil),+ }++ err := s.connector.GraphQLClient().Query(ctx, &query, reviewThreadVars)+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {+ continue+ }++ if err != nil {+ return fmt.Errorf("single-thread query failed: %w", err)+ }++ node := query.Node+ if err := s.chunkComments(ctx, repoInfo, node.Comments.Nodes, reporter, cutoffTime); err != nil {+ return err+ }++ if !node.Comments.PageInfo.HasNextPage {+ break+ }++ // update thread comments pagination+ reviewThreadVars[commentsPagination] = &node.Comments.PageInfo.EndCursor+ }+ }++ return nil+}++// chunkIDs splits a slice of IDs into multiple chunks of at most `size` elements.+// For example, if you have 250 IDs and size=100, this will return+//+// [][]string{+// {id0 ... id99}, // first 100+// {id100 ... id199},// next 100+// {id200 ... id249} // last 50+// }+func chunkIDs(ids []string, size int) [][]string {+ var chunks [][]string+ for size < len(ids) {+ ids, chunks = ids[size:], append(chunks, ids[0:size:size])+ }+ return append(chunks, ids)+}++// handleRateLimitWithChunkReporter is a wrapper around handleRateLimit that includes chunk reporting+func (s *Source) handleGraphqlRateLimitWithChunkReporter(+ ctx context.Context, reporter sources.ChunkReporter,+ rl *rateLimit, errIn error,+) bool {+ return s.handleGraphQLRateLimit(ctx, rl, errIn, &chunkErrorReporter{reporter: reporter})+}++// handleGraphQLRateLimit inspects the rateLimit info returned in GraphQL queries.+func (s *Source) handleGraphQLRateLimit(ctx context.Context, rl *rateLimit, errIn error, reporters ...errorReporter) bool {+ // check global resume time first (in case another request already set it)+ rateLimitMu.RLock()+ resumeTime := rateLimitResumeTime+ rateLimitMu.RUnlock()++ // if resume time is not empty and is after current time, than put the request to sleep till that.+ if !resumeTime.IsZero() && time.Now().Before(resumeTime) {+ retryAfter := time.Until(resumeTime)+ time.Sleep(retryAfter)+ return true+ }++ var retryAfter time.Duration+ if errIn != nil && strings.Contains(errIn.Error(), "rate limit exceeded") {+ now := time.Now()++ rateLimitMu.Lock()+ rateLimitResumeTime = now.Add(1 * time.Minute)+ retryAfter = time.Until(rateLimitResumeTime)+ ctx.Logger().Info("GraphQL RATE_LIMITED error (fallback)",+ "retry_after", retryAfter.String())+ rateLimitMu.Unlock()+ } else if rl != nil {+ // if rate limit remaining is more than 3 continue using graphql api+ if rl.Remaining > 3 {+ return false+ }++ // === only reach here if error is nil and rate limit remaining is less than 3 (safety)+ now := time.Now()+ retryAfter = time.Until(rl.ResetAt)+ // never negative and enforce a sane minimum backoff (avoid thrashing with 1s/2s retries)+ if cmp.Less(retryAfter, 5*time.Second) {+ retryAfter = 5 * time.Second+ }++ jitter := time.Duration(rand.IntN(10)+1) * time.Second+ retryAfter += jitter++ // update global resume time+ rateLimitMu.Lock()+ rateLimitResumeTime = now.Add(retryAfter)+ ctx.Logger().Info("exceeded GraphQL rate limit",+ "retry_after", retryAfter.String(),+ "resume_time", rateLimitResumeTime.Format(time.RFC3339))+ rateLimitMu.Unlock()++ for _, reporter := range reporters {+ _ = reporter.Err(ctx, fmt.Errorf("exceeded GraphQL rate limit"))+ }+ }++ githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()+ time.Sleep(retryAfter)+ githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())++ return true+}++func (s *Source) chunkGraphqlIssues(ctx context.Context, repoInfo repoInfo, issues []issue, reporter sources.ChunkReporter) error {+ for _, issue := range issues {+ // Create chunk and send it to the channel.+ chunk := sources.Chunk{+ SourceName: s.name,+ SourceID: s.SourceID(),+ JobID: s.JobID(),+ SourceType: s.Type(),+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Link: sanitizer.UTF8(issue.URL),+ Username: sanitizer.UTF8(issue.Author.Login),+ Repository: sanitizer.UTF8(repoInfo.fullName),+ Timestamp: sanitizer.UTF8(issue.CreatedAt.String()),+ Visibility: repoInfo.visibility,+ },+ },+ },+ Data: []byte(sanitizer.UTF8(issue.Title + "\n" + issue.Body)),+ Verify: s.verify,+ }++ if err := reporter.ChunkOk(ctx, chunk); err != nil {+ return err+ }+ }+ return nil+}++func (s *Source) chunkComments(ctx context.Context, repoInfo repoInfo, comments []comment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {+ for _, comment := range comments {+ // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted+ if cutoffTime != nil && comment.UpdatedAt.Before(*cutoffTime) {+ continue+ }++ // Create chunk and send it to the channel.+ chunk := sources.Chunk{+ SourceName: s.name,+ SourceID: s.SourceID(),+ JobID: s.JobID(),+ SourceType: s.Type(),+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Link: sanitizer.UTF8(comment.URL),+ Username: sanitizer.UTF8(comment.Author.Login),+ Repository: sanitizer.UTF8(repoInfo.fullName),+ Timestamp: sanitizer.UTF8(comment.CreatedAt.String()),+ Visibility: repoInfo.visibility,+ },+ },+ },+ Data: []byte(sanitizer.UTF8(comment.Body)),+ Verify: s.verify,+ }++ if err := reporter.ChunkOk(ctx, chunk); err != nil {+ return err+ }+ }+ return nil+}++func (s *Source) chunkGraphqlPullRequests(ctx context.Context, repoInfo repoInfo, prs []pullRequest, reporter sources.ChunkReporter) error {+ for _, pr := range prs {+ // Create chunk and send it to the channel.+ chunk := sources.Chunk{+ SourceName: s.name,+ SourceID: s.SourceID(),+ SourceType: s.Type(),+ JobID: s.JobID(),+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Link: sanitizer.UTF8(pr.URL),+ Username: sanitizer.UTF8(pr.Author.Login),+ Repository: sanitizer.UTF8(repoInfo.fullName),+ Timestamp: sanitizer.UTF8(pr.CreatedAt.String()),+ Visibility: repoInfo.visibility,+ },+ },+ },+ Data: []byte(sanitizer.UTF8(pr.Title + "\n" + pr.Body)),+ Verify: s.verify,+ }++ if err := reporter.ChunkOk(ctx, chunk); err != nil {+ return err+ }+ }+ return nil+}diff --git a/pkg/sources/github/queries.go b/pkg/sources/github/queries.gonew file mode 100644index 000000000..dacaa7ad4--- /dev/null+++ b/pkg/sources/github/queries.go@@ -0,0 +1,248 @@+package github++import (+ "time"++ "github.com/shurcooL/githubv4"+)++const (+ // query variable keys+ owner = "owner"+ repository = "repo"++ pullRequestNumber = "number"+ pullRequestPerPage = "first"+ pullRequestPagination = "after"++ commentsPerPage = "commentsFirst"+ commentsPagination = "commentsAfter"++ threadID = "threadID"+ threadPerPage = "threadsFirst"+ threadPagination = "threadsAfter"++ issueNumber = "number"+ issuesPerPage = "issuesFirst"+ issuesPagination = "issuesAfter"+)++// === Pull Requests with Comments ===++// pullRequestWithComments represent a repository pull request nodes+type pullRequestWithComments struct {+ Repository struct {+ PullRequests pullRequestNodes `graphql:"pullRequests(first: $first, after: $after, orderBy: {field: UPDATED_AT, direction: DESC})"`+ } `graphql:"repository(owner: $owner, name: $repo)"`+ RateLimit rateLimit `graphql:"rateLimit"`+}++// pullRequestNodes represents a paginated list of pull requests+type pullRequestNodes struct {+ Nodes []pullRequest+ PageInfo pageInfo+}++// pullRequest represents a single pull request with comment nodes+type pullRequest struct {+ Title string+ Number int+ URL string+ Author author+ CreatedAt time.Time+ Body string+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter, orderBy: {field: UPDATED_AT, direction: DESC})"`+}++// commentNodes represents a paginated list of comments+type commentNodes struct {+ Nodes []comment+ PageInfo pageInfo+}++// comment represents a single comment+type comment struct {+ Body string+ CreatedAt time.Time+ UpdatedAt time.Time+ Author author+ URL string+}++// singlePRComments represents a single PR comments+type singlePRComments struct {+ Repository struct {+ PullRequest struct {+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter, orderBy: {field: UPDATED_AT, direction: DESC})"`+ } `graphql:"pullRequest(number: $number)"`+ } `graphql:"repository(owner: $owner, name: $repo)"`+ RateLimit rateLimit `graphql:"rateLimit"`+}++// GetPullRequests return list of pull requests+func (p pullRequestWithComments) GetPullRequests() []pullRequest {+ return p.Repository.PullRequests.Nodes+}++// GetPRComments return list of comments of the PR+func (p pullRequest) GetPRComments() []comment {+ return p.Comments.Nodes+}++// GetPRComments return list of comments of the PR+func (p singlePRComments) GetPRComments() []comment {+ return p.Repository.PullRequest.Comments.Nodes+}++// === Pull Request Review Threads IDs ===++// prWithReviewThreadIDs represents repository pull requests with review threads IDs+type prWithReviewThreadIDs struct {+ Repository struct {+ PullRequests minimalPullRequestNodes `graphql:"pullRequests(first: $first, after: $after, orderBy: {field: UPDATED_AT, direction: DESC})"`+ } `graphql:"repository(owner: $owner, name: $repo)"`+ RateLimit rateLimit `graphql:"rateLimit"`+}++// minimalPullRequestNodes represents a paginated list of pull request+type minimalPullRequestNodes struct {+ Nodes []minimalPullRequest+ PageInfo pageInfo+}++// minimalPullRequest represents a pull request with no and review threads+type minimalPullRequest struct {+ Number int+ ReviewThreads reviewThreadIDNodes `graphql:"reviewThreads(first: $threadsFirst, after: $threadsAfter)"`+}++// reviewThreadIDNodes represents a paginated list of pr review threads+type reviewThreadIDNodes struct {+ Nodes []reviewThreadID+ PageInfo pageInfo+}++// reviewThreadID represents a single review thread with ID+type reviewThreadID struct {+ ID string+}++func (p prWithReviewThreadIDs) GetMinimalPullRequests() []minimalPullRequest {+ return p.Repository.PullRequests.Nodes+}++func (r reviewThreadIDNodes) GetThreadIDs() []string {+ var ids = make([]string, 0)++ for _, id := range r.Nodes {+ ids = append(ids, id.ID)+ }++ return ids+}++// === Review Threads with comments ===++// multiReviewThreadComments fetches multiple review threads (by IDs) and the first page of their comments.+type multiReviewThreadComments struct {+ Nodes []struct {+ PullRequestReviewThread reviewThreadComments `graphql:"... on PullRequestReviewThread"`+ } `graphql:"nodes(ids: $ids)"`+ RateLimit rateLimit `graphql:"rateLimit"`+}++// reviewThreadComments represents a single thread and its comments.+type reviewThreadComments struct {+ ID string+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"`+}++// singleReviewThreadComments fetches one review thread with comment pagination+type singleReviewThreadComments struct {+ Node struct {+ ID string+ Comments commentNodes `graphql:"... on PullRequestReviewThread"`+ } `graphql:"node(id: $id)"`+ RateLimit rateLimit `graphql:"rateLimit"`+}++// GetThreads returns all review thread nodes+func (r multiReviewThreadComments) GetThreads() []reviewThreadComments {+ threads := make([]reviewThreadComments, 0)+ for _, n := range r.Nodes {+ if n.PullRequestReviewThread.ID != "" {+ threads = append(threads, n.PullRequestReviewThread)+ }+ }++ return threads+}++func (r reviewThreadComments) GetThreadComments() []comment {+ return r.Comments.Nodes+}++// === Issues with comments ===++// issuesWithComments represents a repository issues with comments+type issuesWithComments struct {+ Repository struct {+ Issues issueNodes `graphql:"issues(first: $issuesFirst, after: $issuesAfter, orderBy: {field: UPDATED_AT, direction: DESC})"`+ } `graphql:"repository(owner: $owner, name: $repo)"`+ RateLimit rateLimit `graphql:"rateLimit"`+}++// issueNodes represents a paginated list of issues+type issueNodes struct {+ Nodes []issue+ PageInfo pageInfo+}++// issue represents a single PR issue+type issue struct {+ Number int+ Title string+ Body string+ URL string+ Author author+ CreatedAt time.Time+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"`+}++// singleIssueComments represents a single PR issue with comments+type singleIssueComments struct {+ Repository struct {+ Issue struct {+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"`+ } `graphql:"issue(number: $number)"`+ } `graphql:"repository(owner: $owner, name: $repo)"`+ RateLimit rateLimit+}++func (i issuesWithComments) GetIssues() []issue {+ return i.Repository.Issues.Nodes+}++func (i issue) GetIssueComments() []comment {+ return i.Comments.Nodes+}++func (i singleIssueComments) GetIssueComments() []comment {+ return i.Repository.Issue.Comments.Nodes+}++// === Others ===++type author struct {+ Login string `graphql:"login"`+}++type pageInfo struct {+ HasNextPage bool `graphql:"hasNextPage"`+ EndCursor githubv4.String `graphql:"endCursor"`+}++type rateLimit struct {+ Remaining int `graphql:"remaining"`+ ResetAt time.Time `graphql:"resetAt"`+}