Skip to content

Find, verify, and analyze leaked credentials

Implemented Graphql requests for Github PR's, Issues and comments scanning (#4431)

Kashif Khandaf7e5626bc8
diff --git a/main.go b/main.go
index d212fa7da..992c5b0cb 100644
--- a/main.go
+++ b/main.go
@@ -467,6 +467,9 @@ func run(state overseer.State) {
feature.UseSimplifiedGitlabEnumeration.Store(true)
feature.GitlabProjectsPerPage.Store(100)
+ // OSS Default using github graphql api for issues, pr's and comments
+ feature.UseGithubGraphQLAPI.Store(false)
+
conf := &config.Config{}
if *configFilename != "" {
var err error
diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go
index 3c030a631..a3642b417 100644
--- a/pkg/feature/feature.go
+++ b/pkg/feature/feature.go
@@ -11,6 +11,7 @@ var (
UseSimplifiedGitlabEnumeration atomic.Bool
UseGitMirror atomic.Bool
GitlabProjectsPerPage atomic.Int64
+ UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments
)
type AtomicString struct {
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index b9699a065..b11740b82 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -27,6 +27,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/feature"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -1105,8 +1106,14 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo rep
if s.includeGistComments && isGistUrl(urlParts) && !s.ignoreGists {
return s.processGistComments(ctx, urlString, urlParts, repoInfo, reporter, cutoffTime)
} else if s.includeIssueComments || s.includePRComments {
- return s.processRepoComments(ctx, repoInfo, reporter, cutoffTime)
+ // if we need to use graphql api for repo issues, prs and comments
+ if feature.UseGithubGraphQLAPI.Load() {
+ return s.processRepoIssueandPRsWithCommentsGraphql(ctx, repoInfo, reporter, cutoffTime)
+ }
+
+ return s.processIssueandPRsWithCommentsREST(ctx, repoInfo, reporter, cutoffTime)
}
+
return nil
}
@@ -1273,7 +1280,10 @@ var (
state = "all"
)
-func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+func (s *Source) processIssueandPRsWithCommentsREST(
+ ctx context.Context, repoInfo repoInfo,
+ reporter sources.ChunkReporter, cutoffTime *time.Time,
+) error {
if s.includeIssueComments {
ctx.Logger().V(2).Info("Scanning issues")
if err := s.processIssues(ctx, repoInfo, reporter); err != nil {
@@ -1297,6 +1307,31 @@ func (s *Source) processRepoComments(ctx context.Context, repoInfo repoInfo, rep
return nil
}
+func (s *Source) processRepoIssueandPRsWithCommentsGraphql(
+ ctx context.Context, repoInfo repoInfo,
+ reporter sources.ChunkReporter, cutoffTime *time.Time,
+) error {
+ if s.includeIssueComments {
+ ctx.Logger().V(2).Info("Scanning issues")
+ if err := s.processIssuesWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil {
+ return err
+ }
+ }
+
+ if s.includePRComments {
+ ctx.Logger().V(2).Info("Scanning pull requests")
+ if err := s.processPRWithComments(ctx, repoInfo, reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ if err := s.processReviewThreads(ctx, repoInfo, reporter, cutoffTime); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter) error {
bodyTextsOpts := &github.IssueListByRepoOptions{
Sort: sortType,
diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go
index 42af8f4e2..6e5ce443b 100644
--- a/pkg/sources/github/github_integration_test.go
+++ b/pkg/sources/github/github_integration_test.go
@@ -17,6 +17,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/feature"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -968,6 +969,64 @@ func TestSource_Validate(t *testing.T) {
}
}
+func TestSource_ScanCommentsWithGraphql(t *testing.T) {
+ feature.UseGithubGraphQLAPI.Store(true)
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+
+ source := &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
+ IncludeIssueComments: true,
+ IncludePullRequestComments: true,
+ Credential: &sourcespb.GitHub_Unauthenticated{},
+ }
+
+ wantChunk := sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Link: "https://github.com/trufflesecurity/driftwood.git/issues/1",
+ Username: "truffle-sandbox",
+ Timestamp: "2023-06-22 23:33:46 +0000 UTC",
+ },
+ },
+ },
+ Verify: false,
+ }
+
+ s := Source{}
+
+ conn, err := anypb.New(source)
+ assert.NoError(t, err)
+
+ err = s.Init(ctx, "test-source", 0, 0, false, conn, 4)
+ assert.NoError(t, err)
+
+ chunksCh := make(chan *sources.Chunk, 1)
+ go func() {
+ // Close the channel
+ defer close(chunksCh)
+ err = s.Chunks(ctx, chunksCh)
+ assert.NoError(t, err)
+ }()
+
+ i := 0
+ for gotChunk := range chunksCh {
+ // Skip chunks that are not comments.
+ if gotChunk.SourceMetadata.GetGithub().GetCommit() != "" {
+ continue
+ }
+ i++
+ githubCommentCheckFunc(gotChunk, &wantChunk, i, t, "test-source")
+ }
+
+ // Confirm all comments were processed.
+ assert.Equal(t, i, 5)
+}
+
type countChunkReporter struct {
chunkCount int
errCount int
diff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.go
new file mode 100644
index 000000000..e7e2f08cf
--- /dev/null
+++ b/pkg/sources/github/graphql.go
@@ -0,0 +1,495 @@
+package github
+
+import (
+ "cmp"
+ "fmt"
+ "math/rand/v2"
+ "strings"
+ "time"
+
+ "github.com/shurcooL/githubv4"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+// processIssuesWithComments process github repo issues with comments using graphql API
+func (s *Source) processIssuesWithComments(
+ ctx context.Context, repoInfo repoInfo,
+ reporter sources.ChunkReporter, cutoffTime *time.Time,
+) error {
+ vars := map[string]any{
+ owner: githubv4.String(repoInfo.owner),
+ repository: githubv4.String(repoInfo.name),
+ issuesPerPage: githubv4.Int(defaultPagination),
+ issuesPagination: (*githubv4.String)(nil),
+ commentsPerPage: githubv4.Int(defaultPagination),
+ commentsPagination: (*githubv4.String)(nil),
+ }
+
+ var totalIssues int
+
+ // loop will continue as long as there are issues in the repository
+ for {
+ var query issuesWithComments
+ err := s.connector.GraphQLClient().Query(ctx, &query, vars)
+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {
+ continue
+ }
+
+ if err != nil {
+ return fmt.Errorf("error fetching issues: %w", err)
+ }
+
+ totalIssues += len(query.GetIssues())
+
+ ctx.Logger().V(5).Info("Scanning Issues",
+ "total_issues", len(query.GetIssues()))
+
+ if err := s.chunkGraphqlIssues(ctx, repoInfo, query.Repository.Issues.Nodes, reporter); err != nil {
+ return err
+ }
+
+ // process each issue comments
+ for _, issue := range query.GetIssues() {
+ ctx.Logger().V(5).Info("Scanning Issue Comments",
+ "issue_id", issue.Number,
+ "total_comments", len(issue.GetIssueComments()),
+ )
+
+ if err := s.chunkComments(ctx, repoInfo, issue.GetIssueComments(), reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ // if issue has more than 100 comments, we need to send another request for that specific issue to pull more comments
+ for issue.Comments.PageInfo.HasNextPage {
+ commentVars := map[string]any{
+ owner: githubv4.String(repoInfo.owner),
+ repository: githubv4.String(repoInfo.name),
+ issueNumber: githubv4.Int(issue.Number),
+ commentsPerPage: githubv4.Int(defaultPagination),
+ commentsPagination: issue.Comments.PageInfo.EndCursor,
+ }
+
+ // request this issue more comments
+ var commentsQuery singleIssueComments
+ err := s.connector.GraphQLClient().Query(ctx, &commentsQuery, commentVars)
+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentsQuery.RateLimit, err) {
+ continue
+ }
+
+ if err != nil {
+ return fmt.Errorf("error fetching issue: %w", err)
+ }
+
+ ctx.Logger().V(5).Info("Scanning additional issue comments",
+ "issue_id", issue.Number,
+ "total_comments", len(commentsQuery.GetIssueComments()))
+
+ if err := s.chunkComments(ctx, repoInfo, commentsQuery.GetIssueComments(), reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ // update page info for loop
+ issue.Comments.PageInfo = commentsQuery.Repository.Issue.Comments.PageInfo
+ }
+ }
+
+ // paginate issues
+ if !query.Repository.Issues.PageInfo.HasNextPage {
+ ctx.Logger().V(4).Info("Scanned all repository issues with comments", "total_issues_scanned", totalIssues)
+ break
+ }
+
+ // update issues pagination to go to next page
+ vars[issuesPagination] = githubv4.NewString(query.Repository.Issues.PageInfo.EndCursor)
+ }
+
+ return nil
+}
+
+// processPRWithComments process github repo pull requests with inline comments
+func (s *Source) processPRWithComments(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+ vars := map[string]any{
+ owner: githubv4.String(repoInfo.owner),
+ repository: githubv4.String(repoInfo.name),
+ pullRequestPerPage: githubv4.Int(defaultPagination),
+ pullRequestPagination: (*githubv4.String)(nil),
+ commentsPerPage: githubv4.Int(defaultPagination),
+ commentsPagination: (*githubv4.String)(nil),
+ }
+
+ var totalPRs int
+
+ // continue loop as long as there are pull requests remaining
+ for {
+ var query pullRequestWithComments
+ err := s.connector.GraphQLClient().Query(ctx, &query, vars)
+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {
+ continue
+ }
+
+ if err != nil {
+ return fmt.Errorf("error fetching pull requests with comments: %w", err)
+ }
+
+ totalPRs += len(query.GetPullRequests())
+
+ ctx.Logger().V(5).Info("Scanning pull requests",
+ "total_pull_requests", len(query.GetPullRequests()))
+
+ if err := s.chunkGraphqlPullRequests(ctx, repoInfo, query.GetPullRequests(), reporter); err != nil {
+ return err
+ }
+
+ // process each pr comments
+ for _, pr := range query.GetPullRequests() {
+ ctx.Logger().V(5).Info("Scanning pull request comments",
+ "pull_request_no", pr.Number,
+ "total_comments", len(pr.Comments.Nodes))
+
+ if err := s.chunkComments(ctx, repoInfo, pr.GetPRComments(), reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ // if a pull request has more than 100 comments - some interns pull request might endup here :)
+ for pr.Comments.PageInfo.HasNextPage {
+ // request this pull request more comments
+ var commentQuery singlePRComments
+ singlePRVars := map[string]any{
+ owner: githubv4.String(repoInfo.owner),
+ repository: githubv4.String(repoInfo.name),
+ pullRequestNumber: pr.Number,
+ commentsPerPage: githubv4.Int(defaultPagination),
+ commentsPagination: pr.Comments.PageInfo.EndCursor,
+ }
+
+ err := s.connector.GraphQLClient().Query(ctx, &commentQuery, singlePRVars)
+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &commentQuery.RateLimit, err) {
+ continue
+ }
+
+ if err != nil {
+ return fmt.Errorf("error fetching pull request with comments: %w", err)
+ }
+
+ ctx.Logger().V(5).Info("Scanning additional comments",
+ "pull_request_no", pr.Number,
+ "total_comments", len(commentQuery.GetPRComments()))
+
+ if err := s.chunkComments(ctx, repoInfo, commentQuery.GetPRComments(), reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ // update pr.Comments.PageInfo so loop condition reflects new state
+ pr.Comments.PageInfo = commentQuery.Repository.PullRequest.Comments.PageInfo
+ }
+ }
+
+ // move to next page of PRs
+ if !query.Repository.PullRequests.PageInfo.HasNextPage {
+ ctx.Logger().V(4).Info("Scanned all repository pull requests with comments", "total_pullrequests_scanned", totalPRs)
+ break
+ }
+
+ // update pull request pagination to go to next page
+ vars[pullRequestPagination] = githubv4.NewString(query.Repository.PullRequests.PageInfo.EndCursor)
+ }
+
+ return nil
+}
+
+// processReviewThreads process github repo pull request review threads
+func (s *Source) processReviewThreads(ctx context.Context, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+ vars := map[string]any{
+ owner: githubv4.String(repoInfo.owner),
+ repository: githubv4.String(repoInfo.name),
+ pullRequestPerPage: githubv4.Int(defaultPagination),
+ pullRequestPagination: (*githubv4.String)(nil),
+ threadPerPage: githubv4.Int(defaultPagination),
+ threadPagination: (*githubv4.String)(nil),
+ }
+
+ var threadIDs = make([]string, 0)
+
+ // continue as long as pull requests have threads
+ for {
+ var query prWithReviewThreadIDs
+ err := s.connector.GraphQLClient().Query(ctx, &query, vars)
+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {
+ continue
+ }
+
+ if err != nil {
+ return fmt.Errorf("error fetching pr thread reviews: %w", err)
+ }
+
+ // collect thread ids
+ for _, pr := range query.GetMinimalPullRequests() {
+ prThreadIDs := pr.ReviewThreads.GetThreadIDs()
+ threadIDs = append(threadIDs, prThreadIDs...)
+ }
+
+ if !query.Repository.PullRequests.PageInfo.HasNextPage {
+ ctx.Logger().V(4).Info("Pulled all repository PR's threads IDs")
+ break
+ }
+
+ // update pull request pagination to go to next page
+ vars[pullRequestPagination] = githubv4.NewString(query.Repository.PullRequests.PageInfo.EndCursor)
+ }
+
+ ctx.Logger().V(4).Info("Pulled all thread IDs", "total_threads", len(threadIDs))
+
+ // if we got more than 0 threads unfortunately :( than we need to pull their comments in batches
+ if len(threadIDs) > 0 {
+ // process in batches of max 100
+ for _, batch := range chunkIDs(threadIDs, 100) {
+ ctx.Logger().V(5).Info("Processing Thread comments in Batches", "batch_length", len(batch))
+ // fetch comments for the batch of threads
+ if err := s.fetchThreadComments(ctx, batch, repoInfo, reporter, cutoffTime); err != nil {
+ return fmt.Errorf("error fetching thread review comments: %w", err)
+ }
+ }
+ }
+
+ return nil
+}
+
+// fetchThreadComments process github repo pull request threads and their comments
+func (s *Source) fetchThreadComments(ctx context.Context, threadIDs []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+ // Process in batches of 100
+ var query multiReviewThreadComments
+ vars := map[string]any{
+ "ids": threadIDs,
+ commentsPerPage: githubv4.Int(defaultPagination),
+ commentsPagination: (*githubv4.String)(nil),
+ }
+
+ if err := s.connector.GraphQLClient().Query(ctx, &query, vars); err != nil {
+ return fmt.Errorf("multi-thread query failed: %w", err)
+ }
+
+ // process each thread in batch
+ for _, thread := range query.GetThreads() {
+ if err := s.chunkComments(ctx, repoInfo, thread.GetThreadComments(), reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ // if a thread has more than 100 comments :)
+ for thread.Comments.PageInfo.HasNextPage {
+ // request this thread more comments
+ var query singleReviewThreadComments
+ reviewThreadVars := map[string]any{
+ threadID: thread.ID,
+ commentsPerPage: githubv4.Int(defaultPagination),
+ commentsPagination: (*githubv4.String)(nil),
+ }
+
+ err := s.connector.GraphQLClient().Query(ctx, &query, reviewThreadVars)
+ if s.handleGraphqlRateLimitWithChunkReporter(ctx, reporter, &query.RateLimit, err) {
+ continue
+ }
+
+ if err != nil {
+ return fmt.Errorf("single-thread query failed: %w", err)
+ }
+
+ node := query.Node
+ if err := s.chunkComments(ctx, repoInfo, node.Comments.Nodes, reporter, cutoffTime); err != nil {
+ return err
+ }
+
+ if !node.Comments.PageInfo.HasNextPage {
+ break
+ }
+
+ // update thread comments pagination
+ reviewThreadVars[commentsPagination] = &node.Comments.PageInfo.EndCursor
+ }
+ }
+
+ return nil
+}
+
+// chunkIDs splits a slice of IDs into multiple chunks of at most `size` elements.
+// For example, if you have 250 IDs and size=100, this will return
+//
+// [][]string{
+// {id0 ... id99}, // first 100
+// {id100 ... id199},// next 100
+// {id200 ... id249} // last 50
+// }
+func chunkIDs(ids []string, size int) [][]string {
+ var chunks [][]string
+ for size < len(ids) {
+ ids, chunks = ids[size:], append(chunks, ids[0:size:size])
+ }
+ return append(chunks, ids)
+}
+
+// handleRateLimitWithChunkReporter is a wrapper around handleRateLimit that includes chunk reporting
+func (s *Source) handleGraphqlRateLimitWithChunkReporter(
+ ctx context.Context, reporter sources.ChunkReporter,
+ rl *rateLimit, errIn error,
+) bool {
+ return s.handleGraphQLRateLimit(ctx, rl, errIn, &chunkErrorReporter{reporter: reporter})
+}
+
+// handleGraphQLRateLimit inspects the rateLimit info returned in GraphQL queries.
+func (s *Source) handleGraphQLRateLimit(ctx context.Context, rl *rateLimit, errIn error, reporters ...errorReporter) bool {
+ // check global resume time first (in case another request already set it)
+ rateLimitMu.RLock()
+ resumeTime := rateLimitResumeTime
+ rateLimitMu.RUnlock()
+
+ // if resume time is not empty and is after current time, than put the request to sleep till that.
+ if !resumeTime.IsZero() && time.Now().Before(resumeTime) {
+ retryAfter := time.Until(resumeTime)
+ time.Sleep(retryAfter)
+ return true
+ }
+
+ var retryAfter time.Duration
+ if errIn != nil && strings.Contains(errIn.Error(), "rate limit exceeded") {
+ now := time.Now()
+
+ rateLimitMu.Lock()
+ rateLimitResumeTime = now.Add(1 * time.Minute)
+ retryAfter = time.Until(rateLimitResumeTime)
+ ctx.Logger().Info("GraphQL RATE_LIMITED error (fallback)",
+ "retry_after", retryAfter.String())
+ rateLimitMu.Unlock()
+ } else if rl != nil {
+ // if rate limit remaining is more than 3 continue using graphql api
+ if rl.Remaining > 3 {
+ return false
+ }
+
+ // === only reach here if error is nil and rate limit remaining is less than 3 (safety)
+ now := time.Now()
+ retryAfter = time.Until(rl.ResetAt)
+ // never negative and enforce a sane minimum backoff (avoid thrashing with 1s/2s retries)
+ if cmp.Less(retryAfter, 5*time.Second) {
+ retryAfter = 5 * time.Second
+ }
+
+ jitter := time.Duration(rand.IntN(10)+1) * time.Second
+ retryAfter += jitter
+
+ // update global resume time
+ rateLimitMu.Lock()
+ rateLimitResumeTime = now.Add(retryAfter)
+ ctx.Logger().Info("exceeded GraphQL rate limit",
+ "retry_after", retryAfter.String(),
+ "resume_time", rateLimitResumeTime.Format(time.RFC3339))
+ rateLimitMu.Unlock()
+
+ for _, reporter := range reporters {
+ _ = reporter.Err(ctx, fmt.Errorf("exceeded GraphQL rate limit"))
+ }
+ }
+
+ githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
+ time.Sleep(retryAfter)
+ githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
+
+ return true
+}
+
+func (s *Source) chunkGraphqlIssues(ctx context.Context, repoInfo repoInfo, issues []issue, reporter sources.ChunkReporter) error {
+ for _, issue := range issues {
+ // Create chunk and send it to the channel.
+ chunk := sources.Chunk{
+ SourceName: s.name,
+ SourceID: s.SourceID(),
+ JobID: s.JobID(),
+ SourceType: s.Type(),
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Link: sanitizer.UTF8(issue.URL),
+ Username: sanitizer.UTF8(issue.Author.Login),
+ Repository: sanitizer.UTF8(repoInfo.fullName),
+ Timestamp: sanitizer.UTF8(issue.CreatedAt.String()),
+ Visibility: repoInfo.visibility,
+ },
+ },
+ },
+ Data: []byte(sanitizer.UTF8(issue.Title + "\n" + issue.Body)),
+ Verify: s.verify,
+ }
+
+ if err := reporter.ChunkOk(ctx, chunk); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *Source) chunkComments(ctx context.Context, repoInfo repoInfo, comments []comment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
+ for _, comment := range comments {
+ // Stop processing comments as soon as one created before the cutoff time is detected, as these are sorted
+ if cutoffTime != nil && comment.UpdatedAt.Before(*cutoffTime) {
+ continue
+ }
+
+ // Create chunk and send it to the channel.
+ chunk := sources.Chunk{
+ SourceName: s.name,
+ SourceID: s.SourceID(),
+ JobID: s.JobID(),
+ SourceType: s.Type(),
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Link: sanitizer.UTF8(comment.URL),
+ Username: sanitizer.UTF8(comment.Author.Login),
+ Repository: sanitizer.UTF8(repoInfo.fullName),
+ Timestamp: sanitizer.UTF8(comment.CreatedAt.String()),
+ Visibility: repoInfo.visibility,
+ },
+ },
+ },
+ Data: []byte(sanitizer.UTF8(comment.Body)),
+ Verify: s.verify,
+ }
+
+ if err := reporter.ChunkOk(ctx, chunk); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *Source) chunkGraphqlPullRequests(ctx context.Context, repoInfo repoInfo, prs []pullRequest, reporter sources.ChunkReporter) error {
+ for _, pr := range prs {
+ // Create chunk and send it to the channel.
+ chunk := sources.Chunk{
+ SourceName: s.name,
+ SourceID: s.SourceID(),
+ SourceType: s.Type(),
+ JobID: s.JobID(),
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Link: sanitizer.UTF8(pr.URL),
+ Username: sanitizer.UTF8(pr.Author.Login),
+ Repository: sanitizer.UTF8(repoInfo.fullName),
+ Timestamp: sanitizer.UTF8(pr.CreatedAt.String()),
+ Visibility: repoInfo.visibility,
+ },
+ },
+ },
+ Data: []byte(sanitizer.UTF8(pr.Title + "\n" + pr.Body)),
+ Verify: s.verify,
+ }
+
+ if err := reporter.ChunkOk(ctx, chunk); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/pkg/sources/github/queries.go b/pkg/sources/github/queries.go
new file mode 100644
index 000000000..dacaa7ad4
--- /dev/null
+++ b/pkg/sources/github/queries.go
@@ -0,0 +1,248 @@
+package github
+
+import (
+ "time"
+
+ "github.com/shurcooL/githubv4"
+)
+
+const (
+ // query variable keys
+ owner = "owner"
+ repository = "repo"
+
+ pullRequestNumber = "number"
+ pullRequestPerPage = "first"
+ pullRequestPagination = "after"
+
+ commentsPerPage = "commentsFirst"
+ commentsPagination = "commentsAfter"
+
+ threadID = "threadID"
+ threadPerPage = "threadsFirst"
+ threadPagination = "threadsAfter"
+
+ issueNumber = "number"
+ issuesPerPage = "issuesFirst"
+ issuesPagination = "issuesAfter"
+)
+
+// === Pull Requests with Comments ===
+
+// pullRequestWithComments represent a repository pull request nodes
+type pullRequestWithComments struct {
+ Repository struct {
+ PullRequests pullRequestNodes `graphql:"pullRequests(first: $first, after: $after, orderBy: {field: UPDATED_AT, direction: DESC})"`
+ } `graphql:"repository(owner: $owner, name: $repo)"`
+ RateLimit rateLimit `graphql:"rateLimit"`
+}
+
+// pullRequestNodes represents a paginated list of pull requests
+type pullRequestNodes struct {
+ Nodes []pullRequest
+ PageInfo pageInfo
+}
+
+// pullRequest represents a single pull request with comment nodes
+type pullRequest struct {
+ Title string
+ Number int
+ URL string
+ Author author
+ CreatedAt time.Time
+ Body string
+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter, orderBy: {field: UPDATED_AT, direction: DESC})"`
+}
+
+// commentNodes represents a paginated list of comments
+type commentNodes struct {
+ Nodes []comment
+ PageInfo pageInfo
+}
+
+// comment represents a single comment
+type comment struct {
+ Body string
+ CreatedAt time.Time
+ UpdatedAt time.Time
+ Author author
+ URL string
+}
+
+// singlePRComments represents a single PR comments
+type singlePRComments struct {
+ Repository struct {
+ PullRequest struct {
+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter, orderBy: {field: UPDATED_AT, direction: DESC})"`
+ } `graphql:"pullRequest(number: $number)"`
+ } `graphql:"repository(owner: $owner, name: $repo)"`
+ RateLimit rateLimit `graphql:"rateLimit"`
+}
+
+// GetPullRequests return list of pull requests
+func (p pullRequestWithComments) GetPullRequests() []pullRequest {
+ return p.Repository.PullRequests.Nodes
+}
+
+// GetPRComments return list of comments of the PR
+func (p pullRequest) GetPRComments() []comment {
+ return p.Comments.Nodes
+}
+
+// GetPRComments return list of comments of the PR
+func (p singlePRComments) GetPRComments() []comment {
+ return p.Repository.PullRequest.Comments.Nodes
+}
+
+// === Pull Request Review Threads IDs ===
+
+// prWithReviewThreadIDs represents repository pull requests with review threads IDs
+type prWithReviewThreadIDs struct {
+ Repository struct {
+ PullRequests minimalPullRequestNodes `graphql:"pullRequests(first: $first, after: $after, orderBy: {field: UPDATED_AT, direction: DESC})"`
+ } `graphql:"repository(owner: $owner, name: $repo)"`
+ RateLimit rateLimit `graphql:"rateLimit"`
+}
+
+// minimalPullRequestNodes represents a paginated list of pull request
+type minimalPullRequestNodes struct {
+ Nodes []minimalPullRequest
+ PageInfo pageInfo
+}
+
+// minimalPullRequest represents a pull request with no and review threads
+type minimalPullRequest struct {
+ Number int
+ ReviewThreads reviewThreadIDNodes `graphql:"reviewThreads(first: $threadsFirst, after: $threadsAfter)"`
+}
+
+// reviewThreadIDNodes represents a paginated list of pr review threads
+type reviewThreadIDNodes struct {
+ Nodes []reviewThreadID
+ PageInfo pageInfo
+}
+
+// reviewThreadID represents a single review thread with ID
+type reviewThreadID struct {
+ ID string
+}
+
+func (p prWithReviewThreadIDs) GetMinimalPullRequests() []minimalPullRequest {
+ return p.Repository.PullRequests.Nodes
+}
+
+func (r reviewThreadIDNodes) GetThreadIDs() []string {
+ var ids = make([]string, 0)
+
+ for _, id := range r.Nodes {
+ ids = append(ids, id.ID)
+ }
+
+ return ids
+}
+
+// === Review Threads with comments ===
+
+// multiReviewThreadComments fetches multiple review threads (by IDs) and the first page of their comments.
+type multiReviewThreadComments struct {
+ Nodes []struct {
+ PullRequestReviewThread reviewThreadComments `graphql:"... on PullRequestReviewThread"`
+ } `graphql:"nodes(ids: $ids)"`
+ RateLimit rateLimit `graphql:"rateLimit"`
+}
+
+// reviewThreadComments represents a single thread and its comments.
+type reviewThreadComments struct {
+ ID string
+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"`
+}
+
+// singleReviewThreadComments fetches one review thread with comment pagination
+type singleReviewThreadComments struct {
+ Node struct {
+ ID string
+ Comments commentNodes `graphql:"... on PullRequestReviewThread"`
+ } `graphql:"node(id: $id)"`
+ RateLimit rateLimit `graphql:"rateLimit"`
+}
+
+// GetThreads returns all review thread nodes
+func (r multiReviewThreadComments) GetThreads() []reviewThreadComments {
+ threads := make([]reviewThreadComments, 0)
+ for _, n := range r.Nodes {
+ if n.PullRequestReviewThread.ID != "" {
+ threads = append(threads, n.PullRequestReviewThread)
+ }
+ }
+
+ return threads
+}
+
+func (r reviewThreadComments) GetThreadComments() []comment {
+ return r.Comments.Nodes
+}
+
+// === Issues with comments ===
+
+// issuesWithComments represents a repository issues with comments
+type issuesWithComments struct {
+ Repository struct {
+ Issues issueNodes `graphql:"issues(first: $issuesFirst, after: $issuesAfter, orderBy: {field: UPDATED_AT, direction: DESC})"`
+ } `graphql:"repository(owner: $owner, name: $repo)"`
+ RateLimit rateLimit `graphql:"rateLimit"`
+}
+
+// issueNodes represents a paginated list of issues
+type issueNodes struct {
+ Nodes []issue
+ PageInfo pageInfo
+}
+
+// issue represents a single PR issue
+type issue struct {
+ Number int
+ Title string
+ Body string
+ URL string
+ Author author
+ CreatedAt time.Time
+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"`
+}
+
+// singleIssueComments represents a single PR issue with comments
+type singleIssueComments struct {
+ Repository struct {
+ Issue struct {
+ Comments commentNodes `graphql:"comments(first: $commentsFirst, after: $commentsAfter)"`
+ } `graphql:"issue(number: $number)"`
+ } `graphql:"repository(owner: $owner, name: $repo)"`
+ RateLimit rateLimit
+}
+
+func (i issuesWithComments) GetIssues() []issue {
+ return i.Repository.Issues.Nodes
+}
+
+func (i issue) GetIssueComments() []comment {
+ return i.Comments.Nodes
+}
+
+func (i singleIssueComments) GetIssueComments() []comment {
+ return i.Repository.Issue.Comments.Nodes
+}
+
+// === Others ===
+
+type author struct {
+ Login string `graphql:"login"`
+}
+
+type pageInfo struct {
+ HasNextPage bool `graphql:"hasNextPage"`
+ EndCursor githubv4.String `graphql:"endCursor"`
+}
+
+type rateLimit struct {
+ Remaining int `graphql:"remaining"`
+ ResetAt time.Time `graphql:"resetAt"`
+}