RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat[syncer]: add checkpoint resume run for Github (#18260)
Haruko386d77b514d3e01
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.goindex 7984c192c3..323f38c385 100644--- a/internal/syncer/connector/github.go+++ b/internal/syncer/connector/github.go@@ -99,7 +99,9 @@ func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (Syif err != nil {return nil, err}- return &githubSyncSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1, windowStart: request.WindowStart, windowEnd: request.WindowEnd}, nil+ session := &githubSyncSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1, windowStart: request.WindowStart, windowEnd: request.WindowEnd}+ session.applyResume(request.Resume)+ return session, nil}// OpenPrune opens one complete GitHub prune snapshot session.@@ -151,14 +153,15 @@ func (c *GitHubConnector) listRepoEndpoint(ctx context.Context, path string) ([]}// listPullRequestPage returns one page of GitHub pull requests.-func (c *GitHubConnector) listPullRequestPage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]SourceDocument, bool, error) {+func (c *GitHubConnector) listPullRequestPage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]githubBufferedDocument, bool, error) {var batch []githubPullRequestheaders, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/pulls", githubListQuery(page, pageSize)), &batch)if err != nil {return nil, false, err}- documents := make([]SourceDocument, 0, len(batch))+ documents := make([]githubBufferedDocument, 0, len(batch))doneByWindow := false+ pageOffset := 0for _, pr := range batch {if beforeOrAtWindowStart(pr.UpdatedAt, windowStart) {doneByWindow = true@@ -167,21 +170,29 @@ func (c *GitHubConnector) listPullRequestPage(ctx context.Context, fullName striif afterWindowEnd(pr.UpdatedAt, windowEnd) {continue}- documents = append(documents, pr.toSourceDocument(fullName))+ doc := pr.toSourceDocument(fullName)+ pageOffset+++ documents = append(documents, githubBufferedDocument{+ document: doc,+ checkpoint: githubSyncCheckpoint(fullName, githubStagePRs, page, pageOffset, doc),+ offset: pageOffset,+ sourceID: doc.SourceID,+ })}done := doneByWindow || !hasNextPage(headers) || len(batch) == 0return documents, done, nil}// listIssuePage returns one page of GitHub issues.-func (c *GitHubConnector) listIssuePage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]SourceDocument, bool, error) {+func (c *GitHubConnector) listIssuePage(ctx context.Context, fullName string, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]githubBufferedDocument, bool, error) {var batch []githubIssueheaders, err := c.getJSON(ctx, c.apiURL("/repos/"+fullName+"/issues", githubListQuery(page, pageSize)), &batch)if err != nil {return nil, false, err}- documents := make([]SourceDocument, 0, len(batch))+ documents := make([]githubBufferedDocument, 0, len(batch))doneByWindow := false+ pageOffset := 0for _, issue := range batch {if issue.PullRequest != nil {continue@@ -193,7 +204,14 @@ func (c *GitHubConnector) listIssuePage(ctx context.Context, fullName string, paif afterWindowEnd(issue.UpdatedAt, windowEnd) {continue}- documents = append(documents, issue.toSourceDocument(fullName))+ doc := issue.toSourceDocument(fullName)+ pageOffset+++ documents = append(documents, githubBufferedDocument{+ document: doc,+ checkpoint: githubSyncCheckpoint(fullName, githubStageIssues, page, pageOffset, doc),+ offset: pageOffset,+ sourceID: doc.SourceID,+ })}done := doneByWindow || !hasNextPage(headers) || len(batch) == 0return documents, done, nil@@ -287,26 +305,35 @@ func (c *GitHubConnector) apiURL(path string, query url.Values) string {}type githubSyncSession struct {- connector *GitHubConnector- repos []githubRepo- repoIndex int- stage string- page int- batchSize int- windowStart *time.Time- windowEnd time.Time- buffer []SourceDocument+ connector *GitHubConnector+ repos []githubRepo+ repoIndex int+ stage string+ page int+ batchSize int+ windowStart *time.Time+ windowEnd time.Time+ buffer []githubBufferedDocument+ resumeRepo string+ resumeStage string+ resumePage int+ resumeOffset int+ resumeSourceID string}// NextBatch returns the next GitHub document batch.func (s *githubSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpointif len(s.buffer) > 0 {n := s.batchSizeif n > len(s.buffer) {n = len(s.buffer)}- documents = append(documents, s.buffer[:n]...)+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }s.buffer = s.buffer[n:]}@@ -323,13 +350,19 @@ func (s *githubSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {}remaining := s.batchSize - len(documents)if len(batch) > remaining {- documents = append(documents, batch[:remaining]...)+ for _, buffered := range batch[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }s.buffer = append(s.buffer, batch[remaining:]...)break}- documents = append(documents, batch...)+ for _, buffered := range batch {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }}- return SyncBatch{Documents: documents}, nil+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil}// Close closes the GitHub sync session.@@ -391,7 +424,7 @@ const ()// nextDocumentPage fetches one GitHub API page for sync.-func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocument, error) {+func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]githubBufferedDocument, error) {repo := s.repos[s.repoIndex]switch s.stage {case githubStagePRs:@@ -403,6 +436,7 @@ func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocumif err != nil {return nil, err}+ docs = s.filterResumedDocuments(repo.FullName, githubStagePRs, s.page, docs)if done {s.advanceStage()} else {@@ -418,6 +452,7 @@ func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]SourceDocumif err != nil {return nil, err}+ docs = s.filterResumedDocuments(repo.FullName, githubStageIssues, s.page, docs)if done {s.advanceRepo()} else {@@ -474,6 +509,7 @@ func (s *githubPruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument,func (s *githubSyncSession) advanceStage() {s.stage = githubStageIssuess.page = 1+ s.clearResume()}// advanceRepo moves a GitHub session to the next repository.@@ -481,6 +517,71 @@ func (s *githubSyncSession) advanceRepo() {s.repoIndex++s.stage = githubStagePRss.page = 1+ s.clearResume()+}++// applyResume advances a sync session to the last committed GitHub page.+func (s *githubSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }++ var cursor githubSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return+ }+ if cursor.Repo == "" || cursor.Stage == "" || cursor.Page <= 0 {+ return+ }+ for index, repo := range s.repos {+ if repo.FullName != cursor.Repo {+ continue+ }+ s.repoIndex = index+ s.stage = cursor.Stage+ s.page = cursor.Page+ s.resumeRepo = cursor.Repo+ s.resumeStage = cursor.Stage+ s.resumePage = cursor.Page+ s.resumeOffset = cursor.Offset+ s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)+ return+ }+}++// filterResumedDocuments drops documents through the committed checkpoint.+func (s *githubSyncSession) filterResumedDocuments(repo, stage string, page int, candidates []githubBufferedDocument) []githubBufferedDocument {+ if s.resumeRepo == "" || repo != s.resumeRepo || stage != s.resumeStage || page != s.resumePage {+ return candidates+ }+ if s.resumeSourceID != "" {+ for index, candidate := range candidates {+ if candidate.sourceID == s.resumeSourceID {+ s.clearResume()+ return candidates[index+1:]+ }+ }+ }+ if s.resumeOffset <= 0 {+ s.clearResume()+ return candidates+ }+ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset > s.resumeOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResume()+ return filtered+}++func (s *githubSyncSession) clearResume() {+ s.resumeRepo = ""+ s.resumeStage = ""+ s.resumePage = 0+ s.resumeOffset = 0+ s.resumeSourceID = ""}// advanceStage moves a GitHub prune session from PRs to issues.@@ -500,6 +601,40 @@ type githubRepo struct {FullName string `json:"full_name"`}+type githubSyncCursor struct {+ Repo string `json:"repo"`+ Stage string `json:"stage"`+ Page int `json:"page"`+ Offset int `json:"offset"`+ SourceID string `json:"source_id"`+}++type githubBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+ offset int+ sourceID string+}++func githubSyncCheckpoint(repo, stage string, page, offset int, doc SourceDocument) *SyncCheckpoint {+ cursor, err := json.Marshal(githubSyncCursor{+ Repo: repo,+ Stage: stage,+ Page: page,+ Offset: offset,+ SourceID: doc.SourceID,+ })+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: string(cursor),+ SourceID: doc.SourceID,+ UpdatedAt: &updatedAt,+ }+}+type githubPullRequest struct {HTMLURL string `json:"html_url"`Number int `json:"number"`diff --git a/internal/syncer/connector/github_test.go b/internal/syncer/connector/github_test.goindex 18aa5915cf..afd40ede6d 100644--- a/internal/syncer/connector/github_test.go+++ b/internal/syncer/connector/github_test.go@@ -134,6 +134,123 @@ func TestGitHubConnectorOpenPrune(t *testing.T) {}}+// TestGitHubConnectorOpenSyncResumesAfterCheckpoint verifies retry skips committed GitHub documents.+func TestGitHubConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "ragflow",+ "include_pull_requests": true,+ "include_issues": true,+ "batch_size": 1,+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubFixtureDoJSON(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "https://github.com/openai/ragflow/pull/7" {+ t.Fatalf("first documents = %+v, want PR 7", first.Documents)+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "https://github.com/openai/ragflow/pull/7" {+ t.Fatalf("first checkpoint = %+v, want PR 7", first.Checkpoint)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://github.com/openai/ragflow/issues/3" {+ t.Fatalf("resume documents = %+v, want issue 3", second.Documents)+ }+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "https://github.com/openai/ragflow/issues/3" {+ t.Fatalf("resume checkpoint = %+v, want issue 3", second.Checkpoint)+ }+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resume EOF = %v", err)+ }+}++// TestGitHubConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument verifies offset fallback excludes the committed offset.+func TestGitHubConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {+ connector, err := NewGitHubConnector(map[string]any{+ "repository_owner": "openai",+ "repository_name": "ragflow",+ "include_pull_requests": true,+ "include_issues": true,+ "batch_size": 1,+ "credentials": map[string]any{"github_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitHubConnector failed: %v", err)+ }+ connector.baseURL = "https://api.github.test"+ connector.doJSON = githubFixtureDoJSON(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "https://github.com/openai/ragflow/pull/7" {+ t.Fatalf("first documents = %+v, want PR 7", first.Documents)+ }+ if first.Checkpoint == nil {+ t.Fatalf("first checkpoint is nil")+ }++ resumeCheckpoint := cloneGitHubCheckpointWithMissingSourceID(t, first.Checkpoint)+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://github.com/openai/ragflow/issues/3" {+ t.Fatalf("resume documents = %+v, want issue 3", second.Documents)+ }+ if second.Documents[0].SourceID == first.Documents[0].SourceID {+ t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)+ }+}++func cloneGitHubCheckpointWithMissingSourceID(t *testing.T, checkpoint *SyncCheckpoint) *SyncCheckpoint {+ t.Helper()+ var cursor githubSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ t.Fatalf("decode checkpoint cursor: %v", err)+ }+ cursor.SourceID = ""+ data, err := json.Marshal(cursor)+ if err != nil {+ t.Fatalf("encode checkpoint cursor: %v", err)+ }+ clone := *checkpoint+ clone.Cursor = string(data)+ clone.SourceID = ""+ return &clone+}+// githubFixtureDoJSON returns a fixture GitHub JSON transport.func githubFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {t.Helper()