Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feat: sync anchor resume (#18795)

mkaaad73cdb81bded0
diff --git a/internal/syncer/README.md b/internal/syncer/README.md
index dd1a9026cc..d1b8c9a325 100644
--- a/internal/syncer/README.md
+++ b/internal/syncer/README.md
@@ -439,7 +439,7 @@ The models are defined in `internal/syncer/connector/models.go`.
Here "retry/resume" refers to task-level continuation at batch/document granularity, not resumable byte-range downloads of a single HTTP download.
> [!NOTE]
-> The current design guarantees continuation from after the last successfully committed batch after a task failure. It does not guarantee that a partially downloaded remote file resumes from a byte range.
+> The current design continues from after the last successfully committed batch only while the saved anchor still exists in the current source listing. It does not guarantee that a partially downloaded remote file resumes from a byte range.
Happy path:
@@ -457,6 +457,17 @@ Resume after failure:
- The last batch that was not committed successfully is reprocessed. This is expected behavior.
- The upsert path must stay idempotent; reprocessing the same `SourceID` must not produce duplicate documents.
+Anchor invalidation:
+
+Continuation is only trustworthy while the saved source anchor still exists in
+the current listing. If a connector returns `ErrSyncResumeInvalid` from
+`OpenSync` or `NextBatch`, `SyncRunner` clears the connector checkpoint, resets
+the accumulated stats, and restarts the same fixed sync window from the
+beginning with `Resume=nil`. The restart is bounded by
+`MaxAnchorRestartCount` (default 2); after that the task fails rather than
+guessing a new offset. Connectors enforce this through
+`ErrSyncResumeInvalid` in `connector/models.go`.
+
Requirements when implementing checkpoints:
- Batch output order must be stable. Common approaches: sort by update time, remote pagination order, or source ID.
@@ -567,4 +578,3 @@ Tests that need real MySQL, MinIO, Elasticsearch, Infinity, LLMs, or external Sa
> + [feat[Go]: complete the base for data Syncer - #17890](https://github.com/infiniflow/ragflow/pull/17890)
> + [feat[Go]: monitoring NATs and refactoring concurrency logic - #18049](https://github.com/infiniflow/ragflow/pull/18049)
> + [feat[Go]: resuming transmission from the point of interruption during data source synchronisation - #18176](https://github.com/infiniflow/ragflow/pull/18176)
-
diff --git a/internal/syncer/config.go b/internal/syncer/config.go
index b87656b2b6..de226cf5e0 100644
--- a/internal/syncer/config.go
+++ b/internal/syncer/config.go
@@ -22,23 +22,25 @@ import (
// Config contains runtime limits for the datasource syncer.
type Config struct {
- TaskWorkerCount int
- TaskQueueSize int
- JobWorkerCount int
- JobQueueSize int
- ItemRetryCount int
- ItemRetryBaseDelay time.Duration
+ TaskWorkerCount int
+ TaskQueueSize int
+ JobWorkerCount int
+ JobQueueSize int
+ ItemRetryCount int
+ ItemRetryBaseDelay time.Duration
+ MaxAnchorRestartCount int
}
// DefaultConfig returns the first-version syncer defaults.
func DefaultConfig() Config {
return Config{
- TaskWorkerCount: 5,
- TaskQueueSize: 10,
- JobWorkerCount: 400,
- JobQueueSize: 450,
- ItemRetryCount: 3,
- ItemRetryBaseDelay: time.Second,
+ TaskWorkerCount: 5,
+ TaskQueueSize: 10,
+ JobWorkerCount: 400,
+ JobQueueSize: 450,
+ ItemRetryCount: 3,
+ ItemRetryBaseDelay: time.Second,
+ MaxAnchorRestartCount: 2,
}
}
@@ -69,5 +71,9 @@ func (c Config) Normalize() Config {
c.ItemRetryBaseDelay = def.ItemRetryBaseDelay
}
+ if c.MaxAnchorRestartCount <= 0 {
+ c.MaxAnchorRestartCount = def.MaxAnchorRestartCount
+ }
+
return c
}
diff --git a/internal/syncer/connector/azure_blob.go b/internal/syncer/connector/azure_blob.go
index 143899306d..964fa343f8 100644
--- a/internal/syncer/connector/azure_blob.go
+++ b/internal/syncer/connector/azure_blob.go
@@ -180,7 +180,9 @@ func (c *AzureBlobStorageConnector) OpenSync(ctx context.Context, request SyncRe
batchSize: c.batchSize,
}
if request.Resume != nil {
- session.applyResume(request.Resume)
+ if err := session.applyResume(ctx, request.Resume); err != nil {
+ return nil, err
+ }
}
return session, nil
}
@@ -512,10 +514,39 @@ func (s *azureBlobSyncSession) Fetch(ctx context.Context, ref FetchReference) ([
}
// applyResume configures the session to skip every blob at or before the
-// checkpoint cursor. Azure lists blobs in lexicographic name order, so skipping
-// <= cursor and re-listing guarantees no blob is dropped across a resume.
-func (s *azureBlobSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- s.skipUntil = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+// checkpoint anchor. The anchor must still exist in the current container
+// listing; a missing anchor means the saved progress no longer describes the
+// same source state.
+func (s *azureBlobSyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ anchor := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ if anchor == "" {
+ return fmt.Errorf("azure blob sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+
+ marker := ""
+ for {
+ objects, next, hasMore, err := s.connector.listBlobPage(ctx, s.connector.prefix, marker, int32(s.batchSize))
+ if err != nil {
+ return err
+ }
+ for _, object := range objects {
+ if object.Name == anchor {
+ s.skipUntil = anchor
+ return nil
+ }
+ if object.Name > anchor {
+ return fmt.Errorf("azure blob resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
+ }
+ }
+ if !hasMore {
+ break
+ }
+ marker = next
+ }
+ return fmt.Errorf("azure blob resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
}
type azureBlobPruneSession struct {
diff --git a/internal/syncer/connector/azure_blob_test.go b/internal/syncer/connector/azure_blob_test.go
index f2689fb17c..e46c34c2a3 100644
--- a/internal/syncer/connector/azure_blob_test.go
+++ b/internal/syncer/connector/azure_blob_test.go
@@ -362,6 +362,41 @@ func TestAzureBlobStorageConnectorResumeSkipsBlobs(t *testing.T) {
}
}
+func TestAzureBlobStorageConnectorResumeRejectsMissingAnchor(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }, []azureBlobObject{
+ {Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Name: "c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: "b.txt", SourceID: "b.txt"},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
+func TestAzureBlobStorageConnectorResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }, []azureBlobObject{
+ {Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestAzureBlobStorageConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
connector := newTestAzureBlobStorageConnector(t, map[string]any{
"auth_mode": "account_key",
diff --git a/internal/syncer/connector/azure_devops.go b/internal/syncer/connector/azure_devops.go
index b5944ea6d1..a1976d3951 100644
--- a/internal/syncer/connector/azure_devops.go
+++ b/internal/syncer/connector/azure_devops.go
@@ -306,7 +306,9 @@ func (c *AzureDevOpsConnector) OpenSync(ctx context.Context, request SyncRequest
request: request,
stage: c.initialStage(),
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -973,7 +975,9 @@ func (s *azureDevOpsSyncSession) nextCodeDocuments(ctx context.Context, repo azu
}
s.items = items
s.itemsRepo = repo.Key()
- s.applyFileAnchor(repo)
+ if err := s.applyFileAnchor(repo); err != nil {
+ return nil, err
+ }
}
documents := make([]SourceDocument, 0, limit)
@@ -1005,28 +1009,28 @@ func (s *azureDevOpsSyncSession) nextCodeDocuments(ctx context.Context, repo azu
//
// The listing is re-fetched on every resume and can have shifted since the
// checkpoint was written, so the stored offset is treated as a hint: it is
-// checked first, then the whole listing is searched for the anchor. Only when
-// the anchor is gone — the file was deleted or renamed — does the offset stand
-// on its own.
-func (s *azureDevOpsSyncSession) applyFileAnchor(repo azureDevOpsRepository) {
+// checked first, then the whole listing is searched for the anchor. A missing
+// anchor means the saved progress no longer maps to the source listing.
+func (s *azureDevOpsSyncSession) applyFileAnchor(repo azureDevOpsRepository) error {
if s.resumeSourceID == "" || s.resumeRepoKey != repo.Key() || s.resumeStage != azureDevOpsStageCode {
- return
+ return nil
}
defer s.clearResume()
if s.fileOffset > 0 && s.fileOffset <= len(s.items) {
previous := s.items[s.fileOffset-1]
if azureDevOpsCodeSourceID(s.connector.organization, repo, strings.TrimPrefix(previous.Path, "/")) == s.resumeSourceID {
- return
+ return nil
}
}
for index, item := range s.items {
if azureDevOpsCodeSourceID(s.connector.organization, repo, strings.TrimPrefix(item.Path, "/")) == s.resumeSourceID {
s.fileOffset = index + 1
- return
+ return nil
}
}
+ return fmt.Errorf("azure devops file resume anchor %q was not found in repo %s: %w", s.resumeSourceID, repo.Key(), ErrSyncResumeInvalid)
}
// filterResumedPullRequests drops the pull requests already committed.
@@ -1034,18 +1038,18 @@ func (s *azureDevOpsSyncSession) applyFileAnchor(repo azureDevOpsRepository) {
// $skip indexes into a listing that shifts as pull requests are opened, so the
// anchor decides where the page really resumes; the skip value only positions
// the request.
-func (s *azureDevOpsSyncSession) filterResumedPullRequests(repo azureDevOpsRepository, pullRequests []azureDevOpsPullRequest) []azureDevOpsPullRequest {
+func (s *azureDevOpsSyncSession) filterResumedPullRequests(repo azureDevOpsRepository, pullRequests []azureDevOpsPullRequest) ([]azureDevOpsPullRequest, error) {
if s.resumeSourceID == "" || s.resumeRepoKey != repo.Key() || s.resumeStage != azureDevOpsStagePullRequests {
- return pullRequests
+ return pullRequests, nil
}
defer s.clearResume()
for index, pullRequest := range pullRequests {
if azureDevOpsPullRequestSourceID(s.connector.organization, repo, pullRequest.PullRequestID) == s.resumeSourceID {
- return pullRequests[index+1:]
+ return pullRequests[index+1:], nil
}
}
- return pullRequests
+ return nil, fmt.Errorf("azure devops pull request resume anchor %q was not found in repo %s: %w", s.resumeSourceID, repo.Key(), ErrSyncResumeInvalid)
}
func (s *azureDevOpsSyncSession) clearResume() {
@@ -1062,7 +1066,10 @@ func (s *azureDevOpsSyncSession) nextPullRequestDocuments(ctx context.Context, r
return nil, err
}
pageSize := len(pullRequests)
- pullRequests = s.filterResumedPullRequests(repo, pullRequests)
+ pullRequests, err = s.filterResumedPullRequests(repo, pullRequests)
+ if err != nil {
+ return nil, err
+ }
documents := make([]SourceDocument, 0, len(pullRequests))
for _, pullRequest := range pullRequests {
@@ -1109,13 +1116,26 @@ func (s *azureDevOpsSyncSession) advanceRepo() {
}
// applyResume advances the session to the last committed position.
-func (s *azureDevOpsSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *azureDevOpsSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("azure devops sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor azureDevOpsSyncCursor
- if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil || cursor.RepoKey == "" {
- return
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return fmt.Errorf("azure devops sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ if cursor.RepoKey == "" {
+ return fmt.Errorf("azure devops sync cursor has no repo anchor: %w", ErrSyncResumeInvalid)
+ }
+ if cursor.Stage != azureDevOpsStageCode && cursor.Stage != azureDevOpsStagePullRequests {
+ return fmt.Errorf("azure devops sync cursor has no valid stage: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" {
+ return fmt.Errorf("azure devops sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
for index, repo := range s.repos {
if repo.Key() != cursor.RepoKey {
@@ -1129,9 +1149,10 @@ func (s *azureDevOpsSyncSession) applyResume(checkpoint *SyncCheckpoint) {
s.prSkip = cursor.PRSkip
s.resumeRepoKey = cursor.RepoKey
s.resumeStage = s.stage
- s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
- return
+ s.resumeSourceID = sourceID
+ return nil
}
+ return fmt.Errorf("azure devops resume repo %q was not found in the current listing: %w", cursor.RepoKey, ErrSyncResumeInvalid)
}
type azureDevOpsPruneSession struct {
diff --git a/internal/syncer/connector/azure_devops_test.go b/internal/syncer/connector/azure_devops_test.go
index c3cf2a72cd..4ca43b0669 100644
--- a/internal/syncer/connector/azure_devops_test.go
+++ b/internal/syncer/connector/azure_devops_test.go
@@ -437,12 +437,17 @@ func TestAzureDevOpsOpenSyncResumesFromCursor(t *testing.T) {
}))
defer server.Close()
- cursor, _ := json.Marshal(azureDevOpsSyncCursor{RepoKey: "iddaa/repo-a", Stage: azureDevOpsStageCode, FileOffset: 1})
+ cursor, _ := json.Marshal(azureDevOpsSyncCursor{
+ RepoKey: "iddaa/repo-a",
+ Stage: azureDevOpsStageCode,
+ FileOffset: 1,
+ SourceID: "azure_devops:contoso:iddaa:repo-a:file:a.cs",
+ })
connector := newTestAzureDevOpsConnector(t, server.URL, nil)
session, err := connector.OpenSync(context.Background(), SyncRequest{
FromBeginning: true,
WindowEnd: time.Now().UTC(),
- Resume: &SyncCheckpoint{Cursor: string(cursor)},
+ Resume: &SyncCheckpoint{Cursor: string(cursor), SourceID: "azure_devops:contoso:iddaa:repo-a:file:a.cs"},
})
if err != nil {
t.Fatalf("OpenSync returned error: %v", err)
@@ -461,6 +466,32 @@ func TestAzureDevOpsOpenSyncResumesFromCursor(t *testing.T) {
}
}
+func TestAzureDevOpsOpenSyncRejectsMissingSourceAnchor(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.Contains(r.URL.Path, "/pullrequests"):
+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})
+ case strings.Contains(r.URL.Path, "/items"):
+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})
+ default:
+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{
+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},
+ }})
+ }
+ }))
+ defer server.Close()
+
+ cursor, _ := json.Marshal(azureDevOpsSyncCursor{RepoKey: "iddaa/repo-a", Stage: azureDevOpsStageCode, FileOffset: 1})
+ connector := newTestAzureDevOpsConnector(t, server.URL, nil)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: string(cursor)},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestAzureDevOpsOpenPruneEmitsCodeAndPullRequestIDs(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
diff --git a/internal/syncer/connector/bitbucket.go b/internal/syncer/connector/bitbucket.go
index c5bcdd2505..4412c3c997 100644
--- a/internal/syncer/connector/bitbucket.go
+++ b/internal/syncer/connector/bitbucket.go
@@ -161,7 +161,9 @@ func (c *BitbucketConnector) OpenSync(ctx context.Context, request SyncRequest)
windowStart: request.WindowStart,
windowEnd: request.WindowEnd,
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -592,7 +594,10 @@ func (s *bitbucketSyncSession) nextDocumentPage(ctx context.Context) ([]bitbucke
sourceID: doc.SourceID,
})
}
- documents = s.filterResumedDocuments(repo, pageURL, documents)
+ documents, err = s.filterResumedDocuments(repo, pageURL, documents)
+ if err != nil {
+ return nil, err
+ }
if page.Next == "" || len(page.Values) == 0 {
s.advanceRepo()
} else {
@@ -609,16 +614,19 @@ func (s *bitbucketSyncSession) advanceRepo() {
}
// applyResume advances a sync session to the last committed Bitbucket position.
-func (s *bitbucketSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *bitbucketSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("bitbucket sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor bitbucketSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("bitbucket sync cursor is invalid: %w", ErrSyncResumeInvalid)
}
if cursor.RepoSlug == "" {
- return
+ return fmt.Errorf("bitbucket sync cursor has no resume anchor: %w", ErrSyncResumeInvalid)
}
for index, repo := range s.repos {
if repo != cursor.RepoSlug {
@@ -630,35 +638,32 @@ func (s *bitbucketSyncSession) applyResume(checkpoint *SyncCheckpoint) {
s.resumePageURL = cursor.PageURL
s.resumeOffset = cursor.PageOffset
s.resumeSource = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
- return
+ if s.resumeSource == "" {
+ return fmt.Errorf("bitbucket sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ return nil
}
+ return fmt.Errorf("bitbucket resume repo %q was not found in the current listing: %w", cursor.RepoSlug, ErrSyncResumeInvalid)
}
// filterResumedDocuments drops documents through the committed checkpoint.
-func (s *bitbucketSyncSession) filterResumedDocuments(repo, pageURL string, candidates []bitbucketBufferedDocument) []bitbucketBufferedDocument {
- if s.resumeRepo == "" || repo != s.resumeRepo || pageURL != s.resumePageURL {
- return candidates
+func (s *bitbucketSyncSession) filterResumedDocuments(repo, pageURL string, candidates []bitbucketBufferedDocument) ([]bitbucketBufferedDocument, error) {
+ if s.resumeRepo == "" {
+ return candidates, nil
+ }
+ if repo != s.resumeRepo || pageURL != s.resumePageURL {
+ return nil, fmt.Errorf("bitbucket resume page no longer matches checkpoint page: %w", ErrSyncResumeInvalid)
}
if s.resumeSource != "" {
for index, candidate := range candidates {
if candidate.sourceID == s.resumeSource {
s.clearResume()
- return candidates[index+1:]
+ return candidates[index+1:], nil
}
}
+ return nil, fmt.Errorf("bitbucket resume anchor %q was not found on %s: %w", s.resumeSource, pageURL, ErrSyncResumeInvalid)
}
- if s.resumeOffset > 0 {
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumeOffset {
- filtered = append(filtered, candidate)
- }
- }
- s.clearResume()
- return filtered
- }
- s.clearResume()
- return candidates
+ return nil, fmt.Errorf("bitbucket sync cursor has no source anchor: %w", ErrSyncResumeInvalid)
}
func (s *bitbucketSyncSession) clearResume() {
diff --git a/internal/syncer/connector/bitbucket_test.go b/internal/syncer/connector/bitbucket_test.go
index 846184c10e..071528e402 100644
--- a/internal/syncer/connector/bitbucket_test.go
+++ b/internal/syncer/connector/bitbucket_test.go
@@ -329,7 +329,7 @@ func TestBitbucketConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
}
}
-func TestBitbucketConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {
+func TestBitbucketConnectorOpenSyncResumeRejectsMissingSourceAnchor(t *testing.T) {
connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")
connector.doJSON = bitbucketFixtureDoJSON(t)
@@ -348,18 +348,8 @@ func TestBitbucketConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t
resumeCheckpoint := cloneBitbucketCheckpointWithoutSourceID(t, first.Checkpoint)
resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})
- if err != nil {
- t.Fatalf("resume OpenSync failed: %v", err)
- }
- second, err := resumed.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("resume NextBatch failed: %v", err)
- }
- if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {
- t.Fatalf("resume documents = %+v, want PR 2", second.Documents)
- }
- if second.Documents[0].SourceID == first.Documents[0].SourceID {
- t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)
+ if resumed != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", resumed, err)
}
}
diff --git a/internal/syncer/connector/confluence.go b/internal/syncer/connector/confluence.go
index a56aae514e..1bc022f935 100644
--- a/internal/syncer/connector/confluence.go
+++ b/internal/syncer/connector/confluence.go
@@ -175,7 +175,9 @@ func (c *ConfluenceConnector) OpenSync(ctx context.Context, request SyncRequest)
pageCursor: newConfluenceSearchCursor(c, c.pageCQL(request.WindowStart, end, request.FromBeginning), strings.Join(confluencePageExpansionFields, ",")),
nameCounts: map[string]int{},
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -555,7 +557,7 @@ func (s *confluenceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error
doc, err := s.nextDocument(ctx)
if errors.Is(err, io.EOF) {
if s.resumeSourceID != "" && !s.resumeMatched {
- return SyncBatch{}, fmt.Errorf("confluence sync resume checkpoint %q was not found in the source; refusing to discard unprocessed documents", s.resumeSourceID)
+ return SyncBatch{}, fmt.Errorf("confluence sync resume checkpoint %q was not found in the source: %w", s.resumeSourceID, ErrSyncResumeInvalid)
}
if len(documents) == 0 {
return SyncBatch{}, io.EOF
@@ -640,19 +642,20 @@ func (s *confluenceSyncSession) includeResumed(doc SourceDocument) bool {
s.resumeMatched = true
return false
}
- if s.resumeUpdatedAt != nil && doc.UpdatedAt.After(*s.resumeUpdatedAt) {
- s.resumeMatched = true
- return true
- }
return false
}
-func (s *confluenceSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *confluenceSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
- s.resumeSourceID = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ if sourceID == "" {
+ return fmt.Errorf("confluence sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ s.resumeSourceID = sourceID
s.resumeUpdatedAt = checkpoint.UpdatedAt
+ return nil
}
func confluenceSyncCheckpoint(doc SourceDocument) *SyncCheckpoint {
diff --git a/internal/syncer/connector/confluence_test.go b/internal/syncer/connector/confluence_test.go
index 7cc887df42..e2ef8fecb6 100644
--- a/internal/syncer/connector/confluence_test.go
+++ b/internal/syncer/connector/confluence_test.go
@@ -118,6 +118,21 @@ func TestConfluenceConnectorResume(t *testing.T) {
}
}
+func TestConfluenceConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ server := newConfluenceFixtureServer(t, http.StatusOK)
+ defer server.Close()
+
+ connector := newTestConfluenceConnector(t, server.URL, map[string]any{"batch_size": 1})
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ WindowEnd: confluenceTestTime(t, "2026-01-03T00:00:00Z"),
+ Resume: &SyncCheckpoint{},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestConfluenceConnectorPrune(t *testing.T) {
server := newConfluenceFixtureServer(t, http.StatusOK)
defer server.Close()
diff --git a/internal/syncer/connector/dingtalk_ai_table.go b/internal/syncer/connector/dingtalk_ai_table.go
index 9a92220b62..974edf371a 100644
--- a/internal/syncer/connector/dingtalk_ai_table.go
+++ b/internal/syncer/connector/dingtalk_ai_table.go
@@ -115,7 +115,9 @@ func (c *DingTalkAITableConnector) OpenSync(ctx context.Context, request SyncReq
return nil, err
}
session := &dingTalkAITableSyncSession{documents: documents, batchSize: c.batchSize}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -469,20 +471,21 @@ func (s *dingTalkAITableSyncSession) Close() error {
return nil
}
-func (s *dingTalkAITableSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *dingTalkAITableSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
if sourceID == "" {
- return
+ return fmt.Errorf("dingtalk AI table sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
for index, doc := range s.documents {
if doc.SourceID == sourceID {
s.index = index + 1
- return
+ return nil
}
}
+ return fmt.Errorf("dingtalk AI table resume anchor %q was not found in the current listing: %w", sourceID, ErrSyncResumeInvalid)
}
func dingTalkAITableCheckpoint(doc SourceDocument) *SyncCheckpoint {
diff --git a/internal/syncer/connector/dingtalk_ai_table_test.go b/internal/syncer/connector/dingtalk_ai_table_test.go
index 080792c239..1e852621c3 100644
--- a/internal/syncer/connector/dingtalk_ai_table_test.go
+++ b/internal/syncer/connector/dingtalk_ai_table_test.go
@@ -209,6 +209,29 @@ func TestDingTalkAITableConnectorOpenSyncFiltersByRecordLastModifiedTime(t *test
}
}
+func TestDingTalkAITableConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := &DingTalkAITableConnector{
+ tableID: "table-1",
+ operatorID: "operator-1",
+ accessToken: "token-1",
+ batchSize: 10,
+ apiBaseURL: dingTalkAITableAPIBaseURL,
+ getSheets: func(ctx context.Context) ([]dingTalkAITableSheet, error) {
+ return []dingTalkAITableSheet{{ID: "sheet-1", Name: "Projects"}}, nil
+ },
+ listRecords: func(ctx context.Context, sheetID, nextToken string, maxResults int) ([]dingTalkAITableRecord, string, error) {
+ return []dingTalkAITableRecord{
+ {ID: "rec-1", LastModifiedTime: time.UnixMilli(1780000000000).UnixMilli(), Fields: map[string]any{"Title": "Roadmap"}},
+ }, "", nil
+ },
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestDingTalkAITableConnectorValidateConnectorSetting(t *testing.T) {
config := map[string]any{
"table_id": "table-1",
diff --git a/internal/syncer/connector/discord.go b/internal/syncer/connector/discord.go
index 15448621e2..03a66b90f2 100644
--- a/internal/syncer/connector/discord.go
+++ b/internal/syncer/connector/discord.go
@@ -283,7 +283,10 @@ func (c *DiscordConnector) OpenSync(ctx context.Context, request SyncRequest) (S
upperBound = request.WindowEnd
}
- resumeTarget, resumeBefore := discordResumePosition(targets, request.Resume)
+ resumeTarget, resumeBefore, err := discordResumePosition(targets, request.Resume)
+ if err != nil {
+ return nil, err
+ }
return &discordSyncSession{
connector: c,
iter: newDiscordMessageIterator(c, targets, lowerBound, upperBound, resumeTarget, resumeBefore),
@@ -292,27 +295,34 @@ func (c *DiscordConnector) OpenSync(ctx context.Context, request SyncRequest) (S
}
// discordResumePosition returns the target and before cursor to continue from,
-// or empty values when the checkpoint does not match the current enumeration.
-func discordResumePosition(targets []discordTarget, checkpoint *SyncCheckpoint) (string, string) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return "", ""
+// or ErrSyncResumeInvalid when the checkpoint does not match the current
+// enumeration.
+func discordResumePosition(targets []discordTarget, checkpoint *SyncCheckpoint) (string, string, error) {
+ if checkpoint == nil {
+ return "", "", nil
+ }
+ if checkpoint.Cursor == "" {
+ return "", "", fmt.Errorf("discord sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor discordSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return "", ""
+ return "", "", fmt.Errorf("discord sync cursor is invalid: %w", ErrSyncResumeInvalid)
}
if cursor.Target == "" || cursor.Message == "" {
- return "", ""
+ return "", "", fmt.Errorf("discord sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ if cursor.Targets == "" {
+ return "", "", fmt.Errorf("discord sync cursor has no target listing: %w", ErrSyncResumeInvalid)
}
if cursor.Targets != "" && cursor.Targets != discordCursorFingerprint(targets) {
- return "", ""
+ return "", "", fmt.Errorf("discord resume anchor %q was not found in the current target listing: %w", cursor.Target, ErrSyncResumeInvalid)
}
for _, target := range targets {
if target.channelID == cursor.Target {
- return cursor.Target, cursor.Message
+ return cursor.Target, cursor.Message, nil
}
}
- return "", ""
+ return "", "", fmt.Errorf("discord resume anchor %q was not found in the current target listing: %w", cursor.Target, ErrSyncResumeInvalid)
}
// discordSyncCursor is the resume position serialized into SyncCheckpoint.Cursor.
@@ -325,7 +335,7 @@ type discordSyncCursor struct {
}
// discordCursorFingerprint summarizes the ordered target list so a resume can
-// detect enumeration changes and fall back to a full re-scan.
+// detect enumeration changes and reject a stale resume.
func discordCursorFingerprint(targets []discordTarget) string {
ids := make([]string, 0, len(targets))
for _, target := range targets {
diff --git a/internal/syncer/connector/discord_test.go b/internal/syncer/connector/discord_test.go
index 9c17750ba5..58efff7a9f 100644
--- a/internal/syncer/connector/discord_test.go
+++ b/internal/syncer/connector/discord_test.go
@@ -647,7 +647,8 @@ func TestDiscordResumeFingerprintMismatch(t *testing.T) {
}
session.Close()
- // Run 2 filters to one channel, changing the enumeration: resume is ignored.
+ // Run 2 filters to one channel, changing the enumeration: resume must be
+ // rejected so the runner can restart the same fixed window.
secondConnector := newDiscordTestConnector(t, map[string]any{
"server_ids": []any{"1001"},
"channels": []any{"general"},
@@ -659,17 +660,8 @@ func TestDiscordResumeFingerprintMismatch(t *testing.T) {
FromBeginning: true,
Resume: batch.Checkpoint,
})
- if err != nil {
- t.Fatalf("resumed OpenSync: %v", err)
- }
- defer resumed.Close()
-
- first, err := resumed.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("resumed NextBatch: %v", err)
- }
- if first.Documents[0].SourceID != "DISCORD_m1" {
- t.Fatalf("resumed doc = %q, want full re-scan from DISCORD_m1", first.Documents[0].SourceID)
+ if resumed != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resumed OpenSync = session %v, err %v, want ErrSyncResumeInvalid", resumed, err)
}
}
@@ -690,17 +682,8 @@ func TestDiscordResumeInvalidCursor(t *testing.T) {
FromBeginning: true,
Resume: &SyncCheckpoint{Cursor: "not-json"},
})
- if err != nil {
- t.Fatalf("OpenSync: %v", err)
- }
- defer session.Close()
-
- batch, err := session.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("NextBatch: %v", err)
- }
- if batch.Documents[0].SourceID != "DISCORD_m1" {
- t.Fatalf("doc = %q, want full re-scan from DISCORD_m1", batch.Documents[0].SourceID)
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
}
}
diff --git a/internal/syncer/connector/dropbox.go b/internal/syncer/connector/dropbox.go
index 9f039e4c43..e53b3864e3 100644
--- a/internal/syncer/connector/dropbox.go
+++ b/internal/syncer/connector/dropbox.go
@@ -138,7 +138,9 @@ func (c *DropboxConnector) OpenSync(ctx context.Context, request SyncRequest) (S
nameCounts: nameCounts,
batchSize: positiveDropboxBatchSize(c.batchSize),
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -322,20 +324,21 @@ func (s *dropboxSyncSession) Close() error {
return nil
}
-func (s *dropboxSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *dropboxSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
if sourceID == "" {
- return
+ return fmt.Errorf("dropbox sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
for index, file := range s.files {
if file.sourceID() == sourceID {
s.index = index + 1
- return
+ return nil
}
}
+ return fmt.Errorf("dropbox resume anchor %q was not found in the current listing: %w", sourceID, ErrSyncResumeInvalid)
}
func dropboxSyncCheckpoint(file dropboxFileMetadata) *SyncCheckpoint {
diff --git a/internal/syncer/connector/dropbox_test.go b/internal/syncer/connector/dropbox_test.go
index da04a5284f..7d7256e457 100644
--- a/internal/syncer/connector/dropbox_test.go
+++ b/internal/syncer/connector/dropbox_test.go
@@ -19,6 +19,7 @@ package connector
import (
"context"
"encoding/json"
+ "errors"
"io"
"net/http"
"net/http/httptest"
@@ -245,6 +246,20 @@ func TestDropboxConnectorOpenPruneAndResume(t *testing.T) {
}
}
+func TestDropboxConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ fixture := newDropboxTestServer(t)
+ connector := newDropboxTestConnector(t, fixture, false)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ WindowEnd: time.Date(2026, 1, 5, 0, 0, 0, 0, time.UTC),
+ Resume: &SyncCheckpoint{},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestDropboxConnectorDownloadFileEscapesAPIArgHeader(t *testing.T) {
var gotHeader string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.go
index 2185fd60fb..3bdc29e181 100644
--- a/internal/syncer/connector/github.go
+++ b/internal/syncer/connector/github.go
@@ -216,7 +216,9 @@ func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (Sy
return nil, err
}
session := &githubSyncSession{connector: c, repos: repos, batchSize: c.batchSize, stage: githubStagePRs, page: 1, windowStart: request.WindowStart, windowEnd: request.WindowEnd}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -552,7 +554,10 @@ func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]githubBuffe
if err != nil {
return nil, err
}
- docs = s.filterResumedDocuments(repo.FullName, githubStagePRs, s.page, docs)
+ docs, err = s.filterResumedDocuments(repo.FullName, githubStagePRs, s.page, docs)
+ if err != nil {
+ return nil, err
+ }
if done {
s.advanceStage()
} else {
@@ -568,7 +573,10 @@ func (s *githubSyncSession) nextDocumentPage(ctx context.Context) ([]githubBuffe
if err != nil {
return nil, err
}
- docs = s.filterResumedDocuments(repo.FullName, githubStageIssues, s.page, docs)
+ docs, err = s.filterResumedDocuments(repo.FullName, githubStageIssues, s.page, docs)
+ if err != nil {
+ return nil, err
+ }
if done {
s.advanceRepo()
} else {
@@ -637,17 +645,20 @@ func (s *githubSyncSession) advanceRepo() {
}
// applyResume advances a sync session to the last committed GitHub page.
-func (s *githubSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *githubSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
}
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("github sync cursor is missing: %w", ErrSyncResumeInvalid)
+ }
var cursor githubSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("github sync cursor is invalid: %w", ErrSyncResumeInvalid)
}
if cursor.Repo == "" || cursor.Stage == "" || cursor.Page <= 0 {
- return
+ return fmt.Errorf("github sync cursor has no resume anchor: %w", ErrSyncResumeInvalid)
}
for index, repo := range s.repos {
if repo.FullName != cursor.Repo {
@@ -661,35 +672,29 @@ func (s *githubSyncSession) applyResume(checkpoint *SyncCheckpoint) {
s.resumePage = cursor.Page
s.resumeOffset = cursor.Offset
s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
- return
+ if s.resumeSourceID == "" {
+ return fmt.Errorf("github sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ return nil
}
+ return fmt.Errorf("github resume repo %q was not found in the current listing: %w", cursor.Repo, ErrSyncResumeInvalid)
}
// filterResumedDocuments drops documents through the committed checkpoint.
-func (s *githubSyncSession) filterResumedDocuments(repo, stage string, page int, candidates []githubBufferedDocument) []githubBufferedDocument {
+func (s *githubSyncSession) filterResumedDocuments(repo, stage string, page int, candidates []githubBufferedDocument) ([]githubBufferedDocument, error) {
if s.resumeRepo == "" || repo != s.resumeRepo || stage != s.resumeStage || page != s.resumePage {
- return candidates
+ return candidates, nil
}
if s.resumeSourceID != "" {
for index, candidate := range candidates {
if candidate.sourceID == s.resumeSourceID {
s.clearResume()
- return candidates[index+1:]
+ return candidates[index+1:], nil
}
}
+ return nil, fmt.Errorf("github resume anchor %q was not found on page %d: %w", s.resumeSourceID, page, ErrSyncResumeInvalid)
}
- if s.resumeOffset <= 0 {
- s.clearResume()
- return candidates
- }
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumeOffset {
- filtered = append(filtered, candidate)
- }
- }
- s.clearResume()
- return filtered
+ return nil, fmt.Errorf("github sync cursor has no source anchor: %w", ErrSyncResumeInvalid)
}
func (s *githubSyncSession) clearResume() {
diff --git a/internal/syncer/connector/github_test.go b/internal/syncer/connector/github_test.go
index ad386b13bf..411b12049f 100644
--- a/internal/syncer/connector/github_test.go
+++ b/internal/syncer/connector/github_test.go
@@ -186,8 +186,8 @@ func TestGitHubConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
}
}
-// TestGitHubConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument verifies offset fallback excludes the committed offset.
-func TestGitHubConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {
+// TestGitHubConnectorOpenSyncResumeRejectsMissingSourceAnchor verifies a checkpoint without a source anchor is invalid.
+func TestGitHubConnectorOpenSyncResumeRejectsMissingSourceAnchor(t *testing.T) {
connector, err := NewGitHubConnector(map[string]any{
"repository_owner": "openai",
"repository_name": "ragflow",
@@ -220,18 +220,8 @@ func TestGitHubConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *te
resumeCheckpoint := cloneGitHubCheckpointWithMissingSourceID(t, first.Checkpoint)
resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})
- if err != nil {
- t.Fatalf("resume OpenSync failed: %v", err)
- }
- second, err := resumed.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("resume NextBatch failed: %v", err)
- }
- if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://github.com/openai/ragflow/issues/3" {
- t.Fatalf("resume documents = %+v, want issue 3", second.Documents)
- }
- if second.Documents[0].SourceID == first.Documents[0].SourceID {
- t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)
+ if resumed != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", resumed, err)
}
}
diff --git a/internal/syncer/connector/gitlab.go b/internal/syncer/connector/gitlab.go
index 12a783951f..44eb79ce09 100644
--- a/internal/syncer/connector/gitlab.go
+++ b/internal/syncer/connector/gitlab.go
@@ -144,7 +144,9 @@ func (c *GitlabConnector) OpenSync(ctx context.Context, request SyncRequest) (Sy
session.stage = gitlabStageMRs
session.treeQueue = nil
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -473,7 +475,10 @@ func (s *gitlabSyncSession) nextDocumentPage(ctx context.Context) ([]gitlabBuffe
if err != nil {
return nil, err
}
- docs = s.filterResumedDocuments(gitlabStageMRs, s.page, docs)
+ docs, err = s.filterResumedDocuments(gitlabStageMRs, s.page, docs)
+ if err != nil {
+ return nil, err
+ }
if done {
s.advanceStage()
} else {
@@ -489,7 +494,10 @@ func (s *gitlabSyncSession) nextDocumentPage(ctx context.Context) ([]gitlabBuffe
if err != nil {
return nil, err
}
- docs = s.filterResumedDocuments(gitlabStageIssues, s.page, docs)
+ docs, err = s.filterResumedDocuments(gitlabStageIssues, s.page, docs)
+ if err != nil {
+ return nil, err
+ }
if done {
s.advanceStage()
} else {
@@ -579,7 +587,10 @@ func (s *gitlabSyncSession) nextCodeFilesPage(ctx context.Context) ([]gitlabBuff
sourceID: doc.SourceID,
})
}
- documents = s.filterResumedDocuments(gitlabStageCodeFiles, s.treePage, documents)
+ documents, err = s.filterResumedDocuments(gitlabStageCodeFiles, s.treePage, documents)
+ if err != nil {
+ return nil, err
+ }
if done {
s.treeQueue = s.treeQueue[1:]
s.treePage = 1
@@ -609,55 +620,52 @@ func (s *gitlabSyncSession) advanceStage() {
}
// applyResume advances a sync session to the last committed GitLab position.
-func (s *gitlabSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *gitlabSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("gitlab sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor gitlabSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("gitlab sync cursor is invalid: %w", ErrSyncResumeInvalid)
}
if cursor.Stage == "" || cursor.Page <= 0 {
- return
+ return fmt.Errorf("gitlab sync cursor has no resume anchor: %w", ErrSyncResumeInvalid)
}
s.stage = cursor.Stage
s.resumeStage = cursor.Stage
s.resumePage = cursor.Page
s.resumeOffset = cursor.Offset
s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if s.resumeSourceID == "" {
+ return fmt.Errorf("gitlab sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
if cursor.Stage == gitlabStageCodeFiles {
s.treeQueue = append([]string{cursor.TreePath}, cursor.PendingPaths...)
s.treePage = cursor.Page
} else {
s.page = cursor.Page
}
+ return nil
}
// filterResumedDocuments drops documents through the committed checkpoint.
-func (s *gitlabSyncSession) filterResumedDocuments(stage string, page int, candidates []gitlabBufferedDocument) []gitlabBufferedDocument {
+func (s *gitlabSyncSession) filterResumedDocuments(stage string, page int, candidates []gitlabBufferedDocument) ([]gitlabBufferedDocument, error) {
if s.resumeStage == "" || stage != s.resumeStage || page != s.resumePage {
- return candidates
+ return candidates, nil
}
if s.resumeSourceID != "" {
for index, candidate := range candidates {
if candidate.sourceID == s.resumeSourceID {
s.clearResume()
- return candidates[index+1:]
+ return candidates[index+1:], nil
}
}
+ return nil, fmt.Errorf("gitlab resume anchor %q was not found on page %d: %w", s.resumeSourceID, page, ErrSyncResumeInvalid)
}
- if s.resumeOffset <= 0 {
- s.clearResume()
- return candidates
- }
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumeOffset {
- filtered = append(filtered, candidate)
- }
- }
- s.clearResume()
- return filtered
+ return nil, fmt.Errorf("gitlab sync cursor has no source anchor: %w", ErrSyncResumeInvalid)
}
func (s *gitlabSyncSession) clearResume() {
diff --git a/internal/syncer/connector/gitlab_test.go b/internal/syncer/connector/gitlab_test.go
index c2863ca49d..c07f6b1be7 100644
--- a/internal/syncer/connector/gitlab_test.go
+++ b/internal/syncer/connector/gitlab_test.go
@@ -190,8 +190,8 @@ func TestGitlabConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
}
}
-// TestGitlabConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument verifies offset fallback excludes the committed offset.
-func TestGitlabConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {
+// TestGitlabConnectorOpenSyncResumeRejectsMissingSourceAnchor verifies a checkpoint without a source anchor is invalid.
+func TestGitlabConnectorOpenSyncResumeRejectsMissingSourceAnchor(t *testing.T) {
connector, err := NewGitlabConnector(map[string]any{
"project_owner": "owner",
"project_name": "repo",
@@ -225,18 +225,8 @@ func TestGitlabConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *te
resumeCheckpoint := cloneGitlabCheckpointWithMissingSourceID(t, first.Checkpoint)
resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})
- if err != nil {
- t.Fatalf("resume OpenSync failed: %v", err)
- }
- second, err := resumed.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("resume NextBatch failed: %v", err)
- }
- if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://gitlab.com/owner/repo/-/issues/3" {
- t.Fatalf("resume documents = %+v, want issue 3", second.Documents)
- }
- if second.Documents[0].SourceID == first.Documents[0].SourceID {
- t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)
+ if resumed != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", resumed, err)
}
}
diff --git a/internal/syncer/connector/gmail.go b/internal/syncer/connector/gmail.go
index 39cd2e3489..7044cf5785 100644
--- a/internal/syncer/connector/gmail.go
+++ b/internal/syncer/connector/gmail.go
@@ -113,7 +113,9 @@ func (c *GmailConnector) OpenSync(ctx context.Context, request SyncRequest) (Syn
batchSize: c.batchSize,
query: query,
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -311,7 +313,7 @@ type gmailSyncSession struct {
buffer []gmailBufferedDocument
resumePageToken string
resumeOffset int
- resumeFilename string
+ resumeSourceID string
}
// NextBatch returns the next Gmail document batch.
@@ -388,11 +390,13 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]gmailBuffere
document: doc,
checkpoint: gmailSyncCheckpoint(userEmail, requestPageToken, pageOffset, doc),
offset: pageOffset,
- filename: syncSourceDocumentFilenameFromDocument(doc),
})
}
}
- documents := s.filterResumedDocuments(requestPageToken, candidates)
+ documents, err := s.filterResumedDocuments(requestPageToken, candidates)
+ if err != nil {
+ return nil, err
+ }
if page.NextPageToken == "" {
s.advanceUser()
} else {
@@ -401,18 +405,25 @@ func (s *gmailSyncSession) nextDocumentPage(ctx context.Context) ([]gmailBuffere
return documents, nil
}
-// applyResume apply resume if resume is available (adjust session's token and pageOffset)
-func (s *gmailSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+// applyResume restores the Gmail source anchor when a checkpoint is available.
+func (s *gmailSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("gmail sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor gmailSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("gmail sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" {
+ return fmt.Errorf("gmail sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
if cursor.UserEmail == "" {
- return
+ return fmt.Errorf("gmail sync cursor has no user anchor: %w", ErrSyncResumeInvalid)
}
for index, userEmail := range s.users {
if userEmail != cursor.UserEmail {
@@ -421,48 +432,36 @@ func (s *gmailSyncSession) applyResume(checkpoint *SyncCheckpoint) {
s.userIndex = index
s.pageToken = cursor.PageToken
s.resumePageToken = cursor.PageToken
- s.resumeFilename = cursor.Filename
+ s.resumeSourceID = sourceID
if cursor.Offset > 0 {
s.resumeOffset = cursor.Offset
}
- return
+ return nil
}
+ return fmt.Errorf("gmail resume user %q was not found in the current listing: %w", cursor.UserEmail, ErrSyncResumeInvalid)
}
-// filterResumedDocuments applies checkpoint offset using the filename at that offset.
-func (s *gmailSyncSession) filterResumedDocuments(pageToken string, candidates []gmailBufferedDocument) []gmailBufferedDocument {
- if s.resumeOffset <= 0 {
- return candidates
+// filterResumedDocuments continues after the committed Gmail source anchor.
+func (s *gmailSyncSession) filterResumedDocuments(pageToken string, candidates []gmailBufferedDocument) ([]gmailBufferedDocument, error) {
+ if s.resumeSourceID == "" {
+ return candidates, nil
}
if pageToken != s.resumePageToken {
- s.clearResumeOffset()
- return candidates
- }
- if s.resumeOffset <= len(candidates) && candidates[s.resumeOffset-1].filename == s.resumeFilename {
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumeOffset {
- filtered = append(filtered, candidate)
- }
- }
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("gmail resume page %q no longer matches checkpoint page %q: %w", pageToken, s.resumePageToken, ErrSyncResumeInvalid)
}
-
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset >= s.resumeOffset {
- filtered = append(filtered, candidate)
+ for index, candidate := range candidates {
+ if candidate.document.SourceID == s.resumeSourceID {
+ s.clearResumeOffset()
+ return candidates[index+1:], nil
}
}
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("gmail resume anchor %q was not found on page %q: %w", s.resumeSourceID, pageToken, ErrSyncResumeInvalid)
}
func (s *gmailSyncSession) clearResumeOffset() {
s.resumeOffset = 0
s.resumePageToken = ""
- s.resumeFilename = ""
+ s.resumeSourceID = ""
}
// advanceUser moves a Gmail session to the next mailbox.
@@ -476,19 +475,17 @@ type gmailBufferedDocument struct {
document SourceDocument
checkpoint *SyncCheckpoint
offset int
- filename string
}
type gmailSyncCursor struct {
UserEmail string `json:"user_email"`
PageToken string `json:"page_token,omitempty"`
Offset int `json:"offset"`
- Filename string `json:"filename"`
+ SourceID string `json:"source_id,omitempty"`
}
func gmailSyncCheckpoint(userEmail, pageToken string, offset int, doc SourceDocument) *SyncCheckpoint {
- filename := syncSourceDocumentFilenameFromDocument(doc)
- cursor, err := json.Marshal(gmailSyncCursor{UserEmail: userEmail, PageToken: pageToken, Offset: offset, Filename: filename})
+ cursor, err := json.Marshal(gmailSyncCursor{UserEmail: userEmail, PageToken: pageToken, Offset: offset, SourceID: doc.SourceID})
if err != nil {
return nil
}
diff --git a/internal/syncer/connector/gmail_test.go b/internal/syncer/connector/gmail_test.go
index 22c3f05e16..ee5293809e 100644
--- a/internal/syncer/connector/gmail_test.go
+++ b/internal/syncer/connector/gmail_test.go
@@ -103,8 +103,17 @@ func TestGmailConnectorOpenSyncResumesWithinPage(t *testing.T) {
}
}
-// TestGmailConnectorResumeUsesFilenameWhenOffsetChanged verifies page deletions resume from the current filename offset.
-func TestGmailConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {
+func TestGmailConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := newFixtureGmailConnector()
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
+// TestGmailConnectorResumeRejectsMissingRemoteAnchor verifies a deleted list item invalidates the saved anchor.
+func TestGmailConnectorResumeRejectsMissingRemoteAnchor(t *testing.T) {
connector := newFixtureGmailConnector()
connector.GmailConnector.batchSize = 2
connector.GmailConnector.listThreadPage = func(ctx context.Context, userEmail, query, pageToken string, pageSize int) (gmailThreadListPage, error) {
@@ -141,15 +150,12 @@ func TestGmailConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {
if err != nil {
t.Fatalf("resume OpenSync failed: %v", err)
}
- second, err := resumed.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("resume NextBatch failed: %v", err)
- }
- if len(second.Documents) != 1 || second.Documents[0].SourceID != "thread-3" {
- t.Fatalf("resume documents = %+v, want thread-3", second.Documents)
+ _, err = resumed.NextBatch(context.Background())
+ if err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
}
- if resumeListCalls != 1 {
- t.Fatalf("resume list calls = %d, want one checkpoint page", resumeListCalls)
+ if resumeListCalls == 0 {
+ t.Fatalf("resume should have listed the current remote page")
}
}
diff --git a/internal/syncer/connector/google_cloud_storage.go b/internal/syncer/connector/google_cloud_storage.go
index 94e518fc02..8f86e70f00 100644
--- a/internal/syncer/connector/google_cloud_storage.go
+++ b/internal/syncer/connector/google_cloud_storage.go
@@ -117,7 +117,9 @@ func (c *GoogleCloudStorageConnector) OpenSync(ctx context.Context, request Sync
batchSize: c.batchSize,
}
if request.Resume != nil {
- session.applyResume(request.Resume)
+ if err := session.applyResume(ctx, request.Resume); err != nil {
+ return nil, err
+ }
}
return session, nil
}
@@ -333,13 +335,41 @@ func (s *googleCloudStorageSyncSession) Fetch(ctx context.Context, ref FetchRefe
return s.connector.Fetch(ctx, ref)
}
-func (s *googleCloudStorageSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *googleCloudStorageSyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
prefix := googleCloudStorageSourceID(s.connector.bucketName, "")
if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
- return
+ return fmt.Errorf("google cloud storage sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ anchor := strings.TrimPrefix(sourceID, prefix)
+ if anchor == "" {
+ return fmt.Errorf("google cloud storage sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+
+ startAfter := ""
+ for {
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, startAfter, int32(s.batchSize))
+ if err != nil {
+ return err
+ }
+ for _, object := range objects {
+ if object.Key == anchor {
+ s.startAfter = anchor
+ return nil
+ }
+ if object.Key > anchor {
+ return fmt.Errorf("google cloud storage resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
+ }
+ }
+ if !hasMore {
+ break
+ }
+ startAfter = strings.TrimPrefix(nextStartAfter, prefix)
}
- s.startAfter = strings.TrimPrefix(sourceID, prefix)
+ return fmt.Errorf("google cloud storage resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
}
func (c *GoogleCloudStorageConnector) sourceDocument(sourceID string, object googleCloudStorageObject) (SourceDocument, bool) {
diff --git a/internal/syncer/connector/google_cloud_storage_test.go b/internal/syncer/connector/google_cloud_storage_test.go
index dedfb8351c..e6e406768a 100644
--- a/internal/syncer/connector/google_cloud_storage_test.go
+++ b/internal/syncer/connector/google_cloud_storage_test.go
@@ -126,6 +126,64 @@ func TestGoogleCloudStorageConnectorOpenSyncFiltersImagesUnlessAllowed(t *testin
}
}
+func TestGoogleCloudStorageConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
+ connector := newTestGoogleCloudStorageConnector(t, []googleCloudStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: googleCloudStorageSourceID("bucket", "docs/b.txt"),
+ SourceID: googleCloudStorageSourceID("bucket", "docs/b.txt"),
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != googleCloudStorageSourceID("bucket", "docs/c.txt") {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestGoogleCloudStorageConnectorOpenSyncResumeRejectsMissingAnchor(t *testing.T) {
+ connector := newTestGoogleCloudStorageConnector(t, []googleCloudStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: googleCloudStorageSourceID("bucket", "docs/b.txt"),
+ SourceID: googleCloudStorageSourceID("bucket", "docs/b.txt"),
+ },
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
+func TestGoogleCloudStorageConnectorOpenSyncResumeRejectsForeignCheckpoint(t *testing.T) {
+ connector := newTestGoogleCloudStorageConnector(t, []googleCloudStorageObject{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "other:bucket:docs/a.txt",
+ },
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestGoogleCloudStorageConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
connector := newTestGoogleCloudStorageConnector(t, []googleCloudStorageObject{
{Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
diff --git a/internal/syncer/connector/google_drive.go b/internal/syncer/connector/google_drive.go
index 3a9af58dca..861cabc5c0 100644
--- a/internal/syncer/connector/google_drive.go
+++ b/internal/syncer/connector/google_drive.go
@@ -167,7 +167,10 @@ func (c *GoogleDriveConnector) OpenSync(ctx context.Context, request SyncRequest
windowStart: request.WindowStart,
windowEnd: request.WindowEnd,
}
- return session.applyResume(request.Resume), nil
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
+ return session, nil
}
// OpenPrune opens one complete Google Drive prune snapshot session.
@@ -568,7 +571,7 @@ type googleDriveSyncSession struct {
folderSeen map[string]struct{}
resumePageToken string
resumePageOffset int
- resumeFilename string
+ resumeSourceID string
}
// NextBatch returns the next Google Drive document batch.
@@ -640,10 +643,13 @@ func (s *googleDriveSyncSession) nextDocumentPage(ctx context.Context) ([]google
document: doc,
checkpoint: s.syncCheckpoint(scope, requestPageToken, pageOffset, doc),
offset: pageOffset,
- filename: syncSourceDocumentFilenameFromDocument(doc),
})
}
- docs := s.filterSeenDocuments(s.filterResumedDocuments(requestPageToken, candidates))
+ docs, err := s.filterResumedDocuments(requestPageToken, candidates)
+ if err != nil {
+ return nil, err
+ }
+ docs = s.filterSeenDocuments(docs)
if page.NextPageToken == "" {
if err = s.finishScope(ctx, scope); err != nil {
return nil, err
@@ -702,14 +708,21 @@ func (s *googleDriveSyncSession) filterSeenDocuments(candidates []googleDriveBuf
return filtered
}
-// applyResume apply resume if checkpoint is available
-func (s *googleDriveSyncSession) applyResume(checkpoint *SyncCheckpoint) *googleDriveSyncSession {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return s
+// applyResume restores the Google Drive source anchor when a checkpoint is available.
+func (s *googleDriveSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("google drive sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor googleDriveSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return s
+ return fmt.Errorf("google drive sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" {
+ return fmt.Errorf("google drive sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
if len(cursor.Scopes) > 0 && cursor.ScopeIndex >= 0 && cursor.ScopeIndex < len(cursor.Scopes) {
scopes := make([]googleDriveScope, 0, len(cursor.Scopes))
@@ -726,49 +739,38 @@ func (s *googleDriveSyncSession) applyResume(checkpoint *SyncCheckpoint) *google
s.scopes = append(s.scopes, scope)
s.scopeIndex = len(s.scopes) - 1
}
+ } else {
+ return fmt.Errorf("google drive sync cursor has no valid scope: %w", ErrSyncResumeInvalid)
}
s.pageToken = cursor.PageToken
s.resumePageToken = cursor.PageToken
- s.resumeFilename = cursor.Filename
+ s.resumeSourceID = sourceID
if cursor.Offset > 0 {
s.resumePageOffset = cursor.Offset
}
- return s
+ return nil
}
-func (s *googleDriveSyncSession) filterResumedDocuments(pageToken string, candidates []googleDriveBufferedDocument) []googleDriveBufferedDocument {
- if s.resumePageOffset <= 0 {
- return candidates
+func (s *googleDriveSyncSession) filterResumedDocuments(pageToken string, candidates []googleDriveBufferedDocument) ([]googleDriveBufferedDocument, error) {
+ if s.resumeSourceID == "" {
+ return candidates, nil
}
if pageToken != s.resumePageToken {
- s.clearResumeOffset()
- return candidates
- }
- if s.resumePageOffset <= len(candidates) && candidates[s.resumePageOffset-1].filename == s.resumeFilename {
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumePageOffset {
- filtered = append(filtered, candidate)
- }
- }
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("google drive resume page %q no longer matches checkpoint page %q: %w", pageToken, s.resumePageToken, ErrSyncResumeInvalid)
}
-
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset >= s.resumePageOffset {
- filtered = append(filtered, candidate)
+ for index, candidate := range candidates {
+ if candidate.document.SourceID == s.resumeSourceID {
+ s.clearResumeOffset()
+ return candidates[index+1:], nil
}
}
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("google drive resume anchor %q was not found on page %q: %w", s.resumeSourceID, pageToken, ErrSyncResumeInvalid)
}
func (s *googleDriveSyncSession) clearResumeOffset() {
s.resumePageOffset = 0
s.resumePageToken = ""
- s.resumeFilename = ""
+ s.resumeSourceID = ""
}
func (s *googleDriveSyncSession) syncCheckpoint(scope googleDriveScope, pageToken string, offset int, doc SourceDocument) *SyncCheckpoint {
@@ -782,7 +784,7 @@ func (s *googleDriveSyncSession) syncCheckpoint(scope googleDriveScope, pageToke
ScopeIndex: s.scopeIndex,
PageToken: pageToken,
Offset: offset,
- Filename: syncSourceDocumentFilenameFromDocument(doc),
+ SourceID: doc.SourceID,
})
if err != nil {
return nil
@@ -810,7 +812,6 @@ type googleDriveBufferedDocument struct {
document SourceDocument
checkpoint *SyncCheckpoint
offset int
- filename string
}
type googleDriveSyncCursor struct {
@@ -819,7 +820,7 @@ type googleDriveSyncCursor struct {
ScopeIndex int `json:"scope_index"`
PageToken string `json:"page_token,omitempty"`
Offset int `json:"offset"`
- Filename string `json:"filename"`
+ SourceID string `json:"source_id,omitempty"`
}
type googleDriveScopeCursor struct {
diff --git a/internal/syncer/connector/google_drive_test.go b/internal/syncer/connector/google_drive_test.go
index 5673caf66e..1b04faf761 100644
--- a/internal/syncer/connector/google_drive_test.go
+++ b/internal/syncer/connector/google_drive_test.go
@@ -150,8 +150,8 @@ func TestGoogleDriveConnectorOpenSyncResumesWithinPage(t *testing.T) {
}
}
-// TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged verifies page deletions resume from the current filename offset.
-func TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {
+// TestGoogleDriveConnectorResumeRejectsMissingRemoteAnchor verifies a deleted list item invalidates the saved anchor.
+func TestGoogleDriveConnectorResumeRejectsMissingRemoteAnchor(t *testing.T) {
connector, err := NewGoogleDriveConnector(map[string]any{
"my_drive_emails": "[email protected]",
"batch_size": 2,
@@ -199,15 +199,31 @@ func TestGoogleDriveConnectorResumeUsesFilenameWhenOffsetChanged(t *testing.T) {
if err != nil {
t.Fatalf("resume OpenSync failed: %v", err)
}
- second, err := resumed.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("resume NextBatch failed: %v", err)
+ _, err = resumed.NextBatch(context.Background())
+ if err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
}
- if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://drive.google.com/file/d/file-3" {
- t.Fatalf("resume documents = %+v, want file-3", second.Documents)
+ if resumeListCalls == 0 {
+ t.Fatalf("resume should have listed the current remote page")
}
- if resumeListCalls != 1 {
- t.Fatalf("resume list calls = %d, want one checkpoint page", resumeListCalls)
+}
+
+func TestGoogleDriveConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector, err := NewGoogleDriveConnector(map[string]any{
+ "my_drive_emails": "[email protected]",
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "google_primary_admin": "[email protected]",
+ "google_tokens": `{"client_id":"client","client_secret":"secret","refresh_token":"refresh"}`,
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewGoogleDriveConnector failed: %v", err)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
}
}
diff --git a/internal/syncer/connector/imap.go b/internal/syncer/connector/imap.go
index 6402f97889..d19d430b08 100644
--- a/internal/syncer/connector/imap.go
+++ b/internal/syncer/connector/imap.go
@@ -144,7 +144,10 @@ func (c *IMAPConnector) OpenSync(ctx context.Context, request SyncRequest) (Sync
windowEnd: request.WindowEnd,
hasMore: true,
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ _ = client.Close()
+ return nil, err
+ }
return session, nil
}
@@ -170,11 +173,12 @@ type imapSyncSession struct {
windowStart *time.Time
windowEnd time.Time
- todoMailboxes []string
- currentMailbox string
- todoEmailIDs []string
- selected string
- hasMore bool
+ todoMailboxes []string
+ currentMailbox string
+ todoEmailIDs []string
+ selected string
+ hasMore bool
+ resumeValidated bool
}
// NextBatch returns the next IMAP document batch.
@@ -225,6 +229,9 @@ func (s *imapSyncSession) Close() error {
// ensureCurrentEmail makes sure the session has a list of mailboxes and a
// current mailbox with remaining email IDs, advancing as needed.
func (s *imapSyncSession) ensureCurrentEmail(ctx context.Context) error {
+ if err := s.validateResume(ctx); err != nil {
+ return err
+ }
if s.todoMailboxes == nil {
mailboxes, err := s.listMailboxes(ctx)
if err != nil {
@@ -264,6 +271,52 @@ func (s *imapSyncSession) ensureCurrentEmail(ctx context.Context) error {
}
}
+func (s *imapSyncSession) validateResume(ctx context.Context) error {
+ if s.resumeValidated || s.todoMailboxes == nil {
+ return nil
+ }
+ mailboxes, err := s.listMailboxes(ctx)
+ if err != nil {
+ return err
+ }
+ current := make(map[string]struct{}, len(mailboxes))
+ for _, mailbox := range mailboxes {
+ current[mailbox] = struct{}{}
+ }
+ for _, mailbox := range s.todoMailboxes {
+ if _, ok := current[mailbox]; !ok {
+ return fmt.Errorf("imap resume mailbox %q was not found in the current listing: %w", mailbox, ErrSyncResumeInvalid)
+ }
+ }
+ if s.currentMailbox != "" {
+ if _, ok := current[s.currentMailbox]; !ok {
+ return fmt.Errorf("imap resume mailbox %q was not found in the current listing: %w", s.currentMailbox, ErrSyncResumeInvalid)
+ }
+ emailIDs, err := s.searchMailbox(ctx, s.currentMailbox)
+ if err != nil {
+ return err
+ }
+ if !imapTodoEmailsMatch(s.todoEmailIDs, emailIDs) {
+ return fmt.Errorf("imap resume email state no longer matches mailbox %q: %w", s.currentMailbox, ErrSyncResumeInvalid)
+ }
+ }
+ s.resumeValidated = true
+ return nil
+}
+
+func imapTodoEmailsMatch(todo, current []string) bool {
+ if len(todo) > len(current) {
+ return false
+ }
+ start := len(current) - len(todo)
+ for index, emailID := range todo {
+ if emailID != current[start+index] {
+ return false
+ }
+ }
+ return true
+}
+
// listMailboxes returns configured mailboxes or discovers all mailboxes.
func (s *imapSyncSession) listMailboxes(ctx context.Context) ([]string, error) {
if len(s.connector.mailboxes) > 0 {
@@ -331,13 +384,27 @@ func (s *imapSyncSession) checkpoint(lastDoc *SourceDocument) *SyncCheckpoint {
}
// applyResume restores a sync session from a previously committed checkpoint.
-func (s *imapSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *imapSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("imap sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor imapCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("imap sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ if !cursor.HasMore {
+ return fmt.Errorf("imap sync checkpoint has no remaining work: %w", ErrSyncResumeInvalid)
+ }
+ if len(cursor.TodoMailboxes) == 0 && cursor.CurrentMailbox == nil {
+ return fmt.Errorf("imap sync checkpoint has no mailbox anchor: %w", ErrSyncResumeInvalid)
+ }
+ if cursor.CurrentMailbox != nil {
+ if cursor.CurrentMailbox.Mailbox == "" || len(cursor.CurrentMailbox.TodoEmailIDs) == 0 {
+ return fmt.Errorf("imap sync checkpoint has no email anchor: %w", ErrSyncResumeInvalid)
+ }
}
s.todoMailboxes = cursor.TodoMailboxes
s.hasMore = cursor.HasMore
@@ -345,6 +412,8 @@ func (s *imapSyncSession) applyResume(checkpoint *SyncCheckpoint) {
s.currentMailbox = cursor.CurrentMailbox.Mailbox
s.todoEmailIDs = cursor.CurrentMailbox.TodoEmailIDs
}
+ s.resumeValidated = false
+ return nil
}
type imapCursor struct {
diff --git a/internal/syncer/connector/imap_test.go b/internal/syncer/connector/imap_test.go
index 7b8a2e394e..881c49906e 100644
--- a/internal/syncer/connector/imap_test.go
+++ b/internal/syncer/connector/imap_test.go
@@ -20,6 +20,7 @@ import (
"context"
"encoding/base64"
"encoding/json"
+ "errors"
"io"
"strings"
"testing"
@@ -285,6 +286,9 @@ func TestIMAPSyncResumeSelectsMailboxBeforeFetch(t *testing.T) {
// select call before fetching, matching a real resumed IMAP connection.
client := &fakeIMAPClient{
mailboxes: []string{"INBOX"},
+ searchByMailbox: map[string][]uint32{
+ "INBOX": {2},
+ },
rawBySeq: map[uint32][]byte{
2: rawIMAPEmail("msg2", "Hello two", "Mon, 2 Jan 2026 22:00:00 +0000", "body two"),
},
@@ -313,6 +317,83 @@ func TestIMAPSyncResumeSelectsMailboxBeforeFetch(t *testing.T) {
}
}
+func TestIMAPSyncResumeRejectsInvalidCheckpoint(t *testing.T) {
+ client := &fakeIMAPClient{}
+ connector := newTestIMAPConnector(client, 32)
+ cases := map[string]*SyncCheckpoint{
+ "missing": {},
+ "malformed": {Cursor: "not-json"},
+ "no-mailbox": {Cursor: `{"todo_mailboxes":[],"has_more":true}`},
+ "no-email": {Cursor: `{"todo_mailboxes":[],"has_more":true,"current_mailbox":{"mailbox":"INBOX","todo_email_ids":[]}}`},
+ "completed": {Cursor: `{"todo_mailboxes":["INBOX"],"has_more":false}`},
+ }
+ for name, checkpoint := range cases {
+ t.Run(name, func(t *testing.T) {
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: checkpoint,
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+ })
+ }
+}
+
+func TestIMAPSyncResumeRejectsStaleMailbox(t *testing.T) {
+ cursorData, err := json.Marshal(imapCursor{
+ TodoMailboxes: []string{"MISSING"},
+ HasMore: true,
+ })
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ connector := newTestIMAPConnector(&fakeIMAPClient{mailboxes: []string{"INBOX"}}, 32)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: string(cursorData)},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer session.Close()
+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestIMAPSyncResumeRejectsStaleEmailState(t *testing.T) {
+ cursorData, err := json.Marshal(imapCursor{
+ TodoMailboxes: []string{},
+ HasMore: true,
+ CurrentMailbox: &imapMailboxCursor{
+ Mailbox: "INBOX",
+ TodoEmailIDs: []string{"3"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ client := &fakeIMAPClient{
+ mailboxes: []string{"INBOX"},
+ searchByMailbox: map[string][]uint32{
+ "INBOX": {1, 2},
+ },
+ }
+ connector := newTestIMAPConnector(client, 32)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: string(cursorData)},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer session.Close()
+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
func TestIMAPSyncFiltersOutsideWindow(t *testing.T) {
start := mustTime(t, "2026-01-02T00:00:00Z")
end := mustTime(t, "2026-01-03T00:00:00Z")
diff --git a/internal/syncer/connector/jira.go b/internal/syncer/connector/jira.go
index 51d3bd220d..73caf58a6f 100644
--- a/internal/syncer/connector/jira.go
+++ b/internal/syncer/connector/jira.go
@@ -181,7 +181,9 @@ func (c *JiraConnector) OpenSync(ctx context.Context, request SyncRequest) (Sync
windowEnd: end,
fromBeginning: request.FromBeginning,
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -600,12 +602,17 @@ type jiraSyncSession struct {
nextPageToken string
done bool
resumeSource string
+ resumeSkip bool
+ resumeChecked bool
}
func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
if s.done {
return SyncBatch{}, io.EOF
}
+ if err := s.validateResumeSource(ctx); err != nil {
+ return SyncBatch{}, err
+ }
documents := make([]SourceDocument, 0, s.batchSize)
var checkpoint *SyncCheckpoint
for len(documents) < s.batchSize && !s.done {
@@ -614,7 +621,7 @@ func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
return SyncBatch{}, err
}
for _, issue := range page.Issues {
- if s.resumeSource != "" {
+ if s.resumeSkip && s.resumeSource != "" {
if s.connector.issueURL(issue.Key) == s.resumeSource {
s.resumeSource = ""
}
@@ -649,6 +656,28 @@ func (s *jiraSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
}
+func (s *jiraSyncSession) validateResumeSource(ctx context.Context) error {
+ if s.resumeSource == "" || s.resumeChecked {
+ return nil
+ }
+ s.resumeChecked = true
+ prefix := strings.TrimRight(s.connector.baseURL, "/") + "/browse/"
+ key := strings.TrimPrefix(s.resumeSource, prefix)
+ if key == "" || key == s.resumeSource {
+ return fmt.Errorf("jira sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ var page jiraSearchPage
+ if err := s.connector.searchJQL(ctx, fmt.Sprintf(`key = "%s"`, strings.ReplaceAll(key, `"`, `\"`)), 0, 1, jiraSlimFields, "", &page); err != nil {
+ return err
+ }
+ for _, issue := range page.Issues {
+ if s.connector.issueURL(issue.Key) == s.resumeSource {
+ return nil
+ }
+ }
+ return fmt.Errorf("jira resume anchor %q was not found in the current listing: %w", s.resumeSource, ErrSyncResumeInvalid)
+}
+
func (s *jiraSyncSession) Close() error { return nil }
func (s *jiraSyncSession) nextIssuePage(ctx context.Context) (jiraSearchPage, error) {
@@ -667,20 +696,27 @@ func (s *jiraSyncSession) nextIssuePage(ctx context.Context) (jiraSearchPage, er
return page, nil
}
-func (s *jiraSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *jiraSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
- var cursor jiraSyncCursor
- if checkpoint.Cursor != "" && json.Unmarshal([]byte(checkpoint.Cursor), &cursor) == nil {
- s.startAt = cursor.StartAt
- s.nextPageToken = cursor.NextPageToken
- if s.startAt == 0 && s.nextPageToken == "" {
- s.resumeSource = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
- }
- return
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("jira sync cursor is missing: %w", ErrSyncResumeInvalid)
}
- s.resumeSource = checkpoint.SourceID
+ var cursor jiraSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return fmt.Errorf("jira sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ prefix := strings.TrimRight(s.connector.baseURL, "/") + "/browse/"
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return fmt.Errorf("jira sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ s.resumeSource = sourceID
+ s.startAt = cursor.StartAt
+ s.nextPageToken = cursor.NextPageToken
+ s.resumeSkip = s.startAt == 0 && s.nextPageToken == ""
+ return nil
}
type jiraPruneSession struct {
diff --git a/internal/syncer/connector/jira_test.go b/internal/syncer/connector/jira_test.go
index 4723d22a42..8f160f0995 100644
--- a/internal/syncer/connector/jira_test.go
+++ b/internal/syncer/connector/jira_test.go
@@ -71,6 +71,106 @@ func TestJiraConnectorOpenSyncCloud(t *testing.T) {
}
}
+func TestJiraConnectorOpenSyncResumeSkipsCommittedSource(t *testing.T) {
+ server := jiraResumeFixtureServer(t, []map[string]any{
+ {"id": "10000", "key": "RAG-7"},
+ {"id": "10001", "key": "RAG-8"},
+ })
+ defer server.Close()
+
+ connector, err := NewJiraConnector(map[string]any{
+ "base_url": server.URL,
+ "project_key": "RAG",
+ "batch_size": 10,
+ "timezone_offset": 0,
+ "credentials": map[string]any{
+ "jira_api_token": "token",
+ "rest_api_version": "3",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewJiraConnector failed: %v", err)
+ }
+ cursorData, err := json.Marshal(jiraSyncCursor{
+ StartAt: 0,
+ SourceID: server.URL + "/browse/RAG-7",
+ })
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: string(cursorData),
+ SourceID: server.URL + "/browse/RAG-7",
+ },
+ })
+ if err != nil {
+ t.Fatalf("resumed OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resumed NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != server.URL+"/browse/RAG-8" {
+ t.Fatalf("resumed documents = %+v, want RAG-8 only", batch.Documents)
+ }
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("resumed final NextBatch = %v, want io.EOF", err)
+ }
+}
+
+func TestJiraConnectorOpenSyncResumeRejectsInvalidCheckpoint(t *testing.T) {
+ server := jiraFixtureServer(t)
+ defer server.Close()
+ connector := mustJiraResumeConnector(t, server.URL)
+ cases := map[string]*SyncCheckpoint{
+ "missing": {},
+ "malformed": {Cursor: "not-json"},
+ "no-anchor": {Cursor: `{"start_at":1}`},
+ "foreign": {Cursor: `{"start_at":1,"source_id":"https://other.example/browse/RAG-7"}`},
+ }
+ for name, checkpoint := range cases {
+ t.Run(name, func(t *testing.T) {
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: checkpoint,
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+ })
+ }
+}
+
+func TestJiraConnectorOpenSyncResumeRejectsMissingIssue(t *testing.T) {
+ server := jiraResumeFixtureServer(t, []map[string]any{})
+ defer server.Close()
+ connector := mustJiraResumeConnector(t, server.URL)
+ cursorData, err := json.Marshal(jiraSyncCursor{
+ StartAt: 0,
+ SourceID: server.URL + "/browse/RAG-999",
+ })
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: string(cursorData),
+ SourceID: server.URL + "/browse/RAG-999",
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer session.Close()
+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
func TestJiraConnectorOpenPrune(t *testing.T) {
server := jiraFixtureServer(t)
defer server.Close()
@@ -273,6 +373,48 @@ func jiraFixtureServer(t *testing.T) *httptest.Server {
return server
}
+func jiraResumeFixtureServer(t *testing.T, keys []map[string]any) *httptest.Server {
+ t.Helper()
+ var server *httptest.Server
+ mux := http.NewServeMux()
+ mux.HandleFunc("/rest/api/3/project/RAG", func(w http.ResponseWriter, r *http.Request) {
+ jiraWriteJSON(t, w, map[string]any{"key": "RAG"})
+ })
+ mux.HandleFunc("/rest/api/3/search/jql", func(w http.ResponseWriter, r *http.Request) {
+ jiraWriteJSON(t, w, map[string]any{"issues": keys})
+ })
+ mux.HandleFunc("/rest/api/3/issue/bulkfetch", func(w http.ResponseWriter, r *http.Request) {
+ issues := make([]map[string]any, 0, len(keys))
+ for _, meta := range keys {
+ issue := jiraFixtureIssue(server.URL)
+ issue["id"] = meta["id"]
+ issue["key"] = meta["key"]
+ issues = append(issues, issue)
+ }
+ jiraWriteJSON(t, w, map[string]any{"issues": issues})
+ })
+ server = httptest.NewServer(mux)
+ return server
+}
+
+func mustJiraResumeConnector(t *testing.T, baseURL string) *JiraConnector {
+ t.Helper()
+ connector, err := NewJiraConnector(map[string]any{
+ "base_url": baseURL,
+ "project_key": "RAG",
+ "batch_size": 10,
+ "timezone_offset": 0,
+ "credentials": map[string]any{
+ "jira_api_token": "token",
+ "rest_api_version": "3",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewJiraConnector failed: %v", err)
+ }
+ return connector
+}
+
func jiraFixtureIssue(baseURL string) map[string]any {
return map[string]any{
"id": "10000",
diff --git a/internal/syncer/connector/mock/mock.go b/internal/syncer/connector/mock/mock.go
index 425a8bdd2d..af4e17dda2 100644
--- a/internal/syncer/connector/mock/mock.go
+++ b/internal/syncer/connector/mock/mock.go
@@ -28,12 +28,17 @@ type Connector struct {
ValidateConnectorSettingErr error
SyncBatches []syncerconnector.SyncBatch
SyncErrAt int
+ NextBatchResumeInvalidAt int
+ NextBatchResumeFirstSession bool
PruneBatches []syncerconnector.PruneBatch
PruneErrAt int
FetchBlobs map[string][]byte
OnSyncBatch func(index int)
OnPruneBatch func(index int)
SyncRequests []syncerconnector.SyncRequest
+ OpenSyncResumeInvalidAt int
+ OpenSyncResumeInvalidAlways bool
+ openCount int
}
// Validate returns the configured validation error.
@@ -48,8 +53,12 @@ func (c *Connector) ValidateConnectorSetting(ctx context.Context, request map[st
// OpenSync opens a mock sync session.
func (c *Connector) OpenSync(ctx context.Context, request syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {
+ c.openCount++
c.SyncRequests = append(c.SyncRequests, request)
- return &SyncSession{connector: c}, nil
+ if c.OpenSyncResumeInvalidAlways || (c.OpenSyncResumeInvalidAt > 0 && c.openCount == c.OpenSyncResumeInvalidAt) {
+ return nil, syncerconnector.ErrSyncResumeInvalid
+ }
+ return &SyncSession{connector: c, openCount: c.openCount}, nil
}
// OpenPrune opens a mock prune session.
@@ -60,6 +69,7 @@ func (c *Connector) OpenPrune(ctx context.Context, request syncerconnector.Prune
// SyncSession streams configured sync batches.
type SyncSession struct {
connector *Connector
+ openCount int
index int
}
@@ -68,6 +78,11 @@ func (s *SyncSession) NextBatch(ctx context.Context) (syncerconnector.SyncBatch,
if s.connector.SyncErrAt > 0 && s.index+1 == s.connector.SyncErrAt {
return syncerconnector.SyncBatch{}, io.ErrUnexpectedEOF
}
+ if s.connector.NextBatchResumeInvalidAt > 0 && s.index+1 == s.connector.NextBatchResumeInvalidAt {
+ if !s.connector.NextBatchResumeFirstSession || s.openCount == 1 {
+ return syncerconnector.SyncBatch{}, syncerconnector.ErrSyncResumeInvalid
+ }
+ }
if s.index >= len(s.connector.SyncBatches) {
return syncerconnector.SyncBatch{}, io.EOF
}
diff --git a/internal/syncer/connector/models.go b/internal/syncer/connector/models.go
index f064a40871..3d58ad0467 100644
--- a/internal/syncer/connector/models.go
+++ b/internal/syncer/connector/models.go
@@ -16,7 +16,15 @@
package connector
-import "time"
+import (
+ "errors"
+ "time"
+)
+
+// ErrSyncResumeInvalid reports that a saved sync resume anchor no longer exists
+// in the current source listing. The runner treats this as invalid progress and
+// restarts the current task window instead of silently guessing a new offset.
+var ErrSyncResumeInvalid = errors.New("sync resume checkpoint is no longer valid")
// SourceDocument is the normalized document emitted by datasource connectors.
type SourceDocument struct {
@@ -54,6 +62,7 @@ type SyncCheckpointState struct {
WindowEnd time.Time `json:"window_end"`
NextCommitSeq int64 `json:"next_commit_seq"`
Checkpoint *SyncCheckpoint `json:"checkpoint,omitempty"`
+ RestartCount int `json:"restart_count,omitempty"`
Added int64 `json:"added,omitempty"`
Updated int64 `json:"updated,omitempty"`
Skipped int64 `json:"skipped,omitempty"`
diff --git a/internal/syncer/connector/moodle.go b/internal/syncer/connector/moodle.go
index 1770427e65..4ccc81aec0 100644
--- a/internal/syncer/connector/moodle.go
+++ b/internal/syncer/connector/moodle.go
@@ -125,7 +125,9 @@ func (c *MoodleConnector) OpenSync(ctx context.Context, request SyncRequest) (Sy
batchSize: c.batchSize,
}
if request.Resume != nil {
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
}
return session, nil
}
@@ -668,6 +670,8 @@ type moodleSyncSession struct {
request SyncRequest
batchSize int
resumeAfterCourseID int64
+ resumeCourseSet bool
+ resumeCourseChecked bool
coursesLoaded bool
courses []moodleCourse
@@ -691,6 +695,19 @@ func (s *moodleSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
}
s.courses = courses
s.coursesLoaded = true
+ if s.resumeCourseSet && !s.resumeCourseChecked {
+ found := false
+ for _, course := range s.courses {
+ if course.ID == s.resumeAfterCourseID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return SyncBatch{}, fmt.Errorf("moodle resume course %d was not found in the current listing: %w", s.resumeAfterCourseID, ErrSyncResumeInvalid)
+ }
+ s.resumeCourseChecked = true
+ }
}
if s.courseIndex >= len(s.courses) {
s.done = true
@@ -743,17 +760,22 @@ func (s *moodleSyncSession) Close() error {
return nil
}
-func (s *moodleSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *moodleSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
const prefix = "moodle_course_"
- if !strings.HasPrefix(sourceID, prefix) {
- return
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return fmt.Errorf("moodle sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
courseID, err := strconv.ParseInt(strings.TrimPrefix(sourceID, prefix), 10, 64)
if err != nil {
- return
+ return fmt.Errorf("moodle sync cursor is invalid: %w", ErrSyncResumeInvalid)
}
s.resumeAfterCourseID = courseID
+ s.resumeCourseSet = true
+ return nil
}
type moodlePruneSession struct {
diff --git a/internal/syncer/connector/moodle_test.go b/internal/syncer/connector/moodle_test.go
index bc80d12602..f0432ae206 100644
--- a/internal/syncer/connector/moodle_test.go
+++ b/internal/syncer/connector/moodle_test.go
@@ -365,6 +365,40 @@ func TestMoodleConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
}
}
+func TestMoodleConnectorOpenSyncResumeRejectsMissingCourse(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "moodle_course_999",
+ SourceID: "moodle_course_999",
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestMoodleConnectorOpenSyncResumeRejectsMalformedCursor(t *testing.T) {
+ withMoodleTestHooks(t)
+ server := newTestMoodleServer(t, fullSyncMoodleFixtures)
+ connector := mustMoodleConnector(t, server.URL)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "moodle_course_not-a-number",
+ },
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestMoodleConnectorOpenSyncDeferredCoursesUntilNextBatch(t *testing.T) {
withMoodleTestHooks(t)
var courseListCount atomic.Int64
diff --git a/internal/syncer/connector/notion.go b/internal/syncer/connector/notion.go
index 6c22c0c5da..d2de01c3da 100644
--- a/internal/syncer/connector/notion.go
+++ b/internal/syncer/connector/notion.go
@@ -112,11 +112,15 @@ func (c *NotionConnector) OpenSync(ctx context.Context, request SyncRequest) (Sy
}
c.indexedPages = map[string]bool{}
c.pagePathCache = map[string]string{}
+ resumeSourceID, err := notionResumeSourceID(request.Resume)
+ if err != nil {
+ return nil, err
+ }
session := &notionSyncSession{
connector: c,
request: request,
batchSize: c.batchSize,
- resumeSourceID: notionResumeSourceID(request.Resume),
+ resumeSourceID: resumeSourceID,
searchRequest: notionSearchRequest{
Filter: map[string]any{"property": "object", "value": "page"},
PageSize: 100,
@@ -618,7 +622,7 @@ func (s *notionSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
pageDocs, err := s.nextPageDocuments(ctx)
if errors.Is(err, io.EOF) {
if s.resumeSourceID != "" && !s.resumeMatched {
- return SyncBatch{}, fmt.Errorf("notion sync resume checkpoint %q was not found in the source; refusing to discard unprocessed documents", s.resumeSourceID)
+ return SyncBatch{}, fmt.Errorf("notion sync resume checkpoint %q was not found in the source: %w", s.resumeSourceID, ErrSyncResumeInvalid)
}
if len(documents) == 0 {
return SyncBatch{}, io.EOF
@@ -869,11 +873,15 @@ func notionSearchResultsOlderThanWindowStart(pages []notionPage, request SyncReq
return true
}
-func notionResumeSourceID(checkpoint *SyncCheckpoint) string {
+func notionResumeSourceID(checkpoint *SyncCheckpoint) (string, error) {
if checkpoint == nil {
- return ""
+ return "", nil
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ if sourceID == "" {
+ return "", fmt.Errorf("notion sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
- return firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ return sourceID, nil
}
func includeNotionUpdatedAt(updatedAt time.Time, request SyncRequest) bool {
diff --git a/internal/syncer/connector/notion_test.go b/internal/syncer/connector/notion_test.go
index 5b45778d82..e35e18ea70 100644
--- a/internal/syncer/connector/notion_test.go
+++ b/internal/syncer/connector/notion_test.go
@@ -152,8 +152,8 @@ func TestNotionConnectorResumeMissingCheckpointFails(t *testing.T) {
if err != nil {
t.Fatalf("OpenSync: %v", err)
}
- if _, err = session.NextBatch(context.Background()); err == nil || errors.Is(err, io.EOF) {
- t.Fatalf("NextBatch = %v, want missing checkpoint error", err)
+ if _, err = session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch = %v, want ErrSyncResumeInvalid", err)
}
}
diff --git a/internal/syncer/connector/outlook.go b/internal/syncer/connector/outlook.go
index 456d77ef63..4966d5c459 100644
--- a/internal/syncer/connector/outlook.go
+++ b/internal/syncer/connector/outlook.go
@@ -136,7 +136,9 @@ func (c *OutlookConnector) OpenSync(ctx context.Context, request SyncRequest) (S
windowEnd: request.WindowEnd,
deltaLinks: map[string]string{},
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -451,7 +453,10 @@ func (s *outlookSyncSession) nextDocumentPage(ctx context.Context) ([]outlookBuf
})
}
- documents := s.filterResumedDocuments(requestURL, candidates)
+ documents, err := s.filterResumedDocuments(requestURL, candidates)
+ if err != nil {
+ return nil, err
+ }
if page.NextLink != "" {
s.pageURL = page.NextLink
return documents, nil
@@ -481,19 +486,26 @@ func (s *outlookSyncSession) inWindow(updatedAt time.Time) bool {
return true
}
-func (s *outlookSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *outlookSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("outlook sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor outlookSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("outlook sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" {
+ return fmt.Errorf("outlook sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
if len(cursor.DeltaLinks) > 0 {
s.deltaLinks = cursor.DeltaLinks
}
if cursor.UserID == "" {
- return
+ return fmt.Errorf("outlook sync cursor has no user anchor: %w", ErrSyncResumeInvalid)
}
for index, userID := range s.users {
if userID != cursor.UserID {
@@ -503,38 +515,32 @@ func (s *outlookSyncSession) applyResume(checkpoint *SyncCheckpoint) {
s.pageURL = cursor.PageURL
s.resumePageURL = cursor.PageURL
s.resumeOffset = cursor.Offset
- s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
- return
+ s.resumeSourceID = sourceID
+ return nil
}
+ return fmt.Errorf("outlook resume user %q was not found in the current listing: %w", cursor.UserID, ErrSyncResumeInvalid)
}
-func (s *outlookSyncSession) filterResumedDocuments(pageURL string, candidates []outlookBufferedDocument) []outlookBufferedDocument {
- if s.resumeOffset <= 0 {
- return candidates
+func (s *outlookSyncSession) filterResumedDocuments(pageURL string, candidates []outlookBufferedDocument) ([]outlookBufferedDocument, error) {
+ if s.resumeSourceID == "" {
+ return candidates, nil
}
if pageURL != s.resumePageURL {
- s.clearResumeOffset()
- return candidates
- }
- if s.resumeOffset <= len(candidates) && candidates[s.resumeOffset-1].document.SourceID == s.resumeSourceID {
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumeOffset {
- filtered = append(filtered, candidate)
- }
- }
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("outlook resume page no longer matches checkpoint page: %w", ErrSyncResumeInvalid)
}
- filtered := candidates[:0]
for _, candidate := range candidates {
- if candidate.offset >= s.resumeOffset {
- filtered = append(filtered, candidate)
- continue
+ if candidate.document.SourceID == s.resumeSourceID {
+ filtered := candidates[:0]
+ for _, remaining := range candidates {
+ if remaining.offset > candidate.offset {
+ filtered = append(filtered, remaining)
+ }
+ }
+ s.clearResumeOffset()
+ return filtered, nil
}
}
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("outlook resume anchor %q was not found on %s: %w", s.resumeSourceID, pageURL, ErrSyncResumeInvalid)
}
func (s *outlookSyncSession) checkpoint(cursor outlookSyncCursor, doc SourceDocument) *SyncCheckpoint {
diff --git a/internal/syncer/connector/outlook_test.go b/internal/syncer/connector/outlook_test.go
index e2e7c17906..dcb3efd4af 100644
--- a/internal/syncer/connector/outlook_test.go
+++ b/internal/syncer/connector/outlook_test.go
@@ -103,6 +103,60 @@ func TestOutlookConnectorOpenSyncResumesWithinPage(t *testing.T) {
}
}
+func TestOutlookConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := newFixtureOutlookConnector()
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
+func TestOutlookConnectorOpenSyncResumeRejectsMissingRemoteAnchor(t *testing.T) {
+ connector := newFixtureOutlookConnector()
+ connector.OutlookConnector.batchSize = 2
+ connector.OutlookConnector.getDeltaPage = func(ctx context.Context, apiURL string) (outlookDeltaPage, error) {
+ return outlookDeltaPage{
+ DeltaLink: "delta-1",
+ Value: []outlookMessage{
+ outlookTestMessage("msg-1", "One", "2026-01-02T03:04:05Z"),
+ outlookTestMessage("msg-2", "Two", "2026-01-02T04:04:05Z"),
+ outlookTestMessage("msg-3", "Three", "2026-01-02T05:04:05Z"),
+ },
+ }, nil
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "msg-2" {
+ t.Fatalf("first checkpoint = %+v, want msg-2", first.Checkpoint)
+ }
+
+ connector.OutlookConnector.getDeltaPage = func(ctx context.Context, apiURL string) (outlookDeltaPage, error) {
+ return outlookDeltaPage{
+ DeltaLink: "delta-1",
+ Value: []outlookMessage{
+ outlookTestMessage("msg-1", "One", "2026-01-02T03:04:05Z"),
+ outlookTestMessage("msg-3", "Three", "2026-01-02T05:04:05Z"),
+ },
+ }, nil
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ if _, err = resumed.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
func TestOutlookConnectorOpenPrune(t *testing.T) {
connector := newFixtureOutlookConnector()
session, err := connector.OpenPrune(context.Background(), PruneRequest{})
diff --git a/internal/syncer/connector/r2.go b/internal/syncer/connector/r2.go
index 3705f7f03c..effc88fcfd 100644
--- a/internal/syncer/connector/r2.go
+++ b/internal/syncer/connector/r2.go
@@ -128,7 +128,9 @@ func (c *R2Connector) OpenSync(ctx context.Context, request SyncRequest) (SyncSe
batchSize: c.batchSize,
}
if request.Resume != nil {
- session.applyResume(request.Resume)
+ if err := session.applyResume(ctx, request.Resume); err != nil {
+ return nil, err
+ }
}
return session, nil
}
@@ -306,13 +308,41 @@ func (s *r2SyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte,
return s.connector.Fetch(ctx, ref)
}
-func (s *r2SyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *r2SyncSession) applyResume(ctx context.Context, checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
prefix := r2SourceID(s.connector.bucketName, "")
if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
- return
+ return fmt.Errorf("cloudflare r2 sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ anchor := strings.TrimPrefix(sourceID, prefix)
+ if anchor == "" {
+ return fmt.Errorf("cloudflare r2 sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+
+ startAfter := ""
+ for {
+ objects, nextStartAfter, hasMore, err := s.connector.listObjectPage(ctx, startAfter, int32(s.batchSize))
+ if err != nil {
+ return err
+ }
+ for _, object := range objects {
+ if object.Key == anchor {
+ s.startAfter = anchor
+ return nil
+ }
+ if object.Key > anchor {
+ return fmt.Errorf("cloudflare r2 resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
+ }
+ }
+ if !hasMore {
+ break
+ }
+ startAfter = strings.TrimPrefix(nextStartAfter, prefix)
}
- s.startAfter = strings.TrimPrefix(sourceID, prefix)
+ return fmt.Errorf("cloudflare r2 resume anchor %q was not found in the current listing: %w", anchor, ErrSyncResumeInvalid)
}
func (c *R2Connector) sourceDocument(sourceID string, object r2Object) (SourceDocument, bool) {
diff --git a/internal/syncer/connector/r2_test.go b/internal/syncer/connector/r2_test.go
index 21bfbfec8e..5340ad4443 100644
--- a/internal/syncer/connector/r2_test.go
+++ b/internal/syncer/connector/r2_test.go
@@ -162,7 +162,7 @@ func TestR2ConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
}
}
-func TestR2ConnectorOpenSyncResumeIgnoresForeignCheckpoint(t *testing.T) {
+func TestR2ConnectorOpenSyncResumeRejectsForeignCheckpoint(t *testing.T) {
connector := newTestR2Connector(t, []r2Object{
{Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
})
@@ -172,15 +172,38 @@ func TestR2ConnectorOpenSyncResumeIgnoresForeignCheckpoint(t *testing.T) {
Cursor: "other:bucket:docs/a.txt",
},
})
- if err != nil {
- t.Fatalf("OpenSync: %v", err)
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
}
- batch, err := session.NextBatch(context.Background())
- if err != nil {
- t.Fatalf("NextBatch: %v", err)
+}
+
+func TestR2ConnectorOpenSyncResumeRejectsMissingAnchor(t *testing.T) {
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Key: "docs/c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: r2SourceID("bucket", "docs/b.txt"),
+ SourceID: r2SourceID("bucket", "docs/b.txt"),
+ },
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
}
- if len(batch.Documents) != 1 || batch.Documents[0].SourceID != r2SourceID("bucket", "docs/a.txt") {
- t.Fatalf("documents = %+v", batch.Documents)
+}
+
+func TestR2ConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := newTestR2Connector(t, []r2Object{
+ {Key: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ })
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
}
}
diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.go
index 2f33a42cfc..6b3002c7de 100644
--- a/internal/syncer/connector/rest_api.go
+++ b/internal/syncer/connector/rest_api.go
@@ -546,7 +546,9 @@ func (c *RestAPIConnector) OpenSync(ctx context.Context, request SyncRequest) (S
windowStart: windowStart,
windowEnd: request.WindowEnd.UTC(),
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -1112,8 +1114,9 @@ type restAPIItemIterator struct {
limit int
perPage int
cursor string
- seenCursors map[string]struct{}
finished bool
+ lastPos *restAPISyncCursor
+ seenCursors map[string]struct{}
}
func newRestAPIItemIterator(c *RestAPIConnector) *restAPIItemIterator {
@@ -1150,6 +1153,7 @@ func (it *restAPIItemIterator) nextPage(ctx context.Context) ([]map[string]any,
if it.finished || it.pageCount >= it.c.cfg.MaxPages {
return nil, nil
}
+ it.lastPos = it.positionCursor()
params := map[string]any{}
switch it.c.cfg.PaginationType {
case restAPIPaginationPage:
@@ -1213,7 +1217,7 @@ func (it *restAPIItemIterator) nextPage(ctx context.Context) ([]map[string]any,
if next == "" {
it.finished = true
} else if _, repeated := it.seenCursors[next]; repeated {
- it.finished = true
+ return nil, fmt.Errorf("rest api pagination repeated cursor %q: %w", next, ErrSyncResumeInvalid)
} else {
it.seenCursors[next] = struct{}{}
it.cursor = next
@@ -1222,21 +1226,30 @@ func (it *restAPIItemIterator) nextPage(ctx context.Context) ([]map[string]any,
return items, nil
}
-// positionAfterPage returns the resume position after the page that was just
-// consumed, i.e. the next page to fetch. It is nil for pagination_type=none,
-// which never emits checkpoints.
-func (it *restAPIItemIterator) positionAfterPage() *restAPISyncCursor {
+// positionCursor returns the position of the page that was just fetched. It is
+// nil for pagination_type=none, which never emits checkpoints.
+func (it *restAPIItemIterator) positionCursor() *restAPISyncCursor {
switch it.c.cfg.PaginationType {
case restAPIPaginationPage:
- return &restAPISyncCursor{Page: it.page}
+ return &restAPISyncCursor{Page: it.page, SourceID: ""}
case restAPIPaginationOffset:
- return &restAPISyncCursor{Offset: it.offset}
+ return &restAPISyncCursor{Offset: it.offset, SourceID: ""}
case restAPIPaginationCursor:
- return &restAPISyncCursor{Cursor: it.cursor}
+ return &restAPISyncCursor{Cursor: it.cursor, SourceID: ""}
}
return nil
}
+// currentPagePosition returns the position of the page fetched by the most
+// recent nextPage call. It keeps the page position, not the next page.
+func (it *restAPIItemIterator) currentPagePosition() *restAPISyncCursor {
+ if it.lastPos == nil {
+ return nil
+ }
+ pos := *it.lastPos
+ return &pos
+}
+
// ---------------------------------------------------------------------------
// JSON extraction
// ---------------------------------------------------------------------------
@@ -2108,68 +2121,131 @@ func restAPITimeISO(t time.Time) string {
// Sync session
// ---------------------------------------------------------------------------
-// restAPISyncCursor is the page-boundary resume position serialized into
-// SyncCheckpoint.Cursor. It stores the next page to fetch for the configured
-// pagination type; pagination_type=none never produces a checkpoint.
+// restAPISyncCursor is the resume position serialized into SyncCheckpoint.Cursor.
+// It stores the current page to fetch and the source document that was last
+// committed from that page; pagination_type=none never produces a checkpoint.
type restAPISyncCursor struct {
- Page int `json:"page,omitempty"`
- Offset int `json:"offset,omitempty"`
- Cursor string `json:"cursor,omitempty"`
+ Page int `json:"page,omitempty"`
+ Offset int `json:"offset,omitempty"`
+ Cursor string `json:"cursor,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
}
-// restAPIPageState tracks one fetched page until all of its documents have
-// been handed out in emitted batches. nextPos is the resume position after the
-// page (nil for pagination_type=none).
-type restAPIPageState struct {
- remaining int
- nextPos *restAPISyncCursor
+// restAPIBufferedDocument carries the checkpoint position for an individual
+// source document so a retried task can resume at the last emitted document.
+type restAPIBufferedDocument struct {
+ document SourceDocument
+ checkpoint *SyncCheckpoint
}
// restAPISyncSession streams items lazily so large APIs are not fully
// materialized in memory.
type restAPISyncSession struct {
- connector *RestAPIConnector
- iterator *restAPIItemIterator
- batchSize int
- fromBeginning bool
- windowStart *time.Time
- windowEnd time.Time
- pending []SourceDocument
- pages []restAPIPageState
- drained *restAPISyncCursor
-}
-
-// applyResume restores the next page to fetch from a saved checkpoint. Invalid
-// or empty cursors are ignored, mirroring the tolerant resume of other
-// connectors.
-func (s *restAPISyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ connector *RestAPIConnector
+ iterator *restAPIItemIterator
+ batchSize int
+ fromBeginning bool
+ windowStart *time.Time
+ windowEnd time.Time
+ pending []restAPIBufferedDocument
+ resumePosition *restAPISyncCursor
+ resumeSourceID string
+}
+
+// applyResume restores the checkpoint page and source anchor. Cursors that are
+// malformed, mismatch the configured pagination type, or carry no source anchor
+// are invalid because resuming would silently skip changed data.
+func (s *restAPISyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil || checkpoint.Cursor == "" {
- return
+ if checkpoint == nil {
+ return nil
+ }
+ return fmt.Errorf("rest api sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor restAPISyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("rest api sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" {
+ return fmt.Errorf("rest api sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
switch s.connector.cfg.PaginationType {
case restAPIPaginationPage:
- if cursor.Page > 0 {
- s.iterator.page = cursor.Page
+ if cursor.Page <= 0 || cursor.Offset != 0 || cursor.Cursor != "" {
+ return fmt.Errorf("rest api sync cursor does not match page pagination: %w", ErrSyncResumeInvalid)
}
+ s.iterator.page = cursor.Page
case restAPIPaginationOffset:
- if cursor.Offset > 0 {
- s.iterator.offset = cursor.Offset
+ if cursor.Page != 0 || cursor.Cursor != "" {
+ return fmt.Errorf("rest api sync cursor does not match offset pagination: %w", ErrSyncResumeInvalid)
}
+ s.iterator.offset = cursor.Offset
case restAPIPaginationCursor:
- if cursor.Cursor != "" {
- s.iterator.cursor = cursor.Cursor
+ if cursor.Page != 0 || cursor.Offset != 0 {
+ return fmt.Errorf("rest api sync cursor does not match cursor pagination: %w", ErrSyncResumeInvalid)
+ }
+ s.iterator.cursor = cursor.Cursor
+ default:
+ return fmt.Errorf("rest api sync checkpoint cannot resume pagination_type=none: %w", ErrSyncResumeInvalid)
+ }
+ position := cursor
+ position.SourceID = ""
+ s.resumePosition = &position
+ s.resumeSourceID = sourceID
+ return nil
+}
+
+// filterResumedDocuments drops every document at or before the checkpoint anchor.
+// The anchor must still exist on the checkpoint page, otherwise the listing has
+// changed and the runner should restart the whole fixed window.
+func (s *restAPISyncSession) filterResumedDocuments(candidates []SourceDocument) ([]SourceDocument, error) {
+ if s.resumeSourceID == "" {
+ return candidates, nil
+ }
+ if s.resumePosition == nil || !restAPISyncCursorPositionEqual(s.iterator.currentPagePosition(), s.resumePosition) {
+ return nil, fmt.Errorf("rest api sync resume page no longer matches checkpoint: %w", ErrSyncResumeInvalid)
+ }
+ for index, doc := range candidates {
+ if doc.SourceID == s.resumeSourceID {
+ s.resumePosition = nil
+ s.resumeSourceID = ""
+ return candidates[index+1:], nil
}
}
+ return nil, fmt.Errorf("rest api resume anchor %q was not found on the checkpoint page: %w", s.resumeSourceID, ErrSyncResumeInvalid)
}
-// NextBatch returns the next batch of source documents. The batch carries a
-// checkpoint only when at least one page has been fully handed out: the
-// checkpoint points at the next page to fetch, so a retried task re-fetches at
-// most the page where the last batch ended.
+func restAPISyncCursorPositionEqual(left, right *restAPISyncCursor) bool {
+ if left == nil || right == nil {
+ return false
+ }
+ return left.Page == right.Page && left.Offset == right.Offset && left.Cursor == right.Cursor
+}
+
+// restAPISyncCheckpoint records the current page position plus the document
+// that was last emitted from it.
+func restAPISyncCheckpoint(position *restAPISyncCursor, doc SourceDocument) *SyncCheckpoint {
+ if position == nil {
+ return nil
+ }
+ cursor := *position
+ cursor.SourceID = doc.SourceID
+ raw, err := json.Marshal(cursor)
+ if err != nil {
+ return nil
+ }
+ updatedAt := doc.UpdatedAt.UTC()
+ return &SyncCheckpoint{
+ Cursor: string(raw),
+ SourceID: doc.SourceID,
+ UpdatedAt: &updatedAt,
+ }
+}
+
+// NextBatch returns the next batch of source documents. The batch checkpoint
+// points back to the page of its last document, so a retried task re-fetches
+// that page and skips through the saved source anchor.
func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
for len(s.pending) < s.batchSize {
items, err := s.iterator.nextPage(ctx)
@@ -2179,7 +2255,7 @@ func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
if items == nil {
break
}
- pageDocs := 0
+ candidates := make([]SourceDocument, 0, len(items))
for _, item := range items {
doc, err := s.connector.itemToDocument(item)
if err != nil {
@@ -2191,10 +2267,19 @@ func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
continue
}
}
- s.pending = append(s.pending, doc)
- pageDocs++
+ candidates = append(candidates, doc)
+ }
+ candidates, err = s.filterResumedDocuments(candidates)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ position := s.iterator.currentPagePosition()
+ for _, doc := range candidates {
+ s.pending = append(s.pending, restAPIBufferedDocument{
+ document: doc,
+ checkpoint: restAPISyncCheckpoint(position, doc),
+ })
}
- s.pages = append(s.pages, restAPIPageState{remaining: pageDocs, nextPos: s.iterator.positionAfterPage()})
}
if len(s.pending) == 0 {
return SyncBatch{}, io.EOF
@@ -2203,32 +2288,16 @@ func (s *restAPISyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
if end > len(s.pending) {
end = len(s.pending)
}
- batch := SyncBatch{Documents: s.pending[:end]}
- s.pending = s.pending[end:]
-
- // Advance the resume point past every page fully covered by the batches
- // emitted so far. A page whose tail is still pending keeps the checkpoint
- // at the page start, so a retry re-fetches at most that one page.
- for len(s.pages) > 0 {
- if s.pages[0].remaining > end {
- s.pages[0].remaining -= end
- break
- }
- end -= s.pages[0].remaining
- if s.pages[0].nextPos != nil {
- s.drained = s.pages[0].nextPos
- }
- s.pages = s.pages[1:]
- if end == 0 {
- break
+ documents := make([]SourceDocument, 0, end)
+ var checkpoint *SyncCheckpoint
+ for _, buffered := range s.pending[:end] {
+ documents = append(documents, buffered.document)
+ if buffered.checkpoint != nil {
+ checkpoint = buffered.checkpoint
}
}
- if s.drained != nil {
- if raw, err := json.Marshal(s.drained); err == nil {
- batch.Checkpoint = &SyncCheckpoint{Cursor: string(raw)}
- }
- }
- return batch, nil
+ s.pending = s.pending[end:]
+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
}
// Close releases the session.
diff --git a/internal/syncer/connector/rest_api_test.go b/internal/syncer/connector/rest_api_test.go
index abfb8d72c4..73308a4a11 100644
--- a/internal/syncer/connector/rest_api_test.go
+++ b/internal/syncer/connector/rest_api_test.go
@@ -977,8 +977,8 @@ func TestRestAPISyncSessionPageResume(t *testing.T) {
t.Fatalf("documents=%v want ids 1,2", batch.Documents)
}
cursor := restAPICheckpointCursor(t, batch)
- if cursor.Page != 2 {
- t.Fatalf("cursor=%+v want page 2", cursor)
+ if cursor.Page != 1 || cursor.SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("cursor=%+v want page 1 source id 2", cursor)
}
resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
@@ -1006,8 +1006,8 @@ func TestRestAPISyncSessionPageResume(t *testing.T) {
mu.Lock()
got := append([]string(nil), requested...)
mu.Unlock()
- if len(got) != 3 || got[0] != "1" || got[1] != "2" || got[2] != "3" {
- t.Fatalf("requested pages=%v want [1 2 3]", got)
+ if len(got) != 4 || got[0] != "1" || got[1] != "1" || got[2] != "2" || got[3] != "3" {
+ t.Fatalf("requested pages=%v want [1 1 2 3]", got)
}
}
@@ -1064,8 +1064,8 @@ func TestRestAPISyncSessionOffsetResume(t *testing.T) {
t.Fatalf("documents=%v want ids 1,2", batch.Documents)
}
cursor := restAPICheckpointCursor(t, batch)
- if cursor.Offset != 2 {
- t.Fatalf("cursor=%+v want offset 2", cursor)
+ if cursor.Offset != 0 || cursor.SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("cursor=%+v want offset 0 source id 2", cursor)
}
resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
@@ -1093,8 +1093,8 @@ func TestRestAPISyncSessionOffsetResume(t *testing.T) {
mu.Lock()
got := append([]string(nil), requested...)
mu.Unlock()
- if len(got) != 3 || got[0] != "0" || got[1] != "2" || got[2] != "4" {
- t.Fatalf("requested offsets=%v want [0 2 4]", got)
+ if len(got) != 4 || got[0] != "0" || got[1] != "0" || got[2] != "2" || got[3] != "4" {
+ t.Fatalf("requested offsets=%v want [0 0 2 4]", got)
}
}
@@ -1153,8 +1153,8 @@ func TestRestAPISyncSessionCursorResume(t *testing.T) {
t.Fatalf("documents=%v want ids 1,2", batch.Documents)
}
cursor := restAPICheckpointCursor(t, batch)
- if cursor.Cursor != "t2" {
- t.Fatalf("cursor=%+v want cursor t2", cursor)
+ if cursor.Cursor != "" || cursor.SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("cursor=%+v want empty cursor source id 2", cursor)
}
resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
@@ -1182,15 +1182,20 @@ func TestRestAPISyncSessionCursorResume(t *testing.T) {
mu.Lock()
got := append([]string(nil), requested...)
mu.Unlock()
- if len(got) != 3 || got[0] != "" || got[1] != "t2" || got[2] != "t3" {
- t.Fatalf("requested cursors=%v want [\"\" t2 t3]", got)
+ if len(got) != 4 || got[0] != "" || got[1] != "" || got[2] != "t2" || got[3] != "t3" {
+ t.Fatalf("requested cursors=%v want [\"\" \"\" t2 t3]", got)
}
}
-func TestRestAPISyncSessionCheckpointWaitsForPageBoundary(t *testing.T) {
+func TestRestAPISyncSessionCheckpointResumesInsidePage(t *testing.T) {
withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var requested []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
page := r.URL.Query().Get("page")
+ mu.Lock()
+ requested = append(requested, page)
+ mu.Unlock()
items := []any{}
switch page {
case "1":
@@ -1228,8 +1233,9 @@ func TestRestAPISyncSessionCheckpointWaitsForPageBoundary(t *testing.T) {
if len(first.Documents) != 2 {
t.Fatalf("documents=%d want 2", len(first.Documents))
}
- if first.Checkpoint != nil {
- t.Fatalf("mid-page checkpoint=%+v want nil", first.Checkpoint)
+ firstCursor := restAPICheckpointCursor(t, first)
+ if firstCursor.Page != 1 || firstCursor.SourceID != restAPIHash128("rest_api:2") {
+ t.Fatalf("first cursor=%+v want page 1 source id 2", firstCursor)
}
second, err := session.NextBatch(context.Background())
session.Close()
@@ -1240,8 +1246,8 @@ func TestRestAPISyncSessionCheckpointWaitsForPageBoundary(t *testing.T) {
t.Fatalf("documents=%d want 2", len(second.Documents))
}
cursor := restAPICheckpointCursor(t, second)
- if cursor.Page != 2 {
- t.Fatalf("cursor=%+v want page 2", cursor)
+ if cursor.Page != 1 || cursor.SourceID != restAPIHash128("rest_api:4") {
+ t.Fatalf("cursor=%+v want page 1 source id 4", cursor)
}
resumed, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: second.Checkpoint})
@@ -1259,4 +1265,100 @@ func TestRestAPISyncSessionCheckpointWaitsForPageBoundary(t *testing.T) {
if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
t.Fatalf("err=%v want EOF", err)
}
+ mu.Lock()
+ got := append([]string(nil), requested...)
+ mu.Unlock()
+ if len(got) != 3 || got[0] != "1" || got[1] != "1" || got[2] != "2" {
+ t.Fatalf("requested pages=%v want [1 1 2]", got)
+ }
+}
+
+func TestRestAPISyncSessionResumeRejectsInvalidCheckpoint(t *testing.T) {
+ withRestAPITestHooks(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"items": []any{}})
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "page",
+ "pagination_config": map[string]any{"page_size": 2},
+ "request_delay": 0,
+ })
+ mismatched, _ := json.Marshal(restAPISyncCursor{Offset: 2, SourceID: "anchor"})
+ noAnchor, _ := json.Marshal(restAPISyncCursor{Page: 1})
+ cases := []struct {
+ name string
+ checkpoint *SyncCheckpoint
+ }{
+ {name: "missing cursor", checkpoint: &SyncCheckpoint{}},
+ {name: "malformed cursor", checkpoint: &SyncCheckpoint{Cursor: "{"}},
+ {name: "pagination mismatch", checkpoint: &SyncCheckpoint{Cursor: string(mismatched)}},
+ {name: "missing source anchor", checkpoint: &SyncCheckpoint{Cursor: string(noAnchor)}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: tc.checkpoint})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+ })
+ }
+}
+
+func TestRestAPISyncSessionResumeRejectsMissingAnchor(t *testing.T) {
+ withRestAPITestHooks(t)
+ var mu sync.Mutex
+ var requested []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ page := r.URL.Query().Get("page")
+ mu.Lock()
+ requested = append(requested, page)
+ mu.Unlock()
+ items := []any{}
+ switch page {
+ case "1":
+ items = []any{
+ map[string]any{"id": "10", "title": "Ten"},
+ map[string]any{"id": "11", "title": "Eleven"},
+ }
+ default:
+ items = []any{
+ map[string]any{"id": "2", "title": "Two"},
+ map[string]any{"id": "3", "title": "Three"},
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ json.NewEncoder(w).Encode(map[string]any{"items": items})
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ "id_field": "id",
+ "pagination_type": "page",
+ "pagination_config": map[string]any{"page_size": 2},
+ "batch_size": 2,
+ "request_delay": 0,
+ })
+ raw, _ := json.Marshal(restAPISyncCursor{Page: 1, SourceID: restAPIHash128("rest_api:2")})
+ session, err := c.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{Cursor: string(raw)}})
+ if err != nil {
+ t.Fatalf("resume OpenSync: %v", err)
+ }
+ defer session.Close()
+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch = %v, want ErrSyncResumeInvalid", err)
+ }
+ mu.Lock()
+ got := append([]string(nil), requested...)
+ mu.Unlock()
+ if len(got) != 1 || got[0] != "1" {
+ t.Fatalf("requested pages=%v want [1]", got)
+ }
}
diff --git a/internal/syncer/connector/rss.go b/internal/syncer/connector/rss.go
index 44af351f67..722a8f59bd 100644
--- a/internal/syncer/connector/rss.go
+++ b/internal/syncer/connector/rss.go
@@ -118,7 +118,9 @@ func (c *RSSConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncS
documents = append(documents, entry.toSourceDocument(c.feedURL))
}
session := &rssSyncSession{documents: documents, batchSize: c.batchSize}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -213,25 +215,21 @@ func (s *rssSyncSession) Close() error {
}
// applyResume advances past the last committed RSS document when retrying a task.
-func (s *rssSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *rssSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
- if sourceID != "" {
- for index, doc := range s.documents {
- if doc.SourceID == sourceID {
- s.index = index + 1
- return
- }
- }
- }
- if checkpoint.UpdatedAt == nil {
- return
+ if sourceID == "" {
+ return fmt.Errorf("rss sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
- for s.index < len(s.documents) && s.documents[s.index].UpdatedAt.Before(*checkpoint.UpdatedAt) {
- s.index++
+ for index, doc := range s.documents {
+ if doc.SourceID == sourceID {
+ s.index = index + 1
+ return nil
+ }
}
+ return fmt.Errorf("rss resume anchor %q was not found in the current feed: %w", sourceID, ErrSyncResumeInvalid)
}
// rssSyncCheckpoint returns a resume point after a committed RSS document.
diff --git a/internal/syncer/connector/rss_test.go b/internal/syncer/connector/rss_test.go
index c10d793d02..a590febcd0 100644
--- a/internal/syncer/connector/rss_test.go
+++ b/internal/syncer/connector/rss_test.go
@@ -137,6 +137,19 @@ func TestRSSConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
}
}
+func TestRSSConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml", "batch_size": 2})
+ if err != nil {
+ t.Fatalf("NewRSSConnector failed: %v", err)
+ }
+ connector.fetchFeed = staticRSSFeed
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestRSSConnectorValidateConnectorSetting(t *testing.T) {
connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml"})
if err != nil {
diff --git a/internal/syncer/connector/salesforce.go b/internal/syncer/connector/salesforce.go
index 91d3b2ecf7..2db18483dc 100644
--- a/internal/syncer/connector/salesforce.go
+++ b/internal/syncer/connector/salesforce.go
@@ -250,7 +250,9 @@ func (c *SalesforceConnector) OpenSync(ctx context.Context, request SyncRequest)
windowEnd: request.WindowEnd,
cursors: map[string]salesforceObjectCursor{},
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -725,6 +727,9 @@ type salesforceSyncSession struct {
latestISO string
latestID string
buffer []salesforceBufferedDocument
+
+ resumeAnchor *salesforceResumeAnchor
+ resumeChecked bool
}
type salesforceBufferedDocument struct {
@@ -732,22 +737,159 @@ type salesforceBufferedDocument struct {
checkpoint *SyncCheckpoint
}
+type salesforceResumeAnchor struct {
+ sourceID string
+ object string
+ recordID string
+ objectCursor salesforceObjectCursor
+}
+
// applyResume restores the per-object cursor map from a saved checkpoint.
-func (s *salesforceSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+// A checkpoint must carry a source anchor and a valid cursor for that object;
+// remote anchor existence is checked by validateResume on the first NextBatch.
+func (s *salesforceSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("salesforce sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor salesforceSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("salesforce sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ object, recordID, ok := salesforceSourceIDParts(checkpoint.SourceID)
+ if !ok {
+ return fmt.Errorf("salesforce sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ if !salesforceHasObject(s.objects, object) {
+ return fmt.Errorf("salesforce resume anchor %q was not found in the current object listing: %w", checkpoint.SourceID, ErrSyncResumeInvalid)
+ }
+ if len(cursor.Cursors) == 0 {
+ return fmt.Errorf("salesforce sync cursor has no object positions: %w", ErrSyncResumeInvalid)
+ }
+ for name, objectCursor := range cursor.Cursors {
+ if !salesforceHasObject(s.objects, name) {
+ return fmt.Errorf("salesforce sync cursor references unknown object %q: %w", name, ErrSyncResumeInvalid)
+ }
+ if objectCursor.SystemModstamp == "" || objectCursor.Id == "" {
+ return fmt.Errorf("salesforce sync cursor has an invalid position for object %q: %w", name, ErrSyncResumeInvalid)
+ }
+ if _, err := parseSalesforceTime(objectCursor.SystemModstamp); err != nil {
+ return fmt.Errorf("salesforce sync cursor has an invalid timestamp for object %q: %w", name, ErrSyncResumeInvalid)
+ }
+ }
+ objectCursor, ok := cursor.Cursors[object]
+ if !ok {
+ return fmt.Errorf("salesforce sync cursor has no position for object %q: %w", object, ErrSyncResumeInvalid)
}
- if len(cursor.Cursors) > 0 {
- s.cursors = cursor.Cursors
+ if objectCursor.Id != recordID {
+ return fmt.Errorf("salesforce sync cursor does not match source anchor %q: %w", checkpoint.SourceID, ErrSyncResumeInvalid)
}
+ s.cursors = cursor.Cursors
+ s.resumeAnchor = &salesforceResumeAnchor{
+ sourceID: checkpoint.SourceID,
+ object: object,
+ recordID: recordID,
+ objectCursor: objectCursor,
+ }
+ return nil
+}
+
+// validateResume verifies the saved anchor still exists at the same position
+// before the resumed session emits any documents.
+func (s *salesforceSyncSession) validateResume(ctx context.Context) error {
+ if s.resumeAnchor == nil || s.resumeChecked {
+ return nil
+ }
+ s.resumeChecked = true
+ anchor := s.resumeAnchor
+ expectedModified, err := parseSalesforceTime(anchor.objectCursor.SystemModstamp)
+ if err != nil {
+ return fmt.Errorf("salesforce sync cursor has an invalid timestamp for object %q: %w", anchor.object, ErrSyncResumeInvalid)
+ }
+ if s.windowStart != nil && expectedModified.Before(*s.windowStart) {
+ return fmt.Errorf("salesforce resume anchor %q is outside the sync window: %w", anchor.sourceID, ErrSyncResumeInvalid)
+ }
+ if !s.windowEnd.IsZero() && expectedModified.After(s.windowEnd) {
+ return fmt.Errorf("salesforce resume anchor %q is outside the sync window: %w", anchor.sourceID, ErrSyncResumeInvalid)
+ }
+
+ soql := fmt.Sprintf("SELECT Id,SystemModstamp FROM %s WHERE Id = '%s'", anchor.object, salesforceDataLiteral(anchor.recordID))
+ var page salesforceQueryPage
+ if err := s.connector.getJSON(ctx, "/query?q="+url.QueryEscape(soql), &page); err != nil {
+ var unavailable *salesforceObjectUnavailableError
+ if errors.As(err, &unavailable) {
+ return fmt.Errorf("salesforce resume object %q is no longer available: %w", anchor.object, ErrSyncResumeInvalid)
+ }
+ return err
+ }
+ for _, record := range page.Records {
+ if stringRecordValue(record, "Id") != anchor.recordID {
+ continue
+ }
+ modified, err := parseSalesforceTime(stringRecordValue(record, "SystemModstamp"))
+ if err != nil || !modified.Equal(expectedModified) {
+ return fmt.Errorf("salesforce resume anchor %q is no longer at the saved position: %w", anchor.sourceID, ErrSyncResumeInvalid)
+ }
+ return nil
+ }
+ return fmt.Errorf("salesforce resume anchor %q was not found in the current source: %w", anchor.sourceID, ErrSyncResumeInvalid)
+}
+
+// salesforceSourceIDParts splits the document SourceID anchor into an SObject
+// name and record ID. Salesforce IDs do not contain slashes, so a SourceID with
+// any extra separator or invalid character is not a usable resume anchor.
+func salesforceSourceIDParts(sourceID string) (string, string, bool) {
+ object, recordID, ok := strings.Cut(sourceID, "/")
+ if !ok || !salesforceObjectNameValid(object) || !salesforceRecordIDValid(recordID) {
+ return "", "", false
+ }
+ return object, recordID, true
+}
+
+func salesforceObjectNameValid(value string) bool {
+ if value == "" {
+ return false
+ }
+ for _, r := range value {
+ if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_') {
+ return false
+ }
+ }
+ return true
+}
+
+func salesforceRecordIDValid(value string) bool {
+ if value == "" {
+ return false
+ }
+ for _, r := range value {
+ if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9') {
+ return false
+ }
+ }
+ return true
+}
+
+func salesforceDataLiteral(value string) string {
+ return strings.ReplaceAll(value, "'", "\\'")
+}
+
+func salesforceHasObject(objects []string, name string) bool {
+ for _, object := range objects {
+ if object == name {
+ return true
+ }
+ }
+ return false
}
// NextBatch returns the next Salesforce document batch.
func (s *salesforceSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ if err := s.validateResume(ctx); err != nil {
+ return SyncBatch{}, err
+ }
documents := make([]SourceDocument, 0, s.batchSize)
var checkpoint *SyncCheckpoint
if len(s.buffer) > 0 {
diff --git a/internal/syncer/connector/salesforce_test.go b/internal/syncer/connector/salesforce_test.go
index 12f9c3d6d5..413afe85c4 100644
--- a/internal/syncer/connector/salesforce_test.go
+++ b/internal/syncer/connector/salesforce_test.go
@@ -356,6 +356,172 @@ func TestSalesforceConnectorOpenSyncResume(t *testing.T) {
}
}
+func TestSalesforceConnectorOpenSyncResumeRejectsInvalidCheckpoint(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.objects = []string{"Account", "Contact"}
+ validCursor, err := json.Marshal(salesforceSyncCursor{Cursors: map[string]salesforceObjectCursor{
+ "Account": {SystemModstamp: "2026-01-03T01:00:00.000+0000", Id: "0015g00000Example2"},
+ }})
+ if err != nil {
+ t.Fatalf("marshal valid cursor: %v", err)
+ }
+ validCursorJSON := string(validCursor)
+ cases := []struct {
+ name string
+ checkpoint *SyncCheckpoint
+ }{
+ {
+ name: "missing cursor",
+ checkpoint: &SyncCheckpoint{SourceID: "Account/0015g00000Example2"},
+ },
+ {
+ name: "invalid cursor json",
+ checkpoint: &SyncCheckpoint{Cursor: "not-json", SourceID: "Account/0015g00000Example2"},
+ },
+ {
+ name: "missing source anchor",
+ checkpoint: &SyncCheckpoint{Cursor: validCursorJSON},
+ },
+ {
+ name: "malformed source anchor",
+ checkpoint: &SyncCheckpoint{Cursor: validCursorJSON, SourceID: "Account"},
+ },
+ {
+ name: "anchor object not configured",
+ checkpoint: &SyncCheckpoint{Cursor: validCursorJSON, SourceID: "Bogus/0015g00000Example2"},
+ },
+ {
+ name: "no object positions",
+ checkpoint: &SyncCheckpoint{
+ Cursor: `{"cursors":{}}`,
+ SourceID: "Account/0015g00000Example2",
+ },
+ },
+ {
+ name: "missing anchor object cursor",
+ checkpoint: &SyncCheckpoint{
+ Cursor: `{"cursors":{"Contact":{"system_modstamp":"2026-01-03T01:00:00.000+0000","id":"0035g00000Example2"}}}`,
+ SourceID: "Account/0015g00000Example2",
+ },
+ },
+ {
+ name: "cursor references unknown object",
+ checkpoint: &SyncCheckpoint{
+ Cursor: `{"cursors":{"Account":{"system_modstamp":"2026-01-03T01:00:00.000+0000","id":"0015g00000Example2"},"Bogus":{"system_modstamp":"2026-01-03T01:00:00.000+0000","id":"x"}}}`,
+ SourceID: "Account/0015g00000Example2",
+ },
+ },
+ {
+ name: "object cursor missing fields",
+ checkpoint: &SyncCheckpoint{
+ Cursor: `{"cursors":{"Account":{}}}`,
+ SourceID: "Account/0015g00000Example2",
+ },
+ },
+ {
+ name: "object cursor invalid timestamp",
+ checkpoint: &SyncCheckpoint{
+ Cursor: `{"cursors":{"Account":{"system_modstamp":"bad","id":"0015g00000Example2"}}}`,
+ SourceID: "Account/0015g00000Example2",
+ },
+ },
+ {
+ name: "object cursor record id mismatch",
+ checkpoint: &SyncCheckpoint{
+ Cursor: `{"cursors":{"Account":{"system_modstamp":"2026-01-03T01:00:00.000+0000","id":"0015g00000Other"}}}`,
+ SourceID: "Account/0015g00000Example2",
+ },
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: tc.checkpoint})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+ })
+ }
+}
+
+func TestSalesforceConnectorOpenSyncResumeRejectsMissingAnchor(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ page := salesforceQueryPage{}
+ data, _ := json.Marshal(page)
+ return json.Unmarshal(data, out)
+ }
+ cursor, err := json.Marshal(salesforceSyncCursor{Cursors: map[string]salesforceObjectCursor{
+ "Account": {SystemModstamp: "2026-01-03T01:00:00.000+0000", Id: "0015g00000Example2"},
+ }})
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: string(cursor), SourceID: "Account/0015g00000Example2"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ if _, err = session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestSalesforceConnectorOpenSyncResumeRejectsChangedAnchor(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ if !strings.Contains(apiURL, "/query?") {
+ t.Fatalf("unexpected url %s", apiURL)
+ }
+ page := salesforceQueryPage{Records: []map[string]any{{
+ "Id": "0015g00000Example2",
+ "SystemModstamp": "2026-01-03T02:00:00.000+0000",
+ }}}
+ data, _ := json.Marshal(page)
+ return json.Unmarshal(data, out)
+ }
+ cursor, err := json.Marshal(salesforceSyncCursor{Cursors: map[string]salesforceObjectCursor{
+ "Account": {SystemModstamp: "2026-01-03T01:00:00.000+0000", Id: "0015g00000Example2"},
+ }})
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: string(cursor), SourceID: "Account/0015g00000Example2"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ if _, err = session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestSalesforceConnectorOpenSyncResumeRejectsUnavailableObject(t *testing.T) {
+ connector := newSalesforceFixtureConnector()
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ return &salesforceObjectUnavailableError{message: "object gone"}
+ }
+ cursor, err := json.Marshal(salesforceSyncCursor{Cursors: map[string]salesforceObjectCursor{
+ "Account": {SystemModstamp: "2026-01-03T01:00:00.000+0000", Id: "0015g00000Example2"},
+ }})
+ if err != nil {
+ t.Fatalf("marshal cursor: %v", err)
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: string(cursor), SourceID: "Account/0015g00000Example2"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ if _, err = session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
func TestSalesforceConnectorOpenSyncPaginatedResume(t *testing.T) {
connector := newSalesforceFixtureConnector()
connector.batchSize = 2
@@ -453,9 +619,14 @@ func TestSalesforceConnectorOpenSyncPaginatedResume(t *testing.T) {
]}`), out)
}
if strings.Contains(apiURL, "/query?") {
- resumedQueryCalls++
parsed, _ := url.Parse(apiURL)
soql, _ := url.QueryUnescape(parsed.Query().Get("q"))
+ if strings.Contains(soql, "WHERE Id = '0015g00000Example3'") {
+ page := salesforceQueryPage{Records: records[2:3]}
+ data, _ := json.Marshal(page)
+ return json.Unmarshal(data, out)
+ }
+ resumedQueryCalls++
if !strings.Contains(soql, "SystemModstamp > 2026-01-03T02:00:00Z") {
t.Fatalf("resumed soql missing since predicate: %q", soql)
}
@@ -1070,6 +1241,11 @@ func salesforceFixtureRecordsDoJSON(t *testing.T, records []map[string]any) func
func salesforceFixtureMatchesSOQL(t *testing.T, soql string, record map[string]any) bool {
t.Helper()
lower := strings.ToLower(soql)
+ if idx := strings.Index(lower, "where id = '"); idx >= 0 {
+ value := strings.Trim(strings.TrimSpace(soql[idx+len("WHERE Id = '"):]), "'")
+ id, _ := record["Id"].(string)
+ return id == value
+ }
since := time.Time{}
until := time.Time{}
equalSince := false
diff --git a/internal/syncer/connector/sharepoint.go b/internal/syncer/connector/sharepoint.go
index 34da15c154..2fd2c47c50 100644
--- a/internal/syncer/connector/sharepoint.go
+++ b/internal/syncer/connector/sharepoint.go
@@ -139,7 +139,9 @@ func (c *SharePointConnector) OpenSync(ctx context.Context, request SyncRequest)
drives: drives,
batchSize: c.batchSize,
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -679,20 +681,26 @@ func (s *sharePointSyncSession) Close() error {
return nil
}
-func (s *sharePointSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *sharePointSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
const prefix = "sharepoint_drive_"
- if !strings.HasPrefix(sourceID, prefix) {
- return
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return fmt.Errorf("sharepoint sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
driveID := strings.TrimPrefix(sourceID, prefix)
if driveID == "" {
- return
+ return fmt.Errorf("sharepoint sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ for _, drive := range s.drives {
+ if drive.ID == driveID {
+ s.resumeAfterDriveID = driveID
+ return nil
+ }
}
- s.resumeAfterDriveID = driveID
+ return fmt.Errorf("sharepoint resume drive %q was not found in the current listing: %w", driveID, ErrSyncResumeInvalid)
}
func (s *sharePointSyncSession) maxUpdatedAt(documents []SourceDocument) time.Time {
diff --git a/internal/syncer/connector/sharepoint_test.go b/internal/syncer/connector/sharepoint_test.go
index 05ee017b90..94230cf013 100644
--- a/internal/syncer/connector/sharepoint_test.go
+++ b/internal/syncer/connector/sharepoint_test.go
@@ -509,6 +509,24 @@ func TestSharePointResumeSkipsSyncedDrives(t *testing.T) {
}
}
+func TestSharePointResumeRejectsMissingDrive(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[
+ {"id":"drv-A","name":"A","driveType":"documentLibrary"},
+ {"id":"drv-B","name":"B","driveType":"documentLibrary"}
+ ]}`,
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Resume: &SyncCheckpoint{Cursor: "sharepoint_drive_drv-C"},
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestSharePointCheckpointAdvancesAfterDriveFlush(t *testing.T) {
fixtures := &sharePointTestFixtures{
drivesBody: `{"value":[
diff --git a/internal/syncer/connector/slack.go b/internal/syncer/connector/slack.go
index 8effb14004..d3538f52c2 100644
--- a/internal/syncer/connector/slack.go
+++ b/internal/syncer/connector/slack.go
@@ -150,7 +150,9 @@ func (c *SlackConnector) OpenSync(ctx context.Context, request SyncRequest) (Syn
seenThreads: map[string]struct{}{},
}
if request.Resume != nil {
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
}
return session, nil
}
@@ -960,17 +962,26 @@ func (s *slackSyncSession) Close() error {
return nil
}
-func (s *slackSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *slackSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
const prefix = "slack_channel_"
- if !strings.HasPrefix(sourceID, prefix) {
- return
+ if sourceID == "" || !strings.HasPrefix(sourceID, prefix) {
+ return fmt.Errorf("slack sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
channelID := strings.TrimPrefix(sourceID, prefix)
if channelID == "" {
- return
+ return fmt.Errorf("slack sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ for _, channel := range s.channels {
+ if channel.ID == channelID {
+ s.resumeAfterChannelID = channelID
+ return nil
+ }
}
- s.resumeAfterChannelID = channelID
+ return fmt.Errorf("slack resume channel %q was not found in the current listing: %w", channelID, ErrSyncResumeInvalid)
}
// channelDocuments builds the documents for one channel.
diff --git a/internal/syncer/connector/slack_test.go b/internal/syncer/connector/slack_test.go
index d0a1be22c1..d827492761 100644
--- a/internal/syncer/connector/slack_test.go
+++ b/internal/syncer/connector/slack_test.go
@@ -548,6 +548,29 @@ func TestSlackConnectorOpenSyncResumesFromCheckpoint(t *testing.T) {
}
}
+func TestSlackConnectorOpenSyncResumeRejectsMissingChannel(t *testing.T) {
+ withSlackTestHooks(t)
+ fixtures := &slackTestFixtures{
+ channelsJSON: `{"ok":true,"channels":[
+ {"id":"C1","name":"general","is_member":true},
+ {"id":"C2","name":"random","is_member":true}
+ ],"response_metadata":{"next_cursor":""}}`,
+ }
+ server := newTestSlackServer(t, fixtures)
+ connector := mustSlackConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{
+ Cursor: "slack_channel_C3",
+ SourceID: "slack_channel_C3",
+ },
+ })
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestSlackConnectorOpenSyncCheckpointAdvancesPerChannel(t *testing.T) {
withSlackTestHooks(t)
fixtures := &slackTestFixtures{
diff --git a/internal/syncer/connector/teams.go b/internal/syncer/connector/teams.go
index e05a71c7cd..c7163c0fe5 100644
--- a/internal/syncer/connector/teams.go
+++ b/internal/syncer/connector/teams.go
@@ -156,7 +156,9 @@ func (c *TeamsConnector) OpenSync(ctx context.Context, request SyncRequest) (Syn
if request.FromBeginning {
session.windowStart = nil
}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -439,23 +441,31 @@ type teamsBufferedDocument struct {
}
// applyResume advances the session to the last committed Teams position.
-func (s *teamsSyncSession) applyResume(checkpoint *SyncCheckpoint) {
- if checkpoint == nil || checkpoint.Cursor == "" {
- return
+func (s *teamsSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("teams sync cursor is missing: %w", ErrSyncResumeInvalid)
}
var cursor teamsSyncCursor
if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
- return
+ return fmt.Errorf("teams sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" {
+ return fmt.Errorf("teams sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
if cursor.TeamIndex < 0 || cursor.TeamIndex >= len(s.teams) {
- return
+ return fmt.Errorf("teams resume team %d was not found in the current listing: %w", cursor.TeamIndex, ErrSyncResumeInvalid)
}
s.teamIndex = cursor.TeamIndex
s.resumeChannelIndex = cursor.ChannelIndex
s.resumePageURL = cursor.MessagesPage
s.resumeOffset = cursor.MessageOffset
- s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ s.resumeSourceID = sourceID
s.pageURL = cursor.MessagesPage
+ return nil
}
// NextBatch returns the next Teams document batch.
@@ -522,6 +532,9 @@ func (s *teamsSyncSession) nextDocumentPage(ctx context.Context) ([]teamsBuffere
s.resumeChannelIndex = 0
}
}
+ if s.resumeSourceID != "" && s.channelIndex >= len(s.channels) {
+ return nil, fmt.Errorf("teams resume channel %d was not found in team %s: %w", s.channelIndex, s.teams[s.teamIndex].ID, ErrSyncResumeInvalid)
+ }
if s.channelIndex >= len(s.channels) {
s.advanceTeam()
continue
@@ -564,7 +577,10 @@ func (s *teamsSyncSession) nextDocumentPage(ctx context.Context) ([]teamsBuffere
})
}
- documents := s.filterResumedDocuments(requestURL, candidates)
+ documents, err := s.filterResumedDocuments(requestURL, candidates)
+ if err != nil {
+ return nil, err
+ }
if page.NextLink != "" {
s.pageURL = page.NextLink
return documents, nil
@@ -625,34 +641,20 @@ func (s *teamsSyncSession) checkpoint(cursor teamsSyncCursor, doc SourceDocument
}
// filterResumedDocuments drops documents through the committed checkpoint.
-func (s *teamsSyncSession) filterResumedDocuments(pageURL string, candidates []teamsBufferedDocument) []teamsBufferedDocument {
- if s.resumePageURL == "" {
- return candidates
+func (s *teamsSyncSession) filterResumedDocuments(pageURL string, candidates []teamsBufferedDocument) ([]teamsBufferedDocument, error) {
+ if s.resumeSourceID == "" {
+ return candidates, nil
}
if pageURL != s.resumePageURL {
- s.clearResumeOffset()
- return candidates
- }
- if s.resumeOffset <= 0 {
- s.clearResumeOffset()
- return candidates
- }
- if s.resumeSourceID != "" {
- for index, candidate := range candidates {
- if candidate.document.SourceID == s.resumeSourceID {
- s.clearResumeOffset()
- return candidates[index+1:]
- }
- }
+ return nil, fmt.Errorf("teams resume page no longer matches checkpoint page: %w", ErrSyncResumeInvalid)
}
- filtered := candidates[:0]
- for _, candidate := range candidates {
- if candidate.offset > s.resumeOffset {
- filtered = append(filtered, candidate)
+ for index, candidate := range candidates {
+ if candidate.document.SourceID == s.resumeSourceID {
+ s.clearResumeOffset()
+ return candidates[index+1:], nil
}
}
- s.clearResumeOffset()
- return filtered
+ return nil, fmt.Errorf("teams resume anchor %q was not found on %s: %w", s.resumeSourceID, pageURL, ErrSyncResumeInvalid)
}
func (s *teamsSyncSession) clearResumeOffset() {
diff --git a/internal/syncer/connector/teams_test.go b/internal/syncer/connector/teams_test.go
index 67381b463c..d9b5d40d68 100644
--- a/internal/syncer/connector/teams_test.go
+++ b/internal/syncer/connector/teams_test.go
@@ -281,6 +281,72 @@ func TestTeamsConnectorOpenSyncResume(t *testing.T) {
}
}
+func TestTeamsConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := newFixtureTeamsConnector()
+ connector.doJSON = teamsFixtureDoJSON(t)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
+func TestTeamsConnectorOpenSyncResumeRejectsMissingRemoteAnchor(t *testing.T) {
+ connector := newFixtureTeamsConnector()
+ connector.batchSize = 1
+ connector.doJSON = teamsFixtureDoJSON(t)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "team-1__channel-1__msg-1" {
+ t.Fatalf("first documents = %+v", first.Documents)
+ }
+ if first.Checkpoint == nil {
+ t.Fatalf("first checkpoint is nil")
+ }
+
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) error {
+ switch {
+ case strings.Contains(apiURL, "/channels/channel-1/messages/msg-2/replies"):
+ *out.(*teamsMessagesPage) = teamsMessagesPage{}
+ case strings.Contains(apiURL, "/channels/channel-1/messages"):
+ *out.(*teamsMessagesPage) = teamsMessagesPage{
+ Value: []teamsMessage{{
+ ID: "msg-2",
+ Body: teamsMessageBody{Content: "Second message", ContentType: "text"},
+ LastModifiedDateTime: "2026-01-03T02:00:00Z",
+ WebURL: "https://teams.example/message/msg-2",
+ }},
+ }
+ case strings.Contains(apiURL, "/teams/team-1/channels"):
+ *out.(*teamsChannelsPage) = teamsChannelsPage{
+ Value: []teamsChannel{{ID: "channel-1", DisplayName: "General"}},
+ }
+ case strings.Contains(apiURL, "/teams"):
+ *out.(*teamsPage) = teamsPage{
+ Value: []teamsTeam{{ID: "team-1", DisplayName: "Engineering"}},
+ }
+ default:
+ t.Fatalf("unexpected api url %s", apiURL)
+ }
+ return nil
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ if _, err = resumed.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
func TestTeamsConnectorOpenPrune(t *testing.T) {
connector := newFixtureTeamsConnector()
connector.doJSON = teamsFixtureDoJSON(t)
diff --git a/internal/syncer/connector/webdav.go b/internal/syncer/connector/webdav.go
index ba8de70240..fe8e8c43fc 100644
--- a/internal/syncer/connector/webdav.go
+++ b/internal/syncer/connector/webdav.go
@@ -177,7 +177,9 @@ func (c *WebDAVConnector) OpenSync(ctx context.Context, request SyncRequest) (Sy
sort.Slice(documents, func(i, j int) bool { return documents[i].SourceID < documents[j].SourceID })
session := &webdavSyncSession{documents: documents, batchSize: c.batchSize}
- session.applyResume(request.Resume)
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
return session, nil
}
@@ -257,20 +259,21 @@ func (s *webdavSyncSession) Close() error {
}
// applyResume advances past the last committed WebDAV document when retrying a task.
-func (s *webdavSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+func (s *webdavSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
if checkpoint == nil {
- return
+ return nil
}
sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
if sourceID == "" {
- return
+ return fmt.Errorf("webdav sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
}
for index, doc := range s.documents {
if doc.SourceID == sourceID {
s.index = index + 1
- return
+ return nil
}
}
+ return fmt.Errorf("webdav resume anchor %q was not found in the current listing: %w", sourceID, ErrSyncResumeInvalid)
}
// webdavSyncCheckpoint returns a resume point after a committed WebDAV document.
diff --git a/internal/syncer/connector/webdav_test.go b/internal/syncer/connector/webdav_test.go
index 328fcac7ce..c171cdaa20 100644
--- a/internal/syncer/connector/webdav_test.go
+++ b/internal/syncer/connector/webdav_test.go
@@ -509,6 +509,17 @@ func TestWebDAVConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
}
}
+func TestWebDAVConnectorOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ t.Setenv("BLOB_STORAGE_SIZE_THRESHOLD", "20")
+ server, _, _ := newWebDAVTestServer(t, webDAVTestTree())
+ connector := webDAVTestConnector(t, server.URL, false, 2)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
func TestWebDAVConnectorOpenPrune(t *testing.T) {
t.Setenv("BLOB_STORAGE_SIZE_THRESHOLD", "20")
server, _, getRequests := newWebDAVTestServer(t, webDAVTestTree())
diff --git a/internal/syncer/connector/xquik_test.go b/internal/syncer/connector/xquik_test.go
index 80a5df7d4d..46f1923139 100644
--- a/internal/syncer/connector/xquik_test.go
+++ b/internal/syncer/connector/xquik_test.go
@@ -201,8 +201,8 @@ func TestXquikOpenSyncUsesWindowAndResumesCursor(t *testing.T) {
mu.Lock()
gotRequests := append([]url.Values(nil), requests...)
mu.Unlock()
- if len(gotRequests) != 2 {
- t.Fatalf("request count = %d, want 2", len(gotRequests))
+ if len(gotRequests) != 3 {
+ t.Fatalf("request count = %d, want 3", len(gotRequests))
}
for _, query := range gotRequests {
if query.Get("q") != "ragflow lang:en" || query.Get("queryType") != "Top" || query.Get("limit") != "2" {
@@ -212,8 +212,8 @@ func TestXquikOpenSyncUsesWindowAndResumesCursor(t *testing.T) {
t.Fatalf("window query = %v", query)
}
}
- if gotRequests[0].Get("cursor") != "" || gotRequests[1].Get("cursor") != "cursor-2" {
- t.Fatalf("cursor sequence = [%q %q]", gotRequests[0].Get("cursor"), gotRequests[1].Get("cursor"))
+ if gotRequests[0].Get("cursor") != "" || gotRequests[1].Get("cursor") != "" || gotRequests[2].Get("cursor") != "cursor-2" {
+ t.Fatalf("cursor sequence = [%q %q %q]", gotRequests[0].Get("cursor"), gotRequests[1].Get("cursor"), gotRequests[2].Get("cursor"))
}
}
@@ -285,7 +285,7 @@ func TestXquikContinuesAfterEmptyPageWhenResponseHasMore(t *testing.T) {
}
}
-func TestXquikStopsAfterRepeatedCursor(t *testing.T) {
+func TestXquikRejectsRepeatedCursor(t *testing.T) {
config := validXquikConfig()
config["batch_size"] = 1
connector := newXquikTestConnector(t, config)
@@ -311,17 +311,14 @@ func TestXquikStopsAfterRepeatedCursor(t *testing.T) {
t.Fatalf("OpenSync: %v", err)
}
defer func() { _ = session.Close() }()
- for {
- _, err = session.NextBatch(context.Background())
- if errors.Is(err, io.EOF) {
- break
- }
- if err != nil {
- t.Fatalf("NextBatch: %v", err)
- }
+ if _, err = session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("first NextBatch: %v", err)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("second NextBatch err = %v, want ErrSyncResumeInvalid", err)
}
if requests != 2 {
- t.Fatalf("requests = %d, want 2 before repeated cursor stops pagination", requests)
+ t.Fatalf("requests = %d, want 2 before repeated cursor aborts pagination", requests)
}
}
diff --git a/internal/syncer/sync_runner.go b/internal/syncer/sync_runner.go
index 7f905ff720..98934599c4 100644
--- a/internal/syncer/sync_runner.go
+++ b/internal/syncer/sync_runner.go
@@ -37,8 +37,9 @@ const syncCancelCheckInterval = time.Second
// SyncRunnerConfig controls per-task document processing.
type SyncRunnerConfig struct {
- ItemRetryCount int
- ItemRetryBaseDelay time.Duration
+ ItemRetryCount int
+ ItemRetryBaseDelay time.Duration
+ MaxAnchorRestartCount int
}
func checkTaskCanceled(taskDAO *dao.SyncTaskDAO, ctx context.Context, taskID string) error {
@@ -74,6 +75,9 @@ func NewSyncRunner(config SyncRunnerConfig, taskDAO *dao.SyncTaskDAO, taskServic
if config.ItemRetryBaseDelay <= 0 {
config.ItemRetryBaseDelay = time.Second
}
+ if config.MaxAnchorRestartCount <= 0 {
+ config.MaxAnchorRestartCount = 2
+ }
if checkpoints == nil {
checkpoints = newMemorySyncCheckpointStore()
}
@@ -104,21 +108,15 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext dao.SyncTaskContext, c
if err != nil {
return "", err
}
- session, err := connector.OpenSync(ctx, syncerconnector.SyncRequest{
- TaskID: taskContext.Task.ID,
- ConnectorID: taskContext.Connector.ID,
- KBID: taskContext.Knowledgebase.ID,
- SourceType: sourceType,
- Fingerprints: fingerprints,
- FromBeginning: checkpointState.WindowStart == nil,
- WindowStart: checkpointState.WindowStart,
- WindowEnd: checkpointState.WindowEnd,
- Resume: checkpointState.Checkpoint,
- })
+ session, err := r.openSyncSession(ctx, taskContext, sourceType, fingerprints, &checkpointState, connector)
if err != nil {
return "", err
}
- defer session.Close()
+ defer func() {
+ if session != nil {
+ _ = session.Close()
+ }
+ }()
// prepare sourceType, waterline, stats, resultChan
stats := statsFromCheckpointState(checkpointState) // count `add`, `updated`, `skipped`
@@ -139,6 +137,20 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext dao.SyncTaskContext, c
if err = r.collectResults(ctx, taskContext.Task.ID, resultChans, &checkpointState, &stats); err != nil {
return "", err
}
+ if errors.Is(nextErr, syncerconnector.ErrSyncResumeInvalid) {
+ if checkpointState.RestartCount >= r.config.MaxAnchorRestartCount {
+ return "", nextErr
+ }
+ _ = session.Close()
+ session = nil
+ resultChans = nil
+ session, err = r.restartSyncSession(ctx, taskContext, sourceType, fingerprints, &checkpointState, connector)
+ if err != nil {
+ return "", err
+ }
+ stats = service.SyncStats{}
+ continue
+ }
return "", nextErr
}
@@ -169,6 +181,52 @@ func (r *SyncRunner) Run(ctx context.Context, taskContext dao.SyncTaskContext, c
return nextTaskID, nil
}
+func (r *SyncRunner) openSyncSession(ctx context.Context, taskContext dao.SyncTaskContext, sourceType string, fingerprints map[string]string, checkpointState *syncerconnector.SyncCheckpointState, connector syncerconnector.Connector) (syncerconnector.SyncSession, error) {
+ session, err := connector.OpenSync(ctx, syncerconnector.SyncRequest{
+ TaskID: taskContext.Task.ID,
+ ConnectorID: taskContext.Connector.ID,
+ KBID: taskContext.Knowledgebase.ID,
+ SourceType: sourceType,
+ Fingerprints: fingerprints,
+ FromBeginning: checkpointState.WindowStart == nil,
+ WindowStart: checkpointState.WindowStart,
+ WindowEnd: checkpointState.WindowEnd,
+ Resume: checkpointState.Checkpoint,
+ })
+ if err != nil {
+ if errors.Is(err, syncerconnector.ErrSyncResumeInvalid) && checkpointState.RestartCount < r.config.MaxAnchorRestartCount {
+ restartErr := r.resetCheckpointForRestart(ctx, taskContext.Task.ID, checkpointState)
+ if restartErr != nil {
+ return nil, restartErr
+ }
+ return r.openSyncSession(ctx, taskContext, sourceType, fingerprints, checkpointState, connector)
+ }
+ return nil, err
+ }
+ return session, nil
+}
+
+func (r *SyncRunner) restartSyncSession(ctx context.Context, taskContext dao.SyncTaskContext, sourceType string, fingerprints map[string]string, checkpointState *syncerconnector.SyncCheckpointState, connector syncerconnector.Connector) (syncerconnector.SyncSession, error) {
+ if err := r.resetCheckpointForRestart(ctx, taskContext.Task.ID, checkpointState); err != nil {
+ return nil, err
+ }
+ return r.openSyncSession(ctx, taskContext, sourceType, fingerprints, checkpointState, connector)
+}
+
+// resetCheckpointForRestart discards committed progress and restarts the same
+// source window when the resume anchor is no longer valid.
+func (r *SyncRunner) resetCheckpointForRestart(ctx context.Context, taskID string, checkpointState *syncerconnector.SyncCheckpointState) error {
+ checkpointState.RestartCount++
+ checkpointState.Checkpoint = nil
+ checkpointState.NextCommitSeq = 1
+ checkpointState.Added = 0
+ checkpointState.Updated = 0
+ checkpointState.Skipped = 0
+ checkpointState.ErrorCount = 0
+ checkpointState.ErrorMsg = ""
+ return r.checkpoints.SaveSyncCheckpoint(ctx, taskID, *checkpointState)
+}
+
func (r *SyncRunner) collectResults(ctx context.Context, taskID string, resultChans []<-chan syncJobResult, checkpointState *syncerconnector.SyncCheckpointState, stats *service.SyncStats) error {
var firstErr error
for _, resultChan := range resultChans {
diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go
index 22d837f026..b68d551cd7 100644
--- a/internal/syncer/syncer.go
+++ b/internal/syncer/syncer.go
@@ -83,8 +83,9 @@ func New(config Config, taskDAO *dao.SyncTaskDAO, registry ConnectorRegistry, si
}
coordinator := NewTaskCoordinator(SyncRunnerConfig{
- ItemRetryCount: config.ItemRetryCount,
- ItemRetryBaseDelay: config.ItemRetryBaseDelay,
+ ItemRetryCount: config.ItemRetryCount,
+ ItemRetryBaseDelay: config.ItemRetryBaseDelay,
+ MaxAnchorRestartCount: config.MaxAnchorRestartCount,
}, taskDAO, taskService, registry, sink, pruneService, idResolver, executor, checkpoints)
scheduler := NewScheduler(queue, taskDAO)
diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go
index 1135c9e31c..9d2a7d29bc 100644
--- a/internal/syncer/syncer_test.go
+++ b/internal/syncer/syncer_test.go
@@ -340,8 +340,12 @@ func newCoordinator(taskDAO *dao.SyncTaskDAO, taskService *service.SyncTaskServi
}
func newCoordinatorWithCheckpoints(taskDAO *dao.SyncTaskDAO, taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore, checkpoints SyncCheckpointStore) *TaskCoordinator {
+ return newCoordinatorWithConfigAndCheckpoints(taskDAO, taskService, registry, sink, pruneService, store, checkpoints, SyncRunnerConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond})
+}
+
+func newCoordinatorWithConfigAndCheckpoints(taskDAO *dao.SyncTaskDAO, taskService *service.SyncTaskService, registry *syncerconnector.Registry, sink service.DocumentSink, pruneService *service.SyncPruneService, store service.DocumentStore, checkpoints SyncCheckpointStore, syncRunnerConfig SyncRunnerConfig) *TaskCoordinator {
executor := NewSyncJobExecutor(SyncJobExecutorConfig{WorkerCount: 16})
- return NewTaskCoordinator(SyncRunnerConfig{ItemRetryCount: 1, ItemRetryBaseDelay: time.Millisecond}, taskDAO, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), executor, checkpoints)
+ return NewTaskCoordinator(syncRunnerConfig, taskDAO, taskService, registry, sink, pruneService, service.NewDocumentIDResolver(store), executor, checkpoints)
}
// TestClaimBlocksSameConnectorKBRunningTasks verifies DB-backed task mutual exclusion.
@@ -1384,6 +1388,224 @@ func TestSyncCheckpointStopsBeforeFailedBatch(t *testing.T) {
}
}
+// TestSyncRunnerRestartsAfterInvalidOpenSyncResume verifies a stale resume anchor restarts
+// the same task window with a cleared checkpoint.
+func TestSyncRunnerRestartsAfterInvalidOpenSyncResume(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ checkpoints := newMemorySyncCheckpointStore()
+ windowStart := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+ windowEnd := windowStart.Add(time.Hour)
+ updatedAt := windowStart.Add(30 * time.Minute)
+ initialCheckpoint := &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", SourceID: "source-1", UpdatedAt: &updatedAt}
+ if err := checkpoints.SaveSyncCheckpoint(t.Context(), "task-1", syncerconnector.SyncCheckpointState{
+ Version: 1,
+ TaskID: "task-1",
+ ConnectorID: "conn-1",
+ KBID: "kb-1",
+ WindowStart: &windowStart,
+ WindowEnd: windowEnd,
+ NextCommitSeq: 2,
+ Checkpoint: initialCheckpoint,
+ Added: 1,
+ }); err != nil {
+ t.Fatalf("save checkpoint: %v", err)
+ }
+
+ source := syncerconnector.SourceDocument{SourceID: "source-2", Blob: []byte("b"), UpdatedAt: windowEnd}
+ connector := &connectormock.Connector{
+ OpenSyncResumeInvalidAt: 1,
+ SyncBatches: []syncerconnector.SyncBatch{{
+ Documents: []syncerconnector.SourceDocument{source},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-2", SourceID: "source-2", UpdatedAt: &windowEnd},
+ }},
+ }
+ sink := &fakeSink{}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinatorWithConfigAndCheckpoints(taskDAO, taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, checkpoints, SyncRunnerConfig{
+ ItemRetryCount: 1,
+ ItemRetryBaseDelay: time.Millisecond,
+ MaxAnchorRestartCount: 2,
+ }), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ if len(connector.SyncRequests) != 2 {
+ t.Fatalf("sync requests = %d, want 2", len(connector.SyncRequests))
+ }
+ initialRequest := connector.SyncRequests[0]
+ if initialRequest.Resume == nil || initialRequest.Resume.Cursor != "cursor-1" || initialRequest.Resume.SourceID != "source-1" {
+ t.Fatalf("initial resume = %+v, want cursor-1/source-1", initialRequest.Resume)
+ }
+ restartRequest := connector.SyncRequests[1]
+ if restartRequest.Resume != nil {
+ t.Fatalf("restart resume = %+v, want nil", restartRequest.Resume)
+ }
+ if initialRequest.TaskID != "task-1" || initialRequest.ConnectorID != "conn-1" || initialRequest.KBID != "kb-1" {
+ t.Fatalf("initial request identity = %+v", initialRequest)
+ }
+ if restartRequest.TaskID != initialRequest.TaskID || restartRequest.ConnectorID != initialRequest.ConnectorID || restartRequest.KBID != initialRequest.KBID {
+ t.Fatalf("restart request identity = %+v, want %+v", restartRequest, initialRequest)
+ }
+ if restartRequest.WindowStart == nil || !restartRequest.WindowStart.Equal(*initialRequest.WindowStart) || !restartRequest.WindowEnd.Equal(initialRequest.WindowEnd) {
+ t.Fatalf("restart window = %+v, want same as %+v", restartRequest.WindowStart, initialRequest.WindowStart)
+ }
+ if sink.callCount() != 1 {
+ t.Fatalf("sink calls = %d, want 1", sink.callCount())
+ }
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusDone {
+ t.Fatalf("status = %s, want done", task.Status)
+ }
+ if task.TotalDocsIndexed != 1 {
+ t.Fatalf("total docs indexed = %d, want 1", task.TotalDocsIndexed)
+ }
+}
+
+// TestSyncRunnerRestartsAfterInvalidNextBatchResume verifies pending work is collected
+// before the same source window is reprocessed from the beginning.
+func TestSyncRunnerRestartsAfterInvalidNextBatchResume(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ checkpoints := newMemorySyncCheckpointStore()
+ windowStart := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+ windowEnd := windowStart.Add(time.Hour)
+ if err := checkpoints.SaveSyncCheckpoint(t.Context(), "task-1", syncerconnector.SyncCheckpointState{
+ Version: 1,
+ TaskID: "task-1",
+ ConnectorID: "conn-1",
+ KBID: "kb-1",
+ WindowStart: &windowStart,
+ WindowEnd: windowEnd,
+ NextCommitSeq: 1,
+ }); err != nil {
+ t.Fatalf("save checkpoint: %v", err)
+ }
+
+ firstTime := windowStart.Add(10 * time.Minute)
+ secondTime := windowStart.Add(20 * time.Minute)
+ connector := &connectormock.Connector{
+ NextBatchResumeInvalidAt: 2,
+ NextBatchResumeFirstSession: true,
+ SyncBatches: []syncerconnector.SyncBatch{
+ {
+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-1", Blob: []byte("a"), UpdatedAt: firstTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", SourceID: "source-1", UpdatedAt: &firstTime},
+ },
+ {
+ Documents: []syncerconnector.SourceDocument{{SourceID: "source-2", Blob: []byte("b"), UpdatedAt: secondTime}},
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-2", SourceID: "source-2", UpdatedAt: &secondTime},
+ },
+ },
+ }
+ sink := &fakeSink{}
+ worker := NewTaskWorker(make(chan TaskEnvelope, 1), taskDAO, taskService, newCoordinatorWithConfigAndCheckpoints(taskDAO, taskService, newTestRegistry(map[string]*connectormock.Connector{"conn-1": connector}), sink, nil, fakeStore{}, checkpoints, SyncRunnerConfig{
+ ItemRetryCount: 1,
+ ItemRetryBaseDelay: time.Millisecond,
+ MaxAnchorRestartCount: 2,
+ }), NewConnectorLock())
+ worker.handle(t.Context(), TaskEnvelope{TaskID: "task-1"})
+
+ if len(connector.SyncRequests) != 2 {
+ t.Fatalf("sync requests = %d, want 2", len(connector.SyncRequests))
+ }
+ for _, request := range connector.SyncRequests {
+ if request.Resume != nil {
+ t.Fatalf("resume = %+v, want nil for full-window restart", request.Resume)
+ }
+ if request.WindowStart == nil || !request.WindowStart.Equal(windowStart) || !request.WindowEnd.Equal(windowEnd) {
+ t.Fatalf("window = %+v/%s, want %s/%s", request.WindowStart, request.WindowEnd, windowStart, windowEnd)
+ }
+ }
+ if sink.callCount() != 3 {
+ t.Fatalf("sink calls = %d, want first-pass document plus 2 reprocessed documents", sink.callCount())
+ }
+ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("load checkpoint: %v", err)
+ }
+ if state != nil {
+ t.Fatalf("checkpoint = %+v, want deleted after success", state)
+ }
+ var task entity.SyncLogs
+ if err := db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load task: %v", err)
+ }
+ if task.Status != dao.SyncStatusDone {
+ t.Fatalf("status = %s, want done", task.Status)
+ }
+ if task.TotalDocsIndexed != 2 {
+ t.Fatalf("total docs indexed = %d, want 2", task.TotalDocsIndexed)
+ }
+}
+
+// TestSyncRunnerStopsAfterRepeatedInvalidResumeAnchors verifies the restart budget is bounded.
+func TestSyncRunnerStopsAfterRepeatedInvalidResumeAnchors(t *testing.T) {
+ db := setupSyncerDB(t)
+ insertTaskContext(t, db, "conn-1", "kb-1", "task-1", dao.TaskTypeSync)
+ _ = db.Model(&entity.SyncLogs{}).Where("id = ?", "task-1").Update("status", dao.SyncStatusRunning).Error
+ taskDAO := dao.NewSyncTaskDAO(db)
+ taskService := service.NewSyncTaskService(taskDAO)
+ checkpoints := newMemorySyncCheckpointStore()
+ windowStart := time.Date(2026, 8, 12, 1, 0, 0, 0, time.UTC)
+ windowEnd := windowStart.Add(time.Hour)
+ updatedAt := windowStart.Add(30 * time.Minute)
+ if err := checkpoints.SaveSyncCheckpoint(t.Context(), "task-1", syncerconnector.SyncCheckpointState{
+ Version: 1,
+ TaskID: "task-1",
+ ConnectorID: "conn-1",
+ KBID: "kb-1",
+ WindowStart: &windowStart,
+ WindowEnd: windowEnd,
+ NextCommitSeq: 2,
+ Checkpoint: &syncerconnector.SyncCheckpoint{Cursor: "cursor-1", SourceID: "source-1", UpdatedAt: &updatedAt},
+ }); err != nil {
+ t.Fatalf("save checkpoint: %v", err)
+ }
+
+ taskContext, err := taskDAO.GetTaskContext(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("get context: %v", err)
+ }
+ executor := NewSyncJobExecutor(SyncJobExecutorConfig{WorkerCount: 1})
+ queue, err := executor.RegisterTask(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("register task: %v", err)
+ }
+ defer executor.Close()
+ defer queue.Close()
+
+ connector := &connectormock.Connector{OpenSyncResumeInvalidAlways: true}
+ runner := NewSyncRunner(SyncRunnerConfig{
+ ItemRetryCount: 1,
+ ItemRetryBaseDelay: time.Millisecond,
+ MaxAnchorRestartCount: 1,
+ }, taskDAO, taskService, &fakeSink{}, service.NewDocumentIDResolver(fakeStore{}), queue, checkpoints)
+ if _, err := runner.Run(t.Context(), taskContext, connector); !errors.Is(err, syncerconnector.ErrSyncResumeInvalid) {
+ t.Fatalf("Run error = %v, want ErrSyncResumeInvalid", err)
+ }
+ if len(connector.SyncRequests) != 2 {
+ t.Fatalf("sync requests = %d, want 2", len(connector.SyncRequests))
+ }
+ if connector.SyncRequests[0].Resume == nil || connector.SyncRequests[1].Resume != nil {
+ t.Fatalf("sync request resumes = %+v / %+v, want checkpoint then nil", connector.SyncRequests[0].Resume, connector.SyncRequests[1].Resume)
+ }
+ state, err := checkpoints.LoadSyncCheckpoint(t.Context(), "task-1")
+ if err != nil {
+ t.Fatalf("load checkpoint: %v", err)
+ }
+ if state == nil || state.RestartCount != 1 || state.Checkpoint != nil {
+ t.Fatalf("checkpoint = %+v, want restart_count 1 and cleared resume", state)
+ }
+}
+
// TestSyncRunnerClampsWaterlineToWindowEnd verifies future source timestamps do not poison the next window.
func TestSyncRunnerClampsWaterlineToWindowEnd(t *testing.T) {
db := setupSyncerDB(t)