Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(syncer): retain Dropbox checkpoints on download failure (#21809)

Xiangba54d3ee64c5
diff --git a/internal/syncer/connector/dropbox.go b/internal/syncer/connector/dropbox.go
index e53b3864e3..5a38ed3c25 100644
--- a/internal/syncer/connector/dropbox.go
+++ b/internal/syncer/connector/dropbox.go
@@ -303,19 +303,23 @@ func (s *dropboxSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
batchSize := positiveDropboxBatchSize(s.batchSize)
documents := make([]SourceDocument, 0, batchSize)
var lastAccepted dropboxFileMetadata
- for attempts := 0; attempts < batchSize && s.index < len(s.files); attempts++ {
- file := s.files[s.index]
- s.index++
+ nextIndex := s.index
+ for attempts := 0; attempts < batchSize && nextIndex < len(s.files); attempts++ {
+ file := s.files[nextIndex]
+ nextIndex++
lastAccepted = file
blob, err := s.connector.downloadFile(ctx, file.downloadPath())
- if err != nil || len(blob) == 0 {
+ if err != nil {
+ // Do not commit any part of this batch: the runner ignores a
+ // batch returned with an error and retains its previous checkpoint.
+ return SyncBatch{}, fmt.Errorf("download Dropbox file %q: %w", file.sourceID(), err)
+ }
+ if len(blob) == 0 {
continue
}
documents = append(documents, s.connector.buildDocument(file, blob, s.nameCounts))
}
- if len(documents) == 0 {
- return SyncBatch{Checkpoint: dropboxSyncCheckpoint(lastAccepted)}, nil
- }
+ s.index = nextIndex
return SyncBatch{Documents: documents, Checkpoint: dropboxSyncCheckpoint(lastAccepted)}, nil
}
diff --git a/internal/syncer/connector/dropbox_test.go b/internal/syncer/connector/dropbox_test.go
index 7d7256e457..1c073d2bd3 100644
--- a/internal/syncer/connector/dropbox_test.go
+++ b/internal/syncer/connector/dropbox_test.go
@@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
"net/http/httptest"
@@ -29,11 +30,12 @@ import (
)
type dropboxTestServer struct {
- server *httptest.Server
- listRequests []dropboxListFolderRequest
- continueCalls int
- downloadPaths []string
- downloadArgs []string
+ server *httptest.Server
+ listRequests []dropboxListFolderRequest
+ continueCalls int
+ downloadPaths []string
+ downloadArgs []string
+ downloadRecovered bool
}
func newDropboxTestServer(t *testing.T) *dropboxTestServer {
@@ -93,7 +95,7 @@ func newDropboxTestServer(t *testing.T) *dropboxTestServer {
return
}
fixture.downloadPaths = append(fixture.downloadPaths, arg["path"])
- if arg["path"] == "/broken.txt" {
+ if arg["path"] == "/broken.txt" && !fixture.downloadRecovered {
w.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(w, `{"error_summary":"broken"}`)
return
@@ -171,11 +173,17 @@ func TestDropboxConnectorOpenSync(t *testing.T) {
t.Fatalf("OpenSync downloaded files eagerly: %#v", fixture.downloadPaths)
}
+ failed, err := session.NextBatch(context.Background())
+ if err == nil || failed.Checkpoint != nil {
+ t.Fatalf("download failure: %+v, %v", failed, err)
+ }
+ fixture.downloadRecovered = true
+ fixture.downloadPaths = nil
first, err := session.NextBatch(context.Background())
if err != nil {
t.Fatalf("NextBatch first failed: %v", err)
}
- if len(first.Documents) != 1 {
+ if len(first.Documents) != 2 {
t.Fatalf("first batch size = %d", len(first.Documents))
}
if first.Documents[0].SourceID != "dropbox:id:alpha" || string(first.Documents[0].Blob) != "body:/alpha.txt" {
@@ -288,6 +296,7 @@ func TestDropboxConnectorDownloadFileEscapesAPIArgHeader(t *testing.T) {
func TestDropboxConnectorSessionsNormalizeNonPositiveBatchSize(t *testing.T) {
fixture := newDropboxTestServer(t)
+ fixture.downloadRecovered = true
connector := newDropboxTestConnector(t, fixture, true)
connector.batchSize = 0
@@ -318,3 +327,72 @@ func TestDropboxConnectorSessionsNormalizeNonPositiveBatchSize(t *testing.T) {
t.Fatalf("prune batch should advance with a positive effective size")
}
}
+
+// A failed download must not produce a checkpoint that makes resume skip it.
+func TestDropboxSyncDownloadFailureKeepsBatchRetryable(t *testing.T) {
+ for _, batchSize := range []int{1, 2} {
+ t.Run(fmt.Sprint(batchSize), func(t *testing.T) {
+ failing := true
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/list_folder") {
+ writeDropboxTestListResponse(w, []map[string]any{
+ {".tag": "file", "id": "id:a", "name": "a.txt", "path_display": "/a.txt", "client_modified": "2026-01-02T00:00:00Z", "size": 4},
+ {".tag": "file", "id": "id:b", "name": "b.txt", "path_display": "/b.txt", "client_modified": "2026-01-02T00:00:00Z", "size": 4},
+ }, "", false)
+ return
+ }
+ if failing && strings.Contains(r.Header.Get("Dropbox-API-Arg"), "/b.txt") {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+ }
+ _, _ = io.WriteString(w, "body")
+ }))
+ defer server.Close()
+ c, err := NewDropboxConnector(map[string]any{"batch_size": batchSize})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c.apiBaseURL, c.contentBaseURL = server.URL, server.URL
+ request := SyncRequest{WindowEnd: time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)}
+ session, err := c.OpenSync(context.Background(), request)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer session.Close()
+ if batchSize == 1 {
+ first, err := session.NextBatch(context.Background())
+ if err != nil || len(first.Documents) != 1 {
+ t.Fatalf("first batch: %+v, %v", first, err)
+ }
+ request.Resume = first.Checkpoint
+ }
+ failed, err := session.NextBatch(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "503") {
+ t.Fatalf("download failure lost: batch=%+v err=%v", failed, err)
+ }
+ if failed.Checkpoint != nil || len(failed.Documents) != 0 {
+ t.Fatalf("failed batch must not commit: %+v", failed)
+ }
+ failing = false
+ // Both retrying the session and reopening from the last committed
+ // checkpoint must replay every uncommitted document.
+ reopened, err := c.OpenSync(context.Background(), request)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reopened.Close()
+ for _, retrySession := range []SyncSession{session, reopened} {
+ batch, err := retrySession.NextBatch(context.Background())
+ if err != nil || len(batch.Documents) != batchSize {
+ t.Fatalf("retry: %+v, %v", batch, err)
+ }
+ if batch.Documents[len(batch.Documents)-1].SourceID != "dropbox:id:b" {
+ t.Fatalf("failed file missing: %+v", batch)
+ }
+ if _, err := retrySession.NextBatch(context.Background()); err != io.EOF {
+ t.Fatalf("end: %v", err)
+ }
+ }
+ })
+ }
+}