Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(sync): reject incomplete Google Drive snapshots before pruning (#20629)

Luciene01c5d475c0e
diff --git a/internal/syncer/connector/google_drive.go b/internal/syncer/connector/google_drive.go
index 861cabc5c0..721aaaff1a 100644
--- a/internal/syncer/connector/google_drive.go
+++ b/internal/syncer/connector/google_drive.go
@@ -332,7 +332,7 @@ func (c *GoogleDriveConnector) listFilePage(ctx context.Context, scope googleDri
return googleDriveFilePage{}, err
}
query := url.Values{
- "fields": {"nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,shortcutDetails,owners(emailAddress),size,md5Checksum,fileExtension)"},
+ "fields": {"nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,shortcutDetails,owners(emailAddress),size,md5Checksum,fileExtension)"},
"pageSize": {strconv.Itoa(googleDriveItemsPerPage)},
"orderBy": {"modifiedTime"},
"q": {googleDriveFileQuery(scope, windowStart, windowEnd)},
@@ -361,11 +361,15 @@ func (c *GoogleDriveConnector) listFilePage(ctx context.Context, scope googleDri
return page, err
}
+// listFilePageWithRetry retries rate limits and rejects incomplete search results.
func (c *GoogleDriveConnector) listFilePageWithRetry(ctx context.Context, scope googleDriveScope, pageToken string, windowStart *time.Time, windowEnd time.Time) (googleDriveFilePage, error) {
var lastErr error
for attempt := 1; attempt <= googleDriveListRetryCount; attempt++ {
page, err := c.listFilePage(ctx, scope, pageToken, windowStart, windowEnd)
if err == nil {
+ if page.IncompleteSearch {
+ return googleDriveFilePage{}, fmt.Errorf("Google Drive file search is incomplete; narrow the connector's retrieval scope and retry")
+ }
return page, nil
}
lastErr = err
@@ -382,6 +386,7 @@ func (c *GoogleDriveConnector) listFilePageWithRetry(ctx context.Context, scope
return googleDriveFilePage{}, lastErr
}
+// listFolderIDs lists child folders, failing if any page reports an incomplete search.
func (c *GoogleDriveConnector) listFolderIDs(ctx context.Context, userEmail, parentID string) ([]string, error) {
if c.listFolders != nil {
return c.listFolders(ctx, userEmail, parentID)
@@ -395,7 +400,7 @@ func (c *GoogleDriveConnector) listFolderIDs(ctx context.Context, userEmail, par
for {
query := url.Values{
"corpora": {"allDrives"},
- "fields": {"nextPageToken,files(id)"},
+ "fields": {"nextPageToken,incompleteSearch,files(id)"},
"includeItemsFromAllDrives": {"true"},
"supportsAllDrives": {"true"},
"pageSize": {strconv.Itoa(googleDriveItemsPerPage)},
@@ -408,6 +413,9 @@ func (c *GoogleDriveConnector) listFolderIDs(ctx context.Context, userEmail, par
if err = c.getJSON(ctx, client, "https://www.googleapis.com/drive/v3/files?"+query.Encode(), &page); err != nil {
return nil, err
}
+ if page.IncompleteSearch {
+ return nil, fmt.Errorf("Google Drive folder search for %s is incomplete; narrow the connector's retrieval scope and retry", parentID)
+ }
for _, file := range page.Files {
if file.ID != "" {
ids = append(ids, file.ID)
@@ -997,8 +1005,9 @@ type googleDriveListRequest struct {
}
type googleDriveFilePage struct {
- NextPageToken string `json:"nextPageToken"`
- Files []googleDriveFile `json:"files"`
+ NextPageToken string `json:"nextPageToken"`
+ IncompleteSearch bool `json:"incompleteSearch"`
+ Files []googleDriveFile `json:"files"`
}
type googleDriveDrivePage struct {
diff --git a/internal/syncer/connector/google_drive_test.go b/internal/syncer/connector/google_drive_test.go
index bbbb5a42c0..f6717a358e 100644
--- a/internal/syncer/connector/google_drive_test.go
+++ b/internal/syncer/connector/google_drive_test.go
@@ -2,9 +2,11 @@ package connector
import (
"context"
+ "encoding/json"
"errors"
"io"
"net/http"
+ "net/http/httptest"
"strings"
"testing"
"time"
@@ -403,3 +405,134 @@ func googleDriveTestFile(id, name, modifiedTime string) googleDriveFile {
WebViewLink: "https://drive.google.com/file/d/" + id + "/view",
}
}
+
+// TestGoogleDriveSessionsRequireCompleteSearch verifies partial listings fail both sessions.
+func TestGoogleDriveSessionsRequireCompleteSearch(t *testing.T) {
+ tests := []struct {
+ name string
+ folderScope bool
+ incompleteListing string
+ incompletePageToken string
+ empty bool
+ wantDocuments int
+ }{
+ {name: "files first page", incompleteListing: "files"},
+ {name: "files middle page", incompleteListing: "files", incompletePageToken: "second"},
+ {name: "folders first page", folderScope: true, incompleteListing: "folders"},
+ {name: "folders middle page", folderScope: true, incompleteListing: "folders", incompletePageToken: "second"},
+ {name: "complete empty", empty: true},
+ {name: "complete files", wantDocuments: 2},
+ {name: "complete recursive folders", folderScope: true, wantDocuments: 3},
+ }
+ for _, mode := range []string{"sync", "prune"} {
+ for _, test := range tests {
+ t.Run(mode+"/"+test.name, func(t *testing.T) {
+ requests := 0
+ client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
+ requests++
+ if request.Method != http.MethodGet || request.URL.Path != "/drive/v3/files" {
+ t.Fatalf("unexpected request: %s %s", request.Method, request.URL)
+ }
+ query := request.URL.Query()
+ if !strings.Contains(query.Get("fields"), "incompleteSearch") {
+ t.Errorf("list request must include incompleteSearch: %s", query.Get("fields"))
+ }
+ listing := "files"
+ if strings.HasPrefix(query.Get("q"), "mimeType = '") {
+ listing = "folders"
+ }
+ pageToken := query.Get("pageToken")
+ page := map[string]any{}
+ if !test.empty {
+ page["incompleteSearch"] = false
+ id := "first"
+ if listing == "folders" {
+ if strings.Contains(query.Get("q"), "'root' in parents") {
+ id = "child-" + id
+ if pageToken == "" {
+ page["nextPageToken"] = "second"
+ } else {
+ id = "child-second"
+ }
+ } else {
+ id = ""
+ }
+ } else if test.folderScope {
+ for _, folderID := range []string{"root", "child-first", "child-second"} {
+ if strings.Contains(query.Get("q"), "'"+folderID+"' in parents") {
+ id = folderID + "-file"
+ }
+ }
+ } else if pageToken == "" {
+ page["nextPageToken"] = "second"
+ } else {
+ id = "second"
+ }
+ if id != "" {
+ page["files"] = []map[string]string{{"id": id, "name": id + ".txt", "mimeType": "text/plain"}}
+ }
+ }
+ if listing == test.incompleteListing && pageToken == test.incompletePageToken {
+ page["incompleteSearch"] = true
+ // A terminal partial page is still not a complete snapshot.
+ delete(page, "nextPageToken")
+ }
+ recorder := httptest.NewRecorder()
+ if err := json.NewEncoder(recorder).Encode(page); err != nil {
+ t.Fatal(err)
+ }
+ return recorder.Result(), nil
+ })}
+ connector := &GoogleDriveConnector{httpClientForUser: func(context.Context, string) (*http.Client, error) {
+ return client, nil
+ }}
+ scope := googleDriveScope{userEmail: "[email protected]", corpora: "user", includeSharedWithMe: true}
+ if test.folderScope {
+ scope.corpora = "folder"
+ scope.folderID = "root"
+ }
+ var next func(context.Context) (int, error)
+ if mode == "prune" {
+ session := &googleDrivePruneSession{connector: connector, scopes: []googleDriveScope{scope}, batchSize: 1}
+ next = func(ctx context.Context) (int, error) {
+ batch, err := session.NextBatch(ctx)
+ return len(batch.Documents), err
+ }
+ } else {
+ session := &googleDriveSyncSession{connector: connector, scopes: []googleDriveScope{scope}, batchSize: 1}
+ next = func(ctx context.Context) (int, error) {
+ batch, err := session.NextBatch(ctx)
+ return len(batch.Documents), err
+ }
+ }
+ documents := 0
+ for calls := 0; calls < 10; calls++ {
+ count, err := next(t.Context())
+ if test.incompleteListing != "" && err != nil {
+ if errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "search") || !strings.Contains(err.Error(), "incomplete") {
+ t.Fatalf("partial snapshot ended with %v, want incomplete-search error", err)
+ }
+ if count != 0 {
+ t.Fatalf("failed batch returned %d documents", count)
+ }
+ if test.incompletePageToken != "" && requests < 2 {
+ t.Fatal("middle-page case did not reach pagination")
+ }
+ return
+ }
+ if errors.Is(err, io.EOF) {
+ if documents != test.wantDocuments {
+ t.Fatalf("documents = %d, want %d", documents, test.wantDocuments)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ documents += count
+ }
+ t.Fatal("session did not terminate")
+ })
+ }
+ }
+}