RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(sync): reject incomplete Google Drive snapshots before pruning (#20629)
Luciene01c5d475c0e
diff --git a/internal/syncer/connector/google_drive.go b/internal/syncer/connector/google_drive.goindex 861cabc5c0..721aaaff1a 100644--- a/internal/syncer/connector/google_drive.go+++ b/internal/syncer/connector/google_drive.go@@ -332,7 +332,7 @@ func (c *GoogleDriveConnector) listFilePage(ctx context.Context, scope googleDrireturn googleDriveFilePage{}, err}query := url.Values{- "fields": {"nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,shortcutDetails,owners(emailAddress),size,md5Checksum,fileExtension)"},+ "fields": {"nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,shortcutDetails,owners(emailAddress),size,md5Checksum,fileExtension)"},"pageSize": {strconv.Itoa(googleDriveItemsPerPage)},"orderBy": {"modifiedTime"},"q": {googleDriveFileQuery(scope, windowStart, windowEnd)},@@ -361,11 +361,15 @@ func (c *GoogleDriveConnector) listFilePage(ctx context.Context, scope googleDrireturn page, err}+// listFilePageWithRetry retries rate limits and rejects incomplete search results.func (c *GoogleDriveConnector) listFilePageWithRetry(ctx context.Context, scope googleDriveScope, pageToken string, windowStart *time.Time, windowEnd time.Time) (googleDriveFilePage, error) {var lastErr errorfor attempt := 1; attempt <= googleDriveListRetryCount; attempt++ {page, err := c.listFilePage(ctx, scope, pageToken, windowStart, windowEnd)if err == nil {+ if page.IncompleteSearch {+ return googleDriveFilePage{}, fmt.Errorf("Google Drive file search is incomplete; narrow the connector's retrieval scope and retry")+ }return page, nil}lastErr = err@@ -382,6 +386,7 @@ func (c *GoogleDriveConnector) listFilePageWithRetry(ctx context.Context, scopereturn googleDriveFilePage{}, lastErr}+// listFolderIDs lists child folders, failing if any page reports an incomplete search.func (c *GoogleDriveConnector) listFolderIDs(ctx context.Context, userEmail, parentID string) ([]string, error) {if c.listFolders != nil {return c.listFolders(ctx, userEmail, parentID)@@ -395,7 +400,7 @@ func (c *GoogleDriveConnector) listFolderIDs(ctx context.Context, userEmail, parfor {query := url.Values{"corpora": {"allDrives"},- "fields": {"nextPageToken,files(id)"},+ "fields": {"nextPageToken,incompleteSearch,files(id)"},"includeItemsFromAllDrives": {"true"},"supportsAllDrives": {"true"},"pageSize": {strconv.Itoa(googleDriveItemsPerPage)},@@ -408,6 +413,9 @@ func (c *GoogleDriveConnector) listFolderIDs(ctx context.Context, userEmail, parif err = c.getJSON(ctx, client, "https://www.googleapis.com/drive/v3/files?"+query.Encode(), &page); err != nil {return nil, err}+ if page.IncompleteSearch {+ return nil, fmt.Errorf("Google Drive folder search for %s is incomplete; narrow the connector's retrieval scope and retry", parentID)+ }for _, file := range page.Files {if file.ID != "" {ids = append(ids, file.ID)@@ -997,8 +1005,9 @@ type googleDriveListRequest struct {}type googleDriveFilePage struct {- NextPageToken string `json:"nextPageToken"`- Files []googleDriveFile `json:"files"`+ NextPageToken string `json:"nextPageToken"`+ IncompleteSearch bool `json:"incompleteSearch"`+ Files []googleDriveFile `json:"files"`}type googleDriveDrivePage struct {diff --git a/internal/syncer/connector/google_drive_test.go b/internal/syncer/connector/google_drive_test.goindex bbbb5a42c0..f6717a358e 100644--- a/internal/syncer/connector/google_drive_test.go+++ b/internal/syncer/connector/google_drive_test.go@@ -2,9 +2,11 @@ package connectorimport ("context"+ "encoding/json""errors""io""net/http"+ "net/http/httptest""strings""testing""time"@@ -403,3 +405,134 @@ func googleDriveTestFile(id, name, modifiedTime string) googleDriveFile {WebViewLink: "https://drive.google.com/file/d/" + id + "/view",}}++// TestGoogleDriveSessionsRequireCompleteSearch verifies partial listings fail both sessions.+func TestGoogleDriveSessionsRequireCompleteSearch(t *testing.T) {+ tests := []struct {+ name string+ folderScope bool+ incompleteListing string+ incompletePageToken string+ empty bool+ wantDocuments int+ }{+ {name: "files first page", incompleteListing: "files"},+ {name: "files middle page", incompleteListing: "files", incompletePageToken: "second"},+ {name: "folders first page", folderScope: true, incompleteListing: "folders"},+ {name: "folders middle page", folderScope: true, incompleteListing: "folders", incompletePageToken: "second"},+ {name: "complete empty", empty: true},+ {name: "complete files", wantDocuments: 2},+ {name: "complete recursive folders", folderScope: true, wantDocuments: 3},+ }+ for _, mode := range []string{"sync", "prune"} {+ for _, test := range tests {+ t.Run(mode+"/"+test.name, func(t *testing.T) {+ requests := 0+ client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {+ requests+++ if request.Method != http.MethodGet || request.URL.Path != "/drive/v3/files" {+ t.Fatalf("unexpected request: %s %s", request.Method, request.URL)+ }+ query := request.URL.Query()+ if !strings.Contains(query.Get("fields"), "incompleteSearch") {+ t.Errorf("list request must include incompleteSearch: %s", query.Get("fields"))+ }+ listing := "files"+ if strings.HasPrefix(query.Get("q"), "mimeType = '") {+ listing = "folders"+ }+ pageToken := query.Get("pageToken")+ page := map[string]any{}+ if !test.empty {+ page["incompleteSearch"] = false+ id := "first"+ if listing == "folders" {+ if strings.Contains(query.Get("q"), "'root' in parents") {+ id = "child-" + id+ if pageToken == "" {+ page["nextPageToken"] = "second"+ } else {+ id = "child-second"+ }+ } else {+ id = ""+ }+ } else if test.folderScope {+ for _, folderID := range []string{"root", "child-first", "child-second"} {+ if strings.Contains(query.Get("q"), "'"+folderID+"' in parents") {+ id = folderID + "-file"+ }+ }+ } else if pageToken == "" {+ page["nextPageToken"] = "second"+ } else {+ id = "second"+ }+ if id != "" {+ page["files"] = []map[string]string{{"id": id, "name": id + ".txt", "mimeType": "text/plain"}}+ }+ }+ if listing == test.incompleteListing && pageToken == test.incompletePageToken {+ page["incompleteSearch"] = true+ // A terminal partial page is still not a complete snapshot.+ delete(page, "nextPageToken")+ }+ recorder := httptest.NewRecorder()+ if err := json.NewEncoder(recorder).Encode(page); err != nil {+ t.Fatal(err)+ }+ return recorder.Result(), nil+ })}+ connector := &GoogleDriveConnector{httpClientForUser: func(context.Context, string) (*http.Client, error) {+ return client, nil+ }}+ scope := googleDriveScope{userEmail: "[email protected]", corpora: "user", includeSharedWithMe: true}+ if test.folderScope {+ scope.corpora = "folder"+ scope.folderID = "root"+ }+ var next func(context.Context) (int, error)+ if mode == "prune" {+ session := &googleDrivePruneSession{connector: connector, scopes: []googleDriveScope{scope}, batchSize: 1}+ next = func(ctx context.Context) (int, error) {+ batch, err := session.NextBatch(ctx)+ return len(batch.Documents), err+ }+ } else {+ session := &googleDriveSyncSession{connector: connector, scopes: []googleDriveScope{scope}, batchSize: 1}+ next = func(ctx context.Context) (int, error) {+ batch, err := session.NextBatch(ctx)+ return len(batch.Documents), err+ }+ }+ documents := 0+ for calls := 0; calls < 10; calls++ {+ count, err := next(t.Context())+ if test.incompleteListing != "" && err != nil {+ if errors.Is(err, io.EOF) || !strings.Contains(err.Error(), "search") || !strings.Contains(err.Error(), "incomplete") {+ t.Fatalf("partial snapshot ended with %v, want incomplete-search error", err)+ }+ if count != 0 {+ t.Fatalf("failed batch returned %d documents", count)+ }+ if test.incompletePageToken != "" && requests < 2 {+ t.Fatal("middle-page case did not reach pagination")+ }+ return+ }+ if errors.Is(err, io.EOF) {+ if documents != test.wantDocuments {+ t.Fatalf("documents = %d, want %d", documents, test.wantDocuments)+ }+ return+ }+ if err != nil {+ t.Fatal(err)+ }+ documents += count+ }+ t.Fatal("session did not terminate")+ })+ }+ }+}