Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(feishu): 云盘分页补上 #3645 给 wiki 加的重复 page token 防护 (#3770)

linus-liu-weba282d1091ebf
diff --git a/internal/datasource/connector/feishu/core/client.go b/internal/datasource/connector/feishu/core/client.go
index d77e3941f..f6484c217 100644
--- a/internal/datasource/connector/feishu/core/client.go
+++ b/internal/datasource/connector/feishu/core/client.go
@@ -787,14 +787,19 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err
// listDriveFiles lists files in a Drive folder (non-recursive), one page at a
// time. Pass pageToken="" for the first page; the returned nextPageToken is ""
-// when there are no more pages.
+// when there are no more pages. hasMore mirrors the API's has_more field and
+// is the authoritative stop signal: nextPageToken can be non-empty even when
+// has_more is false.
//
// folderToken == "" is rejected: the root folder is not paginated and does
// not return shortcuts (Feishu API limitation), which would silently drop
// content and risk an unbounded single response. See ADR-0004.
-func (c *Client) listDriveFiles(ctx context.Context, folderToken, pageToken string) ([]DriveFile, string, error) {
+func (c *Client) listDriveFiles(
+ ctx context.Context, folderToken, pageToken string,
+) ([]DriveFile, string, bool, error) {
if folderToken == "" {
- return nil, "", fmt.Errorf("root folder not supported; specify a concrete folder_token (root folder is not paginated and does not return shortcuts)")
+ return nil, "", false, fmt.Errorf("root folder not supported; specify a concrete folder_token " +
+ "(root folder is not paginated and does not return shortcuts)")
}
path := "/open-apis/drive/v1/files?folder_token=" + url.QueryEscape(folderToken)
@@ -806,15 +811,15 @@ func (c *Client) listDriveFiles(ctx context.Context, folderToken, pageToken stri
var resp DriveFileListResponse
if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil {
- return nil, "", fmt.Errorf("list drive files: %w", err)
+ return nil, "", false, fmt.Errorf("list drive files: %w", err)
}
if resp.Code != 0 {
- return nil, "", fmt.Errorf("list drive files error: code=%d msg=%s", resp.Code, resp.Msg)
+ return nil, "", false, fmt.Errorf("list drive files error: code=%d msg=%s", resp.Code, resp.Msg)
}
logger.Infof(ctx, "[FeishuDrive] listDriveFiles: folder=%s got %d files, has_more=%v",
folderToken, len(resp.Data.Files), resp.Data.HasMore)
- return resp.Data.Files, resp.Data.NextPageToken, nil
+ return resp.Data.Files, resp.Data.NextPageToken, resp.Data.HasMore, nil
}
// GetDriveFolderMeta returns the metadata (name, owner, etc.) of a single Drive
@@ -841,15 +846,20 @@ func (c *Client) GetDriveFolderMeta(ctx context.Context, folderToken string) (dr
func (c *Client) ListDriveFilesAllPages(ctx context.Context, folderToken string) ([]DriveFile, error) {
var all []DriveFile
pageToken := ""
+ seenPageTokens := make(map[string]struct{})
for {
- files, next, err := c.listDriveFiles(ctx, folderToken, pageToken)
+ files, next, hasMore, err := c.listDriveFiles(ctx, folderToken, pageToken)
if err != nil {
return nil, err
}
all = append(all, files...)
- if next == "" {
+ if !hasMore || next == "" {
break
}
+ if _, exists := seenPageTokens[next]; exists {
+ return nil, fmt.Errorf("feishu drive file pagination repeated page token %q", next)
+ }
+ seenPageTokens[next] = struct{}{}
pageToken = next
}
return all, nil
diff --git a/internal/datasource/connector/feishu/drive/pagination_test.go b/internal/datasource/connector/feishu/drive/pagination_test.go
new file mode 100644
index 000000000..2a3bf0ab3
--- /dev/null
+++ b/internal/datasource/connector/feishu/drive/pagination_test.go
@@ -0,0 +1,145 @@
+package drive
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core"
+)
+
+// TestClientDriveListFilesPagination is the Drive (云盘) counterpart of the wiki
+// test added in #3645 (TestClientWikiPaginationRejectsRepeatedPageToken):
+// ListDriveFilesAllPages used to stop only on an empty next_page_token, so a
+// repeated token (or has_more=false with a stale token) made it re-request the
+// same page forever. Each case runs under a short deadline so a regression
+// fails fast instead of hanging the suite.
+func TestClientDriveListFilesPagination(t *testing.T) {
+ tests := []struct {
+ name string
+ response func(page int) core.DriveFileListResponse
+ wantErr string
+ wantRequests int
+ wantFiles []string
+ }{
+ {
+ // Feishu can hand back has_more=true with a token it already
+ // returned; that is exactly what #3645 hit on the wiki endpoints.
+ name: "repeated page token",
+ response: func(page int) core.DriveFileListResponse {
+ return core.DriveFileListResponse{
+ ApiResponse: core.ApiResponse{Code: 0},
+ Data: core.DriveFileListData{
+ Files: []core.DriveFile{{Token: fmt.Sprintf("file-%d", page)}},
+ HasMore: true,
+ NextPageToken: "repeated-token",
+ },
+ }
+ },
+ wantErr: "repeated page token",
+ wantRequests: 2,
+ },
+ {
+ // has_more=false is the authoritative stop signal: a residual
+ // non-empty token must not trigger another round trip.
+ name: "has_more false with stale token",
+ response: func(int) core.DriveFileListResponse {
+ return core.DriveFileListResponse{
+ ApiResponse: core.ApiResponse{Code: 0},
+ Data: core.DriveFileListData{
+ Files: []core.DriveFile{{Token: "file-1"}},
+ HasMore: false,
+ NextPageToken: "stale-token",
+ },
+ }
+ },
+ wantRequests: 1,
+ wantFiles: []string{"file-1"},
+ },
+ {
+ // The guard must not truncate a genuinely paginated folder.
+ name: "two real pages are still merged",
+ response: func(page int) core.DriveFileListResponse {
+ if page == 1 {
+ return core.DriveFileListResponse{
+ ApiResponse: core.ApiResponse{Code: 0},
+ Data: core.DriveFileListData{
+ Files: []core.DriveFile{{Token: "file-1"}},
+ HasMore: true,
+ NextPageToken: "token-2",
+ },
+ }
+ }
+ return core.DriveFileListResponse{
+ ApiResponse: core.ApiResponse{Code: 0},
+ Data: core.DriveFileListData{
+ Files: []core.DriveFile{{Token: "file-2"}},
+ HasMore: false,
+ NextPageToken: "",
+ },
+ }
+ },
+ wantRequests: 2,
+ wantFiles: []string{"file-1", "file-2"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mux := http.NewServeMux()
+ authPath := "/open-apis/auth/v3/tenant_access_token/internal"
+ mux.HandleFunc(authPath, func(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, core.TokenResponse{
+ ApiResponse: core.ApiResponse{Code: 0},
+ TenantAccessToken: "fake-token",
+ Expire: 7200,
+ })
+ })
+ var requests atomic.Int64
+ mux.HandleFunc("/open-apis/drive/v1/files", func(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, tt.response(int(requests.Add(1))))
+ })
+ ts := httptest.NewServer(mux)
+ defer ts.Close()
+
+ client := core.NewClient(&core.Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL})
+
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ files, err := client.ListDriveFilesAllPages(ctx, "folder1")
+ if ctx.Err() != nil {
+ t.Fatalf("pagination never terminated (requests=%d, err=%v)", requests.Load(), err)
+ }
+ if tt.wantErr == "" {
+ if err != nil {
+ t.Fatalf("ListDriveFilesAllPages() error = %v, want nil", err)
+ }
+ } else if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
+ t.Fatalf("ListDriveFilesAllPages() error = %v, want error containing %q", err, tt.wantErr)
+ }
+ if got := int(requests.Load()); got != tt.wantRequests {
+ t.Fatalf("drive list requests = %d, want %d", got, tt.wantRequests)
+ }
+
+ gotFiles := make([]string, 0, len(files))
+ seen := make(map[string]bool, len(files))
+ for _, f := range files {
+ if seen[f.Token] {
+ t.Fatalf("file token %q was appended more than once", f.Token)
+ }
+ seen[f.Token] = true
+ gotFiles = append(gotFiles, f.Token)
+ }
+ if tt.wantFiles != nil && !reflect.DeepEqual(gotFiles, tt.wantFiles) {
+ t.Fatalf("files = %v, want %v", gotFiles, tt.wantFiles)
+ }
+ })
+ }
+}