Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(notion): 分页循环补重复游标防护与跳数上限 (#3890)
linus-liu-webdabbdaf156b8
diff --git a/internal/datasource/connector/notion/client.go b/internal/datasource/connector/notion/client.goindex 9b862b3f4..ae4befee3 100644--- a/internal/datasource/connector/notion/client.go+++ b/internal/datasource/connector/notion/client.go@@ -238,8 +238,9 @@ func (c *notionClient) GetDataSourceInfo(ctx context.Context, dsID string) (*notfunc (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string) ([]notionBlock, error) {var allBlocks []notionBlockvar startCursor string+ seenCursors := make(map[string]struct{})- for {+ for page := 1; ; page++ {path := fmt.Sprintf("/v1/blocks/%s/children", blockID)if startCursor != "" {path += "?start_cursor=" + startCursor@@ -265,7 +266,11 @@ func (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string)if !resp.HasMore || resp.NextCursor == "" {break}- startCursor = resp.NextCursor+ next, err := advancePaginationCursor(ctx, seenCursors, resp.NextCursor, page)+ if err != nil {+ return nil, fmt.Errorf("get block children for %s: %w", blockID, err)+ }+ startCursor = next}return allBlocks, nil@@ -469,6 +474,32 @@ func (c *notionClient) DownloadFile(ctx context.Context, fileURL string) ([]byte// --- Shared pagination helper ---+// maxPaginationHops bounds every cursor-paginated loop in this file. A vendor+// (or gateway) that keeps answering has_more=true would otherwise keep the+// client paging until the sync task hits its deadline; the value mirrors the+// guard Confluence and DingTalk already carry.+const maxPaginationHops = 10000++// advancePaginationCursor validates the progress of a cursor-paginated loop+// after page `page` (1-based) has been fetched: a cursor handed back twice+// means the vendor is repeating a page, and page >= maxPaginationHops means+// the listing is unbounded. Both are reported instead of being followed+// forever, and the hop cap is also logged because it is the one failure that+// looks like a healthy, still-running sync from the outside.+func advancePaginationCursor(+ ctx context.Context, seen map[string]struct{}, cursor string, page int,+) (string, error) {+ if page >= maxPaginationHops {+ logger.Warnf(ctx, "[Notion] pagination exceeded %d pages; aborting", maxPaginationHops)+ return "", fmt.Errorf("pagination exceeded %d pages", maxPaginationHops)+ }+ if _, exists := seen[cursor]; exists {+ return "", fmt.Errorf("pagination repeated next_cursor %q", cursor)+ }+ seen[cursor] = struct{}{}+ return cursor, nil+}+// errQueryResultTruncated reports that a paginated response was cut short by a// vendor-side limit, so the rows collected so far are a prefix of the result set// rather than all of it. Notion signals this with has_more=false plus@@ -485,8 +516,9 @@ var errQueryResultTruncated = errors.New("notion paginated response incomplete:func (c *notionClient) paginatePages(ctx context.Context, method, path string) ([]notionPage, error) {var allPages []notionPagevar startCursor string+ seenCursors := make(map[string]struct{})- for {+ for page := 1; ; page++ {body := map[string]interface{}{"page_size": 100,}@@ -538,7 +570,11 @@ func (c *notionClient) paginatePages(ctx context.Context, method, path string) (if !resp.HasMore || resp.NextCursor == "" {break}- startCursor = resp.NextCursor+ next, err := advancePaginationCursor(ctx, seenCursors, resp.NextCursor, page)+ if err != nil {+ return nil, fmt.Errorf("paginate %s: %w", path, err)+ }+ startCursor = next}return allPages, nil@@ -639,8 +675,9 @@ func (c *notionClient) queryDataSourceWindow(ctx context.Context, path string, windowStart *time.Time,) (rows []notionPage, lastCreatedTime time.Time, incompleteReason string, err error) {var startCursor string+ seenCursors := make(map[string]struct{})- for {+ for page := 1; ; page++ {body := map[string]interface{}{"page_size": dataSourceQueryPageSize,"sorts": []map[string]string{@@ -695,7 +732,11 @@ func (c *notionClient) queryDataSourceWindow(if !resp.HasMore || resp.NextCursor == "" {return rows, lastCreatedTime, incompleteReason, nil}- startCursor = resp.NextCursor+ next, cursorErr := advancePaginationCursor(ctx, seenCursors, resp.NextCursor, page)+ if cursorErr != nil {+ return nil, time.Time{}, "", fmt.Errorf("paginate %s: %w", path, cursorErr)+ }+ startCursor = next}}diff --git a/internal/datasource/connector/notion/pagination_guard_test.go b/internal/datasource/connector/notion/pagination_guard_test.gonew file mode 100644index 000000000..a2d5d5c8a--- /dev/null+++ b/internal/datasource/connector/notion/pagination_guard_test.go@@ -0,0 +1,172 @@+package notion++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "sync/atomic"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "golang.org/x/time/rate"++ "github.com/Tencent/WeKnora/internal/logger"+)++// paginationHopCap mirrors maxPaginationHops in client.go. It is spelled as a+// literal so this file also compiles — and then fails on behaviour — against a+// tree that has no guard at all.+const paginationHopCap = 10000++// fakeLoopBudget bounds an unguarded run a little above the guard's own cap:+// the fake answers 400 once the budget is gone, so a regression fails with the+// wrong error instead of paging until the test times out.+const fakeLoopBudget = paginationHopCap + 5++// paginationTestClient returns a client pointed at the fake server. The+// production 3 req/s limiter would turn a 10000-hop regression into 55 minutes+// of waiting, so it is replaced with an unlimited one: this test is about the+// loop's exit conditions, not its pacing.+func paginationTestClient(t *testing.T, server *httptest.Server) *notionClient {+ t.Helper()+ client := mustTestClient(t, "test-token", server.URL)+ client.limiter = rate.NewLimiter(rate.Inf, 1)+ return client+}++func writeListPage(w http.ResponseWriter, results string, hasMore bool, nextCursor string) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprintf(w, `{"object":"list","results":%s,"has_more":%t,"next_cursor":%q}`,+ results, hasMore, nextCursor)+}++// Notion can answer has_more=true with a next_cursor it already returned. The+// loop used to follow it forever; it must now stop as soon as a cursor repeats.+func TestPaginatePagesRejectsRepeatedCursor(t *testing.T) {+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ if requests.Add(1) > fakeLoopBudget {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ writeListPage(w, `[]`, true, "same-cursor")+ }))+ defer server.Close()++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ _, err := paginationTestClient(t, server).paginatePages(ctx, http.MethodPost, "/v1/search")+ require.Error(t, err)+ require.Equal(t, int64(2), requests.Load(),+ "the guard must reject the repeated cursor on the second page; err=%v", err)+ require.Contains(t, err.Error(), `repeated next_cursor "same-cursor"`)+}++// A vendor that invents a new cursor for every page never repeats one, so the+// repeat guard cannot stop it: the hop cap must, and it must log why.+func TestPaginatePagesStopsAtPaginationCap(t *testing.T) {+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ n := requests.Add(1)+ if n > fakeLoopBudget {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ writeListPage(w, `[]`, true, fmt.Sprintf("cursor-%d", n))+ }))+ defer server.Close()++ var logs bytes.Buffer+ logger.SetOutput(&logs)+ defer logger.SetOutput(os.Stdout)++ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ _, err := paginationTestClient(t, server).paginatePages(ctx, http.MethodPost, "/v1/search")+ require.Error(t, err)+ require.Contains(t, err.Error(), "pagination exceeded 10000 pages")+ require.Equal(t, int64(paginationHopCap), requests.Load())+ require.Contains(t, logs.String(), "pagination exceeded 10000 pages",+ "hitting the hop cap must be visible in the log")+}++// The guard must not truncate a listing that genuinely spans several pages.+func TestPaginatePagesMergesRealPages(t *testing.T) {+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ body, _ := io.ReadAll(r.Body)+ var req map[string]interface{}+ _ = json.Unmarshal(body, &req)+ if cursor, _ := req["start_cursor"].(string); cursor == "" {+ writeListPage(w, `[{"id":"page-1","object":"page"}]`, true, "cursor-2")+ return+ }+ writeListPage(w, `[{"id":"page-2","object":"page"}]`, false, "")+ }))+ defer server.Close()++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ pages, err := paginationTestClient(t, server).paginatePages(ctx, http.MethodPost, "/v1/search")+ require.NoError(t, err)+ require.Len(t, pages, 2)+ require.Equal(t, "page-1", pages[0].ID)+ require.Equal(t, "page-2", pages[1].ID)+ require.Equal(t, int64(2), requests.Load())+}++// GetBlockChildrenFlat is the same cursor loop on the block endpoint and used+// to spin the same way during resource discovery.+func TestGetBlockChildrenFlatRejectsRepeatedCursor(t *testing.T) {+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ if requests.Add(1) > fakeLoopBudget {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ writeListPage(w, `[]`, true, "same-cursor")+ }))+ defer server.Close()++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ _, err := paginationTestClient(t, server).GetBlockChildrenFlat(ctx, "block-1")+ require.Error(t, err)+ require.Equal(t, int64(2), requests.Load(),+ "the guard must reject the repeated cursor on the second page; err=%v", err)+ require.Contains(t, err.Error(), `repeated next_cursor "same-cursor"`)+}++// Data source queries page through queryDataSourceWindow since #3845, so the+// repeated-cursor guard has to cover that loop too.+func TestQueryDataSourceAllRejectsRepeatedCursor(t *testing.T) {+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ if requests.Add(1) > fakeLoopBudget {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ writeListPage(w, `[]`, true, "same-cursor")+ }))+ defer server.Close()++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()++ _, err := paginationTestClient(t, server).queryDataSourceAll(ctx, "/v1/data_sources/ds-1/query")+ require.Error(t, err)+ require.Equal(t, int64(2), requests.Load(),+ "the guard must reject the repeated cursor on the second page; err=%v", err)+ require.Contains(t, err.Error(), `repeated next_cursor "same-cursor"`)+}