Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(notion): 分页循环补重复游标防护与跳数上限 (#3890)

linus-liu-webdabbdaf156b8
diff --git a/internal/datasource/connector/notion/client.go b/internal/datasource/connector/notion/client.go
index 9b862b3f4..ae4befee3 100644
--- a/internal/datasource/connector/notion/client.go
+++ b/internal/datasource/connector/notion/client.go
@@ -238,8 +238,9 @@ func (c *notionClient) GetDataSourceInfo(ctx context.Context, dsID string) (*not
func (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string) ([]notionBlock, error) {
var allBlocks []notionBlock
var startCursor string
+ seenCursors := make(map[string]struct{})
- for {
+ for page := 1; ; page++ {
path := fmt.Sprintf("/v1/blocks/%s/children", blockID)
if startCursor != "" {
path += "?start_cursor=" + startCursor
@@ -265,7 +266,11 @@ func (c *notionClient) GetBlockChildrenFlat(ctx context.Context, blockID string)
if !resp.HasMore || resp.NextCursor == "" {
break
}
- startCursor = resp.NextCursor
+ next, err := advancePaginationCursor(ctx, seenCursors, resp.NextCursor, page)
+ if err != nil {
+ return nil, fmt.Errorf("get block children for %s: %w", blockID, err)
+ }
+ startCursor = next
}
return allBlocks, nil
@@ -469,6 +474,32 @@ func (c *notionClient) DownloadFile(ctx context.Context, fileURL string) ([]byte
// --- Shared pagination helper ---
+// maxPaginationHops bounds every cursor-paginated loop in this file. A vendor
+// (or gateway) that keeps answering has_more=true would otherwise keep the
+// client paging until the sync task hits its deadline; the value mirrors the
+// guard Confluence and DingTalk already carry.
+const maxPaginationHops = 10000
+
+// advancePaginationCursor validates the progress of a cursor-paginated loop
+// after page `page` (1-based) has been fetched: a cursor handed back twice
+// means the vendor is repeating a page, and page >= maxPaginationHops means
+// the listing is unbounded. Both are reported instead of being followed
+// forever, and the hop cap is also logged because it is the one failure that
+// looks like a healthy, still-running sync from the outside.
+func advancePaginationCursor(
+ ctx context.Context, seen map[string]struct{}, cursor string, page int,
+) (string, error) {
+ if page >= maxPaginationHops {
+ logger.Warnf(ctx, "[Notion] pagination exceeded %d pages; aborting", maxPaginationHops)
+ return "", fmt.Errorf("pagination exceeded %d pages", maxPaginationHops)
+ }
+ if _, exists := seen[cursor]; exists {
+ return "", fmt.Errorf("pagination repeated next_cursor %q", cursor)
+ }
+ seen[cursor] = struct{}{}
+ return cursor, nil
+}
+
// errQueryResultTruncated reports that a paginated response was cut short by a
// vendor-side limit, so the rows collected so far are a prefix of the result set
// rather than all of it. Notion signals this with has_more=false plus
@@ -485,8 +516,9 @@ var errQueryResultTruncated = errors.New("notion paginated response incomplete:
func (c *notionClient) paginatePages(ctx context.Context, method, path string) ([]notionPage, error) {
var allPages []notionPage
var startCursor string
+ seenCursors := make(map[string]struct{})
- for {
+ for page := 1; ; page++ {
body := map[string]interface{}{
"page_size": 100,
}
@@ -538,7 +570,11 @@ func (c *notionClient) paginatePages(ctx context.Context, method, path string) (
if !resp.HasMore || resp.NextCursor == "" {
break
}
- startCursor = resp.NextCursor
+ next, err := advancePaginationCursor(ctx, seenCursors, resp.NextCursor, page)
+ if err != nil {
+ return nil, fmt.Errorf("paginate %s: %w", path, err)
+ }
+ startCursor = next
}
return allPages, nil
@@ -639,8 +675,9 @@ func (c *notionClient) queryDataSourceWindow(
ctx context.Context, path string, windowStart *time.Time,
) (rows []notionPage, lastCreatedTime time.Time, incompleteReason string, err error) {
var startCursor string
+ seenCursors := make(map[string]struct{})
- for {
+ for page := 1; ; page++ {
body := map[string]interface{}{
"page_size": dataSourceQueryPageSize,
"sorts": []map[string]string{
@@ -695,7 +732,11 @@ func (c *notionClient) queryDataSourceWindow(
if !resp.HasMore || resp.NextCursor == "" {
return rows, lastCreatedTime, incompleteReason, nil
}
- startCursor = resp.NextCursor
+ next, cursorErr := advancePaginationCursor(ctx, seenCursors, resp.NextCursor, page)
+ if cursorErr != nil {
+ return nil, time.Time{}, "", fmt.Errorf("paginate %s: %w", path, cursorErr)
+ }
+ startCursor = next
}
}
diff --git a/internal/datasource/connector/notion/pagination_guard_test.go b/internal/datasource/connector/notion/pagination_guard_test.go
new file mode 100644
index 000000000..a2d5d5c8a
--- /dev/null
+++ b/internal/datasource/connector/notion/pagination_guard_test.go
@@ -0,0 +1,172 @@
+package notion
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "golang.org/x/time/rate"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+)
+
+// paginationHopCap mirrors maxPaginationHops in client.go. It is spelled as a
+// literal so this file also compiles — and then fails on behaviour — against a
+// tree that has no guard at all.
+const paginationHopCap = 10000
+
+// fakeLoopBudget bounds an unguarded run a little above the guard's own cap:
+// the fake answers 400 once the budget is gone, so a regression fails with the
+// wrong error instead of paging until the test times out.
+const fakeLoopBudget = paginationHopCap + 5
+
+// paginationTestClient returns a client pointed at the fake server. The
+// production 3 req/s limiter would turn a 10000-hop regression into 55 minutes
+// of waiting, so it is replaced with an unlimited one: this test is about the
+// loop's exit conditions, not its pacing.
+func paginationTestClient(t *testing.T, server *httptest.Server) *notionClient {
+ t.Helper()
+ client := mustTestClient(t, "test-token", server.URL)
+ client.limiter = rate.NewLimiter(rate.Inf, 1)
+ return client
+}
+
+func writeListPage(w http.ResponseWriter, results string, hasMore bool, nextCursor string) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = fmt.Fprintf(w, `{"object":"list","results":%s,"has_more":%t,"next_cursor":%q}`,
+ results, hasMore, nextCursor)
+}
+
+// Notion can answer has_more=true with a next_cursor it already returned. The
+// loop used to follow it forever; it must now stop as soon as a cursor repeats.
+func TestPaginatePagesRejectsRepeatedCursor(t *testing.T) {
+ var requests atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if requests.Add(1) > fakeLoopBudget {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ writeListPage(w, `[]`, true, "same-cursor")
+ }))
+ defer server.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ _, err := paginationTestClient(t, server).paginatePages(ctx, http.MethodPost, "/v1/search")
+ require.Error(t, err)
+ require.Equal(t, int64(2), requests.Load(),
+ "the guard must reject the repeated cursor on the second page; err=%v", err)
+ require.Contains(t, err.Error(), `repeated next_cursor "same-cursor"`)
+}
+
+// A vendor that invents a new cursor for every page never repeats one, so the
+// repeat guard cannot stop it: the hop cap must, and it must log why.
+func TestPaginatePagesStopsAtPaginationCap(t *testing.T) {
+ var requests atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ n := requests.Add(1)
+ if n > fakeLoopBudget {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ writeListPage(w, `[]`, true, fmt.Sprintf("cursor-%d", n))
+ }))
+ defer server.Close()
+
+ var logs bytes.Buffer
+ logger.SetOutput(&logs)
+ defer logger.SetOutput(os.Stdout)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ _, err := paginationTestClient(t, server).paginatePages(ctx, http.MethodPost, "/v1/search")
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "pagination exceeded 10000 pages")
+ require.Equal(t, int64(paginationHopCap), requests.Load())
+ require.Contains(t, logs.String(), "pagination exceeded 10000 pages",
+ "hitting the hop cap must be visible in the log")
+}
+
+// The guard must not truncate a listing that genuinely spans several pages.
+func TestPaginatePagesMergesRealPages(t *testing.T) {
+ var requests atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ body, _ := io.ReadAll(r.Body)
+ var req map[string]interface{}
+ _ = json.Unmarshal(body, &req)
+ if cursor, _ := req["start_cursor"].(string); cursor == "" {
+ writeListPage(w, `[{"id":"page-1","object":"page"}]`, true, "cursor-2")
+ return
+ }
+ writeListPage(w, `[{"id":"page-2","object":"page"}]`, false, "")
+ }))
+ defer server.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ pages, err := paginationTestClient(t, server).paginatePages(ctx, http.MethodPost, "/v1/search")
+ require.NoError(t, err)
+ require.Len(t, pages, 2)
+ require.Equal(t, "page-1", pages[0].ID)
+ require.Equal(t, "page-2", pages[1].ID)
+ require.Equal(t, int64(2), requests.Load())
+}
+
+// GetBlockChildrenFlat is the same cursor loop on the block endpoint and used
+// to spin the same way during resource discovery.
+func TestGetBlockChildrenFlatRejectsRepeatedCursor(t *testing.T) {
+ var requests atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if requests.Add(1) > fakeLoopBudget {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ writeListPage(w, `[]`, true, "same-cursor")
+ }))
+ defer server.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ _, err := paginationTestClient(t, server).GetBlockChildrenFlat(ctx, "block-1")
+ require.Error(t, err)
+ require.Equal(t, int64(2), requests.Load(),
+ "the guard must reject the repeated cursor on the second page; err=%v", err)
+ require.Contains(t, err.Error(), `repeated next_cursor "same-cursor"`)
+}
+
+// Data source queries page through queryDataSourceWindow since #3845, so the
+// repeated-cursor guard has to cover that loop too.
+func TestQueryDataSourceAllRejectsRepeatedCursor(t *testing.T) {
+ var requests atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if requests.Add(1) > fakeLoopBudget {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ writeListPage(w, `[]`, true, "same-cursor")
+ }))
+ defer server.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ _, err := paginationTestClient(t, server).queryDataSourceAll(ctx, "/v1/data_sources/ds-1/query")
+ require.Error(t, err)
+ require.Equal(t, int64(2), requests.Load(),
+ "the guard must reject the repeated cursor on the second page; err=%v", err)
+ require.Contains(t, err.Error(), `repeated next_cursor "same-cursor"`)
+}