Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(yuque): 两处 offset 分页补跳数上限、重复页检测与 meta.total 提前结束 (#4415)
linus-liu-web4d767d9dc4ea
diff --git a/internal/datasource/connector/yuque/client.go b/internal/datasource/connector/yuque/client.goindex 7fb517074..f51fae198 100644--- a/internal/datasource/connector/yuque/client.go+++ b/internal/datasource/connector/yuque/client.go@@ -23,6 +23,15 @@ const (// maxJSONResponseBytes bounds every API response body. Listings and document// payloads are small; a larger body means a broken or hostile server.maxJSONResponseBytes = 16 << 20++ // maxPaginationHops bounds every offset-paginated walk in this file. The+ // only natural exit is a short page, which a gateway, CDN, or self-hosted+ // deployment that answers every offset with a full page never produces;+ // /users/{login}/repos carries no meta.total to stop on either. Without the+ // cap such a walk keeps calling the API until the sync task hits its+ // deadline, with `all` growing all the way. Mirrors the guard Notion,+ // Confluence, GitLab, and IMA already carry.+ maxPaginationHops = 10000)// client wraps the Yuque Open API.@@ -273,7 +282,12 @@ func (c *client) ListGroupRepos(ctx context.Context, login string) ([]v2Repo, erfunc (c *client) listReposPaginated(ctx context.Context, basePath string) ([]v2Repo, error) {var all []v2Repooffset := 0- for {+ lastFirstID := int64(0)+ for hop := 0; ; hop++ {+ if hop >= maxPaginationHops {+ logger.Warnf(ctx, "[Yuque] %s pagination exceeded %d pages; aborting", basePath, maxPaginationHops)+ return nil, fmt.Errorf("yuque %s pagination exceeded %d pages", basePath, maxPaginationHops)+ }q := buildQuery(map[string]string{"type": "Book","offset": fmt.Sprintf("%d", offset),@@ -287,6 +301,15 @@ func (c *client) listReposPaginated(ctx context.Context, basePath string) ([]v2Rif len(resp.Data) < defaultPageSize {break}+ // A full page whose first id repeats the previous page means the server+ // is answering the same page for a growing offset; following it would+ // append the same repos forever.+ firstID := resp.Data[0].ID+ if firstID != 0 && firstID == lastFirstID {+ logger.Warnf(ctx, "[Yuque] %s pagination repeated page at offset %d; aborting", basePath, offset)+ return nil, fmt.Errorf("yuque %s pagination repeated page at offset %d", basePath, offset)+ }+ lastFirstID = firstIDoffset += defaultPageSize}return all, nil@@ -298,7 +321,12 @@ func (c *client) ListBookDocs(ctx context.Context, bookID int64) ([]v2Doc, errorbasePath := fmt.Sprintf("/api/v2/repos/%d/docs", bookID)var all []v2Docoffset := 0- for {+ lastFirstID := int64(0)+ for hop := 0; ; hop++ {+ if hop >= maxPaginationHops {+ logger.Warnf(ctx, "[Yuque] %s pagination exceeded %d pages; aborting", basePath, maxPaginationHops)+ return nil, fmt.Errorf("yuque %s pagination exceeded %d pages", basePath, maxPaginationHops)+ }q := buildQuery(map[string]string{"offset": fmt.Sprintf("%d", offset),"limit": fmt.Sprintf("%d", defaultPageSize),@@ -311,6 +339,19 @@ func (c *client) ListBookDocs(ctx context.Context, bookID int64) ([]v2Doc, errorif len(resp.Data) < defaultPageSize {break}+ // meta.total counts the whole listing, so a page that reaches it is the+ // last one; asking for the next offset would only fetch an empty page.+ if resp.Meta.Total > 0 && offset+len(resp.Data) >= resp.Meta.Total {+ break+ }+ // Same repeated-page guard as the repo listing: a gateway that ignores+ // offset hands back the identical full page forever.+ firstID := resp.Data[0].ID+ if firstID != 0 && firstID == lastFirstID {+ logger.Warnf(ctx, "[Yuque] %s pagination repeated page at offset %d; aborting", basePath, offset)+ return nil, fmt.Errorf("yuque %s pagination repeated page at offset %d", basePath, offset)+ }+ lastFirstID = firstIDoffset += defaultPageSize}return all, nildiff --git a/internal/datasource/connector/yuque/pagination_guard_test.go b/internal/datasource/connector/yuque/pagination_guard_test.gonew file mode 100644index 000000000..905dcf9e2--- /dev/null+++ b/internal/datasource/connector/yuque/pagination_guard_test.go@@ -0,0 +1,299 @@+package yuque++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/Tencent/WeKnora/internal/logger"+)++// paginationHopCap mirrors maxPaginationHops in client.go. It is spelled as a+// literal so this file also compiles — and then fails on behaviour — against a+// tree that has no guard at all.+const paginationHopCap = 10000++// paginationBudget is how many requests the "always a full page" fakes below+// serve before answering 400. A walk that lacks a guard therefore fails on the+// budget error after paginationBudget+1 requests (and on the request-count+// assertion) instead of spinning until the go test timeout.+const paginationBudget = 300++// fullDocPage builds a full-ish page of docs with ids startID..startID+n-1.+func fullDocPage(startID int64, n int) []v2Doc {+ docs := make([]v2Doc, 0, n)+ for i := 0; i < n; i++ {+ docs = append(docs, v2Doc{+ ID: startID + int64(i),+ Type: "Doc",+ Status: "1",+ Title: fmt.Sprintf("D%d", startID+int64(i)),+ ContentUpdatedAt: "2026-04-20T00:00:00Z",+ })+ }+ return docs+}++// fullRepoPage builds a full-ish page of repos with ids startID..startID+n-1.+func fullRepoPage(startID int64, n int) []v2Repo {+ repos := make([]v2Repo, 0, n)+ for i := 0; i < n; i++ {+ repos = append(repos, v2Repo{+ ID: startID + int64(i),+ Type: "Book",+ Namespace: fmt.Sprintf("alice/book-%d", startID+int64(i)),+ })+ }+ return repos+}++// minimalPage renders a page of defaultPageSize id-only entries, so the hop cap+// test can serve 10000 pages without materialising gigabytes of JSON.+func minimalPage(prefix string, startID int64) string {+ var sb strings.Builder+ sb.WriteString(`{"data":[`)+ for i := 0; i < defaultPageSize; i++ {+ if i > 0 {+ sb.WriteByte(',')+ }+ fmt.Fprintf(&sb, `{"%s":%d}`, prefix, startID+int64(i))+ }+ sb.WriteString(`]}`)+ return sb.String()+}++// alwaysFullPageServer answers every offset with the same full page: a gateway+// or self-hosted deployment that ignores `offset`. Without a repeated-page+// guard the walk keeps appending identical entries.+func alwaysFullPageServer(t *testing.T, body interface{}) (*httptest.Server, *atomic.Int64) {+ t.Helper()+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ if requests.Add(1) > paginationBudget {+ http.Error(w, "pagination guard missing: too many requests", http.StatusBadRequest)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(body)+ }))+ t.Cleanup(server.Close)+ return server, &requests+}++// A docs listing whose meta.total is already covered by the first page must end+// there. The fake keeps answering a full page for every offset, so before the+// fix the walk ran until the fake's budget cut it off 300 requests later.+func TestListBookDocs_StopsAtMetaTotal(t *testing.T) {+ resp := v2DocListResponse{Data: fullDocPage(1, defaultPageSize)}+ resp.Meta.Total = 5 // far below what this page alone hands out+ server, requests := alwaysFullPageServer(t, resp)++ c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})+ docs, err := c.ListBookDocs(context.Background(), 4242)++ require.NoError(t, err, "meta.total covered by the first page must end the walk")+ require.Len(t, docs, defaultPageSize)+ require.EqualValues(t, 1, requests.Load(),+ "the walk must stop on the page that reaches meta.total")+}++// A full page whose first id repeats the previous page means the offset walk is+// not making progress. Each endpoint must report that instead of appending the+// same page forever.+func TestPaginationStopsOnRepeatedPage(t *testing.T) {+ for _, tc := range []struct {+ name string+ path string+ call func(context.Context, *client) (int, error)+ }{+ {+ name: "book docs without meta.total",+ path: "/api/v2/repos/4242/docs",+ call: func(ctx context.Context, c *client) (int, error) {+ docs, err := c.ListBookDocs(ctx, 4242)+ return len(docs), err+ },+ },+ {+ name: "user repos",+ path: "/api/v2/users/alice/repos",+ call: func(ctx context.Context, c *client) (int, error) {+ repos, err := c.ListUserRepos(ctx, "alice")+ return len(repos), err+ },+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ var body interface{}+ if strings.HasSuffix(tc.path, "/docs") {+ body = v2DocListResponse{Data: fullDocPage(1, defaultPageSize)}+ } else {+ body = v2RepoListResponse{Data: fullRepoPage(1, defaultPageSize)}+ }+ server, requests := alwaysFullPageServer(t, body)++ c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})+ n, err := tc.call(context.Background(), c)++ require.Error(t, err, "a repeated full page must not be followed forever")+ require.Contains(t, err.Error(), "repeated page")+ require.Zero(t, n, "a stalled walk must not hand back a partial listing")+ require.LessOrEqual(t, requests.Load(), int64(2),+ "the guard must reject the repeated page on the second request")+ })+ }+}++// A server that never repeats an id but also never returns a short page can only+// be stopped by the hop cap. It must be reported, and the log must carry it too:+// visually this failure looks like a healthy, still-running sync.+func TestPaginationHopCapStopsWalk(t *testing.T) {+ for _, tc := range []struct {+ name string+ call func(context.Context, *client) error+ }{+ {+ name: "book docs",+ call: func(ctx context.Context, c *client) error {+ _, err := c.ListBookDocs(ctx, 4242)+ return err+ },+ },+ {+ name: "user repos",+ call: func(ctx context.Context, c *client) error {+ _, err := c.ListUserRepos(ctx, "alice")+ return err+ },+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ n := requests.Add(1)+ if n > paginationHopCap+10 {+ http.Error(w, "hop cap missing: too many requests", http.StatusBadRequest)+ return+ }+ offset, err := strconv.Atoi(r.URL.Query().Get("offset"))+ if err != nil {+ http.Error(w, "unexpected offset", http.StatusBadRequest)+ return+ }+ // Fresh ids on every page and no meta.total, so only the hop cap+ // can end the walk.+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(minimalPage("id", int64(offset)+1)))+ }))+ t.Cleanup(server.Close)++ c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})++ var logs bytes.Buffer+ logger.SetOutput(&logs)+ t.Cleanup(logger.ConfigureFromEnv)++ err := tc.call(context.Background(), c)++ require.ErrorContains(t, err, "pagination exceeded 10000 pages")+ require.EqualValues(t, paginationHopCap, requests.Load())+ require.Contains(t, logs.String(), "pagination exceeded 10000 pages",+ "hitting the hop cap must leave a warning in the log")+ })+ }+}++// pagedServer serves `total` items in defaultPageSize-sized offset pages, with+// the docs response advertising meta.total. It is the well-behaved listing the+// guards must not disturb.+func pagedServer(t *testing.T, kind string, total int) (*httptest.Server, *atomic.Int64) {+ t.Helper()+ var requests atomic.Int64+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ offset, err := strconv.Atoi(r.URL.Query().Get("offset"))+ if err != nil {+ http.Error(w, "unexpected offset", http.StatusBadRequest)+ return+ }+ size := total - offset+ if size > defaultPageSize {+ size = defaultPageSize+ }+ if size < 0 {+ size = 0+ }+ w.Header().Set("Content-Type", "application/json")+ if kind == "docs" {+ resp := v2DocListResponse{Data: fullDocPage(int64(offset)+1, size)}+ resp.Meta.Total = total+ _ = json.NewEncoder(w).Encode(resp)+ return+ }+ _ = json.NewEncoder(w).Encode(v2RepoListResponse{Data: fullRepoPage(int64(offset)+1, size)})+ }))+ t.Cleanup(server.Close)+ return server, &requests+}++// The guards must not truncate a listing that genuinely spans several pages.+func TestPaginationCollectsRealPages(t *testing.T) {+ for _, tc := range []struct {+ name string+ kind string+ total int+ wantReqs int64+ call func(context.Context, *client) (int, error)+ }{+ {+ name: "docs reaching meta.total exactly",+ kind: "docs",+ total: 2 * defaultPageSize,+ wantReqs: 2,+ call: func(ctx context.Context, c *client) (int, error) {+ docs, err := c.ListBookDocs(ctx, 4242)+ return len(docs), err+ },+ },+ {+ name: "docs with a short final page",+ kind: "docs",+ total: defaultPageSize + 30,+ wantReqs: 2,+ call: func(ctx context.Context, c *client) (int, error) {+ docs, err := c.ListBookDocs(ctx, 4242)+ return len(docs), err+ },+ },+ {+ name: "repos with a short final page",+ kind: "repos",+ total: defaultPageSize + 30,+ wantReqs: 2,+ call: func(ctx context.Context, c *client) (int, error) {+ repos, err := c.ListUserRepos(ctx, "alice")+ return len(repos), err+ },+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ server, requests := pagedServer(t, tc.kind, tc.total)++ c := newClient(&Config{APIToken: "tok", BaseURL: server.URL})+ got, err := tc.call(context.Background(), c)++ require.NoError(t, err)+ require.Equal(t, tc.total, got, "every item of a real multi-page listing must survive")+ require.EqualValues(t, tc.wantReqs, requests.Load())+ })+ }+}