Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(gitlab): 分页循环补重复页防护与跳数上限 (#3841)

linus-liu-weba2aeb97a0120
diff --git a/internal/datasource/connector/gitlab/client.go b/internal/datasource/connector/gitlab/client.go
index 5a26de625..3ebefb947 100644
--- a/internal/datasource/connector/gitlab/client.go
+++ b/internal/datasource/connector/gitlab/client.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/datasource"
+ "github.com/Tencent/WeKnora/internal/logger"
)
const (
@@ -39,6 +40,11 @@ func base64FileBytes(rawLimit int64) int64 {
return rawLimit/3*4 + (1 << 20)
}
+// maxPaginationHops bounds a single paginated walk. GitLab reports the next
+// page through the X-Next-Page header, so a broken instance or reverse proxy
+// could keep advertising pages indefinitely.
+const maxPaginationHops = 10000
+
type client struct {
baseURL, token string
http *http.Client
@@ -206,13 +212,33 @@ func (c *client) project(ctx context.Context, id string) (*project, error) {
return &p, err
}
+// trackPage records the page a paginated walk is about to request. A page that
+// was already requested means X-Next-Page is not advancing, so the walk stops
+// with an error instead of re-reading the same page; the hop cap bounds a
+// server that keeps advertising fresh page numbers.
+func trackPage(ctx context.Context, scope string, seen map[string]struct{}, page string) error {
+ if _, dup := seen[page]; dup {
+ return fmt.Errorf("gitlab %s pagination repeated page %q", scope, page)
+ }
+ if len(seen) >= maxPaginationHops {
+ logger.Warnf(ctx, "[GitLab] %s pagination exceeded %d pages; aborting", scope, maxPaginationHops)
+ return fmt.Errorf("gitlab %s pagination exceeded %d pages", scope, maxPaginationHops)
+ }
+ seen[page] = struct{}{}
+ return nil
+}
+
func (c *client) projects(ctx context.Context) ([]project, error) {
q := url.Values{
"membership": {"true"}, "per_page": {"100"}, "page": {"1"},
"order_by": {"path_with_namespace"}, "sort": {"asc"},
}
var all []project
+ seen := make(map[string]struct{})
for {
+ if err := trackPage(ctx, "project", seen, q.Get("page")); err != nil {
+ return nil, err
+ }
var page []project
nextPage, err := c.getPage(ctx, "/projects", q, &page)
if err != nil {
@@ -251,7 +277,11 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er
}
endpoint := "/projects/" + projectPath(id) + "/repository/tree"
var all []treeEntry
+ seen := make(map[string]struct{})
for {
+ if err := trackPage(ctx, "tree", seen, q.Get("page")); err != nil {
+ return nil, err
+ }
var page []treeEntry
nextPage, err := c.getPage(ctx, endpoint, q, &page)
if err != nil {
diff --git a/internal/datasource/connector/gitlab/pagination_guard_test.go b/internal/datasource/connector/gitlab/pagination_guard_test.go
new file mode 100644
index 000000000..651e95711
--- /dev/null
+++ b/internal/datasource/connector/gitlab/pagination_guard_test.go
@@ -0,0 +1,146 @@
+package gitlab
+
+import (
+ "bytes"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/stretchr/testify/require"
+)
+
+// guardTestTimeout bounds every walk here so that a missing guard fails the
+// test instead of hanging it.
+const guardTestTimeout = 5 * time.Second
+
+// repeatedPageServer always advertises the same X-Next-Page value and refuses
+// to serve more than three requests: a walk without a repeated-page guard then
+// fails quickly with an unexpected error instead of spinning until the context
+// deadline.
+func repeatedPageServer(t *testing.T) (*httptest.Server, *int64) {
+ t.Helper()
+ allowLocalGitLabServer(t)
+ var requests int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if atomic.AddInt64(&requests, 1) > 3 {
+ http.Error(w, "pagination guard missing: too many requests", http.StatusTooManyRequests)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Next-Page", "2")
+ if strings.HasSuffix(r.URL.Path, "/repository/tree") {
+ _, _ = w.Write([]byte(`[{"name":"one.md","type":"blob","path":"one.md"}]`))
+ return
+ }
+ _, _ = w.Write([]byte(`[{"id":1,"path_with_namespace":"group/project-001"}]`))
+ }))
+ t.Cleanup(server.Close)
+ return server, &requests
+}
+
+func TestProjectsStopsOnRepeatedNextPage(t *testing.T) {
+ server, requests := repeatedPageServer(t)
+ c, err := newClient(server.URL, "test-token")
+ require.NoError(t, err)
+ ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout)
+ defer cancel()
+
+ projects, err := c.projects(ctx)
+
+ require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list")
+ require.Nil(t, projects)
+ require.Contains(t, err.Error(), `gitlab project pagination repeated page "2"`)
+ require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page")
+}
+
+func TestTreeStopsOnRepeatedNextPage(t *testing.T) {
+ server, requests := repeatedPageServer(t)
+ c, err := newClient(server.URL, "test-token")
+ require.NoError(t, err)
+ ctx, cancel := context.WithTimeout(context.Background(), guardTestTimeout)
+ defer cancel()
+
+ entries, err := c.tree(ctx, "1", "main", "")
+
+ require.Error(t, err, "a repeated X-Next-Page must not be treated as the end of the list")
+ require.Nil(t, entries)
+ require.Contains(t, err.Error(), `gitlab tree pagination repeated page "2"`)
+ require.LessOrEqual(t, atomic.LoadInt64(requests), int64(3), "must stop before re-reading the repeated page")
+}
+
+// TestPaginationHopCapStopsWalk drives each walk with a server that always
+// advertises a fresh page number, so only the hop cap can end it.
+func TestPaginationHopCapStopsWalk(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ want string
+ walk func(context.Context, *client) error
+ }{
+ {
+ name: "projects",
+ want: "gitlab project pagination exceeded 10000 pages",
+ walk: func(ctx context.Context, c *client) error {
+ _, err := c.projects(ctx)
+ return err
+ },
+ },
+ {
+ name: "tree",
+ want: "gitlab tree pagination exceeded 10000 pages",
+ walk: func(ctx context.Context, c *client) error {
+ _, err := c.tree(ctx, "1", "main", "")
+ return err
+ },
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ allowLocalGitLabServer(t)
+ var requests int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ n := atomic.AddInt64(&requests, 1)
+ if n > maxPaginationHops+10 {
+ http.Error(w, "hop cap missing: too many requests", http.StatusTooManyRequests)
+ return
+ }
+ page, convErr := strconv.Atoi(r.URL.Query().Get("page"))
+ if convErr != nil {
+ http.Error(w, "unexpected page", http.StatusBadRequest)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Next-Page", strconv.Itoa(page+1))
+ _, _ = w.Write([]byte(`[]`))
+ }))
+ t.Cleanup(server.Close)
+ c, err := newClient(server.URL, "test-token")
+ require.NoError(t, err)
+
+ var logs bytes.Buffer
+ logger.SetOutput(&logs)
+ t.Cleanup(logger.ConfigureFromEnv)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ err = tc.walk(ctx, c)
+
+ require.EqualError(t, err, tc.want)
+ require.Equal(t, int64(maxPaginationHops), atomic.LoadInt64(&requests))
+ require.Contains(t, logs.String(), "pagination exceeded 10000 pages",
+ "hitting the cap must leave a warning in the log")
+ })
+ }
+}
+
+func TestTrackPageAllowsForwardPages(t *testing.T) {
+ seen := make(map[string]struct{})
+ for _, page := range []string{"1", "2", "3"} {
+ require.NoError(t, trackPage(context.Background(), "project", seen, page))
+ }
+ require.Error(t, trackPage(context.Background(), "project", seen, "2"))
+}