Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(gitlab): 补齐项目资源列表分页

yeliheng-010608027179fb0
diff --git a/internal/datasource/connector/gitlab/client.go b/internal/datasource/connector/gitlab/client.go
index 710d51f89..53c38692b 100644
--- a/internal/datasource/connector/gitlab/client.go
+++ b/internal/datasource/connector/gitlab/client.go
@@ -142,9 +142,23 @@ func (c *client) project(ctx context.Context, id string) (*project, error) {
return &p, err
}
func (c *client) projects(ctx context.Context) ([]project, error) {
- var p []project
- err := c.get(ctx, "/projects?membership=true&per_page=100&order_by=path_with_namespace&sort=asc", &p)
- return p, err
+ q := url.Values{
+ "membership": {"true"}, "per_page": {"100"}, "page": {"1"},
+ "order_by": {"path_with_namespace"}, "sort": {"asc"},
+ }
+ var all []project
+ for {
+ var page []project
+ nextPage, err := c.getPage(ctx, "/projects", q, &page)
+ if err != nil {
+ return nil, err
+ }
+ all = append(all, page...)
+ if nextPage == "" {
+ return all, nil
+ }
+ q.Set("page", nextPage)
+ }
}
// ping verifies that the supplied private token is accepted by this GitLab
@@ -172,7 +186,7 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er
var all []treeEntry
for {
var page []treeEntry
- nextPage, err := c.getTreePage(ctx, endpoint, q, &page)
+ nextPage, err := c.getPage(ctx, endpoint, q, &page)
if err != nil {
return nil, err
}
@@ -184,7 +198,7 @@ func (c *client) tree(ctx context.Context, id, ref, dir string) ([]treeEntry, er
}
}
-func (c *client) getTreePage(ctx context.Context, endpoint string, query url.Values, out interface{}) (string, error) {
+func (c *client) getPage(ctx context.Context, endpoint string, query url.Values, out interface{}) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+endpoint+"?"+query.Encode(), nil)
if err != nil {
return "", err
diff --git a/internal/datasource/connector/gitlab/projects_test.go b/internal/datasource/connector/gitlab/projects_test.go
new file mode 100644
index 000000000..383f42910
--- /dev/null
+++ b/internal/datasource/connector/gitlab/projects_test.go
@@ -0,0 +1,148 @@
+package gitlab
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestListResourcesIncludesProjectsAfterFirstPage(t *testing.T) {
+ const pageSize = 100
+ const projectCount = 2*pageSize + 1
+ server, requests := projectListServer(t, map[string]projectListPage{
+ "1": {body: projectListJSON(t, 1, pageSize), next: "2"},
+ "2": {body: projectListJSON(t, pageSize+1, pageSize), next: "3"},
+ "3": {body: projectListJSON(t, projectCount, 1)},
+ })
+ ds := &types.DataSourceConfig{
+ Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "test-token"},
+ Settings: map[string]interface{}{"projects": []interface{}{
+ map[string]interface{}{"project_id": "1"},
+ }},
+ }
+
+ resources, err := NewConnector().ListResources(context.Background(), ds, "")
+ require.NoError(t, err)
+ require.Equal(t, projectCount, len(resources), "every GitLab project must be available for selection")
+ for i, resource := range resources {
+ require.Equal(t, fmt.Sprint(i+1), resource.ExternalID)
+ require.Equal(t, fmt.Sprintf("group/project-%03d", i+1), resource.Name)
+ require.Equal(t, fmt.Sprintf("https://gitlab.example.com/group/project-%03d", i+1), resource.URL)
+ require.Equal(t, "project", resource.Type)
+ require.True(t, resource.HasChildren)
+ }
+ require.Equal(t, []string{"1", "2", "3"}, recordedProjectPages(requests))
+}
+
+func TestProjectsStopsAtFinalPage(t *testing.T) {
+ const pageSize = 100
+ for _, count := range []int{0, pageSize} {
+ t.Run(fmt.Sprintf("%d projects", count), func(t *testing.T) {
+ server, requests := projectListServer(t, map[string]projectListPage{
+ "1": {body: projectListJSON(t, 1, count)},
+ })
+ c, err := newClient(server.URL, "test-token")
+ require.NoError(t, err)
+ projects, err := c.projects(context.Background())
+ require.NoError(t, err)
+ require.Len(t, projects, count)
+ require.Equal(t, []string{"1"}, recordedProjectPages(requests))
+ })
+ }
+}
+
+func TestProjectsRejectsIncompleteList(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ page projectListPage
+ }{
+ {name: "HTTP error", page: projectListPage{status: http.StatusServiceUnavailable}},
+ {name: "malformed JSON", page: projectListPage{body: "not JSON"}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ server, requests := projectListServer(t, map[string]projectListPage{
+ "1": {body: projectListJSON(t, 1, 1), next: "2"},
+ "2": tc.page,
+ })
+ c, err := newClient(server.URL, "test-token")
+ require.NoError(t, err)
+ projects, err := c.projects(context.Background())
+ require.Error(t, err)
+ require.Nil(t, projects, "do not return a partial list as a successful result")
+ require.Equal(t, []string{"1", "2"}, recordedProjectPages(requests))
+ })
+ }
+}
+
+type projectListPage struct {
+ body string
+ next string
+ status int
+}
+
+func projectListServer(t *testing.T, pages map[string]projectListPage) (*httptest.Server, <-chan string) {
+ t.Helper()
+ allowLocalGitLabServer(t)
+ requests := make(chan string, len(pages)+1)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ query := r.URL.Query()
+ if r.Method != http.MethodGet || r.URL.Path != "/api/v4/projects" ||
+ r.Header.Get("PRIVATE-TOKEN") != "test-token" || query.Get("membership") != "true" ||
+ query.Get("per_page") != "100" || query.Get("order_by") != "path_with_namespace" ||
+ query.Get("sort") != "asc" {
+ http.Error(w, "unexpected project-list request", http.StatusBadRequest)
+ return
+ }
+ page := query.Get("page")
+ if page == "" {
+ page = "1" // GitLab defaults to its first page when the parameter is omitted.
+ }
+ response, ok := pages[page]
+ if !ok {
+ http.Error(w, "unexpected page", http.StatusBadRequest)
+ return
+ }
+ select {
+ case requests <- page:
+ default:
+ http.Error(w, "too many page requests", http.StatusBadRequest)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Next-Page", response.next)
+ if response.status != 0 {
+ w.WriteHeader(response.status)
+ }
+ _, _ = fmt.Fprint(w, response.body)
+ }))
+ t.Cleanup(server.Close)
+ return server, requests
+}
+
+func projectListJSON(t *testing.T, start, count int) string {
+ t.Helper()
+ projects := make([]project, 0, count)
+ for id := start; id < start+count; id++ {
+ projects = append(projects, project{
+ ID: int64(id), PathWithNamespace: fmt.Sprintf("group/project-%03d", id),
+ WebURL: fmt.Sprintf("https://gitlab.example.com/group/project-%03d", id),
+ })
+ }
+ data, err := json.Marshal(projects)
+ require.NoError(t, err)
+ return string(data)
+}
+
+func recordedProjectPages(requests <-chan string) []string {
+ var pages []string
+ for len(requests) > 0 {
+ pages = append(pages, <-requests)
+ }
+ return pages
+}