Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

CLI dataset document pagination is sent as HTTP headers (#20493)

Taranum Wasu1dc6f3ea638b
diff --git a/internal/cli/filesystem/dataset.go b/internal/cli/filesystem/dataset.go
index e68157775c..c8050c2ccb 100644
--- a/internal/cli/filesystem/dataset.go
+++ b/internal/cli/filesystem/dataset.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "net/url"
"ragflow/internal/cli/utils"
"strconv"
"strings"
@@ -499,19 +500,21 @@ func (p *DatasetProvider) listDocuments(ctx stdctx.Context, datasetName string,
return nil, fmt.Errorf("dataset ID not found")
}
- // Build query parameters
- params := make(map[string]string)
+ params := url.Values{}
if opts != nil {
- if opts.Limit > 0 {
- params["page_size"] = fmt.Sprintf("%d", opts.Limit)
- }
- if opts.Offset > 0 {
- params["page"] = fmt.Sprintf("%d", opts.Offset/opts.Limit+1)
+ pageSize := opts.Limit
+ if pageSize <= 0 {
+ pageSize = 100
}
+ params.Set("page_size", strconv.Itoa(pageSize))
+ params.Set("page", strconv.Itoa(max(opts.Offset, 0)/pageSize+1))
}
path := utils.APIPath("/datasets", datasetID, "documents")
- resp, err := p.httpClient.Request("GET", path, "auto", params, nil)
+ if len(params) > 0 {
+ path += "?" + params.Encode()
+ }
+ resp, err := p.httpClient.Request("GET", path, "auto", nil, nil)
if err != nil {
return nil, err
}
diff --git a/internal/cli/filesystem/r5_regression_test.go b/internal/cli/filesystem/r5_regression_test.go
new file mode 100644
index 0000000000..ea46f7fed1
--- /dev/null
+++ b/internal/cli/filesystem/r5_regression_test.go
@@ -0,0 +1,47 @@
+package filesystem
+
+import (
+ stdctx "context"
+ "io"
+ "net/url"
+ "testing"
+)
+
+type datasetClient struct {
+ paths []string
+ headers []map[string]string
+}
+
+func (f *datasetClient) Request(method, path, authKind string, headers map[string]string, body map[string]interface{}) (*HTTPResponse, error) {
+ f.paths = append(f.paths, path)
+ f.headers = append(f.headers, headers)
+ response := `{"code":0,"data":{"docs":[]}}`
+ if path == "/datasets" {
+ response = `{"code":0,"data":[{"id":"kb1","name":"test"}]}`
+ }
+ return &HTTPResponse{StatusCode: 200, Body: []byte(response)}, nil
+}
+func (f *datasetClient) UploadMultipart(path, contentType string, body io.Reader) error { return nil }
+func TestDatasetDocumentPaginationUsesQueryNotHeaders(t *testing.T) {
+ client := &datasetClient{}
+ p := NewDatasetProvider(client)
+ _, err := p.listDocuments(stdctx.Background(), "test", &ListOptions{Limit: 10, Offset: 20})
+ if err != nil {
+ t.Fatal(err)
+ }
+ u, _ := url.Parse(client.paths[len(client.paths)-1])
+ if u.Query().Get("page") != "3" || u.Query().Get("page_size") != "10" {
+ t.Fatalf("pagination missing from URL: %s", u)
+ }
+ if len(client.headers[len(client.headers)-1]) != 0 {
+ t.Fatal("pagination must not be sent as HTTP headers")
+ }
+}
+func TestDatasetDocumentOffsetWithoutLimitDoesNotPanic(t *testing.T) {
+ client := &datasetClient{}
+ p := NewDatasetProvider(client)
+ _, err := p.listDocuments(stdctx.Background(), "test", &ListOptions{Offset: 100})
+ if err != nil {
+ t.Fatal(err)
+ }
+}