Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(sdk-dataset): fix dataset validation (#21734)

Sbaaoui Idrissd221bf593920
diff --git a/internal/common/app_name.go b/internal/common/app_name.go
index 7d7a70f18c..f90eb5f039 100644
--- a/internal/common/app_name.go
+++ b/internal/common/app_name.go
@@ -22,6 +22,7 @@ import (
"regexp"
"strconv"
"strings"
+ "unicode/utf8"
)
// maxNameRetries bounds how many numbered suffixes UniqueName may try before
@@ -64,27 +65,34 @@ func splitNameCounter(stem string) (string, int, bool) {
// exists reports whether a candidate already collides. A lookup error is
// returned to the caller instead of being treated as "free".
func UniqueName(name string, maxLen int, exists func(candidate string) (bool, error)) (string, error) {
- return uniqueName(name, maxLen, false, exists)
+ return uniqueName(name, maxLen, false, exists, lenBytes)
}
+// UniqueNameCharacters applies a Unicode character limit when choosing a name.
+func UniqueNameCharacters(name string, maxLen int, exists func(string) (bool, error)) (string, error) {
+ return uniqueName(name, maxLen, false, exists, utf8.RuneCountInString)
+}
+
+func lenBytes(name string) int { return len(name) }
+
// UniqueFileName is UniqueName for file and document names: the suffix is
// inserted before the file extension instead of at the end of the name.
//
// UniqueFileName("report.pdf", 0, ...) -> "report(1).pdf"
// UniqueFileName("report(1).pdf", 0, ...) -> "report(2).pdf"
func UniqueFileName(name string, maxLen int, exists func(candidate string) (bool, error)) (string, error) {
- return uniqueName(name, maxLen, true, exists)
+ return uniqueName(name, maxLen, true, exists, lenBytes)
}
// uniqueName appends an incrementing "(N)" suffix until exists reports the
// candidate as free. keepExtension inserts the suffix before the file
// extension instead of at the end of the name.
-func uniqueName(name string, maxLen int, keepExtension bool, exists func(candidate string) (bool, error)) (string, error) {
+func uniqueName(name string, maxLen int, keepExtension bool, exists func(candidate string) (bool, error), length func(string) int) (string, error) {
if maxLen <= 0 {
maxLen = AppNameLimit
}
current := name
- if err := validateNameLimit(current, maxLen); err != nil {
+ if err := validateNameLength(current, maxLen, length(current)); err != nil {
return "", err
}
for i := 0; i < maxNameRetries; i++ {
@@ -107,7 +115,7 @@ func uniqueName(name string, maxLen int, keepExtension bool, exists func(candida
base = stem
}
current = fmt.Sprintf("%s(%d)%s", base, counter+1, ext)
- if err := validateNameLimit(current, maxLen); err != nil {
+ if err := validateNameLength(current, maxLen, length(current)); err != nil {
return "", err
}
}
@@ -144,15 +152,18 @@ func ValidateName(name string) error {
// validateNameLimit rejects empty names and names longer than maxLen bytes.
func validateNameLimit(name string, maxLen int) error {
+ return validateNameLength(name, maxLen, len(name))
+}
+
+func validateNameLength(name string, maxLen, length int) error {
// Validate name is not empty after trimming
trimmedName := strings.TrimSpace(name)
if trimmedName == "" {
return fmt.Errorf("name cannot be empty or whitespace")
}
- // Validate name length in bytes (not characters) - same as Python len(search_name.encode("utf-8"))
- if len([]byte(name)) > maxLen {
- return fmt.Errorf("name length is %d which is large than %d", len([]byte(name)), maxLen)
+ if length > maxLen {
+ return fmt.Errorf("name length is %d which is large than %d", length, maxLen)
}
return nil
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index d3f32d486f..c9096e2f3c 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"time"
+ "unicode/utf8"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -63,8 +64,8 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
if name == "" {
return nil, common.CodeDataError, errors.New("dataset name can't be empty")
}
- if len(name) > entity.DatasetNameLimit {
- return nil, common.CodeDataError, fmt.Errorf("Dataset name length is %d which is large than %d", len(name), entity.DatasetNameLimit)
+ if utf8.RuneCountInString(name) > entity.DatasetNameLimit {
+ return nil, common.CodeDataError, fmt.Errorf("Dataset name length is %d which is large than %d", utf8.RuneCountInString(name), entity.DatasetNameLimit)
}
tenant, err := d.tenantDAO.GetByID(ctx, dao.DB, tenantID)
@@ -307,7 +308,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
// dedupeDatasetName returns a non-colliding dataset name within the tenant,
// appending (1), (2), ... when the requested name is already taken.
func (d *DatasetService) dedupeDatasetName(ctx context.Context, name, tenantID string) (string, error) {
- return common.UniqueName(name, entity.DatasetNameLimit, func(candidate string) (bool, error) {
+ return common.UniqueNameCharacters(name, entity.DatasetNameLimit, func(candidate string) (bool, error) {
return d.kbDAO.NameExists(ctx, dao.DB, tenantID, candidate)
})
}
diff --git a/internal/service/dataset/sdk_contract_test.go b/internal/service/dataset/sdk_contract_test.go
new file mode 100644
index 0000000000..6e7f688cdf
--- /dev/null
+++ b/internal/service/dataset/sdk_contract_test.go
@@ -0,0 +1,71 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dataset
+
+import (
+ "ragflow/internal/common"
+ "ragflow/internal/service"
+ "strings"
+ "testing"
+)
+
+func TestSDKDatasetUnicodeCharacterLimit(t *testing.T) {
+ for _, n := range []int{128, 129} {
+ t.Run(strings.Repeat("数", n), func(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCreateDatasetTenant(t, "tenant-1")
+ name := strings.Repeat("数", n)
+ _, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{Name: name, ParseType: new(1), ParserID: new("general")}, testDatasetSubject("tenant-1"))
+ if (err == nil) != (n == 128) {
+ t.Fatalf("create characters=%d code=%d err=%v", n, code, err)
+ }
+ })
+ t.Run("update", func(t *testing.T) {
+ db := setupDatasetUpdateTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
+ ctx := withDatasetTestContext(t.Context(), "tenant-1", "tenant-1")
+ _, code, err := testDatasetUpdateService(t).UpdateDataset(ctx, "kb-1", "tenant-1", service.UpdateDatasetRequest{Name: new(strings.Repeat("资", n))})
+ if (err == nil) != (n == 128) {
+ t.Fatalf("update characters=%d code=%d err=%v", n, code, err)
+ }
+ })
+ }
+}
+func TestSDKDatasetNegativePagerankDiagnostic(t *testing.T) {
+ db := setupDatasetUpdateTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
+ _, code, err := testDatasetUpdateService(t).UpdateDataset(withDatasetTestContext(t.Context(), "tenant-1", "tenant-1"), "kb-1", "tenant-1", service.UpdateDatasetRequest{Pagerank: new(int64(-1))})
+ if code != common.CodeDataError || err == nil || !strings.Contains(err.Error(), "greater than or equal to 0") {
+ t.Fatalf("code=%d err=%v", code, err)
+ }
+}
+
+func TestSDKDatasetUnicodeDeduplicationLimit(t *testing.T) {
+ for _, n := range []int{125, 128} {
+ name := strings.Repeat("数", n)
+ got, err := common.UniqueNameCharacters(name, 128, func(candidate string) (bool, error) { return candidate == name, nil })
+ if n == 125 {
+ if err != nil || got != name+"(1)" {
+ t.Fatalf("name=%q err=%v", got, err)
+ }
+ } else if err == nil {
+ t.Fatal("deduplicated name exceeded character limit")
+ }
+ }
+}
diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.go
index 4d7f47932e..7a5486e6c6 100644
--- a/internal/service/dataset/update.go
+++ b/internal/service/dataset/update.go
@@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"time"
+ "unicode/utf8"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -68,7 +69,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, userID st
if name == "" {
return nil, common.CodeDataError, errors.New("String should have at least 1 character")
}
- if len(name) > 128 {
+ if utf8.RuneCountInString(name) > 128 {
return nil, common.CodeDataError, errors.New("String should have at most 128 characters")
}
simpleUpdates["name"] = name
@@ -160,7 +161,10 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, userID st
pagerankRequested := req.Pagerank != nil
if pagerankRequested {
requestedPagerank = *req.Pagerank
- if *req.Pagerank < 0 || *req.Pagerank > 100 {
+ if *req.Pagerank < 0 {
+ return nil, common.CodeDataError, errors.New("input should be greater than or equal to 0")
+ }
+ if *req.Pagerank > 100 {
return nil, common.CodeDataError, errors.New("input should be less than or equal to 100")
}
if d.docEngine == nil {
diff --git a/sdk/python/ragflow_sdk/ragflow.py b/sdk/python/ragflow_sdk/ragflow.py
index f27feec3aa..fac148863d 100644
--- a/sdk/python/ragflow_sdk/ragflow.py
+++ b/sdk/python/ragflow_sdk/ragflow.py
@@ -1,5 +1,5 @@
#
-# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -63,12 +63,14 @@ class RAGFlow:
permission: str = "me",
parser_config: Optional[DataSet.ParserConfig] = None,
language: Optional[str] = None,
+ *,
+ pipeline_version_id: str | None = None,
) -> DataSet:
"""Create a dataset with a builtin parser or an ingestion pipeline.
``parse_type=1`` selects a builtin parser and defaults ``parser_id`` to
``"general"``. ``parse_type=2`` selects an ingestion pipeline and requires
- ``pipeline_id``.
+ ``pipeline_id`` and a pinned ``pipeline_version_id``.
"""
if parser_id is not None and not isinstance(parser_id, str):
raise ValueError("parser_id must be a string")
@@ -105,6 +107,9 @@ class RAGFlow:
if language is not None:
payload["language"] = language
+ if pipeline_version_id is not None:
+ payload["pipeline_version_id"] = pipeline_version_id
+
res = self.post("/datasets", payload)
res = res.json()
if res.get("code") == 0:
diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
index 1776174767..aaec0297c0 100644
--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
@@ -39,7 +39,6 @@ def test_name(dataset_factory, client, name):
assert persisted[0].name == name
[email protected](reason="internal/service/dataset/crud.go CreateDataset uses len(name) bytes despite the documented 128-character limit")
def test_name_unicode_limit(dataset_factory):
name = "数" * 128
assert dataset_factory(name=name).name == name
@@ -194,12 +193,13 @@ def test_builtin_parser_default(dataset_factory):
assert dataset.pipeline_id is None
[email protected](reason="sdk/python/ragflow_sdk/ragflow.py create_dataset cannot send pipeline_version_id required by internal/service/dataset/crud.go CreateDataset (empty_version_test.go)")
def test_pipeline_parser(dataset_factory, client, integration_settings):
pipeline_id = integration_settings.get("pipeline_id")
if not pipeline_id:
pytest.fail("RAGFLOW_SDK_PIPELINE_ID must identify an accessible versioned ingestion pipeline")
- dataset = dataset_factory(parse_type=2, pipeline_id=pipeline_id)
+ version_id = integration_settings.get("pipeline_version_id")
+ assert version_id, "RAGFLOW_SDK_PIPELINE_VERSION_ID must pin an accessible ingestion pipeline version"
+ dataset = dataset_factory(parse_type=2, pipeline_id=pipeline_id, pipeline_version_id=version_id)
assert dataset.pipeline_id == pipeline_id
assert dataset.parse_type == 2
persisted = client.list_datasets(id=dataset.id)[0]
diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
index f967cde3d6..7c03558942 100644
--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
@@ -49,7 +49,6 @@ class TestDatasetUpdate:
assert dataset.name == name.strip()
assert client.list_datasets(id=dataset.id)[0].name == name.strip()
- @pytest.mark.skip(reason="Go internal/service/dataset/update.go uses len(name) bytes, rejecting documented 128-character BMP names")
def test_name_unicode_character_limit(self, client, add_dataset_func):
name = "资" * 128
add_dataset_func.update({"name": name})
@@ -291,7 +290,6 @@ class TestDatasetUpdate:
pytest.param(
-1,
"greater than or equal to 0",
- marks=pytest.mark.skip(reason="internal/service/dataset/update.go reports the upper bound for negative pagerank; preserve the correct lower-bound diagnostic"),
),
(101, "less than or equal to 100"),
(3.14, "cannot unmarshal"),