RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(sdk-dataset): fix dataset validation (#21734)
Sbaaoui Idrissd221bf593920
diff --git a/internal/common/app_name.go b/internal/common/app_name.goindex 7d7a70f18c..f90eb5f039 100644--- a/internal/common/app_name.go+++ b/internal/common/app_name.go@@ -22,6 +22,7 @@ import ("regexp""strconv""strings"+ "unicode/utf8")// maxNameRetries bounds how many numbered suffixes UniqueName may try before@@ -64,27 +65,34 @@ func splitNameCounter(stem string) (string, int, bool) {// exists reports whether a candidate already collides. A lookup error is// returned to the caller instead of being treated as "free".func UniqueName(name string, maxLen int, exists func(candidate string) (bool, error)) (string, error) {- return uniqueName(name, maxLen, false, exists)+ return uniqueName(name, maxLen, false, exists, lenBytes)}+// UniqueNameCharacters applies a Unicode character limit when choosing a name.+func UniqueNameCharacters(name string, maxLen int, exists func(string) (bool, error)) (string, error) {+ return uniqueName(name, maxLen, false, exists, utf8.RuneCountInString)+}++func lenBytes(name string) int { return len(name) }+// UniqueFileName is UniqueName for file and document names: the suffix is// inserted before the file extension instead of at the end of the name.//// UniqueFileName("report.pdf", 0, ...) -> "report(1).pdf"// UniqueFileName("report(1).pdf", 0, ...) -> "report(2).pdf"func UniqueFileName(name string, maxLen int, exists func(candidate string) (bool, error)) (string, error) {- return uniqueName(name, maxLen, true, exists)+ return uniqueName(name, maxLen, true, exists, lenBytes)}// uniqueName appends an incrementing "(N)" suffix until exists reports the// candidate as free. keepExtension inserts the suffix before the file// extension instead of at the end of the name.-func uniqueName(name string, maxLen int, keepExtension bool, exists func(candidate string) (bool, error)) (string, error) {+func uniqueName(name string, maxLen int, keepExtension bool, exists func(candidate string) (bool, error), length func(string) int) (string, error) {if maxLen <= 0 {maxLen = AppNameLimit}current := name- if err := validateNameLimit(current, maxLen); err != nil {+ if err := validateNameLength(current, maxLen, length(current)); err != nil {return "", err}for i := 0; i < maxNameRetries; i++ {@@ -107,7 +115,7 @@ func uniqueName(name string, maxLen int, keepExtension bool, exists func(candidabase = stem}current = fmt.Sprintf("%s(%d)%s", base, counter+1, ext)- if err := validateNameLimit(current, maxLen); err != nil {+ if err := validateNameLength(current, maxLen, length(current)); err != nil {return "", err}}@@ -144,15 +152,18 @@ func ValidateName(name string) error {// validateNameLimit rejects empty names and names longer than maxLen bytes.func validateNameLimit(name string, maxLen int) error {+ return validateNameLength(name, maxLen, len(name))+}++func validateNameLength(name string, maxLen, length int) error {// Validate name is not empty after trimmingtrimmedName := strings.TrimSpace(name)if trimmedName == "" {return fmt.Errorf("name cannot be empty or whitespace")}- // Validate name length in bytes (not characters) - same as Python len(search_name.encode("utf-8"))- if len([]byte(name)) > maxLen {- return fmt.Errorf("name length is %d which is large than %d", len([]byte(name)), maxLen)+ if length > maxLen {+ return fmt.Errorf("name length is %d which is large than %d", length, maxLen)}return nildiff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex d3f32d486f..c9096e2f3c 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -6,6 +6,7 @@ import ("fmt""strings""time"+ "unicode/utf8""ragflow/internal/common""ragflow/internal/dao"@@ -63,8 +64,8 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDif name == "" {return nil, common.CodeDataError, errors.New("dataset name can't be empty")}- if len(name) > entity.DatasetNameLimit {- return nil, common.CodeDataError, fmt.Errorf("Dataset name length is %d which is large than %d", len(name), entity.DatasetNameLimit)+ if utf8.RuneCountInString(name) > entity.DatasetNameLimit {+ return nil, common.CodeDataError, fmt.Errorf("Dataset name length is %d which is large than %d", utf8.RuneCountInString(name), entity.DatasetNameLimit)}tenant, err := d.tenantDAO.GetByID(ctx, dao.DB, tenantID)@@ -307,7 +308,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD// dedupeDatasetName returns a non-colliding dataset name within the tenant,// appending (1), (2), ... when the requested name is already taken.func (d *DatasetService) dedupeDatasetName(ctx context.Context, name, tenantID string) (string, error) {- return common.UniqueName(name, entity.DatasetNameLimit, func(candidate string) (bool, error) {+ return common.UniqueNameCharacters(name, entity.DatasetNameLimit, func(candidate string) (bool, error) {return d.kbDAO.NameExists(ctx, dao.DB, tenantID, candidate)})}diff --git a/internal/service/dataset/sdk_contract_test.go b/internal/service/dataset/sdk_contract_test.gonew file mode 100644index 0000000000..6e7f688cdf--- /dev/null+++ b/internal/service/dataset/sdk_contract_test.go@@ -0,0 +1,71 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dataset++import (+ "ragflow/internal/common"+ "ragflow/internal/service"+ "strings"+ "testing"+)++func TestSDKDatasetUnicodeCharacterLimit(t *testing.T) {+ for _, n := range []int{128, 129} {+ t.Run(strings.Repeat("数", n), func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")+ name := strings.Repeat("数", n)+ _, code, err := testDatasetCreateService(t).CreateDataset(t.Context(), &service.CreateDatasetRequest{Name: name, ParseType: new(1), ParserID: new("general")}, testDatasetSubject("tenant-1"))+ if (err == nil) != (n == 128) {+ t.Fatalf("create characters=%d code=%d err=%v", n, code, err)+ }+ })+ t.Run("update", func(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")+ ctx := withDatasetTestContext(t.Context(), "tenant-1", "tenant-1")+ _, code, err := testDatasetUpdateService(t).UpdateDataset(ctx, "kb-1", "tenant-1", service.UpdateDatasetRequest{Name: new(strings.Repeat("资", n))})+ if (err == nil) != (n == 128) {+ t.Fatalf("update characters=%d code=%d err=%v", n, code, err)+ }+ })+ }+}+func TestSDKDatasetNegativePagerankDiagnostic(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")+ _, code, err := testDatasetUpdateService(t).UpdateDataset(withDatasetTestContext(t.Context(), "tenant-1", "tenant-1"), "kb-1", "tenant-1", service.UpdateDatasetRequest{Pagerank: new(int64(-1))})+ if code != common.CodeDataError || err == nil || !strings.Contains(err.Error(), "greater than or equal to 0") {+ t.Fatalf("code=%d err=%v", code, err)+ }+}++func TestSDKDatasetUnicodeDeduplicationLimit(t *testing.T) {+ for _, n := range []int{125, 128} {+ name := strings.Repeat("数", n)+ got, err := common.UniqueNameCharacters(name, 128, func(candidate string) (bool, error) { return candidate == name, nil })+ if n == 125 {+ if err != nil || got != name+"(1)" {+ t.Fatalf("name=%q err=%v", got, err)+ }+ } else if err == nil {+ t.Fatal("deduplicated name exceeded character limit")+ }+ }+}diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.goindex 4d7f47932e..7a5486e6c6 100644--- a/internal/service/dataset/update.go+++ b/internal/service/dataset/update.go@@ -6,6 +6,7 @@ import ("fmt""strings""time"+ "unicode/utf8""ragflow/internal/common""ragflow/internal/dao"@@ -68,7 +69,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, userID stif name == "" {return nil, common.CodeDataError, errors.New("String should have at least 1 character")}- if len(name) > 128 {+ if utf8.RuneCountInString(name) > 128 {return nil, common.CodeDataError, errors.New("String should have at most 128 characters")}simpleUpdates["name"] = name@@ -160,7 +161,10 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, userID stpagerankRequested := req.Pagerank != nilif pagerankRequested {requestedPagerank = *req.Pagerank- if *req.Pagerank < 0 || *req.Pagerank > 100 {+ if *req.Pagerank < 0 {+ return nil, common.CodeDataError, errors.New("input should be greater than or equal to 0")+ }+ if *req.Pagerank > 100 {return nil, common.CodeDataError, errors.New("input should be less than or equal to 100")}if d.docEngine == nil {diff --git a/sdk/python/ragflow_sdk/ragflow.py b/sdk/python/ragflow_sdk/ragflow.pyindex f27feec3aa..fac148863d 100644--- a/sdk/python/ragflow_sdk/ragflow.py+++ b/sdk/python/ragflow_sdk/ragflow.py@@ -1,5 +1,5 @@#-# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.@@ -63,12 +63,14 @@ class RAGFlow:permission: str = "me",parser_config: Optional[DataSet.ParserConfig] = None,language: Optional[str] = None,+ *,+ pipeline_version_id: str | None = None,) -> DataSet:"""Create a dataset with a builtin parser or an ingestion pipeline.``parse_type=1`` selects a builtin parser and defaults ``parser_id`` to``"general"``. ``parse_type=2`` selects an ingestion pipeline and requires- ``pipeline_id``.+ ``pipeline_id`` and a pinned ``pipeline_version_id``."""if parser_id is not None and not isinstance(parser_id, str):raise ValueError("parser_id must be a string")@@ -105,6 +107,9 @@ class RAGFlow:if language is not None:payload["language"] = language+ if pipeline_version_id is not None:+ payload["pipeline_version_id"] = pipeline_version_id+res = self.post("/datasets", payload)res = res.json()if res.get("code") == 0:diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.pyindex 1776174767..aaec0297c0 100644--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py@@ -39,7 +39,6 @@ def test_name(dataset_factory, client, name):assert persisted[0].name == name[email protected](reason="internal/service/dataset/crud.go CreateDataset uses len(name) bytes despite the documented 128-character limit")def test_name_unicode_limit(dataset_factory):name = "数" * 128assert dataset_factory(name=name).name == name@@ -194,12 +193,13 @@ def test_builtin_parser_default(dataset_factory):assert dataset.pipeline_id is None[email protected](reason="sdk/python/ragflow_sdk/ragflow.py create_dataset cannot send pipeline_version_id required by internal/service/dataset/crud.go CreateDataset (empty_version_test.go)")def test_pipeline_parser(dataset_factory, client, integration_settings):pipeline_id = integration_settings.get("pipeline_id")if not pipeline_id:pytest.fail("RAGFLOW_SDK_PIPELINE_ID must identify an accessible versioned ingestion pipeline")- dataset = dataset_factory(parse_type=2, pipeline_id=pipeline_id)+ version_id = integration_settings.get("pipeline_version_id")+ assert version_id, "RAGFLOW_SDK_PIPELINE_VERSION_ID must pin an accessible ingestion pipeline version"+ dataset = dataset_factory(parse_type=2, pipeline_id=pipeline_id, pipeline_version_id=version_id)assert dataset.pipeline_id == pipeline_idassert dataset.parse_type == 2persisted = client.list_datasets(id=dataset.id)[0]diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.pyindex f967cde3d6..7c03558942 100644--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py@@ -49,7 +49,6 @@ class TestDatasetUpdate:assert dataset.name == name.strip()assert client.list_datasets(id=dataset.id)[0].name == name.strip()- @pytest.mark.skip(reason="Go internal/service/dataset/update.go uses len(name) bytes, rejecting documented 128-character BMP names")def test_name_unicode_character_limit(self, client, add_dataset_func):name = "资" * 128add_dataset_func.update({"name": name})@@ -291,7 +290,6 @@ class TestDatasetUpdate:pytest.param(-1,"greater than or equal to 0",- marks=pytest.mark.skip(reason="internal/service/dataset/update.go reports the upper bound for negative pagerank; preserve the correct lower-bound diagnostic"),),(101, "less than or equal to 100"),(3.14, "cannot unmarshal"),