RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(dataset): canonicalize naive parser IDs to general (#18839)
Sbaaoui Idriss7652cd9802a5
diff --git a/internal/entity/dataset.go b/internal/entity/dataset.goindex 26826bfe9d..37ed80226e 100644--- a/internal/entity/dataset.go+++ b/internal/entity/dataset.go@@ -52,6 +52,7 @@ const (ParserTypeBook ParserType = "book"ParserTypeQA ParserType = "qa"ParserTypeTable ParserType = "table"+ ParserTypeGeneral ParserType = "general"ParserTypeNaive ParserType = "naive"ParserTypePicture ParserType = "picture"ParserTypeOne ParserType = "one"@@ -124,7 +125,7 @@ type Knowledgebase struct {ChunkNum int64 `gorm:"column:chunk_num;default:0;index" json:"chunk_num"`SimilarityThreshold float64 `gorm:"column:similarity_threshold;default:0.2;index" json:"similarity_threshold"`VectorSimilarityWeight float64 `gorm:"column:vector_similarity_weight;default:0.3;index" json:"vector_similarity_weight"`- ParserID string `gorm:"column:parser_id;size:32;not null;default:naive;index" json:"parser_id"`+ ParserID string `gorm:"column:parser_id;size:32;not null;default:general;index" json:"parser_id"`PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`ParserConfig JSONMap `gorm:"column:parser_config;type:json" json:"parser_config"`Pagerank int64 `gorm:"column:pagerank;default:0" json:"pagerank"`diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.goindex 74b8fdacfd..89d8b92712 100644--- a/internal/service/dataset/create_test.go+++ b/internal/service/dataset/create_test.go@@ -1,7 +1,6 @@package datasetimport (- "strings""testing""ragflow/internal/common"@@ -57,8 +56,8 @@ func TestCreateDataset_NoComponentParams(t *testing.T) {if code != common.CodeSuccess {t.Fatalf("expected success code, got %d", code)}- if result["parser_id"] != strings.TrimSpace(chunkMethod) {- t.Fatalf("expected parser_id %q, got %#v", chunkMethod, result["parser_id"])+ if result["parser_id"] != string(entity.ParserTypeGeneral) {+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])}}@@ -115,8 +114,8 @@ func TestCreateDataset_ParseTypeBuiltinClearsPipelineID(t *testing.T) {if code != common.CodeSuccess {t.Fatalf("expected success code, got %d", code)}- if result["parser_id"] != chunkMethod {- t.Fatalf("expected parser_id %q, got %#v", chunkMethod, result["parser_id"])+ if result["parser_id"] != string(entity.ParserTypeGeneral) {+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])}if v, ok := result["pipeline_id"]; ok && v != nil {t.Fatalf("expected pipeline_id to be nil for BuiltIn mode, got %#v", v)diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex 74145d7596..b6ea5b8d05 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -47,7 +47,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD}}- parserID := ""+ parserID := string(entity.ParserTypeGeneral)permission := "me"embeddingModel := ""pipelineID := req.PipelineID@@ -59,10 +59,11 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD}}if req.ParserID != nil {- parserID = strings.TrimSpace(*req.ParserID)- if err := validateParserID(parserID); err != nil {+ canonicalID, err := canonicalDatasetParserID(strings.TrimSpace(*req.ParserID))+ if err != nil {return nil, common.CodeDataError, err}+ parserID = canonicalIDpipelineID = nil}if req.PipelineID != nil {@@ -71,6 +72,9 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDreturn nil, common.CodeDataError, err}pipelineID = normalizedPipelineID+ if pipelineID != nil && strings.TrimSpace(*pipelineID) != "" {+ parserID = ""+ }}if req.EmbeddingModel != nil {embeddingModel = strings.TrimSpace(*req.EmbeddingModel)diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 4dcc22b4f2..7e05690d7d 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -46,19 +46,37 @@ const (graphPhaseCommunityDone = "community_done")-// validateParserID validates parser_id against the built-in pipeline registry.-func validateParserID(chunkMethod string) error {- if chunkMethod == "knowledge_graph" {- return nil+// canonicalDatasetParserID resolves a parser ID to its canonical builtin ID.+// The registry retains legacy aliases such as naive -> general for old clients.+func canonicalDatasetParserID(parserID string) (string, error) {+ if parserID == "knowledge_graph" {+ return parserID, nil}registry, err := pipelinepkg.DefaultRegistry()if err != nil || registry == nil {- return errors.New("parser_id validation unavailable: builtin pipeline registry not loaded")+ return "", errors.New("parser_id validation unavailable: builtin pipeline registry not loaded")}- if registry.IsValid(chunkMethod) {- return nil+ template, ok := registry.Get(parserID)+ if ok {+ return template.ParserID, nil}- return parserIDError()+ return "", parserIDError()+}++// validateParserID validates parser_id against the built-in pipeline registry.+func validateParserID(parserID string) error {+ _, err := canonicalDatasetParserID(parserID)+ return err+}++// datasetParserIDForResponse returns the canonical parser ID when a legacy+// persisted value remains resolvable. Unknown stored values are preserved.+func datasetParserIDForResponse(parserID string) string {+ canonicalID, err := canonicalDatasetParserID(parserID)+ if err != nil {+ return parserID+ }+ return canonicalID}func parserIDError() error {@@ -223,10 +241,11 @@ func datasetUpdateParserID(req service.UpdateDatasetRequest) (string, bool, erroif !provided {return "", false, nil}- if err := validateParserID(parserID); err != nil {+ canonicalID, err := canonicalDatasetParserID(parserID)+ if err != nil {return "", true, err}- return parserID, true, nil+ return canonicalID, true, nil}func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, error) {diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.goindex bcea8c9032..1dcf09425d 100644--- a/internal/service/dataset/update_test.go+++ b/internal/service/dataset/update_test.go@@ -1003,8 +1003,8 @@ func TestUpdateDataset_StripsUnknownParam_Builtin(t *testing.T) {if _, exists := rhyme["no_such_param"]; exists {t.Fatal("expected no_such_param to be stripped")}- if result["parser_id"] != string(entity.ParserTypeNaive) {- t.Fatalf("expected parser_id preserved, got %#v", result["parser_id"])+ if result["parser_id"] != string(entity.ParserTypeGeneral) {+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])}}@@ -1027,8 +1027,8 @@ func TestUpdateDataset_AcceptsValidComponentParams_Builtin(t *testing.T) {if code != common.CodeSuccess {t.Fatalf("expected success code, got %d", code)}- if result["parser_id"] != string(entity.ParserTypeNaive) {- t.Fatalf("expected parser_id preserved, got %#v", result["parser_id"])+ if result["parser_id"] != string(entity.ParserTypeGeneral) {+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])}persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")diff --git a/internal/service/dataset/utils.go b/internal/service/dataset/utils.goindex 3a953b2983..7c251b7c44 100644--- a/internal/service/dataset/utils.go+++ b/internal/service/dataset/utils.go@@ -28,7 +28,7 @@ func datasetListItemToMap(kb *entity.KnowledgebaseListItem) map[string]interface"document_count": kb.DocNum,"token_num": kb.TokenNum,"chunk_count": kb.ChunkNum,- "parser_id": kb.ParserID,+ "parser_id": datasetParserIDForResponse(kb.ParserID),"parser_config": jsonMapValue(kb.ParserConfig),"pagerank": kb.Pagerank,"embedding_model": kb.EmbdID,@@ -56,7 +56,7 @@ func datasetToMap(kb *entity.Knowledgebase) map[string]interface{} {"chunk_count": kb.ChunkNum,"similarity_threshold": kb.SimilarityThreshold,"vector_similarity_weight": kb.VectorSimilarityWeight,- "parser_id": kb.ParserID,+ "parser_id": datasetParserIDForResponse(kb.ParserID),"parser_config": kb.ParserConfig,"pagerank": kb.Pagerank,"create_time": kb.CreateTime,diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex f67b97f71d..bd3ac347b7 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -48,6 +48,13 @@ def _parser_id_fields(chunk_method):return fields+def _expected_chunk_method(chunk_method):+ """Return the API-visible parser ID for the active proxy."""+ if IS_GO_PROXY and chunk_method == "naive":+ return "general"+ return chunk_method++def _is_infinity_doc_engine(rest_client: RestClient) -> bool:env_engine = (os.getenv("DOC_ENGINE") or "").strip().lower()if env_engine:@@ -227,7 +234,7 @@ def test_dataset_update_chunk_method_contract(rest_client, clear_datasets, chunkassert update_res.status_code == 200update_payload = update_res.json()assert update_payload["code"] == 0, update_payload- assert update_payload["data"][PARSER_ID_FIELD] == chunk_method, update_payload+ assert update_payload["data"][PARSER_ID_FIELD] == _expected_chunk_method(chunk_method), update_payload@pytest.mark.p1@@ -1181,7 +1188,7 @@ def test_dataset_create_chunk_method_contract(rest_client, clear_datasets, name,assert res.status_code == 200payload = res.json()assert payload["code"] == 0, payload- assert payload["data"][PARSER_ID_FIELD] == chunk_method, payload+ assert payload["data"][PARSER_ID_FIELD] == _expected_chunk_method(chunk_method), payload@pytest.mark.p2@@ -1696,7 +1703,7 @@ def test_dataset_create_permission_and_chunk_method_contract(rest_client, clear_assert chunk_method_unset_res.status_code == 200chunk_method_unset_payload = chunk_method_unset_res.json()assert chunk_method_unset_payload["code"] == 0, chunk_method_unset_payload- assert chunk_method_unset_payload["data"][PARSER_ID_FIELD] == "naive", chunk_method_unset_payload+ assert chunk_method_unset_payload["data"][PARSER_ID_FIELD] == _expected_chunk_method("naive"), chunk_method_unset_payload@pytest.mark.p2