Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(dataset): canonicalize naive parser IDs to general (#18839)

Sbaaoui Idriss7652cd9802a5
diff --git a/internal/entity/dataset.go b/internal/entity/dataset.go
index 26826bfe9d..37ed80226e 100644
--- a/internal/entity/dataset.go
+++ b/internal/entity/dataset.go
@@ -52,6 +52,7 @@ const (
ParserTypeBook ParserType = "book"
ParserTypeQA ParserType = "qa"
ParserTypeTable ParserType = "table"
+ ParserTypeGeneral ParserType = "general"
ParserTypeNaive ParserType = "naive"
ParserTypePicture ParserType = "picture"
ParserTypeOne ParserType = "one"
@@ -124,7 +125,7 @@ type Knowledgebase struct {
ChunkNum int64 `gorm:"column:chunk_num;default:0;index" json:"chunk_num"`
SimilarityThreshold float64 `gorm:"column:similarity_threshold;default:0.2;index" json:"similarity_threshold"`
VectorSimilarityWeight float64 `gorm:"column:vector_similarity_weight;default:0.3;index" json:"vector_similarity_weight"`
- ParserID string `gorm:"column:parser_id;size:32;not null;default:naive;index" json:"parser_id"`
+ ParserID string `gorm:"column:parser_id;size:32;not null;default:general;index" json:"parser_id"`
PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`
ParserConfig JSONMap `gorm:"column:parser_config;type:json" json:"parser_config"`
Pagerank int64 `gorm:"column:pagerank;default:0" json:"pagerank"`
diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.go
index 74b8fdacfd..89d8b92712 100644
--- a/internal/service/dataset/create_test.go
+++ b/internal/service/dataset/create_test.go
@@ -1,7 +1,6 @@
package dataset
import (
- "strings"
"testing"
"ragflow/internal/common"
@@ -57,8 +56,8 @@ func TestCreateDataset_NoComponentParams(t *testing.T) {
if code != common.CodeSuccess {
t.Fatalf("expected success code, got %d", code)
}
- if result["parser_id"] != strings.TrimSpace(chunkMethod) {
- t.Fatalf("expected parser_id %q, got %#v", chunkMethod, result["parser_id"])
+ if result["parser_id"] != string(entity.ParserTypeGeneral) {
+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])
}
}
@@ -115,8 +114,8 @@ func TestCreateDataset_ParseTypeBuiltinClearsPipelineID(t *testing.T) {
if code != common.CodeSuccess {
t.Fatalf("expected success code, got %d", code)
}
- if result["parser_id"] != chunkMethod {
- t.Fatalf("expected parser_id %q, got %#v", chunkMethod, result["parser_id"])
+ if result["parser_id"] != string(entity.ParserTypeGeneral) {
+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])
}
if v, ok := result["pipeline_id"]; ok && v != nil {
t.Fatalf("expected pipeline_id to be nil for BuiltIn mode, got %#v", v)
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index 74145d7596..b6ea5b8d05 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -47,7 +47,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
}
}
- parserID := ""
+ parserID := string(entity.ParserTypeGeneral)
permission := "me"
embeddingModel := ""
pipelineID := req.PipelineID
@@ -59,10 +59,11 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
}
}
if req.ParserID != nil {
- parserID = strings.TrimSpace(*req.ParserID)
- if err := validateParserID(parserID); err != nil {
+ canonicalID, err := canonicalDatasetParserID(strings.TrimSpace(*req.ParserID))
+ if err != nil {
return nil, common.CodeDataError, err
}
+ parserID = canonicalID
pipelineID = nil
}
if req.PipelineID != nil {
@@ -71,6 +72,9 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
return nil, common.CodeDataError, err
}
pipelineID = normalizedPipelineID
+ if pipelineID != nil && strings.TrimSpace(*pipelineID) != "" {
+ parserID = ""
+ }
}
if req.EmbeddingModel != nil {
embeddingModel = strings.TrimSpace(*req.EmbeddingModel)
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 4dcc22b4f2..7e05690d7d 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -46,19 +46,37 @@ const (
graphPhaseCommunityDone = "community_done"
)
-// validateParserID validates parser_id against the built-in pipeline registry.
-func validateParserID(chunkMethod string) error {
- if chunkMethod == "knowledge_graph" {
- return nil
+// canonicalDatasetParserID resolves a parser ID to its canonical builtin ID.
+// The registry retains legacy aliases such as naive -> general for old clients.
+func canonicalDatasetParserID(parserID string) (string, error) {
+ if parserID == "knowledge_graph" {
+ return parserID, nil
}
registry, err := pipelinepkg.DefaultRegistry()
if err != nil || registry == nil {
- return errors.New("parser_id validation unavailable: builtin pipeline registry not loaded")
+ return "", errors.New("parser_id validation unavailable: builtin pipeline registry not loaded")
}
- if registry.IsValid(chunkMethod) {
- return nil
+ template, ok := registry.Get(parserID)
+ if ok {
+ return template.ParserID, nil
}
- return parserIDError()
+ return "", parserIDError()
+}
+
+// validateParserID validates parser_id against the built-in pipeline registry.
+func validateParserID(parserID string) error {
+ _, err := canonicalDatasetParserID(parserID)
+ return err
+}
+
+// datasetParserIDForResponse returns the canonical parser ID when a legacy
+// persisted value remains resolvable. Unknown stored values are preserved.
+func datasetParserIDForResponse(parserID string) string {
+ canonicalID, err := canonicalDatasetParserID(parserID)
+ if err != nil {
+ return parserID
+ }
+ return canonicalID
}
func parserIDError() error {
@@ -223,10 +241,11 @@ func datasetUpdateParserID(req service.UpdateDatasetRequest) (string, bool, erro
if !provided {
return "", false, nil
}
- if err := validateParserID(parserID); err != nil {
+ canonicalID, err := canonicalDatasetParserID(parserID)
+ if err != nil {
return "", true, err
}
- return parserID, true, nil
+ return canonicalID, true, nil
}
func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, error) {
diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go
index bcea8c9032..1dcf09425d 100644
--- a/internal/service/dataset/update_test.go
+++ b/internal/service/dataset/update_test.go
@@ -1003,8 +1003,8 @@ func TestUpdateDataset_StripsUnknownParam_Builtin(t *testing.T) {
if _, exists := rhyme["no_such_param"]; exists {
t.Fatal("expected no_such_param to be stripped")
}
- if result["parser_id"] != string(entity.ParserTypeNaive) {
- t.Fatalf("expected parser_id preserved, got %#v", result["parser_id"])
+ if result["parser_id"] != string(entity.ParserTypeGeneral) {
+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])
}
}
@@ -1027,8 +1027,8 @@ func TestUpdateDataset_AcceptsValidComponentParams_Builtin(t *testing.T) {
if code != common.CodeSuccess {
t.Fatalf("expected success code, got %d", code)
}
- if result["parser_id"] != string(entity.ParserTypeNaive) {
- t.Fatalf("expected parser_id preserved, got %#v", result["parser_id"])
+ if result["parser_id"] != string(entity.ParserTypeGeneral) {
+ t.Fatalf("expected canonical parser_id %q, got %#v", entity.ParserTypeGeneral, result["parser_id"])
}
persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
diff --git a/internal/service/dataset/utils.go b/internal/service/dataset/utils.go
index 3a953b2983..7c251b7c44 100644
--- a/internal/service/dataset/utils.go
+++ b/internal/service/dataset/utils.go
@@ -28,7 +28,7 @@ func datasetListItemToMap(kb *entity.KnowledgebaseListItem) map[string]interface
"document_count": kb.DocNum,
"token_num": kb.TokenNum,
"chunk_count": kb.ChunkNum,
- "parser_id": kb.ParserID,
+ "parser_id": datasetParserIDForResponse(kb.ParserID),
"parser_config": jsonMapValue(kb.ParserConfig),
"pagerank": kb.Pagerank,
"embedding_model": kb.EmbdID,
@@ -56,7 +56,7 @@ func datasetToMap(kb *entity.Knowledgebase) map[string]interface{} {
"chunk_count": kb.ChunkNum,
"similarity_threshold": kb.SimilarityThreshold,
"vector_similarity_weight": kb.VectorSimilarityWeight,
- "parser_id": kb.ParserID,
+ "parser_id": datasetParserIDForResponse(kb.ParserID),
"parser_config": kb.ParserConfig,
"pagerank": kb.Pagerank,
"create_time": kb.CreateTime,
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index f67b97f71d..bd3ac347b7 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -48,6 +48,13 @@ def _parser_id_fields(chunk_method):
return fields
+def _expected_chunk_method(chunk_method):
+ """Return the API-visible parser ID for the active proxy."""
+ if IS_GO_PROXY and chunk_method == "naive":
+ return "general"
+ return chunk_method
+
+
def _is_infinity_doc_engine(rest_client: RestClient) -> bool:
env_engine = (os.getenv("DOC_ENGINE") or "").strip().lower()
if env_engine:
@@ -227,7 +234,7 @@ def test_dataset_update_chunk_method_contract(rest_client, clear_datasets, chunk
assert update_res.status_code == 200
update_payload = update_res.json()
assert update_payload["code"] == 0, update_payload
- assert update_payload["data"][PARSER_ID_FIELD] == chunk_method, update_payload
+ assert update_payload["data"][PARSER_ID_FIELD] == _expected_chunk_method(chunk_method), update_payload
@pytest.mark.p1
@@ -1181,7 +1188,7 @@ def test_dataset_create_chunk_method_contract(rest_client, clear_datasets, name,
assert res.status_code == 200
payload = res.json()
assert payload["code"] == 0, payload
- assert payload["data"][PARSER_ID_FIELD] == chunk_method, payload
+ assert payload["data"][PARSER_ID_FIELD] == _expected_chunk_method(chunk_method), payload
@pytest.mark.p2
@@ -1696,7 +1703,7 @@ def test_dataset_create_permission_and_chunk_method_contract(rest_client, clear_
assert chunk_method_unset_res.status_code == 200
chunk_method_unset_payload = chunk_method_unset_res.json()
assert chunk_method_unset_payload["code"] == 0, chunk_method_unset_payload
- assert chunk_method_unset_payload["data"][PARSER_ID_FIELD] == "naive", chunk_method_unset_payload
+ assert chunk_method_unset_payload["data"][PARSER_ID_FIELD] == _expected_chunk_method("naive"), chunk_method_unset_payload
@pytest.mark.p2