Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix create build-in dataset parser_config should be pipeline config (#19860)

Wang Qif13261e0b2f9
diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.go
index 4646c3fea7..43a5c45ea1 100644
--- a/internal/service/dataset/create_test.go
+++ b/internal/service/dataset/create_test.go
@@ -87,9 +87,10 @@ func TestCreateDataset_ComponentParamsPopulated(t *testing.T) {
chunkMethod := "general"
parseType := 1
result, code, err := testDatasetCreateService(t).CreateDataset(ctx, &service.CreateDatasetRequest{
- Name: "ds-with-cp",
- ParserID: &chunkMethod,
- ParseType: &parseType,
+ Name: "ds-with-cp",
+ ParserID: &chunkMethod,
+ ParserConfig: map[string]interface{}{},
+ ParseType: &parseType,
}, "tenant-1")
if err != nil {
t.Fatalf("CreateDataset failed: %v", err)
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index af79ef61b8..d3a390feaa 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -102,12 +102,6 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
}
}
- parserConfig, cpErr := service.ResolveComponentParamsDefaults(ctx, parserID, pipelineID)
- if cpErr != nil {
- common.Warn("failed to resolve component params defaults for dataset",
- zap.String("parserID", parserID), zap.Error(cpErr))
- parserConfig = entity.JSONMap{}
- }
if req.ParserConfig != nil {
if err := validateDatasetParserConfig(req.ParserConfig); err != nil {
return nil, common.CodeArgumentError, err
@@ -118,17 +112,15 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
if err := pipelinepkg.NormalizeParserConfigPages(req.ParserConfig); err != nil {
return nil, common.CodeArgumentError, err
}
- flatParserID := parserID
- if flatParserID == "general" {
- flatParserID = "naive"
- }
- flat := common.GetParserConfig(flatParserID, req.ParserConfig)
- delete(flat, "raptor")
- delete(flat, "graphrag")
- flat["llm_id"] = tenant.LLMID
- flat["parent_child"] = map[string]interface{}{"use_parent_child": false, "children_delimiter": "\n"}
- flat["children_delimiter"] = ""
- parserConfig = entity.JSONMap(flat)
+ }
+ isPipeline := pipelineID != nil && strings.TrimSpace(*pipelineID) != ""
+ dslJSON, dslErr := service.LoadPipelineDSL(ctx, isPipeline, parserID, pipelineID)
+ parserConfig := entity.JSONMap{}
+ if dslErr != nil {
+ common.Warn("failed to load pipeline DSL for building parser_config",
+ zap.String("parserID", parserID), zap.Error(dslErr))
+ } else {
+ parserConfig = pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)
}
var parserConfigMap map[string]interface{} = parserConfig
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index 87fd0140c8..4f53449ff4 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -55,6 +55,17 @@ def _expected_chunk_method(chunk_method):
return chunk_method
+def _assert_go_pipeline_parser_config(parser_config, chunker_prefix="GeneralChunker:"):
+ assert parser_config
+ assert "File" in parser_config, parser_config
+ assert any(key.startswith("Parser:") for key in parser_config), parser_config
+ assert any(key.startswith(chunker_prefix) for key in parser_config), parser_config
+ assert all(isinstance(params, dict) for params in parser_config.values()), parser_config
+ assert "chunk_token_num" not in parser_config, parser_config
+ assert "raptor" not in parser_config, parser_config
+ assert "graphrag" not in parser_config, parser_config
+
+
def _is_infinity_doc_engine(rest_client: RestClient) -> bool:
env_engine = (os.getenv("DOC_ENGINE") or "").strip().lower()
if env_engine:
@@ -1315,9 +1326,12 @@ def test_dataset_create_parser_config_missing_raptor_and_graphrag(rest_client, c
body = res.json()
assert body["code"] == 0, body
parser_config = body["data"]["parser_config"]
- assert "raptor" not in parser_config, body
- assert "graphrag" not in parser_config, body
- assert parser_config["chunk_token_num"] == 1024, body
+ if IS_GO_PROXY:
+ _assert_go_pipeline_parser_config(parser_config)
+ else:
+ assert "raptor" not in parser_config, body
+ assert "graphrag" not in parser_config, body
+ assert parser_config["chunk_token_num"] == 1024, body
@pytest.mark.p3
@@ -1457,7 +1471,7 @@ def test_dataset_create_concurrent_contract(rest_client, clear_datasets):
)
def test_dataset_create_parser_config_valid_matrix_contract(rest_client, clear_datasets, name, parser_config):
if IS_GO_PROXY:
- pytest.skip("Go CreateDataset does not accept parser_config")
+ pytest.skip("Go CreateDataset does not accept legacy flat parser_config")
payload = {"name": name, "parser_config": parser_config}
res = rest_client.post("/datasets", json=payload)
assert res.status_code == 200
@@ -1494,9 +1508,12 @@ def test_dataset_create_parser_config_bugfix_contract(rest_client, clear_dataset
body = res.json()
assert body["code"] == 0, body
actual_parser_config = body["data"]["parser_config"]
- assert "raptor" not in actual_parser_config, body
- assert "graphrag" not in actual_parser_config, body
- assert actual_parser_config["chunk_token_num"] == 1024, body
+ if IS_GO_PROXY:
+ _assert_go_pipeline_parser_config(actual_parser_config)
+ else:
+ assert "raptor" not in actual_parser_config, body
+ assert "graphrag" not in actual_parser_config, body
+ assert actual_parser_config["chunk_token_num"] == 1024, body
@pytest.mark.p2
@@ -1516,9 +1533,20 @@ def test_dataset_create_parser_config_different_chunk_methods_contract(rest_clie
body = res.json()
assert body["code"] == 0, body
parser_config = body["data"]["parser_config"]
- assert parser_config["chunk_token_num"] == 512, body
- assert "raptor" not in parser_config, body
- assert "graphrag" not in parser_config, body
+ if IS_GO_PROXY:
+ chunker_prefix = {
+ "qa": "QAChunker:",
+ "manual": "ManualChunker:",
+ "paper": "TitleChunker:",
+ "book": "TitleChunker:",
+ "laws": "TitleChunker:",
+ "presentation": "PageChunker:",
+ }[chunk_method]
+ _assert_go_pipeline_parser_config(parser_config, chunker_prefix)
+ else:
+ assert parser_config["chunk_token_num"] == 512, body
+ assert "raptor" not in parser_config, body
+ assert "graphrag" not in parser_config, body
def test_dataset_create_name_invalid_and_duplicate_contract(rest_client, clear_datasets):
@@ -1803,8 +1831,11 @@ def test_dataset_create_parser_config_defaults_and_extra_fields_contract(rest_cl
unset_parser_config = unset_payload["data"]["parser_config"]
none_parser_config = none_payload["data"]["parser_config"]
assert empty_parser_config == unset_parser_config == none_parser_config
- for key in DEFAULT_PARSER_CONFIG:
- assert key in empty_parser_config, empty_payload
+ if IS_GO_PROXY:
+ _assert_go_pipeline_parser_config(empty_parser_config)
+ else:
+ for key in DEFAULT_PARSER_CONFIG:
+ assert key in empty_parser_config, empty_payload
unsupported_field_payloads = [
{"name": "id", "id": "id"},