RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix create build-in dataset parser_config should be pipeline config (#19860)
Wang Qif13261e0b2f9
diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.goindex 4646c3fea7..43a5c45ea1 100644--- a/internal/service/dataset/create_test.go+++ b/internal/service/dataset/create_test.go@@ -87,9 +87,10 @@ func TestCreateDataset_ComponentParamsPopulated(t *testing.T) {chunkMethod := "general"parseType := 1result, code, err := testDatasetCreateService(t).CreateDataset(ctx, &service.CreateDatasetRequest{- Name: "ds-with-cp",- ParserID: &chunkMethod,- ParseType: &parseType,+ Name: "ds-with-cp",+ ParserID: &chunkMethod,+ ParserConfig: map[string]interface{}{},+ ParseType: &parseType,}, "tenant-1")if err != nil {t.Fatalf("CreateDataset failed: %v", err)diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex af79ef61b8..d3a390feaa 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -102,12 +102,6 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD}}- parserConfig, cpErr := service.ResolveComponentParamsDefaults(ctx, parserID, pipelineID)- if cpErr != nil {- common.Warn("failed to resolve component params defaults for dataset",- zap.String("parserID", parserID), zap.Error(cpErr))- parserConfig = entity.JSONMap{}- }if req.ParserConfig != nil {if err := validateDatasetParserConfig(req.ParserConfig); err != nil {return nil, common.CodeArgumentError, err@@ -118,17 +112,15 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDif err := pipelinepkg.NormalizeParserConfigPages(req.ParserConfig); err != nil {return nil, common.CodeArgumentError, err}- flatParserID := parserID- if flatParserID == "general" {- flatParserID = "naive"- }- flat := common.GetParserConfig(flatParserID, req.ParserConfig)- delete(flat, "raptor")- delete(flat, "graphrag")- flat["llm_id"] = tenant.LLMID- flat["parent_child"] = map[string]interface{}{"use_parent_child": false, "children_delimiter": "\n"}- flat["children_delimiter"] = ""- parserConfig = entity.JSONMap(flat)+ }+ isPipeline := pipelineID != nil && strings.TrimSpace(*pipelineID) != ""+ dslJSON, dslErr := service.LoadPipelineDSL(ctx, isPipeline, parserID, pipelineID)+ parserConfig := entity.JSONMap{}+ if dslErr != nil {+ common.Warn("failed to load pipeline DSL for building parser_config",+ zap.String("parserID", parserID), zap.Error(dslErr))+ } else {+ parserConfig = pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)}var parserConfigMap map[string]interface{} = parserConfigdiff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex 87fd0140c8..4f53449ff4 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -55,6 +55,17 @@ def _expected_chunk_method(chunk_method):return chunk_method+def _assert_go_pipeline_parser_config(parser_config, chunker_prefix="GeneralChunker:"):+ assert parser_config+ assert "File" in parser_config, parser_config+ assert any(key.startswith("Parser:") for key in parser_config), parser_config+ assert any(key.startswith(chunker_prefix) for key in parser_config), parser_config+ assert all(isinstance(params, dict) for params in parser_config.values()), parser_config+ assert "chunk_token_num" not in parser_config, parser_config+ assert "raptor" not in parser_config, parser_config+ assert "graphrag" not in parser_config, parser_config++def _is_infinity_doc_engine(rest_client: RestClient) -> bool:env_engine = (os.getenv("DOC_ENGINE") or "").strip().lower()if env_engine:@@ -1315,9 +1326,12 @@ def test_dataset_create_parser_config_missing_raptor_and_graphrag(rest_client, cbody = res.json()assert body["code"] == 0, bodyparser_config = body["data"]["parser_config"]- assert "raptor" not in parser_config, body- assert "graphrag" not in parser_config, body- assert parser_config["chunk_token_num"] == 1024, body+ if IS_GO_PROXY:+ _assert_go_pipeline_parser_config(parser_config)+ else:+ assert "raptor" not in parser_config, body+ assert "graphrag" not in parser_config, body+ assert parser_config["chunk_token_num"] == 1024, body@pytest.mark.p3@@ -1457,7 +1471,7 @@ def test_dataset_create_concurrent_contract(rest_client, clear_datasets):)def test_dataset_create_parser_config_valid_matrix_contract(rest_client, clear_datasets, name, parser_config):if IS_GO_PROXY:- pytest.skip("Go CreateDataset does not accept parser_config")+ pytest.skip("Go CreateDataset does not accept legacy flat parser_config")payload = {"name": name, "parser_config": parser_config}res = rest_client.post("/datasets", json=payload)assert res.status_code == 200@@ -1494,9 +1508,12 @@ def test_dataset_create_parser_config_bugfix_contract(rest_client, clear_datasetbody = res.json()assert body["code"] == 0, bodyactual_parser_config = body["data"]["parser_config"]- assert "raptor" not in actual_parser_config, body- assert "graphrag" not in actual_parser_config, body- assert actual_parser_config["chunk_token_num"] == 1024, body+ if IS_GO_PROXY:+ _assert_go_pipeline_parser_config(actual_parser_config)+ else:+ assert "raptor" not in actual_parser_config, body+ assert "graphrag" not in actual_parser_config, body+ assert actual_parser_config["chunk_token_num"] == 1024, body@pytest.mark.p2@@ -1516,9 +1533,20 @@ def test_dataset_create_parser_config_different_chunk_methods_contract(rest_cliebody = res.json()assert body["code"] == 0, bodyparser_config = body["data"]["parser_config"]- assert parser_config["chunk_token_num"] == 512, body- assert "raptor" not in parser_config, body- assert "graphrag" not in parser_config, body+ if IS_GO_PROXY:+ chunker_prefix = {+ "qa": "QAChunker:",+ "manual": "ManualChunker:",+ "paper": "TitleChunker:",+ "book": "TitleChunker:",+ "laws": "TitleChunker:",+ "presentation": "PageChunker:",+ }[chunk_method]+ _assert_go_pipeline_parser_config(parser_config, chunker_prefix)+ else:+ assert parser_config["chunk_token_num"] == 512, body+ assert "raptor" not in parser_config, body+ assert "graphrag" not in parser_config, bodydef test_dataset_create_name_invalid_and_duplicate_contract(rest_client, clear_datasets):@@ -1803,8 +1831,11 @@ def test_dataset_create_parser_config_defaults_and_extra_fields_contract(rest_clunset_parser_config = unset_payload["data"]["parser_config"]none_parser_config = none_payload["data"]["parser_config"]assert empty_parser_config == unset_parser_config == none_parser_config- for key in DEFAULT_PARSER_CONFIG:- assert key in empty_parser_config, empty_payload+ if IS_GO_PROXY:+ _assert_go_pipeline_parser_config(empty_parser_config)+ else:+ for key in DEFAULT_PARSER_CONFIG:+ assert key in empty_parser_config, empty_payloadunsupported_field_payloads = [{"name": "id", "id": "id"},