RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Drop unscoped (flat) parser_config keys instead of rejecting them (#20445)
xugangqiang4ef147e7acc6
diff --git a/internal/handler/document.go b/internal/handler/document.goindex 6efddefbbc..eafc322eb9 100644--- a/internal/handler/document.go+++ b/internal/handler/document.go@@ -1837,7 +1837,16 @@ func (h *DocumentHandler) UpdateDatasetDocument(c *gin.Context) {return}if present["parser_config"] && req.ParserConfig != nil {- if err := dataset.ValidateDocumentParserConfig(req.ParserConfig); err != nil {+ dropped, err := dataset.ValidateDocumentParserConfig(req.ParserConfig)+ if len(dropped) > 0 {+ common.Warn("dropping unscoped (flat) parser_config keys; keys must be component-scoped (contain ':')",+ zap.Strings("keys", dropped),+ zap.String("user_id", user.ID),+ zap.String("dataset_id", datasetID),+ zap.String("document_id", documentID),+ )+ }+ if err != nil {common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())return}diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex 2835e89fd9..9e85b583c5 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -105,10 +105,14 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD}if req.ParserConfig != nil {- if err := validateDatasetParserConfig(req.ParserConfig); err != nil {- return nil, common.CodeArgumentError, err+ dropped, err := ValidateParserConfig(req.ParserConfig)+ if len(dropped) > 0 {+ common.Warn("dropping unscoped (flat) parser_config keys; keys must be component-scoped (contain ':')",+ zap.Strings("keys", dropped),+ zap.String("tenant_id", tenantID),+ )}- if err := validateDatasetParserConfigSize(req.ParserConfig); err != nil {+ if err != nil {return nil, common.CodeArgumentError, err}if err := pipelinepkg.NormalizeParserConfigPages(req.ParserConfig); err != nil {diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 8d2ab4801b..8c0d706105 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -198,46 +198,53 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) errorreturn nil}-// validateDatasetParserConfig enforces the Go backend's "component-scoped only"-// contract: every top-level parser_config key must be keyed by a component id-// (it contains ":"), never a flat (non-component-scoped) key. The Python backend-// is fully retired (SystemHandler.Language always returns "go"), so the dataset-// parser_config is always built from a DSL whose params live on component nodes-// (e.g. chunk size is "chunk_token_size" on the chunker node, not the legacy-// flat "chunk_token_num"). Any flat key would be silently dropped by-// CleanComponentParams downstream and is never consumed, so we reject it loudly-// here instead.-//-// ValidateDocumentParserConfig delegates to this same check, so documents and-// datasets share one contract: no flat keys are ever permitted on either path.-func validateDatasetParserConfig(parserConfig map[string]interface{}) error {+// DropUnscopedParserConfigKeys removes top-level parser_config keys that are+// not component-scoped (i.e. do not contain ":"), since the Go backend only+// consumes keys keyed by a component id (e.g. "Parser:abc123" or+// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts+// that downstream consumers (CleanComponentParams) never read, so they are+// dropped rather than rejected. Only top-level keys are affected; a key nested+// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps+// its existing name. It mutates the provided map in place and returns the names+// of the dropped keys so callers can log the silent drop. A nil or empty map is+// returned untouched with a nil result.+func DropUnscopedParserConfigKeys(parserConfig map[string]any) []string {if len(parserConfig) == 0 {return nil}+ var dropped []stringfor key := range parserConfig {if !strings.Contains(key, ":") {- return fmt.Errorf(- "parser_config key %q must be component-scoped (e.g. under an Extractor or GeneralChunker node), not a flat top-level key",- key,- )+ dropped = append(dropped, key)}}- return nil+ for _, key := range dropped {+ delete(parserConfig, key)+ }+ return dropped}-// ValidateParserConfig validates the shared REST parser_config schema.-func ValidateParserConfig(parserConfig map[string]interface{}) error {- return validateDatasetParserConfig(parserConfig)+// ValidateParserConfig validates the shared REST parser_config schema. Flat+// (non-component-scoped) keys are no longer rejected: they are silently dropped+// because downstream consumers never read them. The size limit is still+// enforced. It returns the names of the dropped (unscoped) keys so the caller+// can log the silent drop; this is the single drop point shared by every entry+// path, so callers should not call DropUnscopedParserConfigKeys again.+func ValidateParserConfig(parserConfig map[string]interface{}) ([]string, error) {+ dropped := DropUnscopedParserConfigKeys(parserConfig)+ return dropped, validateDatasetParserConfigSize(parserConfig)}// ValidateDocumentParserConfig validates the parser_config attached to a// document. Documents follow the same component-scoped contract as datasets:// every key must be scoped under a node id (e.g. "Extractor:AutoExtractDefault"// or "GeneralChunker:SixApplesFall"). A document's Extractor/GeneralChunker-// nodes come from the same pipeline DSL as the dataset, so there is no flat-key-// fallback to keep. validateDatasetParserConfig enforces the no-flat-key rule.-func ValidateDocumentParserConfig(parserConfig map[string]interface{}) error {- return validateDatasetParserConfig(parserConfig)+// nodes come from the same pipeline DSL as the dataset, so flat keys are dropped+// (not kept) and the size limit is enforced. It returns the dropped key names+// for logging, mirroring ValidateParserConfig.+func ValidateDocumentParserConfig(parserConfig map[string]interface{}) ([]string, error) {+ dropped := DropUnscopedParserConfigKeys(parserConfig)+ return dropped, validateDatasetParserConfigSize(parserConfig)}// NormalizeDatasetID validates the dataset ID format and returns itsdiff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.goindex 4ae3e07e6f..e33c3ec317 100644--- a/internal/service/dataset/helpers_test.go+++ b/internal/service/dataset/helpers_test.go@@ -1,6 +1,8 @@package datasetimport (+ "reflect"+ "sort""strings""testing"@@ -205,113 +207,155 @@ func TestValidateDatasetParserConfigSize_OverLimit(t *testing.T) {}}-func TestValidateDatasetParserConfig_RejectsFlatNullableOptionalFields(t *testing.T) {- // A flat key is rejected regardless of its value (even nil): datasets are- // component-scoped only, so "task_page_size"/"pages" must live on a node.- for _, config := range []map[string]interface{}{- {"task_page_size": nil},- {"pages": nil},- } {- if err := validateDatasetParserConfig(config); err == nil {- t.Fatalf("validateDatasetParserConfig(%#v): expected error for flat key, got nil", config)- }+// --- DropUnscopedParserConfigKeys ---++func TestDropUnscopedParserConfigKeys_NilAndEmpty(t *testing.T) {+ if got := DropUnscopedParserConfigKeys(nil); got != nil {+ t.Fatalf("expected nil for nil map, got %#v", got)+ }+ if got := DropUnscopedParserConfigKeys(map[string]any{}); got != nil {+ t.Fatalf("expected nil for empty map, got %#v", got)}}-func TestValidateDatasetParserConfig_DelimiterType(t *testing.T) {- err := validateDatasetParserConfig(map[string]interface{}{"delimiter": float64(1)})- if err == nil {- t.Fatal("expected error for flat delimiter key")+func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {+ cfg := map[string]any{+ "File": map[string]any{},+ "chunk_token_num": float64(128),+ "delimiter": "\n",+ "metadata": map[string]any{},+ "parent_child": map[string]any{},+ "Parser:abc": map[string]any{"chunk_size": float64(512)},+ "Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{}},+ }+ dropped := DropUnscopedParserConfigKeys(cfg)+ sort.Strings(dropped)+ want := []string{"File", "chunk_token_num", "delimiter", "metadata", "parent_child"}+ if !reflect.DeepEqual(dropped, want) {+ t.Fatalf("dropped = %#v, want %#v", dropped, want)+ }+ // Component-scoped keys survive.+ if _, ok := cfg["Parser:abc"]; !ok {+ t.Error("component-scoped Parser:abc was dropped")+ }+ if _, ok := cfg["Extractor:AutoExtractDefault"]; !ok {+ t.Error("component-scoped Extractor:AutoExtractDefault was dropped")+ }+ // A flat-looking key nested inside a component node is NOT a top-level flat+ // key and must be preserved.+ if inner, ok := cfg["Extractor:AutoExtractDefault"].(map[string]any); !ok || inner["metadata"] == nil {+ t.Error("nested metadata inside a component node was dropped")+ }+ if len(cfg) != 2 {+ t.Fatalf("expected 2 surviving keys, got %d: %#v", len(cfg), cfg)}}-func TestValidateDocumentParserConfig_RejectsAnyFlatKey(t *testing.T) {- // Documents follow the same component-scoped contract as datasets: no flat- // top-level key is accepted (the Go backend scopes every setting under a- // node id such as "Extractor:AutoExtractDefault" or "GeneralChunker:SixApplesFall").- for _, flat := range []map[string]interface{}{- {"parser_specific": "value"},- {"delimiter": float64(1)},- {"children_delimiter": "|"},- {"metadata": map[string]interface{}{}},- {"parent_child": map[string]interface{}{}},- } {- if err := ValidateDocumentParserConfig(flat); err == nil {- t.Fatalf("expected error for flat document config %#v", flat)- }+func TestValidateParserConfig_DropsFlatKeys(t *testing.T) {+ flat := map[string]any{"chunk_token_num": float64(128), "delimiter": "\n"}+ dropped, err := ValidateParserConfig(flat)+ if err != nil {+ t.Fatalf("expected nil after dropping flat keys, got %v", err)+ }+ if len(dropped) != 2 {+ t.Fatalf("expected 2 dropped keys, got %#v", dropped)}- // Component-scoped keys always pass.- if err := ValidateDocumentParserConfig(map[string]interface{}{- "Extractor:AutoExtractDefault": map[string]interface{}{"metadata": map[string]interface{}{}},- "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(512)},- "GeneralChunker:SixApplesFallPC": map[string]interface{}{"parent_child": map[string]interface{}{}},- }); err != nil {+ if len(flat) != 0 {+ t.Fatalf("expected flat keys to be dropped, got %#v", flat)+ }+ // Component-scoped keys pass and are preserved.+ scoped := map[string]any{+ "GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": float64(512)},+ "Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{}},+ }+ dropped, err = ValidateParserConfig(scoped)+ if err != nil {t.Fatalf("expected nil for component-scoped keys, got %v", err)}-}--func TestValidateParserConfigRejectsFlatParentChildDelimiter(t *testing.T) {- config := map[string]interface{}{"children_delimiter": "|"}- // Datasets reject every flat key, including parser-level ones.- if err := validateDatasetParserConfig(config); err == nil {- t.Fatal("expected error for flat children_delimiter on dataset")+ if len(dropped) != 0 {+ t.Fatalf("expected no dropped keys for component-scoped config, got %#v", dropped)}- // Documents share the same no-flat-key contract.- if err := ValidateDocumentParserConfig(config); err == nil {- t.Fatal("expected error for flat children_delimiter on document")+ if len(scoped) != 2 {+ t.Fatalf("expected component-scoped keys preserved, got %#v", scoped)}}-func TestValidateDatasetParserConfig_RejectsAnyFlatKey(t *testing.T) {- for _, flat := range []map[string]interface{}{- {"chunk_token_num": float64(128)},- {"delimiter": "\n"},- {"children_delimiter": "|"},- {"layout_recognize": "DeepDOC"},- {"auto_keywords": float64(0)},- {"task_page_size": float64(1)},- {"pages": nil},+func TestValidateDocumentParserConfig_DropsFlatKeys(t *testing.T) {+ for _, flat := range []map[string]any{{"parser_specific": "value"},+ {"delimiter": float64(1)},+ {"children_delimiter": "|"},+ {"metadata": map[string]any{}},+ {"parent_child": map[string]any{}},+ {"File": map[string]any{}},} {- if err := validateDatasetParserConfig(flat); err == nil {- t.Fatalf("expected error for flat dataset config %#v", flat)+ cfg := make(map[string]any, len(flat))+ for k, v := range flat {+ cfg[k] = v+ }+ dropped, err := ValidateDocumentParserConfig(cfg)+ if err != nil {+ t.Fatalf("expected nil after dropping flat key %#v, got %v", flat, err)+ }+ if len(dropped) != 1 {+ t.Fatalf("expected exactly 1 dropped key for %#v, got %#v", flat, dropped)+ }+ if len(cfg) != 0 {+ t.Fatalf("expected flat key dropped, got %#v", cfg)}}- // Component-scoped keys (containing ":") always pass.- if err := validateDatasetParserConfig(map[string]interface{}{- "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(512)},- }); err != nil {- t.Fatalf("expected nil for component-scoped keys, got %v", err)+}++// TestValidateParserConfig_ReturnsDroppedKeys asserts that ValidateParserConfig+// returns the names of the flat (unscoped) keys it dropped so the caller can log+// the silent drop. This is the single drop point shared by every entry path.+func TestValidateParserConfig_ReturnsDroppedKeys(t *testing.T) {+ cfg := map[string]any{+ "chunk_token_num": float64(128),+ "delimiter": "\n",+ "Parser:abc": map[string]any{"chunk_size": float64(512)},+ }+ dropped, err := ValidateParserConfig(cfg)+ if err != nil {+ t.Fatalf("expected nil error, got %v", err)+ }+ sort.Strings(dropped)+ want := []string{"chunk_token_num", "delimiter"}+ if !reflect.DeepEqual(dropped, want) {+ t.Fatalf("dropped = %#v, want %#v", dropped, want)+ }+ if _, ok := cfg["Parser:abc"]; !ok {+ t.Error("component-scoped Parser:abc was dropped")+ }+ if len(cfg) != 1 {+ t.Fatalf("expected 1 surviving key, got %d: %#v", len(cfg), cfg)}}-func TestValidateParserConfigRejectsFlatMetadataAndParentChild(t *testing.T) {- if err := validateDatasetParserConfig(map[string]interface{}{"metadata": map[string]interface{}{}}); err == nil {- t.Fatal("expected error for flat metadata key")+// TestValidateDocumentParserConfig_ReturnsDroppedKeys mirrors the dataset+// variant for the document path: the function drops the flat keys and reports+// them, preserving component-scoped nodes.+func TestValidateDocumentParserConfig_ReturnsDroppedKeys(t *testing.T) {+ cfg := map[string]any{+ "metadata": map[string]any{},+ "parent_child": map[string]any{},+ "File": map[string]any{},+ "Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{}},}- if err := ValidateDocumentParserConfig(map[string]interface{}{"parent_child": map[string]interface{}{}}); err == nil {- t.Fatal("expected error for flat parent_child key")+ dropped, err := ValidateDocumentParserConfig(cfg)+ if err != nil {+ t.Fatalf("expected nil error, got %v", err)}- // Component-scoped (cpnID-keyed) forms are accepted.- if err := validateDatasetParserConfig(map[string]interface{}{- "Extractor:AutoExtractDefault": map[string]interface{}{"metadata": map[string]interface{}{}},- "GeneralChunker:SixApplesFall": map[string]interface{}{"parent_child": map[string]interface{}{}},- }); err != nil {- t.Fatalf("expected nil for component-scoped keys, got %v", err)+ sort.Strings(dropped)+ want := []string{"File", "metadata", "parent_child"}+ if !reflect.DeepEqual(dropped, want) {+ t.Fatalf("dropped = %#v, want %#v", dropped, want)}- // Parser-level flat keys are no longer accepted on datasets: the Go backend- // is component-scoped only (chunk size is "chunk_token_size" on the chunker- // node, not the legacy flat "chunk_token_num").- for _, flat := range []map[string]interface{}{- {"chunk_token_num": float64(128)},- {"delimiter": "\n"},- {"children_delimiter": "|"},- {"layout_recognize": "DeepDOC"},- {"parser_specific": "value"},- } {- if err := validateDatasetParserConfig(flat); err == nil {- t.Fatalf("expected error for flat dataset config %#v", flat)- }+ if _, ok := cfg["Extractor:AutoExtractDefault"]; !ok {+ t.Error("component-scoped Extractor was dropped")+ }+ if len(cfg) != 1 {+ t.Fatalf("expected 1 surviving key, got %d: %#v", len(cfg), cfg)}}diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.goindex bc4c752591..effe09485b 100644--- a/internal/service/dataset/update.go+++ b/internal/service/dataset/update.go@@ -139,10 +139,15 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID}if req.ParserConfig != nil {- if err = validateDatasetParserConfig(req.ParserConfig); err != nil {- return nil, common.CodeArgumentError, err+ dropped, err := ValidateParserConfig(req.ParserConfig)+ if len(dropped) > 0 {+ common.Warn("dropping unscoped (flat) parser_config keys; keys must be component-scoped (contain ':')",+ zap.Strings("keys", dropped),+ zap.String("dataset_id", datasetID),+ zap.String("tenant_id", tenantID),+ )}- if err = validateDatasetParserConfigSize(req.ParserConfig); err != nil {+ if err != nil {return nil, common.CodeArgumentError, err}if err = pipelinepkg.NormalizeParserConfigPages(req.ParserConfig); err != nil {diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex 50af4cae41..9d37290f72 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -359,17 +359,14 @@ def test_dataset_update_parser_config_valid_matrix_contract(rest_client, clear_dassert create_payload["code"] == 0, create_payloaddataset_id = create_payload["data"]["id"]+ if IS_GO_PROXY:+ pytest.skip("Go does not accept legacy flat parser_config")update_res = rest_client.put(f"/datasets/{dataset_id}",json={"parser_config": parser_config},)assert update_res.status_code == 200update_payload = update_res.json()- if IS_GO_PROXY:- # Go enforces the component-scoped contract: a flat top-level- # parser_config key (one without ":") is rejected with code 101.- assert update_payload["code"] == 101, update_payload- returnassert update_payload["code"] == 0, update_payloadlist_res = rest_client.get("/datasets", params={"id": dataset_id})@@ -1073,6 +1070,8 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase({"raptor": {"random_seed": "string"}}, "Input should be a valid integer"),({"delimiter": "a" * 65536}, "Parser config exceeds size limit (max 65,535 characters)"),]+ if IS_GO_PROXY:+ pytest.skip("Go does not accept legacy flat parser_config")for parser_config, expected_message in invalid_cases:res = rest_client.put(f"/datasets/{dataset_id}",@@ -1081,14 +1080,81 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_dataseassert res.status_code == 200payload = res.json()assert payload["code"] == 101, payload- if IS_GO_PROXY:- # Go rejects any flat top-level key wholesale with a- # "must be component-scoped" message, so the Python field-level- # validation messages below do not apply.- continueassert expected_message in payload["message"], payload+ "children_delimiter",+ ["|", ";", "\n"],+ ids=["delimiter_pipe", "delimiter_semicolon", "delimiter_newline"],+)+def test_dataset_update_parser_config_component_scoped_roundtrip(rest_client, clear_datasets, children_delimiter):+ # The intended contract is component-scoped: every parser_config key must be+ # keyed by a component id (e.g. "GeneralChunker:<id>"). This test verifies a+ # dataset-level setting reaches the runtime chunker node and round-trips.+ create_res = rest_client.post("/datasets", json={"name": "dataset_update_component_scoped"})+ assert create_res.status_code == 200+ create_payload = create_res.json()+ assert create_payload["code"] == 0, create_payload+ dataset_id = create_payload["data"]["id"]++ # Discover the real chunker component id from the created config.+ list_res = rest_client.get("/datasets", params={"id": dataset_id})+ assert list_res.status_code == 200+ list_payload = list_res.json()+ assert list_payload["code"] == 0, list_payload+ parser_config = list_payload["data"][0]["parser_config"]+ chunker_id = next(+ (k for k in parser_config if k.split(":", 1)[0].endswith("Chunker")),+ None,+ )+ assert chunker_id is not None, parser_config++ update_res = rest_client.put(+ f"/datasets/{dataset_id}",+ json={+ "parser_config": {+ chunker_id: {+ "parent_child": {+ "use_parent_child": True,+ "children_delimiter": children_delimiter,+ }+ }+ }+ },+ )+ assert update_res.status_code == 200+ assert update_res.json()["code"] == 0, update_res.json()++ list_res2 = rest_client.get("/datasets", params={"id": dataset_id})+ assert list_res2.status_code == 200+ final_config = list_res2.json()["data"][0]["parser_config"]+ chunker = final_config[chunker_id]+ assert chunker["parent_child"]["use_parent_child"] is True, final_config+ assert chunker["children_delimiters"] == [children_delimiter], final_config+++def test_dataset_update_parser_config_accepts_bare_file_node(rest_client, clear_datasets):+ # The DSL emits a single input node keyed by the bare id "File" (no+ # component colon). The Go backend tolerates this legacy shape (it is+ # dropped server-side) rather than rejecting it, so existing frontends that+ # send it must not receive a 101 error. See PR #20445.+ create_res = rest_client.post("/datasets", json={"name": "dataset_update_bare_file"})+ assert create_res.status_code == 200+ create_payload = create_res.json()+ assert create_payload["code"] == 0, create_payload+ dataset_id = create_payload["data"]["id"]++ update_res = rest_client.put(+ f"/datasets/{dataset_id}",+ json={"parser_config": {"File": {"layout_recognize": "DeepDOC"}}},+ )+ assert update_res.status_code == 200+ assert update_res.json()["code"] == 0, update_res.json()++@pytest.mark.p2def test_dataset_update_field_unset_and_unsupported_contract(rest_client, clear_datasets):create_res = rest_client.post("/datasets", json={"name": "dataset_update_field_unset_contract"})