Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Drop unscoped (flat) parser_config keys instead of rejecting them (#20445)

xugangqiang4ef147e7acc6
diff --git a/internal/handler/document.go b/internal/handler/document.go
index 6efddefbbc..eafc322eb9 100644
--- a/internal/handler/document.go
+++ b/internal/handler/document.go
@@ -1837,7 +1837,16 @@ func (h *DocumentHandler) UpdateDatasetDocument(c *gin.Context) {
return
}
if present["parser_config"] && req.ParserConfig != nil {
- if err := dataset.ValidateDocumentParserConfig(req.ParserConfig); err != nil {
+ dropped, err := dataset.ValidateDocumentParserConfig(req.ParserConfig)
+ if len(dropped) > 0 {
+ common.Warn("dropping unscoped (flat) parser_config keys; keys must be component-scoped (contain ':')",
+ zap.Strings("keys", dropped),
+ zap.String("user_id", user.ID),
+ zap.String("dataset_id", datasetID),
+ zap.String("document_id", documentID),
+ )
+ }
+ if err != nil {
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
return
}
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index 2835e89fd9..9e85b583c5 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -105,10 +105,14 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
}
if req.ParserConfig != nil {
- if err := validateDatasetParserConfig(req.ParserConfig); err != nil {
- return nil, common.CodeArgumentError, err
+ dropped, err := ValidateParserConfig(req.ParserConfig)
+ if len(dropped) > 0 {
+ common.Warn("dropping unscoped (flat) parser_config keys; keys must be component-scoped (contain ':')",
+ zap.Strings("keys", dropped),
+ zap.String("tenant_id", tenantID),
+ )
}
- if err := validateDatasetParserConfigSize(req.ParserConfig); err != nil {
+ if err != nil {
return nil, common.CodeArgumentError, err
}
if err := pipelinepkg.NormalizeParserConfigPages(req.ParserConfig); err != nil {
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 8d2ab4801b..8c0d706105 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -198,46 +198,53 @@ func validateDatasetParserConfigSize(parserConfig map[string]interface{}) error
return nil
}
-// validateDatasetParserConfig enforces the Go backend's "component-scoped only"
-// contract: every top-level parser_config key must be keyed by a component id
-// (it contains ":"), never a flat (non-component-scoped) key. The Python backend
-// is fully retired (SystemHandler.Language always returns "go"), so the dataset
-// parser_config is always built from a DSL whose params live on component nodes
-// (e.g. chunk size is "chunk_token_size" on the chunker node, not the legacy
-// flat "chunk_token_num"). Any flat key would be silently dropped by
-// CleanComponentParams downstream and is never consumed, so we reject it loudly
-// here instead.
-//
-// ValidateDocumentParserConfig delegates to this same check, so documents and
-// datasets share one contract: no flat keys are ever permitted on either path.
-func validateDatasetParserConfig(parserConfig map[string]interface{}) error {
+// DropUnscopedParserConfigKeys removes top-level parser_config keys that are
+// not component-scoped (i.e. do not contain ":"), since the Go backend only
+// consumes keys keyed by a component id (e.g. "Parser:abc123" or
+// "Extractor:AutoExtractDefault"). Flat keys are legacy transport artifacts
+// that downstream consumers (CleanComponentParams) never read, so they are
+// dropped rather than rejected. Only top-level keys are affected; a key nested
+// inside a component node (e.g. "Extractor:AutoExtractDefault.metadata") keeps
+// its existing name. It mutates the provided map in place and returns the names
+// of the dropped keys so callers can log the silent drop. A nil or empty map is
+// returned untouched with a nil result.
+func DropUnscopedParserConfigKeys(parserConfig map[string]any) []string {
if len(parserConfig) == 0 {
return nil
}
+ var dropped []string
for key := range parserConfig {
if !strings.Contains(key, ":") {
- return fmt.Errorf(
- "parser_config key %q must be component-scoped (e.g. under an Extractor or GeneralChunker node), not a flat top-level key",
- key,
- )
+ dropped = append(dropped, key)
}
}
- return nil
+ for _, key := range dropped {
+ delete(parserConfig, key)
+ }
+ return dropped
}
-// ValidateParserConfig validates the shared REST parser_config schema.
-func ValidateParserConfig(parserConfig map[string]interface{}) error {
- return validateDatasetParserConfig(parserConfig)
+// ValidateParserConfig validates the shared REST parser_config schema. Flat
+// (non-component-scoped) keys are no longer rejected: they are silently dropped
+// because downstream consumers never read them. The size limit is still
+// enforced. It returns the names of the dropped (unscoped) keys so the caller
+// can log the silent drop; this is the single drop point shared by every entry
+// path, so callers should not call DropUnscopedParserConfigKeys again.
+func ValidateParserConfig(parserConfig map[string]interface{}) ([]string, error) {
+ dropped := DropUnscopedParserConfigKeys(parserConfig)
+ return dropped, validateDatasetParserConfigSize(parserConfig)
}
// ValidateDocumentParserConfig validates the parser_config attached to a
// document. Documents follow the same component-scoped contract as datasets:
// every key must be scoped under a node id (e.g. "Extractor:AutoExtractDefault"
// or "GeneralChunker:SixApplesFall"). A document's Extractor/GeneralChunker
-// nodes come from the same pipeline DSL as the dataset, so there is no flat-key
-// fallback to keep. validateDatasetParserConfig enforces the no-flat-key rule.
-func ValidateDocumentParserConfig(parserConfig map[string]interface{}) error {
- return validateDatasetParserConfig(parserConfig)
+// nodes come from the same pipeline DSL as the dataset, so flat keys are dropped
+// (not kept) and the size limit is enforced. It returns the dropped key names
+// for logging, mirroring ValidateParserConfig.
+func ValidateDocumentParserConfig(parserConfig map[string]interface{}) ([]string, error) {
+ dropped := DropUnscopedParserConfigKeys(parserConfig)
+ return dropped, validateDatasetParserConfigSize(parserConfig)
}
// NormalizeDatasetID validates the dataset ID format and returns its
diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go
index 4ae3e07e6f..e33c3ec317 100644
--- a/internal/service/dataset/helpers_test.go
+++ b/internal/service/dataset/helpers_test.go
@@ -1,6 +1,8 @@
package dataset
import (
+ "reflect"
+ "sort"
"strings"
"testing"
@@ -205,113 +207,155 @@ func TestValidateDatasetParserConfigSize_OverLimit(t *testing.T) {
}
}
-func TestValidateDatasetParserConfig_RejectsFlatNullableOptionalFields(t *testing.T) {
- // A flat key is rejected regardless of its value (even nil): datasets are
- // component-scoped only, so "task_page_size"/"pages" must live on a node.
- for _, config := range []map[string]interface{}{
- {"task_page_size": nil},
- {"pages": nil},
- } {
- if err := validateDatasetParserConfig(config); err == nil {
- t.Fatalf("validateDatasetParserConfig(%#v): expected error for flat key, got nil", config)
- }
+// --- DropUnscopedParserConfigKeys ---
+
+func TestDropUnscopedParserConfigKeys_NilAndEmpty(t *testing.T) {
+ if got := DropUnscopedParserConfigKeys(nil); got != nil {
+ t.Fatalf("expected nil for nil map, got %#v", got)
+ }
+ if got := DropUnscopedParserConfigKeys(map[string]any{}); got != nil {
+ t.Fatalf("expected nil for empty map, got %#v", got)
}
}
-func TestValidateDatasetParserConfig_DelimiterType(t *testing.T) {
- err := validateDatasetParserConfig(map[string]interface{}{"delimiter": float64(1)})
- if err == nil {
- t.Fatal("expected error for flat delimiter key")
+func TestDropUnscopedParserConfigKeys_DropsFlatKeys(t *testing.T) {
+ cfg := map[string]any{
+ "File": map[string]any{},
+ "chunk_token_num": float64(128),
+ "delimiter": "\n",
+ "metadata": map[string]any{},
+ "parent_child": map[string]any{},
+ "Parser:abc": map[string]any{"chunk_size": float64(512)},
+ "Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{}},
+ }
+ dropped := DropUnscopedParserConfigKeys(cfg)
+ sort.Strings(dropped)
+ want := []string{"File", "chunk_token_num", "delimiter", "metadata", "parent_child"}
+ if !reflect.DeepEqual(dropped, want) {
+ t.Fatalf("dropped = %#v, want %#v", dropped, want)
+ }
+ // Component-scoped keys survive.
+ if _, ok := cfg["Parser:abc"]; !ok {
+ t.Error("component-scoped Parser:abc was dropped")
+ }
+ if _, ok := cfg["Extractor:AutoExtractDefault"]; !ok {
+ t.Error("component-scoped Extractor:AutoExtractDefault was dropped")
+ }
+ // A flat-looking key nested inside a component node is NOT a top-level flat
+ // key and must be preserved.
+ if inner, ok := cfg["Extractor:AutoExtractDefault"].(map[string]any); !ok || inner["metadata"] == nil {
+ t.Error("nested metadata inside a component node was dropped")
+ }
+ if len(cfg) != 2 {
+ t.Fatalf("expected 2 surviving keys, got %d: %#v", len(cfg), cfg)
}
}
-func TestValidateDocumentParserConfig_RejectsAnyFlatKey(t *testing.T) {
- // Documents follow the same component-scoped contract as datasets: no flat
- // top-level key is accepted (the Go backend scopes every setting under a
- // node id such as "Extractor:AutoExtractDefault" or "GeneralChunker:SixApplesFall").
- for _, flat := range []map[string]interface{}{
- {"parser_specific": "value"},
- {"delimiter": float64(1)},
- {"children_delimiter": "|"},
- {"metadata": map[string]interface{}{}},
- {"parent_child": map[string]interface{}{}},
- } {
- if err := ValidateDocumentParserConfig(flat); err == nil {
- t.Fatalf("expected error for flat document config %#v", flat)
- }
+func TestValidateParserConfig_DropsFlatKeys(t *testing.T) {
+ flat := map[string]any{"chunk_token_num": float64(128), "delimiter": "\n"}
+ dropped, err := ValidateParserConfig(flat)
+ if err != nil {
+ t.Fatalf("expected nil after dropping flat keys, got %v", err)
+ }
+ if len(dropped) != 2 {
+ t.Fatalf("expected 2 dropped keys, got %#v", dropped)
}
- // Component-scoped keys always pass.
- if err := ValidateDocumentParserConfig(map[string]interface{}{
- "Extractor:AutoExtractDefault": map[string]interface{}{"metadata": map[string]interface{}{}},
- "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(512)},
- "GeneralChunker:SixApplesFallPC": map[string]interface{}{"parent_child": map[string]interface{}{}},
- }); err != nil {
+ if len(flat) != 0 {
+ t.Fatalf("expected flat keys to be dropped, got %#v", flat)
+ }
+ // Component-scoped keys pass and are preserved.
+ scoped := map[string]any{
+ "GeneralChunker:SixApplesFall": map[string]any{"chunk_token_size": float64(512)},
+ "Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{}},
+ }
+ dropped, err = ValidateParserConfig(scoped)
+ if err != nil {
t.Fatalf("expected nil for component-scoped keys, got %v", err)
}
-}
-
-func TestValidateParserConfigRejectsFlatParentChildDelimiter(t *testing.T) {
- config := map[string]interface{}{"children_delimiter": "|"}
- // Datasets reject every flat key, including parser-level ones.
- if err := validateDatasetParserConfig(config); err == nil {
- t.Fatal("expected error for flat children_delimiter on dataset")
+ if len(dropped) != 0 {
+ t.Fatalf("expected no dropped keys for component-scoped config, got %#v", dropped)
}
- // Documents share the same no-flat-key contract.
- if err := ValidateDocumentParserConfig(config); err == nil {
- t.Fatal("expected error for flat children_delimiter on document")
+ if len(scoped) != 2 {
+ t.Fatalf("expected component-scoped keys preserved, got %#v", scoped)
}
}
-func TestValidateDatasetParserConfig_RejectsAnyFlatKey(t *testing.T) {
- for _, flat := range []map[string]interface{}{
- {"chunk_token_num": float64(128)},
- {"delimiter": "\n"},
- {"children_delimiter": "|"},
- {"layout_recognize": "DeepDOC"},
- {"auto_keywords": float64(0)},
- {"task_page_size": float64(1)},
- {"pages": nil},
+func TestValidateDocumentParserConfig_DropsFlatKeys(t *testing.T) {
+ for _, flat := range []map[string]any{
{"parser_specific": "value"},
+ {"delimiter": float64(1)},
+ {"children_delimiter": "|"},
+ {"metadata": map[string]any{}},
+ {"parent_child": map[string]any{}},
+ {"File": map[string]any{}},
} {
- if err := validateDatasetParserConfig(flat); err == nil {
- t.Fatalf("expected error for flat dataset config %#v", flat)
+ cfg := make(map[string]any, len(flat))
+ for k, v := range flat {
+ cfg[k] = v
+ }
+ dropped, err := ValidateDocumentParserConfig(cfg)
+ if err != nil {
+ t.Fatalf("expected nil after dropping flat key %#v, got %v", flat, err)
+ }
+ if len(dropped) != 1 {
+ t.Fatalf("expected exactly 1 dropped key for %#v, got %#v", flat, dropped)
+ }
+ if len(cfg) != 0 {
+ t.Fatalf("expected flat key dropped, got %#v", cfg)
}
}
- // Component-scoped keys (containing ":") always pass.
- if err := validateDatasetParserConfig(map[string]interface{}{
- "GeneralChunker:SixApplesFall": map[string]interface{}{"chunk_token_size": float64(512)},
- }); err != nil {
- t.Fatalf("expected nil for component-scoped keys, got %v", err)
+}
+
+// TestValidateParserConfig_ReturnsDroppedKeys asserts that ValidateParserConfig
+// returns the names of the flat (unscoped) keys it dropped so the caller can log
+// the silent drop. This is the single drop point shared by every entry path.
+func TestValidateParserConfig_ReturnsDroppedKeys(t *testing.T) {
+ cfg := map[string]any{
+ "chunk_token_num": float64(128),
+ "delimiter": "\n",
+ "Parser:abc": map[string]any{"chunk_size": float64(512)},
+ }
+ dropped, err := ValidateParserConfig(cfg)
+ if err != nil {
+ t.Fatalf("expected nil error, got %v", err)
+ }
+ sort.Strings(dropped)
+ want := []string{"chunk_token_num", "delimiter"}
+ if !reflect.DeepEqual(dropped, want) {
+ t.Fatalf("dropped = %#v, want %#v", dropped, want)
+ }
+ if _, ok := cfg["Parser:abc"]; !ok {
+ t.Error("component-scoped Parser:abc was dropped")
+ }
+ if len(cfg) != 1 {
+ t.Fatalf("expected 1 surviving key, got %d: %#v", len(cfg), cfg)
}
}
-func TestValidateParserConfigRejectsFlatMetadataAndParentChild(t *testing.T) {
- if err := validateDatasetParserConfig(map[string]interface{}{"metadata": map[string]interface{}{}}); err == nil {
- t.Fatal("expected error for flat metadata key")
+// TestValidateDocumentParserConfig_ReturnsDroppedKeys mirrors the dataset
+// variant for the document path: the function drops the flat keys and reports
+// them, preserving component-scoped nodes.
+func TestValidateDocumentParserConfig_ReturnsDroppedKeys(t *testing.T) {
+ cfg := map[string]any{
+ "metadata": map[string]any{},
+ "parent_child": map[string]any{},
+ "File": map[string]any{},
+ "Extractor:AutoExtractDefault": map[string]any{"metadata": map[string]any{}},
}
- if err := ValidateDocumentParserConfig(map[string]interface{}{"parent_child": map[string]interface{}{}}); err == nil {
- t.Fatal("expected error for flat parent_child key")
+ dropped, err := ValidateDocumentParserConfig(cfg)
+ if err != nil {
+ t.Fatalf("expected nil error, got %v", err)
}
- // Component-scoped (cpnID-keyed) forms are accepted.
- if err := validateDatasetParserConfig(map[string]interface{}{
- "Extractor:AutoExtractDefault": map[string]interface{}{"metadata": map[string]interface{}{}},
- "GeneralChunker:SixApplesFall": map[string]interface{}{"parent_child": map[string]interface{}{}},
- }); err != nil {
- t.Fatalf("expected nil for component-scoped keys, got %v", err)
+ sort.Strings(dropped)
+ want := []string{"File", "metadata", "parent_child"}
+ if !reflect.DeepEqual(dropped, want) {
+ t.Fatalf("dropped = %#v, want %#v", dropped, want)
}
- // Parser-level flat keys are no longer accepted on datasets: the Go backend
- // is component-scoped only (chunk size is "chunk_token_size" on the chunker
- // node, not the legacy flat "chunk_token_num").
- for _, flat := range []map[string]interface{}{
- {"chunk_token_num": float64(128)},
- {"delimiter": "\n"},
- {"children_delimiter": "|"},
- {"layout_recognize": "DeepDOC"},
- {"parser_specific": "value"},
- } {
- if err := validateDatasetParserConfig(flat); err == nil {
- t.Fatalf("expected error for flat dataset config %#v", flat)
- }
+ if _, ok := cfg["Extractor:AutoExtractDefault"]; !ok {
+ t.Error("component-scoped Extractor was dropped")
+ }
+ if len(cfg) != 1 {
+ t.Fatalf("expected 1 surviving key, got %d: %#v", len(cfg), cfg)
}
}
diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.go
index bc4c752591..effe09485b 100644
--- a/internal/service/dataset/update.go
+++ b/internal/service/dataset/update.go
@@ -139,10 +139,15 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
}
if req.ParserConfig != nil {
- if err = validateDatasetParserConfig(req.ParserConfig); err != nil {
- return nil, common.CodeArgumentError, err
+ dropped, err := ValidateParserConfig(req.ParserConfig)
+ if len(dropped) > 0 {
+ common.Warn("dropping unscoped (flat) parser_config keys; keys must be component-scoped (contain ':')",
+ zap.Strings("keys", dropped),
+ zap.String("dataset_id", datasetID),
+ zap.String("tenant_id", tenantID),
+ )
}
- if err = validateDatasetParserConfigSize(req.ParserConfig); err != nil {
+ if err != nil {
return nil, common.CodeArgumentError, err
}
if err = pipelinepkg.NormalizeParserConfigPages(req.ParserConfig); err != nil {
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index 50af4cae41..9d37290f72 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -359,17 +359,14 @@ def test_dataset_update_parser_config_valid_matrix_contract(rest_client, clear_d
assert create_payload["code"] == 0, create_payload
dataset_id = create_payload["data"]["id"]
+ if IS_GO_PROXY:
+ pytest.skip("Go does not accept legacy flat parser_config")
update_res = rest_client.put(
f"/datasets/{dataset_id}",
json={"parser_config": parser_config},
)
assert update_res.status_code == 200
update_payload = update_res.json()
- if IS_GO_PROXY:
- # Go enforces the component-scoped contract: a flat top-level
- # parser_config key (one without ":") is rejected with code 101.
- assert update_payload["code"] == 101, update_payload
- return
assert update_payload["code"] == 0, update_payload
list_res = rest_client.get("/datasets", params={"id": dataset_id})
@@ -1073,6 +1070,8 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase
({"raptor": {"random_seed": "string"}}, "Input should be a valid integer"),
({"delimiter": "a" * 65536}, "Parser config exceeds size limit (max 65,535 characters)"),
]
+ if IS_GO_PROXY:
+ pytest.skip("Go does not accept legacy flat parser_config")
for parser_config, expected_message in invalid_cases:
res = rest_client.put(
f"/datasets/{dataset_id}",
@@ -1081,14 +1080,81 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase
assert res.status_code == 200
payload = res.json()
assert payload["code"] == 101, payload
- if IS_GO_PROXY:
- # Go rejects any flat top-level key wholesale with a
- # "must be component-scoped" message, so the Python field-level
- # validation messages below do not apply.
- continue
assert expected_message in payload["message"], payload
+ "children_delimiter",
+ ["|", ";", "\n"],
+ ids=["delimiter_pipe", "delimiter_semicolon", "delimiter_newline"],
+)
+def test_dataset_update_parser_config_component_scoped_roundtrip(rest_client, clear_datasets, children_delimiter):
+ # The intended contract is component-scoped: every parser_config key must be
+ # keyed by a component id (e.g. "GeneralChunker:<id>"). This test verifies a
+ # dataset-level setting reaches the runtime chunker node and round-trips.
+ create_res = rest_client.post("/datasets", json={"name": "dataset_update_component_scoped"})
+ assert create_res.status_code == 200
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, create_payload
+ dataset_id = create_payload["data"]["id"]
+
+ # Discover the real chunker component id from the created config.
+ list_res = rest_client.get("/datasets", params={"id": dataset_id})
+ assert list_res.status_code == 200
+ list_payload = list_res.json()
+ assert list_payload["code"] == 0, list_payload
+ parser_config = list_payload["data"][0]["parser_config"]
+ chunker_id = next(
+ (k for k in parser_config if k.split(":", 1)[0].endswith("Chunker")),
+ None,
+ )
+ assert chunker_id is not None, parser_config
+
+ update_res = rest_client.put(
+ f"/datasets/{dataset_id}",
+ json={
+ "parser_config": {
+ chunker_id: {
+ "parent_child": {
+ "use_parent_child": True,
+ "children_delimiter": children_delimiter,
+ }
+ }
+ }
+ },
+ )
+ assert update_res.status_code == 200
+ assert update_res.json()["code"] == 0, update_res.json()
+
+ list_res2 = rest_client.get("/datasets", params={"id": dataset_id})
+ assert list_res2.status_code == 200
+ final_config = list_res2.json()["data"][0]["parser_config"]
+ chunker = final_config[chunker_id]
+ assert chunker["parent_child"]["use_parent_child"] is True, final_config
+ assert chunker["children_delimiters"] == [children_delimiter], final_config
+
+
+def test_dataset_update_parser_config_accepts_bare_file_node(rest_client, clear_datasets):
+ # The DSL emits a single input node keyed by the bare id "File" (no
+ # component colon). The Go backend tolerates this legacy shape (it is
+ # dropped server-side) rather than rejecting it, so existing frontends that
+ # send it must not receive a 101 error. See PR #20445.
+ create_res = rest_client.post("/datasets", json={"name": "dataset_update_bare_file"})
+ assert create_res.status_code == 200
+ create_payload = create_res.json()
+ assert create_payload["code"] == 0, create_payload
+ dataset_id = create_payload["data"]["id"]
+
+ update_res = rest_client.put(
+ f"/datasets/{dataset_id}",
+ json={"parser_config": {"File": {"layout_recognize": "DeepDOC"}}},
+ )
+ assert update_res.status_code == 200
+ assert update_res.json()["code"] == 0, update_res.json()
+
+
@pytest.mark.p2
def test_dataset_update_field_unset_and_unsupported_contract(rest_client, clear_datasets):
create_res = rest_client.post("/datasets", json={"name": "dataset_update_field_unset_contract"})