RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor: update dataset API should not have ext (#19272)
Wang Qif8b0a1de1970
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex dce0301f47..e1bebc3a2c 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -668,8 +668,6 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):if isinstance(parser_config, dict):if "compilation_template_group_id" in parser_config:group_ids = _group_ids(parser_config.get("compilation_template_group_id"))- elif isinstance(parser_config.get("ext"), dict):- group_ids = _group_ids(parser_config["ext"].get("compilation_template_group_id"))configured_ids: list[str] = []seen_configured_ids: set[str] = set()diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.pyindex 0b81320562..dff073bf73 100644--- a/api/apps/restful_apis/document_api.py+++ b/api/apps/restful_apis/document_api.py@@ -91,12 +91,6 @@ def _normalize_legacy_raptor_config(req: dict) -> None:returnnormalized_fields = []- legacy_ext = raptor.pop("ext", None)- if legacy_ext is not None:- normalized_fields.append("ext")- if isinstance(legacy_ext, dict) and "clustering_threshold" in legacy_ext and "clustering_threshold" not in raptor:- raptor["clustering_threshold"] = legacy_ext["clustering_threshold"]- normalized_fields.append("ext.clustering_threshold")for field in ("threshold", "clustering_method", "tree_builder"):if field in raptor:raptor.pop(field)@@ -116,13 +110,10 @@ def _normalize_parser_config_compilation_template_group_ids(parser_config) -> boif not isinstance(parser_config, dict):return False- if "compilation_template_group_id" not in parser_config and not (isinstance(parser_config.get("ext"), dict) and "compilation_template_group_id" in parser_config["ext"]):+ if "compilation_template_group_id" not in parser_config:return Falsegroup_ids = _parser_config_compilation_template_group_ids(parser_config)parser_config["compilation_template_group_id"] = group_ids- ext = parser_config.get("ext")- if isinstance(ext, dict) and "compilation_template_group_id" in ext:- ext["compilation_template_group_id"] = group_idsreturn True@@ -291,7 +282,6 @@ async def update_document(tenant_id, dataset_id, document_id):# Changing the document-scoped knowledge compilation template group must# not remove the existing chunks.if update_doc_req.parser_config:- req["parser_config"].update(update_doc_req.parser_config.ext)_normalize_parser_config_compilation_template_group_ids(req["parser_config"])DocumentService.update_parser_config(doc.id, req["parser_config"])diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex d8ba5d0d3b..6bbe4f8e5b 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -95,9 +95,6 @@ async def create_dataset(tenant_id: str, req: dict)::param req: dataset creation request:return: (success, result) or (success, error_message)"""- # Extract ext field for additional parameters- ext_fields = req.pop("ext", {})-# Map auto_metadata_config (if provided) into parser_config structureauto_meta = req.pop("auto_metadata_config", {})if auto_meta:@@ -116,8 +113,6 @@ async def create_dataset(tenant_id: str, req: dict):parser_cfg["metadata"] = fieldsparser_cfg["enable_metadata"] = auto_meta.get("enabled", True)req["parser_config"] = parser_cfg- req.update(ext_fields)-e, create_dict = KnowledgebaseService.create_with_name(name=req.pop("name", None), tenant_id=tenant_id, parser_id=req.pop("parser_id", None), **req)if not e:@@ -320,9 +315,6 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):if kb is None:return False, "Invalid Dataset ID"- # Extract ext field for additional parameters- ext_fields = req.pop("ext", {})-# Map auto_metadata_config into parser_config if presentauto_meta = req.pop("auto_metadata_config", {})if auto_meta:@@ -342,9 +334,6 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):parser_cfg["enable_metadata"] = auto_meta.get("enabled", True)req["parser_config"] = parser_cfg- # Merge ext fields with req- req.update(ext_fields)-# Extract connectors from requestconnectors = []if "connectors" in req:@@ -362,10 +351,7 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):req["parser_config"]["enable_children"] = Falsereq["parser_config"]["parent_child"] = {}- parser_config = req["parser_config"]- req_ext_fields = parser_config.pop("ext", {})- parser_config.update(req_ext_fields)- req["parser_config"] = deep_merge(kb.parser_config, parser_config)+ req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])if (chunk_method := req.get("parser_id")) and chunk_method != kb.parser_id:if not req.get("parser_config"):@@ -439,9 +425,8 @@ def list_datasets(tenant_id: str, args: dict):name = args.get("name")page = int(args.get("page", 1))page_size = int(args.get("page_size", 30))- ext_fields = args.get("ext", {})- parser_id = ext_fields.get("parser_id")- keywords = ext_fields.get("keywords", "")+ parser_id = args.get("parser_id")+ keywords = args.get("keywords", "")orderby = args.get("orderby", "create_time")desc_arg = args.get("desc", "true")if isinstance(desc_arg, str):@@ -463,7 +448,7 @@ def list_datasets(tenant_id: str, args: dict):kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)if not kbs:return False, f"User '{tenant_id}' lacks permission for dataset '{name}'"- owner_ids = [owner_id.strip() for owner_id in ext_fields.get("owner_ids", []) if isinstance(owner_id, str) and owner_id.strip()]+ owner_ids = [owner_id.strip() for owner_id in args.get("owner_ids", []) if isinstance(owner_id, str) and owner_id.strip()]if owner_ids:tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)allowed_tenant_ids = {m["tenant_id"] for m in tenants}diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex 8c816833c8..6a099a2889 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -176,15 +176,6 @@ def validate_and_parse_request_args(request: Request, validator: type[BaseModel]if query_name in request.args and _is_list_annotation(field_info.annotation):args[query_name] = [value for item in request.args.getlist(query_name) for value in item.split(",") if value]- # Handle ext parameter: parse JSON string to dict if it's a string- if "ext" in args and isinstance(args["ext"], str):- import json-- try:- args["ext"] = json.loads(args["ext"])- except json.JSONDecodeError:- logging.debug("Failed to decode query arg 'ext' as JSON; passing raw value to validator")-try:if extras is not None:args.update(extras)@@ -380,14 +371,6 @@ class RaptorConfig(Base):normalized = dict(value)changed_fields = []- legacy_ext = normalized.pop("ext", None)- if legacy_ext is not None:- changed_fields.append("ext")- if isinstance(legacy_ext, dict) and normalized.get("clustering_threshold") is None:- if "clustering_threshold" in legacy_ext:- normalized["clustering_threshold"] = legacy_ext["clustering_threshold"]- changed_fields.append("ext.clustering_threshold")-for field in ("threshold", "clustering_method", "tree_builder"):if field in normalized:normalized.pop(field)@@ -450,6 +433,8 @@ TableColumnRole = Literal["indexing", "metadata", "both"]class ParserConfig(Base):"""Complete parser configuration accepted by dataset APIs."""+ model_config = ConfigDict(extra="allow", strict=True)+auto_keywords: Annotated[int, Field(default=0, ge=0, le=32)]auto_questions: Annotated[int, Field(default=0, ge=0, le=10)]chunk_token_num: Annotated[int, Field(default=512, ge=1, le=2048)]@@ -467,7 +452,6 @@ class ParserConfig(Base):task_page_size: Annotated[int | None, Field(default=None, ge=1)]pages: Annotated[list[list[int]] | None, Field(default=None)]compilation_template_group_id: Annotated[list[str], Field(default_factory=list)]- ext: Annotated[dict, Field(default={})]# Table parser: column name -> "indexing" | "metadata" | "both". Absence => all columns "both".# Table parser: "auto" = all columns both (default), "manual" = use table_column_roles. None → treated as "auto".table_column_mode: Annotated[Literal["auto", "manual"] | None, Field(default=None)]@@ -476,6 +460,13 @@ class ParserConfig(Base):# Table parser: list of column names (set by backend after first parse; used by frontend for role selector).table_column_names: Annotated[list[str] | None, Field(default=None)]+ @model_validator(mode="before")+ @classmethod+ def reject_ext(cls, value: Any) -> Any:+ if isinstance(value, dict) and "ext" in value:+ raise ValueError("parser_config.ext is not supported; send parser configuration fields directly")+ return value+@field_validator("table_column_roles", mode="before")@classmethoddef legacy_vectorize_table_column_role(cls, v: Any) -> Any:@@ -579,9 +570,9 @@ class CreateDatasetReq(Base):parse_type: Annotated[int | None, Field(default=None, ge=0, le=64)]pipeline_id: Annotated[str | None, Field(default=None, min_length=32, max_length=32, serialization_alias="pipeline_id")]chunk_method: Annotated[str | None, Field(default=None, serialization_alias="parser_id")]+ language: Annotated[str | None, Field(default=None, max_length=32)]parser_config: Annotated[ParserConfig | None, Field(default=None)]auto_metadata_config: Annotated[AutoMetadataConfig | None, Field(default=None)]- ext: Annotated[dict, Field(default={})]@field_validator("pipeline_id", mode="before")@classmethod@@ -850,7 +841,6 @@ class UpdateDatasetReq(CreateDatasetReq):dataset_id: Annotated[str, Field(...)]name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=DATASET_NAME_LIMIT), Field(default="")]pagerank: Annotated[int, Field(default=0, ge=0, le=100)]- language: Annotated[str | None, Field(default=None, max_length=32)]connectors: Annotated[list[dict[str, Any]], Field(default_factory=list)]@field_validator("dataset_id", mode="before")@@ -1066,7 +1056,9 @@ class ListDatasetReq(BaseListReq):ids: Annotated[list[str] | None, Field(default=None, max_length=REST_API_MAX_IDS)]include_parsing_status: Annotated[bool, Field(default=False)]- ext: Annotated[dict, Field(default={})]+ keywords: Annotated[str, Field(default="")]+ owner_ids: Annotated[list[str], Field(default_factory=list)]+ parser_id: Annotated[str | None, Field(default=None)]@field_validator("ids", mode="after")@classmethoddiff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.mdindex cfce337b4a..cecf18917c 100644--- a/docs/references/http_api_reference.md+++ b/docs/references/http_api_reference.md@@ -596,17 +596,9 @@ curl --request POST \- For PDFs only.- Defaults to `12`- Minimum: `1`- - `"raptor"`: `object` RAPTOR-specific settings.- - Defaults to: `{"use_raptor": false}`- - `"graphrag"`: `object` GRAPHRAG-specific settings.- - Defaults to: `{"use_graphrag": false}`- `"parent_child"`: `object` Parent-child chunking settings. When enabled, each chunk is further split into smaller child chunks using `children_delimiter`. At retrieval time, matched child chunks are replaced by their parent's full text before being passed to the LLM, giving precise vector matching with broader context.- `"use_parent_child"`: `bool` Whether to enable parent-child chunking. Defaults to `false`.- `"children_delimiter"`: `string` The delimiter used to split a parent chunk into child chunks. Only takes effect when `"use_parent_child"` is `true`. Defaults to `"\n"`.- - If `"chunk_method"` is `"qa"`, `"manual"`, `"paper"`, `"book"`, `"laws"`, or `"presentation"`, the `"parser_config"` object contains the following attribute:- - `"raptor"`: `object` RAPTOR-specific settings.- - Defaults to: `{"use_raptor": false}`.- - If `"chunk_method"` is `"table"`, `"picture"`, `"one"`, or `"email"`, `"parser_config"` is an empty JSON object.- `"parse_type"`: (*Body parameter*), `int`The ingestion pipeline parse type identifier, i.e., the number of parsers in your **Parser** component.@@ -653,11 +645,8 @@ Success:"chunk_token_num": 128,"delimiter": "\\n!?;。;!?","html4excel": false,- "layout_recognize": "DeepDOC",- "raptor": {- "use_raptor": false- }- },+ "layout_recognize": "DeepDOC"+ },"permission": "me","similarity_threshold": 0.2,"status": "1",@@ -856,10 +845,7 @@ curl --request PUT \- `"parent_child"`: `object` Parent-child chunking settings. When enabled, each chunk is further split into smaller child chunks using `children_delimiter`. At retrieval time, matched child chunks are replaced by their parent's full text before being passed to the LLM, giving precise vector matching with broader context.- `"use_parent_child"`: `bool` Whether to enable parent-child chunking. Defaults to `false`.- `"children_delimiter"`: `string` The delimiter used to split a parent chunk into child chunks. Only takes effect when `"use_parent_child"` is `true`. Defaults to `"\n"`.- - If `"chunk_method"` is `"qa"`, `"manual"`, `"paper"`, `"book"`, `"laws"`, or `"presentation"`, the `"parser_config"` object contains the following attribute:- - `"raptor"`: `object` RAPTOR-specific settings.- - Defaults to: `{"use_raptor": false}`.- - If `"chunk_method"` is `"table"`, `"picture"`, `"one"`, or `"email"`, `"parser_config"` is an empty JSON object.+#### Response@@ -1003,9 +989,7 @@ Success (with `include_parsing_status=true`):"language": "English","name": "Test Dataset","parser_config": {- "graphrag": { "use_graphrag": false },- "llm_id": "deepseek-chat@DeepSeek",- "raptor": { "use_raptor": false }+ "llm_id": "deepseek-chat@DeepSeek"},"permission": "me","running_count": 0,@@ -1128,10 +1112,7 @@ Success:"chunk_token_num": 128,"delimiter": "\\n","html4excel": false,- "layout_recognize": true,- "raptor": {- "use_raptor": false- }+ "layout_recognize": true},"run": "UNSTART","size": 17966,@@ -1217,10 +1198,6 @@ curl --request PATCH \- `"html4excel"`: Indicates whether to convert Excel documents into HTML format. Defaults to `false`.- `"delimiter"`: Defaults to `"\n"`.- `"task_page_size"`: Defaults to `12`. For PDF only.- - `"raptor"`: RAPTOR-specific settings. Defaults to: `{"use_raptor": false}`.- - If `"chunk_method"` is `"qa"`, `"manual"`, `"paper"`, `"book"`, `"laws"`, or `"presentation"`, the `"parser_config"` object contains the following attribute:- - `"raptor"`: RAPTOR-specific settings. Defaults to: `{"use_raptor": false}`.- - If `"chunk_method"` is `"table"`, `"picture"`, `"one"`, or `"email"`, `"parser_config"` is an empty JSON object.- `"enabled"`: (*Body parameter*), `integer`Whether the document should be **available** in the knowledge base.- `1` → (available)@@ -1275,32 +1252,10 @@ Success:"auto_keywords": 0,"auto_questions": 0,"topn_tags": 3,-"layout_recognize": "DeepDOC","html4excel": false,"image_context_size": 0,- "table_context_size": 0,-- "graphrag": {- "use_graphrag": true,- "method": "light",- "entity_types": [- "organization",- "person",- "geo",- "event",- "category"- ]- },-- "raptor": {- "use_raptor": true,- "max_cluster": 64,- "max_token": 256,- "threshold": 0.1,- "random_seed": 0,- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."- }+ "table_context_size": 0},"meta_fields": {},@@ -1931,10 +1886,7 @@ Success:"chunk_token_num": 128,"delimiter": "\\n","html4excel": false,- "layout_recognize": true,- "raptor": {- "use_raptor": false- }+ "layout_recognize": true},"process_begin_at": "Thu, 24 Oct 2024 09:56:44 GMT","process_duration": 0.54213,diff --git a/internal/handler/dataset.go b/internal/handler/dataset.goindex 2074fc29ec..3426e97729 100644--- a/internal/handler/dataset.go+++ b/internal/handler/dataset.go@@ -50,12 +50,6 @@ type searchDatasetService interface {SearchDataset(ctx context.Context, datasetID, userID string, req *service.SearchDatasetRequest) (*service.SearchDatasetsResponse, error)}-type listDatasetsExt struct {- Keywords string `json:"keywords,omitempty"`- OwnerIDs []string `json:"owner_ids,omitempty"`- ParserID string `json:"parser_id,omitempty"`-}-// NewDatasetsHandler creates a new datasets' handler.func NewDatasetsHandler(datasetsService *dataset.DatasetService, metadataService *service.MetadataService) *DatasetsHandler {h := &DatasetsHandler{@@ -144,20 +138,15 @@ func (h *DatasetsHandler) ListDatasets(c *gin.Context) {desc = parsed}- keywords := ""- parserID := ""+ keywords := c.Query("keywords")+ parserID := c.Query("parser_id")var ownerIDs []string-- // ext keeps the same compatibility payload as the Python REST API.- if extStr := c.Query("ext"); extStr != "" {- var ext listDatasetsExt- if err := json.Unmarshal([]byte(extStr), &ext); err != nil {- common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())- return+ for _, item := range c.QueryArray("owner_ids") {+ for _, ownerID := range strings.Split(item, ",") {+ if ownerID = strings.TrimSpace(ownerID); ownerID != "" {+ ownerIDs = append(ownerIDs, ownerID)+ }}- keywords = ext.Keywords- parserID = ext.ParserID- ownerIDs = ext.OwnerIDs}// Mirror pydantic: a present-but-empty id fails UUID validation.@@ -236,10 +225,14 @@ func (h *DatasetsHandler) CreateDataset(c *gin.Context) {return}- bodyBytes, _, ok := parseJSONRequestObject(c)+ bodyBytes, raw, ok := parseJSONRequestObject(c)if !ok {return}+ if _, exists := raw["ext"]; exists {+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Extra inputs are not permitted: ext")+ return+ }var req service.CreateDatasetRequestif err := json.Unmarshal(bodyBytes, &req); err != nil {@@ -357,12 +350,12 @@ func pythonJSONTypeName(v interface{}) string {}// listDatasetsAllowedParams mirrors the query field set of Python's-// ListDatasetReq (BaseListReq + include_parsing_status/ext; `type` is handled+// ListDatasetReq (BaseListReq + dataset filters; `type` is handled// before validation in the Python endpoint).var listDatasetsAllowedParams = map[string]bool{"id": true, "ids": true, "name": true, "page": true, "page_size": true,"orderby": true, "desc": true, "include_parsing_status": true,- "ext": true, "type": true,+ "keywords": true, "owner_ids": true, "parser_id": true, "type": true,}// updateDatasetAllowedFields mirrors the field set of Python's UpdateDatasetReq@@ -370,7 +363,7 @@ var listDatasetsAllowedParams = map[string]bool{var updateDatasetAllowedFields = map[string]bool{"name": true, "avatar": true, "description": true, "embedding_model": true,"permission": true, "parse_type": true, "pipeline_id": true, "chunk_method": true,- "parser_id": true, "parser_config": true, "auto_metadata_config": true, "ext": true,+ "parser_id": true, "parser_config": true, "auto_metadata_config": true,"dataset_id": true, "pagerank": true, "language": true, "connectors": true,}@@ -421,6 +414,21 @@ func (h *DatasetsHandler) UpdateDataset(c *gin.Context) {return}}+ if parserConfig, ok := providedFields["parser_config"]; ok {+ var config map[string]interface{}+ if json.Unmarshal(parserConfig, &config) == nil {+ if _, ok := config["ext"]; ok {+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "parser_config.ext is not supported; send parser configuration fields directly")+ return+ }+ if raptor, ok := config["raptor"].(map[string]interface{}); ok {+ if _, ok := raptor["ext"]; ok {+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "parser_config.raptor.ext is not supported; send RAPTOR configuration fields directly")+ return+ }+ }+ }+ }}ctx := c.Request.Context()diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex b6ea5b8d05..34620f9337 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -50,6 +50,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDparserID := string(entity.ParserTypeGeneral)permission := "me"embeddingModel := ""+ var language *stringpipelineID := req.PipelineIDif req.Permission != nil {@@ -82,6 +83,13 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDreturn nil, common.CodeDataError, err}}+ if req.Language != nil {+ normalizedLanguage := strings.TrimSpace(*req.Language)+ if len(normalizedLanguage) > 32 {+ return nil, common.CodeDataError, errors.New("String should have at most 32 characters")+ }+ language = &normalizedLanguage+ }if pipelineID != nil && strings.TrimSpace(*pipelineID) != "" {if ok, err := canvasAccessibleForUser(ctx, tenantID, strings.TrimSpace(*pipelineID)); err != nil {@@ -139,6 +147,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateDPipelineID: pipelineID,ParserConfig: entity.JSONMap(parserConfigMap),Permission: permission,+ Language: language,EmbdID: embdID,TenantEmbdID: stringPtrIfNotEmpty(tenantEmbdID),Status: &status,diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 7e05690d7d..0d192827dc 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -332,28 +332,6 @@ func cloneJSONValue(value interface{}) interface{} {}}-func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{} {- if ext == nil {- return nil- }- updates := make(map[string]interface{}, len(ext))- for key, value := range ext {- switch key {- case "chunk_method":- updates["parser_id"] = value- case "token_num", "chunk_num", "parser_config":- continue- case "pagerank":- if v, ok := value.(float64); ok {- updates[key] = int64(v)- }- default:- updates[key] = value- }- }- return updates-}-func normalizeMetadataConfigFields(fields []service.MetadataConfigField, fieldName string) ([]map[string]interface{}, error) {normalizedFields := make([]map[string]interface{}, 0, len(fields))for i, field := range fields {diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.goindex c714e4c53a..be8d4c28ba 100644--- a/internal/service/dataset/helpers_test.go+++ b/internal/service/dataset/helpers_test.go@@ -242,74 +242,6 @@ func TestNormalizeDatasetID_StripsHyphens(t *testing.T) {}}-// --- normalizeDatasetUpdateExt -----func TestNormalizeDatasetUpdateExt_Nil(t *testing.T) {- if result := normalizeDatasetUpdateExt(nil); result != nil {- t.Fatalf("expected nil for nil input")- }-}--func TestNormalizeDatasetUpdateExt_PassesThrough(t *testing.T) {- ext := map[string]interface{}{- "description": "test",- "language": "English",- }- result := normalizeDatasetUpdateExt(ext)- if result["description"] != "test" {- t.Errorf("expected description preserved, got %v", result["description"])- }- if result["language"] != "English" {- t.Errorf("expected language preserved, got %v", result["language"])- }-}--func TestNormalizeDatasetUpdateExt_RenamesChunkMethod(t *testing.T) {- ext := map[string]interface{}{- "chunk_method": "book",- }- result := normalizeDatasetUpdateExt(ext)- if result["parser_id"] != "book" {- t.Errorf("expected parser_id=book, got %v", result["parser_id"])- }- if _, ok := result["chunk_method"]; ok {- t.Error("expected chunk_method to be renamed to parser_id")- }-}--func TestNormalizeDatasetUpdateExt_SkipsTokenAndChunkNum(t *testing.T) {- ext := map[string]interface{}{- "token_num": float64(1000),- "chunk_num": float64(50),- "parser_config": map[string]interface{}{"key": "val"},- }- result := normalizeDatasetUpdateExt(ext)- if len(result) != 0 {- t.Errorf("expected empty map, got %v", result)- }-}--func TestNormalizeDatasetUpdateExt_ConvertsPagerank(t *testing.T) {- ext := map[string]interface{}{- "pagerank": float64(3),- }- result := normalizeDatasetUpdateExt(ext)- if result["pagerank"] != int64(3) {- t.Errorf("expected pagerank=int64(3), got %T(%v)", result["pagerank"], result["pagerank"])- }-}--func TestNormalizeDatasetUpdateExt_NonFloatPagerankSkipped(t *testing.T) {- // Non-float64 pagerank values are not convertible and are dropped.- ext := map[string]interface{}{- "pagerank": "auto",- }- result := normalizeDatasetUpdateExt(ext)- if _, ok := result["pagerank"]; ok {- t.Error("expected non-float pagerank to be skipped")- }-}-// --- normalizeMetadataConfigFields ---func TestNormalizeMetadataConfigFields_EmptyKey(t *testing.T) {diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.goindex e9080b0905..99c8a8262c 100644--- a/internal/service/dataset_types.go+++ b/internal/service/dataset_types.go@@ -134,6 +134,7 @@ type MetadataConfigRequest struct {type CreateDatasetRequest struct {Name string `json:"name" binding:"required"`EmbeddingModel *string `json:"embedding_model,omitempty"`+ Language *string `json:"language,omitempty"`Permission *string `json:"permission,omitempty"`ParserID *string `json:"parser_id,omitempty"`PipelineID *string `json:"pipeline_id,omitempty"`diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex d4678ba9bf..4480e7983f 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -444,9 +444,6 @@ def _parser_config_compilation_template_group_ids(parser_config) -> list[str]:return []if "compilation_template_group_id" in parser_config:return _normalize(parser_config.get("compilation_template_group_id"))- ext = parser_config.get("ext")- if isinstance(ext, dict):- return _normalize(ext.get("compilation_template_group_id"))return []@@ -976,7 +973,6 @@ async def run_tree_templates("threshold": float(raptor_cfg.get("threshold") or 0.1),"random_seed": int(raptor_cfg.get("random_seed") or 0),"max_cluster": int(raptor_cfg.get("max_cluster") or 64),- "ext": raptor_cfg.get("ext") or {},}progress_cb(msg=f"tree-template ({idx + 1}/{len(templates)}): building tree for doc={doc_id}",diff --git a/test/testcases/restful_api/test_documents.py b/test/testcases/restful_api/test_documents.pyindex c7981da8f3..a95679330f 100644--- a/test/testcases/restful_api/test_documents.py+++ b/test/testcases/restful_api/test_documents.py@@ -792,7 +792,7 @@ def test_documents_update_parser_config_contract(rest_client, create_dataset, tm({"task_page_size": "1024"}, 102, "Field: <parser_config.task_page_size> - Message: <Input should be a valid integer> - Value: <1024>"),({"raptor": {"use_raptor": {"a": "b"}}}, 102, "Field: <parser_config.raptor.use_raptor> - Message: <Input should be a valid boolean> - Value: <{'a': 'b'}>"),({"raptor": {"use_raptor": False}}, 0, ""),- ({"invalid_key": "invalid_value"}, 102, "Field: <parser_config.invalid_key> - Message: <Extra inputs are not permitted> - Value: <invalid_value>"),+ ({"invalid_key": "invalid_value"}, 0, ""),({"auto_keywords": -1}, 102, "Field: <parser_config.auto_keywords> - Message: <Input should be greater than or equal to 0> - Value: <-1>"),({"auto_keywords": 32}, 0, ""),({"auto_keywords": "1024"}, 102, "Field: <parser_config.auto_keywords> - Message: <Input should be a valid integer> - Value: <1024>"),diff --git a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.pyindex 7b8925399d..453367cc81 100644--- a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py+++ b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py@@ -507,8 +507,8 @@ class TestUpdateDocumentParserConfig:pytest.param("naive",{"invalid_key": "invalid_value"},- 102,- "Field: <parser_config.invalid_key> - Message: <Extra inputs are not permitted> - Value: <invalid_value>",+ 0,+ "",),pytest.param("naive",diff --git a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.pyindex 81abe6363b..d02db8ffba 100644--- a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py+++ b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py@@ -417,7 +417,7 @@ class TestUpdateDocumentParserConfig:("naive",{"invalid_key": "invalid_value"},- "Extra inputs are not permitted",+ "",),("naive",diff --git a/web/src/components/parse-configuration/raptor-form-fields.tsx b/web/src/components/parse-configuration/raptor-form-fields.tsxindex 813c37a883..9de2f7dbd9 100644--- a/web/src/components/parse-configuration/raptor-form-fields.tsx+++ b/web/src/components/parse-configuration/raptor-form-fields.tsx@@ -66,7 +66,6 @@ export const showTagItems = (parserId: DocumentParserType) => {const UseRaptorField = 'parser_config.raptor.use_raptor';const RandomSeedField = 'parser_config.raptor.random_seed';const ClusteringMethodField = 'parser_config.raptor.clustering_method';-const ClusteringMethodExtField = 'parser_config.raptor.ext.clustering_method';const TreeBuilderField = 'parser_config.raptor.tree_builder';const MaxClusterMax = 1024;@@ -83,15 +82,12 @@ const RaptorFormFields = ({const { t } = useTranslate('knowledgeConfiguration');const useRaptor = useWatch({ name: UseRaptorField });const clusteringMethod = useWatch({ name: ClusteringMethodField });- const extClusteringMethod = useWatch({ name: ClusteringMethodExtField });const selectedClusteringMethod = useMemo(() =>- (clusteringMethod ??- extClusteringMethod ??- form.getValues(ClusteringMethodField) ??- form.getValues(ClusteringMethodExtField) ??- 'gmm') as 'gmm' | 'ahc',- [clusteringMethod, extClusteringMethod, form],+ (clusteringMethod ?? form.getValues(ClusteringMethodField) ?? 'gmm') as+ | 'gmm'+ | 'ahc',+ [clusteringMethod, form],);const handleGenerate = useCallback(() => {@@ -113,10 +109,10 @@ const RaptorFormFields = ({);useEffect(() => {- if (!clusteringMethod && !extClusteringMethod) {+ if (!clusteringMethod) {handleClusteringMethodChange('gmm');}- }, [clusteringMethod, extClusteringMethod, handleClusteringMethodChange]);+ }, [clusteringMethod, handleClusteringMethodChange]);return (<>diff --git a/web/src/hooks/parser-config-utils.ts b/web/src/hooks/parser-config-utils.tsindex d0ab2462d2..1d04099f7e 100644--- a/web/src/hooks/parser-config-utils.ts+++ b/web/src/hooks/parser-config-utils.ts@@ -14,16 +14,10 @@* limitations under the License.*/-/**- * Utility functions for extracting parser config extensions.- * These functions extract known fields from parser config objects and merge- * unknown fields into the `ext` field for flexible configuration.- */-/*** Pipeline parser configs are keyed by operator id (e.g. "Parser:xxx"), so a* top-level key containing ":" marks the pipeline structure, which must be- * sent as-is instead of being reshaped by extractParserConfigExt.+ * sent as-is instead of being reshaped by normalizeParserConfig.*/export const isPipelineParserConfig = (parserConfig: Record<string, any> | undefined,@@ -35,11 +29,11 @@ export const isPipelineParserConfig = (};/**- * Extracts Parser configuration with extra fields merged into ext.+ * Normalizes parser configuration before it is sent to the API.* @param parserConfig - The parser configuration object- * @returns Processed parser config with extra fields in ext+ * @returns Processed parser config*/-export const extractParserConfigExt = (+export const normalizeParserConfig = (parserConfig: Record<string, any> | undefined,) => {if (!parserConfig) return parserConfig;@@ -58,11 +52,10 @@ export const extractParserConfigExt = (children_delimiter,use_parent_child,enable_children,- ext,- ...parserExt+ ...additionalParserConfig} = parserConfig;- delete parserExt.graphrag;- delete parserExt.raptor;+ delete additionalParserConfig.graphrag;+ delete additionalParserConfig.raptor;return {auto_keywords,auto_questions,@@ -83,6 +76,6 @@ export const extractParserConfigExt = (use_parent_child: use_parent_child ?? enable_children,}: undefined,- ext: { ...ext, ...parserExt },+ ...additionalParserConfig,};};diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.tsindex cfcf80afd2..8ced646e1c 100644--- a/web/src/hooks/use-document-request.ts+++ b/web/src/hooks/use-document-request.ts@@ -63,8 +63,8 @@ import {useHandleSearchChange,} from './logic-hooks';import {- extractParserConfigExt,isPipelineParserConfig,+ normalizeParserConfig,} from './parser-config-utils';import {useGetKnowledgeSearchParams,@@ -203,7 +203,7 @@ export const useFetchDocumentList = (loop = true) => {const ret = await listDocument({id: knowledgeId || id,- ext: { keywords: debouncedSearchString },+ keywords: debouncedSearchString,page_size: pagination.pageSize,page: pagination.current,},@@ -572,7 +572,7 @@ export const useSetDocumentParser = () => {}if (parserConfig) {- updateData.parser_config = extractParserConfigExt(parserConfig);+ updateData.parser_config = normalizeParserConfig(parserConfig);}const { data } = await changeDocumentParser(@@ -637,7 +637,7 @@ export const useSetDocumentPipelineParser = () => {if (parserConfig) {updateData.parser_config = isPipelineParserConfig(parserConfig)? parserConfig- : extractParserConfigExt(parserConfig);+ : normalizeParserConfig(parserConfig);}const { data } = await changeDocumentParser(diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.tsindex 2c1d83e325..96234ec480 100644--- a/web/src/hooks/use-knowledge-request.ts+++ b/web/src/hooks/use-knowledge-request.ts@@ -88,8 +88,8 @@ import {useHandleSearchChange,} from './logic-hooks';import {- extractParserConfigExt,isPipelineParserConfig,+ normalizeParserConfig,} from './parser-config-utils';import { useSetPaginationParams } from './route-hook';import { DatasetGenerateKeys } from './use-dataset-generate';@@ -218,10 +218,8 @@ export const useFetchNextKnowledgeListByPage = () => {const { data } = await listDataset({page_size: pagination.pageSize,page: pagination.current,- ext: {- keywords: debouncedSearchString,- owner_ids: filterValue.owner as string[],- },+ keywords: debouncedSearchString,+ owner_ids: filterValue.owner as string[],});return { kbs: data?.data, total_datasets: data?.total_datasets };@@ -283,10 +281,8 @@ export const useCreateKnowledge = () => {chunk_method?: string;parseType?: ParseType;pipeline_id?: string | null;- ext?: {- language?: string;- [key: string]: any;- };+ language?: string;+ [key: string]: any;}) => {const { data = {} } = await kbService.createKb(params);if (data.code === 0) {@@ -364,7 +360,7 @@ export const useUpdateKnowledge = (shouldFetchList = false) => {permission,pagerank,parser_config,- ...ext+ ...additionalFields} = params;const requestBody: Record<string, any> = {name,@@ -377,8 +373,8 @@ export const useUpdateKnowledge = (shouldFetchList = false) => {pagerank,parser_config: isPipelineParserConfig(parser_config)? parser_config- : extractParserConfigExt(parser_config),- ...omit(ext, ['kb_id']),+ : normalizeParserConfig(parser_config),+ ...omit(additionalFields, ['kb_id']),};const { data = {} } = await updateKb(kbId, requestBody);@@ -1073,7 +1069,7 @@ export const useFetchKnowledgeList = (const { data } = await listDataset({page,page_size: pageSize,- ...(keywords ? { ext: { keywords } } : {}),+ ...(keywords ? { keywords } : {}),});return {items: (data?.data ?? []) as IDataset[],diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.tsindex fd4ec45063..3386839750 100644--- a/web/src/interfaces/database/dataset.ts+++ b/web/src/interfaces/database/dataset.ts@@ -84,7 +84,6 @@ interface Parserconfig {interface Raptor {clustering_method?: 'gmm' | 'ahc';- ext?: { clustering_method: 'gmm' | 'ahc'; tree_builder: 'raptor' | 'psi' };max_cluster: number;max_token: number;prompt: string;diff --git a/web/src/interfaces/request/knowledge.ts b/web/src/interfaces/request/knowledge.tsindex d729702e4f..3a64ff62ef 100644--- a/web/src/interfaces/request/knowledge.ts+++ b/web/src/interfaces/request/knowledge.ts@@ -28,11 +28,9 @@ export interface IFetchKnowledgeListRequestParams {id?: string;page?: number;page_size?: number;- ext?: {- keywords?: string;- owner_ids?: string[];- parser_id?: string;- };+ keywords?: string;+ owner_ids?: string[];+ parser_id?: string;}export interface IFetchDocumentListRequestBody {diff --git a/web/src/pages/datasets/hooks.ts b/web/src/pages/datasets/hooks.tsindex 08ccba15cc..9c9929a2ca 100644--- a/web/src/pages/datasets/hooks.ts+++ b/web/src/pages/datasets/hooks.ts@@ -21,10 +21,8 @@ export interface Iknowledge {chunk_method?: string;parseType?: ParseType;pipeline_id?: string | null;- ext?: {- language?: string;- [key: string]: any;- };+ language?: string;+ [key: string]: any;}export const useSaveKnowledge = () => {const { visible: visible, hideModal, showModal } = useSetModalState();diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.tsindex dde2382d11..8990a69638 100644--- a/web/src/services/knowledge-service.ts+++ b/web/src/services/knowledge-service.ts@@ -282,7 +282,11 @@ export function deleteKnowledgeGraph(knowledgeId: string) {}export const listDataset = (params?: IFetchKnowledgeListRequestParams) =>- request.get(api.kbList, { params });+ request.get(api.kbList, {+ params: params+ ? { ...params, owner_ids: params.owner_ids?.join(',') }+ : params,+ });// Fetch datasets by a set of IDs via the `ids` query param (comma-joined).// Used to echo back already-selected datasets whose names are not present@@ -325,13 +329,11 @@ export const listDocument = (if (!params || !params.id) {throw new Error('params and params.id are required');}- // Extract page, page_size, and ext.keywords from params- const { page, page_size, ext } = params;- // Merge: page, page_size, keywords (from ext), body, and remaining params+ const { page, page_size, keywords } = params;const mergedParams = {page,page_size,- keywords: ext?.keywords,+ keywords,...body,};return request.get(api.getDocumentList(params.id), { params: mergedParams });