Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: update dataset API should not have ext (#19272)

Wang Qif8b0a1de1970
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.py
index dce0301f47..e1bebc3a2c 100644
--- a/api/apps/restful_apis/chunk_api.py
+++ b/api/apps/restful_apis/chunk_api.py
@@ -668,8 +668,6 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):
if isinstance(parser_config, dict):
if "compilation_template_group_id" in parser_config:
group_ids = _group_ids(parser_config.get("compilation_template_group_id"))
- elif isinstance(parser_config.get("ext"), dict):
- group_ids = _group_ids(parser_config["ext"].get("compilation_template_group_id"))
configured_ids: list[str] = []
seen_configured_ids: set[str] = set()
diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.py
index 0b81320562..dff073bf73 100644
--- a/api/apps/restful_apis/document_api.py
+++ b/api/apps/restful_apis/document_api.py
@@ -91,12 +91,6 @@ def _normalize_legacy_raptor_config(req: dict) -> None:
return
normalized_fields = []
- legacy_ext = raptor.pop("ext", None)
- if legacy_ext is not None:
- normalized_fields.append("ext")
- if isinstance(legacy_ext, dict) and "clustering_threshold" in legacy_ext and "clustering_threshold" not in raptor:
- raptor["clustering_threshold"] = legacy_ext["clustering_threshold"]
- normalized_fields.append("ext.clustering_threshold")
for field in ("threshold", "clustering_method", "tree_builder"):
if field in raptor:
raptor.pop(field)
@@ -116,13 +110,10 @@ def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bo
if not isinstance(parser_config, dict):
return False
- if "compilation_template_group_id" not in parser_config and not (isinstance(parser_config.get("ext"), dict) and "compilation_template_group_id" in parser_config["ext"]):
+ if "compilation_template_group_id" not in parser_config:
return False
group_ids = _parser_config_compilation_template_group_ids(parser_config)
parser_config["compilation_template_group_id"] = group_ids
- ext = parser_config.get("ext")
- if isinstance(ext, dict) and "compilation_template_group_id" in ext:
- ext["compilation_template_group_id"] = group_ids
return True
@@ -291,7 +282,6 @@ async def update_document(tenant_id, dataset_id, document_id):
# Changing the document-scoped knowledge compilation template group must
# not remove the existing chunks.
if update_doc_req.parser_config:
- req["parser_config"].update(update_doc_req.parser_config.ext)
_normalize_parser_config_compilation_template_group_ids(req["parser_config"])
DocumentService.update_parser_config(doc.id, req["parser_config"])
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index d8ba5d0d3b..6bbe4f8e5b 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -95,9 +95,6 @@ async def create_dataset(tenant_id: str, req: dict):
:param req: dataset creation request
:return: (success, result) or (success, error_message)
"""
- # Extract ext field for additional parameters
- ext_fields = req.pop("ext", {})
-
# Map auto_metadata_config (if provided) into parser_config structure
auto_meta = req.pop("auto_metadata_config", {})
if auto_meta:
@@ -116,8 +113,6 @@ async def create_dataset(tenant_id: str, req: dict):
parser_cfg["metadata"] = fields
parser_cfg["enable_metadata"] = auto_meta.get("enabled", True)
req["parser_config"] = parser_cfg
- req.update(ext_fields)
-
e, create_dict = KnowledgebaseService.create_with_name(name=req.pop("name", None), tenant_id=tenant_id, parser_id=req.pop("parser_id", None), **req)
if not e:
@@ -320,9 +315,6 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):
if kb is None:
return False, "Invalid Dataset ID"
- # Extract ext field for additional parameters
- ext_fields = req.pop("ext", {})
-
# Map auto_metadata_config into parser_config if present
auto_meta = req.pop("auto_metadata_config", {})
if auto_meta:
@@ -342,9 +334,6 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):
parser_cfg["enable_metadata"] = auto_meta.get("enabled", True)
req["parser_config"] = parser_cfg
- # Merge ext fields with req
- req.update(ext_fields)
-
# Extract connectors from request
connectors = []
if "connectors" in req:
@@ -362,10 +351,7 @@ async def update_dataset(tenant_id: str, dataset_id: str, req: dict):
req["parser_config"]["enable_children"] = False
req["parser_config"]["parent_child"] = {}
- parser_config = req["parser_config"]
- req_ext_fields = parser_config.pop("ext", {})
- parser_config.update(req_ext_fields)
- req["parser_config"] = deep_merge(kb.parser_config, parser_config)
+ req["parser_config"] = deep_merge(kb.parser_config, req["parser_config"])
if (chunk_method := req.get("parser_id")) and chunk_method != kb.parser_id:
if not req.get("parser_config"):
@@ -439,9 +425,8 @@ def list_datasets(tenant_id: str, args: dict):
name = args.get("name")
page = int(args.get("page", 1))
page_size = int(args.get("page_size", 30))
- ext_fields = args.get("ext", {})
- parser_id = ext_fields.get("parser_id")
- keywords = ext_fields.get("keywords", "")
+ parser_id = args.get("parser_id")
+ keywords = args.get("keywords", "")
orderby = args.get("orderby", "create_time")
desc_arg = args.get("desc", "true")
if isinstance(desc_arg, str):
@@ -463,7 +448,7 @@ def list_datasets(tenant_id: str, args: dict):
kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)
if not kbs:
return False, f"User '{tenant_id}' lacks permission for dataset '{name}'"
- owner_ids = [owner_id.strip() for owner_id in ext_fields.get("owner_ids", []) if isinstance(owner_id, str) and owner_id.strip()]
+ owner_ids = [owner_id.strip() for owner_id in args.get("owner_ids", []) if isinstance(owner_id, str) and owner_id.strip()]
if owner_ids:
tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)
allowed_tenant_ids = {m["tenant_id"] for m in tenants}
diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py
index 8c816833c8..6a099a2889 100644
--- a/api/utils/validation_utils.py
+++ b/api/utils/validation_utils.py
@@ -176,15 +176,6 @@ def validate_and_parse_request_args(request: Request, validator: type[BaseModel]
if query_name in request.args and _is_list_annotation(field_info.annotation):
args[query_name] = [value for item in request.args.getlist(query_name) for value in item.split(",") if value]
- # Handle ext parameter: parse JSON string to dict if it's a string
- if "ext" in args and isinstance(args["ext"], str):
- import json
-
- try:
- args["ext"] = json.loads(args["ext"])
- except json.JSONDecodeError:
- logging.debug("Failed to decode query arg 'ext' as JSON; passing raw value to validator")
-
try:
if extras is not None:
args.update(extras)
@@ -380,14 +371,6 @@ class RaptorConfig(Base):
normalized = dict(value)
changed_fields = []
- legacy_ext = normalized.pop("ext", None)
- if legacy_ext is not None:
- changed_fields.append("ext")
- if isinstance(legacy_ext, dict) and normalized.get("clustering_threshold") is None:
- if "clustering_threshold" in legacy_ext:
- normalized["clustering_threshold"] = legacy_ext["clustering_threshold"]
- changed_fields.append("ext.clustering_threshold")
-
for field in ("threshold", "clustering_method", "tree_builder"):
if field in normalized:
normalized.pop(field)
@@ -450,6 +433,8 @@ TableColumnRole = Literal["indexing", "metadata", "both"]
class ParserConfig(Base):
"""Complete parser configuration accepted by dataset APIs."""
+ model_config = ConfigDict(extra="allow", strict=True)
+
auto_keywords: Annotated[int, Field(default=0, ge=0, le=32)]
auto_questions: Annotated[int, Field(default=0, ge=0, le=10)]
chunk_token_num: Annotated[int, Field(default=512, ge=1, le=2048)]
@@ -467,7 +452,6 @@ class ParserConfig(Base):
task_page_size: Annotated[int | None, Field(default=None, ge=1)]
pages: Annotated[list[list[int]] | None, Field(default=None)]
compilation_template_group_id: Annotated[list[str], Field(default_factory=list)]
- ext: Annotated[dict, Field(default={})]
# Table parser: column name -> "indexing" | "metadata" | "both". Absence => all columns "both".
# Table parser: "auto" = all columns both (default), "manual" = use table_column_roles. None → treated as "auto".
table_column_mode: Annotated[Literal["auto", "manual"] | None, Field(default=None)]
@@ -476,6 +460,13 @@ class ParserConfig(Base):
# Table parser: list of column names (set by backend after first parse; used by frontend for role selector).
table_column_names: Annotated[list[str] | None, Field(default=None)]
+ @model_validator(mode="before")
+ @classmethod
+ def reject_ext(cls, value: Any) -> Any:
+ if isinstance(value, dict) and "ext" in value:
+ raise ValueError("parser_config.ext is not supported; send parser configuration fields directly")
+ return value
+
@field_validator("table_column_roles", mode="before")
@classmethod
def legacy_vectorize_table_column_role(cls, v: Any) -> Any:
@@ -579,9 +570,9 @@ class CreateDatasetReq(Base):
parse_type: Annotated[int | None, Field(default=None, ge=0, le=64)]
pipeline_id: Annotated[str | None, Field(default=None, min_length=32, max_length=32, serialization_alias="pipeline_id")]
chunk_method: Annotated[str | None, Field(default=None, serialization_alias="parser_id")]
+ language: Annotated[str | None, Field(default=None, max_length=32)]
parser_config: Annotated[ParserConfig | None, Field(default=None)]
auto_metadata_config: Annotated[AutoMetadataConfig | None, Field(default=None)]
- ext: Annotated[dict, Field(default={})]
@field_validator("pipeline_id", mode="before")
@classmethod
@@ -850,7 +841,6 @@ class UpdateDatasetReq(CreateDatasetReq):
dataset_id: Annotated[str, Field(...)]
name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=DATASET_NAME_LIMIT), Field(default="")]
pagerank: Annotated[int, Field(default=0, ge=0, le=100)]
- language: Annotated[str | None, Field(default=None, max_length=32)]
connectors: Annotated[list[dict[str, Any]], Field(default_factory=list)]
@field_validator("dataset_id", mode="before")
@@ -1066,7 +1056,9 @@ class ListDatasetReq(BaseListReq):
ids: Annotated[list[str] | None, Field(default=None, max_length=REST_API_MAX_IDS)]
include_parsing_status: Annotated[bool, Field(default=False)]
- ext: Annotated[dict, Field(default={})]
+ keywords: Annotated[str, Field(default="")]
+ owner_ids: Annotated[list[str], Field(default_factory=list)]
+ parser_id: Annotated[str | None, Field(default=None)]
@field_validator("ids", mode="after")
@classmethod
diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md
index cfce337b4a..cecf18917c 100644
--- a/docs/references/http_api_reference.md
+++ b/docs/references/http_api_reference.md
@@ -596,17 +596,9 @@ curl --request POST \
- For PDFs only.
- Defaults to `12`
- Minimum: `1`
- - `"raptor"`: `object` RAPTOR-specific settings.
- - Defaults to: `{"use_raptor": false}`
- - `"graphrag"`: `object` GRAPHRAG-specific settings.
- - Defaults to: `{"use_graphrag": false}`
- `"parent_child"`: `object` Parent-child chunking settings. When enabled, each chunk is further split into smaller child chunks using `children_delimiter`. At retrieval time, matched child chunks are replaced by their parent's full text before being passed to the LLM, giving precise vector matching with broader context.
- `"use_parent_child"`: `bool` Whether to enable parent-child chunking. Defaults to `false`.
- `"children_delimiter"`: `string` The delimiter used to split a parent chunk into child chunks. Only takes effect when `"use_parent_child"` is `true`. Defaults to `"\n"`.
- - If `"chunk_method"` is `"qa"`, `"manual"`, `"paper"`, `"book"`, `"laws"`, or `"presentation"`, the `"parser_config"` object contains the following attribute:
- - `"raptor"`: `object` RAPTOR-specific settings.
- - Defaults to: `{"use_raptor": false}`.
- - If `"chunk_method"` is `"table"`, `"picture"`, `"one"`, or `"email"`, `"parser_config"` is an empty JSON object.
- `"parse_type"`: (*Body parameter*), `int`
The ingestion pipeline parse type identifier, i.e., the number of parsers in your **Parser** component.
@@ -653,11 +645,8 @@ Success:
"chunk_token_num": 128,
"delimiter": "\\n!?;。;!?",
"html4excel": false,
- "layout_recognize": "DeepDOC",
- "raptor": {
- "use_raptor": false
- }
- },
+ "layout_recognize": "DeepDOC"
+ },
"permission": "me",
"similarity_threshold": 0.2,
"status": "1",
@@ -856,10 +845,7 @@ curl --request PUT \
- `"parent_child"`: `object` Parent-child chunking settings. When enabled, each chunk is further split into smaller child chunks using `children_delimiter`. At retrieval time, matched child chunks are replaced by their parent's full text before being passed to the LLM, giving precise vector matching with broader context.
- `"use_parent_child"`: `bool` Whether to enable parent-child chunking. Defaults to `false`.
- `"children_delimiter"`: `string` The delimiter used to split a parent chunk into child chunks. Only takes effect when `"use_parent_child"` is `true`. Defaults to `"\n"`.
- - If `"chunk_method"` is `"qa"`, `"manual"`, `"paper"`, `"book"`, `"laws"`, or `"presentation"`, the `"parser_config"` object contains the following attribute:
- - `"raptor"`: `object` RAPTOR-specific settings.
- - Defaults to: `{"use_raptor": false}`.
- - If `"chunk_method"` is `"table"`, `"picture"`, `"one"`, or `"email"`, `"parser_config"` is an empty JSON object.
+
#### Response
@@ -1003,9 +989,7 @@ Success (with `include_parsing_status=true`):
"language": "English",
"name": "Test Dataset",
"parser_config": {
- "graphrag": { "use_graphrag": false },
- "llm_id": "deepseek-chat@DeepSeek",
- "raptor": { "use_raptor": false }
+ "llm_id": "deepseek-chat@DeepSeek"
},
"permission": "me",
"running_count": 0,
@@ -1128,10 +1112,7 @@ Success:
"chunk_token_num": 128,
"delimiter": "\\n",
"html4excel": false,
- "layout_recognize": true,
- "raptor": {
- "use_raptor": false
- }
+ "layout_recognize": true
},
"run": "UNSTART",
"size": 17966,
@@ -1217,10 +1198,6 @@ curl --request PATCH \
- `"html4excel"`: Indicates whether to convert Excel documents into HTML format. Defaults to `false`.
- `"delimiter"`: Defaults to `"\n"`.
- `"task_page_size"`: Defaults to `12`. For PDF only.
- - `"raptor"`: RAPTOR-specific settings. Defaults to: `{"use_raptor": false}`.
- - If `"chunk_method"` is `"qa"`, `"manual"`, `"paper"`, `"book"`, `"laws"`, or `"presentation"`, the `"parser_config"` object contains the following attribute:
- - `"raptor"`: RAPTOR-specific settings. Defaults to: `{"use_raptor": false}`.
- - If `"chunk_method"` is `"table"`, `"picture"`, `"one"`, or `"email"`, `"parser_config"` is an empty JSON object.
- `"enabled"`: (*Body parameter*), `integer`
Whether the document should be **available** in the knowledge base.
- `1` → (available)
@@ -1275,32 +1252,10 @@ Success:
"auto_keywords": 0,
"auto_questions": 0,
"topn_tags": 3,
-
"layout_recognize": "DeepDOC",
"html4excel": false,
"image_context_size": 0,
- "table_context_size": 0,
-
- "graphrag": {
- "use_graphrag": true,
- "method": "light",
- "entity_types": [
- "organization",
- "person",
- "geo",
- "event",
- "category"
- ]
- },
-
- "raptor": {
- "use_raptor": true,
- "max_cluster": 64,
- "max_token": 256,
- "threshold": 0.1,
- "random_seed": 0,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
- }
+ "table_context_size": 0
},
"meta_fields": {},
@@ -1931,10 +1886,7 @@ Success:
"chunk_token_num": 128,
"delimiter": "\\n",
"html4excel": false,
- "layout_recognize": true,
- "raptor": {
- "use_raptor": false
- }
+ "layout_recognize": true
},
"process_begin_at": "Thu, 24 Oct 2024 09:56:44 GMT",
"process_duration": 0.54213,
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index 2074fc29ec..3426e97729 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -50,12 +50,6 @@ type searchDatasetService interface {
SearchDataset(ctx context.Context, datasetID, userID string, req *service.SearchDatasetRequest) (*service.SearchDatasetsResponse, error)
}
-type listDatasetsExt struct {
- Keywords string `json:"keywords,omitempty"`
- OwnerIDs []string `json:"owner_ids,omitempty"`
- ParserID string `json:"parser_id,omitempty"`
-}
-
// NewDatasetsHandler creates a new datasets' handler.
func NewDatasetsHandler(datasetsService *dataset.DatasetService, metadataService *service.MetadataService) *DatasetsHandler {
h := &DatasetsHandler{
@@ -144,20 +138,15 @@ func (h *DatasetsHandler) ListDatasets(c *gin.Context) {
desc = parsed
}
- keywords := ""
- parserID := ""
+ keywords := c.Query("keywords")
+ parserID := c.Query("parser_id")
var ownerIDs []string
-
- // ext keeps the same compatibility payload as the Python REST API.
- if extStr := c.Query("ext"); extStr != "" {
- var ext listDatasetsExt
- if err := json.Unmarshal([]byte(extStr), &ext); err != nil {
- common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
- return
+ for _, item := range c.QueryArray("owner_ids") {
+ for _, ownerID := range strings.Split(item, ",") {
+ if ownerID = strings.TrimSpace(ownerID); ownerID != "" {
+ ownerIDs = append(ownerIDs, ownerID)
+ }
}
- keywords = ext.Keywords
- parserID = ext.ParserID
- ownerIDs = ext.OwnerIDs
}
// Mirror pydantic: a present-but-empty id fails UUID validation.
@@ -236,10 +225,14 @@ func (h *DatasetsHandler) CreateDataset(c *gin.Context) {
return
}
- bodyBytes, _, ok := parseJSONRequestObject(c)
+ bodyBytes, raw, ok := parseJSONRequestObject(c)
if !ok {
return
}
+ if _, exists := raw["ext"]; exists {
+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Extra inputs are not permitted: ext")
+ return
+ }
var req service.CreateDatasetRequest
if err := json.Unmarshal(bodyBytes, &req); err != nil {
@@ -357,12 +350,12 @@ func pythonJSONTypeName(v interface{}) string {
}
// listDatasetsAllowedParams mirrors the query field set of Python's
-// ListDatasetReq (BaseListReq + include_parsing_status/ext; `type` is handled
+// ListDatasetReq (BaseListReq + dataset filters; `type` is handled
// before validation in the Python endpoint).
var listDatasetsAllowedParams = map[string]bool{
"id": true, "ids": true, "name": true, "page": true, "page_size": true,
"orderby": true, "desc": true, "include_parsing_status": true,
- "ext": true, "type": true,
+ "keywords": true, "owner_ids": true, "parser_id": true, "type": true,
}
// updateDatasetAllowedFields mirrors the field set of Python's UpdateDatasetReq
@@ -370,7 +363,7 @@ var listDatasetsAllowedParams = map[string]bool{
var updateDatasetAllowedFields = map[string]bool{
"name": true, "avatar": true, "description": true, "embedding_model": true,
"permission": true, "parse_type": true, "pipeline_id": true, "chunk_method": true,
- "parser_id": true, "parser_config": true, "auto_metadata_config": true, "ext": true,
+ "parser_id": true, "parser_config": true, "auto_metadata_config": true,
"dataset_id": true, "pagerank": true, "language": true, "connectors": true,
}
@@ -421,6 +414,21 @@ func (h *DatasetsHandler) UpdateDataset(c *gin.Context) {
return
}
}
+ if parserConfig, ok := providedFields["parser_config"]; ok {
+ var config map[string]interface{}
+ if json.Unmarshal(parserConfig, &config) == nil {
+ if _, ok := config["ext"]; ok {
+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "parser_config.ext is not supported; send parser configuration fields directly")
+ return
+ }
+ if raptor, ok := config["raptor"].(map[string]interface{}); ok {
+ if _, ok := raptor["ext"]; ok {
+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "parser_config.raptor.ext is not supported; send RAPTOR configuration fields directly")
+ return
+ }
+ }
+ }
+ }
}
ctx := c.Request.Context()
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index b6ea5b8d05..34620f9337 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -50,6 +50,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
parserID := string(entity.ParserTypeGeneral)
permission := "me"
embeddingModel := ""
+ var language *string
pipelineID := req.PipelineID
if req.Permission != nil {
@@ -82,6 +83,13 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
return nil, common.CodeDataError, err
}
}
+ if req.Language != nil {
+ normalizedLanguage := strings.TrimSpace(*req.Language)
+ if len(normalizedLanguage) > 32 {
+ return nil, common.CodeDataError, errors.New("String should have at most 32 characters")
+ }
+ language = &normalizedLanguage
+ }
if pipelineID != nil && strings.TrimSpace(*pipelineID) != "" {
if ok, err := canvasAccessibleForUser(ctx, tenantID, strings.TrimSpace(*pipelineID)); err != nil {
@@ -139,6 +147,7 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD
PipelineID: pipelineID,
ParserConfig: entity.JSONMap(parserConfigMap),
Permission: permission,
+ Language: language,
EmbdID: embdID,
TenantEmbdID: stringPtrIfNotEmpty(tenantEmbdID),
Status: &status,
diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go
index 7e05690d7d..0d192827dc 100644
--- a/internal/service/dataset/helpers.go
+++ b/internal/service/dataset/helpers.go
@@ -332,28 +332,6 @@ func cloneJSONValue(value interface{}) interface{} {
}
}
-func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{} {
- if ext == nil {
- return nil
- }
- updates := make(map[string]interface{}, len(ext))
- for key, value := range ext {
- switch key {
- case "chunk_method":
- updates["parser_id"] = value
- case "token_num", "chunk_num", "parser_config":
- continue
- case "pagerank":
- if v, ok := value.(float64); ok {
- updates[key] = int64(v)
- }
- default:
- updates[key] = value
- }
- }
- return updates
-}
-
func normalizeMetadataConfigFields(fields []service.MetadataConfigField, fieldName string) ([]map[string]interface{}, error) {
normalizedFields := make([]map[string]interface{}, 0, len(fields))
for i, field := range fields {
diff --git a/internal/service/dataset/helpers_test.go b/internal/service/dataset/helpers_test.go
index c714e4c53a..be8d4c28ba 100644
--- a/internal/service/dataset/helpers_test.go
+++ b/internal/service/dataset/helpers_test.go
@@ -242,74 +242,6 @@ func TestNormalizeDatasetID_StripsHyphens(t *testing.T) {
}
}
-// --- normalizeDatasetUpdateExt ---
-
-func TestNormalizeDatasetUpdateExt_Nil(t *testing.T) {
- if result := normalizeDatasetUpdateExt(nil); result != nil {
- t.Fatalf("expected nil for nil input")
- }
-}
-
-func TestNormalizeDatasetUpdateExt_PassesThrough(t *testing.T) {
- ext := map[string]interface{}{
- "description": "test",
- "language": "English",
- }
- result := normalizeDatasetUpdateExt(ext)
- if result["description"] != "test" {
- t.Errorf("expected description preserved, got %v", result["description"])
- }
- if result["language"] != "English" {
- t.Errorf("expected language preserved, got %v", result["language"])
- }
-}
-
-func TestNormalizeDatasetUpdateExt_RenamesChunkMethod(t *testing.T) {
- ext := map[string]interface{}{
- "chunk_method": "book",
- }
- result := normalizeDatasetUpdateExt(ext)
- if result["parser_id"] != "book" {
- t.Errorf("expected parser_id=book, got %v", result["parser_id"])
- }
- if _, ok := result["chunk_method"]; ok {
- t.Error("expected chunk_method to be renamed to parser_id")
- }
-}
-
-func TestNormalizeDatasetUpdateExt_SkipsTokenAndChunkNum(t *testing.T) {
- ext := map[string]interface{}{
- "token_num": float64(1000),
- "chunk_num": float64(50),
- "parser_config": map[string]interface{}{"key": "val"},
- }
- result := normalizeDatasetUpdateExt(ext)
- if len(result) != 0 {
- t.Errorf("expected empty map, got %v", result)
- }
-}
-
-func TestNormalizeDatasetUpdateExt_ConvertsPagerank(t *testing.T) {
- ext := map[string]interface{}{
- "pagerank": float64(3),
- }
- result := normalizeDatasetUpdateExt(ext)
- if result["pagerank"] != int64(3) {
- t.Errorf("expected pagerank=int64(3), got %T(%v)", result["pagerank"], result["pagerank"])
- }
-}
-
-func TestNormalizeDatasetUpdateExt_NonFloatPagerankSkipped(t *testing.T) {
- // Non-float64 pagerank values are not convertible and are dropped.
- ext := map[string]interface{}{
- "pagerank": "auto",
- }
- result := normalizeDatasetUpdateExt(ext)
- if _, ok := result["pagerank"]; ok {
- t.Error("expected non-float pagerank to be skipped")
- }
-}
-
// --- normalizeMetadataConfigFields ---
func TestNormalizeMetadataConfigFields_EmptyKey(t *testing.T) {
diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.go
index e9080b0905..99c8a8262c 100644
--- a/internal/service/dataset_types.go
+++ b/internal/service/dataset_types.go
@@ -134,6 +134,7 @@ type MetadataConfigRequest struct {
type CreateDatasetRequest struct {
Name string `json:"name" binding:"required"`
EmbeddingModel *string `json:"embedding_model,omitempty"`
+ Language *string `json:"language,omitempty"`
Permission *string `json:"permission,omitempty"`
ParserID *string `json:"parser_id,omitempty"`
PipelineID *string `json:"pipeline_id,omitempty"`
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index d4678ba9bf..4480e7983f 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -444,9 +444,6 @@ def _parser_config_compilation_template_group_ids(parser_config) -> list[str]:
return []
if "compilation_template_group_id" in parser_config:
return _normalize(parser_config.get("compilation_template_group_id"))
- ext = parser_config.get("ext")
- if isinstance(ext, dict):
- return _normalize(ext.get("compilation_template_group_id"))
return []
@@ -976,7 +973,6 @@ async def run_tree_templates(
"threshold": float(raptor_cfg.get("threshold") or 0.1),
"random_seed": int(raptor_cfg.get("random_seed") or 0),
"max_cluster": int(raptor_cfg.get("max_cluster") or 64),
- "ext": raptor_cfg.get("ext") or {},
}
progress_cb(
msg=f"tree-template ({idx + 1}/{len(templates)}): building tree for doc={doc_id}",
diff --git a/test/testcases/restful_api/test_documents.py b/test/testcases/restful_api/test_documents.py
index c7981da8f3..a95679330f 100644
--- a/test/testcases/restful_api/test_documents.py
+++ b/test/testcases/restful_api/test_documents.py
@@ -792,7 +792,7 @@ def test_documents_update_parser_config_contract(rest_client, create_dataset, tm
({"task_page_size": "1024"}, 102, "Field: <parser_config.task_page_size> - Message: <Input should be a valid integer> - Value: <1024>"),
({"raptor": {"use_raptor": {"a": "b"}}}, 102, "Field: <parser_config.raptor.use_raptor> - Message: <Input should be a valid boolean> - Value: <{'a': 'b'}>"),
({"raptor": {"use_raptor": False}}, 0, ""),
- ({"invalid_key": "invalid_value"}, 102, "Field: <parser_config.invalid_key> - Message: <Extra inputs are not permitted> - Value: <invalid_value>"),
+ ({"invalid_key": "invalid_value"}, 0, ""),
({"auto_keywords": -1}, 102, "Field: <parser_config.auto_keywords> - Message: <Input should be greater than or equal to 0> - Value: <-1>"),
({"auto_keywords": 32}, 0, ""),
({"auto_keywords": "1024"}, 102, "Field: <parser_config.auto_keywords> - Message: <Input should be a valid integer> - Value: <1024>"),
diff --git a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
index 7b8925399d..453367cc81 100644
--- a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
+++ b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
@@ -507,8 +507,8 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"invalid_key": "invalid_value"},
- 102,
- "Field: <parser_config.invalid_key> - Message: <Extra inputs are not permitted> - Value: <invalid_value>",
+ 0,
+ "",
),
pytest.param(
"naive",
diff --git a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
index 81abe6363b..d02db8ffba 100644
--- a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
+++ b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
@@ -417,7 +417,7 @@ class TestUpdateDocumentParserConfig:
(
"naive",
{"invalid_key": "invalid_value"},
- "Extra inputs are not permitted",
+ "",
),
(
"naive",
diff --git a/web/src/components/parse-configuration/raptor-form-fields.tsx b/web/src/components/parse-configuration/raptor-form-fields.tsx
index 813c37a883..9de2f7dbd9 100644
--- a/web/src/components/parse-configuration/raptor-form-fields.tsx
+++ b/web/src/components/parse-configuration/raptor-form-fields.tsx
@@ -66,7 +66,6 @@ export const showTagItems = (parserId: DocumentParserType) => {
const UseRaptorField = 'parser_config.raptor.use_raptor';
const RandomSeedField = 'parser_config.raptor.random_seed';
const ClusteringMethodField = 'parser_config.raptor.clustering_method';
-const ClusteringMethodExtField = 'parser_config.raptor.ext.clustering_method';
const TreeBuilderField = 'parser_config.raptor.tree_builder';
const MaxClusterMax = 1024;
@@ -83,15 +82,12 @@ const RaptorFormFields = ({
const { t } = useTranslate('knowledgeConfiguration');
const useRaptor = useWatch({ name: UseRaptorField });
const clusteringMethod = useWatch({ name: ClusteringMethodField });
- const extClusteringMethod = useWatch({ name: ClusteringMethodExtField });
const selectedClusteringMethod = useMemo(
() =>
- (clusteringMethod ??
- extClusteringMethod ??
- form.getValues(ClusteringMethodField) ??
- form.getValues(ClusteringMethodExtField) ??
- 'gmm') as 'gmm' | 'ahc',
- [clusteringMethod, extClusteringMethod, form],
+ (clusteringMethod ?? form.getValues(ClusteringMethodField) ?? 'gmm') as
+ | 'gmm'
+ | 'ahc',
+ [clusteringMethod, form],
);
const handleGenerate = useCallback(() => {
@@ -113,10 +109,10 @@ const RaptorFormFields = ({
);
useEffect(() => {
- if (!clusteringMethod && !extClusteringMethod) {
+ if (!clusteringMethod) {
handleClusteringMethodChange('gmm');
}
- }, [clusteringMethod, extClusteringMethod, handleClusteringMethodChange]);
+ }, [clusteringMethod, handleClusteringMethodChange]);
return (
<>
diff --git a/web/src/hooks/parser-config-utils.ts b/web/src/hooks/parser-config-utils.ts
index d0ab2462d2..1d04099f7e 100644
--- a/web/src/hooks/parser-config-utils.ts
+++ b/web/src/hooks/parser-config-utils.ts
@@ -14,16 +14,10 @@
* limitations under the License.
*/
-/**
- * Utility functions for extracting parser config extensions.
- * These functions extract known fields from parser config objects and merge
- * unknown fields into the `ext` field for flexible configuration.
- */
-
/**
* Pipeline parser configs are keyed by operator id (e.g. "Parser:xxx"), so a
* top-level key containing ":" marks the pipeline structure, which must be
- * sent as-is instead of being reshaped by extractParserConfigExt.
+ * sent as-is instead of being reshaped by normalizeParserConfig.
*/
export const isPipelineParserConfig = (
parserConfig: Record<string, any> | undefined,
@@ -35,11 +29,11 @@ export const isPipelineParserConfig = (
};
/**
- * Extracts Parser configuration with extra fields merged into ext.
+ * Normalizes parser configuration before it is sent to the API.
* @param parserConfig - The parser configuration object
- * @returns Processed parser config with extra fields in ext
+ * @returns Processed parser config
*/
-export const extractParserConfigExt = (
+export const normalizeParserConfig = (
parserConfig: Record<string, any> | undefined,
) => {
if (!parserConfig) return parserConfig;
@@ -58,11 +52,10 @@ export const extractParserConfigExt = (
children_delimiter,
use_parent_child,
enable_children,
- ext,
- ...parserExt
+ ...additionalParserConfig
} = parserConfig;
- delete parserExt.graphrag;
- delete parserExt.raptor;
+ delete additionalParserConfig.graphrag;
+ delete additionalParserConfig.raptor;
return {
auto_keywords,
auto_questions,
@@ -83,6 +76,6 @@ export const extractParserConfigExt = (
use_parent_child: use_parent_child ?? enable_children,
}
: undefined,
- ext: { ...ext, ...parserExt },
+ ...additionalParserConfig,
};
};
diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.ts
index cfcf80afd2..8ced646e1c 100644
--- a/web/src/hooks/use-document-request.ts
+++ b/web/src/hooks/use-document-request.ts
@@ -63,8 +63,8 @@ import {
useHandleSearchChange,
} from './logic-hooks';
import {
- extractParserConfigExt,
isPipelineParserConfig,
+ normalizeParserConfig,
} from './parser-config-utils';
import {
useGetKnowledgeSearchParams,
@@ -203,7 +203,7 @@ export const useFetchDocumentList = (loop = true) => {
const ret = await listDocument(
{
id: knowledgeId || id,
- ext: { keywords: debouncedSearchString },
+ keywords: debouncedSearchString,
page_size: pagination.pageSize,
page: pagination.current,
},
@@ -572,7 +572,7 @@ export const useSetDocumentParser = () => {
}
if (parserConfig) {
- updateData.parser_config = extractParserConfigExt(parserConfig);
+ updateData.parser_config = normalizeParserConfig(parserConfig);
}
const { data } = await changeDocumentParser(
@@ -637,7 +637,7 @@ export const useSetDocumentPipelineParser = () => {
if (parserConfig) {
updateData.parser_config = isPipelineParserConfig(parserConfig)
? parserConfig
- : extractParserConfigExt(parserConfig);
+ : normalizeParserConfig(parserConfig);
}
const { data } = await changeDocumentParser(
diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts
index 2c1d83e325..96234ec480 100644
--- a/web/src/hooks/use-knowledge-request.ts
+++ b/web/src/hooks/use-knowledge-request.ts
@@ -88,8 +88,8 @@ import {
useHandleSearchChange,
} from './logic-hooks';
import {
- extractParserConfigExt,
isPipelineParserConfig,
+ normalizeParserConfig,
} from './parser-config-utils';
import { useSetPaginationParams } from './route-hook';
import { DatasetGenerateKeys } from './use-dataset-generate';
@@ -218,10 +218,8 @@ export const useFetchNextKnowledgeListByPage = () => {
const { data } = await listDataset({
page_size: pagination.pageSize,
page: pagination.current,
- ext: {
- keywords: debouncedSearchString,
- owner_ids: filterValue.owner as string[],
- },
+ keywords: debouncedSearchString,
+ owner_ids: filterValue.owner as string[],
});
return { kbs: data?.data, total_datasets: data?.total_datasets };
@@ -283,10 +281,8 @@ export const useCreateKnowledge = () => {
chunk_method?: string;
parseType?: ParseType;
pipeline_id?: string | null;
- ext?: {
- language?: string;
- [key: string]: any;
- };
+ language?: string;
+ [key: string]: any;
}) => {
const { data = {} } = await kbService.createKb(params);
if (data.code === 0) {
@@ -364,7 +360,7 @@ export const useUpdateKnowledge = (shouldFetchList = false) => {
permission,
pagerank,
parser_config,
- ...ext
+ ...additionalFields
} = params;
const requestBody: Record<string, any> = {
name,
@@ -377,8 +373,8 @@ export const useUpdateKnowledge = (shouldFetchList = false) => {
pagerank,
parser_config: isPipelineParserConfig(parser_config)
? parser_config
- : extractParserConfigExt(parser_config),
- ...omit(ext, ['kb_id']),
+ : normalizeParserConfig(parser_config),
+ ...omit(additionalFields, ['kb_id']),
};
const { data = {} } = await updateKb(kbId, requestBody);
@@ -1073,7 +1069,7 @@ export const useFetchKnowledgeList = (
const { data } = await listDataset({
page,
page_size: pageSize,
- ...(keywords ? { ext: { keywords } } : {}),
+ ...(keywords ? { keywords } : {}),
});
return {
items: (data?.data ?? []) as IDataset[],
diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.ts
index fd4ec45063..3386839750 100644
--- a/web/src/interfaces/database/dataset.ts
+++ b/web/src/interfaces/database/dataset.ts
@@ -84,7 +84,6 @@ interface Parserconfig {
interface Raptor {
clustering_method?: 'gmm' | 'ahc';
- ext?: { clustering_method: 'gmm' | 'ahc'; tree_builder: 'raptor' | 'psi' };
max_cluster: number;
max_token: number;
prompt: string;
diff --git a/web/src/interfaces/request/knowledge.ts b/web/src/interfaces/request/knowledge.ts
index d729702e4f..3a64ff62ef 100644
--- a/web/src/interfaces/request/knowledge.ts
+++ b/web/src/interfaces/request/knowledge.ts
@@ -28,11 +28,9 @@ export interface IFetchKnowledgeListRequestParams {
id?: string;
page?: number;
page_size?: number;
- ext?: {
- keywords?: string;
- owner_ids?: string[];
- parser_id?: string;
- };
+ keywords?: string;
+ owner_ids?: string[];
+ parser_id?: string;
}
export interface IFetchDocumentListRequestBody {
diff --git a/web/src/pages/datasets/hooks.ts b/web/src/pages/datasets/hooks.ts
index 08ccba15cc..9c9929a2ca 100644
--- a/web/src/pages/datasets/hooks.ts
+++ b/web/src/pages/datasets/hooks.ts
@@ -21,10 +21,8 @@ export interface Iknowledge {
chunk_method?: string;
parseType?: ParseType;
pipeline_id?: string | null;
- ext?: {
- language?: string;
- [key: string]: any;
- };
+ language?: string;
+ [key: string]: any;
}
export const useSaveKnowledge = () => {
const { visible: visible, hideModal, showModal } = useSetModalState();
diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts
index dde2382d11..8990a69638 100644
--- a/web/src/services/knowledge-service.ts
+++ b/web/src/services/knowledge-service.ts
@@ -282,7 +282,11 @@ export function deleteKnowledgeGraph(knowledgeId: string) {
}
export const listDataset = (params?: IFetchKnowledgeListRequestParams) =>
- request.get(api.kbList, { params });
+ request.get(api.kbList, {
+ params: params
+ ? { ...params, owner_ids: params.owner_ids?.join(',') }
+ : params,
+ });
// Fetch datasets by a set of IDs via the `ids` query param (comma-joined).
// Used to echo back already-selected datasets whose names are not present
@@ -325,13 +329,11 @@ export const listDocument = (
if (!params || !params.id) {
throw new Error('params and params.id are required');
}
- // Extract page, page_size, and ext.keywords from params
- const { page, page_size, ext } = params;
- // Merge: page, page_size, keywords (from ext), body, and remaining params
+ const { page, page_size, keywords } = params;
const mergedParams = {
page,
page_size,
- keywords: ext?.keywords,
+ keywords,
...body,
};
return request.get(api.getDocumentList(params.id), { params: mergedParams });