RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor: combine 3 retrieval/search API as one (#19303)
Wang Qib7e577ce48d0
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex 757b3dae81..3865cf942b 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -36,6 +36,7 @@ from api.db.services.document_counter_service import release_reparse_countersfrom api.db.services.document_service import DocumentServicefrom api.db.services.knowledgebase_service import KnowledgebaseService, validate_dataset_embedding_modelsfrom api.db.services.llm_service import LLMBundle+from api.db.services.search_service import SearchServicefrom api.db.services.task_service import TaskService, cancel_all_task_offrom api.db.services.tenant_llm_service import TenantLLMServicefrom common.llm_request_context import normalize_llm_user_id, reset_llm_request_context, set_llm_request_context@@ -57,7 +58,7 @@ from api.utils.reference_metadata_utils import (from common import settingsfrom common.constants import LLMType, ParserType, RetCode, TaskStatusfrom common.doc_store.doc_store_base import OrderByExpr-from common.metadata_utils import convert_conditions, filter_doc_ids_by_metadata+from common.metadata_utils import apply_meta_data_filter, convert_conditions, filter_doc_ids_by_metadatafrom common.misc_utils import thread_pool_execfrom common.string_utils import is_content_empty, remove_redundant_spacesfrom common.tag_feature_utils import validate_tag_features@@ -329,11 +330,33 @@ async def stop_parsing(tenant_id, dataset_id):return get_result()[email protected]("/datasets/<dataset_id>/search", methods=["POST"]) # noqa: F821[email protected]("/datasets/search", methods=["POST"]) # noqa: F821@manager.route("/retrieval", methods=["POST"]) # noqa: F821@login_required@add_tenant_id_to_kwargs-async def retrieval_test(tenant_id):+async def retrieval_test(tenant_id, dataset_id=None):req = await get_request_json()+ if dataset_id:+ req["dataset_ids"] = [dataset_id]+ if "document_ids" not in req and "doc_ids" in req:+ req["document_ids"] = req["doc_ids"]+ if "page_size" not in req and "size" in req:+ req["page_size"] = req["size"]+ request_fields = set(req)+ search_id = req.get("search_id", "")+ search_config = {}+ if search_id:+ search_detail = SearchService.get_detail(search_id)+ if not search_detail or search_detail.get("tenant_id") != tenant_id:+ return get_error_data_result("Invalid search_id")+ search_config = dict(search_detail.get("search_config") or {})+ search_config.setdefault("rerank_candidates_count", 100)+ if "kb_ids" in search_config:+ search_config["dataset_ids"] = search_config["kb_ids"]+ if "doc_ids" in search_config:+ search_config["document_ids"] = search_config["doc_ids"]+ req = {**search_config, **req}if not req.get("dataset_ids"):return get_error_data_result("`dataset_ids` is required.")kb_ids = req["dataset_ids"]@@ -358,30 +381,59 @@ async def retrieval_test(tenant_id):toc_enhance = req.get("toc_enhance", False)langs = req.get("cross_languages", [])if not isinstance(doc_ids, list):- return get_error_data_result("`documents` should be a list")+ return get_error_data_result("`document_ids` should be a list")if doc_ids:doc_ids_list = KnowledgebaseService.list_documents_by_ids(kb_ids)for doc_id in doc_ids:if doc_id not in doc_ids_list:return get_error_data_result(f"The datasets don't own the document {doc_id}")- if not doc_ids:- metadata_condition = req.get("metadata_condition")- if metadata_condition:- doc_ids = filter_doc_ids_by_metadata(- kb_ids,- convert_conditions(metadata_condition),- metadata_condition.get("logic", "and"),- lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),- )- if not doc_ids and metadata_condition.get("conditions"):- return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})- if metadata_condition and not doc_ids:- doc_ids = ["-999"]+ metadata_condition = req.get("metadata_condition")+ meta_data_filter = None if "metadata_condition" in request_fields else req.get("meta_data_filter")+ if meta_data_filter:+ chat_mdl = None+ if meta_data_filter.get("method") in ["auto", "semi_auto"]:+ chat_id = req.get("chat_id", "")+ if chat_id:+ chat_model_config = resolve_model_config(tenant_id, LLMType.CHAT, chat_id)+ else:+ chat_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.CHAT)+ chat_mdl = LLMBundle(tenant_id, chat_model_config)+ doc_ids = await apply_meta_data_filter(+ meta_data_filter,+ None,+ question,+ chat_mdl,+ doc_ids,+ kb_ids=kb_ids,+ metas_loader=lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),+ )+ elif metadata_condition:+ filtered_doc_ids = filter_doc_ids_by_metadata(+ kb_ids,+ convert_conditions(metadata_condition),+ metadata_condition.get("logic", "and"),+ lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),+ )+ if doc_ids:+ filtered_doc_id_set = set(filtered_doc_ids)+ doc_ids = [doc_id for doc_id in doc_ids if doc_id in filtered_doc_id_set]else:- doc_ids = None- similarity_threshold = float(req.get("similarity_threshold", 0.2))- vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))- if "top_k" in req:+ doc_ids = filtered_doc_ids+ if not doc_ids and metadata_condition.get("conditions"):+ return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})+ if metadata_condition and not doc_ids:+ doc_ids = ["-999"]+ elif not doc_ids:+ doc_ids = None+ try:+ similarity_threshold = float(req.get("similarity_threshold", 0.2))+ except (TypeError, ValueError):+ return get_error_data_result("`similarity_threshold` should be a number")+ try:+ vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))+ except (TypeError, ValueError):+ return get_error_data_result("`vector_similarity_weight` should be a number")+ if "top_k" in request_fields:logging.warning("`top_k` is deprecated for POST /api/v1/retrieval; use `knn_top_k` instead.")knn_top_k_parameter = "knn_top_k" if "knn_top_k" in req else "top_k"try:@@ -451,6 +503,7 @@ async def retrieval_test(tenant_id):rerank_mdl=rerank_mdl,highlight=highlight,rank_feature=label_question(question, kbs),+ trace_id=search_id,must_not=None if include_knowledge_compilation else {"exists": "compile_kwd"},rerank_candidates_count=rerank_candidates_count,)diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex d3216be7b8..bf25dad294 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -26,8 +26,6 @@ from api.utils.validation_utils import (CreateDatasetReq,DeleteDatasetReq,ListDatasetReq,- SearchDatasetReq,- SearchDatasetsReq,UpdateDatasetReq,validate_and_parse_json_request,validate_and_parse_request_args,@@ -510,63 +508,6 @@ async def rename_tag(tenant_id, dataset_id):return get_error_data_result(message="Internal server error")[email protected]("/datasets/search", methods=["POST"]) # noqa: F821-@login_required-@add_tenant_id_to_kwargs-async def search_datasets(tenant_id):- """Search (retrieval test) across multiple datasets.-- POST /api/v1/datasets/search- JSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "knn_top_k": int (default 1024), "knn_num_candidates": int (default 2048), "page": int, "page_size": int, "size": int (fallback),- "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool, "highlight": bool,- "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}- The legacy "top_k" parameter is accepted as an alias for "knn_top_k".- "knn_num_candidates" currently applies only to Elasticsearch.- Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}- Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.- """- req, err = await validate_and_parse_json_request(request, SearchDatasetsReq)- if err is not None:- return get_error_argument_result(err)- success, result = await dataset_api_service.search_datasets(tenant_id, req)- if success:- return get_result(data=result)- else:- return get_error_data_result(message=result)--[email protected]("/datasets/<dataset_id>/search", methods=["POST"]) # noqa: F821-@login_required-@add_tenant_id_to_kwargs-async def search(tenant_id, dataset_id):- """Search (retrieval test) within a dataset.-- POST /api/v1/datasets/<dataset_id>/search- JSON body: {"question": str (required), "doc_ids": list[str], "knn_top_k": int (default 1024), "knn_num_candidates": int (default 2048), "page": int, "page_size": int, "size": int (fallback),- "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,- "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}- The legacy "top_k" parameter is accepted as an alias for "knn_top_k".- "knn_num_candidates" currently applies only to Elasticsearch.- Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}- Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.- """- req, err = await validate_and_parse_json_request(request, SearchDatasetReq)- if err is not None:- return get_error_argument_result(err)- req["dataset_ids"] = [dataset_id]- try:- success, result = await dataset_api_service.search_datasets(tenant_id, req)- if success:- return get_result(data=result)- else:- return get_error_data_result(message=result)- except Exception as e:- logging.exception(e)- if "not_found" in str(e):- return get_error_data_result(message="No chunk found! Check the chunk status please!")- return get_error_data_result(message="Internal server error")--@manager.route("/datasets/<dataset_id>/graph", methods=["GET"]) # noqa: F821@login_required@add_tenant_id_to_kwargsdiff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex df6722f520..8ef171de32 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -24,7 +24,7 @@ from typing import Annotated, Any, Literal, Union, get_args, get_originfrom uuid import UUIDfrom quart import Request-from pydantic import AliasChoices, BaseModel, ConfigDict, Field, StringConstraints, ValidationError, field_validator, model_validator, ValidationInfo+from pydantic import BaseModel, ConfigDict, Field, StringConstraints, ValidationError, field_validator, model_validator, ValidationInfofrom pydantic_core import PydanticCustomErrorfrom werkzeug.exceptions import BadRequest, UnsupportedMediaType@@ -963,70 +963,6 @@ class DeleteDocumentReq(DeleteReq):return v_list-class SearchDatasetReq(BaseModel):- """Request model for searching one dataset."""-- model_config = ConfigDict(extra="ignore")-- question: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1), Field(...)]- doc_ids: Annotated[list[str], Field(default=[])]- page: Annotated[int, Field(default=1, ge=1)]- page_size: Annotated[int | None, Field(default=None, ge=1, le=100)]- size: Annotated[int, Field(default=30, ge=1, le=100)]- rerank_candidates_count: Annotated[int, Field(default=64, ge=1)]- knn_top_k: Annotated[int, Field(default=1024, ge=1, le=2048, validation_alias=AliasChoices("knn_top_k", "top_k"))]- knn_num_candidates: Annotated[int, Field(default=2048, ge=1, le=10000)]- similarity_threshold: Annotated[float, Field(default=0.0, ge=0.0, le=1.0)]- vector_similarity_weight: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]- use_kg: Annotated[bool, Field(default=False)]- cross_languages: Annotated[list[str], Field(default=[])]- keyword: Annotated[bool, Field(default=False)]- search_id: Annotated[str | None, Field(default=None)]- rerank_id: Annotated[str | None, Field(default=None)]- tenant_rerank_id: Annotated[str | None, Field(default=None)]- meta_data_filter: Annotated[dict | None, Field(default=None)]- include_knowledge_compilation: Annotated[bool, Field(default=True)]-- @model_validator(mode="after")- def validate_knn_parameters(self):- if self.knn_num_candidates < self.knn_top_k:- raise ValueError("knn_num_candidates must be greater than or equal to knn_top_k")- return self---class SearchDatasetsReq(BaseModel):- """Request model for searching multiple datasets."""-- model_config = ConfigDict(extra="ignore")-- dataset_ids: Annotated[list[str], Field(..., min_length=1)]- question: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1), Field(...)]- doc_ids: Annotated[list[str], Field(default=[])]- page: Annotated[int, Field(default=1, ge=1)]- page_size: Annotated[int | None, Field(default=None, ge=1, le=100)]- size: Annotated[int, Field(default=30, ge=1, le=100)]- rerank_candidates_count: Annotated[int, Field(default=64, ge=1)]- knn_top_k: Annotated[int, Field(default=1024, ge=1, le=2048, validation_alias=AliasChoices("knn_top_k", "top_k"))]- knn_num_candidates: Annotated[int, Field(default=2048, ge=1, le=10000)]- similarity_threshold: Annotated[float, Field(default=0.0, ge=0.0, le=1.0)]- vector_similarity_weight: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]- use_kg: Annotated[bool, Field(default=False)]- cross_languages: Annotated[list[str], Field(default=[])]- keyword: Annotated[bool, Field(default=False)]- highlight: Annotated[bool, Field(default=False)]- search_id: Annotated[str | None, Field(default=None)]- rerank_id: Annotated[str | None, Field(default=None)]- tenant_rerank_id: Annotated[str | None, Field(default=None)]- meta_data_filter: Annotated[dict | None, Field(default=None)]- include_knowledge_compilation: Annotated[bool, Field(default=True)]-- @model_validator(mode="after")- def validate_knn_parameters(self):- if self.knn_num_candidates < self.knn_top_k:- raise ValueError("knn_num_candidates must be greater than or equal to knn_top_k")- return self--class BaseListReq(BaseModel):"""Shared pagination and sorting fields for list APIs."""diff --git a/docker/nginx/ragflow.conf.hybrid b/docker/nginx/ragflow.conf.hybridindex 3745c788d3..3de5e1e0e8 100644--- a/docker/nginx/ragflow.conf.hybrid+++ b/docker/nginx/ragflow.conf.hybrid@@ -45,11 +45,6 @@ server {include proxy.conf;}- location ~ ^/api/v1/datasets/search {- proxy_pass http://127.0.0.1:9384;- include proxy.conf;- }-location ~ ^/api/v1/chat/completions {proxy_pass http://127.0.0.1:9384;include proxy.conf;@@ -80,4 +75,4 @@ server {expires 10y;access_log off;}-}\ No newline at end of file+}diff --git a/internal/cli/cli.go b/internal/cli/cli.goindex dca88f429b..08e7b0525c 100644--- a/internal/cli/cli.go+++ b/internal/cli/cli.go@@ -983,18 +983,17 @@ Datasets syntax (full filter set):keyword true|false Enable keyword extraction via LLMuse_kg true|false Enable knowledge-graph augmentationrerank_id 'id' Rerank model to apply- tenant_rerank_id 'id' Tenant-scoped rerank model- search_id 'id' Idempotency / search-session id- meta_data_filter '<json>' Metadata filter (must be valid JSON)+ search_id 'id' Apply a saved search configuration+ metadata_condition '<json>' Metadata filter (must be valid JSON)cross_languages ['a','b'] Source languages to translate from- doc_ids ['d1',...] Restrict to specific document ids+ document_ids ['d1',...] Restrict to specific document idsExamples:search 'AI' on datasets 'kb_chinese' with top_k 10;search 'AI' on datasets 'kb1' 'kb2' with top_k 20 similarity_threshold 0.3 cross_languages ['Chinese']- doc_ids ['d1', 'd2'];+ document_ids ['d1', 'd2'];search 'manual' on datasets 'kb1' with- meta_data_filter '{"method":"manual","conditions":[{"key":"author","op":"eq","value":"Luo"}]}';+ metadata_condition '{"logic":"and","conditions":[{"name":"author","comparison_operator":"=","value":"Luo"}]}';`fmt.Println(help)}diff --git a/internal/cli/filesystem/dataset.go b/internal/cli/filesystem/dataset.goindex 8383743412..594aca7312 100644--- a/internal/cli/filesystem/dataset.go+++ b/internal/cli/filesystem/dataset.go@@ -369,9 +369,9 @@ func (p *DatasetProvider) searchWithRetrieval(ctx stdctx.Context, opts *SearchOpfunc (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{}, kbIDToName map[string]string) *Node {// Extract chunk content - try multiple field namescontent := ""- if v, ok := chunk["content_with_weight"].(string); ok && v != "" {+ if v, ok := chunk["content"].(string); ok && v != "" {content = v- } else if v, ok := chunk["content"].(string); ok && v != "" {+ } else if v, ok := chunk["content_with_weight"].(string); ok && v != "" {content = v} else if v, ok := chunk["content_ltks"].(string); ok && v != "" {content = v@@ -381,15 +381,17 @@ func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{},// Get chunk_id for URIchunkID := ""- if v, ok := chunk["chunk_id"].(string); ok {+ if v, ok := chunk["id"].(string); ok {chunkID = v- } else if v, ok := chunk["id"].(string); ok {+ } else if v, ok := chunk["chunk_id"].(string); ok {chunkID = v}// Get document name and IDdocName := ""- if v, ok := chunk["docnm_kwd"].(string); ok && v != "" {+ if v, ok := chunk["document_keyword"].(string); ok && v != "" {+ docName = v+ } else if v, ok := chunk["docnm_kwd"].(string); ok && v != "" {docName = v} else if v, ok := chunk["docnm"].(string); ok && v != "" {docName = v@@ -398,7 +400,9 @@ func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{},}docID := ""- if v, ok := chunk["doc_id"].(string); ok && v != "" {+ if v, ok := chunk["document_id"].(string); ok && v != "" {+ docID = v+ } else if v, ok := chunk["doc_id"].(string); ok && v != "" {docID = v}@@ -407,7 +411,9 @@ func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{},datasetID := ""// First try to get kb_id from chunk (could be string or array)- if v, ok := chunk["kb_id"].(string); ok && v != "" {+ if v, ok := chunk["dataset_id"].(string); ok && v != "" {+ datasetID = v+ } else if v, ok := chunk["kb_id"].(string); ok && v != "" {datasetID = v} else if v, ok := chunk["kb_id"].([]interface{}); ok && len(v) > 0 {if s, ok := v[0].(string); ok {diff --git a/internal/cli/user_command.go b/internal/cli/user_command.goindex a8b3277320..caea23aa45 100644--- a/internal/cli/user_command.go+++ b/internal/cli/user_command.go@@ -636,6 +636,13 @@ func formatEmptyArray(v interface{}) string {return fmt.Sprintf("%v", v)}+func retrievalChunkValue(chunk map[string]interface{}, key, fallbackKey string) interface{} {+ if value, ok := chunk[key]; ok {+ return value+ }+ return chunk[fallbackKey]+}+// SearchOnDatasets searches for chunks in specified datasets// Returns (result_map, error) - result_map is non-nil for benchmark modefunc (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {@@ -697,9 +704,6 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {if val, ok := cmd.Params["rerank_id"]; ok {payload["rerank_id"] = val}- if val, ok := cmd.Params["tenant_rerank_id"]; ok {- payload["tenant_rerank_id"] = val- }if val, ok := cmd.Params["page_size"]; ok {payload["page_size"] = val}@@ -707,21 +711,19 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {payload["page"] = val}if val, ok := cmd.Params["search_id"]; ok {- if s, ok := val.(string); ok {- payload["search_id"] = s- }+ payload["search_id"] = val}if val, ok := cmd.Params["cross_languages"]; ok {if list, ok := val.([]string); ok {payload["cross_languages"] = list}}- if val, ok := cmd.Params["doc_ids"]; ok {+ if val, ok := cmd.Params["document_ids"]; ok {if list, ok := val.([]string); ok {- payload["doc_ids"] = list+ payload["document_ids"] = list}}- if val, ok := cmd.Params["meta_data_filter"]; ok {+ if val, ok := cmd.Params["metadata_condition"]; ok {// Accept either a raw JSON string from the CLI or a pre-decoded// map[string]interface{} (future-proofing for callers that// construct the command programmatically). The string form is@@ -730,13 +732,13 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {case string:var decoded map[string]interface{}if err := json.Unmarshal([]byte(v), &decoded); err != nil {- return nil, fmt.Errorf("invalid meta_data_filter JSON: %w", err)+ return nil, fmt.Errorf("invalid metadata_condition JSON: %w", err)}- payload["meta_data_filter"] = decoded+ payload["metadata_condition"] = decodedcase map[string]interface{}:- payload["meta_data_filter"] = v+ payload["metadata_condition"] = vdefault:- return nil, fmt.Errorf("meta_data_filter must be JSON string or object")+ return nil, fmt.Errorf("metadata_condition must be JSON string or object")}}@@ -782,12 +784,12 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {for _, chunk := range chunks {if chunkMap, ok := chunk.(map[string]interface{}); ok {row := map[string]interface{}{- "id": chunkMap["chunk_id"],- "content": chunkMap["content_with_weight"],- "document_id": chunkMap["doc_id"],- "dataset_id": chunkMap["kb_id"],- "docnm_kwd": chunkMap["docnm_kwd"],- "image_id": chunkMap["image_id"],+ "id": retrievalChunkValue(chunkMap, "id", "chunk_id"),+ "content": retrievalChunkValue(chunkMap, "content", "content_with_weight"),+ "document_id": retrievalChunkValue(chunkMap, "document_id", "doc_id"),+ "dataset_id": retrievalChunkValue(chunkMap, "dataset_id", "kb_id"),+ "document_keyword": retrievalChunkValue(chunkMap, "document_keyword", "docnm_kwd"),+ "image_id": retrievalChunkValue(chunkMap, "image_id", "img_id"),"similarity": chunkMap["similarity"],"term_similarity": chunkMap["term_similarity"],"vector_similarity": chunkMap["vector_similarity"],@@ -796,8 +798,8 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {if v, ok := chunkMap["doc_type_kwd"]; ok {row["doc_type_kwd"] = formatEmptyArray(v)}- if v, ok := chunkMap["important_kwd"]; ok {- row["important_kwd"] = formatEmptyArray(v)+ if v := retrievalChunkValue(chunkMap, "important_keywords", "important_kwd"); v != nil {+ row["important_keywords"] = formatEmptyArray(v)}if v, ok := chunkMap["mom_id"]; ok {row["mom_id"] = formatEmptyArray(v)diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.goindex eecc6b846e..8eb149b881 100644--- a/internal/cli/user_parser.go+++ b/internal/cli/user_parser.go@@ -2279,7 +2279,7 @@ func (p *Parser) parseAPIRetrieve() (*Command, error) {p.nextToken()case TokenLBracket:// List value: parsed inside the switch below by- // cross_languages / doc_ids. No value is captured here.+ // cross_languages / document_ids. No value is captured here.paramValue = nildefault:// EOF, ';', or any other non-value token: the option@@ -2316,16 +2316,21 @@ func (p *Parser) parseAPIRetrieve() (*Command, error) {default:return nil, fmt.Errorf("WITH option %q must be true or false, got %q", paramName, s)}- case "rerank_id", "tenant_rerank_id", "search_id", "meta_data_filter":+ case "rerank_id", "search_id":if valueToken != TokenQuotedString {return nil, fmt.Errorf("WITH option %q must be a quoted string, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))}- // meta_data_filter JSON string is decoded into a map in+ cmd.Params[paramName] = paramValue+ case "metadata_condition":+ if valueToken != TokenQuotedString {+ return nil, fmt.Errorf("WITH option %q must be a quoted string, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))+ }+ // metadata_condition JSON string is decoded into a map in// the SearchOnDatasets handler; parser stores the raw// string so the handler can surface a clean error on// invalid JSON.cmd.Params[paramName] = paramValue- case "cross_languages", "doc_ids":+ case "cross_languages", "document_ids":if p.curToken.Type != TokenLBracket {return nil, fmt.Errorf("WITH option %q must be a list, e.g. %q ['a', 'b']", paramName, paramName)}diff --git a/internal/handler/dataset.go b/internal/handler/dataset.goindex 3426e97729..d9d6b3cdcd 100644--- a/internal/handler/dataset.go+++ b/internal/handler/dataset.go@@ -1156,9 +1156,9 @@ func (h *DatasetsHandler) SearchDatasets(c *gin.Context) {defaultPage := 1req.Page = &defaultPage}- if req.Size == nil {+ if req.PageSize == nil && req.Size == nil {defaultSize := 30- req.Size = &defaultSize+ req.PageSize = &defaultSize}if req.KNNTopK == nil && req.TopK == nil {defaultTopK := 1024@@ -1269,17 +1269,20 @@ func (h *DatasetsHandler) SearchDataset(c *gin.Context) {}func validateSearchDatasetsRequest(req *service.SearchDatasetsRequest) error {- return validateSearchParams(req.Page, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)+ return validateSearchParams(req.Page, req.PageSize, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)}func validateSearchDatasetRequest(req *service.SearchDatasetRequest) error {- return validateSearchParams(req.Page, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)+ return validateSearchParams(req.Page, req.PageSize, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)}-func validateSearchParams(page, size, knnTopK, topK, knnNumCandidates *int, similarityThreshold, vectorSimilarityWeight *float64) error {+func validateSearchParams(page, pageSize, size, knnTopK, topK, knnNumCandidates *int, similarityThreshold, vectorSimilarityWeight *float64) error {if page != nil && *page < 1 {return fmt.Errorf("page must be greater than or equal to 1")}+ if pageSize != nil && *pageSize < 1 {+ return fmt.Errorf("page_size must be greater than or equal to 1")+ }if size != nil && *size < 1 {return fmt.Errorf("size must be greater than or equal to 1")}diff --git a/internal/handler/dataset_search_test.go b/internal/handler/dataset_search_test.goindex b093d08a76..2ccc795a58 100644--- a/internal/handler/dataset_search_test.go+++ b/internal/handler/dataset_search_test.go@@ -50,7 +50,7 @@ func TestDatasetsHandlerSearchDataset(t *testing.T) {h := &DatasetsHandler{searchDatasetService: fake}rec := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/ds-1/search", strings.NewReader(`{"question":"hello","doc_ids":["doc-1"],"knn_top_k":7,"knn_num_candidates":14}`))+ req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/ds-1/search", strings.NewReader(`{"question":"hello","document_ids":["doc-1"],"page_size":9,"metadata_condition":{"logic":"and","conditions":[{"name":"author","comparison_operator":"=","value":"Luo"}]},"knn_top_k":7,"knn_num_candidates":14}`))req.Header.Set("Content-Type", "application/json")c, _ := gin.CreateTestContext(rec)c.Request = req@@ -65,9 +65,12 @@ func TestDatasetsHandlerSearchDataset(t *testing.T) {if fake.datasetID != "ds-1" || fake.userID != "user-1" {t.Fatalf("call args = (%q,%q), want (ds-1,user-1)", fake.datasetID, fake.userID)}- if fake.req == nil || fake.req.Question != "hello" || len(fake.req.DocIDs) != 1 || fake.req.DocIDs[0] != "doc-1" {+ if fake.req == nil || fake.req.Question != "hello" || len(fake.req.DocumentIDs) != 1 || fake.req.DocumentIDs[0] != "doc-1" {t.Fatalf("request = %#v", fake.req)}+ if fake.req.PageSize == nil || *fake.req.PageSize != 9 || fake.req.MetadataCondition["logic"] != "and" {+ t.Fatalf("public request fields = %#v", fake.req)+ }if fake.req.KNNTopK == nil || *fake.req.KNNTopK != 7 || fake.req.KNNNumCandidates == nil || *fake.req.KNNNumCandidates != 14 {t.Fatalf("KNN parameters = (%v, %v), want (7, 14)", fake.req.KNNTopK, fake.req.KNNNumCandidates)}@@ -160,7 +163,7 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {h := &DatasetsHandler{searchDatasetsService: fake}rec := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7,"include_knowledge_compilation":false}`))+ req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"document_ids":["doc-1"],"page_size":9,"metadata_condition":{"logic":"and","conditions":[]},"top_k":7,"include_knowledge_compilation":false}`))req.Header.Set("Content-Type", "application/json")c, _ := gin.CreateTestContext(rec)c.Request = req@@ -180,6 +183,9 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {if fake.req.TopK == nil || *fake.req.TopK != 7 {t.Fatalf("legacy top_k alias=%v want 7", fake.req.TopK)}+ if fake.req.PageSize == nil || *fake.req.PageSize != 9 || len(fake.req.DocumentIDs) != 1 || fake.req.MetadataCondition["logic"] != "and" {+ t.Fatalf("public request fields = %#v", fake.req)+ }body := decodeSearchResponse(t, rec)if body["code"] != float64(common.CodeSuccess) {t.Fatalf("code=%v want=%d body=%s", body["code"], common.CodeSuccess, rec.Body.String())@@ -232,12 +238,12 @@ func TestDatasetsHandlerSearchDatasetsValidationErrorsUseArgumentEnvelope(t *tesfunc TestValidateSearchParamsKNNBounds(t *testing.T) {knnTopK := 2048- if err := validateSearchParams(nil, nil, &knnTopK, nil, nil, nil, nil); err != nil {+ if err := validateSearchParams(nil, nil, nil, &knnTopK, nil, nil, nil, nil); err != nil {t.Fatalf("knn_top_k=2048 with default knn_num_candidates should be valid: %v", err)}legacyTopK := 2049- err := validateSearchParams(nil, nil, nil, &legacyTopK, nil, nil, nil)+ err := validateSearchParams(nil, nil, nil, nil, &legacyTopK, nil, nil, nil)if err == nil || err.Error() != "top_k (alias for knn_top_k) must be between 1 and 2048" {t.Fatalf("legacy top_k error = %v", err)}diff --git a/internal/handler/mcp_server.go b/internal/handler/mcp_server.goindex 721bb7e0bf..9346363bb3 100644--- a/internal/handler/mcp_server.go+++ b/internal/handler/mcp_server.go@@ -169,7 +169,7 @@ func MCPRetrieval(ctx context.Context, ds *dataset.DatasetService, userID stringsearchReq := &service.SearchDatasetsRequest{DatasetIDs: datasetIDs,Question: req.Question,- DocIDs: req.DocumentIDs,+ DocumentIDs: req.DocumentIDs,ForceRefresh: req.ForceRefresh,}@@ -179,7 +179,7 @@ func MCPRetrieval(ctx context.Context, ds *dataset.DatasetService, userID string}if req.PageSize > 0 {v := req.PageSize- searchReq.Size = &v+ searchReq.PageSize = &v}if req.TopK > 0 {v := req.TopKdiff --git a/internal/router/datasets_search_route_test.go b/internal/router/datasets_search_route_test.goindex 7920b27ba1..9e8fef3a17 100644--- a/internal/router/datasets_search_route_test.go+++ b/internal/router/datasets_search_route_test.go@@ -10,7 +10,7 @@ import ("ragflow/internal/handler")-func TestRouterSetupRegistersSearchDatasetRoute(t *testing.T) {+func TestRouterSetupRegistersDatasetSearchRoutes(t *testing.T) {gin.SetMode(gin.TestMode)engine := gin.New()@@ -20,14 +20,15 @@ func TestRouterSetupRegistersSearchDatasetRoute(t *testing.T) {}r.Setup(engine)- resp := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/dataset-1/search", nil)- engine.ServeHTTP(resp, req)-- if resp.Code == http.StatusNotFound {- t.Fatalf("POST /api/v1/datasets/:dataset_id/search returned 404; SearchDataset route is not registered")- }- if resp.Code != http.StatusUnauthorized {- t.Fatalf("status=%d body=%s; want auth middleware to handle registered SearchDataset route", resp.Code, resp.Body.String())+ for _, path := range []string{"/api/v1/datasets/dataset-1/search", "/api/v1/datasets/search", "/api/v1/retrieval"} {+ resp := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodPost, path, nil)+ engine.ServeHTTP(resp, req)+ if resp.Code == http.StatusNotFound {+ t.Fatalf("POST %s returned 404", path)+ }+ if resp.Code != http.StatusUnauthorized {+ t.Fatalf("POST %s status=%d body=%s; want auth middleware to handle registered route", path, resp.Code, resp.Body.String())+ }}}diff --git a/internal/router/router.go b/internal/router/router.goindex 14d96e4cf4..736b71abe9 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -340,6 +340,7 @@ func (r *Router) Setup(engine *gin.Engine) {chat.POST("/audio/transcription", r.chatHandler.ChatAudioTranscription)}v1.POST("/openai/:chat_id/chat/completions", r.openaiChatHandler.OpenAIChatCompletions)+ v1.POST("/retrieval", r.datasetsHandler.SearchDatasets)// Dataset routesdatasets := v1.Group("/datasets")diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 9f8827bc63..3c89ff890a 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -35,7 +35,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchpage = *req.Page}pageSize := 30- if req.Size != nil {+ if req.PageSize != nil {+ pageSize = *req.PageSize+ } else if req.Size != nil {pageSize = *req.Size}rerankCandidatesCount := 64@@ -46,7 +48,7 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchif req.UseKG != nil {useKG = *req.UseKG}- similarityThreshold := 0.0+ similarityThreshold := 0.2if req.SimilarityThreshold != nil {similarityThreshold = *req.SimilarityThreshold}@@ -85,7 +87,23 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchquestion := req.QuestiondatasetIDs := req.DatasetIDsmetadataFilter := req.MetadataFilter+ hasMetadataCondition := req.MetadataCondition != nil+ if req.MetadataCondition != nil {+ manual := make([]interface{}, 0)+ if conditions, ok := req.MetadataCondition["conditions"].([]interface{}); ok {+ for _, item := range conditions {+ if condition, ok := item.(map[string]interface{}); ok {+ manual = append(manual, map[string]interface{}{"key": condition["name"], "op": condition["comparison_operator"], "value": condition["value"]})+ }+ }+ }+ metadataFilter = map[string]interface{}{"method": "manual", "logic": req.MetadataCondition["logic"], "manual": manual}+ }crossLanguages := req.CrossLanguages+ documentIDs := req.DocumentIDs+ if documentIDs == nil {+ documentIDs = req.DocIDs+ }modelProviderSvc := service.NewModelProviderService()@@ -135,6 +153,11 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchcommon.Warn("Invalid search_id", zap.String("searchID", searchID))return nil, fmt.Errorf("invalid search_id")}+ searchTenantID, ok := searchDetail["tenant_id"].(string)+ if !ok || searchTenantID != userID {+ common.Warn("Invalid search_id", zap.String("searchID", searchID))+ return nil, fmt.Errorf("invalid search_id")+ }if searchConfig, ok := searchDetail["search_config"].(map[string]interface{}); ok && searchConfig != nil {rerankCandidatesCount = 100@@ -209,8 +232,8 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search}// Apply meta_data_filter to get filtered doc_ids- docIDs := make([]string, len(req.DocIDs))- copy(docIDs, req.DocIDs)+ docIDs := make([]string, len(documentIDs))+ copy(docIDs, documentIDs)if len(metadataFilter) > 0 {metadataSvc := service.NewMetadataService()flattedMeta, err := metadataSvc.GetFlattedMetaByKBs(ctx, datasetIDs)@@ -218,8 +241,15 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchcommon.Warn("Failed to get flatted metadata, using empty metadata for filter", zap.Error(err))flattedMeta = make(common.MetaData)}- filteredDocIDs, _ := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, req.DocIDs, datasetIDs)- docIDs = filteredDocIDs+ if hasMetadataCondition {+ filteredDocIDs, _ := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, documentIDs, datasetIDs)+ docIDs = filteredDocIDs+ } else {+ filteredDocIDs, filterReturnedEmpty := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, nil, datasetIDs)+ if !filterReturnedEmpty {+ docIDs = append(docIDs, filteredDocIDs...)+ }+ }}// Apply cross_languages and keyword extraction@@ -305,8 +335,23 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.SearchfilteredChunks = nlp.RetrievalByChildren(filteredChunks, tenantIDs, d.docEngine, ctx)+ keyMapping := map[string]string{+ "chunk_id": "id",+ "content_with_weight": "content",+ "doc_id": "document_id",+ "important_kwd": "important_keywords",+ "question_kwd": "questions",+ "docnm_kwd": "document_keyword",+ "kb_id": "dataset_id",+ }for i := range filteredChunks {delete(filteredChunks[i], "vector")+ for oldKey, newKey := range keyMapping {+ if value, ok := filteredChunks[i][oldKey]; ok {+ filteredChunks[i][newKey] = value+ delete(filteredChunks[i], oldKey)+ }+ }}common.Info("SearchDatasets completed", zap.String("userID", userID), zap.Any("kbID", datasetIDs), zap.String("question", question), zap.Int64("chunkCount", int64(len(filteredChunks))))diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.goindex b6057b1763..08e48c0284 100644--- a/internal/service/dataset/search_test.go+++ b/internal/service/dataset/search_test.go@@ -8,7 +8,7 @@ import (func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {page := 2- size := 15+ pageSize := 15knnTopK := 128knnNumCandidates := 256useKG := true@@ -21,14 +21,14 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {req := &service.SearchDatasetRequest{Question: "hello world",Page: &page,- Size: &size,- DocIDs: []string{"doc-1", "doc-2"},+ PageSize: &pageSize,+ DocumentIDs: []string{"doc-1", "doc-2"},UseKG: &useKG,KNNTopK: &knnTopK,KNNNumCandidates: &knnNumCandidates,CrossLanguages: []string{"en", "zh"},SearchID: &searchID,- MetadataFilter: map[string]interface{}{"method": "manual"},+ MetadataCondition: map[string]interface{}{"logic": "and"},RerankID: &rerankID,Keyword: &keyword,SimilarityThreshold: &similarityThreshold,@@ -40,16 +40,16 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {if len(converted.DatasetIDs) != 1 || converted.DatasetIDs[0] != "dataset-1" {t.Fatalf("dataset_ids=%v want [dataset-1]", converted.DatasetIDs)}- if converted.Question != req.Question || converted.Page != req.Page || converted.Size != req.Size {+ if converted.Question != req.Question || converted.Page != req.Page || converted.PageSize != req.PageSize {t.Fatalf("converted request did not preserve pagination/question fields: %#v", converted)}- if len(converted.DocIDs) != 2 || converted.DocIDs[0] != "doc-1" || converted.DocIDs[1] != "doc-2" {- t.Fatalf("doc_ids=%v want [doc-1 doc-2]", converted.DocIDs)+ if len(converted.DocumentIDs) != 2 || converted.DocumentIDs[0] != "doc-1" || converted.DocumentIDs[1] != "doc-2" {+ t.Fatalf("document_ids=%v want [doc-1 doc-2]", converted.DocumentIDs)}if converted.UseKG != req.UseKG || converted.KNNTopK != req.KNNTopK || converted.KNNNumCandidates != req.KNNNumCandidates || converted.SearchID != req.SearchID {t.Fatalf("converted request did not preserve optional fields: %#v", converted)}- if converted.MetadataFilter["method"] != "manual" || converted.RerankID != req.RerankID || converted.Keyword != req.Keyword {+ if converted.MetadataCondition["logic"] != "and" || converted.RerankID != req.RerankID || converted.Keyword != req.Keyword {t.Fatalf("converted request did not preserve search config fields: %#v", converted)}if converted.SimilarityThreshold != req.SimilarityThreshold || converted.VectorSimilarityWeight != req.VectorSimilarityWeight {diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.goindex 99c8a8262c..aa52adb5a6 100644--- a/internal/service/dataset_types.go+++ b/internal/service/dataset_types.go@@ -40,8 +40,10 @@ type SearchDatasetsRequest struct {DatasetIDs []string `json:"dataset_ids" binding:"required"`Question string `json:"question" binding:"required"`Page *int `json:"page,omitempty"`+ PageSize *int `json:"page_size,omitempty"`Size *int `json:"size,omitempty"`RerankCandidatesCount *int `json:"rerank_candidates_count,omitempty"`+ DocumentIDs []string `json:"document_ids,omitempty"`DocIDs []string `json:"doc_ids,omitempty"`UseKG *bool `json:"use_kg,omitempty"`KNNTopK *int `json:"knn_top_k,omitempty"`@@ -49,6 +51,7 @@ type SearchDatasetsRequest struct {KNNNumCandidates *int `json:"knn_num_candidates,omitempty"`CrossLanguages []string `json:"cross_languages,omitempty"`SearchID *string `json:"search_id,omitempty"`+ MetadataCondition map[string]interface{} `json:"metadata_condition,omitempty"`MetadataFilter map[string]interface{} `json:"meta_data_filter,omitempty"`RerankID *string `json:"rerank_id,omitempty"`Keyword *bool `json:"keyword,omitempty"`@@ -71,8 +74,10 @@ type SearchDatasetsResponse struct {type SearchDatasetRequest struct {Question string `json:"question"`Page *int `json:"page,omitempty"`+ PageSize *int `json:"page_size,omitempty"`Size *int `json:"size,omitempty"`RerankCandidatesCount *int `json:"rerank_candidates_count,omitempty"`+ DocumentIDs []string `json:"document_ids,omitempty"`DocIDs []string `json:"doc_ids,omitempty"`UseKG *bool `json:"use_kg,omitempty"`KNNTopK *int `json:"knn_top_k,omitempty"`@@ -80,6 +85,7 @@ type SearchDatasetRequest struct {KNNNumCandidates *int `json:"knn_num_candidates,omitempty"`CrossLanguages []string `json:"cross_languages,omitempty"`SearchID *string `json:"search_id,omitempty"`+ MetadataCondition map[string]interface{} `json:"metadata_condition,omitempty"`MetadataFilter map[string]interface{} `json:"meta_data_filter,omitempty"`RerankID *string `json:"rerank_id,omitempty"`Keyword *bool `json:"keyword,omitempty"`@@ -97,8 +103,10 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *SearDatasetIDs: []string{datasetID},Question: req.Question,Page: req.Page,+ PageSize: req.PageSize,Size: req.Size,RerankCandidatesCount: req.RerankCandidatesCount,+ DocumentIDs: req.DocumentIDs,DocIDs: req.DocIDs,UseKG: req.UseKG,KNNTopK: req.KNNTopK,@@ -106,6 +114,7 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *SearKNNNumCandidates: req.KNNNumCandidates,CrossLanguages: req.CrossLanguages,SearchID: req.SearchID,+ MetadataCondition: req.MetadataCondition,MetadataFilter: req.MetadataFilter,RerankID: req.RerankID,Keyword: req.Keyword,diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex 77233bec89..6cc96d88bc 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -2427,7 +2427,7 @@ def test_dataset_search_endpoint(rest_client, ensure_parsed_document):dataset_id, _ = ensure_parsed_document()res = rest_client.post(f"/datasets/{dataset_id}/search",- json={"question": "test TXT file", "page": 1, "size": 10},+ json={"question": "test TXT file", "page": 1, "page_size": 10},)assert res.status_code == 200payload = res.json()@@ -2439,17 +2439,17 @@ def test_dataset_search_endpoint(rest_client, ensure_parsed_document):@pytest.mark.parametrize("payload",[- {"question": "test TXT file", "page": 1, "size": 2},+ {"question": "test TXT file", "page": 1, "page_size": 2},{"question": "test TXT file", "similarity_threshold": 0.5},{"question": "test TXT file", "vector_similarity_weight": 0.7},{"question": "test TXT file", "top_k": 10},],ids=["page_size", "similarity_threshold", "vector_similarity_weight", "top_k"],)-def test_dataset_search_params_and_doc_ids_contract(rest_client, ensure_parsed_document, payload):+def test_dataset_search_params_and_document_ids_contract(rest_client, ensure_parsed_document, payload):dataset_id, document_id = ensure_parsed_document()search_payload = dict(payload)- search_payload["doc_ids"] = [document_id]+ search_payload["document_ids"] = [document_id]res = rest_client.post(f"/datasets/{dataset_id}/search", json=search_payload)assert res.status_code == 200body = res.json()@@ -2463,7 +2463,7 @@ def test_dataset_search_requires_question(rest_client, create_dataset):res = rest_client.post(f"/datasets/{dataset_id}/search", json={})assert res.status_code == 200payload = res.json()- assert payload["code"] == 101, payload+ assert payload["code"] in [101, 102], payloadassert "question" in payload["message"], payloaddiff --git a/test/testcases/restful_api/test_retrieval.py b/test/testcases/restful_api/test_retrieval.pyindex 19936aa89d..b4514b5163 100644--- a/test/testcases/restful_api/test_retrieval.py+++ b/test/testcases/restful_api/test_retrieval.py@@ -24,27 +24,38 @@ from test.testcases.utils import wait_for@pytest.mark.p3def test_dataset_search_rest_endpoint(rest_client, ensure_parsed_document):dataset_id, _ = ensure_parsed_document()+ request_body = {"question": "test TXT file", "top_k": 5}res = rest_client.post(f"/datasets/{dataset_id}/search",- json={"question": "test TXT file", "top_k": 5},+ json=request_body,)assert res.status_code == 200payload = res.json()assert payload["code"] == 0, payloadassert "chunks" in payload["data"], payload+ retrieval_payload = rest_client.post("/retrieval", json={"dataset_ids": [dataset_id], **request_body}).json()+ assert payload == retrieval_payload@pytest.mark.p3def test_multi_dataset_search_rest_endpoint(rest_client, ensure_parsed_document):dataset_id, _ = ensure_parsed_document()+ request_body = {"dataset_ids": [dataset_id], "question": "test TXT file", "top_k": 5}res = rest_client.post("/datasets/search",- json={"dataset_ids": [dataset_id], "question": "test TXT file", "top_k": 5},+ json=request_body,)assert res.status_code == 200payload = res.json()assert payload["code"] == 0, payloadassert "chunks" in payload["data"], payload+ for chunk in payload["data"]["chunks"]:+ assert "id" in chunk, payload+ assert "content" in chunk, payload+ assert "document_id" in chunk, payload+ assert "dataset_id" in chunk, payload+ retrieval_payload = rest_client.post("/retrieval", json=request_body).json()+ assert payload == retrieval_payload@pytest.mark.p3@@ -67,10 +78,9 @@ def test_multi_dataset_search_with_metadata_filter(rest_client, ensure_parsed_dojson={"dataset_ids": [dataset_id],"question": "test TXT file",- "meta_data_filter": {- "method": "manual",+ "metadata_condition": {"logic": "and",- "manual": [{"key": "author", "op": "=", "value": "qa_batch2"}],+ "conditions": [{"name": "author", "comparison_operator": "=", "value": "qa_batch2"}],},},)@@ -81,9 +91,8 @@ def test_multi_dataset_search_with_metadata_filter(rest_client, ensure_parsed_do@pytest.mark.p3-def test_retrieval_compatibility_endpoint(rest_client, ensure_parsed_document):+def test_retrieval_endpoint(rest_client, ensure_parsed_document):dataset_id, _ = ensure_parsed_document()- # /api/v1/retrieval is SDK compatibility endpoint registered from chunk_api.py.res = rest_client.post("/retrieval",json={"dataset_ids": [dataset_id], "question": "test TXT file", "top_k": 5},@@ -255,7 +264,7 @@ def test_retrieval_vector_similarity_and_top_k_contract(rest_client, ensure_pars("vector 0", {"vector_similarity_weight": 0}, 0, ""),("vector 0.5", {"vector_similarity_weight": 0.5}, 0, ""),("vector 10", {"vector_similarity_weight": 10}, 0, ""),- ("vector alpha", {"vector_similarity_weight": "a"}, 100, "could not convert string to float"),+ ("vector alpha", {"vector_similarity_weight": "a"}, 102, "`vector_similarity_weight` should be a number"),("top_k 10", {"top_k": 10}, 0, ""),("top_k 1", {"top_k": 1}, 0, ""),("top_k -1", {"top_k": -1}, 102, "`top_k` must be greater than 0"),diff --git a/web/src/components/top-select.tsx b/web/src/components/top-select.tsxindex 3b017f1006..6ae1ec76c0 100644--- a/web/src/components/top-select.tsx+++ b/web/src/components/top-select.tsx@@ -55,7 +55,7 @@ export function TopSelectFormItem() {return (<SliderInputFormField- name="size"+ name="page_size"label={t('chat.topN')}min={1}max={100}diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.tsindex 96234ec480..9d765bc160 100644--- a/web/src/hooks/use-knowledge-request.ts+++ b/web/src/hooks/use-knowledge-request.ts@@ -137,7 +137,7 @@ export const useTestRetrieval = () => {...values,kb_id: values?.kb_id || knowledgeBaseId,page: 1,- doc_ids: filterValue.doc_ids,+ document_ids: filterValue.doc_ids,highlight: true,include_knowledge_compilation: false,};@@ -164,7 +164,7 @@ export const useTestRetrieval = () => {if (mutation.data && queryParams.question) {const newParams = {...queryParams,- doc_ids: value.doc_ids ?? [],+ document_ids: value.doc_ids ?? [],page: 1,};mutation.mutate(newParams);@@ -1295,7 +1295,7 @@ export const useTestChunkRetrieval = (): ResponsePostType<ITestingResult> & {...values,kb_id: values.kb_id ?? knowledgeBaseId,page,- size: pageSize,+ page_size: pageSize,});if (data.code === 0) {const res = data.data;@@ -1338,9 +1338,9 @@ export const useTestChunkAllRetrieval = (): ResponsePostType<ITestingResult> & {const { data } = await kbService.retrievalTest({...values,kb_id: values.kb_id ?? knowledgeBaseId,- doc_ids: [],+ document_ids: [],page,- size: pageSize,+ page_size: pageSize,});if (data.code === 0) {const res = data.data;diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.tsindex 3386839750..f9468c0ed3 100644--- a/web/src/interfaces/database/dataset.ts+++ b/web/src/interfaces/database/dataset.ts@@ -171,22 +171,20 @@ export interface IChunk {}export interface ITestingChunk {- chunk_id: string;+ id: string;content_ltks: string;- content_with_weight: string;- doc_id: string;- doc_name: string;- img_id: string;+ content: string;+ document_id: string;+ document_keyword: string;image_id: string;- important_kwd: any[];- kb_id: string;+ important_keywords: any[];+ questions?: any[];+ dataset_id: string;similarity: number;term_similarity: number;- vector: number[];vector_similarity: number;highlight: string;positions: number[][];- docnm_kwd: string;doc_type_kwd: string;document_metadata?: Record<string, any>;}diff --git a/web/src/interfaces/request/knowledge.ts b/web/src/interfaces/request/knowledge.tsindex 3a64ff62ef..1d8c09db59 100644--- a/web/src/interfaces/request/knowledge.ts+++ b/web/src/interfaces/request/knowledge.ts@@ -1,6 +1,6 @@export interface ITestRetrievalRequestBody {question: string;- size: number;+ page_size: number;rerank_candidates_count: number;similarity_threshold: number;vector_similarity_weight: number;diff --git a/web/src/pages/dataset/testing/testing-form.tsx b/web/src/pages/dataset/testing/testing-form.tsxindex 0232115fc7..adddae6601 100644--- a/web/src/pages/dataset/testing/testing-form.tsx+++ b/web/src/pages/dataset/testing/testing-form.tsx@@ -66,10 +66,10 @@ export default function TestingForm({...vectorSimilarityWeightSchema,dataset_ids: z.array(z.string()).optional(),...MetadataFilterSchema,- size: z.number().int().min(1).max(100),+ page_size: z.number().int().min(1).max(100),...rerankCandidatesCountSchema,})- .refine((values) => values.rerank_candidates_count >= values.size, {+ .refine((values) => values.rerank_candidates_count >= values.page_size, {message: t('chat.rerankCandidatesCountValidation'),path: ['rerank_candidates_count'],});@@ -80,7 +80,7 @@ export default function TestingForm({...initialSimilarityThresholdValue,...initialVectorSimilarityWeightValue,dataset_ids: [knowledgeBaseId],- size: 10,+ page_size: 10,rerank_candidates_count: 64,},});diff --git a/web/src/pages/dataset/testing/testing-result.tsx b/web/src/pages/dataset/testing/testing-result.tsxindex c15022ecb3..fda9964f7c 100644--- a/web/src/pages/dataset/testing/testing-result.tsx+++ b/web/src/pages/dataset/testing/testing-result.tsx@@ -82,19 +82,19 @@ export function TestingResult({<><section className="px-5 pb-5 flex flex-col gap-5 overflow-auto scrollbar-thin min-h-0">{data.chunks?.map((x) => (- <article key={x.chunk_id}>+ <article key={x.id}><Card className="px-5 py-2.5 bg-transparent shadow-none"><ChunkTitle item={x}></ChunkTitle><divclassName="!mt-2.5 whitespace-pre-wrap [&_em]:text-accent-primary [&_em]:not-italic"dangerouslySetInnerHTML={{__html: sanitizeHtmlWithImagesAsText(- x.highlight || x.content_with_weight,+ x.highlight || x.content,),}}/><div className="mt-2.5 text-right text-xs text-text-sub-title-invert">- {x.doc_name || x.docnm_kwd}+ {x.document_keyword}</div></Card></article>diff --git a/web/src/pages/next-search/document-preview-modal/index.tsx b/web/src/pages/next-search/document-preview-modal/index.tsxindex 2f7800abd3..c50a730efe 100644--- a/web/src/pages/next-search/document-preview-modal/index.tsx+++ b/web/src/pages/next-search/document-preview-modal/index.tsx@@ -30,6 +30,7 @@ interface IProps extends IModalProps<any> {documentId: string;chunk: {docnm_kwd?: string;+ document_keyword?: string;document_name?: string;positions?: number[][];content_with_weight?: string;@@ -53,7 +54,8 @@ const PdfDrawer = ({);// const ref = useRef<(highlight: IHighlight) => void>(() => {});// const [loaded, setLoaded] = useState(false);- const documentName = chunk.docnm_kwd || chunk.document_name;+ const documentName =+ chunk.document_keyword || chunk.docnm_kwd || chunk.document_name;const fileType = documentName ? getFileExtensionRegex(documentName) : '';const isWebPage = !fileType && !!chunk.document_url;const url = isWebPage ? (chunk.document_url as string) : getDocumentUrl();diff --git a/web/src/pages/next-search/hooks.ts b/web/src/pages/next-search/hooks.tsindex f81c4f6ec0..f2f149d6f8 100644--- a/web/src/pages/next-search/hooks.ts+++ b/web/src/pages/next-search/hooks.ts@@ -168,7 +168,7 @@ export const useTestChunkRetrieval = (mutationFn: async (values: any) => {const { data } = await retrievalTestFunc({page,- size: pageSize,+ page_size: pageSize,...values,kb_id: values.kb_id ?? knowledgeBaseId,tenant_id: tenantId,@@ -218,7 +218,7 @@ export const useTestChunkAllRetrieval = (mutationFn: async (values: any) => {const { data } = await retrievalTestFunc({page,- size: pageSize,+ page_size: pageSize,...values,kb_id: values.kb_id ?? knowledgeBaseId,tenant_id: tenantId,@@ -265,9 +265,11 @@ export const useTestRetrieval = (kb_id: kbIds,highlight: true,question: q,- doc_ids: Array.isArray(selectedDocumentIds) ? selectedDocumentIds : [],+ document_ids: Array.isArray(selectedDocumentIds)+ ? selectedDocumentIds+ : [],page: pagination.current,- size: pagination.pageSize,+ page_size: pagination.pageSize,});}, [sendingLoading,@@ -368,7 +370,7 @@ export const useSendQuestion = (highlight: true,question: q,page: 1,- size: pageSize,+ page_size: pageSize,search_id: searchId,});@@ -415,9 +417,9 @@ export const useSendQuestion = (kb_id: kbIds,highlight: true,question: q,- doc_ids: documentIds ?? selectedDocumentIds,+ document_ids: documentIds ?? selectedDocumentIds,page,- size,+ page_size: size,search_id: searchId,});@@ -425,9 +427,9 @@ export const useSendQuestion = (kb_id: kbIds,highlight: true,question: q,- doc_ids: [],+ document_ids: [],page,- size,+ page_size: size,search_id: searchId,});},diff --git a/web/src/pages/next-search/search-view.tsx b/web/src/pages/next-search/search-view.tsxindex 336816ea11..3d7d983c17 100644--- a/web/src/pages/next-search/search-view.tsx+++ b/web/src/pages/next-search/search-view.tsx@@ -251,15 +251,15 @@ export default function SearchingView({<div key={index}><div className="w-full flex flex-col"><div className="w-full">- {(chunk.image_id || chunk.img_id) && (+ {chunk.image_id && (<ImageWithPopover- id={chunk.image_id || chunk.img_id}+ id={chunk.image_id}></ImageWithPopover>)}<divdangerouslySetInnerHTML={{__html: sanitizeHtmlWithImagesAsText(- chunk.highlight || chunk.content_with_weight,+ chunk.highlight || chunk.content,).trim(),}}className={classNames(@@ -292,11 +292,14 @@ export default function SearchingView({<divclassName="flex gap-2 items-center text-xs text-text-secondary border p-1 rounded-lg w-fit mt-3"onClick={() =>- clickDocumentButton(chunk.doc_id, chunk as any)+ clickDocumentButton(+ chunk.document_id,+ chunk as any,+ )}>- <FileIcon name={chunk.docnm_kwd}></FileIcon>- {chunk.docnm_kwd}+ <FileIcon name={chunk.document_keyword}></FileIcon>+ {chunk.document_keyword}</div></div>{index < chunks.length - 1 && (diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.tsindex 8990a69638..2c6339e402 100644--- a/web/src/services/knowledge-service.ts+++ b/web/src/services/knowledge-service.ts@@ -115,6 +115,42 @@ const mapChunkToLegacy = (chunk: Record<string, any>) => ({positions: chunk.positions || chunk.position_int || [],});+const mapChunkToRetrieval = (chunk: Record<string, any>) => ({+ ...chunk,+ id: chunk.id || chunk.chunk_id,+ content: chunk.content ?? chunk.content_with_weight,+ document_id: chunk.document_id || chunk.doc_id,+ document_keyword: chunk.document_keyword || chunk.docnm_kwd || chunk.doc_name,+ dataset_id: chunk.dataset_id || chunk.kb_id,+ important_keywords: chunk.important_keywords || chunk.important_kwd || [],+ questions: chunk.questions || chunk.question_kwd || [],+});++const mapRetrievalResponse = (response: any) => {+ if (response.data?.code === 0) {+ response.data.data = {+ ...response.data.data,+ chunks: (response.data.data?.chunks || []).map(mapChunkToRetrieval),+ };+ }+ return response;+};++const toLegacyMetadataFilter = (condition?: Record<string, any>) => {+ if (!condition?.conditions?.length) {+ return undefined;+ }+ return {+ method: 'manual',+ logic: condition.logic,+ manual: condition.conditions.map((item: Record<string, any>) => ({+ key: item.name,+ op: item.comparison_operator,+ value: item.value,+ })),+ };+};+const mapDocumentToLegacy = (doc: Record<string, any>) => ({...doc,chunk_num: doc.chunk_num ?? doc.chunk_count,@@ -157,9 +193,45 @@ const chunkService = {delete rest.dataset_id;delete rest.kb_id;delete rest.knowledge_id;- return request.post(api.retrievalTest, {- data: { ...rest, dataset_ids: datasetIds },+ const data = {+ dataset_ids: datasetIds,+ document_ids: rest.document_ids ?? rest.doc_ids,+ question: rest.question,+ page: rest.page,+ page_size: rest.page_size ?? rest.size,+ similarity_threshold: rest.similarity_threshold,+ vector_similarity_weight: rest.vector_similarity_weight,+ top_k: rest.top_k,+ knn_top_k: rest.knn_top_k,+ knn_num_candidates: rest.knn_num_candidates,+ rerank_candidates_count: rest.rerank_candidates_count,+ rerank_id: rest.rerank_id,+ search_id: rest.search_id,+ keyword: rest.keyword,+ highlight: rest.highlight,+ cross_languages: rest.cross_languages,+ meta_data_filter: rest.meta_data_filter,+ chat_id: rest.chat_id,+ use_kg: rest.use_kg,+ toc_enhance: rest.toc_enhance,+ include_knowledge_compilation: rest.include_knowledge_compilation,+ reference_metadata: rest.reference_metadata,+ };+ const response = await request.post(api.retrievalTest, {+ data,+ });+ return mapRetrievalResponse(response);+ },+ retrievalTestShare: async (params: Record<string, any>) => {+ const response = await baseKbService.retrievalTestShare({+ ...params,+ doc_ids: params.doc_ids ?? params.document_ids,+ size: params.size ?? params.page_size,+ meta_data_filter:+ params.meta_data_filter ??+ toLegacyMetadataFilter(params.metadata_condition),});+ return mapRetrievalResponse(response);},chunkList: async (params: Record<string, any>) => {const datasetId = getDatasetId(params);diff --git a/web/vite.config.ts b/web/vite.config.tsindex 8bcd60a7b2..71a3e50118 100644--- a/web/vite.config.ts+++ b/web/vite.config.ts@@ -103,7 +103,7 @@ export default defineConfig(({ mode }) => {changeOrigin: true,ws: true,},- '^(/api/v1/datasets/search)|^(/api/v1/chat/completions)': {+ '^(/api/v1/chat/completions)': {target: 'http://127.0.0.1:9384/',changeOrigin: true,ws: true,