Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: combine 3 retrieval/search API as one (#19303)

Wang Qib7e577ce48d0
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.py
index 757b3dae81..3865cf942b 100644
--- a/api/apps/restful_apis/chunk_api.py
+++ b/api/apps/restful_apis/chunk_api.py
@@ -36,6 +36,7 @@ from api.db.services.document_counter_service import release_reparse_counters
from api.db.services.document_service import DocumentService
from api.db.services.knowledgebase_service import KnowledgebaseService, validate_dataset_embedding_models
from api.db.services.llm_service import LLMBundle
+from api.db.services.search_service import SearchService
from api.db.services.task_service import TaskService, cancel_all_task_of
from api.db.services.tenant_llm_service import TenantLLMService
from common.llm_request_context import normalize_llm_user_id, reset_llm_request_context, set_llm_request_context
@@ -57,7 +58,7 @@ from api.utils.reference_metadata_utils import (
from common import settings
from common.constants import LLMType, ParserType, RetCode, TaskStatus
from common.doc_store.doc_store_base import OrderByExpr
-from common.metadata_utils import convert_conditions, filter_doc_ids_by_metadata
+from common.metadata_utils import apply_meta_data_filter, convert_conditions, filter_doc_ids_by_metadata
from common.misc_utils import thread_pool_exec
from common.string_utils import is_content_empty, remove_redundant_spaces
from common.tag_feature_utils import validate_tag_features
@@ -329,11 +330,33 @@ async def stop_parsing(tenant_id, dataset_id):
return get_result()
[email protected]("/datasets/<dataset_id>/search", methods=["POST"]) # noqa: F821
[email protected]("/datasets/search", methods=["POST"]) # noqa: F821
@manager.route("/retrieval", methods=["POST"]) # noqa: F821
@login_required
@add_tenant_id_to_kwargs
-async def retrieval_test(tenant_id):
+async def retrieval_test(tenant_id, dataset_id=None):
req = await get_request_json()
+ if dataset_id:
+ req["dataset_ids"] = [dataset_id]
+ if "document_ids" not in req and "doc_ids" in req:
+ req["document_ids"] = req["doc_ids"]
+ if "page_size" not in req and "size" in req:
+ req["page_size"] = req["size"]
+ request_fields = set(req)
+ search_id = req.get("search_id", "")
+ search_config = {}
+ if search_id:
+ search_detail = SearchService.get_detail(search_id)
+ if not search_detail or search_detail.get("tenant_id") != tenant_id:
+ return get_error_data_result("Invalid search_id")
+ search_config = dict(search_detail.get("search_config") or {})
+ search_config.setdefault("rerank_candidates_count", 100)
+ if "kb_ids" in search_config:
+ search_config["dataset_ids"] = search_config["kb_ids"]
+ if "doc_ids" in search_config:
+ search_config["document_ids"] = search_config["doc_ids"]
+ req = {**search_config, **req}
if not req.get("dataset_ids"):
return get_error_data_result("`dataset_ids` is required.")
kb_ids = req["dataset_ids"]
@@ -358,30 +381,59 @@ async def retrieval_test(tenant_id):
toc_enhance = req.get("toc_enhance", False)
langs = req.get("cross_languages", [])
if not isinstance(doc_ids, list):
- return get_error_data_result("`documents` should be a list")
+ return get_error_data_result("`document_ids` should be a list")
if doc_ids:
doc_ids_list = KnowledgebaseService.list_documents_by_ids(kb_ids)
for doc_id in doc_ids:
if doc_id not in doc_ids_list:
return get_error_data_result(f"The datasets don't own the document {doc_id}")
- if not doc_ids:
- metadata_condition = req.get("metadata_condition")
- if metadata_condition:
- doc_ids = filter_doc_ids_by_metadata(
- kb_ids,
- convert_conditions(metadata_condition),
- metadata_condition.get("logic", "and"),
- lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),
- )
- if not doc_ids and metadata_condition.get("conditions"):
- return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})
- if metadata_condition and not doc_ids:
- doc_ids = ["-999"]
+ metadata_condition = req.get("metadata_condition")
+ meta_data_filter = None if "metadata_condition" in request_fields else req.get("meta_data_filter")
+ if meta_data_filter:
+ chat_mdl = None
+ if meta_data_filter.get("method") in ["auto", "semi_auto"]:
+ chat_id = req.get("chat_id", "")
+ if chat_id:
+ chat_model_config = resolve_model_config(tenant_id, LLMType.CHAT, chat_id)
+ else:
+ chat_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.CHAT)
+ chat_mdl = LLMBundle(tenant_id, chat_model_config)
+ doc_ids = await apply_meta_data_filter(
+ meta_data_filter,
+ None,
+ question,
+ chat_mdl,
+ doc_ids,
+ kb_ids=kb_ids,
+ metas_loader=lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),
+ )
+ elif metadata_condition:
+ filtered_doc_ids = filter_doc_ids_by_metadata(
+ kb_ids,
+ convert_conditions(metadata_condition),
+ metadata_condition.get("logic", "and"),
+ lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),
+ )
+ if doc_ids:
+ filtered_doc_id_set = set(filtered_doc_ids)
+ doc_ids = [doc_id for doc_id in doc_ids if doc_id in filtered_doc_id_set]
else:
- doc_ids = None
- similarity_threshold = float(req.get("similarity_threshold", 0.2))
- vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
- if "top_k" in req:
+ doc_ids = filtered_doc_ids
+ if not doc_ids and metadata_condition.get("conditions"):
+ return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})
+ if metadata_condition and not doc_ids:
+ doc_ids = ["-999"]
+ elif not doc_ids:
+ doc_ids = None
+ try:
+ similarity_threshold = float(req.get("similarity_threshold", 0.2))
+ except (TypeError, ValueError):
+ return get_error_data_result("`similarity_threshold` should be a number")
+ try:
+ vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
+ except (TypeError, ValueError):
+ return get_error_data_result("`vector_similarity_weight` should be a number")
+ if "top_k" in request_fields:
logging.warning("`top_k` is deprecated for POST /api/v1/retrieval; use `knn_top_k` instead.")
knn_top_k_parameter = "knn_top_k" if "knn_top_k" in req else "top_k"
try:
@@ -451,6 +503,7 @@ async def retrieval_test(tenant_id):
rerank_mdl=rerank_mdl,
highlight=highlight,
rank_feature=label_question(question, kbs),
+ trace_id=search_id,
must_not=None if include_knowledge_compilation else {"exists": "compile_kwd"},
rerank_candidates_count=rerank_candidates_count,
)
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py
index d3216be7b8..bf25dad294 100644
--- a/api/apps/restful_apis/dataset_api.py
+++ b/api/apps/restful_apis/dataset_api.py
@@ -26,8 +26,6 @@ from api.utils.validation_utils import (
CreateDatasetReq,
DeleteDatasetReq,
ListDatasetReq,
- SearchDatasetReq,
- SearchDatasetsReq,
UpdateDatasetReq,
validate_and_parse_json_request,
validate_and_parse_request_args,
@@ -510,63 +508,6 @@ async def rename_tag(tenant_id, dataset_id):
return get_error_data_result(message="Internal server error")
[email protected]("/datasets/search", methods=["POST"]) # noqa: F821
-@login_required
-@add_tenant_id_to_kwargs
-async def search_datasets(tenant_id):
- """Search (retrieval test) across multiple datasets.
-
- POST /api/v1/datasets/search
- JSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "knn_top_k": int (default 1024), "knn_num_candidates": int (default 2048), "page": int, "page_size": int, "size": int (fallback),
- "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool, "highlight": bool,
- "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}
- The legacy "top_k" parameter is accepted as an alias for "knn_top_k".
- "knn_num_candidates" currently applies only to Elasticsearch.
- Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}
- Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.
- """
- req, err = await validate_and_parse_json_request(request, SearchDatasetsReq)
- if err is not None:
- return get_error_argument_result(err)
- success, result = await dataset_api_service.search_datasets(tenant_id, req)
- if success:
- return get_result(data=result)
- else:
- return get_error_data_result(message=result)
-
-
[email protected]("/datasets/<dataset_id>/search", methods=["POST"]) # noqa: F821
-@login_required
-@add_tenant_id_to_kwargs
-async def search(tenant_id, dataset_id):
- """Search (retrieval test) within a dataset.
-
- POST /api/v1/datasets/<dataset_id>/search
- JSON body: {"question": str (required), "doc_ids": list[str], "knn_top_k": int (default 1024), "knn_num_candidates": int (default 2048), "page": int, "page_size": int, "size": int (fallback),
- "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,
- "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}
- The legacy "top_k" parameter is accepted as an alias for "knn_top_k".
- "knn_num_candidates" currently applies only to Elasticsearch.
- Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}
- Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.
- """
- req, err = await validate_and_parse_json_request(request, SearchDatasetReq)
- if err is not None:
- return get_error_argument_result(err)
- req["dataset_ids"] = [dataset_id]
- try:
- success, result = await dataset_api_service.search_datasets(tenant_id, req)
- if success:
- return get_result(data=result)
- else:
- return get_error_data_result(message=result)
- except Exception as e:
- logging.exception(e)
- if "not_found" in str(e):
- return get_error_data_result(message="No chunk found! Check the chunk status please!")
- return get_error_data_result(message="Internal server error")
-
-
@manager.route("/datasets/<dataset_id>/graph", methods=["GET"]) # noqa: F821
@login_required
@add_tenant_id_to_kwargs
diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py
index df6722f520..8ef171de32 100644
--- a/api/utils/validation_utils.py
+++ b/api/utils/validation_utils.py
@@ -24,7 +24,7 @@ from typing import Annotated, Any, Literal, Union, get_args, get_origin
from uuid import UUID
from quart import Request
-from pydantic import AliasChoices, BaseModel, ConfigDict, Field, StringConstraints, ValidationError, field_validator, model_validator, ValidationInfo
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints, ValidationError, field_validator, model_validator, ValidationInfo
from pydantic_core import PydanticCustomError
from werkzeug.exceptions import BadRequest, UnsupportedMediaType
@@ -963,70 +963,6 @@ class DeleteDocumentReq(DeleteReq):
return v_list
-class SearchDatasetReq(BaseModel):
- """Request model for searching one dataset."""
-
- model_config = ConfigDict(extra="ignore")
-
- question: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1), Field(...)]
- doc_ids: Annotated[list[str], Field(default=[])]
- page: Annotated[int, Field(default=1, ge=1)]
- page_size: Annotated[int | None, Field(default=None, ge=1, le=100)]
- size: Annotated[int, Field(default=30, ge=1, le=100)]
- rerank_candidates_count: Annotated[int, Field(default=64, ge=1)]
- knn_top_k: Annotated[int, Field(default=1024, ge=1, le=2048, validation_alias=AliasChoices("knn_top_k", "top_k"))]
- knn_num_candidates: Annotated[int, Field(default=2048, ge=1, le=10000)]
- similarity_threshold: Annotated[float, Field(default=0.0, ge=0.0, le=1.0)]
- vector_similarity_weight: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]
- use_kg: Annotated[bool, Field(default=False)]
- cross_languages: Annotated[list[str], Field(default=[])]
- keyword: Annotated[bool, Field(default=False)]
- search_id: Annotated[str | None, Field(default=None)]
- rerank_id: Annotated[str | None, Field(default=None)]
- tenant_rerank_id: Annotated[str | None, Field(default=None)]
- meta_data_filter: Annotated[dict | None, Field(default=None)]
- include_knowledge_compilation: Annotated[bool, Field(default=True)]
-
- @model_validator(mode="after")
- def validate_knn_parameters(self):
- if self.knn_num_candidates < self.knn_top_k:
- raise ValueError("knn_num_candidates must be greater than or equal to knn_top_k")
- return self
-
-
-class SearchDatasetsReq(BaseModel):
- """Request model for searching multiple datasets."""
-
- model_config = ConfigDict(extra="ignore")
-
- dataset_ids: Annotated[list[str], Field(..., min_length=1)]
- question: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1), Field(...)]
- doc_ids: Annotated[list[str], Field(default=[])]
- page: Annotated[int, Field(default=1, ge=1)]
- page_size: Annotated[int | None, Field(default=None, ge=1, le=100)]
- size: Annotated[int, Field(default=30, ge=1, le=100)]
- rerank_candidates_count: Annotated[int, Field(default=64, ge=1)]
- knn_top_k: Annotated[int, Field(default=1024, ge=1, le=2048, validation_alias=AliasChoices("knn_top_k", "top_k"))]
- knn_num_candidates: Annotated[int, Field(default=2048, ge=1, le=10000)]
- similarity_threshold: Annotated[float, Field(default=0.0, ge=0.0, le=1.0)]
- vector_similarity_weight: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]
- use_kg: Annotated[bool, Field(default=False)]
- cross_languages: Annotated[list[str], Field(default=[])]
- keyword: Annotated[bool, Field(default=False)]
- highlight: Annotated[bool, Field(default=False)]
- search_id: Annotated[str | None, Field(default=None)]
- rerank_id: Annotated[str | None, Field(default=None)]
- tenant_rerank_id: Annotated[str | None, Field(default=None)]
- meta_data_filter: Annotated[dict | None, Field(default=None)]
- include_knowledge_compilation: Annotated[bool, Field(default=True)]
-
- @model_validator(mode="after")
- def validate_knn_parameters(self):
- if self.knn_num_candidates < self.knn_top_k:
- raise ValueError("knn_num_candidates must be greater than or equal to knn_top_k")
- return self
-
-
class BaseListReq(BaseModel):
"""Shared pagination and sorting fields for list APIs."""
diff --git a/docker/nginx/ragflow.conf.hybrid b/docker/nginx/ragflow.conf.hybrid
index 3745c788d3..3de5e1e0e8 100644
--- a/docker/nginx/ragflow.conf.hybrid
+++ b/docker/nginx/ragflow.conf.hybrid
@@ -45,11 +45,6 @@ server {
include proxy.conf;
}
- location ~ ^/api/v1/datasets/search {
- proxy_pass http://127.0.0.1:9384;
- include proxy.conf;
- }
-
location ~ ^/api/v1/chat/completions {
proxy_pass http://127.0.0.1:9384;
include proxy.conf;
@@ -80,4 +75,4 @@ server {
expires 10y;
access_log off;
}
-}
\ No newline at end of file
+}
diff --git a/internal/cli/cli.go b/internal/cli/cli.go
index dca88f429b..08e7b0525c 100644
--- a/internal/cli/cli.go
+++ b/internal/cli/cli.go
@@ -983,18 +983,17 @@ Datasets syntax (full filter set):
keyword true|false Enable keyword extraction via LLM
use_kg true|false Enable knowledge-graph augmentation
rerank_id 'id' Rerank model to apply
- tenant_rerank_id 'id' Tenant-scoped rerank model
- search_id 'id' Idempotency / search-session id
- meta_data_filter '<json>' Metadata filter (must be valid JSON)
+ search_id 'id' Apply a saved search configuration
+ metadata_condition '<json>' Metadata filter (must be valid JSON)
cross_languages ['a','b'] Source languages to translate from
- doc_ids ['d1',...] Restrict to specific document ids
+ document_ids ['d1',...] Restrict to specific document ids
Examples:
search 'AI' on datasets 'kb_chinese' with top_k 10;
search 'AI' on datasets 'kb1' 'kb2' with top_k 20 similarity_threshold 0.3 cross_languages ['Chinese']
- doc_ids ['d1', 'd2'];
+ document_ids ['d1', 'd2'];
search 'manual' on datasets 'kb1' with
- meta_data_filter '{"method":"manual","conditions":[{"key":"author","op":"eq","value":"Luo"}]}';
+ metadata_condition '{"logic":"and","conditions":[{"name":"author","comparison_operator":"=","value":"Luo"}]}';
`
fmt.Println(help)
}
diff --git a/internal/cli/filesystem/dataset.go b/internal/cli/filesystem/dataset.go
index 8383743412..594aca7312 100644
--- a/internal/cli/filesystem/dataset.go
+++ b/internal/cli/filesystem/dataset.go
@@ -369,9 +369,9 @@ func (p *DatasetProvider) searchWithRetrieval(ctx stdctx.Context, opts *SearchOp
func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{}, kbIDToName map[string]string) *Node {
// Extract chunk content - try multiple field names
content := ""
- if v, ok := chunk["content_with_weight"].(string); ok && v != "" {
+ if v, ok := chunk["content"].(string); ok && v != "" {
content = v
- } else if v, ok := chunk["content"].(string); ok && v != "" {
+ } else if v, ok := chunk["content_with_weight"].(string); ok && v != "" {
content = v
} else if v, ok := chunk["content_ltks"].(string); ok && v != "" {
content = v
@@ -381,15 +381,17 @@ func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{},
// Get chunk_id for URI
chunkID := ""
- if v, ok := chunk["chunk_id"].(string); ok {
+ if v, ok := chunk["id"].(string); ok {
chunkID = v
- } else if v, ok := chunk["id"].(string); ok {
+ } else if v, ok := chunk["chunk_id"].(string); ok {
chunkID = v
}
// Get document name and ID
docName := ""
- if v, ok := chunk["docnm_kwd"].(string); ok && v != "" {
+ if v, ok := chunk["document_keyword"].(string); ok && v != "" {
+ docName = v
+ } else if v, ok := chunk["docnm_kwd"].(string); ok && v != "" {
docName = v
} else if v, ok := chunk["docnm"].(string); ok && v != "" {
docName = v
@@ -398,7 +400,9 @@ func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{},
}
docID := ""
- if v, ok := chunk["doc_id"].(string); ok && v != "" {
+ if v, ok := chunk["document_id"].(string); ok && v != "" {
+ docID = v
+ } else if v, ok := chunk["doc_id"].(string); ok && v != "" {
docID = v
}
@@ -407,7 +411,9 @@ func (p *DatasetProvider) chunkToNodeWithKBMapping(chunk map[string]interface{},
datasetID := ""
// First try to get kb_id from chunk (could be string or array)
- if v, ok := chunk["kb_id"].(string); ok && v != "" {
+ if v, ok := chunk["dataset_id"].(string); ok && v != "" {
+ datasetID = v
+ } else if v, ok := chunk["kb_id"].(string); ok && v != "" {
datasetID = v
} else if v, ok := chunk["kb_id"].([]interface{}); ok && len(v) > 0 {
if s, ok := v[0].(string); ok {
diff --git a/internal/cli/user_command.go b/internal/cli/user_command.go
index a8b3277320..caea23aa45 100644
--- a/internal/cli/user_command.go
+++ b/internal/cli/user_command.go
@@ -636,6 +636,13 @@ func formatEmptyArray(v interface{}) string {
return fmt.Sprintf("%v", v)
}
+func retrievalChunkValue(chunk map[string]interface{}, key, fallbackKey string) interface{} {
+ if value, ok := chunk[key]; ok {
+ return value
+ }
+ return chunk[fallbackKey]
+}
+
// SearchOnDatasets searches for chunks in specified datasets
// Returns (result_map, error) - result_map is non-nil for benchmark mode
func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
@@ -697,9 +704,6 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
if val, ok := cmd.Params["rerank_id"]; ok {
payload["rerank_id"] = val
}
- if val, ok := cmd.Params["tenant_rerank_id"]; ok {
- payload["tenant_rerank_id"] = val
- }
if val, ok := cmd.Params["page_size"]; ok {
payload["page_size"] = val
}
@@ -707,21 +711,19 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
payload["page"] = val
}
if val, ok := cmd.Params["search_id"]; ok {
- if s, ok := val.(string); ok {
- payload["search_id"] = s
- }
+ payload["search_id"] = val
}
if val, ok := cmd.Params["cross_languages"]; ok {
if list, ok := val.([]string); ok {
payload["cross_languages"] = list
}
}
- if val, ok := cmd.Params["doc_ids"]; ok {
+ if val, ok := cmd.Params["document_ids"]; ok {
if list, ok := val.([]string); ok {
- payload["doc_ids"] = list
+ payload["document_ids"] = list
}
}
- if val, ok := cmd.Params["meta_data_filter"]; ok {
+ if val, ok := cmd.Params["metadata_condition"]; ok {
// Accept either a raw JSON string from the CLI or a pre-decoded
// map[string]interface{} (future-proofing for callers that
// construct the command programmatically). The string form is
@@ -730,13 +732,13 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
case string:
var decoded map[string]interface{}
if err := json.Unmarshal([]byte(v), &decoded); err != nil {
- return nil, fmt.Errorf("invalid meta_data_filter JSON: %w", err)
+ return nil, fmt.Errorf("invalid metadata_condition JSON: %w", err)
}
- payload["meta_data_filter"] = decoded
+ payload["metadata_condition"] = decoded
case map[string]interface{}:
- payload["meta_data_filter"] = v
+ payload["metadata_condition"] = v
default:
- return nil, fmt.Errorf("meta_data_filter must be JSON string or object")
+ return nil, fmt.Errorf("metadata_condition must be JSON string or object")
}
}
@@ -782,12 +784,12 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
for _, chunk := range chunks {
if chunkMap, ok := chunk.(map[string]interface{}); ok {
row := map[string]interface{}{
- "id": chunkMap["chunk_id"],
- "content": chunkMap["content_with_weight"],
- "document_id": chunkMap["doc_id"],
- "dataset_id": chunkMap["kb_id"],
- "docnm_kwd": chunkMap["docnm_kwd"],
- "image_id": chunkMap["image_id"],
+ "id": retrievalChunkValue(chunkMap, "id", "chunk_id"),
+ "content": retrievalChunkValue(chunkMap, "content", "content_with_weight"),
+ "document_id": retrievalChunkValue(chunkMap, "document_id", "doc_id"),
+ "dataset_id": retrievalChunkValue(chunkMap, "dataset_id", "kb_id"),
+ "document_keyword": retrievalChunkValue(chunkMap, "document_keyword", "docnm_kwd"),
+ "image_id": retrievalChunkValue(chunkMap, "image_id", "img_id"),
"similarity": chunkMap["similarity"],
"term_similarity": chunkMap["term_similarity"],
"vector_similarity": chunkMap["vector_similarity"],
@@ -796,8 +798,8 @@ func (c *CLI) SearchOnDatasets(cmd *Command) (ResponseIf, error) {
if v, ok := chunkMap["doc_type_kwd"]; ok {
row["doc_type_kwd"] = formatEmptyArray(v)
}
- if v, ok := chunkMap["important_kwd"]; ok {
- row["important_kwd"] = formatEmptyArray(v)
+ if v := retrievalChunkValue(chunkMap, "important_keywords", "important_kwd"); v != nil {
+ row["important_keywords"] = formatEmptyArray(v)
}
if v, ok := chunkMap["mom_id"]; ok {
row["mom_id"] = formatEmptyArray(v)
diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.go
index eecc6b846e..8eb149b881 100644
--- a/internal/cli/user_parser.go
+++ b/internal/cli/user_parser.go
@@ -2279,7 +2279,7 @@ func (p *Parser) parseAPIRetrieve() (*Command, error) {
p.nextToken()
case TokenLBracket:
// List value: parsed inside the switch below by
- // cross_languages / doc_ids. No value is captured here.
+ // cross_languages / document_ids. No value is captured here.
paramValue = nil
default:
// EOF, ';', or any other non-value token: the option
@@ -2316,16 +2316,21 @@ func (p *Parser) parseAPIRetrieve() (*Command, error) {
default:
return nil, fmt.Errorf("WITH option %q must be true or false, got %q", paramName, s)
}
- case "rerank_id", "tenant_rerank_id", "search_id", "meta_data_filter":
+ case "rerank_id", "search_id":
if valueToken != TokenQuotedString {
return nil, fmt.Errorf("WITH option %q must be a quoted string, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))
}
- // meta_data_filter JSON string is decoded into a map in
+ cmd.Params[paramName] = paramValue
+ case "metadata_condition":
+ if valueToken != TokenQuotedString {
+ return nil, fmt.Errorf("WITH option %q must be a quoted string, got %s", paramName, tokenTypeDescription(valueToken, p.curToken))
+ }
+ // metadata_condition JSON string is decoded into a map in
// the SearchOnDatasets handler; parser stores the raw
// string so the handler can surface a clean error on
// invalid JSON.
cmd.Params[paramName] = paramValue
- case "cross_languages", "doc_ids":
+ case "cross_languages", "document_ids":
if p.curToken.Type != TokenLBracket {
return nil, fmt.Errorf("WITH option %q must be a list, e.g. %q ['a', 'b']", paramName, paramName)
}
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index 3426e97729..d9d6b3cdcd 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -1156,9 +1156,9 @@ func (h *DatasetsHandler) SearchDatasets(c *gin.Context) {
defaultPage := 1
req.Page = &defaultPage
}
- if req.Size == nil {
+ if req.PageSize == nil && req.Size == nil {
defaultSize := 30
- req.Size = &defaultSize
+ req.PageSize = &defaultSize
}
if req.KNNTopK == nil && req.TopK == nil {
defaultTopK := 1024
@@ -1269,17 +1269,20 @@ func (h *DatasetsHandler) SearchDataset(c *gin.Context) {
}
func validateSearchDatasetsRequest(req *service.SearchDatasetsRequest) error {
- return validateSearchParams(req.Page, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)
+ return validateSearchParams(req.Page, req.PageSize, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)
}
func validateSearchDatasetRequest(req *service.SearchDatasetRequest) error {
- return validateSearchParams(req.Page, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)
+ return validateSearchParams(req.Page, req.PageSize, req.Size, req.KNNTopK, req.TopK, req.KNNNumCandidates, req.SimilarityThreshold, req.VectorSimilarityWeight)
}
-func validateSearchParams(page, size, knnTopK, topK, knnNumCandidates *int, similarityThreshold, vectorSimilarityWeight *float64) error {
+func validateSearchParams(page, pageSize, size, knnTopK, topK, knnNumCandidates *int, similarityThreshold, vectorSimilarityWeight *float64) error {
if page != nil && *page < 1 {
return fmt.Errorf("page must be greater than or equal to 1")
}
+ if pageSize != nil && *pageSize < 1 {
+ return fmt.Errorf("page_size must be greater than or equal to 1")
+ }
if size != nil && *size < 1 {
return fmt.Errorf("size must be greater than or equal to 1")
}
diff --git a/internal/handler/dataset_search_test.go b/internal/handler/dataset_search_test.go
index b093d08a76..2ccc795a58 100644
--- a/internal/handler/dataset_search_test.go
+++ b/internal/handler/dataset_search_test.go
@@ -50,7 +50,7 @@ func TestDatasetsHandlerSearchDataset(t *testing.T) {
h := &DatasetsHandler{searchDatasetService: fake}
rec := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/ds-1/search", strings.NewReader(`{"question":"hello","doc_ids":["doc-1"],"knn_top_k":7,"knn_num_candidates":14}`))
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/ds-1/search", strings.NewReader(`{"question":"hello","document_ids":["doc-1"],"page_size":9,"metadata_condition":{"logic":"and","conditions":[{"name":"author","comparison_operator":"=","value":"Luo"}]},"knn_top_k":7,"knn_num_candidates":14}`))
req.Header.Set("Content-Type", "application/json")
c, _ := gin.CreateTestContext(rec)
c.Request = req
@@ -65,9 +65,12 @@ func TestDatasetsHandlerSearchDataset(t *testing.T) {
if fake.datasetID != "ds-1" || fake.userID != "user-1" {
t.Fatalf("call args = (%q,%q), want (ds-1,user-1)", fake.datasetID, fake.userID)
}
- if fake.req == nil || fake.req.Question != "hello" || len(fake.req.DocIDs) != 1 || fake.req.DocIDs[0] != "doc-1" {
+ if fake.req == nil || fake.req.Question != "hello" || len(fake.req.DocumentIDs) != 1 || fake.req.DocumentIDs[0] != "doc-1" {
t.Fatalf("request = %#v", fake.req)
}
+ if fake.req.PageSize == nil || *fake.req.PageSize != 9 || fake.req.MetadataCondition["logic"] != "and" {
+ t.Fatalf("public request fields = %#v", fake.req)
+ }
if fake.req.KNNTopK == nil || *fake.req.KNNTopK != 7 || fake.req.KNNNumCandidates == nil || *fake.req.KNNNumCandidates != 14 {
t.Fatalf("KNN parameters = (%v, %v), want (7, 14)", fake.req.KNNTopK, fake.req.KNNNumCandidates)
}
@@ -160,7 +163,7 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {
h := &DatasetsHandler{searchDatasetsService: fake}
rec := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7,"include_knowledge_compilation":false}`))
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"document_ids":["doc-1"],"page_size":9,"metadata_condition":{"logic":"and","conditions":[]},"top_k":7,"include_knowledge_compilation":false}`))
req.Header.Set("Content-Type", "application/json")
c, _ := gin.CreateTestContext(rec)
c.Request = req
@@ -180,6 +183,9 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {
if fake.req.TopK == nil || *fake.req.TopK != 7 {
t.Fatalf("legacy top_k alias=%v want 7", fake.req.TopK)
}
+ if fake.req.PageSize == nil || *fake.req.PageSize != 9 || len(fake.req.DocumentIDs) != 1 || fake.req.MetadataCondition["logic"] != "and" {
+ t.Fatalf("public request fields = %#v", fake.req)
+ }
body := decodeSearchResponse(t, rec)
if body["code"] != float64(common.CodeSuccess) {
t.Fatalf("code=%v want=%d body=%s", body["code"], common.CodeSuccess, rec.Body.String())
@@ -232,12 +238,12 @@ func TestDatasetsHandlerSearchDatasetsValidationErrorsUseArgumentEnvelope(t *tes
func TestValidateSearchParamsKNNBounds(t *testing.T) {
knnTopK := 2048
- if err := validateSearchParams(nil, nil, &knnTopK, nil, nil, nil, nil); err != nil {
+ if err := validateSearchParams(nil, nil, nil, &knnTopK, nil, nil, nil, nil); err != nil {
t.Fatalf("knn_top_k=2048 with default knn_num_candidates should be valid: %v", err)
}
legacyTopK := 2049
- err := validateSearchParams(nil, nil, nil, &legacyTopK, nil, nil, nil)
+ err := validateSearchParams(nil, nil, nil, nil, &legacyTopK, nil, nil, nil)
if err == nil || err.Error() != "top_k (alias for knn_top_k) must be between 1 and 2048" {
t.Fatalf("legacy top_k error = %v", err)
}
diff --git a/internal/handler/mcp_server.go b/internal/handler/mcp_server.go
index 721bb7e0bf..9346363bb3 100644
--- a/internal/handler/mcp_server.go
+++ b/internal/handler/mcp_server.go
@@ -169,7 +169,7 @@ func MCPRetrieval(ctx context.Context, ds *dataset.DatasetService, userID string
searchReq := &service.SearchDatasetsRequest{
DatasetIDs: datasetIDs,
Question: req.Question,
- DocIDs: req.DocumentIDs,
+ DocumentIDs: req.DocumentIDs,
ForceRefresh: req.ForceRefresh,
}
@@ -179,7 +179,7 @@ func MCPRetrieval(ctx context.Context, ds *dataset.DatasetService, userID string
}
if req.PageSize > 0 {
v := req.PageSize
- searchReq.Size = &v
+ searchReq.PageSize = &v
}
if req.TopK > 0 {
v := req.TopK
diff --git a/internal/router/datasets_search_route_test.go b/internal/router/datasets_search_route_test.go
index 7920b27ba1..9e8fef3a17 100644
--- a/internal/router/datasets_search_route_test.go
+++ b/internal/router/datasets_search_route_test.go
@@ -10,7 +10,7 @@ import (
"ragflow/internal/handler"
)
-func TestRouterSetupRegistersSearchDatasetRoute(t *testing.T) {
+func TestRouterSetupRegistersDatasetSearchRoutes(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
@@ -20,14 +20,15 @@ func TestRouterSetupRegistersSearchDatasetRoute(t *testing.T) {
}
r.Setup(engine)
- resp := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/dataset-1/search", nil)
- engine.ServeHTTP(resp, req)
-
- if resp.Code == http.StatusNotFound {
- t.Fatalf("POST /api/v1/datasets/:dataset_id/search returned 404; SearchDataset route is not registered")
- }
- if resp.Code != http.StatusUnauthorized {
- t.Fatalf("status=%d body=%s; want auth middleware to handle registered SearchDataset route", resp.Code, resp.Body.String())
+ for _, path := range []string{"/api/v1/datasets/dataset-1/search", "/api/v1/datasets/search", "/api/v1/retrieval"} {
+ resp := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, path, nil)
+ engine.ServeHTTP(resp, req)
+ if resp.Code == http.StatusNotFound {
+ t.Fatalf("POST %s returned 404", path)
+ }
+ if resp.Code != http.StatusUnauthorized {
+ t.Fatalf("POST %s status=%d body=%s; want auth middleware to handle registered route", path, resp.Code, resp.Body.String())
+ }
}
}
diff --git a/internal/router/router.go b/internal/router/router.go
index 14d96e4cf4..736b71abe9 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -340,6 +340,7 @@ func (r *Router) Setup(engine *gin.Engine) {
chat.POST("/audio/transcription", r.chatHandler.ChatAudioTranscription)
}
v1.POST("/openai/:chat_id/chat/completions", r.openaiChatHandler.OpenAIChatCompletions)
+ v1.POST("/retrieval", r.datasetsHandler.SearchDatasets)
// Dataset routes
datasets := v1.Group("/datasets")
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 9f8827bc63..3c89ff890a 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -35,7 +35,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
page = *req.Page
}
pageSize := 30
- if req.Size != nil {
+ if req.PageSize != nil {
+ pageSize = *req.PageSize
+ } else if req.Size != nil {
pageSize = *req.Size
}
rerankCandidatesCount := 64
@@ -46,7 +48,7 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
if req.UseKG != nil {
useKG = *req.UseKG
}
- similarityThreshold := 0.0
+ similarityThreshold := 0.2
if req.SimilarityThreshold != nil {
similarityThreshold = *req.SimilarityThreshold
}
@@ -85,7 +87,23 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
question := req.Question
datasetIDs := req.DatasetIDs
metadataFilter := req.MetadataFilter
+ hasMetadataCondition := req.MetadataCondition != nil
+ if req.MetadataCondition != nil {
+ manual := make([]interface{}, 0)
+ if conditions, ok := req.MetadataCondition["conditions"].([]interface{}); ok {
+ for _, item := range conditions {
+ if condition, ok := item.(map[string]interface{}); ok {
+ manual = append(manual, map[string]interface{}{"key": condition["name"], "op": condition["comparison_operator"], "value": condition["value"]})
+ }
+ }
+ }
+ metadataFilter = map[string]interface{}{"method": "manual", "logic": req.MetadataCondition["logic"], "manual": manual}
+ }
crossLanguages := req.CrossLanguages
+ documentIDs := req.DocumentIDs
+ if documentIDs == nil {
+ documentIDs = req.DocIDs
+ }
modelProviderSvc := service.NewModelProviderService()
@@ -135,6 +153,11 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
common.Warn("Invalid search_id", zap.String("searchID", searchID))
return nil, fmt.Errorf("invalid search_id")
}
+ searchTenantID, ok := searchDetail["tenant_id"].(string)
+ if !ok || searchTenantID != userID {
+ common.Warn("Invalid search_id", zap.String("searchID", searchID))
+ return nil, fmt.Errorf("invalid search_id")
+ }
if searchConfig, ok := searchDetail["search_config"].(map[string]interface{}); ok && searchConfig != nil {
rerankCandidatesCount = 100
@@ -209,8 +232,8 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
}
// Apply meta_data_filter to get filtered doc_ids
- docIDs := make([]string, len(req.DocIDs))
- copy(docIDs, req.DocIDs)
+ docIDs := make([]string, len(documentIDs))
+ copy(docIDs, documentIDs)
if len(metadataFilter) > 0 {
metadataSvc := service.NewMetadataService()
flattedMeta, err := metadataSvc.GetFlattedMetaByKBs(ctx, datasetIDs)
@@ -218,8 +241,15 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
common.Warn("Failed to get flatted metadata, using empty metadata for filter", zap.Error(err))
flattedMeta = make(common.MetaData)
}
- filteredDocIDs, _ := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, req.DocIDs, datasetIDs)
- docIDs = filteredDocIDs
+ if hasMetadataCondition {
+ filteredDocIDs, _ := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, documentIDs, datasetIDs)
+ docIDs = filteredDocIDs
+ } else {
+ filteredDocIDs, filterReturnedEmpty := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, nil, datasetIDs)
+ if !filterReturnedEmpty {
+ docIDs = append(docIDs, filteredDocIDs...)
+ }
+ }
}
// Apply cross_languages and keyword extraction
@@ -305,8 +335,23 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
filteredChunks = nlp.RetrievalByChildren(filteredChunks, tenantIDs, d.docEngine, ctx)
+ keyMapping := map[string]string{
+ "chunk_id": "id",
+ "content_with_weight": "content",
+ "doc_id": "document_id",
+ "important_kwd": "important_keywords",
+ "question_kwd": "questions",
+ "docnm_kwd": "document_keyword",
+ "kb_id": "dataset_id",
+ }
for i := range filteredChunks {
delete(filteredChunks[i], "vector")
+ for oldKey, newKey := range keyMapping {
+ if value, ok := filteredChunks[i][oldKey]; ok {
+ filteredChunks[i][newKey] = value
+ delete(filteredChunks[i], oldKey)
+ }
+ }
}
common.Info("SearchDatasets completed", zap.String("userID", userID), zap.Any("kbID", datasetIDs), zap.String("question", question), zap.Int64("chunkCount", int64(len(filteredChunks))))
diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.go
index b6057b1763..08e48c0284 100644
--- a/internal/service/dataset/search_test.go
+++ b/internal/service/dataset/search_test.go
@@ -8,7 +8,7 @@ import (
func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
page := 2
- size := 15
+ pageSize := 15
knnTopK := 128
knnNumCandidates := 256
useKG := true
@@ -21,14 +21,14 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
req := &service.SearchDatasetRequest{
Question: "hello world",
Page: &page,
- Size: &size,
- DocIDs: []string{"doc-1", "doc-2"},
+ PageSize: &pageSize,
+ DocumentIDs: []string{"doc-1", "doc-2"},
UseKG: &useKG,
KNNTopK: &knnTopK,
KNNNumCandidates: &knnNumCandidates,
CrossLanguages: []string{"en", "zh"},
SearchID: &searchID,
- MetadataFilter: map[string]interface{}{"method": "manual"},
+ MetadataCondition: map[string]interface{}{"logic": "and"},
RerankID: &rerankID,
Keyword: &keyword,
SimilarityThreshold: &similarityThreshold,
@@ -40,16 +40,16 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
if len(converted.DatasetIDs) != 1 || converted.DatasetIDs[0] != "dataset-1" {
t.Fatalf("dataset_ids=%v want [dataset-1]", converted.DatasetIDs)
}
- if converted.Question != req.Question || converted.Page != req.Page || converted.Size != req.Size {
+ if converted.Question != req.Question || converted.Page != req.Page || converted.PageSize != req.PageSize {
t.Fatalf("converted request did not preserve pagination/question fields: %#v", converted)
}
- if len(converted.DocIDs) != 2 || converted.DocIDs[0] != "doc-1" || converted.DocIDs[1] != "doc-2" {
- t.Fatalf("doc_ids=%v want [doc-1 doc-2]", converted.DocIDs)
+ if len(converted.DocumentIDs) != 2 || converted.DocumentIDs[0] != "doc-1" || converted.DocumentIDs[1] != "doc-2" {
+ t.Fatalf("document_ids=%v want [doc-1 doc-2]", converted.DocumentIDs)
}
if converted.UseKG != req.UseKG || converted.KNNTopK != req.KNNTopK || converted.KNNNumCandidates != req.KNNNumCandidates || converted.SearchID != req.SearchID {
t.Fatalf("converted request did not preserve optional fields: %#v", converted)
}
- if converted.MetadataFilter["method"] != "manual" || converted.RerankID != req.RerankID || converted.Keyword != req.Keyword {
+ if converted.MetadataCondition["logic"] != "and" || converted.RerankID != req.RerankID || converted.Keyword != req.Keyword {
t.Fatalf("converted request did not preserve search config fields: %#v", converted)
}
if converted.SimilarityThreshold != req.SimilarityThreshold || converted.VectorSimilarityWeight != req.VectorSimilarityWeight {
diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.go
index 99c8a8262c..aa52adb5a6 100644
--- a/internal/service/dataset_types.go
+++ b/internal/service/dataset_types.go
@@ -40,8 +40,10 @@ type SearchDatasetsRequest struct {
DatasetIDs []string `json:"dataset_ids" binding:"required"`
Question string `json:"question" binding:"required"`
Page *int `json:"page,omitempty"`
+ PageSize *int `json:"page_size,omitempty"`
Size *int `json:"size,omitempty"`
RerankCandidatesCount *int `json:"rerank_candidates_count,omitempty"`
+ DocumentIDs []string `json:"document_ids,omitempty"`
DocIDs []string `json:"doc_ids,omitempty"`
UseKG *bool `json:"use_kg,omitempty"`
KNNTopK *int `json:"knn_top_k,omitempty"`
@@ -49,6 +51,7 @@ type SearchDatasetsRequest struct {
KNNNumCandidates *int `json:"knn_num_candidates,omitempty"`
CrossLanguages []string `json:"cross_languages,omitempty"`
SearchID *string `json:"search_id,omitempty"`
+ MetadataCondition map[string]interface{} `json:"metadata_condition,omitempty"`
MetadataFilter map[string]interface{} `json:"meta_data_filter,omitempty"`
RerankID *string `json:"rerank_id,omitempty"`
Keyword *bool `json:"keyword,omitempty"`
@@ -71,8 +74,10 @@ type SearchDatasetsResponse struct {
type SearchDatasetRequest struct {
Question string `json:"question"`
Page *int `json:"page,omitempty"`
+ PageSize *int `json:"page_size,omitempty"`
Size *int `json:"size,omitempty"`
RerankCandidatesCount *int `json:"rerank_candidates_count,omitempty"`
+ DocumentIDs []string `json:"document_ids,omitempty"`
DocIDs []string `json:"doc_ids,omitempty"`
UseKG *bool `json:"use_kg,omitempty"`
KNNTopK *int `json:"knn_top_k,omitempty"`
@@ -80,6 +85,7 @@ type SearchDatasetRequest struct {
KNNNumCandidates *int `json:"knn_num_candidates,omitempty"`
CrossLanguages []string `json:"cross_languages,omitempty"`
SearchID *string `json:"search_id,omitempty"`
+ MetadataCondition map[string]interface{} `json:"metadata_condition,omitempty"`
MetadataFilter map[string]interface{} `json:"meta_data_filter,omitempty"`
RerankID *string `json:"rerank_id,omitempty"`
Keyword *bool `json:"keyword,omitempty"`
@@ -97,8 +103,10 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *Sear
DatasetIDs: []string{datasetID},
Question: req.Question,
Page: req.Page,
+ PageSize: req.PageSize,
Size: req.Size,
RerankCandidatesCount: req.RerankCandidatesCount,
+ DocumentIDs: req.DocumentIDs,
DocIDs: req.DocIDs,
UseKG: req.UseKG,
KNNTopK: req.KNNTopK,
@@ -106,6 +114,7 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *Sear
KNNNumCandidates: req.KNNNumCandidates,
CrossLanguages: req.CrossLanguages,
SearchID: req.SearchID,
+ MetadataCondition: req.MetadataCondition,
MetadataFilter: req.MetadataFilter,
RerankID: req.RerankID,
Keyword: req.Keyword,
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index 77233bec89..6cc96d88bc 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -2427,7 +2427,7 @@ def test_dataset_search_endpoint(rest_client, ensure_parsed_document):
dataset_id, _ = ensure_parsed_document()
res = rest_client.post(
f"/datasets/{dataset_id}/search",
- json={"question": "test TXT file", "page": 1, "size": 10},
+ json={"question": "test TXT file", "page": 1, "page_size": 10},
)
assert res.status_code == 200
payload = res.json()
@@ -2439,17 +2439,17 @@ def test_dataset_search_endpoint(rest_client, ensure_parsed_document):
@pytest.mark.parametrize(
"payload",
[
- {"question": "test TXT file", "page": 1, "size": 2},
+ {"question": "test TXT file", "page": 1, "page_size": 2},
{"question": "test TXT file", "similarity_threshold": 0.5},
{"question": "test TXT file", "vector_similarity_weight": 0.7},
{"question": "test TXT file", "top_k": 10},
],
ids=["page_size", "similarity_threshold", "vector_similarity_weight", "top_k"],
)
-def test_dataset_search_params_and_doc_ids_contract(rest_client, ensure_parsed_document, payload):
+def test_dataset_search_params_and_document_ids_contract(rest_client, ensure_parsed_document, payload):
dataset_id, document_id = ensure_parsed_document()
search_payload = dict(payload)
- search_payload["doc_ids"] = [document_id]
+ search_payload["document_ids"] = [document_id]
res = rest_client.post(f"/datasets/{dataset_id}/search", json=search_payload)
assert res.status_code == 200
body = res.json()
@@ -2463,7 +2463,7 @@ def test_dataset_search_requires_question(rest_client, create_dataset):
res = rest_client.post(f"/datasets/{dataset_id}/search", json={})
assert res.status_code == 200
payload = res.json()
- assert payload["code"] == 101, payload
+ assert payload["code"] in [101, 102], payload
assert "question" in payload["message"], payload
diff --git a/test/testcases/restful_api/test_retrieval.py b/test/testcases/restful_api/test_retrieval.py
index 19936aa89d..b4514b5163 100644
--- a/test/testcases/restful_api/test_retrieval.py
+++ b/test/testcases/restful_api/test_retrieval.py
@@ -24,27 +24,38 @@ from test.testcases.utils import wait_for
@pytest.mark.p3
def test_dataset_search_rest_endpoint(rest_client, ensure_parsed_document):
dataset_id, _ = ensure_parsed_document()
+ request_body = {"question": "test TXT file", "top_k": 5}
res = rest_client.post(
f"/datasets/{dataset_id}/search",
- json={"question": "test TXT file", "top_k": 5},
+ json=request_body,
)
assert res.status_code == 200
payload = res.json()
assert payload["code"] == 0, payload
assert "chunks" in payload["data"], payload
+ retrieval_payload = rest_client.post("/retrieval", json={"dataset_ids": [dataset_id], **request_body}).json()
+ assert payload == retrieval_payload
@pytest.mark.p3
def test_multi_dataset_search_rest_endpoint(rest_client, ensure_parsed_document):
dataset_id, _ = ensure_parsed_document()
+ request_body = {"dataset_ids": [dataset_id], "question": "test TXT file", "top_k": 5}
res = rest_client.post(
"/datasets/search",
- json={"dataset_ids": [dataset_id], "question": "test TXT file", "top_k": 5},
+ json=request_body,
)
assert res.status_code == 200
payload = res.json()
assert payload["code"] == 0, payload
assert "chunks" in payload["data"], payload
+ for chunk in payload["data"]["chunks"]:
+ assert "id" in chunk, payload
+ assert "content" in chunk, payload
+ assert "document_id" in chunk, payload
+ assert "dataset_id" in chunk, payload
+ retrieval_payload = rest_client.post("/retrieval", json=request_body).json()
+ assert payload == retrieval_payload
@pytest.mark.p3
@@ -67,10 +78,9 @@ def test_multi_dataset_search_with_metadata_filter(rest_client, ensure_parsed_do
json={
"dataset_ids": [dataset_id],
"question": "test TXT file",
- "meta_data_filter": {
- "method": "manual",
+ "metadata_condition": {
"logic": "and",
- "manual": [{"key": "author", "op": "=", "value": "qa_batch2"}],
+ "conditions": [{"name": "author", "comparison_operator": "=", "value": "qa_batch2"}],
},
},
)
@@ -81,9 +91,8 @@ def test_multi_dataset_search_with_metadata_filter(rest_client, ensure_parsed_do
@pytest.mark.p3
-def test_retrieval_compatibility_endpoint(rest_client, ensure_parsed_document):
+def test_retrieval_endpoint(rest_client, ensure_parsed_document):
dataset_id, _ = ensure_parsed_document()
- # /api/v1/retrieval is SDK compatibility endpoint registered from chunk_api.py.
res = rest_client.post(
"/retrieval",
json={"dataset_ids": [dataset_id], "question": "test TXT file", "top_k": 5},
@@ -255,7 +264,7 @@ def test_retrieval_vector_similarity_and_top_k_contract(rest_client, ensure_pars
("vector 0", {"vector_similarity_weight": 0}, 0, ""),
("vector 0.5", {"vector_similarity_weight": 0.5}, 0, ""),
("vector 10", {"vector_similarity_weight": 10}, 0, ""),
- ("vector alpha", {"vector_similarity_weight": "a"}, 100, "could not convert string to float"),
+ ("vector alpha", {"vector_similarity_weight": "a"}, 102, "`vector_similarity_weight` should be a number"),
("top_k 10", {"top_k": 10}, 0, ""),
("top_k 1", {"top_k": 1}, 0, ""),
("top_k -1", {"top_k": -1}, 102, "`top_k` must be greater than 0"),
diff --git a/web/src/components/top-select.tsx b/web/src/components/top-select.tsx
index 3b017f1006..6ae1ec76c0 100644
--- a/web/src/components/top-select.tsx
+++ b/web/src/components/top-select.tsx
@@ -55,7 +55,7 @@ export function TopSelectFormItem() {
return (
<SliderInputFormField
- name="size"
+ name="page_size"
label={t('chat.topN')}
min={1}
max={100}
diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts
index 96234ec480..9d765bc160 100644
--- a/web/src/hooks/use-knowledge-request.ts
+++ b/web/src/hooks/use-knowledge-request.ts
@@ -137,7 +137,7 @@ export const useTestRetrieval = () => {
...values,
kb_id: values?.kb_id || knowledgeBaseId,
page: 1,
- doc_ids: filterValue.doc_ids,
+ document_ids: filterValue.doc_ids,
highlight: true,
include_knowledge_compilation: false,
};
@@ -164,7 +164,7 @@ export const useTestRetrieval = () => {
if (mutation.data && queryParams.question) {
const newParams = {
...queryParams,
- doc_ids: value.doc_ids ?? [],
+ document_ids: value.doc_ids ?? [],
page: 1,
};
mutation.mutate(newParams);
@@ -1295,7 +1295,7 @@ export const useTestChunkRetrieval = (): ResponsePostType<ITestingResult> & {
...values,
kb_id: values.kb_id ?? knowledgeBaseId,
page,
- size: pageSize,
+ page_size: pageSize,
});
if (data.code === 0) {
const res = data.data;
@@ -1338,9 +1338,9 @@ export const useTestChunkAllRetrieval = (): ResponsePostType<ITestingResult> & {
const { data } = await kbService.retrievalTest({
...values,
kb_id: values.kb_id ?? knowledgeBaseId,
- doc_ids: [],
+ document_ids: [],
page,
- size: pageSize,
+ page_size: pageSize,
});
if (data.code === 0) {
const res = data.data;
diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.ts
index 3386839750..f9468c0ed3 100644
--- a/web/src/interfaces/database/dataset.ts
+++ b/web/src/interfaces/database/dataset.ts
@@ -171,22 +171,20 @@ export interface IChunk {
}
export interface ITestingChunk {
- chunk_id: string;
+ id: string;
content_ltks: string;
- content_with_weight: string;
- doc_id: string;
- doc_name: string;
- img_id: string;
+ content: string;
+ document_id: string;
+ document_keyword: string;
image_id: string;
- important_kwd: any[];
- kb_id: string;
+ important_keywords: any[];
+ questions?: any[];
+ dataset_id: string;
similarity: number;
term_similarity: number;
- vector: number[];
vector_similarity: number;
highlight: string;
positions: number[][];
- docnm_kwd: string;
doc_type_kwd: string;
document_metadata?: Record<string, any>;
}
diff --git a/web/src/interfaces/request/knowledge.ts b/web/src/interfaces/request/knowledge.ts
index 3a64ff62ef..1d8c09db59 100644
--- a/web/src/interfaces/request/knowledge.ts
+++ b/web/src/interfaces/request/knowledge.ts
@@ -1,6 +1,6 @@
export interface ITestRetrievalRequestBody {
question: string;
- size: number;
+ page_size: number;
rerank_candidates_count: number;
similarity_threshold: number;
vector_similarity_weight: number;
diff --git a/web/src/pages/dataset/testing/testing-form.tsx b/web/src/pages/dataset/testing/testing-form.tsx
index 0232115fc7..adddae6601 100644
--- a/web/src/pages/dataset/testing/testing-form.tsx
+++ b/web/src/pages/dataset/testing/testing-form.tsx
@@ -66,10 +66,10 @@ export default function TestingForm({
...vectorSimilarityWeightSchema,
dataset_ids: z.array(z.string()).optional(),
...MetadataFilterSchema,
- size: z.number().int().min(1).max(100),
+ page_size: z.number().int().min(1).max(100),
...rerankCandidatesCountSchema,
})
- .refine((values) => values.rerank_candidates_count >= values.size, {
+ .refine((values) => values.rerank_candidates_count >= values.page_size, {
message: t('chat.rerankCandidatesCountValidation'),
path: ['rerank_candidates_count'],
});
@@ -80,7 +80,7 @@ export default function TestingForm({
...initialSimilarityThresholdValue,
...initialVectorSimilarityWeightValue,
dataset_ids: [knowledgeBaseId],
- size: 10,
+ page_size: 10,
rerank_candidates_count: 64,
},
});
diff --git a/web/src/pages/dataset/testing/testing-result.tsx b/web/src/pages/dataset/testing/testing-result.tsx
index c15022ecb3..fda9964f7c 100644
--- a/web/src/pages/dataset/testing/testing-result.tsx
+++ b/web/src/pages/dataset/testing/testing-result.tsx
@@ -82,19 +82,19 @@ export function TestingResult({
<>
<section className="px-5 pb-5 flex flex-col gap-5 overflow-auto scrollbar-thin min-h-0">
{data.chunks?.map((x) => (
- <article key={x.chunk_id}>
+ <article key={x.id}>
<Card className="px-5 py-2.5 bg-transparent shadow-none">
<ChunkTitle item={x}></ChunkTitle>
<div
className="!mt-2.5 whitespace-pre-wrap [&_em]:text-accent-primary [&_em]:not-italic"
dangerouslySetInnerHTML={{
__html: sanitizeHtmlWithImagesAsText(
- x.highlight || x.content_with_weight,
+ x.highlight || x.content,
),
}}
/>
<div className="mt-2.5 text-right text-xs text-text-sub-title-invert">
- {x.doc_name || x.docnm_kwd}
+ {x.document_keyword}
</div>
</Card>
</article>
diff --git a/web/src/pages/next-search/document-preview-modal/index.tsx b/web/src/pages/next-search/document-preview-modal/index.tsx
index 2f7800abd3..c50a730efe 100644
--- a/web/src/pages/next-search/document-preview-modal/index.tsx
+++ b/web/src/pages/next-search/document-preview-modal/index.tsx
@@ -30,6 +30,7 @@ interface IProps extends IModalProps<any> {
documentId: string;
chunk: {
docnm_kwd?: string;
+ document_keyword?: string;
document_name?: string;
positions?: number[][];
content_with_weight?: string;
@@ -53,7 +54,8 @@ const PdfDrawer = ({
);
// const ref = useRef<(highlight: IHighlight) => void>(() => {});
// const [loaded, setLoaded] = useState(false);
- const documentName = chunk.docnm_kwd || chunk.document_name;
+ const documentName =
+ chunk.document_keyword || chunk.docnm_kwd || chunk.document_name;
const fileType = documentName ? getFileExtensionRegex(documentName) : '';
const isWebPage = !fileType && !!chunk.document_url;
const url = isWebPage ? (chunk.document_url as string) : getDocumentUrl();
diff --git a/web/src/pages/next-search/hooks.ts b/web/src/pages/next-search/hooks.ts
index f81c4f6ec0..f2f149d6f8 100644
--- a/web/src/pages/next-search/hooks.ts
+++ b/web/src/pages/next-search/hooks.ts
@@ -168,7 +168,7 @@ export const useTestChunkRetrieval = (
mutationFn: async (values: any) => {
const { data } = await retrievalTestFunc({
page,
- size: pageSize,
+ page_size: pageSize,
...values,
kb_id: values.kb_id ?? knowledgeBaseId,
tenant_id: tenantId,
@@ -218,7 +218,7 @@ export const useTestChunkAllRetrieval = (
mutationFn: async (values: any) => {
const { data } = await retrievalTestFunc({
page,
- size: pageSize,
+ page_size: pageSize,
...values,
kb_id: values.kb_id ?? knowledgeBaseId,
tenant_id: tenantId,
@@ -265,9 +265,11 @@ export const useTestRetrieval = (
kb_id: kbIds,
highlight: true,
question: q,
- doc_ids: Array.isArray(selectedDocumentIds) ? selectedDocumentIds : [],
+ document_ids: Array.isArray(selectedDocumentIds)
+ ? selectedDocumentIds
+ : [],
page: pagination.current,
- size: pagination.pageSize,
+ page_size: pagination.pageSize,
});
}, [
sendingLoading,
@@ -368,7 +370,7 @@ export const useSendQuestion = (
highlight: true,
question: q,
page: 1,
- size: pageSize,
+ page_size: pageSize,
search_id: searchId,
});
@@ -415,9 +417,9 @@ export const useSendQuestion = (
kb_id: kbIds,
highlight: true,
question: q,
- doc_ids: documentIds ?? selectedDocumentIds,
+ document_ids: documentIds ?? selectedDocumentIds,
page,
- size,
+ page_size: size,
search_id: searchId,
});
@@ -425,9 +427,9 @@ export const useSendQuestion = (
kb_id: kbIds,
highlight: true,
question: q,
- doc_ids: [],
+ document_ids: [],
page,
- size,
+ page_size: size,
search_id: searchId,
});
},
diff --git a/web/src/pages/next-search/search-view.tsx b/web/src/pages/next-search/search-view.tsx
index 336816ea11..3d7d983c17 100644
--- a/web/src/pages/next-search/search-view.tsx
+++ b/web/src/pages/next-search/search-view.tsx
@@ -251,15 +251,15 @@ export default function SearchingView({
<div key={index}>
<div className="w-full flex flex-col">
<div className="w-full">
- {(chunk.image_id || chunk.img_id) && (
+ {chunk.image_id && (
<ImageWithPopover
- id={chunk.image_id || chunk.img_id}
+ id={chunk.image_id}
></ImageWithPopover>
)}
<div
dangerouslySetInnerHTML={{
__html: sanitizeHtmlWithImagesAsText(
- chunk.highlight || chunk.content_with_weight,
+ chunk.highlight || chunk.content,
).trim(),
}}
className={classNames(
@@ -292,11 +292,14 @@ export default function SearchingView({
<div
className="flex gap-2 items-center text-xs text-text-secondary border p-1 rounded-lg w-fit mt-3"
onClick={() =>
- clickDocumentButton(chunk.doc_id, chunk as any)
+ clickDocumentButton(
+ chunk.document_id,
+ chunk as any,
+ )
}
>
- <FileIcon name={chunk.docnm_kwd}></FileIcon>
- {chunk.docnm_kwd}
+ <FileIcon name={chunk.document_keyword}></FileIcon>
+ {chunk.document_keyword}
</div>
</div>
{index < chunks.length - 1 && (
diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts
index 8990a69638..2c6339e402 100644
--- a/web/src/services/knowledge-service.ts
+++ b/web/src/services/knowledge-service.ts
@@ -115,6 +115,42 @@ const mapChunkToLegacy = (chunk: Record<string, any>) => ({
positions: chunk.positions || chunk.position_int || [],
});
+const mapChunkToRetrieval = (chunk: Record<string, any>) => ({
+ ...chunk,
+ id: chunk.id || chunk.chunk_id,
+ content: chunk.content ?? chunk.content_with_weight,
+ document_id: chunk.document_id || chunk.doc_id,
+ document_keyword: chunk.document_keyword || chunk.docnm_kwd || chunk.doc_name,
+ dataset_id: chunk.dataset_id || chunk.kb_id,
+ important_keywords: chunk.important_keywords || chunk.important_kwd || [],
+ questions: chunk.questions || chunk.question_kwd || [],
+});
+
+const mapRetrievalResponse = (response: any) => {
+ if (response.data?.code === 0) {
+ response.data.data = {
+ ...response.data.data,
+ chunks: (response.data.data?.chunks || []).map(mapChunkToRetrieval),
+ };
+ }
+ return response;
+};
+
+const toLegacyMetadataFilter = (condition?: Record<string, any>) => {
+ if (!condition?.conditions?.length) {
+ return undefined;
+ }
+ return {
+ method: 'manual',
+ logic: condition.logic,
+ manual: condition.conditions.map((item: Record<string, any>) => ({
+ key: item.name,
+ op: item.comparison_operator,
+ value: item.value,
+ })),
+ };
+};
+
const mapDocumentToLegacy = (doc: Record<string, any>) => ({
...doc,
chunk_num: doc.chunk_num ?? doc.chunk_count,
@@ -157,9 +193,45 @@ const chunkService = {
delete rest.dataset_id;
delete rest.kb_id;
delete rest.knowledge_id;
- return request.post(api.retrievalTest, {
- data: { ...rest, dataset_ids: datasetIds },
+ const data = {
+ dataset_ids: datasetIds,
+ document_ids: rest.document_ids ?? rest.doc_ids,
+ question: rest.question,
+ page: rest.page,
+ page_size: rest.page_size ?? rest.size,
+ similarity_threshold: rest.similarity_threshold,
+ vector_similarity_weight: rest.vector_similarity_weight,
+ top_k: rest.top_k,
+ knn_top_k: rest.knn_top_k,
+ knn_num_candidates: rest.knn_num_candidates,
+ rerank_candidates_count: rest.rerank_candidates_count,
+ rerank_id: rest.rerank_id,
+ search_id: rest.search_id,
+ keyword: rest.keyword,
+ highlight: rest.highlight,
+ cross_languages: rest.cross_languages,
+ meta_data_filter: rest.meta_data_filter,
+ chat_id: rest.chat_id,
+ use_kg: rest.use_kg,
+ toc_enhance: rest.toc_enhance,
+ include_knowledge_compilation: rest.include_knowledge_compilation,
+ reference_metadata: rest.reference_metadata,
+ };
+ const response = await request.post(api.retrievalTest, {
+ data,
+ });
+ return mapRetrievalResponse(response);
+ },
+ retrievalTestShare: async (params: Record<string, any>) => {
+ const response = await baseKbService.retrievalTestShare({
+ ...params,
+ doc_ids: params.doc_ids ?? params.document_ids,
+ size: params.size ?? params.page_size,
+ meta_data_filter:
+ params.meta_data_filter ??
+ toLegacyMetadataFilter(params.metadata_condition),
});
+ return mapRetrievalResponse(response);
},
chunkList: async (params: Record<string, any>) => {
const datasetId = getDatasetId(params);
diff --git a/web/vite.config.ts b/web/vite.config.ts
index 8bcd60a7b2..71a3e50118 100644
--- a/web/vite.config.ts
+++ b/web/vite.config.ts
@@ -103,7 +103,7 @@ export default defineConfig(({ mode }) => {
changeOrigin: true,
ws: true,
},
- '^(/api/v1/datasets/search)|^(/api/v1/chat/completions)': {
+ '^(/api/v1/chat/completions)': {
target: 'http://127.0.0.1:9384/',
changeOrigin: true,
ws: true,