RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: restrict xxx_ids to limit 100 (#18074)
Wang Qibd876c796ec9
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.pyindex 3b93217a6e..703961d101 100644--- a/api/apps/restful_apis/agent_api.py+++ b/api/apps/restful_apis/agent_api.py@@ -64,7 +64,7 @@ from api.utils.api_utils import (server_error_response,validate_request,)-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_sizefrom common import settingsfrom common.ssrf_guard import assert_host_is_safefrom common.constants import RetCode@@ -698,6 +698,10 @@ def list_agents(tenant_id):canvas_type = request.args.get("canvas_type")owner_ids = [item for item in request.args.get("owner_ids", "").strip().split(",") if item]tags = [item for item in request.args.get("tags", "").strip().split(",") if item]+ try:+ validate_rest_api_ids(owner_ids, "owner_ids")+ except ValueError as e:+ return get_result(code=RetCode.ARGUMENT_ERROR, message=str(e))page_number = validate_rest_api_page(request.args.get("page", DEFAULT_PAGE))items_per_page = validate_rest_api_page_size(request.args.get("page_size", DEFAULT_PAGE_SIZE))diff --git a/api/apps/restful_apis/chat_api.py b/api/apps/restful_apis/chat_api.pyindex 140750ba0d..1d9658595a 100644--- a/api/apps/restful_apis/chat_api.py+++ b/api/apps/restful_apis/chat_api.py@@ -51,7 +51,7 @@ from api.utils.api_utils import (server_error_response,validate_request,)-from api.utils.pagination_utils import validate_rest_api_page, validate_rest_api_page_size, DEFAULT_PAGE, DEFAULT_PAGE_SIZE+from api.utils.pagination_utils import validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_size, DEFAULT_PAGE, DEFAULT_PAGE_SIZEfrom common.constants import LLMType, RetCode, StatusEnumfrom common import settingsfrom common.misc_utils import get_uuid, thread_pool_exec@@ -510,6 +510,11 @@ async def list_chats():if orderby not in ("create_time", "update_time", "name"):return get_json_result(code=RetCode.ARGUMENT_ERROR, message=f"invalid orderby field: {orderby}")+ try:+ validate_rest_api_ids(owner_ids, "owner_ids")+ except ValueError as ex:+ return get_json_result(code=RetCode.ARGUMENT_ERROR, message=str(ex))+try:# Invalid or negative pagination values fall back to defaults# instead of leaking internal conversion/SQL errors.diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex 2d254b8d96..8e72874825 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -49,7 +49,7 @@ from api.utils.api_utils import (get_result,server_error_response,)-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_sizefrom api.utils.image_utils import store_chunk_imagefrom api.utils.reference_metadata_utils import (enrich_chunks_with_document_metadata,@@ -478,6 +478,10 @@ async def list_chunks(tenant_id, dataset_id, document_id):size = validate_rest_api_page_size(req.get("page_size", DEFAULT_PAGE_SIZE))question = req.get("keywords", "")chunk_ids = _get_query_id_list(req, "chunk_ids")+ try:+ validate_rest_api_ids(chunk_ids, "chunk_ids")+ except ValueError as e:+ return get_result(code=RetCode.ARGUMENT_ERROR, message=str(e))query = {"doc_ids": [document_id],"page": page,diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex 54e68a3b35..ccd79fd095 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -20,7 +20,7 @@ from quart import request, make_responsefrom common.constants import RetCodefrom api.apps import login_required, current_userfrom api.utils.api_utils import get_error_argument_result, get_error_data_result, get_json_result, get_result, add_tenant_id_to_kwargs-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_sizefrom api.utils.validation_utils import (CreateDatasetReq,DeleteDatasetReq,@@ -44,6 +44,7 @@ def aggregate_tags(tenant_id):return get_error_data_result(message="Lack of dataset_ids in query parameters")try:+ validate_rest_api_ids(dataset_ids, "dataset_ids")success, result = dataset_api_service.aggregate_tags(dataset_ids, tenant_id)if success:return get_result(data=result)@@ -66,6 +67,7 @@ def get_flattened_metadata(tenant_id):return get_error_data_result(message="Lack of dataset_ids in query parameters")try:+ validate_rest_api_ids(dataset_ids, "dataset_ids")success, result = dataset_api_service.get_flattened_metadata(dataset_ids, tenant_id)if success:return get_result(data=result)@@ -602,7 +604,7 @@ async def has_any_wiki(tenant_id, dataset_id):async def list_wiki_pages(tenant_id, dataset_id):"""List artifact pages for the dataset Artifact tab.- GET /api/v1/datasets/<dataset_id>/artifacts?page=1&page_size=200&page_type=entity&topic=topic&keywords=query+ GET /api/v1/datasets/<dataset_id>/artifacts?page=1&page_size=100&page_type=entity&topic=topic&keywords=querySuccess: {"code": 0, "data": {"total": int, "items": [{slug, title, page_type}]}}"""try:@@ -638,7 +640,7 @@ async def list_wiki_pages(tenant_id, dataset_id):async def list_wiki_topics(tenant_id, dataset_id):"""List wiki topics for the dataset Artifact tab.- GET /api/v1/datasets/<dataset_id>/artifacts/topics?page=1&page_size=200&keywords=query+ GET /api/v1/datasets/<dataset_id>/artifacts/topics?page=1&page_size=100&keywords=querySuccess: {"code": 0, "data": {"total": int, "items": [{topic, title, slug}]}}"""try:diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.pyindex 3e92e4ecd8..9d25875238 100644--- a/api/apps/restful_apis/document_api.py+++ b/api/apps/restful_apis/document_api.py@@ -61,7 +61,7 @@ from api.utils.api_utils import (get_error_argument_result,check_duplicate_ids,)-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_sizefrom api.utils.validation_utils import (UpdateDocumentReq,format_validation_error_message,@@ -352,6 +352,11 @@ async def metadata_summary(dataset_id, tenant_id):# Get doc_ids from query parameters (comma-separated string)doc_ids_param = request.args.get("doc_ids", "")doc_ids = doc_ids_param.split(",") if doc_ids_param else None+ try:+ validate_rest_api_ids(doc_ids, "doc_ids")+ except ValueError as e:+ return get_error_argument_result(str(e))+try:summary = DocMetadataService.get_metadata_summary(dataset_id, doc_ids)return get_result(data={"summary": summary})@@ -930,6 +935,10 @@ def _get_docs_with_request(req, dataset_id: str):return RetCode.DATA_ERROR, f"you don't own the document {doc_name}", [], 0doc_ids = q.getlist("ids")+ try:+ validate_rest_api_ids(doc_ids)+ except ValueError as e:+ return RetCode.ARGUMENT_ERROR, str(e), [], 0if doc_id and len(doc_ids) > 0:return RetCode.DATA_ERROR, f"Should not provide both 'id':{doc_id} and 'ids'{doc_ids}"if len(doc_ids) > 0:@@ -1318,8 +1327,12 @@ def list_thumbnails():return get_json_result(data=False, message='Lack of "Document ID"', code=RetCode.ARGUMENT_ERROR)try:- docs = DocumentService.get_thumbnails(doc_ids)+ validate_rest_api_ids(doc_ids, "doc_ids")+ except ValueError as e:+ return get_error_argument_result(str(e))+ try:+ docs = DocumentService.get_thumbnails(doc_ids)for doc_item in docs:if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):doc_item["thumbnail"] = f"/api/v1/documents/images/{doc_item['kb_id']}-{doc_item['thumbnail']}"diff --git a/api/apps/restful_apis/mcp_api.py b/api/apps/restful_apis/mcp_api.pyindex bb71ed21d9..f2988b0865 100644--- a/api/apps/restful_apis/mcp_api.py+++ b/api/apps/restful_apis/mcp_api.py@@ -21,7 +21,7 @@ from api.db.db_models import MCPServerfrom api.db.services.mcp_server_service import MCPServerServicefrom api.db.services.user_service import TenantServicefrom api.utils.api_utils import get_data_error_result, get_json_result, get_mcp_tools, get_request_json, server_error_response, validate_request-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_sizefrom api.utils.web_utils import get_float, safe_json_parsefrom common.constants import VALID_MCP_SERVER_TYPESfrom common.mcp_tool_call_conn import MCPToolCallSession, close_multiple_mcp_toolcall_sessions@@ -80,6 +80,13 @@ async def list_mcp() -> Response:desc = Truemcp_ids = _get_mcp_ids_from_args()+ try:+ validate_rest_api_ids(mcp_ids, "mcp_ids")+ except ValueError as e:+ from api.utils.api_utils import get_error_argument_result++ return get_error_argument_result(str(e))+try:servers = MCPServerService.get_servers(current_user.id, mcp_ids, 0, 0, orderby, desc, keywords) or []total = len(servers)diff --git a/api/apps/restful_apis/memory_api.py b/api/apps/restful_apis/memory_api.pyindex 5958924882..90d18c0c6f 100644--- a/api/apps/restful_apis/memory_api.py+++ b/api/apps/restful_apis/memory_api.py@@ -24,7 +24,7 @@ from api.apps import login_required, current_userfrom api.utils.api_utils import validate_request, get_request_json, get_error_argument_result, get_json_resultfrom api.apps.services import memory_api_servicefrom api.db.joint_services.tenant_model_service import ensure_tenant_model_ids_for_params-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_size@manager.route("/memories", methods=["POST"]) # noqa: F821@@ -143,6 +143,13 @@ async def list_memory():keywords = request.args.get("keywords")page = validate_rest_api_page(request.args.get("page", DEFAULT_PAGE))page_size = validate_rest_api_page_size(request.args.get("page_size", DEFAULT_PAGE_SIZE))+ try:+ for field_name in ("owner_ids", "ids"):+ values = [item.strip() for item in str(filter_params.get(field_name) or "").split(",") if item.strip()]+ validate_rest_api_ids(values, field_name)+ except ValueError as e:+ return get_error_argument_result(str(e))+try:res = await memory_api_service.list_memory(filter_params, keywords, page, page_size)return get_json_result(message=True, data=res)@@ -176,6 +183,11 @@ async def get_memory_messages(memory_id):keywords = keywords.strip()page = validate_rest_api_page(args.get("page", 1))page_size = validate_rest_api_page_size(args.get("page_size", 50))+ try:+ validate_rest_api_ids(agent_ids, "agent_id")+ except ValueError as e:+ return get_error_argument_result(str(e))+try:res = await memory_api_service.get_memory_messages(memory_id, agent_ids, keywords, page, page_size)return get_json_result(message=True, data=res)@@ -267,6 +279,7 @@ async def search_message():similarity_threshold = float(args.get("similarity_threshold", 0.2))keywords_similarity_weight = float(args.get("keywords_similarity_weight", 0.7))try:+ validate_rest_api_ids(memory_ids, "memory_id")top_n = validate_rest_api_page_size(int(args.get("top_n", 5)))except ValueError as exc:return get_error_argument_result(str(exc))@@ -290,6 +303,7 @@ async def get_messages():agent_id = args.get("agent_id", "")session_id = args.get("session_id", "")try:+ validate_rest_api_ids(memory_ids, "memory_id")limit = validate_rest_api_page_size(int(args.get("limit", 10)))except ValueError as exc:return get_error_argument_result(str(exc))diff --git a/api/apps/restful_apis/search_api.py b/api/apps/restful_apis/search_api.pyindex ef85626aab..939a8ca488 100644--- a/api/apps/restful_apis/search_api.py+++ b/api/apps/restful_apis/search_api.py@@ -31,7 +31,7 @@ from api.db.services.user_service import TenantService, UserTenantServicefrom common.misc_utils import get_uuidfrom common.constants import RetCode, StatusEnumfrom api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, server_error_response, validate_request-from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size+from api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_ids, validate_rest_api_page, validate_rest_api_page_sizedef _full_text_weight(vector_similarity_weight):@@ -84,6 +84,11 @@ def list_searches():desc = request.args.get("desc", "true").lower() != "false"owner_ids = request.args.getlist("owner_ids")+ try:+ validate_rest_api_ids(owner_ids, "owner_ids")+ except ValueError as e:+ return get_json_result(code=RetCode.ARGUMENT_ERROR, message=str(e))+try:if not owner_ids:tenants = []diff --git a/api/utils/pagination_utils.py b/api/utils/pagination_utils.pyindex 2edf5fd074..03ddf2e818 100644--- a/api/utils/pagination_utils.py+++ b/api/utils/pagination_utils.py@@ -17,6 +17,7 @@DEFAULT_PAGE = 1DEFAULT_PAGE_SIZE = 30REST_API_MAX_PAGE_SIZE = 100+REST_API_MAX_IDS = 100def validate_rest_api_page(page) -> int:@@ -41,3 +42,10 @@ def validate_rest_api_page_size(page_size) -> int:if int_page_size > REST_API_MAX_PAGE_SIZE:raise ValueError(f"page_size must be less than or equal to {REST_API_MAX_PAGE_SIZE}")return int_page_size+++def validate_rest_api_ids(ids: list | None, field_name: str = "ids") -> list | None:+ """Validate REST API ID lists against the public maximum."""+ if ids is not None and len(ids) > REST_API_MAX_IDS:+ raise ValueError(f"{field_name} must contain at most {REST_API_MAX_IDS} IDs")+ return idsdiff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex 0f7b22f970..ddb455762c 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -30,7 +30,7 @@ from werkzeug.exceptions import BadRequest, UnsupportedMediaTypefrom api.constants import DATASET_NAME_LIMIT, FILE_NAME_LEN_LIMITfrom api.db import FileType-from api.utils.pagination_utils import validate_rest_api_page_size+from api.utils.pagination_utils import REST_API_MAX_IDS, validate_rest_api_page_sizefrom common.constants import RetCode@@ -1043,7 +1043,7 @@ class BaseListReq(BaseModel):class ListDatasetReq(BaseListReq):"""Request model for listing datasets."""- ids: Annotated[list[str] | None, Field(default=None)]+ ids: Annotated[list[str] | None, Field(default=None, max_length=REST_API_MAX_IDS)]include_parsing_status: Annotated[bool, Field(default=False)]ext: Annotated[dict, Field(default={})]