RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feature: GET /datasets to support ids array (#17606)
Wang Qi0969c04eca1f
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex eaaf0d44d3..d540958b63 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -321,6 +321,14 @@ def list_datasets(tenant_id):type: stringrequired: falsedescription: Dataset ID to filter.+ - in: query+ name: ids+ type: array+ required: false+ items:+ type: string+ collectionFormat: multi+ description: Dataset IDs to filter.- in: queryname: nametype: stringdiff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 264f6dccfb..63f1a0eaac 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -403,6 +403,7 @@ def list_datasets(tenant_id: str, args: dict)::return: (success, result) or (success, error_message)"""kb_id = args.get("id")+ kb_ids = args.get("ids")name = args.get("name")page = int(args.get("page", 1))page_size = int(args.get("page_size", 30))@@ -419,10 +420,13 @@ def list_datasets(tenant_id: str, args: dict):# unknown type, default to Truedesc = True+ if kb_id and kb_ids:+ return False, f"Should not provide both 'id':{kb_id} and 'ids'{kb_ids}"if kb_id:kbs = KnowledgebaseService.get_kb_by_id(kb_id, tenant_id)if not kbs:return False, f"User '{tenant_id}' lacks permission for dataset '{kb_id}'"+if name:kbs = KnowledgebaseService.get_kb_by_name(name, tenant_id)if not kbs:@@ -438,7 +442,12 @@ def list_datasets(tenant_id: str, args: dict):tenants = TenantService.get_joined_tenants_by_user_id(tenant_id)tenant_ids = [m["tenant_id"] for m in tenants]query_user_id = tenant_id- kbs, total = KnowledgebaseService.get_list(tenant_ids, query_user_id, page, page_size, orderby, desc, kb_id, name, keywords, parser_id)+ if kb_ids:+ accessible_ids = KnowledgebaseService.get_accessible_ids([m["tenant_id"] for m in tenants], tenant_id, kb_ids)+ if len(accessible_ids) != len(kb_ids):+ denied_ids = [kb_id for kb_id in kb_ids if kb_id not in accessible_ids]+ return False, f"""User '{tenant_id}' lacks permission for datasets: '{", ".join(denied_ids)}'"""+ kbs, total = KnowledgebaseService.get_list(tenant_ids, query_user_id, page, page_size, orderby, desc, kb_id, name, keywords, parser_id, kb_ids)users = UserService.get_by_ids([m["tenant_id"] for m in kbs])user_map = {m.id: m.to_dict() for m in users}diff --git a/api/db/services/knowledgebase_service.py b/api/db/services/knowledgebase_service.pyindex 0cf2500360..dfa668f18d 100644--- a/api/db/services/knowledgebase_service.py+++ b/api/db/services/knowledgebase_service.py@@ -463,7 +463,7 @@ class KnowledgebaseService(CommonService):@classmethod@DB.connection_context()- def get_list(cls, joined_tenant_ids, user_id, page_number, items_per_page, orderby, desc, id, name, keywords, parser_id=None):+ def get_list(cls, joined_tenant_ids, user_id, page_number, items_per_page, orderby, desc, id, name, keywords, parser_id=None, ids=None):# Get list of knowledge bases with filtering and pagination# Args:# joined_tenant_ids: List of tenant IDs@@ -482,6 +482,8 @@ class KnowledgebaseService(CommonService):kbs = cls.model.select()if id:kbs = kbs.where(cls.model.id == id)+ if ids:+ kbs = kbs.where(cls.model.id.in_(ids))if name:kbs = kbs.where(cls.model.name == name)if keywords:@@ -501,6 +503,12 @@ class KnowledgebaseService(CommonService):return list(kbs.dicts()), total+ @classmethod+ @DB.connection_context()+ def get_accessible_ids(cls, joined_tenant_ids, user_id, ids):+ kbs = cls.model.select(cls.model.id).where(cls.model.id.in_(ids), cls._visibility_and_status_filter(joined_tenant_ids, user_id))+ return {kb.id for kb in kbs}+@classmethod@DB.connection_context()def get_owner_filter(cls, joined_tenant_ids, user_id):diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex 20847b6f4f..450ac95045 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -19,7 +19,8 @@ import pathlibimport refrom collections import Counterimport string-from typing import Annotated, Any, Literal+from types import UnionType+from typing import Annotated, Any, Literal, Union, get_args, get_originfrom uuid import UUIDfrom quart import Request@@ -33,6 +34,15 @@ from api.utils.pagination_utils import validate_rest_api_page_sizefrom common.constants import RetCode+def _is_list_annotation(annotation: Any) -> bool:+ origin = get_origin(annotation)+ if origin is list:+ return True+ if origin in (Union, UnionType):+ return any(_is_list_annotation(arg) for arg in get_args(annotation))+ return False++async def validate_and_parse_json_request(request: Request, validator: type[BaseModel], *, extras: dict[str, Any] | None = None, exclude_unset: bool = False) -> tuple[dict[str, Any] | None, str | None]:@@ -161,6 +171,10 @@ def validate_and_parse_request_args(request: Request, validator: type[BaseModel]- Preserves type conversion from Pydantic validation"""args = request.args.to_dict(flat=True)+ for field_name, field_info in validator.model_fields.items():+ query_name = field_info.alias or field_name+ if query_name in request.args and _is_list_annotation(field_info.annotation):+ args[query_name] = [value for item in request.args.getlist(query_name) for value in item.split(",") if value]# Handle ext parameter: parse JSON string to dict if it's a stringif "ext" in args and isinstance(args["ext"], str):@@ -1002,9 +1016,26 @@ class BaseListReq(BaseModel):class ListDatasetReq(BaseListReq):"""Request model for listing datasets."""+ ids: Annotated[list[str] | None, Field(default=None)]include_parsing_status: Annotated[bool, Field(default=False)]ext: Annotated[dict, Field(default={})]+ @field_validator("ids", mode="after")+ @classmethod+ def validate_ids(cls, v_list: list[str] | None) -> list[str] | None:+ if v_list is None:+ return None++ ids_list = []+ for v in v_list:+ ids_list.append(validate_uuid1_hex(v))++ duplicates = [item for item, count in Counter(ids_list).items() if count > 1]+ if duplicates:+ raise PydanticCustomError("duplicate_uuids", "Duplicate ids: '{duplicate_ids}'", {"duplicate_ids": ", ".join(duplicates)})++ return ids_list+# ---- File Management Request Models ----diff --git a/sdk/python/ragflow_sdk/ragflow.py b/sdk/python/ragflow_sdk/ragflow.pyindex a339fe1d3f..4932e02d4d 100644--- a/sdk/python/ragflow_sdk/ragflow.py+++ b/sdk/python/ragflow_sdk/ragflow.py@@ -95,7 +95,9 @@ class RAGFlow:return _list[0]raise Exception("Dataset %s not found" % name)- def list_datasets(self, page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True, id: str | None = None, name: str | None = None) -> list[DataSet]:+ def list_datasets(+ self, page: int = 1, page_size: int = 30, orderby: str = "create_time", desc: bool = True, id: str | None = None, ids: list[str] | None = None, name: str | None = None+ ) -> list[DataSet]:res = self.get("/datasets",{@@ -104,6 +106,7 @@ class RAGFlow:"orderby": orderby,"desc": desc,"id": id,+ "ids": ids,"name": name,},)diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.pyindex 89bf7d217b..f7bedd44d5 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py@@ -77,6 +77,7 @@ def _identity_remap(source_data, key_aliases=None):def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):parsing_status_mock = MagicMock(return_value=parsing_status_by_kb)get_list_mock = MagicMock(return_value=(list(kbs), len(kbs)))+ get_accessible_ids_mock = MagicMock(return_value={kb["id"] for kb in kbs})_stub(monkeypatch,@@ -127,6 +128,7 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):"api.db.services.knowledgebase_service",KnowledgebaseService=SimpleNamespace(get_list=get_list_mock,+ get_accessible_ids=get_accessible_ids_mock,),validate_dataset_embedding_models=lambda kbs: None,)@@ -168,6 +170,11 @@ def _load_list_datasets_module(monkeypatch, *, kbs, parsing_status_by_kb):"common.misc_utils",thread_pool_exec=MagicMock(),)+ _stub(+ monkeypatch,+ "rag.advanced_rag.knowlege_compile.wiki",+ WIKI_PAGE_COMPILE_KWD="wiki",+ )repo_root = Path(__file__).resolve().parents[5]module_path = repo_root / "api" / "apps" / "services" / "dataset_api_service.py"@@ -235,6 +242,7 @@ def test_list_datasets_with_include_parsing_status_true_attaches_counts(monkeypaassert ok is Trueparsing_status_mock.assert_called_once_with(["kb-a", "kb-b"])+ assert "parsing_status" in payload["data"][0]by_id = {r["id"]: r for r in payload["data"]}assert by_id["kb-a"]["parsing_status"] == status_by_kb["kb-a"]assert by_id["kb-b"]["parsing_status"] == status_by_kb["kb-b"]@@ -258,7 +266,25 @@ def test_list_datasets_with_include_parsing_status_string_true(monkeypatch):assert ok is Trueparsing_status_mock.assert_called_once_with(["kb-a", "kb-b"])- assert "parsing_status" in payload["data"][0]+++def test_list_datasets_with_ids_filters_query_once(monkeypatch):+ """ids filter is checked once and pushed into the list query."""+ module, _, get_list_mock = _load_list_datasets_module(+ monkeypatch,+ kbs=_stub_kbs(),+ parsing_status_by_kb={},+ )++ ok, payload = module.list_datasets(+ "tenant-1",+ {"page": 1, "page_size": 30, "ids": ["kb-a", "kb-b"]},+ )++ assert ok is True+ assert payload["total"] == 2+ module.KnowledgebaseService.get_accessible_ids.assert_called_once_with(["tenant-1"], "tenant-1", ["kb-a", "kb-b"])+ get_list_mock.assert_called_once_with(["tenant-1"], "tenant-1", 1, 30, "create_time", True, None, None, "", None, ["kb-a", "kb-b"])def test_list_datasets_with_include_parsing_status_false_skips_helper(monkeypatch):