Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
perf(api): batch dataset list detail fields (#42279)
Mesrop Harutyunyancb4d8ab5eaf2
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.pyindex 4e07b814713..ed8c7280446 100644--- a/api/controllers/console/datasets/datasets.py+++ b/api/controllers/console/datasets/datasets.py@@ -42,7 +42,11 @@ from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfofrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethodfrom fields.base import ResponseModel-from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source+from fields.dataset_fields import (+ DatasetDetailResponse,+ build_dataset_detail_prefetch,+ dataset_detail_response_source,+)from graphon.model_runtime.entities.model_entities import ModelTypefrom libs.helper import build_icon_url, dump_response, to_timestampfrom libs.login import login_required@@ -538,8 +542,11 @@ class DatasetListApi(Resource):for embedding_model in embedding_models:model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")+ prefetch = build_dataset_detail_prefetch(datasets, session=session)data = [- dump_response(DatasetDetailResponse, dataset_detail_response_source(dataset, session=session))+ dump_response(+ DatasetDetailResponse, dataset_detail_response_source(dataset, session=session, prefetch=prefetch)+ )for dataset in datasets]dataset_ids = [item["id"] for item in data if item.get("permission") == "partial_members"]diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.pyindex 13379b1e5cf..155b2c7850b 100644--- a/api/controllers/service_api/dataset/dataset.py+++ b/api/controllers/service_api/dataset/dataset.py@@ -35,8 +35,12 @@ from controllers.service_api.wraps import (from core.plugin.impl.model_runtime_factory import create_plugin_provider_managerfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom fields.base import ResponseModel+from fields.dataset_fields import (+ DatasetDetailPrefetch,+ build_dataset_detail_prefetch,+ dataset_detail_response_source,+)from fields.dataset_fields import DatasetDetailResponse as BaseDatasetDetailResponse-from fields.dataset_fields import dataset_detail_response_sourcefrom graphon.model_runtime.entities.model_entities import ModelTypefrom libs.helper import dump_responsefrom libs.login import current_user@@ -98,9 +102,11 @@ _SERVICE_DATASET_DETAIL_EXCLUDE = {"permission_keys"}_SERVICE_DATASET_LIST_EXCLUDE = {"data": {"__all__": _SERVICE_DATASET_DETAIL_EXCLUDE}}-def _dump_service_dataset_detail(dataset: Any, *, session: Session) -> dict[str, Any]:+def _dump_service_dataset_detail(+ dataset: Any, *, session: Session, prefetch: DatasetDetailPrefetch | None = None+) -> dict[str, Any]:return DatasetDetailResponse.model_validate(- dataset_detail_response_source(dataset, session=session), from_attributes=True+ dataset_detail_response_source(dataset, session=session, prefetch=prefetch), from_attributes=True).model_dump(mode="json",exclude=_SERVICE_DATASET_DETAIL_EXCLUDE,@@ -461,7 +467,8 @@ class DatasetListApi(DatasetApiResource):for embedding_model in embedding_models:model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")- data = [_dump_service_dataset_detail(dataset, session=session) for dataset in datasets]+ prefetch = build_dataset_detail_prefetch(datasets, session=session)+ data = [_dump_service_dataset_detail(dataset, session=session, prefetch=prefetch) for dataset in datasets]for item in data:if item["indexing_technique"] == IndexTechniqueType.HIGH_QUALITY and item["embedding_model_provider"]:item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))diff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.pyindex 8f5541647e7..d70cea6d694 100644--- a/api/fields/dataset_fields.py+++ b/api/fields/dataset_fields.py@@ -1,12 +1,26 @@-from dataclasses import dataclass+from collections.abc import Sequence+from dataclasses import dataclass, fieldfrom datetime import datetimefrom typing import Any+import sqlalchemy as safrom pydantic import Field, field_validator+from sqlalchemy import func, selectfrom sqlalchemy.orm import Sessionfrom fields.base import ResponseModelfrom libs.helper import to_timestamp+from models.account import Account+from models.dataset import (+ AppDatasetJoin,+ Dataset,+ DatasetMetadata,+ Document,+ ExternalKnowledgeApis,+ ExternalKnowledgeBindings,+ Pipeline,+)+from models.model import App, Tag, TagBindingclass DatasetMetadataResponse(ResponseModel):@@ -194,60 +208,268 @@ class DatasetDetailResponse(ResponseModel):return {} if value is None else value+@dataclass(frozen=True)+class DatasetDetailPrefetch:+ """Batch-loaded values backing the session-scoped fields of a page of datasets.++ Building a detail response for a single dataset issues about ten queries for+ that row alone, so a list endpoint costs one query per dataset per field.+ Loading the same values for every dataset on the page up front keeps the+ query count bounded regardless of the page size.+ """++ app_counts: dict[str, int] = field(default_factory=dict)+ document_counts: dict[str, int] = field(default_factory=dict)+ word_counts: dict[str, int] = field(default_factory=dict)+ available_document_counts: dict[str, int] = field(default_factory=dict)+ author_names: dict[str, str] = field(default_factory=dict)+ tags: dict[tuple[str, str], list[Tag]] = field(default_factory=dict)+ doc_forms: dict[tuple[str, str], str | None] = field(default_factory=dict)+ external_knowledge_infos: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)+ doc_metadata: dict[str, list[DatasetMetadata]] = field(default_factory=dict)+ pipeline_published: dict[str, bool] = field(default_factory=dict)+++def build_dataset_detail_prefetch(datasets: Sequence[Dataset], *, session: Session) -> DatasetDetailPrefetch:+ """Load every session-scoped detail field for the given datasets in a fixed number of queries."""+ dataset_ids = [dataset.id for dataset in datasets]+ if not dataset_ids:+ return DatasetDetailPrefetch()++ tenant_ids = {dataset.tenant_id for dataset in datasets}++ app_counts = dict(+ session.execute(+ select(AppDatasetJoin.dataset_id, func.count(AppDatasetJoin.id))+ .where(AppDatasetJoin.dataset_id.in_(dataset_ids), App.id == AppDatasetJoin.app_id)+ .group_by(AppDatasetJoin.dataset_id)+ )+ .tuples()+ .all()+ )++ document_counts: dict[str, int] = {}+ word_counts: dict[str, int] = {}+ available_document_counts: dict[str, int] = {}+ available_document_case = sa.case(+ (+ sa.and_(+ Document.indexing_status == "completed",+ Document.enabled == True,+ Document.archived == False,+ ),+ 1,+ ),+ else_=0,+ )+ document_rows = session.execute(+ select(+ Document.dataset_id,+ func.count(Document.id),+ func.coalesce(func.sum(Document.word_count), 0),+ func.coalesce(func.sum(available_document_case), 0),+ )+ .where(Document.dataset_id.in_(dataset_ids))+ .group_by(Document.dataset_id)+ ).all()+ for dataset_id, document_count, word_count, available_count in document_rows:+ document_counts[dataset_id] = document_count+ word_counts[dataset_id] = word_count+ available_document_counts[dataset_id] = available_count++ # doc_form only falls back to the documents table for datasets without a+ # chunk structure of their own.+ doc_forms: dict[tuple[str, str], str | None] = {}+ doc_form_dataset_ids = [dataset.id for dataset in datasets if not dataset.chunk_structure]+ if doc_form_dataset_ids:+ doc_forms = {+ (dataset_id, tenant_id): doc_form+ for dataset_id, tenant_id, doc_form in session.execute(+ select(Document.dataset_id, Document.tenant_id, func.min(Document.doc_form))+ .where(Document.dataset_id.in_(doc_form_dataset_ids), Document.tenant_id.in_(tenant_ids))+ .group_by(Document.dataset_id, Document.tenant_id)+ ).all()+ }++ author_ids = {dataset.created_by for dataset in datasets if dataset.created_by}+ author_names = dict(+ session.execute(select(Account.id, Account.name).where(Account.id.in_(author_ids))).tuples().all()+ )++ tags: dict[tuple[str, str], list[Tag]] = {}+ tag_rows = session.execute(+ select(TagBinding.target_id, TagBinding.tenant_id, Tag)+ .join(TagBinding, Tag.id == TagBinding.tag_id)+ .where(+ TagBinding.target_id.in_(dataset_ids),+ TagBinding.tenant_id.in_(tenant_ids),+ Tag.tenant_id == TagBinding.tenant_id,+ Tag.type == "knowledge",+ )+ ).all()+ for target_id, tenant_id, tag in tag_rows:+ tags.setdefault((target_id, tenant_id), []).append(tag)++ doc_metadata: dict[str, list[DatasetMetadata]] = {}+ for dataset_metadata in session.scalars(+ select(DatasetMetadata).where(DatasetMetadata.dataset_id.in_(dataset_ids))+ ).all():+ doc_metadata.setdefault(dataset_metadata.dataset_id, []).append(dataset_metadata)++ pipeline_ids = {dataset.pipeline_id for dataset in datasets if dataset.pipeline_id}+ pipeline_published: dict[str, bool] = {}+ if pipeline_ids:+ pipeline_published = dict(+ session.execute(select(Pipeline.id, Pipeline.is_published).where(Pipeline.id.in_(pipeline_ids)))+ .tuples()+ .all()+ )++ return DatasetDetailPrefetch(+ app_counts=app_counts,+ document_counts=document_counts,+ word_counts=word_counts,+ available_document_counts=available_document_counts,+ author_names=author_names,+ tags=tags,+ doc_forms=doc_forms,+ external_knowledge_infos=_load_external_knowledge_infos(datasets, session=session),+ doc_metadata=doc_metadata,+ pipeline_published=pipeline_published,+ )+++def _load_external_knowledge_infos(+ datasets: Sequence[Dataset], *, session: Session+) -> dict[tuple[str, str], dict[str, Any]]:+ external_datasets = [dataset for dataset in datasets if dataset.provider == "external"]+ if not external_datasets:+ return {}++ external_dataset_ids = [dataset.id for dataset in external_datasets]+ external_tenant_ids = {dataset.tenant_id for dataset in external_datasets}+ bindings = session.scalars(+ select(ExternalKnowledgeBindings).where(+ ExternalKnowledgeBindings.dataset_id.in_(external_dataset_ids),+ ExternalKnowledgeBindings.tenant_id.in_(external_tenant_ids),+ )+ ).all()+ if not bindings:+ return {}++ apis = {+ (api.id, api.tenant_id): api+ for api in session.scalars(+ select(ExternalKnowledgeApis).where(+ ExternalKnowledgeApis.id.in_({binding.external_knowledge_api_id for binding in bindings}),+ ExternalKnowledgeApis.tenant_id.in_(external_tenant_ids),+ )+ ).all()+ }++ infos: dict[tuple[str, str], dict[str, Any]] = {}+ for binding in bindings:+ info = Dataset.build_external_knowledge_info(+ binding, apis.get((binding.external_knowledge_api_id, binding.tenant_id))+ )+ if info is not None:+ infos[(binding.dataset_id, binding.tenant_id)] = info+ return infos++@dataclass(frozen=True)class DatasetDetailResponseSource:- """Expose session-backed dataset fields during response validation."""+ """Expose session-backed dataset fields during response validation.++ With a prefetch the values come from the batch load for the whole page;+ without one each field is resolved against the session on access.+ """dataset: Anysession: Session+ prefetch: DatasetDetailPrefetch | None = None++ @property+ def _prefetch_key(self) -> tuple[str, str]:+ return (self.dataset.id, self.dataset.tenant_id)@propertydef app_count(self) -> int:+ if self.prefetch is not None:+ return self.prefetch.app_counts.get(self.dataset.id, 0)return self.dataset.get_app_count(session=self.session)@propertydef document_count(self) -> int:+ if self.prefetch is not None:+ return self.prefetch.document_counts.get(self.dataset.id, 0)return self.dataset.get_document_count(session=self.session)@propertydef word_count(self) -> int:+ if self.prefetch is not None:+ return self.prefetch.word_counts.get(self.dataset.id, 0)return self.dataset.get_word_count(session=self.session)@propertydef author_name(self) -> str | None:+ if self.prefetch is not None:+ return self.prefetch.author_names.get(self.dataset.created_by)return self.dataset.get_author_name(session=self.session)@propertydef tags(self) -> Any:+ if self.prefetch is not None:+ return self.prefetch.tags.get(self._prefetch_key, [])return self.dataset.get_tags(session=self.session)@propertydef doc_form(self) -> str | None:+ if self.prefetch is not None:+ if self.dataset.chunk_structure:+ return self.dataset.chunk_structure+ return self.prefetch.doc_forms.get(self._prefetch_key)return self.dataset.get_doc_form(session=self.session)@propertydef external_knowledge_info(self) -> Any:+ if self.prefetch is not None:+ if self.dataset.provider != "external":+ return None+ return self.prefetch.external_knowledge_infos.get(self._prefetch_key)return self.dataset.get_external_knowledge_info(session=self.session)@propertydef doc_metadata(self) -> Any:+ if self.prefetch is not None:+ return self.dataset.build_doc_metadata(self.prefetch.doc_metadata.get(self.dataset.id, []))return self.dataset.get_doc_metadata(session=self.session)@propertydef is_published(self) -> bool:+ if self.prefetch is not None:+ if not self.dataset.pipeline_id:+ return False+ return self.prefetch.pipeline_published.get(self.dataset.pipeline_id, False)return self.dataset.get_is_published(session=self.session)@propertydef total_documents(self) -> int:+ if self.prefetch is not None:+ return self.prefetch.document_counts.get(self.dataset.id, 0)return self.dataset.get_total_documents(session=self.session)@propertydef total_available_documents(self) -> int:+ if self.prefetch is not None:+ return self.prefetch.available_document_counts.get(self.dataset.id, 0)return self.dataset.get_total_available_documents(session=self.session)def __getattr__(self, name: str) -> Any:return getattr(self.dataset, name) # guard-ignore: no-new-getattr -- delegates model fields-def dataset_detail_response_source(dataset: Any, *, session: Session) -> DatasetDetailResponseSource:- return DatasetDetailResponseSource(dataset=dataset, session=session)+def dataset_detail_response_source(+ dataset: Any, *, session: Session, prefetch: DatasetDetailPrefetch | None = None+) -> DatasetDetailResponseSource:+ return DatasetDetailResponseSource(dataset=dataset, session=session, prefetch=prefetch)diff --git a/api/models/dataset.py b/api/models/dataset.pyindex 87e6d3e5635..28f4ce0ebe5 100644--- a/api/models/dataset.py+++ b/api/models/dataset.py@@ -367,6 +367,16 @@ class Dataset(Base):ExternalKnowledgeApis.tenant_id == self.tenant_id,))+ return self.build_external_knowledge_info(external_knowledge_binding, external_knowledge_api)++ @staticmethod+ def build_external_knowledge_info(+ external_knowledge_binding: "ExternalKnowledgeBindings | None",+ external_knowledge_api: "ExternalKnowledgeApis | None",+ ) -> dict[str, Any] | None:+ """Format an external knowledge binding and its API into the response payload."""+ if external_knowledge_binding is None:+ return Noneif external_knowledge_api is None or external_knowledge_api.settings is None:return Nonereturn {@@ -386,6 +396,10 @@ class Dataset(Base):def get_doc_metadata(self, *, session: Session) -> list[dict[str, str]]:dataset_metadatas = session.scalars(select(DatasetMetadata).where(DatasetMetadata.dataset_id == self.id)).all()+ return self.build_doc_metadata(dataset_metadatas)++ def build_doc_metadata(self, dataset_metadatas: "Sequence[DatasetMetadata]") -> list[dict[str, str]]:+ """Format metadata rows, appending the built-in fields when they are enabled."""doc_metadata = [{"id": dataset_metadata.id,diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.pyindex 1f31d150a5b..238e551eeef 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py@@ -45,6 +45,7 @@ from core.rag.datasource.vdb.vector_type import VectorTypefrom core.rag.index_processor.constant.index_type import IndexStructureTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethodfrom extensions.storage.storage_type import StorageType+from fields.dataset_fields import build_dataset_detail_prefetchfrom models.account import Account, TenantAccountRolefrom models.dataset import AppDatasetJoin, Dataset, DatasetPermission, DatasetQuery, Document, DocumentSegmentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus@@ -229,7 +230,7 @@ class TestDatasetList(_UsesSQLiteSession):"icon_url": None,}- def test_get_serializes_database_fields_with_caller_session(self, app: Flask, dataset_model_property_defaults):+ def test_get_batch_loads_database_fields_with_caller_session(self, app: Flask, dataset_model_property_defaults):api = DatasetListApi()method = unwrap(api.get)current_user = self._mock_user()@@ -239,11 +240,21 @@ class TestDatasetList(_UsesSQLiteSession):with (patch.object(DatasetService, "get_datasets", return_value=([dataset], 1)),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),+ patch(+ "controllers.console.datasets.datasets.build_dataset_detail_prefetch",+ wraps=build_dataset_detail_prefetch,+ ) as prefetch_mock,):- method(api, session, "tenant-1", current_user)+ resp, status = method(api, session, "tenant-1", current_user)+ assert status == 200+ prefetch_mock.assert_called_once_with([dataset], session=session)+ # The page is served from that single batch load, so no field may fall back+ # to a per-dataset query.for getter in dataset_model_property_defaults.values():- getter.assert_called_once_with(dataset, session=session)+ getter.assert_not_called()+ assert resp["data"][0]["document_count"] == 0+ assert resp["data"][0]["tags"] == []def test_get_with_ids_filter(self, app: Flask):api = DatasetListApi()diff --git a/api/tests/unit_tests/fields/test_dataset_fields.py b/api/tests/unit_tests/fields/test_dataset_fields.pyindex 71d4ef0a735..ecde29d57fe 100644--- a/api/tests/unit_tests/fields/test_dataset_fields.py+++ b/api/tests/unit_tests/fields/test_dataset_fields.py@@ -1,10 +1,30 @@+import json+from collections.abc import Iterator+from contextlib import contextmanager+import pytest+from sqlalchemy import eventfrom sqlalchemy.orm import Session-from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source+from core.rag.index_processor.constant.index_type import IndexStructureType+from fields.dataset_fields import (+ DatasetDetailPrefetch,+ DatasetDetailResponse,+ build_dataset_detail_prefetch,+ dataset_detail_response_source,+)from models.account import Account-from models.dataset import AppDatasetJoin, Dataset-from models.model import App, AppMode, IconType+from models.dataset import (+ AppDatasetJoin,+ Dataset,+ DatasetMetadata,+ Document,+ ExternalKnowledgeApis,+ ExternalKnowledgeBindings,+ Pipeline,+)+from models.enums import DatasetMetadataType, DataSourceType, DocumentCreatedFrom, TagType+from models.model import App, AppMode, IconType, Tag, TagBindingdef _dataset_detail_payload(**overrides):@@ -243,3 +263,202 @@ def test_dataset_detail_response_source_uses_caller_session_for_database_fields(assert response.is_published is Falseassert response.total_documents == 0assert response.total_available_documents == 0+++def _dataset(index: int, account_id: str, **overrides: object) -> Dataset:+ dataset = Dataset(+ id=f"ds-{index}",+ tenant_id="tenant-1",+ name=f"Dataset {index}",+ description="desc",+ provider="vendor",+ permission="only_me",+ data_source_type=None,+ indexing_technique="economy",+ created_by=account_id,+ retrieval_model=None,+ summary_index_setting=None,+ built_in_field_enabled=False,+ icon_info=None,+ runtime_mode="general",+ enable_api=False,+ is_multimodal=False,+ )+ for name, value in overrides.items():+ setattr(dataset, name, value)+ return dataset+++def _document(dataset_id: str, *, word_count: int, indexing_status: str, archived: bool = False) -> Document:+ return Document(+ tenant_id="tenant-1",+ dataset_id=dataset_id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch",+ name="doc",+ created_from=DocumentCreatedFrom.WEB,+ created_by="account-1",+ word_count=word_count,+ indexing_status=indexing_status,+ enabled=True,+ archived=archived,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ )+++def _seed_dataset_page(session: Session) -> list[Dataset]:+ """Create a page of datasets covering every session-scoped detail field."""+ account = Account(name="Ada", email="[email protected]")+ account.id = "account-1"+ session.add(account)++ pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", is_published=True, created_by=account.id)+ external_api = ExternalKnowledgeApis(+ name="api",+ description="",+ tenant_id="tenant-1",+ settings=json.dumps({"endpoint": "https://example.com"}),+ created_by=account.id,+ updated_by=None,+ )+ session.add_all([pipeline, external_api])+ session.flush()++ plain = _dataset(1, account.id, built_in_field_enabled=True)+ piped = _dataset(2, account.id, pipeline_id=pipeline.id)+ external = _dataset(3, account.id, provider="external")+ structured = _dataset(4, account.id, chunk_structure=IndexStructureType.QA_INDEX)+ empty = _dataset(5, account.id)+ datasets = [plain, piped, external, structured, empty]+ session.add_all(datasets)++ session.add_all(+ [+ _document(plain.id, word_count=10, indexing_status="completed"),+ _document(plain.id, word_count=20, indexing_status="completed", archived=True),+ _document(plain.id, word_count=5, indexing_status="waiting"),+ _document(piped.id, word_count=7, indexing_status="completed"),+ ]+ )++ tag = Tag(tenant_id="tenant-1", type=TagType.KNOWLEDGE, name="tag", created_by=account.id)+ other_tag = Tag(tenant_id="tenant-1", type=TagType.APP, name="app tag", created_by=account.id)+ session.add_all([tag, other_tag])+ session.flush()+ session.add_all(+ [+ TagBinding(tenant_id="tenant-1", tag_id=tag.id, target_id=plain.id, created_by=account.id),+ TagBinding(tenant_id="tenant-1", tag_id=other_tag.id, target_id=plain.id, created_by=account.id),+ ]+ )++ session.add(+ DatasetMetadata(+ tenant_id="tenant-1",+ dataset_id=plain.id,+ type=DatasetMetadataType.STRING,+ name="author",+ created_by=account.id,+ )+ )+ session.add(+ ExternalKnowledgeBindings(+ tenant_id="tenant-1",+ external_knowledge_api_id=external_api.id,+ dataset_id=external.id,+ external_knowledge_id="external-1",+ created_by=account.id,+ )+ )++ app = App(+ id="app-1",+ tenant_id="tenant-1",+ name="App",+ description="",+ mode=AppMode.CHAT,+ icon_type=IconType.EMOJI,+ icon="app",+ icon_background="#FFFFFF",+ enable_site=False,+ enable_api=False,+ max_active_requests=0,+ )+ session.add_all([app, AppDatasetJoin(app_id=app.id, dataset_id=plain.id)])+ session.flush()+ return datasets+++def _dump(dataset: Dataset, *, session: Session, prefetch: DatasetDetailPrefetch | None = None) -> dict[str, object]:+ return DatasetDetailResponse.model_validate(+ dataset_detail_response_source(dataset, session=session, prefetch=prefetch),+ from_attributes=True,+ ).model_dump(mode="json")+++@contextmanager+def _count_selects(session: Session) -> Iterator[list[str]]:+ """Record every SELECT issued against the session's engine."""+ statements: list[str] = []+ engine = session.get_bind()++ @event.listens_for(engine, "before_cursor_execute")+ def _before(_conn, _cursor, statement, _parameters, _context, _executemany):+ if statement.lstrip().upper().startswith("SELECT"):+ statements.append(statement)++ try:+ yield statements+ finally:+ event.remove(engine, "before_cursor_execute", _before)+++def test_dataset_detail_prefetch_matches_per_dataset_lookups(sqlite_session: Session):+ datasets = _seed_dataset_page(sqlite_session)++ expected = [_dump(dataset, session=sqlite_session) for dataset in datasets]++ prefetch = build_dataset_detail_prefetch(datasets, session=sqlite_session)+ prefetched = [_dump(dataset, session=sqlite_session, prefetch=prefetch) for dataset in datasets]++ assert prefetched == expected++ plain = prefetched[0]+ assert plain["app_count"] == 1+ assert plain["document_count"] == 3+ assert plain["total_documents"] == 3+ assert plain["total_available_documents"] == 1+ assert plain["word_count"] == 35+ assert plain["author_name"] == "Ada"+ assert [tag["name"] for tag in plain["tags"]] == ["tag"]+ assert plain["doc_form"] == IndexStructureType.PARAGRAPH_INDEX+ assert [metadata["name"] for metadata in plain["doc_metadata"]][0] == "author"+ assert len(plain["doc_metadata"]) == 6+ assert prefetched[1]["is_published"] is True+ assert prefetched[2]["external_knowledge_info"]["external_knowledge_api_endpoint"] == "https://example.com"+ assert prefetched[3]["doc_form"] == IndexStructureType.QA_INDEX+ assert prefetched[4]["document_count"] == 0+ assert prefetched[4]["doc_form"] is None+++def test_dataset_detail_prefetch_keeps_query_count_independent_of_page_size(sqlite_session: Session):+ account = Account(name="Ada", email="[email protected]")+ account.id = "account-1"+ sqlite_session.add(account)+ datasets = [_dataset(index, account.id) for index in range(20)]+ sqlite_session.add_all(datasets)+ sqlite_session.flush()++ def select_count(page: list[Dataset]) -> int:+ with _count_selects(sqlite_session) as statements:+ prefetch = build_dataset_detail_prefetch(page, session=sqlite_session)+ for dataset in page:+ _dump(dataset, session=sqlite_session, prefetch=prefetch)+ return len(statements)++ small_page = select_count(datasets[:2])+ full_page = select_count(datasets)++ assert full_page == small_page+ assert full_page <= 8