Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

perf(api): batch dataset list detail fields (#42279)

Mesrop Harutyunyancb4d8ab5eaf2
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
index 4e07b814713..ed8c7280446 100644
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -42,7 +42,11 @@ from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from fields.base import ResponseModel
-from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source
+from fields.dataset_fields import (
+ DatasetDetailResponse,
+ build_dataset_detail_prefetch,
+ dataset_detail_response_source,
+)
from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import build_icon_url, dump_response, to_timestamp
from libs.login import login_required
@@ -538,8 +542,11 @@ class DatasetListApi(Resource):
for embedding_model in embedding_models:
model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")
+ prefetch = build_dataset_detail_prefetch(datasets, session=session)
data = [
- dump_response(DatasetDetailResponse, dataset_detail_response_source(dataset, session=session))
+ dump_response(
+ DatasetDetailResponse, dataset_detail_response_source(dataset, session=session, prefetch=prefetch)
+ )
for dataset in datasets
]
dataset_ids = [item["id"] for item in data if item.get("permission") == "partial_members"]
diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py
index 13379b1e5cf..155b2c7850b 100644
--- a/api/controllers/service_api/dataset/dataset.py
+++ b/api/controllers/service_api/dataset/dataset.py
@@ -35,8 +35,12 @@ from controllers.service_api.wraps import (
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from fields.base import ResponseModel
+from fields.dataset_fields import (
+ DatasetDetailPrefetch,
+ build_dataset_detail_prefetch,
+ dataset_detail_response_source,
+)
from fields.dataset_fields import DatasetDetailResponse as BaseDatasetDetailResponse
-from fields.dataset_fields import dataset_detail_response_source
from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import dump_response
from libs.login import current_user
@@ -98,9 +102,11 @@ _SERVICE_DATASET_DETAIL_EXCLUDE = {"permission_keys"}
_SERVICE_DATASET_LIST_EXCLUDE = {"data": {"__all__": _SERVICE_DATASET_DETAIL_EXCLUDE}}
-def _dump_service_dataset_detail(dataset: Any, *, session: Session) -> dict[str, Any]:
+def _dump_service_dataset_detail(
+ dataset: Any, *, session: Session, prefetch: DatasetDetailPrefetch | None = None
+) -> dict[str, Any]:
return DatasetDetailResponse.model_validate(
- dataset_detail_response_source(dataset, session=session), from_attributes=True
+ dataset_detail_response_source(dataset, session=session, prefetch=prefetch), from_attributes=True
).model_dump(
mode="json",
exclude=_SERVICE_DATASET_DETAIL_EXCLUDE,
@@ -461,7 +467,8 @@ class DatasetListApi(DatasetApiResource):
for embedding_model in embedding_models:
model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")
- data = [_dump_service_dataset_detail(dataset, session=session) for dataset in datasets]
+ prefetch = build_dataset_detail_prefetch(datasets, session=session)
+ data = [_dump_service_dataset_detail(dataset, session=session, prefetch=prefetch) for dataset in datasets]
for item in data:
if item["indexing_technique"] == IndexTechniqueType.HIGH_QUALITY and item["embedding_model_provider"]:
item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
diff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.py
index 8f5541647e7..d70cea6d694 100644
--- a/api/fields/dataset_fields.py
+++ b/api/fields/dataset_fields.py
@@ -1,12 +1,26 @@
-from dataclasses import dataclass
+from collections.abc import Sequence
+from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
+import sqlalchemy as sa
from pydantic import Field, field_validator
+from sqlalchemy import func, select
from sqlalchemy.orm import Session
from fields.base import ResponseModel
from libs.helper import to_timestamp
+from models.account import Account
+from models.dataset import (
+ AppDatasetJoin,
+ Dataset,
+ DatasetMetadata,
+ Document,
+ ExternalKnowledgeApis,
+ ExternalKnowledgeBindings,
+ Pipeline,
+)
+from models.model import App, Tag, TagBinding
class DatasetMetadataResponse(ResponseModel):
@@ -194,60 +208,268 @@ class DatasetDetailResponse(ResponseModel):
return {} if value is None else value
+@dataclass(frozen=True)
+class DatasetDetailPrefetch:
+ """Batch-loaded values backing the session-scoped fields of a page of datasets.
+
+ Building a detail response for a single dataset issues about ten queries for
+ that row alone, so a list endpoint costs one query per dataset per field.
+ Loading the same values for every dataset on the page up front keeps the
+ query count bounded regardless of the page size.
+ """
+
+ app_counts: dict[str, int] = field(default_factory=dict)
+ document_counts: dict[str, int] = field(default_factory=dict)
+ word_counts: dict[str, int] = field(default_factory=dict)
+ available_document_counts: dict[str, int] = field(default_factory=dict)
+ author_names: dict[str, str] = field(default_factory=dict)
+ tags: dict[tuple[str, str], list[Tag]] = field(default_factory=dict)
+ doc_forms: dict[tuple[str, str], str | None] = field(default_factory=dict)
+ external_knowledge_infos: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)
+ doc_metadata: dict[str, list[DatasetMetadata]] = field(default_factory=dict)
+ pipeline_published: dict[str, bool] = field(default_factory=dict)
+
+
+def build_dataset_detail_prefetch(datasets: Sequence[Dataset], *, session: Session) -> DatasetDetailPrefetch:
+ """Load every session-scoped detail field for the given datasets in a fixed number of queries."""
+ dataset_ids = [dataset.id for dataset in datasets]
+ if not dataset_ids:
+ return DatasetDetailPrefetch()
+
+ tenant_ids = {dataset.tenant_id for dataset in datasets}
+
+ app_counts = dict(
+ session.execute(
+ select(AppDatasetJoin.dataset_id, func.count(AppDatasetJoin.id))
+ .where(AppDatasetJoin.dataset_id.in_(dataset_ids), App.id == AppDatasetJoin.app_id)
+ .group_by(AppDatasetJoin.dataset_id)
+ )
+ .tuples()
+ .all()
+ )
+
+ document_counts: dict[str, int] = {}
+ word_counts: dict[str, int] = {}
+ available_document_counts: dict[str, int] = {}
+ available_document_case = sa.case(
+ (
+ sa.and_(
+ Document.indexing_status == "completed",
+ Document.enabled == True,
+ Document.archived == False,
+ ),
+ 1,
+ ),
+ else_=0,
+ )
+ document_rows = session.execute(
+ select(
+ Document.dataset_id,
+ func.count(Document.id),
+ func.coalesce(func.sum(Document.word_count), 0),
+ func.coalesce(func.sum(available_document_case), 0),
+ )
+ .where(Document.dataset_id.in_(dataset_ids))
+ .group_by(Document.dataset_id)
+ ).all()
+ for dataset_id, document_count, word_count, available_count in document_rows:
+ document_counts[dataset_id] = document_count
+ word_counts[dataset_id] = word_count
+ available_document_counts[dataset_id] = available_count
+
+ # doc_form only falls back to the documents table for datasets without a
+ # chunk structure of their own.
+ doc_forms: dict[tuple[str, str], str | None] = {}
+ doc_form_dataset_ids = [dataset.id for dataset in datasets if not dataset.chunk_structure]
+ if doc_form_dataset_ids:
+ doc_forms = {
+ (dataset_id, tenant_id): doc_form
+ for dataset_id, tenant_id, doc_form in session.execute(
+ select(Document.dataset_id, Document.tenant_id, func.min(Document.doc_form))
+ .where(Document.dataset_id.in_(doc_form_dataset_ids), Document.tenant_id.in_(tenant_ids))
+ .group_by(Document.dataset_id, Document.tenant_id)
+ ).all()
+ }
+
+ author_ids = {dataset.created_by for dataset in datasets if dataset.created_by}
+ author_names = dict(
+ session.execute(select(Account.id, Account.name).where(Account.id.in_(author_ids))).tuples().all()
+ )
+
+ tags: dict[tuple[str, str], list[Tag]] = {}
+ tag_rows = session.execute(
+ select(TagBinding.target_id, TagBinding.tenant_id, Tag)
+ .join(TagBinding, Tag.id == TagBinding.tag_id)
+ .where(
+ TagBinding.target_id.in_(dataset_ids),
+ TagBinding.tenant_id.in_(tenant_ids),
+ Tag.tenant_id == TagBinding.tenant_id,
+ Tag.type == "knowledge",
+ )
+ ).all()
+ for target_id, tenant_id, tag in tag_rows:
+ tags.setdefault((target_id, tenant_id), []).append(tag)
+
+ doc_metadata: dict[str, list[DatasetMetadata]] = {}
+ for dataset_metadata in session.scalars(
+ select(DatasetMetadata).where(DatasetMetadata.dataset_id.in_(dataset_ids))
+ ).all():
+ doc_metadata.setdefault(dataset_metadata.dataset_id, []).append(dataset_metadata)
+
+ pipeline_ids = {dataset.pipeline_id for dataset in datasets if dataset.pipeline_id}
+ pipeline_published: dict[str, bool] = {}
+ if pipeline_ids:
+ pipeline_published = dict(
+ session.execute(select(Pipeline.id, Pipeline.is_published).where(Pipeline.id.in_(pipeline_ids)))
+ .tuples()
+ .all()
+ )
+
+ return DatasetDetailPrefetch(
+ app_counts=app_counts,
+ document_counts=document_counts,
+ word_counts=word_counts,
+ available_document_counts=available_document_counts,
+ author_names=author_names,
+ tags=tags,
+ doc_forms=doc_forms,
+ external_knowledge_infos=_load_external_knowledge_infos(datasets, session=session),
+ doc_metadata=doc_metadata,
+ pipeline_published=pipeline_published,
+ )
+
+
+def _load_external_knowledge_infos(
+ datasets: Sequence[Dataset], *, session: Session
+) -> dict[tuple[str, str], dict[str, Any]]:
+ external_datasets = [dataset for dataset in datasets if dataset.provider == "external"]
+ if not external_datasets:
+ return {}
+
+ external_dataset_ids = [dataset.id for dataset in external_datasets]
+ external_tenant_ids = {dataset.tenant_id for dataset in external_datasets}
+ bindings = session.scalars(
+ select(ExternalKnowledgeBindings).where(
+ ExternalKnowledgeBindings.dataset_id.in_(external_dataset_ids),
+ ExternalKnowledgeBindings.tenant_id.in_(external_tenant_ids),
+ )
+ ).all()
+ if not bindings:
+ return {}
+
+ apis = {
+ (api.id, api.tenant_id): api
+ for api in session.scalars(
+ select(ExternalKnowledgeApis).where(
+ ExternalKnowledgeApis.id.in_({binding.external_knowledge_api_id for binding in bindings}),
+ ExternalKnowledgeApis.tenant_id.in_(external_tenant_ids),
+ )
+ ).all()
+ }
+
+ infos: dict[tuple[str, str], dict[str, Any]] = {}
+ for binding in bindings:
+ info = Dataset.build_external_knowledge_info(
+ binding, apis.get((binding.external_knowledge_api_id, binding.tenant_id))
+ )
+ if info is not None:
+ infos[(binding.dataset_id, binding.tenant_id)] = info
+ return infos
+
+
@dataclass(frozen=True)
class DatasetDetailResponseSource:
- """Expose session-backed dataset fields during response validation."""
+ """Expose session-backed dataset fields during response validation.
+
+ With a prefetch the values come from the batch load for the whole page;
+ without one each field is resolved against the session on access.
+ """
dataset: Any
session: Session
+ prefetch: DatasetDetailPrefetch | None = None
+
+ @property
+ def _prefetch_key(self) -> tuple[str, str]:
+ return (self.dataset.id, self.dataset.tenant_id)
@property
def app_count(self) -> int:
+ if self.prefetch is not None:
+ return self.prefetch.app_counts.get(self.dataset.id, 0)
return self.dataset.get_app_count(session=self.session)
@property
def document_count(self) -> int:
+ if self.prefetch is not None:
+ return self.prefetch.document_counts.get(self.dataset.id, 0)
return self.dataset.get_document_count(session=self.session)
@property
def word_count(self) -> int:
+ if self.prefetch is not None:
+ return self.prefetch.word_counts.get(self.dataset.id, 0)
return self.dataset.get_word_count(session=self.session)
@property
def author_name(self) -> str | None:
+ if self.prefetch is not None:
+ return self.prefetch.author_names.get(self.dataset.created_by)
return self.dataset.get_author_name(session=self.session)
@property
def tags(self) -> Any:
+ if self.prefetch is not None:
+ return self.prefetch.tags.get(self._prefetch_key, [])
return self.dataset.get_tags(session=self.session)
@property
def doc_form(self) -> str | None:
+ if self.prefetch is not None:
+ if self.dataset.chunk_structure:
+ return self.dataset.chunk_structure
+ return self.prefetch.doc_forms.get(self._prefetch_key)
return self.dataset.get_doc_form(session=self.session)
@property
def external_knowledge_info(self) -> Any:
+ if self.prefetch is not None:
+ if self.dataset.provider != "external":
+ return None
+ return self.prefetch.external_knowledge_infos.get(self._prefetch_key)
return self.dataset.get_external_knowledge_info(session=self.session)
@property
def doc_metadata(self) -> Any:
+ if self.prefetch is not None:
+ return self.dataset.build_doc_metadata(self.prefetch.doc_metadata.get(self.dataset.id, []))
return self.dataset.get_doc_metadata(session=self.session)
@property
def is_published(self) -> bool:
+ if self.prefetch is not None:
+ if not self.dataset.pipeline_id:
+ return False
+ return self.prefetch.pipeline_published.get(self.dataset.pipeline_id, False)
return self.dataset.get_is_published(session=self.session)
@property
def total_documents(self) -> int:
+ if self.prefetch is not None:
+ return self.prefetch.document_counts.get(self.dataset.id, 0)
return self.dataset.get_total_documents(session=self.session)
@property
def total_available_documents(self) -> int:
+ if self.prefetch is not None:
+ return self.prefetch.available_document_counts.get(self.dataset.id, 0)
return self.dataset.get_total_available_documents(session=self.session)
def __getattr__(self, name: str) -> Any:
return getattr(self.dataset, name) # guard-ignore: no-new-getattr -- delegates model fields
-def dataset_detail_response_source(dataset: Any, *, session: Session) -> DatasetDetailResponseSource:
- return DatasetDetailResponseSource(dataset=dataset, session=session)
+def dataset_detail_response_source(
+ dataset: Any, *, session: Session, prefetch: DatasetDetailPrefetch | None = None
+) -> DatasetDetailResponseSource:
+ return DatasetDetailResponseSource(dataset=dataset, session=session, prefetch=prefetch)
diff --git a/api/models/dataset.py b/api/models/dataset.py
index 87e6d3e5635..28f4ce0ebe5 100644
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@@ -367,6 +367,16 @@ class Dataset(Base):
ExternalKnowledgeApis.tenant_id == self.tenant_id,
)
)
+ return self.build_external_knowledge_info(external_knowledge_binding, external_knowledge_api)
+
+ @staticmethod
+ def build_external_knowledge_info(
+ external_knowledge_binding: "ExternalKnowledgeBindings | None",
+ external_knowledge_api: "ExternalKnowledgeApis | None",
+ ) -> dict[str, Any] | None:
+ """Format an external knowledge binding and its API into the response payload."""
+ if external_knowledge_binding is None:
+ return None
if external_knowledge_api is None or external_knowledge_api.settings is None:
return None
return {
@@ -386,6 +396,10 @@ class Dataset(Base):
def get_doc_metadata(self, *, session: Session) -> list[dict[str, str]]:
dataset_metadatas = session.scalars(select(DatasetMetadata).where(DatasetMetadata.dataset_id == self.id)).all()
+ return self.build_doc_metadata(dataset_metadatas)
+
+ def build_doc_metadata(self, dataset_metadatas: "Sequence[DatasetMetadata]") -> list[dict[str, str]]:
+ """Format metadata rows, appending the built-in fields when they are enabled."""
doc_metadata = [
{
"id": dataset_metadata.id,
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
index 1f31d150a5b..238e551eeef 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
@@ -45,6 +45,7 @@ from core.rag.datasource.vdb.vector_type import VectorType
from core.rag.index_processor.constant.index_type import IndexStructureType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
from extensions.storage.storage_type import StorageType
+from fields.dataset_fields import build_dataset_detail_prefetch
from models.account import Account, TenantAccountRole
from models.dataset import AppDatasetJoin, Dataset, DatasetPermission, DatasetQuery, Document, DocumentSegment
from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus, SegmentStatus
@@ -229,7 +230,7 @@ class TestDatasetList(_UsesSQLiteSession):
"icon_url": None,
}
- def test_get_serializes_database_fields_with_caller_session(self, app: Flask, dataset_model_property_defaults):
+ def test_get_batch_loads_database_fields_with_caller_session(self, app: Flask, dataset_model_property_defaults):
api = DatasetListApi()
method = unwrap(api.get)
current_user = self._mock_user()
@@ -239,11 +240,21 @@ class TestDatasetList(_UsesSQLiteSession):
with (
patch.object(DatasetService, "get_datasets", return_value=([dataset], 1)),
patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),
+ patch(
+ "controllers.console.datasets.datasets.build_dataset_detail_prefetch",
+ wraps=build_dataset_detail_prefetch,
+ ) as prefetch_mock,
):
- method(api, session, "tenant-1", current_user)
+ resp, status = method(api, session, "tenant-1", current_user)
+ assert status == 200
+ prefetch_mock.assert_called_once_with([dataset], session=session)
+ # The page is served from that single batch load, so no field may fall back
+ # to a per-dataset query.
for getter in dataset_model_property_defaults.values():
- getter.assert_called_once_with(dataset, session=session)
+ getter.assert_not_called()
+ assert resp["data"][0]["document_count"] == 0
+ assert resp["data"][0]["tags"] == []
def test_get_with_ids_filter(self, app: Flask):
api = DatasetListApi()
diff --git a/api/tests/unit_tests/fields/test_dataset_fields.py b/api/tests/unit_tests/fields/test_dataset_fields.py
index 71d4ef0a735..ecde29d57fe 100644
--- a/api/tests/unit_tests/fields/test_dataset_fields.py
+++ b/api/tests/unit_tests/fields/test_dataset_fields.py
@@ -1,10 +1,30 @@
+import json
+from collections.abc import Iterator
+from contextlib import contextmanager
+
import pytest
+from sqlalchemy import event
from sqlalchemy.orm import Session
-from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source
+from core.rag.index_processor.constant.index_type import IndexStructureType
+from fields.dataset_fields import (
+ DatasetDetailPrefetch,
+ DatasetDetailResponse,
+ build_dataset_detail_prefetch,
+ dataset_detail_response_source,
+)
from models.account import Account
-from models.dataset import AppDatasetJoin, Dataset
-from models.model import App, AppMode, IconType
+from models.dataset import (
+ AppDatasetJoin,
+ Dataset,
+ DatasetMetadata,
+ Document,
+ ExternalKnowledgeApis,
+ ExternalKnowledgeBindings,
+ Pipeline,
+)
+from models.enums import DatasetMetadataType, DataSourceType, DocumentCreatedFrom, TagType
+from models.model import App, AppMode, IconType, Tag, TagBinding
def _dataset_detail_payload(**overrides):
@@ -243,3 +263,202 @@ def test_dataset_detail_response_source_uses_caller_session_for_database_fields(
assert response.is_published is False
assert response.total_documents == 0
assert response.total_available_documents == 0
+
+
+def _dataset(index: int, account_id: str, **overrides: object) -> Dataset:
+ dataset = Dataset(
+ id=f"ds-{index}",
+ tenant_id="tenant-1",
+ name=f"Dataset {index}",
+ description="desc",
+ provider="vendor",
+ permission="only_me",
+ data_source_type=None,
+ indexing_technique="economy",
+ created_by=account_id,
+ retrieval_model=None,
+ summary_index_setting=None,
+ built_in_field_enabled=False,
+ icon_info=None,
+ runtime_mode="general",
+ enable_api=False,
+ is_multimodal=False,
+ )
+ for name, value in overrides.items():
+ setattr(dataset, name, value)
+ return dataset
+
+
+def _document(dataset_id: str, *, word_count: int, indexing_status: str, archived: bool = False) -> Document:
+ return Document(
+ tenant_id="tenant-1",
+ dataset_id=dataset_id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch",
+ name="doc",
+ created_from=DocumentCreatedFrom.WEB,
+ created_by="account-1",
+ word_count=word_count,
+ indexing_status=indexing_status,
+ enabled=True,
+ archived=archived,
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ )
+
+
+def _seed_dataset_page(session: Session) -> list[Dataset]:
+ """Create a page of datasets covering every session-scoped detail field."""
+ account = Account(name="Ada", email="[email protected]")
+ account.id = "account-1"
+ session.add(account)
+
+ pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", is_published=True, created_by=account.id)
+ external_api = ExternalKnowledgeApis(
+ name="api",
+ description="",
+ tenant_id="tenant-1",
+ settings=json.dumps({"endpoint": "https://example.com"}),
+ created_by=account.id,
+ updated_by=None,
+ )
+ session.add_all([pipeline, external_api])
+ session.flush()
+
+ plain = _dataset(1, account.id, built_in_field_enabled=True)
+ piped = _dataset(2, account.id, pipeline_id=pipeline.id)
+ external = _dataset(3, account.id, provider="external")
+ structured = _dataset(4, account.id, chunk_structure=IndexStructureType.QA_INDEX)
+ empty = _dataset(5, account.id)
+ datasets = [plain, piped, external, structured, empty]
+ session.add_all(datasets)
+
+ session.add_all(
+ [
+ _document(plain.id, word_count=10, indexing_status="completed"),
+ _document(plain.id, word_count=20, indexing_status="completed", archived=True),
+ _document(plain.id, word_count=5, indexing_status="waiting"),
+ _document(piped.id, word_count=7, indexing_status="completed"),
+ ]
+ )
+
+ tag = Tag(tenant_id="tenant-1", type=TagType.KNOWLEDGE, name="tag", created_by=account.id)
+ other_tag = Tag(tenant_id="tenant-1", type=TagType.APP, name="app tag", created_by=account.id)
+ session.add_all([tag, other_tag])
+ session.flush()
+ session.add_all(
+ [
+ TagBinding(tenant_id="tenant-1", tag_id=tag.id, target_id=plain.id, created_by=account.id),
+ TagBinding(tenant_id="tenant-1", tag_id=other_tag.id, target_id=plain.id, created_by=account.id),
+ ]
+ )
+
+ session.add(
+ DatasetMetadata(
+ tenant_id="tenant-1",
+ dataset_id=plain.id,
+ type=DatasetMetadataType.STRING,
+ name="author",
+ created_by=account.id,
+ )
+ )
+ session.add(
+ ExternalKnowledgeBindings(
+ tenant_id="tenant-1",
+ external_knowledge_api_id=external_api.id,
+ dataset_id=external.id,
+ external_knowledge_id="external-1",
+ created_by=account.id,
+ )
+ )
+
+ app = App(
+ id="app-1",
+ tenant_id="tenant-1",
+ name="App",
+ description="",
+ mode=AppMode.CHAT,
+ icon_type=IconType.EMOJI,
+ icon="app",
+ icon_background="#FFFFFF",
+ enable_site=False,
+ enable_api=False,
+ max_active_requests=0,
+ )
+ session.add_all([app, AppDatasetJoin(app_id=app.id, dataset_id=plain.id)])
+ session.flush()
+ return datasets
+
+
+def _dump(dataset: Dataset, *, session: Session, prefetch: DatasetDetailPrefetch | None = None) -> dict[str, object]:
+ return DatasetDetailResponse.model_validate(
+ dataset_detail_response_source(dataset, session=session, prefetch=prefetch),
+ from_attributes=True,
+ ).model_dump(mode="json")
+
+
+@contextmanager
+def _count_selects(session: Session) -> Iterator[list[str]]:
+ """Record every SELECT issued against the session's engine."""
+ statements: list[str] = []
+ engine = session.get_bind()
+
+ @event.listens_for(engine, "before_cursor_execute")
+ def _before(_conn, _cursor, statement, _parameters, _context, _executemany):
+ if statement.lstrip().upper().startswith("SELECT"):
+ statements.append(statement)
+
+ try:
+ yield statements
+ finally:
+ event.remove(engine, "before_cursor_execute", _before)
+
+
+def test_dataset_detail_prefetch_matches_per_dataset_lookups(sqlite_session: Session):
+ datasets = _seed_dataset_page(sqlite_session)
+
+ expected = [_dump(dataset, session=sqlite_session) for dataset in datasets]
+
+ prefetch = build_dataset_detail_prefetch(datasets, session=sqlite_session)
+ prefetched = [_dump(dataset, session=sqlite_session, prefetch=prefetch) for dataset in datasets]
+
+ assert prefetched == expected
+
+ plain = prefetched[0]
+ assert plain["app_count"] == 1
+ assert plain["document_count"] == 3
+ assert plain["total_documents"] == 3
+ assert plain["total_available_documents"] == 1
+ assert plain["word_count"] == 35
+ assert plain["author_name"] == "Ada"
+ assert [tag["name"] for tag in plain["tags"]] == ["tag"]
+ assert plain["doc_form"] == IndexStructureType.PARAGRAPH_INDEX
+ assert [metadata["name"] for metadata in plain["doc_metadata"]][0] == "author"
+ assert len(plain["doc_metadata"]) == 6
+ assert prefetched[1]["is_published"] is True
+ assert prefetched[2]["external_knowledge_info"]["external_knowledge_api_endpoint"] == "https://example.com"
+ assert prefetched[3]["doc_form"] == IndexStructureType.QA_INDEX
+ assert prefetched[4]["document_count"] == 0
+ assert prefetched[4]["doc_form"] is None
+
+
+def test_dataset_detail_prefetch_keeps_query_count_independent_of_page_size(sqlite_session: Session):
+ account = Account(name="Ada", email="[email protected]")
+ account.id = "account-1"
+ sqlite_session.add(account)
+ datasets = [_dataset(index, account.id) for index in range(20)]
+ sqlite_session.add_all(datasets)
+ sqlite_session.flush()
+
+ def select_count(page: list[Dataset]) -> int:
+ with _count_selects(sqlite_session) as statements:
+ prefetch = build_dataset_detail_prefetch(page, session=sqlite_session)
+ for dataset in page:
+ _dump(dataset, session=sqlite_session, prefetch=prefetch)
+ return len(statements)
+
+ small_page = select_count(datasets[:2])
+ full_page = select_count(datasets)
+
+ assert full_page == small_page
+ assert full_page <= 8