Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): report Service API knowledge list has_more from the page the query served (#42531)

Jeremy Schoemakerad9b85932b52
diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py
index 1989cb0b427..653aaa97197 100644
--- a/api/controllers/service_api/dataset/dataset.py
+++ b/api/controllers/service_api/dataset/dataset.py
@@ -44,6 +44,7 @@ from fields.dataset_fields import DatasetDetailResponse as BaseDatasetDetailResp
from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import dump_response
from libs.login import current_user
+from libs.pagination import clamp_pagination
from models.account import Account
from models.dataset import DatasetPermissionEnum
from models.enums import TagType
@@ -441,10 +442,10 @@ class DatasetListApi(DatasetApiResource):
query_params["tag_ids"] = request.args.getlist("tag_ids")
query = DatasetListQuery.model_validate(query_params)
# provider = request.args.get("provider", default="vendor")
- effective_limit = min(query.limit, 100)
+ effective_page, effective_limit = clamp_pagination(query.page, query.limit, 100)
datasets, total = DatasetService.get_datasets(
- query.page,
+ effective_page,
effective_limit,
session,
tenant_id,
@@ -480,10 +481,10 @@ class DatasetListApi(DatasetApiResource):
item["embedding_available"] = True
response = {
"data": data,
- "has_more": query.page * effective_limit < total,
+ "has_more": effective_page * effective_limit < total,
"limit": effective_limit,
"total": total,
- "page": query.page,
+ "page": effective_page,
}
return _dump_service_dataset_list(response), 200
diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py
index b18bb485971..50e5ebb3308 100644
--- a/api/controllers/service_api/dataset/document.py
+++ b/api/controllers/service_api/dataset/document.py
@@ -75,7 +75,7 @@ from fields.document_fields import (
)
from libs.helper import dump_response
from libs.login import current_user
-from libs.pagination import paginate_query
+from libs.pagination import clamp_pagination, paginate_query
from models.dataset import Dataset, Document
from services.dataset_service import DatasetService, DocumentService
from services.entities.knowledge_entities.knowledge_entities import (
@@ -1032,9 +1032,9 @@ class DocumentListApi(DatasetApiResource):
query = query.order_by(desc(Document.created_at), desc(Document.position))
- effective_limit = min(query_params.limit, 100)
+ effective_page, effective_limit = clamp_pagination(query_params.page, query_params.limit, 100)
paginated_documents = paginate_query(
- query, session=session, page=query_params.page, per_page=effective_limit, max_per_page=100
+ query, session=session, page=effective_page, per_page=effective_limit, max_per_page=100
)
documents = paginated_documents.items
@@ -1047,10 +1047,12 @@ class DocumentListApi(DatasetApiResource):
response = {
"data": document_responses(documents, session=session),
- "has_more": query_params.page * effective_limit < paginated_documents.total,
- "limit": effective_limit,
+ # The result object already knows: it was built from the page the query
+ # ran with, while the requested values are only ever a request.
+ "has_more": paginated_documents.has_next,
+ "limit": paginated_documents.per_page,
"total": paginated_documents.total,
- "page": query_params.page,
+ "page": paginated_documents.page,
}
return dump_response(DocumentListResponse, response)
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
index 8b3c7d42c79..89da1671549 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py
@@ -317,6 +317,119 @@ class TestDatasetListApiGet:
assert response["page"] == 1
assert mock_dataset_svc.get_datasets.call_args.args[1] == 100
+ @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager")
+ @patch("controllers.service_api.dataset.dataset.DatasetService")
+ def test_list_datasets_reports_the_page_size_the_query_used_for_limit_zero(
+ self,
+ mock_dataset_svc: MagicMock,
+ mock_provider_mgr: MagicMock,
+ app: Flask,
+ account: Account,
+ tenant: Tenant,
+ controller_session: Session,
+ ) -> None:
+ """limit=0 is floored to one row by the query, so the response must report one."""
+ from controllers.service_api.dataset.dataset import DatasetListApi
+
+ total = 7
+ dataset = make_dataset(controller_session, tenant, account)
+ mock_dataset_svc.get_datasets.return_value = ([dataset], total)
+ mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]()
+
+ with app.test_request_context("/datasets?page=1&limit=0", method="GET"):
+ api = DatasetListApi()
+ response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id)
+
+ assert status == 200
+ assert response["limit"] == 1
+ assert response["page"] == 1
+ assert response["has_more"] is True
+ assert mock_dataset_svc.get_datasets.call_args.args[:2] == (1, 1)
+
+ @pytest.mark.usefixtures("account")
+ @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager")
+ @patch("controllers.service_api.dataset.dataset.DatasetService")
+ def test_list_datasets_has_more_false_past_the_last_page_for_limit_zero(
+ self,
+ mock_dataset_svc: MagicMock,
+ mock_provider_mgr: MagicMock,
+ app: Flask,
+ tenant: Tenant,
+ controller_session: Session,
+ ) -> None:
+ """A page past the end must end the walk instead of reporting more rows forever."""
+ from controllers.service_api.dataset.dataset import DatasetListApi
+
+ total = 7
+ mock_dataset_svc.get_datasets.return_value = (list[Dataset](), total)
+ mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]()
+
+ with app.test_request_context("/datasets?page=8&limit=0", method="GET"):
+ api = DatasetListApi()
+ response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id)
+
+ assert status == 200
+ assert response["has_more"] is False
+ assert response["limit"] == 1
+ assert response["page"] == 8
+ assert response["data"] == []
+
+ @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager")
+ @patch("controllers.service_api.dataset.dataset.DatasetService")
+ def test_list_datasets_reports_the_page_size_the_query_used_for_negative_limit(
+ self,
+ mock_dataset_svc: MagicMock,
+ mock_provider_mgr: MagicMock,
+ app: Flask,
+ account: Account,
+ tenant: Tenant,
+ controller_session: Session,
+ ) -> None:
+ """A negative limit is floored the same way, and is never echoed back."""
+ from controllers.service_api.dataset.dataset import DatasetListApi
+
+ total = 7
+ dataset = make_dataset(controller_session, tenant, account)
+ mock_dataset_svc.get_datasets.return_value = ([dataset], total)
+ mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]()
+
+ with app.test_request_context("/datasets?page=1&limit=-1", method="GET"):
+ api = DatasetListApi()
+ response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id)
+
+ assert status == 200
+ assert response["limit"] == 1
+ assert response["page"] == 1
+ assert mock_dataset_svc.get_datasets.call_args.args[:2] == (1, 1)
+
+ @patch("controllers.service_api.dataset.dataset.create_plugin_provider_manager")
+ @patch("controllers.service_api.dataset.dataset.DatasetService")
+ def test_list_datasets_reports_the_page_the_query_used_for_page_zero(
+ self,
+ mock_dataset_svc: MagicMock,
+ mock_provider_mgr: MagicMock,
+ app: Flask,
+ account: Account,
+ tenant: Tenant,
+ controller_session: Session,
+ ) -> None:
+ """page=0 is served as page 1, so the response must not offer page 1 as the next one."""
+ from controllers.service_api.dataset.dataset import DatasetListApi
+
+ total = 7
+ dataset = make_dataset(controller_session, tenant, account)
+ mock_dataset_svc.get_datasets.return_value = ([dataset] * total, total)
+ mock_provider_mgr.return_value.get_configurations.return_value.get_models.return_value = list[object]()
+
+ with app.test_request_context("/datasets?page=0&limit=20", method="GET"):
+ api = DatasetListApi()
+ response, status = unwrap(api.get)(api, controller_session, tenant_id=tenant.id)
+
+ assert status == 200
+ assert response["page"] == 1
+ assert response["has_more"] is False
+ assert mock_dataset_svc.get_datasets.call_args.args[:2] == (1, 20)
+
class TestDatasetListApiPost:
"""Test suite for DatasetListApi.post() endpoint."""
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
index 8ea26e001f8..665d4cb0607 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/test_document.py
@@ -19,7 +19,6 @@ import inspect
import json
import uuid
from collections.abc import Iterator
-from dataclasses import dataclass
from datetime import UTC, datetime
from unittest.mock import Mock, PropertyMock, patch
@@ -49,6 +48,7 @@ from controllers.service_api.dataset.document import (
from controllers.service_api.dataset.error import ArchivedDocumentImmutableError
from core.rag.index_processor.constant.index_type import IndexStructureType
from enums import DeploymentEdition
+from libs.pagination import PaginatedResult, clamp_pagination
from models.account import Account
from models.dataset import Dataset, Document, DocumentSegment
from models.enums import (
@@ -95,10 +95,21 @@ def _unwrap_non_wrapped_controller(view):
return inspect.unwrap(view)
-@dataclass
-class _PaginationRecord:
- items: list[Document]
- total: int
+def _pagination_record(
+ items: list[Document],
+ total: int,
+ *,
+ page: int = 1,
+ per_page: int = 20,
+) -> PaginatedResult:
+ """Build the result `paginate_query` would return for these request values.
+
+ The clamp is part of that contract: the query floors both numbers at 1 and
+ caps the page size at 100 before it runs, and the handler reports what it ran
+ with, so the stand-in has to carry the same numbers.
+ """
+ page, per_page = clamp_pagination(page, per_page, 100)
+ return PaginatedResult(items=items, total=total, page=page, per_page=per_page)
@pytest.fixture
@@ -1155,7 +1166,7 @@ class TestDocumentListApi(SQLiteControllerTest):
id="doc-2", name="Document 2", tenant_id=mock_tenant, dataset_id=mock_dataset.id
),
]
- mock_paginate.return_value = _PaginationRecord(items=documents, total=2)
+ mock_paginate.return_value = _pagination_record(items=documents, total=2)
mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
@@ -1197,7 +1208,7 @@ class TestDocumentListApi(SQLiteControllerTest):
)
for index in range(page_size)
]
- mock_paginate.return_value = _PaginationRecord(items=documents, total=page_size)
+ mock_paginate.return_value = _pagination_record(items=documents, total=page_size)
mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
with app.test_request_context(
@@ -1232,7 +1243,7 @@ class TestDocumentListApi(SQLiteControllerTest):
)
for index in range(returned_count)
]
- mock_paginate.return_value = _PaginationRecord(items=documents, total=total)
+ mock_paginate.return_value = _pagination_record(items=documents, total=total, per_page=200)
mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
with app.test_request_context(
@@ -1250,6 +1261,116 @@ class TestDocumentListApi(SQLiteControllerTest):
assert response["page"] == 1
assert mock_paginate.call_args.kwargs["per_page"] == 100
+ @patch("controllers.service_api.dataset.document.paginate_query")
+ @patch("controllers.service_api.dataset.document.DocumentService")
+ def test_list_documents_reports_the_page_size_the_query_used_for_limit_zero(
+ self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset
+ ):
+ """limit=0 is floored to one row by the query, so the response must report one."""
+ self._persist_dataset(mock_dataset)
+ total = 7
+ documents = [
+ make_serializable_document(id="doc-1", name="Document 1", tenant_id=mock_tenant, dataset_id=mock_dataset.id)
+ ]
+ mock_paginate.return_value = _pagination_record(items=documents, total=total, page=1, per_page=0)
+ mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
+
+ with app.test_request_context(
+ f"/datasets/{mock_dataset.id}/documents?page=1&limit=0",
+ method="GET",
+ ):
+ api = DocumentListApi()
+ response = inspect.unwrap(type(api).get)(
+ api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id
+ )
+
+ assert response["limit"] == 1
+ assert response["page"] == 1
+ assert response["has_more"] is True
+ assert mock_paginate.call_args.kwargs["per_page"] == 1
+
+ @patch("controllers.service_api.dataset.document.paginate_query")
+ @patch("controllers.service_api.dataset.document.DocumentService")
+ def test_list_documents_has_more_false_past_the_last_page_for_limit_zero(
+ self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset
+ ):
+ """A page past the end must end the walk instead of reporting more rows forever."""
+ self._persist_dataset(mock_dataset)
+ total = 7
+ mock_paginate.return_value = _pagination_record(items=[], total=total, page=8, per_page=0)
+ mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
+
+ with app.test_request_context(
+ f"/datasets/{mock_dataset.id}/documents?page=8&limit=0",
+ method="GET",
+ ):
+ api = DocumentListApi()
+ response = inspect.unwrap(type(api).get)(
+ api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id
+ )
+
+ assert response["has_more"] is False
+ assert response["limit"] == 1
+ assert response["page"] == 8
+ assert response["data"] == []
+
+ @patch("controllers.service_api.dataset.document.paginate_query")
+ @patch("controllers.service_api.dataset.document.DocumentService")
+ def test_list_documents_reports_the_page_size_the_query_used_for_negative_limit(
+ self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset
+ ):
+ """A negative limit is floored the same way, and is never echoed back."""
+ self._persist_dataset(mock_dataset)
+ total = 7
+ documents = [
+ make_serializable_document(id="doc-1", name="Document 1", tenant_id=mock_tenant, dataset_id=mock_dataset.id)
+ ]
+ mock_paginate.return_value = _pagination_record(items=documents, total=total, page=1, per_page=-1)
+ mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
+
+ with app.test_request_context(
+ f"/datasets/{mock_dataset.id}/documents?page=1&limit=-1",
+ method="GET",
+ ):
+ api = DocumentListApi()
+ response = inspect.unwrap(type(api).get)(
+ api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id
+ )
+
+ assert response["limit"] == 1
+ assert response["page"] == 1
+ assert mock_paginate.call_args.kwargs["per_page"] == 1
+
+ @patch("controllers.service_api.dataset.document.paginate_query")
+ @patch("controllers.service_api.dataset.document.DocumentService")
+ def test_list_documents_reports_the_page_the_query_used_for_page_zero(
+ self, mock_doc_svc, mock_paginate, app: Flask, mock_tenant, mock_dataset
+ ):
+ """page=0 is served as page 1, so the response must not offer page 1 as the next one."""
+ self._persist_dataset(mock_dataset)
+ total = 7
+ documents = [
+ make_serializable_document(
+ id=f"doc-{index}", name=f"Document {index}", tenant_id=mock_tenant, dataset_id=mock_dataset.id
+ )
+ for index in range(total)
+ ]
+ mock_paginate.return_value = _pagination_record(items=documents, total=total, page=0, per_page=20)
+ mock_doc_svc.enrich_documents_with_summary_index_status.return_value = None
+
+ with app.test_request_context(
+ f"/datasets/{mock_dataset.id}/documents?page=0&limit=20",
+ method="GET",
+ ):
+ api = DocumentListApi()
+ response = inspect.unwrap(type(api).get)(
+ api, self.session, tenant_id=mock_tenant, dataset_id=mock_dataset.id
+ )
+
+ assert response["page"] == 1
+ assert response["has_more"] is False
+ assert mock_paginate.call_args.kwargs["page"] == 1
+
def test_list_documents_dataset_not_found(self, app: Flask, mock_tenant, mock_dataset):
"""Test 404 when dataset not found."""
# Arrange