Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): report has_more from the page the query served (#42323)

Lukas Buck38f9d85d5a2b
diff --git a/api/controllers/console/app/annotation.py b/api/controllers/console/app/annotation.py
index bbf343669d2..6f709ed2146 100644
--- a/api/controllers/console/app/annotation.py
+++ b/api/controllers/console/app/annotation.py
@@ -37,6 +37,7 @@ from fields.annotation_fields import (
from fields.base import ResponseModel
from libs.helper import dump_response, uuid_value
from libs.login import current_account_with_tenant, login_required
+from libs.pagination import clamp_pagination
from models.model import App
from services.annotation_service import (
AppAnnotationService,
@@ -301,18 +302,18 @@ class AnnotationApi(Resource):
page = req_data.page
limit = req_data.limit
keyword = req_data.keyword
- effective_limit = min(limit, 100)
+ effective_page, effective_limit = clamp_pagination(page, limit, 100)
annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(
- str(app_id), page, effective_limit, keyword, session
+ str(app_id), effective_page, effective_limit, keyword, session
)
annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
return AnnotationList(
data=annotation_models,
- has_more=page * effective_limit < total,
+ has_more=effective_page * effective_limit < total,
limit=effective_limit,
total=total,
- page=page,
+ page=effective_page,
).model_dump(mode="json"), 200
@console_ns.doc("create_annotation")
@@ -556,12 +557,14 @@ class AnnotationHitHistoryListApi(Resource):
def get(self, session: Session, app_id: UUID, annotation_id: UUID):
page = request.args.get("page", default=1, type=int)
limit = request.args.get("limit", default=20, type=int)
- effective_limit = min(limit, 100)
+ # This route takes both straight off the query string with no bounds, so
+ # clamp them the way the query itself will.
+ effective_page, effective_limit = clamp_pagination(page, limit, 100)
app_ref = _get_app_ref(session, str(app_id))
annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
annotation_hit_history_list, total = AppAnnotationService.get_annotation_hit_histories(
annotation_ref,
- page,
+ effective_page,
effective_limit,
session,
)
@@ -569,5 +572,9 @@ class AnnotationHitHistoryListApi(Resource):
annotation_hit_history_list, from_attributes=True
)
return AnnotationHitHistoryList(
- data=history_models, has_more=page * effective_limit < total, limit=effective_limit, total=total, page=page
+ data=history_models,
+ has_more=effective_page * effective_limit < total,
+ limit=effective_limit,
+ total=total,
+ page=effective_page,
).model_dump(mode="json")
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
index ed8c7280446..0b6986b1972 100644
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -50,6 +50,7 @@ from fields.dataset_fields import (
from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import build_icon_url, dump_response, to_timestamp
from libs.login import login_required
+from libs.pagination import clamp_pagination
from libs.url_utils import normalize_api_base_url
from models import Account, ApiToken, App, Dataset, Document, UploadFile
from models.dataset import DatasetPermission, DatasetPermissionEnum, DatasetQuery
@@ -504,6 +505,7 @@ class DatasetListApi(Resource):
if filtered_dataset_ids is not None:
accessible_dataset_ids = sorted(filtered_dataset_ids)
+ effective_page, effective_limit = clamp_pagination(query.page, query.limit, 100)
if query.ids:
datasets, total = DatasetService.get_datasets_by_ids(
query.ids,
@@ -513,10 +515,13 @@ class DatasetListApi(Resource):
include_own_datasets=include_own_datasets,
session=session,
)
+ # This branch resolves the ids it was handed in a single page
+ # (`per_page=len(ids)`), so there is never a next one to ask for.
+ has_more = False
else:
datasets, total = DatasetService.get_datasets(
- query.page,
- query.limit,
+ effective_page,
+ effective_limit,
session,
current_tenant_id,
current_user,
@@ -526,6 +531,7 @@ class DatasetListApi(Resource):
accessible_dataset_ids=accessible_dataset_ids,
include_own_datasets=include_own_datasets,
)
+ has_more = effective_page * effective_limit < total
permission_keys_map = {}
if datasets:
@@ -581,10 +587,10 @@ class DatasetListApi(Resource):
response = {
"data": data,
- "has_more": len(datasets) == query.limit,
- "limit": query.limit,
+ "has_more": has_more,
+ "limit": effective_limit,
"total": total,
- "page": query.page,
+ "page": effective_page,
}
return dump_response(DatasetListResponse, response), 200
@@ -874,16 +880,17 @@ class DatasetQueryApi(Resource):
page = request.args.get("page", default=1, type=int)
limit = request.args.get("limit", default=20, type=int)
+ effective_page, effective_limit = clamp_pagination(page, limit, 100)
dataset_queries, total = DatasetService.get_dataset_queries(
- dataset_id=dataset.id, page=page, per_page=limit, session=session
+ dataset_id=dataset.id, page=effective_page, per_page=effective_limit, session=session
)
response = {
"data": [_DatasetQueryResponseSource(query=query, session=session) for query in dataset_queries],
- "has_more": len(dataset_queries) == limit,
- "limit": limit,
+ "has_more": effective_page * effective_limit < total,
+ "limit": effective_limit,
"total": total,
- "page": page,
+ "page": effective_page,
}
return dump_response(DatasetQueryListResponse, response), 200
diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py
index 2f34de91f2e..0d69e9ce754 100644
--- a/api/controllers/console/datasets/datasets_document.py
+++ b/api/controllers/console/datasets/datasets_document.py
@@ -519,10 +519,13 @@ class DatasetDocumentListApi(Resource):
document.total_segments = total_segments
response = {
"data": document_with_segments_responses(documents, session=session),
- "has_more": len(documents) == limit,
- "limit": limit,
+ # The result object already knows: it was built from the page the query
+ # ran with, where `len(documents) == limit` is only ever a guess that a
+ # full page means another one follows.
+ "has_more": paginated_documents.has_next,
+ "limit": paginated_documents.per_page,
"total": paginated_documents.total,
- "page": page,
+ "page": paginated_documents.page,
}
return dump_response(DocumentWithSegmentsListResponse, response)
diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.py
index 371b9d537c0..fd93100d9fc 100644
--- a/api/controllers/console/datasets/external.py
+++ b/api/controllers/console/datasets/external.py
@@ -35,6 +35,7 @@ from fields.base import ResponseModel
from fields.dataset_fields import DatasetDetailResponse, dataset_detail_response_source
from libs.helper import dump_response
from libs.login import login_required
+from libs.pagination import clamp_pagination
from models import Account
from models.dataset import ExternalKnowledgeApis
from services.dataset_service import DatasetService
@@ -168,15 +169,16 @@ class ExternalApiTemplateListApi(Resource):
@model_validate(ExternalApiTemplateListQuery)
def get(self, req_data: ExternalApiTemplateListQuery, session: Session, current_tenant_id: str):
+ effective_page, effective_limit = clamp_pagination(req_data.page, req_data.limit, 100)
external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
- req_data.page, req_data.limit, current_tenant_id, req_data.keyword, session=session
+ effective_page, effective_limit, current_tenant_id, req_data.keyword, session=session
)
return ExternalKnowledgeApiListResponse(
data=[external_knowledge_api_response(item, session=session) for item in external_knowledge_apis],
- has_more=len(external_knowledge_apis) == req_data.limit,
- limit=req_data.limit,
+ has_more=effective_page * effective_limit < total,
+ limit=effective_limit,
total=total,
- page=req_data.page,
+ page=effective_page,
).model_dump(mode="json"), 200
@console_ns.doc("create_external_api_template")
diff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.py
index e481c80b756..f3eae387503 100644
--- a/api/controllers/console/explore/trial.py
+++ b/api/controllers/console/explore/trial.py
@@ -915,7 +915,10 @@ class DatasetListApi(Resource):
else:
raise NeedAddIdsError()
- response = {"data": datasets, "has_more": len(datasets) == limit, "limit": limit, "total": total, "page": page}
+ # `get_datasets_by_ids` resolves the ids it was handed in a single page
+ # (`per_page=len(ids)`), so `limit` never bounded this result and there is
+ # never a next page to ask for.
+ response = {"data": datasets, "has_more": False, "limit": limit, "total": total, "page": page}
return dump_response(TrialDatasetListResponse, response)
diff --git a/api/controllers/service_api/app/annotation.py b/api/controllers/service_api/app/annotation.py
index 2f954dba422..0d431754189 100644
--- a/api/controllers/service_api/app/annotation.py
+++ b/api/controllers/service_api/app/annotation.py
@@ -19,6 +19,7 @@ from fields.annotation_fields import (
AnnotationList,
)
from libs.helper import dump_response
+from libs.pagination import clamp_pagination
from models.model import App
from services.annotation_service import (
AppAnnotationService,
@@ -209,17 +210,17 @@ class AnnotationListApi(Resource):
def get(self, query: AnnotationListQuery, session: Session, app_model: App):
"""List annotations for the application."""
- effective_limit = min(query.limit, 100)
+ effective_page, effective_limit = clamp_pagination(query.page, query.limit, 100)
annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(
- app_model.id, query.page, effective_limit, query.keyword, session
+ app_model.id, effective_page, effective_limit, query.keyword, session
)
annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
return AnnotationList(
data=annotation_models,
- has_more=query.page * effective_limit < total,
+ has_more=effective_page * effective_limit < total,
limit=effective_limit,
total=total,
- page=query.page,
+ page=effective_page,
).model_dump(mode="json")
@service_api_ns.doc(
diff --git a/api/libs/pagination.py b/api/libs/pagination.py
index c6220fbc13e..e417b2ee804 100644
--- a/api/libs/pagination.py
+++ b/api/libs/pagination.py
@@ -35,6 +35,20 @@ class PaginatedResult[T]:
return iter(self.items)
+def clamp_pagination(page: int, per_page: int, max_per_page: int | None = None) -> tuple[int, int]:
+ """Return the ``(page, per_page)`` a paginated query will actually use.
+
+ Callers that report ``has_more`` next to a page of rows have to derive it
+ from the same numbers the query ran with. Computing it from the requested
+ values instead lets a request for ``limit=0`` be served one row while the
+ response claims a page size of zero, which makes ``page * limit < total``
+ true for every page and gives the client a pager that never ends.
+ """
+ if max_per_page is not None:
+ per_page = min(per_page, max_per_page)
+ return max(1, page), max(1, per_page)
+
+
def paginate_query(
stmt: Select,
*,
@@ -58,11 +72,7 @@ def paginate_query(
session:
SQLAlchemy session used to execute the count and page queries.
"""
- if max_per_page is not None:
- per_page = min(per_page, max_per_page)
-
- page = max(1, page)
- per_page = max(1, per_page)
+ page, per_page = clamp_pagination(page, per_page, max_per_page)
# total count — wrap in a scalar subquery so arbitrary selects work
count_stmt = select(func.count()).select_from(stmt.subquery())
diff --git a/api/services/external_knowledge_service.py b/api/services/external_knowledge_service.py
index 03fe274fe89..76620c48f75 100644
--- a/api/services/external_knowledge_service.py
+++ b/api/services/external_knowledge_service.py
@@ -33,7 +33,7 @@ class ExternalDatasetService:
@staticmethod
def get_external_knowledge_apis(
page, per_page, tenant_id, search=None, *, session: Session
- ) -> tuple[list[ExternalKnowledgeApis], int | None]:
+ ) -> tuple[list[ExternalKnowledgeApis], int]:
query = (
select(ExternalKnowledgeApis)
.where(ExternalKnowledgeApis.tenant_id == tenant_id)
diff --git a/api/tests/unit_tests/controllers/console/app/test_annotation_api.py b/api/tests/unit_tests/controllers/console/app/test_annotation_api.py
index b8e5861882a..38ae25284e1 100644
--- a/api/tests/unit_tests/controllers/console/app/test_annotation_api.py
+++ b/api/tests/unit_tests/controllers/console/app/test_annotation_api.py
@@ -264,6 +264,57 @@ class TestConsoleAnnotationRefBoundaries:
AnnotationRef(AppRef("tenant-1", "app-1"), "ann-1"), 2, 5, sqlite_session
)
+ def test_hit_history_pager_ends_for_a_non_positive_limit(self, app: Flask, sqlite_session: Session):
+ """This route reads page and limit straight off the query string with no bounds.
+
+ `paginate_query` floors both at 1, so `limit=0` is served one row. Reporting
+ `has_more` from the requested 0 made `page * limit < total` true on every page,
+ including the pages past the end that return nothing, so a client walking until
+ `has_more` is false never stopped.
+ """
+ api = annotation_module.AnnotationHitHistoryListApi()
+ handler = unwrap(api.get)
+ hit_history_mock = Mock(return_value=([_annotation_hit_history()], 1))
+ _persist_app(sqlite_session)
+ with (
+ app.test_request_context("/hit-histories?page=1&limit=0", method="GET"),
+ patch.object(
+ annotation_module,
+ "current_account_with_tenant",
+ return_value=(_account(), "tenant-1"),
+ ),
+ patch.object(annotation_module.AppAnnotationService, "get_annotation_hit_histories", hit_history_mock),
+ ):
+ response = handler(api, sqlite_session, "app-1", "ann-1")
+ assert response["limit"] == 1
+ assert response["page"] == 1
+ assert response["has_more"] is False
+ hit_history_mock.assert_called_once_with(
+ AnnotationRef(AppRef("tenant-1", "app-1"), "ann-1"), 1, 1, sqlite_session
+ )
+
+ def test_hit_history_clamps_a_non_positive_page(self, app: Flask, sqlite_session: Session):
+ """A page below 1 is served as page 1, so the response has to say page 1."""
+ api = annotation_module.AnnotationHitHistoryListApi()
+ handler = unwrap(api.get)
+ hit_history_mock = Mock(return_value=([_annotation_hit_history()], 9))
+ _persist_app(sqlite_session)
+ with (
+ app.test_request_context("/hit-histories?page=0&limit=5", method="GET"),
+ patch.object(
+ annotation_module,
+ "current_account_with_tenant",
+ return_value=(_account(), "tenant-1"),
+ ),
+ patch.object(annotation_module.AppAnnotationService, "get_annotation_hit_histories", hit_history_mock),
+ ):
+ response = handler(api, sqlite_session, "app-1", "ann-1")
+ assert response["page"] == 1
+ assert response["has_more"] is True
+ hit_history_mock.assert_called_once_with(
+ AnnotationRef(AppRef("tenant-1", "app-1"), "ann-1"), 1, 5, sqlite_session
+ )
+
class TestAnnotationListHasMore:
def test_list_annotations_has_more_false_on_last_page_exact_limit(
diff --git a/api/tests/unit_tests/libs/test_pagination.py b/api/tests/unit_tests/libs/test_pagination.py
index 0034ca20fa8..af6286dcc05 100644
--- a/api/tests/unit_tests/libs/test_pagination.py
+++ b/api/tests/unit_tests/libs/test_pagination.py
@@ -2,7 +2,7 @@ import math
import pytest
-from libs.pagination import PaginatedResult
+from libs.pagination import PaginatedResult, clamp_pagination
class TestPaginatedResultPages:
@@ -50,3 +50,53 @@ def test_paginated_result_is_iterable():
items = [1, 2, 3]
result = PaginatedResult(items=items, total=3, page=1, per_page=20)
assert list(result) == items
+
+
+class TestClampPagination:
+ """The numbers a caller reports `has_more` with must be the ones the query ran with.
+
+ `paginate_query` floors both at 1. A caller that kept the requested values instead
+ could report a page size the query never used, and `page * per_page < total` is then
+ true for every page, which is a pager with no end (#41876).
+ """
+
+ @pytest.mark.parametrize("per_page", [0, -1, -100])
+ def test_per_page_below_one_is_floored(self, per_page):
+ assert clamp_pagination(1, per_page) == (1, 1)
+
+ @pytest.mark.parametrize("page", [0, -1, -100])
+ def test_page_below_one_is_floored(self, page):
+ assert clamp_pagination(page, 20) == (1, 20)
+
+ def test_max_per_page_caps_before_the_floor(self):
+ assert clamp_pagination(3, 500, 100) == (3, 100)
+
+ def test_the_floor_wins_over_a_cap_below_one(self):
+ """Whatever the cap says, a page size of zero is the input that breaks the caller."""
+ assert clamp_pagination(1, 20, 0) == (1, 1)
+
+ def test_no_cap_leaves_a_large_per_page_alone(self):
+ assert clamp_pagination(3, 500) == (3, 500)
+
+ def test_valid_values_pass_through(self):
+ assert clamp_pagination(2, 20, 100) == (2, 20)
+
+ def test_a_full_last_page_is_still_the_last_page(self):
+ """The other guess this replaces: `len(items) == per_page`.
+
+ Forty rows at twenty per page means page two is exactly full and also last.
+ Reading fullness as evidence of a next page costs the caller a round trip that
+ comes back empty.
+ """
+ page, size = clamp_pagination(2, 20, 100)
+ result = PaginatedResult(items=[object()] * size, total=40, page=page, per_page=size)
+ assert len(result.items) == size
+ assert (page * size < 40) is result.has_next is False
+
+ @pytest.mark.parametrize("per_page", [0, -1])
+ def test_clamped_values_agree_with_has_next(self, per_page):
+ """What the helper returns has to make the caller's arithmetic match the truth."""
+ total = 7
+ page, size = clamp_pagination(total, per_page, 100)
+ result = PaginatedResult(items=[object()], total=total, page=page, per_page=size)
+ assert (page * size < total) is result.has_next is False