Self hosted FLOSS fitness/workout, nutrition and weight tracker
Use cursor-based pagination for the ingredient overview
Roland Geiderfbe554c01c82
diff --git a/wger/core/templates/tags/pagination.html b/wger/core/templates/tags/pagination.htmldeleted file mode 100644index d272b2de1..000000000--- a/wger/core/templates/tags/pagination.html+++ /dev/null@@ -1,42 +0,0 @@-{% load i18n %}--<ul class="pagination justify-content-center">- {% if page.has_previous %}- <li class="page-item">- <a href="?page={{ page.previous_page_number }}" class="page-link">- « {% translate "previous" %}- </a>- </li>- {% else %}- <li class="page-item disabled">- <a href="#" class="page-link">« {% translate "previous" %}</a>- </li>- {% endif %}-- {% for i in page_range %}- {% if page.number != i %}- <li class="page-item">- <a href="?page={{ i }}" class="page-link">- {{ i }}- </a>- </li>- {% else %}- <li class="page-item active">- <a href="#" class="page-link">{{ i }}</a>- </li>- {% endif %}- {% endfor %}-- {% if page.has_next %}- <li class="page-item">- <a href="?page={{ page.next_page_number }}" class="page-link">- {% trans "next" %} »- </a>- </li>- {% else %}- <li class="page-item disabled">- <a href="#" class="page-link">{% trans "next" %} »</a>- </li>- {% endif %}-</ul>-diff --git a/wger/core/templatetags/wger_extras.py b/wger/core/templatetags/wger_extras.pyindex 592111c1e..aa7361b6b 100644--- a/wger/core/templatetags/wger_extras.py+++ b/wger/core/templatetags/wger_extras.py@@ -28,46 +28,12 @@ from django.utils.translation import (# wgerfrom wger.core.tests.base_testcase import get_reverse-from wger.utils.constants import (- PAGINATION_MAX_TOTAL_PAGES,- PAGINATION_PAGES_AROUND_CURRENT,-)from wger.utils.language import get_language_dataregister = template.Library()[email protected]_tag('tags/pagination.html')-def pagination(paginator, page):- """- Renders the necessary links to paginating a long list- """-- # For very long lists (e.g. the English ingredient with more than 8000 items)- # we muck around here to remove the pages not inmediately 'around' the current- # one, otherwise we end up with a useless block with 300 pages.- if paginator.num_pages > PAGINATION_MAX_TOTAL_PAGES:- start_page = page.number - PAGINATION_PAGES_AROUND_CURRENT- for i in range(page.number - PAGINATION_PAGES_AROUND_CURRENT, page.number + 1):- if i > 0:- start_page = i- break-- end_page = page.number + PAGINATION_PAGES_AROUND_CURRENT- for i in range(page.number, page.number + PAGINATION_PAGES_AROUND_CURRENT):- if i > paginator.num_pages:- end_page = i- break-- page_range = range(start_page, end_page)- else:- page_range = paginator.page_range-- # Set the template variables- return {'page': page, 'page_range': page_range}--@register.inclusion_tag('tags/language_select.html', takes_context=True)def language_select(context, language):"""diff --git a/wger/nutrition/templates/ingredient/overview.html b/wger/nutrition/templates/ingredient/overview.htmlindex ed054588f..a7db96f1d 100644--- a/wger/nutrition/templates/ingredient/overview.html+++ b/wger/nutrition/templates/ingredient/overview.html@@ -47,7 +47,28 @@<div style="padding-top:3em;"></div>- {% pagination paginator page_obj %}+ <nav aria-label="{% translate 'Ingredient pagination' %}">+ <ul class="pagination justify-content-center">+ {% if is_paginated %}+ <li class="page-item">+ <a href="{{ first_url }}" class="page-link">+ « {% translate "Back to start" %}+ </a>+ </li>+ {% endif %}+ {% if has_next %}+ <li class="page-item">+ <a href="{{ next_url }}" class="page-link">+ {% translate "next" %} »+ </a>+ </li>+ {% else %}+ <li class="page-item disabled">+ <a href="#" class="page-link">{% translate "next" %} »</a>+ </li>+ {% endif %}+ </ul>+ </nav>{% endblock %}{% block sidebar %}diff --git a/wger/nutrition/tests/test_ingredient_overview.py b/wger/nutrition/tests/test_ingredient_overview.pyindex 8f6193145..25c6b0838 100644--- a/wger/nutrition/tests/test_ingredient_overview.py+++ b/wger/nutrition/tests/test_ingredient_overview.py@@ -26,7 +26,10 @@ class IngredientOverviewTestCase(WgerTestCase):Tests the ingredient overview"""- def test_overview(self):+ def test_overview_cursor_pagination(self):+ """+ The overview uses cursor pagination via ?after=<id>.+ """# Add more ingredients so we can test the paginationself.user_login('admin')data = {@@ -46,32 +49,81 @@ class IngredientOverviewTestCase(WgerTestCase):for i in range(0, 50):self.client.post(reverse('nutrition:ingredient:add'), data)- # Page existsself.user_logout()- response = self.client.get(reverse('nutrition:ingredient:list'))+ url = reverse('nutrition:ingredient:list')++ # First page: full PAGE_SIZE, has_next, no is_paginated marker yet.+ response = self.client.get(url)self.assertEqual(response.status_code, 200)self.assertEqual(len(response.context['ingredients_list']), PAGINATION_OBJECTS_PER_PAGE)-- response = self.client.get(reverse('nutrition:ingredient:list'), {'page': 2})+ self.assertTrue(response.context['has_next'])+ self.assertFalse(response.context['is_paginated'])+ self.assertIsNotNone(response.context['next_url'])+ self.assertIn('after=', response.context['next_url'])++ # Walk the catalogue forward until exhausted by following next_url.+ seen_ids = []+ page_count = 0+ next_url = url+ while True:+ response = self.client.get(next_url)+ self.assertEqual(response.status_code, 200)+ seen_ids.extend(i.id for i in response.context['ingredients_list'])+ page_count += 1+ if not response.context['has_next']:+ break+ # next_url is relative ('?after=N&...'); join with the list path.+ next_url = url + response.context['next_url']++ # Every ingredient appears exactly once across the walk.+ self.assertEqual(len(seen_ids), len(set(seen_ids)))++ # Pages are at most PAGE_SIZE; with 50 created + the fixture rows we+ # should have at least 3 pages.+ self.assertGreaterEqual(page_count, 3)++ # An invalid cursor falls back to the first page (no 500, no 404).+ response = self.client.get(url, {'after': 'foobar'})self.assertEqual(response.status_code, 200)self.assertEqual(len(response.context['ingredients_list']), PAGINATION_OBJECTS_PER_PAGE)- rest_ingredients = 11- response = self.client.get(reverse('nutrition:ingredient:list'), {'page': 3})+ # A cursor past the end yields an empty page with no next link.+ response = self.client.get(url, {'after': 999_999_999})self.assertEqual(response.status_code, 200)- self.assertEqual(len(response.context['ingredients_list']), rest_ingredients)+ self.assertEqual(response.context['ingredients_list'], [])+ self.assertFalse(response.context['has_next'])+ self.assertIsNone(response.context['next_url'])- # 'last' is a special case- response = self.client.get(reverse('nutrition:ingredient:list'), {'page': 'last'})- self.assertEqual(response.status_code, 200)- self.assertEqual(len(response.context['ingredients_list']), rest_ingredients)+ def test_next_url_preserves_filter_params(self):+ """The next_url must keep filter parameters like ?is_vegan=1 attached."""+ self.user_login('admin')+ for i in range(0, 30):+ self.client.post(+ reverse('nutrition:ingredient:add'),+ {+ 'name': 'Vegan ingredient',+ 'language': 2,+ 'sodium': 10.54,+ 'energy': 176,+ 'fat': 8.19,+ 'carbohydrates_sugar': 0.0,+ 'fat_saturated': 3.24,+ 'fiber': 0.0,+ 'protein': 25.63,+ 'carbohydrates': 0.0,+ 'license': 1,+ 'license_author': 'internet',+ 'is_vegan': 'on',+ },+ )- # Page does not exist- response = self.client.get(reverse('nutrition:ingredient:list'), {'page': 100})- self.assertEqual(response.status_code, 404)+ self.user_logout()+ response = self.client.get(reverse('nutrition:ingredient:list'), {'is_vegan': '1'})+ self.assertEqual(response.status_code, 200)- response = self.client.get(reverse('nutrition:ingredient:list'), {'page': 'foobar'})- self.assertEqual(response.status_code, 404)+ if response.context['has_next']:+ self.assertIn('is_vegan=1', response.context['next_url'])+ self.assertIn('after=', response.context['next_url'])def ingredient_overview(self):"""diff --git a/wger/nutrition/views/ingredient.py b/wger/nutrition/views/ingredient.pyindex cbe318ec3..61b29ae45 100644--- a/wger/nutrition/views/ingredient.py+++ b/wger/nutrition/views/ingredient.py@@ -30,12 +30,10 @@ from django.shortcuts import (render,)from django.urls import reverse_lazy-from django.utils.decorators import method_decoratorfrom django.utils.translation import (gettext as _,gettext_lazy,)-from django.views.decorators.cache import cache_pagefrom django.views.generic import (CreateView,DeleteView,@@ -65,27 +63,80 @@ logger = logging.getLogger(__name__)# ************************# Ingredient functions# ************************-@method_decorator(cache_page(settings.WGER_SETTINGS['INGREDIENT_CACHE_TTL']), name='dispatch')class IngredientListView(ListView):"""- Show an overview of all ingredients+ Show an overview of all ingredients using cursor-based pagination.++ This is more efficient than OFFSET-based pagination for large lists because it+ can use an index on id to jump directly to the next page, while OFFSET+ requires scanning and counting rows up to the offset.++ This also allows to keep the list public so that crawlers can index it++ The query string interface:+ - no parameter -> first page+ - ?after=<id> -> rows with id > <id>, ascending"""model = Ingredienttemplate_name = 'ingredient/overview.html'context_object_name = 'ingredients_list'- paginate_by = PAGINATION_OBJECTS_PER_PAGE+ paginate_by = None # disabled — cursor handled in get_context_datafilterset_class = IngredientFilterSet+ PAGE_SIZE = PAGINATION_OBJECTS_PER_PAGE+def get_queryset(self):"""- Filter the ingredients the user will see by its language, optionally- also filtering by other properties+ Apply language + filterset, then cursor logic. We fetch one extra row+ beyond PAGE_SIZE so we know whether a next page exists without an+ extra COUNT query."""language = load_language()queryset = Ingredient.objects.filter(language=language)filterset = self.filterset_class(self.request.GET or None, queryset=queryset)- return filterset.qs+ qs = filterset.qs.order_by('id')++ after = self.request.GET.get('after')+ if after:+ try:+ qs = qs.filter(id__gt=int(after))+ except (TypeError, ValueError):+ # fall through to first page+ pass++ return qs[: self.PAGE_SIZE + 1]++ def get_context_data(self, **kwargs):+ context = super().get_context_data(**kwargs)+ ingredients = list(context['ingredients_list'])++ # We fetched PAGE_SIZE + 1 to peek at the next-page existence.+ has_next = len(ingredients) > self.PAGE_SIZE+ if has_next:+ ingredients = ingredients[: self.PAGE_SIZE]++ # Pre-build the pagination URLs here+ next_url = None+ if has_next and ingredients:+ params = self.request.GET.copy()+ params['after'] = str(ingredients[-1].id)+ next_url = f'?{params.urlencode()}'++ is_paginated = bool(self.request.GET.get('after'))+ first_url = None+ if is_paginated:+ params = self.request.GET.copy()+ params.pop('after', None)+ first_url = f'?{params.urlencode()}' if params else '?'++ context['ingredients_list'] = ingredients+ context['has_next'] = has_next+ context['next_url'] = next_url+ context['first_url'] = first_url+ context['is_paginated'] = is_paginated++ return contextdef view(request, pk, slug=None):diff --git a/wger/utils/constants.py b/wger/utils/constants.pyindex 8adcdb147..49fbcefb4 100644--- a/wger/utils/constants.py+++ b/wger/utils/constants.py@@ -59,8 +59,6 @@ HTML_STYLES_WHITELIST = ('text-decoration',)# PaginationPAGINATION_OBJECTS_PER_PAGE = 25-PAGINATION_MAX_TOTAL_PAGES = 10-PAGINATION_PAGES_AROUND_CURRENT = 5# Important license IDsCC_BY_SA_4_LICENSE_ID = 2diff --git a/wger/utils/url.py b/wger/utils/url.pyindex 0f5500362..3232a1694 100644--- a/wger/utils/url.py+++ b/wger/utils/url.py@@ -14,6 +14,7 @@# Standard Libraryfrom typing import (+ Any,Dict,Optional,)@@ -30,7 +31,7 @@ def make_uri(path: str,object_id: Optional[int] = None,object_method: Optional[str] = None,- query: Optional[Dict[str, any]] = None,+ query: Optional[Dict[str, Any]] = None,server_url: str = settings.WGER_SETTINGS['WGER_INSTANCE'],):uri_server = urlparse(server_url)