Self hosted FLOSS fitness/workout, nutrition and weight tracker
Cheapen the dynamic reconcile triggers
Roland Geider9e1f9df94277
diff --git a/wger/measurements/dynamic/engine.py b/wger/measurements/dynamic/engine.pyindex 0e6bd4dd3..117343c7e 100644--- a/wger/measurements/dynamic/engine.py+++ b/wger/measurements/dynamic/engine.py@@ -15,10 +15,10 @@# Standard Libraryimport logging-import threading# Djangofrom django.conf import settings+from django.core.cache import cachefrom django.db import transaction# wger@@ -32,7 +32,18 @@ from wger.measurements.models.measurement import MeasurementSourcelogger = logging.getLogger(__name__)-_pending = threading.local()+# One queued task per category: the marker suppresses further queueing until+# the task starts, and the countdown gives a write burst (e.g. a draining+# upload queue) time to finish so it collapses into that one run+RECONCILE_DEBOUNCE_SECONDS = 15++# A crashed task suppresses reconciles at most this long; the daily catch-all+# repairs anything behind it+RECONCILE_MARKER_TIMEOUT = 5 * 60+++def reconcile_marker_key(category_id) -> str:+ return f'measurements-dynamic-reconcile-{category_id}'def reconcile(category: Category) -> None:@@ -97,27 +108,16 @@ def schedule_reconcile(category_id) -> None:"""Queues a reconcile for after the current transaction.- Every call registers a callback, the deduplication happens when they run:- the first one of a flush does the work and marks the category, the- duplicates of the same batch see the mark and return. Marking on- execution rather than on registration keeps a rolled-back transaction- (whose callbacks Django discards) from blocking later reconciles.+ With celery the queueing is single-flight per category: while a task is+ waiting, further writes queue nothing, and the task removes the marker+ before it computes so a write arriving during the run queues a follow-up.+ Without celery the reconcile runs right here, there is no later moment to+ collapse a burst into."""- done = getattr(_pending, 'done', None)- if done:- # A new write batch begins, the marks of the previous flush are stale- done.clear()transaction.on_commit(lambda: _run_scheduled(category_id))def _run_scheduled(category_id) -> None:- done = getattr(_pending, 'done', None)- if done is None:- done = _pending.done = set()- if category_id in done:- return- done.add(category_id)-# This runs after the commit, so a failure here has nothing to roll back# and would only turn an unrelated request into a 500. The daily task# picks the category up again@@ -126,7 +126,12 @@ def _run_scheduled(category_id) -> None:# wgerfrom wger.measurements.tasks import reconcile_dynamic_category_task- reconcile_dynamic_category_task.delay(str(category_id))+ # add() is atomic across processes: True means no task is waiting+ if cache.add(reconcile_marker_key(category_id), True, RECONCILE_MARKER_TIMEOUT):+ reconcile_dynamic_category_task.apply_async(+ args=[str(category_id)],+ countdown=RECONCILE_DEBOUNCE_SECONDS,+ )else:reconcile_by_id(category_id)except Exception:diff --git a/wger/measurements/tasks.py b/wger/measurements/tasks.pyindex e1c682156..937e177ff 100644--- a/wger/measurements/tasks.py+++ b/wger/measurements/tasks.py@@ -17,6 +17,9 @@import loggingimport random+# Django+from django.core.cache import cache+# Third Partyfrom celery.schedules import crontab@@ -25,6 +28,7 @@ from wger.celery_configuration import appfrom wger.measurements.dynamic.engine import (reconcile,reconcile_by_id,+ reconcile_marker_key,)from wger.measurements.models import Category@@ -37,6 +41,9 @@ def reconcile_dynamic_category_task(category_id: str):"""Recomputes the calculated rows of one dynamic category"""+ # Marker first: a write arriving during the run has to queue a follow-up,+ # or its change would wait for the daily catch-all+ cache.delete(reconcile_marker_key(category_id))reconcile_by_id(category_id)diff --git a/wger/measurements/tests/test_dynamic.py b/wger/measurements/tests/test_dynamic.pyindex 6e7666b9d..5419db251 100644--- a/wger/measurements/tests/test_dynamic.py+++ b/wger/measurements/tests/test_dynamic.py@@ -19,7 +19,9 @@ from decimal import Decimalfrom unittest.mock import patch# Django+from django.conf import settingsfrom django.contrib.auth.models import User+from django.core.cache import cachefrom django.urls import reversefrom django.utils import timezone@@ -30,7 +32,10 @@ from rest_framework import statusfrom wger.core.tests.base_testcase import WgerTestCasefrom wger.manager.consts import WEIGHT_UNIT_LBfrom wger.manager.models import WorkoutLog-from wger.measurements.dynamic.engine import reconcile+from wger.measurements.dynamic.engine import (+ reconcile,+ reconcile_marker_key,+)from wger.measurements.models import (Category,Measurement,@@ -38,7 +43,10 @@ from wger.measurements.models import (from wger.measurements.models.measurement import MeasurementSourcefrom wger.measurements.powersync import MeasurementHandlerfrom wger.measurements.signals import _dispatch_source_change-from wger.measurements.tasks import reconcile_all_dynamic_categories_task+from wger.measurements.tasks import (+ reconcile_all_dynamic_categories_task,+ reconcile_dynamic_category_task,+)# Pinned in test-measurement-categories.json, the official body weight@@ -606,6 +614,37 @@ class ReconcileFailureTestCase(DynamicMeasurementTestCase):self.assertTrue(Category.objects.filter(pk=category.pk).exists())+class SingleFlightSchedulingTestCase(DynamicMeasurementTestCase):+ """+ With celery, a burst of source writes queues one reconcile task per+ category, and the task frees the slot before it computes+ """++ def setUp(self):+ super().setUp()+ cache.clear()++ def test_a_write_burst_queues_one_task(self):+ self.enable_bmi()++ with patch.dict(settings.WGER_SETTINGS, {'USE_CELERY': True}):+ with patch(+ 'wger.measurements.tasks.reconcile_dynamic_category_task.apply_async'+ ) as apply_async:+ self.create_weight('80')+ self.create_weight('81')++ self.assertEqual(apply_async.call_count, 1)++ def test_the_task_frees_the_slot_before_computing(self):+ category = self.enable_bmi()+ cache.add(reconcile_marker_key(category.pk), True, 60)++ reconcile_dynamic_category_task(str(category.pk))++ self.assertIsNone(cache.get(reconcile_marker_key(category.pk)))++class ApiWriteBlockTestCase(DynamicMeasurementTestCase):"""The entries of a calculated category cannot be written through the API