Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Cheapen the dynamic reconcile triggers

Roland Geider9e1f9df94277
diff --git a/wger/measurements/dynamic/engine.py b/wger/measurements/dynamic/engine.py
index 0e6bd4dd3..117343c7e 100644
--- a/wger/measurements/dynamic/engine.py
+++ b/wger/measurements/dynamic/engine.py
@@ -15,10 +15,10 @@
# Standard Library
import logging
-import threading
# Django
from django.conf import settings
+from django.core.cache import cache
from django.db import transaction
# wger
@@ -32,7 +32,18 @@ from wger.measurements.models.measurement import MeasurementSource
logger = logging.getLogger(__name__)
-_pending = threading.local()
+# One queued task per category: the marker suppresses further queueing until
+# the task starts, and the countdown gives a write burst (e.g. a draining
+# upload queue) time to finish so it collapses into that one run
+RECONCILE_DEBOUNCE_SECONDS = 15
+
+# A crashed task suppresses reconciles at most this long; the daily catch-all
+# repairs anything behind it
+RECONCILE_MARKER_TIMEOUT = 5 * 60
+
+
+def reconcile_marker_key(category_id) -> str:
+ return f'measurements-dynamic-reconcile-{category_id}'
def reconcile(category: Category) -> None:
@@ -97,27 +108,16 @@ def schedule_reconcile(category_id) -> None:
"""
Queues a reconcile for after the current transaction.
- Every call registers a callback, the deduplication happens when they run:
- the first one of a flush does the work and marks the category, the
- duplicates of the same batch see the mark and return. Marking on
- execution rather than on registration keeps a rolled-back transaction
- (whose callbacks Django discards) from blocking later reconciles.
+ With celery the queueing is single-flight per category: while a task is
+ waiting, further writes queue nothing, and the task removes the marker
+ before it computes so a write arriving during the run queues a follow-up.
+ Without celery the reconcile runs right here, there is no later moment to
+ collapse a burst into.
"""
- done = getattr(_pending, 'done', None)
- if done:
- # A new write batch begins, the marks of the previous flush are stale
- done.clear()
transaction.on_commit(lambda: _run_scheduled(category_id))
def _run_scheduled(category_id) -> None:
- done = getattr(_pending, 'done', None)
- if done is None:
- done = _pending.done = set()
- if category_id in done:
- return
- done.add(category_id)
-
# This runs after the commit, so a failure here has nothing to roll back
# and would only turn an unrelated request into a 500. The daily task
# picks the category up again
@@ -126,7 +126,12 @@ def _run_scheduled(category_id) -> None:
# wger
from wger.measurements.tasks import reconcile_dynamic_category_task
- reconcile_dynamic_category_task.delay(str(category_id))
+ # add() is atomic across processes: True means no task is waiting
+ if cache.add(reconcile_marker_key(category_id), True, RECONCILE_MARKER_TIMEOUT):
+ reconcile_dynamic_category_task.apply_async(
+ args=[str(category_id)],
+ countdown=RECONCILE_DEBOUNCE_SECONDS,
+ )
else:
reconcile_by_id(category_id)
except Exception:
diff --git a/wger/measurements/tasks.py b/wger/measurements/tasks.py
index e1c682156..937e177ff 100644
--- a/wger/measurements/tasks.py
+++ b/wger/measurements/tasks.py
@@ -17,6 +17,9 @@
import logging
import random
+# Django
+from django.core.cache import cache
+
# Third Party
from celery.schedules import crontab
@@ -25,6 +28,7 @@ from wger.celery_configuration import app
from wger.measurements.dynamic.engine import (
reconcile,
reconcile_by_id,
+ reconcile_marker_key,
)
from wger.measurements.models import Category
@@ -37,6 +41,9 @@ def reconcile_dynamic_category_task(category_id: str):
"""
Recomputes the calculated rows of one dynamic category
"""
+ # Marker first: a write arriving during the run has to queue a follow-up,
+ # or its change would wait for the daily catch-all
+ cache.delete(reconcile_marker_key(category_id))
reconcile_by_id(category_id)
diff --git a/wger/measurements/tests/test_dynamic.py b/wger/measurements/tests/test_dynamic.py
index 6e7666b9d..5419db251 100644
--- a/wger/measurements/tests/test_dynamic.py
+++ b/wger/measurements/tests/test_dynamic.py
@@ -19,7 +19,9 @@ from decimal import Decimal
from unittest.mock import patch
# Django
+from django.conf import settings
from django.contrib.auth.models import User
+from django.core.cache import cache
from django.urls import reverse
from django.utils import timezone
@@ -30,7 +32,10 @@ from rest_framework import status
from wger.core.tests.base_testcase import WgerTestCase
from wger.manager.consts import WEIGHT_UNIT_LB
from wger.manager.models import WorkoutLog
-from wger.measurements.dynamic.engine import reconcile
+from wger.measurements.dynamic.engine import (
+ reconcile,
+ reconcile_marker_key,
+)
from wger.measurements.models import (
Category,
Measurement,
@@ -38,7 +43,10 @@ from wger.measurements.models import (
from wger.measurements.models.measurement import MeasurementSource
from wger.measurements.powersync import MeasurementHandler
from wger.measurements.signals import _dispatch_source_change
-from wger.measurements.tasks import reconcile_all_dynamic_categories_task
+from wger.measurements.tasks import (
+ reconcile_all_dynamic_categories_task,
+ reconcile_dynamic_category_task,
+)
# Pinned in test-measurement-categories.json, the official body weight
@@ -606,6 +614,37 @@ class ReconcileFailureTestCase(DynamicMeasurementTestCase):
self.assertTrue(Category.objects.filter(pk=category.pk).exists())
+class SingleFlightSchedulingTestCase(DynamicMeasurementTestCase):
+ """
+ With celery, a burst of source writes queues one reconcile task per
+ category, and the task frees the slot before it computes
+ """
+
+ def setUp(self):
+ super().setUp()
+ cache.clear()
+
+ def test_a_write_burst_queues_one_task(self):
+ self.enable_bmi()
+
+ with patch.dict(settings.WGER_SETTINGS, {'USE_CELERY': True}):
+ with patch(
+ 'wger.measurements.tasks.reconcile_dynamic_category_task.apply_async'
+ ) as apply_async:
+ self.create_weight('80')
+ self.create_weight('81')
+
+ self.assertEqual(apply_async.call_count, 1)
+
+ def test_the_task_frees_the_slot_before_computing(self):
+ category = self.enable_bmi()
+ cache.add(reconcile_marker_key(category.pk), True, 60)
+
+ reconcile_dynamic_category_task(str(category.pk))
+
+ self.assertIsNone(cache.get(reconcile_marker_key(category.pk)))
+
+
class ApiWriteBlockTestCase(DynamicMeasurementTestCase):
"""
The entries of a calculated category cannot be written through the API