Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Send emails from a celery worker instead of the request

Roland Geidere661111297f3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7dd97702f..6be5f3636 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@
* Improved openAPI spec. The spec now properly describes the different parts of
the API and can be used to generate clients.
+* Emails are now send asynchronously via the celery queue, this should make
+ registration, password resets, etc. feel a bit snappier. If celery is not configured,
+ the emails are send as before
### OAuth2 provider
diff --git a/settings/main.py b/settings/main.py
index b7319c0c7..b362eafea 100644
--- a/settings/main.py
+++ b/settings/main.py
@@ -197,6 +197,13 @@ WGER_SETTINGS['CACHE_API_EXERCISES_CELERY'] = env.bool('CACHE_API_EXERCISES_CELE
WGER_SETTINGS['CACHE_API_EXERCISES_CELERY_FORCE_UPDATE'] = env.bool(
'CACHE_API_EXERCISES_CELERY_FORCE_UPDATE', False
)
+
+# Let a worker do the SMTP round trip, it would otherwise happen inside the
+# request that triggered the email (registration, password reset, ...)
+EMAIL_DELIVERY_BACKEND = EMAIL_BACKEND
+if WGER_SETTINGS['USE_CELERY']:
+ EMAIL_BACKEND = 'wger.core.mail.CeleryEmailBackend'
+
WGER_SHOW_APP_STORE_LINKS = env.bool('WGER_SHOW_APP_STORE_LINKS', True)
#
diff --git a/settings/settings_global.py b/settings/settings_global.py
index 625e7ca2c..1f5988d99 100644
--- a/settings/settings_global.py
+++ b/settings/settings_global.py
@@ -236,6 +236,10 @@ STATICFILES_DIRS = (('node', os.path.join(BASE_DIR, '..', 'node_modules')),)
EMAIL_SUBJECT_PREFIX = '[wger] '
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
+# Backend that performs the delivery when EMAIL_BACKEND only queues it, see
+# wger.core.mail
+EMAIL_DELIVERY_BACKEND = EMAIL_BACKEND
+
#
# django-allauth
#
diff --git a/wger/core/mail.py b/wger/core/mail.py
new file mode 100644
index 000000000..ad6bc4f0d
--- /dev/null
+++ b/wger/core/mail.py
@@ -0,0 +1,74 @@
+# This file is part of wger Workout Manager <https://github.com/wger-project>.
+# Copyright (C) 2013 - 2021 wger Team
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Email delivery through celery.
+
+Sending an email means an SMTP round trip, which otherwise happens inside the
+request that triggered it: registration, password reset, email change. This
+backend only serializes the message and hands it to a worker, the real delivery
+happens in EMAIL_DELIVERY_BACKEND.
+"""
+
+# Standard Library
+import logging
+
+# Django
+from django.core.mail.backends.base import BaseEmailBackend
+
+# Third Party
+from celery.exceptions import OperationalError
+
+# wger
+from wger.core.tasks import send_email_task
+from wger.utils.mail import (
+ get_delivery_connection,
+ message_to_dict,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class CeleryEmailBackend(BaseEmailBackend):
+ """
+ Email backend that queues the delivery instead of doing it in-band
+ """
+
+ def send_messages(self, email_messages) -> int:
+ direct = []
+ queued = 0
+
+ for message in email_messages:
+ payload = message_to_dict(message)
+ if payload is None:
+ direct.append(message)
+ continue
+
+ try:
+ send_email_task.delay(payload)
+ except OperationalError:
+ # No broker. Delivering late is better than not at all, so
+ # fall back to sending the message right here.
+ logger.exception('Could not queue an email, sending it directly')
+ direct.append(message)
+ else:
+ queued += 1
+
+ if direct:
+ queued += get_delivery_connection(self.fail_silently).send_messages(direct) or 0
+
+ return queued
diff --git a/wger/core/tasks.py b/wger/core/tasks.py
index f9f354b7b..a482a8f53 100644
--- a/wger/core/tasks.py
+++ b/wger/core/tasks.py
@@ -31,11 +31,29 @@ from celery.schedules import crontab
# wger
from wger.celery_configuration import app
from wger.core.models import LongLivedSession
+from wger.utils.mail import (
+ get_delivery_connection,
+ message_from_dict,
+)
logger = logging.getLogger(__name__)
+ autoretry_for=(Exception,),
+ retry_backoff=True,
+ retry_kwargs={'max_retries': 5},
+)
+def send_email_task(payload: dict):
+ """
+ Deliver a single email, see wger.core.mail.CeleryEmailBackend
+ """
+ message = message_from_dict(payload)
+ message.connection = get_delivery_connection()
+ message.send()
+
+
@app.task
def flush_expired_jwt_tokens_task():
"""
diff --git a/wger/core/tests/test_mail.py b/wger/core/tests/test_mail.py
new file mode 100644
index 000000000..70d11e10c
--- /dev/null
+++ b/wger/core/tests/test_mail.py
@@ -0,0 +1,192 @@
+# This file is part of wger Workout Manager.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+
+# Standard Library
+from unittest import mock
+
+# Django
+from django.core import mail
+from django.core.mail import EmailMultiAlternatives
+from django.core.mail.backends.base import BaseEmailBackend
+from django.test import (
+ SimpleTestCase,
+ override_settings,
+)
+from django.urls import reverse
+
+# Third Party
+from celery.exceptions import OperationalError
+
+# wger
+from wger.core.tasks import send_email_task
+from wger.core.tests.base_testcase import WgerTestCase
+from wger.utils.mail import (
+ message_from_dict,
+ message_to_dict,
+)
+
+
+LOCMEM = 'django.core.mail.backends.locmem.EmailBackend'
+CELERY = 'wger.core.mail.CeleryEmailBackend'
+EXPLODING = 'wger.core.tests.test_mail.ExplodingBackend'
+
+
+class ExplodingBackend(BaseEmailBackend):
+ """
+ Delivery backend that fails, unless it is told to keep quiet
+ """
+
+ def send_messages(self, email_messages):
+ if not self.fail_silently:
+ raise OSError('the mail server is not answering')
+ return 0
+
+
+def build_message():
+ message = EmailMultiAlternatives(
+ subject='Please confirm',
+ body='plain text',
+ from_email='[email protected]',
+ reply_to=['[email protected]'],
+ headers={'X-Wger': 'yes'},
+ )
+ message.attach_alternative('<p>html</p>', 'text/html')
+ return message
+
+
+@override_settings(EMAIL_BACKEND=CELERY, EMAIL_DELIVERY_BACKEND=LOCMEM)
+class CeleryEmailBackendTestCase(SimpleTestCase):
+ """
+ Tests the email backend that hands the delivery to celery
+ """
+
+ def test_message_survives_the_round_trip(self):
+ """
+ Everything the message carries is restored from the task payload
+ """
+ original = build_message()
+
+ restored = message_from_dict(message_to_dict(original))
+
+ self.assertEqual(restored.subject, original.subject)
+ self.assertEqual(restored.body, original.body)
+ self.assertEqual(restored.from_email, original.from_email)
+ self.assertEqual(restored.to, original.to)
+ self.assertEqual(restored.cc, original.cc)
+ self.assertEqual(restored.bcc, original.bcc)
+ self.assertEqual(restored.reply_to, original.reply_to)
+ self.assertEqual(restored.extra_headers, original.extra_headers)
+ self.assertEqual(list(restored.alternatives), list(original.alternatives))
+
+ def test_sending_only_queues(self):
+ """
+ The backend queues a task and does not talk to the mail server
+ """
+ with mock.patch('wger.core.tasks.send_email_task.delay') as delay:
+ sent = build_message().send()
+
+ self.assertEqual(sent, 1)
+ self.assertEqual(delay.call_count, 1)
+ self.assertEqual(mail.outbox, [])
+
+ def test_task_delivers_the_message(self):
+ """
+ The task hands the rebuilt message to the delivery backend
+ """
+ payload = message_to_dict(build_message())
+
+ send_email_task(payload)
+
+ self.assertEqual(len(mail.outbox), 1)
+ self.assertEqual(mail.outbox[0].subject, 'Please confirm')
+ self.assertEqual(mail.outbox[0].to, ['[email protected]'])
+ self.assertEqual(mail.outbox[0].alternatives[0].content, '<p>html</p>')
+
+ def test_message_with_attachment_is_sent_directly(self):
+ """
+ Attachments do not fit into the payload, those messages are delivered
+ without the detour
+ """
+ message = build_message()
+ message.attach('plan.csv', 'a,b,c', 'text/csv')
+
+ with mock.patch('wger.core.tasks.send_email_task.delay') as delay:
+ sent = message.send()
+
+ self.assertEqual(sent, 1)
+ self.assertEqual(delay.call_count, 0)
+ self.assertEqual(len(mail.outbox), 1)
+
+ @override_settings(EMAIL_DELIVERY_BACKEND=EXPLODING)
+ def test_direct_delivery_keeps_fail_silently(self):
+ """
+ A caller that asked not to be bothered with delivery errors is not
+ bothered with them on the direct path either
+ """
+ quiet = build_message()
+ quiet.attach('plan.csv', 'a,b,c', 'text/csv')
+ self.assertEqual(quiet.send(fail_silently=True), 0)
+
+ # a fresh message, EmailMessage caches the connection it was sent with
+ loud = build_message()
+ loud.attach('plan.csv', 'a,b,c', 'text/csv')
+ with self.assertRaises(OSError):
+ loud.send(fail_silently=False)
+
+ def test_unreachable_broker_falls_back_to_direct_delivery(self):
+ """
+ A message is delivered even when it cannot be queued
+ """
+ with mock.patch(
+ 'wger.core.tasks.send_email_task.delay',
+ side_effect=OperationalError('no broker'),
+ ):
+ with self.assertLogs('wger.core.mail', level='ERROR'):
+ sent = build_message().send()
+
+ self.assertEqual(sent, 1)
+ self.assertEqual(len(mail.outbox), 1)
+
+
+@override_settings(EMAIL_BACKEND=CELERY, EMAIL_DELIVERY_BACKEND=LOCMEM)
+class SignupEmailTestCase(WgerTestCase):
+ """
+ Tests that the emails of the registration flow are queued
+ """
+
+ def test_confirmation_email_is_queued(self):
+ """
+ Registering does not talk to the mail server during the request
+ """
+ data = {
+ 'username': 'mailtest',
+ 'password1': 'quai8fai7Zae',
+ 'password2': 'quai8fai7Zae',
+ 'email': '[email protected]',
+ }
+
+ with mock.patch('wger.core.tasks.send_email_task.delay') as delay:
+ response = self.client.post(reverse('core:user:registration'), data)
+
+ self.assertEqual(response.status_code, 302)
+ self.assertEqual(delay.call_count, 1)
+ self.assertEqual(mail.outbox, [])
+
+ # and the queued payload is the confirmation email
+ payload = delay.call_args[0][0]
+ self.assertEqual(payload['to'], ['[email protected]'])
+ self.assertIn('Confirm', payload['subject'])
diff --git a/wger/utils/mail.py b/wger/utils/mail.py
new file mode 100644
index 000000000..7def1b283
--- /dev/null
+++ b/wger/utils/mail.py
@@ -0,0 +1,83 @@
+# This file is part of wger Workout Manager <https://github.com/wger-project>.
+# Copyright (C) 2013 - 2021 wger Team
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Turning email messages into something a celery task can carry.
+
+See wger.core.mail for the backend that queues them and wger.core.tasks for
+the task that delivers them.
+"""
+
+# Django
+from django.conf import settings
+from django.core.mail import (
+ EmailMultiAlternatives,
+ get_connection,
+)
+
+
+def get_delivery_connection(fail_silently: bool = False):
+ """
+ Connection to the backend that actually talks to the mail server
+ """
+ return get_connection(
+ backend=settings.EMAIL_DELIVERY_BACKEND,
+ fail_silently=fail_silently,
+ )
+
+
+def message_to_dict(message) -> dict | None:
+ """
+ Break a message down into the pieces a task can carry, or None if it
+ contains something that would not survive the round trip.
+ """
+ if message.attachments:
+ return None
+
+ return {
+ 'subject': message.subject,
+ 'body': message.body,
+ 'from_email': message.from_email,
+ 'to': list(message.to),
+ 'cc': list(message.cc),
+ 'bcc': list(message.bcc),
+ 'reply_to': list(message.reply_to),
+ 'headers': dict(message.extra_headers),
+ 'content_subtype': message.content_subtype,
+ 'alternatives': [
+ [content, mimetype] for content, mimetype in getattr(message, 'alternatives', [])
+ ],
+ }
+
+
+def message_from_dict(data: dict) -> EmailMultiAlternatives:
+ """
+ Rebuild a message from the pieces of message_to_dict
+ """
+ message = EmailMultiAlternatives(
+ subject=data['subject'],
+ body=data['body'],
+ from_email=data['from_email'],
+ to=data['to'],
+ cc=data['cc'],
+ bcc=data['bcc'],
+ reply_to=data['reply_to'],
+ headers=data['headers'],
+ )
+ message.content_subtype = data['content_subtype']
+ for content, mimetype in data['alternatives']:
+ message.attach_alternative(content, mimetype)
+ return message