Self hosted FLOSS fitness/workout, nutrition and weight tracker
Send emails from a celery worker instead of the request
Roland Geidere661111297f3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7dd97702f..6be5f3636 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,9 @@* Improved openAPI spec. The spec now properly describes the different parts ofthe API and can be used to generate clients.+* Emails are now send asynchronously via the celery queue, this should make+ registration, password resets, etc. feel a bit snappier. If celery is not configured,+ the emails are send as before### OAuth2 providerdiff --git a/settings/main.py b/settings/main.pyindex b7319c0c7..b362eafea 100644--- a/settings/main.py+++ b/settings/main.py@@ -197,6 +197,13 @@ WGER_SETTINGS['CACHE_API_EXERCISES_CELERY'] = env.bool('CACHE_API_EXERCISES_CELEWGER_SETTINGS['CACHE_API_EXERCISES_CELERY_FORCE_UPDATE'] = env.bool('CACHE_API_EXERCISES_CELERY_FORCE_UPDATE', False)++# Let a worker do the SMTP round trip, it would otherwise happen inside the+# request that triggered the email (registration, password reset, ...)+EMAIL_DELIVERY_BACKEND = EMAIL_BACKEND+if WGER_SETTINGS['USE_CELERY']:+ EMAIL_BACKEND = 'wger.core.mail.CeleryEmailBackend'+WGER_SHOW_APP_STORE_LINKS = env.bool('WGER_SHOW_APP_STORE_LINKS', True)#diff --git a/settings/settings_global.py b/settings/settings_global.pyindex 625e7ca2c..1f5988d99 100644--- a/settings/settings_global.py+++ b/settings/settings_global.py@@ -236,6 +236,10 @@ STATICFILES_DIRS = (('node', os.path.join(BASE_DIR, '..', 'node_modules')),)EMAIL_SUBJECT_PREFIX = '[wger] 'EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'+# Backend that performs the delivery when EMAIL_BACKEND only queues it, see+# wger.core.mail+EMAIL_DELIVERY_BACKEND = EMAIL_BACKEND+## django-allauth#diff --git a/wger/core/mail.py b/wger/core/mail.pynew file mode 100644index 000000000..ad6bc4f0d--- /dev/null+++ b/wger/core/mail.py@@ -0,0 +1,74 @@+# This file is part of wger Workout Manager <https://github.com/wger-project>.+# Copyright (C) 2013 - 2021 wger Team+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <http://www.gnu.org/licenses/>.++"""+Email delivery through celery.++Sending an email means an SMTP round trip, which otherwise happens inside the+request that triggered it: registration, password reset, email change. This+backend only serializes the message and hands it to a worker, the real delivery+happens in EMAIL_DELIVERY_BACKEND.+"""++# Standard Library+import logging++# Django+from django.core.mail.backends.base import BaseEmailBackend++# Third Party+from celery.exceptions import OperationalError++# wger+from wger.core.tasks import send_email_task+from wger.utils.mail import (+ get_delivery_connection,+ message_to_dict,+)+++logger = logging.getLogger(__name__)+++class CeleryEmailBackend(BaseEmailBackend):+ """+ Email backend that queues the delivery instead of doing it in-band+ """++ def send_messages(self, email_messages) -> int:+ direct = []+ queued = 0++ for message in email_messages:+ payload = message_to_dict(message)+ if payload is None:+ direct.append(message)+ continue++ try:+ send_email_task.delay(payload)+ except OperationalError:+ # No broker. Delivering late is better than not at all, so+ # fall back to sending the message right here.+ logger.exception('Could not queue an email, sending it directly')+ direct.append(message)+ else:+ queued += 1++ if direct:+ queued += get_delivery_connection(self.fail_silently).send_messages(direct) or 0++ return queueddiff --git a/wger/core/tasks.py b/wger/core/tasks.pyindex f9f354b7b..a482a8f53 100644--- a/wger/core/tasks.py+++ b/wger/core/tasks.py@@ -31,11 +31,29 @@ from celery.schedules import crontab# wgerfrom wger.celery_configuration import appfrom wger.core.models import LongLivedSession+from wger.utils.mail import (+ get_delivery_connection,+ message_from_dict,+)logger = logging.getLogger(__name__)+ autoretry_for=(Exception,),+ retry_backoff=True,+ retry_kwargs={'max_retries': 5},+)+def send_email_task(payload: dict):+ """+ Deliver a single email, see wger.core.mail.CeleryEmailBackend+ """+ message = message_from_dict(payload)+ message.connection = get_delivery_connection()+ message.send()++@app.taskdef flush_expired_jwt_tokens_task():"""diff --git a/wger/core/tests/test_mail.py b/wger/core/tests/test_mail.pynew file mode 100644index 000000000..70d11e10c--- /dev/null+++ b/wger/core/tests/test_mail.py@@ -0,0 +1,192 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Standard Library+from unittest import mock++# Django+from django.core import mail+from django.core.mail import EmailMultiAlternatives+from django.core.mail.backends.base import BaseEmailBackend+from django.test import (+ SimpleTestCase,+ override_settings,+)+from django.urls import reverse++# Third Party+from celery.exceptions import OperationalError++# wger+from wger.core.tasks import send_email_task+from wger.core.tests.base_testcase import WgerTestCase+from wger.utils.mail import (+ message_from_dict,+ message_to_dict,+)+++LOCMEM = 'django.core.mail.backends.locmem.EmailBackend'+CELERY = 'wger.core.mail.CeleryEmailBackend'+EXPLODING = 'wger.core.tests.test_mail.ExplodingBackend'+++class ExplodingBackend(BaseEmailBackend):+ """+ Delivery backend that fails, unless it is told to keep quiet+ """++ def send_messages(self, email_messages):+ if not self.fail_silently:+ raise OSError('the mail server is not answering')+ return 0+++def build_message():+ message = EmailMultiAlternatives(+ subject='Please confirm',+ body='plain text',+ from_email='[email protected]',+ to=['[email protected]'],+ cc=['[email protected]'],+ bcc=['[email protected]'],+ reply_to=['[email protected]'],+ headers={'X-Wger': 'yes'},+ )+ message.attach_alternative('<p>html</p>', 'text/html')+ return message+++@override_settings(EMAIL_BACKEND=CELERY, EMAIL_DELIVERY_BACKEND=LOCMEM)+class CeleryEmailBackendTestCase(SimpleTestCase):+ """+ Tests the email backend that hands the delivery to celery+ """++ def test_message_survives_the_round_trip(self):+ """+ Everything the message carries is restored from the task payload+ """+ original = build_message()++ restored = message_from_dict(message_to_dict(original))++ self.assertEqual(restored.subject, original.subject)+ self.assertEqual(restored.body, original.body)+ self.assertEqual(restored.from_email, original.from_email)+ self.assertEqual(restored.to, original.to)+ self.assertEqual(restored.cc, original.cc)+ self.assertEqual(restored.bcc, original.bcc)+ self.assertEqual(restored.reply_to, original.reply_to)+ self.assertEqual(restored.extra_headers, original.extra_headers)+ self.assertEqual(list(restored.alternatives), list(original.alternatives))++ def test_sending_only_queues(self):+ """+ The backend queues a task and does not talk to the mail server+ """+ with mock.patch('wger.core.tasks.send_email_task.delay') as delay:+ sent = build_message().send()++ self.assertEqual(sent, 1)+ self.assertEqual(delay.call_count, 1)+ self.assertEqual(mail.outbox, [])++ def test_task_delivers_the_message(self):+ """+ The task hands the rebuilt message to the delivery backend+ """+ payload = message_to_dict(build_message())++ send_email_task(payload)++ self.assertEqual(len(mail.outbox), 1)+ self.assertEqual(mail.outbox[0].subject, 'Please confirm')+ self.assertEqual(mail.outbox[0].to, ['[email protected]'])+ self.assertEqual(mail.outbox[0].alternatives[0].content, '<p>html</p>')++ def test_message_with_attachment_is_sent_directly(self):+ """+ Attachments do not fit into the payload, those messages are delivered+ without the detour+ """+ message = build_message()+ message.attach('plan.csv', 'a,b,c', 'text/csv')++ with mock.patch('wger.core.tasks.send_email_task.delay') as delay:+ sent = message.send()++ self.assertEqual(sent, 1)+ self.assertEqual(delay.call_count, 0)+ self.assertEqual(len(mail.outbox), 1)++ @override_settings(EMAIL_DELIVERY_BACKEND=EXPLODING)+ def test_direct_delivery_keeps_fail_silently(self):+ """+ A caller that asked not to be bothered with delivery errors is not+ bothered with them on the direct path either+ """+ quiet = build_message()+ quiet.attach('plan.csv', 'a,b,c', 'text/csv')+ self.assertEqual(quiet.send(fail_silently=True), 0)++ # a fresh message, EmailMessage caches the connection it was sent with+ loud = build_message()+ loud.attach('plan.csv', 'a,b,c', 'text/csv')+ with self.assertRaises(OSError):+ loud.send(fail_silently=False)++ def test_unreachable_broker_falls_back_to_direct_delivery(self):+ """+ A message is delivered even when it cannot be queued+ """+ with mock.patch(+ 'wger.core.tasks.send_email_task.delay',+ side_effect=OperationalError('no broker'),+ ):+ with self.assertLogs('wger.core.mail', level='ERROR'):+ sent = build_message().send()++ self.assertEqual(sent, 1)+ self.assertEqual(len(mail.outbox), 1)+++@override_settings(EMAIL_BACKEND=CELERY, EMAIL_DELIVERY_BACKEND=LOCMEM)+class SignupEmailTestCase(WgerTestCase):+ """+ Tests that the emails of the registration flow are queued+ """++ def test_confirmation_email_is_queued(self):+ """+ Registering does not talk to the mail server during the request+ """+ data = {+ 'username': 'mailtest',+ 'password1': 'quai8fai7Zae',+ 'password2': 'quai8fai7Zae',+ 'email': '[email protected]',+ }++ with mock.patch('wger.core.tasks.send_email_task.delay') as delay:+ response = self.client.post(reverse('core:user:registration'), data)++ self.assertEqual(response.status_code, 302)+ self.assertEqual(delay.call_count, 1)+ self.assertEqual(mail.outbox, [])++ # and the queued payload is the confirmation email+ payload = delay.call_args[0][0]+ self.assertEqual(payload['to'], ['[email protected]'])+ self.assertIn('Confirm', payload['subject'])diff --git a/wger/utils/mail.py b/wger/utils/mail.pynew file mode 100644index 000000000..7def1b283--- /dev/null+++ b/wger/utils/mail.py@@ -0,0 +1,83 @@+# This file is part of wger Workout Manager <https://github.com/wger-project>.+# Copyright (C) 2013 - 2021 wger Team+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <http://www.gnu.org/licenses/>.++"""+Turning email messages into something a celery task can carry.++See wger.core.mail for the backend that queues them and wger.core.tasks for+the task that delivers them.+"""++# Django+from django.conf import settings+from django.core.mail import (+ EmailMultiAlternatives,+ get_connection,+)+++def get_delivery_connection(fail_silently: bool = False):+ """+ Connection to the backend that actually talks to the mail server+ """+ return get_connection(+ backend=settings.EMAIL_DELIVERY_BACKEND,+ fail_silently=fail_silently,+ )+++def message_to_dict(message) -> dict | None:+ """+ Break a message down into the pieces a task can carry, or None if it+ contains something that would not survive the round trip.+ """+ if message.attachments:+ return None++ return {+ 'subject': message.subject,+ 'body': message.body,+ 'from_email': message.from_email,+ 'to': list(message.to),+ 'cc': list(message.cc),+ 'bcc': list(message.bcc),+ 'reply_to': list(message.reply_to),+ 'headers': dict(message.extra_headers),+ 'content_subtype': message.content_subtype,+ 'alternatives': [+ [content, mimetype] for content, mimetype in getattr(message, 'alternatives', [])+ ],+ }+++def message_from_dict(data: dict) -> EmailMultiAlternatives:+ """+ Rebuild a message from the pieces of message_to_dict+ """+ message = EmailMultiAlternatives(+ subject=data['subject'],+ body=data['body'],+ from_email=data['from_email'],+ to=data['to'],+ cc=data['cc'],+ bcc=data['bcc'],+ reply_to=data['reply_to'],+ headers=data['headers'],+ )+ message.content_subtype = data['content_subtype']+ for content, mimetype in data['alternatives']:+ message.attach_alternative(content, mimetype)+ return message