Self hosted FLOSS fitness/workout, nutrition and weight tracker
Fetch ingredient images in the background with celery
Roland Geider41dd35648eea
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.pyindex bf9115a4a..5113660b6 100644--- a/extras/scripts/create_ingredients_from_foodfacts.py+++ b/extras/scripts/create_ingredients_from_foodfacts.py@@ -25,7 +25,7 @@ django.setup()from django.conf import settings # noqa: E402from wger.nutrition.models import Ingredient # noqa: E402-from wger.nutrition.models.ingredient import Source # noqa: E402+from wger.nutrition.models.sources import Sourcefrom wger.core.models import Language # noqa: E402diff --git a/wger/__init__.py b/wger/__init__.pyindex 7d9b6d3a4..424daade4 100644--- a/wger/__init__.py+++ b/wger/__init__.py@@ -4,6 +4,9 @@:copyright: 2011, 2012 by OpenSlides team, see AUTHORS.:license: GNU GPL, see LICENSE for more details."""+# Local+from .celery import app as celery_app+MIN_APP_VERSION = (1, 5, 0, 'final', 1)diff --git a/wger/celery.py b/wger/celery.pyindex e0a791130..1564e05eb 100644--- a/wger/celery.py+++ b/wger/celery.py@@ -1,13 +1,22 @@+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <http://www.gnu.org/licenses/>."""-https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html+See https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html"""# Standard Libraryimport os-# Django-from django.conf import settings-# Third Partyfrom celery import Celery@@ -20,12 +29,4 @@ app = Celery("wger")app.config_from_object('django.conf:settings', namespace='CELERY')# discover and load tasks.py from all registered Django apps-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)---def divide(x, y):- # Standard Library- import time- time.sleep(5)- return x / y+app.autodiscover_tasks()diff --git a/wger/exercises/management/commands/download-exercise-images.py b/wger/exercises/management/commands/download-exercise-images.pyindex 1a4cca68b..ad00ee58e 100644--- a/wger/exercises/management/commands/download-exercise-images.py+++ b/wger/exercises/management/commands/download-exercise-images.py@@ -115,7 +115,12 @@ class Command(BaseCommand):except ExerciseImage.DoesNotExist:self.stdout.write(' Image not found in local DB, creating now...')retrieved_image = requests.get(image_data['image'], headers=headers)- image = ExerciseImage.from_json(exercise_base, retrieved_image, image_data, headers)+ image = ExerciseImage.from_json(+ exercise_base,+ retrieved_image,+ image_data,+ headers,+ )# Temporary files on windows don't support the delete attributeif os.name == 'nt':diff --git a/wger/nutrition/api/views.py b/wger/nutrition/api/views.pyindex 8ab23bd5d..f8b2d13b5 100644--- a/wger/nutrition/api/views.py+++ b/wger/nutrition/api/views.py@@ -166,7 +166,7 @@ def search(request):name__icontains=term,language__in=languages,status=Ingredient.STATUS_ACCEPTED,- )+ )[:30]for ingredient in ingredients:if hasattr(ingredient, 'image'):@@ -175,6 +175,7 @@ def search(request):t = get_thumbnailer(image_obj.image)thumbnail = t.get_thumbnail(aliases.get('micro_cropped')).urlelse:+ ingredient.get_image(request)image = Nonethumbnail = Nonediff --git a/wger/nutrition/management/commands/download-off-ingredient-images.py b/wger/nutrition/management/commands/download-off-ingredient-images.pyindex 9249ebc9b..21549ec2b 100644--- a/wger/nutrition/management/commands/download-off-ingredient-images.py+++ b/wger/nutrition/management/commands/download-off-ingredient-images.py@@ -15,24 +15,22 @@# You should have received a copy of the GNU Affero General Public License# Standard Library-import datetime# Djangofrom django.conf import settings-from django.contrib.sites.models import Site-from django.core import mailfrom django.core.management.base import BaseCommand-from django.template import loader-from django.utils import translation-from django.utils.translation import gettext_lazy as _# wger+from wger.nutrition.tasks import fetch_ingredient_imagefrom wger.nutrition.models import MealItemclass Command(BaseCommand):"""- Download images of all Open Food Facts ingredients that are used in a nutrition plan+ One off script++ This is intended to download all images from Open Food Facts for the all+ currently used ingredients"""help = 'Download images of all Open Food Facts ingredients that are used in a nutrition plan'@@ -47,7 +45,8 @@ class Command(BaseCommand):meal_item_counter = 0download_counter = 0for meal_item in meal_items:- if meal_item.ingredient.fetch_image():+ if meal_item.ingredient:+ fetch_ingredient_image.delay(meal_item.ingredient.pk)download_counter += 1meal_item_counter += 1diff --git a/wger/nutrition/models/__init__.py b/wger/nutrition/models/__init__.pyindex d91a98078..00e2a1f31 100644--- a/wger/nutrition/models/__init__.py+++ b/wger/nutrition/models/__init__.py@@ -23,4 +23,5 @@ from .log import LogItemfrom .meal import Mealfrom .meal_item import MealItemfrom .plan import NutritionPlan+from .sources import Sourcefrom .weight_unit import WeightUnitdiff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.pyindex 07eff9408..c5fa135ee 100644--- a/wger/nutrition/models/ingredient.py+++ b/wger/nutrition/models/ingredient.py@@ -16,11 +16,8 @@# Standard Libraryimport logging-import osimport uuid as uuidfrom decimal import Decimal-from enum import Enum-from typing import Optional# Djangofrom django.conf import settings@@ -42,12 +39,9 @@ from django.utils import translationfrom django.utils.text import slugifyfrom django.utils.translation import gettext_lazy as _-# Third Party-import requests-from requests.utils import default_user_agent-# wgerfrom wger.core.models import Language+from wger.nutrition.tasks import fetch_ingredient_imagefrom wger.utils.cache import cache_mapperfrom wger.utils.constants import TWOPLACESfrom wger.utils.managers import SubmissionManager@@ -57,20 +51,14 @@ from wger.utils.models import ()# Local-from ... import get_versionfrom ..consts import ENERGY_FACTOR-from .image import Imagefrom .ingredient_category import IngredientCategory+from .sources import Sourcelogger = logging.getLogger(__name__)-class Source(Enum):- WGER = 'wger'- OPEN_FOOD_FACTS = 'Open Food Facts'--class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):"""An ingredient, with some approximate nutrition values@@ -447,73 +435,18 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):if self.source_name == Source.OPEN_FOOD_FACTS.value:return f'https://world.openfoodfacts.org/product/{self.code}/'- def fetch_image(self):- """- Fetches the ingredient image from Open Food Facts servers if it is not available locally-- Returns the image if it was fetched- """- if hasattr(self, 'image'):- return-- if self.source_name != Source.OPEN_FOOD_FACTS.value:- return-- if not settings.WGER_SETTINGS['DOWNLOAD_FROM_OFF']:- return-- if settings.TESTING:- return-- # Everything looks fine, go ahead- logger.info(f'Trying to fetch image from OFF for {self.name} (UUID: {self.uuid})')- headers = {- 'User-agent':- default_user_agent(f'wger/{get_version()} - https://github.com/wger-project')- }-- # Fetch the product data- product_data = requests.get(self.source_url, headers=headers).json()- image_url: Optional[str] = product_data['product'].get('image_front_url')- if not image_url:- return-- image_data = product_data['product']['images']-- # Download the image file- downloaded_image: requests.Response = requests.get(image_url, headers=headers)- if downloaded_image.status_code != 200:- return-- # Parse the file name, looks something like this:- # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg- image_name: str = image_url.rpartition("/")[2].partition(".")[0]-- # Retrieve the uploader name- try:- image_id: str = image_data[image_name]['imgid']- uploader_name: str = image_data[image_id]['uploader']- except KeyError:- return-- # Save to DB- image_data: dict = {- 'image': os.path.basename(image_url),- 'license_author': uploader_name,- 'size': len(downloaded_image.content)- }- return Image.from_json(self, downloaded_image, image_data, headers, generate_uuid=True)-def get_image(self, request: HttpRequest):"""Returns the ingredient imageIf it is not available locally, it is fetched from Open Food Facts servers"""+if hasattr(self, 'image'):return self.imageif not request.user.is_authenticated:return- return self.fetch_image()+ # Let celery fetch the image+ fetch_ingredient_image.delay(self.pk)diff --git a/wger/nutrition/models/sources.py b/wger/nutrition/models/sources.pynew file mode 100644index 000000000..eb7d5d26e--- /dev/null+++ b/wger/nutrition/models/sources.py@@ -0,0 +1,7 @@+# Standard Library+from enum import Enum+++class Source(Enum):+ WGER = 'wger'+ OPEN_FOOD_FACTS = 'Open Food Facts'diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.pynew file mode 100644index 000000000..37b8aa7a7--- /dev/null+++ b/wger/nutrition/tasks.py@@ -0,0 +1,98 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Standard Library+import logging+import os+from typing import Optional++# Django+from django.conf import settings++# Third Party+import requests+from celery import shared_task+from requests.utils import default_user_agent++# wger+from wger import get_version+from wger.nutrition.models import Image+from wger.nutrition.models.sources import Source+++logger = logging.getLogger(__name__)+++@shared_task()+def fetch_ingredient_image(pk: int):+ """+ Fetches the ingredient image from Open Food Facts servers if it is not available locally++ Returns the image if it was fetched+ """+ # wger+ from wger.nutrition.models import Ingredient+ ingredient = Ingredient.objects.get(pk=pk)++ if hasattr(ingredient, 'image'):+ return++ if ingredient.source_name != Source.OPEN_FOOD_FACTS.value:+ return++ if not settings.WGER_SETTINGS['DOWNLOAD_FROM_OFF']:+ return++ if settings.TESTING:+ return++ # Everything looks fine, go ahead+ logger.info(f'Trying to fetch image from OFF for {ingredient.name} (UUID: {ingredient.uuid})')+ headers = {+ 'User-agent': default_user_agent(f'wger/{get_version()} - https://github.com/wger-project')+ }++ # Fetch the product data+ product_data = requests.get(ingredient.source_url, headers=headers).json()+ image_url: Optional[str] = product_data['product'].get('image_front_url')+ if not image_url:+ return++ image_data = product_data['product']['images']++ # Download the image file+ response = requests.get(image_url, headers=headers)+ if response.status_code != 200:+ logger.info('An error occurred!')+ return++ # Parse the file name, looks something like this:+ # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg+ image_name: str = image_url.rpartition("/")[2].partition(".")[0]++ # Retrieve the uploader name+ try:+ image_id: str = image_data[image_name]['imgid']+ uploader_name: str = image_data[image_id]['uploader']+ except KeyError:+ return++ # Save to DB+ image_data: dict = {+ 'image': os.path.basename(image_url),+ 'license_author': uploader_name,+ 'size': len(response.content)+ }+ Image.from_json(ingredient, response, image_data, headers, generate_uuid=True)+ logger.info('Image successfully saved')