Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Fetch ingredient images in the background with celery

Roland Geider41dd35648eea
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.py
index bf9115a4a..5113660b6 100644
--- a/extras/scripts/create_ingredients_from_foodfacts.py
+++ b/extras/scripts/create_ingredients_from_foodfacts.py
@@ -25,7 +25,7 @@ django.setup()
from django.conf import settings # noqa: E402
from wger.nutrition.models import Ingredient # noqa: E402
-from wger.nutrition.models.ingredient import Source # noqa: E402
+from wger.nutrition.models.sources import Source
from wger.core.models import Language # noqa: E402
diff --git a/wger/__init__.py b/wger/__init__.py
index 7d9b6d3a4..424daade4 100644
--- a/wger/__init__.py
+++ b/wger/__init__.py
@@ -4,6 +4,9 @@
:copyright: 2011, 2012 by OpenSlides team, see AUTHORS.
:license: GNU GPL, see LICENSE for more details.
"""
+# Local
+from .celery import app as celery_app
+
MIN_APP_VERSION = (1, 5, 0, 'final', 1)
diff --git a/wger/celery.py b/wger/celery.py
index e0a791130..1564e05eb 100644
--- a/wger/celery.py
+++ b/wger/celery.py
@@ -1,13 +1,22 @@
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
-https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html
+See https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html
"""
# Standard Library
import os
-# Django
-from django.conf import settings
-
# Third Party
from celery import Celery
@@ -20,12 +29,4 @@ app = Celery("wger")
app.config_from_object('django.conf:settings', namespace='CELERY')
# discover and load tasks.py from all registered Django apps
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
-
-
-def divide(x, y):
- # Standard Library
- import time
- time.sleep(5)
- return x / y
+app.autodiscover_tasks()
diff --git a/wger/exercises/management/commands/download-exercise-images.py b/wger/exercises/management/commands/download-exercise-images.py
index 1a4cca68b..ad00ee58e 100644
--- a/wger/exercises/management/commands/download-exercise-images.py
+++ b/wger/exercises/management/commands/download-exercise-images.py
@@ -115,7 +115,12 @@ class Command(BaseCommand):
except ExerciseImage.DoesNotExist:
self.stdout.write(' Image not found in local DB, creating now...')
retrieved_image = requests.get(image_data['image'], headers=headers)
- image = ExerciseImage.from_json(exercise_base, retrieved_image, image_data, headers)
+ image = ExerciseImage.from_json(
+ exercise_base,
+ retrieved_image,
+ image_data,
+ headers,
+ )
# Temporary files on windows don't support the delete attribute
if os.name == 'nt':
diff --git a/wger/nutrition/api/views.py b/wger/nutrition/api/views.py
index 8ab23bd5d..f8b2d13b5 100644
--- a/wger/nutrition/api/views.py
+++ b/wger/nutrition/api/views.py
@@ -166,7 +166,7 @@ def search(request):
name__icontains=term,
language__in=languages,
status=Ingredient.STATUS_ACCEPTED,
- )
+ )[:30]
for ingredient in ingredients:
if hasattr(ingredient, 'image'):
@@ -175,6 +175,7 @@ def search(request):
t = get_thumbnailer(image_obj.image)
thumbnail = t.get_thumbnail(aliases.get('micro_cropped')).url
else:
+ ingredient.get_image(request)
image = None
thumbnail = None
diff --git a/wger/nutrition/management/commands/download-off-ingredient-images.py b/wger/nutrition/management/commands/download-off-ingredient-images.py
index 9249ebc9b..21549ec2b 100644
--- a/wger/nutrition/management/commands/download-off-ingredient-images.py
+++ b/wger/nutrition/management/commands/download-off-ingredient-images.py
@@ -15,24 +15,22 @@
# You should have received a copy of the GNU Affero General Public License
# Standard Library
-import datetime
# Django
from django.conf import settings
-from django.contrib.sites.models import Site
-from django.core import mail
from django.core.management.base import BaseCommand
-from django.template import loader
-from django.utils import translation
-from django.utils.translation import gettext_lazy as _
# wger
+from wger.nutrition.tasks import fetch_ingredient_image
from wger.nutrition.models import MealItem
class Command(BaseCommand):
"""
- Download images of all Open Food Facts ingredients that are used in a nutrition plan
+ One off script
+
+ This is intended to download all images from Open Food Facts for the all
+ currently used ingredients
"""
help = 'Download images of all Open Food Facts ingredients that are used in a nutrition plan'
@@ -47,7 +45,8 @@ class Command(BaseCommand):
meal_item_counter = 0
download_counter = 0
for meal_item in meal_items:
- if meal_item.ingredient.fetch_image():
+ if meal_item.ingredient:
+ fetch_ingredient_image.delay(meal_item.ingredient.pk)
download_counter += 1
meal_item_counter += 1
diff --git a/wger/nutrition/models/__init__.py b/wger/nutrition/models/__init__.py
index d91a98078..00e2a1f31 100644
--- a/wger/nutrition/models/__init__.py
+++ b/wger/nutrition/models/__init__.py
@@ -23,4 +23,5 @@ from .log import LogItem
from .meal import Meal
from .meal_item import MealItem
from .plan import NutritionPlan
+from .sources import Source
from .weight_unit import WeightUnit
diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py
index 07eff9408..c5fa135ee 100644
--- a/wger/nutrition/models/ingredient.py
+++ b/wger/nutrition/models/ingredient.py
@@ -16,11 +16,8 @@
# Standard Library
import logging
-import os
import uuid as uuid
from decimal import Decimal
-from enum import Enum
-from typing import Optional
# Django
from django.conf import settings
@@ -42,12 +39,9 @@ from django.utils import translation
from django.utils.text import slugify
from django.utils.translation import gettext_lazy as _
-# Third Party
-import requests
-from requests.utils import default_user_agent
-
# wger
from wger.core.models import Language
+from wger.nutrition.tasks import fetch_ingredient_image
from wger.utils.cache import cache_mapper
from wger.utils.constants import TWOPLACES
from wger.utils.managers import SubmissionManager
@@ -57,20 +51,14 @@ from wger.utils.models import (
)
# Local
-from ... import get_version
from ..consts import ENERGY_FACTOR
-from .image import Image
from .ingredient_category import IngredientCategory
+from .sources import Source
logger = logging.getLogger(__name__)
-class Source(Enum):
- WGER = 'wger'
- OPEN_FOOD_FACTS = 'Open Food Facts'
-
-
class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):
"""
An ingredient, with some approximate nutrition values
@@ -447,73 +435,18 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):
if self.source_name == Source.OPEN_FOOD_FACTS.value:
return f'https://world.openfoodfacts.org/product/{self.code}/'
- def fetch_image(self):
- """
- Fetches the ingredient image from Open Food Facts servers if it is not available locally
-
- Returns the image if it was fetched
- """
- if hasattr(self, 'image'):
- return
-
- if self.source_name != Source.OPEN_FOOD_FACTS.value:
- return
-
- if not settings.WGER_SETTINGS['DOWNLOAD_FROM_OFF']:
- return
-
- if settings.TESTING:
- return
-
- # Everything looks fine, go ahead
- logger.info(f'Trying to fetch image from OFF for {self.name} (UUID: {self.uuid})')
- headers = {
- 'User-agent':
- default_user_agent(f'wger/{get_version()} - https://github.com/wger-project')
- }
-
- # Fetch the product data
- product_data = requests.get(self.source_url, headers=headers).json()
- image_url: Optional[str] = product_data['product'].get('image_front_url')
- if not image_url:
- return
-
- image_data = product_data['product']['images']
-
- # Download the image file
- downloaded_image: requests.Response = requests.get(image_url, headers=headers)
- if downloaded_image.status_code != 200:
- return
-
- # Parse the file name, looks something like this:
- # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg
- image_name: str = image_url.rpartition("/")[2].partition(".")[0]
-
- # Retrieve the uploader name
- try:
- image_id: str = image_data[image_name]['imgid']
- uploader_name: str = image_data[image_id]['uploader']
- except KeyError:
- return
-
- # Save to DB
- image_data: dict = {
- 'image': os.path.basename(image_url),
- 'license_author': uploader_name,
- 'size': len(downloaded_image.content)
- }
- return Image.from_json(self, downloaded_image, image_data, headers, generate_uuid=True)
-
def get_image(self, request: HttpRequest):
"""
Returns the ingredient image
If it is not available locally, it is fetched from Open Food Facts servers
"""
+
if hasattr(self, 'image'):
return self.image
if not request.user.is_authenticated:
return
- return self.fetch_image()
+ # Let celery fetch the image
+ fetch_ingredient_image.delay(self.pk)
diff --git a/wger/nutrition/models/sources.py b/wger/nutrition/models/sources.py
new file mode 100644
index 000000000..eb7d5d26e
--- /dev/null
+++ b/wger/nutrition/models/sources.py
@@ -0,0 +1,7 @@
+# Standard Library
+from enum import Enum
+
+
+class Source(Enum):
+ WGER = 'wger'
+ OPEN_FOOD_FACTS = 'Open Food Facts'
diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.py
new file mode 100644
index 000000000..37b8aa7a7
--- /dev/null
+++ b/wger/nutrition/tasks.py
@@ -0,0 +1,98 @@
+# This file is part of wger Workout Manager.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+
+# Standard Library
+import logging
+import os
+from typing import Optional
+
+# Django
+from django.conf import settings
+
+# Third Party
+import requests
+from celery import shared_task
+from requests.utils import default_user_agent
+
+# wger
+from wger import get_version
+from wger.nutrition.models import Image
+from wger.nutrition.models.sources import Source
+
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task()
+def fetch_ingredient_image(pk: int):
+ """
+ Fetches the ingredient image from Open Food Facts servers if it is not available locally
+
+ Returns the image if it was fetched
+ """
+ # wger
+ from wger.nutrition.models import Ingredient
+ ingredient = Ingredient.objects.get(pk=pk)
+
+ if hasattr(ingredient, 'image'):
+ return
+
+ if ingredient.source_name != Source.OPEN_FOOD_FACTS.value:
+ return
+
+ if not settings.WGER_SETTINGS['DOWNLOAD_FROM_OFF']:
+ return
+
+ if settings.TESTING:
+ return
+
+ # Everything looks fine, go ahead
+ logger.info(f'Trying to fetch image from OFF for {ingredient.name} (UUID: {ingredient.uuid})')
+ headers = {
+ 'User-agent': default_user_agent(f'wger/{get_version()} - https://github.com/wger-project')
+ }
+
+ # Fetch the product data
+ product_data = requests.get(ingredient.source_url, headers=headers).json()
+ image_url: Optional[str] = product_data['product'].get('image_front_url')
+ if not image_url:
+ return
+
+ image_data = product_data['product']['images']
+
+ # Download the image file
+ response = requests.get(image_url, headers=headers)
+ if response.status_code != 200:
+ logger.info('An error occurred!')
+ return
+
+ # Parse the file name, looks something like this:
+ # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg
+ image_name: str = image_url.rpartition("/")[2].partition(".")[0]
+
+ # Retrieve the uploader name
+ try:
+ image_id: str = image_data[image_name]['imgid']
+ uploader_name: str = image_data[image_id]['uploader']
+ except KeyError:
+ return
+
+ # Save to DB
+ image_data: dict = {
+ 'image': os.path.basename(image_url),
+ 'license_author': uploader_name,
+ 'size': len(response.content)
+ }
+ Image.from_json(ingredient, response, image_data, headers, generate_uuid=True)
+ logger.info('Image successfully saved')