Self hosted FLOSS fitness/workout, nutrition and weight tracker
Small refactoring of ingredient and image import logic
Roland Geiderc47885f07e0d
diff --git a/wger/nutrition/api/serializers.py b/wger/nutrition/api/serializers.pyindex bcc9676d5..8830fefea 100644--- a/wger/nutrition/api/serializers.py+++ b/wger/nutrition/api/serializers.py@@ -119,6 +119,8 @@ class IngredientSerializer(serializers.ModelSerializer):'source_url','code','name',+ 'common_name',+ 'brand','created','last_update','last_imported',diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.pyindex ffefbbdf7..6991f6fc3 100644--- a/wger/nutrition/dataclasses.py+++ b/wger/nutrition/dataclasses.py@@ -48,6 +48,8 @@ class IngredientData:license_author: strlicense_title: strlicense_object_url: str+ license_derivative_source_url: str = ''+ license_author_url: str = ''def sanity_checks(self):if not self.name:diff --git a/wger/nutrition/extract_info/__init__.py b/wger/nutrition/extract_info/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/wger/nutrition/off.py b/wger/nutrition/extract_info/off.pysimilarity index 100%rename from wger/nutrition/off.pyrename to wger/nutrition/extract_info/off.pydiff --git a/wger/nutrition/usda.py b/wger/nutrition/extract_info/usda.pysimilarity index 100%rename from wger/nutrition/usda.pyrename to wger/nutrition/extract_info/usda.pydiff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.pynew file mode 100644index 000000000..7f140f711--- /dev/null+++ b/wger/nutrition/extract_info/wger.py@@ -0,0 +1,83 @@+# This file is part of wger Workout Manager <https://github.com/wger-project>.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <http://www.gnu.org/licenses/>.++# wger+from wger.nutrition.dataclasses import IngredientData+from wger.utils.constants import ODBL_LICENSE_ID+++def extract_info_from_wger_api(product_data: dict) -> IngredientData:+ # Basics+ name = product_data.get('name')+ common_name = product_data.get('common_name', '')+ code = product_data['code']+ energy = float(product_data['energy'])+ protein = float(product_data['protein'])+ carbs = float(product_data['carbohydrates'])+ fat = float(product_data['fat'])+ language = product_data['language']++ # Optional+ sodium = product_data.get('sodium', None)+ sodium = float(sodium) if sodium is not None else None++ saturated_fat = product_data.get('fat_saturated', None)+ saturated_fat = float(saturated_fat) if saturated_fat is not None else None++ sugars = product_data.get('carbohydrates_sugar', None)+ sugars = float(sugars) if sugars is not None else None++ fiber = product_data.get('fiber', None)+ fiber = float(fiber) if fiber is not None else None++ brand = product_data.get('brand', '')++ # License and author info+ source_name = product_data.get('source_name', '')+ source_url = product_data.get('source_url', '')++ license_id = product_data.get('license', ODBL_LICENSE_ID)+ license_title = product_data.get('license_title', '')+ license_object_url = product_data.get('license_object_url', '')+ license_authors = product_data.get('authors', '')+ license_author_url = product_data.get('license_author_url', '')+ license_derivative_source_url = product_data.get('license_derivative_source_url', '')++ ingredient_data = IngredientData(+ remote_id=code,+ name=name,+ language_id=language,+ energy=energy,+ protein=protein,+ carbohydrates=carbs,+ carbohydrates_sugar=sugars,+ fat=fat,+ fat_saturated=saturated_fat,+ fiber=fiber,+ sodium=sodium,+ code=code,+ source_name=source_name,+ source_url=source_url,+ common_name=common_name,+ brand=brand,+ license_id=license_id,+ license_title=license_title,+ license_object_url=license_object_url,+ license_author=license_authors,+ license_author_url=license_author_url,+ license_derivative_source_url=license_derivative_source_url,+ )+ ingredient_data.sanity_checks()+ return ingredient_datadiff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.pyindex 54fde1d16..449764c7d 100644--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@ -21,12 +21,11 @@ import requests# wgerfrom wger.core.models import Language+from wger.nutrition.extract_info.off import extract_info_from_offfrom wger.nutrition.management.products import (ImportProductCommand,Mode,)-from wger.nutrition.off import extract_info_from_off-logger = logging.getLogger(__name__)diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.pyindex 50087128c..145cbcb16 100644--- a/wger/nutrition/management/commands/import-usda-products.py+++ b/wger/nutrition/management/commands/import-usda-products.py@@ -21,14 +21,13 @@ from zipfile import ZipFile# wgerfrom wger.core.models import Language+from wger.nutrition.extract_info.usda import extract_info_from_usdafrom wger.nutrition.management.products import (ImportProductCommand,Mode,)-from wger.nutrition.usda import extract_info_from_usdafrom wger.utils.constants import ENGLISH_SHORT_NAME-logger = logging.getLogger(__name__)@@ -47,8 +46,8 @@ class Command(ImportProductCommand):dest='dataset',type=str,help='What dataset to download, this value will be appended to '- '"https://fdc.nal.usda.gov/fdc-datasets/". Consult '- 'https://fdc.nal.usda.gov/download-datasets.html for current file names',+ '"https://fdc.nal.usda.gov/fdc-datasets/". Consult '+ 'https://fdc.nal.usda.gov/download-datasets.html for current file names',)def handle(self, **options):diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.pyindex a5cec08b2..9aeaf8f0c 100644--- a/wger/nutrition/models/ingredient.py+++ b/wger/nutrition/models/ingredient.py@@ -35,7 +35,6 @@ from django.http import HttpRequestfrom django.urls import reversefrom django.utils.text import slugifyfrom django.utils.translation import gettext_lazy as _-# Third Partyfrom openfoodfacts import APIfrom requests import (@@ -59,7 +58,6 @@ from wger.utils.language import load_languagefrom wger.utils.models import AbstractLicenseModelfrom wger.utils.requests import wger_user_agent-logger = logging.getLogger(__name__)@@ -346,16 +344,16 @@ class Ingredient(AbstractLicenseModel, models.Model):equal = Trueif isinstance(other, self.__class__):for i in (- 'carbohydrates',- 'carbohydrates_sugar',- 'creation_date',- 'energy',- 'fat',- 'fat_saturated',- 'fiber',- 'name',- 'protein',- 'sodium',+ 'carbohydrates',+ 'carbohydrates_sugar',+ 'creation_date',+ 'energy',+ 'fat',+ 'fat_saturated',+ 'fiber',+ 'name',+ 'protein',+ 'sodium',):if (hasattr(self, i)@@ -436,7 +434,7 @@ class Ingredient(AbstractLicenseModel, models.Model):Searches OFF by barcode and creates a local ingredient from the result"""# wger- from wger.nutrition.off import extract_info_from_off+ from wger.nutrition.extract_info.off import extract_info_from_offlogger.info(f'Searching for ingredient {code} in OFF')try:diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex 3db1d9031..8df64caa5 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -20,13 +20,12 @@ from typing import (Optional,)+# Third Party+import requests# Djangofrom django.conf import settingsfrom django.db import IntegrityErrorfrom django.utils import timezone--# Third Party-import requestsfrom openfoodfacts.images import (AWS_S3_BASE_URL,generate_image_path,@@ -38,6 +37,7 @@ from wger.nutrition.api.endpoints import (IMAGE_ENDPOINT,INGREDIENTS_ENDPOINT,)+from wger.nutrition.extract_info.wger import extract_info_from_wger_apifrom wger.nutrition.models import (Image,Ingredient,@@ -56,7 +56,6 @@ from wger.utils.requests import ()from wger.utils.url import make_uri-logger = logging.getLogger(__name__)@@ -67,25 +66,31 @@ def fetch_ingredient_image(pk: int):try:ingredient = Ingredient.objects.get(pk=pk)except Ingredient.DoesNotExist:- logger.info(f'Ingredient with ID {pk} does not exist')+ logger.debug(f'Ingredient with ID {pk} does not exist')returnif hasattr(ingredient, 'image'):+ # logger.debug(f'Ingredient {pk} already has an image, skipping...')returnif ingredient.source_name != Source.OPEN_FOOD_FACTS.value:+ # logger.debug(f'Ingredient {pk} is not from Open Food Facts, skipping...')returnif not ingredient.source_url:+ # logger.debug(f'Ingredient {pk} does not have a source URL, skipping...')returnif (ingredient.last_image_checkand (- ingredient.last_image_check + settings.WGER_SETTINGS['INGREDIENT_IMAGE_CHECK_INTERVAL']- )+ ingredient.last_image_check + settings.WGER_SETTINGS['INGREDIENT_IMAGE_CHECK_INTERVAL']+ )> timezone.now()):+ # logger.debug(+ # f'Last image check for ingredient {pk} is too recent'+ # f' ({ingredient.last_image_check}), skipping...')returnif settings.TESTING:@@ -105,7 +110,7 @@ def fetch_image_from_wger_instance(ingredient):logger.info(f'Trying to fetch image from WGER for {ingredient.name} (UUID: {ingredient.uuid})')result = requests.get(url, headers=wger_headers()).json()if result['count'] == 0:- logger.info('No ingredient matches UUID in the remote server')+ logger.info('No image for ingredient found in the remote server')returnimage_data = result['results'][0]@@ -116,7 +121,9 @@ def fetch_image_from_wger_instance(ingredient):returnexcept Image.DoesNotExist:retrieved_image = requests.get(image_data['image'], headers=wger_headers())- Image.from_json(ingredient, retrieved_image, image_data)+ image = Image.from_json(ingredient, retrieved_image, image_data)+ image.ingredient.last_image_check = timezone.now()+ image.ingredient.save()def fetch_image_from_off(ingredient: Ingredient):@@ -258,28 +265,11 @@ def sync_ingredients(uuid = data['uuid']name = data['name']+ ingredient_data = extract_info_from_wger_api(data).dict()+ ingredient_data['uuid'] = uuid+ingredient, created = Ingredient.objects.update_or_create(- uuid=uuid,- defaults={- 'name': name,- 'code': data['code'],- 'language_id': data['language'],- 'created': data['created'],- 'license_id': data['license'],- 'license_object_url': data['license_object_url'],- 'license_author': data['license_author_url'],- 'license_author_url': data['license_author_url'],- 'license_title': data['license_title'],- 'license_derivative_source_url': data['license_derivative_source_url'],- 'energy': data['energy'],- 'carbohydrates': data['carbohydrates'],- 'carbohydrates_sugar': data['carbohydrates_sugar'],- 'fat': data['fat'],- 'fat_saturated': data['fat_saturated'],- 'protein': data['protein'],- 'fiber': data['fiber'],- 'sodium': data['sodium'],- },+ uuid=uuid, defaults=ingredient_data)print_fn(f'{"created" if created else "updated"} ingredient {uuid} - {name}')diff --git a/wger/nutrition/tests/test_helpers.py b/wger/nutrition/tests/test_helpers.pyindex 59c59ec2d..65acf14cd 100644--- a/wger/nutrition/tests/test_helpers.py+++ b/wger/nutrition/tests/test_helpers.py@@ -17,7 +17,6 @@from django.test import SimpleTestCase# wger-from wger.nutrition.dataclasses import IngredientDatafrom wger.nutrition.helpers import (change_html_entities_to_human_readable,remove_problematic_characters,diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.pyindex 5000cb0da..093106b70 100644--- a/wger/nutrition/tests/test_off.py+++ b/wger/nutrition/tests/test_off.py@@ -18,7 +18,7 @@ from django.test import SimpleTestCase# wgerfrom wger.nutrition.dataclasses import IngredientData-from wger.nutrition.off import extract_info_from_off+from wger.nutrition.extract_info.off import extract_info_from_offfrom wger.utils.constants import ODBL_LICENSE_IDdiff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.pyindex 8aab7d43c..7bba179d4 100644--- a/wger/nutrition/tests/test_usda.py+++ b/wger/nutrition/tests/test_usda.py@@ -18,7 +18,7 @@ from django.test import SimpleTestCase# wgerfrom wger.nutrition.dataclasses import IngredientData-from wger.nutrition.usda import (+from wger.nutrition.extract_info.usda import (convert_to_grams,extract_info_from_usda,)@@ -226,7 +226,7 @@ class ExtractInfoFromUSDATestCase(SimpleTestCase):brand='',license_id=CC_0_LICENSE_ID,license_author='U.S. Department of Agriculture, Agricultural Research Service, '- 'Beltsville Human Nutrition Research Center. FoodData Central.',+ 'Beltsville Human Nutrition Research Center. FoodData Central.',license_title='Foo With Chocolate',license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients',)