Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Make import logic more robust

Roland Geider3ea0b7754b69
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.py
index 00c17ddea..e1af8af2e 100644
--- a/extras/scripts/create_ingredients_from_foodfacts.py
+++ b/extras/scripts/create_ingredients_from_foodfacts.py
@@ -140,41 +140,21 @@ print('***********************************')
print(languages.keys())
print('***********************************')
-required_top_level = [
- 'product_name',
- 'code',
- 'nutriments'
-]
-required_nutriments = [
- 'energy-kcal_100g',
- 'proteins_100g',
- 'carbohydrates_100g',
- 'sugars_100g',
- 'fat_100g',
- 'saturated-fat_100g'
-]
-
for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'complete': 1}):
- if not all(req in product for req in required_top_level):
+ try:
+ ingredient_data = extract_info_from_off(product, languages[product['lang']])
+ except KeyError as e:
+ # print('--> KeyError while extracting info from OFF', e)
counter['skipped'] += 1
continue
- if not all(req in product['nutriments'] for req in required_nutriments):
- # print(f'-> skipping due to required nutriments')
- counter['skipped'] += 1
- continue
-
- ingredient_data = extract_info_from_off(product, languages[product['lang']])
-
# Some products have no name or name is too long, skipping
- name = ingredient_data['name']
- if not name or len(name) > 200:
+ if not ingredient_data['name']:
counter['skipped'] += 1
continue
- common_name = ingredient_data['common_name']
- if not common_name or len(common_name) > 200:
+ if not ingredient_data['common_name']:
counter['skipped'] += 1
continue
diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py
index ed36f88ea..55bb4d20a 100644
--- a/wger/nutrition/models/ingredient.py
+++ b/wger/nutrition/models/ingredient.py
@@ -464,9 +464,19 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):
result = openfoodfacts.products.get_product(code)
if result['status'] != OFF_SEARCH_PRODUCT_FOUND:
return None
-
product = result['product']
- ingredient_data = extract_info_from_off(product, load_language(product['lang']))
+
+ try:
+ ingredient_data = extract_info_from_off(product, load_language(product['lang']))
+ except KeyError:
+ return None
+
+ if not ingredient_data['name']:
+ return
+
+ if not ingredient_data['common_name']:
+ return
+
ingredient = cls(**ingredient_data)
ingredient.save()
return ingredient
diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py
index f1082e808..e79bf63a9 100644
--- a/wger/nutrition/off.py
+++ b/wger/nutrition/off.py
@@ -18,11 +18,38 @@ from wger.nutrition.models import Source
from wger.utils.constants import CC_ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel
+OFF_REQUIRED_TOP_LEVEL = [
+ 'product_name',
+ 'code',
+ 'nutriments'
+]
+OFF_REQUIRED_NUTRIMENTS = [
+ 'energy-kcal_100g',
+ 'proteins_100g',
+ 'carbohydrates_100g',
+ 'sugars_100g',
+ 'fat_100g',
+ 'saturated-fat_100g'
+]
+
def extract_info_from_off(product, language):
+ if not all(req in product for req in OFF_REQUIRED_TOP_LEVEL):
+ raise KeyError(f'Missing required top-level key')
+
+ if not all(req in product['nutriments'] for req in OFF_REQUIRED_NUTRIMENTS):
+ raise KeyError(f'Missing required nutrition key')
+
# Basics
name = product['product_name']
+ if len(name) > 200:
+ name = name[:200]
+
+ common_name = product.get('generic_name', None)
+ if len(common_name) > 200:
+ common_name = common_name[:200]
+
code = product['code']
energy = product['nutriments']['energy-kcal_100g']
protein = product['nutriments']['proteins_100g']
@@ -34,7 +61,6 @@ def extract_info_from_off(product, language):
# these are optional
sodium = product['nutriments'].get('sodium_100g', None)
fibre = product['nutriments'].get('fiber_100g', None)
- common_name = product.get('generic_name', None)
brand = product.get('brands', None)
# License and author info
diff --git a/wger/nutrition/tests/test_ingredient.py b/wger/nutrition/tests/test_ingredient.py
index ca86d2d2b..ca5f34745 100644
--- a/wger/nutrition/tests/test_ingredient.py
+++ b/wger/nutrition/tests/test_ingredient.py
@@ -462,32 +462,34 @@ class IngredientModelTestCase(WgerTestCase):
Tests the ingredient model functions
"""
- off_response = {
- 'status': OFF_SEARCH_PRODUCT_FOUND,
- 'product': {
- 'code': '1234',
- 'lang': 'de',
- 'product_name': 'Foo with chocolate',
- 'generic_name': 'Foo with chocolate, 250g package',
- 'brands': 'The bar company',
- 'editors_tags': ['open food facts', 'MrX'],
- 'nutriments': {
- 'energy-kcal_100g': 120,
- 'proteins_100g': 10,
- 'carbohydrates_100g': 20,
- 'sugars_100g': 30,
- 'fat_100g': 40,
- 'saturated-fat_100g': 11,
- 'sodium_100g': 5,
- 'fiber_100g': None
- },
+ def setUp(self):
+ super().setUp()
+ self.off_response = {
+ 'status': OFF_SEARCH_PRODUCT_FOUND,
+ 'product': {
+ 'code': '1234',
+ 'lang': 'de',
+ 'product_name': 'Foo with chocolate',
+ 'generic_name': 'Foo with chocolate, 250g package',
+ 'brands': 'The bar company',
+ 'editors_tags': ['open food facts', 'MrX'],
+ 'nutriments': {
+ 'energy-kcal_100g': 120,
+ 'proteins_100g': 10,
+ 'carbohydrates_100g': 20,
+ 'sugars_100g': 30,
+ 'fat_100g': 40,
+ 'saturated-fat_100g': 11,
+ 'sodium_100g': 5,
+ 'fiber_100g': None
+ },
+ }
}
- }
- off_response_no_results = {
- 'status': OFF_SEARCH_PRODUCT_NOT_FOUND,
- 'status_verbose': 'product not found'
- }
+ self.off_response_no_results = {
+ 'status': OFF_SEARCH_PRODUCT_NOT_FOUND,
+ 'status_verbose': 'product not found'
+ }
@patch('openfoodfacts.products.get_product')
def test_fetch_from_off_success(self, mock_get_product):
@@ -510,6 +512,43 @@ class IngredientModelTestCase(WgerTestCase):
self.assertEqual(ingredient.brand, 'The bar company')
self.assertEqual(ingredient.license_author, 'open food facts, MrX')
+ @patch('openfoodfacts.products.get_product')
+ def test_fetch_from_off_success_long_name(self, mock_get_product):
+ """
+ Tests creating an ingredient from OFF - name gets truncated
+ """
+ self.off_response['product']['product_name'] = """
+ The Shiba Inu (柴犬, Japanese: [ɕiba inɯ]) is a breed of hunting dog from Japan. A
+ small-to-medium breed, it is the smallest of the six original and distinct spitz
+ breeds of dog native to Japan.[1] Its name literally translates to "brushwood dog",
+ as it is used to flush game."""
+ mock_get_product.return_value = self.off_response
+
+ ingredient = Ingredient.fetch_ingredient_from_off('1234')
+ self.assertEqual(len(ingredient.name), 200)
+
+ @patch('openfoodfacts.products.get_product')
+ def test_fetch_from_off_key_missing_1(self, mock_get_product):
+ """
+ Tests creating an ingredient from OFF - missing key in nutriments
+ """
+ del self.off_response['product']['nutriments']['energy-kcal_100g']
+ mock_get_product.return_value = self.off_response
+
+ ingredient = Ingredient.fetch_ingredient_from_off('1234')
+ self.assertIsNone(ingredient)
+
+ @patch('openfoodfacts.products.get_product')
+ def test_fetch_from_off_key_missing_2(self, mock_get_product):
+ """
+ Tests creating an ingredient from OFF - missing name
+ """
+ del self.off_response['product']['product_name']
+ mock_get_product.return_value = self.off_response
+
+ ingredient = Ingredient.fetch_ingredient_from_off('1234')
+ self.assertIsNone(ingredient)
+
@patch('openfoodfacts.products.get_product')
def test_fetch_from_off_no_results(self, mock_get_product):
"""