Self hosted FLOSS fitness/workout, nutrition and weight tracker
Make import logic more robust
Roland Geider3ea0b7754b69
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.pyindex 00c17ddea..e1af8af2e 100644--- a/extras/scripts/create_ingredients_from_foodfacts.py+++ b/extras/scripts/create_ingredients_from_foodfacts.py@@ -140,41 +140,21 @@ print('***********************************')print(languages.keys())print('***********************************')-required_top_level = [- 'product_name',- 'code',- 'nutriments'-]-required_nutriments = [- 'energy-kcal_100g',- 'proteins_100g',- 'carbohydrates_100g',- 'sugars_100g',- 'fat_100g',- 'saturated-fat_100g'-]-for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'complete': 1}):- if not all(req in product for req in required_top_level):+ try:+ ingredient_data = extract_info_from_off(product, languages[product['lang']])+ except KeyError as e:+ # print('--> KeyError while extracting info from OFF', e)counter['skipped'] += 1continue- if not all(req in product['nutriments'] for req in required_nutriments):- # print(f'-> skipping due to required nutriments')- counter['skipped'] += 1- continue-- ingredient_data = extract_info_from_off(product, languages[product['lang']])-# Some products have no name or name is too long, skipping- name = ingredient_data['name']- if not name or len(name) > 200:+ if not ingredient_data['name']:counter['skipped'] += 1continue- common_name = ingredient_data['common_name']- if not common_name or len(common_name) > 200:+ if not ingredient_data['common_name']:counter['skipped'] += 1continuediff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.pyindex ed36f88ea..55bb4d20a 100644--- a/wger/nutrition/models/ingredient.py+++ b/wger/nutrition/models/ingredient.py@@ -464,9 +464,19 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):result = openfoodfacts.products.get_product(code)if result['status'] != OFF_SEARCH_PRODUCT_FOUND:return None-product = result['product']- ingredient_data = extract_info_from_off(product, load_language(product['lang']))++ try:+ ingredient_data = extract_info_from_off(product, load_language(product['lang']))+ except KeyError:+ return None++ if not ingredient_data['name']:+ return++ if not ingredient_data['common_name']:+ return+ingredient = cls(**ingredient_data)ingredient.save()return ingredientdiff --git a/wger/nutrition/off.py b/wger/nutrition/off.pyindex f1082e808..e79bf63a9 100644--- a/wger/nutrition/off.py+++ b/wger/nutrition/off.py@@ -18,11 +18,38 @@ from wger.nutrition.models import Sourcefrom wger.utils.constants import CC_ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel+OFF_REQUIRED_TOP_LEVEL = [+ 'product_name',+ 'code',+ 'nutriments'+]+OFF_REQUIRED_NUTRIMENTS = [+ 'energy-kcal_100g',+ 'proteins_100g',+ 'carbohydrates_100g',+ 'sugars_100g',+ 'fat_100g',+ 'saturated-fat_100g'+]+def extract_info_from_off(product, language):+ if not all(req in product for req in OFF_REQUIRED_TOP_LEVEL):+ raise KeyError(f'Missing required top-level key')++ if not all(req in product['nutriments'] for req in OFF_REQUIRED_NUTRIMENTS):+ raise KeyError(f'Missing required nutrition key')+# Basicsname = product['product_name']+ if len(name) > 200:+ name = name[:200]++ common_name = product.get('generic_name', None)+ if len(common_name) > 200:+ common_name = common_name[:200]+code = product['code']energy = product['nutriments']['energy-kcal_100g']protein = product['nutriments']['proteins_100g']@@ -34,7 +61,6 @@ def extract_info_from_off(product, language):# these are optionalsodium = product['nutriments'].get('sodium_100g', None)fibre = product['nutriments'].get('fiber_100g', None)- common_name = product.get('generic_name', None)brand = product.get('brands', None)# License and author infodiff --git a/wger/nutrition/tests/test_ingredient.py b/wger/nutrition/tests/test_ingredient.pyindex ca86d2d2b..ca5f34745 100644--- a/wger/nutrition/tests/test_ingredient.py+++ b/wger/nutrition/tests/test_ingredient.py@@ -462,32 +462,34 @@ class IngredientModelTestCase(WgerTestCase):Tests the ingredient model functions"""- off_response = {- 'status': OFF_SEARCH_PRODUCT_FOUND,- 'product': {- 'code': '1234',- 'lang': 'de',- 'product_name': 'Foo with chocolate',- 'generic_name': 'Foo with chocolate, 250g package',- 'brands': 'The bar company',- 'editors_tags': ['open food facts', 'MrX'],- 'nutriments': {- 'energy-kcal_100g': 120,- 'proteins_100g': 10,- 'carbohydrates_100g': 20,- 'sugars_100g': 30,- 'fat_100g': 40,- 'saturated-fat_100g': 11,- 'sodium_100g': 5,- 'fiber_100g': None- },+ def setUp(self):+ super().setUp()+ self.off_response = {+ 'status': OFF_SEARCH_PRODUCT_FOUND,+ 'product': {+ 'code': '1234',+ 'lang': 'de',+ 'product_name': 'Foo with chocolate',+ 'generic_name': 'Foo with chocolate, 250g package',+ 'brands': 'The bar company',+ 'editors_tags': ['open food facts', 'MrX'],+ 'nutriments': {+ 'energy-kcal_100g': 120,+ 'proteins_100g': 10,+ 'carbohydrates_100g': 20,+ 'sugars_100g': 30,+ 'fat_100g': 40,+ 'saturated-fat_100g': 11,+ 'sodium_100g': 5,+ 'fiber_100g': None+ },+ }}- }- off_response_no_results = {- 'status': OFF_SEARCH_PRODUCT_NOT_FOUND,- 'status_verbose': 'product not found'- }+ self.off_response_no_results = {+ 'status': OFF_SEARCH_PRODUCT_NOT_FOUND,+ 'status_verbose': 'product not found'+ }@patch('openfoodfacts.products.get_product')def test_fetch_from_off_success(self, mock_get_product):@@ -510,6 +512,43 @@ class IngredientModelTestCase(WgerTestCase):self.assertEqual(ingredient.brand, 'The bar company')self.assertEqual(ingredient.license_author, 'open food facts, MrX')+ @patch('openfoodfacts.products.get_product')+ def test_fetch_from_off_success_long_name(self, mock_get_product):+ """+ Tests creating an ingredient from OFF - name gets truncated+ """+ self.off_response['product']['product_name'] = """+ The Shiba Inu (柴犬, Japanese: [ɕiba inɯ]) is a breed of hunting dog from Japan. A+ small-to-medium breed, it is the smallest of the six original and distinct spitz+ breeds of dog native to Japan.[1] Its name literally translates to "brushwood dog",+ as it is used to flush game."""+ mock_get_product.return_value = self.off_response++ ingredient = Ingredient.fetch_ingredient_from_off('1234')+ self.assertEqual(len(ingredient.name), 200)++ @patch('openfoodfacts.products.get_product')+ def test_fetch_from_off_key_missing_1(self, mock_get_product):+ """+ Tests creating an ingredient from OFF - missing key in nutriments+ """+ del self.off_response['product']['nutriments']['energy-kcal_100g']+ mock_get_product.return_value = self.off_response++ ingredient = Ingredient.fetch_ingredient_from_off('1234')+ self.assertIsNone(ingredient)++ @patch('openfoodfacts.products.get_product')+ def test_fetch_from_off_key_missing_2(self, mock_get_product):+ """+ Tests creating an ingredient from OFF - missing name+ """+ del self.off_response['product']['product_name']+ mock_get_product.return_value = self.off_response++ ingredient = Ingredient.fetch_ingredient_from_off('1234')+ self.assertIsNone(ingredient)+@patch('openfoodfacts.products.get_product')def test_fetch_from_off_no_results(self, mock_get_product):"""