Self hosted FLOSS fitness/workout, nutrition and weight tracker
Add sanity checks for ingredient sync
Roland Geider4e7c0c344619
diff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.pyindex 4845455e2..af914bbbe 100644--- a/wger/nutrition/extract_info/wger.py+++ b/wger/nutrition/extract_info/wger.py@@ -112,4 +112,5 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:is_vegetarian=is_vegetarian,nutriscore=nutriscore,)+ ingredient_data.sanity_checks()return ingredient_datadiff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex 9ebf668ef..4bb51d4d4 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -387,8 +387,15 @@ def sync_ingredients()count = 0+ errors = 0for data in get_paginated(url, headers=wger_headers()):- _sync_ingredient_from_api_data(data)+ try:+ _sync_ingredient_from_api_data(data)+ except (KeyError, ValueError, TypeError) as e:+ # A single malformed or invalid record must not abort the whole run+ errors += 1+ logger.warning(f'Skipping malformed ingredient during sync: {e}')+ continuecount += 1pbar.update(1)if not show_progress_bar and count % API_MAX_ITEMS == 0:@@ -396,7 +403,7 @@ def sync_ingredients(pbar.close()- print_fn(style_fn(f'done! Processed {count} ingredients.\n'))+ print_fn(style_fn(f'done! Processed {count} ingredients ({errors} errors).\n'))return count@@ -520,7 +527,12 @@ def sync_ingredients_from_dump(errors += 1continue- ingredient_data = extract_info_from_wger_api(data).dict()+ try:+ ingredient_data = extract_info_from_wger_api(data).dict()+ except (KeyError, ValueError, TypeError):+ # A single malformed or invalid record must not abort the import+ errors += 1+ continueingredient_data['uuid'] = uuidweight_units_data = extract_weight_unit_info_from_wger_api(data)diff --git a/wger/nutrition/tests/test_sync.py b/wger/nutrition/tests/test_sync.pyindex c86efe128..2ed5230ac 100644--- a/wger/nutrition/tests/test_sync.py+++ b/wger/nutrition/tests/test_sync.py@@ -76,7 +76,7 @@ class MockIngredientResponse:'language': 2,'is_vegan': True,'is_vegetarian': True,- 'nutriscore': 'D',+ 'nutriscore': 'd','weight_units': [{'id': 100,@@ -156,7 +156,7 @@ class TestSyncMethods(WgerTestCase):self.assertEqual(ingredient.uuid, UUID('7908c204-907f-4b1e-ad4e-f482e9769ade'))self.assertTrue(ingredient.is_vegan)self.assertTrue(ingredient.is_vegetarian)- self.assertEqual(ingredient.nutriscore, 'D')+ self.assertEqual(ingredient.nutriscore, 'd')# Weight units syncedunits = ingredient.ingredientweightunit_set.order_by('gram')@@ -194,6 +194,66 @@ class TestSyncMethods(WgerTestCase):# Assert - local-only unit should be removedself.assertEqual(ingredient.ingredientweightunit_set.count(), 2)+ @patch('requests.get')+ def test_sync_skips_ingredient_failing_sanity_checks(self, mock_request):+ """A record that fails sanity_checks() is skipped without aborting the sync."""+ response = MagicMock()+ response.json.return_value = {+ 'count': 2,+ 'next': None,+ 'previous': None,+ 'results': [+ {+ # protein > 100 -> fails IngredientData.sanity_checks()+ 'id': 1,+ 'uuid': 'baaaaaad-0000-0000-0000-000000000001',+ 'code': 'bogus-code',+ 'name': 'Bogus ingredient',+ 'created': '2020-12-20T01:00:00+01:00',+ 'last_update': '2020-12-20T01:00:00+01:00',+ 'energy': 200,+ 'protein': '150.000',+ 'carbohydrates': '10.000',+ 'fat': '10.000',+ 'language': 2,+ 'license': 5,+ 'license_author': 'Open Food Facts',+ 'weight_units': [],+ },+ {+ 'id': 2,+ 'uuid': '582f1b7f-a8bd-4951-9edd-247bc68b28f4',+ 'code': '3181238941963',+ 'name': 'Maxi Hot Dog New York Style',+ 'created': '2020-12-20T01:00:00+01:00',+ 'last_update': '2020-12-20T01:00:00+01:00',+ 'energy': 256,+ 'protein': '11.000',+ 'carbohydrates': '27.000',+ 'fat': '11.000',+ 'language': 3,+ 'license': 5,+ 'license_author': 'Open Food Facts',+ 'weight_units': [],+ },+ ],+ }+ mock_request.return_value = response++ count_before = Ingredient.objects.count()++ # A single malformed record must not abort the whole run+ sync_ingredients(lambda x: x)++ # The valid ingredient was synced, the malformed one was skipped+ self.assertTrue(+ Ingredient.objects.filter(uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4').exists()+ )+ self.assertFalse(+ Ingredient.objects.filter(uuid='baaaaaad-0000-0000-0000-000000000001').exists()+ )+ self.assertEqual(Ingredient.objects.count(), count_before + 1)+@patch('requests.get', return_value=MockIngredientResponse())def test_ingredient_sync_languages(self, mock_request):# Call the function with the language_codes parameterdiff --git a/wger/nutrition/tests/test_sync_bulk.py b/wger/nutrition/tests/test_sync_bulk.pyindex b0c05ff82..fb1721090 100644--- a/wger/nutrition/tests/test_sync_bulk.py+++ b/wger/nutrition/tests/test_sync_bulk.py@@ -35,10 +35,7 @@ from django.test import SimpleTestCase# wgerfrom wger.core.tests.base_testcase import WgerTestCasefrom wger.nutrition.consts import SyncMode-from wger.nutrition.models import (- Ingredient,- IngredientWeightUnit,-)+from wger.nutrition.models import Ingredientfrom wger.nutrition.sync import (_open_jsonl,download_ingredient_dump,@@ -339,6 +336,31 @@ class TestSyncIngredientsFromDump(WgerTestCase):finally:Path(tmp.name).unlink()+ def test_skips_ingredient_failing_sanity_checks(self):+ """An entry that fails sanity_checks() is skipped and the import continues."""+ bad_entry = {**SAMPLE_INGREDIENTS[0]}+ bad_entry['uuid'] = 'baaaaaad-0000-0000-0000-000000000001'+ bad_entry['protein'] = '150.000' # > 100 -> fails sanity_checks()++ tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w')+ tmp.write(json.dumps(bad_entry) + '\n')+ tmp.write(json.dumps(SAMPLE_INGREDIENTS[1]) + '\n')+ tmp.close()++ try:+ count = sync_ingredients_from_dump(lambda x: x, tmp.name)++ # Only the valid entry was imported; the malformed one was skipped+ self.assertEqual(count, 1)+ self.assertFalse(+ Ingredient.objects.filter(uuid='baaaaaad-0000-0000-0000-000000000001').exists()+ )+ self.assertTrue(+ Ingredient.objects.filter(uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4').exists()+ )+ finally:+ Path(tmp.name).unlink()+class TestDownloadIngredientDump(WgerTestCase):"""Test downloading the ingredient dump from a remote server."""