Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Add sanity checks for ingredient sync

Roland Geider4e7c0c344619
diff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.py
index 4845455e2..af914bbbe 100644
--- a/wger/nutrition/extract_info/wger.py
+++ b/wger/nutrition/extract_info/wger.py
@@ -112,4 +112,5 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:
is_vegetarian=is_vegetarian,
nutriscore=nutriscore,
)
+ ingredient_data.sanity_checks()
return ingredient_data
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py
index 9ebf668ef..4bb51d4d4 100644
--- a/wger/nutrition/sync.py
+++ b/wger/nutrition/sync.py
@@ -387,8 +387,15 @@ def sync_ingredients(
)
count = 0
+ errors = 0
for data in get_paginated(url, headers=wger_headers()):
- _sync_ingredient_from_api_data(data)
+ try:
+ _sync_ingredient_from_api_data(data)
+ except (KeyError, ValueError, TypeError) as e:
+ # A single malformed or invalid record must not abort the whole run
+ errors += 1
+ logger.warning(f'Skipping malformed ingredient during sync: {e}')
+ continue
count += 1
pbar.update(1)
if not show_progress_bar and count % API_MAX_ITEMS == 0:
@@ -396,7 +403,7 @@ def sync_ingredients(
pbar.close()
- print_fn(style_fn(f'done! Processed {count} ingredients.\n'))
+ print_fn(style_fn(f'done! Processed {count} ingredients ({errors} errors).\n'))
return count
@@ -520,7 +527,12 @@ def sync_ingredients_from_dump(
errors += 1
continue
- ingredient_data = extract_info_from_wger_api(data).dict()
+ try:
+ ingredient_data = extract_info_from_wger_api(data).dict()
+ except (KeyError, ValueError, TypeError):
+ # A single malformed or invalid record must not abort the import
+ errors += 1
+ continue
ingredient_data['uuid'] = uuid
weight_units_data = extract_weight_unit_info_from_wger_api(data)
diff --git a/wger/nutrition/tests/test_sync.py b/wger/nutrition/tests/test_sync.py
index c86efe128..2ed5230ac 100644
--- a/wger/nutrition/tests/test_sync.py
+++ b/wger/nutrition/tests/test_sync.py
@@ -76,7 +76,7 @@ class MockIngredientResponse:
'language': 2,
'is_vegan': True,
'is_vegetarian': True,
- 'nutriscore': 'D',
+ 'nutriscore': 'd',
'weight_units': [
{
'id': 100,
@@ -156,7 +156,7 @@ class TestSyncMethods(WgerTestCase):
self.assertEqual(ingredient.uuid, UUID('7908c204-907f-4b1e-ad4e-f482e9769ade'))
self.assertTrue(ingredient.is_vegan)
self.assertTrue(ingredient.is_vegetarian)
- self.assertEqual(ingredient.nutriscore, 'D')
+ self.assertEqual(ingredient.nutriscore, 'd')
# Weight units synced
units = ingredient.ingredientweightunit_set.order_by('gram')
@@ -194,6 +194,66 @@ class TestSyncMethods(WgerTestCase):
# Assert - local-only unit should be removed
self.assertEqual(ingredient.ingredientweightunit_set.count(), 2)
+ @patch('requests.get')
+ def test_sync_skips_ingredient_failing_sanity_checks(self, mock_request):
+ """A record that fails sanity_checks() is skipped without aborting the sync."""
+ response = MagicMock()
+ response.json.return_value = {
+ 'count': 2,
+ 'next': None,
+ 'previous': None,
+ 'results': [
+ {
+ # protein > 100 -> fails IngredientData.sanity_checks()
+ 'id': 1,
+ 'uuid': 'baaaaaad-0000-0000-0000-000000000001',
+ 'code': 'bogus-code',
+ 'name': 'Bogus ingredient',
+ 'created': '2020-12-20T01:00:00+01:00',
+ 'last_update': '2020-12-20T01:00:00+01:00',
+ 'energy': 200,
+ 'protein': '150.000',
+ 'carbohydrates': '10.000',
+ 'fat': '10.000',
+ 'language': 2,
+ 'license': 5,
+ 'license_author': 'Open Food Facts',
+ 'weight_units': [],
+ },
+ {
+ 'id': 2,
+ 'uuid': '582f1b7f-a8bd-4951-9edd-247bc68b28f4',
+ 'code': '3181238941963',
+ 'name': 'Maxi Hot Dog New York Style',
+ 'created': '2020-12-20T01:00:00+01:00',
+ 'last_update': '2020-12-20T01:00:00+01:00',
+ 'energy': 256,
+ 'protein': '11.000',
+ 'carbohydrates': '27.000',
+ 'fat': '11.000',
+ 'language': 3,
+ 'license': 5,
+ 'license_author': 'Open Food Facts',
+ 'weight_units': [],
+ },
+ ],
+ }
+ mock_request.return_value = response
+
+ count_before = Ingredient.objects.count()
+
+ # A single malformed record must not abort the whole run
+ sync_ingredients(lambda x: x)
+
+ # The valid ingredient was synced, the malformed one was skipped
+ self.assertTrue(
+ Ingredient.objects.filter(uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4').exists()
+ )
+ self.assertFalse(
+ Ingredient.objects.filter(uuid='baaaaaad-0000-0000-0000-000000000001').exists()
+ )
+ self.assertEqual(Ingredient.objects.count(), count_before + 1)
+
@patch('requests.get', return_value=MockIngredientResponse())
def test_ingredient_sync_languages(self, mock_request):
# Call the function with the language_codes parameter
diff --git a/wger/nutrition/tests/test_sync_bulk.py b/wger/nutrition/tests/test_sync_bulk.py
index b0c05ff82..fb1721090 100644
--- a/wger/nutrition/tests/test_sync_bulk.py
+++ b/wger/nutrition/tests/test_sync_bulk.py
@@ -35,10 +35,7 @@ from django.test import SimpleTestCase
# wger
from wger.core.tests.base_testcase import WgerTestCase
from wger.nutrition.consts import SyncMode
-from wger.nutrition.models import (
- Ingredient,
- IngredientWeightUnit,
-)
+from wger.nutrition.models import Ingredient
from wger.nutrition.sync import (
_open_jsonl,
download_ingredient_dump,
@@ -339,6 +336,31 @@ class TestSyncIngredientsFromDump(WgerTestCase):
finally:
Path(tmp.name).unlink()
+ def test_skips_ingredient_failing_sanity_checks(self):
+ """An entry that fails sanity_checks() is skipped and the import continues."""
+ bad_entry = {**SAMPLE_INGREDIENTS[0]}
+ bad_entry['uuid'] = 'baaaaaad-0000-0000-0000-000000000001'
+ bad_entry['protein'] = '150.000' # > 100 -> fails sanity_checks()
+
+ tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w')
+ tmp.write(json.dumps(bad_entry) + '\n')
+ tmp.write(json.dumps(SAMPLE_INGREDIENTS[1]) + '\n')
+ tmp.close()
+
+ try:
+ count = sync_ingredients_from_dump(lambda x: x, tmp.name)
+
+ # Only the valid entry was imported; the malformed one was skipped
+ self.assertEqual(count, 1)
+ self.assertFalse(
+ Ingredient.objects.filter(uuid='baaaaaad-0000-0000-0000-000000000001').exists()
+ )
+ self.assertTrue(
+ Ingredient.objects.filter(uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4').exists()
+ )
+ finally:
+ Path(tmp.name).unlink()
+
class TestDownloadIngredientDump(WgerTestCase):
"""Test downloading the ingredient dump from a remote server."""