Self hosted FLOSS fitness/workout, nutrition and weight tracker
Don't let a single bad product abort the whole import run
Roland Geider0db5edd2a322
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.pyindex fe0a1a87b..6b999346b 100644--- a/wger/nutrition/dataclasses.py+++ b/wger/nutrition/dataclasses.py@@ -17,6 +17,7 @@ from dataclasses import (asdict,dataclass,)+from decimal import Decimalfrom typing import Optional# wger@@ -88,6 +89,14 @@ class IngredientData:'sodium','fiber',]+ # The dumps are user generated, anything can turn up where a number is+ # expected. Reject it like any other invalid value instead of letting a+ # TypeError escape and abort the whole import run.+ for field in ['energy'] + macros:+ value = getattr(self, field)+ if value is not None and not isinstance(value, (int, float, Decimal)):+ raise ValueError(f'Value for {field} is not a number: {value!r}')+for macro in macros:value = getattr(self, macro)if value and value > 100:diff --git a/wger/nutrition/tests/test_import_products.py b/wger/nutrition/tests/test_import_products.pynew file mode 100644index 000000000..2782028e4--- /dev/null+++ b/wger/nutrition/tests/test_import_products.py@@ -0,0 +1,166 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Standard Library+import gzip+import json+import os+import tempfile++# Django+from django.core.management import call_command++# wger+from wger.core.tests.base_testcase import WgerTestCase+from wger.nutrition.consts import OFF_FULL_DUMP_URL+from wger.nutrition.models import Ingredient+++def off_product(code: str, name: str, lang: str = 'de', **overrides) -> dict:+ """+ A minimal product as it appears in the Open Food Facts dump+ """+ product = {+ 'code': code,+ 'lang': lang,+ 'product_name': name,+ 'nutriments': {+ 'energy-kcal_100g': 600,+ 'proteins_100g': 10,+ 'carbohydrates_100g': 30,+ 'fat_100g': 40,+ },+ }+ product.update(overrides)+ return product+++class ImportOffProductsTestCase(WgerTestCase):+ """+ Tests importing a full Open Food Facts dump++ The dump is written to a folder passed with --folder, so the command+ finds the file in place and never downloads anything.+ """++ def run_import(self, lines: list, **options) -> str:+ """+ Writes the lines as a gzipped dump and runs the import over it+ """+ with tempfile.TemporaryDirectory() as folder:+ path = os.path.join(folder, os.path.basename(OFF_FULL_DUMP_URL))+ with gzip.open(path, 'wt') as dump:+ for line in lines:+ dump.write(line if isinstance(line, str) else json.dumps(line))+ dump.write('\n')++ call_command('import-off-products', '--jsonl', folder=folder, **options)+ return folder++ def test_products_are_imported(self):+ count_before = Ingredient.objects.count()++ self.run_import([off_product('111', 'Imported product')])++ self.assertEqual(Ingredient.objects.count(), count_before + 1)+ ingredient = Ingredient.objects.get(remote_id='111')+ self.assertEqual(ingredient.name, 'Imported product')+ self.assertEqual(ingredient.energy, 600)++ def test_existing_products_are_updated(self):+ self.run_import([off_product('222', 'First name')])+ count_after_first = Ingredient.objects.count()++ self.run_import([off_product('222', 'Second name')])++ self.assertEqual(Ingredient.objects.count(), count_after_first)+ self.assertEqual(Ingredient.objects.get(remote_id='222').name, 'Second name')++ def test_products_in_other_languages_are_skipped(self):+ count_before = Ingredient.objects.count()++ self.run_import([off_product('333', 'Klingon food', lang='tlh')])++ self.assertEqual(Ingredient.objects.count(), count_before)++ def test_malformed_json_does_not_abort_the_run(self):+ count_before = Ingredient.objects.count()++ self.run_import(+ [+ off_product('444', 'Before the broken line'),+ 'this is not json',+ off_product('555', 'After the broken line'),+ ]+ )++ self.assertEqual(Ingredient.objects.count(), count_before + 2)+ self.assertTrue(Ingredient.objects.filter(remote_id='555').exists())++ def test_incomplete_product_does_not_abort_the_run(self):+ """+ Products missing required keys are skipped, the import continues+ """+ count_before = Ingredient.objects.count()+ incomplete = off_product('666', 'No nutriments')+ del incomplete['nutriments']++ self.run_import(+ [+ incomplete,+ off_product('777', 'Complete product'),+ ]+ )++ self.assertEqual(Ingredient.objects.count(), count_before + 1)+ self.assertTrue(Ingredient.objects.filter(remote_id='777').exists())++ def test_implausible_product_does_not_abort_the_run(self):+ """+ Products failing the sanity checks are skipped, the import continues+ """+ count_before = Ingredient.objects.count()+ implausible = off_product('888', 'Impossible macros')+ implausible['nutriments']['proteins_100g'] = 100+ implausible['nutriments']['carbohydrates_100g'] = 100+ implausible['nutriments']['fat_100g'] = 100++ self.run_import(+ [+ implausible,+ off_product('999', 'Sane product'),+ ]+ )++ self.assertEqual(Ingredient.objects.count(), count_before + 1)+ self.assertTrue(Ingredient.objects.filter(remote_id='999').exists())++ def test_non_numeric_nutriments_do_not_abort_the_run(self):+ """+ The dump is user-generated, a string where a number belongs must not+ take down the whole import+ """+ count_before = Ingredient.objects.count()+ broken = off_product('1000', 'String nutriments')+ broken['nutriments']['proteins_100g'] = 'a lot'++ self.run_import(+ [+ broken,+ off_product('1001', 'Sane product'),+ ]+ )++ self.assertEqual(Ingredient.objects.count(), count_before + 1)+ self.assertTrue(Ingredient.objects.filter(remote_id='1001').exists())