Self hosted FLOSS fitness/workout, nutrition and weight tracker
Properly handle units in the ingredient sync
Roland Geider9b7c9316de8e
diff --git a/wger/nutrition/api/serializers.py b/wger/nutrition/api/serializers.pyindex 14f81cabb..c46fe0a29 100644--- a/wger/nutrition/api/serializers.py+++ b/wger/nutrition/api/serializers.py@@ -56,6 +56,7 @@ class IngredientWeightUnitInfoSerializer(serializers.ModelSerializer):model = IngredientWeightUnitfields = ('id',+ 'uuid','gram','name',)diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex 38b941a99..620944c63 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -56,6 +56,7 @@ from wger.nutrition.extract_info.wger import extract_info_from_wger_apifrom wger.nutrition.models import (Image,Ingredient,+ IngredientWeightUnit,Source,)from wger.utils.constants import (@@ -76,6 +77,32 @@ from wger.utils.url import make_urilogger = logging.getLogger(__name__)+def sync_weight_units(ingredient: Ingredient, weight_units_data: list[dict]):+ """+ Synchronize weight units for an ingredient from remote data.++ Each entry in weight_units_data is expected to have: uuid, name, gram.+ """+ remote_uuids = set()+ for unit_data in weight_units_data:+ uuid = unit_data.get('uuid')+ if not uuid:+ continue++ remote_uuids.add(uuid)+ IngredientWeightUnit.objects.update_or_create(+ uuid=uuid,+ defaults={+ 'ingredient': ingredient,+ 'name': unit_data['name'],+ 'gram': unit_data['gram'],+ },+ )++ # Remove local units that no longer exist on the remote+ ingredient.ingredientweightunit_set.exclude(uuid__in=remote_uuids).delete()++def fetch_ingredient_image(pk: int):# wgerfrom wger.nutrition.models import Ingredient@@ -289,7 +316,8 @@ def sync_ingredients_page(uuid = data['uuid']ingredient_data = extract_info_from_wger_api(data).dict()ingredient_data['uuid'] = uuid- Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)+ ingredient, _ = Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)+ sync_weight_units(ingredient, data.get('weight_units', []))processed += 1logger.debug(f'Page {page} synced ({processed} ingredients)')return processed@@ -348,7 +376,8 @@ def sync_ingredients(ingredient_data = extract_info_from_wger_api(data).dict()ingredient_data['uuid'] = uuid- Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)+ ingredient, _ = Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)+ sync_weight_units(ingredient, data.get('weight_units', []))if show_progress_bar:pbar.update(1)@@ -384,11 +413,10 @@ def export_ingredient_dump((local filesystem, S3, GCS, etc.)."""- # TODO: handle units here as wellqueryset = Ingredient.objects.select_related('language','license',- )+ ).prefetch_related('ingredientweightunit_set')total = queryset.count()queryset = queryset.iterator(chunk_size=2000)@@ -475,25 +503,32 @@ def sync_ingredients_from_dump(ingredient_data = extract_info_from_wger_api(data).dict()ingredient_data['uuid'] = uuid+ weight_units_data = data.get('weight_units', [])if mode == SyncMode.INSERT:- bulk_bucket.append(Ingredient(**ingredient_data))+ bulk_bucket.append((Ingredient(**ingredient_data), weight_units_data))if len(bulk_bucket) >= BULK_SIZE:try:- Ingredient.objects.bulk_create(bulk_bucket)+ Ingredient.objects.bulk_create([item[0] for item in bulk_bucket])+ for ingredient, units_data in bulk_bucket:+ sync_weight_units(ingredient, units_data)count += len(bulk_bucket)except Exception:# Fall back to individual saves- for ingredient in bulk_bucket:+ for ingredient, units_data in bulk_bucket:try:ingredient.save()+ sync_weight_units(ingredient, units_data)count += 1except Exception:errors += 1bulk_bucket = []else:try:- Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)+ ingredient, _ = Ingredient.objects.update_or_create(+ uuid=uuid, defaults=ingredient_data+ )+ sync_weight_units(ingredient, weight_units_data)count += 1except Exception:errors += 1@@ -503,12 +538,15 @@ def sync_ingredients_from_dump(# Flush remaining items in INSERT modeif mode == SyncMode.INSERT and bulk_bucket:try:- Ingredient.objects.bulk_create(bulk_bucket)+ Ingredient.objects.bulk_create([item[0] for item in bulk_bucket])+ for ingredient, units_data in bulk_bucket:+ sync_weight_units(ingredient, units_data)count += len(bulk_bucket)except Exception:- for ingredient in bulk_bucket:+ for ingredient, units_data in bulk_bucket:try:ingredient.save()+ sync_weight_units(ingredient, units_data)count += 1except Exception:errors += 1diff --git a/wger/nutrition/tests/test_sync.py b/wger/nutrition/tests/test_sync.pyindex 7cc9083e1..4ac5bbf07 100644--- a/wger/nutrition/tests/test_sync.py+++ b/wger/nutrition/tests/test_sync.py@@ -19,7 +19,10 @@ from uuid import UUID# wgerfrom wger.core.tests.base_testcase import WgerTestCase-from wger.nutrition.models import Ingredient+from wger.nutrition.models import (+ Ingredient,+ IngredientWeightUnit,+)from wger.nutrition.sync import (sync_ingredients,sync_ingredients_page,@@ -61,6 +64,20 @@ class MockIngredientResponse:'license_author_url': '','license_derivative_source_url': '','language': 2,+ 'weight_units': [+ {+ 'id': 100,+ 'uuid': 'a1b2c3d4-0000-0000-0000-000000000001',+ 'name': 'Serving',+ 'gram': 85,+ },+ {+ 'id': 101,+ 'uuid': 'a1b2c3d4-0000-0000-0000-000000000002',+ 'name': '1 Portion (2 slices)',+ 'gram': 60,+ },+ ],},{'id': 22634,@@ -84,6 +101,7 @@ class MockIngredientResponse:'license_author_url': '','license_derivative_source_url': '','language': 3,+ 'weight_units': [],},],}@@ -119,11 +137,38 @@ class TestSyncMethods(WgerTestCase):self.assertEqual(ingredient.license.pk, 5)self.assertEqual(ingredient.uuid, UUID('7908c204-907f-4b1e-ad4e-f482e9769ade'))+ # Weight units synced+ units = ingredient.ingredientweightunit_set.order_by('gram')+ self.assertEqual(units.count(), 2)+ self.assertEqual(units[0].name, '1 Portion (2 slices)')+ self.assertEqual(units[0].gram, 60)+ self.assertEqual(units[1].name, 'Serving')+ self.assertEqual(units[1].gram, 85)+new_ingredient = Ingredient.objects.get(uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4')self.assertEqual(new_ingredient.name, 'Maxi Hot Dog New York Style')self.assertEqual(new_ingredient.energy, 256)self.assertAlmostEqual(new_ingredient.protein, Decimal(11), 2)self.assertEqual(new_ingredient.code, '3181238941963')+ self.assertEqual(new_ingredient.ingredientweightunit_set.count(), 0)++ @patch('requests.get', return_value=MockIngredientResponse())+ def test_sync_removes_deleted_weight_units(self, mock_request):+ """Weight units that no longer exist on the remote are deleted locally"""+ # Arrange - sync once to create the weight units+ sync_ingredients(lambda x: x)+ ingredient = Ingredient.objects.get(pk=1)+ self.assertEqual(ingredient.ingredientweightunit_set.count(), 2)++ # Add a local-only unit that doesn't exist on the remote+ IngredientWeightUnit.objects.create(ingredient=ingredient, name='Local only', gram=999)+ self.assertEqual(ingredient.ingredientweightunit_set.count(), 3)++ # Act - sync again+ sync_ingredients(lambda x: x)++ # Assert - local-only unit should be removed+ self.assertEqual(ingredient.ingredientweightunit_set.count(), 2)@patch('requests.get', return_value=MockIngredientResponse())def test_sync_ingredients_page(self, mock_request):@@ -159,6 +204,9 @@ class TestSyncMethods(WgerTestCase):self.assertEqual(ingredient.license.pk, 5)self.assertEqual(ingredient.uuid, UUID('7908c204-907f-4b1e-ad4e-f482e9769ade'))+ # Weight units synced+ self.assertEqual(ingredient.ingredientweightunit_set.count(), 2)+new_ingredient = Ingredient.objects.get(uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4')self.assertEqual(new_ingredient.name, 'Maxi Hot Dog New York Style')self.assertEqual(new_ingredient.energy, 256)diff --git a/wger/nutrition/tests/test_sync_bulk.py b/wger/nutrition/tests/test_sync_bulk.pyindex 60b10682a..854271ec9 100644--- a/wger/nutrition/tests/test_sync_bulk.py+++ b/wger/nutrition/tests/test_sync_bulk.py@@ -35,7 +35,10 @@ from django.test import SimpleTestCase# wgerfrom wger.core.tests.base_testcase import WgerTestCasefrom wger.nutrition.consts import SyncMode-from wger.nutrition.models import Ingredient+from wger.nutrition.models import (+ Ingredient,+ IngredientWeightUnit,+)from wger.nutrition.sync import (_open_jsonl,download_ingredient_dump,@@ -67,6 +70,14 @@ SAMPLE_INGREDIENTS = ['license_author_url': '','license_derivative_source_url': '','language': 2,+ 'weight_units': [+ {+ 'id': 100,+ 'uuid': 'a1b2c3d4-0000-0000-0000-000000000001',+ 'name': 'Serving',+ 'gram': 85,+ },+ ],},{'id': 22634,@@ -90,6 +101,7 @@ SAMPLE_INGREDIENTS = ['license_author_url': '','license_derivative_source_url': '','language': 3,+ 'weight_units': [],},]@@ -223,6 +235,12 @@ class TestSyncIngredientsFromDump(WgerTestCase):self.assertEqual(ingredient.energy, 360)self.assertAlmostEqual(ingredient.protein, Decimal(5), 2)self.assertEqual(count, 2)++ # Weight units synced+ units = ingredient.ingredientweightunit_set.all()+ self.assertEqual(units.count(), 1)+ self.assertEqual(units[0].name, 'Serving')+ self.assertEqual(units[0].gram, 85)finally:Path(file_path).unlink()@@ -263,6 +281,10 @@ class TestSyncIngredientsFromDump(WgerTestCase):self.assertEqual(count, 2)self.assertEqual(Ingredient.objects.count(), initial_count + 2)++ # Weight units synced for the ingredient that has them+ ingredient = Ingredient.objects.get(uuid='7908c204-907f-4b1e-ad4e-f482e9769ade')+ self.assertEqual(ingredient.ingredientweightunit_set.count(), 1)finally:Path(file_path).unlink()