Self hosted FLOSS fitness/workout, nutrition and weight tracker
Cleanup
Roland Geiderb76861b0b9b5
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ad84174d1..54b269f07 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,9 +2,9 @@Improved ingredient sync. Instead of loading the ingredients over the paginatedAPI (which was taking a long time with almost 3 million entries), wger instances-can generate and read a dump file, which can be downloaded and read much more-effectively. The setting `SYNC_INGREDIENTS_CELERY` will automatically prefer the-bulk dump over the paginated API, falling back to the API if no dump is available.+can generate and read a dump file, which can be processed much more effectively.+The setting `SYNC_INGREDIENTS_CELERY` will automatically prefer the bulk dump+over the paginated API, falling back to the API if no dump is available.<https://github.com/wger-project/wger/issues/2231> Add vegan and vegetarian flags tothe ingredient model. This allows to filter products based on dietary preferences.diff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.pyindex f64e190f0..77e511528 100644--- a/wger/nutrition/extract_info/wger.py+++ b/wger/nutrition/extract_info/wger.py@@ -20,7 +20,7 @@ from wger.utils.constants import ODBL_LICENSE_IDdef extract_info_from_wger_api(product_data: dict) -> IngredientData:# Basics- name = product_data.get('name')+ name = product_data.get('name', '')common_name = product_data.get('common_name', '')code = product_data['code']energy = float(product_data['energy'])@@ -47,6 +47,7 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:# Dietary propertiesis_vegan = product_data.get('is_vegan', None)is_vegetarian = product_data.get('is_vegetarian', None)+ nutriscore = product_data.get('nutriscore', None)# License and author infosource_name = product_data.get('source_name', '')@@ -84,5 +85,6 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:license_derivative_source_url=license_derivative_source_url,is_vegan=is_vegan,is_vegetarian=is_vegetarian,+ nutriscore=nutriscore,)return ingredient_datadiff --git a/wger/nutrition/management/commands/bulk-sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients-bulk.pysimilarity index 100%rename from wger/nutrition/management/commands/bulk-sync-ingredients.pyrename to wger/nutrition/management/commands/sync-ingredients-bulk.pydiff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex 22356c4c5..6ed25a76b 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -26,6 +26,8 @@ from typing import (# Djangofrom django.conf import settings+from django.core.files.base import File+from django.core.files.storage import default_storagefrom django.db import IntegrityErrorfrom django.utils import timezone@@ -45,8 +47,10 @@ from tqdm import tqdmfrom wger.core.models.language import Languagefrom wger.nutrition.api.endpoints import (IMAGE_ENDPOINT,+ INGREDIENT_BULK_EXPORT_PATH,INGREDIENTS_ENDPOINT,)+from wger.nutrition.api.serializers import IngredientSerializerfrom wger.nutrition.consts import SyncModefrom wger.nutrition.extract_info.wger import extract_info_from_wger_apifrom wger.nutrition.models import (@@ -149,7 +153,7 @@ def fetch_image_from_off(ingredient: Ingredient):logger.info(f'Trying to fetch image from OFF for "{ingredient.name}" ({ingredient.uuid})')# We always update the last check time, no matter if we found an image or there were- # errors in the response (keys missing, etc) since in any case we don't want to retry+ # errors in the response (keys missing, etc.) since in any case we don't want to retry# too often.ingredient.last_image_check = timezone.now()ingredient.save()@@ -305,14 +309,16 @@ def sync_ingredients(if language_codes is not None:language_ids = []for code in language_codes.split(','):- # Leaving the try except in here even though we've already validated on the sync-ingredients command itself.- # This is in case we ever want to re-use this function for anything else where user can input language codes.+ # Leaving the try except in here even though we've already validated on the+ # sync-ingredients command itself. This is in case we ever want to re-use+ # this function for anything else where user can input language codes.try:lang = load_language(code, default_to_english=False)language_ids.append(str(lang.id))- except Language.DoesNotExist as e:+ except Language.DoesNotExist:print_fn(- f'Error: The language code you provided ("{code}") does not exist in this database. Please try again.'+ f'Error: The language code you provided ("{code}") does not exist in '+ f'this database. Please try again.')return 0@@ -378,15 +384,6 @@ def export_ingredient_dump((local filesystem, S3, GCS, etc.)."""- # Import here to avoid circular imports- # Django- from django.core.files.base import File- from django.core.files.storage import default_storage-- # wger- from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH- from wger.nutrition.api.serializers import IngredientSerializer-# TODO: handle units here as wellqueryset = Ingredient.objects.select_related('language',@@ -427,7 +424,7 @@ def export_ingredient_dump(return saved_name-def _open_jsonl(file_path: str):+def _open_jsonl(file_path: Path):"""Open a JSONL file, transparently handling both gzip and plain text."""with open(file_path, 'rb') as f:magic = f.read(2)@@ -438,7 +435,7 @@ def _open_jsonl(file_path: str):def sync_ingredients_from_dump(print_fn,- file_path: str,+ file_path: Path,mode: SyncMode = SyncMode.UPDATE,style_fn=lambda x: x,show_progress_bar: bool = False,@@ -525,14 +522,12 @@ def download_ingredient_dump(remote_url: str = settings.WGER_SETTINGS['WGER_INSTANCE'],folder: str = '',style_fn=lambda x: x,-) -> str:+) -> Path:"""Download the ingredient JSONL dump from a remote wger instance.Returns the path to the downloaded file."""- # wger- from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATHdump_url = f'{remote_url}{settings.MEDIA_URL}{INGREDIENT_BULK_EXPORT_PATH}'print_fn(f'*** Downloading ingredient dump from {dump_url}...')@@ -546,7 +541,7 @@ def download_ingredient_dump(if file_path.exists():print_fn(f'File already downloaded at {file_path}')- return str(file_path)+ return file_pathresponse = requests.get(dump_url, stream=True, headers=wger_headers())if response.status_code == 404:@@ -570,4 +565,4 @@ def download_ingredient_dump(pbar.update(len(chunk))print_fn(style_fn(f'Download complete: {file_path}\n'))- return str(file_path)+ return file_pathdiff --git a/wger/nutrition/tests/test_sync_bulk.py b/wger/nutrition/tests/test_sync_bulk.pyindex 822bad795..60b10682a 100644--- a/wger/nutrition/tests/test_sync_bulk.py+++ b/wger/nutrition/tests/test_sync_bulk.py@@ -14,17 +14,20 @@# Standard Libraryimport gzip+import importlibimport json-import os+import shutilimport tempfilefrom decimal import Decimalfrom io import StringIO+from pathlib import Pathfrom unittest.mock import (MagicMock,patch,)# Django+from django.core.files.storage import default_storagefrom django.core.management import call_commandfrom django.core.management.base import CommandErrorfrom django.test import SimpleTestCase@@ -93,10 +96,12 @@ SAMPLE_INGREDIENTS = [def _write_dump(path, ingredients, use_gzip=True):"""Helper to write a JSONL dump file."""+if use_gzip:with gzip.open(path, 'wt', encoding='utf-8') as f:for item in ingredients:f.write(json.dumps(item) + '\n')+else:with open(path, 'w', encoding='utf-8') as f:for item in ingredients:@@ -117,7 +122,7 @@ class TestOpenJsonl(SimpleTestCase):self.assertEqual(len(lines), 2)self.assertEqual(json.loads(lines[0])['name'], 'Gâteau double chocolat')finally:- os.unlink(tmp_path)+ Path(tmp_path).unlink()def test_opens_plain_file(self):with tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w') as tmp:@@ -130,7 +135,7 @@ class TestOpenJsonl(SimpleTestCase):self.assertEqual(len(lines), 2)self.assertEqual(json.loads(lines[1])['name'], 'Maxi Hot Dog New York Style')finally:- os.unlink(tmp_path)+ Path(tmp_path).unlink()class TestExportIngredientDump(WgerTestCase):@@ -141,9 +146,6 @@ class TestExportIngredientDump(WgerTestCase):self.tmp_media = tempfile.mkdtemp()def tearDown(self):- # Standard Library- import shutil-shutil.rmtree(self.tmp_media, ignore_errors=True)super().tearDown()@@ -156,9 +158,6 @@ class TestExportIngredientDump(WgerTestCase):with self.settings(MEDIA_ROOT=self.tmp_media):saved_name = self._export()- # Django- from django.core.files.storage import default_storage-self.assertTrue(default_storage.exists(saved_name))# Read back and verify content@@ -194,9 +193,6 @@ class TestExportIngredientDump(WgerTestCase):self.assertEqual(saved_name_1, saved_name_2)with self.settings(MEDIA_ROOT=self.tmp_media):- # Django- from django.core.files.storage import default_storage-self.assertTrue(default_storage.exists(saved_name_2))@@ -217,9 +213,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):try:# Ingredient with this UUID already exists in fixtures- ingredient = Ingredient.objects.get(- uuid='7908c204-907f-4b1e-ad4e-f482e9769ade',- )+ ingredient = Ingredient.objects.get(uuid='7908c204-907f-4b1e-ad4e-f482e9769ade')self.assertEqual(ingredient.name, 'Test ingredient 1')count = sync_ingredients_from_dump(lambda x: x, file_path)@@ -230,7 +224,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):self.assertAlmostEqual(ingredient.protein, Decimal(5), 2)self.assertEqual(count, 2)finally:- os.unlink(file_path)+ Path(file_path).unlink()def test_update_mode_creates_new(self):"""Test that UPDATE mode creates new ingredients for unknown UUIDs."""@@ -248,7 +242,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):self.assertEqual(new_ingredient.name, 'Maxi Hot Dog New York Style')self.assertEqual(new_ingredient.energy, 256)finally:- os.unlink(file_path)+ Path(file_path).unlink()def test_insert_mode(self):"""Test that INSERT mode bulk-creates ingredients."""@@ -270,7 +264,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):self.assertEqual(count, 2)self.assertEqual(Ingredient.objects.count(), initial_count + 2)finally:- os.unlink(file_path)+ Path(file_path).unlink()def test_handles_plain_jsonl(self):"""Test that import works with plain (non-gzipped) JSONL files."""@@ -280,7 +274,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):count = sync_ingredients_from_dump(lambda x: x, file_path)self.assertEqual(count, 2)finally:- os.unlink(file_path)+ Path(file_path).unlink()def test_handles_invalid_json(self):"""Test that invalid JSON lines are skipped."""@@ -293,7 +287,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):count = sync_ingredients_from_dump(lambda x: x, tmp.name)self.assertEqual(count, 1)finally:- os.unlink(tmp.name)+ Path(tmp.name).unlink()def test_handles_missing_uuid(self):"""Test that entries without UUID are skipped."""@@ -309,7 +303,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):count = sync_ingredients_from_dump(lambda x: x, tmp.name)self.assertEqual(count, 1)finally:- os.unlink(tmp.name)+ Path(tmp.name).unlink()def test_returns_error_count(self):"""Test that the function correctly tracks errors."""@@ -325,7 +319,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):count = sync_ingredients_from_dump(lambda x: x, tmp.name)self.assertEqual(count, 1)finally:- os.unlink(tmp.name)+ Path(tmp.name).unlink()class TestDownloadIngredientDump(WgerTestCase):@@ -345,8 +339,8 @@ class TestDownloadIngredientDump(WgerTestCase):with tempfile.TemporaryDirectory() as folder:path = download_ingredient_dump(lambda x: x, folder=folder)- self.assertTrue(os.path.exists(path))- self.assertTrue(path.endswith('ingredients.jsonl.gz'))+ self.assertTrue(path.exists())+ self.assertEqual(path.name, 'ingredients.jsonl.gz')@patch('wger.nutrition.sync.requests.get')def test_download_404_raises_file_not_found(self, mock_get: MagicMock):@@ -371,9 +365,8 @@ class TestDownloadIngredientDump(WgerTestCase):def test_download_skips_existing_file(self, mock_get: MagicMock):"""Test that an already-downloaded file is reused."""with tempfile.TemporaryDirectory() as folder:- file_path = os.path.join(folder, 'ingredients.jsonl.gz')- with open(file_path, 'w') as f:- f.write('existing')+ file_path = Path(folder) / 'ingredients.jsonl.gz'+ file_path.write_text('existing')path = download_ingredient_dump(lambda x: x, folder=folder)@@ -407,26 +400,25 @@ class TestBulkSyncManagementCommands(WgerTestCase):self.assertIn('done!', output)def _get_command_module(self):- """Get the bulk-sync-ingredients command module (hyphenated name needs importlib)."""- # Standard Library- import importlib-- return importlib.import_module('wger.nutrition.management.commands.bulk-sync-ingredients')+ """Get the sync-ingredients-bulk command module (hyphenated name needs importlib)."""+ return importlib.import_module('wger.nutrition.management.commands.sync-ingredients-bulk')@patch('wger.core.api.min_server_version.check_min_server_version')def test_bulk_sync_command_calls_functions(self, mock_version_check: MagicMock):- """Test that the bulk-sync-ingredients command calls the right functions."""+ """Test that the sync-ingredients-bulk command calls the right functions."""mod = self._get_command_module()with (patch.object(- mod, 'download_ingredient_dump', return_value='/tmp/fake.jsonl.gz'+ mod,+ 'download_ingredient_dump',+ return_value=Path('/tmp/fake.jsonl.gz'),) as mock_download,patch.object(mod, 'sync_ingredients_from_dump') as mock_sync,):out = StringIO()call_command(- 'bulk-sync-ingredients',+ 'sync-ingredients-bulk','--remote-url','https://example.com',stdout=out,@@ -448,7 +440,7 @@ class TestBulkSyncManagementCommands(WgerTestCase):out = StringIO()with self.assertRaises(CommandError):call_command(- 'bulk-sync-ingredients',+ 'sync-ingredients-bulk','--remote-url','https://example.com',stdout=out,