Self hosted FLOSS fitness/workout, nutrition and weight tracker
Add tasks to periodically generate the ingredient export
Roland Geidere2e257dc24d3
diff --git a/settings/main.py b/settings/main.pyindex 31d8f8e63..b503d4b00 100644--- a/settings/main.py+++ b/settings/main.py@@ -127,6 +127,7 @@ WGER_SETTINGS['SYNC_EXERCISE_IMAGES_CELERY'] = env.bool('SYNC_EXERCISE_IMAGES_CEWGER_SETTINGS['SYNC_EXERCISE_VIDEOS_CELERY'] = env.bool('SYNC_EXERCISE_VIDEOS_CELERY', False)WGER_SETTINGS['SYNC_INGREDIENTS_CELERY'] = env.bool('SYNC_INGREDIENTS_CELERY', False)WGER_SETTINGS['SYNC_OFF_DAILY_DELTA_CELERY'] = env.bool('SYNC_OFF_DAILY_DELTA_CELERY', False)+WGER_SETTINGS['EXPORT_INGREDIENTS_BULK_CELERY'] = env.bool('EXPORT_INGREDIENTS_BULK_CELERY', False)WGER_SETTINGS['USE_RECAPTCHA'] = env.bool('USE_RECAPTCHA', False)WGER_SETTINGS['USE_CELERY'] = env.bool('USE_CELERY', False)WGER_SETTINGS['CACHE_API_EXERCISES_CELERY'] = env.bool('CACHE_API_EXERCISES_CELERY', False)diff --git a/settings/settings_global.py b/settings/settings_global.pyindex bd3e877ce..6a33ea4ee 100644--- a/settings/settings_global.py+++ b/settings/settings_global.py@@ -478,6 +478,7 @@ WGER_SETTINGS = {'SYNC_EXERCISE_VIDEOS_CELERY': False,'SYNC_INGREDIENTS_CELERY': False,'SYNC_OFF_DAILY_DELTA_CELERY': False,+ 'EXPORT_INGREDIENTS_BULK_CELERY': False,'CACHE_API_EXERCISES_CELERY': False,'CACHE_API_EXERCISES_CELERY_FORCE_UPDATE': False,'TWITTER': False,diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.pyindex 40e617910..03edf0ef2 100644--- a/wger/nutrition/tasks.py+++ b/wger/nutrition/tasks.py@@ -31,8 +31,11 @@ from wger.celery_configuration import appfrom wger.nutrition.api.endpoints import INGREDIENTS_ENDPOINTfrom wger.nutrition.sync import (download_ingredient_images,+ download_ingredient_dump,+ export_ingredient_dump,fetch_ingredient_image,sync_ingredients,+ sync_ingredients_from_dump,sync_ingredients_page,)from wger.utils.cache import CacheKeyMapper@@ -140,6 +143,31 @@ def sync_all_ingredients_chunked_task(return None+def sync_ingredients_bulk_or_api_task():+ """+ Sync ingredients from a remote wger instance.++ Tries to download a bulk JSONL dump first. If the dump is not available+ (e.g. the remote server hasn't generated one), falls back to the paginated+ API sync.+ """+ try:+ file_path = download_ingredient_dump(logger.info)+ sync_ingredients_from_dump(logger.info, file_path)+ except FileNotFoundError:+ logger.info('Bulk dump not available, falling back to API sync.')+ sync_all_ingredients_chunked_task.delay()+++def export_ingredients_dump_task():+ """+ Export all ingredients as a gzipped JSONL file for bulk synchronization.+ """+ export_ingredient_dump(logger.info)++@app.taskdef sync_off_daily_delta():"""@@ -150,6 +178,8 @@ def sync_off_daily_delta():@app.on_after_finalize.connectdef setup_periodic_tasks(sender, **kwargs):++ # Sync ingredients via celery once a month (prefers bulk dump, falls back to API)if settings.WGER_SETTINGS['SYNC_INGREDIENTS_CELERY']:sender.add_periodic_task(crontab(@@ -157,10 +187,23 @@ def setup_periodic_tasks(sender, **kwargs):minute=str(randint(0, 59)),day_of_month=str(randint(1, 28)),),- sync_all_ingredients_chunked_task.s(),+ sync_ingredients_bulk_or_api_task.s(),name='Sync ingredients',)+ # Generate the ingredient export on the 1st and 15th of every month+ if settings.WGER_SETTINGS['EXPORT_INGREDIENTS_BULK_CELERY']:+ sender.add_periodic_task(+ crontab(+ hour=str(randint(0, 23)),+ minute=str(randint(0, 59)),+ day_of_month='1,15',+ ),+ export_ingredients_dump_task.s(),+ name='Export ingredients bulk dump',+ )++ # Sync the OFF daily delta updates once a dayif settings.WGER_SETTINGS['SYNC_OFF_DAILY_DELTA_CELERY']:sender.add_periodic_task(crontab(diff --git a/wger/nutrition/tests/test_sync_bulk.py b/wger/nutrition/tests/test_sync_bulk.pynew file mode 100644index 000000000..aa2205dde--- /dev/null+++ b/wger/nutrition/tests/test_sync_bulk.py@@ -0,0 +1,448 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Standard Library+import gzip+import json+import os+import tempfile+from decimal import Decimal+from io import StringIO+from unittest.mock import (+ MagicMock,+ patch,+)++# Django+from django.core.management import call_command+from django.core.management.base import CommandError+from django.test import SimpleTestCase++# wger+from wger.core.tests.base_testcase import WgerTestCase+from wger.nutrition.models import Ingredient+from wger.nutrition.consts import SyncMode+from wger.nutrition.sync import (+ _open_jsonl,+ download_ingredient_dump,+ export_ingredient_dump,+ sync_ingredients_from_dump,+)+++SAMPLE_INGREDIENTS = [+ {+ 'id': 1,+ 'uuid': '7908c204-907f-4b1e-ad4e-f482e9769ade',+ 'code': '0013087245950',+ 'name': 'Gâteau double chocolat',+ 'created': '2020-12-20T01:00:00+01:00',+ 'last_update': '2020-12-20T01:00:00+01:00',+ 'energy': 360,+ 'protein': '5.000',+ 'carbohydrates': '45.000',+ 'carbohydrates_sugar': '27.000',+ 'fat': '18.000',+ 'fat_saturated': '4.500',+ 'fiber': '2.000',+ 'sodium': '0.356',+ 'license': 5,+ 'license_title': 'Gâteau double chocolat',+ 'license_object_url': '',+ 'license_author': 'Open Food Facts',+ 'license_author_url': '',+ 'license_derivative_source_url': '',+ 'language': 2,+ },+ {+ 'id': 22634,+ 'uuid': '582f1b7f-a8bd-4951-9edd-247bc68b28f4',+ 'code': '3181238941963',+ 'name': 'Maxi Hot Dog New York Style',+ 'created': '2020-12-20T01:00:00+01:00',+ 'last_update': '2020-12-20T01:00:00+01:00',+ 'energy': 256,+ 'protein': '11.000',+ 'carbohydrates': '27.000',+ 'carbohydrates_sugar': '5.600',+ 'fat': '11.000',+ 'fat_saturated': '4.600',+ 'fiber': None,+ 'sodium': '0.820',+ 'license': 5,+ 'license_title': 'Maxi Hot Dog New York Style',+ 'license_object_url': '',+ 'license_author': 'Open Food Facts',+ 'license_author_url': '',+ 'license_derivative_source_url': '',+ 'language': 3,+ },+]+++def _write_dump(path, ingredients, use_gzip=True):+ """Helper to write a JSONL dump file."""+ if use_gzip:+ with gzip.open(path, 'wt', encoding='utf-8') as f:+ for item in ingredients:+ f.write(json.dumps(item) + '\n')+ else:+ with open(path, 'w', encoding='utf-8') as f:+ for item in ingredients:+ f.write(json.dumps(item) + '\n')+++class TestOpenJsonl(SimpleTestCase):+ """Test the _open_jsonl helper that handles both gzip and plain files."""++ def test_opens_gzip_file(self):+ with tempfile.NamedTemporaryFile(suffix='.jsonl.gz', delete=False) as tmp:+ tmp_path = tmp.name++ try:+ _write_dump(tmp_path, SAMPLE_INGREDIENTS, use_gzip=True)+ with _open_jsonl(tmp_path) as f:+ lines = f.readlines()+ self.assertEqual(len(lines), 2)+ self.assertEqual(json.loads(lines[0])['name'], 'Gâteau double chocolat')+ finally:+ os.unlink(tmp_path)++ def test_opens_plain_file(self):+ with tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w') as tmp:+ tmp_path = tmp.name++ try:+ _write_dump(tmp_path, SAMPLE_INGREDIENTS, use_gzip=False)+ with _open_jsonl(tmp_path) as f:+ lines = f.readlines()+ self.assertEqual(len(lines), 2)+ self.assertEqual(json.loads(lines[1])['name'], 'Maxi Hot Dog New York Style')+ finally:+ os.unlink(tmp_path)+++class TestExportIngredientDump(WgerTestCase):+ """Test exporting ingredients to a JSONL dump via default_storage."""++ def setUp(self):+ super().setUp()+ self.tmp_media = tempfile.mkdtemp()++ def tearDown(self):+ import shutil++ shutil.rmtree(self.tmp_media, ignore_errors=True)+ super().tearDown()++ def _export(self):+ with self.settings(MEDIA_ROOT=self.tmp_media):+ return export_ingredient_dump(lambda x: x)++ def test_export_creates_file(self):+ """Test that export creates a valid gzipped JSONL file."""+ with self.settings(MEDIA_ROOT=self.tmp_media):+ saved_name = self._export()++ from django.core.files.storage import default_storage++ self.assertTrue(default_storage.exists(saved_name))++ # Read back and verify content+ with default_storage.open(saved_name, 'rb') as f:+ raw = f.read()++ lines = gzip.decompress(raw).decode('utf-8').strip().split('\n')++ # The test fixtures have 14 ingredients+ self.assertEqual(len(lines), 14)++ # Verify each line is valid JSON with expected fields+ for line in lines:+ data = json.loads(line)+ self.assertIn('uuid', data)+ self.assertIn('name', data)+ self.assertIn('energy', data)+ self.assertIn('protein', data)+ self.assertIn('language', data)++ def test_export_returns_storage_path(self):+ """Test that export returns the storage path."""+ saved_name = self._export()++ self.assertIn('ingredients', saved_name)+ self.assertTrue(saved_name.endswith('.jsonl.gz'))++ def test_export_overwrites_existing(self):+ """Test that a second export replaces the first."""+ saved_name_1 = self._export()+ saved_name_2 = self._export()++ self.assertEqual(saved_name_1, saved_name_2)++ with self.settings(MEDIA_ROOT=self.tmp_media):+ from django.core.files.storage import default_storage++ self.assertTrue(default_storage.exists(saved_name_2))+++class TestSyncIngredientsFromDump(WgerTestCase):+ """Test importing ingredients from a JSONL dump file."""++ def _create_dump_file(self, ingredients=None, use_gzip=True):+ """Create a temporary dump file and return its path."""+ suffix = '.jsonl.gz' if use_gzip else '.jsonl'+ tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)+ tmp.close()+ _write_dump(tmp.name, ingredients or SAMPLE_INGREDIENTS, use_gzip=use_gzip)+ return tmp.name++ def test_update_mode_updates_existing(self):+ """Test that UPDATE mode updates existing ingredients by UUID."""+ file_path = self._create_dump_file()++ try:+ # Ingredient with this UUID already exists in fixtures+ ingredient = Ingredient.objects.get(+ uuid='7908c204-907f-4b1e-ad4e-f482e9769ade',+ )+ self.assertEqual(ingredient.name, 'Test ingredient 1')++ count = sync_ingredients_from_dump(lambda x: x, file_path)++ ingredient.refresh_from_db()+ self.assertEqual(ingredient.name, 'Gâteau double chocolat')+ self.assertEqual(ingredient.energy, 360)+ self.assertAlmostEqual(ingredient.protein, Decimal(5), 2)+ self.assertEqual(count, 2)+ finally:+ os.unlink(file_path)++ def test_update_mode_creates_new(self):+ """Test that UPDATE mode creates new ingredients for unknown UUIDs."""+ file_path = self._create_dump_file()++ try:+ self.assertEqual(Ingredient.objects.count(), 14)++ sync_ingredients_from_dump(lambda x: x, file_path)++ self.assertEqual(Ingredient.objects.count(), 15)+ new_ingredient = Ingredient.objects.get(+ uuid='582f1b7f-a8bd-4951-9edd-247bc68b28f4',+ )+ self.assertEqual(new_ingredient.name, 'Maxi Hot Dog New York Style')+ self.assertEqual(new_ingredient.energy, 256)+ finally:+ os.unlink(file_path)++ def test_insert_mode(self):+ """Test that INSERT mode bulk-creates ingredients."""+ file_path = self._create_dump_file()++ try:+ # Delete the ingredient that would conflict+ Ingredient.objects.filter(+ uuid='7908c204-907f-4b1e-ad4e-f482e9769ade',+ ).delete()++ initial_count = Ingredient.objects.count()+ count = sync_ingredients_from_dump(+ lambda x: x,+ file_path,+ mode=SyncMode.INSERT,+ )++ self.assertEqual(count, 2)+ self.assertEqual(Ingredient.objects.count(), initial_count + 2)+ finally:+ os.unlink(file_path)++ def test_handles_plain_jsonl(self):+ """Test that import works with plain (non-gzipped) JSONL files."""+ file_path = self._create_dump_file(use_gzip=False)++ try:+ count = sync_ingredients_from_dump(lambda x: x, file_path)+ self.assertEqual(count, 2)+ finally:+ os.unlink(file_path)++ def test_handles_invalid_json(self):+ """Test that invalid JSON lines are skipped."""+ tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w')+ tmp.write('not valid json\n')+ tmp.write(json.dumps(SAMPLE_INGREDIENTS[1]) + '\n')+ tmp.close()++ try:+ count = sync_ingredients_from_dump(lambda x: x, tmp.name)+ self.assertEqual(count, 1)+ finally:+ os.unlink(tmp.name)++ def test_handles_missing_uuid(self):+ """Test that entries without UUID are skipped."""+ bad_entry = {**SAMPLE_INGREDIENTS[0]}+ del bad_entry['uuid']++ tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w')+ tmp.write(json.dumps(bad_entry) + '\n')+ tmp.write(json.dumps(SAMPLE_INGREDIENTS[1]) + '\n')+ tmp.close()++ try:+ count = sync_ingredients_from_dump(lambda x: x, tmp.name)+ self.assertEqual(count, 1)+ finally:+ os.unlink(tmp.name)++ def test_returns_error_count(self):+ """Test that the function correctly tracks errors."""+ tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w')+ tmp.write('bad json\n')+ tmp.write(json.dumps({'no_uuid': True}) + '\n')+ tmp.write(json.dumps(SAMPLE_INGREDIENTS[1]) + '\n')+ tmp.close()++ try:+ # We can't directly get the error count from return value,+ # but we can verify only valid entries are processed+ count = sync_ingredients_from_dump(lambda x: x, tmp.name)+ self.assertEqual(count, 1)+ finally:+ os.unlink(tmp.name)+++class TestDownloadIngredientDump(WgerTestCase):+ """Test downloading the ingredient dump from a remote server."""++ def _mock_response(self, status_code=200, content=b'fake gzip content'):+ mock = MagicMock()+ mock.status_code = status_code+ mock.headers = {'content-length': str(len(content))}+ mock.raw.stream.return_value = [content]+ return mock++ @patch('wger.nutrition.sync.requests.get')+ def test_download_success(self, mock_get: MagicMock):+ mock_get.return_value = self._mock_response()++ with tempfile.TemporaryDirectory() as folder:+ path = download_ingredient_dump(lambda x: x, folder=folder)++ self.assertTrue(os.path.exists(path))+ self.assertTrue(path.endswith('ingredients.jsonl.gz'))++ @patch('wger.nutrition.sync.requests.get')+ def test_download_404_raises_file_not_found(self, mock_get: MagicMock):+ mock_get.return_value = self._mock_response(status_code=404)++ with tempfile.TemporaryDirectory() as folder:+ with self.assertRaises(FileNotFoundError) as ctx:+ download_ingredient_dump(lambda x: x, folder=folder)++ self.assertIn('not found', str(ctx.exception).lower())+ self.assertIn('export-ingredients', str(ctx.exception))++ @patch('wger.nutrition.sync.requests.get')+ def test_download_server_error_raises(self, mock_get: MagicMock):+ mock_get.return_value = self._mock_response(status_code=500)++ with tempfile.TemporaryDirectory() as folder:+ with self.assertRaises(Exception):+ download_ingredient_dump(lambda x: x, folder=folder)++ @patch('wger.nutrition.sync.requests.get')+ def test_download_skips_existing_file(self, mock_get: MagicMock):+ """Test that an already-downloaded file is reused."""+ with tempfile.TemporaryDirectory() as folder:+ file_path = os.path.join(folder, 'ingredients.jsonl.gz')+ with open(file_path, 'w') as f:+ f.write('existing')++ path = download_ingredient_dump(lambda x: x, folder=folder)++ self.assertEqual(path, file_path)+ mock_get.assert_not_called()++ @patch('wger.nutrition.sync.requests.get')+ def test_download_disables_content_decoding(self, mock_get: MagicMock):+ """Test that raw response decoding is disabled to preserve gzip."""+ mock_response = self._mock_response()+ mock_get.return_value = mock_response++ with tempfile.TemporaryDirectory() as folder:+ download_ingredient_dump(lambda x: x, folder=folder)++ self.assertFalse(mock_response.raw.decode_content)+++class TestBulkSyncManagementCommands(WgerTestCase):+ """Test the management commands for bulk export and import."""++ def test_export_ingredients_command(self):+ """Test the export-ingredients management command."""+ with tempfile.TemporaryDirectory() as tmp_media:+ with self.settings(MEDIA_ROOT=tmp_media):+ out = StringIO()+ call_command('export-ingredients', stdout=out)++ output = out.getvalue()+ self.assertIn('Exporting', output)+ self.assertIn('done!', output)++ def _get_command_module(self):+ """Get the bulk-sync-ingredients command module (hyphenated name needs importlib)."""+ import importlib++ return importlib.import_module(+ 'wger.nutrition.management.commands.bulk-sync-ingredients'+ )++ @patch('wger.core.api.min_server_version.check_min_server_version')+ def test_bulk_sync_command_calls_functions(self, mock_version_check: MagicMock):+ """Test that the bulk-sync-ingredients command calls the right functions."""+ mod = self._get_command_module()++ with (+ patch.object(mod, 'download_ingredient_dump', return_value='/tmp/fake.jsonl.gz') as mock_download,+ patch.object(mod, 'sync_ingredients_from_dump') as mock_sync,+ ):+ out = StringIO()+ call_command(+ 'bulk-sync-ingredients',+ '--remote-url', 'https://example.com',+ stdout=out,+ )++ mock_download.assert_called_once()+ mock_sync.assert_called_once()++ @patch('wger.core.api.min_server_version.check_min_server_version')+ def test_bulk_sync_command_handles_404(self, mock_version_check: MagicMock):+ """Test that the command shows a helpful error when dump is not found."""+ mod = self._get_command_module()++ with patch.object(+ mod, 'download_ingredient_dump',+ side_effect=FileNotFoundError('Bulk ingredient dump not found'),+ ):+ out = StringIO()+ with self.assertRaises(CommandError):+ call_command(+ 'bulk-sync-ingredients',+ '--remote-url', 'https://example.com',+ stdout=out,+ )