Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Cleanup

Roland Geiderb76861b0b9b5
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ad84174d1..54b269f07 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,9 +2,9 @@
Improved ingredient sync. Instead of loading the ingredients over the paginated
API (which was taking a long time with almost 3 million entries), wger instances
-can generate and read a dump file, which can be downloaded and read much more
-effectively. The setting `SYNC_INGREDIENTS_CELERY` will automatically prefer the
-bulk dump over the paginated API, falling back to the API if no dump is available.
+can generate and read a dump file, which can be processed much more effectively.
+The setting `SYNC_INGREDIENTS_CELERY` will automatically prefer the bulk dump
+over the paginated API, falling back to the API if no dump is available.
<https://github.com/wger-project/wger/issues/2231> Add vegan and vegetarian flags to
the ingredient model. This allows to filter products based on dietary preferences.
diff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.py
index f64e190f0..77e511528 100644
--- a/wger/nutrition/extract_info/wger.py
+++ b/wger/nutrition/extract_info/wger.py
@@ -20,7 +20,7 @@ from wger.utils.constants import ODBL_LICENSE_ID
def extract_info_from_wger_api(product_data: dict) -> IngredientData:
# Basics
- name = product_data.get('name')
+ name = product_data.get('name', '')
common_name = product_data.get('common_name', '')
code = product_data['code']
energy = float(product_data['energy'])
@@ -47,6 +47,7 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:
# Dietary properties
is_vegan = product_data.get('is_vegan', None)
is_vegetarian = product_data.get('is_vegetarian', None)
+ nutriscore = product_data.get('nutriscore', None)
# License and author info
source_name = product_data.get('source_name', '')
@@ -84,5 +85,6 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:
license_derivative_source_url=license_derivative_source_url,
is_vegan=is_vegan,
is_vegetarian=is_vegetarian,
+ nutriscore=nutriscore,
)
return ingredient_data
diff --git a/wger/nutrition/management/commands/bulk-sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients-bulk.py
similarity index 100%
rename from wger/nutrition/management/commands/bulk-sync-ingredients.py
rename to wger/nutrition/management/commands/sync-ingredients-bulk.py
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py
index 22356c4c5..6ed25a76b 100644
--- a/wger/nutrition/sync.py
+++ b/wger/nutrition/sync.py
@@ -26,6 +26,8 @@ from typing import (
# Django
from django.conf import settings
+from django.core.files.base import File
+from django.core.files.storage import default_storage
from django.db import IntegrityError
from django.utils import timezone
@@ -45,8 +47,10 @@ from tqdm import tqdm
from wger.core.models.language import Language
from wger.nutrition.api.endpoints import (
IMAGE_ENDPOINT,
+ INGREDIENT_BULK_EXPORT_PATH,
INGREDIENTS_ENDPOINT,
)
+from wger.nutrition.api.serializers import IngredientSerializer
from wger.nutrition.consts import SyncMode
from wger.nutrition.extract_info.wger import extract_info_from_wger_api
from wger.nutrition.models import (
@@ -149,7 +153,7 @@ def fetch_image_from_off(ingredient: Ingredient):
logger.info(f'Trying to fetch image from OFF for "{ingredient.name}" ({ingredient.uuid})')
# We always update the last check time, no matter if we found an image or there were
- # errors in the response (keys missing, etc) since in any case we don't want to retry
+ # errors in the response (keys missing, etc.) since in any case we don't want to retry
# too often.
ingredient.last_image_check = timezone.now()
ingredient.save()
@@ -305,14 +309,16 @@ def sync_ingredients(
if language_codes is not None:
language_ids = []
for code in language_codes.split(','):
- # Leaving the try except in here even though we've already validated on the sync-ingredients command itself.
- # This is in case we ever want to re-use this function for anything else where user can input language codes.
+ # Leaving the try except in here even though we've already validated on the
+ # sync-ingredients command itself. This is in case we ever want to re-use
+ # this function for anything else where user can input language codes.
try:
lang = load_language(code, default_to_english=False)
language_ids.append(str(lang.id))
- except Language.DoesNotExist as e:
+ except Language.DoesNotExist:
print_fn(
- f'Error: The language code you provided ("{code}") does not exist in this database. Please try again.'
+ f'Error: The language code you provided ("{code}") does not exist in '
+ f'this database. Please try again.'
)
return 0
@@ -378,15 +384,6 @@ def export_ingredient_dump(
(local filesystem, S3, GCS, etc.).
"""
- # Import here to avoid circular imports
- # Django
- from django.core.files.base import File
- from django.core.files.storage import default_storage
-
- # wger
- from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH
- from wger.nutrition.api.serializers import IngredientSerializer
-
# TODO: handle units here as well
queryset = Ingredient.objects.select_related(
'language',
@@ -427,7 +424,7 @@ def export_ingredient_dump(
return saved_name
-def _open_jsonl(file_path: str):
+def _open_jsonl(file_path: Path):
"""Open a JSONL file, transparently handling both gzip and plain text."""
with open(file_path, 'rb') as f:
magic = f.read(2)
@@ -438,7 +435,7 @@ def _open_jsonl(file_path: str):
def sync_ingredients_from_dump(
print_fn,
- file_path: str,
+ file_path: Path,
mode: SyncMode = SyncMode.UPDATE,
style_fn=lambda x: x,
show_progress_bar: bool = False,
@@ -525,14 +522,12 @@ def download_ingredient_dump(
remote_url: str = settings.WGER_SETTINGS['WGER_INSTANCE'],
folder: str = '',
style_fn=lambda x: x,
-) -> str:
+) -> Path:
"""
Download the ingredient JSONL dump from a remote wger instance.
Returns the path to the downloaded file.
"""
- # wger
- from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH
dump_url = f'{remote_url}{settings.MEDIA_URL}{INGREDIENT_BULK_EXPORT_PATH}'
print_fn(f'*** Downloading ingredient dump from {dump_url}...')
@@ -546,7 +541,7 @@ def download_ingredient_dump(
if file_path.exists():
print_fn(f'File already downloaded at {file_path}')
- return str(file_path)
+ return file_path
response = requests.get(dump_url, stream=True, headers=wger_headers())
if response.status_code == 404:
@@ -570,4 +565,4 @@ def download_ingredient_dump(
pbar.update(len(chunk))
print_fn(style_fn(f'Download complete: {file_path}\n'))
- return str(file_path)
+ return file_path
diff --git a/wger/nutrition/tests/test_sync_bulk.py b/wger/nutrition/tests/test_sync_bulk.py
index 822bad795..60b10682a 100644
--- a/wger/nutrition/tests/test_sync_bulk.py
+++ b/wger/nutrition/tests/test_sync_bulk.py
@@ -14,17 +14,20 @@
# Standard Library
import gzip
+import importlib
import json
-import os
+import shutil
import tempfile
from decimal import Decimal
from io import StringIO
+from pathlib import Path
from unittest.mock import (
MagicMock,
patch,
)
# Django
+from django.core.files.storage import default_storage
from django.core.management import call_command
from django.core.management.base import CommandError
from django.test import SimpleTestCase
@@ -93,10 +96,12 @@ SAMPLE_INGREDIENTS = [
def _write_dump(path, ingredients, use_gzip=True):
"""Helper to write a JSONL dump file."""
+
if use_gzip:
with gzip.open(path, 'wt', encoding='utf-8') as f:
for item in ingredients:
f.write(json.dumps(item) + '\n')
+
else:
with open(path, 'w', encoding='utf-8') as f:
for item in ingredients:
@@ -117,7 +122,7 @@ class TestOpenJsonl(SimpleTestCase):
self.assertEqual(len(lines), 2)
self.assertEqual(json.loads(lines[0])['name'], 'Gâteau double chocolat')
finally:
- os.unlink(tmp_path)
+ Path(tmp_path).unlink()
def test_opens_plain_file(self):
with tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False, mode='w') as tmp:
@@ -130,7 +135,7 @@ class TestOpenJsonl(SimpleTestCase):
self.assertEqual(len(lines), 2)
self.assertEqual(json.loads(lines[1])['name'], 'Maxi Hot Dog New York Style')
finally:
- os.unlink(tmp_path)
+ Path(tmp_path).unlink()
class TestExportIngredientDump(WgerTestCase):
@@ -141,9 +146,6 @@ class TestExportIngredientDump(WgerTestCase):
self.tmp_media = tempfile.mkdtemp()
def tearDown(self):
- # Standard Library
- import shutil
-
shutil.rmtree(self.tmp_media, ignore_errors=True)
super().tearDown()
@@ -156,9 +158,6 @@ class TestExportIngredientDump(WgerTestCase):
with self.settings(MEDIA_ROOT=self.tmp_media):
saved_name = self._export()
- # Django
- from django.core.files.storage import default_storage
-
self.assertTrue(default_storage.exists(saved_name))
# Read back and verify content
@@ -194,9 +193,6 @@ class TestExportIngredientDump(WgerTestCase):
self.assertEqual(saved_name_1, saved_name_2)
with self.settings(MEDIA_ROOT=self.tmp_media):
- # Django
- from django.core.files.storage import default_storage
-
self.assertTrue(default_storage.exists(saved_name_2))
@@ -217,9 +213,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
try:
# Ingredient with this UUID already exists in fixtures
- ingredient = Ingredient.objects.get(
- uuid='7908c204-907f-4b1e-ad4e-f482e9769ade',
- )
+ ingredient = Ingredient.objects.get(uuid='7908c204-907f-4b1e-ad4e-f482e9769ade')
self.assertEqual(ingredient.name, 'Test ingredient 1')
count = sync_ingredients_from_dump(lambda x: x, file_path)
@@ -230,7 +224,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
self.assertAlmostEqual(ingredient.protein, Decimal(5), 2)
self.assertEqual(count, 2)
finally:
- os.unlink(file_path)
+ Path(file_path).unlink()
def test_update_mode_creates_new(self):
"""Test that UPDATE mode creates new ingredients for unknown UUIDs."""
@@ -248,7 +242,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
self.assertEqual(new_ingredient.name, 'Maxi Hot Dog New York Style')
self.assertEqual(new_ingredient.energy, 256)
finally:
- os.unlink(file_path)
+ Path(file_path).unlink()
def test_insert_mode(self):
"""Test that INSERT mode bulk-creates ingredients."""
@@ -270,7 +264,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
self.assertEqual(count, 2)
self.assertEqual(Ingredient.objects.count(), initial_count + 2)
finally:
- os.unlink(file_path)
+ Path(file_path).unlink()
def test_handles_plain_jsonl(self):
"""Test that import works with plain (non-gzipped) JSONL files."""
@@ -280,7 +274,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
count = sync_ingredients_from_dump(lambda x: x, file_path)
self.assertEqual(count, 2)
finally:
- os.unlink(file_path)
+ Path(file_path).unlink()
def test_handles_invalid_json(self):
"""Test that invalid JSON lines are skipped."""
@@ -293,7 +287,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
count = sync_ingredients_from_dump(lambda x: x, tmp.name)
self.assertEqual(count, 1)
finally:
- os.unlink(tmp.name)
+ Path(tmp.name).unlink()
def test_handles_missing_uuid(self):
"""Test that entries without UUID are skipped."""
@@ -309,7 +303,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
count = sync_ingredients_from_dump(lambda x: x, tmp.name)
self.assertEqual(count, 1)
finally:
- os.unlink(tmp.name)
+ Path(tmp.name).unlink()
def test_returns_error_count(self):
"""Test that the function correctly tracks errors."""
@@ -325,7 +319,7 @@ class TestSyncIngredientsFromDump(WgerTestCase):
count = sync_ingredients_from_dump(lambda x: x, tmp.name)
self.assertEqual(count, 1)
finally:
- os.unlink(tmp.name)
+ Path(tmp.name).unlink()
class TestDownloadIngredientDump(WgerTestCase):
@@ -345,8 +339,8 @@ class TestDownloadIngredientDump(WgerTestCase):
with tempfile.TemporaryDirectory() as folder:
path = download_ingredient_dump(lambda x: x, folder=folder)
- self.assertTrue(os.path.exists(path))
- self.assertTrue(path.endswith('ingredients.jsonl.gz'))
+ self.assertTrue(path.exists())
+ self.assertEqual(path.name, 'ingredients.jsonl.gz')
@patch('wger.nutrition.sync.requests.get')
def test_download_404_raises_file_not_found(self, mock_get: MagicMock):
@@ -371,9 +365,8 @@ class TestDownloadIngredientDump(WgerTestCase):
def test_download_skips_existing_file(self, mock_get: MagicMock):
"""Test that an already-downloaded file is reused."""
with tempfile.TemporaryDirectory() as folder:
- file_path = os.path.join(folder, 'ingredients.jsonl.gz')
- with open(file_path, 'w') as f:
- f.write('existing')
+ file_path = Path(folder) / 'ingredients.jsonl.gz'
+ file_path.write_text('existing')
path = download_ingredient_dump(lambda x: x, folder=folder)
@@ -407,26 +400,25 @@ class TestBulkSyncManagementCommands(WgerTestCase):
self.assertIn('done!', output)
def _get_command_module(self):
- """Get the bulk-sync-ingredients command module (hyphenated name needs importlib)."""
- # Standard Library
- import importlib
-
- return importlib.import_module('wger.nutrition.management.commands.bulk-sync-ingredients')
+ """Get the sync-ingredients-bulk command module (hyphenated name needs importlib)."""
+ return importlib.import_module('wger.nutrition.management.commands.sync-ingredients-bulk')
@patch('wger.core.api.min_server_version.check_min_server_version')
def test_bulk_sync_command_calls_functions(self, mock_version_check: MagicMock):
- """Test that the bulk-sync-ingredients command calls the right functions."""
+ """Test that the sync-ingredients-bulk command calls the right functions."""
mod = self._get_command_module()
with (
patch.object(
- mod, 'download_ingredient_dump', return_value='/tmp/fake.jsonl.gz'
+ mod,
+ 'download_ingredient_dump',
+ return_value=Path('/tmp/fake.jsonl.gz'),
) as mock_download,
patch.object(mod, 'sync_ingredients_from_dump') as mock_sync,
):
out = StringIO()
call_command(
- 'bulk-sync-ingredients',
+ 'sync-ingredients-bulk',
'--remote-url',
'https://example.com',
stdout=out,
@@ -448,7 +440,7 @@ class TestBulkSyncManagementCommands(WgerTestCase):
out = StringIO()
with self.assertRaises(CommandError):
call_command(
- 'bulk-sync-ingredients',
+ 'sync-ingredients-bulk',
'--remote-url',
'https://example.com',
stdout=out,