Self hosted FLOSS fitness/workout, nutrition and weight tracker
Add bulk export and sync functionality for ingredients
Roland Geider2fd0579d5850
diff --git a/wger/nutrition/api/endpoints.py b/wger/nutrition/api/endpoints.pyindex 3e7f8301e..ab8e267b1 100644--- a/wger/nutrition/api/endpoints.py+++ b/wger/nutrition/api/endpoints.py@@ -1,2 +1,3 @@IMAGE_ENDPOINT = 'ingredient-image'INGREDIENTS_ENDPOINT = 'ingredient'+INGREDIENT_BULK_EXPORT_PATH = 'ingredients/ingredients.jsonl.gz'diff --git a/wger/nutrition/consts.py b/wger/nutrition/consts.pyindex d5c4f2d95..e55ec57ed 100644--- a/wger/nutrition/consts.py+++ b/wger/nutrition/consts.py@@ -14,6 +14,22 @@# You should have received a copy of the GNU Affero General Public License# along with this program. If not, see <http://www.gnu.org/licenses/>.+# Standard Library+import enum+++class SyncMode(enum.Enum):+ """+ Mode for ingredient sync/import scripts.++ INSERT: bulk-insert new ingredients (very fast, use for empty databases)+ UPDATE: update existing ingredients or create new ones (slower, 2 queries per product)+ """++ INSERT = enum.auto()+ UPDATE = enum.auto()++MEALITEM_WEIGHT_GRAM = '1'MEALITEM_WEIGHT_UNIT = '2'diff --git a/wger/nutrition/management/commands/bulk-sync-ingredients.py b/wger/nutrition/management/commands/bulk-sync-ingredients.pynew file mode 100644index 000000000..32a269cbf--- /dev/null+++ b/wger/nutrition/management/commands/bulk-sync-ingredients.py@@ -0,0 +1,90 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Django+from django.core.management.base import CommandError++# wger+from wger.core.api.min_server_version import check_min_server_version+from wger.core.management.wger_command import WgerCommand+from wger.nutrition.consts import SyncMode+from wger.nutrition.sync import (+ download_ingredient_dump,+ sync_ingredients_from_dump,+)+++class Command(WgerCommand):+ """+ Download a JSONL dump from a wger instance and bulk-import the ingredients.++ This is much faster than the paginated API sync for initial imports or+ full re-syncs of large ingredient databases.+ """++ help = 'Bulk-sync ingredients by downloading a JSONL dump from a wger instance'++ def add_arguments(self, parser):+ super().add_arguments(parser)++ parser.add_argument(+ '--set-mode',+ action='store',+ default='update',+ dest='mode',+ type=str,+ help=(+ 'Script mode, "insert" or "update". Insert will bulk-insert the ingredients as '+ 'new entries (fastest, use for empty databases). Update will update existing '+ 'entries or create new ones. Default: update'+ ),+ )++ parser.add_argument(+ '--folder',+ action='store',+ default='',+ dest='folder',+ type=str,+ help=(+ 'Folder for storing the downloaded dump file. If the file already exists, '+ 'it will be reused. Default: use a temporary folder.'+ ),+ )++ def handle(self, **options):+ super().handle(**options)++ remote_url = self.remote_url+ mode = SyncMode.INSERT if options['mode'] == 'insert' else SyncMode.UPDATE++ check_min_server_version(remote_url)++ try:+ file_path = download_ingredient_dump(+ self.stdout.write,+ remote_url=remote_url,+ folder=options['folder'],+ style_fn=self.style.SUCCESS,+ )+ except FileNotFoundError as e:+ raise CommandError(str(e))++ sync_ingredients_from_dump(+ self.stdout.write,+ file_path=file_path,+ mode=mode,+ style_fn=self.style.SUCCESS,+ show_progress_bar=True,+ )diff --git a/wger/nutrition/management/commands/export-ingredients.py b/wger/nutrition/management/commands/export-ingredients.pynew file mode 100644index 000000000..b94849dd5--- /dev/null+++ b/wger/nutrition/management/commands/export-ingredients.py@@ -0,0 +1,37 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Django+from django.core.management.base import BaseCommand++# wger+from wger.nutrition.sync import export_ingredient_dump+++class Command(BaseCommand):+ """+ Export all ingredients as a gzipped JSONL file for bulk synchronization.++ The generated file is placed in MEDIA_ROOT/ingredients/ and can be+ served to other wger instances for fast bulk imports.+ """++ help = 'Export all ingredients as a gzipped JSONL dump file for bulk synchronization'++ def handle(self, **options):+ export_ingredient_dump(+ self.stdout.write,+ style_fn=self.style.SUCCESS,+ show_progress_bar=True,+ )diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.pyindex e848730b8..49a625e42 100644--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@ -22,10 +22,8 @@ import requests# wgerfrom wger.core.models import Languagefrom wger.nutrition.extract_info.off import extract_info_from_off-from wger.nutrition.management.products import (- ImportProductCommand,- Mode,-)+from wger.nutrition.consts import SyncMode+from wger.nutrition.management.products import ImportProductCommandlogger = logging.getLogger(__name__)@@ -133,7 +131,7 @@ class Command(ImportProductCommand):def handle(self, **options):if options['mode'] == 'insert':- self.mode = Mode.INSERT+ self.mode = SyncMode.INSERTlanguages = {lang.short_name: lang.pk for lang in Language.objects.all()}diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.pyindex 53e238cbe..ff3498355 100644--- a/wger/nutrition/management/commands/import-usda-products.py+++ b/wger/nutrition/management/commands/import-usda-products.py@@ -22,10 +22,8 @@ from zipfile import ZipFile# wgerfrom wger.core.models import Languagefrom wger.nutrition.extract_info.usda import extract_info_from_usda-from wger.nutrition.management.products import (- ImportProductCommand,- Mode,-)+from wger.nutrition.consts import SyncMode+from wger.nutrition.management.products import ImportProductCommandfrom wger.utils.constants import ENGLISH_SHORT_NAME@@ -53,7 +51,7 @@ class Command(ImportProductCommand):def handle(self, **options):if options['mode'] == 'insert':- self.mode = Mode.INSERT+ self.mode = SyncMode.INSERTdataset_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}'download_folder, tmp_folder = self.get_download_folder(options['folder'])diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.pyindex 4530b1407..596eec3a3 100644--- a/wger/nutrition/management/products.py+++ b/wger/nutrition/management/products.py@@ -13,7 +13,6 @@# You should have received a copy of the GNU Affero General Public License# Standard Library-import enumimport jsonimport loggingimport os@@ -31,6 +30,7 @@ import requestsfrom tqdm import tqdm# wger+from wger.nutrition.consts import SyncModefrom wger.nutrition.dataclasses import IngredientDatafrom wger.nutrition.models import Ingredientfrom wger.utils.requests import wger_headers@@ -39,22 +39,12 @@ from wger.utils.requests import wger_headerslogger = logging.getLogger(__name__)-# Mode for this script. When using 'insert', the script will bulk-insert the new-# ingredients, which is very efficient. Importing the whole database will require-# barely a minute. When using 'update', existing ingredients will be updated, which-# requires two queries per product and is needed when there are already existing-# entries in the local ingredient table.-class Mode(enum.Enum):- INSERT = enum.auto()- UPDATE = enum.auto()--class ImportProductCommand(BaseCommand):"""Import an Open Food facts Dump"""- mode = Mode.UPDATE+ mode = SyncMode.UPDATEbulk_update_bucket: list[Ingredient] = []bulk_size = 500counter: Counter@@ -100,7 +90,7 @@ class ImportProductCommand(BaseCommand):## Add entries as new productsingredient_data.clean_name()- if self.mode == Mode.INSERT:+ if self.mode == SyncMode.INSERT:self.bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))if len(self.bulk_update_bucket) > self.bulk_size:try:diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex a30038d18..c430cf23c 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -13,8 +13,12 @@# You should have received a copy of the GNU Affero General Public License# Standard Library+import gzip+import jsonimport loggingimport os+import tempfile+from pathlib import Pathfrom typing import (List,Optional,@@ -39,6 +43,7 @@ from tqdm import tqdm# wgerfrom wger.core.models.language import Language+from wger.nutrition.consts import SyncModefrom wger.nutrition.api.endpoints import (IMAGE_ENDPOINT,INGREDIENTS_ENDPOINT,@@ -353,3 +358,213 @@ def sync_ingredients(print_fn(style_fn('done!\n'))return None+++BULK_SIZE = 500+++def export_ingredient_dump(+ print_fn,+ style_fn=lambda x: x,+ show_progress_bar: bool = False,+):+ """+ Export all ingredients as a gzipped JSONL file via Django's default storage.++ Each line is the same JSON format as the ingredient API endpoint,+ so clients can parse it with extract_info_from_wger_api().++ Uses default_storage so the dump works with any configured backend+ (local filesystem, S3, GCS, etc.).+ """++ # Import here to avoid circular imports+ from django.core.files.base import File+ from django.core.files.storage import default_storage++ from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH+ from wger.nutrition.api.serializers import IngredientSerializer++ # TODO: handle units here as well+ queryset = Ingredient.objects.select_related(+ 'language',+ 'license',+ )+ total = queryset.count()+ queryset = queryset.iterator(chunk_size=2000)++ print_fn('*** Exporting ingredients to JSONL dump...')++ pbar = tqdm(total=total, unit='ingredients', desc='Exporting', disable=not show_progress_bar)++ # Write to a local temp file first, then upload via storage backend+ count = 0+ with tempfile.NamedTemporaryFile(suffix='.jsonl.gz', delete=False) as tmp:+ tmp_path = tmp.name++ try:+ with gzip.open(tmp_path, 'wt', encoding='utf-8') as f:+ for ingredient in queryset:+ data = IngredientSerializer(ingredient).data+ f.write(json.dumps(data) + '\n')+ count += 1+ pbar.update(1)++ pbar.close()++ # Upload to the configured storage backend+ with open(tmp_path, 'rb') as f:+ # delete old file first if it exists, then save the new one+ if default_storage.exists(INGREDIENT_BULK_EXPORT_PATH):+ default_storage.delete(INGREDIENT_BULK_EXPORT_PATH)+ saved_name = default_storage.save(INGREDIENT_BULK_EXPORT_PATH, File(f))+ finally:+ Path(tmp_path).unlink()++ print_fn(style_fn(f'done! Exported {count} ingredients to {saved_name}\n'))+ return saved_name+++def _open_jsonl(file_path: str):+ """Open a JSONL file, transparently handling both gzip and plain text."""+ with open(file_path, 'rb') as f:+ magic = f.read(2)+ if magic == b'\x1f\x8b':+ return gzip.open(file_path, 'rt', encoding='utf-8')+ return open(file_path, 'r', encoding='utf-8')+++def sync_ingredients_from_dump(+ print_fn,+ file_path: str,+ mode: SyncMode = SyncMode.UPDATE,+ style_fn=lambda x: x,+ show_progress_bar: bool = False,+):+ """+ Import ingredients from a JSONL dump file (gzipped or plain).++ Each line must be in the same JSON format as the ingredient API endpoint.+ Reuses extract_info_from_wger_api() for data transformation.+ """+ print_fn(f'*** Importing ingredients from {file_path} (mode: {mode.name})...')++ # Count lines for the progress bar+ total = 0+ if show_progress_bar:+ with _open_jsonl(file_path) as f:+ total = sum(1 for _ in f)++ pbar = tqdm(total=total, unit='ingredients', desc='Importing', disable=not show_progress_bar)++ bulk_bucket: list[Ingredient] = []+ count = 0+ errors = 0++ with _open_jsonl(file_path) as f:+ for line in f:+ try:+ data = json.loads(line)+ except json.JSONDecodeError:+ errors += 1+ continue++ uuid = data.get('uuid')+ if not uuid:+ errors += 1+ continue++ ingredient_data = extract_info_from_wger_api(data).dict()+ ingredient_data['uuid'] = uuid++ if mode == SyncMode.INSERT:+ bulk_bucket.append(Ingredient(**ingredient_data))+ if len(bulk_bucket) >= BULK_SIZE:+ try:+ Ingredient.objects.bulk_create(bulk_bucket)+ except Exception:+ # Fall back to individual saves+ for ingredient in bulk_bucket:+ try:+ ingredient.save()+ except Exception:+ errors += 1+ count += len(bulk_bucket)+ bulk_bucket = []+ else:+ try:+ Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)+ count += 1+ except Exception:+ errors += 1++ pbar.update(1)++ # Flush remaining items in INSERT mode+ if mode == SyncMode.INSERT and bulk_bucket:+ try:+ Ingredient.objects.bulk_create(bulk_bucket)+ except Exception:+ for ingredient in bulk_bucket:+ try:+ ingredient.save()+ except Exception:+ errors += 1+ count += len(bulk_bucket)++ pbar.close()++ print_fn(style_fn(f'done! Processed {count} ingredients ({errors} errors)\n'))+ return count+++def download_ingredient_dump(+ print_fn,+ remote_url: str = settings.WGER_SETTINGS['WGER_INSTANCE'],+ folder: str = '',+ style_fn=lambda x: x,+) -> str:+ """+ Download the ingredient JSONL dump from a remote wger instance.++ Returns the path to the downloaded file.+ """+ from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH++ dump_url = f'{remote_url}{settings.MEDIA_URL}{INGREDIENT_BULK_EXPORT_PATH}'+ print_fn(f'*** Downloading ingredient dump from {dump_url}...')++ if folder:+ download_folder = folder+ else:+ download_folder = tempfile.mkdtemp()++ file_path = Path(download_folder) / 'ingredients.jsonl.gz'++ if file_path.exists():+ print_fn(f'File already downloaded at {file_path}')+ return str(file_path)++ response = requests.get(dump_url, stream=True, headers=wger_headers())+ if response.status_code == 404:+ raise FileNotFoundError(+ f'Bulk ingredient dump not found at {dump_url}. '+ f'The remote server may not have generated a dump yet. '+ f'Please ask the server admin to run "python manage.py export-ingredients" first, '+ f'or use "python manage.py sync-ingredients" to sync via the API instead.'+ )+ if response.status_code != 200:+ raise Exception(f'Could not download dump from {dump_url} (status {response.status_code})')++ # Prevent requests from transparently decompressing the gzip file+ response.raw.decode_content = False+ total_size = int(response.headers.get('content-length', 0))++ with open(file_path, 'wb') as fid:+ with tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') as pbar:+ for chunk in response.raw.stream(50 * 1024):+ fid.write(chunk)+ pbar.update(len(chunk))++ print_fn(style_fn(f'Download complete: {file_path}\n'))+ return str(file_path)