Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Add bulk export and sync functionality for ingredients

Roland Geider2fd0579d5850
diff --git a/wger/nutrition/api/endpoints.py b/wger/nutrition/api/endpoints.py
index 3e7f8301e..ab8e267b1 100644
--- a/wger/nutrition/api/endpoints.py
+++ b/wger/nutrition/api/endpoints.py
@@ -1,2 +1,3 @@
IMAGE_ENDPOINT = 'ingredient-image'
INGREDIENTS_ENDPOINT = 'ingredient'
+INGREDIENT_BULK_EXPORT_PATH = 'ingredients/ingredients.jsonl.gz'
diff --git a/wger/nutrition/consts.py b/wger/nutrition/consts.py
index d5c4f2d95..e55ec57ed 100644
--- a/wger/nutrition/consts.py
+++ b/wger/nutrition/consts.py
@@ -14,6 +14,22 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# Standard Library
+import enum
+
+
+class SyncMode(enum.Enum):
+ """
+ Mode for ingredient sync/import scripts.
+
+ INSERT: bulk-insert new ingredients (very fast, use for empty databases)
+ UPDATE: update existing ingredients or create new ones (slower, 2 queries per product)
+ """
+
+ INSERT = enum.auto()
+ UPDATE = enum.auto()
+
+
MEALITEM_WEIGHT_GRAM = '1'
MEALITEM_WEIGHT_UNIT = '2'
diff --git a/wger/nutrition/management/commands/bulk-sync-ingredients.py b/wger/nutrition/management/commands/bulk-sync-ingredients.py
new file mode 100644
index 000000000..32a269cbf
--- /dev/null
+++ b/wger/nutrition/management/commands/bulk-sync-ingredients.py
@@ -0,0 +1,90 @@
+# This file is part of wger Workout Manager.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+
+# Django
+from django.core.management.base import CommandError
+
+# wger
+from wger.core.api.min_server_version import check_min_server_version
+from wger.core.management.wger_command import WgerCommand
+from wger.nutrition.consts import SyncMode
+from wger.nutrition.sync import (
+ download_ingredient_dump,
+ sync_ingredients_from_dump,
+)
+
+
+class Command(WgerCommand):
+ """
+ Download a JSONL dump from a wger instance and bulk-import the ingredients.
+
+ This is much faster than the paginated API sync for initial imports or
+ full re-syncs of large ingredient databases.
+ """
+
+ help = 'Bulk-sync ingredients by downloading a JSONL dump from a wger instance'
+
+ def add_arguments(self, parser):
+ super().add_arguments(parser)
+
+ parser.add_argument(
+ '--set-mode',
+ action='store',
+ default='update',
+ dest='mode',
+ type=str,
+ help=(
+ 'Script mode, "insert" or "update". Insert will bulk-insert the ingredients as '
+ 'new entries (fastest, use for empty databases). Update will update existing '
+ 'entries or create new ones. Default: update'
+ ),
+ )
+
+ parser.add_argument(
+ '--folder',
+ action='store',
+ default='',
+ dest='folder',
+ type=str,
+ help=(
+ 'Folder for storing the downloaded dump file. If the file already exists, '
+ 'it will be reused. Default: use a temporary folder.'
+ ),
+ )
+
+ def handle(self, **options):
+ super().handle(**options)
+
+ remote_url = self.remote_url
+ mode = SyncMode.INSERT if options['mode'] == 'insert' else SyncMode.UPDATE
+
+ check_min_server_version(remote_url)
+
+ try:
+ file_path = download_ingredient_dump(
+ self.stdout.write,
+ remote_url=remote_url,
+ folder=options['folder'],
+ style_fn=self.style.SUCCESS,
+ )
+ except FileNotFoundError as e:
+ raise CommandError(str(e))
+
+ sync_ingredients_from_dump(
+ self.stdout.write,
+ file_path=file_path,
+ mode=mode,
+ style_fn=self.style.SUCCESS,
+ show_progress_bar=True,
+ )
diff --git a/wger/nutrition/management/commands/export-ingredients.py b/wger/nutrition/management/commands/export-ingredients.py
new file mode 100644
index 000000000..b94849dd5
--- /dev/null
+++ b/wger/nutrition/management/commands/export-ingredients.py
@@ -0,0 +1,37 @@
+# This file is part of wger Workout Manager.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+
+# Django
+from django.core.management.base import BaseCommand
+
+# wger
+from wger.nutrition.sync import export_ingredient_dump
+
+
+class Command(BaseCommand):
+ """
+ Export all ingredients as a gzipped JSONL file for bulk synchronization.
+
+ The generated file is placed in MEDIA_ROOT/ingredients/ and can be
+ served to other wger instances for fast bulk imports.
+ """
+
+ help = 'Export all ingredients as a gzipped JSONL dump file for bulk synchronization'
+
+ def handle(self, **options):
+ export_ingredient_dump(
+ self.stdout.write,
+ style_fn=self.style.SUCCESS,
+ show_progress_bar=True,
+ )
diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py
index e848730b8..49a625e42 100644
--- a/wger/nutrition/management/commands/import-off-products.py
+++ b/wger/nutrition/management/commands/import-off-products.py
@@ -22,10 +22,8 @@ import requests
# wger
from wger.core.models import Language
from wger.nutrition.extract_info.off import extract_info_from_off
-from wger.nutrition.management.products import (
- ImportProductCommand,
- Mode,
-)
+from wger.nutrition.consts import SyncMode
+from wger.nutrition.management.products import ImportProductCommand
logger = logging.getLogger(__name__)
@@ -133,7 +131,7 @@ class Command(ImportProductCommand):
def handle(self, **options):
if options['mode'] == 'insert':
- self.mode = Mode.INSERT
+ self.mode = SyncMode.INSERT
languages = {lang.short_name: lang.pk for lang in Language.objects.all()}
diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py
index 53e238cbe..ff3498355 100644
--- a/wger/nutrition/management/commands/import-usda-products.py
+++ b/wger/nutrition/management/commands/import-usda-products.py
@@ -22,10 +22,8 @@ from zipfile import ZipFile
# wger
from wger.core.models import Language
from wger.nutrition.extract_info.usda import extract_info_from_usda
-from wger.nutrition.management.products import (
- ImportProductCommand,
- Mode,
-)
+from wger.nutrition.consts import SyncMode
+from wger.nutrition.management.products import ImportProductCommand
from wger.utils.constants import ENGLISH_SHORT_NAME
@@ -53,7 +51,7 @@ class Command(ImportProductCommand):
def handle(self, **options):
if options['mode'] == 'insert':
- self.mode = Mode.INSERT
+ self.mode = SyncMode.INSERT
dataset_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}'
download_folder, tmp_folder = self.get_download_folder(options['folder'])
diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py
index 4530b1407..596eec3a3 100644
--- a/wger/nutrition/management/products.py
+++ b/wger/nutrition/management/products.py
@@ -13,7 +13,6 @@
# You should have received a copy of the GNU Affero General Public License
# Standard Library
-import enum
import json
import logging
import os
@@ -31,6 +30,7 @@ import requests
from tqdm import tqdm
# wger
+from wger.nutrition.consts import SyncMode
from wger.nutrition.dataclasses import IngredientData
from wger.nutrition.models import Ingredient
from wger.utils.requests import wger_headers
@@ -39,22 +39,12 @@ from wger.utils.requests import wger_headers
logger = logging.getLogger(__name__)
-# Mode for this script. When using 'insert', the script will bulk-insert the new
-# ingredients, which is very efficient. Importing the whole database will require
-# barely a minute. When using 'update', existing ingredients will be updated, which
-# requires two queries per product and is needed when there are already existing
-# entries in the local ingredient table.
-class Mode(enum.Enum):
- INSERT = enum.auto()
- UPDATE = enum.auto()
-
-
class ImportProductCommand(BaseCommand):
"""
Import an Open Food facts Dump
"""
- mode = Mode.UPDATE
+ mode = SyncMode.UPDATE
bulk_update_bucket: list[Ingredient] = []
bulk_size = 500
counter: Counter
@@ -100,7 +90,7 @@ class ImportProductCommand(BaseCommand):
#
# Add entries as new products
ingredient_data.clean_name()
- if self.mode == Mode.INSERT:
+ if self.mode == SyncMode.INSERT:
self.bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))
if len(self.bulk_update_bucket) > self.bulk_size:
try:
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py
index a30038d18..c430cf23c 100644
--- a/wger/nutrition/sync.py
+++ b/wger/nutrition/sync.py
@@ -13,8 +13,12 @@
# You should have received a copy of the GNU Affero General Public License
# Standard Library
+import gzip
+import json
import logging
import os
+import tempfile
+from pathlib import Path
from typing import (
List,
Optional,
@@ -39,6 +43,7 @@ from tqdm import tqdm
# wger
from wger.core.models.language import Language
+from wger.nutrition.consts import SyncMode
from wger.nutrition.api.endpoints import (
IMAGE_ENDPOINT,
INGREDIENTS_ENDPOINT,
@@ -353,3 +358,213 @@ def sync_ingredients(
print_fn(style_fn('done!\n'))
return None
+
+
+BULK_SIZE = 500
+
+
+def export_ingredient_dump(
+ print_fn,
+ style_fn=lambda x: x,
+ show_progress_bar: bool = False,
+):
+ """
+ Export all ingredients as a gzipped JSONL file via Django's default storage.
+
+ Each line is the same JSON format as the ingredient API endpoint,
+ so clients can parse it with extract_info_from_wger_api().
+
+ Uses default_storage so the dump works with any configured backend
+ (local filesystem, S3, GCS, etc.).
+ """
+
+ # Import here to avoid circular imports
+ from django.core.files.base import File
+ from django.core.files.storage import default_storage
+
+ from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH
+ from wger.nutrition.api.serializers import IngredientSerializer
+
+ # TODO: handle units here as well
+ queryset = Ingredient.objects.select_related(
+ 'language',
+ 'license',
+ )
+ total = queryset.count()
+ queryset = queryset.iterator(chunk_size=2000)
+
+ print_fn('*** Exporting ingredients to JSONL dump...')
+
+ pbar = tqdm(total=total, unit='ingredients', desc='Exporting', disable=not show_progress_bar)
+
+ # Write to a local temp file first, then upload via storage backend
+ count = 0
+ with tempfile.NamedTemporaryFile(suffix='.jsonl.gz', delete=False) as tmp:
+ tmp_path = tmp.name
+
+ try:
+ with gzip.open(tmp_path, 'wt', encoding='utf-8') as f:
+ for ingredient in queryset:
+ data = IngredientSerializer(ingredient).data
+ f.write(json.dumps(data) + '\n')
+ count += 1
+ pbar.update(1)
+
+ pbar.close()
+
+ # Upload to the configured storage backend
+ with open(tmp_path, 'rb') as f:
+ # delete old file first if it exists, then save the new one
+ if default_storage.exists(INGREDIENT_BULK_EXPORT_PATH):
+ default_storage.delete(INGREDIENT_BULK_EXPORT_PATH)
+ saved_name = default_storage.save(INGREDIENT_BULK_EXPORT_PATH, File(f))
+ finally:
+ Path(tmp_path).unlink()
+
+ print_fn(style_fn(f'done! Exported {count} ingredients to {saved_name}\n'))
+ return saved_name
+
+
+def _open_jsonl(file_path: str):
+ """Open a JSONL file, transparently handling both gzip and plain text."""
+ with open(file_path, 'rb') as f:
+ magic = f.read(2)
+ if magic == b'\x1f\x8b':
+ return gzip.open(file_path, 'rt', encoding='utf-8')
+ return open(file_path, 'r', encoding='utf-8')
+
+
+def sync_ingredients_from_dump(
+ print_fn,
+ file_path: str,
+ mode: SyncMode = SyncMode.UPDATE,
+ style_fn=lambda x: x,
+ show_progress_bar: bool = False,
+):
+ """
+ Import ingredients from a JSONL dump file (gzipped or plain).
+
+ Each line must be in the same JSON format as the ingredient API endpoint.
+ Reuses extract_info_from_wger_api() for data transformation.
+ """
+ print_fn(f'*** Importing ingredients from {file_path} (mode: {mode.name})...')
+
+ # Count lines for the progress bar
+ total = 0
+ if show_progress_bar:
+ with _open_jsonl(file_path) as f:
+ total = sum(1 for _ in f)
+
+ pbar = tqdm(total=total, unit='ingredients', desc='Importing', disable=not show_progress_bar)
+
+ bulk_bucket: list[Ingredient] = []
+ count = 0
+ errors = 0
+
+ with _open_jsonl(file_path) as f:
+ for line in f:
+ try:
+ data = json.loads(line)
+ except json.JSONDecodeError:
+ errors += 1
+ continue
+
+ uuid = data.get('uuid')
+ if not uuid:
+ errors += 1
+ continue
+
+ ingredient_data = extract_info_from_wger_api(data).dict()
+ ingredient_data['uuid'] = uuid
+
+ if mode == SyncMode.INSERT:
+ bulk_bucket.append(Ingredient(**ingredient_data))
+ if len(bulk_bucket) >= BULK_SIZE:
+ try:
+ Ingredient.objects.bulk_create(bulk_bucket)
+ except Exception:
+ # Fall back to individual saves
+ for ingredient in bulk_bucket:
+ try:
+ ingredient.save()
+ except Exception:
+ errors += 1
+ count += len(bulk_bucket)
+ bulk_bucket = []
+ else:
+ try:
+ Ingredient.objects.update_or_create(uuid=uuid, defaults=ingredient_data)
+ count += 1
+ except Exception:
+ errors += 1
+
+ pbar.update(1)
+
+ # Flush remaining items in INSERT mode
+ if mode == SyncMode.INSERT and bulk_bucket:
+ try:
+ Ingredient.objects.bulk_create(bulk_bucket)
+ except Exception:
+ for ingredient in bulk_bucket:
+ try:
+ ingredient.save()
+ except Exception:
+ errors += 1
+ count += len(bulk_bucket)
+
+ pbar.close()
+
+ print_fn(style_fn(f'done! Processed {count} ingredients ({errors} errors)\n'))
+ return count
+
+
+def download_ingredient_dump(
+ print_fn,
+ remote_url: str = settings.WGER_SETTINGS['WGER_INSTANCE'],
+ folder: str = '',
+ style_fn=lambda x: x,
+) -> str:
+ """
+ Download the ingredient JSONL dump from a remote wger instance.
+
+ Returns the path to the downloaded file.
+ """
+ from wger.nutrition.api.endpoints import INGREDIENT_BULK_EXPORT_PATH
+
+ dump_url = f'{remote_url}{settings.MEDIA_URL}{INGREDIENT_BULK_EXPORT_PATH}'
+ print_fn(f'*** Downloading ingredient dump from {dump_url}...')
+
+ if folder:
+ download_folder = folder
+ else:
+ download_folder = tempfile.mkdtemp()
+
+ file_path = Path(download_folder) / 'ingredients.jsonl.gz'
+
+ if file_path.exists():
+ print_fn(f'File already downloaded at {file_path}')
+ return str(file_path)
+
+ response = requests.get(dump_url, stream=True, headers=wger_headers())
+ if response.status_code == 404:
+ raise FileNotFoundError(
+ f'Bulk ingredient dump not found at {dump_url}. '
+ f'The remote server may not have generated a dump yet. '
+ f'Please ask the server admin to run "python manage.py export-ingredients" first, '
+ f'or use "python manage.py sync-ingredients" to sync via the API instead.'
+ )
+ if response.status_code != 200:
+ raise Exception(f'Could not download dump from {dump_url} (status {response.status_code})')
+
+ # Prevent requests from transparently decompressing the gzip file
+ response.raw.decode_content = False
+ total_size = int(response.headers.get('content-length', 0))
+
+ with open(file_path, 'wb') as fid:
+ with tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') as pbar:
+ for chunk in response.raw.stream(50 * 1024):
+ fid.write(chunk)
+ pbar.update(len(chunk))
+
+ print_fn(style_fn(f'Download complete: {file_path}\n'))
+ return str(file_path)