Self hosted FLOSS fitness/workout, nutrition and weight tracker
Properly handle temp files and other small fixes
Roland Geider410b1d8e6e5a
diff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.pyindex 77e511528..ebca085f8 100644--- a/wger/nutrition/extract_info/wger.py+++ b/wger/nutrition/extract_info/wger.py@@ -56,7 +56,7 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:license_id = product_data.get('license', ODBL_LICENSE_ID)license_title = product_data.get('license_title', '')license_object_url = product_data.get('license_object_url', '')- license_authors = product_data.get('authors', '')+ license_authors = product_data.get('license_author', '')license_author_url = product_data.get('license_author_url', '')license_derivative_source_url = product_data.get('license_derivative_source_url', '')diff --git a/wger/nutrition/management/commands/export-ingredients.py b/wger/nutrition/management/commands/export-ingredients.pyindex b94849dd5..1f1c95800 100644--- a/wger/nutrition/management/commands/export-ingredients.py+++ b/wger/nutrition/management/commands/export-ingredients.py@@ -23,8 +23,8 @@ class Command(BaseCommand):"""Export all ingredients as a gzipped JSONL file for bulk synchronization.- The generated file is placed in MEDIA_ROOT/ingredients/ and can be- served to other wger instances for fast bulk imports.+ The generated file is written using the configured storage backend and can+ be served to other wger instances for fast bulk imports."""help = 'Export all ingredients as a gzipped JSONL dump file for bulk synchronization'diff --git a/wger/nutrition/management/commands/sync-ingredients-bulk.py b/wger/nutrition/management/commands/sync-ingredients-bulk.pyindex 6a028d648..19e3b90b5 100644--- a/wger/nutrition/management/commands/sync-ingredients-bulk.py+++ b/wger/nutrition/management/commands/sync-ingredients-bulk.py@@ -16,7 +16,6 @@from django.core.management.base import CommandError# wger-from wger.core.api.min_server_version import check_min_server_versionfrom wger.core.management.wger_command import WgerCommandfrom wger.nutrition.consts import SyncModefrom wger.nutrition.sync import (@@ -69,8 +68,6 @@ class Command(WgerCommand):remote_url = self.remote_urlmode = SyncMode.INSERT if options['mode'] == 'insert' else SyncMode.UPDATE- check_min_server_version(remote_url)-try:file_path = download_ingredient_dump(self.stdout.write,diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex d9c862a85..38b941a99 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -481,14 +481,15 @@ def sync_ingredients_from_dump(if len(bulk_bucket) >= BULK_SIZE:try:Ingredient.objects.bulk_create(bulk_bucket)+ count += len(bulk_bucket)except Exception:# Fall back to individual savesfor ingredient in bulk_bucket:try:ingredient.save()+ count += 1except Exception:errors += 1- count += len(bulk_bucket)bulk_bucket = []else:try:@@ -503,13 +504,14 @@ def sync_ingredients_from_dump(if mode == SyncMode.INSERT and bulk_bucket:try:Ingredient.objects.bulk_create(bulk_bucket)+ count += len(bulk_bucket)except Exception:for ingredient in bulk_bucket:try:ingredient.save()+ count += 1except Exception:errors += 1- count += len(bulk_bucket)pbar.close()@@ -536,23 +538,19 @@ def download_ingredient_dump(print_fn(f'*** Downloading ingredient dump from {dump_url}...')if folder:- download_folder = folder+ file_path = Path(folder) / 'ingredients.jsonl.gz'+ if file_path.exists():+ print_fn(f'File already downloaded at {file_path}')+ return file_pathelse:- download_folder = tempfile.mkdtemp()-- file_path = Path(download_folder) / 'ingredients.jsonl.gz'-- if file_path.exists():- print_fn(f'File already downloaded at {file_path}')- return file_path+ file_path = Path(tempfile.NamedTemporaryFile(delete=False, suffix='.jsonl.gz').name)- response = requests.get(dump_url, stream=True, headers=wger_headers())+ response = requests.get(dump_url, stream=True, headers=wger_headers(), timeout=(10, 60))if response.status_code == 404:raise FileNotFoundError(f'Bulk ingredient dump not found at {dump_url}. 'f'The remote server may not have generated a dump yet. '- f'Please ask the server admin to run "python manage.py export-ingredients" first, '- f'or use "python manage.py sync-ingredients" to sync via the API instead.'+ f'Please ask the server admin to run "python manage.py export-ingredients" first.')if response.status_code != 200:raise Exception(f'Could not download dump from {dump_url} (status {response.status_code})')diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.pyindex 88af4fdfa..c1f851ef7 100644--- a/wger/nutrition/tasks.py+++ b/wger/nutrition/tasks.py@@ -145,7 +145,12 @@ def sync_all_ingredients_chunked_task(return None+ autoretry_for=(requests.exceptions.RequestException,),+ retry_backoff=True,+ retry_jitter=True,+ retry_kwargs={'max_retries': 3},+)def sync_ingredients_bulk_or_api_task():"""Sync ingredients from a remote wger instance.