Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Properly handle temp files and other small fixes

Roland Geider410b1d8e6e5a
diff --git a/wger/nutrition/extract_info/wger.py b/wger/nutrition/extract_info/wger.py
index 77e511528..ebca085f8 100644
--- a/wger/nutrition/extract_info/wger.py
+++ b/wger/nutrition/extract_info/wger.py
@@ -56,7 +56,7 @@ def extract_info_from_wger_api(product_data: dict) -> IngredientData:
license_id = product_data.get('license', ODBL_LICENSE_ID)
license_title = product_data.get('license_title', '')
license_object_url = product_data.get('license_object_url', '')
- license_authors = product_data.get('authors', '')
+ license_authors = product_data.get('license_author', '')
license_author_url = product_data.get('license_author_url', '')
license_derivative_source_url = product_data.get('license_derivative_source_url', '')
diff --git a/wger/nutrition/management/commands/export-ingredients.py b/wger/nutrition/management/commands/export-ingredients.py
index b94849dd5..1f1c95800 100644
--- a/wger/nutrition/management/commands/export-ingredients.py
+++ b/wger/nutrition/management/commands/export-ingredients.py
@@ -23,8 +23,8 @@ class Command(BaseCommand):
"""
Export all ingredients as a gzipped JSONL file for bulk synchronization.
- The generated file is placed in MEDIA_ROOT/ingredients/ and can be
- served to other wger instances for fast bulk imports.
+ The generated file is written using the configured storage backend and can
+ be served to other wger instances for fast bulk imports.
"""
help = 'Export all ingredients as a gzipped JSONL dump file for bulk synchronization'
diff --git a/wger/nutrition/management/commands/sync-ingredients-bulk.py b/wger/nutrition/management/commands/sync-ingredients-bulk.py
index 6a028d648..19e3b90b5 100644
--- a/wger/nutrition/management/commands/sync-ingredients-bulk.py
+++ b/wger/nutrition/management/commands/sync-ingredients-bulk.py
@@ -16,7 +16,6 @@
from django.core.management.base import CommandError
# wger
-from wger.core.api.min_server_version import check_min_server_version
from wger.core.management.wger_command import WgerCommand
from wger.nutrition.consts import SyncMode
from wger.nutrition.sync import (
@@ -69,8 +68,6 @@ class Command(WgerCommand):
remote_url = self.remote_url
mode = SyncMode.INSERT if options['mode'] == 'insert' else SyncMode.UPDATE
- check_min_server_version(remote_url)
-
try:
file_path = download_ingredient_dump(
self.stdout.write,
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py
index d9c862a85..38b941a99 100644
--- a/wger/nutrition/sync.py
+++ b/wger/nutrition/sync.py
@@ -481,14 +481,15 @@ def sync_ingredients_from_dump(
if len(bulk_bucket) >= BULK_SIZE:
try:
Ingredient.objects.bulk_create(bulk_bucket)
+ count += len(bulk_bucket)
except Exception:
# Fall back to individual saves
for ingredient in bulk_bucket:
try:
ingredient.save()
+ count += 1
except Exception:
errors += 1
- count += len(bulk_bucket)
bulk_bucket = []
else:
try:
@@ -503,13 +504,14 @@ def sync_ingredients_from_dump(
if mode == SyncMode.INSERT and bulk_bucket:
try:
Ingredient.objects.bulk_create(bulk_bucket)
+ count += len(bulk_bucket)
except Exception:
for ingredient in bulk_bucket:
try:
ingredient.save()
+ count += 1
except Exception:
errors += 1
- count += len(bulk_bucket)
pbar.close()
@@ -536,23 +538,19 @@ def download_ingredient_dump(
print_fn(f'*** Downloading ingredient dump from {dump_url}...')
if folder:
- download_folder = folder
+ file_path = Path(folder) / 'ingredients.jsonl.gz'
+ if file_path.exists():
+ print_fn(f'File already downloaded at {file_path}')
+ return file_path
else:
- download_folder = tempfile.mkdtemp()
-
- file_path = Path(download_folder) / 'ingredients.jsonl.gz'
-
- if file_path.exists():
- print_fn(f'File already downloaded at {file_path}')
- return file_path
+ file_path = Path(tempfile.NamedTemporaryFile(delete=False, suffix='.jsonl.gz').name)
- response = requests.get(dump_url, stream=True, headers=wger_headers())
+ response = requests.get(dump_url, stream=True, headers=wger_headers(), timeout=(10, 60))
if response.status_code == 404:
raise FileNotFoundError(
f'Bulk ingredient dump not found at {dump_url}. '
f'The remote server may not have generated a dump yet. '
- f'Please ask the server admin to run "python manage.py export-ingredients" first, '
- f'or use "python manage.py sync-ingredients" to sync via the API instead.'
+ f'Please ask the server admin to run "python manage.py export-ingredients" first.'
)
if response.status_code != 200:
raise Exception(f'Could not download dump from {dump_url} (status {response.status_code})')
diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.py
index 88af4fdfa..c1f851ef7 100644
--- a/wger/nutrition/tasks.py
+++ b/wger/nutrition/tasks.py
@@ -145,7 +145,12 @@ def sync_all_ingredients_chunked_task(
return None
+ autoretry_for=(requests.exceptions.RequestException,),
+ retry_backoff=True,
+ retry_jitter=True,
+ retry_kwargs={'max_retries': 3},
+)
def sync_ingredients_bulk_or_api_task():
"""
Sync ingredients from a remote wger instance.