Self hosted FLOSS fitness/workout, nutrition and weight tracker
Add command to sync all the ingredients from a wger instance
Roland Geider97d1e1d00c17
diff --git a/extras/docker/development/settings.py b/extras/docker/development/settings.pyindex 19dfc3276..08a17ca7d 100644--- a/extras/docker/development/settings.py+++ b/extras/docker/development/settings.py@@ -98,6 +98,7 @@ WGER_SETTINGS["MIN_ACCOUNT_AGE_TO_TRUST"] = env.int("MIN_ACCOUNT_AGE_TO_TRUST",WGER_SETTINGS["SYNC_EXERCISES_CELERY"] = env.bool("SYNC_EXERCISES_CELERY", False)WGER_SETTINGS["SYNC_EXERCISE_IMAGES_CELERY"] = env.bool("SYNC_EXERCISE_IMAGES_CELERY", False)WGER_SETTINGS["SYNC_EXERCISE_VIDEOS_CELERY"] = env.bool("SYNC_EXERCISE_VIDEOS_CELERY", False)+WGER_SETTINGS["SYNC_INGREDIENTS_CELERY"] = env.bool("SYNC_INGREDIENTS_CELERY", False)WGER_SETTINGS["USE_CELERY"] = env.bool("USE_CELERY", False)# Cachediff --git a/wger/exercises/sync.py b/wger/exercises/sync.pyindex 15372cc15..6544f0d20 100644--- a/wger/exercises/sync.py+++ b/wger/exercises/sync.py@@ -73,9 +73,7 @@ def sync_exercises(print_fn('*** Synchronizing exercises...')url = make_uri(EXERCISE_ENDPOINT, server_url=remote_url, query={'limit': 100})- result = get_paginated(url, headers=wger_headers())-- for data in result:+ for data in get_paginated(url, headers=wger_headers()):uuid = data['uuid']created = data['created']@@ -166,8 +164,8 @@ def sync_languages(print_fn('*** Synchronizing languages...')headers = wger_headers()url = make_uri(LANGUAGE_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=headers)- for data in result:++ for data in get_paginated(url, headers=headers):short_name = data['short_name']full_name = data['full_name']@@ -190,8 +188,8 @@ def sync_licenses("""Synchronize the licenses from the remote server"""print_fn('*** Synchronizing licenses...')url = make_uri(LICENSE_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=wger_headers())- for data in result:++ for data in get_paginated(url, headers=wger_headers()):short_name = data['short_name']full_name = data['full_name']license_url = data['url']@@ -219,8 +217,8 @@ def sync_categories(print_fn('*** Synchronizing categories...')url = make_uri(CATEGORY_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=wger_headers())- for data in result:++ for data in get_paginated(url, headers=wger_headers()):category_id = data['id']category_name = data['name']@@ -244,9 +242,8 @@ def sync_muscles(print_fn('*** Synchronizing muscles...')url = make_uri(MUSCLE_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=wger_headers())- for data in result:+ for data in get_paginated(url, headers=wger_headers()):muscle_id = data['id']muscle_name = data['name']muscle_is_front = data['is_front']@@ -280,9 +277,8 @@ def sync_equipment(print_fn('*** Synchronizing equipment...')url = make_uri(EQUIPMENT_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=wger_headers())- for data in result:+ for data in get_paginated(url, headers=wger_headers()):equipment_id = data['id']equipment_name = data['name']@@ -303,7 +299,6 @@ def handle_deleted_entries(style_fn=lambda x: x,):if not print_fn:-def print_fn(_):return None@@ -311,9 +306,8 @@ def handle_deleted_entries(print_fn('*** Deleting exercise data that was removed on the server...')url = make_uri(DELETION_LOG_ENDPOINT, server_url=remote_url, query={'limit': 100})- result = get_paginated(url, headers=wger_headers())- for data in result:+ for data in get_paginated(url, headers=wger_headers()):uuid = data['uuid']replaced_by_uuid = data['replaced_by']model_type = data['model_type']@@ -382,7 +376,6 @@ def download_exercise_images():headers = wger_headers()url = make_uri(IMAGE_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=headers)print_fn('*** Processing images ***')@@ -396,7 +389,7 @@ def download_exercise_images(if deleted:print_fn(f'Deleted {deleted} images without associated image files')- for image_data in result:+ for image_data in get_paginated(url, headers=headers):image_uuid = image_data['uuid']print_fn(f'Processing image {image_uuid}')@@ -426,11 +419,10 @@ def download_exercise_videos():headers = wger_headers()url = make_uri(VIDEO_ENDPOINT, server_url=remote_url)- result = get_paginated(url, headers=headers)print_fn('*** Processing videos ***')- for video_data in result:+ for video_data in get_paginated(url, headers=headers):video_uuid = video_data['uuid']print_fn(f'Processing video {video_uuid}')diff --git a/wger/nutrition/api/endpoints.py b/wger/nutrition/api/endpoints.pyindex 6f225e199..4e0fe8e49 100644--- a/wger/nutrition/api/endpoints.py+++ b/wger/nutrition/api/endpoints.py@@ -1 +1,2 @@IMAGE_ENDPOINT = "ingredient-image"+INGREDIENTS_ENDPOINT = "ingredient"diff --git a/wger/nutrition/management/commands/sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients.pynew file mode 100644index 000000000..0965c222b--- /dev/null+++ b/wger/nutrition/management/commands/sync-ingredients.py@@ -0,0 +1,65 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Django+from django.conf import settings+from django.core.exceptions import ValidationError+from django.core.management.base import (+ BaseCommand,+ CommandError,+)+from django.core.validators import URLValidator++# wger+from wger.exercises.sync import (+ handle_deleted_entries,+ sync_categories,+ sync_equipment,+ sync_exercises,+ sync_languages,+ sync_licenses,+ sync_muscles,+)+from wger.nutrition.sync import sync_ingredients+++class Command(BaseCommand):+ """+ Synchronizes ingredient data from a wger instance to the local database+ """+ remote_url = settings.WGER_SETTINGS['WGER_INSTANCE']++ help = """Synchronizes ingredient data from a wger instance to the local database"""++ def add_arguments(self, parser):+ parser.add_argument(+ '--remote-url',+ action='store',+ dest='remote_url',+ default=settings.WGER_SETTINGS['WGER_INSTANCE'],+ help=f'Remote URL to fetch the ingredients from (default: WGER_SETTINGS'+ f'["WGER_INSTANCE"] - {settings.WGER_SETTINGS["WGER_INSTANCE"]})'+ )++ def handle(self, **options):+ remote_url = options['remote_url']++ try:+ val = URLValidator()+ val(remote_url)+ self.remote_url = remote_url+ except ValidationError:+ raise CommandError('Please enter a valid URL')++ sync_ingredients(self.stdout.write, self.remote_url, self.style.SUCCESS)diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex ffc0b2599..a64b69c81 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -25,7 +25,7 @@ from django.db import IntegrityErrorimport requests# wger-from wger.nutrition.api.endpoints import IMAGE_ENDPOINT+from wger.nutrition.api.endpoints import IMAGE_ENDPOINT, INGREDIENTS_ENDPOINTfrom wger.nutrition.models import (Image,Ingredient,@@ -35,6 +35,7 @@ from wger.utils.constants import (CC_BY_SA_3_LICENSE_ID,DOWNLOAD_INGREDIENT_OFF,DOWNLOAD_INGREDIENT_WGER,+ API_MAX_ITEMS,)from wger.utils.requests import (get_paginated_generator,@@ -42,7 +43,6 @@ from wger.utils.requests import ()from wger.utils.url import make_uri-logger = logging.getLogger(__name__)@@ -162,30 +162,71 @@ def download_ingredient_images(headers = wger_headers()url = make_uri(IMAGE_ENDPOINT, server_url=remote_url, query={'limit': 100})- print_fn('*** Processing images ***')- for result in get_paginated_generator(url, headers=headers):+ print_fn('*** Processing ingredient images ***')+ for image_data in get_paginated_generator(url, headers=headers):+ image_uuid = image_data['uuid']+ print_fn(f'Processing image {image_uuid}')- for image_data in result:- image_uuid = image_data['uuid']+ try:+ ingredient = Ingredient.objects.get(uuid=image_data['ingredient_uuid'])+ except Ingredient.DoesNotExist:+ print_fn(' Remote ingredient not found in local DB, skipping...')+ continue- print_fn(f'Processing image {image_uuid}')+ if hasattr(ingredient, 'image'):+ continue- try:- ingredient = Ingredient.objects.get(uuid=image_data['ingredient_uuid'])- except Ingredient.DoesNotExist:- print_fn(' Remote ingredient not found in local DB, skipping...')- continue+ try:+ Image.objects.get(uuid=image_uuid)+ print_fn(' Image already present locally, skipping...')+ continue+ except Image.DoesNotExist:+ print_fn(' Image not found in local DB, creating now...')+ retrieved_image = requests.get(image_data['image'], headers=headers)+ Image.from_json(ingredient, retrieved_image, image_data)- if hasattr(ingredient, 'image'):- continue+ print_fn(style_fn(' successfully saved'))- try:- Image.objects.get(uuid=image_uuid)- print_fn(' Image already present locally, skipping...')- continue- except Image.DoesNotExist:- print_fn(' Image not found in local DB, creating now...')- retrieved_image = requests.get(image_data['image'], headers=headers)- Image.from_json(ingredient, retrieved_image, image_data)- print_fn(style_fn(' successfully saved'))+def sync_ingredients(+ print_fn,+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],+ style_fn=lambda x: x,+):+ """Synchronize the ingredients from the remote server"""+ print_fn('*** Synchronizing ingredients...')++ url = make_uri(INGREDIENTS_ENDPOINT, server_url=remote_url, query={'limit': API_MAX_ITEMS})+ for data in get_paginated_generator(url, headers=wger_headers()):+ uuid = data['uuid']+ name = data['name']++ ingredient, created = Ingredient.objects.update_or_create(+ uuid=uuid,+ defaults={+ 'name': name,+ 'code': data['code'],+ 'language_id': data['language'],+ 'created': data['created'],+ 'license_id': data['license'],++ 'license_object_url': data['license_object_url'],+ 'license_author': data['license_author_url'],+ 'license_author_url': data['license_author_url'],+ 'license_title': data['license_title'],+ 'license_derivative_source_url': data['license_derivative_source_url'],++ 'energy': data['energy'],+ 'carbohydrates': data['carbohydrates'],+ 'carbohydrates_sugar': data['carbohydrates_sugar'],+ 'fat': data['fat'],+ 'fat_saturated': data['fat_saturated'],+ 'protein': data['protein'],+ 'fibres': data['fibres'],+ 'sodium': data['sodium'],+ },+ )++ print_fn(f"{'created' if created else 'updated'} ingredient {uuid} - {name}")++ print_fn(style_fn('done!\n'))diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.pyindex 40352ee3c..8e3b569c0 100644--- a/wger/nutrition/tasks.py+++ b/wger/nutrition/tasks.py@@ -14,15 +14,22 @@# Standard Libraryimport logging+import random++# Django+from django.conf import settings++# Third Party+from celery.schedules import crontab# wgerfrom wger.celery_configuration import appfrom wger.nutrition.sync import (download_ingredient_images,fetch_ingredient_image,+ sync_ingredients,)-logger = logging.getLogger(__name__)@@ -44,3 +51,25 @@ def fetch_all_ingredient_images_task():Returns the image if it is already present in the DB"""download_ingredient_images(logger.info)+++def sync_all_ingredients_task():+ """+ Fetches the current ingredients from the default wger instance+ """+ sync_ingredients(logger.info)++[email protected]_after_finalize.connect+def setup_periodic_tasks(sender, **kwargs):+ if settings.WGER_SETTINGS['SYNC_INGREDIENTS_CELERY']:+ sender.add_periodic_task(+ crontab(+ hour=random.randint(0, 23),+ minute=random.randint(0, 59),+ day_of_week=random.randint(0, 6),+ ),+ sync_all_ingredients_task.s(),+ name='Sync exercises',+ )diff --git a/wger/settings_global.py b/wger/settings_global.pyindex 2b2374128..2cf80e560 100644--- a/wger/settings_global.py+++ b/wger/settings_global.py@@ -536,6 +536,7 @@ WGER_SETTINGS = {'SYNC_EXERCISES_CELERY': False,'SYNC_EXERCISE_IMAGES_CELERY': False,'SYNC_EXERCISE_VIDEOS_CELERY': False,+ 'SYNC_INGREDIENTS_CELERY': False,'TWITTER': False,'MASTODON': 'https://fosstodon.org/@wger','USE_CELERY': False,diff --git a/wger/utils/constants.py b/wger/utils/constants.pyindex 58d4fa47b..ce9242860 100644--- a/wger/utils/constants.py+++ b/wger/utils/constants.py@@ -59,8 +59,16 @@ ENGLISH_SHORT_NAME = 'en'# Possible values for ingredient image downloadDOWNLOAD_INGREDIENT_WGER = 'WGER'DOWNLOAD_INGREDIENT_OFF = 'OFF'-DOWNLOAD_INGREDIENT_OPTIONS = (DOWNLOAD_INGREDIENT_WGER, DOWNLOAD_INGREDIENT_OFF, None)+DOWNLOAD_INGREDIENT_NONE = 'None'+DOWNLOAD_INGREDIENT_OPTIONS = (+ DOWNLOAD_INGREDIENT_WGER,+ DOWNLOAD_INGREDIENT_OFF,+ DOWNLOAD_INGREDIENT_NONE,+)# OFF ApiOFF_SEARCH_PRODUCT_FOUND = 1OFF_SEARCH_PRODUCT_NOT_FOUND = 0++# API+API_MAX_ITEMS = 999diff --git a/wger/utils/requests.py b/wger/utils/requests.pyindex abcf11834..2a41d18f9 100644--- a/wger/utils/requests.py+++ b/wger/utils/requests.py@@ -62,7 +62,9 @@ def get_paginated_generator(url: str, headers=None):while True:response = requests.get(url, headers=headers).json()- yield response['results']++ for result in response['results']:+ yield resulturl = response['next']if not url: