Self hosted FLOSS fitness/workout, nutrition and weight tracker
Move actual sync logic to celery tasks
Roland Geidera9e204c61496
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c0b3303d5..83cf502f8 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -56,4 +56,5 @@ jobs:with:github-token: ${{ secrets.GITHUB_TOKEN }}path-to-lcov: coverage.lcov- if: matrix.python-version == 3.10+ if: matrix.python-version == 3.11+diff --git a/wger/exercises/management/commands/download-exercise-images.py b/wger/exercises/management/commands/download-exercise-images.pyindex dc44e1309..c993d8ad5 100644--- a/wger/exercises/management/commands/download-exercise-images.py+++ b/wger/exercises/management/commands/download-exercise-images.py@@ -14,35 +14,20 @@## You should have received a copy of the GNU Affero General Public License-# Standard Library-import os-# Djangofrom django.conf import settingsfrom django.core.exceptions import (ImproperlyConfigured,ValidationError,)-from django.core.files import File-from django.core.files.temp import NamedTemporaryFilefrom django.core.management.base import (BaseCommand,CommandError,)from django.core.validators import URLValidator-# Third Party-import requests-# wger-from wger.exercises.models import (- ExerciseBase,- ExerciseImage,-)-from wger.utils.requests import wger_headers---IMAGE_API = "{0}/api/v2/exerciseimage/"+from wger.exercises.sync import download_exercise_imagesclass Command(BaseCommand):@@ -84,62 +69,4 @@ class Command(BaseCommand):except ValidationError:raise CommandError('Please enter a valid URL')- headers = wger_headers()-- # Get all images- page = 1- all_images_processed = False- result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()- self.stdout.write('*** Processing images ***')- while not all_images_processed:- self.stdout.write('')- self.stdout.write(f'*** Page {page}')- self.stdout.write('')-- for image_data in result['results']:- image_uuid = image_data['uuid']-- self.stdout.write(f'Processing image {image_uuid}')-- try:- exercise_base = ExerciseBase.objects.get(id=image_data['exercise_base'])- except ExerciseBase.DoesNotExist:- self.stdout.write(' Remote exercise base not found in local DB, skipping...')- continue-- try:- ExerciseImage.objects.get(uuid=image_uuid)- self.stdout.write(' Image already present locally, skipping...')- continue- except ExerciseImage.DoesNotExist:- self.stdout.write(' Image not found in local DB, creating now...')- retrieved_image = requests.get(image_data['image'], headers=headers)- image = ExerciseImage.from_json(- exercise_base,- retrieved_image,- image_data,- headers,- )-- # Temporary files on windows don't support the delete attribute- if os.name == 'nt':- img_temp = NamedTemporaryFile()- else:- img_temp = NamedTemporaryFile(delete=True)- img_temp.write(retrieved_image.content)- img_temp.flush()-- image.exercise_base = exercise_base- image.is_main = image_data['is_main']- image.image.save(- os.path.basename(os.path.basename(image_data['image'])),- File(img_temp),- )- image.save()- self.stdout.write(self.style.SUCCESS(' successfully saved'))-- if result['next']:- page += 1- result = requests.get(result['next'], headers=headers).json()- else:- all_images_processed = True+ download_exercise_images(self.stdout.write, remote_url, self.style.SUCCESS)diff --git a/wger/exercises/management/commands/download-exercise-videos.py b/wger/exercises/management/commands/download-exercise-videos.pyindex 4f3d0dc24..76b007cfe 100644--- a/wger/exercises/management/commands/download-exercise-videos.py+++ b/wger/exercises/management/commands/download-exercise-videos.py@@ -14,35 +14,20 @@## You should have received a copy of the GNU Affero General Public License-# Standard Library-import os-# Djangofrom django.conf import settingsfrom django.core.exceptions import (ImproperlyConfigured,ValidationError,)-from django.core.files import File-from django.core.files.temp import NamedTemporaryFilefrom django.core.management.base import (BaseCommand,CommandError,)from django.core.validators import URLValidator-# Third Party-import requests-# wger-from wger.exercises.models import (- ExerciseBase,- ExerciseVideo,-)-from wger.utils.requests import wger_headers---VIDEO_API = "{0}/api/v2/video/"+from wger.exercises.sync import download_exercise_videosclass Command(BaseCommand):@@ -77,68 +62,4 @@ class Command(BaseCommand):except ValidationError:raise CommandError('Please enter a valid URL')- headers = wger_headers()-- # Get all videos- page = 1- all_videos_processed = False- result = requests.get(VIDEO_API.format(remote_url), headers=headers).json()- self.stdout.write('*** Processing videos ***')- while not all_videos_processed:- self.stdout.write('')- self.stdout.write(f'*** Page {page}')- self.stdout.write('')-- for video_data in result['results']:- video_uuid = video_data['uuid']- self.stdout.write(f'Processing video {video_uuid}')-- try:- exercise_base = ExerciseBase.objects.get(uuid=video_data['exercise_base_uuid'])- except ExerciseBase.DoesNotExist:- self.stdout.write(' Remote exercise base not found in local DB, skipping...')- continue-- try:- ExerciseVideo.objects.get(uuid=video_uuid)- self.stdout.write(' Video already present locally, skipping...')- continue- except ExerciseVideo.DoesNotExist:- self.stdout.write(' Video not found in local DB, creating now...')- video = ExerciseVideo()- video.exercise_base = exercise_base- video.uuid = video_uuid- video.is_main = video_data['is_main']- video.license_id = video_data['license']- video.license_author = video_data['license_author']- video.size = video_data['size']- video.width = video_data['width']- video.height = video_data['height']- video.codec = video_data['codec']- video.codec_long = video_data['codec_long']- video.duration = video_data['duration']-- # Save the downloaded video- # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-- retrieved_video = requests.get(video_data['video'], headers=headers)-- # Temporary files on Windows don't support the delete attribute- if os.name == 'nt':- img_temp = NamedTemporaryFile()- else:- img_temp = NamedTemporaryFile(delete=True)- img_temp.write(retrieved_video.content)- img_temp.flush()-- video.video.save(- os.path.basename(os.path.basename(video_data['video'])),- File(img_temp),- )- video.save()- self.stdout.write(self.style.SUCCESS(' saved successfully'))-- if result['next']:- page += 1- result = requests.get(result['next'], headers=headers).json()- else:- all_videos_processed = True+ download_exercise_videos(self.stdout.write, remote_url)diff --git a/wger/exercises/models/image.py b/wger/exercises/models/image.pyindex 62b4ec1c2..bedadcb03 100644--- a/wger/exercises/models/image.py+++ b/wger/exercises/models/image.py@@ -165,15 +165,16 @@ class ExerciseImage(AbstractLicenseModel, AbstractHistoryMixin, models.Model, Baconnect_to: ExerciseBase,retrieved_image,json_data: dict,- headers,- generate_uuid: bool = False+ generate_uuid: bool = False,):image: cls = super().from_json(- connect_to, retrieved_image, json_data, headers, generate_uuid+ connect_to,+ retrieved_image,+ json_data,+ generate_uuid,)image.exercise_base = connect_toimage.is_main = json_data['is_main']- image.status = json_data['status']image.save_image(retrieved_image, json_data)diff --git a/wger/exercises/sync.py b/wger/exercises/sync.pyindex 7eaa391a0..f79403c4d 100644--- a/wger/exercises/sync.py+++ b/wger/exercises/sync.py@@ -11,9 +11,13 @@# GNU General Public License for more details.## You should have received a copy of the GNU Affero General Public License+# Standard Library+import os# Djangofrom django.conf import settings+from django.core.files import File+from django.core.files.temp import NamedTemporaryFile# Third Partyimport requests@@ -39,6 +43,8 @@ CATEGORY_API = "{0}/api/v2/exercisecategory/"MUSCLE_API = "{0}/api/v2/muscle/"LANGUAGE_API = "{0}/api/v2/language/"EQUIPMENT_API = "{0}/api/v2/equipment/"+IMAGE_API = "{0}/api/v2/exerciseimage/"+VIDEO_API = "{0}/api/v2/video/"def sync_exercises(@@ -279,3 +285,141 @@ def delete_entries(else:all_entries_processed = Trueprint_fn(style_fn('done!\n'))+++def download_exercise_images(+ print_fn,+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],+ style_fn=lambda x: x,+):+ headers = wger_headers()++ # Get all images+ page = 1+ all_images_processed = False+ result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()+ print_fn('*** Processing images ***')+ while not all_images_processed:+ print_fn('')+ print_fn(f'*** Page {page}')+ print_fn('')++ for image_data in result['results']:+ image_uuid = image_data['uuid']++ print_fn(f'Processing image {image_uuid}')++ try:+ exercise_base = ExerciseBase.objects.get(id=image_data['exercise_base'])+ except ExerciseBase.DoesNotExist:+ print_fn(' Remote exercise base not found in local DB, skipping...')+ continue++ try:+ ExerciseImage.objects.get(uuid=image_uuid)+ print_fn(' Image already present locally, skipping...')+ continue+ except ExerciseImage.DoesNotExist:+ print_fn(' Image not found in local DB, creating now...')+ retrieved_image = requests.get(image_data['image'], headers=headers)+ image = ExerciseImage.from_json(+ exercise_base,+ retrieved_image,+ image_data,+ headers,+ )++ # Temporary files on Windows don't support the delete attribute+ if os.name == 'nt':+ img_temp = NamedTemporaryFile()+ else:+ img_temp = NamedTemporaryFile(delete=True)+ img_temp.write(retrieved_image.content)+ img_temp.flush()++ image.exercise_base = exercise_base+ image.is_main = image_data['is_main']+ image.image.save(+ os.path.basename(os.path.basename(image_data['image'])),+ File(img_temp),+ )+ image.save()+ print_fn(style_fn(' successfully saved'))++ if result['next']:+ page += 1+ result = requests.get(result['next'], headers=headers).json()+ else:+ all_images_processed = True+++def download_exercise_videos(+ print_fn,+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],+ style_fn=lambda x: x,+):+ headers = wger_headers()++ # Get all videos+ page = 1+ all_videos_processed = False+ result = requests.get(VIDEO_API.format(remote_url), headers=headers).json()+ print_fn('*** Processing videos ***')+ while not all_videos_processed:+ print_fn('')+ print_fn(f'*** Page {page}')+ print_fn('')++ for video_data in result['results']:+ video_uuid = video_data['uuid']+ print_fn(f'Processing video {video_uuid}')++ try:+ exercise_base = ExerciseBase.objects.get(uuid=video_data['exercise_base_uuid'])+ except ExerciseBase.DoesNotExist:+ print_fn(' Remote exercise base not found in local DB, skipping...')+ continue++ try:+ ExerciseVideo.objects.get(uuid=video_uuid)+ print_fn(' Video already present locally, skipping...')+ continue+ except ExerciseVideo.DoesNotExist:+ print_fn(' Video not found in local DB, creating now...')+ video = ExerciseVideo()+ video.exercise_base = exercise_base+ video.uuid = video_uuid+ video.is_main = video_data['is_main']+ video.license_id = video_data['license']+ video.license_author = video_data['license_author']+ video.size = video_data['size']+ video.width = video_data['width']+ video.height = video_data['height']+ video.codec = video_data['codec']+ video.codec_long = video_data['codec_long']+ video.duration = video_data['duration']++ # Save the downloaded video+ # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-+ retrieved_video = requests.get(video_data['video'], headers=headers)++ # Temporary files on Windows don't support the delete attribute+ if os.name == 'nt':+ img_temp = NamedTemporaryFile()+ else:+ img_temp = NamedTemporaryFile(delete=True)+ img_temp.write(retrieved_video.content)+ img_temp.flush()++ video.video.save(+ os.path.basename(os.path.basename(video_data['video'])),+ File(img_temp),+ )+ video.save()+ print_fn(style_fn(' saved successfully'))++ if result['next']:+ page += 1+ result = requests.get(result['next'], headers=headers).json()+ else:+ all_videos_processed = Truediff --git a/wger/exercises/tasks.py b/wger/exercises/tasks.pyindex be42becc1..ad183612a 100644--- a/wger/exercises/tasks.py+++ b/wger/exercises/tasks.py@@ -26,6 +26,8 @@ from celery.schedules import crontabfrom wger.celery_configuration import appfrom wger.exercises.sync import (delete_entries,+ download_exercise_images,+ download_exercise_videos,sync_categories,sync_equipment,sync_exercises,@@ -33,6 +35,7 @@ from wger.exercises.sync import (sync_muscles,)+logger = logging.getLogger(__name__)@@ -55,7 +58,7 @@ def sync_images_task():"""Fetches the exercise images from the default wger instance"""- pass+ download_exercise_images(logger.info)@app.task@@ -63,7 +66,7 @@ def sync_videos_task():"""Fetches the exercise videos from the default wger instance"""- pass+ download_exercise_videos(logger.info)@app.on_after_finalize.connectdiff --git a/wger/nutrition/management/commands/download-ingredient-images.py b/wger/nutrition/management/commands/download-ingredient-images.pyindex 3b4aebc04..6e647a020 100644--- a/wger/nutrition/management/commands/download-ingredient-images.py+++ b/wger/nutrition/management/commands/download-ingredient-images.py@@ -26,19 +26,8 @@ from django.core.management.base import ()from django.core.validators import URLValidator-# Third Party-import requests-# wger-from wger import get_version-from wger.nutrition.models import (- Image,- Ingredient,-)-from wger.utils.requests import wger_headers---IMAGE_API = "{0}/api/v2/ingredient-image/"+from wger.nutrition.sync import download_ingredient_imagesclass Command(BaseCommand):@@ -80,42 +69,4 @@ class Command(BaseCommand):except ValidationError:raise CommandError('Please enter a valid URL')- headers = wger_headers()-- # Get all images- page = 1- all_images_processed = False- result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()- self.stdout.write('*** Processing images ***')- while not all_images_processed:- self.stdout.write('')- self.stdout.write(f'*** Page {page}')- self.stdout.write('')-- for image_data in result['results']:- image_uuid = image_data['uuid']-- self.stdout.write(f'Processing image {image_uuid}')-- try:- ingredient = Ingredient.objects.get(uuid=image_data['ingredient_uuid'])- except Ingredient.DoesNotExist:- self.stdout.write(' Remote ingredient not found in local DB, skipping...')- continue-- try:- Image.objects.get(uuid=image_uuid)- self.stdout.write(' Image already present locally, skipping...')- continue- except Image.DoesNotExist:- self.stdout.write(' Image not found in local DB, creating now...')- retrieved_image = requests.get(image_data['image'], headers=headers)- Image.from_json(ingredient, retrieved_image, image_data)-- self.stdout.write(self.style.SUCCESS(' successfully saved'))-- if result['next']:- page += 1- result = requests.get(result['next'], headers=headers).json()- else:- all_images_processed = True+ download_ingredient_images(self.stdout.write, remote_url, self.style.SUCCESS)diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex 7d63e7be6..139a6b3e6 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -27,6 +27,7 @@ import requests# wgerfrom wger.nutrition.models import (Image,+ Ingredient,Source,)from wger.utils.constants import (@@ -38,6 +39,8 @@ from wger.utils.requests import wger_headerslogger = logging.getLogger(__name__)+IMAGE_API = "{0}/api/v2/ingredient-image/"+def fetch_ingredient_image(pk: int):# wger@@ -123,3 +126,48 @@ def fetch_image_from_off(ingredient):logger.info('Ingredient has already an image, skipping...')returnlogger.info('Image successfully saved')+++def download_ingredient_images(+ print_fn,+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],+ style_fn=lambda x: x,+):+ headers = wger_headers()+ # Get all images+ page = 1+ all_images_processed = False+ result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()+ print_fn('*** Processing images ***')+ while not all_images_processed:+ print_fn('')+ print_fn(f'*** Page {page}')+ print_fn('')++ for image_data in result['results']:+ image_uuid = image_data['uuid']++ print_fn(f'Processing image {image_uuid}')++ try:+ ingredient = Ingredient.objects.get(uuid=image_data['ingredient_uuid'])+ except Ingredient.DoesNotExist:+ print_fn(' Remote ingredient not found in local DB, skipping...')+ continue++ try:+ Image.objects.get(uuid=image_uuid)+ print_fn(' Image already present locally, skipping...')+ continue+ except Image.DoesNotExist:+ print_fn(' Image not found in local DB, creating now...')+ retrieved_image = requests.get(image_data['image'], headers=headers)+ Image.from_json(ingredient, retrieved_image, image_data)++ print_fn(style_fn(' successfully saved'))++ if result['next']:+ page += 1+ result = requests.get(result['next'], headers=headers).json()+ else:+ all_images_processed = Truediff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.pyindex 618a9051b..40352ee3c 100644--- a/wger/nutrition/tasks.py+++ b/wger/nutrition/tasks.py@@ -12,9 +12,18 @@## You should have received a copy of the GNU Affero General Public License+# Standard Library+import logging+# wgerfrom wger.celery_configuration import app-from wger.nutrition.sync import fetch_ingredient_image+from wger.nutrition.sync import (+ download_ingredient_images,+ fetch_ingredient_image,+)+++logger = logging.getLogger(__name__)@app.task@@ -25,3 +34,13 @@ def fetch_ingredient_image_task(pk: int):Returns the image if it is already present in the DB"""fetch_ingredient_image(pk)+++def fetch_all_ingredient_images_task():+ """+ Fetches the ingredient image from Open Food Facts servers if it is not available locally++ Returns the image if it is already present in the DB+ """+ download_ingredient_images(logger.info)diff --git a/wger/settings_global.py b/wger/settings_global.pyindex b317049d6..9ca05bb40 100644--- a/wger/settings_global.py+++ b/wger/settings_global.py@@ -24,6 +24,7 @@ from datetime import timedelta# wgerfrom wger.utils.constants import DOWNLOAD_INGREDIENT_WGER+"""This file contains the global settings that don't usually need to be changed.For a full list of options, visit:@@ -169,7 +170,7 @@ TEMPLATES = ['django.template.loaders.app_directories.Loader',],'debug':- False+ False},},]@@ -239,7 +240,7 @@ LANGUAGES = (LANGUAGE_CODE = 'en'# All translation files are in one place-LOCALE_PATHS = (os.path.join(SITE_ROOT, 'locale'),)+LOCALE_PATHS = (os.path.join(SITE_ROOT, 'locale'), )# Primary keys are AutoFieldsDEFAULT_AUTO_FIELD = 'django.db.models.AutoField'@@ -411,15 +412,11 @@ COMPRESS_ROOT = STATIC_ROOT# Django Rest Framework#REST_FRAMEWORK = {- 'DEFAULT_PERMISSION_CLASSES': ('wger.utils.permissions.WgerPermission',),- 'DEFAULT_PAGINATION_CLASS':- 'rest_framework.pagination.LimitOffsetPagination',- 'PAGE_SIZE':- 20,- 'PAGINATE_BY_PARAM':- 'limit', # Allow client to override, using `?limit=xxx`.- 'TEST_REQUEST_DEFAULT_FORMAT':- 'json',+ 'DEFAULT_PERMISSION_CLASSES': ('wger.utils.permissions.WgerPermission', ),+ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',+ 'PAGE_SIZE': 20,+ 'PAGINATE_BY_PARAM': 'limit', # Allow client to override, using `?limit=xxx`.+ 'TEST_REQUEST_DEFAULT_FORMAT': 'json','DEFAULT_AUTHENTICATION_CLASSES': ('rest_framework.authentication.SessionAuthentication','rest_framework.authentication.TokenAuthentication',@@ -431,7 +428,7 @@ REST_FRAMEWORK = {),'DEFAULT_THROTTLE_CLASSES': ['rest_framework.throttling.ScopedRateThrottle'],'DEFAULT_THROTTLE_RATES': {- 'login': '3/min'+ 'login': '10/min'}}@@ -455,7 +452,7 @@ CORS_URLS_REGEX = r'^/api/.*$'## Ignore these URLs if they cause 404#-IGNORABLE_404_URLS = (re.compile(r'^/favicon\.ico$'),)+IGNORABLE_404_URLS = (re.compile(r'^/favicon\.ico$'), )## Password rules