Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Move actual sync logic to celery tasks

Roland Geidera9e204c61496
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c0b3303d5..83cf502f8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,4 +56,5 @@ jobs:
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
path-to-lcov: coverage.lcov
- if: matrix.python-version == 3.10
+ if: matrix.python-version == 3.11
+
diff --git a/wger/exercises/management/commands/download-exercise-images.py b/wger/exercises/management/commands/download-exercise-images.py
index dc44e1309..c993d8ad5 100644
--- a/wger/exercises/management/commands/download-exercise-images.py
+++ b/wger/exercises/management/commands/download-exercise-images.py
@@ -14,35 +14,20 @@
#
# You should have received a copy of the GNU Affero General Public License
-# Standard Library
-import os
-
# Django
from django.conf import settings
from django.core.exceptions import (
ImproperlyConfigured,
ValidationError,
)
-from django.core.files import File
-from django.core.files.temp import NamedTemporaryFile
from django.core.management.base import (
BaseCommand,
CommandError,
)
from django.core.validators import URLValidator
-# Third Party
-import requests
-
# wger
-from wger.exercises.models import (
- ExerciseBase,
- ExerciseImage,
-)
-from wger.utils.requests import wger_headers
-
-
-IMAGE_API = "{0}/api/v2/exerciseimage/"
+from wger.exercises.sync import download_exercise_images
class Command(BaseCommand):
@@ -84,62 +69,4 @@ class Command(BaseCommand):
except ValidationError:
raise CommandError('Please enter a valid URL')
- headers = wger_headers()
-
- # Get all images
- page = 1
- all_images_processed = False
- result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()
- self.stdout.write('*** Processing images ***')
- while not all_images_processed:
- self.stdout.write('')
- self.stdout.write(f'*** Page {page}')
- self.stdout.write('')
-
- for image_data in result['results']:
- image_uuid = image_data['uuid']
-
- self.stdout.write(f'Processing image {image_uuid}')
-
- try:
- exercise_base = ExerciseBase.objects.get(id=image_data['exercise_base'])
- except ExerciseBase.DoesNotExist:
- self.stdout.write(' Remote exercise base not found in local DB, skipping...')
- continue
-
- try:
- ExerciseImage.objects.get(uuid=image_uuid)
- self.stdout.write(' Image already present locally, skipping...')
- continue
- except ExerciseImage.DoesNotExist:
- self.stdout.write(' Image not found in local DB, creating now...')
- retrieved_image = requests.get(image_data['image'], headers=headers)
- image = ExerciseImage.from_json(
- exercise_base,
- retrieved_image,
- image_data,
- headers,
- )
-
- # Temporary files on windows don't support the delete attribute
- if os.name == 'nt':
- img_temp = NamedTemporaryFile()
- else:
- img_temp = NamedTemporaryFile(delete=True)
- img_temp.write(retrieved_image.content)
- img_temp.flush()
-
- image.exercise_base = exercise_base
- image.is_main = image_data['is_main']
- image.image.save(
- os.path.basename(os.path.basename(image_data['image'])),
- File(img_temp),
- )
- image.save()
- self.stdout.write(self.style.SUCCESS(' successfully saved'))
-
- if result['next']:
- page += 1
- result = requests.get(result['next'], headers=headers).json()
- else:
- all_images_processed = True
+ download_exercise_images(self.stdout.write, remote_url, self.style.SUCCESS)
diff --git a/wger/exercises/management/commands/download-exercise-videos.py b/wger/exercises/management/commands/download-exercise-videos.py
index 4f3d0dc24..76b007cfe 100644
--- a/wger/exercises/management/commands/download-exercise-videos.py
+++ b/wger/exercises/management/commands/download-exercise-videos.py
@@ -14,35 +14,20 @@
#
# You should have received a copy of the GNU Affero General Public License
-# Standard Library
-import os
-
# Django
from django.conf import settings
from django.core.exceptions import (
ImproperlyConfigured,
ValidationError,
)
-from django.core.files import File
-from django.core.files.temp import NamedTemporaryFile
from django.core.management.base import (
BaseCommand,
CommandError,
)
from django.core.validators import URLValidator
-# Third Party
-import requests
-
# wger
-from wger.exercises.models import (
- ExerciseBase,
- ExerciseVideo,
-)
-from wger.utils.requests import wger_headers
-
-
-VIDEO_API = "{0}/api/v2/video/"
+from wger.exercises.sync import download_exercise_videos
class Command(BaseCommand):
@@ -77,68 +62,4 @@ class Command(BaseCommand):
except ValidationError:
raise CommandError('Please enter a valid URL')
- headers = wger_headers()
-
- # Get all videos
- page = 1
- all_videos_processed = False
- result = requests.get(VIDEO_API.format(remote_url), headers=headers).json()
- self.stdout.write('*** Processing videos ***')
- while not all_videos_processed:
- self.stdout.write('')
- self.stdout.write(f'*** Page {page}')
- self.stdout.write('')
-
- for video_data in result['results']:
- video_uuid = video_data['uuid']
- self.stdout.write(f'Processing video {video_uuid}')
-
- try:
- exercise_base = ExerciseBase.objects.get(uuid=video_data['exercise_base_uuid'])
- except ExerciseBase.DoesNotExist:
- self.stdout.write(' Remote exercise base not found in local DB, skipping...')
- continue
-
- try:
- ExerciseVideo.objects.get(uuid=video_uuid)
- self.stdout.write(' Video already present locally, skipping...')
- continue
- except ExerciseVideo.DoesNotExist:
- self.stdout.write(' Video not found in local DB, creating now...')
- video = ExerciseVideo()
- video.exercise_base = exercise_base
- video.uuid = video_uuid
- video.is_main = video_data['is_main']
- video.license_id = video_data['license']
- video.license_author = video_data['license_author']
- video.size = video_data['size']
- video.width = video_data['width']
- video.height = video_data['height']
- video.codec = video_data['codec']
- video.codec_long = video_data['codec_long']
- video.duration = video_data['duration']
-
- # Save the downloaded video
- # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
- retrieved_video = requests.get(video_data['video'], headers=headers)
-
- # Temporary files on Windows don't support the delete attribute
- if os.name == 'nt':
- img_temp = NamedTemporaryFile()
- else:
- img_temp = NamedTemporaryFile(delete=True)
- img_temp.write(retrieved_video.content)
- img_temp.flush()
-
- video.video.save(
- os.path.basename(os.path.basename(video_data['video'])),
- File(img_temp),
- )
- video.save()
- self.stdout.write(self.style.SUCCESS(' saved successfully'))
-
- if result['next']:
- page += 1
- result = requests.get(result['next'], headers=headers).json()
- else:
- all_videos_processed = True
+ download_exercise_videos(self.stdout.write, remote_url)
diff --git a/wger/exercises/models/image.py b/wger/exercises/models/image.py
index 62b4ec1c2..bedadcb03 100644
--- a/wger/exercises/models/image.py
+++ b/wger/exercises/models/image.py
@@ -165,15 +165,16 @@ class ExerciseImage(AbstractLicenseModel, AbstractHistoryMixin, models.Model, Ba
connect_to: ExerciseBase,
retrieved_image,
json_data: dict,
- headers,
- generate_uuid: bool = False
+ generate_uuid: bool = False,
):
image: cls = super().from_json(
- connect_to, retrieved_image, json_data, headers, generate_uuid
+ connect_to,
+ retrieved_image,
+ json_data,
+ generate_uuid,
)
image.exercise_base = connect_to
image.is_main = json_data['is_main']
- image.status = json_data['status']
image.save_image(retrieved_image, json_data)
diff --git a/wger/exercises/sync.py b/wger/exercises/sync.py
index 7eaa391a0..f79403c4d 100644
--- a/wger/exercises/sync.py
+++ b/wger/exercises/sync.py
@@ -11,9 +11,13 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
+# Standard Library
+import os
# Django
from django.conf import settings
+from django.core.files import File
+from django.core.files.temp import NamedTemporaryFile
# Third Party
import requests
@@ -39,6 +43,8 @@ CATEGORY_API = "{0}/api/v2/exercisecategory/"
MUSCLE_API = "{0}/api/v2/muscle/"
LANGUAGE_API = "{0}/api/v2/language/"
EQUIPMENT_API = "{0}/api/v2/equipment/"
+IMAGE_API = "{0}/api/v2/exerciseimage/"
+VIDEO_API = "{0}/api/v2/video/"
def sync_exercises(
@@ -279,3 +285,141 @@ def delete_entries(
else:
all_entries_processed = True
print_fn(style_fn('done!\n'))
+
+
+def download_exercise_images(
+ print_fn,
+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],
+ style_fn=lambda x: x,
+):
+ headers = wger_headers()
+
+ # Get all images
+ page = 1
+ all_images_processed = False
+ result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()
+ print_fn('*** Processing images ***')
+ while not all_images_processed:
+ print_fn('')
+ print_fn(f'*** Page {page}')
+ print_fn('')
+
+ for image_data in result['results']:
+ image_uuid = image_data['uuid']
+
+ print_fn(f'Processing image {image_uuid}')
+
+ try:
+ exercise_base = ExerciseBase.objects.get(id=image_data['exercise_base'])
+ except ExerciseBase.DoesNotExist:
+ print_fn(' Remote exercise base not found in local DB, skipping...')
+ continue
+
+ try:
+ ExerciseImage.objects.get(uuid=image_uuid)
+ print_fn(' Image already present locally, skipping...')
+ continue
+ except ExerciseImage.DoesNotExist:
+ print_fn(' Image not found in local DB, creating now...')
+ retrieved_image = requests.get(image_data['image'], headers=headers)
+ image = ExerciseImage.from_json(
+ exercise_base,
+ retrieved_image,
+ image_data,
+ headers,
+ )
+
+ # Temporary files on Windows don't support the delete attribute
+ if os.name == 'nt':
+ img_temp = NamedTemporaryFile()
+ else:
+ img_temp = NamedTemporaryFile(delete=True)
+ img_temp.write(retrieved_image.content)
+ img_temp.flush()
+
+ image.exercise_base = exercise_base
+ image.is_main = image_data['is_main']
+ image.image.save(
+ os.path.basename(os.path.basename(image_data['image'])),
+ File(img_temp),
+ )
+ image.save()
+ print_fn(style_fn(' successfully saved'))
+
+ if result['next']:
+ page += 1
+ result = requests.get(result['next'], headers=headers).json()
+ else:
+ all_images_processed = True
+
+
+def download_exercise_videos(
+ print_fn,
+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],
+ style_fn=lambda x: x,
+):
+ headers = wger_headers()
+
+ # Get all videos
+ page = 1
+ all_videos_processed = False
+ result = requests.get(VIDEO_API.format(remote_url), headers=headers).json()
+ print_fn('*** Processing videos ***')
+ while not all_videos_processed:
+ print_fn('')
+ print_fn(f'*** Page {page}')
+ print_fn('')
+
+ for video_data in result['results']:
+ video_uuid = video_data['uuid']
+ print_fn(f'Processing video {video_uuid}')
+
+ try:
+ exercise_base = ExerciseBase.objects.get(uuid=video_data['exercise_base_uuid'])
+ except ExerciseBase.DoesNotExist:
+ print_fn(' Remote exercise base not found in local DB, skipping...')
+ continue
+
+ try:
+ ExerciseVideo.objects.get(uuid=video_uuid)
+ print_fn(' Video already present locally, skipping...')
+ continue
+ except ExerciseVideo.DoesNotExist:
+ print_fn(' Video not found in local DB, creating now...')
+ video = ExerciseVideo()
+ video.exercise_base = exercise_base
+ video.uuid = video_uuid
+ video.is_main = video_data['is_main']
+ video.license_id = video_data['license']
+ video.license_author = video_data['license_author']
+ video.size = video_data['size']
+ video.width = video_data['width']
+ video.height = video_data['height']
+ video.codec = video_data['codec']
+ video.codec_long = video_data['codec_long']
+ video.duration = video_data['duration']
+
+ # Save the downloaded video
+ # http://stackoverflow.com/questions/1308386/programmatically-saving-image-to-
+ retrieved_video = requests.get(video_data['video'], headers=headers)
+
+ # Temporary files on Windows don't support the delete attribute
+ if os.name == 'nt':
+ img_temp = NamedTemporaryFile()
+ else:
+ img_temp = NamedTemporaryFile(delete=True)
+ img_temp.write(retrieved_video.content)
+ img_temp.flush()
+
+ video.video.save(
+ os.path.basename(os.path.basename(video_data['video'])),
+ File(img_temp),
+ )
+ video.save()
+ print_fn(style_fn(' saved successfully'))
+
+ if result['next']:
+ page += 1
+ result = requests.get(result['next'], headers=headers).json()
+ else:
+ all_videos_processed = True
diff --git a/wger/exercises/tasks.py b/wger/exercises/tasks.py
index be42becc1..ad183612a 100644
--- a/wger/exercises/tasks.py
+++ b/wger/exercises/tasks.py
@@ -26,6 +26,8 @@ from celery.schedules import crontab
from wger.celery_configuration import app
from wger.exercises.sync import (
delete_entries,
+ download_exercise_images,
+ download_exercise_videos,
sync_categories,
sync_equipment,
sync_exercises,
@@ -33,6 +35,7 @@ from wger.exercises.sync import (
sync_muscles,
)
+
logger = logging.getLogger(__name__)
@@ -55,7 +58,7 @@ def sync_images_task():
"""
Fetches the exercise images from the default wger instance
"""
- pass
+ download_exercise_images(logger.info)
@app.task
@@ -63,7 +66,7 @@ def sync_videos_task():
"""
Fetches the exercise videos from the default wger instance
"""
- pass
+ download_exercise_videos(logger.info)
@app.on_after_finalize.connect
diff --git a/wger/nutrition/management/commands/download-ingredient-images.py b/wger/nutrition/management/commands/download-ingredient-images.py
index 3b4aebc04..6e647a020 100644
--- a/wger/nutrition/management/commands/download-ingredient-images.py
+++ b/wger/nutrition/management/commands/download-ingredient-images.py
@@ -26,19 +26,8 @@ from django.core.management.base import (
)
from django.core.validators import URLValidator
-# Third Party
-import requests
-
# wger
-from wger import get_version
-from wger.nutrition.models import (
- Image,
- Ingredient,
-)
-from wger.utils.requests import wger_headers
-
-
-IMAGE_API = "{0}/api/v2/ingredient-image/"
+from wger.nutrition.sync import download_ingredient_images
class Command(BaseCommand):
@@ -80,42 +69,4 @@ class Command(BaseCommand):
except ValidationError:
raise CommandError('Please enter a valid URL')
- headers = wger_headers()
-
- # Get all images
- page = 1
- all_images_processed = False
- result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()
- self.stdout.write('*** Processing images ***')
- while not all_images_processed:
- self.stdout.write('')
- self.stdout.write(f'*** Page {page}')
- self.stdout.write('')
-
- for image_data in result['results']:
- image_uuid = image_data['uuid']
-
- self.stdout.write(f'Processing image {image_uuid}')
-
- try:
- ingredient = Ingredient.objects.get(uuid=image_data['ingredient_uuid'])
- except Ingredient.DoesNotExist:
- self.stdout.write(' Remote ingredient not found in local DB, skipping...')
- continue
-
- try:
- Image.objects.get(uuid=image_uuid)
- self.stdout.write(' Image already present locally, skipping...')
- continue
- except Image.DoesNotExist:
- self.stdout.write(' Image not found in local DB, creating now...')
- retrieved_image = requests.get(image_data['image'], headers=headers)
- Image.from_json(ingredient, retrieved_image, image_data)
-
- self.stdout.write(self.style.SUCCESS(' successfully saved'))
-
- if result['next']:
- page += 1
- result = requests.get(result['next'], headers=headers).json()
- else:
- all_images_processed = True
+ download_ingredient_images(self.stdout.write, remote_url, self.style.SUCCESS)
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py
index 7d63e7be6..139a6b3e6 100644
--- a/wger/nutrition/sync.py
+++ b/wger/nutrition/sync.py
@@ -27,6 +27,7 @@ import requests
# wger
from wger.nutrition.models import (
Image,
+ Ingredient,
Source,
)
from wger.utils.constants import (
@@ -38,6 +39,8 @@ from wger.utils.requests import wger_headers
logger = logging.getLogger(__name__)
+IMAGE_API = "{0}/api/v2/ingredient-image/"
+
def fetch_ingredient_image(pk: int):
# wger
@@ -123,3 +126,48 @@ def fetch_image_from_off(ingredient):
logger.info('Ingredient has already an image, skipping...')
return
logger.info('Image successfully saved')
+
+
+def download_ingredient_images(
+ print_fn,
+ remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'],
+ style_fn=lambda x: x,
+):
+ headers = wger_headers()
+ # Get all images
+ page = 1
+ all_images_processed = False
+ result = requests.get(IMAGE_API.format(remote_url), headers=headers).json()
+ print_fn('*** Processing images ***')
+ while not all_images_processed:
+ print_fn('')
+ print_fn(f'*** Page {page}')
+ print_fn('')
+
+ for image_data in result['results']:
+ image_uuid = image_data['uuid']
+
+ print_fn(f'Processing image {image_uuid}')
+
+ try:
+ ingredient = Ingredient.objects.get(uuid=image_data['ingredient_uuid'])
+ except Ingredient.DoesNotExist:
+ print_fn(' Remote ingredient not found in local DB, skipping...')
+ continue
+
+ try:
+ Image.objects.get(uuid=image_uuid)
+ print_fn(' Image already present locally, skipping...')
+ continue
+ except Image.DoesNotExist:
+ print_fn(' Image not found in local DB, creating now...')
+ retrieved_image = requests.get(image_data['image'], headers=headers)
+ Image.from_json(ingredient, retrieved_image, image_data)
+
+ print_fn(style_fn(' successfully saved'))
+
+ if result['next']:
+ page += 1
+ result = requests.get(result['next'], headers=headers).json()
+ else:
+ all_images_processed = True
diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.py
index 618a9051b..40352ee3c 100644
--- a/wger/nutrition/tasks.py
+++ b/wger/nutrition/tasks.py
@@ -12,9 +12,18 @@
#
# You should have received a copy of the GNU Affero General Public License
+# Standard Library
+import logging
+
# wger
from wger.celery_configuration import app
-from wger.nutrition.sync import fetch_ingredient_image
+from wger.nutrition.sync import (
+ download_ingredient_images,
+ fetch_ingredient_image,
+)
+
+
+logger = logging.getLogger(__name__)
@app.task
@@ -25,3 +34,13 @@ def fetch_ingredient_image_task(pk: int):
Returns the image if it is already present in the DB
"""
fetch_ingredient_image(pk)
+
+
+def fetch_all_ingredient_images_task():
+ """
+ Fetches the ingredient image from Open Food Facts servers if it is not available locally
+
+ Returns the image if it is already present in the DB
+ """
+ download_ingredient_images(logger.info)
diff --git a/wger/settings_global.py b/wger/settings_global.py
index b317049d6..9ca05bb40 100644
--- a/wger/settings_global.py
+++ b/wger/settings_global.py
@@ -24,6 +24,7 @@ from datetime import timedelta
# wger
from wger.utils.constants import DOWNLOAD_INGREDIENT_WGER
+
"""
This file contains the global settings that don't usually need to be changed.
For a full list of options, visit:
@@ -169,7 +170,7 @@ TEMPLATES = [
'django.template.loaders.app_directories.Loader',
],
'debug':
- False
+ False
},
},
]
@@ -239,7 +240,7 @@ LANGUAGES = (
LANGUAGE_CODE = 'en'
# All translation files are in one place
-LOCALE_PATHS = (os.path.join(SITE_ROOT, 'locale'),)
+LOCALE_PATHS = (os.path.join(SITE_ROOT, 'locale'), )
# Primary keys are AutoFields
DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
@@ -411,15 +412,11 @@ COMPRESS_ROOT = STATIC_ROOT
# Django Rest Framework
#
REST_FRAMEWORK = {
- 'DEFAULT_PERMISSION_CLASSES': ('wger.utils.permissions.WgerPermission',),
- 'DEFAULT_PAGINATION_CLASS':
- 'rest_framework.pagination.LimitOffsetPagination',
- 'PAGE_SIZE':
- 20,
- 'PAGINATE_BY_PARAM':
- 'limit', # Allow client to override, using `?limit=xxx`.
- 'TEST_REQUEST_DEFAULT_FORMAT':
- 'json',
+ 'DEFAULT_PERMISSION_CLASSES': ('wger.utils.permissions.WgerPermission', ),
+ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+ 'PAGE_SIZE': 20,
+ 'PAGINATE_BY_PARAM': 'limit', # Allow client to override, using `?limit=xxx`.
+ 'TEST_REQUEST_DEFAULT_FORMAT': 'json',
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework.authentication.SessionAuthentication',
'rest_framework.authentication.TokenAuthentication',
@@ -431,7 +428,7 @@ REST_FRAMEWORK = {
),
'DEFAULT_THROTTLE_CLASSES': ['rest_framework.throttling.ScopedRateThrottle'],
'DEFAULT_THROTTLE_RATES': {
- 'login': '3/min'
+ 'login': '10/min'
}
}
@@ -455,7 +452,7 @@ CORS_URLS_REGEX = r'^/api/.*$'
#
# Ignore these URLs if they cause 404
#
-IGNORABLE_404_URLS = (re.compile(r'^/favicon\.ico$'),)
+IGNORABLE_404_URLS = (re.compile(r'^/favicon\.ico$'), )
#
# Password rules