Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Use the openfoodfacts skd to download the product images

Roland Geidereeb18f0e1b4a
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py
index 66f01acaf..b5970f7d4 100644
--- a/wger/nutrition/sync.py
+++ b/wger/nutrition/sync.py
@@ -27,10 +27,14 @@ from django.utils import timezone
# Third Party
import requests
-from openfoodfacts.images import (
- AWS_S3_BASE_URL,
- generate_image_path,
+from openfoodfacts import (
+ API,
+ APIVersion,
+ Country,
+ Environment,
+ Flavor,
)
+from openfoodfacts.images import download_image
from tqdm import tqdm
# wger
@@ -55,6 +59,7 @@ from wger.utils.language import load_language
from wger.utils.requests import (
get_paginated,
wger_headers,
+ wger_user_agent,
)
from wger.utils.url import make_uri
@@ -135,55 +140,47 @@ def fetch_image_from_off(ingredient: Ingredient):
- https://openfoodfacts.github.io/openfoodfacts-server/api/how-to-download-images/
- https://openfoodfacts.github.io/openfoodfacts-server/api/ref-v2/
"""
- logger.info(f'Trying to fetch image from OFF for {ingredient.name} (UUID: {ingredient.uuid})')
-
- url = ingredient.source_url + '?fields=images,image_front_url'
- headers = wger_headers()
- try:
- product_data = requests.get(url, headers=headers, timeout=3).json()
- except requests.JSONDecodeError:
- logger.warning(f'Could not decode JSON response from {url}')
- return
- except requests.ConnectTimeout as e:
- logger.warning(f'Connection timeout while trying to fetch {url}: {e}')
- return
- except requests.ReadTimeout as e:
- logger.warning(f'Read timeout while trying to fetch {url}: {e}')
- return
+ logger.info(f'Trying to fetch image from OFF for "{ingredient.name}" ({ingredient.uuid})')
+ off_api = API(
+ user_agent=wger_user_agent(),
+ country=Country.world,
+ flavor=Flavor.off,
+ version=APIVersion.v2,
+ environment=Environment.org,
+ )
+ product_data = off_api.product.get(ingredient.code, fields=['images', 'image_front_url'])
- try:
- image_url: Optional[str] = product_data['product'].get('image_front_url')
- except KeyError:
- logger.info('No "product" key found, exiting...')
+ if product_data is None:
+ logger.info('No product data found for this ingredient')
return
+ image_url: Optional[str] = product_data.get('image_front_url')
if not image_url:
logger.info('Product data has no "image_front_url" key')
return
- image_data = product_data['product']['images']
+ image_data = product_data['images']
+ if not image_data:
+ logger.info('Product data has no "images" key')
+ return
# Extract the image key from the url:
# https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg -> "front_en"
- image_id: str = image_url.rpartition('/')[2].partition('.')[0]
+ image_key: str = image_url.rpartition('/')[2].partition('.')[0]
- # Extract the uploader name
+ # Extract the uploader name and numerical image id
try:
- image_id: str = image_data[image_id]['imgid']
+ image_id: str = image_data[image_key]['imgid']
uploader_name: str = image_data[image_id]['uploader']
except KeyError as e:
logger.info('could not load all image information, skipping...', e)
return
- # Download image from amazon
- image_s3_url = f'{AWS_S3_BASE_URL}{generate_image_path(ingredient.code, image_id)}'
- response = requests.get(image_s3_url, headers=headers)
- if not response.ok:
- logger.info(f'Could not locate image on AWS! Status code: {response.status_code}')
- return
+ off_response = download_image(image_url, return_struct=True)
# Save to DB
- url = (
- f'https://world.openfoodfacts.org/cgi/product_image.pl?code={ingredient.code}&id={image_id}'
+ url = make_uri(
+ 'https://world.openfoodfacts.org/cgi/product_image.pl',
+ query={'code': ingredient.code, 'id': image_id},
)
uploader_url = f'https://world.openfoodfacts.org/photographer/{uploader_name}'
image_data = {
@@ -194,15 +191,15 @@ def fetch_image_from_off(ingredient: Ingredient):
'license_author_url': uploader_url,
'license_object_url': url,
'license_derivative_source_url': '',
- 'size': len(response.content),
+ 'size': len(off_response.image_bytes),
}
try:
- Image.from_json(ingredient, response, image_data, generate_uuid=True)
+ Image.from_json(ingredient, off_response.response, image_data, generate_uuid=True)
# Due to a race condition (e.g. when adding tasks over the search), we might
# try to save an image to an ingredient that already has one. In that case,
# just ignore the error
except IntegrityError:
- logger.info('Ingredient has already an image, skipping...')
+ logger.debug('Ingredient has already an image, skipping...')
return
ingredient.last_image_check = timezone.now()
ingredient.save()
diff --git a/wger/nutrition/tests/test_tasks.py b/wger/nutrition/tests/test_tasks.py
index 09d36a85d..e72dd9e78 100644
--- a/wger/nutrition/tests/test_tasks.py
+++ b/wger/nutrition/tests/test_tasks.py
@@ -15,10 +15,7 @@
# Standard Library
import datetime
-from unittest.mock import (
- ANY,
- patch,
-)
+from unittest.mock import patch
# Django
from django.utils import timezone
@@ -37,30 +34,20 @@ from wger.utils.constants import (
from wger.utils.requests import wger_headers
-class MockOffResponse:
- def __init__(self):
- self.status_code = 200
- self.ok = True
- self.content = b'2000'
+mock_off_response = {
+ 'image_front_url': 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
+ 'images': {
+ 'front_en': {
+ 'imgid': '12345',
+ },
+ '12345': {'uploader': 'Mr Foobar'},
+ },
+}
- # yapf: disable
- @staticmethod
- def json():
- return {
- "product": {
- 'image_front_url':
- 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
- 'images': {
- 'front_en': {
- 'imgid': '12345',
- },
- '12345': {
- 'uploader': 'Mr Foobar'
- }
- }
- },
- }
- # yapf: disable
+
+class MockOffImageResponse:
+ image_bytes = [0, 1, 2, 3]
+ response = type('obj', (object,), {'content': b'mock_content'})()
class MockWgerApiResponse:
@@ -68,35 +55,33 @@ class MockWgerApiResponse:
self.status_code = 200
self.content = b'2000'
- # yapf: disable
@staticmethod
def json():
return {
- "count": 1,
- "next": None,
- "previous": None,
- "results": [
+ 'count': 1,
+ 'next': None,
+ 'previous': None,
+ 'results': [
{
- "id": 1,
- "uuid": "188324b5-587f-42d7-9abc-d2ca64c73d45",
- "ingredient_id": "12345",
- "ingredient_uuid": "e9baa8bd-84fc-4756-8d90-5b9739b06cf8",
- "image": "http://localhost:8000/media/ingredients/1/188324b5-587f-42d7-9abc-d2ca64c73d45.jpg",
- "created": "2023-03-15T23:20:10.969369+01:00",
- "last_update": "2023-03-15T23:20:10.969369+01:00",
- "size": 20179,
- "width": 400,
- "height": 166,
- "license": 1,
- "license_author": "Tester McTest",
- "license_author_url": "https://example.com/editors/mcLovin",
- "license_title": "",
- "license_object_url": "",
- "license_derivative_source_url": ""
+ 'id': 1,
+ 'uuid': '188324b5-587f-42d7-9abc-d2ca64c73d45',
+ 'ingredient_id': '12345',
+ 'ingredient_uuid': 'e9baa8bd-84fc-4756-8d90-5b9739b06cf8',
+ 'image': 'http://localhost:8000/media/ingredients/1/188324b5-587f-42d7-9abc-d2ca64c73d45.jpg',
+ 'created': '2023-03-15T23:20:10.969369+01:00',
+ 'last_update': '2023-03-15T23:20:10.969369+01:00',
+ 'size': 20179,
+ 'width': 400,
+ 'height': 166,
+ 'license': 1,
+ 'license_author': 'Tester McTest',
+ 'license_author_url': 'https://example.com/editors/mcLovin',
+ 'license_title': '',
+ 'license_object_url': '',
+ 'license_derivative_source_url': '',
}
- ]
+ ],
}
- # yapf: enable
class FetchIngredientImageTestCase(WgerTestCase):
@@ -104,7 +89,7 @@ class FetchIngredientImageTestCase(WgerTestCase):
Test fetching an ingredient image
"""
- @patch('requests.get')
+ @patch('openfoodfacts.api.ProductResource.get')
@patch.object(logger, 'info')
def test_source(self, mock_logger, mock_request):
"""
@@ -120,7 +105,7 @@ class FetchIngredientImageTestCase(WgerTestCase):
mock_request.assert_not_called()
self.assertEqual(result, None)
- @patch('requests.get')
+ @patch('openfoodfacts.api.ProductResource.get')
@patch.object(logger, 'info')
def test_download_off_setting(self, mock_logger, mock_request):
"""
@@ -136,7 +121,7 @@ class FetchIngredientImageTestCase(WgerTestCase):
mock_request.assert_not_called()
self.assertEqual(result, None)
- @patch('requests.get', return_value=MockOffResponse())
+ @patch('openfoodfacts.api.ProductResource.get')
@patch('wger.nutrition.models.Image.from_json')
@patch.object(logger, 'info')
def test_last_new_image_date(self, mock_logger, mock_from_json, mock_request):
@@ -160,10 +145,17 @@ class FetchIngredientImageTestCase(WgerTestCase):
mock_request.assert_not_called()
self.assertEqual(result, None)
- @patch('requests.get', return_value=MockOffResponse())
+ @patch('wger.nutrition.sync.download_image', return_value=MockOffImageResponse)
+ @patch('openfoodfacts.api.ProductResource.get', return_value=mock_off_response)
@patch('wger.nutrition.models.Image.from_json')
@patch.object(logger, 'info')
- def test_last_old_image_date(self, mock_logger, mock_from_json, mock_request):
+ def test_last_old_image_date(
+ self,
+ mock_logger,
+ mock_from_json,
+ mock_request,
+ mock_download_image,
+ ):
"""
Test that images are fetched if we checked a long time ago
"""
@@ -180,14 +172,27 @@ class FetchIngredientImageTestCase(WgerTestCase):
):
result = fetch_ingredient_image(1)
mock_from_json.assert_called()
+ mock_download_image.assert_called_with(
+ 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
+ return_struct=True,
+ )
mock_logger.assert_any_call('Fetching image for ingredient 1')
- mock_request.assert_called()
+ mock_request.assert_called_with(
+ '1234567890987654321', fields=['images', 'image_front_url']
+ )
self.assertEqual(result, None)
- @patch('requests.get', return_value=MockOffResponse())
+ @patch('wger.nutrition.sync.download_image', return_value=MockOffImageResponse)
+ @patch('openfoodfacts.api.ProductResource.get', return_value=mock_off_response)
@patch('wger.nutrition.models.Image.from_json')
@patch.object(logger, 'info')
- def test_download_ingredient_off(self, mock_logger, mock_from_json, mock_request):
+ def test_download_ingredient_off(
+ self,
+ mock_logger,
+ mock_from_json,
+ mock_request,
+ mock_download_image,
+ ):
"""
Test that the image is correctly downloaded
@@ -202,23 +207,21 @@ class FetchIngredientImageTestCase(WgerTestCase):
):
result = fetch_ingredient_image(1)
+ # print(mock_request.mock_calls)
+ mock_request.assert_called_with(
+ '1234567890987654321',
+ fields=['images', 'image_front_url'],
+ )
+ mock_download_image.assert_called_with(
+ 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
+ return_struct=True,
+ )
mock_logger.assert_any_call('Fetching image for ingredient 1')
mock_logger.assert_any_call(
- 'Trying to fetch image from OFF for Test ingredient 1 (UUID: '
- '7908c204-907f-4b1e-ad4e-f482e9769ade)'
+ 'Trying to fetch image from OFF for "Test ingredient 1" '
+ '(7908c204-907f-4b1e-ad4e-f482e9769ade)'
)
mock_logger.assert_any_call('Image successfully saved')
-
- # print(mock_request.mock_calls)
- mock_request.assert_any_call(
- 'https://world.openfoodfacts.org/api/v2/product/5055365635003.json?fields=images,image_front_url',
- headers=wger_headers(),
- timeout=ANY,
- )
- mock_request.assert_any_call(
- 'https://openfoodfacts-images.s3.eu-west-3.amazonaws.com/data/123/456/789/0987654321/12345.jpg',
- headers=wger_headers(),
- )
mock_from_json.assert_called()
self.assertEqual(result, None)