Self hosted FLOSS fitness/workout, nutrition and weight tracker
Use the openfoodfacts skd to download the product images
Roland Geidereeb18f0e1b4a
diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.pyindex 66f01acaf..b5970f7d4 100644--- a/wger/nutrition/sync.py+++ b/wger/nutrition/sync.py@@ -27,10 +27,14 @@ from django.utils import timezone# Third Partyimport requests-from openfoodfacts.images import (- AWS_S3_BASE_URL,- generate_image_path,+from openfoodfacts import (+ API,+ APIVersion,+ Country,+ Environment,+ Flavor,)+from openfoodfacts.images import download_imagefrom tqdm import tqdm# wger@@ -55,6 +59,7 @@ from wger.utils.language import load_languagefrom wger.utils.requests import (get_paginated,wger_headers,+ wger_user_agent,)from wger.utils.url import make_uri@@ -135,55 +140,47 @@ def fetch_image_from_off(ingredient: Ingredient):- https://openfoodfacts.github.io/openfoodfacts-server/api/how-to-download-images/- https://openfoodfacts.github.io/openfoodfacts-server/api/ref-v2/"""- logger.info(f'Trying to fetch image from OFF for {ingredient.name} (UUID: {ingredient.uuid})')-- url = ingredient.source_url + '?fields=images,image_front_url'- headers = wger_headers()- try:- product_data = requests.get(url, headers=headers, timeout=3).json()- except requests.JSONDecodeError:- logger.warning(f'Could not decode JSON response from {url}')- return- except requests.ConnectTimeout as e:- logger.warning(f'Connection timeout while trying to fetch {url}: {e}')- return- except requests.ReadTimeout as e:- logger.warning(f'Read timeout while trying to fetch {url}: {e}')- return+ logger.info(f'Trying to fetch image from OFF for "{ingredient.name}" ({ingredient.uuid})')+ off_api = API(+ user_agent=wger_user_agent(),+ country=Country.world,+ flavor=Flavor.off,+ version=APIVersion.v2,+ environment=Environment.org,+ )+ product_data = off_api.product.get(ingredient.code, fields=['images', 'image_front_url'])- try:- image_url: Optional[str] = product_data['product'].get('image_front_url')- except KeyError:- logger.info('No "product" key found, exiting...')+ if product_data is None:+ logger.info('No product data found for this ingredient')return+ image_url: Optional[str] = product_data.get('image_front_url')if not image_url:logger.info('Product data has no "image_front_url" key')return- image_data = product_data['product']['images']+ image_data = product_data['images']+ if not image_data:+ logger.info('Product data has no "images" key')+ return# Extract the image key from the url:# https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg -> "front_en"- image_id: str = image_url.rpartition('/')[2].partition('.')[0]+ image_key: str = image_url.rpartition('/')[2].partition('.')[0]- # Extract the uploader name+ # Extract the uploader name and numerical image idtry:- image_id: str = image_data[image_id]['imgid']+ image_id: str = image_data[image_key]['imgid']uploader_name: str = image_data[image_id]['uploader']except KeyError as e:logger.info('could not load all image information, skipping...', e)return- # Download image from amazon- image_s3_url = f'{AWS_S3_BASE_URL}{generate_image_path(ingredient.code, image_id)}'- response = requests.get(image_s3_url, headers=headers)- if not response.ok:- logger.info(f'Could not locate image on AWS! Status code: {response.status_code}')- return+ off_response = download_image(image_url, return_struct=True)# Save to DB- url = (- f'https://world.openfoodfacts.org/cgi/product_image.pl?code={ingredient.code}&id={image_id}'+ url = make_uri(+ 'https://world.openfoodfacts.org/cgi/product_image.pl',+ query={'code': ingredient.code, 'id': image_id},)uploader_url = f'https://world.openfoodfacts.org/photographer/{uploader_name}'image_data = {@@ -194,15 +191,15 @@ def fetch_image_from_off(ingredient: Ingredient):'license_author_url': uploader_url,'license_object_url': url,'license_derivative_source_url': '',- 'size': len(response.content),+ 'size': len(off_response.image_bytes),}try:- Image.from_json(ingredient, response, image_data, generate_uuid=True)+ Image.from_json(ingredient, off_response.response, image_data, generate_uuid=True)# Due to a race condition (e.g. when adding tasks over the search), we might# try to save an image to an ingredient that already has one. In that case,# just ignore the errorexcept IntegrityError:- logger.info('Ingredient has already an image, skipping...')+ logger.debug('Ingredient has already an image, skipping...')returningredient.last_image_check = timezone.now()ingredient.save()diff --git a/wger/nutrition/tests/test_tasks.py b/wger/nutrition/tests/test_tasks.pyindex 09d36a85d..e72dd9e78 100644--- a/wger/nutrition/tests/test_tasks.py+++ b/wger/nutrition/tests/test_tasks.py@@ -15,10 +15,7 @@# Standard Libraryimport datetime-from unittest.mock import (- ANY,- patch,-)+from unittest.mock import patch# Djangofrom django.utils import timezone@@ -37,30 +34,20 @@ from wger.utils.constants import (from wger.utils.requests import wger_headers-class MockOffResponse:- def __init__(self):- self.status_code = 200- self.ok = True- self.content = b'2000'+mock_off_response = {+ 'image_front_url': 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',+ 'images': {+ 'front_en': {+ 'imgid': '12345',+ },+ '12345': {'uploader': 'Mr Foobar'},+ },+}- # yapf: disable- @staticmethod- def json():- return {- "product": {- 'image_front_url':- 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',- 'images': {- 'front_en': {- 'imgid': '12345',- },- '12345': {- 'uploader': 'Mr Foobar'- }- }- },- }- # yapf: disable++class MockOffImageResponse:+ image_bytes = [0, 1, 2, 3]+ response = type('obj', (object,), {'content': b'mock_content'})()class MockWgerApiResponse:@@ -68,35 +55,33 @@ class MockWgerApiResponse:self.status_code = 200self.content = b'2000'- # yapf: disable@staticmethoddef json():return {- "count": 1,- "next": None,- "previous": None,- "results": [+ 'count': 1,+ 'next': None,+ 'previous': None,+ 'results': [{- "id": 1,- "uuid": "188324b5-587f-42d7-9abc-d2ca64c73d45",- "ingredient_id": "12345",- "ingredient_uuid": "e9baa8bd-84fc-4756-8d90-5b9739b06cf8",- "image": "http://localhost:8000/media/ingredients/1/188324b5-587f-42d7-9abc-d2ca64c73d45.jpg",- "created": "2023-03-15T23:20:10.969369+01:00",- "last_update": "2023-03-15T23:20:10.969369+01:00",- "size": 20179,- "width": 400,- "height": 166,- "license": 1,- "license_author": "Tester McTest",- "license_author_url": "https://example.com/editors/mcLovin",- "license_title": "",- "license_object_url": "",- "license_derivative_source_url": ""+ 'id': 1,+ 'uuid': '188324b5-587f-42d7-9abc-d2ca64c73d45',+ 'ingredient_id': '12345',+ 'ingredient_uuid': 'e9baa8bd-84fc-4756-8d90-5b9739b06cf8',+ 'image': 'http://localhost:8000/media/ingredients/1/188324b5-587f-42d7-9abc-d2ca64c73d45.jpg',+ 'created': '2023-03-15T23:20:10.969369+01:00',+ 'last_update': '2023-03-15T23:20:10.969369+01:00',+ 'size': 20179,+ 'width': 400,+ 'height': 166,+ 'license': 1,+ 'license_author': 'Tester McTest',+ 'license_author_url': 'https://example.com/editors/mcLovin',+ 'license_title': '',+ 'license_object_url': '',+ 'license_derivative_source_url': '',}- ]+ ],}- # yapf: enableclass FetchIngredientImageTestCase(WgerTestCase):@@ -104,7 +89,7 @@ class FetchIngredientImageTestCase(WgerTestCase):Test fetching an ingredient image"""- @patch('requests.get')+ @patch('openfoodfacts.api.ProductResource.get')@patch.object(logger, 'info')def test_source(self, mock_logger, mock_request):"""@@ -120,7 +105,7 @@ class FetchIngredientImageTestCase(WgerTestCase):mock_request.assert_not_called()self.assertEqual(result, None)- @patch('requests.get')+ @patch('openfoodfacts.api.ProductResource.get')@patch.object(logger, 'info')def test_download_off_setting(self, mock_logger, mock_request):"""@@ -136,7 +121,7 @@ class FetchIngredientImageTestCase(WgerTestCase):mock_request.assert_not_called()self.assertEqual(result, None)- @patch('requests.get', return_value=MockOffResponse())+ @patch('openfoodfacts.api.ProductResource.get')@patch('wger.nutrition.models.Image.from_json')@patch.object(logger, 'info')def test_last_new_image_date(self, mock_logger, mock_from_json, mock_request):@@ -160,10 +145,17 @@ class FetchIngredientImageTestCase(WgerTestCase):mock_request.assert_not_called()self.assertEqual(result, None)- @patch('requests.get', return_value=MockOffResponse())+ @patch('wger.nutrition.sync.download_image', return_value=MockOffImageResponse)+ @patch('openfoodfacts.api.ProductResource.get', return_value=mock_off_response)@patch('wger.nutrition.models.Image.from_json')@patch.object(logger, 'info')- def test_last_old_image_date(self, mock_logger, mock_from_json, mock_request):+ def test_last_old_image_date(+ self,+ mock_logger,+ mock_from_json,+ mock_request,+ mock_download_image,+ ):"""Test that images are fetched if we checked a long time ago"""@@ -180,14 +172,27 @@ class FetchIngredientImageTestCase(WgerTestCase):):result = fetch_ingredient_image(1)mock_from_json.assert_called()+ mock_download_image.assert_called_with(+ 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',+ return_struct=True,+ )mock_logger.assert_any_call('Fetching image for ingredient 1')- mock_request.assert_called()+ mock_request.assert_called_with(+ '1234567890987654321', fields=['images', 'image_front_url']+ )self.assertEqual(result, None)- @patch('requests.get', return_value=MockOffResponse())+ @patch('wger.nutrition.sync.download_image', return_value=MockOffImageResponse)+ @patch('openfoodfacts.api.ProductResource.get', return_value=mock_off_response)@patch('wger.nutrition.models.Image.from_json')@patch.object(logger, 'info')- def test_download_ingredient_off(self, mock_logger, mock_from_json, mock_request):+ def test_download_ingredient_off(+ self,+ mock_logger,+ mock_from_json,+ mock_request,+ mock_download_image,+ ):"""Test that the image is correctly downloaded@@ -202,23 +207,21 @@ class FetchIngredientImageTestCase(WgerTestCase):):result = fetch_ingredient_image(1)+ # print(mock_request.mock_calls)+ mock_request.assert_called_with(+ '1234567890987654321',+ fields=['images', 'image_front_url'],+ )+ mock_download_image.assert_called_with(+ 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',+ return_struct=True,+ )mock_logger.assert_any_call('Fetching image for ingredient 1')mock_logger.assert_any_call(- 'Trying to fetch image from OFF for Test ingredient 1 (UUID: '- '7908c204-907f-4b1e-ad4e-f482e9769ade)'+ 'Trying to fetch image from OFF for "Test ingredient 1" '+ '(7908c204-907f-4b1e-ad4e-f482e9769ade)')mock_logger.assert_any_call('Image successfully saved')-- # print(mock_request.mock_calls)- mock_request.assert_any_call(- 'https://world.openfoodfacts.org/api/v2/product/5055365635003.json?fields=images,image_front_url',- headers=wger_headers(),- timeout=ANY,- )- mock_request.assert_any_call(- 'https://openfoodfacts-images.s3.eu-west-3.amazonaws.com/data/123/456/789/0987654321/12345.jpg',- headers=wger_headers(),- )mock_from_json.assert_called()self.assertEqual(result, None)