Self hosted FLOSS fitness/workout, nutrition and weight tracker
Some polishing, add tests for the USDA extraction function
Roland Geider1a0ab64681c6
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.pyindex c135f30cf..b190d5c6f 100644--- a/wger/nutrition/dataclasses.py+++ b/wger/nutrition/dataclasses.py@@ -28,9 +28,9 @@ class IngredientData:energy: floatprotein: floatcarbohydrates: float- carbohydrates_sugar: float+ carbohydrates_sugar: Optional[float]fat: float- fat_saturated: float+ fat_saturated: Optional[float]fibres: Optional[float]sodium: Optional[float]code: Optional[str]diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.pyindex ae0ad9272..d91dbc930 100644--- a/wger/nutrition/management/commands/import-usda-products.py+++ b/wger/nutrition/management/commands/import-usda-products.py@@ -31,6 +31,7 @@ from wger.nutrition.management.products import ()from wger.nutrition.usda import extract_info_from_usdafrom wger.utils.constants import ENGLISH_SHORT_NAME+from wger.utils.requests import wger_headerslogger = logging.getLogger(__name__)@@ -45,7 +46,9 @@ class Command(ImportProductCommand):if options['mode'] == 'insert':self.mode = Mode.INSERT- usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip'+ current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip'++ usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}'folder = '/Users/roland/Entwicklung/wger/server/extras/usda'self.stdout.write('Importing entries from USDA')@@ -55,12 +58,12 @@ class Command(ImportProductCommand):english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)- zip_file = os.path.join(folder, 'usda.zip')+ zip_file = os.path.join(folder, current_file)if os.path.exists(zip_file):self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')else:self.stdout.write(f'Downloading {zip_file}... (this may take a while)')- req = requests.get(usda_url, stream=True)+ req = requests.get(usda_url, stream=True, headers=wger_headers())with open(zip_file, 'wb') as fid:for chunk in req.iter_content(chunk_size=50 * 1024):fid.write(chunk)diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.pyindex aead23dd9..50965e549 100644--- a/wger/nutrition/management/products.py+++ b/wger/nutrition/management/products.py@@ -118,4 +118,5 @@ class ImportProductCommand(BaseCommand):except Exception as e:self.stdout.write('--> Error while performing update_or_create')self.stdout.write(repr(e))+ # self.stdout.write(repr(ingredient_data))self.counter['error'] += 1diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.pynew file mode 100644index 000000000..eac565e46--- /dev/null+++ b/wger/nutrition/tests/test_usda.py@@ -0,0 +1,282 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with Workout Manager. If not, see <http://www.gnu.org/licenses/>.++# Django+from django.test import SimpleTestCase++# wger+from wger.nutrition.dataclasses import IngredientData+from wger.nutrition.usda import (+ convert_to_g_per_100g,+ extract_info_from_usda,+)+from wger.utils.constants import CC_0_LICENSE_ID+from wger.utils.models import AbstractSubmissionModel+++class ExtractInfoFromUSDATestCase(SimpleTestCase):+ """+ Test the extract_info_from_usda function+ """++ usda_data1 = {}++ def setUp(self):+ self.usda_data1 = {+ 'foodClass': 'FinalFood',+ 'description': 'Foo with chocolate',+ 'foodNutrients': [+ {+ 'type': 'FoodNutrient',+ 'id': 2259514,+ 'nutrient': {+ 'id': 1170,+ 'number': '410',+ 'name': 'Pantothenic acid',+ 'rank': 6700,+ 'unitName': 'mg',+ },+ 'dataPoints': 2,+ 'foodNutrientDerivation': {+ 'code': 'A',+ 'description': 'Analytical',+ 'foodNutrientSource': {+ 'id': 1,+ 'code': '1',+ 'description': 'Analytical or derived from analytical',+ },+ },+ 'max': 1.64,+ 'min': 1.53,+ 'median': 1.58,+ 'amount': 1.58,+ },+ {+ 'type': 'FoodNutrient',+ 'id': 2259524,+ 'nutrient': {+ 'id': 1004,+ 'number': '204',+ 'name': 'Total lipid (fat)',+ 'rank': 800,+ 'unitName': 'g',+ },+ 'dataPoints': 6,+ 'foodNutrientDerivation': {+ 'code': 'A',+ 'description': 'Analytical',+ 'foodNutrientSource': {+ 'id': 1,+ 'code': '1',+ 'description': 'Analytical or derived from analytical',+ },+ },+ 'max': 3.99,+ 'min': 2.17,+ 'median': 3.26,+ 'amount': 3.24,+ },+ {+ 'type': 'FoodNutrient',+ 'id': 2259525,+ 'nutrient': {+ 'id': 1005,+ 'number': '205',+ 'name': 'Carbohydrate, by difference',+ 'rank': 1110,+ 'unitName': 'g',+ },+ 'foodNutrientDerivation': {+ 'code': 'NC',+ 'description': 'Calculated',+ 'foodNutrientSource': {+ 'id': 2,+ 'code': '4',+ 'description': 'Calculated or imputed',+ },+ },+ 'amount': 0.000,+ },+ {+ 'type': 'FoodNutrient',+ 'id': 2259526,+ 'nutrient': {+ 'id': 1008,+ 'number': '208',+ 'name': 'Energy',+ 'rank': 300,+ 'unitName': 'kcal',+ },+ 'foodNutrientDerivation': {+ 'code': 'NC',+ 'description': 'Calculated',+ 'foodNutrientSource': {+ 'id': 2,+ 'code': '4',+ 'description': 'Calculated or imputed',+ },+ },+ 'amount': 166,+ },+ {+ 'type': 'FoodNutrient',+ 'id': 2259565,+ 'nutrient': {+ 'id': 1003,+ 'number': '203',+ 'name': 'Protein',+ 'rank': 600,+ 'unitName': 'g',+ },+ 'foodNutrientDerivation': {+ 'code': 'NC',+ 'description': 'Calculated',+ 'foodNutrientSource': {+ 'id': 2,+ 'code': '4',+ 'description': 'Calculated or imputed',+ },+ },+ 'max': 32.9,+ 'min': 31.3,+ 'median': 32.1,+ 'amount': 32.1,+ },+ ],+ 'foodAttributes': [],+ 'nutrientConversionFactors': [+ {+ 'type': '.CalorieConversionFactor',+ 'proteinValue': 4.27,+ 'fatValue': 9.02,+ 'carbohydrateValue': 3.87,+ },+ {'type': '.ProteinConversionFactor', 'value': 6.25},+ ],+ 'isHistoricalReference': False,+ 'ndbNumber': 5746,+ 'foodPortions': [+ {+ 'id': 121343,+ 'value': 1.0,+ 'measureUnit': {'id': 1043, 'name': 'piece', 'abbreviation': 'piece'},+ 'gramWeight': 174.0,+ 'sequenceNumber': 1,+ 'minYearAcquired': 2012,+ 'amount': 1.0,+ }+ ],+ 'publicationDate': '4/1/2019',+ 'inputFoods': [+ {+ 'id': 11213,+ 'foodDescription': 'Lorem ipsum',+ 'inputFood': {+ 'foodClass': 'Composite',+ 'description': '......',+ 'publicationDate': '4/1/2019',+ 'foodCategory': {+ 'id': 5,+ 'code': '0500',+ 'description': 'Poultry Products',+ },+ 'fdcId': 331904,+ 'dataType': 'Sample',+ },+ },+ ],+ 'foodCategory': {'description': 'Poultry Products'},+ 'fdcId': 1234567,+ 'dataType': 'Foundation',+ }++ def test_regular_response(self):+ """+ Test that the function can read the regular case+ """++ result = extract_info_from_usda(self.usda_data1, 1)+ data = IngredientData(+ name='Foo with chocolate',+ remote_id='1234567',+ language_id=1,+ energy=166.0,+ protein=32.1,+ carbohydrates=3.24,+ carbohydrates_sugar=None,+ fat=0.0,+ fat_saturated=None,+ fibres=None,+ sodium=None,+ code=None,+ source_name='USDA',+ source_url='https://fdc.nal.usda.gov/',+ common_name='Foo with chocolate',+ brand='',+ status=AbstractSubmissionModel.STATUS_ACCEPTED,+ license_id=CC_0_LICENSE_ID,+ license_author='U.S. Department of Agriculture, Agricultural Research Service, '+ 'Beltsville Human Nutrition Research Center. FoodData Central.',+ license_title='Foo with chocolate',+ license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients',+ )++ self.assertEqual(result, data)++ def test_no_energy(self):+ """+ No energy available+ """+ del self.usda_data1['foodNutrients'][3]+ self.assertRaises(KeyError, extract_info_from_usda, self.usda_data1, 1)++ def test_no_nutrients(self):+ """+ No nutrients available+ """+ del self.usda_data1['foodNutrients']+ self.assertRaises(KeyError, extract_info_from_usda, self.usda_data1, 1)++ def test_converting_grams(self):+ """+ Convert from grams (nothing changes)+ """++ entry = {'nutrient': {'unitName': 'g'}, 'amount': '5.0'}+ self.assertEqual(convert_to_g_per_100g(entry), 5.0)++ def test_converting_milligrams(self):+ """+ Convert from milligrams+ """++ entry = {'nutrient': {'unitName': 'mg'}, 'amount': '5000'}+ self.assertEqual(convert_to_g_per_100g(entry), 5.0)++ def test_converting_unknown_unit(self):+ """+ Convert from unknown unit+ """++ entry = {'nutrient': {'unitName': 'kg'}, 'amount': '5.0'}+ self.assertRaises(ValueError, convert_to_g_per_100g, entry)++ def test_converting_invalid_amount(self):+ """+ Convert from invalid amount+ """++ entry = {'nutrient': {'unitName': 'g'}, 'amount': 'invalid'}+ self.assertRaises(ValueError, convert_to_g_per_100g, entry)diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.pyindex cb5c93e86..ee67832ab 100644--- a/wger/nutrition/usda.py+++ b/wger/nutrition/usda.py@@ -20,53 +20,74 @@ from wger.utils.constants import CC_0_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel+def convert_to_g_per_100g(entry: dict) -> float:+ """+ Convert a nutrient entry to grams per 100g+ """++ nutrient = entry['nutrient']+ amount = float(entry['amount'])++ if nutrient['unitName'] == 'g':+ return amount++ elif nutrient['unitName'] == 'mg':+ return amount / 1000++ else:+ raise ValueError(f'Unknown unit: {nutrient["unitName"]}')++def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:if not product_data.get('foodNutrients'):raise KeyError('Missing key "foodNutrients"')- energy = 0- protein = 0- carbs = 0- fats = 0- sugars = 0- saturated = 0- fat = 0+ energy = None+ protein = None+ carbs = None+ sugars = None+ fats = None+ fats_saturated = Nonesodium = Nonefibre = None- for d in product_data['foodNutrients']:- if not d.get('nutrient'):+ for nutrient in product_data['foodNutrients']:+ if not nutrient.get('nutrient'):raise KeyError('Missing key "nutrient"')- nutrient = d.get('nutrient')- nutrient_id = nutrient.get('id')-+ nutrient_id = nutrient['nutrient']['id']match nutrient_id:+ case 1008:+ energy = float(nutrient.get('amount'))+case 1003:- protein = float(d.get('amount'))+ protein = convert_to_g_per_100g(nutrient)case 1004:- carbs = float(d.get('amount'))+ carbs = convert_to_g_per_100g(nutrient)case 1005:- fats = float(d.get('amount'))-- case 2048:- energy = float(d.get('amount'))+ fats = convert_to_g_per_100g(nutrient)+ # These are optionalcase 1093:- sodium = float(d.get('amount'))+ sodium = convert_to_g_per_100g(nutrient)++ case 1079:+ fibre = convert_to_g_per_100g(nutrient)- if not all([protein, carbs, fats, energy]):- raise KeyError('Could not extract all nutrition information')+ macros = [energy, protein, carbs, fats]+ for value in macros:+ if value is None:+ raise KeyError(f'Could not extract all basic macros: {macros=}')name = product_data['description']if len(name) > 200:name = name[:200]code = None- remote_id = product_data['fdcId']+ remote_id = str(product_data['fdcId'])brand = ''# License and author info@@ -76,7 +97,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:'U.S. Department of Agriculture, Agricultural Research Service, ''Beltsville Human Nutrition Research Center. FoodData Central.')- object_url = f'https://api.nal.usda.gov/fdc/v1/food/{remote_id}'+ object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients'return IngredientData(code=code,@@ -88,8 +109,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:protein=protein,carbohydrates=carbs,carbohydrates_sugar=sugars,- fat=fat,- fat_saturated=saturated,+ fat=fats,+ fat_saturated=fats_saturated,fibres=fibre,sodium=sodium,source_name=source_name,diff --git a/wger/utils/requests.py b/wger/utils/requests.pyindex d59020e63..aec1fd6f6 100644--- a/wger/utils/requests.py+++ b/wger/utils/requests.py@@ -24,7 +24,7 @@ def wger_user_agent():def wger_headers():- return {'User-agent': wger_user_agent()}+ return {'User-Agent': wger_user_agent()}def get_all_paginated(url: str, headers=None):