Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Some polishing, add tests for the USDA extraction function

Roland Geider1a0ab64681c6
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py
index c135f30cf..b190d5c6f 100644
--- a/wger/nutrition/dataclasses.py
+++ b/wger/nutrition/dataclasses.py
@@ -28,9 +28,9 @@ class IngredientData:
energy: float
protein: float
carbohydrates: float
- carbohydrates_sugar: float
+ carbohydrates_sugar: Optional[float]
fat: float
- fat_saturated: float
+ fat_saturated: Optional[float]
fibres: Optional[float]
sodium: Optional[float]
code: Optional[str]
diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py
index ae0ad9272..d91dbc930 100644
--- a/wger/nutrition/management/commands/import-usda-products.py
+++ b/wger/nutrition/management/commands/import-usda-products.py
@@ -31,6 +31,7 @@ from wger.nutrition.management.products import (
)
from wger.nutrition.usda import extract_info_from_usda
from wger.utils.constants import ENGLISH_SHORT_NAME
+from wger.utils.requests import wger_headers
logger = logging.getLogger(__name__)
@@ -45,7 +46,9 @@ class Command(ImportProductCommand):
if options['mode'] == 'insert':
self.mode = Mode.INSERT
- usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip'
+ current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip'
+
+ usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}'
folder = '/Users/roland/Entwicklung/wger/server/extras/usda'
self.stdout.write('Importing entries from USDA')
@@ -55,12 +58,12 @@ class Command(ImportProductCommand):
english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)
- zip_file = os.path.join(folder, 'usda.zip')
+ zip_file = os.path.join(folder, current_file)
if os.path.exists(zip_file):
self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')
else:
self.stdout.write(f'Downloading {zip_file}... (this may take a while)')
- req = requests.get(usda_url, stream=True)
+ req = requests.get(usda_url, stream=True, headers=wger_headers())
with open(zip_file, 'wb') as fid:
for chunk in req.iter_content(chunk_size=50 * 1024):
fid.write(chunk)
diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py
index aead23dd9..50965e549 100644
--- a/wger/nutrition/management/products.py
+++ b/wger/nutrition/management/products.py
@@ -118,4 +118,5 @@ class ImportProductCommand(BaseCommand):
except Exception as e:
self.stdout.write('--> Error while performing update_or_create')
self.stdout.write(repr(e))
+ # self.stdout.write(repr(ingredient_data))
self.counter['error'] += 1
diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.py
new file mode 100644
index 000000000..eac565e46
--- /dev/null
+++ b/wger/nutrition/tests/test_usda.py
@@ -0,0 +1,282 @@
+# This file is part of wger Workout Manager.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Workout Manager. If not, see <http://www.gnu.org/licenses/>.
+
+# Django
+from django.test import SimpleTestCase
+
+# wger
+from wger.nutrition.dataclasses import IngredientData
+from wger.nutrition.usda import (
+ convert_to_g_per_100g,
+ extract_info_from_usda,
+)
+from wger.utils.constants import CC_0_LICENSE_ID
+from wger.utils.models import AbstractSubmissionModel
+
+
+class ExtractInfoFromUSDATestCase(SimpleTestCase):
+ """
+ Test the extract_info_from_usda function
+ """
+
+ usda_data1 = {}
+
+ def setUp(self):
+ self.usda_data1 = {
+ 'foodClass': 'FinalFood',
+ 'description': 'Foo with chocolate',
+ 'foodNutrients': [
+ {
+ 'type': 'FoodNutrient',
+ 'id': 2259514,
+ 'nutrient': {
+ 'id': 1170,
+ 'number': '410',
+ 'name': 'Pantothenic acid',
+ 'rank': 6700,
+ 'unitName': 'mg',
+ },
+ 'dataPoints': 2,
+ 'foodNutrientDerivation': {
+ 'code': 'A',
+ 'description': 'Analytical',
+ 'foodNutrientSource': {
+ 'id': 1,
+ 'code': '1',
+ 'description': 'Analytical or derived from analytical',
+ },
+ },
+ 'max': 1.64,
+ 'min': 1.53,
+ 'median': 1.58,
+ 'amount': 1.58,
+ },
+ {
+ 'type': 'FoodNutrient',
+ 'id': 2259524,
+ 'nutrient': {
+ 'id': 1004,
+ 'number': '204',
+ 'name': 'Total lipid (fat)',
+ 'rank': 800,
+ 'unitName': 'g',
+ },
+ 'dataPoints': 6,
+ 'foodNutrientDerivation': {
+ 'code': 'A',
+ 'description': 'Analytical',
+ 'foodNutrientSource': {
+ 'id': 1,
+ 'code': '1',
+ 'description': 'Analytical or derived from analytical',
+ },
+ },
+ 'max': 3.99,
+ 'min': 2.17,
+ 'median': 3.26,
+ 'amount': 3.24,
+ },
+ {
+ 'type': 'FoodNutrient',
+ 'id': 2259525,
+ 'nutrient': {
+ 'id': 1005,
+ 'number': '205',
+ 'name': 'Carbohydrate, by difference',
+ 'rank': 1110,
+ 'unitName': 'g',
+ },
+ 'foodNutrientDerivation': {
+ 'code': 'NC',
+ 'description': 'Calculated',
+ 'foodNutrientSource': {
+ 'id': 2,
+ 'code': '4',
+ 'description': 'Calculated or imputed',
+ },
+ },
+ 'amount': 0.000,
+ },
+ {
+ 'type': 'FoodNutrient',
+ 'id': 2259526,
+ 'nutrient': {
+ 'id': 1008,
+ 'number': '208',
+ 'name': 'Energy',
+ 'rank': 300,
+ 'unitName': 'kcal',
+ },
+ 'foodNutrientDerivation': {
+ 'code': 'NC',
+ 'description': 'Calculated',
+ 'foodNutrientSource': {
+ 'id': 2,
+ 'code': '4',
+ 'description': 'Calculated or imputed',
+ },
+ },
+ 'amount': 166,
+ },
+ {
+ 'type': 'FoodNutrient',
+ 'id': 2259565,
+ 'nutrient': {
+ 'id': 1003,
+ 'number': '203',
+ 'name': 'Protein',
+ 'rank': 600,
+ 'unitName': 'g',
+ },
+ 'foodNutrientDerivation': {
+ 'code': 'NC',
+ 'description': 'Calculated',
+ 'foodNutrientSource': {
+ 'id': 2,
+ 'code': '4',
+ 'description': 'Calculated or imputed',
+ },
+ },
+ 'max': 32.9,
+ 'min': 31.3,
+ 'median': 32.1,
+ 'amount': 32.1,
+ },
+ ],
+ 'foodAttributes': [],
+ 'nutrientConversionFactors': [
+ {
+ 'type': '.CalorieConversionFactor',
+ 'proteinValue': 4.27,
+ 'fatValue': 9.02,
+ 'carbohydrateValue': 3.87,
+ },
+ {'type': '.ProteinConversionFactor', 'value': 6.25},
+ ],
+ 'isHistoricalReference': False,
+ 'ndbNumber': 5746,
+ 'foodPortions': [
+ {
+ 'id': 121343,
+ 'value': 1.0,
+ 'measureUnit': {'id': 1043, 'name': 'piece', 'abbreviation': 'piece'},
+ 'gramWeight': 174.0,
+ 'sequenceNumber': 1,
+ 'minYearAcquired': 2012,
+ 'amount': 1.0,
+ }
+ ],
+ 'publicationDate': '4/1/2019',
+ 'inputFoods': [
+ {
+ 'id': 11213,
+ 'foodDescription': 'Lorem ipsum',
+ 'inputFood': {
+ 'foodClass': 'Composite',
+ 'description': '......',
+ 'publicationDate': '4/1/2019',
+ 'foodCategory': {
+ 'id': 5,
+ 'code': '0500',
+ 'description': 'Poultry Products',
+ },
+ 'fdcId': 331904,
+ 'dataType': 'Sample',
+ },
+ },
+ ],
+ 'foodCategory': {'description': 'Poultry Products'},
+ 'fdcId': 1234567,
+ 'dataType': 'Foundation',
+ }
+
+ def test_regular_response(self):
+ """
+ Test that the function can read the regular case
+ """
+
+ result = extract_info_from_usda(self.usda_data1, 1)
+ data = IngredientData(
+ name='Foo with chocolate',
+ remote_id='1234567',
+ language_id=1,
+ energy=166.0,
+ protein=32.1,
+ carbohydrates=3.24,
+ carbohydrates_sugar=None,
+ fat=0.0,
+ fat_saturated=None,
+ fibres=None,
+ sodium=None,
+ code=None,
+ source_name='USDA',
+ source_url='https://fdc.nal.usda.gov/',
+ common_name='Foo with chocolate',
+ brand='',
+ status=AbstractSubmissionModel.STATUS_ACCEPTED,
+ license_id=CC_0_LICENSE_ID,
+ license_author='U.S. Department of Agriculture, Agricultural Research Service, '
+ 'Beltsville Human Nutrition Research Center. FoodData Central.',
+ license_title='Foo with chocolate',
+ license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients',
+ )
+
+ self.assertEqual(result, data)
+
+ def test_no_energy(self):
+ """
+ No energy available
+ """
+ del self.usda_data1['foodNutrients'][3]
+ self.assertRaises(KeyError, extract_info_from_usda, self.usda_data1, 1)
+
+ def test_no_nutrients(self):
+ """
+ No nutrients available
+ """
+ del self.usda_data1['foodNutrients']
+ self.assertRaises(KeyError, extract_info_from_usda, self.usda_data1, 1)
+
+ def test_converting_grams(self):
+ """
+ Convert from grams (nothing changes)
+ """
+
+ entry = {'nutrient': {'unitName': 'g'}, 'amount': '5.0'}
+ self.assertEqual(convert_to_g_per_100g(entry), 5.0)
+
+ def test_converting_milligrams(self):
+ """
+ Convert from milligrams
+ """
+
+ entry = {'nutrient': {'unitName': 'mg'}, 'amount': '5000'}
+ self.assertEqual(convert_to_g_per_100g(entry), 5.0)
+
+ def test_converting_unknown_unit(self):
+ """
+ Convert from unknown unit
+ """
+
+ entry = {'nutrient': {'unitName': 'kg'}, 'amount': '5.0'}
+ self.assertRaises(ValueError, convert_to_g_per_100g, entry)
+
+ def test_converting_invalid_amount(self):
+ """
+ Convert from invalid amount
+ """
+
+ entry = {'nutrient': {'unitName': 'g'}, 'amount': 'invalid'}
+ self.assertRaises(ValueError, convert_to_g_per_100g, entry)
diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py
index cb5c93e86..ee67832ab 100644
--- a/wger/nutrition/usda.py
+++ b/wger/nutrition/usda.py
@@ -20,53 +20,74 @@ from wger.utils.constants import CC_0_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel
+def convert_to_g_per_100g(entry: dict) -> float:
+ """
+ Convert a nutrient entry to grams per 100g
+ """
+
+ nutrient = entry['nutrient']
+ amount = float(entry['amount'])
+
+ if nutrient['unitName'] == 'g':
+ return amount
+
+ elif nutrient['unitName'] == 'mg':
+ return amount / 1000
+
+ else:
+ raise ValueError(f'Unknown unit: {nutrient["unitName"]}')
+
+
def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
if not product_data.get('foodNutrients'):
raise KeyError('Missing key "foodNutrients"')
- energy = 0
- protein = 0
- carbs = 0
- fats = 0
- sugars = 0
- saturated = 0
- fat = 0
+ energy = None
+ protein = None
+ carbs = None
+ sugars = None
+ fats = None
+ fats_saturated = None
sodium = None
fibre = None
- for d in product_data['foodNutrients']:
- if not d.get('nutrient'):
+ for nutrient in product_data['foodNutrients']:
+ if not nutrient.get('nutrient'):
raise KeyError('Missing key "nutrient"')
- nutrient = d.get('nutrient')
- nutrient_id = nutrient.get('id')
-
+ nutrient_id = nutrient['nutrient']['id']
match nutrient_id:
+ case 1008:
+ energy = float(nutrient.get('amount'))
+
case 1003:
- protein = float(d.get('amount'))
+ protein = convert_to_g_per_100g(nutrient)
case 1004:
- carbs = float(d.get('amount'))
+ carbs = convert_to_g_per_100g(nutrient)
case 1005:
- fats = float(d.get('amount'))
-
- case 2048:
- energy = float(d.get('amount'))
+ fats = convert_to_g_per_100g(nutrient)
+ # These are optional
case 1093:
- sodium = float(d.get('amount'))
+ sodium = convert_to_g_per_100g(nutrient)
+
+ case 1079:
+ fibre = convert_to_g_per_100g(nutrient)
- if not all([protein, carbs, fats, energy]):
- raise KeyError('Could not extract all nutrition information')
+ macros = [energy, protein, carbs, fats]
+ for value in macros:
+ if value is None:
+ raise KeyError(f'Could not extract all basic macros: {macros=}')
name = product_data['description']
if len(name) > 200:
name = name[:200]
code = None
- remote_id = product_data['fdcId']
+ remote_id = str(product_data['fdcId'])
brand = ''
# License and author info
@@ -76,7 +97,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
'U.S. Department of Agriculture, Agricultural Research Service, '
'Beltsville Human Nutrition Research Center. FoodData Central.'
)
- object_url = f'https://api.nal.usda.gov/fdc/v1/food/{remote_id}'
+ object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients'
return IngredientData(
code=code,
@@ -88,8 +109,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
protein=protein,
carbohydrates=carbs,
carbohydrates_sugar=sugars,
- fat=fat,
- fat_saturated=saturated,
+ fat=fats,
+ fat_saturated=fats_saturated,
fibres=fibre,
sodium=sodium,
source_name=source_name,
diff --git a/wger/utils/requests.py b/wger/utils/requests.py
index d59020e63..aec1fd6f6 100644
--- a/wger/utils/requests.py
+++ b/wger/utils/requests.py
@@ -24,7 +24,7 @@ def wger_user_agent():
def wger_headers():
- return {'User-agent': wger_user_agent()}
+ return {'User-Agent': wger_user_agent()}
def get_all_paginated(url: str, headers=None):