Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Better handling branded products

Roland Geider248f28bafd7f
diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py
index d91dbc930..de623380a 100644
--- a/wger/nutrition/management/commands/import-usda-products.py
+++ b/wger/nutrition/management/commands/import-usda-products.py
@@ -33,9 +33,13 @@ from wger.nutrition.usda import extract_info_from_usda
from wger.utils.constants import ENGLISH_SHORT_NAME
from wger.utils.requests import wger_headers
-
logger = logging.getLogger(__name__)
+# Check https://fdc.nal.usda.gov/download-datasets.html for current file names
+# current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip'
+current_file = 'FoodData_Central_branded_food_json_2024-04-18.zip'
+download_folder = '/Users/roland/Entwicklung/wger/server/extras/usda'
+
class Command(ImportProductCommand):
"""
@@ -46,19 +50,18 @@ class Command(ImportProductCommand):
if options['mode'] == 'insert':
self.mode = Mode.INSERT
- current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip'
-
usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}'
- folder = '/Users/roland/Entwicklung/wger/server/extras/usda'
self.stdout.write('Importing entries from USDA')
self.stdout.write(f' - {self.mode}')
- self.stdout.write(f' - {folder=}')
+ self.stdout.write(f' - dataset: {usda_url}')
+ self.stdout.write(f' - download folder: {download_folder}')
self.stdout.write('')
english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)
- zip_file = os.path.join(folder, current_file)
+ # Download the dataset
+ zip_file = os.path.join(download_folder, current_file)
if os.path.exists(zip_file):
self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')
else:
@@ -70,20 +73,21 @@ class Command(ImportProductCommand):
self.stdout.write('download successful')
+ # Extract the first file from the ZIP archive
with ZipFile(zip_file, 'r') as zip_ref:
file_list = zip_ref.namelist()
if not file_list:
raise Exception('No files found in the ZIP archive')
first_file = file_list[0]
- self.stdout.write(f'Extracting {first_file=}')
- extracted_file_path = zip_ref.extract(first_file, path=folder)
+ self.stdout.write(f'Extracting {first_file}...')
+ extracted_file_path = zip_ref.extract(first_file, path=download_folder)
# Since the file is almost JSONL, just process each line individually
with open(extracted_file_path, 'r') as extracted_file:
for line in extracted_file.readlines():
- # Skip the first and last lines in the file
- if 'FoundationFoods' in line:
+ # Try to skip the first and last lines in the file
+ if 'FoundationFoods' in line or 'BrandedFoods' in line:
continue
if line.strip() == '}':
@@ -98,9 +102,10 @@ class Command(ImportProductCommand):
try:
ingredient_data = extract_info_from_usda(json_data, english.pk)
except KeyError as e:
- self.stdout.write(f'--> KeyError while extracting info from USDA: {e}')
+ self.stdout.write(f'--> KeyError while extracting ingredient info: {e}')
self.counter['skipped'] += 1
else:
+ # pass
self.handle_data(ingredient_data)
self.stdout.write(self.style.SUCCESS('Finished!'))
diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.py
index eac565e46..746228b0d 100644
--- a/wger/nutrition/tests/test_usda.py
+++ b/wger/nutrition/tests/test_usda.py
@@ -36,7 +36,7 @@ class ExtractInfoFromUSDATestCase(SimpleTestCase):
def setUp(self):
self.usda_data1 = {
'foodClass': 'FinalFood',
- 'description': 'Foo with chocolate',
+ 'description': 'FOO WITH CHOCOLATE',
'foodNutrients': [
{
'type': 'FoodNutrient',
@@ -209,27 +209,27 @@ class ExtractInfoFromUSDATestCase(SimpleTestCase):
result = extract_info_from_usda(self.usda_data1, 1)
data = IngredientData(
- name='Foo with chocolate',
+ name='Foo With Chocolate',
remote_id='1234567',
language_id=1,
energy=166.0,
protein=32.1,
- carbohydrates=3.24,
+ carbohydrates=0.0,
carbohydrates_sugar=None,
- fat=0.0,
+ fat=3.24,
fat_saturated=None,
fibres=None,
sodium=None,
code=None,
source_name='USDA',
source_url='https://fdc.nal.usda.gov/',
- common_name='Foo with chocolate',
+ common_name='Foo With Chocolate',
brand='',
status=AbstractSubmissionModel.STATUS_ACCEPTED,
license_id=CC_0_LICENSE_ID,
license_author='U.S. Department of Agriculture, Agricultural Research Service, '
'Beltsville Human Nutrition Research Center. FoodData Central.',
- license_title='Foo with chocolate',
+ license_title='Foo With Chocolate',
license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients',
)
diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py
index ee67832ab..a5b01d816 100644
--- a/wger/nutrition/usda.py
+++ b/wger/nutrition/usda.py
@@ -42,6 +42,9 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
if not product_data.get('foodNutrients'):
raise KeyError('Missing key "foodNutrients"')
+ remote_id = str(product_data['fdcId'])
+ barcode = None
+
energy = None
protein = None
carbs = None
@@ -52,6 +55,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
sodium = None
fibre = None
+ brand = product_data.get('brandName', '')
+
for nutrient in product_data['foodNutrients']:
if not nutrient.get('nutrient'):
raise KeyError('Missing key "nutrient"')
@@ -59,18 +64,18 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
nutrient_id = nutrient['nutrient']['id']
match nutrient_id:
case 1008:
- energy = float(nutrient.get('amount'))
+ energy = float(nutrient.get('amount')) # in kcal
case 1003:
protein = convert_to_g_per_100g(nutrient)
- case 1004:
+ case 1005:
carbs = convert_to_g_per_100g(nutrient)
- case 1005:
+ # Total lipid (fat) | Total fat (NLEA)
+ case 1004 | 1085:
fats = convert_to_g_per_100g(nutrient)
- # These are optional
case 1093:
sodium = convert_to_g_per_100g(nutrient)
@@ -80,16 +85,12 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
macros = [energy, protein, carbs, fats]
for value in macros:
if value is None:
- raise KeyError(f'Could not extract all basic macros: {macros=}')
+ raise KeyError(f'Could not extract all basic macros: {macros=} {remote_id=}')
- name = product_data['description']
+ name = product_data['description'].title()
if len(name) > 200:
name = name[:200]
- code = None
- remote_id = str(product_data['fdcId'])
- brand = ''
-
# License and author info
source_name = Source.USDA.value
source_url = f'https://fdc.nal.usda.gov/'
@@ -100,7 +101,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients'
return IngredientData(
- code=code,
+ code=barcode,
remote_id=remote_id,
name=name,
common_name=name,