Self hosted FLOSS fitness/workout, nutrition and weight tracker
Better handling branded products
Roland Geider248f28bafd7f
diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.pyindex d91dbc930..de623380a 100644--- a/wger/nutrition/management/commands/import-usda-products.py+++ b/wger/nutrition/management/commands/import-usda-products.py@@ -33,9 +33,13 @@ from wger.nutrition.usda import extract_info_from_usdafrom wger.utils.constants import ENGLISH_SHORT_NAMEfrom wger.utils.requests import wger_headers-logger = logging.getLogger(__name__)+# Check https://fdc.nal.usda.gov/download-datasets.html for current file names+# current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip'+current_file = 'FoodData_Central_branded_food_json_2024-04-18.zip'+download_folder = '/Users/roland/Entwicklung/wger/server/extras/usda'+class Command(ImportProductCommand):"""@@ -46,19 +50,18 @@ class Command(ImportProductCommand):if options['mode'] == 'insert':self.mode = Mode.INSERT- current_file = 'FoodData_Central_foundation_food_json_2024-04-18.zip'-usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{current_file}'- folder = '/Users/roland/Entwicklung/wger/server/extras/usda'self.stdout.write('Importing entries from USDA')self.stdout.write(f' - {self.mode}')- self.stdout.write(f' - {folder=}')+ self.stdout.write(f' - dataset: {usda_url}')+ self.stdout.write(f' - download folder: {download_folder}')self.stdout.write('')english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)- zip_file = os.path.join(folder, current_file)+ # Download the dataset+ zip_file = os.path.join(download_folder, current_file)if os.path.exists(zip_file):self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')else:@@ -70,20 +73,21 @@ class Command(ImportProductCommand):self.stdout.write('download successful')+ # Extract the first file from the ZIP archivewith ZipFile(zip_file, 'r') as zip_ref:file_list = zip_ref.namelist()if not file_list:raise Exception('No files found in the ZIP archive')first_file = file_list[0]- self.stdout.write(f'Extracting {first_file=}')- extracted_file_path = zip_ref.extract(first_file, path=folder)+ self.stdout.write(f'Extracting {first_file}...')+ extracted_file_path = zip_ref.extract(first_file, path=download_folder)# Since the file is almost JSONL, just process each line individuallywith open(extracted_file_path, 'r') as extracted_file:for line in extracted_file.readlines():- # Skip the first and last lines in the file- if 'FoundationFoods' in line:+ # Try to skip the first and last lines in the file+ if 'FoundationFoods' in line or 'BrandedFoods' in line:continueif line.strip() == '}':@@ -98,9 +102,10 @@ class Command(ImportProductCommand):try:ingredient_data = extract_info_from_usda(json_data, english.pk)except KeyError as e:- self.stdout.write(f'--> KeyError while extracting info from USDA: {e}')+ self.stdout.write(f'--> KeyError while extracting ingredient info: {e}')self.counter['skipped'] += 1else:+ # passself.handle_data(ingredient_data)self.stdout.write(self.style.SUCCESS('Finished!'))diff --git a/wger/nutrition/tests/test_usda.py b/wger/nutrition/tests/test_usda.pyindex eac565e46..746228b0d 100644--- a/wger/nutrition/tests/test_usda.py+++ b/wger/nutrition/tests/test_usda.py@@ -36,7 +36,7 @@ class ExtractInfoFromUSDATestCase(SimpleTestCase):def setUp(self):self.usda_data1 = {'foodClass': 'FinalFood',- 'description': 'Foo with chocolate',+ 'description': 'FOO WITH CHOCOLATE','foodNutrients': [{'type': 'FoodNutrient',@@ -209,27 +209,27 @@ class ExtractInfoFromUSDATestCase(SimpleTestCase):result = extract_info_from_usda(self.usda_data1, 1)data = IngredientData(- name='Foo with chocolate',+ name='Foo With Chocolate',remote_id='1234567',language_id=1,energy=166.0,protein=32.1,- carbohydrates=3.24,+ carbohydrates=0.0,carbohydrates_sugar=None,- fat=0.0,+ fat=3.24,fat_saturated=None,fibres=None,sodium=None,code=None,source_name='USDA',source_url='https://fdc.nal.usda.gov/',- common_name='Foo with chocolate',+ common_name='Foo With Chocolate',brand='',status=AbstractSubmissionModel.STATUS_ACCEPTED,license_id=CC_0_LICENSE_ID,license_author='U.S. Department of Agriculture, Agricultural Research Service, ''Beltsville Human Nutrition Research Center. FoodData Central.',- license_title='Foo with chocolate',+ license_title='Foo With Chocolate',license_object_url='https://fdc.nal.usda.gov/fdc-app.html#/food-details/1234567/nutrients',)diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.pyindex ee67832ab..a5b01d816 100644--- a/wger/nutrition/usda.py+++ b/wger/nutrition/usda.py@@ -42,6 +42,9 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:if not product_data.get('foodNutrients'):raise KeyError('Missing key "foodNutrients"')+ remote_id = str(product_data['fdcId'])+ barcode = None+energy = Noneprotein = Nonecarbs = None@@ -52,6 +55,8 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:sodium = Nonefibre = None+ brand = product_data.get('brandName', '')+for nutrient in product_data['foodNutrients']:if not nutrient.get('nutrient'):raise KeyError('Missing key "nutrient"')@@ -59,18 +64,18 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:nutrient_id = nutrient['nutrient']['id']match nutrient_id:case 1008:- energy = float(nutrient.get('amount'))+ energy = float(nutrient.get('amount')) # in kcalcase 1003:protein = convert_to_g_per_100g(nutrient)- case 1004:+ case 1005:carbs = convert_to_g_per_100g(nutrient)- case 1005:+ # Total lipid (fat) | Total fat (NLEA)+ case 1004 | 1085:fats = convert_to_g_per_100g(nutrient)- # These are optionalcase 1093:sodium = convert_to_g_per_100g(nutrient)@@ -80,16 +85,12 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:macros = [energy, protein, carbs, fats]for value in macros:if value is None:- raise KeyError(f'Could not extract all basic macros: {macros=}')+ raise KeyError(f'Could not extract all basic macros: {macros=} {remote_id=}')- name = product_data['description']+ name = product_data['description'].title()if len(name) > 200:name = name[:200]- code = None- remote_id = str(product_data['fdcId'])- brand = ''-# License and author infosource_name = Source.USDA.valuesource_url = f'https://fdc.nal.usda.gov/'@@ -100,7 +101,7 @@ def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:object_url = f'https://fdc.nal.usda.gov/fdc-app.html#/food-details/{remote_id}/nutrients'return IngredientData(- code=code,+ code=barcode,remote_id=remote_id,name=name,common_name=name,