Self hosted FLOSS fitness/workout, nutrition and weight tracker
Make USDA food import actually import the products
Roland Geiderd5a796eb3b86
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.pyindex b5c443c4e..c135f30cf 100644--- a/wger/nutrition/dataclasses.py+++ b/wger/nutrition/dataclasses.py@@ -12,7 +12,11 @@## You should have received a copy of the GNU Affero General Public License-from dataclasses import dataclass, asdict+# Standard Library+from dataclasses import (+ asdict,+ dataclass,+)from typing import Optional@@ -29,7 +33,7 @@ class IngredientData:fat_saturated: floatfibres: Optional[float]sodium: Optional[float]- code: str+ code: Optional[str]source_name: strsource_url: strcommon_name: strdiff --git a/wger/nutrition/management/__init__.py b/wger/nutrition/management/__init__.pynew file mode 100644index 000000000..1292caeab--- /dev/null+++ b/wger/nutrition/management/__init__.py@@ -0,0 +1,15 @@+# This file is part of wger Workout Manager <https://github.com/wger-project>.+# Copyright (C) 2013 - 2021 wger Team+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <http://www.gnu.org/licenses/>.diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.pyindex 17857211f..c40d9486c 100644--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@ -14,15 +14,16 @@# Standard Libraryimport logging-from collections import Counter--# Django-from wger.nutrition.management.commands.products import ImportProductCommand, Mode# wgerfrom wger.core.models import Language+from wger.nutrition.management.products import (+ ImportProductCommand,+ Mode,+)from wger.nutrition.off import extract_info_from_off+logger = logging.getLogger(__name__)@@ -31,8 +32,6 @@ class Command(ImportProductCommand):Import an Open Food facts Dump"""- completeness = 0.7-help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'def handle(self, **options):@@ -43,8 +42,6 @@ class Command(ImportProductCommand):self.stdout.write('Please install pymongo, `pip install pymongo`')return- self.counter = Counter()-if options['mode'] == 'insert':self.mode = Mode.INSERT@@ -55,7 +52,7 @@ class Command(ImportProductCommand):client = MongoClient('mongodb://off:[email protected]', port=27017)db = client.admin- languages = {l.short_name: l.pk for l in Language.objects.all()}+ languages = {lang.short_name: lang.pk for lang in Language.objects.all()}for product in db.products.find({'lang': {'$in': list(languages.keys())}}):try:@@ -65,9 +62,8 @@ class Command(ImportProductCommand):# self.stdout.write(repr(e))# pprint(product)self.counter['skipped'] += 1- continue-- self.handle_data(ingredient_data)+ else:+ self.handle_data(ingredient_data)self.stdout.write(self.style.SUCCESS('Finished!'))self.stdout.write(self.style.SUCCESS(str(self.counter)))diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.pyindex d59b11ea0..ae0ad9272 100644--- a/wger/nutrition/management/commands/import-usda-products.py+++ b/wger/nutrition/management/commands/import-usda-products.py@@ -13,7 +13,6 @@# You should have received a copy of the GNU Affero General Public License# Standard Library-import enumimport jsonimport loggingimport os@@ -21,131 +20,85 @@ import tempfilefrom json import JSONDecodeErrorfrom zipfile import ZipFile+# Third Partyimport requests-# Django-from django.core.management.base import BaseCommand--logger = logging.getLogger(__name__)+# wger+from wger.core.models import Language+from wger.nutrition.management.products import (+ ImportProductCommand,+ Mode,+)+from wger.nutrition.usda import extract_info_from_usda+from wger.utils.constants import ENGLISH_SHORT_NAME-# Mode for this script. When using 'insert', the script will bulk-insert the new-# ingredients, which is very efficient. Importing the whole database will require-# barely a minute. When using 'update', existing ingredients will be updated, which-# requires two queries per product and is needed when there are already existing-# entries in the local ingredient table.-class Mode(enum.Enum):- INSERT = enum.auto()- UPDATE = enum.auto()+logger = logging.getLogger(__name__)-class Command(BaseCommand):+class Command(ImportProductCommand):"""Import an Open Food facts Dump"""- mode = Mode.UPDATE-- def add_arguments(self, parser):- parser.add_argument(- '--set-mode',- action='store',- default=10,- dest='mode',- type=str,- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '- 'entries in the database, while update will try to update them if they are '- 'already present. Deault: insert',- )-def handle(self, **options):if options['mode'] == 'insert':self.mode = Mode.INSERT+ usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip'+ folder = '/Users/roland/Entwicklung/wger/server/extras/usda'+self.stdout.write('Importing entries from USDA')self.stdout.write(f' - {self.mode}')+ self.stdout.write(f' - {folder=}')self.stdout.write('')- usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip'-- with tempfile.TemporaryDirectory() as folder:- folder = '/Users/roland/Entwicklung/wger/server/extras/usda'-- print(f'{folder=}')- zip_file = os.path.join(folder, 'usda.zip')- if os.path.exists(zip_file):- self.stdout.write(f'Already downloaded {zip_file}, skipping download')- else:- self.stdout.write(f'downloading {zip_file}... (this may take a while)')- req = requests.get(usda_url, stream=True)- with open(zip_file, 'wb') as fid:- for chunk in req.iter_content(chunk_size=50 * 1024):- fid.write(chunk)-- self.stdout.write('download successful')-- with ZipFile(zip_file, 'r') as zip_ref:- file_list = zip_ref.namelist()- if not file_list:- raise Exception("No files found in the ZIP archive")-- first_file = file_list[0]- self.stdout.write(f'Extracting {first_file=}')- extracted_file_path = zip_ref.extract(first_file, path=folder)-- with open(extracted_file_path, "r") as extracted_file:- for line in extracted_file:- self.process_product(line.strip().strip(','))-- def process_product(self, json_data):- try:- data = json.loads(json_data)- except JSONDecodeError as e:- # print(e)- # print(json_data)- # print('---------------')- return-- name = data['description']- fdc_id = data['fdcId']-- if not data.get('foodNutrients'):- return-- proteins = None- carbs = None- fats = None- energy = None- for d in data['foodNutrients']:-- if not d.get("nutrient"):- return-- nutrient = d.get("nutrient")- nutrient_id = nutrient.get("id")-- match nutrient_id:- case 1003:- proteins = float(d.get("amount"))-- case 1004:- carbs = float(d.get("amount"))-- case 1005:- fats = float(d.get("amount"))-- case 2048:- energy = float(d.get("amount"))-- if not all([proteins, carbs, fats, energy]):- return-- self.stdout.write(f' - {fdc_id}')- self.stdout.write(f' - {name}')- self.stdout.write(f' - {proteins=}')- self.stdout.write(f' - {carbs=}')- self.stdout.write(f' - {fats=}')- self.stdout.write(f' - {energy=}')-- self.stdout.write('')- return+ english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)++ zip_file = os.path.join(folder, 'usda.zip')+ if os.path.exists(zip_file):+ self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')+ else:+ self.stdout.write(f'Downloading {zip_file}... (this may take a while)')+ req = requests.get(usda_url, stream=True)+ with open(zip_file, 'wb') as fid:+ for chunk in req.iter_content(chunk_size=50 * 1024):+ fid.write(chunk)++ self.stdout.write('download successful')++ with ZipFile(zip_file, 'r') as zip_ref:+ file_list = zip_ref.namelist()+ if not file_list:+ raise Exception('No files found in the ZIP archive')++ first_file = file_list[0]+ self.stdout.write(f'Extracting {first_file=}')+ extracted_file_path = zip_ref.extract(first_file, path=folder)++ # Since the file is almost JSONL, just process each line individually+ with open(extracted_file_path, 'r') as extracted_file:+ for line in extracted_file.readlines():+ # Skip the first and last lines in the file+ if 'FoundationFoods' in line:+ continue++ if line.strip() == '}':+ continue++ try:+ json_data = json.loads(line.strip().strip(','))+ except JSONDecodeError as e:+ self.stdout.write(f'--> Error while decoding JSON: {e}')+ continue++ try:+ ingredient_data = extract_info_from_usda(json_data, english.pk)+ except KeyError as e:+ self.stdout.write(f'--> KeyError while extracting info from USDA: {e}')+ self.counter['skipped'] += 1+ else:+ self.handle_data(ingredient_data)++ self.stdout.write(self.style.SUCCESS('Finished!'))+ self.stdout.write(self.style.SUCCESS(str(self.counter)))diff --git a/wger/nutrition/management/commands/sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients.pyindex 4f4a92fa8..5e12aaff1 100644--- a/wger/nutrition/management/commands/sync-ingredients.py+++ b/wger/nutrition/management/commands/sync-ingredients.py@@ -22,15 +22,6 @@ from django.core.management.base import (from django.core.validators import URLValidator# wger-from wger.exercises.sync import (- handle_deleted_entries,- sync_categories,- sync_equipment,- sync_exercises,- sync_languages,- sync_licenses,- sync_muscles,-)from wger.nutrition.sync import sync_ingredientsdiff --git a/wger/nutrition/management/commands/products.py b/wger/nutrition/management/products.pysimilarity index 93%rename from wger/nutrition/management/commands/products.pyrename to wger/nutrition/management/products.pyindex c4fdecb71..aead23dd9 100644--- a/wger/nutrition/management/commands/products.py+++ b/wger/nutrition/management/products.py@@ -24,6 +24,7 @@ from django.core.management.base import BaseCommandfrom wger.nutrition.dataclasses import IngredientDatafrom wger.nutrition.models import Ingredient+logger = logging.getLogger(__name__)@@ -49,6 +50,10 @@ class ImportProductCommand(BaseCommand):help = "Don't run this command directly. Use either import-off-products or import-usda-products"+ def __init__(self, stdout=None, stderr=None, no_color=False, force_color=False):+ super().__init__(stdout, stderr, no_color, force_color)+ self.counter = Counter()+def add_arguments(self, parser):parser.add_argument('--set-mode',@@ -57,15 +62,14 @@ class ImportProductCommand(BaseCommand):dest='mode',type=str,help='Script mode, "insert" or "update". Insert will insert the ingredients as new '- 'entries in the database, while update will try to update them if they are '- 'already present. Deault: insert',+ 'entries in the database, while update will try to update them if they are '+ 'already present. Default: update',)def handle(self, **options):raise NotImplementedError('Do not run this command on its own!')def handle_data(self, ingredient_data: IngredientData):-## Add entries as new productsif self.mode == Mode.INSERT:diff --git a/wger/nutrition/migrations/0021_add_remote_id.py b/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.pysimilarity index 59%rename from wger/nutrition/migrations/0021_add_remote_id.pyrename to wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.pyindex 6feefbc22..9d11dfcce 100644--- a/wger/nutrition/migrations/0021_add_remote_id.py+++ b/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py@@ -34,4 +34,26 @@ class Migration(migrations.Migration):set_external_id,reverse_code=migrations.RunPython.noop,),+ migrations.AlterField(+ model_name='image',+ name='license_author',+ field=models.CharField(+ blank=True,+ help_text='If you are not the author, enter the name or source here.',+ max_length=2500,+ null=True,+ verbose_name='Author(s)',+ ),+ ),+ migrations.AlterField(+ model_name='ingredient',+ name='license_author',+ field=models.CharField(+ blank=True,+ help_text='If you are not the author, enter the name or source here.',+ max_length=2500,+ null=True,+ verbose_name='Author(s)',+ ),+ ),]diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.pyindex d39e74d0d..35d66ddde 100644--- a/wger/nutrition/models/ingredient.py+++ b/wger/nutrition/models/ingredient.py@@ -60,9 +60,11 @@ from wger.utils.models import (AbstractSubmissionModel,)from wger.utils.requests import wger_user_agent+# Localfrom .ingredient_category import IngredientCategory+logger = logging.getLogger(__name__)diff --git a/wger/nutrition/models/sources.py b/wger/nutrition/models/sources.pyindex 2d276966e..478d3e63f 100644--- a/wger/nutrition/models/sources.py+++ b/wger/nutrition/models/sources.py@@ -20,3 +20,4 @@ from enum import Enumclass Source(Enum):WGER = 'wger'OPEN_FOOD_FACTS = 'Open Food Facts'+ USDA = 'USDA'diff --git a/wger/nutrition/off.py b/wger/nutrition/off.pyindex e7225d076..9423fa216 100644--- a/wger/nutrition/off.py+++ b/wger/nutrition/off.py@@ -22,6 +22,7 @@ from wger.nutrition.models import Sourcefrom wger.utils.constants import ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel+OFF_REQUIRED_TOP_LEVEL = ['product_name','code',@@ -34,7 +35,7 @@ OFF_REQUIRED_NUTRIMENTS = []-def extract_info_from_off(product_data: dict, language: int):+def extract_info_from_off(product_data: dict, language: int) -> IngredientData:if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL):raise KeyError('Missing required top-level key')@@ -70,7 +71,7 @@ def extract_info_from_off(product_data: dict, language: int):sodium = product_data['nutriments'].get('sodium_100g', None)sugars = product_data['nutriments'].get('sugars_100g', None)fibre = product_data['nutriments'].get('fiber_100g', None)- brand = product_data.get('brands', None)+ brand = product_data.get('brands', '')# License and author infosource_name = Source.OPEN_FOOD_FACTS.valuediff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.pyindex de9ed19d5..5583b2306 100644--- a/wger/nutrition/tests/test_off.py+++ b/wger/nutrition/tests/test_off.py@@ -17,10 +17,8 @@from django.test import SimpleTestCase# wger-from wger.nutrition.off import (- extract_info_from_off,-)from wger.nutrition.dataclasses import IngredientData+from wger.nutrition.off import extract_info_from_offfrom wger.utils.constants import ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModeldiff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.pynew file mode 100644index 000000000..cb5c93e86--- /dev/null+++ b/wger/nutrition/usda.py@@ -0,0 +1,103 @@+# This file is part of wger Workout Manager <https://github.com/wger-project>.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <http://www.gnu.org/licenses/>.++# wger+from wger.nutrition.dataclasses import IngredientData+from wger.nutrition.models import Source+from wger.utils.constants import CC_0_LICENSE_ID+from wger.utils.models import AbstractSubmissionModel+++def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:+ if not product_data.get('foodNutrients'):+ raise KeyError('Missing key "foodNutrients"')++ energy = 0+ protein = 0+ carbs = 0+ fats = 0+ sugars = 0+ saturated = 0+ fat = 0++ sodium = None+ fibre = None++ for d in product_data['foodNutrients']:+ if not d.get('nutrient'):+ raise KeyError('Missing key "nutrient"')++ nutrient = d.get('nutrient')+ nutrient_id = nutrient.get('id')++ match nutrient_id:+ case 1003:+ protein = float(d.get('amount'))++ case 1004:+ carbs = float(d.get('amount'))++ case 1005:+ fats = float(d.get('amount'))++ case 2048:+ energy = float(d.get('amount'))++ case 1093:+ sodium = float(d.get('amount'))++ if not all([protein, carbs, fats, energy]):+ raise KeyError('Could not extract all nutrition information')++ name = product_data['description']+ if len(name) > 200:+ name = name[:200]++ code = None+ remote_id = product_data['fdcId']+ brand = ''++ # License and author info+ source_name = Source.USDA.value+ source_url = f'https://fdc.nal.usda.gov/'+ author = (+ 'U.S. Department of Agriculture, Agricultural Research Service, '+ 'Beltsville Human Nutrition Research Center. FoodData Central.'+ )+ object_url = f'https://api.nal.usda.gov/fdc/v1/food/{remote_id}'++ return IngredientData(+ code=code,+ remote_id=remote_id,+ name=name,+ common_name=name,+ language_id=language,+ energy=energy,+ protein=protein,+ carbohydrates=carbs,+ carbohydrates_sugar=sugars,+ fat=fat,+ fat_saturated=saturated,+ fibres=fibre,+ sodium=sodium,+ source_name=source_name,+ source_url=source_url,+ brand=brand,+ status=AbstractSubmissionModel.STATUS_ACCEPTED,+ license_id=CC_0_LICENSE_ID,+ license_author=author,+ license_title=name,+ license_object_url=object_url,+ )diff --git a/wger/utils/constants.py b/wger/utils/constants.pyindex d327a380c..1a24c0a13 100644--- a/wger/utils/constants.py+++ b/wger/utils/constants.py@@ -15,6 +15,7 @@# Standard Libraryfrom decimal import Decimal+# NavigationWORKOUT_TAB = 'workout'EXERCISE_TAB = 'exercises'@@ -50,6 +51,7 @@ PAGINATION_PAGES_AROUND_CURRENT = 5# Important license IDsCC_BY_SA_4_ID = 2CC_BY_SA_3_LICENSE_ID = 1+CC_0_LICENSE_ID = 3ODBL_LICENSE_ID = 5# Default/fallback languagediff --git a/wger/utils/models.py b/wger/utils/models.pyindex bfd80c8af..5e7fe89bc 100644--- a/wger/utils/models.py+++ b/wger/utils/models.py@@ -61,7 +61,7 @@ class AbstractLicenseModel(models.Model):license_author = models.CharField(verbose_name=_('Author(s)'),- max_length=600,+ max_length=2500,blank=True,null=True,help_text=_('If you are not the author, enter the name or source here.'),