Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Make USDA food import actually import the products

Roland Geiderd5a796eb3b86
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py
index b5c443c4e..c135f30cf 100644
--- a/wger/nutrition/dataclasses.py
+++ b/wger/nutrition/dataclasses.py
@@ -12,7 +12,11 @@
#
# You should have received a copy of the GNU Affero General Public License
-from dataclasses import dataclass, asdict
+# Standard Library
+from dataclasses import (
+ asdict,
+ dataclass,
+)
from typing import Optional
@@ -29,7 +33,7 @@ class IngredientData:
fat_saturated: float
fibres: Optional[float]
sodium: Optional[float]
- code: str
+ code: Optional[str]
source_name: str
source_url: str
common_name: str
diff --git a/wger/nutrition/management/__init__.py b/wger/nutrition/management/__init__.py
new file mode 100644
index 000000000..1292caeab
--- /dev/null
+++ b/wger/nutrition/management/__init__.py
@@ -0,0 +1,15 @@
+# This file is part of wger Workout Manager <https://github.com/wger-project>.
+# Copyright (C) 2013 - 2021 wger Team
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py
index 17857211f..c40d9486c 100644
--- a/wger/nutrition/management/commands/import-off-products.py
+++ b/wger/nutrition/management/commands/import-off-products.py
@@ -14,15 +14,16 @@
# Standard Library
import logging
-from collections import Counter
-
-# Django
-from wger.nutrition.management.commands.products import ImportProductCommand, Mode
# wger
from wger.core.models import Language
+from wger.nutrition.management.products import (
+ ImportProductCommand,
+ Mode,
+)
from wger.nutrition.off import extract_info_from_off
+
logger = logging.getLogger(__name__)
@@ -31,8 +32,6 @@ class Command(ImportProductCommand):
Import an Open Food facts Dump
"""
- completeness = 0.7
-
help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'
def handle(self, **options):
@@ -43,8 +42,6 @@ class Command(ImportProductCommand):
self.stdout.write('Please install pymongo, `pip install pymongo`')
return
- self.counter = Counter()
-
if options['mode'] == 'insert':
self.mode = Mode.INSERT
@@ -55,7 +52,7 @@ class Command(ImportProductCommand):
client = MongoClient('mongodb://off:[email protected]', port=27017)
db = client.admin
- languages = {l.short_name: l.pk for l in Language.objects.all()}
+ languages = {lang.short_name: lang.pk for lang in Language.objects.all()}
for product in db.products.find({'lang': {'$in': list(languages.keys())}}):
try:
@@ -65,9 +62,8 @@ class Command(ImportProductCommand):
# self.stdout.write(repr(e))
# pprint(product)
self.counter['skipped'] += 1
- continue
-
- self.handle_data(ingredient_data)
+ else:
+ self.handle_data(ingredient_data)
self.stdout.write(self.style.SUCCESS('Finished!'))
self.stdout.write(self.style.SUCCESS(str(self.counter)))
diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py
index d59b11ea0..ae0ad9272 100644
--- a/wger/nutrition/management/commands/import-usda-products.py
+++ b/wger/nutrition/management/commands/import-usda-products.py
@@ -13,7 +13,6 @@
# You should have received a copy of the GNU Affero General Public License
# Standard Library
-import enum
import json
import logging
import os
@@ -21,131 +20,85 @@ import tempfile
from json import JSONDecodeError
from zipfile import ZipFile
+# Third Party
import requests
-# Django
-from django.core.management.base import BaseCommand
-
-logger = logging.getLogger(__name__)
+# wger
+from wger.core.models import Language
+from wger.nutrition.management.products import (
+ ImportProductCommand,
+ Mode,
+)
+from wger.nutrition.usda import extract_info_from_usda
+from wger.utils.constants import ENGLISH_SHORT_NAME
-# Mode for this script. When using 'insert', the script will bulk-insert the new
-# ingredients, which is very efficient. Importing the whole database will require
-# barely a minute. When using 'update', existing ingredients will be updated, which
-# requires two queries per product and is needed when there are already existing
-# entries in the local ingredient table.
-class Mode(enum.Enum):
- INSERT = enum.auto()
- UPDATE = enum.auto()
+logger = logging.getLogger(__name__)
-class Command(BaseCommand):
+class Command(ImportProductCommand):
"""
Import an Open Food facts Dump
"""
- mode = Mode.UPDATE
-
- def add_arguments(self, parser):
- parser.add_argument(
- '--set-mode',
- action='store',
- default=10,
- dest='mode',
- type=str,
- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '
- 'entries in the database, while update will try to update them if they are '
- 'already present. Deault: insert',
- )
-
def handle(self, **options):
if options['mode'] == 'insert':
self.mode = Mode.INSERT
+ usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip'
+ folder = '/Users/roland/Entwicklung/wger/server/extras/usda'
+
self.stdout.write('Importing entries from USDA')
self.stdout.write(f' - {self.mode}')
+ self.stdout.write(f' - {folder=}')
self.stdout.write('')
- usda_url = 'https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_foundation_food_json_2024-04-18.zip'
-
- with tempfile.TemporaryDirectory() as folder:
- folder = '/Users/roland/Entwicklung/wger/server/extras/usda'
-
- print(f'{folder=}')
- zip_file = os.path.join(folder, 'usda.zip')
- if os.path.exists(zip_file):
- self.stdout.write(f'Already downloaded {zip_file}, skipping download')
- else:
- self.stdout.write(f'downloading {zip_file}... (this may take a while)')
- req = requests.get(usda_url, stream=True)
- with open(zip_file, 'wb') as fid:
- for chunk in req.iter_content(chunk_size=50 * 1024):
- fid.write(chunk)
-
- self.stdout.write('download successful')
-
- with ZipFile(zip_file, 'r') as zip_ref:
- file_list = zip_ref.namelist()
- if not file_list:
- raise Exception("No files found in the ZIP archive")
-
- first_file = file_list[0]
- self.stdout.write(f'Extracting {first_file=}')
- extracted_file_path = zip_ref.extract(first_file, path=folder)
-
- with open(extracted_file_path, "r") as extracted_file:
- for line in extracted_file:
- self.process_product(line.strip().strip(','))
-
- def process_product(self, json_data):
- try:
- data = json.loads(json_data)
- except JSONDecodeError as e:
- # print(e)
- # print(json_data)
- # print('---------------')
- return
-
- name = data['description']
- fdc_id = data['fdcId']
-
- if not data.get('foodNutrients'):
- return
-
- proteins = None
- carbs = None
- fats = None
- energy = None
- for d in data['foodNutrients']:
-
- if not d.get("nutrient"):
- return
-
- nutrient = d.get("nutrient")
- nutrient_id = nutrient.get("id")
-
- match nutrient_id:
- case 1003:
- proteins = float(d.get("amount"))
-
- case 1004:
- carbs = float(d.get("amount"))
-
- case 1005:
- fats = float(d.get("amount"))
-
- case 2048:
- energy = float(d.get("amount"))
-
- if not all([proteins, carbs, fats, energy]):
- return
-
- self.stdout.write(f' - {fdc_id}')
- self.stdout.write(f' - {name}')
- self.stdout.write(f' - {proteins=}')
- self.stdout.write(f' - {carbs=}')
- self.stdout.write(f' - {fats=}')
- self.stdout.write(f' - {energy=}')
-
- self.stdout.write('')
- return
+ english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)
+
+ zip_file = os.path.join(folder, 'usda.zip')
+ if os.path.exists(zip_file):
+ self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')
+ else:
+ self.stdout.write(f'Downloading {zip_file}... (this may take a while)')
+ req = requests.get(usda_url, stream=True)
+ with open(zip_file, 'wb') as fid:
+ for chunk in req.iter_content(chunk_size=50 * 1024):
+ fid.write(chunk)
+
+ self.stdout.write('download successful')
+
+ with ZipFile(zip_file, 'r') as zip_ref:
+ file_list = zip_ref.namelist()
+ if not file_list:
+ raise Exception('No files found in the ZIP archive')
+
+ first_file = file_list[0]
+ self.stdout.write(f'Extracting {first_file=}')
+ extracted_file_path = zip_ref.extract(first_file, path=folder)
+
+ # Since the file is almost JSONL, just process each line individually
+ with open(extracted_file_path, 'r') as extracted_file:
+ for line in extracted_file.readlines():
+ # Skip the first and last lines in the file
+ if 'FoundationFoods' in line:
+ continue
+
+ if line.strip() == '}':
+ continue
+
+ try:
+ json_data = json.loads(line.strip().strip(','))
+ except JSONDecodeError as e:
+ self.stdout.write(f'--> Error while decoding JSON: {e}')
+ continue
+
+ try:
+ ingredient_data = extract_info_from_usda(json_data, english.pk)
+ except KeyError as e:
+ self.stdout.write(f'--> KeyError while extracting info from USDA: {e}')
+ self.counter['skipped'] += 1
+ else:
+ self.handle_data(ingredient_data)
+
+ self.stdout.write(self.style.SUCCESS('Finished!'))
+ self.stdout.write(self.style.SUCCESS(str(self.counter)))
diff --git a/wger/nutrition/management/commands/sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients.py
index 4f4a92fa8..5e12aaff1 100644
--- a/wger/nutrition/management/commands/sync-ingredients.py
+++ b/wger/nutrition/management/commands/sync-ingredients.py
@@ -22,15 +22,6 @@ from django.core.management.base import (
from django.core.validators import URLValidator
# wger
-from wger.exercises.sync import (
- handle_deleted_entries,
- sync_categories,
- sync_equipment,
- sync_exercises,
- sync_languages,
- sync_licenses,
- sync_muscles,
-)
from wger.nutrition.sync import sync_ingredients
diff --git a/wger/nutrition/management/commands/products.py b/wger/nutrition/management/products.py
similarity index 93%
rename from wger/nutrition/management/commands/products.py
rename to wger/nutrition/management/products.py
index c4fdecb71..aead23dd9 100644
--- a/wger/nutrition/management/commands/products.py
+++ b/wger/nutrition/management/products.py
@@ -24,6 +24,7 @@ from django.core.management.base import BaseCommand
from wger.nutrition.dataclasses import IngredientData
from wger.nutrition.models import Ingredient
+
logger = logging.getLogger(__name__)
@@ -49,6 +50,10 @@ class ImportProductCommand(BaseCommand):
help = "Don't run this command directly. Use either import-off-products or import-usda-products"
+ def __init__(self, stdout=None, stderr=None, no_color=False, force_color=False):
+ super().__init__(stdout, stderr, no_color, force_color)
+ self.counter = Counter()
+
def add_arguments(self, parser):
parser.add_argument(
'--set-mode',
@@ -57,15 +62,14 @@ class ImportProductCommand(BaseCommand):
dest='mode',
type=str,
help='Script mode, "insert" or "update". Insert will insert the ingredients as new '
- 'entries in the database, while update will try to update them if they are '
- 'already present. Deault: insert',
+ 'entries in the database, while update will try to update them if they are '
+ 'already present. Default: update',
)
def handle(self, **options):
raise NotImplementedError('Do not run this command on its own!')
def handle_data(self, ingredient_data: IngredientData):
-
#
# Add entries as new products
if self.mode == Mode.INSERT:
diff --git a/wger/nutrition/migrations/0021_add_remote_id.py b/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py
similarity index 59%
rename from wger/nutrition/migrations/0021_add_remote_id.py
rename to wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py
index 6feefbc22..9d11dfcce 100644
--- a/wger/nutrition/migrations/0021_add_remote_id.py
+++ b/wger/nutrition/migrations/0021_add_remote_id_increase_author_field_length.py
@@ -34,4 +34,26 @@ class Migration(migrations.Migration):
set_external_id,
reverse_code=migrations.RunPython.noop,
),
+ migrations.AlterField(
+ model_name='image',
+ name='license_author',
+ field=models.CharField(
+ blank=True,
+ help_text='If you are not the author, enter the name or source here.',
+ max_length=2500,
+ null=True,
+ verbose_name='Author(s)',
+ ),
+ ),
+ migrations.AlterField(
+ model_name='ingredient',
+ name='license_author',
+ field=models.CharField(
+ blank=True,
+ help_text='If you are not the author, enter the name or source here.',
+ max_length=2500,
+ null=True,
+ verbose_name='Author(s)',
+ ),
+ ),
]
diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py
index d39e74d0d..35d66ddde 100644
--- a/wger/nutrition/models/ingredient.py
+++ b/wger/nutrition/models/ingredient.py
@@ -60,9 +60,11 @@ from wger.utils.models import (
AbstractSubmissionModel,
)
from wger.utils.requests import wger_user_agent
+
# Local
from .ingredient_category import IngredientCategory
+
logger = logging.getLogger(__name__)
diff --git a/wger/nutrition/models/sources.py b/wger/nutrition/models/sources.py
index 2d276966e..478d3e63f 100644
--- a/wger/nutrition/models/sources.py
+++ b/wger/nutrition/models/sources.py
@@ -20,3 +20,4 @@ from enum import Enum
class Source(Enum):
WGER = 'wger'
OPEN_FOOD_FACTS = 'Open Food Facts'
+ USDA = 'USDA'
diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py
index e7225d076..9423fa216 100644
--- a/wger/nutrition/off.py
+++ b/wger/nutrition/off.py
@@ -22,6 +22,7 @@ from wger.nutrition.models import Source
from wger.utils.constants import ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel
+
OFF_REQUIRED_TOP_LEVEL = [
'product_name',
'code',
@@ -34,7 +35,7 @@ OFF_REQUIRED_NUTRIMENTS = [
]
-def extract_info_from_off(product_data: dict, language: int):
+def extract_info_from_off(product_data: dict, language: int) -> IngredientData:
if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL):
raise KeyError('Missing required top-level key')
@@ -70,7 +71,7 @@ def extract_info_from_off(product_data: dict, language: int):
sodium = product_data['nutriments'].get('sodium_100g', None)
sugars = product_data['nutriments'].get('sugars_100g', None)
fibre = product_data['nutriments'].get('fiber_100g', None)
- brand = product_data.get('brands', None)
+ brand = product_data.get('brands', '')
# License and author info
source_name = Source.OPEN_FOOD_FACTS.value
diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.py
index de9ed19d5..5583b2306 100644
--- a/wger/nutrition/tests/test_off.py
+++ b/wger/nutrition/tests/test_off.py
@@ -17,10 +17,8 @@
from django.test import SimpleTestCase
# wger
-from wger.nutrition.off import (
- extract_info_from_off,
-)
from wger.nutrition.dataclasses import IngredientData
+from wger.nutrition.off import extract_info_from_off
from wger.utils.constants import ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel
diff --git a/wger/nutrition/usda.py b/wger/nutrition/usda.py
new file mode 100644
index 000000000..cb5c93e86
--- /dev/null
+++ b/wger/nutrition/usda.py
@@ -0,0 +1,103 @@
+# This file is part of wger Workout Manager <https://github.com/wger-project>.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# wger
+from wger.nutrition.dataclasses import IngredientData
+from wger.nutrition.models import Source
+from wger.utils.constants import CC_0_LICENSE_ID
+from wger.utils.models import AbstractSubmissionModel
+
+
+def extract_info_from_usda(product_data: dict, language: int) -> IngredientData:
+ if not product_data.get('foodNutrients'):
+ raise KeyError('Missing key "foodNutrients"')
+
+ energy = 0
+ protein = 0
+ carbs = 0
+ fats = 0
+ sugars = 0
+ saturated = 0
+ fat = 0
+
+ sodium = None
+ fibre = None
+
+ for d in product_data['foodNutrients']:
+ if not d.get('nutrient'):
+ raise KeyError('Missing key "nutrient"')
+
+ nutrient = d.get('nutrient')
+ nutrient_id = nutrient.get('id')
+
+ match nutrient_id:
+ case 1003:
+ protein = float(d.get('amount'))
+
+ case 1004:
+ carbs = float(d.get('amount'))
+
+ case 1005:
+ fats = float(d.get('amount'))
+
+ case 2048:
+ energy = float(d.get('amount'))
+
+ case 1093:
+ sodium = float(d.get('amount'))
+
+ if not all([protein, carbs, fats, energy]):
+ raise KeyError('Could not extract all nutrition information')
+
+ name = product_data['description']
+ if len(name) > 200:
+ name = name[:200]
+
+ code = None
+ remote_id = product_data['fdcId']
+ brand = ''
+
+ # License and author info
+ source_name = Source.USDA.value
+ source_url = f'https://fdc.nal.usda.gov/'
+ author = (
+ 'U.S. Department of Agriculture, Agricultural Research Service, '
+ 'Beltsville Human Nutrition Research Center. FoodData Central.'
+ )
+ object_url = f'https://api.nal.usda.gov/fdc/v1/food/{remote_id}'
+
+ return IngredientData(
+ code=code,
+ remote_id=remote_id,
+ name=name,
+ common_name=name,
+ language_id=language,
+ energy=energy,
+ protein=protein,
+ carbohydrates=carbs,
+ carbohydrates_sugar=sugars,
+ fat=fat,
+ fat_saturated=saturated,
+ fibres=fibre,
+ sodium=sodium,
+ source_name=source_name,
+ source_url=source_url,
+ brand=brand,
+ status=AbstractSubmissionModel.STATUS_ACCEPTED,
+ license_id=CC_0_LICENSE_ID,
+ license_author=author,
+ license_title=name,
+ license_object_url=object_url,
+ )
diff --git a/wger/utils/constants.py b/wger/utils/constants.py
index d327a380c..1a24c0a13 100644
--- a/wger/utils/constants.py
+++ b/wger/utils/constants.py
@@ -15,6 +15,7 @@
# Standard Library
from decimal import Decimal
+
# Navigation
WORKOUT_TAB = 'workout'
EXERCISE_TAB = 'exercises'
@@ -50,6 +51,7 @@ PAGINATION_PAGES_AROUND_CURRENT = 5
# Important license IDs
CC_BY_SA_4_ID = 2
CC_BY_SA_3_LICENSE_ID = 1
+CC_0_LICENSE_ID = 3
ODBL_LICENSE_ID = 5
# Default/fallback language
diff --git a/wger/utils/models.py b/wger/utils/models.py
index bfd80c8af..5e7fe89bc 100644
--- a/wger/utils/models.py
+++ b/wger/utils/models.py
@@ -61,7 +61,7 @@ class AbstractLicenseModel(models.Model):
license_author = models.CharField(
verbose_name=_('Author(s)'),
- max_length=600,
+ max_length=2500,
blank=True,
null=True,
help_text=_('If you are not the author, enter the name or source here.'),