Self hosted FLOSS fitness/workout, nutrition and weight tracker
Add remote_id to ingredients.
Roland Geiderebd9534382ba
diff --git a/wger/nutrition/api/serializers.py b/wger/nutrition/api/serializers.pyindex ad5e0a551..98e5755bb 100644--- a/wger/nutrition/api/serializers.py+++ b/wger/nutrition/api/serializers.py@@ -114,6 +114,7 @@ class IngredientSerializer(serializers.ModelSerializer):fields = ['id','uuid',+ 'remote_id','code','name','created',diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.pynew file mode 100644index 000000000..b5c443c4e--- /dev/null+++ b/wger/nutrition/dataclasses.py@@ -0,0 +1,44 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++from dataclasses import dataclass, asdict+from typing import Optional+++@dataclass+class IngredientData:+ name: str+ remote_id: str+ language_id: int+ energy: float+ protein: float+ carbohydrates: float+ carbohydrates_sugar: float+ fat: float+ fat_saturated: float+ fibres: Optional[float]+ sodium: Optional[float]+ code: str+ source_name: str+ source_url: str+ common_name: str+ brand: str+ status: str+ license_id: int+ license_author: str+ license_title: str+ license_object_url: str++ def dict(self):+ return asdict(self)diff --git a/wger/nutrition/fixtures/test-ingredients.json b/wger/nutrition/fixtures/test-ingredients.jsonindex d4152ebef..da61a7814 100644--- a/wger/nutrition/fixtures/test-ingredients.json+++ b/wger/nutrition/fixtures/test-ingredients.json@@ -4,6 +4,7 @@"model": "nutrition.ingredient","fields": {"uuid": "7908c204-907f-4b1e-ad4e-f482e9769ade",+ "remote_id": "1234567890987654321","status": "2","license_author": "wger test","last_update": "2013-02-12T01:00:00+01:00",@@ -31,6 +32,7 @@"model": "nutrition.ingredient","fields": {"uuid": "44dc5966-73a2-4df7-8b15-f6d37a8990d9",+ "remote_id": "1234567890","status": "2","license_author": "wger test","name": "Ingredient, test, 2, organic, raw",@@ -58,6 +60,7 @@"model": "nutrition.ingredient","fields": {"uuid": "f28c8a68-ecda-42db-a9c4-781ee2aee472",+ "remote_id": "1223334444","status": "2","license_author": "wger test","last_update": "2013-03-22T18:32:15+01:00",@@ -85,6 +88,7 @@"model": "nutrition.ingredient","fields": {"uuid": "fdd662aa-696c-46c6-acae-b192ccc25f11",+ "remote_id": "0987654321","status": "2","license_author": "wger test","last_update": "2013-02-12T12:17:45+01:00",@@ -112,6 +116,7 @@"model": "nutrition.ingredient","fields": {"uuid": "293957df-b5bb-4ca6-b883-2234185c5372",+ "remote_id": "555555555","status": "2","license_author": "wger test","last_update": "2013-02-12T14:14:00+01:00",@@ -139,6 +144,7 @@"model": "nutrition.ingredient","fields": {"uuid": "dfc5c622-027b-4f17-8141-dadd1ce7e3f1",+ "remote_id": "1112222333333334444444","status": "2","license_author": "wger test","last_update": "2013-02-12T01:00:00+01:00",@@ -166,6 +172,7 @@"model": "nutrition.ingredient","fields": {"uuid": "42a41805-2caf-4c7e-9ff3-decbfee5f695",+ "remote_id": "11222222333","status": "1","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -193,6 +200,7 @@"model": "nutrition.ingredient","fields": {"uuid": "db9e0502-2a54-472d-97e2-3761d1b286ee",+ "remote_id": "9988877766655","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -220,6 +228,7 @@"model": "nutrition.ingredient","fields": {"uuid": "7164629a-9f58-450b-8316-ad7954564171",+ "remote_id": "123454321","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -247,6 +256,7 @@"model": "nutrition.ingredient","fields": {"uuid": "9d53cf0e-5100-480f-a320-2de51c70603b",+ "remote_id": "0009999888777","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -274,6 +284,7 @@"model": "nutrition.ingredient","fields": {"uuid": "0187406a-7290-4199-a4dc-ccceb8cb55ea",+ "remote_id": "7787878787878787","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -301,6 +312,7 @@"model": "nutrition.ingredient","fields": {"uuid": "16ade127-00b0-4bcf-906e-dd9730f5ade1",+ "remote_id": "29292929292929","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -328,6 +340,7 @@"model": "nutrition.ingredient","fields": {"uuid": "59a73b2c-ead1-4001-b6a0-355c9cb366a7",+ "remote_id": "19191919191","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",@@ -355,6 +368,7 @@"model": "nutrition.ingredient","fields": {"uuid": "a18665d0-24e3-4255-88aa-92f2831d568a",+ "remote_id": "4444444444444","status": "2","license_author": "test","last_update": "2013-02-12T01:00:00+01:00",diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.pyindex 06397f26d..17857211f 100644--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@ -13,64 +13,28 @@# You should have received a copy of the GNU Affero General Public License# Standard Library-import enumimport loggingfrom collections import Counter# Django-from django.core.management.base import BaseCommand+from wger.nutrition.management.commands.products import ImportProductCommand, Mode# wgerfrom wger.core.models import Language-from wger.nutrition.models import Ingredientfrom wger.nutrition.off import extract_info_from_off-logger = logging.getLogger(__name__)-# Mode for this script. When using 'insert', the script will bulk-insert the new-# ingredients, which is very efficient. Importing the whole database will require-# barely a minute. When using 'update', existing ingredients will be updated, which-# requires two queries per product and is needed when there are already existing-# entries in the local ingredient table.-class Mode(enum.Enum):- INSERT = enum.auto()- UPDATE = enum.auto()---class Command(BaseCommand):+class Command(ImportProductCommand):"""Import an Open Food facts Dump"""- mode = Mode.UPDATE- bulk_size = 500completeness = 0.7help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'- def add_arguments(self, parser):- parser.add_argument(- '--set-mode',- action='store',- default=10,- dest='mode',- type=str,- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '- 'entries in the database, while update will try to update them if they are '- 'already present. Deault: insert',- )- parser.add_argument(- '--completeness',- action='store',- default=0.7,- dest='completeness',- type=float,- help='Completeness threshold for importing the products. Products in OFF have '- 'completeness score that ranges from 0 to 1.1. Default: 0.7',- )-def handle(self, **options):try:# Third Party@@ -79,16 +43,12 @@ class Command(BaseCommand):self.stdout.write('Please install pymongo, `pip install pymongo`')return+ self.counter = Counter()+if options['mode'] == 'insert':self.mode = Mode.INSERT- if options['completeness'] < 0 or options['completeness'] > 1.1:- self.stdout.write('Completeness must be between 0 and 1.1')- return- self.completeness = options['completeness']-self.stdout.write('Importing entries from Open Food Facts')- self.stdout.write(f' - Completeness threshold: {self.completeness}')self.stdout.write(f' - {self.mode}')self.stdout.write('')@@ -97,88 +57,17 @@ class Command(BaseCommand):languages = {l.short_name: l.pk for l in Language.objects.all()}- bulk_update_bucket = []- counter = Counter()-- for product in db.products.find(- {'lang': {'$in': list(languages.keys())}, 'completeness': {'$gt': self.completeness}}- ):+ for product in db.products.find({'lang': {'$in': list(languages.keys())}}):try:ingredient_data = extract_info_from_off(product, languages[product['lang']])except KeyError as e:# self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')- # self.stdout.write(- # '***********************************************************************************************')- # self.stdout.write(- # '***********************************************************************************************')- # self.stdout.write(- # '***********************************************************************************************')+ # self.stdout.write(repr(e))# pprint(product)- # self.stdout.write(f'--> Product: {product}')- counter['skipped'] += 1- continue-- # Some products have no name or name is too long, skipping- if not ingredient_data.name:- # self.stdout.write('--> Ingredient has no name field')- counter['skipped'] += 1- continue-- if not ingredient_data.common_name:- # self.stdout.write('--> Ingredient has no common name field')- counter['skipped'] += 1+ self.counter['skipped'] += 1continue- #- # Add entries as new products- if self.mode == Mode.INSERT:- bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))- if len(bulk_update_bucket) > self.bulk_size:- try:- Ingredient.objects.bulk_create(bulk_update_bucket)- self.stdout.write('***** Bulk adding products *****')- except Exception as e:- self.stdout.write(- '--> Error while saving the product bucket. Saving individually'- )- self.stdout.write(e)-- # Try saving the ingredients individually as most will be correct- for ingredient in bulk_update_bucket:- try:- ingredient.save()-- # ¯\_(ツ)_/¯- except Exception as e:- self.stdout.write('--> Error while saving the product individually')- self.stdout.write(e)-- counter['new'] += self.bulk_size- bulk_update_bucket = []-- # Update existing entries- else:- try:- # Update an existing product (look-up key is the code) or create a new- # one. While this might not be the most efficient query (there will always- # be a SELECT first), it's ok because this script is run very rarely.- obj, created = Ingredient.objects.update_or_create(- code=ingredient_data.code,- defaults=ingredient_data.dict(),- )-- if created:- counter['new'] += 1- # self.stdout.write('-> added to the database')- else:- counter['edited'] += 1- # self.stdout.write('-> updated')-- except Exception as e:- self.stdout.write('--> Error while performing update_or_create')- self.stdout.write(str(e))- counter['error'] += 1- continue+ self.handle_data(ingredient_data)self.stdout.write(self.style.SUCCESS('Finished!'))- self.stdout.write(self.style.SUCCESS(str(counter)))+ self.stdout.write(self.style.SUCCESS(str(self.counter)))diff --git a/wger/nutrition/management/commands/products.py b/wger/nutrition/management/commands/products.pynew file mode 100644index 000000000..c4fdecb71--- /dev/null+++ b/wger/nutrition/management/commands/products.py@@ -0,0 +1,117 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License++# Standard Library+import enum+import logging+from collections import Counter++# Django+from django.core.management.base import BaseCommand++# wger+from wger.nutrition.dataclasses import IngredientData+from wger.nutrition.models import Ingredient++logger = logging.getLogger(__name__)+++# Mode for this script. When using 'insert', the script will bulk-insert the new+# ingredients, which is very efficient. Importing the whole database will require+# barely a minute. When using 'update', existing ingredients will be updated, which+# requires two queries per product and is needed when there are already existing+# entries in the local ingredient table.+class Mode(enum.Enum):+ INSERT = enum.auto()+ UPDATE = enum.auto()+++class ImportProductCommand(BaseCommand):+ """+ Import an Open Food facts Dump+ """++ mode = Mode.UPDATE+ bulk_update_bucket: list[Ingredient] = []+ bulk_size = 500+ counter: Counter++ help = "Don't run this command directly. Use either import-off-products or import-usda-products"++ def add_arguments(self, parser):+ parser.add_argument(+ '--set-mode',+ action='store',+ default='update',+ dest='mode',+ type=str,+ help='Script mode, "insert" or "update". Insert will insert the ingredients as new '+ 'entries in the database, while update will try to update them if they are '+ 'already present. Deault: insert',+ )++ def handle(self, **options):+ raise NotImplementedError('Do not run this command on its own!')++ def handle_data(self, ingredient_data: IngredientData):++ #+ # Add entries as new products+ if self.mode == Mode.INSERT:+ self.bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))+ if len(self.bulk_update_bucket) > self.bulk_size:+ try:+ Ingredient.objects.bulk_create(self.bulk_update_bucket)+ self.stdout.write('***** Bulk adding products *****')+ except Exception as e:+ self.stdout.write(+ '--> Error while saving the product bucket. Saving individually'+ )+ self.stdout.write(e)++ # Try saving the ingredients individually as most will be correct+ for ingredient in self.bulk_update_bucket:+ try:+ ingredient.save()++ # ¯\_(ツ)_/¯+ except Exception as e:+ self.stdout.write('--> Error while saving the product individually')+ self.stdout.write(e)++ self.counter['new'] += self.bulk_size+ self.bulk_update_bucket = []++ # Update existing entries+ else:+ try:+ # Update an existing product (look-up key is the code) or create a new+ # one. While this might not be the most efficient query (there will always+ # be a SELECT first), it's ok because this script is run very rarely.+ obj, created = Ingredient.objects.update_or_create(+ remote_id=ingredient_data.remote_id,+ defaults=ingredient_data.dict(),+ )++ if created:+ self.counter['new'] += 1+ # self.stdout.write('-> added to the database')+ else:+ self.counter['edited'] += 1+ # self.stdout.write('-> updated')++ except Exception as e:+ self.stdout.write('--> Error while performing update_or_create')+ self.stdout.write(repr(e))+ self.counter['error'] += 1diff --git a/wger/nutrition/migrations/0021_add_remote_id.py b/wger/nutrition/migrations/0021_add_remote_id.pynew file mode 100644index 000000000..6feefbc22--- /dev/null+++ b/wger/nutrition/migrations/0021_add_remote_id.py@@ -0,0 +1,37 @@+# Generated by Django 4.2.6 on 2024-05-18 07:31++from django.db import migrations, models++from wger.nutrition.models import Source+++def set_external_id(apps, schema_editor):+ """Set remote_id, used by imported ingredients (OFF, etc.)"""++ Ingredient = apps.get_model('nutrition', 'Ingredient')+ for ingredient in Ingredient.objects.filter(source_name=Source.OPEN_FOOD_FACTS.value):+ ingredient.remote_id = ingredient.code+ ingredient.save()+++class Migration(migrations.Migration):+ dependencies = [+ ('nutrition', '0020_full_text_search'),+ ]++ operations = [+ migrations.RenameIndex(+ model_name='ingredient',+ new_name='nutrition_i_name_8f538f_gin',+ old_name='nutrition_i_search__f274b7_gin',+ ),+ migrations.AddField(+ model_name='ingredient',+ name='remote_id',+ field=models.CharField(blank=True, db_index=True, max_length=200, null=True),+ ),+ migrations.RunPython(+ set_external_id,+ reverse_code=migrations.RunPython.noop,+ ),+ ]diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.pyindex 252bbf085..4cf48975a 100644--- a/wger/nutrition/models/ingredient.py+++ b/wger/nutrition/models/ingredient.py@@ -24,7 +24,6 @@ from json import JSONDecodeErrorfrom django.conf import settingsfrom django.contrib.auth.models import Userfrom django.contrib.postgres.indexes import GinIndex-from django.contrib.postgres.search import SearchVectorFieldfrom django.contrib.sites.models import Sitefrom django.core import mailfrom django.core.cache import cache@@ -53,10 +52,7 @@ from wger.nutrition.consts import ()from wger.nutrition.models.sources import Sourcefrom wger.utils.cache import cache_mapper-from wger.utils.constants import (- OFF_SEARCH_PRODUCT_FOUND,- TWOPLACES,-)+from wger.utils.constants import TWOPLACESfrom wger.utils.language import load_languagefrom wger.utils.managers import SubmissionManagerfrom wger.utils.models import (@@ -64,11 +60,9 @@ from wger.utils.models import (AbstractSubmissionModel,)from wger.utils.requests import wger_user_agent-# Localfrom .ingredient_category import IngredientCategory-logger = logging.getLogger(__name__)@@ -197,7 +191,15 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):blank=True,db_index=True,)- """Internal ID of the source database, e.g. a barcode or similar"""+ """The product's barcode"""++ remote_id = models.CharField(+ max_length=200,+ null=True,+ blank=True,+ db_index=True,+ )+ """ID of the product in the external source database. Used for updated during imports."""source_name = models.CharField(max_length=200,diff --git a/wger/nutrition/off.py b/wger/nutrition/off.pyindex 9e99f141c..e7225d076 100644--- a/wger/nutrition/off.py+++ b/wger/nutrition/off.py@@ -14,19 +14,14 @@# along with this program. If not, see <http://www.gnu.org/licenses/>.# Standard Library-from dataclasses import (- asdict,- dataclass,-)-from typing import Optional# wgerfrom wger.nutrition.consts import KJ_PER_KCAL+from wger.nutrition.dataclasses import IngredientDatafrom wger.nutrition.models import Sourcefrom wger.utils.constants import ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel-OFF_REQUIRED_TOP_LEVEL = ['product_name','code',@@ -39,34 +34,7 @@ OFF_REQUIRED_NUTRIMENTS = []-@dataclass-class IngredientData:- name: str- language_id: int- energy: float- protein: float- carbohydrates: float- carbohydrates_sugar: float- fat: float- fat_saturated: float- fibres: Optional[float]- sodium: Optional[float]- code: str- source_name: str- source_url: str- common_name: str- brand: str- status: str- license_id: int- license_author: str- license_title: str- license_object_url: str-- def dict(self):- return asdict(self)---def extract_info_from_off(product_data, language: int):+def extract_info_from_off(product_data: dict, language: int):if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL):raise KeyError('Missing required top-level key')@@ -95,12 +63,12 @@ def extract_info_from_off(product_data, language: int):code = product_data['code']protein = product_data['nutriments']['proteins_100g']carbs = product_data['nutriments']['carbohydrates_100g']- sugars = product_data['nutriments'].get('sugars_100g', 0)fat = product_data['nutriments']['fat_100g']- saturated = product_data['nutriments'].get('saturated-fat_100g', 0)# these are optional+ saturated = product_data['nutriments'].get('saturated-fat_100g', None)sodium = product_data['nutriments'].get('sodium_100g', None)+ sugars = product_data['nutriments'].get('sugars_100g', None)fibre = product_data['nutriments'].get('fiber_100g', None)brand = product_data.get('brands', None)@@ -111,6 +79,7 @@ def extract_info_from_off(product_data, language: int):object_url = f'https://world.openfoodfacts.org/product/{code}/'return IngredientData(+ remote_id=code,name=name,language_id=language,energy=energy,diff --git a/wger/nutrition/tests/test_ingredient.py b/wger/nutrition/tests/test_ingredient.pyindex 51afabd4d..5d8c8a549 100644--- a/wger/nutrition/tests/test_ingredient.py+++ b/wger/nutrition/tests/test_ingredient.py@@ -41,11 +41,7 @@ from wger.nutrition.models import (Ingredient,Meal,)-from wger.utils.constants import (- NUTRITION_TAB,- OFF_SEARCH_PRODUCT_FOUND,- OFF_SEARCH_PRODUCT_NOT_FOUND,-)+from wger.utils.constants import NUTRITION_TABclass IngredientRepresentationTestCase(WgerTestCase):@@ -463,7 +459,6 @@ class IngredientModelTestCase(WgerTestCase):def setUp(self):super().setUp()self.off_response = {- 'status': OFF_SEARCH_PRODUCT_FOUND,'product': {'code': '1234','lang': 'de',@@ -484,10 +479,7 @@ class IngredientModelTestCase(WgerTestCase):},}- self.off_response_no_results = {- 'status': OFF_SEARCH_PRODUCT_NOT_FOUND,- 'status_verbose': 'product not found',- }+ self.off_response_no_results = None@patch('openfoodfacts.api.ProductResource.get')def test_fetch_from_off_success(self, mock_api):diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.pyindex de652c26e..de9ed19d5 100644--- a/wger/nutrition/tests/test_off.py+++ b/wger/nutrition/tests/test_off.py@@ -18,9 +18,9 @@ from django.test import SimpleTestCase# wgerfrom wger.nutrition.off import (- IngredientData,extract_info_from_off,)+from wger.nutrition.dataclasses import IngredientDatafrom wger.utils.constants import ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel@@ -60,6 +60,7 @@ class ExtractInfoFromOffTestCase(SimpleTestCase):result = extract_info_from_off(self.off_data1, 1)data = IngredientData(name='Foo with chocolate',+ remote_id='1234',language_id=1,energy=120,protein=10,@@ -112,5 +113,5 @@ class ExtractInfoFromOffTestCase(SimpleTestCase):del self.off_data1['nutriments']['saturated-fat_100g']result = extract_info_from_off(self.off_data1, 1)- self.assertEqual(result.carbohydrates_sugar, 0)- self.assertEqual(result.fat_saturated, 0)+ self.assertEqual(result.carbohydrates_sugar, None)+ self.assertEqual(result.fat_saturated, None)diff --git a/wger/utils/constants.py b/wger/utils/constants.pyindex 144be1f2d..d327a380c 100644--- a/wger/utils/constants.py+++ b/wger/utils/constants.py@@ -15,7 +15,6 @@# Standard Libraryfrom decimal import Decimal-# NavigationWORKOUT_TAB = 'workout'EXERCISE_TAB = 'exercises'@@ -66,9 +65,5 @@ DOWNLOAD_INGREDIENT_OPTIONS = (DOWNLOAD_INGREDIENT_NONE,)-# OFF Api-OFF_SEARCH_PRODUCT_FOUND = 1-OFF_SEARCH_PRODUCT_NOT_FOUND = 0-# APIAPI_MAX_ITEMS = 999