Self hosted FLOSS fitness/workout, nutrition and weight tracker
Check name similarity with other ingredients
Roland Geider2007a0419c18
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.pyindex ad40feca5..f14ed20ab 100644--- a/extras/scripts/create_ingredients_from_foodfacts.py+++ b/extras/scripts/create_ingredients_from_foodfacts.py@@ -1,4 +1,19 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+from pymongo import MongoClient+from Levenshtein import distanceimport osimport djangoimport sys@@ -11,6 +26,39 @@ from django.conf import settings # noqa: E402from wger.nutrition.models import Ingredient # noqa: E402from wger.core.models import Language # noqa: E402++"""+Simple script that imports and loads the Open Food Facts database into the+ingredients database.++NOTE: The file is VERY large (17 GB), so it takes a long time (> 3 hours) to run+and create all ingredients.+++* Requirements:+pip3 install python-Levenshtein+apt-get install mongo-tools # (for mongorestore)++* Steps:+wget https://static.openfoodfacts.org/data/openfoodfacts-mongodbdump.tar.gz+tar xzvf openfoodfacts-mongodbdump.tar.gz++# Import+docker pull mongo+docker run -it --name wger_mongo -p 27017:27017 -d mongo:latest+mongorestore -d off -c products tmp/mongo_dump/off/products.bson++# Process+python create_ingredients_from_foodfacts.py++# Cleanup+docker stop wger_mongo+docker rm wger_mongo+rm openfoodfacts-mongodbdump.tar.gz+rm -r dump+"""++client = MongoClient(port=27017)db = client.off@@ -34,8 +82,20 @@ lang_objects = [Language.objects.get(short_name=lang) for lang in langs]# Lang sv has 1340 completed products out of 2878# Lang pt has 526 completed products out of 3541+print('***********************************')+print(langs)+print('***********************************')++stats = {'levenshtein': 0,+ 'new': 0,+ 'edited': 0,+ 'skipped': 0}-for product in db.products.find({'lang': {"$in": langs}, 'complete': 1}):+product_names = {}+for lang in langs:+ product_names[lang] = []++for product in db.products.find({'lang': {"$in": langs}}).limit(500):lang = product['lang']main_details = ['product_name', 'code']@@ -43,6 +103,14 @@ for product in db.products.find({'lang': {"$in": langs}, 'complete': 1}):name = product['product_name']code = product['code']+ # Some products have no name or name is too long, skipping+ if not name or len(name) > 200:+ print(f'-> skipping due to name requirements')+ stats['skipped'] += 1+ continue++ print(f'Processing "{name}"...')+required = ['energy-kcal_100g','proteins_100g','carbohydrates_100g',@@ -57,14 +125,10 @@ for product in db.products.find({'lang': {"$in": langs}, 'complete': 1}):fat = product['nutriments']['fat_100g']saturated = product['nutriments']['saturated-fat_100g']else:+ print(f'-> skipping due to required nutriments')+ stats['skipped'] += 1continue- # Some products have no name, skipping- if not name:- continue-- print(f'Processing {name}...')-# these are optionalsodium = product['nutriments'].get('sodium_100g', None)fibre = product['nutriments'].get('fiber_100g', None)@@ -74,6 +138,20 @@ for product in db.products.find({'lang': {"$in": langs}, 'complete': 1}):source_name = "Open Food Facts"source_url = f'https://world.openfoodfacts.org/api/v0/product/{code}.json'+ # Check if there are already names with similar names in the same language+ found_similar = False+ for i in product_names[lang]:+ if distance(name, i) <= 3:+ stats['levenshtein'] += 1+ print(f'-> skipping due to similarity with "{i}"')+ found_similar = True+ break++ if found_similar:+ continue++ product_names[lang].append(name)+ingredient_data = {'language': lang_objects[langs.index(lang)],'name': name,@@ -100,9 +178,15 @@ for product in db.products.find({'lang': {"$in": langs}, 'complete': 1}):obj, created = Ingredient.objects.update_or_create(code=code, defaults=ingredient_data)if created:- print(' -> added to the database')+ stats['new'] += 1+ print('-> added to the database')else:- print(' -> updated')+ stats['edited'] += 1+ print('-> updated')except: # noqa: E722continue++print('***********************************')+print(stats)+print('***********************************')diff --git a/extras/scripts/import_foodfacts.sh b/extras/scripts/import_foodfacts.shdeleted file mode 100644index 48cc6fa7e..000000000--- a/extras/scripts/import_foodfacts.sh+++ /dev/null@@ -1,38 +0,0 @@-#!/bin/sh--# -*- coding: utf-8 -*---# This file is part of wger Workout Manager.-#-# wger Workout Manager is free software: you can redistribute it and/or modify-# it under the terms of the GNU Affero General Public License as published by-# the Free Software Foundation, either version 3 of the License, or-# (at your option) any later version.-#-# wger Workout Manager is distributed in the hope that it will be useful,-# but WITHOUT ANY WARRANTY; without even the implied warranty of-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-# GNU General Public License for more details.-#-# You should have received a copy of the GNU Affero General Public License--"""-Simple script that imports and loads the Open Food Facts database into the-ingredients database--NOTE: The file is VERY large (17 GB), so it takes a long time (> 3 hours) to run-and create all ingredients-"""--wget https://static.openfoodfacts.org/data/openfoodfacts-mongodbdump.tar.gz--tar xzvf openfoodfacts-mongodbdump.tar.gz-docker pull mongo-docker run -it --name wger_mongo -v $(pwd)/dump:/tmp/mongo_dump -p 27017:27017 -d mongo:latest-mongorestore -d off -c products tmp/mongo_dump/off/products.bson--python create_ingredients_from_foodfacts.py--rm $(pwd)/openfoodfacts-mongodbdump.tar.gz-rm -r $(pwd)/dump-rm -r /tmp\ No newline at end of file