Self hosted FLOSS fitness/workout, nutrition and weight tracker
Small improvements to OFF import script
Roland Geider82255c951e08
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.pyindex 38335545f..50af0dfee 100644--- a/extras/scripts/create_ingredients_from_foodfacts.py+++ b/extras/scripts/create_ingredients_from_foodfacts.py@@ -11,6 +11,8 @@# GNU General Public License for more details.## You should have received a copy of the GNU Affero General Public License+import enum+from enum import Enumfrom pymongo import MongoClientimport os@@ -30,13 +32,17 @@ from wger.core.models import Language # noqa: E402Simple script that imports and loads the Open Food Facts database into theingredients database.-NOTE: The file is VERY large (17 GB), so it takes a long time (> 3 hours) to run-and create all ingredients.+NOTE: The file is VERY large (17 GB), so it takes a long time (> 3 hours) to+import the data and create all the ingredients.* Requirements:-pip3 install pymongo zip-apt-get install mongo-tools # (for mongorestore)+(note that the local mongo version needs to be compatible with the one used to+ create the dump, otherwise the indices won't be compatible, it is best to use+ a newer version than the one found in the ubuntu/debian repos)++pip3 install pymongo+apt-get install mongo-tools zip* Steps:wget https://static.openfoodfacts.org/data/openfoodfacts-mongodbdump.tar.gz@@ -62,7 +68,7 @@ rm -r dump# Update ingredient fixturepython3 manage.py dumpdata nutrition.ingredient > extras/scripts/data.jsoncd extras/scripts/-python3 python3 filter-fixtures.py+python3 filter-fixtures.pyzip ingredients.json.zip ingredients.json"""@@ -70,11 +76,17 @@ zip ingredients.json.zip ingredients.jsonclient = MongoClient('mongodb://off:[email protected]', port=27017)db = client.admin+# Mode for this script. When using 'insert', the script will bulk-insert the new# ingredients, which is very efficient. Importing the whole database will require# barely a minute. When using 'update', existing ingredients will be updated, which# requires two queries per product.-MODE = 'insert'+class Mode(Enum):+ INSERT = enum.auto()+ UPDATE = enum.auto()+++MODE = Mode.INSERTlanguages = {i[0]: Language.objects.get(short_name=i[0]) for i in settings.LANGUAGES}@@ -89,17 +101,24 @@ stats = {'new': 0,# total = db.products.count_documents({'lang': lang})# print(f'Lang {lang} has {count} completed products out of {total}')-# Lang de has 15667 completed products out of 52669-# Lang es has 8757 completed products out of 186171-# Lang ru has 505 completed products out of 2918-# Lang fr has 85058 completed products out of 811715-# Lang bg has 15 completed products out of 718-# Lang el has 20 completed products out of 251-# Lang nl has 750 completed products out of 5611-# Lang no has 9 completed products out of 141-# Lang cs has 123 completed products out of 789-# Lang sv has 1340 completed products out of 2878-# Lang pt has 526 completed products out of 3541+# Completeness status of ingredients as of 2021-11-18+#+# Lang en has 5154 completed products out of 579495+# Lang de has 8882 completed products out of 90885+# Lang bg has 16 completed products out of 897+# Lang es has 6785 completed products out of 236210+# Lang ru has 481 completed products out of 8729+# Lang nl has 628 completed products out of 7428+# Lang pt has 482 completed products out of 5024+# Lang el has 27 completed products out of 544+# Lang cs has 154 completed products out of 1451+# Lang sv has 1234 completed products out of 3459+# Lang no has 9 completed products out of 155+# Lang fr has 45358 completed products out of 960096+# Lang it has 821 completed products out of 118270+# Lang pl has 2032 completed products out of 4978+# Lang uk has 4 completed products out of 158+# Lang tr has 8 completed products out of 425print('***********************************')print(languages.keys())@@ -174,7 +193,7 @@ for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'compl}# Add entries as new products- if MODE == 'insert':+ if MODE == Mode.INSERT:bulk_update_bucket.append(Ingredient(**ingredient_data))if len(bulk_update_bucket) > BULK_SIZE:try:@@ -190,8 +209,9 @@ for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'complingredient.save()# ¯\_(ツ)_/¯- except Exception:- pass+ except Exception as e:+ print('--> Error while saving the product individually')+ print(e)stats['new'] += BULK_SIZEbulk_update_bucket = []@@ -212,7 +232,9 @@ for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'complstats['edited'] += 1# print('-> updated')- except: # noqa: E722+ except Exception as e:+ print('--> Error while performing update_or_create')+ print(e)continueprint('***********************************')diff --git a/requirements_dev.txt b/requirements_dev.txtindex 7ee5bf89b..efc84270f 100644--- a/requirements_dev.txt+++ b/requirements_dev.txt@@ -8,6 +8,9 @@# Building/installingwheel==0.37.0+# for ingredient import script from OFF+pymongo==3.12.1+# Development packagescoverage==6.1.2django-debug-toolbar==3.2.2