Self hosted FLOSS fitness/workout, nutrition and weight tracker
Use bulk inserts when possible
Roland Geider79b0472f4c15
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.pyindex a641f23b6..b46c74040 100644--- a/extras/scripts/create_ingredients_from_foodfacts.py+++ b/extras/scripts/create_ingredients_from_foodfacts.py@@ -65,9 +65,25 @@ rm -r dumpclient = MongoClient('mongodb://off:[email protected]', port=27017)db = client.admin+# Mode for this script. When using 'insert', the script will bulk-insert the new+# ingredients, which is very efficient. Importing the whole database will require+# around 2 hours. When using 'update', existing ingredients will be updated, which+# requires two queries per product(!!!) and takes probably a week to complete.+MODE = 'insert'+langs = [i[0] for i in settings.LANGUAGES]lang_objects = [Language.objects.get(short_name=lang) for lang in langs]+BULK_SIZE = 500+bulk_update_bucket = []+stats = {'levenshtein': 0,+ 'new': 0,+ 'edited': 0,+ 'skipped': 0}+product_names = {}+for lang in langs:+ product_names[lang] = []+# for lang in langs:# count = db.products.count_documents({'lang': lang, 'complete': 1})# total = db.products.count_documents({'lang': lang})@@ -89,14 +105,6 @@ print('***********************************')print(langs)print('***********************************')-stats = {'levenshtein': 0,- 'new': 0,- 'edited': 0,- 'skipped': 0}--product_names = {}-for lang in langs:- product_names[lang] = []for product in db.products.find({'lang': {"$in": langs}}):lang = product['lang']@@ -107,12 +115,12 @@ for product in db.products.find({'lang': {"$in": langs}}):code = product['code']# Some products have no name or name is too long, skipping- if not name or len(name) > 200:- print(f'-> skipping due to name requirements')+ if not name or len(name) > 200 :+ #print(f'-> skipping due to name requirements')stats['skipped'] += 1continue- print(f'Processing "{name}"...')+ #print(f'Processing "{name}"...')required = ['energy-kcal_100g','proteins_100g',@@ -128,7 +136,7 @@ for product in db.products.find({'lang': {"$in": langs}}):fat = product['nutriments']['fat_100g']saturated = product['nutriments']['saturated-fat_100g']else:- print(f'-> skipping due to required nutriments')+ #print(f'-> skipping due to required nutriments')stats['skipped'] += 1continue@@ -138,6 +146,9 @@ for product in db.products.find({'lang': {"$in": langs}}):common_name = product.get('generic_name', None)brand = product.get('brands', None)+ if common_name and len(common_name) > 200:+ continue+source_name = "Open Food Facts"source_url = f'https://world.openfoodfacts.org/api/v0/product/{code}.json'@@ -146,7 +157,7 @@ for product in db.products.find({'lang': {"$in": langs}}):for i in product_names[lang]:if distance(name, i) <= 3:stats['levenshtein'] += 1- print(f'-> skipping due to similarity with "{i}"')+ #print(f'-> skipping due to similarity with "{i}"')found_similar = Truebreak@@ -176,21 +187,46 @@ for product in db.products.find({'lang': {"$in": langs}}):'license_author': 'Open Food Facts',}- try:- # Update an existing product (look-up key is the code) or create a new- # one. While this might not be the most efficient query (there will always- # be a SELECT first), it's ok because this script is run very rarely.- obj, created = Ingredient.objects.update_or_create(code=code, defaults=ingredient_data)-- if created:- stats['new'] += 1- print('-> added to the database')- else:- stats['edited'] += 1- print('-> updated')-- except: # noqa: E722- continue+ # Add entries as new products+ if MODE == 'insert':+ bulk_update_bucket.append(Ingredient(**ingredient_data))+ if len(bulk_update_bucket) > BULK_SIZE:+ try:+ Ingredient.objects.bulk_create(bulk_update_bucket)+ print('***** Bulk adding products *****')+ except Exception as e:+ print('--> Error while saving the product bucket. Saving individually')+ print(e)++ # Try saving the ingredients individually as most will be correct+ for ingredient in bulk_update_bucket:+ try:+ ingredient.save()++ # ¯\_(ツ)_/¯+ except Exception as e:+ pass++ bulk_update_bucket = []++ # Update existing entries+ else:+ try:++ # Update an existing product (look-up key is the code) or create a new+ # one. While this might not be the most efficient query (there will always+ # be a SELECT first), it's ok because this script is run very rarely.+ obj, created = Ingredient.objects.update_or_create(code=code, defaults=ingredient_data)++ if created:+ stats['new'] += 1+ print('-> added to the database')+ else:+ stats['edited'] += 1+ print('-> updated')++ except: # noqa: E722+ continueprint('***********************************')print(stats)