Self hosted FLOSS fitness/workout, nutrition and weight tracker
Import products with a completeness value higher than 0.7
Roland Geider23dca42a8b19
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.pyindex e1af8af2e..a71a32c54 100644--- a/extras/scripts/create_ingredients_from_foodfacts.py+++ b/extras/scripts/create_ingredients_from_foodfacts.py@@ -20,15 +20,13 @@ import osimport djangoimport sys-from wger.nutrition.models.off import extract_info_from_off-sys.path.insert(0, os.path.join('..', '..'))os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")django.setup()from django.conf import settings # noqa: E402from wger.nutrition.models import Ingredient # noqa: E402-from wger.nutrition.models.sources import Source+from wger.nutrition.off import extract_info_from_offfrom wger.core.models import Language # noqa: E402"""@@ -95,21 +93,23 @@ class Mode(enum.Enum):MODE = Mode.UPDATE+BULK_SIZE = 500-languages = {l.short_name: l for l in Language.objects.all()}+# The completeness is a value between 0 and 1.1 and shows how much product information+# is in the open food facts DB+COMPLETENESS = 0.7-BULK_SIZE = 500-bulk_update_bucket = []-counter = Counter()-# for lang in languages.keys():-# count = db.products.count_documents({'lang': lang, 'complete': 1})-# total = db.products.count_documents({'lang': lang})-# print(f'Lang {lang} has {count} completed products out of {total}')+# Get some completeness statistics+# for lang in ['de', 'es', 'en']:+# count = db.products.count_documents({'lang': lang, 'completeness': {"$gt": COMPLETENESS}})+# # count = db.products.count_documents({'lang': lang, 'complete': 1})+# total = db.products.count_documents({'lang': lang})+# print(f'Lang {lang} has {count} completed products out of {total}')# import sys# sys.exit()-# Completeness status of ingredients as of 2023-04-08+# Completed ingredients as of 2023-04-08## Lang az has 0 completed products out of 40# Lang id has 6 completed products out of 981@@ -136,77 +136,99 @@ counter = Counter()# Lang fa has 0 completed products out of 575# Lang zh has 2 completed products out of 935-print('***********************************')-print(languages.keys())-print('***********************************')--for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'complete': 1}):-- try:- ingredient_data = extract_info_from_off(product, languages[product['lang']])- except KeyError as e:- # print('--> KeyError while extracting info from OFF', e)- counter['skipped'] += 1- continue-- # Some products have no name or name is too long, skipping- if not ingredient_data['name']:- counter['skipped'] += 1- continue-- if not ingredient_data['common_name']:- counter['skipped'] += 1- continue-- #- # Add entries as new products- if MODE == Mode.INSERT:- bulk_update_bucket.append(Ingredient(**ingredient_data))- if len(bulk_update_bucket) > BULK_SIZE:- try:- Ingredient.objects.bulk_create(bulk_update_bucket)- print('***** Bulk adding products *****')- except Exception as e:- print('--> Error while saving the product bucket. Saving individually')- print(e)+# 2023-11-22+# completeness > 0.7+# Lang de has 47605 completed products out of 195056+# Lang es has 23187 completed products out of 311365+# Lang en has 39555 completed products out of 979534++def main():+ languages = {l.short_name: l for l in Language.objects.all()}- # Try saving the ingredients individually as most will be correct- for ingredient in bulk_update_bucket:- try:- ingredient.save()+ bulk_update_bucket = []+ counter = Counter()- # ¯\_(ツ)_/¯- except Exception as e:- print('--> Error while saving the product individually')- print(e)+ print('***********************************')+ print(languages.keys())+ print('***********************************')- counter['new'] += BULK_SIZE- bulk_update_bucket = []+ for product in db.products.find({+ 'lang': {"$in": list(languages.keys())},+ 'completeness': {"$gt": COMPLETENESS}+ }):- # Update existing entries- else:try:+ ingredient_data = extract_info_from_off(product, languages[product['lang']])+ except KeyError as e:+ print('--> KeyError while extracting info from OFF', e)+ counter['skipped'] += 1+ continue++ # Some products have no name or name is too long, skipping+ if not ingredient_data['name']:+ counter['skipped'] += 1+ continue- # Update an existing product (look-up key is the code) or create a new- # one. While this might not be the most efficient query (there will always- # be a SELECT first), it's ok because this script is run very rarely.- obj, created = Ingredient.objects.update_or_create(- code=ingredient_data['code'],- defaults=ingredient_data- )-- if created:- counter['new'] += 1- # print('-> added to the database')- else:- counter['edited'] += 1- # print('-> updated')-- except Exception as e:- print('--> Error while performing update_or_create')- print(e)+ if not ingredient_data['common_name']:+ counter['skipped'] += 1continue-print('***********************************')-print(counter)-print('***********************************')+ #+ # Add entries as new products+ if MODE == Mode.INSERT:+ bulk_update_bucket.append(Ingredient(**ingredient_data))+ if len(bulk_update_bucket) > BULK_SIZE:+ try:+ Ingredient.objects.bulk_create(bulk_update_bucket)+ print('***** Bulk adding products *****')+ except Exception as e:+ print('--> Error while saving the product bucket. Saving individually')+ print(e)++ # Try saving the ingredients individually as most will be correct+ for ingredient in bulk_update_bucket:+ try:+ ingredient.save()++ # ¯\_(ツ)_/¯+ except Exception as e:+ print('--> Error while saving the product individually')+ print(e)++ counter['new'] += BULK_SIZE+ bulk_update_bucket = []++ # Update existing entries+ else:+ try:++ # Update an existing product (look-up key is the code) or create a new+ # one. While this might not be the most efficient query (there will always+ # be a SELECT first), it's ok because this script is run very rarely.+ obj, created = Ingredient.objects.update_or_create(+ code=ingredient_data['code'],+ defaults=ingredient_data+ )++ if created:+ counter['new'] += 1+ # print('-> added to the database')+ else:+ counter['edited'] += 1+ # print('-> updated')++ except Exception as e:+ print('--> Error while performing update_or_create')+ print(f' ingredient: {ingredient_data["name"]}')+ print(e)+ print(ingredient_data)+ counter['error'] += 1+ continue++ print('***********************************')+ print(counter)+ print('***********************************')+++if __name__ == "__main__":+ main()diff --git a/wger/nutrition/management/commands/download-ingredient-images.py b/wger/nutrition/management/commands/download-ingredient-images.pyindex f3b16201e..32cc87b6b 100644--- a/wger/nutrition/management/commands/download-ingredient-images.py+++ b/wger/nutrition/management/commands/download-ingredient-images.py@@ -30,7 +30,7 @@ from wger.nutrition.sync import download_ingredient_imagesclass Command(BaseCommand):"""- Download ingredient images from wger.de and updates the local database+ Download ingredient images from a wger instance and updates the local databaseBoth the ingredients and the images are identified by their UUID, which can'tbe modified via the GUI.diff --git a/wger/nutrition/off.py b/wger/nutrition/off.pyindex 114f6e0dc..36ef4e392 100644--- a/wger/nutrition/off.py+++ b/wger/nutrition/off.py@@ -12,10 +12,10 @@## You should have received a copy of the GNU Affero General Public License# along with this program. If not, see <http://www.gnu.org/licenses/>.+# wger-from wger.core.models import Languagefrom wger.nutrition.models import Source-from wger.utils.constants import CC_ODBL_LICENSE_ID+from wger.utils.constants import ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel@@ -50,7 +50,6 @@ def extract_info_from_off(product, language):name = name[:200]common_name = product.get('generic_name', '')- common_name = '' if common_name is None else common_nameif len(common_name) > 200:common_name = common_name[:200]@@ -71,6 +70,7 @@ def extract_info_from_off(product, language):source_name = Source.OPEN_FOOD_FACTS.valuesource_url = f'https://world.openfoodfacts.org/api/v2/product/{code}.json'authors = ', '.join(product.get('editors_tags', ['open food facts']))+ object_url = f'https://world.openfoodfacts.org/product/{code}/'return {'name': name,@@ -89,8 +89,8 @@ def extract_info_from_off(product, language):'common_name': common_name,'brand': brand,'status': AbstractSubmissionModel.STATUS_ACCEPTED,- 'license_id': CC_ODBL_LICENSE_ID,+ 'license_id': ODBL_LICENSE_ID,'license_author': authors,'license_title': name,- 'license_object_url': f'https://world.openfoodfacts.org/product/{code}/'+ 'license_object_url': object_url}diff --git a/wger/utils/constants.py b/wger/utils/constants.pyindex 449831b03..58d4fa47b 100644--- a/wger/utils/constants.py+++ b/wger/utils/constants.py@@ -51,7 +51,7 @@ PAGINATION_PAGES_AROUND_CURRENT = 5# Important license IDsCC_BY_SA_4_ID = 2CC_BY_SA_3_LICENSE_ID = 1-CC_ODBL_LICENSE_ID = 5+ODBL_LICENSE_ID = 5# Default/fallback languageENGLISH_SHORT_NAME = 'en'