Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Import products with a completeness value higher than 0.7

Roland Geider23dca42a8b19
diff --git a/extras/scripts/create_ingredients_from_foodfacts.py b/extras/scripts/create_ingredients_from_foodfacts.py
index e1af8af2e..a71a32c54 100644
--- a/extras/scripts/create_ingredients_from_foodfacts.py
+++ b/extras/scripts/create_ingredients_from_foodfacts.py
@@ -20,15 +20,13 @@ import os
import django
import sys
-from wger.nutrition.models.off import extract_info_from_off
-
sys.path.insert(0, os.path.join('..', '..'))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
django.setup()
from django.conf import settings # noqa: E402
from wger.nutrition.models import Ingredient # noqa: E402
-from wger.nutrition.models.sources import Source
+from wger.nutrition.off import extract_info_from_off
from wger.core.models import Language # noqa: E402
"""
@@ -95,21 +93,23 @@ class Mode(enum.Enum):
MODE = Mode.UPDATE
+BULK_SIZE = 500
-languages = {l.short_name: l for l in Language.objects.all()}
+# The completeness is a value between 0 and 1.1 and shows how much product information
+# is in the open food facts DB
+COMPLETENESS = 0.7
-BULK_SIZE = 500
-bulk_update_bucket = []
-counter = Counter()
-# for lang in languages.keys():
-# count = db.products.count_documents({'lang': lang, 'complete': 1})
-# total = db.products.count_documents({'lang': lang})
-# print(f'Lang {lang} has {count} completed products out of {total}')
+# Get some completeness statistics
+# for lang in ['de', 'es', 'en']:
+# count = db.products.count_documents({'lang': lang, 'completeness': {"$gt": COMPLETENESS}})
+# # count = db.products.count_documents({'lang': lang, 'complete': 1})
+# total = db.products.count_documents({'lang': lang})
+# print(f'Lang {lang} has {count} completed products out of {total}')
# import sys
# sys.exit()
-# Completeness status of ingredients as of 2023-04-08
+# Completed ingredients as of 2023-04-08
#
# Lang az has 0 completed products out of 40
# Lang id has 6 completed products out of 981
@@ -136,77 +136,99 @@ counter = Counter()
# Lang fa has 0 completed products out of 575
# Lang zh has 2 completed products out of 935
-print('***********************************')
-print(languages.keys())
-print('***********************************')
-
-for product in db.products.find({'lang': {"$in": list(languages.keys())}, 'complete': 1}):
-
- try:
- ingredient_data = extract_info_from_off(product, languages[product['lang']])
- except KeyError as e:
- # print('--> KeyError while extracting info from OFF', e)
- counter['skipped'] += 1
- continue
-
- # Some products have no name or name is too long, skipping
- if not ingredient_data['name']:
- counter['skipped'] += 1
- continue
-
- if not ingredient_data['common_name']:
- counter['skipped'] += 1
- continue
-
- #
- # Add entries as new products
- if MODE == Mode.INSERT:
- bulk_update_bucket.append(Ingredient(**ingredient_data))
- if len(bulk_update_bucket) > BULK_SIZE:
- try:
- Ingredient.objects.bulk_create(bulk_update_bucket)
- print('***** Bulk adding products *****')
- except Exception as e:
- print('--> Error while saving the product bucket. Saving individually')
- print(e)
+# 2023-11-22
+# completeness > 0.7
+# Lang de has 47605 completed products out of 195056
+# Lang es has 23187 completed products out of 311365
+# Lang en has 39555 completed products out of 979534
+
+def main():
+ languages = {l.short_name: l for l in Language.objects.all()}
- # Try saving the ingredients individually as most will be correct
- for ingredient in bulk_update_bucket:
- try:
- ingredient.save()
+ bulk_update_bucket = []
+ counter = Counter()
- # ¯\_(ツ)_/¯
- except Exception as e:
- print('--> Error while saving the product individually')
- print(e)
+ print('***********************************')
+ print(languages.keys())
+ print('***********************************')
- counter['new'] += BULK_SIZE
- bulk_update_bucket = []
+ for product in db.products.find({
+ 'lang': {"$in": list(languages.keys())},
+ 'completeness': {"$gt": COMPLETENESS}
+ }):
- # Update existing entries
- else:
try:
+ ingredient_data = extract_info_from_off(product, languages[product['lang']])
+ except KeyError as e:
+ print('--> KeyError while extracting info from OFF', e)
+ counter['skipped'] += 1
+ continue
+
+ # Some products have no name or name is too long, skipping
+ if not ingredient_data['name']:
+ counter['skipped'] += 1
+ continue
- # Update an existing product (look-up key is the code) or create a new
- # one. While this might not be the most efficient query (there will always
- # be a SELECT first), it's ok because this script is run very rarely.
- obj, created = Ingredient.objects.update_or_create(
- code=ingredient_data['code'],
- defaults=ingredient_data
- )
-
- if created:
- counter['new'] += 1
- # print('-> added to the database')
- else:
- counter['edited'] += 1
- # print('-> updated')
-
- except Exception as e:
- print('--> Error while performing update_or_create')
- print(e)
+ if not ingredient_data['common_name']:
+ counter['skipped'] += 1
continue
-print('***********************************')
-print(counter)
-print('***********************************')
+ #
+ # Add entries as new products
+ if MODE == Mode.INSERT:
+ bulk_update_bucket.append(Ingredient(**ingredient_data))
+ if len(bulk_update_bucket) > BULK_SIZE:
+ try:
+ Ingredient.objects.bulk_create(bulk_update_bucket)
+ print('***** Bulk adding products *****')
+ except Exception as e:
+ print('--> Error while saving the product bucket. Saving individually')
+ print(e)
+
+ # Try saving the ingredients individually as most will be correct
+ for ingredient in bulk_update_bucket:
+ try:
+ ingredient.save()
+
+ # ¯\_(ツ)_/¯
+ except Exception as e:
+ print('--> Error while saving the product individually')
+ print(e)
+
+ counter['new'] += BULK_SIZE
+ bulk_update_bucket = []
+
+ # Update existing entries
+ else:
+ try:
+
+ # Update an existing product (look-up key is the code) or create a new
+ # one. While this might not be the most efficient query (there will always
+ # be a SELECT first), it's ok because this script is run very rarely.
+ obj, created = Ingredient.objects.update_or_create(
+ code=ingredient_data['code'],
+ defaults=ingredient_data
+ )
+
+ if created:
+ counter['new'] += 1
+ # print('-> added to the database')
+ else:
+ counter['edited'] += 1
+ # print('-> updated')
+
+ except Exception as e:
+ print('--> Error while performing update_or_create')
+ print(f' ingredient: {ingredient_data["name"]}')
+ print(e)
+ print(ingredient_data)
+ counter['error'] += 1
+ continue
+
+ print('***********************************')
+ print(counter)
+ print('***********************************')
+
+
+if __name__ == "__main__":
+ main()
diff --git a/wger/nutrition/management/commands/download-ingredient-images.py b/wger/nutrition/management/commands/download-ingredient-images.py
index f3b16201e..32cc87b6b 100644
--- a/wger/nutrition/management/commands/download-ingredient-images.py
+++ b/wger/nutrition/management/commands/download-ingredient-images.py
@@ -30,7 +30,7 @@ from wger.nutrition.sync import download_ingredient_images
class Command(BaseCommand):
"""
- Download ingredient images from wger.de and updates the local database
+ Download ingredient images from a wger instance and updates the local database
Both the ingredients and the images are identified by their UUID, which can't
be modified via the GUI.
diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py
index 114f6e0dc..36ef4e392 100644
--- a/wger/nutrition/off.py
+++ b/wger/nutrition/off.py
@@ -12,10 +12,10 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
# wger
-from wger.core.models import Language
from wger.nutrition.models import Source
-from wger.utils.constants import CC_ODBL_LICENSE_ID
+from wger.utils.constants import ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel
@@ -50,7 +50,6 @@ def extract_info_from_off(product, language):
name = name[:200]
common_name = product.get('generic_name', '')
- common_name = '' if common_name is None else common_name
if len(common_name) > 200:
common_name = common_name[:200]
@@ -71,6 +70,7 @@ def extract_info_from_off(product, language):
source_name = Source.OPEN_FOOD_FACTS.value
source_url = f'https://world.openfoodfacts.org/api/v2/product/{code}.json'
authors = ', '.join(product.get('editors_tags', ['open food facts']))
+ object_url = f'https://world.openfoodfacts.org/product/{code}/'
return {
'name': name,
@@ -89,8 +89,8 @@ def extract_info_from_off(product, language):
'common_name': common_name,
'brand': brand,
'status': AbstractSubmissionModel.STATUS_ACCEPTED,
- 'license_id': CC_ODBL_LICENSE_ID,
+ 'license_id': ODBL_LICENSE_ID,
'license_author': authors,
'license_title': name,
- 'license_object_url': f'https://world.openfoodfacts.org/product/{code}/'
+ 'license_object_url': object_url
}
diff --git a/wger/utils/constants.py b/wger/utils/constants.py
index 449831b03..58d4fa47b 100644
--- a/wger/utils/constants.py
+++ b/wger/utils/constants.py
@@ -51,7 +51,7 @@ PAGINATION_PAGES_AROUND_CURRENT = 5
# Important license IDs
CC_BY_SA_4_ID = 2
CC_BY_SA_3_LICENSE_ID = 1
-CC_ODBL_LICENSE_ID = 5
+ODBL_LICENSE_ID = 5
# Default/fallback language
ENGLISH_SHORT_NAME = 'en'