Self hosted FLOSS fitness/workout, nutrition and weight tracker
Merge branch 'refs/heads/fork/import_off_jsonl' into feature/import-usda-products
Roland Geidera6733be59181
diff --cc AUTHORS.rstindex 02dfe98e1,7dd2dfd1d..59efd849f--- a/AUTHORS.rst+++ b/AUTHORS.rst@@@ -75,9 -75,10 +75,11 @@@ Developer* Gabriel Liss - https://github.com/gabeliss* Alexandra Rhodes - https://github.com/arhodes130* Jayanth Bontha - https://github.com/JayanthBontha+ * Roelof Rietbroek https://github.com/strawpants* Ethan Winters - https://github.com/ebwinters+* Dieter Plaetinck - https://github.com/Dieterbe+Translators-----------diff --cc wger/nutrition/management/commands/import-off-products.pyindex c40d9486c,817561562..e76d4af05--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@@ -13,7 -13,13 +13,8 @@@# You should have received a copy of the GNU Affero General Public License# Standard Library-import enumimport logging+ import os-from collections import Counter--# Django-from django.core.management.base import BaseCommand# wgerfrom wger.core.models import Language@@@ -34,13 -47,86 +35,54 @@@ class Command(ImportProductCommand)help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'+ def add_arguments(self, parser):- parser.add_argument(- '--set-mode',- action='store',- default=10,- dest='mode',- type=str,- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '- 'entries in the database, while update will try to update them if they are '- 'already present. Default: insert'- )- parser.add_argument(- '--completeness',- action='store',- default=0.7,- dest='completeness',- type=float,- help='Completeness threshold for importing the products. Products in OFF have '- 'completeness score that ranges from 0 to 1.1. Default: 0.7'- )++ super().add_arguments(parser)++ parser.add_argument(+ '--jsonl',+ action='store_true',+ default=False,+ dest='usejsonl',+ help='Use the JSONL dump of the Open Food Facts database.'+ '(this option does not require mongo)'+ )-- def products_mongo(self,filterdict):- """returns a mongo iterator with filtered prodcuts"""- try:- # Third Party- from pymongo import MongoClient- except ImportError:- self.stdout.write('Please install pymongo, `pip install pymongo`')- return+- client = MongoClient('mongodb://off:[email protected]', port=27017)- db = client.admin- return db.products.find(filterdict)-+ def products_jsonl(self,languages,completeness):+ import json+ import requests+ from gzip import GzipFile+ off_url='https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'+ download_dir=os.path.expanduser('~/.cache/off_cache')+ os.makedirs(download_dir,exist_ok=True)+ gzipdb=os.path.join(download_dir,os.path.basename(off_url))+ if os.path.exists(gzipdb):+ self.stdout.write(f'Already downloaded {gzipdb}, skipping download')+ else:+ self.stdout.write(f'downloading {gzipdb}... (this may take a while)')+ req=requests.get(off_url,stream=True)+ with open(gzipdb,'wb') as fid:+ for chunk in req.iter_content(chunk_size=50*1024):+ fid.write(chunk)+ with GzipFile(gzipdb,'rb') as gzid:+ for line in gzid:+ try:+ product=json.loads(line)+ if product['completeness'] < completeness:+ continue+ if not product['lang'] in languages:+ continue+ yield product+ except:+ self.stdout.write(f' Error parsing and/or filtering json record, skipping')+ continue+---def handle(self, **options):-+ try:+ # Third Party+ from pymongo import MongoClient+ except ImportError:+ self.stdout.write('Please install pymongo, `pip install pymongo`')+ returnif options['mode'] == 'insert':self.mode = Mode.INSERTdiff --cc wger/utils/models.pyindex 699560924,72dcce863..21ba70eab--- a/wger/utils/models.py+++ b/wger/utils/models.py@@@ -58,12 -59,12 +58,12 @@@ class AbstractLicenseModel(models.Modelblank=True,)- license_author = models.CharField(+ license_author = models.TextField(verbose_name=_('Author(s)'),- max_length=600,+ max_length=3500,blank=True,null=True,- help_text=_('If you are not the author, enter the name or source here.')+ help_text=_('If you are not the author, enter the name or source here.'),)license_author_url = models.URLField(