Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Merge branch 'refs/heads/fork/import_off_jsonl' into feature/import-usda-products

Roland Geidera6733be59181
diff --cc AUTHORS.rst
index 02dfe98e1,7dd2dfd1d..59efd849f
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@@ -75,9 -75,10 +75,11 @@@ Developer
* Gabriel Liss - https://github.com/gabeliss
* Alexandra Rhodes - https://github.com/arhodes130
* Jayanth Bontha - https://github.com/JayanthBontha
+ * Roelof Rietbroek https://github.com/strawpants
* Ethan Winters - https://github.com/ebwinters
+* Dieter Plaetinck - https://github.com/Dieterbe
+
Translators
-----------
diff --cc wger/nutrition/management/commands/import-off-products.py
index c40d9486c,817561562..e76d4af05
--- a/wger/nutrition/management/commands/import-off-products.py
+++ b/wger/nutrition/management/commands/import-off-products.py
@@@ -13,7 -13,13 +13,8 @@@
# You should have received a copy of the GNU Affero General Public License
# Standard Library
-import enum
import logging
+ import os
-from collections import Counter
-
-# Django
-from django.core.management.base import BaseCommand
# wger
from wger.core.models import Language
@@@ -34,13 -47,86 +35,54 @@@ class Command(ImportProductCommand)
help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'
+ def add_arguments(self, parser):
- parser.add_argument(
- '--set-mode',
- action='store',
- default=10,
- dest='mode',
- type=str,
- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '
- 'entries in the database, while update will try to update them if they are '
- 'already present. Default: insert'
- )
- parser.add_argument(
- '--completeness',
- action='store',
- default=0.7,
- dest='completeness',
- type=float,
- help='Completeness threshold for importing the products. Products in OFF have '
- 'completeness score that ranges from 0 to 1.1. Default: 0.7'
- )
++ super().add_arguments(parser)
+
+ parser.add_argument(
+ '--jsonl',
+ action='store_true',
+ default=False,
+ dest='usejsonl',
+ help='Use the JSONL dump of the Open Food Facts database.'
+ '(this option does not require mongo)'
+ )
-
- def products_mongo(self,filterdict):
- """returns a mongo iterator with filtered prodcuts"""
- try:
- # Third Party
- from pymongo import MongoClient
- except ImportError:
- self.stdout.write('Please install pymongo, `pip install pymongo`')
- return
+
- client = MongoClient('mongodb://off:[email protected]', port=27017)
- db = client.admin
- return db.products.find(filterdict)
-
+ def products_jsonl(self,languages,completeness):
+ import json
+ import requests
+ from gzip import GzipFile
+ off_url='https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'
+ download_dir=os.path.expanduser('~/.cache/off_cache')
+ os.makedirs(download_dir,exist_ok=True)
+ gzipdb=os.path.join(download_dir,os.path.basename(off_url))
+ if os.path.exists(gzipdb):
+ self.stdout.write(f'Already downloaded {gzipdb}, skipping download')
+ else:
+ self.stdout.write(f'downloading {gzipdb}... (this may take a while)')
+ req=requests.get(off_url,stream=True)
+ with open(gzipdb,'wb') as fid:
+ for chunk in req.iter_content(chunk_size=50*1024):
+ fid.write(chunk)
+ with GzipFile(gzipdb,'rb') as gzid:
+ for line in gzid:
+ try:
+ product=json.loads(line)
+ if product['completeness'] < completeness:
+ continue
+ if not product['lang'] in languages:
+ continue
+ yield product
+ except:
+ self.stdout.write(f' Error parsing and/or filtering json record, skipping')
+ continue
+
-
-
-
def handle(self, **options):
-
+ try:
+ # Third Party
+ from pymongo import MongoClient
+ except ImportError:
+ self.stdout.write('Please install pymongo, `pip install pymongo`')
+ return
if options['mode'] == 'insert':
self.mode = Mode.INSERT
diff --cc wger/utils/models.py
index 699560924,72dcce863..21ba70eab
--- a/wger/utils/models.py
+++ b/wger/utils/models.py
@@@ -58,12 -59,12 +58,12 @@@ class AbstractLicenseModel(models.Model
blank=True,
)
- license_author = models.CharField(
+ license_author = models.TextField(
verbose_name=_('Author(s)'),
- max_length=600,
+ max_length=3500,
blank=True,
null=True,
- help_text=_('If you are not the author, enter the name or source here.')
+ help_text=_('If you are not the author, enter the name or source here.'),
)
license_author_url = models.URLField(