Self hosted FLOSS fitness/workout, nutrition and weight tracker
Refactor import scripts
Roland Geider3dbeb62777bd
diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.pyindex e76d4af05..1e8eed13b 100644--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@ -16,6 +16,10 @@import loggingimport os+# Third Party+import requests+from pymongo import MongoClient+# wgerfrom wger.core.models import Languagefrom wger.nutrition.management.products import (@@ -42,39 +46,87 @@ class Command(ImportProductCommand):'--jsonl',action='store_true',default=False,- dest='usejsonl',+ dest='use_jsonl',help='Use the JSONL dump of the Open Food Facts database.'- '(this option does not require mongo)'+ '(this option does not require mongo)',)- def products_jsonl(self,languages,completeness):- import json- import requests- from gzip import GzipFile- off_url='https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'- download_dir=os.path.expanduser('~/.cache/off_cache')- os.makedirs(download_dir,exist_ok=True)- gzipdb=os.path.join(download_dir,os.path.basename(off_url))- if os.path.exists(gzipdb):- self.stdout.write(f'Already downloaded {gzipdb}, skipping download')- else:- self.stdout.write(f'downloading {gzipdb}... (this may take a while)')- req=requests.get(off_url,stream=True)- with open(gzipdb,'wb') as fid:- for chunk in req.iter_content(chunk_size=50*1024):- fid.write(chunk)- with GzipFile(gzipdb,'rb') as gzid:- for line in gzid:- try:- product=json.loads(line)- if product['completeness'] < completeness:- continue- if not product['lang'] in languages:- continue- yield product- except:- self.stdout.write(f' Error parsing and/or filtering json record, skipping')- continue+ parser.add_argument(+ '--delta-updates',+ action='store_true',+ default=False,+ dest='delta_updates',+ help='Downloads and imports the most recent delta file',+ )++ def import_mongo(self, languages: dict[str:int]):+ client = MongoClient('mongodb://off:[email protected]', port=27017)+ db = client.admin+ for product in db.products.find({'lang': {'$in': list(languages.keys())}}):+ try:+ ingredient_data = extract_info_from_off(product, languages[product['lang']])+ except KeyError as e:+ # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')+ # self.stdout.write(repr(e))+ # pprint(product)+ self.counter['skipped'] += 1+ else:+ self.process_ingredient(ingredient_data)++ def import_daily_delta(self, languages: dict[str:int]):+ download_folder, tmp_folder = self.get_download_folder(+ '/Users/roland/Entwicklung/wger/server/extras/usda'+ )++ base_url = 'https://static.openfoodfacts.org/data/delta/'++ # Fetch the index page with requests and read the result+ index_url = base_url + 'index.txt'+ req = requests.get(index_url)+ index_content = req.text+ newest_entry = index_content.split('\n')[0]++ file_path = os.path.join(download_folder, newest_entry)++ # Fetch the newest entry and extract the contents+ delta_url = base_url + newest_entry+ self.download_file(delta_url, file_path)++ for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):+ try:+ ingredient_data = extract_info_from_off(entry, languages[entry['lang']])+ except KeyError as e:+ # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')+ self.counter['skipped'] += 1+ else:+ self.process_ingredient(ingredient_data)++ if tmp_folder:+ self.stdout.write(f'Removing temporary folder {download_folder}')+ tmp_folder.cleanup()++ def import_full_dump(self, languages: dict[str:int]):+ off_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'++ download_folder, tmp_folder = self.get_download_folder(+ '/Users/roland/Entwicklung/wger/server/extras/open-food-facts/dump'+ )++ file_path = os.path.join(download_folder, os.path.basename(off_url))+ self.download_file(off_url, file_path)++ for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):+ try:+ ingredient_data = extract_info_from_off(entry, languages[entry['lang']])+ except KeyError as e:+ # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')+ self.counter['skipped'] += 1+ else:+ self.process_ingredient(ingredient_data)++ if tmp_folder:+ self.stdout.write(f'Removing temporary folder {download_folder}')+ tmp_folder.cleanup()def handle(self, **options):try:@@ -91,21 +143,13 @@ class Command(ImportProductCommand):self.stdout.write(f' - {self.mode}')self.stdout.write('')- client = MongoClient('mongodb://off:[email protected]', port=27017)- db = client.admin-languages = {lang.short_name: lang.pk for lang in Language.objects.all()}-- for product in db.products.find({'lang': {'$in': list(languages.keys())}}):- try:- ingredient_data = extract_info_from_off(product, languages[product['lang']])- except KeyError as e:- # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')- # self.stdout.write(repr(e))- # pprint(product)- self.counter['skipped'] += 1- else:- self.handle_data(ingredient_data)+ if options['delta_updates']:+ self.import_daily_delta(languages)+ elif options['use_jsonl']:+ self.import_full_dump(languages)+ else:+ self.import_mongo(languages)self.stdout.write(self.style.SUCCESS('Finished!'))self.stdout.write(self.style.SUCCESS(str(self.counter)))diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.pyindex 831dd55ba..160c9b96b 100644--- a/wger/nutrition/management/commands/import-usda-products.py+++ b/wger/nutrition/management/commands/import-usda-products.py@@ -16,13 +16,9 @@import jsonimport loggingimport os-import tempfilefrom json import JSONDecodeErrorfrom zipfile import ZipFile-# Third Party-import requests-# wgerfrom wger.core.models import Languagefrom wger.nutrition.dataclasses import IngredientData@@ -33,7 +29,6 @@ from wger.nutrition.management.products import (from wger.nutrition.models import Ingredientfrom wger.nutrition.usda import extract_info_from_usdafrom wger.utils.constants import ENGLISH_SHORT_NAME-from wger.utils.requests import wger_headerslogger = logging.getLogger(__name__)@@ -47,18 +42,6 @@ class Command(ImportProductCommand):def add_arguments(self, parser):super().add_arguments(parser)- parser.add_argument(- '--folder',- action='store',- default='',- dest='tmp_folder',- type=str,- help='Controls whether to use a temporary folder created by python (the default) or '- 'the path provided for storing the downloaded dataset. If there are already '- 'downloaded or extracted files here, they will be used instead of fetching them '- 'again.',- )-parser.add_argument('--dataset',action='store',@@ -66,55 +49,31 @@ class Command(ImportProductCommand):dest='dataset',type=str,help='What dataset to download, this value will be appended to '- '"https://fdc.nal.usda.gov/fdc-datasets/". Consult '- 'https://fdc.nal.usda.gov/download-datasets.html for current file names',+ '"https://fdc.nal.usda.gov/fdc-datasets/". Consult '+ 'https://fdc.nal.usda.gov/download-datasets.html for current file names',)def handle(self, **options):if options['mode'] == 'insert':self.mode = Mode.INSERT- usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}'-- if options['folder']:- download_folder = options['folder']-- # Check whether the folder exists- if not os.path.exists(download_folder):- self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!'))- return- else:- tmp_folder = tempfile.TemporaryDirectory()- download_folder = tmp_folder.name+ dataset_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}'+ download_folder, tmp_folder = self.get_download_folder(options['folder'])self.stdout.write('Importing entries from USDA')self.stdout.write(f' - {self.mode}')- self.stdout.write(f' - dataset: {usda_url}')+ self.stdout.write(f' - dataset: {dataset_url}')self.stdout.write(f' - download folder: {download_folder}')self.stdout.write('')english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)# Download the dataset- zip_file = os.path.join(download_folder, options['dataset'])- if os.path.exists(zip_file):- self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')- else:- self.stdout.write(f'Downloading {usda_url}... (this may take a while)')- req = requests.get(usda_url, stream=True, headers=wger_headers())-- if req.status_code == 404:- self.stdout.write(self.style.ERROR(f'Could not open {usda_url}!'))- return-- with open(zip_file, 'wb') as fid:- for chunk in req.iter_content(chunk_size=50 * 1024):- fid.write(chunk)-- self.stdout.write('download successful')+ zip_file_path = os.path.join(download_folder, options['dataset'])+ self.download_file(dataset_url, zip_file_path)# Extract the first file from the ZIP archive- with ZipFile(zip_file, 'r') as zip_ref:+ with ZipFile(zip_file_path, 'r') as zip_ref:file_list = zip_ref.namelist()if not file_list:raise Exception('No files found in the ZIP archive')@@ -154,11 +113,10 @@ class Command(ImportProductCommand):self.stdout.write(f'--> ValueError while extracting ingredient info: {e}')self.counter['skipped'] += 1else:- # passself.match_existing_entry(ingredient_data)# self.handle_data(ingredient_data)- if not options['folder']:+ if tmp_folder:self.stdout.write(f'Removing temporary folder {download_folder}')tmp_folder.cleanup()diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.pyindex 50965e549..fa8205f13 100644--- a/wger/nutrition/management/products.py+++ b/wger/nutrition/management/products.py@@ -14,15 +14,26 @@# Standard Libraryimport enum+import jsonimport logging+import os+import tempfilefrom collections import Counter+from gzip import GzipFile+from json import JSONDecodeError+from typing import Optional# Djangofrom django.core.management.base import BaseCommand+# Third Party+import requests+from tqdm import tqdm+# wgerfrom wger.nutrition.dataclasses import IngredientDatafrom wger.nutrition.models import Ingredient+from wger.utils.requests import wger_headerslogger = logging.getLogger(__name__)@@ -66,10 +77,22 @@ class ImportProductCommand(BaseCommand):'already present. Default: update',)+ parser.add_argument(+ '--folder',+ action='store',+ default='',+ dest='folder',+ type=str,+ help='Controls whether to use a temporary folder created by python (the default) or '+ 'the path provided for storing the downloaded dataset. If there are already '+ 'downloaded or extracted files here, they will be used instead of fetching them '+ 'again.',+ )+def handle(self, **options):raise NotImplementedError('Do not run this command on its own!')- def handle_data(self, ingredient_data: IngredientData):+ def process_ingredient(self, ingredient_data: IngredientData):## Add entries as new productsif self.mode == Mode.INSERT:@@ -120,3 +143,59 @@ class ImportProductCommand(BaseCommand):self.stdout.write(repr(e))# self.stdout.write(repr(ingredient_data))self.counter['error'] += 1++ def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.TemporaryDirectory]]:+ if folder:+ tmp_folder = None+ download_folder = folder++ # Check whether the folder exists+ if not os.path.exists(download_folder):+ self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!'))+ raise Exception('Folder does not exist')+ else:+ tmp_folder = tempfile.TemporaryDirectory()+ download_folder = tmp_folder.name++ return download_folder, tmp_folder++ def parse_file_content(self, path: str):+ with GzipFile(path, 'rb') as gzid:+ for line in gzid:+ try:+ product = json.loads(line)+ yield product+ except JSONDecodeError as e:+ self.stdout.write(f' Error parsing and/or filtering json record, skipping')+ continue++ def iterate_gz_file_contents(self, path: str, languages: list[str]):+ with GzipFile(path, 'rb') as gzid:+ for line in gzid:+ try:+ product = json.loads(line)+ if not product['lang'] in languages:+ continue+ yield product+ except JSONDecodeError as e:+ self.stdout.write(f' Error parsing and/or filtering json record, skipping')+ continue++ def download_file(self, url: str, destination: str) -> None:+ if os.path.exists(destination):+ self.stdout.write(f'File already downloaded {destination}, not downloading it again')+ return++ self.stdout.write(f'Downloading {url}... (this may take a while)')+ response = requests.get(url, stream=True, headers=wger_headers())+ total_size = int(response.headers.get('content-length', 0))+ size = int(response.headers['content-length']) / (1024 * 1024)++ if response.status_code == 404:+ raise Exception(f'Could not open {url}!')++ with open(destination, 'wb') as fid:+ with tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') as pbar:+ for chunk in response.iter_content(chunk_size=50 * 1024):+ fid.write(chunk)+ pbar.update(len(chunk))diff --git a/wger/nutrition/off.py b/wger/nutrition/off.pyindex 57ac8be2e..2b367ee0d 100644--- a/wger/nutrition/off.py+++ b/wger/nutrition/off.py@@ -22,6 +22,7 @@ from wger.nutrition.models import Sourcefrom wger.utils.constants import ODBL_LICENSE_IDfrom wger.utils.models import AbstractSubmissionModel+OFF_REQUIRED_TOP_LEVEL = ['product_name','code',diff --git a/wger/utils/models.py b/wger/utils/models.pyindex 21ba70eab..aa6660b2d 100644--- a/wger/utils/models.py+++ b/wger/utils/models.py@@ -20,6 +20,7 @@ from django.utils.translation import gettext_lazy as _from wger.core.models import Licensefrom wger.utils.constants import CC_BY_SA_4_ID+"""Abstract model classes"""