Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Refactor import scripts

Roland Geider3dbeb62777bd
diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py
index e76d4af05..1e8eed13b 100644
--- a/wger/nutrition/management/commands/import-off-products.py
+++ b/wger/nutrition/management/commands/import-off-products.py
@@ -16,6 +16,10 @@
import logging
import os
+# Third Party
+import requests
+from pymongo import MongoClient
+
# wger
from wger.core.models import Language
from wger.nutrition.management.products import (
@@ -42,39 +46,87 @@ class Command(ImportProductCommand):
'--jsonl',
action='store_true',
default=False,
- dest='usejsonl',
+ dest='use_jsonl',
help='Use the JSONL dump of the Open Food Facts database.'
- '(this option does not require mongo)'
+ '(this option does not require mongo)',
)
- def products_jsonl(self,languages,completeness):
- import json
- import requests
- from gzip import GzipFile
- off_url='https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'
- download_dir=os.path.expanduser('~/.cache/off_cache')
- os.makedirs(download_dir,exist_ok=True)
- gzipdb=os.path.join(download_dir,os.path.basename(off_url))
- if os.path.exists(gzipdb):
- self.stdout.write(f'Already downloaded {gzipdb}, skipping download')
- else:
- self.stdout.write(f'downloading {gzipdb}... (this may take a while)')
- req=requests.get(off_url,stream=True)
- with open(gzipdb,'wb') as fid:
- for chunk in req.iter_content(chunk_size=50*1024):
- fid.write(chunk)
- with GzipFile(gzipdb,'rb') as gzid:
- for line in gzid:
- try:
- product=json.loads(line)
- if product['completeness'] < completeness:
- continue
- if not product['lang'] in languages:
- continue
- yield product
- except:
- self.stdout.write(f' Error parsing and/or filtering json record, skipping')
- continue
+ parser.add_argument(
+ '--delta-updates',
+ action='store_true',
+ default=False,
+ dest='delta_updates',
+ help='Downloads and imports the most recent delta file',
+ )
+
+ def import_mongo(self, languages: dict[str:int]):
+ client = MongoClient('mongodb://off:[email protected]', port=27017)
+ db = client.admin
+ for product in db.products.find({'lang': {'$in': list(languages.keys())}}):
+ try:
+ ingredient_data = extract_info_from_off(product, languages[product['lang']])
+ except KeyError as e:
+ # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
+ # self.stdout.write(repr(e))
+ # pprint(product)
+ self.counter['skipped'] += 1
+ else:
+ self.process_ingredient(ingredient_data)
+
+ def import_daily_delta(self, languages: dict[str:int]):
+ download_folder, tmp_folder = self.get_download_folder(
+ '/Users/roland/Entwicklung/wger/server/extras/usda'
+ )
+
+ base_url = 'https://static.openfoodfacts.org/data/delta/'
+
+ # Fetch the index page with requests and read the result
+ index_url = base_url + 'index.txt'
+ req = requests.get(index_url)
+ index_content = req.text
+ newest_entry = index_content.split('\n')[0]
+
+ file_path = os.path.join(download_folder, newest_entry)
+
+ # Fetch the newest entry and extract the contents
+ delta_url = base_url + newest_entry
+ self.download_file(delta_url, file_path)
+
+ for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):
+ try:
+ ingredient_data = extract_info_from_off(entry, languages[entry['lang']])
+ except KeyError as e:
+ # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
+ self.counter['skipped'] += 1
+ else:
+ self.process_ingredient(ingredient_data)
+
+ if tmp_folder:
+ self.stdout.write(f'Removing temporary folder {download_folder}')
+ tmp_folder.cleanup()
+
+ def import_full_dump(self, languages: dict[str:int]):
+ off_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'
+
+ download_folder, tmp_folder = self.get_download_folder(
+ '/Users/roland/Entwicklung/wger/server/extras/open-food-facts/dump'
+ )
+
+ file_path = os.path.join(download_folder, os.path.basename(off_url))
+ self.download_file(off_url, file_path)
+
+ for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):
+ try:
+ ingredient_data = extract_info_from_off(entry, languages[entry['lang']])
+ except KeyError as e:
+ # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
+ self.counter['skipped'] += 1
+ else:
+ self.process_ingredient(ingredient_data)
+
+ if tmp_folder:
+ self.stdout.write(f'Removing temporary folder {download_folder}')
+ tmp_folder.cleanup()
def handle(self, **options):
try:
@@ -91,21 +143,13 @@ class Command(ImportProductCommand):
self.stdout.write(f' - {self.mode}')
self.stdout.write('')
- client = MongoClient('mongodb://off:[email protected]', port=27017)
- db = client.admin
-
languages = {lang.short_name: lang.pk for lang in Language.objects.all()}
-
- for product in db.products.find({'lang': {'$in': list(languages.keys())}}):
- try:
- ingredient_data = extract_info_from_off(product, languages[product['lang']])
- except KeyError as e:
- # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
- # self.stdout.write(repr(e))
- # pprint(product)
- self.counter['skipped'] += 1
- else:
- self.handle_data(ingredient_data)
+ if options['delta_updates']:
+ self.import_daily_delta(languages)
+ elif options['use_jsonl']:
+ self.import_full_dump(languages)
+ else:
+ self.import_mongo(languages)
self.stdout.write(self.style.SUCCESS('Finished!'))
self.stdout.write(self.style.SUCCESS(str(self.counter)))
diff --git a/wger/nutrition/management/commands/import-usda-products.py b/wger/nutrition/management/commands/import-usda-products.py
index 831dd55ba..160c9b96b 100644
--- a/wger/nutrition/management/commands/import-usda-products.py
+++ b/wger/nutrition/management/commands/import-usda-products.py
@@ -16,13 +16,9 @@
import json
import logging
import os
-import tempfile
from json import JSONDecodeError
from zipfile import ZipFile
-# Third Party
-import requests
-
# wger
from wger.core.models import Language
from wger.nutrition.dataclasses import IngredientData
@@ -33,7 +29,6 @@ from wger.nutrition.management.products import (
from wger.nutrition.models import Ingredient
from wger.nutrition.usda import extract_info_from_usda
from wger.utils.constants import ENGLISH_SHORT_NAME
-from wger.utils.requests import wger_headers
logger = logging.getLogger(__name__)
@@ -47,18 +42,6 @@ class Command(ImportProductCommand):
def add_arguments(self, parser):
super().add_arguments(parser)
- parser.add_argument(
- '--folder',
- action='store',
- default='',
- dest='tmp_folder',
- type=str,
- help='Controls whether to use a temporary folder created by python (the default) or '
- 'the path provided for storing the downloaded dataset. If there are already '
- 'downloaded or extracted files here, they will be used instead of fetching them '
- 'again.',
- )
-
parser.add_argument(
'--dataset',
action='store',
@@ -66,55 +49,31 @@ class Command(ImportProductCommand):
dest='dataset',
type=str,
help='What dataset to download, this value will be appended to '
- '"https://fdc.nal.usda.gov/fdc-datasets/". Consult '
- 'https://fdc.nal.usda.gov/download-datasets.html for current file names',
+ '"https://fdc.nal.usda.gov/fdc-datasets/". Consult '
+ 'https://fdc.nal.usda.gov/download-datasets.html for current file names',
)
def handle(self, **options):
if options['mode'] == 'insert':
self.mode = Mode.INSERT
- usda_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}'
-
- if options['folder']:
- download_folder = options['folder']
-
- # Check whether the folder exists
- if not os.path.exists(download_folder):
- self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!'))
- return
- else:
- tmp_folder = tempfile.TemporaryDirectory()
- download_folder = tmp_folder.name
+ dataset_url = f'https://fdc.nal.usda.gov/fdc-datasets/{options["dataset"]}'
+ download_folder, tmp_folder = self.get_download_folder(options['folder'])
self.stdout.write('Importing entries from USDA')
self.stdout.write(f' - {self.mode}')
- self.stdout.write(f' - dataset: {usda_url}')
+ self.stdout.write(f' - dataset: {dataset_url}')
self.stdout.write(f' - download folder: {download_folder}')
self.stdout.write('')
english = Language.objects.get(short_name=ENGLISH_SHORT_NAME)
# Download the dataset
- zip_file = os.path.join(download_folder, options['dataset'])
- if os.path.exists(zip_file):
- self.stdout.write(f'File already downloaded {zip_file}, not downloading it again')
- else:
- self.stdout.write(f'Downloading {usda_url}... (this may take a while)')
- req = requests.get(usda_url, stream=True, headers=wger_headers())
-
- if req.status_code == 404:
- self.stdout.write(self.style.ERROR(f'Could not open {usda_url}!'))
- return
-
- with open(zip_file, 'wb') as fid:
- for chunk in req.iter_content(chunk_size=50 * 1024):
- fid.write(chunk)
-
- self.stdout.write('download successful')
+ zip_file_path = os.path.join(download_folder, options['dataset'])
+ self.download_file(dataset_url, zip_file_path)
# Extract the first file from the ZIP archive
- with ZipFile(zip_file, 'r') as zip_ref:
+ with ZipFile(zip_file_path, 'r') as zip_ref:
file_list = zip_ref.namelist()
if not file_list:
raise Exception('No files found in the ZIP archive')
@@ -154,11 +113,10 @@ class Command(ImportProductCommand):
self.stdout.write(f'--> ValueError while extracting ingredient info: {e}')
self.counter['skipped'] += 1
else:
- # pass
self.match_existing_entry(ingredient_data)
# self.handle_data(ingredient_data)
- if not options['folder']:
+ if tmp_folder:
self.stdout.write(f'Removing temporary folder {download_folder}')
tmp_folder.cleanup()
diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py
index 50965e549..fa8205f13 100644
--- a/wger/nutrition/management/products.py
+++ b/wger/nutrition/management/products.py
@@ -14,15 +14,26 @@
# Standard Library
import enum
+import json
import logging
+import os
+import tempfile
from collections import Counter
+from gzip import GzipFile
+from json import JSONDecodeError
+from typing import Optional
# Django
from django.core.management.base import BaseCommand
+# Third Party
+import requests
+from tqdm import tqdm
+
# wger
from wger.nutrition.dataclasses import IngredientData
from wger.nutrition.models import Ingredient
+from wger.utils.requests import wger_headers
logger = logging.getLogger(__name__)
@@ -66,10 +77,22 @@ class ImportProductCommand(BaseCommand):
'already present. Default: update',
)
+ parser.add_argument(
+ '--folder',
+ action='store',
+ default='',
+ dest='folder',
+ type=str,
+ help='Controls whether to use a temporary folder created by python (the default) or '
+ 'the path provided for storing the downloaded dataset. If there are already '
+ 'downloaded or extracted files here, they will be used instead of fetching them '
+ 'again.',
+ )
+
def handle(self, **options):
raise NotImplementedError('Do not run this command on its own!')
- def handle_data(self, ingredient_data: IngredientData):
+ def process_ingredient(self, ingredient_data: IngredientData):
#
# Add entries as new products
if self.mode == Mode.INSERT:
@@ -120,3 +143,59 @@ class ImportProductCommand(BaseCommand):
self.stdout.write(repr(e))
# self.stdout.write(repr(ingredient_data))
self.counter['error'] += 1
+
+ def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.TemporaryDirectory]]:
+ if folder:
+ tmp_folder = None
+ download_folder = folder
+
+ # Check whether the folder exists
+ if not os.path.exists(download_folder):
+ self.stdout.write(self.style.ERROR(f'Folder {download_folder} does not exist!'))
+ raise Exception('Folder does not exist')
+ else:
+ tmp_folder = tempfile.TemporaryDirectory()
+ download_folder = tmp_folder.name
+
+ return download_folder, tmp_folder
+
+ def parse_file_content(self, path: str):
+ with GzipFile(path, 'rb') as gzid:
+ for line in gzid:
+ try:
+ product = json.loads(line)
+ yield product
+ except JSONDecodeError as e:
+ self.stdout.write(f' Error parsing and/or filtering json record, skipping')
+ continue
+
+ def iterate_gz_file_contents(self, path: str, languages: list[str]):
+ with GzipFile(path, 'rb') as gzid:
+ for line in gzid:
+ try:
+ product = json.loads(line)
+ if not product['lang'] in languages:
+ continue
+ yield product
+ except JSONDecodeError as e:
+ self.stdout.write(f' Error parsing and/or filtering json record, skipping')
+ continue
+
+ def download_file(self, url: str, destination: str) -> None:
+ if os.path.exists(destination):
+ self.stdout.write(f'File already downloaded {destination}, not downloading it again')
+ return
+
+ self.stdout.write(f'Downloading {url}... (this may take a while)')
+ response = requests.get(url, stream=True, headers=wger_headers())
+ total_size = int(response.headers.get('content-length', 0))
+ size = int(response.headers['content-length']) / (1024 * 1024)
+
+ if response.status_code == 404:
+ raise Exception(f'Could not open {url}!')
+
+ with open(destination, 'wb') as fid:
+ with tqdm(total=total_size, unit='B', unit_scale=True, desc='Downloading') as pbar:
+ for chunk in response.iter_content(chunk_size=50 * 1024):
+ fid.write(chunk)
+ pbar.update(len(chunk))
diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py
index 57ac8be2e..2b367ee0d 100644
--- a/wger/nutrition/off.py
+++ b/wger/nutrition/off.py
@@ -22,6 +22,7 @@ from wger.nutrition.models import Source
from wger.utils.constants import ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel
+
OFF_REQUIRED_TOP_LEVEL = [
'product_name',
'code',
diff --git a/wger/utils/models.py b/wger/utils/models.py
index 21ba70eab..aa6660b2d 100644
--- a/wger/utils/models.py
+++ b/wger/utils/models.py
@@ -20,6 +20,7 @@ from django.utils.translation import gettext_lazy as _
from wger.core.models import License
from wger.utils.constants import CC_BY_SA_4_ID
+
"""
Abstract model classes
"""