Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Refactor and remove hard coded paths

Roland Geider69b53e543c53
diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py
index 1e8eed13b..a841a838c 100644
--- a/wger/nutrition/management/commands/import-off-products.py
+++ b/wger/nutrition/management/commands/import-off-products.py
@@ -28,7 +28,6 @@ from wger.nutrition.management.products import (
)
from wger.nutrition.off import extract_info_from_off
-
logger = logging.getLogger(__name__)
@@ -39,6 +38,9 @@ class Command(ImportProductCommand):
help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'
+ deltas_base_url = 'https://static.openfoodfacts.org/data/delta/'
+ full_off_dump_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'
+
def add_arguments(self, parser):
super().add_arguments(parser)
@@ -48,7 +50,7 @@ class Command(ImportProductCommand):
default=False,
dest='use_jsonl',
help='Use the JSONL dump of the Open Food Facts database.'
- '(this option does not require mongo)',
+ '(this option does not require mongo)',
)
parser.add_argument(
@@ -65,23 +67,17 @@ class Command(ImportProductCommand):
for product in db.products.find({'lang': {'$in': list(languages.keys())}}):
try:
ingredient_data = extract_info_from_off(product, languages[product['lang']])
- except KeyError as e:
+ except (KeyError, ValueError) as e:
# self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
- # self.stdout.write(repr(e))
- # pprint(product)
self.counter['skipped'] += 1
else:
self.process_ingredient(ingredient_data)
- def import_daily_delta(self, languages: dict[str:int]):
- download_folder, tmp_folder = self.get_download_folder(
- '/Users/roland/Entwicklung/wger/server/extras/usda'
- )
-
- base_url = 'https://static.openfoodfacts.org/data/delta/'
+ def import_daily_delta(self, languages: dict[str:int], destination: str):
+ download_folder, tmp_folder = self.get_download_folder(destination)
# Fetch the index page with requests and read the result
- index_url = base_url + 'index.txt'
+ index_url = self.deltas_base_url + 'index.txt'
req = requests.get(index_url)
index_content = req.text
newest_entry = index_content.split('\n')[0]
@@ -89,14 +85,16 @@ class Command(ImportProductCommand):
file_path = os.path.join(download_folder, newest_entry)
# Fetch the newest entry and extract the contents
- delta_url = base_url + newest_entry
+ delta_url = self.deltas_base_url + newest_entry
self.download_file(delta_url, file_path)
for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):
try:
ingredient_data = extract_info_from_off(entry, languages[entry['lang']])
- except KeyError as e:
- # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
+ except (KeyError, ValueError) as e:
+ self.stdout.write(
+ f'--> {ingredient_data.remote_id=} KeyError while extracting info from OFF: {e}'
+ )
self.counter['skipped'] += 1
else:
self.process_ingredient(ingredient_data)
@@ -105,20 +103,16 @@ class Command(ImportProductCommand):
self.stdout.write(f'Removing temporary folder {download_folder}')
tmp_folder.cleanup()
- def import_full_dump(self, languages: dict[str:int]):
- off_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'
+ def import_full_dump(self, languages: dict[str:int], destination: str):
+ download_folder, tmp_folder = self.get_download_folder(destination)
- download_folder, tmp_folder = self.get_download_folder(
- '/Users/roland/Entwicklung/wger/server/extras/open-food-facts/dump'
- )
-
- file_path = os.path.join(download_folder, os.path.basename(off_url))
- self.download_file(off_url, file_path)
+ file_path = os.path.join(download_folder, os.path.basename(self.full_off_dump_url))
+ self.download_file(self.full_off_dump_url, file_path)
for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):
try:
ingredient_data = extract_info_from_off(entry, languages[entry['lang']])
- except KeyError as e:
+ except (KeyError, ValueError) as e:
# self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
self.counter['skipped'] += 1
else:
@@ -141,13 +135,19 @@ class Command(ImportProductCommand):
self.stdout.write('Importing entries from Open Food Facts')
self.stdout.write(f' - {self.mode}')
+ if options['delta_updates']:
+ self.stdout.write(f' - importing only delta updates')
+ elif options['use_jsonl']:
+ self.stdout.write(f' - importing the full dump')
+ else:
+ self.stdout.write(f' - importing from mongo')
self.stdout.write('')
languages = {lang.short_name: lang.pk for lang in Language.objects.all()}
if options['delta_updates']:
- self.import_daily_delta(languages)
+ self.import_daily_delta(languages, options['folder'])
elif options['use_jsonl']:
- self.import_full_dump(languages)
+ self.import_full_dump(languages, options['folder'])
else:
self.import_mongo(languages)
diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py
index fa8205f13..86cc0b611 100644
--- a/wger/nutrition/management/products.py
+++ b/wger/nutrition/management/products.py
@@ -72,9 +72,11 @@ class ImportProductCommand(BaseCommand):
default='update',
dest='mode',
type=str,
- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '
- 'entries in the database, while update will try to update them if they are '
- 'already present. Default: update',
+ help=(
+ 'Script mode, "insert" or "update". Insert will insert the ingredients as new '
+ 'entries in the database, while update will try to update them if they are '
+ 'already present. Default: update'
+ ),
)
parser.add_argument(
@@ -83,10 +85,12 @@ class ImportProductCommand(BaseCommand):
default='',
dest='folder',
type=str,
- help='Controls whether to use a temporary folder created by python (the default) or '
- 'the path provided for storing the downloaded dataset. If there are already '
- 'downloaded or extracted files here, they will be used instead of fetching them '
- 'again.',
+ help=(
+ 'Controls whether to use a temporary folder created by python (the default) or '
+ 'the path provided for storing the downloaded dataset. If there are already '
+ 'downloaded or extracted files here, they will be used instead of fetching them '
+ 'again.'
+ ),
)
def handle(self, **options):
@@ -139,9 +143,8 @@ class ImportProductCommand(BaseCommand):
# self.stdout.write('-> updated')
except Exception as e:
- self.stdout.write('--> Error while performing update_or_create')
- self.stdout.write(repr(e))
- # self.stdout.write(repr(ingredient_data))
+ self.stdout.write(f'--> Error while performing update_or_create: {e}')
+ self.stdout.write(repr(ingredient_data))
self.counter['error'] += 1
def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.TemporaryDirectory]]:
@@ -157,18 +160,9 @@ class ImportProductCommand(BaseCommand):
tmp_folder = tempfile.TemporaryDirectory()
download_folder = tmp_folder.name
+ self.stdout.write(f'Using folder {download_folder} for storing downloaded files')
return download_folder, tmp_folder
- def parse_file_content(self, path: str):
- with GzipFile(path, 'rb') as gzid:
- for line in gzid:
- try:
- product = json.loads(line)
- yield product
- except JSONDecodeError as e:
- self.stdout.write(f' Error parsing and/or filtering json record, skipping')
- continue
-
def iterate_gz_file_contents(self, path: str, languages: list[str]):
with GzipFile(path, 'rb') as gzid:
for line in gzid:
@@ -183,13 +177,12 @@ class ImportProductCommand(BaseCommand):
def download_file(self, url: str, destination: str) -> None:
if os.path.exists(destination):
- self.stdout.write(f'File already downloaded {destination}, not downloading it again')
+ self.stdout.write(f'File already downloaded at {destination}')
return
self.stdout.write(f'Downloading {url}... (this may take a while)')
response = requests.get(url, stream=True, headers=wger_headers())
total_size = int(response.headers.get('content-length', 0))
- size = int(response.headers['content-length']) / (1024 * 1024)
if response.status_code == 404:
raise Exception(f'Could not open {url}!')