Self hosted FLOSS fitness/workout, nutrition and weight tracker
Refactor and remove hard coded paths
Roland Geider69b53e543c53
diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.pyindex 1e8eed13b..a841a838c 100644--- a/wger/nutrition/management/commands/import-off-products.py+++ b/wger/nutrition/management/commands/import-off-products.py@@ -28,7 +28,6 @@ from wger.nutrition.management.products import ()from wger.nutrition.off import extract_info_from_off-logger = logging.getLogger(__name__)@@ -39,6 +38,9 @@ class Command(ImportProductCommand):help = 'Import an Open Food Facts dump. Please consult extras/docker/open-food-facts'+ deltas_base_url = 'https://static.openfoodfacts.org/data/delta/'+ full_off_dump_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'+def add_arguments(self, parser):super().add_arguments(parser)@@ -48,7 +50,7 @@ class Command(ImportProductCommand):default=False,dest='use_jsonl',help='Use the JSONL dump of the Open Food Facts database.'- '(this option does not require mongo)',+ '(this option does not require mongo)',)parser.add_argument(@@ -65,23 +67,17 @@ class Command(ImportProductCommand):for product in db.products.find({'lang': {'$in': list(languages.keys())}}):try:ingredient_data = extract_info_from_off(product, languages[product['lang']])- except KeyError as e:+ except (KeyError, ValueError) as e:# self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')- # self.stdout.write(repr(e))- # pprint(product)self.counter['skipped'] += 1else:self.process_ingredient(ingredient_data)- def import_daily_delta(self, languages: dict[str:int]):- download_folder, tmp_folder = self.get_download_folder(- '/Users/roland/Entwicklung/wger/server/extras/usda'- )-- base_url = 'https://static.openfoodfacts.org/data/delta/'+ def import_daily_delta(self, languages: dict[str:int], destination: str):+ download_folder, tmp_folder = self.get_download_folder(destination)# Fetch the index page with requests and read the result- index_url = base_url + 'index.txt'+ index_url = self.deltas_base_url + 'index.txt'req = requests.get(index_url)index_content = req.textnewest_entry = index_content.split('\n')[0]@@ -89,14 +85,16 @@ class Command(ImportProductCommand):file_path = os.path.join(download_folder, newest_entry)# Fetch the newest entry and extract the contents- delta_url = base_url + newest_entry+ delta_url = self.deltas_base_url + newest_entryself.download_file(delta_url, file_path)for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):try:ingredient_data = extract_info_from_off(entry, languages[entry['lang']])- except KeyError as e:- # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')+ except (KeyError, ValueError) as e:+ self.stdout.write(+ f'--> {ingredient_data.remote_id=} KeyError while extracting info from OFF: {e}'+ )self.counter['skipped'] += 1else:self.process_ingredient(ingredient_data)@@ -105,20 +103,16 @@ class Command(ImportProductCommand):self.stdout.write(f'Removing temporary folder {download_folder}')tmp_folder.cleanup()- def import_full_dump(self, languages: dict[str:int]):- off_url = 'https://static.openfoodfacts.org/data/openfoodfacts-products.jsonl.gz'+ def import_full_dump(self, languages: dict[str:int], destination: str):+ download_folder, tmp_folder = self.get_download_folder(destination)- download_folder, tmp_folder = self.get_download_folder(- '/Users/roland/Entwicklung/wger/server/extras/open-food-facts/dump'- )-- file_path = os.path.join(download_folder, os.path.basename(off_url))- self.download_file(off_url, file_path)+ file_path = os.path.join(download_folder, os.path.basename(self.full_off_dump_url))+ self.download_file(self.full_off_dump_url, file_path)for entry in self.iterate_gz_file_contents(file_path, list(languages.keys())):try:ingredient_data = extract_info_from_off(entry, languages[entry['lang']])- except KeyError as e:+ except (KeyError, ValueError) as e:# self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')self.counter['skipped'] += 1else:@@ -141,13 +135,19 @@ class Command(ImportProductCommand):self.stdout.write('Importing entries from Open Food Facts')self.stdout.write(f' - {self.mode}')+ if options['delta_updates']:+ self.stdout.write(f' - importing only delta updates')+ elif options['use_jsonl']:+ self.stdout.write(f' - importing the full dump')+ else:+ self.stdout.write(f' - importing from mongo')self.stdout.write('')languages = {lang.short_name: lang.pk for lang in Language.objects.all()}if options['delta_updates']:- self.import_daily_delta(languages)+ self.import_daily_delta(languages, options['folder'])elif options['use_jsonl']:- self.import_full_dump(languages)+ self.import_full_dump(languages, options['folder'])else:self.import_mongo(languages)diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.pyindex fa8205f13..86cc0b611 100644--- a/wger/nutrition/management/products.py+++ b/wger/nutrition/management/products.py@@ -72,9 +72,11 @@ class ImportProductCommand(BaseCommand):default='update',dest='mode',type=str,- help='Script mode, "insert" or "update". Insert will insert the ingredients as new '- 'entries in the database, while update will try to update them if they are '- 'already present. Default: update',+ help=(+ 'Script mode, "insert" or "update". Insert will insert the ingredients as new '+ 'entries in the database, while update will try to update them if they are '+ 'already present. Default: update'+ ),)parser.add_argument(@@ -83,10 +85,12 @@ class ImportProductCommand(BaseCommand):default='',dest='folder',type=str,- help='Controls whether to use a temporary folder created by python (the default) or '- 'the path provided for storing the downloaded dataset. If there are already '- 'downloaded or extracted files here, they will be used instead of fetching them '- 'again.',+ help=(+ 'Controls whether to use a temporary folder created by python (the default) or '+ 'the path provided for storing the downloaded dataset. If there are already '+ 'downloaded or extracted files here, they will be used instead of fetching them '+ 'again.'+ ),)def handle(self, **options):@@ -139,9 +143,8 @@ class ImportProductCommand(BaseCommand):# self.stdout.write('-> updated')except Exception as e:- self.stdout.write('--> Error while performing update_or_create')- self.stdout.write(repr(e))- # self.stdout.write(repr(ingredient_data))+ self.stdout.write(f'--> Error while performing update_or_create: {e}')+ self.stdout.write(repr(ingredient_data))self.counter['error'] += 1def get_download_folder(self, folder: str) -> tuple[str, Optional[tempfile.TemporaryDirectory]]:@@ -157,18 +160,9 @@ class ImportProductCommand(BaseCommand):tmp_folder = tempfile.TemporaryDirectory()download_folder = tmp_folder.name+ self.stdout.write(f'Using folder {download_folder} for storing downloaded files')return download_folder, tmp_folder- def parse_file_content(self, path: str):- with GzipFile(path, 'rb') as gzid:- for line in gzid:- try:- product = json.loads(line)- yield product- except JSONDecodeError as e:- self.stdout.write(f' Error parsing and/or filtering json record, skipping')- continue-def iterate_gz_file_contents(self, path: str, languages: list[str]):with GzipFile(path, 'rb') as gzid:for line in gzid:@@ -183,13 +177,12 @@ class ImportProductCommand(BaseCommand):def download_file(self, url: str, destination: str) -> None:if os.path.exists(destination):- self.stdout.write(f'File already downloaded {destination}, not downloading it again')+ self.stdout.write(f'File already downloaded at {destination}')returnself.stdout.write(f'Downloading {url}... (this may take a while)')response = requests.get(url, stream=True, headers=wger_headers())total_size = int(response.headers.get('content-length', 0))- size = int(response.headers['content-length']) / (1024 * 1024)if response.status_code == 404:raise Exception(f'Could not open {url}!')