Self hosted FLOSS fitness/workout, nutrition and weight tracker
OFF product import fix (issue 1956) - HTML entities and control characters are removed/changed to human readable from OFF product name before saving in db.
Bonicki Wojciecha4853a5fefaf
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.pyindex 606a0a70b..390947fc7 100644--- a/wger/nutrition/dataclasses.py+++ b/wger/nutrition/dataclasses.py@@ -19,6 +19,8 @@ from dataclasses import ()from typing import Optional+from wger.nutrition.helpers import change_html_entities_to_human_readable, remove_problematic_characters+@dataclassclass IngredientData:@@ -69,3 +71,7 @@ class IngredientData:def dict(self):return asdict(self)++ def clean_name(self):+ self.name = remove_problematic_characters(self.name)+ self.name = change_html_entities_to_human_readable(self.name)\ No newline at end of filediff --git a/wger/nutrition/helpers.py b/wger/nutrition/helpers.pyindex 634b6121b..28101b0e7 100644--- a/wger/nutrition/helpers.py+++ b/wger/nutrition/helpers.py@@ -21,7 +21,7 @@ from dataclasses import ()from decimal import Decimalfrom typing import Union-+import re# wgerfrom wger.nutrition.consts import (KJ_PER_KCAL,@@ -29,6 +29,11 @@ from wger.nutrition.consts import (MEALITEM_WEIGHT_UNIT,)+from wger.utils.constants import (+ CHARACTERS_TO_REMOVE_FROM_INGREDIENT_NAME,+ HTML_ENTITY_TO_HUMAN_READABLE_MAP,+ HTML_ENTITY_PATTERN,+)class BaseMealItem:"""@@ -143,3 +148,19 @@ class NutritionalValues:@propertydef to_dict(self):return asdict(self)++++def remove_problematic_characters(+ string: str, characters_to_be_removed: set[str] = CHARACTERS_TO_REMOVE_FROM_INGREDIENT_NAME+) -> str:+ string = "".join(c for c in string if c not in characters_to_be_removed)+ return string+++def change_html_entities_to_human_readable(+ string, html_entity_map: dict[str, str] = HTML_ENTITY_TO_HUMAN_READABLE_MAP+) -> str:+ pattern = re.compile(HTML_ENTITY_PATTERN)+ string = pattern.sub(lambda m: html_entity_map.get(m.group(0), m.group(0)), string)+ return string\ No newline at end of filediff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.pyindex e9a96cb27..4ad50e6a9 100644--- a/wger/nutrition/management/products.py+++ b/wger/nutrition/management/products.py@@ -99,6 +99,7 @@ class ImportProductCommand(BaseCommand):def process_ingredient(self, ingredient_data: IngredientData):## Add entries as new products+ ingredient_data.clean_name()if self.mode == Mode.INSERT:self.bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))if len(self.bulk_update_bucket) > self.bulk_size:diff --git a/wger/nutrition/tests/test_helpers.py b/wger/nutrition/tests/test_helpers.pynew file mode 100644index 000000000..6053ec3dd--- /dev/null+++ b/wger/nutrition/tests/test_helpers.py@@ -0,0 +1,63 @@+# This file is part of wger Workout Manager.+#+# wger Workout Manager is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# wger Workout Manager is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with Workout Manager. If not, see <http://www.gnu.org/licenses/>.++# Django+from django.test import SimpleTestCase++# wger+from wger.nutrition.dataclasses import IngredientData+from wger.nutrition.helpers import remove_problematic_characters, change_html_entities_to_human_readable++class HelperFunctionsTestCase(SimpleTestCase):+++ def test_change_html_entities_to_human_readable(self):++ string = "Proper Corn Sweet & Salty"+ result = change_html_entities_to_human_readable(string)+ correct_result = "Proper Corn Sweet & Salty"+ self.assertEqual(result, correct_result)++ string = "Stonebaked Pizza "the american pepperoni""+ result = change_html_entities_to_human_readable(string)+ correct_result = 'Stonebaked Pizza "the american pepperoni"'+ self.assertEqual(result, correct_result)++ string = "Van Houten VH10 Kakao (12,1%) 1KG"+ result = change_html_entities_to_human_readable(string)+ correct_result = "Van Houten VH10 Kakao (12,1%) 1KG"+ self.assertEqual(result, correct_result)++ string = "String with not existing html entitiy ϧ"+ result = change_html_entities_to_human_readable(string)+ correct_result = "String with not existing html entitiy ϧ"+ self.assertEqual(result, correct_result)++ def test_remove_problematic_characters(self):++ string = "Danone Danette Mousse Liégeoise Schokolade 2x(2x80g) 320g"+ result = remove_problematic_characters(string)+ correct_result = "Danone Danette Mousse Liégeoise Schokolade 2x(2x80g) 320g"+ self.assertEqual(result, correct_result)++ string = "POM'POTES Compotes Gourdes BIO Pomme Framboise 4x90g"+ result = remove_problematic_characters(string)+ correct_result = "POM'POTES Compotes Gourdes BIO Pomme Framboise 4x90g"+ self.assertEqual(result, correct_result)++ string = "String with \x9a control characters \t \x85"+ result = remove_problematic_characters(string)+ correct_result = "String with control characters "+ self.assertEqual(result, correct_result)\ No newline at end of filediff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.pyindex a4a5fe71c..8894d1a38 100644--- a/wger/nutrition/tests/test_off.py+++ b/wger/nutrition/tests/test_off.py@@ -111,3 +111,30 @@ class ExtractInfoFromOffTestCase(SimpleTestCase):self.assertEqual(result.carbohydrates_sugar, None)self.assertEqual(result.fat_saturated, None)++ def test_ingredient_clean_name(self):++ data = IngredientData(+ name='Stonebaked Pizza "the amer\x96ican \x99pepperoni"',+ remote_id='1234',+ language_id=1,+ energy=120,+ protein=10,+ carbohydrates=20,+ carbohydrates_sugar=30,+ fat=40,+ fat_saturated=11,+ fiber=None,+ sodium=5,+ code='1234',+ source_name='Open Food Facts',+ source_url='https://world.openfoodfacts.org/api/v2/product/1234.json',+ common_name='',+ brand='The bar company',+ license_id=ODBL_LICENSE_ID,+ license_author='open food facts, MrX',+ license_title='Foo with chocolate',+ license_object_url='https://world.openfoodfacts.org/product/1234/',+ )+ data.clean_name()+ self.assertEqual(data.name, 'Stonebaked Pizza "the american pepperoni"')\ No newline at end of filediff --git a/wger/utils/constants.py b/wger/utils/constants.pyindex f5bb39234..0d1f7c8e5 100644--- a/wger/utils/constants.py+++ b/wger/utils/constants.py@@ -77,3 +77,68 @@ UUID_CURLS = '1ae6a28d-10e7-4ecf-af4f-905f8193e2c6'UUID_FRENCH_PRESS = '95a7e546-e8f8-4521-a76b-983d94161b25'UUID_CRUNCHES = 'b186f1f8-4957-44dc-bf30-d0b00064ce6f'UUID_LEG_RAISES = 'c2078aac-e4e2-4103-a845-6252a3eb795e'++CHARACTERS_GROUPS_TO_REMOVE: dict = {+ "control_characters": [+ "\x8a",+ "\x83",+ "\x8f",+ "\x8d",+ "\x82",+ "\x9a",+ "\x89",+ "\x99",+ "\x87",+ "\x9c",+ "\x80",+ "\x96",+ "\t",+ "\x92",+ "\x8c",+ "\x90",+ "\x9f",+ "\x84",+ "\x97",+ "\x93",+ "\x98",+ "\x85",+ ],+ "format_characters": [],+ "space_separators": [],+ "quotation_or_parenthesis": [],+ "symbols_and_emoji": [],+}++CHARACTERS_TO_REMOVE_FROM_INGREDIENT_NAME: set[str] = {+ char for group in CHARACTERS_GROUPS_TO_REMOVE.values() for char in group+}++HTML_ENTITY_TO_HUMAN_READABLE_MAP = {+ """: '"',+ ">": ">",+ "<": "<",+ "&": "&",+ "°": "°",+ "€": "€",+ "œ": "œ",+ "‘": "‘",+ "(": "(", "(": "(",+ ")": ")", ")": ")",+ """: '"', """: '"',+ "'": "'", "'": "'",+ "/": "/", "/": "/",+ "%": "%", "%": "%",+ "&": "&", "&": "&",+ "ł": "ł",+ "ś": "ś",+ "ń": "ń",+ "ą": "ą",+ "ę": "ę",+ "ç": "ç",+ "´": "´",+ "[": "[", "[": "[",+ "]": "]", "]": "]",+ "'": "'"+}++HTML_ENTITY_PATTERN = r"&[a-zA-Z0-9#x]+;"\ No newline at end of file