Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

OFF product import fix (issue 1956) - HTML entities and control characters are removed/changed to human readable from OFF product name before saving in db.

Bonicki Wojciecha4853a5fefaf
diff --git a/wger/nutrition/dataclasses.py b/wger/nutrition/dataclasses.py
index 606a0a70b..390947fc7 100644
--- a/wger/nutrition/dataclasses.py
+++ b/wger/nutrition/dataclasses.py
@@ -19,6 +19,8 @@ from dataclasses import (
)
from typing import Optional
+from wger.nutrition.helpers import change_html_entities_to_human_readable, remove_problematic_characters
+
@dataclass
class IngredientData:
@@ -69,3 +71,7 @@ class IngredientData:
def dict(self):
return asdict(self)
+
+ def clean_name(self):
+ self.name = remove_problematic_characters(self.name)
+ self.name = change_html_entities_to_human_readable(self.name)
\ No newline at end of file
diff --git a/wger/nutrition/helpers.py b/wger/nutrition/helpers.py
index 634b6121b..28101b0e7 100644
--- a/wger/nutrition/helpers.py
+++ b/wger/nutrition/helpers.py
@@ -21,7 +21,7 @@ from dataclasses import (
)
from decimal import Decimal
from typing import Union
-
+import re
# wger
from wger.nutrition.consts import (
KJ_PER_KCAL,
@@ -29,6 +29,11 @@ from wger.nutrition.consts import (
MEALITEM_WEIGHT_UNIT,
)
+from wger.utils.constants import (
+ CHARACTERS_TO_REMOVE_FROM_INGREDIENT_NAME,
+ HTML_ENTITY_TO_HUMAN_READABLE_MAP,
+ HTML_ENTITY_PATTERN,
+)
class BaseMealItem:
"""
@@ -143,3 +148,19 @@ class NutritionalValues:
@property
def to_dict(self):
return asdict(self)
+
+
+
+def remove_problematic_characters(
+ string: str, characters_to_be_removed: set[str] = CHARACTERS_TO_REMOVE_FROM_INGREDIENT_NAME
+) -> str:
+ string = "".join(c for c in string if c not in characters_to_be_removed)
+ return string
+
+
+def change_html_entities_to_human_readable(
+ string, html_entity_map: dict[str, str] = HTML_ENTITY_TO_HUMAN_READABLE_MAP
+) -> str:
+ pattern = re.compile(HTML_ENTITY_PATTERN)
+ string = pattern.sub(lambda m: html_entity_map.get(m.group(0), m.group(0)), string)
+ return string
\ No newline at end of file
diff --git a/wger/nutrition/management/products.py b/wger/nutrition/management/products.py
index e9a96cb27..4ad50e6a9 100644
--- a/wger/nutrition/management/products.py
+++ b/wger/nutrition/management/products.py
@@ -99,6 +99,7 @@ class ImportProductCommand(BaseCommand):
def process_ingredient(self, ingredient_data: IngredientData):
#
# Add entries as new products
+ ingredient_data.clean_name()
if self.mode == Mode.INSERT:
self.bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))
if len(self.bulk_update_bucket) > self.bulk_size:
diff --git a/wger/nutrition/tests/test_helpers.py b/wger/nutrition/tests/test_helpers.py
new file mode 100644
index 000000000..6053ec3dd
--- /dev/null
+++ b/wger/nutrition/tests/test_helpers.py
@@ -0,0 +1,63 @@
+# This file is part of wger Workout Manager.
+#
+# wger Workout Manager is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# wger Workout Manager is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Workout Manager. If not, see <http://www.gnu.org/licenses/>.
+
+# Django
+from django.test import SimpleTestCase
+
+# wger
+from wger.nutrition.dataclasses import IngredientData
+from wger.nutrition.helpers import remove_problematic_characters, change_html_entities_to_human_readable
+
+class HelperFunctionsTestCase(SimpleTestCase):
+
+
+ def test_change_html_entities_to_human_readable(self):
+
+ string = "Proper Corn Sweet &amp; Salty"
+ result = change_html_entities_to_human_readable(string)
+ correct_result = "Proper Corn Sweet & Salty"
+ self.assertEqual(result, correct_result)
+
+ string = "Stonebaked Pizza &quot;the american pepperoni&quot;"
+ result = change_html_entities_to_human_readable(string)
+ correct_result = 'Stonebaked Pizza "the american pepperoni"'
+ self.assertEqual(result, correct_result)
+
+ string = "Van Houten VH10 Kakao &#40;12,1&#37;&#41; 1KG"
+ result = change_html_entities_to_human_readable(string)
+ correct_result = "Van Houten VH10 Kakao (12,1%) 1KG"
+ self.assertEqual(result, correct_result)
+
+ string = "String with not existing html entitiy &#999;"
+ result = change_html_entities_to_human_readable(string)
+ correct_result = "String with not existing html entitiy &#999;"
+ self.assertEqual(result, correct_result)
+
+ def test_remove_problematic_characters(self):
+
+ string = "Danone Danette Mousse Liégeoise Schokolade 2x(2x80g) 320g"
+ result = remove_problematic_characters(string)
+ correct_result = "Danone Danette Mousse Liégeoise Schokolade 2x(2x80g) 320g"
+ self.assertEqual(result, correct_result)
+
+ string = "POM'POTES Compotes Gourdes BIO Pomme Framboise 4x90g"
+ result = remove_problematic_characters(string)
+ correct_result = "POM'POTES Compotes Gourdes BIO Pomme Framboise 4x90g"
+ self.assertEqual(result, correct_result)
+
+ string = "String with \x9a control characters \t \x85"
+ result = remove_problematic_characters(string)
+ correct_result = "String with control characters "
+ self.assertEqual(result, correct_result)
\ No newline at end of file
diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.py
index a4a5fe71c..8894d1a38 100644
--- a/wger/nutrition/tests/test_off.py
+++ b/wger/nutrition/tests/test_off.py
@@ -111,3 +111,30 @@ class ExtractInfoFromOffTestCase(SimpleTestCase):
self.assertEqual(result.carbohydrates_sugar, None)
self.assertEqual(result.fat_saturated, None)
+
+ def test_ingredient_clean_name(self):
+
+ data = IngredientData(
+ name='Stonebaked Pizza &quot;the amer\x96ican \x99pepperoni&quot;',
+ remote_id='1234',
+ language_id=1,
+ energy=120,
+ protein=10,
+ carbohydrates=20,
+ carbohydrates_sugar=30,
+ fat=40,
+ fat_saturated=11,
+ fiber=None,
+ sodium=5,
+ code='1234',
+ source_name='Open Food Facts',
+ source_url='https://world.openfoodfacts.org/api/v2/product/1234.json',
+ common_name='',
+ brand='The bar company',
+ license_id=ODBL_LICENSE_ID,
+ license_author='open food facts, MrX',
+ license_title='Foo with chocolate',
+ license_object_url='https://world.openfoodfacts.org/product/1234/',
+ )
+ data.clean_name()
+ self.assertEqual(data.name, 'Stonebaked Pizza "the american pepperoni"')
\ No newline at end of file
diff --git a/wger/utils/constants.py b/wger/utils/constants.py
index f5bb39234..0d1f7c8e5 100644
--- a/wger/utils/constants.py
+++ b/wger/utils/constants.py
@@ -77,3 +77,68 @@ UUID_CURLS = '1ae6a28d-10e7-4ecf-af4f-905f8193e2c6'
UUID_FRENCH_PRESS = '95a7e546-e8f8-4521-a76b-983d94161b25'
UUID_CRUNCHES = 'b186f1f8-4957-44dc-bf30-d0b00064ce6f'
UUID_LEG_RAISES = 'c2078aac-e4e2-4103-a845-6252a3eb795e'
+
+CHARACTERS_GROUPS_TO_REMOVE: dict = {
+ "control_characters": [
+ "\x8a",
+ "\x83",
+ "\x8f",
+ "\x8d",
+ "\x82",
+ "\x9a",
+ "\x89",
+ "\x99",
+ "\x87",
+ "\x9c",
+ "\x80",
+ "\x96",
+ "\t",
+ "\x92",
+ "\x8c",
+ "\x90",
+ "\x9f",
+ "\x84",
+ "\x97",
+ "\x93",
+ "\x98",
+ "\x85",
+ ],
+ "format_characters": [],
+ "space_separators": [],
+ "quotation_or_parenthesis": [],
+ "symbols_and_emoji": [],
+}
+
+CHARACTERS_TO_REMOVE_FROM_INGREDIENT_NAME: set[str] = {
+ char for group in CHARACTERS_GROUPS_TO_REMOVE.values() for char in group
+}
+
+HTML_ENTITY_TO_HUMAN_READABLE_MAP = {
+ "&quot;": '"',
+ "&gt;": ">",
+ "&lt;": "<",
+ "&amp;": "&",
+ "&deg;": "°",
+ "&euro;": "€",
+ "&oelig;": "œ",
+ "&lsquo;": "‘",
+ "&#40;": "(", "&#040;": "(",
+ "&#41;": ")", "&#041;": ")",
+ "&#34;": '"', "&#034;": '"',
+ "&#39;": "'", "&#039;": "'",
+ "&#47;": "/", "&#047;": "/",
+ "&#37;": "%", "&#037;": "%",
+ "&#38;": "&", "&#038;": "&",
+ "&#322;": "ł",
+ "&#347;": "ś",
+ "&#324;": "ń",
+ "&#261;": "ą",
+ "&#281;": "ę",
+ "&#231;": "ç",
+ "&#180;": "´",
+ "&#91;": "[", "&#091;": "[",
+ "&#93;": "]", "&#093;": "]",
+ "&#x27;": "'"
+}
+
+HTML_ENTITY_PATTERN = r"&[a-zA-Z0-9#x]+;"
\ No newline at end of file