🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: adding new tests and updating existing ones
Karim shoair3e0ca3311179
diff --git a/tests/fetchers/sync/test_camoufox.py b/tests/fetchers/sync/test_camoufox.pyindex 0c5cc86..9594a9c 100644--- a/tests/fetchers/sync/test_camoufox.py+++ b/tests/fetchers/sync/test_camoufox.py@@ -37,7 +37,7 @@ class TestStealthyFetcher:assert cookies == {"test": "value"}def test_automation(self, fetcher):- """Test if automation break the code or not"""+ """Test if automation breaks the code or not"""def scroll_page(page):page.mouse.wheel(10, 0)@@ -59,6 +59,7 @@ class TestStealthyFetcher:{"network_idle": True,"wait": 10,+ "timeout": 30_000,"cookies": [],"google_search": True,"extra_headers": {"ayo": ""},diff --git a/tests/parser/test_general.py b/tests/parser/test_general.pyindex 266e72a..6c17a61 100644--- a/tests/parser/test_general.py+++ b/tests/parser/test_general.py@@ -1,10 +1,12 @@import pickleimport time+import loggingimport pytestfrom cssselect import SelectorError, SelectorSyntaxErrorfrom scrapling import Selector+logging.getLogger("scrapling").setLevel(logging.DEBUG)@pytest.fixturediff --git a/tests/parser/test_html_utils.py b/tests/parser/test_html_utils.pynew file mode 100644index 0000000..924ef95--- /dev/null+++ b/tests/parser/test_html_utils.py@@ -0,0 +1,194 @@+import pytest++from scrapling.core._html_utils import to_unicode, _replace_entities, name2codepoint+++class TestToUnicode:+ def test_string_input(self):+ """Test to_unicode with string input"""+ text = "hello world"+ assert to_unicode(text) == "hello world"++ def test_bytes_input_default_encoding(self):+ """Test to_unicode with `bytes` input using default UTF-8"""+ text = b"hello world"+ assert to_unicode(text) == "hello world"++ def test_bytes_input_custom_encoding(self):+ """Test to_unicode with custom encoding"""+ text = "café".encode('latin-1')+ assert to_unicode(text, encoding='latin-1') == "café"++ def test_bytes_input_with_errors(self):+ """Test to_unicode with error handling"""+ # Invalid UTF-8 bytes+ text = b'\xff\xfe'+ assert to_unicode(text, errors='ignore') == ""+ assert to_unicode(text, errors='replace') == "��"++ def test_invalid_input_type(self):+ """Test to_unicode with an invalid input type"""+ with pytest.raises(TypeError, match="to_unicode must receive bytes or str"):+ to_unicode(123)++ def test_none_encoding_defaults_to_utf8(self):+ """Test that None encoding defaults to UTF-8"""+ text = "café".encode('utf-8')+ assert to_unicode(text, encoding=None) == "café"+++class TestReplaceEntities:+ def test_named_entities(self):+ """Test replacement of named HTML entities"""+ text = "& < > " "+ result = _replace_entities(text)+ assert result == "& < > \" \xa0"++ def test_decimal_entities(self):+ """Test replacement of decimal numeric entities"""+ text = "& < >"+ result = _replace_entities(text)+ assert result == "& < >"++ def test_hexadecimal_entities(self):+ """Test replacement of hexadecimal numeric entities"""+ text = "& < >"+ result = _replace_entities(text)+ assert result == "& < >"++ def test_mixed_entities(self):+ """Test replacement of mixed entity types"""+ text = "Price: £100 €50 $25"+ result = _replace_entities(text)+ assert result == "Price: £100 €50 $25"++ def test_keep_entities(self):+ """Test keeping specific entities"""+ text = "& < >"+ result = _replace_entities(text, keep=['amp', 'lt'])+ assert result == "& < >"++ def test_windows_1252_range(self):+ """Test handling of Windows-1252 range characters"""+ text = "€ ‚ Ÿ" # Windows-1252 range+ result = _replace_entities(text)+ # These should be decoded using cp1252+ assert "€" in result # 128 -> Euro sign++ def test_remove_illegal_entities_true(self):+ """Test removing illegal entities with remove_illegal=True"""+ text = "&unknown; 󴈿"+ result = _replace_entities(text, remove_illegal=True)+ # The function may convert large numbers to Unicode characters or leave them as-is+ assert "&unknown;" not in result # Unknown entities should be removed or converted++ def test_remove_illegal_entities_false(self):+ """Test keeping illegal entities with remove_illegal=False"""+ text = "&unknown; 󴈿"+ result = _replace_entities(text, remove_illegal=False)+ # Unknown entities should be preserved when remove_illegal=False+ assert "&unknown;" in result+ # Large numeric entities may be converted to Unicode characters++ def test_bytes_input(self):+ """Test with bytes input"""+ text = b"& < >"+ result = _replace_entities(text)+ assert result == "& < >"++ def test_custom_encoding(self):+ """Test with custom encoding"""+ text = "é".encode('latin-1')+ result = _replace_entities(text, encoding='latin-1')+ assert result == "é"++ def test_entities_without_semicolon(self):+ """Test entities without semicolon"""+ text = "& < >"+ result = _replace_entities(text, remove_illegal=True)+ # Should handle entities without a semicolon+ assert len(result) <= len(text)++ def test_case_insensitive_named_entities(self):+ """Test case-insensitive named-entity handling"""+ text = "& ≪ >"+ result = _replace_entities(text)+ assert result == "& < >"++ def test_edge_cases(self):+ """Test edge cases"""+ # Empty string+ assert _replace_entities("") == ""++ # No entities+ assert _replace_entities("plain text") == "plain text"++ # Invalid numeric entity+ text = "&#-1;"+ result = _replace_entities(text, remove_illegal=True)+ # Invalid entities may be left as-is or removed depending on implementation+ assert len(result) >= 0 # Ensure no exception is raised+++class TestName2Codepoint:+ def test_common_entities_exist(self):+ """Test that common HTML entities exist in mapping"""+ common_entities = ['amp', 'lt', 'gt', 'quot', 'nbsp', 'copy', 'reg']+ for entity in common_entities:+ assert entity in name2codepoint++ def test_greek_letters_exist(self):+ """Test that Greek letter entities exist"""+ greek_letters = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']+ for letter in greek_letters:+ assert letter in name2codepoint++ def test_mathematical_symbols_exist(self):+ """Test that mathematical symbol entities exist"""+ math_symbols = ['sum', 'prod', 'int', 'infin', 'plusmn']+ for symbol in math_symbols:+ assert symbol in name2codepoint++ def test_currency_symbols_exist(self):+ """Test that currency symbol entities exist"""+ currencies = ['pound', 'yen', 'euro', 'cent']+ for currency in currencies:+ assert currency in name2codepoint++ def test_codepoint_values(self):+ """Test specific codepoint values"""+ assert name2codepoint['amp'] == 0x0026 # &+ assert name2codepoint['lt'] == 0x003C # <+ assert name2codepoint['gt'] == 0x003E # >+ assert name2codepoint['nbsp'] == 0x00A0 # non-breaking space+ assert name2codepoint['copy'] == 0x00A9 # ©+++class TestIntegration:+ def test_real_world_html(self):+ """Test with real-world HTML content"""+ html = """+ <div class="content">+ © 2024 Company & Associates+ Price: £99.99 (€89.99)+ Math: α + β = γ+ </div>+ """+ result = _replace_entities(html)++ assert '<div class="content">' in result+ assert '© 2024 Company & Associates' in result+ assert 'Price: £99.99 (€89.99)' in result+ assert 'Math: α + β = γ' in result++ def test_performance_with_large_text(self):+ """Test performance with large text containing many entities"""+ # Create large text with repeated entities+ text = ("& < > " " * 1000)+ result = _replace_entities(text)++ # Should complete without issues and have correct content+ assert result.count("&") == 1000+ assert result.count("<") == 1000+ assert result.count(">") == 1000+ assert result.count('"') == 1000