🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: remove the tests for the removed html_utils module
Karim shoaird0f52c38555c
diff --git a/tests/parser/test_html_utils.py b/tests/parser/test_html_utils.pydeleted file mode 100644index 924ef95..0000000--- a/tests/parser/test_html_utils.py+++ /dev/null@@ -1,194 +0,0 @@-import pytest--from scrapling.core._html_utils import to_unicode, _replace_entities, name2codepoint---class TestToUnicode:- def test_string_input(self):- """Test to_unicode with string input"""- text = "hello world"- assert to_unicode(text) == "hello world"-- def test_bytes_input_default_encoding(self):- """Test to_unicode with `bytes` input using default UTF-8"""- text = b"hello world"- assert to_unicode(text) == "hello world"-- def test_bytes_input_custom_encoding(self):- """Test to_unicode with custom encoding"""- text = "café".encode('latin-1')- assert to_unicode(text, encoding='latin-1') == "café"-- def test_bytes_input_with_errors(self):- """Test to_unicode with error handling"""- # Invalid UTF-8 bytes- text = b'\xff\xfe'- assert to_unicode(text, errors='ignore') == ""- assert to_unicode(text, errors='replace') == "��"-- def test_invalid_input_type(self):- """Test to_unicode with an invalid input type"""- with pytest.raises(TypeError, match="to_unicode must receive bytes or str"):- to_unicode(123)-- def test_none_encoding_defaults_to_utf8(self):- """Test that None encoding defaults to UTF-8"""- text = "café".encode('utf-8')- assert to_unicode(text, encoding=None) == "café"---class TestReplaceEntities:- def test_named_entities(self):- """Test replacement of named HTML entities"""- text = "& < > " "- result = _replace_entities(text)- assert result == "& < > \" \xa0"-- def test_decimal_entities(self):- """Test replacement of decimal numeric entities"""- text = "& < >"- result = _replace_entities(text)- assert result == "& < >"-- def test_hexadecimal_entities(self):- """Test replacement of hexadecimal numeric entities"""- text = "& < >"- result = _replace_entities(text)- assert result == "& < >"-- def test_mixed_entities(self):- """Test replacement of mixed entity types"""- text = "Price: £100 €50 $25"- result = _replace_entities(text)- assert result == "Price: £100 €50 $25"-- def test_keep_entities(self):- """Test keeping specific entities"""- text = "& < >"- result = _replace_entities(text, keep=['amp', 'lt'])- assert result == "& < >"-- def test_windows_1252_range(self):- """Test handling of Windows-1252 range characters"""- text = "€ ‚ Ÿ" # Windows-1252 range- result = _replace_entities(text)- # These should be decoded using cp1252- assert "€" in result # 128 -> Euro sign-- def test_remove_illegal_entities_true(self):- """Test removing illegal entities with remove_illegal=True"""- text = "&unknown; 󴈿"- result = _replace_entities(text, remove_illegal=True)- # The function may convert large numbers to Unicode characters or leave them as-is- assert "&unknown;" not in result # Unknown entities should be removed or converted-- def test_remove_illegal_entities_false(self):- """Test keeping illegal entities with remove_illegal=False"""- text = "&unknown; 󴈿"- result = _replace_entities(text, remove_illegal=False)- # Unknown entities should be preserved when remove_illegal=False- assert "&unknown;" in result- # Large numeric entities may be converted to Unicode characters-- def test_bytes_input(self):- """Test with bytes input"""- text = b"& < >"- result = _replace_entities(text)- assert result == "& < >"-- def test_custom_encoding(self):- """Test with custom encoding"""- text = "é".encode('latin-1')- result = _replace_entities(text, encoding='latin-1')- assert result == "é"-- def test_entities_without_semicolon(self):- """Test entities without semicolon"""- text = "& < >"- result = _replace_entities(text, remove_illegal=True)- # Should handle entities without a semicolon- assert len(result) <= len(text)-- def test_case_insensitive_named_entities(self):- """Test case-insensitive named-entity handling"""- text = "& ≪ >"- result = _replace_entities(text)- assert result == "& < >"-- def test_edge_cases(self):- """Test edge cases"""- # Empty string- assert _replace_entities("") == ""-- # No entities- assert _replace_entities("plain text") == "plain text"-- # Invalid numeric entity- text = "&#-1;"- result = _replace_entities(text, remove_illegal=True)- # Invalid entities may be left as-is or removed depending on implementation- assert len(result) >= 0 # Ensure no exception is raised---class TestName2Codepoint:- def test_common_entities_exist(self):- """Test that common HTML entities exist in mapping"""- common_entities = ['amp', 'lt', 'gt', 'quot', 'nbsp', 'copy', 'reg']- for entity in common_entities:- assert entity in name2codepoint-- def test_greek_letters_exist(self):- """Test that Greek letter entities exist"""- greek_letters = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']- for letter in greek_letters:- assert letter in name2codepoint-- def test_mathematical_symbols_exist(self):- """Test that mathematical symbol entities exist"""- math_symbols = ['sum', 'prod', 'int', 'infin', 'plusmn']- for symbol in math_symbols:- assert symbol in name2codepoint-- def test_currency_symbols_exist(self):- """Test that currency symbol entities exist"""- currencies = ['pound', 'yen', 'euro', 'cent']- for currency in currencies:- assert currency in name2codepoint-- def test_codepoint_values(self):- """Test specific codepoint values"""- assert name2codepoint['amp'] == 0x0026 # &- assert name2codepoint['lt'] == 0x003C # <- assert name2codepoint['gt'] == 0x003E # >- assert name2codepoint['nbsp'] == 0x00A0 # non-breaking space- assert name2codepoint['copy'] == 0x00A9 # ©---class TestIntegration:- def test_real_world_html(self):- """Test with real-world HTML content"""- html = """- <div class="content">- © 2024 Company & Associates- Price: £99.99 (€89.99)- Math: α + β = γ- </div>- """- result = _replace_entities(html)-- assert '<div class="content">' in result- assert '© 2024 Company & Associates' in result- assert 'Price: £99.99 (€89.99)' in result- assert 'Math: α + β = γ' in result-- def test_performance_with_large_text(self):- """Test performance with large text containing many entities"""- # Create large text with repeated entities- text = ("& < > " " * 1000)- result = _replace_entities(text)-- # Should complete without issues and have correct content- assert result.count("&") == 1000- assert result.count("<") == 1000- assert result.count(">") == 1000- assert result.count('"') == 1000