🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Adding tests for response encoding and response status text
Karim shoairf33ff247dd02
diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pynew file mode 100644index 0000000..0392388--- /dev/null+++ b/tests/fetchers/test_utils.py@@ -0,0 +1,129 @@+import unittest++from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText+++class TestPlayWrightFetcher(unittest.TestCase):+ def setUp(self):+ self.content_type_map = {+ # A map generated by ChatGPT for most possible `content_type` values and the expected outcome+ 'text/html; charset=UTF-8': 'UTF-8',+ 'text/html; charset=ISO-8859-1': 'ISO-8859-1',+ 'text/html': 'ISO-8859-1',+ 'application/json; charset=UTF-8': 'UTF-8',+ 'application/json': 'utf-8',+ 'text/json': 'utf-8',+ 'application/javascript; charset=UTF-8': 'UTF-8',+ 'application/javascript': 'utf-8',+ 'text/plain; charset=UTF-8': 'UTF-8',+ 'text/plain; charset=ISO-8859-1': 'ISO-8859-1',+ 'text/plain': 'ISO-8859-1',+ 'application/xhtml+xml; charset=UTF-8': 'UTF-8',+ 'application/xhtml+xml': 'utf-8',+ 'text/html; charset=windows-1252': 'windows-1252',+ 'application/json; charset=windows-1252': 'windows-1252',+ 'text/plain; charset=windows-1252': 'windows-1252',+ 'text/html; charset="UTF-8"': 'UTF-8',+ 'text/html; charset="ISO-8859-1"': 'ISO-8859-1',+ 'text/html; charset="windows-1252"': 'windows-1252',+ 'application/json; charset="UTF-8"': 'UTF-8',+ 'application/json; charset="ISO-8859-1"': 'ISO-8859-1',+ 'application/json; charset="windows-1252"': 'windows-1252',+ 'text/json; charset="UTF-8"': 'UTF-8',+ 'application/javascript; charset="UTF-8"': 'UTF-8',+ 'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',+ 'text/plain; charset="UTF-8"': 'UTF-8',+ 'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',+ 'text/plain; charset="windows-1252"': 'windows-1252',+ 'application/xhtml+xml; charset="UTF-8"': 'UTF-8',+ 'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',+ 'application/xhtml+xml; charset="windows-1252"': 'windows-1252',+ 'text/html; charset="US-ASCII"': 'US-ASCII',+ 'application/json; charset="US-ASCII"': 'US-ASCII',+ 'text/plain; charset="US-ASCII"': 'US-ASCII',+ 'text/html; charset="Shift_JIS"': 'Shift_JIS',+ 'application/json; charset="Shift_JIS"': 'Shift_JIS',+ 'text/plain; charset="Shift_JIS"': 'Shift_JIS',+ 'application/xml; charset="UTF-8"': 'UTF-8',+ 'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',+ 'application/xml': 'utf-8',+ 'text/xml; charset="UTF-8"': 'UTF-8',+ 'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',+ 'text/xml': 'utf-8'+ }+ self.status_map = {+ 100: "Continue",+ 101: "Switching Protocols",+ 102: "Processing",+ 103: "Early Hints",+ 200: "OK",+ 201: "Created",+ 202: "Accepted",+ 203: "Non-Authoritative Information",+ 204: "No Content",+ 205: "Reset Content",+ 206: "Partial Content",+ 207: "Multi-Status",+ 208: "Already Reported",+ 226: "IM Used",+ 300: "Multiple Choices",+ 301: "Moved Permanently",+ 302: "Found",+ 303: "See Other",+ 304: "Not Modified",+ 305: "Use Proxy",+ 307: "Temporary Redirect",+ 308: "Permanent Redirect",+ 400: "Bad Request",+ 401: "Unauthorized",+ 402: "Payment Required",+ 403: "Forbidden",+ 404: "Not Found",+ 405: "Method Not Allowed",+ 406: "Not Acceptable",+ 407: "Proxy Authentication Required",+ 408: "Request Timeout",+ 409: "Conflict",+ 410: "Gone",+ 411: "Length Required",+ 412: "Precondition Failed",+ 413: "Payload Too Large",+ 414: "URI Too Long",+ 415: "Unsupported Media Type",+ 416: "Range Not Satisfiable",+ 417: "Expectation Failed",+ 418: "I'm a teapot",+ 421: "Misdirected Request",+ 422: "Unprocessable Entity",+ 423: "Locked",+ 424: "Failed Dependency",+ 425: "Too Early",+ 426: "Upgrade Required",+ 428: "Precondition Required",+ 429: "Too Many Requests",+ 431: "Request Header Fields Too Large",+ 451: "Unavailable For Legal Reasons",+ 500: "Internal Server Error",+ 501: "Not Implemented",+ 502: "Bad Gateway",+ 503: "Service Unavailable",+ 504: "Gateway Timeout",+ 505: "HTTP Version Not Supported",+ 506: "Variant Also Negotiates",+ 507: "Insufficient Storage",+ 508: "Loop Detected",+ 510: "Not Extended",+ 511: "Network Authentication Required"+ }++ def test_parsing_content_type(self):+ """Test if parsing different types of content-type returns the expected result"""+ for header_value, expected_encoding in self.content_type_map.items():+ self.assertEqual(ResponseEncoding.get_value(header_value), expected_encoding)++ def test_parsing_response_status(self):+ """Test if using different http responses' status codes returns the expected result"""+ for status_code, expected_status_text in self.status_map.items():+ self.assertEqual(StatusText.get(status_code), expected_status_text)++ self.assertEqual(1000, "Unknown Status Code")