Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not fail on canonicalizing URLs with wrong netlocs
Paul Tremberth1aec5200bc81
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex c80fc6e70..406eb5843 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions):def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):+ # IDNA encoding can fail for too long labels (>63 characters)+ # or missing labels (e.g. http://.example.com)+ try:+ netloc = parts.netloc.encode('idna')+ except UnicodeError:+ netloc = parts.netloc+return (to_native_str(parts.scheme),- to_native_str(parts.netloc.encode('idna')),+ to_native_str(netloc),# default encoding for path component SHOULD be UTF-8quote(to_bytes(parts.path, path_encoding), _safe_chars),diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 1fc3a3510..b4819874d 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase):# without encoding, already canonicalized URL is canonicalized identicallyself.assertEqual(canonicalize_url(canonicalized), canonicalized)+ def test_canonicalize_url_idna_exceptions(self):+ # missing DNS label+ self.assertEqual(+ canonicalize_url(u"http://.example.com/résumé?q=résumé"),+ "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")++ # DNS label too long+ self.assertEqual(+ canonicalize_url(+ u"http://www.{label}.com/résumé?q=résumé".format(+ label=u"example"*11)),+ "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(+ label=u"example"*11))+class AddHttpIfNoScheme(unittest.TestCase):