Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Do not fail on canonicalizing URLs with wrong netlocs

Paul Tremberth1aec5200bc81
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c80fc6e70..406eb5843 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions):
def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
+ # IDNA encoding can fail for too long labels (>63 characters)
+ # or missing labels (e.g. http://.example.com)
+ try:
+ netloc = parts.netloc.encode('idna')
+ except UnicodeError:
+ netloc = parts.netloc
+
return (
to_native_str(parts.scheme),
- to_native_str(parts.netloc.encode('idna')),
+ to_native_str(netloc),
# default encoding for path component SHOULD be UTF-8
quote(to_bytes(parts.path, path_encoding), _safe_chars),
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1fc3a3510..b4819874d 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase):
# without encoding, already canonicalized URL is canonicalized identically
self.assertEqual(canonicalize_url(canonicalized), canonicalized)
+ def test_canonicalize_url_idna_exceptions(self):
+ # missing DNS label
+ self.assertEqual(
+ canonicalize_url(u"http://.example.com/résumé?q=résumé"),
+ "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+
+ # DNS label too long
+ self.assertEqual(
+ canonicalize_url(
+ u"http://www.{label}.com/résumé?q=résumé".format(
+ label=u"example"*11)),
+ "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
+ label=u"example"*11))
+
class AddHttpIfNoScheme(unittest.TestCase):