Scrapy, a fast high-level web crawling & scraping framework for Python.
LinkExtractor PY3 'unicode' type fix
nyove8205f67339b
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.pyindex dcc261b31..9867e1179 100644--- a/scrapy/linkextractors/htmlparser.py+++ b/scrapy/linkextractors/htmlparser.py@@ -3,6 +3,7 @@ HTMLParser-based link extractor"""import warnings+import sixfrom six.moves.html_parser import HTMLParserfrom six.moves.urllib.parse import urljoin@@ -39,7 +40,7 @@ class HtmlParserLinkExtractor(HTMLParser):ret = []base_url = urljoin(response_url, self.base_url) if self.base_url else response_urlfor link in links:- if isinstance(link.url, unicode):+ if isinstance(link.url, six.text_type):link.url = link.url.encode(response_encoding)try:link.url = urljoin(base_url, link.url)diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.pyindex 9938e071f..c68dae4c8 100644--- a/scrapy/linkextractors/sgml.py+++ b/scrapy/linkextractors/sgml.py@@ -1,6 +1,7 @@"""SGMLParser-based Link extractors"""+import sixfrom six.moves.urllib.parse import urljoinimport warningsfrom sgmllib import SGMLParser@@ -40,7 +41,7 @@ class BaseSgmlLinkExtractor(SGMLParser):if base_url is None:base_url = urljoin(response_url, self.base_url) if self.base_url else response_urlfor link in self.links:- if isinstance(link.url, unicode):+ if isinstance(link.url, six.text_type):link.url = link.url.encode(response_encoding)try:link.url = urljoin(base_url, link.url)