Scrapy, a fast high-level web crawling & scraping framework for Python.
use w3lib.html.strip_html5_whitespace function; expand docs; strip consistently before calling process_value
Mikhail Korobovd09eed7674b5
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rstindex 2486e0982..75bdb4142 100644--- a/docs/topics/link-extractors.rst+++ b/docs/topics/link-extractors.rst@@ -134,10 +134,11 @@ LxmlLinkExtractor:param strip: whether to strip whitespaces from extracted attributes.According to HTML5 standard, leading and trailing whitespaces- must be stripped from ``href`` attributes of ``<a>`` and ``<area>``- elements, so LinkExtractor strips them by default. Set ``strip=False``- to turn it off (e.g. if you're extracting urls from elements or- attributes which allow leading/trailing whitespaces).+ must be stripped from ``href`` attributes of ``<a>``, ``<area>``+ and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``+ elements, etc., so LinkExtractor strips space chars by default.+ Set ``strip=False`` to turn it off (e.g. if you're extracting urls+ from elements or attributes which allow leading/trailing whitespaces).:type strip: boolean.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.pydiff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.pyindex 4841e4a54..27978a8a1 100644--- a/scrapy/linkextractors/htmlparser.py+++ b/scrapy/linkextractors/htmlparser.py@@ -7,10 +7,10 @@ from six.moves.html_parser import HTMLParserfrom six.moves.urllib.parse import urljoinfrom w3lib.url import safe_url_string+from w3lib.html import strip_html5_whitespacefrom scrapy.link import Linkfrom scrapy.utils.python import unique as unique_list-from scrapy.utils.url import trim_href_attributefrom scrapy.exceptions import ScrapyDeprecationWarning@@ -71,9 +71,9 @@ class HtmlParserLinkExtractor(HTMLParser):if self.scan_tag(tag):for attr, value in attrs:if self.scan_attr(attr):- url = self.process_attr(value)if self.strip:- url = trim_href_attribute(url)+ value = strip_html5_whitespace(value)+ url = self.process_attr(value)link = Link(url=url)self.links.append(link)self.current_link = linkdiff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex f753033ab..c284f1905 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -2,15 +2,15 @@Link extractor based on lxml.html"""import six-from six.moves.urllib.parse import urlparse, urljoin+from six.moves.urllib.parse import urljoinimport lxml.etree as etree+from w3lib.html import strip_html5_whitespacefrom scrapy.link import Linkfrom scrapy.utils.misc import arg_to_iter, rel_has_nofollowfrom scrapy.utils.python import unique as unique_list, to_native_strfrom scrapy.utils.response import get_base_url-from scrapy.utils.url import trim_href_attributefrom scrapy.linkextractors import FilteringLinkExtractor@@ -53,7 +53,7 @@ class LxmlParserLinkExtractor(object):# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)try:if self.strip:- attr_val = trim_href_attribute(attr_val)+ attr_val = strip_html5_whitespace(attr_val)attr_val = urljoin(base_url, attr_val)except ValueError:continue # skipping bogus linksdiff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.pyindex 6ecfd52aa..11ff7a261 100644--- a/scrapy/linkextractors/sgml.py+++ b/scrapy/linkextractors/sgml.py@@ -7,12 +7,13 @@ import warningsfrom sgmllib import SGMLParserfrom w3lib.url import safe_url_string+from w3lib.html import strip_html5_whitespace+from scrapy.link import Linkfrom scrapy.linkextractors import FilteringLinkExtractorfrom scrapy.utils.misc import arg_to_iter, rel_has_nofollowfrom scrapy.utils.python import unique as unique_list, to_unicodefrom scrapy.utils.response import get_base_url-from scrapy.utils.url import trim_href_attributefrom scrapy.exceptions import ScrapyDeprecationWarning@@ -81,10 +82,10 @@ class BaseSgmlLinkExtractor(SGMLParser):if self.scan_tag(tag):for attr, value in attrs:if self.scan_attr(attr):+ if self.strip and value is not None:+ value = strip_html5_whitespace(value)url = self.process_value(value)if url is not None:- if self.strip:- url = trim_href_attribute(url)link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))self.links.append(link)self.current_link = linkdiff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 090f65f80..dc1cce4ac 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -103,16 +103,3 @@ def guess_scheme(url):return any_to_uri(url)else:return add_http_if_no_scheme(url)---def trim_href_attribute(href):- """- Process href attribute of ``a`` or ``area`` elements according to HTML5- standards (strip all leading and trailing whitespaces). References:-- * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements- * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces- * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace- * https://www.w3.org/TR/html5/infrastructure.html#space-character- """- return href.strip(' \t\n\r\x0c')