Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

use w3lib.html.strip_html5_whitespace function; expand docs; strip consistently before calling process_value

Mikhail Korobovd09eed7674b5
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 2486e0982..75bdb4142 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -134,10 +134,11 @@ LxmlLinkExtractor
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
- must be stripped from ``href`` attributes of ``<a>`` and ``<area>``
- elements, so LinkExtractor strips them by default. Set ``strip=False``
- to turn it off (e.g. if you're extracting urls from elements or
- attributes which allow leading/trailing whitespaces).
+ must be stripped from ``href`` attributes of ``<a>``, ``<area>``
+ and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
+ elements, etc., so LinkExtractor strips space chars by default.
+ Set ``strip=False`` to turn it off (e.g. if you're extracting urls
+ from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 4841e4a54..27978a8a1 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -7,10 +7,10 @@ from six.moves.html_parser import HTMLParser
from six.moves.urllib.parse import urljoin
from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
from scrapy.utils.python import unique as unique_list
-from scrapy.utils.url import trim_href_attribute
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -71,9 +71,9 @@ class HtmlParserLinkExtractor(HTMLParser):
if self.scan_tag(tag):
for attr, value in attrs:
if self.scan_attr(attr):
- url = self.process_attr(value)
if self.strip:
- url = trim_href_attribute(url)
+ value = strip_html5_whitespace(value)
+ url = self.process_attr(value)
link = Link(url=url)
self.links.append(link)
self.current_link = link
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index f753033ab..c284f1905 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -2,15 +2,15 @@
Link extractor based on lxml.html
"""
import six
-from six.moves.urllib.parse import urlparse, urljoin
+from six.moves.urllib.parse import urljoin
import lxml.etree as etree
+from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list, to_native_str
from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
from scrapy.linkextractors import FilteringLinkExtractor
@@ -53,7 +53,7 @@ class LxmlParserLinkExtractor(object):
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
try:
if self.strip:
- attr_val = trim_href_attribute(attr_val)
+ attr_val = strip_html5_whitespace(attr_val)
attr_val = urljoin(base_url, attr_val)
except ValueError:
continue # skipping bogus links
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 6ecfd52aa..11ff7a261 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -7,12 +7,13 @@ import warnings
from sgmllib import SGMLParser
from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
+
from scrapy.link import Link
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list, to_unicode
from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -81,10 +82,10 @@ class BaseSgmlLinkExtractor(SGMLParser):
if self.scan_tag(tag):
for attr, value in attrs:
if self.scan_attr(attr):
+ if self.strip and value is not None:
+ value = strip_html5_whitespace(value)
url = self.process_value(value)
if url is not None:
- if self.strip:
- url = trim_href_attribute(url)
link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
self.links.append(link)
self.current_link = link
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 090f65f80..dc1cce4ac 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -103,16 +103,3 @@ def guess_scheme(url):
return any_to_uri(url)
else:
return add_http_if_no_scheme(url)
-
-
-def trim_href_attribute(href):
- """
- Process href attribute of ``a`` or ``area`` elements according to HTML5
- standards (strip all leading and trailing whitespaces). References:
-
- * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements
- * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces
- * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace
- * https://www.w3.org/TR/html5/infrastructure.html#space-character
- """
- return href.strip(' \t\n\r\x0c')