Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

RegexLinkExtractor: add \x0c to whitespace characters, as per html5 standard

Mikhail Korobovad36a4a6ae83
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index 0fc7b079f..e689b4727 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -10,9 +10,10 @@ linkre = re.compile(
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
re.DOTALL | re.IGNORECASE)
+
def clean_link(link_text):
"""Remove leading and trailing whitespace and punctuation"""
- return link_text.strip("\t\r\n '\"")
+ return link_text.strip("\t\r\n '\"\x0c")
class RegexLinkExtractor(SgmlLinkExtractor):