Scrapy, a fast high-level web crawling & scraping framework for Python.
RegexLinkExtractor: add \x0c to whitespace characters, as per html5 standard
Mikhail Korobovad36a4a6ae83
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.pyindex 0fc7b079f..e689b4727 100644--- a/scrapy/linkextractors/regex.py+++ b/scrapy/linkextractors/regex.py@@ -10,9 +10,10 @@ linkre = re.compile("<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",re.DOTALL | re.IGNORECASE)+def clean_link(link_text):"""Remove leading and trailing whitespace and punctuation"""- return link_text.strip("\t\r\n '\"")+ return link_text.strip("\t\r\n '\"\x0c")class RegexLinkExtractor(SgmlLinkExtractor):