Scrapy, a fast high-level web crawling & scraping framework for Python.
Skip test_skip_bad_links based on the w3lib version
Adrián Chaves3f0c2fae5e18
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 0d1b76531..23cbd0116 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -1,8 +1,8 @@"""Link extractor based on lxml.html"""-import operatorimport logging+import operatorfrom functools import partialfrom urllib.parse import urljoin, urlparse@@ -94,8 +94,8 @@ class LxmlParserLinkExtractor:try:url = safe_url_string(url, encoding=response_encoding)except ValueError:- logger.debug(f"Skipping extraction of bad link {url}")- continue # Disregard badly formatted urls+ logger.debug(f"Skipping extraction of link with bad URL {url!r}")+ continue# to fix relative links after process_valueurl = urljoin(response_url, url)diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex 784fdb658..e1ec19601 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -1,7 +1,10 @@import pickleimport reimport unittest-import sys++from packaging.version import Version+from pytest import mark+from w3lib import __version__ as w3lib_versionfrom scrapy.http import HtmlResponse, XmlResponsefrom scrapy.link import Link@@ -817,13 +820,16 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):def test_restrict_xpaths_with_html_entities(self):super().test_restrict_xpaths_with_html_entities()- @unittest.skipIf(- sys.version_info < (3, 8),- reason="some library for python 3.7 so is less strict so bad links like htis don't crash scrapy",+ @mark.skipif(+ Version(w3lib_version) < Version("2.0.0"),+ reason=(+ "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "+ "about an invalid port value."+ ),)def test_skip_bad_links(self):html = b"""- <a href="http://Some wierd html : http://example.com/like_this">Why would you do this?</a>+ <a href="http://example.org:non-port">Why would you do this?</a><a href="http://example.org/item2.html">Good Link</a><a href="http://example.org/item3.html">Good Link 2</a>"""diff --git a/tox.ini b/tox.iniindex 5a9d9cf29..873e7662b 100644--- a/tox.ini+++ b/tox.ini@@ -101,6 +101,7 @@ install_command =python -I -m pip install {opts} {packages}[testenv:pinned]+basepython = python3.7deps ={[pinned]deps}PyDispatcher==2.0.5