Scrapy, a fast high-level web crawling & scraping framework for Python.
Move the parsel version check to _deps_compat, drop the w3lib ones (#8001)
Andrey Rakhmatulline810f5fca166
diff --git a/pyproject.toml b/pyproject.tomlindex 81f3c062b..b449dcbe1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -21,7 +21,7 @@ dependencies = ["queuelib>=1.6.1","service_identity>=24.2.0","tldextract",- "w3lib>=2.1.0",+ "w3lib>=2.1.1","zope.interface>=5.1.0",# Platform-specific dependencies'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.pyindex c6391dd8e..519d83017 100644--- a/scrapy/utils/_deps_compat.py+++ b/scrapy/utils/_deps_compat.py@@ -2,9 +2,9 @@ import sysfrom OpenSSL import __version__ as PYOPENSSL_VERSION_STRINGfrom packaging.version import Version+from parsel import __version__ as PARSEL_VERSION_STRINGfrom twisted import version as TWISTED_VERSIONfrom twisted.python.versions import Version as TxVersion-from w3lib import __version__ as W3LIB_VERSION_STRING# improved urllib.robotparser, https://github.com/python/cpython/pull/149374STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (@@ -17,10 +17,10 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)# lowerMaximumSecurityTo off-by-1, https://github.com/twisted/twisted/issues/10232TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)+PARSEL_VERSION = Version(PARSEL_VERSION_STRING)+# Selector.jmespath() support+PARSEL_SUPPORTS_JMESPATH = PARSEL_VERSION >= Version("1.8.0")+PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutablePYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")--W3LIB_VERSION = Version(W3LIB_VERSION_STRING)-# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207-W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1")diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.pyindex d8ef5eaea..d144532c1 100644--- a/tests/test_linkextractors.py+++ b/tests/test_linkextractors.py@@ -3,10 +3,6 @@ from __future__ import annotationsimport pickleimport re-import pytest-from packaging.version import Version-from w3lib import __version__ as w3lib_version-from scrapy.http import HtmlResponse, XmlResponsefrom scrapy.link import Linkfrom scrapy.linkextractors import lxmlhtml@@ -844,13 +840,6 @@ class TestLxmlLinkExtractor:),]- @pytest.mark.skipif(- Version(w3lib_version) < Version("2.0.0"),- reason=(- "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "- "about an invalid port value."- ),- )def test_skip_bad_links(self):html = b"""<a href="http://example.org:non-port">Why would you do this?</a>diff --git a/tests/test_selector.py b/tests/test_selector.pyindex ce6de9524..cf8d9686b 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -1,14 +1,10 @@import weakref-import parselimport pytest-from packaging import versionfrom scrapy.http import HtmlResponse, JsonResponse, TextResponse, XmlResponsefrom scrapy.selector import Selector--PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0"))-PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0")+from scrapy.utils._deps_compat import PARSEL_SUPPORTS_JMESPATHclass TestSelector:@@ -60,7 +56,9 @@ class TestSelector:'<div><img src="a.jpg"><p>Hello</p></div>']- @pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support json")+ @pytest.mark.skipif(+ not PARSEL_SUPPORTS_JMESPATH, reason="parsel < 1.8 doesn't support json"+ )def test_flavor_detection_json(self) -> None:response = JsonResponse("http://example.com", body=b'{"a": "b"}', encoding="utf-8"@@ -68,7 +66,9 @@ class TestSelector:assert Selector(response).type == "json"assert response.jmespath("a").get() == "b"- @pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support json")+ @pytest.mark.skipif(+ not PARSEL_SUPPORTS_JMESPATH, reason="parsel < 1.8 doesn't support json"+ )def test_flavor_detection_json_with_html_body(self) -> None:body = b"<div><p>Hello</p></div>"response = JsonResponse("http://example.com", body=body, encoding="utf-8")@@ -128,7 +128,9 @@ class TestSelector:Selector(TextResponse(url="http://example.com", body=b""), text="")[email protected](not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath")+ not PARSEL_SUPPORTS_JMESPATH, reason="parsel < 1.8 doesn't support jmespath"+)class TestJMESPath:def test_json_has_html(self) -> None:"""Sometimes the information is returned in a json wrapper"""@@ -265,7 +267,7 @@ class TestJMESPath:) == ["18", "32", "22", "25"][email protected](PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath")[email protected](PARSEL_SUPPORTS_JMESPATH, reason="parsel >= 1.8 supports jmespath")def test_jmespath_not_available() -> None:body = """{diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.pyindex 78e14e7b0..d23c3ebe1 100644--- a/tests/utils/bases/http_response.py+++ b/tests/utils/bases/http_response.py@@ -9,7 +9,6 @@ from w3lib.encoding import resolve_encodingfrom scrapy.exceptions import NotSupportedfrom scrapy.http import Headers, Request, Response, TextResponsefrom scrapy.link import Link-from scrapy.utils._deps_compat import W3LIB_STRIPS_URLSfrom tests import get_testdataif TYPE_CHECKING:@@ -267,19 +266,9 @@ class TestResponseBase(ABC):with pytest.raises(ValueError, match="encoding can't be None"):r.follow("foo", encoding=None)- @pytest.mark.xfail(- not W3LIB_STRIPS_URLS,- reason="https://github.com/scrapy/w3lib/pull/207",- strict=True,- )def test_follow_whitespace_url(self):self._assert_followed_url("foo ", "http://example.com/foo")- @pytest.mark.xfail(- not W3LIB_STRIPS_URLS,- reason="https://github.com/scrapy/w3lib/pull/207",- strict=True,- )def test_follow_whitespace_link(self):self._assert_followed_url(Link("http://example.com/foo "), "http://example.com/foo"@@ -344,11 +333,6 @@ class TestResponseBase(ABC):with pytest.raises(ValueError, match="url can't be None"):list(r.follow_all(urls=[None])) # type: ignore[list-item]- @pytest.mark.xfail(- not W3LIB_STRIPS_URLS,- reason="https://github.com/scrapy/w3lib/pull/207",- strict=True,- )def test_follow_all_whitespace(self):relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]absolute = [@@ -359,11 +343,6 @@ class TestResponseBase(ABC):]self._assert_followed_all_urls(relative, absolute)- @pytest.mark.xfail(- not W3LIB_STRIPS_URLS,- reason="https://github.com/scrapy/w3lib/pull/207",- strict=True,- )def test_follow_all_whitespace_links(self):absolute = ["http://example.com/foo ",diff --git a/tox.ini b/tox.iniindex d2a682415..c5e4ff7bf 100644--- a/tox.ini+++ b/tox.ini@@ -149,7 +149,7 @@ deps =pyOpenSSL==24.3.0queuelib==1.6.1service_identity==24.2.0- w3lib==2.1.0+ w3lib==2.1.1zope.interface==5.1.0{[test-requirements]deps}setenv =@@ -307,7 +307,7 @@ deps =pyOpenSSL==24.3.0queuelib==1.6.1service_identity==24.2.0- w3lib==2.1.0+ w3lib==2.1.1zope.interface==5.1.0commands =; disabling coverage