Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Move the parsel version check to _deps_compat, drop the w3lib ones (#8001)

Andrey Rakhmatulline810f5fca166
diff --git a/pyproject.toml b/pyproject.toml
index 81f3c062b..b449dcbe1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,7 +21,7 @@ dependencies = [
"queuelib>=1.6.1",
"service_identity>=24.2.0",
"tldextract",
- "w3lib>=2.1.0",
+ "w3lib>=2.1.1",
"zope.interface>=5.1.0",
# Platform-specific dependencies
'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py
index c6391dd8e..519d83017 100644
--- a/scrapy/utils/_deps_compat.py
+++ b/scrapy/utils/_deps_compat.py
@@ -2,9 +2,9 @@ import sys
from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING
from packaging.version import Version
+from parsel import __version__ as PARSEL_VERSION_STRING
from twisted import version as TWISTED_VERSION
from twisted.python.versions import Version as TxVersion
-from w3lib import __version__ as W3LIB_VERSION_STRING
# improved urllib.robotparser, https://github.com/python/cpython/pull/149374
STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (
@@ -17,10 +17,10 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
# lowerMaximumSecurityTo off-by-1, https://github.com/twisted/twisted/issues/10232
TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)
+PARSEL_VERSION = Version(PARSEL_VERSION_STRING)
+# Selector.jmespath() support
+PARSEL_SUPPORTS_JMESPATH = PARSEL_VERSION >= Version("1.8.0")
+
PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
-
-W3LIB_VERSION = Version(W3LIB_VERSION_STRING)
-# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207
-W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1")
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d8ef5eaea..d144532c1 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -3,10 +3,6 @@ from __future__ import annotations
import pickle
import re
-import pytest
-from packaging.version import Version
-from w3lib import __version__ as w3lib_version
-
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.link import Link
from scrapy.linkextractors import lxmlhtml
@@ -844,13 +840,6 @@ class TestLxmlLinkExtractor:
),
]
- @pytest.mark.skipif(
- Version(w3lib_version) < Version("2.0.0"),
- reason=(
- "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "
- "about an invalid port value."
- ),
- )
def test_skip_bad_links(self):
html = b"""
<a href="http://example.org:non-port">Why would you do this?</a>
diff --git a/tests/test_selector.py b/tests/test_selector.py
index ce6de9524..cf8d9686b 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1,14 +1,10 @@
import weakref
-import parsel
import pytest
-from packaging import version
from scrapy.http import HtmlResponse, JsonResponse, TextResponse, XmlResponse
from scrapy.selector import Selector
-
-PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0"))
-PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0")
+from scrapy.utils._deps_compat import PARSEL_SUPPORTS_JMESPATH
class TestSelector:
@@ -60,7 +56,9 @@ class TestSelector:
'<div><img src="a.jpg"><p>Hello</p></div>'
]
- @pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support json")
+ @pytest.mark.skipif(
+ not PARSEL_SUPPORTS_JMESPATH, reason="parsel < 1.8 doesn't support json"
+ )
def test_flavor_detection_json(self) -> None:
response = JsonResponse(
"http://example.com", body=b'{"a": "b"}', encoding="utf-8"
@@ -68,7 +66,9 @@ class TestSelector:
assert Selector(response).type == "json"
assert response.jmespath("a").get() == "b"
- @pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support json")
+ @pytest.mark.skipif(
+ not PARSEL_SUPPORTS_JMESPATH, reason="parsel < 1.8 doesn't support json"
+ )
def test_flavor_detection_json_with_html_body(self) -> None:
body = b"<div><p>Hello</p></div>"
response = JsonResponse("http://example.com", body=body, encoding="utf-8")
@@ -128,7 +128,9 @@ class TestSelector:
Selector(TextResponse(url="http://example.com", body=b""), text="")
[email protected](not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath")
+ not PARSEL_SUPPORTS_JMESPATH, reason="parsel < 1.8 doesn't support jmespath"
+)
class TestJMESPath:
def test_json_has_html(self) -> None:
"""Sometimes the information is returned in a json wrapper"""
@@ -265,7 +267,7 @@ class TestJMESPath:
) == ["18", "32", "22", "25"]
[email protected](PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath")
[email protected](PARSEL_SUPPORTS_JMESPATH, reason="parsel >= 1.8 supports jmespath")
def test_jmespath_not_available() -> None:
body = """
{
diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py
index 78e14e7b0..d23c3ebe1 100644
--- a/tests/utils/bases/http_response.py
+++ b/tests/utils/bases/http_response.py
@@ -9,7 +9,6 @@ from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response, TextResponse
from scrapy.link import Link
-from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
if TYPE_CHECKING:
@@ -267,19 +266,9 @@ class TestResponseBase(ABC):
with pytest.raises(ValueError, match="encoding can't be None"):
r.follow("foo", encoding=None)
- @pytest.mark.xfail(
- not W3LIB_STRIPS_URLS,
- reason="https://github.com/scrapy/w3lib/pull/207",
- strict=True,
- )
def test_follow_whitespace_url(self):
self._assert_followed_url("foo ", "http://example.com/foo")
- @pytest.mark.xfail(
- not W3LIB_STRIPS_URLS,
- reason="https://github.com/scrapy/w3lib/pull/207",
- strict=True,
- )
def test_follow_whitespace_link(self):
self._assert_followed_url(
Link("http://example.com/foo "), "http://example.com/foo"
@@ -344,11 +333,6 @@ class TestResponseBase(ABC):
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None])) # type: ignore[list-item]
- @pytest.mark.xfail(
- not W3LIB_STRIPS_URLS,
- reason="https://github.com/scrapy/w3lib/pull/207",
- strict=True,
- )
def test_follow_all_whitespace(self):
relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
absolute = [
@@ -359,11 +343,6 @@ class TestResponseBase(ABC):
]
self._assert_followed_all_urls(relative, absolute)
- @pytest.mark.xfail(
- not W3LIB_STRIPS_URLS,
- reason="https://github.com/scrapy/w3lib/pull/207",
- strict=True,
- )
def test_follow_all_whitespace_links(self):
absolute = [
"http://example.com/foo ",
diff --git a/tox.ini b/tox.ini
index d2a682415..c5e4ff7bf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -149,7 +149,7 @@ deps =
pyOpenSSL==24.3.0
queuelib==1.6.1
service_identity==24.2.0
- w3lib==2.1.0
+ w3lib==2.1.1
zope.interface==5.1.0
{[test-requirements]deps}
setenv =
@@ -307,7 +307,7 @@ deps =
pyOpenSSL==24.3.0
queuelib==1.6.1
service_identity==24.2.0
- w3lib==2.1.0
+ w3lib==2.1.1
zope.interface==5.1.0
commands =
; disabling coverage