Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Replace import tests with find_spec() or importorskip(). (#7755)

Andrey Rakhmatullin64358f547bfa
diff --git a/conftest.py b/conftest.py
index 7403b07b2..1bfb002cf 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-import importlib
+from importlib.util import find_spec
from pathlib import Path
from typing import TYPE_CHECKING
@@ -50,9 +50,7 @@ if not H2_ENABLED:
)
)
-try:
- import httpx # noqa: F401
-except ImportError:
+if not find_spec("httpx"):
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
@@ -130,11 +128,8 @@ def pytest_runtest_setup(item):
]
for module in optional_deps:
- if item.get_closest_marker(f"requires_{module}"):
- try:
- importlib.import_module(module)
- except ImportError:
- pytest.skip(f"{module} is not installed")
+ if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:
+ pytest.skip(f"{module} is not installed")
if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
pytest.skip("mitmdump is not available")
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 13a432476..b99c323c5 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import warnings
+from importlib.util import find_spec
from itertools import chain
from logging import getLogger
from typing import TYPE_CHECKING, Any
@@ -51,11 +52,7 @@ else:
else:
ACCEPTED_ENCODINGS.append(b"br")
-try:
- import zstandard # noqa: F401
-except ImportError:
- pass
-else:
+if find_spec("zstandard") is not None:
ACCEPTED_ENCODINGS.append(b"zstd")
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 73f86bc71..2a77ee2ac 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,10 +1,7 @@
"""Boto/botocore helpers"""
+from importlib.util import find_spec
-def is_botocore_available() -> bool:
- try:
- import botocore # noqa: F401,PLC0415
- return True
- except ImportError:
- return False
+def is_botocore_available() -> bool:
+ return find_spec("botocore") is not None
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 311aff7df..ca18c8038 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,4 +1,5 @@
from gzip import GzipFile
+from importlib.util import find_spec
from io import BytesIO
from logging import WARNING
from pathlib import Path
@@ -65,10 +66,7 @@ def _skip_if_no_br() -> None:
def _skip_if_no_zstd() -> None:
- try:
- import zstandard # noqa: F401,PLC0415
- except ImportError:
- pytest.skip("no zstd support (zstandard)")
+ pytest.importorskip("zstandard")
class TestHttpCompression:
@@ -172,17 +170,8 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_br_unsupported(self):
- try:
- try:
- import brotli # noqa: F401,PLC0415
-
- pytest.skip("Requires not having brotli support")
- except ImportError:
- import brotlicffi # noqa: F401,PLC0415
-
- pytest.skip("Requires not having brotli support")
- except ImportError:
- pass
+ if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
+ pytest.skip("Requires not having brotli support")
response = self._getresponse("br")
request = response.request
assert response.headers["Content-Encoding"] == b"br"
@@ -226,12 +215,8 @@ class TestHttpCompression:
assert "Content-Encoding" not in newresponse.headers
def test_process_response_zstd_unsupported(self):
- try:
- import zstandard # noqa: F401,PLC0415
-
+ if find_spec("zstandard") is not None:
pytest.skip("Requires not having zstandard support")
- except ImportError:
- pass
response = self._getresponse("zstd-static-content-size")
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 82e2597a5..bdd0d7614 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -436,10 +436,7 @@ class TestS3FeedStorage:
class TestGCSFeedStorage:
def test_parse_settings(self):
- try:
- from google.cloud.storage import Client # noqa: F401,PLC0415
- except ImportError:
- pytest.skip("GCSFeedStorage requires google-cloud-storage")
+ pytest.importorskip("google.cloud.storage")
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
crawler = get_crawler(settings_dict=settings)
@@ -450,10 +447,7 @@ class TestGCSFeedStorage:
assert storage.blob_name == "export.csv"
def test_parse_empty_acl(self):
- try:
- from google.cloud.storage import Client # noqa: F401,PLC0415
- except ImportError:
- pytest.skip("GCSFeedStorage requires google-cloud-storage")
+ pytest.importorskip("google.cloud.storage")
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
crawler = get_crawler(settings_dict=settings)
@@ -467,10 +461,7 @@ class TestGCSFeedStorage:
@coroutine_test
async def test_store(self):
- try:
- from google.cloud.storage import Client # noqa: F401,PLC0415
- except ImportError:
- pytest.skip("GCSFeedStorage requires google-cloud-storage")
+ pytest.importorskip("google.cloud.storage")
uri = "gs://mybucket/export.csv"
project_id = "myproject-123"
@@ -492,10 +483,7 @@ class TestGCSFeedStorage:
@coroutine_test
async def test_store_closes_file_on_upload_error(self):
- try:
- from google.cloud.storage import Client # noqa: F401,PLC0415
- except ImportError:
- pytest.skip("GCSFeedStorage requires google-cloud-storage")
+ pytest.importorskip("google.cloud.storage")
uri = "gs://mybucket/export.csv"
project_id = "myproject-123"
diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.py
index c1b2f28c6..38eaebf41 100644
--- a/tests/utils/robotstxt.py
+++ b/tests/utils/robotstxt.py
@@ -1,12 +1,7 @@
from __future__ import annotations
+from importlib.util import find_spec
+
def rerp_available() -> bool:
- # check if robotexclusionrulesparser is installed
- try:
- from robotexclusionrulesparser import ( # noqa: PLC0415
- RobotExclusionRulesParser, # noqa: F401
- )
- except ImportError:
- return False
- return True
+ return find_spec("robotexclusionrulesparser") is not None