Scrapy, a fast high-level web crawling & scraping framework for Python.
Replace import tests with find_spec() or importorskip(). (#7755)
Andrey Rakhmatullin64358f547bfa
diff --git a/conftest.py b/conftest.pyindex 7403b07b2..1bfb002cf 100644--- a/conftest.py+++ b/conftest.py@@ -1,6 +1,6 @@from __future__ import annotations-import importlib+from importlib.util import find_specfrom pathlib import Pathfrom typing import TYPE_CHECKING@@ -50,9 +50,7 @@ if not H2_ENABLED:))-try:- import httpx # noqa: F401-except ImportError:+if not find_spec("httpx"):collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")@@ -130,11 +128,8 @@ def pytest_runtest_setup(item):]for module in optional_deps:- if item.get_closest_marker(f"requires_{module}"):- try:- importlib.import_module(module)- except ImportError:- pytest.skip(f"{module} is not installed")+ if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:+ pytest.skip(f"{module} is not installed")if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:pytest.skip("mitmdump is not available")diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 13a432476..b99c323c5 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -1,6 +1,7 @@from __future__ import annotationsimport warnings+from importlib.util import find_specfrom itertools import chainfrom logging import getLoggerfrom typing import TYPE_CHECKING, Any@@ -51,11 +52,7 @@ else:else:ACCEPTED_ENCODINGS.append(b"br")-try:- import zstandard # noqa: F401-except ImportError:- pass-else:+if find_spec("zstandard") is not None:ACCEPTED_ENCODINGS.append(b"zstd")diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.pyindex 73f86bc71..2a77ee2ac 100644--- a/scrapy/utils/boto.py+++ b/scrapy/utils/boto.py@@ -1,10 +1,7 @@"""Boto/botocore helpers"""+from importlib.util import find_spec-def is_botocore_available() -> bool:- try:- import botocore # noqa: F401,PLC0415- return True- except ImportError:- return False+def is_botocore_available() -> bool:+ return find_spec("botocore") is not Nonediff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 311aff7df..ca18c8038 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -1,4 +1,5 @@from gzip import GzipFile+from importlib.util import find_specfrom io import BytesIOfrom logging import WARNINGfrom pathlib import Path@@ -65,10 +66,7 @@ def _skip_if_no_br() -> None:def _skip_if_no_zstd() -> None:- try:- import zstandard # noqa: F401,PLC0415- except ImportError:- pytest.skip("no zstd support (zstandard)")+ pytest.importorskip("zstandard")class TestHttpCompression:@@ -172,17 +170,8 @@ class TestHttpCompression:self.assertStatsEqual("httpcompression/response_bytes", 74837)def test_process_response_br_unsupported(self):- try:- try:- import brotli # noqa: F401,PLC0415-- pytest.skip("Requires not having brotli support")- except ImportError:- import brotlicffi # noqa: F401,PLC0415-- pytest.skip("Requires not having brotli support")- except ImportError:- pass+ if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:+ pytest.skip("Requires not having brotli support")response = self._getresponse("br")request = response.requestassert response.headers["Content-Encoding"] == b"br"@@ -226,12 +215,8 @@ class TestHttpCompression:assert "Content-Encoding" not in newresponse.headersdef test_process_response_zstd_unsupported(self):- try:- import zstandard # noqa: F401,PLC0415-+ if find_spec("zstandard") is not None:pytest.skip("Requires not having zstandard support")- except ImportError:- passresponse = self._getresponse("zstd-static-content-size")request = response.requestassert response.headers["Content-Encoding"] == b"zstd"diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex 82e2597a5..bdd0d7614 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -436,10 +436,7 @@ class TestS3FeedStorage:class TestGCSFeedStorage:def test_parse_settings(self):- try:- from google.cloud.storage import Client # noqa: F401,PLC0415- except ImportError:- pytest.skip("GCSFeedStorage requires google-cloud-storage")+ pytest.importorskip("google.cloud.storage")settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}crawler = get_crawler(settings_dict=settings)@@ -450,10 +447,7 @@ class TestGCSFeedStorage:assert storage.blob_name == "export.csv"def test_parse_empty_acl(self):- try:- from google.cloud.storage import Client # noqa: F401,PLC0415- except ImportError:- pytest.skip("GCSFeedStorage requires google-cloud-storage")+ pytest.importorskip("google.cloud.storage")settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}crawler = get_crawler(settings_dict=settings)@@ -467,10 +461,7 @@ class TestGCSFeedStorage:@coroutine_testasync def test_store(self):- try:- from google.cloud.storage import Client # noqa: F401,PLC0415- except ImportError:- pytest.skip("GCSFeedStorage requires google-cloud-storage")+ pytest.importorskip("google.cloud.storage")uri = "gs://mybucket/export.csv"project_id = "myproject-123"@@ -492,10 +483,7 @@ class TestGCSFeedStorage:@coroutine_testasync def test_store_closes_file_on_upload_error(self):- try:- from google.cloud.storage import Client # noqa: F401,PLC0415- except ImportError:- pytest.skip("GCSFeedStorage requires google-cloud-storage")+ pytest.importorskip("google.cloud.storage")uri = "gs://mybucket/export.csv"project_id = "myproject-123"diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.pyindex c1b2f28c6..38eaebf41 100644--- a/tests/utils/robotstxt.py+++ b/tests/utils/robotstxt.py@@ -1,12 +1,7 @@from __future__ import annotations+from importlib.util import find_spec+def rerp_available() -> bool:- # check if robotexclusionrulesparser is installed- try:- from robotexclusionrulesparser import ( # noqa: PLC0415- RobotExclusionRulesParser, # noqa: F401- )- except ImportError:- return False- return True+ return find_spec("robotexclusionrulesparser") is not None