Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove deprecated test utils.
Andrey Rakhmatullin14f49ab63c54
diff --git a/conftest.py b/conftest.pyindex d49901a7c..4f8d32e1b 100644--- a/conftest.py+++ b/conftest.py@@ -23,9 +23,6 @@ def _py_files(folder):collect_ignore = [# may need extra deps"docs/_ext",- # not a test, but looks like a test- "scrapy/utils/testproc.py",- "scrapy/utils/testsite.py",# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess*_py_files("tests/AsyncCrawlerProcess"),# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocessdiff --git a/pyproject.toml b/pyproject.tomlindex d96790873..d35b333ff 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -127,7 +127,6 @@ ignore_errors = truemodule = ["scrapy.core.downloader.webclient","scrapy.spiders.init",- "scrapy.utils.testsite","tests.test_webclient",]allow_any_generics = truediff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 5c69dc76d..b4e20c3c6 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -7,20 +7,14 @@ from __future__ import annotationsimport asyncioimport osimport warnings-from ftplib import FTPfrom importlib import import_modulefrom pathlib import Path-from posixpath import splitfrom typing import TYPE_CHECKING, Any, TypeVar, cast-from unittest import mock-from twisted.trial.unittest import SkipTestfrom twisted.web.client import Agentfrom scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBasefrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.boto import is_botocore_available-from scrapy.utils.deprecate import create_deprecated_classfrom scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installedfrom scrapy.utils.spider import DefaultSpider@@ -37,80 +31,6 @@ if TYPE_CHECKING:_T = TypeVar("_T")-def assert_gcs_environ() -> None: # pragma: no cover- warnings.warn(- "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy."- " Check GCS_PROJECT_ID directly.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- if "GCS_PROJECT_ID" not in os.environ:- raise SkipTest("GCS_PROJECT_ID not found")---def skip_if_no_boto() -> None: # pragma: no cover- warnings.warn(- "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy."- " Check scrapy.utils.boto.is_botocore_available() directly.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- if not is_botocore_available():- raise SkipTest("missing botocore library")---def get_gcs_content_and_delete(- bucket: Any, path: str-) -> tuple[bytes, list[dict[str, str]], Any]: # pragma: no cover- from google.cloud import storage # noqa: PLC0415-- warnings.warn(- "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))- bucket = client.get_bucket(bucket)- blob = bucket.get_blob(path)- content = blob.download_as_string()- acl = list(blob.acl) # loads acl before it will be deleted- bucket.delete_blob(path)- return content, acl, blob---def get_ftp_content_and_delete(- path: str,- host: str,- port: int,- username: str,- password: str,- use_active_mode: bool = False,-) -> bytes: # pragma: no cover- warnings.warn(- "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )- ftp = FTP()- ftp.connect(host, port)- ftp.login(username, password)- if use_active_mode:- ftp.set_pasv(False)- ftp_data: list[bytes] = []-- def buffer_data(data: bytes) -> None:- ftp_data.append(data)-- ftp.retrbinary(f"RETR {path}", buffer_data)- dirname, filename = split(path)- ftp.cwd(dirname)- ftp.delete(filename)- return b"".join(ftp_data)---TestSpider = create_deprecated_class("TestSpider", DefaultSpider)--def get_reactor_settings() -> dict[str, Any]:"""Return a settings dict that works with the installed reactor.@@ -185,29 +105,6 @@ def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:return getter-def mock_google_cloud_storage() -> tuple[Any, Any, Any]: # pragma: no cover- """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob- classes and set their proper return values.- """- from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415-- warnings.warn(- "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",- category=ScrapyDeprecationWarning,- stacklevel=2,- )-- client_mock = mock.create_autospec(Client)-- bucket_mock = mock.create_autospec(Bucket)- client_mock.get_bucket.return_value = bucket_mock-- blob_mock = mock.create_autospec(Blob)- bucket_mock.blob.return_value = blob_mock-- return (client_mock, bucket_mock, blob_mock)--def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: # pragma: no coverwarnings.warn("The get_web_client_agent_req() function is deprecated"diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pydeleted file mode 100644index 7d548524a..000000000--- a/scrapy/utils/testproc.py+++ /dev/null@@ -1,78 +0,0 @@-# pragma: no file cover-from __future__ import annotations--import os-import sys-import warnings-from typing import TYPE_CHECKING, ClassVar, cast--from twisted.internet.defer import Deferred-from twisted.internet.protocol import ProcessProtocol--from scrapy.exceptions import ScrapyDeprecationWarning--if TYPE_CHECKING:- from collections.abc import Iterable-- from twisted.internet.error import ProcessTerminated- from twisted.python.failure import Failure---warnings.warn(- "The scrapy.utils.testproc module is deprecated.",- ScrapyDeprecationWarning,- stacklevel=2,-)---class ProcessTest:- command: str | None = None- prefix: ClassVar[list[str]] = [sys.executable, "-m", "scrapy.cmdline"]- cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109-- def execute(- self,- args: Iterable[str],- check_code: bool = True,- settings: str | None = None,- ) -> Deferred[TestProcessProtocol]:- from twisted.internet import reactor-- env = os.environ.copy()- if settings is not None:- env["SCRAPY_SETTINGS_MODULE"] = settings- assert self.command- cmd = [*self.prefix, self.command, *args]- pp = TestProcessProtocol()- pp.deferred.addCallback(self._process_finished, cmd, check_code)- reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)- return pp.deferred-- def _process_finished(- self, pp: TestProcessProtocol, cmd: list[str], check_code: bool- ) -> tuple[int, bytes, bytes]:- if pp.exitcode and check_code:- msg = f"process {cmd} exit with code {pp.exitcode}"- msg += f"\n>>> stdout <<<\n{pp.out.decode()}"- msg += "\n"- msg += f"\n>>> stderr <<<\n{pp.err.decode()}"- raise RuntimeError(msg)- return cast("int", pp.exitcode), pp.out, pp.err---class TestProcessProtocol(ProcessProtocol):- def __init__(self) -> None:- self.deferred: Deferred[TestProcessProtocol] = Deferred()- self.out: bytes = b""- self.err: bytes = b""- self.exitcode: int | None = None-- def outReceived(self, data: bytes) -> None:- self.out += data-- def errReceived(self, data: bytes) -> None:- self.err += data-- def processEnded(self, status: Failure) -> None:- self.exitcode = cast("ProcessTerminated", status.value).exitCode- self.deferred.callback(self)diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pydeleted file mode 100644index 4089e86ed..000000000--- a/scrapy/utils/testsite.py+++ /dev/null@@ -1,65 +0,0 @@-# pragma: no file cover-import warnings-from urllib.parse import urljoin--from twisted.web import resource, server, static, util--from scrapy.exceptions import ScrapyDeprecationWarning--warnings.warn(- "The scrapy.utils.testsite module is deprecated.",- ScrapyDeprecationWarning,- stacklevel=2,-)---class SiteTest:- def setUp(self):- from twisted.internet import reactor-- super().setUp()- self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")- self.baseurl = f"http://localhost:{self.site.getHost().port}/"-- def tearDown(self):- super().tearDown()- self.site.stopListening()-- def url(self, path: str) -> str:- return urljoin(self.baseurl, path)---class NoMetaRefreshRedirect(util.Redirect):- def render(self, request: server.Request) -> bytes:- content = util.Redirect.render(self, request)- return content.replace(- b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'- )---def test_site():- r = resource.Resource()- r.putChild(b"text", static.Data(b"Works", "text/plain"))- r.putChild(- b"html",- static.Data(- b"<body><p class='one'>Works</p><p class='two'>World</p></body>",- "text/html",- ),- )- r.putChild(- b"enc-gb18030",- static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),- )- r.putChild(b"redirect", util.Redirect(b"/redirected"))- r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))- r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))- return server.Site(r)---if __name__ == "__main__":- from twisted.internet import reactor # pylint: disable=ungrouped-imports-- port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")- print(f"http://localhost:{port.getHost().port}/")- reactor.run()