Scrapy, a fast high-level web crawling & scraping framework for Python.
Move most of the test utils inside tests.
Andrey Rakhmatullina5731c1944d8
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex e89786103..db1f5c419 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -34,11 +34,23 @@ _T = TypeVar("_T")def assert_gcs_environ() -> None:+ warnings.warn(+ "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy."+ " Check GCS_PROJECT_ID directly.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )if "GCS_PROJECT_ID" not in os.environ:raise SkipTest("GCS_PROJECT_ID not found")def skip_if_no_boto() -> None:+ warnings.warn(+ "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy."+ " Check scrapy.utils.boto.is_botocore_available() directly.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )if not is_botocore_available():raise SkipTest("missing botocore library")@@ -48,6 +60,11 @@ def get_gcs_content_and_delete() -> tuple[bytes, list[dict[str, str]], Any]:from google.cloud import storage+ warnings.warn(+ "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))bucket = client.get_bucket(bucket)blob = bucket.get_blob(path)@@ -67,6 +84,11 @@ def get_ftp_content_and_delete() -> bytes:from ftplib import FTP+ warnings.warn(+ "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )ftp = FTP()ftp.connect(host, port)ftp.login(username, password)@@ -150,6 +172,12 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:"""from google.cloud.storage import Blob, Bucket, Client+ warnings.warn(+ "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+client_mock = mock.create_autospec(Client)bucket_mock = mock.create_autospec(Bucket)diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pyindex 85d7c940f..10f764ab8 100644--- a/scrapy/utils/testproc.py+++ b/scrapy/utils/testproc.py@@ -2,18 +2,27 @@ from __future__ import annotationsimport osimport sys+import warningsfrom typing import TYPE_CHECKING, castfrom twisted.internet.defer import Deferredfrom twisted.internet.error import ProcessTerminatedfrom twisted.internet.protocol import ProcessProtocol+from scrapy.exceptions import ScrapyDeprecationWarning+if TYPE_CHECKING:from collections.abc import Iterablefrom twisted.python.failure import Failure+warnings.warn(+ "The scrapy.utils.testproc module is deprecated.",+ ScrapyDeprecationWarning,+)++class ProcessTest:command: str | None = Noneprefix = [sys.executable, "-m", "scrapy.cmdline"]diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pyindex ca1f68116..f12b301fd 100644--- a/scrapy/utils/testsite.py+++ b/scrapy/utils/testsite.py@@ -1,7 +1,15 @@+import warningsfrom urllib.parse import urljoinfrom twisted.web import resource, server, static, util+from scrapy.exceptions import ScrapyDeprecationWarning++warnings.warn(+ "The scrapy.utils.testsite module is deprecated.",+ ScrapyDeprecationWarning,+)+class SiteTest:def setUp(self):@@ -48,7 +56,7 @@ def test_site():if __name__ == "__main__":- from twisted.internet import reactor+ from twisted.internet import reactor # pylint: disable=ungrouped-importsport = reactor.listenTCP(0, test_site(), interface="127.0.0.1")print(f"http://localhost:{port.getHost().port}/")diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex d2027d1c2..a4d7fdd30 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -1,8 +1,8 @@from twisted.internet import deferfrom twisted.trial import unittest-from scrapy.utils.testproc import ProcessTest-from scrapy.utils.testsite import SiteTest+from tests.utils.testproc import ProcessTest+from tests.utils.testsite import SiteTestclass FetchTest(ProcessTest, SiteTest, unittest.TestCase):diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 9356d6b79..9f2c7fa13 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -7,9 +7,9 @@ from twisted.internet import deferfrom scrapy.commands import parsefrom scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode-from scrapy.utils.testproc import ProcessTest-from scrapy.utils.testsite import SiteTestfrom tests.test_commands import CommandTest+from tests.utils.testproc import ProcessTest+from tests.utils.testsite import SiteTestdef _textmode(bstr):diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 7918d94b2..9ca5e05dc 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -7,10 +7,10 @@ from pexpect.popen_spawn import PopenSpawnfrom twisted.internet import deferfrom twisted.trial import unittest-from scrapy.utils.testproc import ProcessTest-from scrapy.utils.testsite import SiteTestfrom tests import NON_EXISTING_RESOLVABLE, tests_datadirfrom tests.mockserver import MockServer+from tests.utils.testproc import ProcessTest+from tests.utils.testsite import SiteTestclass ShellTest(ProcessTest, SiteTest, unittest.TestCase):diff --git a/tests/test_command_version.py b/tests/test_command_version.pyindex 18c1c531c..917f457cb 100644--- a/tests/test_command_version.py+++ b/tests/test_command_version.py@@ -4,7 +4,7 @@ from twisted.internet import deferfrom twisted.trial import unittestimport scrapy-from scrapy.utils.testproc import ProcessTest+from tests.utils.testproc import ProcessTestclass VersionTest(ProcessTest, unittest.TestCase):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex b4c1b9631..8e008ab98 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -17,7 +17,7 @@ from io import BytesIOfrom logging import getLoggerfrom pathlib import Pathfrom string import ascii_letters, digits-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom unittest import mockfrom urllib.parse import quote, urljoinfrom urllib.request import pathname2url@@ -48,7 +48,7 @@ from scrapy.extensions.feedexport import ()from scrapy.settings import Settingsfrom scrapy.utils.python import to_unicode-from scrapy.utils.test import get_crawler, mock_google_cloud_storage+from scrapy.utils.test import get_crawlerfrom tests.mockserver import MockFTPServer, MockServerfrom tests.spiders import ItemSpider@@ -71,6 +71,23 @@ def build_url(path: str | PathLike) -> str:return urljoin("file:", path_str)+def mock_google_cloud_storage() -> tuple[Any, Any, Any]:+ """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob+ classes and set their proper return values.+ """+ from google.cloud.storage import Blob, Bucket, Client++ client_mock = mock.create_autospec(Client)++ bucket_mock = mock.create_autospec(Bucket)+ client_mock.get_bucket.return_value = bucket_mock++ blob_mock = mock.create_autospec(Blob)+ bucket_mock.blob.return_value = blob_mock++ return (client_mock, bucket_mock, blob_mock)++class FileFeedStorageTest(unittest.TestCase):def test_store_file_uri(self):path = Path(self.mktemp()).resolve()diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 4c59fcfb7..05fd17207 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -6,8 +6,10 @@ import warningsfrom datetime import datetimefrom io import BytesIOfrom pathlib import Path+from posixpath import splitfrom shutil import rmtreefrom tempfile import mkdtemp+from typing import Anyfrom unittest import mockfrom urllib.parse import urlparse@@ -27,16 +29,54 @@ from scrapy.pipelines.files import (S3FilesStore,)from scrapy.utils.test import (- assert_gcs_environ,get_crawler,- get_ftp_content_and_delete,- get_gcs_content_and_delete,)from tests.mockserver import MockFTPServerfrom .test_pipeline_media import _mocked_download_func+def get_gcs_content_and_delete(+ bucket: Any, path: str+) -> tuple[bytes, list[dict[str, str]], Any]:+ from google.cloud import storage++ client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))+ bucket = client.get_bucket(bucket)+ blob = bucket.get_blob(path)+ content = blob.download_as_string()+ acl = list(blob.acl) # loads acl before it will be deleted+ bucket.delete_blob(path)+ return content, acl, blob+++def get_ftp_content_and_delete(+ path: str,+ host: str,+ port: int,+ username: str,+ password: str,+ use_active_mode: bool = False,+) -> bytes:+ from ftplib import FTP++ ftp = FTP()+ ftp.connect(host, port)+ ftp.login(username, password)+ if use_active_mode:+ ftp.set_pasv(False)+ ftp_data: list[bytes] = []++ def buffer_data(data: bytes) -> None:+ ftp_data.append(data)++ ftp.retrbinary(f"RETR {path}", buffer_data)+ dirname, filename = split(path)+ ftp.cwd(dirname)+ ftp.delete(filename)+ return b"".join(ftp_data)++class FilesPipelineTestCase(unittest.TestCase):def setUp(self):self.tempdir = mkdtemp()@@ -597,10 +637,12 @@ class TestS3FilesStore(unittest.TestCase):stub.assert_no_pending_responses()+ "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"+)class TestGCSFilesStore(unittest.TestCase):@defer.inlineCallbacksdef test_persist(self):- assert_gcs_environ()uri = os.environ.get("GCS_TEST_FILE_URI")if not uri:raise unittest.SkipTest("No GCS URI available for testing")@@ -629,7 +671,6 @@ class TestGCSFilesStore(unittest.TestCase):"""Test to make sure that paths used to store files is the same as the one used to getalready uploaded files."""- assert_gcs_environ()try:import google.cloud.storage # noqa: F401except ModuleNotFoundError:diff --git a/tests/utils/__init__.py b/tests/utils/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/utils/testproc.py b/tests/utils/testproc.pynew file mode 100644index 000000000..85d7c940f--- /dev/null+++ b/tests/utils/testproc.py@@ -0,0 +1,67 @@+from __future__ import annotations++import os+import sys+from typing import TYPE_CHECKING, cast++from twisted.internet.defer import Deferred+from twisted.internet.error import ProcessTerminated+from twisted.internet.protocol import ProcessProtocol++if TYPE_CHECKING:+ from collections.abc import Iterable++ from twisted.python.failure import Failure+++class ProcessTest:+ command: str | None = None+ prefix = [sys.executable, "-m", "scrapy.cmdline"]+ cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109++ def execute(+ self,+ args: Iterable[str],+ check_code: bool = True,+ settings: str | None = None,+ ) -> Deferred[TestProcessProtocol]:+ from twisted.internet import reactor++ env = os.environ.copy()+ if settings is not None:+ env["SCRAPY_SETTINGS_MODULE"] = settings+ assert self.command+ cmd = [*self.prefix, self.command, *args]+ pp = TestProcessProtocol()+ pp.deferred.addCallback(self._process_finished, cmd, check_code)+ reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)+ return pp.deferred++ def _process_finished(+ self, pp: TestProcessProtocol, cmd: list[str], check_code: bool+ ) -> tuple[int, bytes, bytes]:+ if pp.exitcode and check_code:+ msg = f"process {cmd} exit with code {pp.exitcode}"+ msg += f"\n>>> stdout <<<\n{pp.out.decode()}"+ msg += "\n"+ msg += f"\n>>> stderr <<<\n{pp.err.decode()}"+ raise RuntimeError(msg)+ return cast(int, pp.exitcode), pp.out, pp.err+++class TestProcessProtocol(ProcessProtocol):+ def __init__(self) -> None:+ self.deferred: Deferred[TestProcessProtocol] = Deferred()+ self.out: bytes = b""+ self.err: bytes = b""+ self.exitcode: int | None = None++ def outReceived(self, data: bytes) -> None:+ self.out += data++ def errReceived(self, data: bytes) -> None:+ self.err += data++ def processEnded(self, status: Failure) -> None:+ self.exitcode = cast(ProcessTerminated, status.value).exitCode+ self.deferred.callback(self)diff --git a/tests/utils/testsite.py b/tests/utils/testsite.pynew file mode 100644index 000000000..473738773--- /dev/null+++ b/tests/utils/testsite.py@@ -0,0 +1,47 @@+from urllib.parse import urljoin++from twisted.web import resource, server, static, util+++class SiteTest:+ def setUp(self):+ from twisted.internet import reactor++ super().setUp()+ self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")+ self.baseurl = f"http://localhost:{self.site.getHost().port}/"++ def tearDown(self):+ super().tearDown()+ self.site.stopListening()++ def url(self, path: str) -> str:+ return urljoin(self.baseurl, path)+++class NoMetaRefreshRedirect(util.Redirect):+ def render(self, request: server.Request) -> bytes:+ content = util.Redirect.render(self, request)+ return content.replace(+ b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'+ )+++def test_site():+ r = resource.Resource()+ r.putChild(b"text", static.Data(b"Works", "text/plain"))+ r.putChild(+ b"html",+ static.Data(+ b"<body><p class='one'>Works</p><p class='two'>World</p></body>",+ "text/html",+ ),+ )+ r.putChild(+ b"enc-gb18030",+ static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),+ )+ r.putChild(b"redirect", util.Redirect(b"/redirected"))+ r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))+ r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))+ return server.Site(r)