Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Move most of the test utils inside tests.

Andrey Rakhmatullina5731c1944d8
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index e89786103..db1f5c419 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -34,11 +34,23 @@ _T = TypeVar("_T")
def assert_gcs_environ() -> None:
+ warnings.warn(
+ "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy."
+ " Check GCS_PROJECT_ID directly.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
if "GCS_PROJECT_ID" not in os.environ:
raise SkipTest("GCS_PROJECT_ID not found")
def skip_if_no_boto() -> None:
+ warnings.warn(
+ "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy."
+ " Check scrapy.utils.boto.is_botocore_available() directly.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
if not is_botocore_available():
raise SkipTest("missing botocore library")
@@ -48,6 +60,11 @@ def get_gcs_content_and_delete(
) -> tuple[bytes, list[dict[str, str]], Any]:
from google.cloud import storage
+ warnings.warn(
+ "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
bucket = client.get_bucket(bucket)
blob = bucket.get_blob(path)
@@ -67,6 +84,11 @@ def get_ftp_content_and_delete(
) -> bytes:
from ftplib import FTP
+ warnings.warn(
+ "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
ftp = FTP()
ftp.connect(host, port)
ftp.login(username, password)
@@ -150,6 +172,12 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""
from google.cloud.storage import Blob, Bucket, Client
+ warnings.warn(
+ "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
client_mock = mock.create_autospec(Client)
bucket_mock = mock.create_autospec(Bucket)
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 85d7c940f..10f764ab8 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -2,18 +2,27 @@ from __future__ import annotations
import os
import sys
+import warnings
from typing import TYPE_CHECKING, cast
from twisted.internet.defer import Deferred
from twisted.internet.error import ProcessTerminated
from twisted.internet.protocol import ProcessProtocol
+from scrapy.exceptions import ScrapyDeprecationWarning
+
if TYPE_CHECKING:
from collections.abc import Iterable
from twisted.python.failure import Failure
+warnings.warn(
+ "The scrapy.utils.testproc module is deprecated.",
+ ScrapyDeprecationWarning,
+)
+
+
class ProcessTest:
command: str | None = None
prefix = [sys.executable, "-m", "scrapy.cmdline"]
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index ca1f68116..f12b301fd 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,7 +1,15 @@
+import warnings
from urllib.parse import urljoin
from twisted.web import resource, server, static, util
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+warnings.warn(
+ "The scrapy.utils.testsite module is deprecated.",
+ ScrapyDeprecationWarning,
+)
+
class SiteTest:
def setUp(self):
@@ -48,7 +56,7 @@ def test_site():
if __name__ == "__main__":
- from twisted.internet import reactor
+ from twisted.internet import reactor # pylint: disable=ungrouped-imports
port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
print(f"http://localhost:{port.getHost().port}/")
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index d2027d1c2..a4d7fdd30 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -1,8 +1,8 @@
from twisted.internet import defer
from twisted.trial import unittest
-from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.testsite import SiteTest
+from tests.utils.testproc import ProcessTest
+from tests.utils.testsite import SiteTest
class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 9356d6b79..9f2c7fa13 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -7,9 +7,9 @@ from twisted.internet import defer
from scrapy.commands import parse
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
-from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.testsite import SiteTest
from tests.test_commands import CommandTest
+from tests.utils.testproc import ProcessTest
+from tests.utils.testsite import SiteTest
def _textmode(bstr):
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 7918d94b2..9ca5e05dc 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -7,10 +7,10 @@ from pexpect.popen_spawn import PopenSpawn
from twisted.internet import defer
from twisted.trial import unittest
-from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.testsite import SiteTest
from tests import NON_EXISTING_RESOLVABLE, tests_datadir
from tests.mockserver import MockServer
+from tests.utils.testproc import ProcessTest
+from tests.utils.testsite import SiteTest
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 18c1c531c..917f457cb 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -4,7 +4,7 @@ from twisted.internet import defer
from twisted.trial import unittest
import scrapy
-from scrapy.utils.testproc import ProcessTest
+from tests.utils.testproc import ProcessTest
class VersionTest(ProcessTest, unittest.TestCase):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b4c1b9631..8e008ab98 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -17,7 +17,7 @@ from io import BytesIO
from logging import getLogger
from pathlib import Path
from string import ascii_letters, digits
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from unittest import mock
from urllib.parse import quote, urljoin
from urllib.request import pathname2url
@@ -48,7 +48,7 @@ from scrapy.extensions.feedexport import (
)
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
-from scrapy.utils.test import get_crawler, mock_google_cloud_storage
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockFTPServer, MockServer
from tests.spiders import ItemSpider
@@ -71,6 +71,23 @@ def build_url(path: str | PathLike) -> str:
return urljoin("file:", path_str)
+def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
+ """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+ classes and set their proper return values.
+ """
+ from google.cloud.storage import Blob, Bucket, Client
+
+ client_mock = mock.create_autospec(Client)
+
+ bucket_mock = mock.create_autospec(Bucket)
+ client_mock.get_bucket.return_value = bucket_mock
+
+ blob_mock = mock.create_autospec(Blob)
+ bucket_mock.blob.return_value = blob_mock
+
+ return (client_mock, bucket_mock, blob_mock)
+
+
class FileFeedStorageTest(unittest.TestCase):
def test_store_file_uri(self):
path = Path(self.mktemp()).resolve()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4c59fcfb7..05fd17207 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -6,8 +6,10 @@ import warnings
from datetime import datetime
from io import BytesIO
from pathlib import Path
+from posixpath import split
from shutil import rmtree
from tempfile import mkdtemp
+from typing import Any
from unittest import mock
from urllib.parse import urlparse
@@ -27,16 +29,54 @@ from scrapy.pipelines.files import (
S3FilesStore,
)
from scrapy.utils.test import (
- assert_gcs_environ,
get_crawler,
- get_ftp_content_and_delete,
- get_gcs_content_and_delete,
)
from tests.mockserver import MockFTPServer
from .test_pipeline_media import _mocked_download_func
+def get_gcs_content_and_delete(
+ bucket: Any, path: str
+) -> tuple[bytes, list[dict[str, str]], Any]:
+ from google.cloud import storage
+
+ client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
+ bucket = client.get_bucket(bucket)
+ blob = bucket.get_blob(path)
+ content = blob.download_as_string()
+ acl = list(blob.acl) # loads acl before it will be deleted
+ bucket.delete_blob(path)
+ return content, acl, blob
+
+
+def get_ftp_content_and_delete(
+ path: str,
+ host: str,
+ port: int,
+ username: str,
+ password: str,
+ use_active_mode: bool = False,
+) -> bytes:
+ from ftplib import FTP
+
+ ftp = FTP()
+ ftp.connect(host, port)
+ ftp.login(username, password)
+ if use_active_mode:
+ ftp.set_pasv(False)
+ ftp_data: list[bytes] = []
+
+ def buffer_data(data: bytes) -> None:
+ ftp_data.append(data)
+
+ ftp.retrbinary(f"RETR {path}", buffer_data)
+ dirname, filename = split(path)
+ ftp.cwd(dirname)
+ ftp.delete(filename)
+ return b"".join(ftp_data)
+
+
class FilesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = mkdtemp()
@@ -597,10 +637,12 @@ class TestS3FilesStore(unittest.TestCase):
stub.assert_no_pending_responses()
+ "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"
+)
class TestGCSFilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
- assert_gcs_environ()
uri = os.environ.get("GCS_TEST_FILE_URI")
if not uri:
raise unittest.SkipTest("No GCS URI available for testing")
@@ -629,7 +671,6 @@ class TestGCSFilesStore(unittest.TestCase):
"""Test to make sure that paths used to store files is the same as the one used to get
already uploaded files.
"""
- assert_gcs_environ()
try:
import google.cloud.storage # noqa: F401
except ModuleNotFoundError:
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/utils/testproc.py b/tests/utils/testproc.py
new file mode 100644
index 000000000..85d7c940f
--- /dev/null
+++ b/tests/utils/testproc.py
@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import os
+import sys
+from typing import TYPE_CHECKING, cast
+
+from twisted.internet.defer import Deferred
+from twisted.internet.error import ProcessTerminated
+from twisted.internet.protocol import ProcessProtocol
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
+ from twisted.python.failure import Failure
+
+
+class ProcessTest:
+ command: str | None = None
+ prefix = [sys.executable, "-m", "scrapy.cmdline"]
+ cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109
+
+ def execute(
+ self,
+ args: Iterable[str],
+ check_code: bool = True,
+ settings: str | None = None,
+ ) -> Deferred[TestProcessProtocol]:
+ from twisted.internet import reactor
+
+ env = os.environ.copy()
+ if settings is not None:
+ env["SCRAPY_SETTINGS_MODULE"] = settings
+ assert self.command
+ cmd = [*self.prefix, self.command, *args]
+ pp = TestProcessProtocol()
+ pp.deferred.addCallback(self._process_finished, cmd, check_code)
+ reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
+ return pp.deferred
+
+ def _process_finished(
+ self, pp: TestProcessProtocol, cmd: list[str], check_code: bool
+ ) -> tuple[int, bytes, bytes]:
+ if pp.exitcode and check_code:
+ msg = f"process {cmd} exit with code {pp.exitcode}"
+ msg += f"\n>>> stdout <<<\n{pp.out.decode()}"
+ msg += "\n"
+ msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
+ raise RuntimeError(msg)
+ return cast(int, pp.exitcode), pp.out, pp.err
+
+
+class TestProcessProtocol(ProcessProtocol):
+ def __init__(self) -> None:
+ self.deferred: Deferred[TestProcessProtocol] = Deferred()
+ self.out: bytes = b""
+ self.err: bytes = b""
+ self.exitcode: int | None = None
+
+ def outReceived(self, data: bytes) -> None:
+ self.out += data
+
+ def errReceived(self, data: bytes) -> None:
+ self.err += data
+
+ def processEnded(self, status: Failure) -> None:
+ self.exitcode = cast(ProcessTerminated, status.value).exitCode
+ self.deferred.callback(self)
diff --git a/tests/utils/testsite.py b/tests/utils/testsite.py
new file mode 100644
index 000000000..473738773
--- /dev/null
+++ b/tests/utils/testsite.py
@@ -0,0 +1,47 @@
+from urllib.parse import urljoin
+
+from twisted.web import resource, server, static, util
+
+
+class SiteTest:
+ def setUp(self):
+ from twisted.internet import reactor
+
+ super().setUp()
+ self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
+ self.baseurl = f"http://localhost:{self.site.getHost().port}/"
+
+ def tearDown(self):
+ super().tearDown()
+ self.site.stopListening()
+
+ def url(self, path: str) -> str:
+ return urljoin(self.baseurl, path)
+
+
+class NoMetaRefreshRedirect(util.Redirect):
+ def render(self, request: server.Request) -> bytes:
+ content = util.Redirect.render(self, request)
+ return content.replace(
+ b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
+ )
+
+
+def test_site():
+ r = resource.Resource()
+ r.putChild(b"text", static.Data(b"Works", "text/plain"))
+ r.putChild(
+ b"html",
+ static.Data(
+ b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
+ "text/html",
+ ),
+ )
+ r.putChild(
+ b"enc-gb18030",
+ static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
+ )
+ r.putChild(b"redirect", util.Redirect(b"/redirected"))
+ r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
+ r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
+ return server.Site(r)