Scrapy, a fast high-level web crawling & scraping framework for Python.
Type tests involving items and exports (#7867)
Adriana5614544a2ae
diff --git a/pyproject.toml b/pyproject.tomlindex 4edfb7ec8..5ec1d05a4 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -123,23 +123,11 @@ module = ["tests.test_downloaderslotssettings","tests.test_dupefilters","tests.test_engine_loop",- "tests.test_exporters","tests.test_extension_statsmailer","tests.test_extension_throttle",- "tests.test_feedexport",- "tests.test_feedexport_postprocess",- "tests.test_feedexport_storages",- "tests.test_feedexport_uri_params",- "tests.test_item","tests.test_linkextractors",- "tests.test_loader","tests.test_logformatter","tests.test_mail",- "tests.test_pipeline_crawl",- "tests.test_pipeline_files",- "tests.test_pipeline_images",- "tests.test_pipeline_media",- "tests.test_pipelines","tests.test_pqueues","tests.test_scheduler_base","tests.test_settings",diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex fc2b2f43f..e4294ac90 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -149,7 +149,7 @@ class BlockingFeedStorage(ABC):return NamedTemporaryFile(prefix="feed-", dir=path)- def store(self, file: IO[bytes]) -> Deferred[None] | None:+ def store(self, file: IO[bytes]) -> Deferred[None]:return deferred_from_coro(run_in_thread(self._store_in_thread, file))@abstractmethoddiff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 79b6c4f27..5e7a4b409 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -29,7 +29,7 @@ from scrapy.utils.defer import ensure_awaitablefrom scrapy.utils.python import to_bytesif TYPE_CHECKING:- from collections.abc import Iterable+ from collections.abc import Iteratorfrom os import PathLikefrom PIL import Image@@ -180,7 +180,7 @@ class ImagesPipeline(FilesPipeline):info: MediaPipeline.SpiderInfo,*,item: Any = None,- ) -> Iterable[tuple[str, Image.Image, BytesIO]]:+ ) -> Iterator[tuple[str, Image.Image, BytesIO]]:path = self.file_path(request, response=response, info=info, item=item)orig_image = self._Image.open(BytesIO(response.body))transposed_image = self._ImageOps.exif_transpose(orig_image)diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.pyindex 1edd64dda..72760b5ac 100644--- a/tests/mockserver/ftp.py+++ b/tests/mockserver/ftp.py@@ -27,11 +27,12 @@ class MockFTPServer:(anonymous) and a temporary root path that you can read from the:attr:`path` attribute."""+ proc: Popen[str]+ port: int+ path: Path+def __init__(self) -> None:- self.proc: Popen[str] | None = Noneself.host: str = "127.0.0.1"- self.port: int | None = None- self.path: Path | None = Nonedef __enter__(self) -> Self:self.path = Path(mkdtemp())@@ -63,7 +64,6 @@ class MockFTPServer:traceback: TracebackType | None,) -> None:rmtree(str(self.path))- assert self.proc is not Noneself.proc.kill()self.proc.communicate()diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 99afb8beb..956697a04 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -4,6 +4,7 @@ import marshalimport pickleimport refrom abc import ABC, abstractmethod+from collections.abc import Mappingfrom datetime import datetimefrom io import BytesIOfrom typing import Any@@ -63,18 +64,18 @@ class TestBaseItemExporter(ABC):self.ie = self._get_exporter()@abstractmethod- def _get_exporter(self, **kwargs) -> BaseItemExporter:+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:raise NotImplementedError- def _check_output(self): # noqa: B027+ def _check_output(self) -> None: # noqa: B027pass- def _assert_expected_item(self, exported_dict):+ def _assert_expected_item(self, exported_dict: dict[str, Any]) -> None:for k, v in exported_dict.items():exported_dict[k] = to_unicode(v)assert self.i == self.item_class(**exported_dict)- def _get_nonstring_types_item(self):+ def _get_nonstring_types_item(self) -> dict[str, Any]:return {"boolean": False,"number": 22,@@ -82,7 +83,7 @@ class TestBaseItemExporter(ABC):"float": 3.14,}- def assertItemExportWorks(self, item):+ def assertItemExportWorks(self, item: Any) -> None:self.ie.start_exporting()self.ie.export_item(item)self.ie.finish_exporting()@@ -92,7 +93,7 @@ class TestBaseItemExporter(ABC):del self.ieself._check_output()- def test_export_item(self):+ def test_export_item(self) -> None:self.assertItemExportWorks(self.i)def test_export_dict_item(self):@@ -142,7 +143,7 @@ class TestBaseItemExporter(ABC):class TestPythonItemExporter(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:return PythonItemExporter(**kwargs)def test_invalid_option(self):@@ -173,6 +174,7 @@ class TestPythonItemExporter(TestBaseItemExporter):"age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}],"name": "Jesus",}+ assert exported is not Noneassert isinstance(exported["age"][0], dict)assert isinstance(exported["age"][0]["age"][0], dict)@@ -186,6 +188,7 @@ class TestPythonItemExporter(TestBaseItemExporter):"age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}],"name": "Jesus",}+ assert exported is not Noneassert isinstance(exported["age"][0], dict)assert isinstance(exported["age"][0]["age"][0], dict)@@ -202,10 +205,10 @@ class TestPythonItemExporterDataclass(TestPythonItemExporter):class TestPprintItemExporter(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:return PprintItemExporter(self.output, **kwargs)- def _check_output(self):+ def _check_output(self) -> None:self._assert_expected_item(eval(self.output.getvalue()))@@ -215,10 +218,10 @@ class TestPprintItemExporterDataclass(TestPprintItemExporter):class TestPickleItemExporter(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:return PickleItemExporter(self.output, **kwargs)- def _check_output(self):+ def _check_output(self) -> None:self._assert_expected_item(pickle.loads(self.output.getvalue()))def test_export_multiple_items(self):@@ -252,10 +255,10 @@ class TestPickleItemExporterDataclass(TestPickleItemExporter):class TestMarshalItemExporter(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:return MarshalItemExporter(self.output, **kwargs)- def _check_output(self):+ def _check_output(self) -> None:self.output.seek(0)self._assert_expected_item(marshal.load(self.output))@@ -279,7 +282,7 @@ class TestMarshalItemExporterDataclass(TestMarshalItemExporter):class TestCsvItemExporter(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:# We need a fresh instance for each exporter, because# CsvItemExporter.stream.__del__() closes the underlying file# (CsvItemExporter.finish_exporting() calls detach() but not all tests@@ -287,8 +290,10 @@ class TestCsvItemExporter(TestBaseItemExporter):self.output = BytesIO()return CsvItemExporter(self.output, **kwargs)- def assertCsvEqual(self, first, second, msg=None):- def split_csv(csv):+ def assertCsvEqual(+ self, first: bytes | str, second: bytes | str, msg: str | None = None+ ) -> None:+ def split_csv(csv: bytes | str) -> list[list[str]]:return [sorted(re.split(r"(,|\s+)", line))for line in to_unicode(csv).splitlines(True)@@ -296,13 +301,15 @@ class TestCsvItemExporter(TestBaseItemExporter):assert split_csv(first) == split_csv(second), msg- def _check_output(self):+ def _check_output(self) -> None:self.output.seek(0)self.assertCsvEqual(to_unicode(self.output.read()), "age,name\r\n22,John\xa3\r\n")- def assertExportResult(self, item, expected, **kwargs):+ def assertExportResult(+ self, item: Any, expected: bytes | str = b"", **kwargs: Any+ ) -> None:fp = BytesIO()ie = CsvItemExporter(fp, **kwargs)ie.start_exporting()@@ -383,7 +390,6 @@ class TestCsvItemExporter(TestBaseItemExporter):with pytest.raises(UnicodeEncodeError):self.assertExportResult(item={"text": "W\u0275\u200brd"},- expected=None,encoding="windows-1251",)@@ -417,7 +423,7 @@ class TestCsvItemExporterDataclass(TestCsvItemExporter):class TestXmlItemExporter(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:# We need a fresh instance for each exporter, because# XmlItemExporter.stream.__del__() closes the underlying file# (XmlItemExporter.finish_exporting() calls detach() but not all tests@@ -425,20 +431,22 @@ class TestXmlItemExporter(TestBaseItemExporter):self.output = BytesIO()return XmlItemExporter(self.output, **kwargs)- def assertXmlEquivalent(self, first, second, msg=None):- def xmltuple(elem):+ def assertXmlEquivalent(+ self, first: bytes, second: bytes, msg: str | None = None+ ) -> None:+ def xmltuple(elem: Any) -> list[Any]:children = list(elem.iterchildren())if children:return [(child.tag, sorted(xmltuple(child))) for child in children]return [(elem.tag, [(elem.text, ())])]- def xmlsplit(xmlcontent):+ def xmlsplit(xmlcontent: bytes) -> list[Any]:doc = lxml.etree.fromstring(xmlcontent)return xmltuple(doc)assert xmlsplit(first) == xmlsplit(second), msg- def assertExportResult(self, item, expected_value):+ def assertExportResult(self, item: Any, expected_value: bytes) -> None:fp = BytesIO()ie = XmlItemExporter(fp)ie.start_exporting()@@ -447,7 +455,7 @@ class TestXmlItemExporter(TestBaseItemExporter):del ie # See the first “del self.ie” in this file for context.self.assertXmlEquivalent(fp.getvalue(), expected_value)- def _check_output(self):+ def _check_output(self) -> None:expected_value = (b'<?xml version="1.0" encoding="utf-8"?>\n'b"<items><item><age>22</age><name>John\xc2\xa3</name></item></items>"@@ -538,10 +546,10 @@ class TestJsonLinesItemExporter(TestBaseItemExporter):"age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}},}- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:return JsonLinesItemExporter(self.output, **kwargs)- def _check_output(self):+ def _check_output(self) -> None:exported = json.loads(to_unicode(self.output.getvalue().strip()))assert exported == ItemAdapter(self.i).asdict()@@ -582,14 +590,14 @@ class TestJsonLinesItemExporterDataclass(TestJsonLinesItemExporter):class TestJsonItemExporter(TestJsonLinesItemExporter):_expected_nested = [TestJsonLinesItemExporter._expected_nested]- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:return JsonItemExporter(self.output, **kwargs)- def _check_output(self):+ def _check_output(self) -> None:exported = json.loads(to_unicode(self.output.getvalue().strip()))assert exported == [ItemAdapter(self.i).asdict()]- def assertTwoItemsExported(self, item):+ def assertTwoItemsExported(self, item: Any) -> None:self.ie.start_exporting()self.ie.export_item(item)self.ie.export_item(item)@@ -658,7 +666,7 @@ class TestJsonItemExporter(TestJsonLinesItemExporter):class TestJsonItemExporterToBytes(TestBaseItemExporter):- def _get_exporter(self, **kwargs):+ def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:kwargs["encoding"] = "latin"return JsonItemExporter(self.output, **kwargs)@@ -690,7 +698,9 @@ class TestCustomExporterItem:def test_exporter_custom_serializer(self):class CustomItemExporter(BaseItemExporter):- def serialize_field(self, field, name, value):+ def serialize_field(+ self, field: Mapping[str, Any] | Field, name: str, value: Any+ ) -> Any:if name == "age":return str(int(value) + 1)return super().serialize_field(field, name, value)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 40a763efd..d0f6a297c 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -100,6 +100,8 @@ class InstrumentedFeedSlot(FeedSlot):"""Instrumented FeedSlot subclass for keeping track of calls tostart_exporting and finish_exporting."""+ update_listener: Callable[[str], None]+def start_exporting(self):self.update_listener("start")super().start_exporting()@@ -109,7 +111,7 @@ class InstrumentedFeedSlot(FeedSlot):super().finish_exporting()@classmethod- def subscribe__listener(cls, listener):+ def subscribe__listener(cls, listener: IsExportingListener) -> None:cls.update_listener = listener.update@@ -119,7 +121,7 @@ class IsExportingListener:finish_exporting and when a call to finish_exporting has been madebefore a call to start_exporting."""- def __init__(self):+ def __init__(self) -> None:self.start_without_finish = Falseself.finish_without_start = False@@ -307,6 +309,7 @@ class TestFeedExport(TestFeedExportBase):}crawler = get_crawler(ItemSpider, settings)yield crawler.crawl(mockserver=self.mockserver)+ assert crawler.stats is not Noneassert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1@@ -330,6 +333,7 @@ class TestFeedExport(TestFeedExportBase):side_effect=store,):yield crawler.crawl(mockserver=self.mockserver)+ assert crawler.stats is not Noneassert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1@@ -347,6 +351,7 @@ class TestFeedExport(TestFeedExportBase):}crawler = get_crawler(ItemSpider, settings)yield crawler.crawl(mockserver=self.mockserver)+ assert crawler.stats is not Noneassert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1@@ -487,7 +492,7 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_start_finish_exporting_no_items(self):- items = []+ items: list[Any] = []settings = {"FEEDS": {self._random_temp_filename(): {"format": "json"},@@ -526,7 +531,7 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_start_finish_exporting_no_items_exception(self):- items = []+ items: list[Any] = []settings = {"FEEDS": {self._random_temp_filename(): {"format": "json"},@@ -611,7 +616,7 @@ class TestFeedExport(TestFeedExportBase):items = [{"foo": "bar"}]header = ["foo"]rows = [{"foo": "bar"}]- settings = {"FEED_EXPORT_FIELDS": []}+ settings: dict[str, Any] = {"FEED_EXPORT_FIELDS": []}await self.assertExportedCsv(items, header, rows)await self.assertExportedJsonLines(items, rows, settings)@@ -727,14 +732,14 @@ class TestFeedExport(TestFeedExportBase):def accepts(self, item):return isinstance(item, MyItem)- class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):+ class CustomFilter2(ItemFilter):def accepts(self, item):return "foo" in item.fields- class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):+ class CustomFilter3(ItemFilter):def accepts(self, item):return (- isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1"+ isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" # type: ignore[index])formats = {@@ -834,7 +839,7 @@ class TestFeedExport(TestFeedExportBase):}for fmt, expected in formats.items():- settings = {+ settings: dict[str, Any] = {"FEEDS": {self._random_temp_filename(): {"format": fmt},},@@ -911,7 +916,7 @@ class TestFeedExport(TestFeedExportBase):{"key": "value"},]- test_cases = [+ test_cases: list[dict[str, Any]] = [# JSON{"format": "json",@@ -1132,7 +1137,7 @@ class TestFeedExport(TestFeedExportBase):expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"expected_without_title_csv = b"FOO,BAR\r\n"- test_cases = [+ test_cases: list[dict[str, Any]] = [# with title{"options": {@@ -1166,6 +1171,9 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_storage_file_no_postprocessing(self):class Storage:+ open_file: IO[bytes]+ store_file: IO[bytes]+def __init__(self, uri, *, feed_options=None):pass@@ -1187,6 +1195,10 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_storage_file_postprocessing(self):class Storage:+ open_file: IO[bytes]+ store_file: IO[bytes]+ file_was_closed: bool+def __init__(self, uri, *, feed_options=None):pass@@ -1299,7 +1311,7 @@ class TestItemFilter:class TestFeedExportInit:def test_unsupported_storage(self):- settings = {+ settings: dict[str, Any] = {"FEEDS": {"unsupported://uri": {},},diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.pyindex 36d8586ce..69696e65e 100644--- a/tests/test_feedexport_postprocess.py+++ b/tests/test_feedexport_postprocess.py@@ -74,7 +74,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase):return content- def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""):+ def get_gzip_compressed(+ self,+ data: bytes,+ compresslevel: int = 9,+ mtime: int = 0,+ filename: str = "",+ ) -> bytes:data_stream = BytesIO()gzipf = gzip.GzipFile(fileobj=data_stream,@@ -539,11 +545,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase):data = await self.exported_data(self.items, settings)- for filename, result in data.items():+ for filename, data_bytes in data.items():+ expected: Any+ result: Anyif "pickle" in filename:- expected, result = self.items[0], pickle.loads(result)+ expected, result = self.items[0], pickle.loads(data_bytes)elif "marshal" in filename:- expected, result = self.items[0], marshal.loads(result)+ expected, result = self.items[0], marshal.loads(data_bytes)else:- expected = filename_to_expected[filename]+ expected, result = filename_to_expected[filename], data_bytesassert result == expecteddiff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex 6f9e33449..d7fb5c9c2 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -95,27 +95,34 @@ class TestFileFeedStorage:assert storage.path == path-class TestFTPFeedStorage:- def get_test_spider(self, settings=None):- class TestSpider(scrapy.Spider):- name = "test_spider"+def get_test_spider(settings: dict[str, Any] | None = None) -> scrapy.Spider:+ class TestSpider(scrapy.Spider):+ name = "test_spider"- crawler = get_crawler(settings_dict=settings)- return TestSpider.from_crawler(crawler)+ crawler = get_crawler(settings_dict=settings)+ return TestSpider.from_crawler(crawler)- async def _store(self, uri, content, feed_options=None, settings=None):++class TestFTPFeedStorage:+ async def _store(+ self,+ uri: str,+ content: bytes,+ feed_options: dict[str, Any] | None = None,+ settings: dict[str, Any] | None = None,+ ) -> None:crawler = get_crawler(settings_dict=settings or {})storage = FTPFeedStorage.from_crawler(crawler,uri,feed_options=feed_options,)- spider = self.get_test_spider()+ spider = get_test_spider()file = storage.open(spider)file.write(content)await maybe_deferred_to_future(storage.store(file))- def _assert_stored(self, path: Path, content):+ def _assert_stored(self, path: Path, content: bytes) -> None:assert path.exists()try:assert path.read_bytes() == content@@ -165,7 +172,7 @@ class TestFTPFeedStorage:def test_uri_auth_quote(self):# RFC3986: 3.2.1. User Informationpw_quoted = quote(string.punctuation, safe="")- st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {})+ st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path")assert st.password == string.punctuationdef test_uri_without_hostname(self):@@ -181,24 +188,17 @@ class MyBlockingFeedStorage(BlockingFeedStorage):class TestBlockingFeedStorage:- def get_test_spider(self, settings=None):- class TestSpider(scrapy.Spider):- name = "test_spider"-- crawler = get_crawler(settings_dict=settings)- return TestSpider.from_crawler(crawler)-def test_default_temp_dir(self):b = MyBlockingFeedStorage()- storage_file = b.open(self.get_test_spider())+ storage_file = b.open(get_test_spider())storage_dir = Path(storage_file.name).parentassert str(storage_dir) == tempfile.gettempdir()def test_temp_file(self, tmp_path):b = MyBlockingFeedStorage()- spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)})+ spider = get_test_spider({"FEED_TEMPDIR": str(tmp_path)})storage_file = b.open(spider)storage_dir = Path(storage_file.name).parentassert storage_dir == tmp_path@@ -207,7 +207,7 @@ class TestBlockingFeedStorage:b = MyBlockingFeedStorage()invalid_path = tmp_path / "invalid_path"- spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})+ spider = get_test_spider({"FEED_TEMPDIR": str(invalid_path)})with pytest.raises(OSError, match="Not a Directory:"):b.open(spider=spider)@@ -311,7 +311,7 @@ class TestS3FeedStorage:assert storage.access_key == "access_key"assert storage.secret_key == "secret_key"assert storage.region_name == region_name- assert storage.s3_client._client_config.region_name == region_name+ assert storage.s3_client._client_config.region_name == region_name # type: ignore[attr-defined]def test_from_crawler_without_acl(self):settings = {@@ -353,7 +353,7 @@ class TestS3FeedStorage:)assert storage.access_key == "access_key"assert storage.secret_key == "secret_key"- assert storage.s3_client._client_config.region_name == "us-east-1"+ assert storage.s3_client._client_config.region_name == "us-east-1" # type: ignore[attr-defined]def test_from_crawler_with_acl(self):settings = {@@ -394,7 +394,7 @@ class TestS3FeedStorage:assert storage.access_key == "access_key"assert storage.secret_key == "secret_key"assert storage.region_name == region_name- assert storage.s3_client._client_config.region_name == region_name+ assert storage.s3_client._client_config.region_name == region_name # type: ignore[attr-defined]def test_init_without_max_pool_connections(self) -> None:storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")@@ -497,7 +497,7 @@ class TestGCSFeedStorage:def test_parse_empty_acl(self):pytest.importorskip("google.cloud.storage")- settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}+ settings: dict[str, Any] = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}crawler = get_crawler(settings_dict=settings)storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")assert storage.acl is Nonediff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.pyindex 150d8449f..21c291732 100644--- a/tests/test_feedexport_uri_params.py+++ b/tests/test_feedexport_uri_params.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport warningsfrom abc import ABC, abstractmethod+from typing import TYPE_CHECKING, Anyimport pytest@@ -10,16 +11,27 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.extensions.feedexport import FeedExporterfrom scrapy.utils.test import get_crawler+if TYPE_CHECKING:+ from collections.abc import Callable++ from scrapy.crawler import Crawler+class TestURIParams(ABC):spider_name = "uri_params_spider"deprecated_options = False@abstractmethod- def build_settings(self, uri="file:///tmp/foobar", uri_params=None):+ def build_settings(+ self,+ uri: str = "file:///tmp/foobar",+ uri_params: Callable[..., dict[str, Any] | None] | None = None,+ ) -> dict[str, Any]:raise NotImplementedError- def _crawler_feed_exporter(self, settings):+ def _crawler_feed_exporter(+ self, settings: dict[str, Any]+ ) -> tuple[Crawler, FeedExporter]:if self.deprecated_options:with pytest.warns(ScrapyDeprecationWarning,@@ -29,6 +41,7 @@ class TestURIParams(ABC):else:crawler = get_crawler(settings_dict=settings)feed_exporter = crawler.get_extension(FeedExporter)+ assert feed_exporter is not Nonereturn crawler, feed_exporterdef test_default(self):@@ -116,8 +129,12 @@ class TestURIParams(ABC):class TestURIParamsSetting(TestURIParams):deprecated_options = True- def build_settings(self, uri="file:///tmp/foobar", uri_params=None):- extra_settings = {}+ def build_settings(+ self,+ uri: str = "file:///tmp/foobar",+ uri_params: Callable[..., dict[str, Any] | None] | None = None,+ ) -> dict[str, Any]:+ extra_settings: dict[str, Any] = {}if uri_params:extra_settings["FEED_URI_PARAMS"] = uri_paramsreturn {@@ -129,8 +146,12 @@ class TestURIParamsSetting(TestURIParams):class TestURIParamsFeedOption(TestURIParams):deprecated_options = False- def build_settings(self, uri="file:///tmp/foobar", uri_params=None):- options = {+ def build_settings(+ self,+ uri: str = "file:///tmp/foobar",+ uri_params: Callable[..., dict[str, Any] | None] | None = None,+ ) -> dict[str, Any]:+ options: dict[str, Any] = {"format": "jl",}if uri_params:diff --git a/tests/test_item.py b/tests/test_item.pyindex 7b4c2e918..45732f157 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -1,4 +1,5 @@from abc import ABCMeta+from typing import Anyfrom unittest import mockimport pytest@@ -7,9 +8,6 @@ from scrapy.item import Field, Item, ItemMetaclass TestItem:- def assertSortedEqual(self, first, second, msg=None):- assert sorted(first) == sorted(second), msg-def test_simple(self):class TestItem(Item):name = Field()@@ -98,16 +96,16 @@ class TestItem:i = TestItem()with pytest.raises(AttributeError):- i.name = "john"+ i.name = "john" # type: ignore[assignment]def test_custom_methods(self):class TestItem(Item):name = Field()- def get_name(self):+ def get_name(self) -> Any:return self["name"]- def change_name(self, name):+ def change_name(self, name: str) -> None:self["name"] = namei = TestItem()@@ -121,40 +119,40 @@ class TestItem:def test_metaclass(self):class TestItem(Item):name = Field()- keys = Field()- values = Field()+ keys = Field() # type: ignore[assignment]+ values = Field() # type: ignore[assignment]i = TestItem()i["name"] = "John"- assert list(i.keys()) == ["name"]- assert list(i.values()) == ["John"]+ assert list(i.keys()) == ["name"] # type: ignore[operator]+ assert list(i.values()) == ["John"] # type: ignore[operator]i["keys"] = "Keys"i["values"] = "Values"- self.assertSortedEqual(list(i.keys()), ["keys", "values", "name"])- self.assertSortedEqual(list(i.values()), ["Keys", "Values", "John"])+ assert sorted(i.keys()) == ["keys", "name", "values"] # type: ignore[operator]+ assert sorted(i.values()) == ["John", "Keys", "Values"] # type: ignore[operator]def test_metaclass_with_fields_attribute(self):class TestItem(Item):fields = {"new": Field(default="X")}item = TestItem(new="New")- self.assertSortedEqual(list(item.keys()), ["new"])- self.assertSortedEqual(list(item.values()), ["New"])+ assert list(item.keys()) == ["new"]+ assert list(item.values()) == ["New"]def test_fields_order(self):class TestItem(Item):name = Field()- keys = Field()- values = Field()+ keys = Field() # type: ignore[assignment]+ values = Field() # type: ignore[assignment]assert list(TestItem.fields) == ["name", "keys", "values"]def test_fields_order_inheritance(self):class ParentItem(Item):name = Field()- keys = Field()- values = Field()+ keys = Field() # type: ignore[assignment]+ values = Field() # type: ignore[assignment]class TestItem(ParentItem):extra = Field()@@ -169,16 +167,16 @@ class TestItem:def test_metaclass_inheritance(self):class ParentItem(Item):name = Field()- keys = Field()- values = Field()+ keys = Field() # type: ignore[assignment]+ values = Field() # type: ignore[assignment]class TestItem(ParentItem):keys = Field()i = TestItem()i["keys"] = 3- assert list(i.keys()) == ["keys"]- assert list(i.values()) == [3]+ assert list(i.keys()) == ["keys"] # type: ignore[operator]+ assert list(i.values()) == [3] # type: ignore[operator]def test_metaclass_multiple_inheritance_simple(self):class A(Item):@@ -314,7 +312,7 @@ class TestItemMeta:def f(self):# For rationale of this see:# https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222- return __class__+ return __class__ # type: ignore[name-defined]MyItem()diff --git a/tests/test_loader.py b/tests/test_loader.pyindex c094d25d8..969bf5b98 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -79,7 +79,7 @@ class TestBasicItemLoader:class InitializationTestMixin:- item_class: type | None = None+ item_class: typedef test_keep_single_value(self):"""Loaded item should contain values from the initial item"""@@ -311,7 +311,7 @@ class TestSelectortemLoader:def test_init_method_with_base_response(self):"""Selector should be None after initialization"""response = Response("https://scrapy.org")- l = ProcessorItemLoader(response=response)+ l = ProcessorItemLoader(response=response) # type: ignore[arg-type]assert l.selector is Nonedef test_init_method_with_response(self):@@ -461,6 +461,7 @@ class TestSubselectorLoader:l = NestedItemLoader(response=self.response)nl = l.nested_xpath("//header")+ assert nl.selector is not Nonenl.add_xpath("name", "div/text()")nl.add_css("name_div", "#id")nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())@@ -476,6 +477,7 @@ class TestSubselectorLoader:def test_nested_css(self):l = NestedItemLoader(response=self.response)nl = l.nested_css("header")+ assert nl.selector is not Nonenl.add_xpath("name", "div/text()")nl.add_css("name_div", "#id")nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 0681371ef..8b522255a 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -23,7 +23,10 @@ if TYPE_CHECKING:class MediaDownloadSpider(SimpleSpider):name = "mediadownload"- def _process_url(self, url):+ media_key: str+ media_urls_key: str++ def _process_url(self, url: str) -> str:return urldef parse(self, response):@@ -44,14 +47,15 @@ class MediaDownloadSpider(SimpleSpider):class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):name = "brokenmedia"- def _process_url(self, url):+ def _process_url(self, url: str) -> str:return url + ".foo"class RedirectedMediaDownloadSpider(MediaDownloadSpider):name = "redirectedmedia"- def _process_url(self, url):+ def _process_url(self, url: str) -> str:+ assert self.mockserverreturn add_or_replace_parameter(self.mockserver.url("/redirect-to"), "goto", url)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 4e7fb118b..3a8a3d0f7 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -22,6 +22,7 @@ from itemadapter import ItemAdapterfrom twisted.internet.defer import Deferredfrom twisted.python.failure import Failure+from scrapy.crawler import Crawlerfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Responsefrom scrapy.item import Field, Item@@ -75,7 +76,7 @@ class DeferredFSFilesStore(FSFilesStore):"""A simple store with persist_file() returning a deferred."""def persist_file(self, path, buf, info, meta=None, headers=None):- deferred = Deferred()+ deferred: Deferred[None] = Deferred()# short-hand super() doesn't work in nested functionsparent_persist_file = super().persist_file@@ -152,7 +153,7 @@ class TestFilesPipeline:file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),response=Response("http://www.dorma.co.uk/images/product_details/2532"),- info=object(),+ info=object(), # type: ignore[arg-type])== "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1")@@ -383,7 +384,7 @@ class TestFilesPipeline:"""class CustomFilesPipeline(FilesPipeline):- def file_path(self, request, response=None, info=None, item=None):+ def file_path(self, request, response=None, info=None, item=None) -> str:return f"full/{item.get('path')}"file_path = CustomFilesPipeline.from_crawler(@@ -476,7 +477,7 @@ class TestFilesPipeline:item["file_urls"] = bad_typewith pytest.raises(TypeError, match="file_urls must be a list of URLs"):- list(pipeline.get_media_requests(item, None))+ list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]class TestFilesPipelineFieldsMixin(ABC):@@ -491,10 +492,10 @@ class TestFilesPipelineFieldsMixin(ABC):pipeline = FilesPipeline.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))- requests = list(pipeline.get_media_requests(item, None))+ requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]assert requests[0].url == urlresults = [(True, {"url": url})]- item = pipeline.item_completed(results, item, None)+ item = pipeline.item_completed(results, item, None) # type: ignore[arg-type]files = ItemAdapter(item).get("files")assert files == [results[0][1]]assert isinstance(item, self.item_class)@@ -512,10 +513,10 @@ class TestFilesPipelineFieldsMixin(ABC):},))- requests = list(pipeline.get_media_requests(item, None))+ requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]assert requests[0].url == urlresults = [(True, {"url": url})]- item = pipeline.item_completed(results, item, None)+ item = pipeline.item_completed(results, item, None) # type: ignore[arg-type]custom_files = ItemAdapter(item).get("custom_files")assert custom_files == [results[0][1]]assert isinstance(item, self.item_class)@@ -581,8 +582,10 @@ class TestFilesPipelineCustomSettings:("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),}- def _generate_fake_settings(self, tmp_path, prefix=None):- def random_string():+ def _generate_fake_settings(+ self, tmp_path: Path, prefix: str | None = None+ ) -> dict[str, Any]:+ def random_string() -> str:return "".join([chr(random.randint(97, 123)) for _ in range(10)])settings = {@@ -599,7 +602,7 @@ class TestFilesPipelineCustomSettings:for k, v in settings.items()}- def _generate_fake_pipeline(self):+ def _generate_fake_pipeline(self) -> type[FilesPipeline]:class UserDefinedFilePipeline(FilesPipeline):EXPIRES = 1001FILES_URLS_FIELD = "alfa"@@ -739,14 +742,14 @@ class TestFilesPipelineCustomSettings:def test_file_pipeline_using_pathlike_objects(self, tmp_path):class CustomFilesPipelineWithPathLikeDir(FilesPipeline):- def file_path(self, request, response=None, info=None, *, item=None):- return Path("subdir") / Path(request.url).name+ def file_path(self, request, response=None, info=None, *, item=None) -> str:+ return str(Path("subdir") / Path(request.url).name)pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))request = Request("http://example.com/image01.jpg")- assert pipeline.file_path(request) == Path("subdir/image01.jpg")+ assert pipeline.file_path(request) == str(Path("subdir/image01.jpg"))class TestFSFilesStore:@@ -1092,7 +1095,7 @@ class TestFTPFileStore:store.port,store.username,store.password,- store.USE_ACTIVE_MODE,+ bool(store.USE_ACTIVE_MODE),)assert data == content@@ -1160,7 +1163,7 @@ class TestBuildFromCrawler:_from_crawler_called = False@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> "Pipeline":settings = crawler.settingsstore_uri = settings["FILES_STORE"]o = cls(store_uri, crawler=crawler)@@ -1179,7 +1182,7 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store):settings = Settings()settings.clear()settings.set("FILES_STORE", store, priority="cmdline")- crawler = get_crawler(settings_dict=settings)+ crawler = get_crawler(settings_dict=dict(settings))with pytest.raises(NotConfigured):FilesPipeline.from_crawler(crawler)diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 19e61579f..43316f125 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -91,7 +91,7 @@ class TestImagesPipeline:file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),response=Response("http://www.dorma.co.uk/images/product_details/2532"),- info=object(),+ info=DUMMY_SPIDER_INFO,)== "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg")@@ -120,7 +120,7 @@ class TestImagesPipeline:Request("file:///tmp/some.name/foo"),name,response=Response("file:///tmp/some.name/foo"),- info=object(),+ info=DUMMY_SPIDER_INFO,)== "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg")@@ -133,7 +133,7 @@ class TestImagesPipeline:class CustomImagesPipeline(ImagesPipeline):def thumb_path(self, request, thumb_id, response=None, info=None, item=None- ):+ ) -> str:return f"thumb/{thumb_id}/{item.get('path')}"thumb_path = CustomImagesPipeline.from_crawler(@@ -159,11 +159,23 @@ class TestImagesPipeline:req = Request(url="https://dev.mydeco.com/mydeco.gif")with pytest.raises(ImageException):- next(self.pipeline.get_images(response=resp1, request=req, info=object()))+ next(+ self.pipeline.get_images(+ response=resp1, request=req, info=DUMMY_SPIDER_INFO+ )+ )with pytest.raises(ImageException):- next(self.pipeline.get_images(response=resp2, request=req, info=object()))+ next(+ self.pipeline.get_images(+ response=resp2, request=req, info=DUMMY_SPIDER_INFO+ )+ )with pytest.raises(ImageException):- next(self.pipeline.get_images(response=resp3, request=req, info=object()))+ next(+ self.pipeline.get_images(+ response=resp3, request=req, info=DUMMY_SPIDER_INFO+ )+ )def test_get_images(self):self.pipeline.min_width = 0@@ -176,7 +188,7 @@ class TestImagesPipeline:req = Request(url="https://dev.mydeco.com/mydeco.gif")get_images_gen = self.pipeline.get_images(- response=resp, request=req, info=object()+ response=resp, request=req, info=DUMMY_SPIDER_INFO)path, new_im, new_buf = next(get_images_gen)@@ -201,7 +213,7 @@ class TestImagesPipeline:req = Request(url="https://dev.mydeco.com/mydeco.gif")get_images_gen = self.pipeline.get_images(- response=resp, request=req, info=object()+ response=resp, request=req, info=DUMMY_SPIDER_INFO)path, new_im, _ = next(get_images_gen)@@ -230,7 +242,7 @@ class TestImagesPipeline:def test_convert_image(self):SIZE = (100, 100)# straight forward case: RGB and JPEG- COLOUR = (0, 127, 255)+ COLOUR: tuple[int, ...] = (0, 127, 255)im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)assert converted.mode == "RGB"@@ -296,7 +308,7 @@ class TestImagesPipeline:item["image_urls"] = bad_typewith pytest.raises(TypeError, match="image_urls must be a list of URLs"):- list(pipeline.get_media_requests(item, None))+ list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))class TestImagesPipelineFieldsMixin(ABC):@@ -311,10 +323,10 @@ class TestImagesPipelineFieldsMixin(ABC):pipeline = ImagesPipeline.from_crawler(get_crawler(None, {"IMAGES_STORE": "s3://example/images/"}))- requests = list(pipeline.get_media_requests(item, None))+ requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))assert requests[0].url == url- results = [(True, {"url": url})]- item = pipeline.item_completed(results, item, None)+ results: Any = [(True, {"url": url})]+ item = pipeline.item_completed(results, item, DUMMY_SPIDER_INFO)images = ItemAdapter(item).get("images")assert images == [results[0][1]]assert isinstance(item, self.item_class)@@ -332,10 +344,10 @@ class TestImagesPipelineFieldsMixin(ABC):},))- requests = list(pipeline.get_media_requests(item, None))+ requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))assert requests[0].url == url- results = [(True, {"url": url})]- item = pipeline.item_completed(results, item, None)+ results: Any = [(True, {"url": url})]+ item = pipeline.item_completed(results, item, DUMMY_SPIDER_INFO)custom_images = ItemAdapter(item).get("custom_images")assert custom_images == [results[0][1]]assert isinstance(item, self.item_class)@@ -410,13 +422,15 @@ class TestImagesPipelineCustomSettings:"IMAGES_RESULT_FIELD": "images",}- def _generate_fake_settings(self, tmp_path, prefix=None):+ def _generate_fake_settings(+ self, tmp_path: Path, prefix: str | None = None+ ) -> dict[str, Any]:""":param prefix: string for setting keys:return: dictionary of image pipeline settings"""- def random_string():+ def random_string() -> str:return "".join([chr(random.randint(97, 123)) for _ in range(10)])settings = {@@ -439,7 +453,7 @@ class TestImagesPipelineCustomSettings:for k, v in settings.items()}- def _generate_fake_pipeline_subclass(self):+ def _generate_fake_pipeline_subclass(self) -> type[ImagesPipeline]:""":return: ImagePipeline class will all uppercase attributes set."""diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex ba1c18006..50787d7b7 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -1,6 +1,7 @@from __future__ import annotationsimport logging+from typing import TYPE_CHECKING, Any, castfrom unittest.mock import MagicMockimport pytest@@ -10,7 +11,12 @@ from scrapy import signalsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.pipelines.files import FileException-from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered+from scrapy.pipelines.media import (+ FileInfo,+ FileInfoOrError,+ MediaPipeline,+ _MediaRequestFiltered,+)from scrapy.utils.defer import _defer_sleep_asyncfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.signal import disconnect_all@@ -19,21 +25,48 @@ from scrapy.utils.test import get_crawlerfrom tests.utils.decorators import coroutine_testfrom tests.utils.media_pipelines import mocked_download_func+if TYPE_CHECKING:+ from collections.abc import Awaitable-class UserDefinedPipeline(MediaPipeline):- def media_to_download(self, request, info, *, item=None):- pass+ from twisted.internet.defer import Deferred- def get_media_requests(self, item, info):- pass+ from scrapy.crawler import Crawler- def media_downloaded(self, response, request, info, *, item=None):- return {}- def media_failed(self, failure, request, info):+class UserDefinedPipeline(MediaPipeline):+ def media_to_download(+ self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None+ ) -> Deferred[FileInfo | None] | None:+ return None++ def get_media_requests(+ self, item: Any, info: MediaPipeline.SpiderInfo+ ) -> list[Request]:+ return []++ def media_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> FileInfo | Awaitable[FileInfo]:+ return cast("FileInfo", {})++ def media_failed(+ self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo+ ) -> Failure:failure.raiseException()- def file_path(self, request, response=None, info=None, *, item=None):+ def file_path(+ self,+ request: Request,+ response: Response | None = None,+ info: MediaPipeline.SpiderInfo | None = None,+ *,+ item: Any = None,+ ) -> str:return ""@@ -48,8 +81,14 @@ class TestBaseMediaPipeline:self.pipe = self.pipeline_class.from_crawler(crawler)self.pipe.open_spider()self.info = self.pipe.spiderinfo+ assert crawler.request_fingerprinter is not Noneself.fingerprint = crawler.request_fingerprinter.fingerprint+ @property+ def mocked_pipe(self) -> MockedMediaPipeline:+ assert isinstance(self.pipe, MockedMediaPipeline)+ return self.pipe+def teardown_method(self):for name, signal in vars(signals).items():if not name.startswith("_"):@@ -121,11 +160,13 @@ class TestBaseMediaPipeline:# When calling the method that caches the Request's result ...self.pipe._cache_result_and_execute_waiters(failure, fp, info)# ... it should store the Twisted Failure ...- assert info.downloaded[fp] == failure+ downloaded = info.downloaded[fp]+ assert downloaded == failure# ... encapsulating the original FileException ...- assert info.downloaded[fp].value == file_exc+ assert isinstance(downloaded, Failure)+ assert downloaded.value == file_exc# ... but it should not store the StopIteration exception on its context- context = getattr(info.downloaded[fp].value, "__context__", None)+ context = getattr(downloaded.value, "__context__", None)assert context is Nonedef test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None:@@ -134,7 +175,7 @@ class TestBaseMediaPipeline:# Check that failures are logged by defaultfail = Failure(Exception())- results = [(True, 1), (False, fail)]+ results: Any = [(True, 1), (False, fail)]caplog.clear()new_item = self.pipe.item_completed(results, item, self.info)@@ -158,7 +199,7 @@ class TestBaseMediaPipeline:by item_completed(), as they are not download errors."""item = {"name": "name"}fail = Failure(_MediaRequestFiltered("Filtered offsite request"))- results = [(True, 1), (False, fail)]+ results: Any = [(True, 1), (False, fail)]with caplog.at_level(logging.DEBUG):new_item = self.pipe.item_completed(results, item, self.info)@@ -174,29 +215,44 @@ class TestBaseMediaPipeline:class MockedMediaPipeline(UserDefinedPipeline):- def __init__(self, *args, crawler=None, **kwargs):+ def __init__(self, *args: Any, crawler: Crawler, **kwargs: Any):super().__init__(*args, crawler=crawler, **kwargs)- self._mockcalled = []+ self._mockcalled: list[str] = []- def media_to_download(self, request, info, *, item=None):+ def media_to_download(+ self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None+ ) -> Deferred[FileInfo | None] | None:self._mockcalled.append("media_to_download")if "result" in request.meta:return request.meta.get("result")return super().media_to_download(request, info)- def get_media_requests(self, item, info):+ def get_media_requests(+ self, item: Any, info: MediaPipeline.SpiderInfo+ ) -> list[Request]:self._mockcalled.append("get_media_requests")- return item.get("requests")-- def media_downloaded(self, response, request, info, *, item=None):+ return item.get("requests") # type: ignore[no-any-return]++ def media_downloaded(+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> FileInfo | Awaitable[FileInfo]:self._mockcalled.append("media_downloaded")return super().media_downloaded(response, request, info)- def media_failed(self, failure, request, info):+ def media_failed(+ self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo+ ) -> Failure:self._mockcalled.append("media_failed")return super().media_failed(failure, request, info)- def item_completed(self, results, item, info):+ def item_completed(+ self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo+ ) -> Any:self._mockcalled.append("item_completed")item = super().item_completed(results, item, info)item["results"] = results@@ -204,7 +260,14 @@ class MockedMediaPipeline(UserDefinedPipeline):class AsyncMediaDownloadedPipeline(MockedMediaPipeline):- async def media_downloaded(self, response, request, info, *, item=None):+ async def media_downloaded( # type: ignore[override]+ self,+ response: Response,+ request: Request,+ info: MediaPipeline.SpiderInfo,+ *,+ item: Any = None,+ ) -> FileInfo | Awaitable[FileInfo]:return super().media_downloaded(response, request, info)@@ -212,7 +275,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):pipeline_class = MockedMediaPipelinedef _errback(self, result):- self.pipe._mockcalled.append("request_errback")+ self.mocked_pipe._mockcalled.append("request_errback")return result@coroutine_test@@ -226,7 +289,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):item = {"requests": req}new_item = await self.pipe.process_item(item)assert new_item["results"] == [(True, {})]- assert self.pipe._mockcalled == [+ assert self.mocked_pipe._mockcalled == ["get_media_requests","media_to_download","media_downloaded",@@ -248,7 +311,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):assert new_item["results"][0][0] is Falseassert isinstance(new_item["results"][0][1], Failure)assert new_item["results"][0][1].value == exc- assert self.pipe._mockcalled == [+ assert self.mocked_pipe._mockcalled == ["get_media_requests","media_to_download","media_failed",@@ -270,7 +333,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):assert new_item["results"][1][0] is Falseassert isinstance(new_item["results"][1][1], Failure)assert new_item["results"][1][1].value == exc- m = self.pipe._mockcalled+ m = self.mocked_pipe._mockcalled# only onceassert m[0] == "get_media_requests" # first hook calledassert m.count("get_media_requests") == 1@@ -294,7 +357,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):# returns iterable of Requestsreq1 = Request("http://url1")req2 = Request("http://url2")- item = {"requests": iter([req1, req2])}+ item = {"requests": iter([req1, req2])} # type: ignore[dict-item]new_item = await self.pipe.process_item(item)assert new_item is itemassert self.fingerprint(req1) in self.info.downloaded@@ -304,7 +367,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):async def test_results_are_cached_across_multiple_items(self):rsp1 = Response("http://url1")req1 = Request("http://url1", meta={"response": rsp1})- item = {"requests": req1}+ item: dict[str, Any] = {"requests": req1}new_item = await self.pipe.process_item(item)assert new_item is itemassert new_item["results"] == [(True, {})]@@ -335,7 +398,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):new_item = await self.pipe.process_item({"requests": req2})assert new_item["results"][0][0] is Falseassert new_item["results"][0][1].value is exc- assert self.pipe._mockcalled.count("media_to_download") == 1+ assert self.mocked_pipe._mockcalled.count("media_to_download") == 1@coroutine_testasync def test_cached_failure_calls_errback(self):@@ -347,13 +410,13 @@ class TestMediaPipeline(TestBaseMediaPipeline):)def errback(failure):- self.pipe._mockcalled.append("request_errback")+ self.mocked_pipe._mockcalled.append("request_errback")return {"recovered": failure.value}req = Request("http://url1", errback=errback)new_item = await self.pipe.process_item({"requests": req})assert new_item["results"] == [(True, {"recovered": exc})]- assert self.pipe._mockcalled.count("request_errback") == 1+ assert self.mocked_pipe._mockcalled.count("request_errback") == 1@coroutine_testasync def test_results_are_cached_for_requests_of_single_item(self):@@ -362,14 +425,14 @@ class TestMediaPipeline(TestBaseMediaPipeline):req2 = Request(req1.url, meta={"response": Response("http://donot.download.me")})- item = {"requests": [req1, req2]}+ item: dict[str, Any] = {"requests": [req1, req2]}new_item = await self.pipe.process_item(item)assert new_item is itemassert new_item["results"] == [(True, {}), (True, {})]@coroutine_testasync def test_wait_if_request_is_downloading(self):- def _check_downloading(response):+ def _check_downloading(response: Response) -> Response:fp = self.fingerprint(req1)assert fp in self.info.downloadingassert fp in self.info.waiting@@ -398,7 +461,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):item = {"requests": req}new_item = await self.pipe.process_item(item)assert new_item["results"] == [(True, "ITSME")]- assert self.pipe._mockcalled == [+ assert self.mocked_pipe._mockcalled == ["get_media_requests","media_to_download","item_completed",@@ -422,7 +485,9 @@ class TestAsyncMediaDownloaded(TestMediaPipeline):class TestMediaPipelineAllowRedirectSettings:- def _assert_request_no3xx(self, pipeline_class, settings):+ def _assert_request_no3xx(+ self, pipeline_class: type[MediaPipeline], settings: dict[str, Any]+ ) -> None:pipe = pipeline_class(crawler=get_crawler(None, settings))request = Request("http://url")pipe._modify_media_request(request)@@ -477,7 +542,7 @@ class TestBuildFromCrawler:self._init_called = True@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Pipeline:settings = crawler.settingsstore_uri = settings["FILES_STORE"]o = cls(store_uri, settings=settings, crawler=crawler)@@ -493,9 +558,10 @@ class TestBuildFromCrawler:def test_has_from_crawler(self):class Pipeline(UserDefinedPipeline):_from_crawler_called = False+ store_uri: str@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Pipeline:settings = crawler.settingso = super().from_crawler(crawler)o._from_crawler_called = True@@ -509,7 +575,9 @@ class TestBuildFromCrawler:class MediaFailedNonePipeline(MockedMediaPipeline):- def media_failed(self, failure, request, info):+ def media_failed( # type: ignore[override]+ self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo+ ) -> None:self._mockcalled.append("media_failed")@@ -524,7 +592,7 @@ class TestMediaFailedNone(TestBaseMediaPipeline):req = Request("http://url1", meta={"response": Exception("foo")})new_item = await self.pipe.process_item({"requests": req})assert new_item["results"] == [(True, None)]- assert self.pipe._mockcalled == [+ assert self.mocked_pipe._mockcalled == ["get_media_requests","media_to_download","media_failed",@@ -533,7 +601,9 @@ class TestMediaFailedNone(TestBaseMediaPipeline):class MediaFailedFailurePipeline(MockedMediaPipeline):- def media_failed(self, failure, request, info):+ def media_failed(+ self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo+ ) -> Failure:self._mockcalled.append("media_failed")return failure # deprecated@@ -544,7 +614,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline):pipeline_class = MediaFailedFailurePipelinedef _errback(self, result):- self.pipe._mockcalled.append("request_errback")+ self.mocked_pipe._mockcalled.append("request_errback")return result@coroutine_test@@ -565,7 +635,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline):assert new_item["results"][0][0] is Falseassert isinstance(new_item["results"][0][1], Failure)assert new_item["results"][0][1].value == exc- assert self.pipe._mockcalled == [+ assert self.mocked_pipe._mockcalled == ["get_media_requests","media_to_download","media_failed",diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex e65b5bc32..4b34946ae 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -47,7 +47,7 @@ class DeferredPipeline:return succeed(None)def process_item(self, item):- d = Deferred()+ d: Deferred[Any] = Deferred()d.addCallback(self.cb)d.callback(item)return d@@ -55,7 +55,7 @@ class DeferredPipeline:class AsyncDefPipeline:async def process_item(self, item):- d = Deferred()+ d: Deferred[Any] = Deferred()call_later(0, d.callback, None)await maybe_deferred_to_future(d)item["pipeline_passed"] = True@@ -64,7 +64,7 @@ class AsyncDefPipeline:class AsyncDefAsyncioPipeline:async def process_item(self, item):- d = Deferred()+ d: Deferred[Any] = Deferred()loop = asyncio.get_event_loop()loop.call_later(0, d.callback, None)await deferred_to_future(d)@@ -75,12 +75,12 @@ class AsyncDefAsyncioPipeline:class AsyncDefNotAsyncioPipeline:async def process_item(self, item):- d1 = Deferred()+ d1: Deferred[Any] = Deferred()from twisted.internet import reactorreactor.callLater(0, d1.callback, None)await d1- d2 = Deferred()+ d2: Deferred[Any] = Deferred()reactor.callLater(0, d2.callback, None)await maybe_deferred_to_future(d2)item["pipeline_passed"] = True@@ -120,6 +120,8 @@ class OpenSpiderExceptionAsyncPipeline:class ItemSpider(Spider):name = "itemspider"+ mockserver: MockServer+async def start(self):yield Request(self.mockserver.url("/status?n=200"))