Scrapy, a fast high-level web crawling & scraping framework for Python.
Type spider-related test modules (#7968)
Adrian71aefab5f77c
diff --git a/pyproject.toml b/pyproject.tomlindex 61c23fbec..13aae01ac 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -116,13 +116,9 @@ allow_incomplete_defs = true # 59 errors# TODO[[tool.mypy.overrides]]module = [- "tests.spiders",- "tests.test_closespider","tests.test_cmdline",- "tests.test_contracts","tests.test_downloaderslotssettings","tests.test_dupefilters",- "tests.test_engine_loop","tests.test_extension_statsmailer","tests.test_extension_throttle","tests.test_linkextractors",@@ -131,15 +127,9 @@ module = ["tests.test_pqueues","tests.test_scheduler_base","tests.test_settings",- "tests.test_spider",- "tests.test_spider_crawl",- "tests.test_spidermiddleware_output_chain",- "tests.test_spidermiddleware_process_start",- "tests.test_spider_sitemap","tests.test_squeues","tests.test_squeues_request","tests.test_stats",- "tests.utils.bases.spider",]check_untyped_defs = falsediff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.pyindex a066ed00a..7504ddf18 100644--- a/tests/benchmarks/__init__.py+++ b/tests/benchmarks/__init__.py@@ -36,7 +36,6 @@ class NullDownloadHandler:async def download_request(self, request: Request) -> Response:self._active += 1- assert self._crawler.statsself._crawler.stats.max_value("benchmark/peak_concurrency", self._active)try:await asyncio.sleep(0)diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.pyindex f79b66a5b..4f68e4f6f 100644--- a/tests/benchmarks/test_crawl.py+++ b/tests/benchmarks/test_crawl.py@@ -128,7 +128,6 @@ class _DelayedPipeline:async def process_item(self, item: Any) -> Any:url = item["url"]self._active[url] += 1- assert self._crawler.statsself._crawler.stats.max_value("benchmark/peak_items", self._active[url])try:await asyncio.sleep(DELAY)@@ -147,7 +146,6 @@ def _crawl_tree(pages=pages,items=items,)- assert crawler.statsassert crawler.stats.get_value("downloader/response_count") == domains * pagesassert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * itemsreturn crawler@@ -166,7 +164,6 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> Ndef run() -> None:crawler = crawl(_FollowSpider, settings, url=url)- assert crawler.statsassert crawler.stats.get_value("item_scraped_count") == PAGES + 1benchmark(run)@@ -177,7 +174,6 @@ def test_overhead_engine(benchmark: BenchmarkFixture) -> None:def run() -> None:crawler = _crawl_tree({}, domains=1, pages=REQUESTS)- assert crawler.statsassert crawler.stats.get_value("benchmark/peak_concurrency") > 1benchmark(run)@@ -262,7 +258,6 @@ def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:crawler = _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE)- assert crawler.statsassert (crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS)diff --git a/tests/spiders.py b/tests/spiders.pyindex 7c7d3007c..3ff30ce03 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -13,22 +13,31 @@ from twisted.internet import deferfrom scrapy import signalsfrom scrapy.exceptions import StopDownload-from scrapy.http import Request+from scrapy.http import Request, TextResponsefrom scrapy.item import Itemfrom scrapy.linkextractors import LinkExtractor+from scrapy.spidermiddlewares.httperror import HttpErrorfrom scrapy.spiders import Spiderfrom scrapy.spiders.crawl import CrawlSpider, Rulefrom scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.test import get_from_asyncio_queueif TYPE_CHECKING:+ from collections.abc import AsyncIterator, Callable, Iterator++ from twisted.python.failure import Failure+ from typing_extensions import Self++ from scrapy.crawler import Crawler+ from scrapy.http import Headers, Response+ from scrapy.http.request import CallbackTfrom tests.mockserver.http import MockServerclass MockServerSpider(Spider):def __init__(self,- *args,+ *args: Any,mockserver: MockServer | None = None,is_secure: bool = False,**kwargs: Any,@@ -74,7 +83,7 @@ class MetaSpider(MockServerSpider):super().__init__(*args, **kwargs)self.meta: dict[str, Any] = {}- def closed(self, reason):+ def closed(self, reason: str) -> None:self.meta["close_reason"] = reason@@ -83,16 +92,24 @@ class FollowAllSpider(MetaSpider):link_extractor = LinkExtractor()def __init__(- self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs+ self,+ total: int = 10,+ show: int = 20,+ order: str = "rand",+ maxlatency: float = 0.0,+ *args: Any,+ **kwargs: Any,):super().__init__(*args, **kwargs)- self.urls_visited = []- self.times = []+ self.urls_visited: list[str] = []+ self.times: list[float] = []qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency}+ assert self.mockserverurl = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")self.start_urls = [url]- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:+ assert isinstance(response, TextResponse)self.urls_visited.append(response.url)self.times.append(time.time())for link in self.link_extractor.extract_links(response):@@ -102,50 +119,52 @@ class FollowAllSpider(MetaSpider):class DelaySpider(MetaSpider):name = "delay"- def __init__(self, n=1, b=0, *args, **kwargs):+ def __init__(self, n: float = 1, b: float = 0, *args: Any, **kwargs: Any):super().__init__(*args, **kwargs)self.n = nself.b = b- self.t1 = self.t2 = self.t2_err = 0+ self.t1 = self.t2 = self.t2_err = 0.0- async def start(self):+ async def start(self) -> AsyncIterator[Any]:self.t1 = time.time()+ assert self.mockserverurl = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")yield Request(url, callback=self.parse, errback=self.errback)- def parse(self, response):+ def parse(self, response: Response) -> Any:self.t2 = time.time()- def errback(self, failure):+ def errback(self, failure: Failure) -> None:self.t2_err = time.time()class LogSpider(MetaSpider):name = "log_spider"- def log_debug(self, message: str, extra: dict[str, Any] | None = None):+ def log_debug(self, message: str, extra: dict[str, Any] | None = None) -> None:self.logger.debug(message, extra=extra)- def log_info(self, message: str, extra: dict[str, Any] | None = None):+ def log_info(self, message: str, extra: dict[str, Any] | None = None) -> None:self.logger.info(message, extra=extra)- def log_warning(self, message: str, extra: dict[str, Any] | None = None):+ def log_warning(self, message: str, extra: dict[str, Any] | None = None) -> None:self.logger.warning(message, extra=extra)- def log_error(self, message: str, extra: dict[str, Any] | None = None):+ def log_error(self, message: str, extra: dict[str, Any] | None = None) -> None:self.logger.error(message, extra=extra)- def log_critical(self, message: str, extra: dict[str, Any] | None = None):+ def log_critical(self, message: str, extra: dict[str, Any] | None = None) -> None:self.logger.critical(message, extra=extra)- def parse(self, response):+ def parse(self, response: Response) -> None:passclass SlowSpider(DelaySpider):name = "slow"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserver# 1st response is fasturl = self.mockserver.url("/delay?n=0&b=0")yield Request(url, callback=self.parse, errback=self.errback)@@ -154,25 +173,25 @@ class SlowSpider(DelaySpider):url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")yield Request(url, callback=self.parse, errback=self.errback)- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:yield Item()class SimpleSpider(MetaSpider):name = "simple"- def __init__(self, url="http://localhost:8998", *args, **kwargs):+ def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):super().__init__(*args, **kwargs)self.start_urls = [url]- def parse(self, response):+ def parse(self, response: Response) -> Any:self.logger.info(f"Got response {response.status}")class AsyncDefSpider(SimpleSpider):name = "asyncdef"- async def parse(self, response):+ async def parse(self, response: Response) -> None:await defer.succeed(42)self.logger.info(f"Got response {response.status}")@@ -180,7 +199,7 @@ class AsyncDefSpider(SimpleSpider):class AsyncDefAsyncioSpider(SimpleSpider):name = "asyncdef_asyncio"- async def parse(self, response):+ async def parse(self, response: Response) -> None:await asyncio.sleep(0.2)status = await get_from_asyncio_queue(response.status)self.logger.info(f"Got response {status}")@@ -189,7 +208,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):class AsyncDefAsyncioReturnSpider(SimpleSpider):name = "asyncdef_asyncio_return"- async def parse(self, response):+ async def parse(self, response: Response) -> Any:await asyncio.sleep(0.2)status = await get_from_asyncio_queue(response.status)self.logger.info(f"Got response {status}")@@ -199,7 +218,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):name = "asyncdef_asyncio_return_single_element"- async def parse(self, response):+ async def parse(self, response: Response) -> Any:await asyncio.sleep(0.1)status = await get_from_asyncio_queue(response.status)self.logger.info(f"Got response {status}")@@ -209,7 +228,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):name = "asyncdef_asyncio_reqs_return"- async def parse(self, response):+ async def parse(self, response: Response) -> Any:await asyncio.sleep(0.2)req_id = response.meta.get("req_id", 0)status = await get_from_asyncio_queue(response.status)@@ -226,7 +245,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):class AsyncDefAsyncioGenExcSpider(SimpleSpider):name = "asyncdef_asyncio_gen_exc"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:for i in range(10):await asyncio.sleep(0.1)yield {"foo": i}@@ -237,7 +256,7 @@ class AsyncDefAsyncioGenExcSpider(SimpleSpider):class AsyncDefDeferredDirectSpider(SimpleSpider):name = "asyncdef_deferred_direct"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:await defer.succeed(None)yield {"code": 200}@@ -245,7 +264,7 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):class AsyncDefDeferredWrappedSpider(SimpleSpider):name = "asyncdef_deferred_wrapped"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:await deferred_to_future(defer.succeed(None))yield {"code": 200}@@ -253,7 +272,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):name = "asyncdef_deferred_maybe_wrapped"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:await maybe_deferred_to_future(defer.succeed(None))yield {"code": 200}@@ -261,7 +280,7 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):class AsyncDefAsyncioGenSpider(SimpleSpider):name = "asyncdef_asyncio_gen"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:await asyncio.sleep(0.2)yield {"foo": 42}self.logger.info(f"Got response {response.status}")@@ -270,7 +289,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):class AsyncDefAsyncioGenLoopSpider(SimpleSpider):name = "asyncdef_asyncio_gen_loop"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:for i in range(10):await asyncio.sleep(0.1)yield {"foo": i}@@ -283,7 +302,8 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):following_reqs = 3depth = 2- def _get_req(self, index, cb=None):+ def _get_req(self, index: int, cb: CallbackT | None = None) -> Request:+ assert self.mockserverreturn Request(self.mockserver.url(f"/status?n=200&request={index}"),meta={"index": index},@@ -291,11 +311,11 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):callback=cb,)- async def start(self):+ async def start(self) -> AsyncIterator[Any]:for i in range(1, self.initial_reqs + 1):yield self._get_req(i)- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]:index = response.meta["index"]yield {"index": index}if index < 10**self.depth:@@ -305,7 +325,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):await asyncio.sleep(0.1)yield {"index": index + 5}- async def parse2(self, response):+ async def parse2(self, response: Response) -> AsyncIterator[Any]:await asyncio.sleep(0.1)yield {"index2": response.meta["index"]}@@ -313,7 +333,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):class ItemSpider(FollowAllSpider):name = "item"- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:for request in super().parse(response):yield requestyield Item()@@ -321,12 +341,18 @@ class ItemSpider(FollowAllSpider):class MaxItemsAndRequestsSpider(FollowAllSpider):- def __init__(self, max_items=10, max_requests=10, *args, **kwargs):+ def __init__(+ self,+ max_items: int = 10,+ max_requests: int = 10,+ *args: Any,+ **kwargs: Any,+ ):super().__init__(*args, **kwargs)self.max_items = max_itemsself.max_requests = max_requests- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:self.items_scraped = 0self.pages_crawled = 1 # account for the start urlfor request in super().parse(response):@@ -344,12 +370,12 @@ class DefaultError(Exception):class ErrorSpider(FollowAllSpider):name = "error"- exception_cls = DefaultError+ exception_cls: type[Exception] = DefaultError- def raise_exception(self):+ def raise_exception(self) -> None:raise self.exception_cls("Expected exception")- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:for request in super().parse(response):yield requestself.raise_exception()@@ -359,14 +385,15 @@ class BrokenStartSpider(FollowAllSpider):fail_before_yield = Falsefail_yielding = False- def __init__(self, *a, **kw):+ def __init__(self, *a: Any, **kw: Any):super().__init__(*a, **kw)- self.seedsseen = []+ self.seedsseen: list[Any] = []- async def start(self):+ async def start(self) -> AsyncIterator[Any]:if self.fail_before_yield:1 / 0+ assert self.mockserverfor s in range(100):qargs = {"total": 10, "seed": s}url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")@@ -376,18 +403,18 @@ class BrokenStartSpider(FollowAllSpider):assert self.seedsseen, "All seeds consumed before any download happened"- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:self.seedsseen.append(response.meta.get("seed"))yield from super().parse(response)class StartItemSpider(FollowAllSpider):- async def start(self):+ async def start(self) -> AsyncIterator[Any]:yield {"name": "test item"}class StartGoodAndBadOutput(FollowAllSpider):- async def start(self):+ async def start(self) -> AsyncIterator[Any]:yield {"a": "a"}yield Request("data:,a")yield "data:,b"@@ -395,17 +422,18 @@ class StartGoodAndBadOutput(FollowAllSpider):class SingleRequestSpider(MetaSpider):- seed = None- callback_func = None- errback_func = None+ seed: Request | str | None = None+ callback_func: Callable[[Response], Any] | None = None+ errback_func: Callable[[Failure], Any] | None = None- async def start(self):+ async def start(self) -> AsyncIterator[Any]:if isinstance(self.seed, Request):yield self.seed.replace(callback=self.parse, errback=self.on_error)else:+ assert self.seedyield Request(self.seed, callback=self.parse, errback=self.on_error)- def parse(self, response):+ def parse(self, response: Response) -> Any:self.meta.setdefault("responses", []).append(response)if callable(self.callback_func):return self.callback_func(response)@@ -413,7 +441,7 @@ class SingleRequestSpider(MetaSpider):return response.meta["next"]return None- def on_error(self, failure):+ def on_error(self, failure: Failure) -> Any:self.meta["failure"] = failureif callable(self.errback_func):return self.errback_func(failure)@@ -426,17 +454,18 @@ class DuplicateStartSpider(MockServerSpider):distinct_urls = 2dupe_factor = 3- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserverfor i in range(self.distinct_urls):for _ in range(self.dupe_factor):url = self.mockserver.url(f"/echo?headers=1&body=test{i}")yield Request(url, dont_filter=self.dont_filter)- def __init__(self, url="http://localhost:8998", *args, **kwargs):+ def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):super().__init__(*args, **kwargs)self.visited = 0- def parse(self, response):+ def parse(self, response: Response) -> None:self.visited += 1@@ -451,7 +480,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):}rules = (Rule(LinkExtractor(), callback="parse", follow=True),)- async def start(self):+ async def start(self) -> AsyncIterator[Any]:test_body = b"""<html><head><title>Page title</title></head>@@ -461,11 +490,13 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):</body></html>"""+ assert self.mockserverurl = self.mockserver.url("/alpayload")yield Request(url, method="POST", body=test_body)- def parse(self, response, foo=None):+ def parse(self, response: Response, foo: str | None = None) -> Iterator[Any]:self.logger.info("[parse] status %i (foo: %s)", response.status, foo)+ assert self.mockserveryield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})@@ -477,8 +508,9 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):name = "crawl_spider_with_async_callback"rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),)- async def parse_async(self, response, foo=None):+ async def parse_async(self, response: Response, foo: str | None = None) -> Request:self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo)+ assert self.mockserverreturn Request(self.mockserver.url("/status?n=202"),self.parse_async,@@ -492,8 +524,11 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):name = "crawl_spider_with_async_generator_callback"rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),)- async def parse_async_gen(self, response, foo=None):+ async def parse_async_gen(+ self, response: Response, foo: str | None = None+ ) -> AsyncIterator[Any]:self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo)+ assert self.mockserveryield Request(self.mockserver.url("/status?n=202"),self.parse_async_gen,@@ -505,7 +540,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):name = "crawl_spider_with_errback"rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)- async def start(self):+ async def start(self) -> AsyncIterator[Any]:test_body = b"""<html><head><title>Page title</title></head>@@ -518,17 +553,19 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):</body></html>"""+ assert self.mockserverurl = self.mockserver.url("/alpayload")yield Request(url, method="POST", body=test_body)- def errback(self, failure):+ def errback(self, failure: Failure) -> None:+ assert isinstance(failure.value, HttpError)self.logger.info("[errback] status %i", failure.value.response.status)class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):name = "crawl_spider_without_errback"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:test_body = b"""<html><head><title>Page title</title></head>@@ -538,6 +575,7 @@ class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):</body></html>"""+ assert self.mockserverurl = self.mockserver.url("/alpayload")yield Request(url, method="POST", body=test_body)@@ -553,7 +591,7 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse),)- def process_request(self, request, response):+ def process_request(self, request: Request, response: Response) -> Request:request.cb_kwargs["foo"] = "process_request"return request@@ -562,59 +600,65 @@ class BytesReceivedCallbackSpider(MetaSpider):full_response_length = 2**18@classmethod- def from_crawler(cls, crawler, *args, **kwargs):+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:spider = super().from_crawler(crawler, *args, **kwargs)crawler.signals.connect(spider.bytes_received, signals.bytes_received)return spider- async def start(self):+ async def start(self) -> AsyncIterator[Any]:body = b"a" * self.full_response_length+ assert self.mockserverurl = self.mockserver.url("/alpayload", is_secure=self.is_secure)yield Request(url, method="POST", body=body, errback=self.errback)- def parse(self, response):+ def parse(self, response: Response) -> None:self.meta["response"] = response- def errback(self, failure):+ def errback(self, failure: Failure) -> None:self.meta["failure"] = failure- def bytes_received(self, data, request, spider):+ def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:self.meta["bytes_received"] = dataraise StopDownload(fail=False)class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):- def bytes_received(self, data, request, spider):+ def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:self.meta["bytes_received"] = dataraise StopDownload(fail=True)class HeadersReceivedCallbackSpider(MetaSpider):@classmethod- def from_crawler(cls, crawler, *args, **kwargs):+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:spider = super().from_crawler(crawler, *args, **kwargs)crawler.signals.connect(spider.headers_received, signals.headers_received)return spider- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(self.mockserver.url("/status", is_secure=self.is_secure),errback=self.errback,)- def parse(self, response):+ def parse(self, response: Response) -> None:self.meta["response"] = response- def errback(self, failure):+ def errback(self, failure: Failure) -> None:self.meta["failure"] = failure- def headers_received(self, headers, body_length, request, spider):+ def headers_received(+ self, headers: Headers, body_length: int, request: Request, spider: Spider+ ) -> None:self.meta["headers_received"] = headersraise StopDownload(fail=False)class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):- def headers_received(self, headers, body_length, request, spider):+ def headers_received(+ self, headers: Headers, body_length: int, request: Request, spider: Spider+ ) -> None:self.meta["headers_received"] = headersraise StopDownload(fail=True)@@ -623,13 +667,13 @@ class ExceptionSpider(Spider):name = "exception"@classmethod- def from_crawler(cls, crawler, *args, **kwargs):+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:raise ValueError("Exception in from_crawler method")class NoRequestsSpider(Spider):name = "no_request"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:returnyielddiff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 9a7447e16..81c122b81 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -11,6 +11,8 @@ from tests.utils.decorators import inline_callbacks_testclass TestCloseSpider:+ mockserver: MockServer+@classmethoddef setup_class(cls):cls.mockserver = MockServer()@@ -25,6 +27,7 @@ class TestCloseSpider:close_on = 5crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})yield crawler.crawl(mockserver=self.mockserver)+ assert isinstance(crawler.spider, ItemSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_itemcount"itemcount = crawler.stats.get_value("item_scraped_count")@@ -35,6 +38,7 @@ class TestCloseSpider:close_on = 5crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})yield crawler.crawl(mockserver=self.mockserver)+ assert isinstance(crawler.spider, FollowAllSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_pagecount"pagecount = crawler.stats.get_value("response_received_count")@@ -54,6 +58,7 @@ class TestCloseSpider:yield crawler.crawl(max_items=max_items, max_requests=max_requests, mockserver=self.mockserver)+ assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_pagecount_no_item"pagecount = crawler.stats.get_value("response_received_count")@@ -72,6 +77,7 @@ class TestCloseSpider:},)yield crawler.crawl(mockserver=self.mockserver)+ assert isinstance(crawler.spider, FollowAllSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_pagecount_no_item"pagecount = crawler.stats.get_value("response_received_count")@@ -82,6 +88,7 @@ class TestCloseSpider:close_on = 5crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})yield crawler.crawl(total=1000000, mockserver=self.mockserver)+ assert isinstance(crawler.spider, ErrorSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_errorcount"key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"@@ -94,6 +101,7 @@ class TestCloseSpider:close_on = 0.1crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})yield crawler.crawl(total=1000000, mockserver=self.mockserver)+ assert isinstance(crawler.spider, FollowAllSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_timeout"total_seconds = crawler.stats.get_value("elapsed_time_seconds")@@ -104,6 +112,7 @@ class TestCloseSpider:timeout = 1crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})yield crawler.crawl(n=3, mockserver=self.mockserver)+ assert isinstance(crawler.spider, SlowSpider)reason = crawler.spider.meta["close_reason"]assert reason == "closespider_timeout_no_item"total_seconds = crawler.stats.get_value("elapsed_time_seconds")diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex 11036eaeb..73a85dd5d 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -16,6 +16,11 @@ if TYPE_CHECKING:from pathlib import Path+# Header for spider classes copied with inspect.getsource(): the future import+# keeps their annotations from being evaluated in the generated module.+SPIDER_HEADER = "from __future__ import annotations\nfrom scrapy import Spider\n"++class TestRunSpiderCommand:spider_filename = "myspider.py"@@ -65,14 +70,14 @@ class BadSpider(scrapy.Spider):def test_run_fail_spider(self, tmp_path: Path) -> None:ret, _, _ = self.runspider(- tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)+ tmp_path, SPIDER_HEADER + inspect.getsource(ExceptionSpider))assert ret != 0def test_run_good_spider(self, tmp_path: Path) -> None:ret, _, _ = self.runspider(tmp_path,- "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),+ SPIDER_HEADER + inspect.getsource(NoRequestsSpider),)assert ret == 0diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex 698044de7..44b621776 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -1,3 +1,6 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Anyfrom unittest import TextTestResultimport pytest@@ -11,7 +14,7 @@ from scrapy.contracts.default import (ScrapesContract,UrlContract,)-from scrapy.http import Request+from scrapy.http import Request, Responsefrom scrapy.item import Field, Itemfrom scrapy.spidermiddlewares.httperror import HttpErrorfrom scrapy.spiders import Spider@@ -19,6 +22,24 @@ from scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServerfrom tests.utils.decorators import inline_callbacks_test+if TYPE_CHECKING:+ from unittest import TestResult++ from scrapy.http.request import CallbackT+++def _request(+ conman: ContractsManager, method: CallbackT, results: TestResult+) -> Request:+ request = conman.from_method(method, results)+ assert request+ return request+++def _call(request: Request, response: Any) -> Any:+ assert request.callback+ return request.callback(response, **request.cb_kwargs)+class DemoItem(Item):name = Field()@@ -30,7 +51,7 @@ class ResponseMock:class ResponseMetaMock(ResponseMock):- meta = None+ meta: Any = Noneclass TaggedRequest(Request):@@ -289,17 +310,19 @@ class TestContractsManager:def setup_method(self):self.conman = ContractsManager(self.contracts)- self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)+ self.results = TextTestResult( # type: ignore[type-var]+ stream=None, descriptions=False, verbosity=0+ )- def should_succeed(self):+ def should_succeed(self) -> None:assert not self.results.failuresassert not self.results.errors- def should_fail(self):+ def should_fail(self) -> None:assert self.results.failuresassert not self.results.errors- def should_error(self):+ def should_error(self) -> None:assert self.results.errorsdef test_contracts(self):@@ -313,12 +336,10 @@ class TestContractsManager:)# returns request for valid method- request = self.conman.from_method(spider.returns_request, self.results)- assert request is not None+ assert self.conman.from_method(spider.returns_request, self.results) is not None# no request for missing url- request = self.conman.from_method(spider.parse_no_url, self.results)- assert request is None+ assert self.conman.from_method(spider.parse_no_url, self.results) is Nonedef test_cb_kwargs(self):spider = DemoSpider()@@ -354,29 +375,31 @@ class TestContractsManager:)# returns_request- request = self.conman.from_method(- spider.returns_request_cb_kwargs, self.results- )- request.callback(response, **request.cb_kwargs)+ request = _request(self.conman, spider.returns_request_cb_kwargs, self.results)+ _call(request, response)self.should_succeed()# returns_item- request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)- request.callback(response, **request.cb_kwargs)+ request = _request(self.conman, spider.returns_item_cb_kwargs, self.results)+ _call(request, response)self.should_succeed()# returns_item (error, callback doesn't take keyword arguments)- request = self.conman.from_method(- spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results+ request = _request(+ self.conman,+ spider.returns_item_cb_kwargs_error_unexpected_keyword,+ self.results,)- request.callback(response, **request.cb_kwargs)+ _call(request, response)self.should_error()# returns_item (error, contract doesn't provide keyword arguments)- request = self.conman.from_method(- spider.returns_item_cb_kwargs_error_missing_argument, self.results+ request = _request(+ self.conman,+ spider.returns_item_cb_kwargs_error_missing_argument,+ self.results,)- request.callback(response, **request.cb_kwargs)+ _call(request, response)self.should_error()def test_meta(self):@@ -398,29 +421,27 @@ class TestContractsManager:response = ResponseMetaMock()# returns_request- request = self.conman.from_method(spider.returns_request_meta, self.results)+ request = _request(self.conman, spider.returns_request_meta, self.results)assert request.meta["cookiejar"] == "session1"response.meta = request.meta- request.callback(response)+ _call(request, response)assert response.meta["cookiejar"] == "session1"self.should_succeed()response = ResponseMetaMock()# returns_item- request = self.conman.from_method(spider.returns_item_meta, self.results)+ request = _request(self.conman, spider.returns_item_meta, self.results)assert request.meta["key"] == "example"response.meta = request.meta- request.callback(response)+ _call(request, response)assert response.meta["key"] == "example"self.should_succeed()response = ResponseMetaMock()- request = self.conman.from_method(- spider.returns_error_missing_meta, self.results- )- request.callback(response)+ request = _request(self.conman, spider.returns_error_missing_meta, self.results)+ _call(request, response)self.should_error()def test_returns(self):@@ -428,36 +449,36 @@ class TestContractsManager:response = ResponseMock()# returns_item- request = self.conman.from_method(spider.returns_item, self.results)- request.callback(response)+ request = _request(self.conman, spider.returns_item, self.results)+ _call(request, response)self.should_succeed()# returns_dict_item- request = self.conman.from_method(spider.returns_dict_item, self.results)- request.callback(response)+ request = _request(self.conman, spider.returns_dict_item, self.results)+ _call(request, response)self.should_succeed()# returns_request- request = self.conman.from_method(spider.returns_request, self.results)- request.callback(response)+ request = _request(self.conman, spider.returns_request, self.results)+ _call(request, response)self.should_succeed()# returns_fail- request = self.conman.from_method(spider.returns_fail, self.results)- request.callback(response)+ request = _request(self.conman, spider.returns_fail, self.results)+ _call(request, response)self.should_fail()# returns_dict_fail- request = self.conman.from_method(spider.returns_dict_fail, self.results)- request.callback(response)+ request = _request(self.conman, spider.returns_dict_fail, self.results)+ _call(request, response)self.should_fail()def test_returns_async(self):spider = DemoSpider()response = ResponseMock()- request = self.conman.from_method(spider.returns_request_async, self.results)- request.callback(response)+ request = _request(self.conman, spider.returns_request_async, self.results)+ _call(request, response)self.should_error()def test_returns_invalid_argument_count(self):@@ -477,10 +498,8 @@ class TestContractsManager:spider = DemoSpider()response = ResponseMock()- request = self.conman.from_method(- spider.returns_request_range_fail, self.results- )- request.callback(response)+ request = _request(self.conman, spider.returns_request_range_fail, self.results)+ _call(request, response)self.should_fail()assert "expected 2..3" in self.results.failures[-1][-1]@@ -490,8 +509,8 @@ class TestContractsManager:# @returns and @scrapes only count matching output objects and skip# the request that is also yielded.- request = self.conman.from_method(spider.yields_item_and_request, self.results)- request.callback(response)+ request = _request(self.conman, spider.yields_item_and_request, self.results)+ _call(request, response)self.should_succeed()def test_testcase_str(self):@@ -507,30 +526,30 @@ class TestContractsManager:response = ResponseMock()# scrapes_item_ok- request = self.conman.from_method(spider.scrapes_item_ok, self.results)- request.callback(response)+ request = _request(self.conman, spider.scrapes_item_ok, self.results)+ _call(request, response)self.should_succeed()# scrapes_dict_item_ok- request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results)- request.callback(response)+ request = _request(self.conman, spider.scrapes_dict_item_ok, self.results)+ _call(request, response)self.should_succeed()# scrapes_item_fail- request = self.conman.from_method(spider.scrapes_item_fail, self.results)- request.callback(response)+ request = _request(self.conman, spider.scrapes_item_fail, self.results)+ _call(request, response)self.should_fail()# scrapes_dict_item_fail- request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)- request.callback(response)+ request = _request(self.conman, spider.scrapes_dict_item_fail, self.results)+ _call(request, response)self.should_fail()# scrapes_multiple_missing_fields- request = self.conman.from_method(- spider.scrapes_multiple_missing_fields, self.results+ request = _request(+ self.conman, spider.scrapes_multiple_missing_fields, self.results)- request.callback(response)+ _call(request, response)self.should_fail()message = "ContractFail: Missing fields: name, url"assert message in self.results.failures[-1][-1]@@ -540,14 +559,13 @@ class TestContractsManager:response = ResponseMock()# invalid regex- request = self.conman.from_method(spider.invalid_regex, self.results)- assert request is None+ assert self.conman.from_method(spider.invalid_regex, self.results) is None# invalid regex with valid contract- request = self.conman.from_method(- spider.invalid_regex_with_valid_contract, self.results+ request = _request(+ self.conman, spider.invalid_regex_with_valid_contract, self.results)- request.callback(response)+ _call(request, response)self.should_succeed()def test_custom_contracts(self):@@ -559,14 +577,14 @@ class TestContractsManager:def test_errback(self):spider = DemoSpider()- response = ResponseMock()try:- raise HttpError(response, "Ignoring non-200 response")+ raise HttpError(Response("http://scrapy.org"), "Ignoring non-200 response")except HttpError:failure_mock = failure.Failure()- request = self.conman.from_method(spider.returns_request, self.results)+ request = _request(self.conman, spider.returns_request, self.results)+ assert request.errbackrequest.errback(failure_mock)assert not self.results.failures@@ -602,11 +620,12 @@ class TestContractsManager:crawler = get_crawler(TestSameUrlSpider)yield crawler.crawl()+ assert isinstance(crawler.spider, TestSameUrlSpider)assert crawler.spider.visited == 2def test_custom_tagged_request_contract(self):spider = DemoSpider()- request = self.conman.from_method(spider.custom_tagged_request, self.results)+ request = _request(self.conman, spider.custom_tagged_request, self.results)assert request.method == "POST"assert isinstance(request, TaggedRequest)assert request.contract_tag == "custom"@@ -672,18 +691,20 @@ class PostProcessErrorContract(Contract):class TestCustomContractPrePostProcess:def setup_method(self):- self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)+ self.results = TextTestResult( # type: ignore[type-var]+ stream=None, descriptions=False, verbosity=0+ )def test_pre_hook_keyboard_interrupt(self):spider = DemoSpider()response = ResponseMock()contract = CustomFailContractPreProcess(spider.returns_request)- conman = ContractsManager([UrlContract, ReturnsContract, contract])+ conman = ContractsManager([UrlContract, ReturnsContract])- request = conman.from_method(spider.returns_request, self.results)+ request = _request(conman, spider.returns_request, self.results)contract.add_pre_hook(request, self.results)with pytest.raises(KeyboardInterrupt, match="Pre-process exception"):- request.callback(response, **request.cb_kwargs)+ _call(request, response)assert not self.results.failuresassert not self.results.errors@@ -692,12 +713,12 @@ class TestCustomContractPrePostProcess:spider = DemoSpider()response = ResponseMock()contract = CustomFailContractPostProcess(spider.returns_request)- conman = ContractsManager([UrlContract, ReturnsContract, contract])+ conman = ContractsManager([UrlContract, ReturnsContract])- request = conman.from_method(spider.returns_request, self.results)+ request = _request(conman, spider.returns_request, self.results)contract.add_post_hook(request, self.results)with pytest.raises(KeyboardInterrupt, match="Post-process exception"):- request.callback(response, **request.cb_kwargs)+ _call(request, response)assert not self.results.failuresassert not self.results.errors@@ -706,11 +727,11 @@ class TestCustomContractPrePostProcess:spider = DemoSpider()response = ResponseMock()contract = PreProcessSuccessContract(spider.returns_request)- conman = ContractsManager([UrlContract, ReturnsContract, contract])+ conman = ContractsManager([UrlContract, ReturnsContract])- request = conman.from_method(spider.returns_request, self.results)+ request = _request(conman, spider.returns_request, self.results)contract.add_pre_hook(request, self.results)- request.callback(response, **request.cb_kwargs)+ _call(request, response)assert not self.results.failuresassert not self.results.errors@@ -719,11 +740,11 @@ class TestCustomContractPrePostProcess:spider = DemoSpider()response = ResponseMock()contract = PreProcessAssertionFailContract(spider.returns_request)- conman = ContractsManager([UrlContract, ReturnsContract, contract])+ conman = ContractsManager([UrlContract, ReturnsContract])- request = conman.from_method(spider.returns_request, self.results)+ request = _request(conman, spider.returns_request, self.results)contract.add_pre_hook(request, self.results)- request.callback(response, **request.cb_kwargs)+ _call(request, response)assert self.results.failuresassert not self.results.errors@@ -732,11 +753,11 @@ class TestCustomContractPrePostProcess:spider = DemoSpider()response = ResponseMock()contract = PreProcessErrorContract(spider.returns_request)- conman = ContractsManager([UrlContract, ReturnsContract, contract])+ conman = ContractsManager([UrlContract, ReturnsContract])- request = conman.from_method(spider.returns_request, self.results)+ request = _request(conman, spider.returns_request, self.results)contract.add_pre_hook(request, self.results)- request.callback(response, **request.cb_kwargs)+ _call(request, response)assert self.results.errors@@ -748,7 +769,7 @@ class TestCustomContractPrePostProcess:contract.add_pre_hook(request, self.results)with pytest.raises(TypeError, match="async callbacks"):- request.callback(response)+ _call(request, response)def test_pre_hook_async_generator(self):spider = DemoSpider()@@ -758,7 +779,7 @@ class TestCustomContractPrePostProcess:contract.add_pre_hook(request, self.results)with pytest.raises(TypeError, match="async callbacks"):- request.callback(response)+ _call(request, response)def test_post_hook_async_generator(self):spider = DemoSpider()@@ -768,16 +789,16 @@ class TestCustomContractPrePostProcess:contract.add_post_hook(request, self.results)with pytest.raises(TypeError, match="async callbacks"):- request.callback(response)+ _call(request, response)def test_post_hook_error(self):spider = DemoSpider()response = ResponseMock()contract = PostProcessErrorContract(spider.returns_request)- conman = ContractsManager([UrlContract, ReturnsContract, contract])+ conman = ContractsManager([UrlContract, ReturnsContract])- request = conman.from_method(spider.returns_request, self.results)+ request = _request(conman, spider.returns_request, self.results)contract.add_post_hook(request, self.results)- request.callback(response, **request.cb_kwargs)+ _call(request, response)assert self.results.errorsdiff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 862714167..ca6d0ef93 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -364,7 +364,6 @@ with multiples linesest: list[list[tuple[str, Any]]] = []def cb(response):- assert crawler.engineest.append(get_engine_status(crawler.engine))crawler = get_crawler(SingleRequestSpider)@@ -399,7 +398,6 @@ with multiples linesest: list[str] = []def cb(response):- assert crawler.engineest.append(format_engine_status(crawler.engine))crawler = get_crawler(SingleRequestSpider)@@ -474,7 +472,6 @@ class TestCrawlSpider:await crawler.crawl_async(mockserver.url("/status?n=200"), mockserver=mockserver)- assert crawler.statsreturn items, crawler.stats@coroutine_test@@ -540,7 +537,6 @@ class TestCrawlSpider:# so the failure is dropped silently and the crawl finishes normally.assert "[parse] status 200 (foo: None)" in caplog.textassert "[errback]" not in caplog.text- assert crawler.statsassert crawler.stats.get_value("downloader/response_status_count/404") == 1@coroutine_testdiff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 17ca02dea..2836a6627 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -536,7 +536,6 @@ class TestSpiderSettings:custom_settings = {"AUTOTHROTTLE_ENABLED": True}crawler = get_crawler(MySpider)- assert crawler.extensionsenabled_exts = [e.__class__ for e in crawler.extensions.middlewares]assert AutoThrottle in enabled_exts@@ -568,7 +567,6 @@ class TestCrawlerLogging:}async def start(self):- assert crawler.statsinfo_count_start = crawler.stats.get_value("log_count/INFO")logging.debug("debug message") # noqa: LOG015logging.info("info message") # noqa: LOG015@@ -601,7 +599,6 @@ class TestCrawlerLogging:assert "info message" in loggedassert "warning message" in loggedassert "error message" in logged- assert crawler.statsassert crawler.stats.get_value("log_count/ERROR") == 1assert crawler.stats.get_value("log_count/WARNING") == 1assert info_count == 1diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 6eabe91ad..75affe343 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -72,7 +72,6 @@ class TestBase:settings = self._get_settings(**new_settings)crawler = get_crawler(Spider, settings)crawler.spider = crawler._create_spider("example.com")- assert crawler.statscrawler.stats.open_spider()try:yield crawler@@ -137,7 +136,6 @@ class StorageTestMixin(TestBase):with self._middleware() as mw:spider = mw.crawler.spiderassert spider- assert mw.crawler.statsmw.storage.store_response(spider, self.request, self.response)self._corrupt_cache_entry(mw.storage, spider, self.request)@@ -159,7 +157,6 @@ class StorageTestMixin(TestBase):with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:spider = mw.crawler.spiderassert spider- assert mw.crawler.statsmw.storage.store_response(spider, self.request, self.response)self._corrupt_cache_entry(mw.storage, spider, self.request)diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 9290f55ae..131b9c4b7 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -63,7 +63,6 @@ class TestHttpCompression:def setup_method(self):self.crawler = get_crawler(Spider)self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler)- assert self.crawler.statsself.crawler.stats.open_spider()def _getresponse(self, coding: str) -> Response:@@ -89,7 +88,6 @@ class TestHttpCompression:return responsedef assertStatsEqual(self, key: str, value: Any) -> None:- assert self.crawler.statsassert self.crawler.stats.get_value(key) == value, str(self.crawler.stats.get_stats())diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.pyindex 40e09be13..575051526 100644--- a/tests/test_downloadermiddleware_offsite.py+++ b/tests/test_downloadermiddleware_offsite.py@@ -230,7 +230,6 @@ def test_repeated_offsite_domain():with pytest.raises(IgnoreRequest):mw.process_request(req1)assert "other.org" in mw.domains_seen- assert crawler.statsassert crawler.stats.get_value("offsite/domains") == 1assert crawler.stats.get_value("offsite/filtered") == 1with pytest.raises(IgnoreRequest):diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 1c8a61aa5..7b684b678 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -82,7 +82,6 @@ class TestRetry:# discard itassert self.mw.process_response(req3, rsp) is rsp- assert self.crawler.statsassert self.crawler.stats.get_value("retry/max_reached") == 1assert (self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")@@ -134,7 +133,6 @@ class TestRetry:self._test_retry_exception(req, exc("foo"))stats = self.crawler.stats- assert statsassert stats.get_value("retry/max_reached") == len(exceptions)assert stats.get_value("retry/count") == len(exceptions) * 2assert (@@ -315,7 +313,6 @@ class TestGetRetryRequest:assert new_request.meta["retry_times"] == expected_retry_timesassert new_request.priority == -1expected_reason = "unspecified"- assert spider.crawler.statsfor stat in ("retry/count", f"retry/reason_count/{expected_reason}"):assert spider.crawler.stats.get_value(stat) == 1assert (@@ -336,7 +333,6 @@ class TestGetRetryRequest:max_retry_times=max_retry_times,)assert new_request is None- assert spider.crawler.statsassert spider.crawler.stats.get_value("retry/max_reached") == 1failure_count = max_retry_times + 1expected_reason = "unspecified"@@ -363,7 +359,6 @@ class TestGetRetryRequest:assert new_request.meta["retry_times"] == expected_retry_timesassert new_request.priority == -1expected_reason = "unspecified"- assert spider.crawler.statsfor stat in ("retry/count", f"retry/reason_count/{expected_reason}"):assert spider.crawler.stats.get_value(stat) == 1assert (@@ -396,7 +391,6 @@ class TestGetRetryRequest:assert new_request.meta["retry_times"] == expected_retry_timesassert new_request.priority == -expected_retry_timesexpected_reason = "unspecified"- assert spider.crawler.statsfor stat in ("retry/count", f"retry/reason_count/{expected_reason}"):value = spider.crawler.stats.get_value(stat)assert value == expected_retry_times@@ -521,7 +515,6 @@ class TestGetRetryRequest:reason=expected_reason,)expected_retry_times = 1- assert spider.crawler.statsfor stat in ("retry/count", f"retry/reason_count/{expected_reason}"):assert spider.crawler.stats.get_value(stat) == 1assert (@@ -543,7 +536,6 @@ class TestGetRetryRequest:reason=expected_reason,)expected_retry_times = 1- assert spider.crawler.statsstat = spider.crawler.stats.get_value(f"retry/reason_count/{expected_reason_string}")@@ -569,7 +561,6 @@ class TestGetRetryRequest:reason=expected_reason,)expected_retry_times = 1- assert spider.crawler.statsstat = spider.crawler.stats.get_value(f"retry/reason_count/{expected_reason_string}")@@ -593,7 +584,6 @@ class TestGetRetryRequest:reason=expected_reason,)expected_retry_times = 1- assert spider.crawler.statsstat = spider.crawler.stats.get_value(f"retry/reason_count/{expected_reason_string}")@@ -619,7 +609,6 @@ class TestGetRetryRequest:reason=expected_reason,)expected_retry_times = 1- assert spider.crawler.statsstat = spider.crawler.stats.get_value(f"retry/reason_count/{expected_reason_string}")@@ -737,7 +726,6 @@ class TestGetRetryRequest:reason=expected_reason,stats_base_key=stats_key,)- assert spider.crawler.statsfor stat in (f"{stats_key}/count",f"{stats_key}/reason_count/{expected_reason}",diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.pyindex 13f6d6fe3..01014ddd6 100644--- a/tests/test_downloadermiddleware_stats.py+++ b/tests/test_downloadermiddleware_stats.py@@ -17,7 +17,6 @@ class MyException(Exception):class TestDownloaderStats:def setup_method(self) -> None:self.crawler = get_crawler(Spider)- assert self.crawler.stats is not Noneself.mw = build_from_crawler(DownloaderStats, self.crawler)self.crawler.stats.open_spider()@@ -26,7 +25,6 @@ class TestDownloaderStats:self.res = Response("http://scrapytest.org", status=400)def assertStatsEqual(self, key: str, value: object) -> None:- assert self.crawler.stats is not Noneassert self.crawler.stats.get_value(key) == value, str(self.crawler.stats.get_stats())@@ -53,7 +51,6 @@ class TestDownloaderStats:build_from_crawler(DownloaderStats, crawler)def teardown_method(self) -> None:- assert self.crawler.stats is not Noneself.crawler.stats.close_spider()diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 6f323a15c..077a92ecf 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -148,7 +148,6 @@ async def test_none_slot_with_priority_queue(assert crawler.spider.default_slot in crawler.spider.timesassert len(crawler.spider.times[crawler.spider.default_slot]) == 2- assert crawler.statsstats = crawler.statsassert stats.get_value("spider_exceptions", 0) == 0assert stats.get_value("downloader/exception_count", 0) == 0diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 348e778cd..4b984406a 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -179,7 +179,6 @@ class TestRFPDupeFilter:dupefilter.log(r1, spider)dupefilter.log(r2, spider)- assert crawler.statsassert crawler.stats.get_value("dupefilter/filtered") == 2assert ("scrapy.dupefilters",@@ -213,7 +212,6 @@ class TestRFPDupeFilter:dupefilter.log(r1, spider)dupefilter.log(r2, spider)- assert crawler.statsassert crawler.stats.get_value("dupefilter/filtered") == 2assert ("scrapy.dupefilters",diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 8e5197df7..5c3c92d3b 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -15,7 +15,7 @@ from tests.mockserver.http import MockServerfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:- from collections.abc import Iterator+ from collections.abc import AsyncIterator, Callable, Iteratorfrom scrapy.http import Response@@ -85,19 +85,23 @@ class TestMain:class TestSpider(Spider):name = "test"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.crawler.engine._slot+ scheduler = self.crawler.engine._slot.scheduler+ assert isinstance(scheduler, MemoryScheduler)+yield Request("data:,a")await sleep(seconds)- self.crawler.engine._slot.scheduler.pause()- self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))+ scheduler.pause()+ scheduler.enqueue_request(Request("data:,b"))# During this time, the scheduler reports having requests but# returns None.await sleep(seconds)- self.crawler.engine._slot.scheduler.unpause()+ scheduler.unpause()# The scheduler request is processed.await sleep(seconds)@@ -106,21 +110,21 @@ class TestMain:await sleep(seconds)- self.crawler.engine._slot.scheduler.pause()- self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))+ scheduler.pause()+ scheduler.enqueue_request(Request("data:,d"))# The last start request is processed during the time until the# delayed call below, proving that the start iteration can# finish before a scheduler “sleep” without causing the# scheduler to finish.- call_later(seconds, self.crawler.engine._slot.scheduler.unpause)+ call_later(seconds, scheduler.unpause)- def parse(self, response):+ def parse(self, response: Response) -> None:passactual_urls = []- def track_url(request, spider):+ def track_url(request: Request, spider: Spider) -> None:actual_urls.append(request.url)settings = {"SCHEDULER": MemoryScheduler}@@ -138,17 +142,16 @@ class TestMain:class TestSpider(Spider):name = "test"- async def start(self):- assert self.crawler.engine is not None+ async def start(self) -> AsyncIterator[Any]:await self.crawler.engine.close_async()yield Request("data:,a")- def parse(self, response):+ def parse(self, response: Response) -> None:passactual_urls = []- def track_url(request, spider):+ def track_url(request: Request, spider: Spider) -> None:actual_urls.append(request.url)settings = {"SCHEDULER": MemoryScheduler}@@ -160,7 +163,6 @@ class TestMain:await crawler.crawl_async()assert not caplog.records- assert crawler.statsassert crawler.stats.get_value("finish_reason") == "shutdown"assert not actual_urls@@ -229,7 +231,6 @@ class TestMain:failure, response = errors[0]assert isinstance(failure.value, ValueError)assert response is None- assert crawler.statsassert crawler.stats.get_value("finish_reason") == "start_error"assert crawler.stats.get_value("spider_exceptions/count") == 1assert crawler.stats.get_value("spider_exceptions/ValueError") == 1@@ -256,12 +257,13 @@ class TestMain:await crawler.crawl_async()assert not caplog.records- assert crawler.statsassert crawler.stats.get_value("finish_reason") == "my_reason"assert crawler.stats.get_value("spider_exceptions/count") is Noneclass TestRequestSendOrder:+ mockserver: MockServer+seconds = 0.1 # increase if flaky@classmethod@@ -273,52 +275,65 @@ class TestRequestSendOrder:def teardown_class(cls):cls.mockserver.__exit__(None, None, None)- def request(self, num, response_seconds, download_slots, priority=0):+ def request(+ self,+ num: int,+ response_seconds: float,+ download_slots: int,+ priority: int = 0,+ ) -> Request:url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")meta = {"download_slot": str(num % download_slots)}return Request(url, meta=meta, priority=priority)- def get_num(self, request_or_response: Request | Response):+ def get_num(self, request_or_response: Request | Response) -> int:return int(request_or_response.url.rsplit("&", maxsplit=1)[1])async def _test_request_order(self,- start_nums,- cb_nums=None,- settings=None,- response_seconds=None,- download_slots=1,- start_fn=None,- parse_fn=None,- ):+ start_nums: list[int],+ cb_nums: list[int] | None = None,+ settings: dict[str, Any] | None = None,+ response_seconds: float | None = None,+ download_slots: int = 1,+ start_fn: Callable[[Spider], AsyncIterator[Any]] | None = None,+ parse_fn: Callable[..., Iterator[Any]] | None = None,+ ) -> None:cb_nums = cb_nums or []settings = settings or {}- response_seconds = response_seconds or self.seconds+ seconds = response_seconds or self.secondscb_requests = deque(- [self.request(num, response_seconds, download_slots) for num in cb_nums]+ [self.request(num, seconds, download_slots) for num in cb_nums])if start_fn is None:- async def start_fn(spider):+ async def default_start(spider: Spider) -> AsyncIterator[Any]:for num in start_nums:- yield self.request(num, response_seconds, download_slots)+ yield self.request(num, seconds, download_slots)++ start_fn = default_startif parse_fn is None:- def parse_fn(spider, response):+ def default_parse(spider: Spider, response: Response) -> Iterator[Any]:while cb_requests:yield cb_requests.popleft()+ parse_fn = default_parse++ spider_start = start_fn+ spider_parse = parse_fn+class TestSpider(Spider):name = "test"- start = start_fn- parse = parse_fn+ start = spider_start+ parse = spider_parseactual_nums = []- def track_num(request, spider):+ def track_num(request: Request, spider: Spider) -> None:actual_nums.append(self.get_num(request))crawler = get_crawler(TestSpider, settings_dict=settings)@@ -337,12 +352,13 @@ class TestRequestSendOrder:response_seconds = 0download_slots = 1- def _request(num, priority=0):+ def _request(num: int, priority: int = 0) -> Request:return self.request(num, response_seconds, download_slots, priority=priority)- async def start(spider):+ async def start(spider: Spider) -> AsyncIterator[Any]:+ assert spider.crawler.engine._slot# The first CONCURRENT_REQUESTS start requests are sent# immediately.yield _request(1)@@ -356,7 +372,7 @@ class TestRequestSendOrder:yield _request(3, priority=1)yield _request(4, priority=1)- def parse(spider, response):+ def parse(spider: Spider, response: Response) -> Iterator[Any]:returnyield@@ -377,12 +393,13 @@ class TestRequestSendOrder:response_seconds = 0download_slots = 1- def _request(num, priority=0):+ def _request(num: int, priority: int = 0) -> Request:return self.request(num, response_seconds, download_slots, priority=priority)- async def start(spider):+ async def start(spider: Spider) -> AsyncIterator[Any]:+ assert spider.crawler.engine._slot# The first CONCURRENT_REQUESTS start requests are sent# immediately.yield _request(1)@@ -396,7 +413,7 @@ class TestRequestSendOrder:yield _request(4, priority=1)yield _request(3, priority=1)- def parse(spider, response):+ def parse(spider: Spider, response: Response) -> Iterator[Any]:returnyield@@ -420,12 +437,13 @@ class TestRequestSendOrder:response_seconds = 0download_slots = 1- def _request(num, priority=0):+ def _request(num: int, priority: int = 0) -> Request:return self.request(num, response_seconds, download_slots, priority=priority)- async def start(spider):+ async def start(spider: Spider) -> AsyncIterator[Any]:+ assert spider.crawler.engine._slot# The first CONCURRENT_REQUESTS start requests are sent# immediately.yield _request(1)@@ -454,7 +472,7 @@ class TestRequestSendOrder:):spider.crawler.engine._slot.scheduler.enqueue_request(request)- def parse(spider, response):+ def parse(spider: Spider, response: Response) -> Iterator[Any]:returnyield@@ -479,7 +497,7 @@ class TestRequestSendOrder:response_seconds = self.seconds * 2**1 # increase if flakydownload_slots = 1- async def start(spider):+ async def start(spider: Spider) -> AsyncIterator[Any]:for num in start_nums:if spider.crawler.engine.needs_backout():await spider.crawler.signals.wait_for(signals.scheduler_empty)diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.pyindex ddcfb5163..0c4f2ad95 100644--- a/tests/test_extension_memdebug.py+++ b/tests/test_extension_memdebug.py@@ -33,7 +33,6 @@ def test_spider_closed_sets_stats() -> None:ext.spider_closed(DefaultSpider.from_crawler(crawler), "finished")- assert crawler.statsassert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked)assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None@@ -47,6 +46,5 @@ async def test_crawl_sets_stats() -> None:crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True})await crawler.crawl_async()- assert crawler.statsassert crawler.stats.get_value("memdebug/gc_garbage_count") is not Noneassert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.pyindex dbe28eded..aa0149ff2 100644--- a/tests/test_extension_memusage.py+++ b/tests/test_extension_memusage.py@@ -99,7 +99,6 @@ async def test_memusage_below_thresholds_logs_peak(with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"):await crawler.crawl_async(url="data:,", loops=1)- assert crawler.statsassert crawler.stats.get_value("memusage/limit_reached") is Noneassert crawler.stats.get_value("memusage/warning_reached") is Noneassert crawler.stats.get_value("memusage/max") == 25 * MB@@ -129,7 +128,6 @@ async def test_memusage_limit_closes_spider_with_reason_and_error_log(with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"):await crawler.crawl_async(url="data:,", loops=100)- assert crawler.statsassert crawler.stats.get_value("memusage/limit_reached") == 1assert crawler.stats.get_value("finish_reason") == "memusage_exceeded"assert any(@@ -167,7 +165,6 @@ async def test_memusage_warning_logs_but_allows_normal_finish(await crawler.crawl_async(url="data:,", loops=60)assert warning_signals == [1]- assert crawler.statsassert crawler.stats.get_value("memusage/warning_reached") == 1assert crawler.stats.get_value("finish_reason") == "finished"warnings_logged = [diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex ad59297b6..1e8771d65 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -119,7 +119,6 @@ async def test_telnet_vars() -> None:start_urls = ["data:,"]async def parse(self, response: Response) -> None:- assert self.crawler.extensionsconsole = next(extfor ext in self.crawler.extensions.middlewaresdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 8560dd364..67de762fc 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -310,7 +310,6 @@ class TestFeedExport(TestFeedExportBase):}crawler = get_crawler(ItemSpider, settings)yield crawler.crawl(mockserver=self.mockserver)- assert crawler.stats is not Noneassert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1@@ -334,7 +333,6 @@ class TestFeedExport(TestFeedExportBase):side_effect=store,):yield crawler.crawl(mockserver=self.mockserver)- assert crawler.stats is not Noneassert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1@@ -352,7 +350,6 @@ class TestFeedExport(TestFeedExportBase):}crawler = get_crawler(ItemSpider, settings)yield crawler.crawl(mockserver=self.mockserver)- assert crawler.stats is not Noneassert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 1958e2935..fecb33d07 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -420,7 +420,6 @@ class TestBatchDeliveries(TestFeedExportBase):}crawler = get_crawler(ItemSpider, settings)yield crawler.crawl(total=2, mockserver=self.mockserver)- assert crawler.statsassert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12@@ -491,7 +490,6 @@ class TestBatchDeliveries(TestFeedExportBase):assert len(CustomS3FeedStorage.stubs) == len(items)for stub in CustomS3FeedStorage.stubs:stub.assert_no_pending_responses()- assert crawler.statsassert ("feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats())diff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex 35fd78283..09cafd218 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -15,7 +15,6 @@ class TestLogStats:def setup_method(self) -> None:self.crawler = get_crawler(SimpleSpider)self.spider = self.crawler._create_spider("spidey")- assert self.crawler.stats is not Noneself.stats = self.crawler.statsself.stats.set_value("response_received_count", 4802)diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 8b522255a..529e7ffcb 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -140,7 +140,6 @@ class TestFileDownloadCrawl:assert not items[0][self.media_key]# check that there was 1 successful fetch and 3 other responses with non-200 code- assert crawler.statsassert crawler.stats.get_value("downloader/request_method_count/GET") == 4assert crawler.stats.get_value("downloader/response_count") == 4assert crawler.stats.get_value("downloader/response_status_count/200") == 1@@ -208,7 +207,6 @@ class TestFileDownloadCrawl:mockserver=self.mockserver,)self._assert_files_downloaded(self.items, caplog.text)- assert crawler.statsassert crawler.stats.get_value("downloader/response_status_count/302") == 3@coroutine_testdiff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 231f6748f..30a42da48 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -82,7 +82,6 @@ class TestBaseMediaPipeline:self.pipe = build_from_crawler(self.pipeline_class, crawler)self.pipe.open_spider()self.info = self.pipe.spiderinfo- assert crawler.request_fingerprinter is not Noneself.fingerprint = crawler.request_fingerprinter.fingerprint@propertydiff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 6c26aa878..97dafc87c 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -70,7 +70,6 @@ class KeywordArgumentsSpider(MockServerSpider):checks: list[bool] = []def _inc_checks(self, count: int = 1) -> None:- assert self.crawler.statsself.crawler.stats.inc_value("boolean_checks", count)async def start(self):@@ -171,7 +170,6 @@ class TestCallbackKeywordArguments:await crawler.crawl_async(mockserver=mockserver)assert isinstance(crawler.spider, KeywordArgumentsSpider)assert all(crawler.spider.checks)- assert crawler.statsassert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")# check exceptions for argument mismatchexceptions = {}diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex ff92b1133..508147a7c 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -54,7 +54,6 @@ async def create_scheduler(finally:await ensure_awaitable(scheduler.close("finished"))await mock_crawler.stop_async()- assert mock_crawler.enginemock_crawler.engine.downloader.close()@@ -254,7 +253,6 @@ class DownloaderAwareSchedulerTestMixin(TestSchedulerBase):dequeued_slots: list[str] = []requests: list[Request] = []assert scheduler.crawler- assert scheduler.crawler.enginedownloader = scheduler.crawler.engine.downloaderassert isinstance(downloader, MockDownloader)while scheduler.has_pending_requests():@@ -320,7 +318,6 @@ class TestIntegrationWithDownloaderAwareInMemory:url = mockserver.url("/status?n=200", is_secure=False)start_urls = [url] * 6yield self.crawler.crawl(start_urls)- assert self.crawler.statsassert self.crawler.stats.get_value("downloader/response_count") == len(start_urls)diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 38cb8da18..6f7f171a3 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -41,7 +41,7 @@ class TestXMLFeedSpider(TestSpiderBase):</urlset>"""response = XmlResponse(url="http://example.com/sitemap.xml", body=body)- class _XMLSpider(self.spider_class):+ class _XMLSpider(self.spider_class): # type: ignore[name-defined,misc]itertag = "url"namespaces = (("a", "http://www.google.com/schemas/sitemap/0.84"),@@ -100,7 +100,6 @@ class TestXMLFeedSpider(TestSpiderBase):items, crawler = await crawl_items(_Spider, mockserver)assert items == []- assert crawler.statsassert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1@coroutine_test@@ -135,7 +134,6 @@ class TestXMLFeedSpider(TestSpiderBase):items, crawler = await crawl_items(_Spider, mockserver)assert items == []- assert crawler.statsassert crawler.stats.get_value("spider_exceptions/NotSupported") == 1@pytest.mark.parametrize("feed_iterator", ["xml", "html"])@@ -155,7 +153,6 @@ class TestXMLFeedSpider(TestSpiderBase):items, crawler = await crawl_items(_Spider, mockserver)assert items == []- assert crawler.statsassert crawler.stats.get_value("spider_exceptions/ValueError") == 1@@ -166,7 +163,7 @@ class TestCSVFeedSpider(TestSpiderBase):body = get_testdata("feeds", "feed-sample6.csv")response = Response("http://example.org/dummy.csv", body=body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"delimiter = ","quotechar = "'"@@ -206,7 +203,6 @@ class TestCSVFeedSpider(TestSpiderBase):items, crawler = await crawl_items(_Spider, mockserver)assert items == []- assert crawler.statsassert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.pyindex 9d5f0548b..ce556d0d1 100644--- a/tests/test_spider_crawl.py+++ b/tests/test_spider_crawl.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport reimport warnings+from typing import TYPE_CHECKING, Anyimport pytestfrom w3lib.url import safe_url_string@@ -14,6 +15,12 @@ from scrapy.utils.test import get_crawlerfrom tests.utils.bases.spider import TestSpiderBasefrom tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from scrapy.http import Response+ from scrapy.http.request import CallbackT++URL = "https://www.example.com"+class TestCrawlSpider(TestSpiderBase):test_body = b"""<html><head><title>Page title</title></head>@@ -33,7 +40,7 @@ class TestCrawlSpider(TestSpiderBase):"http://example.org/somepage/index.html", body=self.test_body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (Rule(),)@@ -53,7 +60,7 @@ class TestCrawlSpider(TestSpiderBase):"http://example.org/somepage/index.html", body=self.test_body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)@@ -76,7 +83,7 @@ class TestCrawlSpider(TestSpiderBase):"http://example.org/somepage/index.html", body=self.test_body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)@@ -99,7 +106,7 @@ class TestCrawlSpider(TestSpiderBase):"http://example.org/somepage/index.html", body=self.test_body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)@@ -125,7 +132,7 @@ class TestCrawlSpider(TestSpiderBase):def process_request_change_domain(request, response):return request.replace(url=request.url.replace(".org", ".com"))- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (@@ -151,7 +158,7 @@ class TestCrawlSpider(TestSpiderBase):request.meta["response_class"] = response.__class__.__name__return request- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (@@ -180,7 +187,7 @@ class TestCrawlSpider(TestSpiderBase):"http://example.org/somepage/index.html", body=self.test_body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)@@ -203,7 +210,7 @@ class TestCrawlSpider(TestSpiderBase):"http://example.org/somepage/index.html", body=self.test_body)- class _CrawlSpider(self.spider_class):+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"allowed_domains = ["example.org"]rules = (@@ -248,7 +255,6 @@ class TestCrawlSpider(TestSpiderBase):def test_parse_response_use(self):class _CrawlSpider(CrawlSpider):name = "test"- start_urls = "https://www.example.com"_follow_links = Falsewith warnings.catch_warnings():@@ -258,17 +264,20 @@ class TestCrawlSpider(TestSpiderBase):ScrapyDeprecationWarning,match=r"CrawlSpider\._parse_response method is deprecated",):- spider._parse_response(- TextResponse(spider.start_urls, body=b""), None, None- )+ spider._parse_response(TextResponse(URL, body=b""), None, {})def test_parse_response_override(self):class _CrawlSpider(CrawlSpider):- def _parse_response(self, response, callback, cb_kwargs, follow=True):+ def _parse_response(+ self,+ response: Response,+ callback: CallbackT | None,+ cb_kwargs: dict[str, Any],+ follow: bool = True,+ ) -> Any:passname = "test"- start_urls = "https://www.example.com"_follow_links = Falsewith pytest.warns(@@ -278,21 +287,16 @@ class TestCrawlSpider(TestSpiderBase):spider = _CrawlSpider()with warnings.catch_warnings():warnings.simplefilter("error", category=ScrapyDeprecationWarning)- spider._parse_response(- TextResponse(spider.start_urls, body=b""), None, None- )+ spider._parse_response(TextResponse(URL, body=b""), None, {})def test_parse_with_rules(self):class _CrawlSpider(CrawlSpider):name = "test"- start_urls = "https://www.example.com"with warnings.catch_warnings():warnings.simplefilter("error", category=ScrapyDeprecationWarning)spider = _CrawlSpider()- spider.parse_with_rules(- TextResponse(spider.start_urls, body=b""), None, None- )+ spider.parse_with_rules(TextResponse(URL, body=b""), None, {})@coroutine_testasync def test_parse_with_rules_without_callback(self):@@ -327,12 +331,13 @@ class TestCrawlSpider(TestSpiderBase):allowed_domains = ["example.org"]rules = (Rule(),)+ def callback(response: Response) -> list[Any]:+ return [item]+spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))results = [r- async for r in spider.parse_with_rules(- response, lambda response: [item], {}, follow=False- )+ async for r in spider.parse_with_rules(response, callback, {}, follow=False)]assert results == [item]diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex 2f1ccab81..630ab5ca4 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -55,7 +55,7 @@ class TestSitemapSpider(TestSpiderBase):assert spider._get_sitemap_body(response) == bodydef test_get_sitemap_body(self):- r = XmlResponse(url="http://www.example.com/", body=self.BODY)+ r: Response = XmlResponse(url="http://www.example.com/", body=self.BODY)self.assertSitemapBody(r, self.BODY)r = HtmlResponse(url="http://www.example.com/", body=self.BODY)@@ -166,7 +166,7 @@ Sitemap: /sitemap-relative-url.xml</url></urlset>"""- class FilteredSitemapSpider(self.spider_class):+ class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]def sitemap_filter(self, entries):for entry in entries:date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")@@ -201,7 +201,7 @@ Sitemap: /sitemap-relative-url.xml</url></urlset>"""- class FilteredSitemapSpider(self.spider_class):+ class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]def sitemap_filter(self, entries):for entry in entries:alternate_links = entry.get("alternate", ())@@ -235,7 +235,7 @@ Sitemap: /sitemap-relative-url.xml</sitemap></sitemapindex>"""- class FilteredSitemapSpider(self.spider_class):+ class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]def sitemap_filter(self, entries):for entry in entries:date_time = datetime.strptime(@@ -285,6 +285,7 @@ Sitemap: /sitemap-relative-url.xmlsitemap_rules = [("", parse_item)]def raw_body(self):+ assert self.mockserverloc = self.mockserver.url("/text")return ('<?xml version="1.0" encoding="UTF-8"?>'@@ -303,6 +304,7 @@ Sitemap: /sitemap-relative-url.xmlyield {"url": response.url}def raw_body(self):+ assert self.mockserverloc = self.mockserver.url("/text")return ('<?xml version="1.0" encoding="UTF-8"?>'@@ -363,7 +365,7 @@ Sitemap: /sitemap-relative-url.xml</sitemapindex>"""r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)- class _FollowSpider(self.spider_class):+ class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]sitemap_follow = [follow]spider = _FollowSpider("example.com")@@ -382,7 +384,7 @@ Sitemap: /sitemap-relative-url.xml@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")def test_compression_bomb_spider_attr(self):- class DownloadMaxSizeSpider(self.spider_class):+ class DownloadMaxSizeSpider(self.spider_class): # type: ignore[name-defined,misc]download_maxsize = 10_000_000crawler = get_crawler()@@ -485,7 +487,7 @@ Sitemap: /sitemap-relative-url.xml@coroutine_testasync def test_sitemap_urls(self):- class TestSpider(self.spider_class):+ class TestSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"sitemap_urls = ["https://toscrape.com/sitemap.xml"]diff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex 7cef2fefa..fb72ece6c 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -34,7 +34,6 @@ class TestMain:crawler = get_crawler(spider)crawler.signals.connect(track_item, signals.item_scraped)await crawler.crawl_async()- assert crawler.statsassert crawler.stats.get_value("finish_reason") == "finished"assert actual_items == expected_itemsdiff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.pyindex 69a41c0ff..9d6f4666b 100644--- a/tests/test_spidermiddleware_depth.py+++ b/tests/test_spidermiddleware_depth.py@@ -26,7 +26,6 @@ def crawler() -> Crawler:@pytest.fixturedef stats(crawler: Crawler) -> Generator[StatsCollector]:- assert crawler.stats is not Nonecrawler.stats.open_spider()yield crawler.stats@@ -120,7 +119,6 @@ def test_priority_and_non_verbose_stats() -> None:Spider,{"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},)- assert crawler.stats is not Nonecrawler.stats.open_spider()try:mw = build_from_crawler(DepthMiddleware, crawler)diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.pyindex d8d5b278b..43b41ce3f 100644--- a/tests/test_spidermiddleware_httperror.py+++ b/tests/test_spidermiddleware_httperror.py@@ -205,7 +205,6 @@ class TestHttpErrorMiddlewareIntegrational:assert crawler.spider.parsed == {"200"}assert crawler.spider.failed == {"404", "402", "500"}- assert crawler.statsget_value = crawler.stats.get_valueassert get_value("httperror/response_ignored_count") == 3assert get_value("httperror/response_ignored_status_count/404") == 1diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex dacc90b27..74051d1db 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -1,16 +1,21 @@from __future__ import annotationsimport logging-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom scrapy import Request, Spiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer+from tests.spiders import MockServerSpiderfrom tests.utils.decorators import coroutine_testif TYPE_CHECKING:+ from collections.abc import AsyncIterator, Iterator+import pytest+ from scrapy.http import Response+class _BaseSpiderMiddleware:def __init__(self, crawler):@@ -41,7 +46,7 @@ class RecoveryMiddleware(_BaseSpiderMiddleware):]-class RecoverySpider(Spider):+class RecoverySpider(MockServerSpider):name = "RecoverySpider"custom_settings = {"SPIDER_MIDDLEWARES_BASE": {},@@ -50,10 +55,11 @@ class RecoverySpider(Spider):},}- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(self.mockserver.url("/status?n=200"))- def parse(self, response):+ def parse(self, response: Response) -> Iterator[Any]:yield {"test": 1}self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))if not response.meta.get("dont_fail"):@@ -63,7 +69,7 @@ class RecoverySpider(Spider):class RecoveryAsyncGenSpider(RecoverySpider):name = "RecoveryAsyncGenSpider"- async def parse(self, response):+ async def parse(self, response: Response) -> AsyncIterator[Any]: # type: ignore[override]for r in super().parse(response):yield r@@ -76,7 +82,7 @@ class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):raise IndexError-class ProcessSpiderInputSpiderWithoutErrback(Spider):+class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider):name = "ProcessSpiderInputSpiderWithoutErrback"custom_settings = {"SPIDER_MIDDLEWARES": {@@ -87,7 +93,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):}}- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)def parse(self, response):@@ -97,7 +104,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):name = "ProcessSpiderInputSpiderWithErrback"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(self.mockserver.url("/status?n=200"), self.parse, errback=self.errback)@@ -109,7 +117,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback# ================================================================================# (2) exceptions from a spider callback (generator)-class GeneratorCallbackSpider(Spider):+class GeneratorCallbackSpider(MockServerSpider):name = "GeneratorCallbackSpider"custom_settings = {"SPIDER_MIDDLEWARES": {@@ -117,7 +125,8 @@ class GeneratorCallbackSpider(Spider):},}- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(self.mockserver.url("/status?n=200"))def parse(self, response):@@ -146,7 +155,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)# ================================================================================# (3) exceptions from a spider callback (not a generator)-class NotGeneratorCallbackSpider(Spider):+class NotGeneratorCallbackSpider(MockServerSpider):name = "NotGeneratorCallbackSpider"custom_settings = {"SPIDER_MIDDLEWARES": {@@ -154,7 +163,8 @@ class NotGeneratorCallbackSpider(Spider):},}- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(self.mockserver.url("/status?n=200"))def parse(self, response):@@ -224,7 +234,7 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):pass-class GeneratorOutputChainSpider(Spider):+class GeneratorOutputChainSpider(MockServerSpider):name = "GeneratorOutputChainSpider"custom_settings = {"SPIDER_MIDDLEWARES": {@@ -235,7 +245,8 @@ class GeneratorOutputChainSpider(Spider):},}- async def start(self):+ async def start(self) -> AsyncIterator[Any]:+ assert self.mockserveryield Request(self.mockserver.url("/status?n=200"))def parse(self, response):diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex 321139da3..46414bb59 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -1,4 +1,7 @@+from __future__ import annotations+from asyncio import sleep+from typing import TYPE_CHECKING, Anyimport pytest@@ -9,6 +12,11 @@ from scrapy.utils.test import get_crawlerfrom .utils import twisted_sleepfrom .utils.decorators import coroutine_test+if TYPE_CHECKING:+ from collections.abc import AsyncIterator++ from scrapy.http import Response+ITEM_A = {"id": "a"}ITEM_B = {"id": "b"}ITEM_C = {"id": "c"}@@ -17,20 +25,20 @@ SLEEP_SECONDS = 0.1class AsyncioSleepSpiderMiddleware:- async def process_start(self, start):+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:await sleep(SLEEP_SECONDS)async for item_or_request in start:yield item_or_requestclass NoOpSpiderMiddleware:- async def process_start(self, start):+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:async for item_or_request in start:yield item_or_requestclass TwistedSleepSpiderMiddleware:- async def process_start(self, start):+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))async for item_or_request in start:yield item_or_request@@ -42,12 +50,12 @@ class TwistedSleepSpiderMiddleware:class ModernWrapSpider(Spider):name = "test"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:yield ITEM_Bclass ModernWrapSpiderMiddleware:- async def process_start(self, start):+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:yield ITEM_Aasync for item_or_request in start:yield item_or_request@@ -55,10 +63,15 @@ class ModernWrapSpiderMiddleware:class TestMain:- async def _test(self, spider_middlewares, spider_cls, expected_items):+ async def _test(+ self,+ spider_middlewares: list[type],+ spider_cls: type[Spider],+ expected_items: list[Any],+ ) -> None:actual_items = []- def track_item(item, response, spider):+ def track_item(item: Any, response: Response, spider: Spider) -> None:actual_items.append(item)settings = {@@ -70,7 +83,12 @@ class TestMain:assert crawler.stats.get_value("finish_reason") == "finished"assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"- async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None):+ async def _test_wrap(+ self,+ spider_middleware: type,+ spider_cls: type[Spider],+ expected_items: list[Any] | None = None,+ ) -> None:expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]await self._test([spider_middleware], spider_cls, expected_items)@@ -78,11 +96,11 @@ class TestMain:async def test_modern_mw_modern_spider(self):await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)- async def _test_sleep(self, spider_middlewares):+ async def _test_sleep(self, spider_middlewares: list[type]) -> None:class TestSpider(Spider):name = "test"- async def start(self):+ async def start(self) -> AsyncIterator[Any]:yield ITEM_Aawait self._test(spider_middlewares, TestSpider, [ITEM_A])diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex 1090b3e84..2620327af 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -31,7 +31,6 @@ def crawler() -> Crawler:@pytest.fixturedef stats(crawler: Crawler) -> StatsCollector:- assert crawler.stats is not Nonereturn crawler.statsdiff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex d1b98d009..4a18d9ff6 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -97,7 +97,6 @@ class TestLogCounterHandler:logger.removeHandler(handler)def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:- assert crawler.statsassert crawler.stats.get_value("log_count/DEBUG") is Noneassert crawler.stats.get_value("log_count/INFO") is Noneassert crawler.stats.get_value("log_count/WARNING") is None@@ -106,12 +105,10 @@ class TestLogCounterHandler:def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:logger.error("test log msg")- assert crawler.statsassert crawler.stats.get_value("log_count/ERROR") == 1def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:logger.debug("test log msg")- assert crawler.statsassert crawler.stats.get_value("log_count/DEBUG") is Nonediff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 935447bc4..6ca608053 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -274,7 +274,6 @@ class TestRequestFingerprinter:def test_fingerprint(self):crawler = get_crawler()request = Request("https://example.com")- assert crawler.request_fingerprinterassert crawler.request_fingerprinter.fingerprint(request) == fingerprint(request)@@ -291,7 +290,6 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)- assert crawler.request_fingerprinterr1 = Request("http://www.example.com", headers={"X-ID": "1"})fp1 = crawler.request_fingerprinter.fingerprint(r1)r2 = Request("http://www.example.com", headers={"X-ID": "2"})@@ -314,7 +312,6 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)- assert crawler.request_fingerprinterr1 = Request("http://www.example.com?a=1&a=2")fp1 = crawler.request_fingerprinter.fingerprint(r1)r2 = Request("http://www.example.com?a=2&a=1")@@ -333,7 +330,6 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)- assert crawler.request_fingerprinterr1 = Request("http://www.example.com")fp1 = crawler.request_fingerprinter.fingerprint(r1)r2 = Request("http://www.example.com", meta={"fingerprint": "a"})@@ -365,7 +361,6 @@ class TestCustomRequestFingerprinter:}crawler = get_crawler(settings_dict=settings)- assert crawler.request_fingerprinterrequest = Request("http://www.example.com")fingerprint = crawler.request_fingerprinter.fingerprint(request)assert fingerprint == settings["FINGERPRINT"]diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.pyindex 799c1820d..224606bf6 100644--- a/tests/utils/bases/spider.py+++ b/tests/utils/bases/spider.py@@ -32,7 +32,7 @@ class TestSpiderBase(ABC):def test_spider_args(self):"""``__init__`` method arguments are assigned to spider attributes"""spider = self.spider_class("example.com", foo="bar")- assert spider.foo == "bar"+ assert spider.foo == "bar" # type: ignore[attr-defined]def test_spider_without_name(self):"""``__init__`` raises when the name is not provided."""@@ -58,7 +58,7 @@ class TestSpiderBase(ABC):mock_init.assert_called_once_with("example.com", foo="bar")def test_closed_signal_call(self):- class TestSpider(self.spider_class):+ class TestSpider(self.spider_class): # type: ignore[name-defined,misc]closed_called = Falsedef closed(self, reason):@@ -92,7 +92,7 @@ class TestSpiderBase(ABC):**get_reactor_settings(),}- class TestSpider(self.spider_class):+ class TestSpider(self.spider_class): # type: ignore[name-defined,misc]name = "test"custom_settings = spider_settings@@ -127,5 +127,5 @@ class TestSpiderBase(ABC):ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"),):- spider.log("test log msg", "INFO")+ spider.log("test log msg", "INFO") # type: ignore[arg-type]mock_logger.log.assert_called_once_with("INFO", "test log msg")