Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Type spider-related test modules (#7968)

Adrian71aefab5f77c
diff --git a/pyproject.toml b/pyproject.toml
index 61c23fbec..13aae01ac 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -116,13 +116,9 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
- "tests.spiders",
- "tests.test_closespider",
"tests.test_cmdline",
- "tests.test_contracts",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
- "tests.test_engine_loop",
"tests.test_extension_statsmailer",
"tests.test_extension_throttle",
"tests.test_linkextractors",
@@ -131,15 +127,9 @@ module = [
"tests.test_pqueues",
"tests.test_scheduler_base",
"tests.test_settings",
- "tests.test_spider",
- "tests.test_spider_crawl",
- "tests.test_spidermiddleware_output_chain",
- "tests.test_spidermiddleware_process_start",
- "tests.test_spider_sitemap",
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
- "tests.utils.bases.spider",
]
check_untyped_defs = false
diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py
index a066ed00a..7504ddf18 100644
--- a/tests/benchmarks/__init__.py
+++ b/tests/benchmarks/__init__.py
@@ -36,7 +36,6 @@ class NullDownloadHandler:
async def download_request(self, request: Request) -> Response:
self._active += 1
- assert self._crawler.stats
self._crawler.stats.max_value("benchmark/peak_concurrency", self._active)
try:
await asyncio.sleep(0)
diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py
index f79b66a5b..4f68e4f6f 100644
--- a/tests/benchmarks/test_crawl.py
+++ b/tests/benchmarks/test_crawl.py
@@ -128,7 +128,6 @@ class _DelayedPipeline:
async def process_item(self, item: Any) -> Any:
url = item["url"]
self._active[url] += 1
- assert self._crawler.stats
self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
try:
await asyncio.sleep(DELAY)
@@ -147,7 +146,6 @@ def _crawl_tree(
pages=pages,
items=items,
)
- assert crawler.stats
assert crawler.stats.get_value("downloader/response_count") == domains * pages
assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
return crawler
@@ -166,7 +164,6 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> N
def run() -> None:
crawler = crawl(_FollowSpider, settings, url=url)
- assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
benchmark(run)
@@ -177,7 +174,6 @@ def test_overhead_engine(benchmark: BenchmarkFixture) -> None:
def run() -> None:
crawler = _crawl_tree({}, domains=1, pages=REQUESTS)
- assert crawler.stats
assert crawler.stats.get_value("benchmark/peak_concurrency") > 1
benchmark(run)
@@ -262,7 +258,6 @@ def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
crawler = _crawl_tree(
settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
)
- assert crawler.stats
assert (
crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
)
diff --git a/tests/spiders.py b/tests/spiders.py
index 7c7d3007c..3ff30ce03 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -13,22 +13,31 @@ from twisted.internet import defer
from scrapy import signals
from scrapy.exceptions import StopDownload
-from scrapy.http import Request
+from scrapy.http import Request, TextResponse
from scrapy.item import Item
from scrapy.linkextractors import LinkExtractor
+from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_from_asyncio_queue
if TYPE_CHECKING:
+ from collections.abc import AsyncIterator, Callable, Iterator
+
+ from twisted.python.failure import Failure
+ from typing_extensions import Self
+
+ from scrapy.crawler import Crawler
+ from scrapy.http import Headers, Response
+ from scrapy.http.request import CallbackT
from tests.mockserver.http import MockServer
class MockServerSpider(Spider):
def __init__(
self,
- *args,
+ *args: Any,
mockserver: MockServer | None = None,
is_secure: bool = False,
**kwargs: Any,
@@ -74,7 +83,7 @@ class MetaSpider(MockServerSpider):
super().__init__(*args, **kwargs)
self.meta: dict[str, Any] = {}
- def closed(self, reason):
+ def closed(self, reason: str) -> None:
self.meta["close_reason"] = reason
@@ -83,16 +92,24 @@ class FollowAllSpider(MetaSpider):
link_extractor = LinkExtractor()
def __init__(
- self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs
+ self,
+ total: int = 10,
+ show: int = 20,
+ order: str = "rand",
+ maxlatency: float = 0.0,
+ *args: Any,
+ **kwargs: Any,
):
super().__init__(*args, **kwargs)
- self.urls_visited = []
- self.times = []
+ self.urls_visited: list[str] = []
+ self.times: list[float] = []
qargs = {"total": total, "show": show, "order": order, "maxlatency": maxlatency}
+ assert self.mockserver
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
self.start_urls = [url]
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
+ assert isinstance(response, TextResponse)
self.urls_visited.append(response.url)
self.times.append(time.time())
for link in self.link_extractor.extract_links(response):
@@ -102,50 +119,52 @@ class FollowAllSpider(MetaSpider):
class DelaySpider(MetaSpider):
name = "delay"
- def __init__(self, n=1, b=0, *args, **kwargs):
+ def __init__(self, n: float = 1, b: float = 0, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.n = n
self.b = b
- self.t1 = self.t2 = self.t2_err = 0
+ self.t1 = self.t2 = self.t2_err = 0.0
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
self.t1 = time.time()
+ assert self.mockserver
url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback)
- def parse(self, response):
+ def parse(self, response: Response) -> Any:
self.t2 = time.time()
- def errback(self, failure):
+ def errback(self, failure: Failure) -> None:
self.t2_err = time.time()
class LogSpider(MetaSpider):
name = "log_spider"
- def log_debug(self, message: str, extra: dict[str, Any] | None = None):
+ def log_debug(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.debug(message, extra=extra)
- def log_info(self, message: str, extra: dict[str, Any] | None = None):
+ def log_info(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.info(message, extra=extra)
- def log_warning(self, message: str, extra: dict[str, Any] | None = None):
+ def log_warning(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.warning(message, extra=extra)
- def log_error(self, message: str, extra: dict[str, Any] | None = None):
+ def log_error(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.error(message, extra=extra)
- def log_critical(self, message: str, extra: dict[str, Any] | None = None):
+ def log_critical(self, message: str, extra: dict[str, Any] | None = None) -> None:
self.logger.critical(message, extra=extra)
- def parse(self, response):
+ def parse(self, response: Response) -> None:
pass
class SlowSpider(DelaySpider):
name = "slow"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
# 1st response is fast
url = self.mockserver.url("/delay?n=0&b=0")
yield Request(url, callback=self.parse, errback=self.errback)
@@ -154,25 +173,25 @@ class SlowSpider(DelaySpider):
url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
yield Request(url, callback=self.parse, errback=self.errback)
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
yield Item()
class SimpleSpider(MetaSpider):
name = "simple"
- def __init__(self, url="http://localhost:8998", *args, **kwargs):
+ def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.start_urls = [url]
- def parse(self, response):
+ def parse(self, response: Response) -> Any:
self.logger.info(f"Got response {response.status}")
class AsyncDefSpider(SimpleSpider):
name = "asyncdef"
- async def parse(self, response):
+ async def parse(self, response: Response) -> None:
await defer.succeed(42)
self.logger.info(f"Got response {response.status}")
@@ -180,7 +199,7 @@ class AsyncDefSpider(SimpleSpider):
class AsyncDefAsyncioSpider(SimpleSpider):
name = "asyncdef_asyncio"
- async def parse(self, response):
+ async def parse(self, response: Response) -> None:
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@@ -189,7 +208,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):
class AsyncDefAsyncioReturnSpider(SimpleSpider):
name = "asyncdef_asyncio_return"
- async def parse(self, response):
+ async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@@ -199,7 +218,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
name = "asyncdef_asyncio_return_single_element"
- async def parse(self, response):
+ async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.1)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {status}")
@@ -209,7 +228,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
name = "asyncdef_asyncio_reqs_return"
- async def parse(self, response):
+ async def parse(self, response: Response) -> Any:
await asyncio.sleep(0.2)
req_id = response.meta.get("req_id", 0)
status = await get_from_asyncio_queue(response.status)
@@ -226,7 +245,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
class AsyncDefAsyncioGenExcSpider(SimpleSpider):
name = "asyncdef_asyncio_gen_exc"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
for i in range(10):
await asyncio.sleep(0.1)
yield {"foo": i}
@@ -237,7 +256,7 @@ class AsyncDefAsyncioGenExcSpider(SimpleSpider):
class AsyncDefDeferredDirectSpider(SimpleSpider):
name = "asyncdef_deferred_direct"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
await defer.succeed(None)
yield {"code": 200}
@@ -245,7 +264,7 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):
class AsyncDefDeferredWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_wrapped"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
await deferred_to_future(defer.succeed(None))
yield {"code": 200}
@@ -253,7 +272,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):
class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
name = "asyncdef_deferred_maybe_wrapped"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
await maybe_deferred_to_future(defer.succeed(None))
yield {"code": 200}
@@ -261,7 +280,7 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
class AsyncDefAsyncioGenSpider(SimpleSpider):
name = "asyncdef_asyncio_gen"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
await asyncio.sleep(0.2)
yield {"foo": 42}
self.logger.info(f"Got response {response.status}")
@@ -270,7 +289,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
name = "asyncdef_asyncio_gen_loop"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
for i in range(10):
await asyncio.sleep(0.1)
yield {"foo": i}
@@ -283,7 +302,8 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
following_reqs = 3
depth = 2
- def _get_req(self, index, cb=None):
+ def _get_req(self, index: int, cb: CallbackT | None = None) -> Request:
+ assert self.mockserver
return Request(
self.mockserver.url(f"/status?n=200&request={index}"),
meta={"index": index},
@@ -291,11 +311,11 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
callback=cb,
)
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
for i in range(1, self.initial_reqs + 1):
yield self._get_req(i)
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]:
index = response.meta["index"]
yield {"index": index}
if index < 10**self.depth:
@@ -305,7 +325,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
await asyncio.sleep(0.1)
yield {"index": index + 5}
- async def parse2(self, response):
+ async def parse2(self, response: Response) -> AsyncIterator[Any]:
await asyncio.sleep(0.1)
yield {"index2": response.meta["index"]}
@@ -313,7 +333,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
class ItemSpider(FollowAllSpider):
name = "item"
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
for request in super().parse(response):
yield request
yield Item()
@@ -321,12 +341,18 @@ class ItemSpider(FollowAllSpider):
class MaxItemsAndRequestsSpider(FollowAllSpider):
- def __init__(self, max_items=10, max_requests=10, *args, **kwargs):
+ def __init__(
+ self,
+ max_items: int = 10,
+ max_requests: int = 10,
+ *args: Any,
+ **kwargs: Any,
+ ):
super().__init__(*args, **kwargs)
self.max_items = max_items
self.max_requests = max_requests
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
self.items_scraped = 0
self.pages_crawled = 1 # account for the start url
for request in super().parse(response):
@@ -344,12 +370,12 @@ class DefaultError(Exception):
class ErrorSpider(FollowAllSpider):
name = "error"
- exception_cls = DefaultError
+ exception_cls: type[Exception] = DefaultError
- def raise_exception(self):
+ def raise_exception(self) -> None:
raise self.exception_cls("Expected exception")
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
for request in super().parse(response):
yield request
self.raise_exception()
@@ -359,14 +385,15 @@ class BrokenStartSpider(FollowAllSpider):
fail_before_yield = False
fail_yielding = False
- def __init__(self, *a, **kw):
+ def __init__(self, *a: Any, **kw: Any):
super().__init__(*a, **kw)
- self.seedsseen = []
+ self.seedsseen: list[Any] = []
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
if self.fail_before_yield:
1 / 0
+ assert self.mockserver
for s in range(100):
qargs = {"total": 10, "seed": s}
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
@@ -376,18 +403,18 @@ class BrokenStartSpider(FollowAllSpider):
assert self.seedsseen, "All seeds consumed before any download happened"
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
self.seedsseen.append(response.meta.get("seed"))
yield from super().parse(response)
class StartItemSpider(FollowAllSpider):
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
yield {"name": "test item"}
class StartGoodAndBadOutput(FollowAllSpider):
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
yield {"a": "a"}
yield Request("data:,a")
yield "data:,b"
@@ -395,17 +422,18 @@ class StartGoodAndBadOutput(FollowAllSpider):
class SingleRequestSpider(MetaSpider):
- seed = None
- callback_func = None
- errback_func = None
+ seed: Request | str | None = None
+ callback_func: Callable[[Response], Any] | None = None
+ errback_func: Callable[[Failure], Any] | None = None
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
if isinstance(self.seed, Request):
yield self.seed.replace(callback=self.parse, errback=self.on_error)
else:
+ assert self.seed
yield Request(self.seed, callback=self.parse, errback=self.on_error)
- def parse(self, response):
+ def parse(self, response: Response) -> Any:
self.meta.setdefault("responses", []).append(response)
if callable(self.callback_func):
return self.callback_func(response)
@@ -413,7 +441,7 @@ class SingleRequestSpider(MetaSpider):
return response.meta["next"]
return None
- def on_error(self, failure):
+ def on_error(self, failure: Failure) -> Any:
self.meta["failure"] = failure
if callable(self.errback_func):
return self.errback_func(failure)
@@ -426,17 +454,18 @@ class DuplicateStartSpider(MockServerSpider):
distinct_urls = 2
dupe_factor = 3
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
for i in range(self.distinct_urls):
for _ in range(self.dupe_factor):
url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
yield Request(url, dont_filter=self.dont_filter)
- def __init__(self, url="http://localhost:8998", *args, **kwargs):
+ def __init__(self, url: str = "http://localhost:8998", *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.visited = 0
- def parse(self, response):
+ def parse(self, response: Response) -> None:
self.visited += 1
@@ -451,7 +480,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
}
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@@ -461,11 +490,13 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
</body>
</html>
"""
+ assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
- def parse(self, response, foo=None):
+ def parse(self, response: Response, foo: str | None = None) -> Iterator[Any]:
self.logger.info("[parse] status %i (foo: %s)", response.status, foo)
+ assert self.mockserver
yield Request(
self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}
)
@@ -477,8 +508,9 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_async_callback"
rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),)
- async def parse_async(self, response, foo=None):
+ async def parse_async(self, response: Response, foo: str | None = None) -> Request:
self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo)
+ assert self.mockserver
return Request(
self.mockserver.url("/status?n=202"),
self.parse_async,
@@ -492,8 +524,11 @@ class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_async_generator_callback"
rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),)
- async def parse_async_gen(self, response, foo=None):
+ async def parse_async_gen(
+ self, response: Response, foo: str | None = None
+ ) -> AsyncIterator[Any]:
self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo)
+ assert self.mockserver
yield Request(
self.mockserver.url("/status?n=202"),
self.parse_async_gen,
@@ -505,7 +540,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_errback"
rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),)
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@@ -518,17 +553,19 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
</body>
</html>
"""
+ assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
- def errback(self, failure):
+ def errback(self, failure: Failure) -> None:
+ assert isinstance(failure.value, HttpError)
self.logger.info("[errback] status %i", failure.value.response.status)
class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
name = "crawl_spider_without_errback"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
test_body = b"""
<html>
<head><title>Page title</title></head>
@@ -538,6 +575,7 @@ class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
</body>
</html>
"""
+ assert self.mockserver
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
@@ -553,7 +591,7 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse
),
)
- def process_request(self, request, response):
+ def process_request(self, request: Request, response: Response) -> Request:
request.cb_kwargs["foo"] = "process_request"
return request
@@ -562,59 +600,65 @@ class BytesReceivedCallbackSpider(MetaSpider):
full_response_length = 2**18
@classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.bytes_received, signals.bytes_received)
return spider
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
body = b"a" * self.full_response_length
+ assert self.mockserver
url = self.mockserver.url("/alpayload", is_secure=self.is_secure)
yield Request(url, method="POST", body=body, errback=self.errback)
- def parse(self, response):
+ def parse(self, response: Response) -> None:
self.meta["response"] = response
- def errback(self, failure):
+ def errback(self, failure: Failure) -> None:
self.meta["failure"] = failure
- def bytes_received(self, data, request, spider):
+ def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.meta["bytes_received"] = data
raise StopDownload(fail=False)
class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
- def bytes_received(self, data, request, spider):
+ def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.meta["bytes_received"] = data
raise StopDownload(fail=True)
class HeadersReceivedCallbackSpider(MetaSpider):
@classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.headers_received, signals.headers_received)
return spider
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(
self.mockserver.url("/status", is_secure=self.is_secure),
errback=self.errback,
)
- def parse(self, response):
+ def parse(self, response: Response) -> None:
self.meta["response"] = response
- def errback(self, failure):
+ def errback(self, failure: Failure) -> None:
self.meta["failure"] = failure
- def headers_received(self, headers, body_length, request, spider):
+ def headers_received(
+ self, headers: Headers, body_length: int, request: Request, spider: Spider
+ ) -> None:
self.meta["headers_received"] = headers
raise StopDownload(fail=False)
class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
- def headers_received(self, headers, body_length, request, spider):
+ def headers_received(
+ self, headers: Headers, body_length: int, request: Request, spider: Spider
+ ) -> None:
self.meta["headers_received"] = headers
raise StopDownload(fail=True)
@@ -623,13 +667,13 @@ class ExceptionSpider(Spider):
name = "exception"
@classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
raise ValueError("Exception in from_crawler method")
class NoRequestsSpider(Spider):
name = "no_request"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
return
yield
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 9a7447e16..81c122b81 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -11,6 +11,8 @@ from tests.utils.decorators import inline_callbacks_test
class TestCloseSpider:
+ mockserver: MockServer
+
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
@@ -25,6 +27,7 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
+ assert isinstance(crawler.spider, ItemSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_itemcount"
itemcount = crawler.stats.get_value("item_scraped_count")
@@ -35,6 +38,7 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
yield crawler.crawl(mockserver=self.mockserver)
+ assert isinstance(crawler.spider, FollowAllSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount"
pagecount = crawler.stats.get_value("response_received_count")
@@ -54,6 +58,7 @@ class TestCloseSpider:
yield crawler.crawl(
max_items=max_items, max_requests=max_requests, mockserver=self.mockserver
)
+ assert isinstance(crawler.spider, MaxItemsAndRequestsSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@@ -72,6 +77,7 @@ class TestCloseSpider:
},
)
yield crawler.crawl(mockserver=self.mockserver)
+ assert isinstance(crawler.spider, FollowAllSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_pagecount_no_item"
pagecount = crawler.stats.get_value("response_received_count")
@@ -82,6 +88,7 @@ class TestCloseSpider:
close_on = 5
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
+ assert isinstance(crawler.spider, ErrorSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_errorcount"
key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
@@ -94,6 +101,7 @@ class TestCloseSpider:
close_on = 0.1
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
+ assert isinstance(crawler.spider, FollowAllSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
@@ -104,6 +112,7 @@ class TestCloseSpider:
timeout = 1
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
yield crawler.crawl(n=3, mockserver=self.mockserver)
+ assert isinstance(crawler.spider, SlowSpider)
reason = crawler.spider.meta["close_reason"]
assert reason == "closespider_timeout_no_item"
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 11036eaeb..73a85dd5d 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -16,6 +16,11 @@ if TYPE_CHECKING:
from pathlib import Path
+# Header for spider classes copied with inspect.getsource(): the future import
+# keeps their annotations from being evaluated in the generated module.
+SPIDER_HEADER = "from __future__ import annotations\nfrom scrapy import Spider\n"
+
+
class TestRunSpiderCommand:
spider_filename = "myspider.py"
@@ -65,14 +70,14 @@ class BadSpider(scrapy.Spider):
def test_run_fail_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
- tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
+ tmp_path, SPIDER_HEADER + inspect.getsource(ExceptionSpider)
)
assert ret != 0
def test_run_good_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path,
- "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
+ SPIDER_HEADER + inspect.getsource(NoRequestsSpider),
)
assert ret == 0
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 698044de7..44b621776 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
from unittest import TextTestResult
import pytest
@@ -11,7 +14,7 @@ from scrapy.contracts.default import (
ScrapesContract,
UrlContract,
)
-from scrapy.http import Request
+from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
@@ -19,6 +22,24 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inline_callbacks_test
+if TYPE_CHECKING:
+ from unittest import TestResult
+
+ from scrapy.http.request import CallbackT
+
+
+def _request(
+ conman: ContractsManager, method: CallbackT, results: TestResult
+) -> Request:
+ request = conman.from_method(method, results)
+ assert request
+ return request
+
+
+def _call(request: Request, response: Any) -> Any:
+ assert request.callback
+ return request.callback(response, **request.cb_kwargs)
+
class DemoItem(Item):
name = Field()
@@ -30,7 +51,7 @@ class ResponseMock:
class ResponseMetaMock(ResponseMock):
- meta = None
+ meta: Any = None
class TaggedRequest(Request):
@@ -289,17 +310,19 @@ class TestContractsManager:
def setup_method(self):
self.conman = ContractsManager(self.contracts)
- self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
+ self.results = TextTestResult( # type: ignore[type-var]
+ stream=None, descriptions=False, verbosity=0
+ )
- def should_succeed(self):
+ def should_succeed(self) -> None:
assert not self.results.failures
assert not self.results.errors
- def should_fail(self):
+ def should_fail(self) -> None:
assert self.results.failures
assert not self.results.errors
- def should_error(self):
+ def should_error(self) -> None:
assert self.results.errors
def test_contracts(self):
@@ -313,12 +336,10 @@ class TestContractsManager:
)
# returns request for valid method
- request = self.conman.from_method(spider.returns_request, self.results)
- assert request is not None
+ assert self.conman.from_method(spider.returns_request, self.results) is not None
# no request for missing url
- request = self.conman.from_method(spider.parse_no_url, self.results)
- assert request is None
+ assert self.conman.from_method(spider.parse_no_url, self.results) is None
def test_cb_kwargs(self):
spider = DemoSpider()
@@ -354,29 +375,31 @@ class TestContractsManager:
)
# returns_request
- request = self.conman.from_method(
- spider.returns_request_cb_kwargs, self.results
- )
- request.callback(response, **request.cb_kwargs)
+ request = _request(self.conman, spider.returns_request_cb_kwargs, self.results)
+ _call(request, response)
self.should_succeed()
# returns_item
- request = self.conman.from_method(spider.returns_item_cb_kwargs, self.results)
- request.callback(response, **request.cb_kwargs)
+ request = _request(self.conman, spider.returns_item_cb_kwargs, self.results)
+ _call(request, response)
self.should_succeed()
# returns_item (error, callback doesn't take keyword arguments)
- request = self.conman.from_method(
- spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results
+ request = _request(
+ self.conman,
+ spider.returns_item_cb_kwargs_error_unexpected_keyword,
+ self.results,
)
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
self.should_error()
# returns_item (error, contract doesn't provide keyword arguments)
- request = self.conman.from_method(
- spider.returns_item_cb_kwargs_error_missing_argument, self.results
+ request = _request(
+ self.conman,
+ spider.returns_item_cb_kwargs_error_missing_argument,
+ self.results,
)
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
self.should_error()
def test_meta(self):
@@ -398,29 +421,27 @@ class TestContractsManager:
response = ResponseMetaMock()
# returns_request
- request = self.conman.from_method(spider.returns_request_meta, self.results)
+ request = _request(self.conman, spider.returns_request_meta, self.results)
assert request.meta["cookiejar"] == "session1"
response.meta = request.meta
- request.callback(response)
+ _call(request, response)
assert response.meta["cookiejar"] == "session1"
self.should_succeed()
response = ResponseMetaMock()
# returns_item
- request = self.conman.from_method(spider.returns_item_meta, self.results)
+ request = _request(self.conman, spider.returns_item_meta, self.results)
assert request.meta["key"] == "example"
response.meta = request.meta
- request.callback(response)
+ _call(request, response)
assert response.meta["key"] == "example"
self.should_succeed()
response = ResponseMetaMock()
- request = self.conman.from_method(
- spider.returns_error_missing_meta, self.results
- )
- request.callback(response)
+ request = _request(self.conman, spider.returns_error_missing_meta, self.results)
+ _call(request, response)
self.should_error()
def test_returns(self):
@@ -428,36 +449,36 @@ class TestContractsManager:
response = ResponseMock()
# returns_item
- request = self.conman.from_method(spider.returns_item, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.returns_item, self.results)
+ _call(request, response)
self.should_succeed()
# returns_dict_item
- request = self.conman.from_method(spider.returns_dict_item, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.returns_dict_item, self.results)
+ _call(request, response)
self.should_succeed()
# returns_request
- request = self.conman.from_method(spider.returns_request, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.returns_request, self.results)
+ _call(request, response)
self.should_succeed()
# returns_fail
- request = self.conman.from_method(spider.returns_fail, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.returns_fail, self.results)
+ _call(request, response)
self.should_fail()
# returns_dict_fail
- request = self.conman.from_method(spider.returns_dict_fail, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.returns_dict_fail, self.results)
+ _call(request, response)
self.should_fail()
def test_returns_async(self):
spider = DemoSpider()
response = ResponseMock()
- request = self.conman.from_method(spider.returns_request_async, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.returns_request_async, self.results)
+ _call(request, response)
self.should_error()
def test_returns_invalid_argument_count(self):
@@ -477,10 +498,8 @@ class TestContractsManager:
spider = DemoSpider()
response = ResponseMock()
- request = self.conman.from_method(
- spider.returns_request_range_fail, self.results
- )
- request.callback(response)
+ request = _request(self.conman, spider.returns_request_range_fail, self.results)
+ _call(request, response)
self.should_fail()
assert "expected 2..3" in self.results.failures[-1][-1]
@@ -490,8 +509,8 @@ class TestContractsManager:
# @returns and @scrapes only count matching output objects and skip
# the request that is also yielded.
- request = self.conman.from_method(spider.yields_item_and_request, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.yields_item_and_request, self.results)
+ _call(request, response)
self.should_succeed()
def test_testcase_str(self):
@@ -507,30 +526,30 @@ class TestContractsManager:
response = ResponseMock()
# scrapes_item_ok
- request = self.conman.from_method(spider.scrapes_item_ok, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.scrapes_item_ok, self.results)
+ _call(request, response)
self.should_succeed()
# scrapes_dict_item_ok
- request = self.conman.from_method(spider.scrapes_dict_item_ok, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.scrapes_dict_item_ok, self.results)
+ _call(request, response)
self.should_succeed()
# scrapes_item_fail
- request = self.conman.from_method(spider.scrapes_item_fail, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.scrapes_item_fail, self.results)
+ _call(request, response)
self.should_fail()
# scrapes_dict_item_fail
- request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)
- request.callback(response)
+ request = _request(self.conman, spider.scrapes_dict_item_fail, self.results)
+ _call(request, response)
self.should_fail()
# scrapes_multiple_missing_fields
- request = self.conman.from_method(
- spider.scrapes_multiple_missing_fields, self.results
+ request = _request(
+ self.conman, spider.scrapes_multiple_missing_fields, self.results
)
- request.callback(response)
+ _call(request, response)
self.should_fail()
message = "ContractFail: Missing fields: name, url"
assert message in self.results.failures[-1][-1]
@@ -540,14 +559,13 @@ class TestContractsManager:
response = ResponseMock()
# invalid regex
- request = self.conman.from_method(spider.invalid_regex, self.results)
- assert request is None
+ assert self.conman.from_method(spider.invalid_regex, self.results) is None
# invalid regex with valid contract
- request = self.conman.from_method(
- spider.invalid_regex_with_valid_contract, self.results
+ request = _request(
+ self.conman, spider.invalid_regex_with_valid_contract, self.results
)
- request.callback(response)
+ _call(request, response)
self.should_succeed()
def test_custom_contracts(self):
@@ -559,14 +577,14 @@ class TestContractsManager:
def test_errback(self):
spider = DemoSpider()
- response = ResponseMock()
try:
- raise HttpError(response, "Ignoring non-200 response")
+ raise HttpError(Response("http://scrapy.org"), "Ignoring non-200 response")
except HttpError:
failure_mock = failure.Failure()
- request = self.conman.from_method(spider.returns_request, self.results)
+ request = _request(self.conman, spider.returns_request, self.results)
+ assert request.errback
request.errback(failure_mock)
assert not self.results.failures
@@ -602,11 +620,12 @@ class TestContractsManager:
crawler = get_crawler(TestSameUrlSpider)
yield crawler.crawl()
+ assert isinstance(crawler.spider, TestSameUrlSpider)
assert crawler.spider.visited == 2
def test_custom_tagged_request_contract(self):
spider = DemoSpider()
- request = self.conman.from_method(spider.custom_tagged_request, self.results)
+ request = _request(self.conman, spider.custom_tagged_request, self.results)
assert request.method == "POST"
assert isinstance(request, TaggedRequest)
assert request.contract_tag == "custom"
@@ -672,18 +691,20 @@ class PostProcessErrorContract(Contract):
class TestCustomContractPrePostProcess:
def setup_method(self):
- self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)
+ self.results = TextTestResult( # type: ignore[type-var]
+ stream=None, descriptions=False, verbosity=0
+ )
def test_pre_hook_keyboard_interrupt(self):
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPreProcess(spider.returns_request)
- conman = ContractsManager([UrlContract, ReturnsContract, contract])
+ conman = ContractsManager([UrlContract, ReturnsContract])
- request = conman.from_method(spider.returns_request, self.results)
+ request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
with pytest.raises(KeyboardInterrupt, match="Pre-process exception"):
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
assert not self.results.failures
assert not self.results.errors
@@ -692,12 +713,12 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPostProcess(spider.returns_request)
- conman = ContractsManager([UrlContract, ReturnsContract, contract])
+ conman = ContractsManager([UrlContract, ReturnsContract])
- request = conman.from_method(spider.returns_request, self.results)
+ request = _request(conman, spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
with pytest.raises(KeyboardInterrupt, match="Post-process exception"):
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
assert not self.results.failures
assert not self.results.errors
@@ -706,11 +727,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessSuccessContract(spider.returns_request)
- conman = ContractsManager([UrlContract, ReturnsContract, contract])
+ conman = ContractsManager([UrlContract, ReturnsContract])
- request = conman.from_method(spider.returns_request, self.results)
+ request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
assert not self.results.failures
assert not self.results.errors
@@ -719,11 +740,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessAssertionFailContract(spider.returns_request)
- conman = ContractsManager([UrlContract, ReturnsContract, contract])
+ conman = ContractsManager([UrlContract, ReturnsContract])
- request = conman.from_method(spider.returns_request, self.results)
+ request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
assert self.results.failures
assert not self.results.errors
@@ -732,11 +753,11 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = PreProcessErrorContract(spider.returns_request)
- conman = ContractsManager([UrlContract, ReturnsContract, contract])
+ conman = ContractsManager([UrlContract, ReturnsContract])
- request = conman.from_method(spider.returns_request, self.results)
+ request = _request(conman, spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
assert self.results.errors
@@ -748,7 +769,7 @@ class TestCustomContractPrePostProcess:
contract.add_pre_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
- request.callback(response)
+ _call(request, response)
def test_pre_hook_async_generator(self):
spider = DemoSpider()
@@ -758,7 +779,7 @@ class TestCustomContractPrePostProcess:
contract.add_pre_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
- request.callback(response)
+ _call(request, response)
def test_post_hook_async_generator(self):
spider = DemoSpider()
@@ -768,16 +789,16 @@ class TestCustomContractPrePostProcess:
contract.add_post_hook(request, self.results)
with pytest.raises(TypeError, match="async callbacks"):
- request.callback(response)
+ _call(request, response)
def test_post_hook_error(self):
spider = DemoSpider()
response = ResponseMock()
contract = PostProcessErrorContract(spider.returns_request)
- conman = ContractsManager([UrlContract, ReturnsContract, contract])
+ conman = ContractsManager([UrlContract, ReturnsContract])
- request = conman.from_method(spider.returns_request, self.results)
+ request = _request(conman, spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
- request.callback(response, **request.cb_kwargs)
+ _call(request, response)
assert self.results.errors
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 862714167..ca6d0ef93 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -364,7 +364,6 @@ with multiples lines
est: list[list[tuple[str, Any]]] = []
def cb(response):
- assert crawler.engine
est.append(get_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@@ -399,7 +398,6 @@ with multiples lines
est: list[str] = []
def cb(response):
- assert crawler.engine
est.append(format_engine_status(crawler.engine))
crawler = get_crawler(SingleRequestSpider)
@@ -474,7 +472,6 @@ class TestCrawlSpider:
await crawler.crawl_async(
mockserver.url("/status?n=200"), mockserver=mockserver
)
- assert crawler.stats
return items, crawler.stats
@coroutine_test
@@ -540,7 +537,6 @@ class TestCrawlSpider:
# so the failure is dropped silently and the crawl finishes normally.
assert "[parse] status 200 (foo: None)" in caplog.text
assert "[errback]" not in caplog.text
- assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/404") == 1
@coroutine_test
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 17ca02dea..2836a6627 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -536,7 +536,6 @@ class TestSpiderSettings:
custom_settings = {"AUTOTHROTTLE_ENABLED": True}
crawler = get_crawler(MySpider)
- assert crawler.extensions
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
assert AutoThrottle in enabled_exts
@@ -568,7 +567,6 @@ class TestCrawlerLogging:
}
async def start(self):
- assert crawler.stats
info_count_start = crawler.stats.get_value("log_count/INFO")
logging.debug("debug message") # noqa: LOG015
logging.info("info message") # noqa: LOG015
@@ -601,7 +599,6 @@ class TestCrawlerLogging:
assert "info message" in logged
assert "warning message" in logged
assert "error message" in logged
- assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
assert crawler.stats.get_value("log_count/WARNING") == 1
assert info_count == 1
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 6eabe91ad..75affe343 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -72,7 +72,6 @@ class TestBase:
settings = self._get_settings(**new_settings)
crawler = get_crawler(Spider, settings)
crawler.spider = crawler._create_spider("example.com")
- assert crawler.stats
crawler.stats.open_spider()
try:
yield crawler
@@ -137,7 +136,6 @@ class StorageTestMixin(TestBase):
with self._middleware() as mw:
spider = mw.crawler.spider
assert spider
- assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@@ -159,7 +157,6 @@ class StorageTestMixin(TestBase):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
assert spider
- assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 9290f55ae..131b9c4b7 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -63,7 +63,6 @@ class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler)
- assert self.crawler.stats
self.crawler.stats.open_spider()
def _getresponse(self, coding: str) -> Response:
@@ -89,7 +88,6 @@ class TestHttpCompression:
return response
def assertStatsEqual(self, key: str, value: Any) -> None:
- assert self.crawler.stats
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index 40e09be13..575051526 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -230,7 +230,6 @@ def test_repeated_offsite_domain():
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
- assert crawler.stats
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 1c8a61aa5..7b684b678 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -82,7 +82,6 @@ class TestRetry:
# discard it
assert self.mw.process_response(req3, rsp) is rsp
- assert self.crawler.stats
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
@@ -134,7 +133,6 @@ class TestRetry:
self._test_retry_exception(req, exc("foo"))
stats = self.crawler.stats
- assert stats
assert stats.get_value("retry/max_reached") == len(exceptions)
assert stats.get_value("retry/count") == len(exceptions) * 2
assert (
@@ -315,7 +313,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -1
expected_reason = "unspecified"
- assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@@ -336,7 +333,6 @@ class TestGetRetryRequest:
max_retry_times=max_retry_times,
)
assert new_request is None
- assert spider.crawler.stats
assert spider.crawler.stats.get_value("retry/max_reached") == 1
failure_count = max_retry_times + 1
expected_reason = "unspecified"
@@ -363,7 +359,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -1
expected_reason = "unspecified"
- assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@@ -396,7 +391,6 @@ class TestGetRetryRequest:
assert new_request.meta["retry_times"] == expected_retry_times
assert new_request.priority == -expected_retry_times
expected_reason = "unspecified"
- assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
value = spider.crawler.stats.get_value(stat)
assert value == expected_retry_times
@@ -521,7 +515,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
- assert spider.crawler.stats
for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
assert spider.crawler.stats.get_value(stat) == 1
assert (
@@ -543,7 +536,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
- assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@@ -569,7 +561,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
- assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@@ -593,7 +584,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
- assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@@ -619,7 +609,6 @@ class TestGetRetryRequest:
reason=expected_reason,
)
expected_retry_times = 1
- assert spider.crawler.stats
stat = spider.crawler.stats.get_value(
f"retry/reason_count/{expected_reason_string}"
)
@@ -737,7 +726,6 @@ class TestGetRetryRequest:
reason=expected_reason,
stats_base_key=stats_key,
)
- assert spider.crawler.stats
for stat in (
f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}",
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 13f6d6fe3..01014ddd6 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -17,7 +17,6 @@ class MyException(Exception):
class TestDownloaderStats:
def setup_method(self) -> None:
self.crawler = get_crawler(Spider)
- assert self.crawler.stats is not None
self.mw = build_from_crawler(DownloaderStats, self.crawler)
self.crawler.stats.open_spider()
@@ -26,7 +25,6 @@ class TestDownloaderStats:
self.res = Response("http://scrapytest.org", status=400)
def assertStatsEqual(self, key: str, value: object) -> None:
- assert self.crawler.stats is not None
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
@@ -53,7 +51,6 @@ class TestDownloaderStats:
build_from_crawler(DownloaderStats, crawler)
def teardown_method(self) -> None:
- assert self.crawler.stats is not None
self.crawler.stats.close_spider()
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 6f323a15c..077a92ecf 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -148,7 +148,6 @@ async def test_none_slot_with_priority_queue(
assert crawler.spider.default_slot in crawler.spider.times
assert len(crawler.spider.times[crawler.spider.default_slot]) == 2
- assert crawler.stats
stats = crawler.stats
assert stats.get_value("spider_exceptions", 0) == 0
assert stats.get_value("downloader/exception_count", 0) == 0
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 348e778cd..4b984406a 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -179,7 +179,6 @@ class TestRFPDupeFilter:
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
- assert crawler.stats
assert crawler.stats.get_value("dupefilter/filtered") == 2
assert (
"scrapy.dupefilters",
@@ -213,7 +212,6 @@ class TestRFPDupeFilter:
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
- assert crawler.stats
assert crawler.stats.get_value("dupefilter/filtered") == 2
assert (
"scrapy.dupefilters",
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 8e5197df7..5c3c92d3b 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -15,7 +15,7 @@ from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
- from collections.abc import Iterator
+ from collections.abc import AsyncIterator, Callable, Iterator
from scrapy.http import Response
@@ -85,19 +85,23 @@ class TestMain:
class TestSpider(Spider):
name = "test"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.crawler.engine._slot
+ scheduler = self.crawler.engine._slot.scheduler
+ assert isinstance(scheduler, MemoryScheduler)
+
yield Request("data:,a")
await sleep(seconds)
- self.crawler.engine._slot.scheduler.pause()
- self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
+ scheduler.pause()
+ scheduler.enqueue_request(Request("data:,b"))
# During this time, the scheduler reports having requests but
# returns None.
await sleep(seconds)
- self.crawler.engine._slot.scheduler.unpause()
+ scheduler.unpause()
# The scheduler request is processed.
await sleep(seconds)
@@ -106,21 +110,21 @@ class TestMain:
await sleep(seconds)
- self.crawler.engine._slot.scheduler.pause()
- self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))
+ scheduler.pause()
+ scheduler.enqueue_request(Request("data:,d"))
# The last start request is processed during the time until the
# delayed call below, proving that the start iteration can
# finish before a scheduler “sleep” without causing the
# scheduler to finish.
- call_later(seconds, self.crawler.engine._slot.scheduler.unpause)
+ call_later(seconds, scheduler.unpause)
- def parse(self, response):
+ def parse(self, response: Response) -> None:
pass
actual_urls = []
- def track_url(request, spider):
+ def track_url(request: Request, spider: Spider) -> None:
actual_urls.append(request.url)
settings = {"SCHEDULER": MemoryScheduler}
@@ -138,17 +142,16 @@ class TestMain:
class TestSpider(Spider):
name = "test"
- async def start(self):
- assert self.crawler.engine is not None
+ async def start(self) -> AsyncIterator[Any]:
await self.crawler.engine.close_async()
yield Request("data:,a")
- def parse(self, response):
+ def parse(self, response: Response) -> None:
pass
actual_urls = []
- def track_url(request, spider):
+ def track_url(request: Request, spider: Spider) -> None:
actual_urls.append(request.url)
settings = {"SCHEDULER": MemoryScheduler}
@@ -160,7 +163,6 @@ class TestMain:
await crawler.crawl_async()
assert not caplog.records
- assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "shutdown"
assert not actual_urls
@@ -229,7 +231,6 @@ class TestMain:
failure, response = errors[0]
assert isinstance(failure.value, ValueError)
assert response is None
- assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "start_error"
assert crawler.stats.get_value("spider_exceptions/count") == 1
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
@@ -256,12 +257,13 @@ class TestMain:
await crawler.crawl_async()
assert not caplog.records
- assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "my_reason"
assert crawler.stats.get_value("spider_exceptions/count") is None
class TestRequestSendOrder:
+ mockserver: MockServer
+
seconds = 0.1 # increase if flaky
@classmethod
@@ -273,52 +275,65 @@ class TestRequestSendOrder:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
- def request(self, num, response_seconds, download_slots, priority=0):
+ def request(
+ self,
+ num: int,
+ response_seconds: float,
+ download_slots: int,
+ priority: int = 0,
+ ) -> Request:
url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")
meta = {"download_slot": str(num % download_slots)}
return Request(url, meta=meta, priority=priority)
- def get_num(self, request_or_response: Request | Response):
+ def get_num(self, request_or_response: Request | Response) -> int:
return int(request_or_response.url.rsplit("&", maxsplit=1)[1])
async def _test_request_order(
self,
- start_nums,
- cb_nums=None,
- settings=None,
- response_seconds=None,
- download_slots=1,
- start_fn=None,
- parse_fn=None,
- ):
+ start_nums: list[int],
+ cb_nums: list[int] | None = None,
+ settings: dict[str, Any] | None = None,
+ response_seconds: float | None = None,
+ download_slots: int = 1,
+ start_fn: Callable[[Spider], AsyncIterator[Any]] | None = None,
+ parse_fn: Callable[..., Iterator[Any]] | None = None,
+ ) -> None:
cb_nums = cb_nums or []
settings = settings or {}
- response_seconds = response_seconds or self.seconds
+ seconds = response_seconds or self.seconds
cb_requests = deque(
- [self.request(num, response_seconds, download_slots) for num in cb_nums]
+ [self.request(num, seconds, download_slots) for num in cb_nums]
)
if start_fn is None:
- async def start_fn(spider):
+ async def default_start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
- yield self.request(num, response_seconds, download_slots)
+ yield self.request(num, seconds, download_slots)
+
+ start_fn = default_start
if parse_fn is None:
- def parse_fn(spider, response):
+ def default_parse(spider: Spider, response: Response) -> Iterator[Any]:
while cb_requests:
yield cb_requests.popleft()
+ parse_fn = default_parse
+
+ spider_start = start_fn
+ spider_parse = parse_fn
+
class TestSpider(Spider):
name = "test"
- start = start_fn
- parse = parse_fn
+ start = spider_start
+ parse = spider_parse
actual_nums = []
- def track_num(request, spider):
+ def track_num(request: Request, spider: Spider) -> None:
actual_nums.append(self.get_num(request))
crawler = get_crawler(TestSpider, settings_dict=settings)
@@ -337,12 +352,13 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
- def _request(num, priority=0):
+ def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
- async def start(spider):
+ async def start(spider: Spider) -> AsyncIterator[Any]:
+ assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@@ -356,7 +372,7 @@ class TestRequestSendOrder:
yield _request(3, priority=1)
yield _request(4, priority=1)
- def parse(spider, response):
+ def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@@ -377,12 +393,13 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
- def _request(num, priority=0):
+ def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
- async def start(spider):
+ async def start(spider: Spider) -> AsyncIterator[Any]:
+ assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@@ -396,7 +413,7 @@ class TestRequestSendOrder:
yield _request(4, priority=1)
yield _request(3, priority=1)
- def parse(spider, response):
+ def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@@ -420,12 +437,13 @@ class TestRequestSendOrder:
response_seconds = 0
download_slots = 1
- def _request(num, priority=0):
+ def _request(num: int, priority: int = 0) -> Request:
return self.request(
num, response_seconds, download_slots, priority=priority
)
- async def start(spider):
+ async def start(spider: Spider) -> AsyncIterator[Any]:
+ assert spider.crawler.engine._slot
# The first CONCURRENT_REQUESTS start requests are sent
# immediately.
yield _request(1)
@@ -454,7 +472,7 @@ class TestRequestSendOrder:
):
spider.crawler.engine._slot.scheduler.enqueue_request(request)
- def parse(spider, response):
+ def parse(spider: Spider, response: Response) -> Iterator[Any]:
return
yield
@@ -479,7 +497,7 @@ class TestRequestSendOrder:
response_seconds = self.seconds * 2**1 # increase if flaky
download_slots = 1
- async def start(spider):
+ async def start(spider: Spider) -> AsyncIterator[Any]:
for num in start_nums:
if spider.crawler.engine.needs_backout():
await spider.crawler.signals.wait_for(signals.scheduler_empty)
diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.py
index ddcfb5163..0c4f2ad95 100644
--- a/tests/test_extension_memdebug.py
+++ b/tests/test_extension_memdebug.py
@@ -33,7 +33,6 @@ def test_spider_closed_sets_stats() -> None:
ext.spider_closed(DefaultSpider.from_crawler(crawler), "finished")
- assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)
assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked)
assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None
@@ -47,6 +46,5 @@ async def test_crawl_sets_stats() -> None:
crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True})
await crawler.crawl_async()
- assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None
assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1
diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py
index dbe28eded..aa0149ff2 100644
--- a/tests/test_extension_memusage.py
+++ b/tests/test_extension_memusage.py
@@ -99,7 +99,6 @@ async def test_memusage_below_thresholds_logs_peak(
with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=1)
- assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") is None
assert crawler.stats.get_value("memusage/warning_reached") is None
assert crawler.stats.get_value("memusage/max") == 25 * MB
@@ -129,7 +128,6 @@ async def test_memusage_limit_closes_spider_with_reason_and_error_log(
with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=100)
- assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") == 1
assert crawler.stats.get_value("finish_reason") == "memusage_exceeded"
assert any(
@@ -167,7 +165,6 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
await crawler.crawl_async(url="data:,", loops=60)
assert warning_signals == [1]
- assert crawler.stats
assert crawler.stats.get_value("memusage/warning_reached") == 1
assert crawler.stats.get_value("finish_reason") == "finished"
warnings_logged = [
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index ad59297b6..1e8771d65 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -119,7 +119,6 @@ async def test_telnet_vars() -> None:
start_urls = ["data:,"]
async def parse(self, response: Response) -> None:
- assert self.crawler.extensions
console = next(
ext
for ext in self.crawler.extensions.middlewares
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8560dd364..67de762fc 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -310,7 +310,6 @@ class TestFeedExport(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
- assert crawler.stats is not None
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
@@ -334,7 +333,6 @@ class TestFeedExport(TestFeedExportBase):
side_effect=store,
):
yield crawler.crawl(mockserver=self.mockserver)
- assert crawler.stats is not None
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
@@ -352,7 +350,6 @@ class TestFeedExport(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
- assert crawler.stats is not None
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 1958e2935..fecb33d07 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -420,7 +420,6 @@ class TestBatchDeliveries(TestFeedExportBase):
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(total=2, mockserver=self.mockserver)
- assert crawler.stats
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@@ -491,7 +490,6 @@ class TestBatchDeliveries(TestFeedExportBase):
assert len(CustomS3FeedStorage.stubs) == len(items)
for stub in CustomS3FeedStorage.stubs:
stub.assert_no_pending_responses()
- assert crawler.stats
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
)
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index 35fd78283..09cafd218 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -15,7 +15,6 @@ class TestLogStats:
def setup_method(self) -> None:
self.crawler = get_crawler(SimpleSpider)
self.spider = self.crawler._create_spider("spidey")
- assert self.crawler.stats is not None
self.stats = self.crawler.stats
self.stats.set_value("response_received_count", 4802)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 8b522255a..529e7ffcb 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -140,7 +140,6 @@ class TestFileDownloadCrawl:
assert not items[0][self.media_key]
# check that there was 1 successful fetch and 3 other responses with non-200 code
- assert crawler.stats
assert crawler.stats.get_value("downloader/request_method_count/GET") == 4
assert crawler.stats.get_value("downloader/response_count") == 4
assert crawler.stats.get_value("downloader/response_status_count/200") == 1
@@ -208,7 +207,6 @@ class TestFileDownloadCrawl:
mockserver=self.mockserver,
)
self._assert_files_downloaded(self.items, caplog.text)
- assert crawler.stats
assert crawler.stats.get_value("downloader/response_status_count/302") == 3
@coroutine_test
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 231f6748f..30a42da48 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -82,7 +82,6 @@ class TestBaseMediaPipeline:
self.pipe = build_from_crawler(self.pipeline_class, crawler)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
- assert crawler.request_fingerprinter is not None
self.fingerprint = crawler.request_fingerprinter.fingerprint
@property
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 6c26aa878..97dafc87c 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -70,7 +70,6 @@ class KeywordArgumentsSpider(MockServerSpider):
checks: list[bool] = []
def _inc_checks(self, count: int = 1) -> None:
- assert self.crawler.stats
self.crawler.stats.inc_value("boolean_checks", count)
async def start(self):
@@ -171,7 +170,6 @@ class TestCallbackKeywordArguments:
await crawler.crawl_async(mockserver=mockserver)
assert isinstance(crawler.spider, KeywordArgumentsSpider)
assert all(crawler.spider.checks)
- assert crawler.stats
assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
# check exceptions for argument mismatch
exceptions = {}
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index ff92b1133..508147a7c 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -54,7 +54,6 @@ async def create_scheduler(
finally:
await ensure_awaitable(scheduler.close("finished"))
await mock_crawler.stop_async()
- assert mock_crawler.engine
mock_crawler.engine.downloader.close()
@@ -254,7 +253,6 @@ class DownloaderAwareSchedulerTestMixin(TestSchedulerBase):
dequeued_slots: list[str] = []
requests: list[Request] = []
assert scheduler.crawler
- assert scheduler.crawler.engine
downloader = scheduler.crawler.engine.downloader
assert isinstance(downloader, MockDownloader)
while scheduler.has_pending_requests():
@@ -320,7 +318,6 @@ class TestIntegrationWithDownloaderAwareInMemory:
url = mockserver.url("/status?n=200", is_secure=False)
start_urls = [url] * 6
yield self.crawler.crawl(start_urls)
- assert self.crawler.stats
assert self.crawler.stats.get_value("downloader/response_count") == len(
start_urls
)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 38cb8da18..6f7f171a3 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -41,7 +41,7 @@ class TestXMLFeedSpider(TestSpiderBase):
</urlset>"""
response = XmlResponse(url="http://example.com/sitemap.xml", body=body)
- class _XMLSpider(self.spider_class):
+ class _XMLSpider(self.spider_class): # type: ignore[name-defined,misc]
itertag = "url"
namespaces = (
("a", "http://www.google.com/schemas/sitemap/0.84"),
@@ -100,7 +100,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
- assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
@coroutine_test
@@ -135,7 +134,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
- assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1
@pytest.mark.parametrize("feed_iterator", ["xml", "html"])
@@ -155,7 +153,6 @@ class TestXMLFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
- assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
@@ -166,7 +163,7 @@ class TestCSVFeedSpider(TestSpiderBase):
body = get_testdata("feeds", "feed-sample6.csv")
response = Response("http://example.org/dummy.csv", body=body)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
delimiter = ","
quotechar = "'"
@@ -206,7 +203,6 @@ class TestCSVFeedSpider(TestSpiderBase):
items, crawler = await crawl_items(_Spider, mockserver)
assert items == []
- assert crawler.stats
assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py
index 9d5f0548b..ce556d0d1 100644
--- a/tests/test_spider_crawl.py
+++ b/tests/test_spider_crawl.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import re
import warnings
+from typing import TYPE_CHECKING, Any
import pytest
from w3lib.url import safe_url_string
@@ -14,6 +15,12 @@ from scrapy.utils.test import get_crawler
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from scrapy.http import Response
+ from scrapy.http.request import CallbackT
+
+URL = "https://www.example.com"
+
class TestCrawlSpider(TestSpiderBase):
test_body = b"""<html><head><title>Page title</title></head>
@@ -33,7 +40,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
@@ -53,7 +60,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
@@ -76,7 +83,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@@ -99,7 +106,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
@@ -125,7 +132,7 @@ class TestCrawlSpider(TestSpiderBase):
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace(".org", ".com"))
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@@ -151,7 +158,7 @@ class TestCrawlSpider(TestSpiderBase):
request.meta["response_class"] = response.__class__.__name__
return request
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@@ -180,7 +187,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
@@ -203,7 +210,7 @@ class TestCrawlSpider(TestSpiderBase):
"http://example.org/somepage/index.html", body=self.test_body
)
- class _CrawlSpider(self.spider_class):
+ class _CrawlSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
allowed_domains = ["example.org"]
rules = (
@@ -248,7 +255,6 @@ class TestCrawlSpider(TestSpiderBase):
def test_parse_response_use(self):
class _CrawlSpider(CrawlSpider):
name = "test"
- start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings():
@@ -258,17 +264,20 @@ class TestCrawlSpider(TestSpiderBase):
ScrapyDeprecationWarning,
match=r"CrawlSpider\._parse_response method is deprecated",
):
- spider._parse_response(
- TextResponse(spider.start_urls, body=b""), None, None
- )
+ spider._parse_response(TextResponse(URL, body=b""), None, {})
def test_parse_response_override(self):
class _CrawlSpider(CrawlSpider):
- def _parse_response(self, response, callback, cb_kwargs, follow=True):
+ def _parse_response(
+ self,
+ response: Response,
+ callback: CallbackT | None,
+ cb_kwargs: dict[str, Any],
+ follow: bool = True,
+ ) -> Any:
pass
name = "test"
- start_urls = "https://www.example.com"
_follow_links = False
with pytest.warns(
@@ -278,21 +287,16 @@ class TestCrawlSpider(TestSpiderBase):
spider = _CrawlSpider()
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
- spider._parse_response(
- TextResponse(spider.start_urls, body=b""), None, None
- )
+ spider._parse_response(TextResponse(URL, body=b""), None, {})
def test_parse_with_rules(self):
class _CrawlSpider(CrawlSpider):
name = "test"
- start_urls = "https://www.example.com"
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider = _CrawlSpider()
- spider.parse_with_rules(
- TextResponse(spider.start_urls, body=b""), None, None
- )
+ spider.parse_with_rules(TextResponse(URL, body=b""), None, {})
@coroutine_test
async def test_parse_with_rules_without_callback(self):
@@ -327,12 +331,13 @@ class TestCrawlSpider(TestSpiderBase):
allowed_domains = ["example.org"]
rules = (Rule(),)
+ def callback(response: Response) -> list[Any]:
+ return [item]
+
spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
results = [
r
- async for r in spider.parse_with_rules(
- response, lambda response: [item], {}, follow=False
- )
+ async for r in spider.parse_with_rules(response, callback, {}, follow=False)
]
assert results == [item]
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index 2f1ccab81..630ab5ca4 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -55,7 +55,7 @@ class TestSitemapSpider(TestSpiderBase):
assert spider._get_sitemap_body(response) == body
def test_get_sitemap_body(self):
- r = XmlResponse(url="http://www.example.com/", body=self.BODY)
+ r: Response = XmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
@@ -166,7 +166,7 @@ Sitemap: /sitemap-relative-url.xml
</url>
</urlset>"""
- class FilteredSitemapSpider(self.spider_class):
+ class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
@@ -201,7 +201,7 @@ Sitemap: /sitemap-relative-url.xml
</url>
</urlset>"""
- class FilteredSitemapSpider(self.spider_class):
+ class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
alternate_links = entry.get("alternate", ())
@@ -235,7 +235,7 @@ Sitemap: /sitemap-relative-url.xml
</sitemap>
</sitemapindex>"""
- class FilteredSitemapSpider(self.spider_class):
+ class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(
@@ -285,6 +285,7 @@ Sitemap: /sitemap-relative-url.xml
sitemap_rules = [("", parse_item)]
def raw_body(self):
+ assert self.mockserver
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
@@ -303,6 +304,7 @@ Sitemap: /sitemap-relative-url.xml
yield {"url": response.url}
def raw_body(self):
+ assert self.mockserver
loc = self.mockserver.url("/text")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
@@ -363,7 +365,7 @@ Sitemap: /sitemap-relative-url.xml
</sitemapindex>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
- class _FollowSpider(self.spider_class):
+ class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]
sitemap_follow = [follow]
spider = _FollowSpider("example.com")
@@ -382,7 +384,7 @@ Sitemap: /sitemap-relative-url.xml
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
- class DownloadMaxSizeSpider(self.spider_class):
+ class DownloadMaxSizeSpider(self.spider_class): # type: ignore[name-defined,misc]
download_maxsize = 10_000_000
crawler = get_crawler()
@@ -485,7 +487,7 @@ Sitemap: /sitemap-relative-url.xml
@coroutine_test
async def test_sitemap_urls(self):
- class TestSpider(self.spider_class):
+ class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
sitemap_urls = ["https://toscrape.com/sitemap.xml"]
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 7cef2fefa..fb72ece6c 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -34,7 +34,6 @@ class TestMain:
crawler = get_crawler(spider)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
- assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 69a41c0ff..9d6f4666b 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -26,7 +26,6 @@ def crawler() -> Crawler:
@pytest.fixture
def stats(crawler: Crawler) -> Generator[StatsCollector]:
- assert crawler.stats is not None
crawler.stats.open_spider()
yield crawler.stats
@@ -120,7 +119,6 @@ def test_priority_and_non_verbose_stats() -> None:
Spider,
{"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},
)
- assert crawler.stats is not None
crawler.stats.open_spider()
try:
mw = build_from_crawler(DepthMiddleware, crawler)
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index d8d5b278b..43b41ce3f 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -205,7 +205,6 @@ class TestHttpErrorMiddlewareIntegrational:
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}
- assert crawler.stats
get_value = crawler.stats.get_value
assert get_value("httperror/response_ignored_count") == 3
assert get_value("httperror/response_ignored_status_count/404") == 1
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index dacc90b27..74051d1db 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,16 +1,21 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
+from tests.spiders import MockServerSpider
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
+ from collections.abc import AsyncIterator, Iterator
+
import pytest
+ from scrapy.http import Response
+
class _BaseSpiderMiddleware:
def __init__(self, crawler):
@@ -41,7 +46,7 @@ class RecoveryMiddleware(_BaseSpiderMiddleware):
]
-class RecoverySpider(Spider):
+class RecoverySpider(MockServerSpider):
name = "RecoverySpider"
custom_settings = {
"SPIDER_MIDDLEWARES_BASE": {},
@@ -50,10 +55,11 @@ class RecoverySpider(Spider):
},
}
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
- def parse(self, response):
+ def parse(self, response: Response) -> Iterator[Any]:
yield {"test": 1}
self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
if not response.meta.get("dont_fail"):
@@ -63,7 +69,7 @@ class RecoverySpider(Spider):
class RecoveryAsyncGenSpider(RecoverySpider):
name = "RecoveryAsyncGenSpider"
- async def parse(self, response):
+ async def parse(self, response: Response) -> AsyncIterator[Any]: # type: ignore[override]
for r in super().parse(response):
yield r
@@ -76,7 +82,7 @@ class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
raise IndexError
-class ProcessSpiderInputSpiderWithoutErrback(Spider):
+class ProcessSpiderInputSpiderWithoutErrback(MockServerSpider):
name = "ProcessSpiderInputSpiderWithoutErrback"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@@ -87,7 +93,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
}
}
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)
def parse(self, response):
@@ -97,7 +104,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
name = "ProcessSpiderInputSpiderWithErrback"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(
self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
)
@@ -109,7 +117,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback
# ================================================================================
# (2) exceptions from a spider callback (generator)
-class GeneratorCallbackSpider(Spider):
+class GeneratorCallbackSpider(MockServerSpider):
name = "GeneratorCallbackSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@@ -117,7 +125,8 @@ class GeneratorCallbackSpider(Spider):
},
}
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
@@ -146,7 +155,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
# ================================================================================
# (3) exceptions from a spider callback (not a generator)
-class NotGeneratorCallbackSpider(Spider):
+class NotGeneratorCallbackSpider(MockServerSpider):
name = "NotGeneratorCallbackSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@@ -154,7 +163,8 @@ class NotGeneratorCallbackSpider(Spider):
},
}
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
@@ -224,7 +234,7 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
pass
-class GeneratorOutputChainSpider(Spider):
+class GeneratorOutputChainSpider(MockServerSpider):
name = "GeneratorOutputChainSpider"
custom_settings = {
"SPIDER_MIDDLEWARES": {
@@ -235,7 +245,8 @@ class GeneratorOutputChainSpider(Spider):
},
}
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
+ assert self.mockserver
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index 321139da3..46414bb59 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -1,4 +1,7 @@
+from __future__ import annotations
+
from asyncio import sleep
+from typing import TYPE_CHECKING, Any
import pytest
@@ -9,6 +12,11 @@ from scrapy.utils.test import get_crawler
from .utils import twisted_sleep
from .utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
+ from scrapy.http import Response
+
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
@@ -17,20 +25,20 @@ SLEEP_SECONDS = 0.1
class AsyncioSleepSpiderMiddleware:
- async def process_start(self, start):
+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
await sleep(SLEEP_SECONDS)
async for item_or_request in start:
yield item_or_request
class NoOpSpiderMiddleware:
- async def process_start(self, start):
+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
async for item_or_request in start:
yield item_or_request
class TwistedSleepSpiderMiddleware:
- async def process_start(self, start):
+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS))
async for item_or_request in start:
yield item_or_request
@@ -42,12 +50,12 @@ class TwistedSleepSpiderMiddleware:
class ModernWrapSpider(Spider):
name = "test"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
yield ITEM_B
class ModernWrapSpiderMiddleware:
- async def process_start(self, start):
+ async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
yield ITEM_A
async for item_or_request in start:
yield item_or_request
@@ -55,10 +63,15 @@ class ModernWrapSpiderMiddleware:
class TestMain:
- async def _test(self, spider_middlewares, spider_cls, expected_items):
+ async def _test(
+ self,
+ spider_middlewares: list[type],
+ spider_cls: type[Spider],
+ expected_items: list[Any],
+ ) -> None:
actual_items = []
- def track_item(item, response, spider):
+ def track_item(item: Any, response: Response, spider: Spider) -> None:
actual_items.append(item)
settings = {
@@ -70,7 +83,12 @@ class TestMain:
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"
- async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None):
+ async def _test_wrap(
+ self,
+ spider_middleware: type,
+ spider_cls: type[Spider],
+ expected_items: list[Any] | None = None,
+ ) -> None:
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
await self._test([spider_middleware], spider_cls, expected_items)
@@ -78,11 +96,11 @@ class TestMain:
async def test_modern_mw_modern_spider(self):
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
- async def _test_sleep(self, spider_middlewares):
+ async def _test_sleep(self, spider_middlewares: list[type]) -> None:
class TestSpider(Spider):
name = "test"
- async def start(self):
+ async def start(self) -> AsyncIterator[Any]:
yield ITEM_A
await self._test(spider_middlewares, TestSpider, [ITEM_A])
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 1090b3e84..2620327af 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -31,7 +31,6 @@ def crawler() -> Crawler:
@pytest.fixture
def stats(crawler: Crawler) -> StatsCollector:
- assert crawler.stats is not None
return crawler.stats
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index d1b98d009..4a18d9ff6 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -97,7 +97,6 @@ class TestLogCounterHandler:
logger.removeHandler(handler)
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
- assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None
assert crawler.stats.get_value("log_count/INFO") is None
assert crawler.stats.get_value("log_count/WARNING") is None
@@ -106,12 +105,10 @@ class TestLogCounterHandler:
def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.error("test log msg")
- assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.debug("test log msg")
- assert crawler.stats
assert crawler.stats.get_value("log_count/DEBUG") is None
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 935447bc4..6ca608053 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -274,7 +274,6 @@ class TestRequestFingerprinter:
def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
- assert crawler.request_fingerprinter
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
@@ -291,7 +290,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
- assert crawler.request_fingerprinter
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@@ -314,7 +312,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
- assert crawler.request_fingerprinter
r1 = Request("http://www.example.com?a=1&a=2")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com?a=2&a=1")
@@ -333,7 +330,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
- assert crawler.request_fingerprinter
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@@ -365,7 +361,6 @@ class TestCustomRequestFingerprinter:
}
crawler = get_crawler(settings_dict=settings)
- assert crawler.request_fingerprinter
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py
index 799c1820d..224606bf6 100644
--- a/tests/utils/bases/spider.py
+++ b/tests/utils/bases/spider.py
@@ -32,7 +32,7 @@ class TestSpiderBase(ABC):
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class("example.com", foo="bar")
- assert spider.foo == "bar"
+ assert spider.foo == "bar" # type: ignore[attr-defined]
def test_spider_without_name(self):
"""``__init__`` raises when the name is not provided."""
@@ -58,7 +58,7 @@ class TestSpiderBase(ABC):
mock_init.assert_called_once_with("example.com", foo="bar")
def test_closed_signal_call(self):
- class TestSpider(self.spider_class):
+ class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
closed_called = False
def closed(self, reason):
@@ -92,7 +92,7 @@ class TestSpiderBase(ABC):
**get_reactor_settings(),
}
- class TestSpider(self.spider_class):
+ class TestSpider(self.spider_class): # type: ignore[name-defined,misc]
name = "test"
custom_settings = spider_settings
@@ -127,5 +127,5 @@ class TestSpiderBase(ABC):
ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
),
):
- spider.log("test log msg", "INFO")
+ spider.log("test log msg", "INFO") # type: ignore[arg-type]
mock_logger.log.assert_called_once_with("INFO", "test log msg")