Scrapy, a fast high-level web crawling & scraping framework for Python.
Some test cleanups. (#8273)
Andrey Rakhmatullin9558c60ed517
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex c494c0e98..cc095ff75 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -1,7 +1,6 @@from __future__ import annotationsimport logging-import osimport stringimport sysimport tempfile@@ -48,14 +47,12 @@ class TestFileFeedStorage:path = tmp_path / "file.txt"self._assert_stores(FileFeedStorage(str(path)), path)- def test_store_direct_path_relative(self, tmp_path):- old_cwd = Path.cwd()- try:- os.chdir(tmp_path)- path = Path("foo", "bar")- self._assert_stores(FileFeedStorage(str(path)), path)- finally:- os.chdir(old_cwd)+ def test_store_direct_path_relative(+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.chdir(tmp_path)+ path = Path("foo", "bar")+ self._assert_stores(FileFeedStorage(str(path)), path)@staticmethoddef _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex a6e5188d7..c8ad2386a 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -1,11 +1,12 @@+from __future__ import annotations+import asyncio-from typing import Any+from typing import TYPE_CHECKING, Anyimport pytestfrom twisted.internet.defer import Deferred, fail, succeedfrom scrapy import Request, Spider, signals-from scrapy.crawler import Crawlerfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.pipelines import ItemPipelineManagerfrom scrapy.utils.asyncio import call_later@@ -14,9 +15,12 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawler, get_from_asyncio_queue-from tests.mockserver.http import MockServerfrom tests.utils.decorators import coroutine_test+if TYPE_CHECKING:+ from scrapy.crawler import Crawler+ from tests.mockserver.http import MockServer+class SimplePipeline:def process_item(self, item):@@ -131,7 +135,7 @@ class ItemSpider(Spider):class TestPipeline:- def _on_item_scraped(self, item):+ def _on_item_scraped(self, item: dict[str, Any]) -> None:assert isinstance(item, dict)assert item.get("pipeline_passed")self.items.append(item)@@ -274,7 +278,7 @@ class TestCustomPipelineManager:items = []- def _on_item_scraped(item):+ def _on_item_scraped(item: dict[str, Any]) -> None:assert isinstance(item, dict)assert item.get("pipeline_passed")items.append(item)@@ -318,7 +322,7 @@ class TestCustomPipelineManager:items = []- def _on_item_scraped(item):+ def _on_item_scraped(item: dict[str, Any]) -> None:assert isinstance(item, dict)assert item.get("pipeline_passed")items.append(item)@@ -380,7 +384,7 @@ class TestCustomPipelineManager:items = []- def _on_item_scraped(item):+ def _on_item_scraped(item: dict[str, Any]) -> None:assert isinstance(item, dict)assert item.get("pipeline_passed")items.append(item)diff --git a/tests/test_poet.py b/tests/test_poet.pyindex 9601c75a1..c85063a78 100644--- a/tests/test_poet.py+++ b/tests/test_poet.py@@ -1,12 +1,14 @@"""Tests that make sure parts needed for the scrapy-poet stack work."""+from __future__ import annotations+from typing import get_type_hintsfrom scrapy import Spiderfrom scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider-def test_callbacks():+def test_callbacks() -> None:"""Making sure annotations on all non-abstract callbacks can be resolved."""for cb in [diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex bcb134135..51a143b7d 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -1,5 +1,6 @@-import tempfile-from typing import Any+from __future__ import annotations++from typing import TYPE_CHECKING, Anyfrom unittest.mock import Mockimport pytest@@ -13,6 +14,12 @@ from scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.test import get_crawlerfrom tests.utils.downloader import MockDownloader+if TYPE_CHECKING:+ from collections.abc import Generator+ from pathlib import Path++ from scrapy.crawler import Crawler+def _pop(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:request = queue.pop()@@ -27,14 +34,13 @@ def _peek(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:class TestPriorityQueue:- def setup_method(self):- self.crawler = get_crawler(Spider)- self.spider = self.crawler._create_spider("foo")+ @pytest.fixture+ def crawler(self) -> Crawler:+ return get_crawler(Spider)- def test_queue_push_pop_one(self):- temp_dir = tempfile.mkdtemp()+ def test_queue_push_pop_one(self, crawler: Crawler, tmp_path: Path) -> None:queue = build_from_crawler(- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir+ ScrapyPriorityQueue, crawler, FifoMemoryQueue, str(tmp_path))assert queue.pop() is Noneassert len(queue) == 0@@ -47,10 +53,9 @@ class TestPriorityQueue:assert dequeued.priority == req1.priorityassert not queue.close()- def test_peek(self):- temp_dir = tempfile.mkdtemp()+ def test_peek(self, crawler: Crawler, tmp_path: Path) -> None:queue = build_from_crawler(- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir+ ScrapyPriorityQueue, crawler, FifoMemoryQueue, str(tmp_path))assert len(queue) == 0assert queue.peek() is None@@ -71,13 +76,14 @@ class TestPriorityQueue:assert _pop(queue).url == req3.urlassert not queue.close()- def test_init_prios_with_start_queue(self):- temp_dir = tempfile.mkdtemp()+ def test_init_prios_with_start_queue(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:queue = build_from_crawler(ScrapyPriorityQueue,- self.crawler,+ crawler,PickleFifoDiskQueue,- temp_dir,+ str(tmp_path),start_queue_cls=PickleFifoDiskQueue,)req = Request("https://example.org/", meta={"is_start_request": True})@@ -86,9 +92,9 @@ class TestPriorityQueue:queue2 = build_from_crawler(ScrapyPriorityQueue,- self.crawler,+ crawler,PickleFifoDiskQueue,- temp_dir,+ str(tmp_path),startprios,start_queue_cls=PickleFifoDiskQueue,)@@ -96,10 +102,9 @@ class TestPriorityQueue:assert _pop(queue2).url == req.urlqueue2.close()- def test_queue_push_pop_priorities(self):- temp_dir = tempfile.mkdtemp()+ def test_queue_push_pop_priorities(self, crawler: Crawler, tmp_path: Path) -> None:queue = build_from_crawler(- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]+ ScrapyPriorityQueue, crawler, FifoMemoryQueue, str(tmp_path), [-1, -2, -3])assert queue.pop() is Noneassert len(queue) == 0@@ -118,58 +123,67 @@ class TestPriorityQueue:class TestDownloaderAwarePriorityQueue:- def setup_method(self):+ @pytest.fixture+ def downloader(self) -> MockDownloader:+ return MockDownloader()++ @pytest.fixture+ def queue(+ self, downloader: MockDownloader+ ) -> Generator[DownloaderAwarePriorityQueue]:crawler = get_crawler(Spider)- self.downloader = MockDownloader()- crawler.engine = Mock(downloader=self.downloader)- self.queue = build_from_crawler(+ crawler.engine = Mock(downloader=downloader)+ queue = build_from_crawler(DownloaderAwarePriorityQueue,crawler,downstream_queue_cls=FifoMemoryQueue,key="foo/bar",)+ try:+ yield queue+ finally:+ queue.close()- def teardown_method(self):- self.queue.close()-- def test_push_pop(self):- assert len(self.queue) == 0- assert self.queue.pop() is None+ def test_push_pop(self, queue: DownloaderAwarePriorityQueue) -> None:+ assert len(queue) == 0+ assert queue.pop() is Nonereq1 = Request("http://www.example.com/1")req2 = Request("http://www.example.com/2")req3 = Request("http://www.example.com/3")- self.queue.push(req1)- self.queue.push(req2)- self.queue.push(req3)- assert len(self.queue) == 3- assert _pop(self.queue).url == req1.url- assert len(self.queue) == 2- assert _pop(self.queue).url == req2.url- assert len(self.queue) == 1- assert _pop(self.queue).url == req3.url- assert len(self.queue) == 0- assert self.queue.pop() is None-- def test_peek(self):- assert len(self.queue) == 0+ queue.push(req1)+ queue.push(req2)+ queue.push(req3)+ assert len(queue) == 3+ assert _pop(queue).url == req1.url+ assert len(queue) == 2+ assert _pop(queue).url == req2.url+ assert len(queue) == 1+ assert _pop(queue).url == req3.url+ assert len(queue) == 0+ assert queue.pop() is None++ def test_peek(self, queue: DownloaderAwarePriorityQueue) -> None:+ assert len(queue) == 0req1 = Request("https://example.org/1")req2 = Request("https://example.org/2")req3 = Request("https://example.org/3")- self.queue.push(req1)- self.queue.push(req2)- self.queue.push(req3)- assert len(self.queue) == 3- assert _peek(self.queue).url == req1.url- assert _pop(self.queue).url == req1.url- assert len(self.queue) == 2- assert _peek(self.queue).url == req2.url- assert _pop(self.queue).url == req2.url- assert len(self.queue) == 1- assert _peek(self.queue).url == req3.url- assert _pop(self.queue).url == req3.url- assert self.queue.peek() is None-- def test_tie_breaking_rotates_slots(self):+ queue.push(req1)+ queue.push(req2)+ queue.push(req3)+ assert len(queue) == 3+ assert _peek(queue).url == req1.url+ assert _pop(queue).url == req1.url+ assert len(queue) == 2+ assert _peek(queue).url == req2.url+ assert _pop(queue).url == req2.url+ assert len(queue) == 1+ assert _peek(queue).url == req3.url+ assert _pop(queue).url == req3.url+ assert queue.peek() is None++ def test_tie_breaking_rotates_slots(+ self, queue: DownloaderAwarePriorityQueue+ ) -> None:# No active downloads are tracked in the downloader, so every slot has# the same score and tie-breaking must not starve a slot.req_a1 = Request("https://example.org/a1")@@ -182,18 +196,20 @@ class TestDownloaderAwarePriorityQueue:req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"for request in (req_a1, req_b1, req_a2, req_b2):- self.queue.push(request)+ queue.push(request)slots = [- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],]assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"]- def test_tie_breaking_keeps_rotation_after_selected_slot_is_deleted(self):+ def test_tie_breaking_keeps_rotation_after_selected_slot_is_deleted(+ self, queue: DownloaderAwarePriorityQueue+ ) -> None:# If the selected slot becomes empty, rotation should continue from# that slot marker to avoid restarting from the smallest slot.req_a1 = Request("https://example.org/a1")@@ -206,18 +222,20 @@ class TestDownloaderAwarePriorityQueue:req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"for request in (req_a1, req_a2, req_b1, req_c1):- self.queue.push(request)+ queue.push(request)slots = [- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],]assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]- def test_pop_prefers_slot_with_fewer_active_downloads(self):+ def test_pop_prefers_slot_with_fewer_active_downloads(+ self, downloader: MockDownloader, queue: DownloaderAwarePriorityQueue+ ) -> None:req_a = Request("https://example.org/a")req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"req_b = Request("https://example.org/b")@@ -226,24 +244,24 @@ class TestDownloaderAwarePriorityQueue:req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"for req in (req_a, req_b, req_c):- self.queue.push(req)+ queue.push(req)- self.downloader.increment("slot-a")- self.downloader.increment("slot-c")+ downloader.increment("slot-a")+ downloader.increment("slot-c")- popped = _pop(self.queue)+ popped = _pop(queue)assert popped.url == req_b.url- def test_contains(self):+ def test_contains(self, queue: DownloaderAwarePriorityQueue) -> None:req = Request("https://example.org/")req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"- assert "example-slot" not in self.queue- self.queue.push(req)- assert "example-slot" in self.queue- assert "other-slot" not in self.queue+ assert "example-slot" not in queue+ queue.push(req)+ assert "example-slot" in queue+ assert "other-slot" not in queue-def test_slot_directory_removed_when_slot_drains(tmp_path):+def test_slot_directory_removed_when_slot_drains(tmp_path: Path) -> None:crawler = get_crawler(Spider)crawler.spider = crawler._create_spider("foo")crawler.engine = Mock(downloader=MockDownloader())@@ -281,7 +299,7 @@ def test_slot_directory_removed_when_slot_drains(tmp_path):([{"start": True}, {}], [2, 1]),],)-def test_pop_order(input_, output):+def test_pop_order(input_: list[dict[str, Any]], output: list[int]) -> None:def make_url(index: int) -> str:return f"https://toscrape.com/{index}"diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 97dafc87c..81f5fe5a4 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -160,32 +160,31 @@ class KeywordArgumentsSpider(MockServerSpider):self._inc_checks()-class TestCallbackKeywordArguments:- @coroutine_test- async def test_callback_kwargs(- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer- ) -> None:- crawler = get_crawler(KeywordArgumentsSpider)- with caplog.at_level(logging.ERROR):- await crawler.crawl_async(mockserver=mockserver)- assert isinstance(crawler.spider, KeywordArgumentsSpider)- assert all(crawler.spider.checks)- assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")- # check exceptions for argument mismatch- exceptions = {}- for line in caplog.records:- for key in ("takes_less", "takes_more"):- if key in line.getMessage():- exceptions[key] = line- takes_less_exc_info = exceptions["takes_less"].exc_info- assert takes_less_exc_info is not None- assert takes_less_exc_info[0] is TypeError- assert str(takes_less_exc_info[1]).endswith(- "parse_takes_less() got an unexpected keyword argument 'number'"- )- takes_more_exc_info = exceptions["takes_more"].exc_info- assert takes_more_exc_info is not None- assert takes_more_exc_info[0] is TypeError- assert str(takes_more_exc_info[1]).endswith(- "parse_takes_more() missing 1 required positional argument: 'other'"- )+@coroutine_test+async def test_callback_kwargs(+ caplog: pytest.LogCaptureFixture, mockserver: MockServer+) -> None:+ crawler = get_crawler(KeywordArgumentsSpider)+ with caplog.at_level(logging.ERROR):+ await crawler.crawl_async(mockserver=mockserver)+ assert isinstance(crawler.spider, KeywordArgumentsSpider)+ assert all(crawler.spider.checks)+ assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")+ # check exceptions for argument mismatch+ exceptions = {}+ for line in caplog.records:+ for key in ("takes_less", "takes_more"):+ if key in line.getMessage():+ exceptions[key] = line+ takes_less_exc_info = exceptions["takes_less"].exc_info+ assert takes_less_exc_info is not None+ assert takes_less_exc_info[0] is TypeError+ assert str(takes_less_exc_info[1]).endswith(+ "parse_takes_less() got an unexpected keyword argument 'number'"+ )+ takes_more_exc_info = exceptions["takes_more"].exc_info+ assert takes_more_exc_info is not None+ assert takes_more_exc_info[0] is TypeError+ assert str(takes_more_exc_info[1]).endswith(+ "parse_takes_more() missing 1 required positional argument: 'other'"+ )diff --git a/tests/test_request_left.py b/tests/test_request_left.pyindex e95f1d7d9..5b37f4b97 100644--- a/tests/test_request_left.py+++ b/tests/test_request_left.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Anyfrom scrapy.signals import request_left_downloaderfrom scrapy.spiders import Spiderfrom scrapy.utils.test import get_crawler-from tests.utils.decorators import inline_callbacks_test+from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from scrapy import Request@@ -32,33 +32,33 @@ class SignalCatcherSpider(Spider):self.caught_times += 1-@inline_callbacks_test-def test_success(mockserver: MockServer):+@coroutine_test+async def test_success(mockserver: MockServer) -> None:crawler = get_crawler(SignalCatcherSpider)- yield crawler.crawl(mockserver.url("/status?n=200"))+ await crawler.crawl_async(mockserver.url("/status?n=200"))assert isinstance(crawler.spider, SignalCatcherSpider)assert crawler.spider.caught_times == 1-@inline_callbacks_test-def test_timeout(mockserver: MockServer):+@coroutine_test+async def test_timeout(mockserver: MockServer) -> None:crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})- yield crawler.crawl(mockserver.url("/delay?n=0.2"))+ await crawler.crawl_async(mockserver.url("/delay?n=0.2"))assert isinstance(crawler.spider, SignalCatcherSpider)assert crawler.spider.caught_times == 1-@inline_callbacks_test-def test_disconnect(mockserver: MockServer):+@coroutine_test+async def test_disconnect(mockserver: MockServer) -> None:crawler = get_crawler(SignalCatcherSpider)- yield crawler.crawl(mockserver.url("/drop"))+ await crawler.crawl_async(mockserver.url("/drop"))assert isinstance(crawler.spider, SignalCatcherSpider)assert crawler.spider.caught_times == 1-@inline_callbacks_test-def test_noconnect():+@coroutine_test+async def test_noconnect() -> None:crawler = get_crawler(SignalCatcherSpider)- yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")+ await crawler.crawl_async("http://thereisdefinetelynosuchdomain.com")assert isinstance(crawler.spider, SignalCatcherSpider)assert crawler.spider.caught_times == 1diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex e4898f8fd..ace6a2f31 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -1,5 +1,6 @@from __future__ import annotations+from abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, Self, castimport pytest@@ -22,16 +23,16 @@ if TYPE_CHECKING:NO_CRAWLER = cast("Crawler", None)-class BaseRobotParserTest:- parser_cls: type[RobotParser]-- def _setUp(self, parser_cls: type[RobotParser]) -> None:- self.parser_cls = parser_cls+class TestRobotParserBase(ABC):+ @property+ @abstractmethod+ def parser_cls(self) -> type[RobotParser]:+ raise NotImplementedErrordef _parse(self, robotstxt_body: bytes) -> RobotParser:return build_from_crawler(self.parser_cls, NO_CRAWLER, robotstxt_body)- def test_allowed(self):+ def test_allowed(self) -> None:robotstxt_robotstxt_body = (b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10")@@ -68,7 +69,7 @@ class BaseRobotParserTest:rp = self._parse(robotstxt_robotstxt_body)assert not rp.allowed("https://www.site.local/page", "*")- def test_empty_response(self):+ def test_empty_response(self) -> None:"""empty response should equal 'allow all'"""rp = self._parse(b"")assert rp.allowed("https://site.local/", "*")@@ -76,7 +77,7 @@ class BaseRobotParserTest:assert rp.allowed("https://site.local/index.html", "*")assert rp.allowed("https://site.local/disallowed", "*")- def test_garbage_response(self):+ def test_garbage_response(self) -> None:"""garbage response should be discarded, equal 'allow all'"""robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"rp = self._parse(robotstxt_robotstxt_body)@@ -85,17 +86,17 @@ class BaseRobotParserTest:assert rp.allowed("https://site.local/index.html", "*")assert rp.allowed("https://site.local/disallowed", "*")- def test_crawl_delay(self):+ def test_crawl_delay(self) -> None:robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"rp = self._parse(robotstxt_body)assert rp.crawl_delay("*") == 10.0- def test_crawl_delay_unset(self):+ def test_crawl_delay_unset(self) -> None:robotstxt_body = b"User-agent: *\nDisallow: /private\n"rp = self._parse(robotstxt_body)assert rp.crawl_delay("*") is None- def test_unicode_url_and_useragent(self):+ def test_unicode_url_and_useragent(self) -> None:robotstxt_robotstxt_body = """User-Agent: *Disallow: /admin/@@ -118,7 +119,7 @@ class BaseRobotParserTest:class TestRobotParser:- def test_crawl_delay_unsupported(self):+ def test_crawl_delay_unsupported(self) -> None:class AllowAllRobotParser(RobotParser):@classmethoddef from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:@@ -134,33 +135,32 @@ class TestRobotParser:class TestDecodeRobotsTxt:- def test_native_string_conversion(self):+ def test_native_string_conversion(self) -> None:robotstxt_body = b"User-agent: *\nDisallow: /\n"decoded_content = decode_robotstxt(robotstxt_body, spider=None, to_native_str_type=True)assert decoded_content == "User-agent: *\nDisallow: /\n"- def test_decode_utf8(self):+ def test_decode_utf8(self) -> None:robotstxt_body = b"User-agent: *\nDisallow: /\n"decoded_content = decode_robotstxt(robotstxt_body, spider=None)assert decoded_content == "User-agent: *\nDisallow: /\n"- def test_decode_non_utf8(self):+ def test_decode_non_utf8(self) -> None:robotstxt_body = b"User-agent: *\n\xffDisallow: /\n"decoded_content = decode_robotstxt(robotstxt_body, spider=None)assert decoded_content == "User-agent: *\nDisallow: /\n"# UTF-8 BOM at the beginning of the file ignored- def test_decode_utf8_bom(self):+ def test_decode_utf8_bom(self) -> None:robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n"decoded_content = decode_robotstxt(robotstxt_body, spider=None)assert decoded_content == "User-agent: *\nDisallow: /\n"-class TestPythonRobotParser(BaseRobotParserTest):- def setup_method(self):- super()._setUp(PythonRobotParser)+class TestPythonRobotParser(TestRobotParserBase):+ parser_cls = PythonRobotParser@pytest.mark.skipif(not STDLIB_IMPROVED_ROBOTFILEPARSER,@@ -185,17 +185,15 @@ class TestPythonRobotParser(BaseRobotParserTest):@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")-class TestRerpRobotParser(BaseRobotParserTest):- def setup_method(self):- super()._setUp(RerpRobotParser)+class TestRerpRobotParser(TestRobotParserBase):+ parser_cls = RerpRobotParserdef test_length_based_precedence(self) -> None:pytest.skip("Rerp does not support length based directives precedence.")-class TestProtegoRobotParser(BaseRobotParserTest):- def setup_method(self):- super()._setUp(ProtegoRobotParser)+class TestProtegoRobotParser(TestRobotParserBase):+ parser_cls = ProtegoRobotParserdef test_order_based_precedence(self) -> None:pytest.skip("Protego does not support order based directives precedence.")diff --git a/tests/test_selector.py b/tests/test_selector.pyindex 802edfd67..443331c69 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -1,3 +1,5 @@+from __future__ import annotations+import weakrefimport pytest@@ -7,7 +9,7 @@ from scrapy.selector import Selectorclass TestSelector:- def test_simple_selection(self):+ def test_simple_selection(self) -> None:"""Simple selector tests"""body = b"<p><input name='a'value='1'/><input name='b'value='2'/></p>"response = TextResponse(url="http://example.com", body=body, encoding="utf-8")@@ -34,14 +36,14 @@ class TestSelector:)] == ["12"]- def test_root_base_url(self):+ def test_root_base_url(self) -> None:body = b'<html><form action="/path"><input name="a" /></form></html>'url = "http://example.com"response = TextResponse(url=url, body=body, encoding="utf-8")sel = Selector(response)assert url == sel.root.base- def test_flavor_detection(self):+ def test_flavor_detection(self) -> None:text = b'<div><img src="a.jpg"><p>Hello</div>'sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8"))assert sel.type == "xml"@@ -77,7 +79,7 @@ class TestSelector:)assert Selector(response).type == "html"- def test_http_header_encoding_precedence(self):+ def test_http_header_encoding_precedence(self) -> None:# '\xa3' = pound symbol in unicode# '\xc2\xa3' = pound symbol in utf-8# '\xa3' = pound symbol in latin-1 (iso-8859-1)@@ -99,7 +101,7 @@ class TestSelector:x = Selector(response)assert x.xpath("//span[@id='blank']/text()").getall() == ["\xa3"]- def test_badly_encoded_body(self):+ def test_badly_encoded_body(self) -> None:# \xe9 alone isn't valid utf8 sequencer1 = TextResponse("http://www.example.com",@@ -122,7 +124,7 @@ class TestSelector:response = HtmlResponse("https://example.com", body=body, headers=headers)assert response.css("p::text").get() == "£€"- def test_weakref_slots(self):+ def test_weakref_slots(self) -> None:"""Check that classes are using slots and are weak-referenceable"""x = Selector(text="")weakref.ref(x)@@ -130,11 +132,11 @@ class TestSelector:f"{x.__class__.__name__} does not use __slots__")- def test_response(self):+ def test_response(self) -> None:response = TextResponse(url="http://example.com", body=b"")assert Selector(response).response is response- def test_selector_bad_args(self):+ def test_selector_bad_args(self) -> None:with pytest.raises(ValueError, match="received both response and text"):Selector(TextResponse(url="http://example.com", body=b""), text="")diff --git a/tests/test_signals.py b/tests/test_signals.pyindex fc9520f49..7e1426ed6 100644--- a/tests/test_signals.py+++ b/tests/test_signals.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyimport pytest@@ -26,34 +26,31 @@ class ItemSpider(Spider):return {"index": response.meta["index"]}-class TestMain:- @coroutine_test- async def test_scheduler_empty(self):- crawler = get_crawler()- calls = []+@coroutine_test+async def test_scheduler_empty() -> None:+ crawler = get_crawler()+ calls = []- def track_call():- calls.append(object())+ def track_call() -> None:+ calls.append(object())- crawler.signals.connect(track_call, signals.scheduler_empty)- await crawler.crawl_async()- assert len(calls) >= 1+ crawler.signals.connect(track_call, signals.scheduler_empty)+ await crawler.crawl_async()+ assert len(calls) >= 1-class TestMockServer:- def setup_method(self):- self.items = [][email protected]_asyncio+@coroutine_test+async def test_simple_pipeline(mockserver: MockServer) -> None:+ items: list[Any] = []- async def _on_item_scraped(self, item):+ async def _on_item_scraped(item: Any) -> None:item = await get_from_asyncio_queue(item)- self.items.append(item)-- @pytest.mark.only_asyncio- @coroutine_test- async def test_simple_pipeline(self, mockserver: MockServer) -> None:- crawler = get_crawler(ItemSpider)- crawler.signals.connect(self._on_item_scraped, signals.item_scraped)- await crawler.crawl_async(mockserver=mockserver)- assert len(self.items) == 10- for index in range(10):- assert {"index": index} in self.items+ items.append(item)++ crawler = get_crawler(ItemSpider)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ await crawler.crawl_async(mockserver=mockserver)+ assert len(items) == 10+ for index in range(10):+ assert {"index": index} in itemsdiff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex 46166815a..e35fc8fd0 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -27,6 +27,7 @@ from scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:+ from collections.abc import Generatorfrom pathlib import Pathimport queuelib@@ -45,7 +46,7 @@ class TestRequestQueueBase(ABC):def is_fifo(self) -> bool:raise NotImplementedError- def test_one_element(self, q: queuelib.queue.BaseQueue):+ def test_one_element(self, q: queuelib.queue.BaseQueue) -> None:assert len(q) == 0assert q.peek() is Noneassert q.pop() is None@@ -62,7 +63,7 @@ class TestRequestQueueBase(ABC):assert q.peek() is Noneassert q.pop() is None- def test_headers(self, q: queuelib.queue.BaseQueue):+ def test_headers(self, q: queuelib.queue.BaseQueue) -> None:req = Request("http://www.example.com",headers=[("X-Multi", "a"), ("x-multi", "b"), ("Accept", None)],@@ -72,7 +73,7 @@ class TestRequestQueueBase(ABC):assert result is not Noneassert result.headers == req.headers- def test_order(self, q: queuelib.queue.BaseQueue):+ def test_order(self, q: queuelib.queue.BaseQueue) -> None:assert len(q) == 0assert q.peek() is Noneassert q.pop() is None@@ -100,7 +101,9 @@ class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):is_fifo = True@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo"))@@ -114,7 +117,9 @@ class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):is_fifo = False@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo"))@@ -128,7 +133,9 @@ class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):is_fifo = True@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo"))@@ -142,7 +149,9 @@ class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):is_fifo = False@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo"))@@ -156,7 +165,9 @@ class TestPickleFifoSQLiteQueueRequest(TestRequestQueueBase):is_fifo = True@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(PickleFifoSQLiteQueue, crawler, key=str(tmp_path / "pickle" / "fifo"))@@ -170,7 +181,9 @@ class TestPickleLifoSQLiteQueueRequest(TestRequestQueueBase):is_fifo = False@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(PickleLifoSQLiteQueue, crawler, key=str(tmp_path / "pickle" / "lifo"))@@ -184,7 +197,9 @@ class TestMarshalFifoSQLiteQueueRequest(TestRequestQueueBase):is_fifo = True@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(MarshalFifoSQLiteQueue, crawler, key=str(tmp_path / "marshal" / "fifo"))@@ -198,7 +213,9 @@ class TestMarshalLifoSQLiteQueueRequest(TestRequestQueueBase):is_fifo = False@pytest.fixture- def q(self, crawler, tmp_path):+ def q(+ self, crawler: Crawler, tmp_path: Path+ ) -> Generator[queuelib.queue.BaseQueue]:queue = build_from_crawler(MarshalLifoSQLiteQueue, crawler, key=str(tmp_path / "marshal" / "lifo"))@@ -242,7 +259,7 @@ class TestFifoMemoryQueueRequest(TestRequestQueueBase):is_fifo = True@pytest.fixture- def q(self, crawler):+ def q(self, crawler: Crawler) -> queuelib.queue.BaseQueue:return build_from_crawler(FifoMemoryQueue, crawler)@@ -250,5 +267,5 @@ class TestLifoMemoryQueueRequest(TestRequestQueueBase):is_fifo = False@pytest.fixture- def q(self, crawler):+ def q(self, crawler: Crawler) -> queuelib.queue.BaseQueue:return build_from_crawler(LifoMemoryQueue, crawler)diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.pyindex d4e9e8129..754702f5f 100644--- a/tests/test_utils_decorators.py+++ b/tests/test_utils_decorators.py@@ -71,7 +71,7 @@ class TestInthread:@coroutine_testasync def test_returns_deferred_with_result(self):@inthread- def multiply(a, b):+ def multiply(a: int, b: int) -> int:return a * bdeferred = multiply(6, 7)diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex 55acfe890..a9e23166a 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -32,6 +32,7 @@ from tests.spiders import LogSpiderif TYPE_CHECKING:from collections.abc import Generator, Mapping, MutableMapping+ from pathlib import Pathfrom scrapy.crawler import Crawlerfrom scrapy.logformatter import LogFormatterResult@@ -78,7 +79,8 @@ class TestTopLevelFormatter:class TestSpiderPlaceholderInLogFormat:- def _log(self, log_format: str, extra: dict[str, Any] | None) -> str:+ @staticmethod+ def _log(log_format: str, extra: dict[str, Any] | None) -> str:stream = StringIO()settings = Settings({"LOG_FORMAT": log_format})handler = _get_handler(settings)@@ -217,7 +219,7 @@ class TestGetFormatter:formatter = _get_formatter(handler, self._settings(LOG_COLOR=False))assert type(formatter) is logging.Formatter- def test_plain_for_file_handler(self, tmp_path: Any) -> None:+ def test_plain_for_file_handler(self, tmp_path: Path) -> None:handler = logging.FileHandler(tmp_path / "log.txt")try:formatter = _get_formatter(handler, self._settings())@@ -342,9 +344,10 @@ class TestLogging:logger.addHandler(handler)logger.setLevel(logging.DEBUG)- yield logger-- logger.removeHandler(handler)+ try:+ yield logger+ finally:+ logger.removeHandler(handler)def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo message"@@ -401,10 +404,10 @@ class TestLoggingWithExtra:logger = logging.getLogger("log_spider")logger.addHandler(handler)logger.setLevel(logging.DEBUG)-- yield logger-- logger.removeHandler(handler)+ try:+ yield logger+ finally:+ logger.removeHandler(handler)def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:log_message = "Foo message"diff --git a/tests/test_utils_project.py b/tests/test_utils_project.pyindex 3cc56b283..0383a1a61 100644--- a/tests/test_utils_project.py+++ b/tests/test_utils_project.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport osimport sysfrom pathlib import Path-from typing import TYPE_CHECKINGimport pytest@@ -17,35 +16,23 @@ from scrapy.utils.project import (project_data_dir,)-if TYPE_CHECKING:- from collections.abc import Generator-@pytest.fixture-def proj_path(tmp_path: Path) -> Generator[Path]:- prev_dir = Path.cwd()+def proj_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:project_dir = tmp_path-- try:- os.chdir(project_dir)- Path("scrapy.cfg").touch()-- yield project_dir- finally:- os.chdir(prev_dir)+ monkeypatch.chdir(project_dir)+ Path("scrapy.cfg").touch()+ return project_dir@pytest.fixture-def no_proj_path(tmp_path: Path) -> Generator[Path]:+def no_proj_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:"""A working directory without a scrapy.cfg file, also isolated from theuser-wide and system-wide Scrapy configuration files."""- prev_dir = Path.cwd()- try:- os.chdir(tmp_path)- with set_environ(HOME=str(tmp_path), XDG_CONFIG_HOME=str(tmp_path)):- yield tmp_path- finally:- os.chdir(prev_dir)+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))+ return tmp_pathdef test_data_path_outside_project() -> None: