Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Some test cleanups. (#8273)

Andrey Rakhmatullin9558c60ed517
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index c494c0e98..cc095ff75 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
-import os
import string
import sys
import tempfile
@@ -48,14 +47,12 @@ class TestFileFeedStorage:
path = tmp_path / "file.txt"
self._assert_stores(FileFeedStorage(str(path)), path)
- def test_store_direct_path_relative(self, tmp_path):
- old_cwd = Path.cwd()
- try:
- os.chdir(tmp_path)
- path = Path("foo", "bar")
- self._assert_stores(FileFeedStorage(str(path)), path)
- finally:
- os.chdir(old_cwd)
+ def test_store_direct_path_relative(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.chdir(tmp_path)
+ path = Path("foo", "bar")
+ self._assert_stores(FileFeedStorage(str(path)), path)
@staticmethod
def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index a6e5188d7..c8ad2386a 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -1,11 +1,12 @@
+from __future__ import annotations
+
import asyncio
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred, fail, succeed
from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
@@ -14,9 +15,12 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test
+if TYPE_CHECKING:
+ from scrapy.crawler import Crawler
+ from tests.mockserver.http import MockServer
+
class SimplePipeline:
def process_item(self, item):
@@ -131,7 +135,7 @@ class ItemSpider(Spider):
class TestPipeline:
- def _on_item_scraped(self, item):
+ def _on_item_scraped(self, item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
self.items.append(item)
@@ -274,7 +278,7 @@ class TestCustomPipelineManager:
items = []
- def _on_item_scraped(item):
+ def _on_item_scraped(item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
items.append(item)
@@ -318,7 +322,7 @@ class TestCustomPipelineManager:
items = []
- def _on_item_scraped(item):
+ def _on_item_scraped(item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
items.append(item)
@@ -380,7 +384,7 @@ class TestCustomPipelineManager:
items = []
- def _on_item_scraped(item):
+ def _on_item_scraped(item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
items.append(item)
diff --git a/tests/test_poet.py b/tests/test_poet.py
index 9601c75a1..c85063a78 100644
--- a/tests/test_poet.py
+++ b/tests/test_poet.py
@@ -1,12 +1,14 @@
"""Tests that make sure parts needed for the scrapy-poet stack work."""
+from __future__ import annotations
+
from typing import get_type_hints
from scrapy import Spider
from scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider
-def test_callbacks():
+def test_callbacks() -> None:
"""Making sure annotations on all non-abstract callbacks can be resolved."""
for cb in [
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index bcb134135..51a143b7d 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -1,5 +1,6 @@
-import tempfile
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
from unittest.mock import Mock
import pytest
@@ -13,6 +14,12 @@ from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
from tests.utils.downloader import MockDownloader
+if TYPE_CHECKING:
+ from collections.abc import Generator
+ from pathlib import Path
+
+ from scrapy.crawler import Crawler
+
def _pop(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:
request = queue.pop()
@@ -27,14 +34,13 @@ def _peek(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:
class TestPriorityQueue:
- def setup_method(self):
- self.crawler = get_crawler(Spider)
- self.spider = self.crawler._create_spider("foo")
+ @pytest.fixture
+ def crawler(self) -> Crawler:
+ return get_crawler(Spider)
- def test_queue_push_pop_one(self):
- temp_dir = tempfile.mkdtemp()
+ def test_queue_push_pop_one(self, crawler: Crawler, tmp_path: Path) -> None:
queue = build_from_crawler(
- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
+ ScrapyPriorityQueue, crawler, FifoMemoryQueue, str(tmp_path)
)
assert queue.pop() is None
assert len(queue) == 0
@@ -47,10 +53,9 @@ class TestPriorityQueue:
assert dequeued.priority == req1.priority
assert not queue.close()
- def test_peek(self):
- temp_dir = tempfile.mkdtemp()
+ def test_peek(self, crawler: Crawler, tmp_path: Path) -> None:
queue = build_from_crawler(
- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
+ ScrapyPriorityQueue, crawler, FifoMemoryQueue, str(tmp_path)
)
assert len(queue) == 0
assert queue.peek() is None
@@ -71,13 +76,14 @@ class TestPriorityQueue:
assert _pop(queue).url == req3.url
assert not queue.close()
- def test_init_prios_with_start_queue(self):
- temp_dir = tempfile.mkdtemp()
+ def test_init_prios_with_start_queue(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> None:
queue = build_from_crawler(
ScrapyPriorityQueue,
- self.crawler,
+ crawler,
PickleFifoDiskQueue,
- temp_dir,
+ str(tmp_path),
start_queue_cls=PickleFifoDiskQueue,
)
req = Request("https://example.org/", meta={"is_start_request": True})
@@ -86,9 +92,9 @@ class TestPriorityQueue:
queue2 = build_from_crawler(
ScrapyPriorityQueue,
- self.crawler,
+ crawler,
PickleFifoDiskQueue,
- temp_dir,
+ str(tmp_path),
startprios,
start_queue_cls=PickleFifoDiskQueue,
)
@@ -96,10 +102,9 @@ class TestPriorityQueue:
assert _pop(queue2).url == req.url
queue2.close()
- def test_queue_push_pop_priorities(self):
- temp_dir = tempfile.mkdtemp()
+ def test_queue_push_pop_priorities(self, crawler: Crawler, tmp_path: Path) -> None:
queue = build_from_crawler(
- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
+ ScrapyPriorityQueue, crawler, FifoMemoryQueue, str(tmp_path), [-1, -2, -3]
)
assert queue.pop() is None
assert len(queue) == 0
@@ -118,58 +123,67 @@ class TestPriorityQueue:
class TestDownloaderAwarePriorityQueue:
- def setup_method(self):
+ @pytest.fixture
+ def downloader(self) -> MockDownloader:
+ return MockDownloader()
+
+ @pytest.fixture
+ def queue(
+ self, downloader: MockDownloader
+ ) -> Generator[DownloaderAwarePriorityQueue]:
crawler = get_crawler(Spider)
- self.downloader = MockDownloader()
- crawler.engine = Mock(downloader=self.downloader)
- self.queue = build_from_crawler(
+ crawler.engine = Mock(downloader=downloader)
+ queue = build_from_crawler(
DownloaderAwarePriorityQueue,
crawler,
downstream_queue_cls=FifoMemoryQueue,
key="foo/bar",
)
+ try:
+ yield queue
+ finally:
+ queue.close()
- def teardown_method(self):
- self.queue.close()
-
- def test_push_pop(self):
- assert len(self.queue) == 0
- assert self.queue.pop() is None
+ def test_push_pop(self, queue: DownloaderAwarePriorityQueue) -> None:
+ assert len(queue) == 0
+ assert queue.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
req3 = Request("http://www.example.com/3")
- self.queue.push(req1)
- self.queue.push(req2)
- self.queue.push(req3)
- assert len(self.queue) == 3
- assert _pop(self.queue).url == req1.url
- assert len(self.queue) == 2
- assert _pop(self.queue).url == req2.url
- assert len(self.queue) == 1
- assert _pop(self.queue).url == req3.url
- assert len(self.queue) == 0
- assert self.queue.pop() is None
-
- def test_peek(self):
- assert len(self.queue) == 0
+ queue.push(req1)
+ queue.push(req2)
+ queue.push(req3)
+ assert len(queue) == 3
+ assert _pop(queue).url == req1.url
+ assert len(queue) == 2
+ assert _pop(queue).url == req2.url
+ assert len(queue) == 1
+ assert _pop(queue).url == req3.url
+ assert len(queue) == 0
+ assert queue.pop() is None
+
+ def test_peek(self, queue: DownloaderAwarePriorityQueue) -> None:
+ assert len(queue) == 0
req1 = Request("https://example.org/1")
req2 = Request("https://example.org/2")
req3 = Request("https://example.org/3")
- self.queue.push(req1)
- self.queue.push(req2)
- self.queue.push(req3)
- assert len(self.queue) == 3
- assert _peek(self.queue).url == req1.url
- assert _pop(self.queue).url == req1.url
- assert len(self.queue) == 2
- assert _peek(self.queue).url == req2.url
- assert _pop(self.queue).url == req2.url
- assert len(self.queue) == 1
- assert _peek(self.queue).url == req3.url
- assert _pop(self.queue).url == req3.url
- assert self.queue.peek() is None
-
- def test_tie_breaking_rotates_slots(self):
+ queue.push(req1)
+ queue.push(req2)
+ queue.push(req3)
+ assert len(queue) == 3
+ assert _peek(queue).url == req1.url
+ assert _pop(queue).url == req1.url
+ assert len(queue) == 2
+ assert _peek(queue).url == req2.url
+ assert _pop(queue).url == req2.url
+ assert len(queue) == 1
+ assert _peek(queue).url == req3.url
+ assert _pop(queue).url == req3.url
+ assert queue.peek() is None
+
+ def test_tie_breaking_rotates_slots(
+ self, queue: DownloaderAwarePriorityQueue
+ ) -> None:
# No active downloads are tracked in the downloader, so every slot has
# the same score and tie-breaking must not starve a slot.
req_a1 = Request("https://example.org/a1")
@@ -182,18 +196,20 @@ class TestDownloaderAwarePriorityQueue:
req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
for request in (req_a1, req_b1, req_a2, req_b2):
- self.queue.push(request)
+ queue.push(request)
slots = [
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
]
assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"]
- def test_tie_breaking_keeps_rotation_after_selected_slot_is_deleted(self):
+ def test_tie_breaking_keeps_rotation_after_selected_slot_is_deleted(
+ self, queue: DownloaderAwarePriorityQueue
+ ) -> None:
# If the selected slot becomes empty, rotation should continue from
# that slot marker to avoid restarting from the smallest slot.
req_a1 = Request("https://example.org/a1")
@@ -206,18 +222,20 @@ class TestDownloaderAwarePriorityQueue:
req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
for request in (req_a1, req_a2, req_b1, req_c1):
- self.queue.push(request)
+ queue.push(request)
slots = [
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
- _pop(self.queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
+ _pop(queue).meta[Downloader.DOWNLOAD_SLOT],
]
assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]
- def test_pop_prefers_slot_with_fewer_active_downloads(self):
+ def test_pop_prefers_slot_with_fewer_active_downloads(
+ self, downloader: MockDownloader, queue: DownloaderAwarePriorityQueue
+ ) -> None:
req_a = Request("https://example.org/a")
req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
req_b = Request("https://example.org/b")
@@ -226,24 +244,24 @@ class TestDownloaderAwarePriorityQueue:
req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
for req in (req_a, req_b, req_c):
- self.queue.push(req)
+ queue.push(req)
- self.downloader.increment("slot-a")
- self.downloader.increment("slot-c")
+ downloader.increment("slot-a")
+ downloader.increment("slot-c")
- popped = _pop(self.queue)
+ popped = _pop(queue)
assert popped.url == req_b.url
- def test_contains(self):
+ def test_contains(self, queue: DownloaderAwarePriorityQueue) -> None:
req = Request("https://example.org/")
req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"
- assert "example-slot" not in self.queue
- self.queue.push(req)
- assert "example-slot" in self.queue
- assert "other-slot" not in self.queue
+ assert "example-slot" not in queue
+ queue.push(req)
+ assert "example-slot" in queue
+ assert "other-slot" not in queue
-def test_slot_directory_removed_when_slot_drains(tmp_path):
+def test_slot_directory_removed_when_slot_drains(tmp_path: Path) -> None:
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider("foo")
crawler.engine = Mock(downloader=MockDownloader())
@@ -281,7 +299,7 @@ def test_slot_directory_removed_when_slot_drains(tmp_path):
([{"start": True}, {}], [2, 1]),
],
)
-def test_pop_order(input_, output):
+def test_pop_order(input_: list[dict[str, Any]], output: list[int]) -> None:
def make_url(index: int) -> str:
return f"https://toscrape.com/{index}"
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 97dafc87c..81f5fe5a4 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -160,32 +160,31 @@ class KeywordArgumentsSpider(MockServerSpider):
self._inc_checks()
-class TestCallbackKeywordArguments:
- @coroutine_test
- async def test_callback_kwargs(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
- ) -> None:
- crawler = get_crawler(KeywordArgumentsSpider)
- with caplog.at_level(logging.ERROR):
- await crawler.crawl_async(mockserver=mockserver)
- assert isinstance(crawler.spider, KeywordArgumentsSpider)
- assert all(crawler.spider.checks)
- assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
- # check exceptions for argument mismatch
- exceptions = {}
- for line in caplog.records:
- for key in ("takes_less", "takes_more"):
- if key in line.getMessage():
- exceptions[key] = line
- takes_less_exc_info = exceptions["takes_less"].exc_info
- assert takes_less_exc_info is not None
- assert takes_less_exc_info[0] is TypeError
- assert str(takes_less_exc_info[1]).endswith(
- "parse_takes_less() got an unexpected keyword argument 'number'"
- )
- takes_more_exc_info = exceptions["takes_more"].exc_info
- assert takes_more_exc_info is not None
- assert takes_more_exc_info[0] is TypeError
- assert str(takes_more_exc_info[1]).endswith(
- "parse_takes_more() missing 1 required positional argument: 'other'"
- )
+@coroutine_test
+async def test_callback_kwargs(
+ caplog: pytest.LogCaptureFixture, mockserver: MockServer
+) -> None:
+ crawler = get_crawler(KeywordArgumentsSpider)
+ with caplog.at_level(logging.ERROR):
+ await crawler.crawl_async(mockserver=mockserver)
+ assert isinstance(crawler.spider, KeywordArgumentsSpider)
+ assert all(crawler.spider.checks)
+ assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
+ # check exceptions for argument mismatch
+ exceptions = {}
+ for line in caplog.records:
+ for key in ("takes_less", "takes_more"):
+ if key in line.getMessage():
+ exceptions[key] = line
+ takes_less_exc_info = exceptions["takes_less"].exc_info
+ assert takes_less_exc_info is not None
+ assert takes_less_exc_info[0] is TypeError
+ assert str(takes_less_exc_info[1]).endswith(
+ "parse_takes_less() got an unexpected keyword argument 'number'"
+ )
+ takes_more_exc_info = exceptions["takes_more"].exc_info
+ assert takes_more_exc_info is not None
+ assert takes_more_exc_info[0] is TypeError
+ assert str(takes_more_exc_info[1]).endswith(
+ "parse_takes_more() missing 1 required positional argument: 'other'"
+ )
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index e95f1d7d9..5b37f4b97 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any
from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy import Request
@@ -32,33 +32,33 @@ class SignalCatcherSpider(Spider):
self.caught_times += 1
-@inline_callbacks_test
-def test_success(mockserver: MockServer):
+@coroutine_test
+async def test_success(mockserver: MockServer) -> None:
crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl(mockserver.url("/status?n=200"))
+ await crawler.crawl_async(mockserver.url("/status?n=200"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
-@inline_callbacks_test
-def test_timeout(mockserver: MockServer):
+@coroutine_test
+async def test_timeout(mockserver: MockServer) -> None:
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
- yield crawler.crawl(mockserver.url("/delay?n=0.2"))
+ await crawler.crawl_async(mockserver.url("/delay?n=0.2"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
-@inline_callbacks_test
-def test_disconnect(mockserver: MockServer):
+@coroutine_test
+async def test_disconnect(mockserver: MockServer) -> None:
crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl(mockserver.url("/drop"))
+ await crawler.crawl_async(mockserver.url("/drop"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
-@inline_callbacks_test
-def test_noconnect():
+@coroutine_test
+async def test_noconnect() -> None:
crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
+ await crawler.crawl_async("http://thereisdefinetelynosuchdomain.com")
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index e4898f8fd..ace6a2f31 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Self, cast
import pytest
@@ -22,16 +23,16 @@ if TYPE_CHECKING:
NO_CRAWLER = cast("Crawler", None)
-class BaseRobotParserTest:
- parser_cls: type[RobotParser]
-
- def _setUp(self, parser_cls: type[RobotParser]) -> None:
- self.parser_cls = parser_cls
+class TestRobotParserBase(ABC):
+ @property
+ @abstractmethod
+ def parser_cls(self) -> type[RobotParser]:
+ raise NotImplementedError
def _parse(self, robotstxt_body: bytes) -> RobotParser:
return build_from_crawler(self.parser_cls, NO_CRAWLER, robotstxt_body)
- def test_allowed(self):
+ def test_allowed(self) -> None:
robotstxt_robotstxt_body = (
b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10"
)
@@ -68,7 +69,7 @@ class BaseRobotParserTest:
rp = self._parse(robotstxt_robotstxt_body)
assert not rp.allowed("https://www.site.local/page", "*")
- def test_empty_response(self):
+ def test_empty_response(self) -> None:
"""empty response should equal 'allow all'"""
rp = self._parse(b"")
assert rp.allowed("https://site.local/", "*")
@@ -76,7 +77,7 @@ class BaseRobotParserTest:
assert rp.allowed("https://site.local/index.html", "*")
assert rp.allowed("https://site.local/disallowed", "*")
- def test_garbage_response(self):
+ def test_garbage_response(self) -> None:
"""garbage response should be discarded, equal 'allow all'"""
robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"
rp = self._parse(robotstxt_robotstxt_body)
@@ -85,17 +86,17 @@ class BaseRobotParserTest:
assert rp.allowed("https://site.local/index.html", "*")
assert rp.allowed("https://site.local/disallowed", "*")
- def test_crawl_delay(self):
+ def test_crawl_delay(self) -> None:
robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") == 10.0
- def test_crawl_delay_unset(self):
+ def test_crawl_delay_unset(self) -> None:
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") is None
- def test_unicode_url_and_useragent(self):
+ def test_unicode_url_and_useragent(self) -> None:
robotstxt_robotstxt_body = """
User-Agent: *
Disallow: /admin/
@@ -118,7 +119,7 @@ class BaseRobotParserTest:
class TestRobotParser:
- def test_crawl_delay_unsupported(self):
+ def test_crawl_delay_unsupported(self) -> None:
class AllowAllRobotParser(RobotParser):
@classmethod
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
@@ -134,33 +135,32 @@ class TestRobotParser:
class TestDecodeRobotsTxt:
- def test_native_string_conversion(self):
+ def test_native_string_conversion(self) -> None:
robotstxt_body = b"User-agent: *\nDisallow: /\n"
decoded_content = decode_robotstxt(
robotstxt_body, spider=None, to_native_str_type=True
)
assert decoded_content == "User-agent: *\nDisallow: /\n"
- def test_decode_utf8(self):
+ def test_decode_utf8(self) -> None:
robotstxt_body = b"User-agent: *\nDisallow: /\n"
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
assert decoded_content == "User-agent: *\nDisallow: /\n"
- def test_decode_non_utf8(self):
+ def test_decode_non_utf8(self) -> None:
robotstxt_body = b"User-agent: *\n\xffDisallow: /\n"
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
assert decoded_content == "User-agent: *\nDisallow: /\n"
# UTF-8 BOM at the beginning of the file ignored
- def test_decode_utf8_bom(self):
+ def test_decode_utf8_bom(self) -> None:
robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n"
decoded_content = decode_robotstxt(robotstxt_body, spider=None)
assert decoded_content == "User-agent: *\nDisallow: /\n"
-class TestPythonRobotParser(BaseRobotParserTest):
- def setup_method(self):
- super()._setUp(PythonRobotParser)
+class TestPythonRobotParser(TestRobotParserBase):
+ parser_cls = PythonRobotParser
@pytest.mark.skipif(
not STDLIB_IMPROVED_ROBOTFILEPARSER,
@@ -185,17 +185,15 @@ class TestPythonRobotParser(BaseRobotParserTest):
@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
-class TestRerpRobotParser(BaseRobotParserTest):
- def setup_method(self):
- super()._setUp(RerpRobotParser)
+class TestRerpRobotParser(TestRobotParserBase):
+ parser_cls = RerpRobotParser
def test_length_based_precedence(self) -> None:
pytest.skip("Rerp does not support length based directives precedence.")
-class TestProtegoRobotParser(BaseRobotParserTest):
- def setup_method(self):
- super()._setUp(ProtegoRobotParser)
+class TestProtegoRobotParser(TestRobotParserBase):
+ parser_cls = ProtegoRobotParser
def test_order_based_precedence(self) -> None:
pytest.skip("Protego does not support order based directives precedence.")
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 802edfd67..443331c69 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import weakref
import pytest
@@ -7,7 +9,7 @@ from scrapy.selector import Selector
class TestSelector:
- def test_simple_selection(self):
+ def test_simple_selection(self) -> None:
"""Simple selector tests"""
body = b"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
response = TextResponse(url="http://example.com", body=body, encoding="utf-8")
@@ -34,14 +36,14 @@ class TestSelector:
)
] == ["12"]
- def test_root_base_url(self):
+ def test_root_base_url(self) -> None:
body = b'<html><form action="/path"><input name="a" /></form></html>'
url = "http://example.com"
response = TextResponse(url=url, body=body, encoding="utf-8")
sel = Selector(response)
assert url == sel.root.base
- def test_flavor_detection(self):
+ def test_flavor_detection(self) -> None:
text = b'<div><img src="a.jpg"><p>Hello</div>'
sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8"))
assert sel.type == "xml"
@@ -77,7 +79,7 @@ class TestSelector:
)
assert Selector(response).type == "html"
- def test_http_header_encoding_precedence(self):
+ def test_http_header_encoding_precedence(self) -> None:
# '\xa3' = pound symbol in unicode
# '\xc2\xa3' = pound symbol in utf-8
# '\xa3' = pound symbol in latin-1 (iso-8859-1)
@@ -99,7 +101,7 @@ class TestSelector:
x = Selector(response)
assert x.xpath("//span[@id='blank']/text()").getall() == ["\xa3"]
- def test_badly_encoded_body(self):
+ def test_badly_encoded_body(self) -> None:
# \xe9 alone isn't valid utf8 sequence
r1 = TextResponse(
"http://www.example.com",
@@ -122,7 +124,7 @@ class TestSelector:
response = HtmlResponse("https://example.com", body=body, headers=headers)
assert response.css("p::text").get() == "£€"
- def test_weakref_slots(self):
+ def test_weakref_slots(self) -> None:
"""Check that classes are using slots and are weak-referenceable"""
x = Selector(text="")
weakref.ref(x)
@@ -130,11 +132,11 @@ class TestSelector:
f"{x.__class__.__name__} does not use __slots__"
)
- def test_response(self):
+ def test_response(self) -> None:
response = TextResponse(url="http://example.com", body=b"")
assert Selector(response).response is response
- def test_selector_bad_args(self):
+ def test_selector_bad_args(self) -> None:
with pytest.raises(ValueError, match="received both response and text"):
Selector(TextResponse(url="http://example.com", body=b""), text="")
diff --git a/tests/test_signals.py b/tests/test_signals.py
index fc9520f49..7e1426ed6 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
import pytest
@@ -26,34 +26,31 @@ class ItemSpider(Spider):
return {"index": response.meta["index"]}
-class TestMain:
- @coroutine_test
- async def test_scheduler_empty(self):
- crawler = get_crawler()
- calls = []
+@coroutine_test
+async def test_scheduler_empty() -> None:
+ crawler = get_crawler()
+ calls = []
- def track_call():
- calls.append(object())
+ def track_call() -> None:
+ calls.append(object())
- crawler.signals.connect(track_call, signals.scheduler_empty)
- await crawler.crawl_async()
- assert len(calls) >= 1
+ crawler.signals.connect(track_call, signals.scheduler_empty)
+ await crawler.crawl_async()
+ assert len(calls) >= 1
-class TestMockServer:
- def setup_method(self):
- self.items = []
+@coroutine_test
+async def test_simple_pipeline(mockserver: MockServer) -> None:
+ items: list[Any] = []
- async def _on_item_scraped(self, item):
+ async def _on_item_scraped(item: Any) -> None:
item = await get_from_asyncio_queue(item)
- self.items.append(item)
-
- @pytest.mark.only_asyncio
- @coroutine_test
- async def test_simple_pipeline(self, mockserver: MockServer) -> None:
- crawler = get_crawler(ItemSpider)
- crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
- await crawler.crawl_async(mockserver=mockserver)
- assert len(self.items) == 10
- for index in range(10):
- assert {"index": index} in self.items
+ items.append(item)
+
+ crawler = get_crawler(ItemSpider)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ await crawler.crawl_async(mockserver=mockserver)
+ assert len(items) == 10
+ for index in range(10):
+ assert {"index": index} in items
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index 46166815a..e35fc8fd0 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -27,6 +27,7 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
+ from collections.abc import Generator
from pathlib import Path
import queuelib
@@ -45,7 +46,7 @@ class TestRequestQueueBase(ABC):
def is_fifo(self) -> bool:
raise NotImplementedError
- def test_one_element(self, q: queuelib.queue.BaseQueue):
+ def test_one_element(self, q: queuelib.queue.BaseQueue) -> None:
assert len(q) == 0
assert q.peek() is None
assert q.pop() is None
@@ -62,7 +63,7 @@ class TestRequestQueueBase(ABC):
assert q.peek() is None
assert q.pop() is None
- def test_headers(self, q: queuelib.queue.BaseQueue):
+ def test_headers(self, q: queuelib.queue.BaseQueue) -> None:
req = Request(
"http://www.example.com",
headers=[("X-Multi", "a"), ("x-multi", "b"), ("Accept", None)],
@@ -72,7 +73,7 @@ class TestRequestQueueBase(ABC):
assert result is not None
assert result.headers == req.headers
- def test_order(self, q: queuelib.queue.BaseQueue):
+ def test_order(self, q: queuelib.queue.BaseQueue) -> None:
assert len(q) == 0
assert q.peek() is None
assert q.pop() is None
@@ -100,7 +101,9 @@ class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):
is_fifo = True
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
)
@@ -114,7 +117,9 @@ class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):
is_fifo = False
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
)
@@ -128,7 +133,9 @@ class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):
is_fifo = True
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
)
@@ -142,7 +149,9 @@ class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):
is_fifo = False
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
)
@@ -156,7 +165,9 @@ class TestPickleFifoSQLiteQueueRequest(TestRequestQueueBase):
is_fifo = True
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
PickleFifoSQLiteQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
)
@@ -170,7 +181,9 @@ class TestPickleLifoSQLiteQueueRequest(TestRequestQueueBase):
is_fifo = False
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
PickleLifoSQLiteQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
)
@@ -184,7 +197,9 @@ class TestMarshalFifoSQLiteQueueRequest(TestRequestQueueBase):
is_fifo = True
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
MarshalFifoSQLiteQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
)
@@ -198,7 +213,9 @@ class TestMarshalLifoSQLiteQueueRequest(TestRequestQueueBase):
is_fifo = False
@pytest.fixture
- def q(self, crawler, tmp_path):
+ def q(
+ self, crawler: Crawler, tmp_path: Path
+ ) -> Generator[queuelib.queue.BaseQueue]:
queue = build_from_crawler(
MarshalLifoSQLiteQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
)
@@ -242,7 +259,7 @@ class TestFifoMemoryQueueRequest(TestRequestQueueBase):
is_fifo = True
@pytest.fixture
- def q(self, crawler):
+ def q(self, crawler: Crawler) -> queuelib.queue.BaseQueue:
return build_from_crawler(FifoMemoryQueue, crawler)
@@ -250,5 +267,5 @@ class TestLifoMemoryQueueRequest(TestRequestQueueBase):
is_fifo = False
@pytest.fixture
- def q(self, crawler):
+ def q(self, crawler: Crawler) -> queuelib.queue.BaseQueue:
return build_from_crawler(LifoMemoryQueue, crawler)
diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py
index d4e9e8129..754702f5f 100644
--- a/tests/test_utils_decorators.py
+++ b/tests/test_utils_decorators.py
@@ -71,7 +71,7 @@ class TestInthread:
@coroutine_test
async def test_returns_deferred_with_result(self):
@inthread
- def multiply(a, b):
+ def multiply(a: int, b: int) -> int:
return a * b
deferred = multiply(6, 7)
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 55acfe890..a9e23166a 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -32,6 +32,7 @@ from tests.spiders import LogSpider
if TYPE_CHECKING:
from collections.abc import Generator, Mapping, MutableMapping
+ from pathlib import Path
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatterResult
@@ -78,7 +79,8 @@ class TestTopLevelFormatter:
class TestSpiderPlaceholderInLogFormat:
- def _log(self, log_format: str, extra: dict[str, Any] | None) -> str:
+ @staticmethod
+ def _log(log_format: str, extra: dict[str, Any] | None) -> str:
stream = StringIO()
settings = Settings({"LOG_FORMAT": log_format})
handler = _get_handler(settings)
@@ -217,7 +219,7 @@ class TestGetFormatter:
formatter = _get_formatter(handler, self._settings(LOG_COLOR=False))
assert type(formatter) is logging.Formatter
- def test_plain_for_file_handler(self, tmp_path: Any) -> None:
+ def test_plain_for_file_handler(self, tmp_path: Path) -> None:
handler = logging.FileHandler(tmp_path / "log.txt")
try:
formatter = _get_formatter(handler, self._settings())
@@ -342,9 +344,10 @@ class TestLogging:
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
- yield logger
-
- logger.removeHandler(handler)
+ try:
+ yield logger
+ finally:
+ logger.removeHandler(handler)
def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
@@ -401,10 +404,10 @@ class TestLoggingWithExtra:
logger = logging.getLogger("log_spider")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
-
- yield logger
-
- logger.removeHandler(handler)
+ try:
+ yield logger
+ finally:
+ logger.removeHandler(handler)
def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None:
log_message = "Foo message"
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 3cc56b283..0383a1a61 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -3,7 +3,6 @@ from __future__ import annotations
import os
import sys
from pathlib import Path
-from typing import TYPE_CHECKING
import pytest
@@ -17,35 +16,23 @@ from scrapy.utils.project import (
project_data_dir,
)
-if TYPE_CHECKING:
- from collections.abc import Generator
-
@pytest.fixture
-def proj_path(tmp_path: Path) -> Generator[Path]:
- prev_dir = Path.cwd()
+def proj_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
project_dir = tmp_path
-
- try:
- os.chdir(project_dir)
- Path("scrapy.cfg").touch()
-
- yield project_dir
- finally:
- os.chdir(prev_dir)
+ monkeypatch.chdir(project_dir)
+ Path("scrapy.cfg").touch()
+ return project_dir
@pytest.fixture
-def no_proj_path(tmp_path: Path) -> Generator[Path]:
+def no_proj_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""A working directory without a scrapy.cfg file, also isolated from the
user-wide and system-wide Scrapy configuration files."""
- prev_dir = Path.cwd()
- try:
- os.chdir(tmp_path)
- with set_environ(HOME=str(tmp_path), XDG_CONFIG_HOME=str(tmp_path)):
- yield tmp_path
- finally:
- os.chdir(prev_dir)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("HOME", str(tmp_path))
+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
+ return tmp_path
def test_data_path_outside_project() -> None: