Scrapy, a fast high-level web crawling & scraping framework for Python.
Decouple queue tests from queuelib.tests. (#8372)
Andrey Rakhmatullin8fbc11e10e71
diff --git a/pyproject.toml b/pyproject.tomlindex 5007475e8..2d7113e94 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -322,15 +322,6 @@ filterwarnings = [# These are triggered by garbage collecting an unclosed AiohttpDownloadHandler."ignore:.*<function (ClientSession|BaseConnector)\\.__del__:pytest.PytestUnraisableExceptionWarning","ignore:Unclosed (client session|connector) <aiohttp\\.:ResourceWarning",- # queuelib's test helpers (reused by tests/test_squeues.py) leave queue- # files and SQLite connections open; see the comment above about the two- # forms this can surface as. sqlite3's warning message has no path to- # match on, so it cannot be scoped to queuelib's temporary directories the- # way the file one is.- "ignore:.*<_io\\.\\w+ name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning",- "ignore:unclosed file <_io\\.\\w+ name=.*queuelib-tests-:ResourceWarning",- "ignore:.*<sqlite3\\.Connection object at:pytest.PytestUnraisableExceptionWarning",- "ignore:unclosed database in <sqlite3\\.Connection object at:ResourceWarning",# itemadapter imports pydantic.v1, which warns on Python 3.14 and higher."ignore:Core Pydantic V1 functionality isn't compatible:UserWarning",# pyftpdlib imports asynchat, removed in Python 3.12, on lower versions.diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 8e4772c8c..33007a5d3 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -1,9 +1,10 @@+from __future__ import annotations+import pickleimport sys-from typing import Any, cast+from typing import TYPE_CHECKING, Any, castimport pytest-from queuelib.tests import test_queue as tfrom scrapy.http import Requestfrom scrapy.item import Field, Item@@ -22,15 +23,32 @@ from scrapy.squeues import (_serializable_queue,)-# The disk and SQLite queue classes are built at run time, so they are untyped.-_MarshalFifoQueue: Any = _MarshalFifoSerializationDiskQueue-_MarshalLifoQueue: Any = _MarshalLifoSerializationDiskQueue-_PickleFifoQueue: Any = _PickleFifoSerializationDiskQueue-_PickleLifoQueue: Any = _PickleLifoSerializationDiskQueue-_MarshalFifoSQLiteQueue: Any = _MarshalFifoSerializationSQLiteQueue-_MarshalLifoSQLiteQueue: Any = _MarshalLifoSerializationSQLiteQueue-_PickleFifoSQLiteQueue: Any = _PickleFifoSerializationSQLiteQueue-_PickleLifoSQLiteQueue: Any = _PickleLifoSerializationSQLiteQueue+if TYPE_CHECKING:+ from collections.abc import Generator+ from pathlib import Path++_PICKLE_QUEUES = [+ pytest.param(_PickleFifoSerializationDiskQueue, id="pickle-fifo-disk"),+ pytest.param(_PickleLifoSerializationDiskQueue, id="pickle-lifo-disk"),+ pytest.param(_PickleFifoSerializationSQLiteQueue, id="pickle-fifo-sqlite"),+ pytest.param(_PickleLifoSerializationSQLiteQueue, id="pickle-lifo-sqlite"),+]+_QUEUES = [+ *_PICKLE_QUEUES,+ pytest.param(_MarshalFifoSerializationDiskQueue, id="marshal-fifo-disk"),+ pytest.param(_MarshalLifoSerializationDiskQueue, id="marshal-lifo-disk"),+ pytest.param(_MarshalFifoSerializationSQLiteQueue, id="marshal-fifo-sqlite"),+ pytest.param(_MarshalLifoSerializationSQLiteQueue, id="marshal-lifo-sqlite"),+]+++def q(queue_cls: Any, tmp_path: Path) -> Generator[Any]:+ queue = queue_cls(str(tmp_path / "queue"))+ try:+ yield queue+ finally:+ queue.close()class MyItem(Item):@@ -68,8 +86,15 @@ class MyLoader(ItemLoader):name_out = staticmethod(_test_procesor)-def nonserializable_object_test(self):- q = self.queue()[email protected]("queue_cls", _QUEUES)+def test_serialize(q: Any) -> None:+ for value in ["a", 123, {"a": "dict"}]:+ q.push(value)+ assert q.pop() == value++[email protected]("queue_cls", _QUEUES)+def test_nonserializable_object(q: Any) -> None:with pytest.raises(ValueError,match=r"unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",@@ -81,193 +106,50 @@ def nonserializable_object_test(self):ValueError, match=r"unmarshallable object|can't pickle Selector objects"):q.push(sel)- q.close()---class FifoQueueTestMixin:- def queue(self) -> Any:- raise NotImplementedError-- def test_serialize(self):- q = self.queue()- q.push("a")- q.push(123)- q.push({"a": "dict"})- assert q.pop() == "a"- assert q.pop() == 123- assert q.pop() == {"a": "dict"}- q.close()-- test_nonserializable_object = nonserializable_object_test---class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin):- chunksize = 100000-- def queue(self) -> Any:- return _MarshalFifoQueue(self.qpath, chunksize=self.chunksize)---class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):- chunksize = 1---class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):- chunksize = 2---class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):- chunksize = 3---class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):- chunksize = 4---class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin):- chunksize = 100000-- def queue(self) -> Any:- return _PickleFifoQueue(self.qpath, chunksize=self.chunksize)-- def test_serialize_item(self):- q = self.queue()- i = MyItem(name="foo")- q.push(i)- i2 = q.pop()- assert isinstance(i2, MyItem)- assert i == i2- q.close()-- def test_serialize_loader(self):- q = self.queue()- loader = MyLoader()- q.push(loader)- loader2 = q.pop()- assert isinstance(loader2, MyLoader)- assert loader2.default_item_class is MyItem- assert loader2.name_out("x") == "xx"- q.close()-- def test_serialize_request_recursive(self):- q = self.queue()- r = Request("http://www.example.com")- r.meta["request"] = r- q.push(r)- r2 = q.pop()- assert isinstance(r2, Request)- assert r.url == r2.url- assert r2.meta["request"] is r2- q.close()-- def test_non_pickable_object(self):- q = self.queue()- with pytest.raises(- ValueError,- match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",- ) as exc_info:- q.push(lambda x: x)- if sys.version_info >= (3, 14):- assert isinstance(exc_info.value.__context__, pickle.PicklingError)- else:- assert isinstance(exc_info.value.__context__, AttributeError)- sel = Selector(text="<html><body><p>some text</p></body></html>")- with pytest.raises(- ValueError, match="can't pickle Selector objects"- ) as exc_info:- q.push(sel)- assert isinstance(exc_info.value.__context__, TypeError)- q.close()-class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):- chunksize = 1[email protected]("queue_cls", _PICKLE_QUEUES)+def test_serialize_item(q: Any) -> None:+ i = MyItem(name="foo")+ q.push(i)+ i2 = q.pop()+ assert isinstance(i2, MyItem)+ assert i == i2-class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):- chunksize = 2[email protected]("queue_cls", _PICKLE_QUEUES)+def test_serialize_loader(q: Any) -> None:+ loader = MyLoader()+ q.push(loader)+ loader2 = q.pop()+ assert isinstance(loader2, MyLoader)+ assert loader2.default_item_class is MyItem+ assert loader2.name_out("x") == "xx"-class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):- chunksize = 3[email protected]("queue_cls", _PICKLE_QUEUES)+def test_serialize_request_recursive(q: Any) -> None:+ r = Request("http://www.example.com")+ r.meta["request"] = r+ q.push(r)+ r2 = q.pop()+ assert isinstance(r2, Request)+ assert r.url == r2.url+ assert r2.meta["request"] is r2-class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):- chunksize = 4---class LifoQueueTestMixin:- def queue(self) -> Any:- raise NotImplementedError-- def test_serialize(self):- q = self.queue()- q.push("a")- q.push(123)- q.push({"a": "dict"})- assert q.pop() == {"a": "dict"}- assert q.pop() == 123- assert q.pop() == "a"- q.close()-- test_nonserializable_object = nonserializable_object_test---class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoQueueTestMixin):- def queue(self) -> Any:- return _MarshalLifoQueue(self.qpath)---class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoQueueTestMixin):- def queue(self) -> Any:- return _PickleLifoQueue(self.qpath)-- def test_serialize_item(self):- q = self.queue()- i = MyItem(name="foo")- q.push(i)- i2 = q.pop()- assert isinstance(i2, MyItem)- assert i == i2- q.close()-- def test_serialize_loader(self):- q = self.queue()- loader = MyLoader()- q.push(loader)- loader2 = q.pop()- assert isinstance(loader2, MyLoader)- assert loader2.default_item_class is MyItem- assert loader2.name_out("x") == "xx"- q.close()-- def test_serialize_request_recursive(self):- q = self.queue()- r = Request("http://www.example.com")- r.meta["request"] = r- q.push(r)- r2 = q.pop()- assert isinstance(r2, Request)- assert r.url == r2.url- assert r2.meta["request"] is r2- q.close()---class MarshalFifoSQLiteQueueTest(t.FifoSQLiteQueueTest, FifoQueueTestMixin):- def queue(self) -> Any:- return _MarshalFifoSQLiteQueue(self.qpath)---class PickleFifoSQLiteQueueTest(t.FifoSQLiteQueueTest, FifoQueueTestMixin):- def queue(self) -> Any:- return _PickleFifoSQLiteQueue(self.qpath)---class MarshalLifoSQLiteQueueTest(t.LifoSQLiteQueueTest, LifoQueueTestMixin):- def queue(self) -> Any:- return _MarshalLifoSQLiteQueue(self.qpath)---class PickleLifoSQLiteQueueTest(t.LifoSQLiteQueueTest, LifoQueueTestMixin):- def queue(self) -> Any:- return _PickleLifoSQLiteQueue(self.qpath)[email protected]("queue_cls", _PICKLE_QUEUES)+def test_non_pickable_object(q: Any) -> None:+ with pytest.raises(+ ValueError,+ match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",+ ) as exc_info:+ q.push(lambda x: x)+ if sys.version_info >= (3, 14):+ assert isinstance(exc_info.value.__context__, pickle.PicklingError)+ else:+ assert isinstance(exc_info.value.__context__, AttributeError)+ sel = Selector(text="<html><body><p>some text</p></body></html>")+ with pytest.raises(ValueError, match="can't pickle Selector objects") as exc_info:+ q.push(sel)+ assert isinstance(exc_info.value.__context__, TypeError)diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex e35fc8fd0..af500aad9 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -255,6 +255,41 @@ def test_sqlite_queue_survives_unclean_shutdown(queue.close()+ ("queue_cls", "is_fifo"),+ [+ (PickleFifoDiskQueue, True),+ (PickleLifoDiskQueue, False),+ (MarshalFifoDiskQueue, True),+ (MarshalLifoDiskQueue, False),+ (PickleFifoSQLiteQueue, True),+ (PickleLifoSQLiteQueue, False),+ (MarshalFifoSQLiteQueue, True),+ (MarshalLifoSQLiteQueue, False),+ ],+)+def test_close_reopen(+ crawler: Crawler, tmp_path: Path, queue_cls: Any, is_fifo: bool+) -> None:+ key = str(tmp_path / "queue")+ urls = [f"https://example.com/{i}" for i in range(4)]+ queue = build_from_crawler(queue_cls, crawler, key=key)+ for url in urls:+ queue.push(Request(url))+ queue.pop()+ queue.close()++ expected = urls[1:] if is_fifo else urls[-2::-1]+ queue = build_from_crawler(queue_cls, crawler, key=key)+ try:+ assert len(queue) == 3+ assert queue.peek().url == expected[0]+ assert [queue.pop().url for _ in expected] == expected+ assert queue.pop() is None+ finally:+ queue.close()++class TestFifoMemoryQueueRequest(TestRequestQueueBase):is_fifo = True