Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Decouple queue tests from queuelib.tests. (#8372)

Andrey Rakhmatullin8fbc11e10e71
diff --git a/pyproject.toml b/pyproject.toml
index 5007475e8..2d7113e94 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -322,15 +322,6 @@ filterwarnings = [
# These are triggered by garbage collecting an unclosed AiohttpDownloadHandler.
"ignore:.*<function (ClientSession|BaseConnector)\\.__del__:pytest.PytestUnraisableExceptionWarning",
"ignore:Unclosed (client session|connector) <aiohttp\\.:ResourceWarning",
- # queuelib's test helpers (reused by tests/test_squeues.py) leave queue
- # files and SQLite connections open; see the comment above about the two
- # forms this can surface as. sqlite3's warning message has no path to
- # match on, so it cannot be scoped to queuelib's temporary directories the
- # way the file one is.
- "ignore:.*<_io\\.\\w+ name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning",
- "ignore:unclosed file <_io\\.\\w+ name=.*queuelib-tests-:ResourceWarning",
- "ignore:.*<sqlite3\\.Connection object at:pytest.PytestUnraisableExceptionWarning",
- "ignore:unclosed database in <sqlite3\\.Connection object at:ResourceWarning",
# itemadapter imports pydantic.v1, which warns on Python 3.14 and higher.
"ignore:Core Pydantic V1 functionality isn't compatible:UserWarning",
# pyftpdlib imports asynchat, removed in Python 3.12, on lower versions.
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 8e4772c8c..33007a5d3 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,9 +1,10 @@
+from __future__ import annotations
+
import pickle
import sys
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
import pytest
-from queuelib.tests import test_queue as t
from scrapy.http import Request
from scrapy.item import Field, Item
@@ -22,15 +23,32 @@ from scrapy.squeues import (
_serializable_queue,
)
-# The disk and SQLite queue classes are built at run time, so they are untyped.
-_MarshalFifoQueue: Any = _MarshalFifoSerializationDiskQueue
-_MarshalLifoQueue: Any = _MarshalLifoSerializationDiskQueue
-_PickleFifoQueue: Any = _PickleFifoSerializationDiskQueue
-_PickleLifoQueue: Any = _PickleLifoSerializationDiskQueue
-_MarshalFifoSQLiteQueue: Any = _MarshalFifoSerializationSQLiteQueue
-_MarshalLifoSQLiteQueue: Any = _MarshalLifoSerializationSQLiteQueue
-_PickleFifoSQLiteQueue: Any = _PickleFifoSerializationSQLiteQueue
-_PickleLifoSQLiteQueue: Any = _PickleLifoSerializationSQLiteQueue
+if TYPE_CHECKING:
+ from collections.abc import Generator
+ from pathlib import Path
+
+_PICKLE_QUEUES = [
+ pytest.param(_PickleFifoSerializationDiskQueue, id="pickle-fifo-disk"),
+ pytest.param(_PickleLifoSerializationDiskQueue, id="pickle-lifo-disk"),
+ pytest.param(_PickleFifoSerializationSQLiteQueue, id="pickle-fifo-sqlite"),
+ pytest.param(_PickleLifoSerializationSQLiteQueue, id="pickle-lifo-sqlite"),
+]
+_QUEUES = [
+ *_PICKLE_QUEUES,
+ pytest.param(_MarshalFifoSerializationDiskQueue, id="marshal-fifo-disk"),
+ pytest.param(_MarshalLifoSerializationDiskQueue, id="marshal-lifo-disk"),
+ pytest.param(_MarshalFifoSerializationSQLiteQueue, id="marshal-fifo-sqlite"),
+ pytest.param(_MarshalLifoSerializationSQLiteQueue, id="marshal-lifo-sqlite"),
+]
+
+
+def q(queue_cls: Any, tmp_path: Path) -> Generator[Any]:
+ queue = queue_cls(str(tmp_path / "queue"))
+ try:
+ yield queue
+ finally:
+ queue.close()
class MyItem(Item):
@@ -68,8 +86,15 @@ class MyLoader(ItemLoader):
name_out = staticmethod(_test_procesor)
-def nonserializable_object_test(self):
- q = self.queue()
[email protected]("queue_cls", _QUEUES)
+def test_serialize(q: Any) -> None:
+ for value in ["a", 123, {"a": "dict"}]:
+ q.push(value)
+ assert q.pop() == value
+
+
[email protected]("queue_cls", _QUEUES)
+def test_nonserializable_object(q: Any) -> None:
with pytest.raises(
ValueError,
match=r"unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
@@ -81,193 +106,50 @@ def nonserializable_object_test(self):
ValueError, match=r"unmarshallable object|can't pickle Selector objects"
):
q.push(sel)
- q.close()
-
-
-class FifoQueueTestMixin:
- def queue(self) -> Any:
- raise NotImplementedError
-
- def test_serialize(self):
- q = self.queue()
- q.push("a")
- q.push(123)
- q.push({"a": "dict"})
- assert q.pop() == "a"
- assert q.pop() == 123
- assert q.pop() == {"a": "dict"}
- q.close()
-
- test_nonserializable_object = nonserializable_object_test
-
-
-class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin):
- chunksize = 100000
-
- def queue(self) -> Any:
- return _MarshalFifoQueue(self.qpath, chunksize=self.chunksize)
-
-
-class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
- chunksize = 1
-
-
-class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
- chunksize = 2
-
-
-class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
- chunksize = 3
-
-
-class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
- chunksize = 4
-
-
-class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin):
- chunksize = 100000
-
- def queue(self) -> Any:
- return _PickleFifoQueue(self.qpath, chunksize=self.chunksize)
-
- def test_serialize_item(self):
- q = self.queue()
- i = MyItem(name="foo")
- q.push(i)
- i2 = q.pop()
- assert isinstance(i2, MyItem)
- assert i == i2
- q.close()
-
- def test_serialize_loader(self):
- q = self.queue()
- loader = MyLoader()
- q.push(loader)
- loader2 = q.pop()
- assert isinstance(loader2, MyLoader)
- assert loader2.default_item_class is MyItem
- assert loader2.name_out("x") == "xx"
- q.close()
-
- def test_serialize_request_recursive(self):
- q = self.queue()
- r = Request("http://www.example.com")
- r.meta["request"] = r
- q.push(r)
- r2 = q.pop()
- assert isinstance(r2, Request)
- assert r.url == r2.url
- assert r2.meta["request"] is r2
- q.close()
-
- def test_non_pickable_object(self):
- q = self.queue()
- with pytest.raises(
- ValueError,
- match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",
- ) as exc_info:
- q.push(lambda x: x)
- if sys.version_info >= (3, 14):
- assert isinstance(exc_info.value.__context__, pickle.PicklingError)
- else:
- assert isinstance(exc_info.value.__context__, AttributeError)
- sel = Selector(text="<html><body><p>some text</p></body></html>")
- with pytest.raises(
- ValueError, match="can't pickle Selector objects"
- ) as exc_info:
- q.push(sel)
- assert isinstance(exc_info.value.__context__, TypeError)
- q.close()
-class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
- chunksize = 1
[email protected]("queue_cls", _PICKLE_QUEUES)
+def test_serialize_item(q: Any) -> None:
+ i = MyItem(name="foo")
+ q.push(i)
+ i2 = q.pop()
+ assert isinstance(i2, MyItem)
+ assert i == i2
-class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
- chunksize = 2
[email protected]("queue_cls", _PICKLE_QUEUES)
+def test_serialize_loader(q: Any) -> None:
+ loader = MyLoader()
+ q.push(loader)
+ loader2 = q.pop()
+ assert isinstance(loader2, MyLoader)
+ assert loader2.default_item_class is MyItem
+ assert loader2.name_out("x") == "xx"
-class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
- chunksize = 3
[email protected]("queue_cls", _PICKLE_QUEUES)
+def test_serialize_request_recursive(q: Any) -> None:
+ r = Request("http://www.example.com")
+ r.meta["request"] = r
+ q.push(r)
+ r2 = q.pop()
+ assert isinstance(r2, Request)
+ assert r.url == r2.url
+ assert r2.meta["request"] is r2
-class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
- chunksize = 4
-
-
-class LifoQueueTestMixin:
- def queue(self) -> Any:
- raise NotImplementedError
-
- def test_serialize(self):
- q = self.queue()
- q.push("a")
- q.push(123)
- q.push({"a": "dict"})
- assert q.pop() == {"a": "dict"}
- assert q.pop() == 123
- assert q.pop() == "a"
- q.close()
-
- test_nonserializable_object = nonserializable_object_test
-
-
-class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoQueueTestMixin):
- def queue(self) -> Any:
- return _MarshalLifoQueue(self.qpath)
-
-
-class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoQueueTestMixin):
- def queue(self) -> Any:
- return _PickleLifoQueue(self.qpath)
-
- def test_serialize_item(self):
- q = self.queue()
- i = MyItem(name="foo")
- q.push(i)
- i2 = q.pop()
- assert isinstance(i2, MyItem)
- assert i == i2
- q.close()
-
- def test_serialize_loader(self):
- q = self.queue()
- loader = MyLoader()
- q.push(loader)
- loader2 = q.pop()
- assert isinstance(loader2, MyLoader)
- assert loader2.default_item_class is MyItem
- assert loader2.name_out("x") == "xx"
- q.close()
-
- def test_serialize_request_recursive(self):
- q = self.queue()
- r = Request("http://www.example.com")
- r.meta["request"] = r
- q.push(r)
- r2 = q.pop()
- assert isinstance(r2, Request)
- assert r.url == r2.url
- assert r2.meta["request"] is r2
- q.close()
-
-
-class MarshalFifoSQLiteQueueTest(t.FifoSQLiteQueueTest, FifoQueueTestMixin):
- def queue(self) -> Any:
- return _MarshalFifoSQLiteQueue(self.qpath)
-
-
-class PickleFifoSQLiteQueueTest(t.FifoSQLiteQueueTest, FifoQueueTestMixin):
- def queue(self) -> Any:
- return _PickleFifoSQLiteQueue(self.qpath)
-
-
-class MarshalLifoSQLiteQueueTest(t.LifoSQLiteQueueTest, LifoQueueTestMixin):
- def queue(self) -> Any:
- return _MarshalLifoSQLiteQueue(self.qpath)
-
-
-class PickleLifoSQLiteQueueTest(t.LifoSQLiteQueueTest, LifoQueueTestMixin):
- def queue(self) -> Any:
- return _PickleLifoSQLiteQueue(self.qpath)
[email protected]("queue_cls", _PICKLE_QUEUES)
+def test_non_pickable_object(q: Any) -> None:
+ with pytest.raises(
+ ValueError,
+ match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+ ) as exc_info:
+ q.push(lambda x: x)
+ if sys.version_info >= (3, 14):
+ assert isinstance(exc_info.value.__context__, pickle.PicklingError)
+ else:
+ assert isinstance(exc_info.value.__context__, AttributeError)
+ sel = Selector(text="<html><body><p>some text</p></body></html>")
+ with pytest.raises(ValueError, match="can't pickle Selector objects") as exc_info:
+ q.push(sel)
+ assert isinstance(exc_info.value.__context__, TypeError)
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index e35fc8fd0..af500aad9 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -255,6 +255,41 @@ def test_sqlite_queue_survives_unclean_shutdown(
queue.close()
+ ("queue_cls", "is_fifo"),
+ [
+ (PickleFifoDiskQueue, True),
+ (PickleLifoDiskQueue, False),
+ (MarshalFifoDiskQueue, True),
+ (MarshalLifoDiskQueue, False),
+ (PickleFifoSQLiteQueue, True),
+ (PickleLifoSQLiteQueue, False),
+ (MarshalFifoSQLiteQueue, True),
+ (MarshalLifoSQLiteQueue, False),
+ ],
+)
+def test_close_reopen(
+ crawler: Crawler, tmp_path: Path, queue_cls: Any, is_fifo: bool
+) -> None:
+ key = str(tmp_path / "queue")
+ urls = [f"https://example.com/{i}" for i in range(4)]
+ queue = build_from_crawler(queue_cls, crawler, key=key)
+ for url in urls:
+ queue.push(Request(url))
+ queue.pop()
+ queue.close()
+
+ expected = urls[1:] if is_fifo else urls[-2::-1]
+ queue = build_from_crawler(queue_cls, crawler, key=key)
+ try:
+ assert len(queue) == 3
+ assert queue.peek().url == expected[0]
+ assert [queue.pop().url for _ in expected] == expected
+ assert queue.pop() is None
+ finally:
+ queue.close()
+
+
class TestFifoMemoryQueueRequest(TestRequestQueueBase):
is_fifo = True