Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate unused squeues (#5117)

Eugenio Lacuesta02ae1deaf499
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 44898ba08..16f7bf4b6 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -8,6 +8,7 @@ import pickle
from queuelib import queue
+from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.reqser import request_to_dict, request_from_dict
@@ -123,30 +124,60 @@ def _pickle_serialize(obj):
raise ValueError(str(e)) from e
-PickleFifoDiskQueueNonRequest = _serializable_queue(
+_PickleFifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.FifoDiskQueue),
_pickle_serialize,
pickle.loads
)
-PickleLifoDiskQueueNonRequest = _serializable_queue(
+_PickleLifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.LifoDiskQueue),
_pickle_serialize,
pickle.loads
)
-MarshalFifoDiskQueueNonRequest = _serializable_queue(
+_MarshalFifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.FifoDiskQueue),
marshal.dumps,
marshal.loads
)
-MarshalLifoDiskQueueNonRequest = _serializable_queue(
+_MarshalLifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.LifoDiskQueue),
marshal.dumps,
marshal.loads
)
-PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest)
-PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest)
-MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest)
-MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest)
+# public queue classes
+PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)
+PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
+MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
+MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
+
+
+# deprecated queue classes
+_subclass_warn_message = "{cls} inherits from deprecated class {old}"
+_instance_warn_message = "{cls} is deprecated"
+PickleFifoDiskQueueNonRequest = create_deprecated_class(
+ name="PickleFifoDiskQueueNonRequest",
+ new_class=_PickleFifoSerializationDiskQueue,
+ subclass_warn_message=_subclass_warn_message,
+ instance_warn_message=_instance_warn_message,
+)
+PickleLifoDiskQueueNonRequest = create_deprecated_class(
+ name="PickleLifoDiskQueueNonRequest",
+ new_class=_PickleLifoSerializationDiskQueue,
+ subclass_warn_message=_subclass_warn_message,
+ instance_warn_message=_instance_warn_message,
+)
+MarshalFifoDiskQueueNonRequest = create_deprecated_class(
+ name="MarshalFifoDiskQueueNonRequest",
+ new_class=_MarshalFifoSerializationDiskQueue,
+ subclass_warn_message=_subclass_warn_message,
+ instance_warn_message=_instance_warn_message,
+)
+MarshalLifoDiskQueueNonRequest = create_deprecated_class(
+ name="MarshalLifoDiskQueueNonRequest",
+ new_class=_MarshalLifoSerializationDiskQueue,
+ subclass_warn_message=_subclass_warn_message,
+ instance_warn_message=_instance_warn_message,
+)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index becacce62..acc821b83 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -3,10 +3,10 @@ import sys
from queuelib.tests import test_queue as t
from scrapy.squeues import (
- MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
- MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue,
- PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue,
- PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue
+ _MarshalFifoSerializationDiskQueue,
+ _MarshalLifoSerializationDiskQueue,
+ _PickleFifoSerializationDiskQueue,
+ _PickleLifoSerializationDiskQueue,
)
from scrapy.item import Item, Field
from scrapy.http import Request
@@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
chunksize = 100000
def queue(self):
- return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+ return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
@@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
chunksize = 100000
def queue(self):
- return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+ return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
def test_serialize_item(self):
q = self.queue()
@@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin:
class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def queue(self):
- return MarshalLifoDiskQueue(self.qpath)
+ return _MarshalLifoSerializationDiskQueue(self.qpath)
class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def queue(self):
- return PickleLifoDiskQueue(self.qpath)
+ return _PickleLifoSerializationDiskQueue(self.qpath)
def test_serialize_item(self):
q = self.queue()