Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate unused squeues (#5117)
Eugenio Lacuesta02ae1deaf499
diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 44898ba08..16f7bf4b6 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -8,6 +8,7 @@ import picklefrom queuelib import queue+from scrapy.utils.deprecate import create_deprecated_classfrom scrapy.utils.reqser import request_to_dict, request_from_dict@@ -123,30 +124,60 @@ def _pickle_serialize(obj):raise ValueError(str(e)) from e-PickleFifoDiskQueueNonRequest = _serializable_queue(+_PickleFifoSerializationDiskQueue = _serializable_queue(_with_mkdir(queue.FifoDiskQueue),_pickle_serialize,pickle.loads)-PickleLifoDiskQueueNonRequest = _serializable_queue(+_PickleLifoSerializationDiskQueue = _serializable_queue(_with_mkdir(queue.LifoDiskQueue),_pickle_serialize,pickle.loads)-MarshalFifoDiskQueueNonRequest = _serializable_queue(+_MarshalFifoSerializationDiskQueue = _serializable_queue(_with_mkdir(queue.FifoDiskQueue),marshal.dumps,marshal.loads)-MarshalLifoDiskQueueNonRequest = _serializable_queue(+_MarshalLifoSerializationDiskQueue = _serializable_queue(_with_mkdir(queue.LifoDiskQueue),marshal.dumps,marshal.loads)-PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest)-PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest)-MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest)-MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest)+# public queue classes+PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)+PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)+MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)+MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)+++# deprecated queue classes+_subclass_warn_message = "{cls} inherits from deprecated class {old}"+_instance_warn_message = "{cls} is deprecated"+PickleFifoDiskQueueNonRequest = create_deprecated_class(+ name="PickleFifoDiskQueueNonRequest",+ new_class=_PickleFifoSerializationDiskQueue,+ subclass_warn_message=_subclass_warn_message,+ instance_warn_message=_instance_warn_message,+)+PickleLifoDiskQueueNonRequest = create_deprecated_class(+ name="PickleLifoDiskQueueNonRequest",+ new_class=_PickleLifoSerializationDiskQueue,+ subclass_warn_message=_subclass_warn_message,+ instance_warn_message=_instance_warn_message,+)+MarshalFifoDiskQueueNonRequest = create_deprecated_class(+ name="MarshalFifoDiskQueueNonRequest",+ new_class=_MarshalFifoSerializationDiskQueue,+ subclass_warn_message=_subclass_warn_message,+ instance_warn_message=_instance_warn_message,+)+MarshalLifoDiskQueueNonRequest = create_deprecated_class(+ name="MarshalLifoDiskQueueNonRequest",+ new_class=_MarshalLifoSerializationDiskQueue,+ subclass_warn_message=_subclass_warn_message,+ instance_warn_message=_instance_warn_message,+)diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex becacce62..acc821b83 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -3,10 +3,10 @@ import sysfrom queuelib.tests import test_queue as tfrom scrapy.squeues import (- MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,- MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue,- PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue,- PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue+ _MarshalFifoSerializationDiskQueue,+ _MarshalLifoSerializationDiskQueue,+ _PickleFifoSerializationDiskQueue,+ _PickleLifoSerializationDiskQueue,)from scrapy.item import Item, Fieldfrom scrapy.http import Request@@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):chunksize = 100000def queue(self):- return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)+ return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):@@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):chunksize = 100000def queue(self):- return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize)+ return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)def test_serialize_item(self):q = self.queue()@@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin:class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):def queue(self):- return MarshalLifoDiskQueue(self.qpath)+ return _MarshalLifoSerializationDiskQueue(self.qpath)class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):def queue(self):- return PickleLifoDiskQueue(self.qpath)+ return _PickleLifoSerializationDiskQueue(self.qpath)def test_serialize_item(self):q = self.queue()