Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix queue serialization test on PyPy

Konstantin Lopuhinc3d17659b334
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 48871ceeb..3a24348b4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,3 +1,5 @@
+import pickle
+
from queuelib.tests import test_queue as t
from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
from scrapy.item import Item, Field
@@ -14,6 +16,22 @@ class TestLoader(ItemLoader):
default_item_class = TestItem
name_out = staticmethod(_test_procesor)
+def nonserializable_object_test(self):
+ try:
+ pickle.dumps(lambda x: x)
+ except Exception:
+ # Trigger Twisted bug #7989
+ import twisted.persisted.styles # NOQA
+ q = self.queue()
+ self.assertRaises(ValueError, q.push, lambda x: x)
+ else:
+ # Use a different unpickleable object
+ class A(object): pass
+ a = A()
+ a.__reduce__ = a.__reduce_ex__ = None
+ q = self.queue()
+ self.assertRaises(ValueError, q.push, a)
+
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
chunksize = 100000
@@ -30,11 +48,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
self.assertEqual(q.pop(), 123)
self.assertEqual(q.pop(), {'a': 'dict'})
- def test_nonserializable_object(self):
- # Trigger Twisted bug #7989
- import twisted.persisted.styles # NOQA
- q = self.queue()
- self.assertRaises(ValueError, q.push, lambda x: x)
+ test_nonserializable_object = nonserializable_object_test
class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 1
@@ -110,11 +124,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
self.assertEqual(q.pop(), 123)
self.assertEqual(q.pop(), 'a')
- def test_nonserializable_object(self):
- # Trigger Twisted bug #7989
- import twisted.persisted.styles # NOQA
- q = self.queue()
- self.assertRaises(ValueError, q.push, lambda x: x)
+ test_nonserializable_object = nonserializable_object_test
class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest):