Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Chain pickling exception, test_squeues.py updates

Eugenio Lacuesta93436f9d3a67
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 8d05bd0d0..c7ad4d53d 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -82,11 +82,10 @@ def _scrapy_non_serialization_queue(queue_class):
def _pickle_serialize(obj):
try:
return pickle.dumps(obj, protocol=4)
- # Python <= 3.4 raises pickle.PicklingError here while
- # 3.5 <= Python < 3.6 raises AttributeError and
- # Python >= 3.6 raises TypeError
+ # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
+ # TypeError is raised from parsel.Selector
except (pickle.PicklingError, AttributeError, TypeError) as e:
- raise ValueError(str(e))
+ raise ValueError(str(e)) from e
PickleFifoDiskQueueNonRequest = _serializable_queue(
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 7e997a25e..a20d242f4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -28,20 +28,7 @@ class TestLoader(ItemLoader):
def nonserializable_object_test(self):
q = self.queue()
- try:
- pickle.dumps(lambda x: x)
- except Exception:
- # Trigger Twisted bug #7989
- import twisted.persisted.styles # NOQA
- self.assertRaises(ValueError, q.push, lambda x: x)
- else:
- # Use a different unpickleable object
- class A:
- pass
-
- a = A()
- a.__reduce__ = a.__reduce_ex__ = None
- self.assertRaises(ValueError, q.push, a)
+ self.assertRaises(ValueError, q.push, lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text='<html><body><p>some text</p></body></html>')
self.assertRaises(ValueError, q.push, sel)
@@ -118,6 +105,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
self.assertEqual(r.url, r2.url)
assert r2.meta['request'] is r2
+ def test_non_pickable_object(self):
+ q = self.queue()
+ try:
+ q.push(lambda x: x)
+ except ValueError as exc:
+ self.assertIsInstance(exc.__context__, AttributeError)
+
+ sel = Selector(text='<html><body><p>some text</p></body></html>')
+ try:
+ q.push(sel)
+ except ValueError as exc:
+ self.assertIsInstance(exc.__context__, TypeError)
+
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 1