Scrapy, a fast high-level web crawling & scraping framework for Python.
Chain pickling exception, test_squeues.py updates
Eugenio Lacuesta93436f9d3a67
diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 8d05bd0d0..c7ad4d53d 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -82,11 +82,10 @@ def _scrapy_non_serialization_queue(queue_class):def _pickle_serialize(obj):try:return pickle.dumps(obj, protocol=4)- # Python <= 3.4 raises pickle.PicklingError here while- # 3.5 <= Python < 3.6 raises AttributeError and- # Python >= 3.6 raises TypeError+ # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)+ # TypeError is raised from parsel.Selectorexcept (pickle.PicklingError, AttributeError, TypeError) as e:- raise ValueError(str(e))+ raise ValueError(str(e)) from ePickleFifoDiskQueueNonRequest = _serializable_queue(diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 7e997a25e..a20d242f4 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -28,20 +28,7 @@ class TestLoader(ItemLoader):def nonserializable_object_test(self):q = self.queue()- try:- pickle.dumps(lambda x: x)- except Exception:- # Trigger Twisted bug #7989- import twisted.persisted.styles # NOQA- self.assertRaises(ValueError, q.push, lambda x: x)- else:- # Use a different unpickleable object- class A:- pass-- a = A()- a.__reduce__ = a.__reduce_ex__ = None- self.assertRaises(ValueError, q.push, a)+ self.assertRaises(ValueError, q.push, lambda x: x)# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)sel = Selector(text='<html><body><p>some text</p></body></html>')self.assertRaises(ValueError, q.push, sel)@@ -118,6 +105,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):self.assertEqual(r.url, r2.url)assert r2.meta['request'] is r2+ def test_non_pickable_object(self):+ q = self.queue()+ try:+ q.push(lambda x: x)+ except ValueError as exc:+ self.assertIsInstance(exc.__context__, AttributeError)++ sel = Selector(text='<html><body><p>some text</p></body></html>')+ try:+ q.push(sel)+ except ValueError as exc:+ self.assertIsInstance(exc.__context__, TypeError)+class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):chunksize = 1