Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not serialize unpickable objects (py3)
Eugenio Lacuestaa56540877c2c
diff --git a/scrapy/squeues.py b/scrapy/squeues.pyindex 21520f454..0b8f6af7d 100644--- a/scrapy/squeues.py+++ b/scrapy/squeues.py@@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize):def _pickle_serialize(obj):try:return pickle.dumps(obj, protocol=2)- # Python>=3.5 raises AttributeError here while- # Python<=3.4 raises pickle.PicklingError- except (pickle.PicklingError, AttributeError) as e:+ # Python<=3.4 raises pickle.PicklingError here while+ # Python>=3.5 raises AttributeError and+ # Python>=3.6 raises TypeError+ except (pickle.PicklingError, AttributeError, TypeError) as e:raise ValueError(str(e))PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 3a24348b4..d2f721241 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -1,3 +1,4 @@+from sys import version_infoimport picklefrom queuelib.tests import test_queue as t@@ -5,6 +6,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFiffrom scrapy.item import Item, Fieldfrom scrapy.http import Requestfrom scrapy.loader import ItemLoader+from scrapy.selector import Selectorclass TestItem(Item):name = Field()@@ -17,20 +19,23 @@ class TestLoader(ItemLoader):name_out = staticmethod(_test_procesor)def nonserializable_object_test(self):+ q = self.queue()try:pickle.dumps(lambda x: x)except Exception:# Trigger Twisted bug #7989import twisted.persisted.styles # NOQA- q = self.queue()self.assertRaises(ValueError, q.push, lambda x: x)else:# Use a different unpickleable objectclass A(object): passa = A()a.__reduce__ = a.__reduce_ex__ = None- q = self.queue()self.assertRaises(ValueError, q.push, a)+ if version_info.major == 3 and version_info.minor >= 6:+ # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)+ sel = Selector(text='<html><body><p>some text</p></body></html>')+ self.assertRaises(ValueError, q.push, sel)class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):