Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Do not serialize unpickable objects (py3)

Eugenio Lacuestaa56540877c2c
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 21520f454..0b8f6af7d 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
def _pickle_serialize(obj):
try:
return pickle.dumps(obj, protocol=2)
- # Python>=3.5 raises AttributeError here while
- # Python<=3.4 raises pickle.PicklingError
- except (pickle.PicklingError, AttributeError) as e:
+ # Python<=3.4 raises pickle.PicklingError here while
+ # Python>=3.5 raises AttributeError and
+ # Python>=3.6 raises TypeError
+ except (pickle.PicklingError, AttributeError, TypeError) as e:
raise ValueError(str(e))
PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 3a24348b4..d2f721241 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -1,3 +1,4 @@
+from sys import version_info
import pickle
from queuelib.tests import test_queue as t
@@ -5,6 +6,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif
from scrapy.item import Item, Field
from scrapy.http import Request
from scrapy.loader import ItemLoader
+from scrapy.selector import Selector
class TestItem(Item):
name = Field()
@@ -17,20 +19,23 @@ class TestLoader(ItemLoader):
name_out = staticmethod(_test_procesor)
def nonserializable_object_test(self):
+ q = self.queue()
try:
pickle.dumps(lambda x: x)
except Exception:
# Trigger Twisted bug #7989
import twisted.persisted.styles # NOQA
- q = self.queue()
self.assertRaises(ValueError, q.push, lambda x: x)
else:
# Use a different unpickleable object
class A(object): pass
a = A()
a.__reduce__ = a.__reduce_ex__ = None
- q = self.queue()
self.assertRaises(ValueError, q.push, a)
+ if version_info.major == 3 and version_info.minor >= 6:
+ # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
+ sel = Selector(text='<html><body><p>some text</p></body></html>')
+ self.assertRaises(ValueError, q.push, sel)
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):