Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Implement Item.deepcopy()

Adrián Chaves7c148fce5acc
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index ae44aecd3..d744fd9ea 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -40,6 +40,7 @@ objects. Here is an example::
name = scrapy.Field()
price = scrapy.Field()
stock = scrapy.Field()
+ tags = scrapy.Field()
last_updated = scrapy.Field(serializer=str)
.. note:: Those familiar with `Django`_ will notice that Scrapy Items are
@@ -155,18 +156,41 @@ To access all populated values, just use the typical `dict API`_::
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
-Other common tasks
-------------------
-Copying items::
+Copying items
+-------------
- >>> product2 = Product(product)
- >>> print(product2)
- Product(name='Desktop PC', price=1000)
+To copy an item, you must first decide whether you want a shallow copy or a
+deep copy.
- >>> product3 = product2.copy()
- >>> print(product3)
- Product(name='Desktop PC', price=1000)
+If your item contains mutable_ values like lists or dictionaries, a shallow
+copy will keep references to the same mutable values across all different
+copies.
+
+.. _mutable: https://docs.python.org/glossary.html#term-mutable
+
+For example, if you have an item with a list of tags, and you create a shallow
+copy of that item, both the original item and the copy have the same list of
+tags. Adding a tag to the list of one of the items will add the tag to the
+other item as well.
+
+If that is not the desired behavior, use a deep copy instead.
+
+See the `documentation of the copy module`_ for more information.
+
+.. _documentation of the copy module: https://docs.python.org/library/copy.html
+
+To create a shallow copy of an item, you can either call
+:meth:`~scrapy.item.Item.copy` on an existing item
+(``product2 = product.copy()``) or instantiate your item class from an existing
+item (``product2 = Product(product)``).
+
+To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
+(``product2 = product.deepcopy()``).
+
+
+Other common tasks
+------------------
Creating dicts from items::
diff --git a/scrapy/item.py b/scrapy/item.py
index aa05e9c69..031b80a2d 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst
from pprint import pformat
from collections import MutableMapping
+from copy import deepcopy
from abc import ABCMeta
import six
@@ -96,6 +97,13 @@ class DictItem(MutableMapping, BaseItem):
def copy(self):
return self.__class__(self)
+ def deepcopy(self):
+ """Return a `deep copy`_ of this item.
+
+ .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
+ """
+ return deepcopy(self)
+
@six.add_metaclass(ItemMeta)
class Item(DictItem):
diff --git a/tests/test_item.py b/tests/test_item.py
index 2c1eb0dd3..010d3b141 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -249,6 +249,14 @@ class ItemTest(unittest.TestCase):
copied_item['name'] = copied_item['name'].upper()
self.assertNotEqual(item['name'], copied_item['name'])
+ def test_deepcopy(self):
+ class TestItem(Item):
+ tags = Field()
+ item = TestItem({'tags': ['tag1']})
+ copied_item = item.deepcopy()
+ item['tags'].append('tag2')
+ assert item['tags'] != copied_item['tags']
+
class ItemMetaTest(unittest.TestCase):