Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Provide complete API documentation coverage of scrapy.item

Adrián Chaves1236e9e81ecd
diff --git a/docs/conf.py b/docs/conf.py
index fa257dead..34dd5bcb7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -262,6 +262,9 @@ coverage_ignore_pyobjects = [
# details that are not documented.
r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
+
+ # Never documented before, and deprecated now.
+ r'^scrapy\.item\.DictItem$',
]
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 60fbc82f8..260f5882c 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -263,3 +263,9 @@ Field objects
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
+Other classes related to Item
+=============================
+
+.. autoclass:: BaseItem
+
+.. autoclass:: ItemMeta
diff --git a/scrapy/item.py b/scrapy/item.py
index 9d4786788..73b8f54b0 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,13 +4,15 @@ Scrapy Item
See documentation in docs/topics/item.rst
"""
+import collections
from abc import ABCMeta
-from pprint import pformat
from copy import deepcopy
-import collections
+from pprint import pformat
+from warnings import warn
import six
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
@@ -21,7 +23,19 @@ else:
class BaseItem(object_ref):
- """Base class for all scraped items."""
+ """Base class for all scraped items.
+
+ In Scrapy, an object is considered an *item* if it is an instance of either
+ :class:`BaseItem` or :class:`dict`. For example, when the output of a
+ spider callback is evaluated, only instances of :class:`BaseItem` or
+ :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
+
+ If you need instances of a custom class to be considered items by Scrapy,
+ you must inherit from either :class:`BaseItem` or :class:`dict`.
+
+ Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
+ :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+ """
pass
@@ -30,6 +44,10 @@ class Field(dict):
class ItemMeta(ABCMeta):
+ """Metaclass_ of :class:`Item` that handles field definitions.
+
+ .. _metaclass: https://realpython.com/python-metaclasses
+ """
def __new__(mcs, class_name, bases, attrs):
classcell = attrs.pop('__classcell__', None)
@@ -56,6 +74,13 @@ class DictItem(MutableMapping, BaseItem):
fields = {}
+ def __new__(cls, *args, **kwargs):
+ if issubclass(cls, DictItem) and not issubclass(cls, Item):
+ warn('scrapy.item.DictItem is deprecated, please use '
+ 'scrapy.item.Item instead',
+ ScrapyDeprecationWarning, stacklevel=2)
+ return super(DictItem, cls).__new__(cls, *args, **kwargs)
+
def __init__(self, *args, **kwargs):
self._values = {}
if args or kwargs: # avoid creating dict for most common case
diff --git a/tests/test_item.py b/tests/test_item.py
index 010d3b141..947566686 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,9 +1,11 @@
import sys
import unittest
+from warnings import catch_warnings
import six
-from scrapy.item import ABCMeta, Item, ItemMeta, Field
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
from tests import mock
@@ -257,6 +259,17 @@ class ItemTest(unittest.TestCase):
item['tags'].append('tag2')
assert item['tags'] != copied_item['tags']
+ def test_dictitem_deprecation_warning(self):
+ """Make sure the DictItem deprecation warning is not issued for
+ Item"""
+ with catch_warnings(record=True) as warnings:
+ item = Item()
+ self.assertEqual(len(warnings), 0)
+ class SubclassedItem(Item):
+ pass
+ subclassed_item = SubclassedItem()
+ self.assertEqual(len(warnings), 0)
+
class ItemMetaTest(unittest.TestCase):
@@ -302,5 +315,20 @@ class ItemMetaClassCellRegression(unittest.TestCase):
super(MyItem, self).__init__(*args, **kwargs)
+class DictItemTest(unittest.TestCase):
+
+ def test_deprecation_warning(self):
+ with catch_warnings(record=True) as warnings:
+ dict_item = DictItem()
+ self.assertEqual(len(warnings), 1)
+ self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+ with catch_warnings(record=True) as warnings:
+ class SubclassedDictItem(DictItem):
+ pass
+ subclassed_dict_item = SubclassedDictItem()
+ self.assertEqual(len(warnings), 1)
+ self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+
if __name__ == "__main__":
unittest.main()