Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Feeds: Item Filters (#5178)

D R Siddhartha4ddc9d6b55fa
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 26c247cdd..7a4b054e9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as
soon as a file reaches the maximum item count, that file is delivered to the
feed URI, allowing item delivery to start way before the end of the crawl.
+.. _item-filter:
+
+Item filtering
+==============
+
+.. versionadded:: VERSION
+
+You can filter items that you want to allow for a particular feed by using the
+``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
+the specified types will be added to the feed.
+
+The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`
+class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.
+
+You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
+method ``accepts`` and taking ``feed_options`` as an argument.
+
+For instance::
+
+ class MyCustomFilter:
+
+ def __init__(self, feed_options):
+ self.feed_options = feed_options
+
+ def accepts(self, item):
+ if "field1" in item and item["field1"] == "expected_data":
+ return True
+ return False
+
+
+You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.
+See :setting:`FEEDS` for examples.
+
+ItemFilter
+----------
+
+.. autoclass:: scrapy.extensions.feedexport.ItemFilter
+ :members:
+
Settings
========
@@ -311,6 +350,7 @@ For instance::
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
+ 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
'fields': None,
'indent': 4,
'item_export_kwargs': {
@@ -320,12 +360,14 @@ For instance::
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],
+ 'item_filter': MyCustomFilter1,
'encoding': 'latin1',
'indent': 8,
},
pathlib.Path('items.csv'): {
'format': 'csv',
'fields': ['price', 'name'],
+ 'item_filter': 'myproject.filters.MyCustomFilter2',
},
}
@@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
+- ``item_classes``: list of :ref:`item classes <topics-items>` to export.
+
+ If undefined or empty, all items are exported.
+
+ .. versionadded:: VERSION
+
+- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
+
+ :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
+
+ .. versionadded:: VERSION
+
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index bec114707..89dca12f4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
return builder(*preargs, uri, *args, **kwargs)
+class ItemFilter:
+ """
+ This will be used by FeedExporter to decide if an item should be allowed
+ to be exported to a particular feed.
+
+ :param feed_options: feed specific options passed from FeedExporter
+ :type feed_options: dict
+ """
+
+ def __init__(self, feed_options):
+ self.feed_options = feed_options
+ self.item_classes = set()
+
+ if 'item_classes' in self.feed_options:
+ for item_class in self.feed_options['item_classes']:
+ self.item_classes.add(load_object(item_class))
+
+ def accepts(self, item):
+ """
+ Return ``True`` if `item` should be exported or ``False`` otherwise.
+
+ :param item: scraped item which user wants to check if is acceptable
+ :type item: :ref:`Scrapy items <topics-items>`
+ :return: `True` if accepted, `False` otherwise
+ :rtype: bool
+ """
+ if self.item_classes:
+ return isinstance(item, tuple(self.item_classes))
+
+ return True # accept all items if none declared in item_classes
+
+
class IFeedStorage(Interface):
"""Interface that all Feed Storages must implement"""
@@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage):
class _FeedSlot:
- def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
+ def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter):
self.file = file
self.exporter = exporter
self.storage = storage
@@ -225,6 +257,7 @@ class _FeedSlot:
self.store_empty = store_empty
self.uri_template = uri_template
self.uri = uri
+ self.filter = filter
# flags
self.itemcount = 0
self._exporting = False
@@ -255,6 +288,7 @@ class FeedExporter:
self.settings = crawler.settings
self.feeds = {}
self.slots = []
+ self.filters = {}
if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
raise NotConfigured
@@ -269,12 +303,14 @@ class FeedExporter:
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
+ self.filters[uri] = self._load_filter(feed_options)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed_options in self.settings.getdict('FEEDS').items():
uri = str(uri) # handle pathlib.Path objects
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
+ self.filters[uri] = self._load_filter(feed_options)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
@@ -368,6 +404,7 @@ class FeedExporter:
store_empty=feed_options['store_empty'],
batch_id=batch_id,
uri_template=uri_template,
+ filter=self.filters[uri_template]
)
if slot.store_empty:
slot.start_exporting()
@@ -376,6 +413,10 @@ class FeedExporter:
def item_scraped(self, item, spider):
slots = []
for slot in self.slots:
+ if not slot.filter.accepts(item):
+ slots.append(slot) # if slot doesn't accept item, continue with next slot
+ continue
+
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
@@ -486,3 +527,8 @@ class FeedExporter:
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
uripar_function(params, spider)
return params
+
+ def _load_filter(self, feed_options):
+ # load the item filter if declared else load the default filter class
+ item_filter_class = load_object(feed_options.get("item_filter", ItemFilter))
+ return item_filter_class(feed_options)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index df7ec4461..81437b011 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):
egg = scrapy.Field()
baz = scrapy.Field()
+ class MyItem2(scrapy.Item):
+ foo = scrapy.Field()
+ hello = scrapy.Field()
+
def _random_temp_filename(self, inter_dir=''):
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
@@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_multiple_item_classes(self):
- class MyItem2(scrapy.Item):
- foo = scrapy.Field()
- hello = scrapy.Field()
-
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
- MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),
{'hello': 'world4', 'egg': 'spam4'},
]
@@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase):
yield self.assertExported(items, header, rows,
settings=settings, ordered=True)
+ @defer.inlineCallbacks
+ def test_export_based_on_item_classes(self):
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+ {'hello': 'world3', 'egg': 'spam3'},
+ ]
+
+ formats = {
+ 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',
+ 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',
+ 'jsonlines': (
+ b'{"foo": "bar1", "egg": "spam1"}\n'
+ b'{"hello": "world2", "foo": "bar2"}\n'
+ ),
+ 'xml': (
+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
+ b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'
+ b'</hello><egg>spam3</egg></item>\n</items>'
+ ),
+ }
+
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {
+ 'format': 'csv',
+ 'item_classes': [self.MyItem],
+ },
+ self._random_temp_filename(): {
+ 'format': 'json',
+ 'item_classes': [self.MyItem2],
+ },
+ self._random_temp_filename(): {
+ 'format': 'jsonlines',
+ 'item_classes': [self.MyItem, self.MyItem2],
+ },
+ self._random_temp_filename(): {
+ 'format': 'xml',
+ },
+ },
+ }
+
+ data = yield self.exported_data(items, settings)
+ for fmt, expected in formats.items():
+ self.assertEqual(expected, data[fmt])
+
+ @defer.inlineCallbacks
+ def test_export_based_on_custom_filters(self):
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+ {'hello': 'world3', 'egg': 'spam3'},
+ ]
+
+ MyItem = self.MyItem
+
+ class CustomFilter1:
+ def __init__(self, feed_options):
+ pass
+
+ def accepts(self, item):
+ return isinstance(item, MyItem)
+
+ class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
+ def accepts(self, item):
+ if 'foo' not in item.fields:
+ return False
+ return True
+
+ class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
+ def accepts(self, item):
+ if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":
+ return True
+ return False
+
+ formats = {
+ 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
+ 'xml': (
+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
+ b'world2</hello><foo>bar2</foo></item>\n</items>'
+ ),
+ 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',
+ }
+
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {
+ 'format': 'json',
+ 'item_filter': CustomFilter1,
+ },
+ self._random_temp_filename(): {
+ 'format': 'xml',
+ 'item_filter': CustomFilter2,
+ },
+ self._random_temp_filename(): {
+ 'format': 'jsonlines',
+ 'item_classes': [self.MyItem, self.MyItem2],
+ 'item_filter': CustomFilter3,
+ },
+ },
+ }
+
+ data = yield self.exported_data(items, settings)
+ for fmt, expected in formats.items():
+ self.assertEqual(expected, data[fmt])
+
@defer.inlineCallbacks
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as