Scrapy, a fast high-level web crawling & scraping framework for Python.
Feeds: Item Filters (#5178)
D R Siddhartha4ddc9d6b55fa
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 26c247cdd..7a4b054e9 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, assoon as a file reaches the maximum item count, that file is delivered to thefeed URI, allowing item delivery to start way before the end of the crawl.+.. _item-filter:++Item filtering+==============++.. versionadded:: VERSION++You can filter items that you want to allow for a particular feed by using the+``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of+the specified types will be added to the feed.++The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`+class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.++You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s+method ``accepts`` and taking ``feed_options`` as an argument.++For instance::++ class MyCustomFilter:++ def __init__(self, feed_options):+ self.feed_options = feed_options++ def accepts(self, item):+ if "field1" in item and item["field1"] == "expected_data":+ return True+ return False+++You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.+See :setting:`FEEDS` for examples.++ItemFilter+----------++.. autoclass:: scrapy.extensions.feedexport.ItemFilter+ :members:+Settings========@@ -311,6 +350,7 @@ For instance::'format': 'json','encoding': 'utf8','store_empty': False,+ 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],'fields': None,'indent': 4,'item_export_kwargs': {@@ -320,12 +360,14 @@ For instance::'/home/user/documents/items.xml': {'format': 'xml','fields': ['name', 'price'],+ 'item_filter': MyCustomFilter1,'encoding': 'latin1','indent': 8,},pathlib.Path('items.csv'): {'format': 'csv','fields': ['price', 'name'],+ 'item_filter': 'myproject.filters.MyCustomFilter2',},}@@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition:- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.+- ``item_classes``: list of :ref:`item classes <topics-items>` to export.++ If undefined or empty, all items are exported.++ .. versionadded:: VERSION++- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.++ :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.++ .. versionadded:: VERSION+- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex bec114707..89dca12f4 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):return builder(*preargs, uri, *args, **kwargs)+class ItemFilter:+ """+ This will be used by FeedExporter to decide if an item should be allowed+ to be exported to a particular feed.++ :param feed_options: feed specific options passed from FeedExporter+ :type feed_options: dict+ """++ def __init__(self, feed_options):+ self.feed_options = feed_options+ self.item_classes = set()++ if 'item_classes' in self.feed_options:+ for item_class in self.feed_options['item_classes']:+ self.item_classes.add(load_object(item_class))++ def accepts(self, item):+ """+ Return ``True`` if `item` should be exported or ``False`` otherwise.++ :param item: scraped item which user wants to check if is acceptable+ :type item: :ref:`Scrapy items <topics-items>`+ :return: `True` if accepted, `False` otherwise+ :rtype: bool+ """+ if self.item_classes:+ return isinstance(item, tuple(self.item_classes))++ return True # accept all items if none declared in item_classes++class IFeedStorage(Interface):"""Interface that all Feed Storages must implement"""@@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage):class _FeedSlot:- def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):+ def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter):self.file = fileself.exporter = exporterself.storage = storage@@ -225,6 +257,7 @@ class _FeedSlot:self.store_empty = store_emptyself.uri_template = uri_templateself.uri = uri+ self.filter = filter# flagsself.itemcount = 0self._exporting = False@@ -255,6 +288,7 @@ class FeedExporter:self.settings = crawler.settingsself.feeds = {}self.slots = []+ self.filters = {}if not self.settings['FEEDS'] and not self.settings['FEED_URI']:raise NotConfigured@@ -269,12 +303,14 @@ class FeedExporter:uri = str(self.settings['FEED_URI']) # handle pathlib.Path objectsfeed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)+ self.filters[uri] = self._load_filter(feed_options)# End: Backward compatibility for FEED_URI and FEED_FORMAT settings# 'FEEDS' setting takes precedence over 'FEED_URI'for uri, feed_options in self.settings.getdict('FEEDS').items():uri = str(uri) # handle pathlib.Path objectsself.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)+ self.filters[uri] = self._load_filter(feed_options)self.storages = self._load_components('FEED_STORAGES')self.exporters = self._load_components('FEED_EXPORTERS')@@ -368,6 +404,7 @@ class FeedExporter:store_empty=feed_options['store_empty'],batch_id=batch_id,uri_template=uri_template,+ filter=self.filters[uri_template])if slot.store_empty:slot.start_exporting()@@ -376,6 +413,10 @@ class FeedExporter:def item_scraped(self, item, spider):slots = []for slot in self.slots:+ if not slot.filter.accepts(item):+ slots.append(slot) # if slot doesn't accept item, continue with next slot+ continue+slot.start_exporting()slot.exporter.export_item(item)slot.itemcount += 1@@ -486,3 +527,8 @@ class FeedExporter:uripar_function = load_object(uri_params) if uri_params else lambda x, y: Noneuripar_function(params, spider)return params++ def _load_filter(self, feed_options):+ # load the item filter if declared else load the default filter class+ item_filter_class = load_object(feed_options.get("item_filter", ItemFilter))+ return item_filter_class(feed_options)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex df7ec4461..81437b011 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):egg = scrapy.Field()baz = scrapy.Field()+ class MyItem2(scrapy.Item):+ foo = scrapy.Field()+ hello = scrapy.Field()+def _random_temp_filename(self, inter_dir=''):chars = [random.choice(ascii_letters + digits) for _ in range(15)]filename = ''.join(chars)@@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_multiple_item_classes(self):- class MyItem2(scrapy.Item):- foo = scrapy.Field()- hello = scrapy.Field()-items = [self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),- MyItem2({'hello': 'world2', 'foo': 'bar2'}),+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),{'hello': 'world4', 'egg': 'spam4'},]@@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase):yield self.assertExported(items, header, rows,settings=settings, ordered=True)+ @defer.inlineCallbacks+ def test_export_based_on_item_classes(self):+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),+ {'hello': 'world3', 'egg': 'spam3'},+ ]++ formats = {+ 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',+ 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',+ 'jsonlines': (+ b'{"foo": "bar1", "egg": "spam1"}\n'+ b'{"hello": "world2", "foo": "bar2"}\n'+ ),+ 'xml': (+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'+ b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'+ b'</hello><egg>spam3</egg></item>\n</items>'+ ),+ }++ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {+ 'format': 'csv',+ 'item_classes': [self.MyItem],+ },+ self._random_temp_filename(): {+ 'format': 'json',+ 'item_classes': [self.MyItem2],+ },+ self._random_temp_filename(): {+ 'format': 'jsonlines',+ 'item_classes': [self.MyItem, self.MyItem2],+ },+ self._random_temp_filename(): {+ 'format': 'xml',+ },+ },+ }++ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ self.assertEqual(expected, data[fmt])++ @defer.inlineCallbacks+ def test_export_based_on_custom_filters(self):+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),+ {'hello': 'world3', 'egg': 'spam3'},+ ]++ MyItem = self.MyItem++ class CustomFilter1:+ def __init__(self, feed_options):+ pass++ def accepts(self, item):+ return isinstance(item, MyItem)++ class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):+ def accepts(self, item):+ if 'foo' not in item.fields:+ return False+ return True++ class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):+ def accepts(self, item):+ if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":+ return True+ return False++ formats = {+ 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',+ 'xml': (+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'+ b'world2</hello><foo>bar2</foo></item>\n</items>'+ ),+ 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',+ }++ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {+ 'format': 'json',+ 'item_filter': CustomFilter1,+ },+ self._random_temp_filename(): {+ 'format': 'xml',+ 'item_filter': CustomFilter2,+ },+ self._random_temp_filename(): {+ 'format': 'jsonlines',+ 'item_classes': [self.MyItem, self.MyItem2],+ 'item_filter': CustomFilter3,+ },+ },+ }++ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ self.assertEqual(expected, data[fmt])+@defer.inlineCallbacksdef test_export_dicts(self):# When dicts are used, only keys from the first row are used as