Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

revise/modernize item exporter example in docs

colinmorris8a7552370de2
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index b6139af92..28f5ad9c2 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -36,38 +36,36 @@ to export
3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
the end of the exporting process
-Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
-Exporter to export scraped items to different files, one per spider::
-
- from scrapy import signals
- from scrapy.exporters import XmlItemExporter
-
- class XmlExportPipeline(object):
-
- def __init__(self):
- self.files = {}
-
- @classmethod
- def from_crawler(cls, crawler):
- pipeline = cls()
- crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
- crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
- return pipeline
-
- def spider_opened(self, spider):
- file = open('%s_products.xml' % spider.name, 'w+b')
- self.files[spider] = file
- self.exporter = XmlItemExporter(file)
- self.exporter.start_exporting()
-
- def spider_closed(self, spider):
- self.exporter.finish_exporting()
- file = self.files.pop(spider)
- file.close()
-
- def process_item(self, item, spider):
- self.exporter.export_item(item)
- return item
+Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
+Item Exporters to group scraped items to different files according to the
+value of one of their fields::
+
+ from scrapy.exporters import XmlItemExporter
+
+ class PerYearXmlExportPipeline(object):
+ """Distribute items across multiple XML files according to their 'year' field"""
+
+ def open_spider(self, spider):
+ self.year_to_exporter = {}
+
+ def close_spider(self, spider):
+ for exporter in self.year_to_exporter.itervalues():
+ exporter.finish_exporting()
+ exporter.file.close()
+
+ def _exporter_for_item(self, item):
+ year = item['year']
+ if year not in self.year_to_exporter:
+ f = open('{}.xml'.format(year), 'w+b')
+ exporter = XmlItemExporter(f)
+ exporter.start_exporting()
+ self.year_to_exporter[year] = exporter
+ return self.year_to_exporter[year]
+
+ def process_item(self, item, spider):
+ exporter = self._exporter_for_item(item)
+ exporter.export_item(item)
+ return item
.. _topics-exporters-field-serialization: