Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

resolve conflicts

BroodingKangaroo77333666f13a
diff --cc scrapy/extensions/feedexport.py
index 45c2971a6,61dad8726..1331782e3
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@@ -270,60 -296,8 +270,69 @@@ class FeedExporter
deferred_list.append(d)
return defer.DeferredList(deferred_list) if deferred_list else None
+ def _close_slot(self, slot, spider):
+ if not slot.itemcount and not slot.store_empty:
+ # We need to call slot.storage.store nonetheless to get the file
+ # properly closed.
+ return defer.maybeDeferred(slot.storage.store, slot.file)
+ slot.finish_exporting()
+ logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
+ log_args = {'format': slot.format,
+ 'itemcount': slot.itemcount,
+ 'uri': slot.uri}
+ d = defer.maybeDeferred(slot.storage.store, slot.file)
- d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
- extra={'spider': spider}))
- d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
- exc_info=failure_to_exc_info(f),
- extra={'spider': spider}))
++
++ # Use `largs=log_args` to copy log_args into function's scope
++ # instead of using `log_args` from the outer scope
++ d.addCallback(
++ lambda _, largs=log_args: logger.info(
++ logfmt % "Stored", largs, extra={'spider': spider}
++ )
++ )
++ d.addErrback(
++ lambda f, largs=log_args: logger.error(
++ logfmt % "Error storing", largs,
++ exc_info=failure_to_exc_info(f), extra={'spider': spider}
++ )
++ )
+ return d
+
+ def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
+ """
+ Redirect the output data stream to a new file.
+ Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
+ :param batch_id: sequence number of current batch
+ :param uri: uri of the new batch to start
+ :param feed: dict with parameters of feed
+ :param spider: user spider
+ :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
+ """
+ storage = self._get_storage(uri)
+ file = storage.open(spider)
+ exporter = self._get_exporter(
+ file=file,
+ format=feed['format'],
+ fields_to_export=feed['fields'],
+ encoding=feed['encoding'],
+ indent=feed['indent'],
+ )
+ slot = _FeedSlot(
+ file=file,
+ exporter=exporter,
+ storage=storage,
+ uri=uri,
+ format=feed['format'],
+ store_empty=feed['store_empty'],
+ batch_id=batch_id,
+ uri_template=uri_template,
+ )
+ if slot.store_empty:
+ slot.start_exporting()
+ return slot
+
def item_scraped(self, item, spider):
- for slot in self.slots:
+ slots = []
+ for idx, slot in enumerate(self.slots):
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
diff --cc tests/test_feedexport.py
index fecb17e29,e38644214..1a6a5624b
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -6,9 -6,8 +6,10 @@@ import shuti
import string
import tempfile
import warnings
+from abc import ABC, abstractmethod
+from collections import defaultdict
from io import BytesIO
+ from logging import getLogger
from pathlib import Path
from string import ascii_letters, digits
from unittest import mock
@@@ -395,8 -393,47 +398,48 @@@ class FromCrawlerFileFeedStorage(FileFe
pass
+ class DummyBlockingFeedStorage(BlockingFeedStorage):
+
+ def __init__(self, uri):
+ self.path = file_uri_to_path(uri)
+
+ def _store_in_thread(self, file):
+ dirname = os.path.dirname(self.path)
+ if dirname and not os.path.exists(dirname):
+ os.makedirs(dirname)
+ with open(self.path, 'ab') as output_file:
+ output_file.write(file.read())
+
+ file.close()
+
+
+ class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
+
+ def _store_in_thread(self, file):
+ raise OSError('Cannot store')
+
+
+ @implementer(IFeedStorage)
+ class LogOnStoreFileStorage:
+ """
+ This storage logs inside `store` method.
+ It can be used to make sure `store` method is invoked.
+ """
+
+ def __init__(self, uri):
+ self.path = file_uri_to_path(uri)
+ self.logger = getLogger()
+
+ def open(self, spider):
+ return tempfile.NamedTemporaryFile(prefix='feed-')
+
+ def store(self, file):
+ self.logger.info('Storage.store is called')
+ file.close()
+
+
-class FeedExportTest(unittest.TestCase):
+class FeedExportTestBase(ABC, unittest.TestCase):
+ __test__ = False
class MyItem(scrapy.Item):
foo = scrapy.Field()
@@@ -636,8 -670,27 +686,27 @@@ class FeedExportTest(FeedExportTestBase
'FEED_EXPORT_INDENT': None,
}
data = yield self.exported_no_data(settings)
- self.assertEqual(data[fmt], expctd)
+ self.assertEqual(expctd, data[fmt])
+ @defer.inlineCallbacks
+ def test_export_no_items_multiple_feeds(self):
+ """ Make sure that `storage.store` is called for every feed. """
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {'format': 'json'},
+ self._random_temp_filename(): {'format': 'xml'},
+ self._random_temp_filename(): {'format': 'csv'},
+ },
+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
+ 'FEED_STORE_EMPTY': False
+ }
+
+ with LogCapture() as log:
+ yield self.exported_no_data(settings)
+
+ print(log)
+ self.assertEqual(str(log).count('Storage.store is called'), 3)
+
@defer.inlineCallbacks
def test_export_multiple_item_classes(self):
@@@ -994,348 -1047,44 +1063,390 @@@
data = yield self.exported_no_data(settings)
self.assertEqual(data['csv'], b'')
+ @defer.inlineCallbacks
+ def test_multiple_feeds_success_logs_blocking_feed_storage(self):
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {'format': 'json'},
+ self._random_temp_filename(): {'format': 'xml'},
+ self._random_temp_filename(): {'format': 'csv'},
+ },
+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},
+ }
+ items = [
+ {'foo': 'bar1', 'baz': ''},
+ {'foo': 'bar2', 'baz': 'quux'},
+ ]
+ with LogCapture() as log:
+ yield self.exported_data(items, settings)
+
+ print(log)
+ for fmt in ['json', 'xml', 'csv']:
+ self.assertIn('Stored %s feed (2 items)' % fmt, str(log))
+
+ @defer.inlineCallbacks
+ def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {'format': 'json'},
+ self._random_temp_filename(): {'format': 'xml'},
+ self._random_temp_filename(): {'format': 'csv'},
+ },
+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},
+ }
+ items = [
+ {'foo': 'bar1', 'baz': ''},
+ {'foo': 'bar2', 'baz': 'quux'},
+ ]
+ with LogCapture() as log:
+ yield self.exported_data(items, settings)
+
+ print(log)
+ for fmt in ['json', 'xml', 'csv']:
+ self.assertIn('Error storing %s feed (2 items)' % fmt, str(log))
++
+
+class BatchDeliveriesTest(FeedExportTestBase):
+ __test__ = True
+ _file_mark = '_%(batch_time)s_#%(batch_id)s_'
+
+ @defer.inlineCallbacks
+ def run_and_export(self, spider_cls, settings):
+ """ Run spider with specified settings; return exported data. """
+
+ FEEDS = settings.get('FEEDS') or {}
+ settings['FEEDS'] = {
+ urljoin('file:', file_path): feed
+ for file_path, feed in FEEDS.items()
+ }
+ content = defaultdict(list)
+ try:
+ with MockServer() as s:
+ runner = CrawlerRunner(Settings(settings))
+ spider_cls.start_urls = [s.url('/')]
+ yield runner.crawl(spider_cls)
+
+ for path, feed in FEEDS.items():
+ dir_name = os.path.dirname(path)
+ for file in sorted(os.listdir(dir_name)):
+ with open(os.path.join(dir_name, file), 'rb') as f:
+ data = f.read()
+ content[feed['format']].append(data)
+ finally:
+ self.tearDown()
+ defer.returnValue(content)
+
+ @defer.inlineCallbacks
+ def assertExportedJsonLines(self, items, rows, settings=None):
+ settings = settings or {}
+ settings.update({
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
+ },
+ })
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+ rows = [{k: v for k, v in row.items() if v} for row in rows]
+ data = yield self.exported_data(items, settings)
+ for batch in data['jl']:
+ got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]
+ expected_batch, rows = rows[:batch_size], rows[batch_size:]
+ self.assertEqual(expected_batch, got_batch)
+
+ @defer.inlineCallbacks
+ def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+ settings = settings or {}
+ settings.update({
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
+ },
+ })
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+ data = yield self.exported_data(items, settings)
+ for batch in data['csv']:
+ got_batch = csv.DictReader(to_unicode(batch).splitlines())
+ self.assertEqual(list(header), got_batch.fieldnames)
+ expected_batch, rows = rows[:batch_size], rows[batch_size:]
+ self.assertEqual(expected_batch, list(got_batch))
+
+ @defer.inlineCallbacks
+ def assertExportedXml(self, items, rows, settings=None):
+ settings = settings or {}
+ settings.update({
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+ },
+ })
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+ rows = [{k: v for k, v in row.items() if v} for row in rows]
+ data = yield self.exported_data(items, settings)
+ for batch in data['xml']:
+ root = lxml.etree.fromstring(batch)
+ got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+ expected_batch, rows = rows[:batch_size], rows[batch_size:]
+ self.assertEqual(expected_batch, got_batch)
+
+ @defer.inlineCallbacks
+ def assertExportedMultiple(self, items, rows, settings=None):
+ settings = settings or {}
+ settings.update({
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
+ os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
+ },
+ })
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+ rows = [{k: v for k, v in row.items() if v} for row in rows]
+ data = yield self.exported_data(items, settings)
+ # XML
+ xml_rows = rows.copy()
+ for batch in data['xml']:
+ root = lxml.etree.fromstring(batch)
+ got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
+ expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
+ self.assertEqual(expected_batch, got_batch)
+ # JSON
+ json_rows = rows.copy()
+ for batch in data['json']:
+ got_batch = json.loads(batch.decode('utf-8'))
+ expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
+ self.assertEqual(expected_batch, got_batch)
+
+ @defer.inlineCallbacks
+ def assertExportedPickle(self, items, rows, settings=None):
+ settings = settings or {}
+ settings.update({
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
+ },
+ })
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+ rows = [{k: v for k, v in row.items() if v} for row in rows]
+ data = yield self.exported_data(items, settings)
+ import pickle
+ for batch in data['pickle']:
+ got_batch = self._load_until_eof(batch, load_func=pickle.load)
+ expected_batch, rows = rows[:batch_size], rows[batch_size:]
+ self.assertEqual(expected_batch, got_batch)
+
+ @defer.inlineCallbacks
+ def assertExportedMarshal(self, items, rows, settings=None):
+ settings = settings or {}
+ settings.update({
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
+ },
+ })
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+ rows = [{k: v for k, v in row.items() if v} for row in rows]
+ data = yield self.exported_data(items, settings)
+ import marshal
+ for batch in data['marshal']:
+ got_batch = self._load_until_eof(batch, load_func=marshal.load)
+ expected_batch, rows = rows[:batch_size], rows[batch_size:]
+ self.assertEqual(expected_batch, got_batch)
+
+ @defer.inlineCallbacks
+ def test_export_items(self):
+ """ Test partial deliveries in all supported formats """
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+ self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+ ]
+ rows = [
+ {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
+ {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
+ {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
+ ]
+ settings = {
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 2
+ }
+ header = self.MyItem.fields.keys()
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ def test_wrong_path(self):
+ """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {'format': 'xml'},
+ },
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+ }
+ crawler = get_crawler(settings_dict=settings)
+ self.assertRaises(NotConfigured, FeedExporter, crawler)
+
+ @defer.inlineCallbacks
+ def test_export_no_items_not_store_empty(self):
+ for fmt in ('json', 'jsonlines', 'xml', 'csv'):
+ settings = {
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+ },
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1
+ }
+ data = yield self.exported_no_data(settings)
+ data = dict(data)
+ self.assertEqual(b'', data[fmt][0])
+
+ @defer.inlineCallbacks
+ def test_export_no_items_store_empty(self):
+ formats = (
+ ('json', b'[]'),
+ ('jsonlines', b''),
+ ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+ ('csv', b''),
+ )
+
+ for fmt, expctd in formats:
+ settings = {
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
+ },
+ 'FEED_STORE_EMPTY': True,
+ 'FEED_EXPORT_INDENT': None,
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+ }
+ data = yield self.exported_no_data(settings)
+ data = dict(data)
+ self.assertEqual(expctd, data[fmt][0])
+
+ @defer.inlineCallbacks
+ def test_export_multiple_configs(self):
+ items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+ formats = {
+ 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
+ '[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
+ 'xml': [
+ (
+ '<?xml version="1.0" encoding="latin-1"?>\n'
+ '<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'
+ ).encode('latin-1'),
+ (
+ '<?xml version="1.0" encoding="latin-1"?>\n'
+ '<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'
+ ).encode('latin-1')
+ ],
+ 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
+ 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+ }
+
+ settings = {
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+ 'format': 'json',
+ 'indent': 0,
+ 'fields': ['bar'],
+ 'encoding': 'utf-8',
+ },
+ os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
+ 'format': 'xml',
+ 'indent': 2,
+ 'fields': ['foo'],
+ 'encoding': 'latin-1',
+ },
+ os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+ 'format': 'csv',
+ 'indent': None,
+ 'fields': ['bar', 'foo'],
+ 'encoding': 'utf-8',
+ },
+ },
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+ }
+ data = yield self.exported_data(items, settings)
+ for fmt, expected in formats.items():
+ for expected_batch, got_batch in zip(expected, data[fmt]):
+ self.assertEqual(expected_batch, got_batch)
+
+ @defer.inlineCallbacks
+ def test_batch_path_differ(self):
+ """
+ Test that the name of all batch files differ from each other.
+ So %(batch_time)s replaced with the current date.
+ """
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+ self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+ ]
+ settings = {
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
+ 'format': 'json',
+ },
+ },
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+ }
+ data = yield self.exported_data(items, settings)
+ self.assertEqual(len(items) + 1, len(data['json']))
+
+ @defer.inlineCallbacks
+ def test_s3_export(self):
+ """
+ Test export of items into s3 bucket.
+ S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
+ to perform this test:
+ [testenv]
+ setenv =
+ AWS_SECRET_ACCESS_KEY = ABCD
+ AWS_ACCESS_KEY_ID = ABCD
+ S3_TEST_BUCKET_NAME = ABCD
+ """
+ try:
+ import boto3
+ except ImportError:
+ raise unittest.SkipTest("S3FeedStorage requires boto3")
+
+ assert_aws_environ()
+ s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
+ access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+ secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+ if not s3_test_bucket_name:
+ raise unittest.SkipTest("No S3 BUCKET available for testing")
+
+ chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+ filename = ''.join(chars)
+ prefix = 'tmp/{filename}'.format(filename=filename)
+ s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(
+ bucket_name=s3_test_bucket_name, prefix=prefix
+ )
+ storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
+ settings = {
+ 'FEEDS': {
+ s3_test_file_uri: {
+ 'format': 'json',
+ },
+ },
+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
+ }
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
+ self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
+ ]
+ verifyObject(IFeedStorage, storage)
+
+ class TestSpider(scrapy.Spider):
+ name = 'testspider'
+
+ def parse(self, response):
+ for item in items:
+ yield item
+
+ s3 = boto3.resource('s3')
+ my_bucket = s3.Bucket(s3_test_bucket_name)
+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
+
+ with MockServer() as s:
+ runner = CrawlerRunner(Settings(settings))
+ TestSpider.start_urls = [s.url('/')]
+ yield runner.crawl(TestSpider)
+
+ for file_uri in my_bucket.objects.filter(Prefix=prefix):
+ content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
+ if not content and not items:
+ break
+ content = json.loads(content.decode('utf-8'))
+ expected_batch, items = items[:batch_size], items[batch_size:]
+ self.assertEqual(expected_batch, content)