Scrapy, a fast high-level web crawling & scraping framework for Python.
resolve conflicts
BroodingKangaroo77333666f13a
diff --cc scrapy/extensions/feedexport.pyindex 45c2971a6,61dad8726..1331782e3--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@@ -270,60 -296,8 +270,69 @@@ class FeedExporterdeferred_list.append(d)return defer.DeferredList(deferred_list) if deferred_list else None+ def _close_slot(self, slot, spider):+ if not slot.itemcount and not slot.store_empty:+ # We need to call slot.storage.store nonetheless to get the file+ # properly closed.+ return defer.maybeDeferred(slot.storage.store, slot.file)+ slot.finish_exporting()+ logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"+ log_args = {'format': slot.format,+ 'itemcount': slot.itemcount,+ 'uri': slot.uri}+ d = defer.maybeDeferred(slot.storage.store, slot.file)- d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,- extra={'spider': spider}))- d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,- exc_info=failure_to_exc_info(f),- extra={'spider': spider}))++++ # Use `largs=log_args` to copy log_args into function's scope++ # instead of using `log_args` from the outer scope++ d.addCallback(++ lambda _, largs=log_args: logger.info(++ logfmt % "Stored", largs, extra={'spider': spider}++ )++ )++ d.addErrback(++ lambda f, largs=log_args: logger.error(++ logfmt % "Error storing", largs,++ exc_info=failure_to_exc_info(f), extra={'spider': spider}++ )++ )+ return d++ def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):+ """+ Redirect the output data stream to a new file.+ Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.+ :param batch_id: sequence number of current batch+ :param uri: uri of the new batch to start+ :param feed: dict with parameters of feed+ :param spider: user spider+ :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri+ """+ storage = self._get_storage(uri)+ file = storage.open(spider)+ exporter = self._get_exporter(+ file=file,+ format=feed['format'],+ fields_to_export=feed['fields'],+ encoding=feed['encoding'],+ indent=feed['indent'],+ )+ slot = _FeedSlot(+ file=file,+ exporter=exporter,+ storage=storage,+ uri=uri,+ format=feed['format'],+ store_empty=feed['store_empty'],+ batch_id=batch_id,+ uri_template=uri_template,+ )+ if slot.store_empty:+ slot.start_exporting()+ return slot+def item_scraped(self, item, spider):- for slot in self.slots:+ slots = []+ for idx, slot in enumerate(self.slots):slot.start_exporting()slot.exporter.export_item(item)slot.itemcount += 1diff --cc tests/test_feedexport.pyindex fecb17e29,e38644214..1a6a5624b--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@@ -6,9 -6,8 +6,10 @@@ import shutiimport stringimport tempfileimport warnings+from abc import ABC, abstractmethod+from collections import defaultdictfrom io import BytesIO+ from logging import getLoggerfrom pathlib import Pathfrom string import ascii_letters, digitsfrom unittest import mock@@@ -395,8 -393,47 +398,48 @@@ class FromCrawlerFileFeedStorage(FileFepass+ class DummyBlockingFeedStorage(BlockingFeedStorage):++ def __init__(self, uri):+ self.path = file_uri_to_path(uri)++ def _store_in_thread(self, file):+ dirname = os.path.dirname(self.path)+ if dirname and not os.path.exists(dirname):+ os.makedirs(dirname)+ with open(self.path, 'ab') as output_file:+ output_file.write(file.read())++ file.close()+++ class FailingBlockingFeedStorage(DummyBlockingFeedStorage):++ def _store_in_thread(self, file):+ raise OSError('Cannot store')+++ @implementer(IFeedStorage)+ class LogOnStoreFileStorage:+ """+ This storage logs inside `store` method.+ It can be used to make sure `store` method is invoked.+ """++ def __init__(self, uri):+ self.path = file_uri_to_path(uri)+ self.logger = getLogger()++ def open(self, spider):+ return tempfile.NamedTemporaryFile(prefix='feed-')++ def store(self, file):+ self.logger.info('Storage.store is called')+ file.close()++-class FeedExportTest(unittest.TestCase):+class FeedExportTestBase(ABC, unittest.TestCase):+ __test__ = Falseclass MyItem(scrapy.Item):foo = scrapy.Field()@@@ -636,8 -670,27 +686,27 @@@ class FeedExportTest(FeedExportTestBase'FEED_EXPORT_INDENT': None,}data = yield self.exported_no_data(settings)- self.assertEqual(data[fmt], expctd)+ self.assertEqual(expctd, data[fmt])+ @defer.inlineCallbacks+ def test_export_no_items_multiple_feeds(self):+ """ Make sure that `storage.store` is called for every feed. """+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'json'},+ self._random_temp_filename(): {'format': 'xml'},+ self._random_temp_filename(): {'format': 'csv'},+ },+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},+ 'FEED_STORE_EMPTY': False+ }++ with LogCapture() as log:+ yield self.exported_no_data(settings)++ print(log)+ self.assertEqual(str(log).count('Storage.store is called'), 3)+@defer.inlineCallbacksdef test_export_multiple_item_classes(self):@@@ -994,348 -1047,44 +1063,390 @@@data = yield self.exported_no_data(settings)self.assertEqual(data['csv'], b'')+ @defer.inlineCallbacks+ def test_multiple_feeds_success_logs_blocking_feed_storage(self):+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'json'},+ self._random_temp_filename(): {'format': 'xml'},+ self._random_temp_filename(): {'format': 'csv'},+ },+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},+ }+ items = [+ {'foo': 'bar1', 'baz': ''},+ {'foo': 'bar2', 'baz': 'quux'},+ ]+ with LogCapture() as log:+ yield self.exported_data(items, settings)++ print(log)+ for fmt in ['json', 'xml', 'csv']:+ self.assertIn('Stored %s feed (2 items)' % fmt, str(log))++ @defer.inlineCallbacks+ def test_multiple_feeds_failing_logs_blocking_feed_storage(self):+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'json'},+ self._random_temp_filename(): {'format': 'xml'},+ self._random_temp_filename(): {'format': 'csv'},+ },+ 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},+ }+ items = [+ {'foo': 'bar1', 'baz': ''},+ {'foo': 'bar2', 'baz': 'quux'},+ ]+ with LogCapture() as log:+ yield self.exported_data(items, settings)++ print(log)+ for fmt in ['json', 'xml', 'csv']:+ self.assertIn('Error storing %s feed (2 items)' % fmt, str(log))++++class BatchDeliveriesTest(FeedExportTestBase):+ __test__ = True+ _file_mark = '_%(batch_time)s_#%(batch_id)s_'++ @defer.inlineCallbacks+ def run_and_export(self, spider_cls, settings):+ """ Run spider with specified settings; return exported data. """++ FEEDS = settings.get('FEEDS') or {}+ settings['FEEDS'] = {+ urljoin('file:', file_path): feed+ for file_path, feed in FEEDS.items()+ }+ content = defaultdict(list)+ try:+ with MockServer() as s:+ runner = CrawlerRunner(Settings(settings))+ spider_cls.start_urls = [s.url('/')]+ yield runner.crawl(spider_cls)++ for path, feed in FEEDS.items():+ dir_name = os.path.dirname(path)+ for file in sorted(os.listdir(dir_name)):+ with open(os.path.join(dir_name, file), 'rb') as f:+ data = f.read()+ content[feed['format']].append(data)+ finally:+ self.tearDown()+ defer.returnValue(content)++ @defer.inlineCallbacks+ def assertExportedJsonLines(self, items, rows, settings=None):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},+ },+ })+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']+ rows = [{k: v for k, v in row.items() if v} for row in rows]+ data = yield self.exported_data(items, settings)+ for batch in data['jl']:+ got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]+ expected_batch, rows = rows[:batch_size], rows[batch_size:]+ self.assertEqual(expected_batch, got_batch)++ @defer.inlineCallbacks+ def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},+ },+ })+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']+ data = yield self.exported_data(items, settings)+ for batch in data['csv']:+ got_batch = csv.DictReader(to_unicode(batch).splitlines())+ self.assertEqual(list(header), got_batch.fieldnames)+ expected_batch, rows = rows[:batch_size], rows[batch_size:]+ self.assertEqual(expected_batch, list(got_batch))++ @defer.inlineCallbacks+ def assertExportedXml(self, items, rows, settings=None):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},+ },+ })+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']+ rows = [{k: v for k, v in row.items() if v} for row in rows]+ data = yield self.exported_data(items, settings)+ for batch in data['xml']:+ root = lxml.etree.fromstring(batch)+ got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]+ expected_batch, rows = rows[:batch_size], rows[batch_size:]+ self.assertEqual(expected_batch, got_batch)++ @defer.inlineCallbacks+ def assertExportedMultiple(self, items, rows, settings=None):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},+ os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},+ },+ })+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']+ rows = [{k: v for k, v in row.items() if v} for row in rows]+ data = yield self.exported_data(items, settings)+ # XML+ xml_rows = rows.copy()+ for batch in data['xml']:+ root = lxml.etree.fromstring(batch)+ got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]+ expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]+ self.assertEqual(expected_batch, got_batch)+ # JSON+ json_rows = rows.copy()+ for batch in data['json']:+ got_batch = json.loads(batch.decode('utf-8'))+ expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]+ self.assertEqual(expected_batch, got_batch)++ @defer.inlineCallbacks+ def assertExportedPickle(self, items, rows, settings=None):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},+ },+ })+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']+ rows = [{k: v for k, v in row.items() if v} for row in rows]+ data = yield self.exported_data(items, settings)+ import pickle+ for batch in data['pickle']:+ got_batch = self._load_until_eof(batch, load_func=pickle.load)+ expected_batch, rows = rows[:batch_size], rows[batch_size:]+ self.assertEqual(expected_batch, got_batch)++ @defer.inlineCallbacks+ def assertExportedMarshal(self, items, rows, settings=None):+ settings = settings or {}+ settings.update({+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},+ },+ })+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']+ rows = [{k: v for k, v in row.items() if v} for row in rows]+ data = yield self.exported_data(items, settings)+ import marshal+ for batch in data['marshal']:+ got_batch = self._load_until_eof(batch, load_func=marshal.load)+ expected_batch, rows = rows[:batch_size], rows[batch_size:]+ self.assertEqual(expected_batch, got_batch)++ @defer.inlineCallbacks+ def test_export_items(self):+ """ Test partial deliveries in all supported formats """+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),+ self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),+ ]+ rows = [+ {'egg': 'spam1', 'foo': 'bar1', 'baz': ''},+ {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},+ {'foo': 'bar3', 'baz': 'quux3', 'egg': ''}+ ]+ settings = {+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 2+ }+ header = self.MyItem.fields.keys()+ yield self.assertExported(items, header, rows, settings=settings)++ def test_wrong_path(self):+ """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """+ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {'format': 'xml'},+ },+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1+ }+ crawler = get_crawler(settings_dict=settings)+ self.assertRaises(NotConfigured, FeedExporter, crawler)++ @defer.inlineCallbacks+ def test_export_no_items_not_store_empty(self):+ for fmt in ('json', 'jsonlines', 'xml', 'csv'):+ settings = {+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},+ },+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1+ }+ data = yield self.exported_no_data(settings)+ data = dict(data)+ self.assertEqual(b'', data[fmt][0])++ @defer.inlineCallbacks+ def test_export_no_items_store_empty(self):+ formats = (+ ('json', b'[]'),+ ('jsonlines', b''),+ ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),+ ('csv', b''),+ )++ for fmt, expctd in formats:+ settings = {+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},+ },+ 'FEED_STORE_EMPTY': True,+ 'FEED_EXPORT_INDENT': None,+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,+ }+ data = yield self.exported_no_data(settings)+ data = dict(data)+ self.assertEqual(expctd, data[fmt][0])++ @defer.inlineCallbacks+ def test_export_multiple_configs(self):+ items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]++ formats = {+ 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),+ '[\n{"bar": "BAR1"}\n]'.encode('utf-8')],+ 'xml': [+ (+ '<?xml version="1.0" encoding="latin-1"?>\n'+ '<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'+ ).encode('latin-1'),+ (+ '<?xml version="1.0" encoding="latin-1"?>\n'+ '<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'+ ).encode('latin-1')+ ],+ 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),+ 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],+ }++ settings = {+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'json', self._file_mark): {+ 'format': 'json',+ 'indent': 0,+ 'fields': ['bar'],+ 'encoding': 'utf-8',+ },+ os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {+ 'format': 'xml',+ 'indent': 2,+ 'fields': ['foo'],+ 'encoding': 'latin-1',+ },+ os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {+ 'format': 'csv',+ 'indent': None,+ 'fields': ['bar', 'foo'],+ 'encoding': 'utf-8',+ },+ },+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,+ }+ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ for expected_batch, got_batch in zip(expected, data[fmt]):+ self.assertEqual(expected_batch, got_batch)++ @defer.inlineCallbacks+ def test_batch_path_differ(self):+ """+ Test that the name of all batch files differ from each other.+ So %(batch_time)s replaced with the current date.+ """+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),+ self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),+ ]+ settings = {+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), '%(batch_time)s'): {+ 'format': 'json',+ },+ },+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,+ }+ data = yield self.exported_data(items, settings)+ self.assertEqual(len(items) + 1, len(data['json']))++ @defer.inlineCallbacks+ def test_s3_export(self):+ """+ Test export of items into s3 bucket.+ S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini+ to perform this test:+ [testenv]+ setenv =+ AWS_SECRET_ACCESS_KEY = ABCD+ AWS_ACCESS_KEY_ID = ABCD+ S3_TEST_BUCKET_NAME = ABCD+ """+ try:+ import boto3+ except ImportError:+ raise unittest.SkipTest("S3FeedStorage requires boto3")++ assert_aws_environ()+ s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')+ access_key = os.environ.get('AWS_ACCESS_KEY_ID')+ secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')+ if not s3_test_bucket_name:+ raise unittest.SkipTest("No S3 BUCKET available for testing")++ chars = [random.choice(ascii_letters + digits) for _ in range(15)]+ filename = ''.join(chars)+ prefix = 'tmp/{filename}'.format(filename=filename)+ s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(+ bucket_name=s3_test_bucket_name, prefix=prefix+ )+ storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)+ settings = {+ 'FEEDS': {+ s3_test_file_uri: {+ 'format': 'json',+ },+ },+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,+ }+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),+ self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),+ ]+ verifyObject(IFeedStorage, storage)++ class TestSpider(scrapy.Spider):+ name = 'testspider'++ def parse(self, response):+ for item in items:+ yield item++ s3 = boto3.resource('s3')+ my_bucket = s3.Bucket(s3_test_bucket_name)+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']++ with MockServer() as s:+ runner = CrawlerRunner(Settings(settings))+ TestSpider.start_urls = [s.url('/')]+ yield runner.crawl(TestSpider)++ for file_uri in my_bucket.objects.filter(Prefix=prefix):+ content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)+ if not content and not items:+ break+ content = json.loads(content.decode('utf-8'))+ expected_batch, items = items[:batch_size], items[batch_size:]+ self.assertEqual(expected_batch, content)