Scrapy, a fast high-level web crawling & scraping framework for Python.
added FEED_TEMPDIR to settings
Aron Bordin9250a5bffa91
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex e42dc6370..1bf428edb 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -178,6 +178,15 @@ Default: ``None``The AWS secret key used by code that requires access to `Amazon Web services`_,such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.+.. setting:: FEED_TEMPDIR++FEED_TEMPDIR+------------++The Feed Temp dir allows you to set a custom folder to save crawler+temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and+:ref:`Amazon S3 <topics-feed-storage-s3>`.+.. setting:: BOT_NAMEBOT_NAMEdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 3dab2d77e..5d90ef738 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -8,7 +8,7 @@ import osimport sysimport loggingimport posixpath-from tempfile import TemporaryFile+from tempfile import NamedTemporaryFilefrom datetime import datetimeimport sixfrom six.moves.urllib.parse import urlparse@@ -47,7 +47,11 @@ class IFeedStorage(Interface):class BlockingFeedStorage(object):def open(self, spider):- return TemporaryFile(prefix='feed-')+ path = spider.crawler.settings['FEED_TEMPDIR']+ if path and not os.path.isdir(path):+ raise OSError('Not a Directory: ' + str(path))++ return NamedTemporaryFile(prefix='feed-', dir=path)def store(self, file):return threads.deferToThread(self._store_in_thread, file)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex d449c4891..b9d01e155 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -134,6 +134,7 @@ EXTENSIONS_BASE = {'scrapy.extensions.throttle.AutoThrottle': 0,}+FEED_TEMPDIR = NoneFEED_URI = NoneFEED_URI_PARAMS = None # a function to extend uri argumentsFEED_FORMAT = 'jsonlines'diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex c76d26b57..05f19d33e 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -18,9 +18,9 @@ from w3lib.url import path_to_file_uriimport scrapyfrom scrapy.extensions.feedexport import (IFeedStorage, FileFeedStorage, FTPFeedStorage,- S3FeedStorage, StdoutFeedStorage-)-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete+ S3FeedStorage, StdoutFeedStorage,+ BlockingFeedStorage)+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawlerfrom scrapy.utils.python import to_native_str@@ -87,6 +87,41 @@ class FTPFeedStorageTest(unittest.TestCase):self.assertEqual(fp.read(), b"new content")+class BlockingFeedStorageTest(unittest.TestCase):++ def get_test_spider(self, settings=None):+ class TestSpider(scrapy.Spider):+ name = 'test_spider'+ crawler = get_crawler(settings_dict=settings)+ spider = TestSpider.from_crawler(crawler)+ return spider++ def test_default_temp_dir(self):+ b = BlockingFeedStorage()++ tmp = b.open(self.get_test_spider())+ tmp_path = os.path.dirname(tmp.name)+ self.assertEqual(tmp_path, tempfile.gettempdir())++ def test_temp_file(self):+ b = BlockingFeedStorage()++ tests_path = os.path.dirname(os.path.abspath(__file__))+ spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})+ tmp = b.open(spider)+ tmp_path = os.path.dirname(tmp.name)+ self.assertEqual(tmp_path, tests_path)++ def test_invalid_folder(self):+ b = BlockingFeedStorage()++ tests_path = os.path.dirname(os.path.abspath(__file__))+ invalid_path = os.path.join(tests_path, 'invalid_path')+ spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})++ self.assertRaises(OSError, b.open, spider=spider)++class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacks