Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

added FEED_TEMPDIR to settings

Aron Bordin9250a5bffa91
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e42dc6370..1bf428edb 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -178,6 +178,15 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+.. setting:: FEED_TEMPDIR
+
+FEED_TEMPDIR
+------------
+
+The Feed Temp dir allows you to set a custom folder to save crawler
+temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
+:ref:`Amazon S3 <topics-feed-storage-s3>`.
+
.. setting:: BOT_NAME
BOT_NAME
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3dab2d77e..5d90ef738 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -8,7 +8,7 @@ import os
import sys
import logging
import posixpath
-from tempfile import TemporaryFile
+from tempfile import NamedTemporaryFile
from datetime import datetime
import six
from six.moves.urllib.parse import urlparse
@@ -47,7 +47,11 @@ class IFeedStorage(Interface):
class BlockingFeedStorage(object):
def open(self, spider):
- return TemporaryFile(prefix='feed-')
+ path = spider.crawler.settings['FEED_TEMPDIR']
+ if path and not os.path.isdir(path):
+ raise OSError('Not a Directory: ' + str(path))
+
+ return NamedTemporaryFile(prefix='feed-', dir=path)
def store(self, file):
return threads.deferToThread(self._store_in_thread, file)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d449c4891..b9d01e155 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -134,6 +134,7 @@ EXTENSIONS_BASE = {
'scrapy.extensions.throttle.AutoThrottle': 0,
}
+FEED_TEMPDIR = None
FEED_URI = None
FEED_URI_PARAMS = None # a function to extend uri arguments
FEED_FORMAT = 'jsonlines'
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c76d26b57..05f19d33e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,9 +18,9 @@ from w3lib.url import path_to_file_uri
import scrapy
from scrapy.extensions.feedexport import (
IFeedStorage, FileFeedStorage, FTPFeedStorage,
- S3FeedStorage, StdoutFeedStorage
-)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+ S3FeedStorage, StdoutFeedStorage,
+ BlockingFeedStorage)
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
from scrapy.utils.python import to_native_str
@@ -87,6 +87,41 @@ class FTPFeedStorageTest(unittest.TestCase):
self.assertEqual(fp.read(), b"new content")
+class BlockingFeedStorageTest(unittest.TestCase):
+
+ def get_test_spider(self, settings=None):
+ class TestSpider(scrapy.Spider):
+ name = 'test_spider'
+ crawler = get_crawler(settings_dict=settings)
+ spider = TestSpider.from_crawler(crawler)
+ return spider
+
+ def test_default_temp_dir(self):
+ b = BlockingFeedStorage()
+
+ tmp = b.open(self.get_test_spider())
+ tmp_path = os.path.dirname(tmp.name)
+ self.assertEqual(tmp_path, tempfile.gettempdir())
+
+ def test_temp_file(self):
+ b = BlockingFeedStorage()
+
+ tests_path = os.path.dirname(os.path.abspath(__file__))
+ spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})
+ tmp = b.open(spider)
+ tmp_path = os.path.dirname(tmp.name)
+ self.assertEqual(tmp_path, tests_path)
+
+ def test_invalid_folder(self):
+ b = BlockingFeedStorage()
+
+ tests_path = os.path.dirname(os.path.abspath(__file__))
+ invalid_path = os.path.join(tests_path, 'invalid_path')
+ spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})
+
+ self.assertRaises(OSError, b.open, spider=spider)
+
+
class S3FeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks