Scrapy, a fast high-level web crawling & scraping framework for Python.
Minimize test reliance on S3; do not install botocore on the default test environments
Adrián Chaves39dbfa1d8276
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex eafe1b334..19ca311c3 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -235,8 +235,10 @@ class BlockingFeedStorageTest(unittest.TestCase):class S3FeedStorageTest(unittest.TestCase):- def test_parse_credentials(self):+ def setUp(self):skip_if_no_boto()++ def test_parse_credentials(self):aws_credentials = {"AWS_ACCESS_KEY_ID": "settings_key","AWS_SECRET_ACCESS_KEY": "settings_secret",@@ -272,8 +274,6 @@ class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacksdef test_store(self):- skip_if_no_boto()-settings = {"AWS_ACCESS_KEY_ID": "access_key","AWS_SECRET_ACCESS_KEY": "secret_key",@@ -392,7 +392,6 @@ class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacksdef test_store_botocore_without_acl(self):- skip_if_no_boto()storage = S3FeedStorage("s3://mybucket/export.csv","access_key",@@ -408,7 +407,6 @@ class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacksdef test_store_botocore_with_acl(self):- skip_if_no_boto()storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl")@@ -888,15 +886,10 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_stats_multiple_file(self):settings = {- "AWS_ACCESS_KEY_ID": "access_key",- "AWS_SECRET_ACCESS_KEY": "secret_key","FEEDS": {printf_escape(path_to_url(str(self._random_temp_filename()))): {"format": "json",},- "s3://bucket/key/foo.csv": {- "format": "csv",- },"stdout:": {"format": "xml",},@@ -908,18 +901,12 @@ class FeedExportTest(FeedExportTestBase):self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())- self.assertIn(- "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()- )self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats())self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)- self.assertEqual(- crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1- )self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1)@@ -2535,7 +2522,6 @@ class BatchDeliveriesTest(FeedExportTestBase):@defer.inlineCallbacksdef test_s3_export(self):skip_if_no_boto()-bucket = "mybucket"items = [self.MyItem({"foo": "bar1", "egg": "spam1"}),@@ -2707,6 +2693,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):maxDiff = None+ def setUp(self):+ skip_if_no_boto()+def test_init(self):settings_dict = {"FEED_URI": "file:///tmp/foobar",diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 9701e5d4e..e0bcfcfea 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -225,12 +225,16 @@ class FilesPipelineTestCase(unittest.TestCase):class FilesPipelineTestCaseFieldsMixin:+ def setUp(self):+ self.tempdir = mkdtemp()++ def tearDown(self):+ rmtree(self.tempdir)+def test_item_fields_default(self):url = "http://www.example.com/files/1.txt"item = self.item_class(name="item1", file_urls=[url])- pipeline = FilesPipeline.from_settings(- Settings({"FILES_STORE": "s3://example/files/"})- )+ pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))requests = list(pipeline.get_media_requests(item, None))self.assertEqual(requests[0].url, url)results = [(True, {"url": url})]@@ -245,7 +249,7 @@ class FilesPipelineTestCaseFieldsMixin:pipeline = FilesPipeline.from_settings(Settings({- "FILES_STORE": "s3://example/files/",+ "FILES_STORE": self.tempdir,"FILES_URLS_FIELD": "custom_file_urls","FILES_RESULT_FIELD": "custom_files",}diff --git a/tox.ini b/tox.iniindex 5a9d9cf29..5c2f583d9 100644--- a/tox.ini+++ b/tox.ini@@ -18,8 +18,6 @@ deps =mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'# newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'- # Extras- botocore>=1.4.87passenv =S3_TEST_FILE_URIAWS_ACCESS_KEY_ID@@ -121,7 +119,7 @@ setenv =basepython = python3deps ={[testenv]deps}- boto+ botocore>=1.4.87google-cloud-storage# Twisted[http2] currently forces old mitmproxy because of h2 version# restrictions in their deps, so we need to pin old markupsafe here too.