Scrapy, a fast high-level web crawling & scraping framework for Python.
add direct test for S3FilesStore
Konstantin Lopuhin32cd8c91654a
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex d2f7c0ae4..43abd64a0 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -2,6 +2,7 @@This module contains some assorted functions used in tests"""+from __future__ import absolute_importimport osfrom importlib import import_module@@ -25,6 +26,25 @@ def skip_if_no_boto():except NotConfigured as e:raise SkipTest(e.message)+def get_s3_content_and_delete(bucket, path):+ """ Get content from s3 key, and delete key afterwards.+ """+ if is_botocore():+ import botocore.session+ session = botocore.session.get_session()+ client = session.create_client('s3')+ key = client.get_object(Bucket=bucket, Key=path)+ content = key['Body'].read()+ client.delete_object(Bucket=bucket, Key=path)+ else:+ import boto+ # assuming boto=2.2.2+ bucket = boto.connect_s3().get_bucket(bucket, validate=False)+ key = bucket.get_key(path)+ content = key.get_contents_as_string()+ bucket.delete_key(path)+ return content+def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project leveldiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex f3cf1c2cb..fd2f5a2ba 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import (IFeedStorage, FileFeedStorage, FTPFeedStorage,S3FeedStorage, StdoutFeedStorage)-from scrapy.utils.test import assert_aws_environ+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_deletefrom scrapy.utils.python import to_native_strfrom scrapy.utils.boto import is_botocore@@ -93,7 +93,7 @@ class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacksdef test_store(self):assert_aws_environ()- uri = os.environ.get('FEEDTEST_S3_URI')+ uri = os.environ.get('S3_TEST_FILE_URI')if not uri:raise unittest.SkipTest("No S3 URI available for testing")storage = S3FeedStorage(uri)@@ -103,25 +103,9 @@ class S3FeedStorageTest(unittest.TestCase):file.write(expected_content)yield storage.store(file)u = urlparse(uri)- content = self._get_content_and_delete(u.hostname, u.path[1:])+ content = get_s3_content_and_delete(u.hostname, u.path[1:])self.assertEqual(content, expected_content)- def _get_content_and_delete(self, bucket, path):- if is_botocore():- import botocore.session- session = botocore.session.get_session()- client = session.create_client('s3')- key = client.get_object(Bucket=bucket, Key=path)- content = key['Body'].read()- client.delete_object(Bucket=bucket, Key=path)- else:- from boto import connect_s3- bucket = connect_s3().get_bucket(bucket, validate=False)- key = bucket.get_key(path)- content = key.get_contents_as_string()- bucket.delete_key(path)- return content-class StdoutFeedStorageTest(unittest.TestCase):diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex c9977f5ca..6ea47086f 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -4,15 +4,18 @@ import hashlibimport warningsfrom tempfile import mkdtempfrom shutil import rmtree+from six.moves.urllib.parse import urlparse+from six import BytesIOfrom twisted.trial import unittestfrom twisted.internet import defer-from scrapy.pipelines.files import FilesPipeline, FSFilesStore+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStorefrom scrapy.item import Item, Fieldfrom scrapy.http import Request, Responsefrom scrapy.settings import Settingsfrom scrapy.utils.python import to_bytes+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_deletefrom tests import mock@@ -179,6 +182,28 @@ class FilesPipelineTestCaseFields(unittest.TestCase):self.assertEqual(item['stored_file'], [results[0][1]])+class TestS3FilesStore(unittest.TestCase):+ @defer.inlineCallbacks+ def test_persist(self):+ assert_aws_environ()+ uri = os.environ.get('S3_TEST_FILE_URI')+ if not uri:+ raise unittest.SkipTest("No S3 URI available for testing")+ data = b"TestS3FilesStore: \xe2\x98\x83"+ buf = BytesIO(data)+ meta = {'foo': 'bar'}+ path = ''+ store = S3FilesStore(uri)+ yield store.persist_file(path, buf, info=None, meta=meta)+ s = yield store.stat_file(path, info=None)+ self.assertIn('last_modified', s)+ self.assertIn('checksum', s)+ self.assertEqual(s['checksum'], b'3187896a9657a28163abb31667df64c8')+ u = urlparse(uri)+ content = get_s3_content_and_delete(u.hostname, u.path[1:])+ self.assertEqual(content, data)++class ItemWithFiles(Item):file_urls = Field()files = Field()diff --git a/tox.ini b/tox.iniindex 4d8236b4e..2a8067618 100644--- a/tox.ini+++ b/tox.ini@@ -15,7 +15,7 @@ deps =leveldb-rtests/requirements.txtpassenv =- FEEDTEST_S3_URI+ S3_TEST_FILE_URIAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYcommands =