Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

add direct test for S3FilesStore

Konstantin Lopuhin32cd8c91654a
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index d2f7c0ae4..43abd64a0 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,6 +2,7 @@
This module contains some assorted functions used in tests
"""
+from __future__ import absolute_import
import os
from importlib import import_module
@@ -25,6 +26,25 @@ def skip_if_no_boto():
except NotConfigured as e:
raise SkipTest(e.message)
+def get_s3_content_and_delete(bucket, path):
+ """ Get content from s3 key, and delete key afterwards.
+ """
+ if is_botocore():
+ import botocore.session
+ session = botocore.session.get_session()
+ client = session.create_client('s3')
+ key = client.get_object(Bucket=bucket, Key=path)
+ content = key['Body'].read()
+ client.delete_object(Bucket=bucket, Key=path)
+ else:
+ import boto
+ # assuming boto=2.2.2
+ bucket = boto.connect_s3().get_bucket(bucket, validate=False)
+ key = bucket.get_key(path)
+ content = key.get_contents_as_string()
+ bucket.delete_key(path)
+ return content
+
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f3cf1c2cb..fd2f5a2ba 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import (
IFeedStorage, FileFeedStorage, FTPFeedStorage,
S3FeedStorage, StdoutFeedStorage
)
-from scrapy.utils.test import assert_aws_environ
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from scrapy.utils.python import to_native_str
from scrapy.utils.boto import is_botocore
@@ -93,7 +93,7 @@ class S3FeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def test_store(self):
assert_aws_environ()
- uri = os.environ.get('FEEDTEST_S3_URI')
+ uri = os.environ.get('S3_TEST_FILE_URI')
if not uri:
raise unittest.SkipTest("No S3 URI available for testing")
storage = S3FeedStorage(uri)
@@ -103,25 +103,9 @@ class S3FeedStorageTest(unittest.TestCase):
file.write(expected_content)
yield storage.store(file)
u = urlparse(uri)
- content = self._get_content_and_delete(u.hostname, u.path[1:])
+ content = get_s3_content_and_delete(u.hostname, u.path[1:])
self.assertEqual(content, expected_content)
- def _get_content_and_delete(self, bucket, path):
- if is_botocore():
- import botocore.session
- session = botocore.session.get_session()
- client = session.create_client('s3')
- key = client.get_object(Bucket=bucket, Key=path)
- content = key['Body'].read()
- client.delete_object(Bucket=bucket, Key=path)
- else:
- from boto import connect_s3
- bucket = connect_s3().get_bucket(bucket, validate=False)
- key = bucket.get_key(path)
- content = key.get_contents_as_string()
- bucket.delete_key(path)
- return content
-
class StdoutFeedStorageTest(unittest.TestCase):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index c9977f5ca..6ea47086f 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -4,15 +4,18 @@ import hashlib
import warnings
from tempfile import mkdtemp
from shutil import rmtree
+from six.moves.urllib.parse import urlparse
+from six import BytesIO
from twisted.trial import unittest
from twisted.internet import defer
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from tests import mock
@@ -179,6 +182,28 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
self.assertEqual(item['stored_file'], [results[0][1]])
+class TestS3FilesStore(unittest.TestCase):
+ @defer.inlineCallbacks
+ def test_persist(self):
+ assert_aws_environ()
+ uri = os.environ.get('S3_TEST_FILE_URI')
+ if not uri:
+ raise unittest.SkipTest("No S3 URI available for testing")
+ data = b"TestS3FilesStore: \xe2\x98\x83"
+ buf = BytesIO(data)
+ meta = {'foo': 'bar'}
+ path = ''
+ store = S3FilesStore(uri)
+ yield store.persist_file(path, buf, info=None, meta=meta)
+ s = yield store.stat_file(path, info=None)
+ self.assertIn('last_modified', s)
+ self.assertIn('checksum', s)
+ self.assertEqual(s['checksum'], b'3187896a9657a28163abb31667df64c8')
+ u = urlparse(uri)
+ content = get_s3_content_and_delete(u.hostname, u.path[1:])
+ self.assertEqual(content, data)
+
+
class ItemWithFiles(Item):
file_urls = Field()
files = Field()
diff --git a/tox.ini b/tox.ini
index 4d8236b4e..2a8067618 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ deps =
leveldb
-rtests/requirements.txt
passenv =
- FEEDTEST_S3_URI
+ S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
commands =