Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix anon test: in this case we do no signing, just change the url

Konstantin Lopuhin467553cc2922
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d3feb9815..6d28f866e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -47,9 +47,10 @@ class S3DownloadHandler(object):
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
# unless 'anon' was set explicitly (True/False).
- anon = kw.get('anon', None)
+ anon = kw.get('anon')
if anon is None and not aws_access_key_id and not aws_secret_access_key:
kw['anon'] = True
+ self.anon = kw.get('anon')
self._signer = None
try:
@@ -80,7 +81,9 @@ class S3DownloadHandler(object):
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
- if self._signer is not None:
+ if self.anon:
+ request = request.replace(url=url)
+ elif self._signer is not None:
import botocore.awsrequest
from botocore.vendored.requests.structures import CaseInsensitiveDict
print(url, request.headers)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 06e232503..0f79a208d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -458,8 +458,10 @@ class S3AnonTestCase(BaseS3TestCase):
def test_anon_request(self):
req = Request('s3://aws-publicdatasets/')
httpreq = self.download_request(req, self.spider)
- self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
- self.assertEqual(self.s3reqh.conn.anon, True)
+ self.assertEqual(hasattr(self.s3reqh, 'anon'), True)
+ self.assertEqual(self.s3reqh.anon, True)
+ self.assertEqual(
+ httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
class S3TestCase(unittest.TestCase):