Scrapy, a fast high-level web crawling & scraping framework for Python.
fix anon test: in this case we do no signing, just change the url
Konstantin Lopuhin467553cc2922
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex d3feb9815..6d28f866e 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -47,9 +47,10 @@ class S3DownloadHandler(object):# If no credentials could be found anywhere,# consider this an anonymous connection request by default;# unless 'anon' was set explicitly (True/False).- anon = kw.get('anon', None)+ anon = kw.get('anon')if anon is None and not aws_access_key_id and not aws_secret_access_key:kw['anon'] = True+ self.anon = kw.get('anon')self._signer = Nonetry:@@ -80,7 +81,9 @@ class S3DownloadHandler(object):bucket = p.hostnamepath = p.path + '?' + p.query if p.query else p.pathurl = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)- if self._signer is not None:+ if self.anon:+ request = request.replace(url=url)+ elif self._signer is not None:import botocore.awsrequestfrom botocore.vendored.requests.structures import CaseInsensitiveDictprint(url, request.headers)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 06e232503..0f79a208d 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -458,8 +458,10 @@ class S3AnonTestCase(BaseS3TestCase):def test_anon_request(self):req = Request('s3://aws-publicdatasets/')httpreq = self.download_request(req, self.spider)- self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)- self.assertEqual(self.s3reqh.conn.anon, True)+ self.assertEqual(hasattr(self.s3reqh, 'anon'), True)+ self.assertEqual(self.s3reqh.anon, True)+ self.assertEqual(+ httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')class S3TestCase(unittest.TestCase):