Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate is_secure=False for s3://. (#7738)
Andrey Rakhmatullin11d7a05a6f07
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 9b7ceeb4c..1fba7f019 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -1,9 +1,10 @@from __future__ import annotations+import warningsfrom typing import TYPE_CHECKING, Any, castfrom scrapy.core.downloader.handlers.base import BaseDownloadHandler-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.utils.boto import is_botocore_availablefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import build_from_crawler, load_object@@ -49,7 +50,16 @@ class S3DownloadHandler(BaseDownloadHandler):async def download_request(self, request: Request) -> Response:p = urlparse_cached(request)- scheme = "http" if request.meta.get("is_secure") is False else "https"+ if request.meta.get("is_secure") is False:+ warnings.warn(+ "Passing is_secure=False for s3:// requests is deprecated."+ " In future Scrapy releases this flag will be ignored.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ scheme = "http"+ else:+ scheme = "https"bucket = p.hostnamepath = p.path + "?" + p.query if p.query else p.pathurl = f"{scheme}://{bucket}.s3.amazonaws.com{path}"diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 4af5ca0e8..8cb9308ff 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -175,7 +175,11 @@ class TestS3Anon:@coroutine_testasync def test_anon_request_insecure(self):req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})- httpreq = await self.download_request(req)+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Passing is_secure=False for s3:// requests is deprecated",+ ):+ httpreq = await self.download_request(req)assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"@@ -215,7 +219,11 @@ class TestS3:@coroutine_testasync def test_insecure_opt_out(self):req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})- httpreq = await self.download_request(req)+ with pytest.warns(+ ScrapyDeprecationWarning,+ match="Passing is_secure=False for s3:// requests is deprecated",+ ):+ httpreq = await self.download_request(req)assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"@coroutine_test