Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate is_secure=False for s3://. (#7738)

Andrey Rakhmatullin11d7a05a6f07
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 9b7ceeb4c..1fba7f019 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,10 @@
from __future__ import annotations
+import warnings
from typing import TYPE_CHECKING, Any, cast
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
@@ -49,7 +50,16 @@ class S3DownloadHandler(BaseDownloadHandler):
async def download_request(self, request: Request) -> Response:
p = urlparse_cached(request)
- scheme = "http" if request.meta.get("is_secure") is False else "https"
+ if request.meta.get("is_secure") is False:
+ warnings.warn(
+ "Passing is_secure=False for s3:// requests is deprecated."
+ " In future Scrapy releases this flag will be ignored.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ scheme = "http"
+ else:
+ scheme = "https"
bucket = p.hostname
path = p.path + "?" + p.query if p.query else p.path
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4af5ca0e8..8cb9308ff 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -175,7 +175,11 @@ class TestS3Anon:
@coroutine_test
async def test_anon_request_insecure(self):
req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})
- httpreq = await self.download_request(req)
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="Passing is_secure=False for s3:// requests is deprecated",
+ ):
+ httpreq = await self.download_request(req)
assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"
@@ -215,7 +219,11 @@ class TestS3:
@coroutine_test
async def test_insecure_opt_out(self):
req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})
- httpreq = await self.download_request(req)
+ with pytest.warns(
+ ScrapyDeprecationWarning,
+ match="Passing is_secure=False for s3:// requests is deprecated",
+ ):
+ httpreq = await self.download_request(req)
assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
@coroutine_test