Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate HTTP/1.0 support.

Andrey Rakhmatullind27c6b46b11c
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 58f7ad577..0fbe5fc23 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -2,8 +2,10 @@
from __future__ import annotations
+import warnings
from typing import TYPE_CHECKING
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
@@ -26,6 +28,11 @@ class HTTP10DownloadHandler:
lazy = False
def __init__(self, settings: BaseSettings, crawler: Crawler):
+ warnings.warn(
+ "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object(
settings["DOWNLOADER_HTTPCLIENTFACTORY"]
)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index ee10ae73b..aaaf68152 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import re
+import warnings
from time import time
from typing import TYPE_CHECKING
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
@@ -9,6 +10,7 @@ from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from twisted.web.http import HTTPClient
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
@@ -49,6 +51,14 @@ def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]:
class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b"\n"
+ def __init__(self):
+ warnings.warn(
+ "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ super().__init__()
+
def connectionMade(self):
self.headers = Headers() # bucket for response headers
@@ -140,6 +150,12 @@ class ScrapyHTTPClientFactory(ClientFactory):
self.path = self.url
def __init__(self, request: Request, timeout: float = 180):
+ warnings.warn(
+ "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
self._url: str = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
self.url: bytes = to_bytes(self._url, encoding="ascii")