Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Full typing for scrapy/core/downloader/handlers/http10.py.

Andrey Rakhmatullinaf3e38ab1f3d
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 2507a4231..9117cb818 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -3,8 +3,13 @@
from __future__ import annotations
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Type
+from twisted.internet.defer import Deferred
+
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
+from scrapy.settings import BaseSettings
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import to_unicode
@@ -12,29 +17,33 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
+ from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
class HTTP10DownloadHandler:
lazy = False
- def __init__(self, settings, crawler):
- self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
- self.ClientContextFactory = load_object(
+ def __init__(self, settings: BaseSettings, crawler: Crawler):
+ self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object(
+ settings["DOWNLOADER_HTTPCLIENTFACTORY"]
+ )
+ self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object(
settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
- self._settings = settings
- self._crawler = crawler
+ self._settings: BaseSettings = settings
+ self._crawler: Crawler = crawler
@classmethod
- def from_crawler(cls, crawler) -> Self:
+ def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings, crawler)
- def download_request(self, request, spider):
+ def download_request(self, request: Request, spider: Spider) -> Deferred:
"""Return a deferred for the HTTP download"""
factory = self.HTTPClientFactory(request)
self._connect(factory)
return factory.deferred
- def _connect(self, factory):
+ def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port