Scrapy, a fast high-level web crawling & scraping framework for Python.
feat: H2Agent, H2ConnectionPool base implementation
Aditya9fffb801ed3d
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pynew file mode 100644index 000000000..0dc06f4d8--- /dev/null+++ b/scrapy/core/downloader/handlers/http2.py@@ -0,0 +1,55 @@+import warnings++from scrapy.core.downloader.tls import openssl_methods+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool+from scrapy.http.request import Request+from scrapy.settings import Settings+from scrapy.utils.misc import create_instance, load_object+++class H2DownloadHandler:+ def __init__(self, settings: Settings, crawler=None):+ self._crawler = crawler++ from twisted.internet import reactor+ self._pool = H2ConnectionPool(reactor, settings)++ self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]+ self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])+ # try method-aware context factory+ try:+ self._context_factory = create_instance(+ objcls=self._context_factory_cls,+ settings=settings,+ crawler=crawler,+ method=self._ssl_method,+ )+ except TypeError:+ # use context factory defaults+ self._context_factory = create_instance(+ objcls=self._context_factory_cls,+ settings=settings,+ crawler=crawler,+ )+ msg = """+ '%s' does not accept `method` argument (type OpenSSL.SSL method,\+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\+ Please upgrade your context factory class to handle them or ignore them.""" % (+ settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)+ warnings.warn(msg)++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler.settings, crawler)++ def download_request(self, request: Request, spider):+ from twisted.internet import reactor++ agent = H2Agent(reactor, self._pool, self._context_factory)+ d = agent.request(request)++ def print_result(result):+ print(result)++ d.addCallback(print_result)+ return ddiff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.pynew file mode 100644index 000000000..e14e5d633--- /dev/null+++ b/scrapy/core/http2/agent.py@@ -0,0 +1,64 @@+from typing import Dict, Tuple++from twisted.internet import defer+from twisted.internet.base import ReactorBase+from twisted.internet.defer import Deferred+from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS+from twisted.web.client import URI, BrowserLikePolicyForHTTPS++from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory+from scrapy.http.request import Request+from scrapy.settings import Settings+from scrapy.utils.python import to_bytes, to_unicode+++class H2ConnectionPool:+ def __init__(self, reactor: ReactorBase, settings: Settings) -> None:+ self._reactor = reactor+ self.settings = settings+ self._connections: Dict[Tuple, H2ClientProtocol] = {}++ def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:+ key = (uri.scheme, uri.host, uri.port)+ conn = self._connections.get(key, None)+ if conn:+ return defer.succeed(conn)+ return self._new_connection(key, uri, endpoint)++ def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:+ factory = H2ClientFactory(uri, self.settings)+ d = endpoint.connect(factory)++ def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol:+ self._connections[key] = conn+ return conn++ d.addCallback(put_connection)+ return d++ def _remove_connection(self, key) -> None:+ conn = self._connections.pop(key)+ conn.loseConnection()+++class H2Agent:+ def __init__(+ self, reactor: ReactorBase, pool: H2ConnectionPool,+ context_factory=BrowserLikePolicyForHTTPS()+ ) -> None:+ self._reactor = reactor+ self._pool = pool+ self._context_factory = context_factory++ def request(self, request: Request) -> Deferred:+ uri = URI.fromBytes(to_bytes(request.url, encoding='ascii'))+ # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2'])+ # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2+ endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory)+ d = self._pool.get_connection(uri, endpoint)++ def cb_connected(conn: H2ClientProtocol):+ return conn.request(request)++ d.addCallback(cb_connected)+ return d