Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

feat: H2Agent, H2ConnectionPool base implementation

Aditya9fffb801ed3d
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
new file mode 100644
index 000000000..0dc06f4d8
--- /dev/null
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -0,0 +1,55 @@
+import warnings
+
+from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
+from scrapy.http.request import Request
+from scrapy.settings import Settings
+from scrapy.utils.misc import create_instance, load_object
+
+
+class H2DownloadHandler:
+ def __init__(self, settings: Settings, crawler=None):
+ self._crawler = crawler
+
+ from twisted.internet import reactor
+ self._pool = H2ConnectionPool(reactor, settings)
+
+ self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
+ self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+ # try method-aware context factory
+ try:
+ self._context_factory = create_instance(
+ objcls=self._context_factory_cls,
+ settings=settings,
+ crawler=crawler,
+ method=self._ssl_method,
+ )
+ except TypeError:
+ # use context factory defaults
+ self._context_factory = create_instance(
+ objcls=self._context_factory_cls,
+ settings=settings,
+ crawler=crawler,
+ )
+ msg = """
+ '%s' does not accept `method` argument (type OpenSSL.SSL method,\
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
+ settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+ warnings.warn(msg)
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler.settings, crawler)
+
+ def download_request(self, request: Request, spider):
+ from twisted.internet import reactor
+
+ agent = H2Agent(reactor, self._pool, self._context_factory)
+ d = agent.request(request)
+
+ def print_result(result):
+ print(result)
+
+ d.addCallback(print_result)
+ return d
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
new file mode 100644
index 000000000..e14e5d633
--- /dev/null
+++ b/scrapy/core/http2/agent.py
@@ -0,0 +1,64 @@
+from typing import Dict, Tuple
+
+from twisted.internet import defer
+from twisted.internet.base import ReactorBase
+from twisted.internet.defer import Deferred
+from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS
+from twisted.web.client import URI, BrowserLikePolicyForHTTPS
+
+from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
+from scrapy.http.request import Request
+from scrapy.settings import Settings
+from scrapy.utils.python import to_bytes, to_unicode
+
+
+class H2ConnectionPool:
+ def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
+ self._reactor = reactor
+ self.settings = settings
+ self._connections: Dict[Tuple, H2ClientProtocol] = {}
+
+ def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
+ key = (uri.scheme, uri.host, uri.port)
+ conn = self._connections.get(key, None)
+ if conn:
+ return defer.succeed(conn)
+ return self._new_connection(key, uri, endpoint)
+
+ def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
+ factory = H2ClientFactory(uri, self.settings)
+ d = endpoint.connect(factory)
+
+ def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol:
+ self._connections[key] = conn
+ return conn
+
+ d.addCallback(put_connection)
+ return d
+
+ def _remove_connection(self, key) -> None:
+ conn = self._connections.pop(key)
+ conn.loseConnection()
+
+
+class H2Agent:
+ def __init__(
+ self, reactor: ReactorBase, pool: H2ConnectionPool,
+ context_factory=BrowserLikePolicyForHTTPS()
+ ) -> None:
+ self._reactor = reactor
+ self._pool = pool
+ self._context_factory = context_factory
+
+ def request(self, request: Request) -> Deferred:
+ uri = URI.fromBytes(to_bytes(request.url, encoding='ascii'))
+ # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2'])
+ # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2
+ endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory)
+ d = self._pool.get_connection(uri, endpoint)
+
+ def cb_connected(conn: H2ClientProtocol):
+ return conn.request(request)
+
+ d.addCallback(cb_connected)
+ return d