Scrapy, a fast high-level web crawling & scraping framework for Python.
feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent
Aditya031bfc9c3bed
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex 452242d47..c0463cfc7 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -1,8 +1,12 @@from OpenSSL import SSL+import warnings+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphersfrom twisted.web.client import BrowserLikePolicyForHTTPSfrom twisted.web.iweb import IPolicyForHTTPSfrom zope.interface.declarations import implementer+from scrapy.core.downloader.tls import openssl_methods+from scrapy.utils.misc import create_instance, load_objectfrom scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS@@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):trustRoot=platformTrust(),extraCertificateOptions={'method': self._ssl_method},)+++def load_context_factory_from_settings(settings, crawler):+ ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]+ context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])+ # try method-aware context factory+ try:+ context_factory = create_instance(+ objcls=context_factory_cls,+ settings=settings,+ crawler=crawler,+ method=ssl_method,+ )+ except TypeError:+ # use context factory defaults+ context_factory = create_instance(+ objcls=context_factory_cls,+ settings=settings,+ crawler=crawler,+ )+ msg = """+ '%s' does not accept `method` argument (type OpenSSL.SSL method,\+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\+ Please upgrade your context factory class to handle them or ignore them.""" % (+ settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)+ warnings.warn(msg)++ return context_factorydiff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 22c9ac520..dac97ad29 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTHfrom zope.interface import implementerfrom scrapy import signals-from scrapy.core.downloader.tls import openssl_methods+from scrapy.core.downloader.contextfactory import load_context_factory_from_settingsfrom scrapy.core.downloader.webclient import _parsefrom scrapy.exceptions import ScrapyDeprecationWarning, StopDownloadfrom scrapy.http import Headersfrom scrapy.responsetypes import responsetypes-from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.python import to_bytes, to_unicode@@ -43,29 +42,7 @@ class HTTP11DownloadHandler:self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')self._pool._factory.noisy = False- self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]- self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])- # try method-aware context factory- try:- self._contextFactory = create_instance(- objcls=self._contextFactoryClass,- settings=settings,- crawler=crawler,- method=self._sslMethod,- )- except TypeError:- # use context factory defaults- self._contextFactory = create_instance(- objcls=self._contextFactoryClass,- settings=settings,- crawler=crawler,- )- msg = """- '%s' does not accept `method` argument (type OpenSSL.SSL method,\- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\- Please upgrade your context factory class to handle them or ignore them.""" % (- settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)- warnings.warn(msg)+ self._contextFactory = load_context_factory_from_settings(settings, crawler)self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex c8b401e80..81ea78e69 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -1,10 +1,19 @@import warnings+from time import time+from typing import Optional, Tuple+from urllib.parse import urldefrag-from scrapy.core.downloader.tls import openssl_methods+from twisted.internet.base import ReactorBase+from twisted.internet.error import TimeoutError+from twisted.web.client import URI++from scrapy.core.downloader.contextfactory import load_context_factory_from_settings+from scrapy.core.downloader.webclient import _parsefrom scrapy.core.http2.agent import H2Agent, H2ConnectionPool-from scrapy.http.request import Request+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.http import Request, Responsefrom scrapy.settings import Settings-from scrapy.utils.misc import create_instance, load_object+from scrapy.spiders import Spiderclass H2DownloadHandler:@@ -13,44 +22,128 @@ class H2DownloadHandler:from twisted.internet import reactorself._pool = H2ConnectionPool(reactor, settings)-- self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]- self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])- # try method-aware context factory- try:- self._context_factory = create_instance(- objcls=self._context_factory_cls,- settings=settings,- crawler=crawler,- method=self._ssl_method,- )- except TypeError:- # use context factory defaults- self._context_factory = create_instance(- objcls=self._context_factory_cls,- settings=settings,- crawler=crawler,- )- msg = """- '%s' does not accept `method` argument (type OpenSSL.SSL method,\- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\- Please upgrade your context factory class to handle them or ignore them.""" % (- settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)- warnings.warn(msg)+ self._context_factory = load_context_factory_from_settings(settings, crawler)+ self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')+ self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')+ self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')@classmethoddef from_crawler(cls, crawler):return cls(crawler.settings, crawler)- def download_request(self, request: Request, spider):+ def download_request(self, request: Request, spider: Spider):+ agent = ScrapyH2Agent(+ context_factory=self._context_factory,+ pool=self._pool,+ maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),+ warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),+ crawler=self._crawler+ )+ return agent.download_request(request, spider)++ def close(self) -> None:+ self._pool.close_connections()+++class ScrapyProxyH2Agent(H2Agent):+ def __init__(+ self, reactor: ReactorBase,+ proxy_uri: URI, pool: H2ConnectionPool,+ connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None+ ) -> None:+ super(ScrapyProxyH2Agent, self).__init__(+ reactor=reactor,+ pool=pool,+ connect_timeout=connect_timeout,+ bind_address=bind_address+ )+ self._proxy_uri = proxy_uri++ @staticmethod+ def get_key(uri: URI) -> Tuple:+ return "http-proxy", uri.host, uri.port+++class ScrapyH2Agent:+ _Agent = H2Agent+ _ProxyAgent = ScrapyProxyH2Agent++ def __init__(+ self, context_factory,+ connect_timeout=10,+ bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,+ maxsize: int = 0, warnsize: int = 0,+ crawler=None+ ) -> None:+ self._context_factory = context_factory+ self._connect_timeout = connect_timeout+ self._bind_address = bind_address+ self._pool = pool+ self._maxsize = maxsize+ self._warnsize = warnsize+ self._crawler = crawler++ def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:from twisted.internet import reactor+ bind_address = request.meta.get('bindaddress') or self._bind_address+ proxy = request.meta.get('proxy')+ if proxy:+ _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)+ scheme = _parse(request.url)[0]+ proxy_host = str(proxy_host, 'utf-8')+ omit_connect_timeout = b'noconnect' in proxy_params+ if omit_connect_timeout:+ warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "+ "If you use Crawlera, it doesn't require this mode anymore, "+ "so you should update scrapy-crawlera to 1.3.0+ "+ "and remove '?noconnect' from the Crawlera URL.",+ ScrapyDeprecationWarning)++ if scheme == b'https' and not omit_connect_timeout:+ proxy_auth = request.headers.get(b'Proxy-Authorization', None)+ proxy_conf = (proxy_host, proxy_port, proxy_auth)- agent = H2Agent(reactor, self._pool, self._context_factory)- d = agent.request(request)+ # TODO: Return TunnelingAgent instance+ else:+ return self._ProxyAgent(+ reactor=reactor,+ proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),+ connect_timeout=timeout,+ bind_address=bind_address,+ pool=self._pool+ )- def print_result(result):- print(result)- return result+ return self._Agent(+ reactor=reactor,+ context_factory=self._context_factory,+ connect_timeout=timeout,+ bind_address=bind_address,+ pool=self._pool+ )++ def download_request(self, request: Request, spider: Spider):+ from twisted.internet import reactor+ timeout = request.meta.get('download_timeout') or self._connect_timeout+ agent = self._get_agent(request, timeout)- d.addCallback(print_result)+ start_time = time()+ d = agent.request(request, spider)+ d.addCallback(self._cb_latency, request, start_time)++ timeout_cl = reactor.callLater(timeout, d.cancel)+ d.addBoth(self._cb_timeout, request, timeout, timeout_cl)return d++ @staticmethod+ def _cb_latency(response: Response, request: Request, start_time: float):+ request.meta['download_latency'] = time() - start_time+ return response++ @staticmethod+ def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl):+ if timeout_cl.active():+ timeout_cl.cancel()+ return response++ url = urldefrag(request.url)[0]+ raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.pyindex 7a8847c38..c7a49fd42 100644--- a/scrapy/core/http2/agent.py+++ b/scrapy/core/http2/agent.py@@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptionfrom twisted.internet.base import ReactorBasefrom twisted.internet.defer import Deferredfrom twisted.internet.endpoints import HostnameEndpoint+from twisted.python.failure import Failurefrom twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory+from twisted.web.error import SchemeNotSupportedfrom twisted.web.iweb import IPolicyForHTTPSfrom zope.interface import implementerfrom zope.interface.verify import verifyObject@@ -14,6 +16,7 @@ from zope.interface.verify import verifyObjectfrom scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactoryfrom scrapy.http.request import Requestfrom scrapy.settings import Settings+from scrapy.spiders import Spiderclass H2ConnectionPool:@@ -28,8 +31,7 @@ class H2ConnectionPool:# Save all requests that arrive before the connection is establishedself._pending_requests: Dict[Tuple, Deque[Deferred]] = {}- def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred:- key = (uri.scheme, uri.host, uri.port)+ def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:if key in self._pending_requests:# Received a request while connecting to remote# Create a deferred which will fire with the H2ClientProtocol@@ -84,6 +86,15 @@ class H2ConnectionPool:d = pending_requests.popleft()d.errback(errors)+ def close_connections(self) -> None:+ """Close all the HTTP/2 connections and remove them from pool++ Returns:+ Deferred that fires when all connections have been closed+ """+ for conn in self._connections.values():+ conn.transport.loseConnection()+@implementer(IPolicyForHTTPS)class H2WrappedContextFactory:@@ -111,9 +122,21 @@ class H2Agent:connect_timeout, bind_address)- def request(self, request: Request) -> Deferred:+ def _get_endpoint(self, uri: URI):+ return self._endpoint_factory.endpointForURI(uri)++ @staticmethod+ def get_key(uri: URI) -> Tuple:+ return uri.scheme, uri.host, uri.port++ def request(self, request: Request, spider: Spider) -> Deferred:uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))- endpoint = self._endpoint_factory.endpointForURI(uri)- d = self._pool.get_connection(uri, endpoint)- d.addCallback(lambda conn: conn.request(request))+ try:+ endpoint = self._get_endpoint(uri)+ except SchemeNotSupported:+ return defer.fail(Failure())++ key = self.get_key(uri)+ d = self._pool.get_connection(key, uri, endpoint)+ d.addCallback(lambda conn: conn.request(request, spider))return d