Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent

Aditya031bfc9c3bed
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 452242d47..c0463cfc7 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,8 +1,12 @@
from OpenSSL import SSL
+import warnings
+
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
+from scrapy.core.downloader.tls import openssl_methods
+from scrapy.utils.misc import create_instance, load_object
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
@@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)
+
+
+def load_context_factory_from_settings(settings, crawler):
+ ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
+ context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+ # try method-aware context factory
+ try:
+ context_factory = create_instance(
+ objcls=context_factory_cls,
+ settings=settings,
+ crawler=crawler,
+ method=ssl_method,
+ )
+ except TypeError:
+ # use context factory defaults
+ context_factory = create_instance(
+ objcls=context_factory_cls,
+ settings=settings,
+ crawler=crawler,
+ )
+ msg = """
+ '%s' does not accept `method` argument (type OpenSSL.SSL method,\
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
+ settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+ warnings.warn(msg)
+
+ return context_factory
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 22c9ac520..dac97ad29 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
-from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
-from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_bytes, to_unicode
@@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
- self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
- self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
- # try method-aware context factory
- try:
- self._contextFactory = create_instance(
- objcls=self._contextFactoryClass,
- settings=settings,
- crawler=crawler,
- method=self._sslMethod,
- )
- except TypeError:
- # use context factory defaults
- self._contextFactory = create_instance(
- objcls=self._contextFactoryClass,
- settings=settings,
- crawler=crawler,
- )
- msg = """
- '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
- Please upgrade your context factory class to handle them or ignore them.""" % (
- settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
- warnings.warn(msg)
+ self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index c8b401e80..81ea78e69 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,10 +1,19 @@
import warnings
+from time import time
+from typing import Optional, Tuple
+from urllib.parse import urldefrag
-from scrapy.core.downloader.tls import openssl_methods
+from twisted.internet.base import ReactorBase
+from twisted.internet.error import TimeoutError
+from twisted.web.client import URI
+
+from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
+from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
-from scrapy.http.request import Request
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.http import Request, Response
from scrapy.settings import Settings
-from scrapy.utils.misc import create_instance, load_object
+from scrapy.spiders import Spider
class H2DownloadHandler:
@@ -13,44 +22,128 @@ class H2DownloadHandler:
from twisted.internet import reactor
self._pool = H2ConnectionPool(reactor, settings)
-
- self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
- self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
- # try method-aware context factory
- try:
- self._context_factory = create_instance(
- objcls=self._context_factory_cls,
- settings=settings,
- crawler=crawler,
- method=self._ssl_method,
- )
- except TypeError:
- # use context factory defaults
- self._context_factory = create_instance(
- objcls=self._context_factory_cls,
- settings=settings,
- crawler=crawler,
- )
- msg = """
- '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
- Please upgrade your context factory class to handle them or ignore them.""" % (
- settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
- warnings.warn(msg)
+ self._context_factory = load_context_factory_from_settings(settings, crawler)
+ self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
+ self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
+ self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings, crawler)
- def download_request(self, request: Request, spider):
+ def download_request(self, request: Request, spider: Spider):
+ agent = ScrapyH2Agent(
+ context_factory=self._context_factory,
+ pool=self._pool,
+ maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
+ warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
+ crawler=self._crawler
+ )
+ return agent.download_request(request, spider)
+
+ def close(self) -> None:
+ self._pool.close_connections()
+
+
+class ScrapyProxyH2Agent(H2Agent):
+ def __init__(
+ self, reactor: ReactorBase,
+ proxy_uri: URI, pool: H2ConnectionPool,
+ connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
+ ) -> None:
+ super(ScrapyProxyH2Agent, self).__init__(
+ reactor=reactor,
+ pool=pool,
+ connect_timeout=connect_timeout,
+ bind_address=bind_address
+ )
+ self._proxy_uri = proxy_uri
+
+ @staticmethod
+ def get_key(uri: URI) -> Tuple:
+ return "http-proxy", uri.host, uri.port
+
+
+class ScrapyH2Agent:
+ _Agent = H2Agent
+ _ProxyAgent = ScrapyProxyH2Agent
+
+ def __init__(
+ self, context_factory,
+ connect_timeout=10,
+ bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,
+ maxsize: int = 0, warnsize: int = 0,
+ crawler=None
+ ) -> None:
+ self._context_factory = context_factory
+ self._connect_timeout = connect_timeout
+ self._bind_address = bind_address
+ self._pool = pool
+ self._maxsize = maxsize
+ self._warnsize = warnsize
+ self._crawler = crawler
+
+ def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
from twisted.internet import reactor
+ bind_address = request.meta.get('bindaddress') or self._bind_address
+ proxy = request.meta.get('proxy')
+ if proxy:
+ _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
+ scheme = _parse(request.url)[0]
+ proxy_host = str(proxy_host, 'utf-8')
+ omit_connect_timeout = b'noconnect' in proxy_params
+ if omit_connect_timeout:
+ warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
+ "If you use Crawlera, it doesn't require this mode anymore, "
+ "so you should update scrapy-crawlera to 1.3.0+ "
+ "and remove '?noconnect' from the Crawlera URL.",
+ ScrapyDeprecationWarning)
+
+ if scheme == b'https' and not omit_connect_timeout:
+ proxy_auth = request.headers.get(b'Proxy-Authorization', None)
+ proxy_conf = (proxy_host, proxy_port, proxy_auth)
- agent = H2Agent(reactor, self._pool, self._context_factory)
- d = agent.request(request)
+ # TODO: Return TunnelingAgent instance
+ else:
+ return self._ProxyAgent(
+ reactor=reactor,
+ proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),
+ connect_timeout=timeout,
+ bind_address=bind_address,
+ pool=self._pool
+ )
- def print_result(result):
- print(result)
- return result
+ return self._Agent(
+ reactor=reactor,
+ context_factory=self._context_factory,
+ connect_timeout=timeout,
+ bind_address=bind_address,
+ pool=self._pool
+ )
+
+ def download_request(self, request: Request, spider: Spider):
+ from twisted.internet import reactor
+ timeout = request.meta.get('download_timeout') or self._connect_timeout
+ agent = self._get_agent(request, timeout)
- d.addCallback(print_result)
+ start_time = time()
+ d = agent.request(request, spider)
+ d.addCallback(self._cb_latency, request, start_time)
+
+ timeout_cl = reactor.callLater(timeout, d.cancel)
+ d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
return d
+
+ @staticmethod
+ def _cb_latency(response: Response, request: Request, start_time: float):
+ request.meta['download_latency'] = time() - start_time
+ return response
+
+ @staticmethod
+ def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl):
+ if timeout_cl.active():
+ timeout_cl.cancel()
+ return response
+
+ url = urldefrag(request.url)[0]
+ raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 7a8847c38..c7a49fd42 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption
from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
from twisted.internet.endpoints import HostnameEndpoint
+from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
+from twisted.web.error import SchemeNotSupported
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface import implementer
from zope.interface.verify import verifyObject
@@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject
from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
from scrapy.http.request import Request
from scrapy.settings import Settings
+from scrapy.spiders import Spider
class H2ConnectionPool:
@@ -28,8 +31,7 @@ class H2ConnectionPool:
# Save all requests that arrive before the connection is established
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
- def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
- key = (uri.scheme, uri.host, uri.port)
+ def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
if key in self._pending_requests:
# Received a request while connecting to remote
# Create a deferred which will fire with the H2ClientProtocol
@@ -84,6 +86,15 @@ class H2ConnectionPool:
d = pending_requests.popleft()
d.errback(errors)
+ def close_connections(self) -> None:
+ """Close all the HTTP/2 connections and remove them from pool
+
+ Returns:
+ Deferred that fires when all connections have been closed
+ """
+ for conn in self._connections.values():
+ conn.transport.loseConnection()
+
@implementer(IPolicyForHTTPS)
class H2WrappedContextFactory:
@@ -111,9 +122,21 @@ class H2Agent:
connect_timeout, bind_address
)
- def request(self, request: Request) -> Deferred:
+ def _get_endpoint(self, uri: URI):
+ return self._endpoint_factory.endpointForURI(uri)
+
+ @staticmethod
+ def get_key(uri: URI) -> Tuple:
+ return uri.scheme, uri.host, uri.port
+
+ def request(self, request: Request, spider: Spider) -> Deferred:
uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
- endpoint = self._endpoint_factory.endpointForURI(uri)
- d = self._pool.get_connection(uri, endpoint)
- d.addCallback(lambda conn: conn.request(request))
+ try:
+ endpoint = self._get_endpoint(uri)
+ except SchemeNotSupported:
+ return defer.fail(Failure())
+
+ key = self.get_key(uri)
+ d = self._pool.get_connection(key, uri, endpoint)
+ d.addCallback(lambda conn: conn.request(request, spider))
return d