Scrapy, a fast high-level web crawling & scraping framework for Python.
feat: H2ClientFactory
Aditya2ea7d82534ca
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 55dbcabec..ee51300cc 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -2,29 +2,38 @@ import ipaddressimport itertoolsimport loggingfrom collections import deque-from typing import Dict, Optional+from typing import Dict, List, Optionalfrom h2.config import H2Configurationfrom h2.connection import H2Connectionfrom h2.events import (- DataReceived, ResponseReceived, SettingsAcknowledged,+ Event, DataReceived, ResponseReceived, SettingsAcknowledged,StreamEnded, StreamReset, WindowUpdated)from h2.exceptions import ProtocolErrorfrom twisted.internet.defer import Deferred-from twisted.internet.protocol import connectionDone, Protocol+from twisted.internet.protocol import connectionDone, Factory, Protocolfrom twisted.internet.ssl import Certificatefrom twisted.python.failure import Failure+from twisted.web.client import URIfrom scrapy.core.http2.stream import Stream, StreamCloseReasonfrom scrapy.core.http2.types import H2ConnectionMetadataDictfrom scrapy.http import Request+from scrapy.settings import Settingslogger = logging.getLogger(__name__)class H2ClientProtocol(Protocol):- def __init__(self) -> None:+ def __init__(self, uri: URI, settings: Settings) -> None:+ """+ Arguments:+ uri -- URI of the base url to which HTTP/2 Connection will be made.+ uri is used to verify that incoming client requests have correct+ base URL.+ settings -- Scrapy project settings+ """config = H2Configuration(client_side=True, header_encoding='utf-8')self.conn = H2Connection(config=config)@@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol):self.metadata: H2ConnectionMetadataDict = {'certificate': None,'ip_address': None,- 'hostname': None,- 'port': None,+ 'uri': uri,+ 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),+ 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),}@property@@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol):destination = self.transport.getPeer()logger.debug('Connection made to {}'.format(destination))self.metadata['ip_address'] = ipaddress.ip_address(destination.host)- self.metadata['port'] = destination.port- self.metadata['hostname'] = self.transport.transport.addr[0]self.conn.initiate_connection()self._write_to_transport()@@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol):self._pending_request_stream_pool.clear()self.conn.close_connection()- def _handle_events(self, events: list) -> None:+ def _handle_events(self, events: List[Event]) -> None:"""Private method which acts as a bridge between the eventsreceived from the HTTP/2 data and IH2EventsHandler@@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol):# Send leftover data for all the streamsfor stream in self.streams.values():stream.receive_window_update()+++class H2ClientFactory(Factory):+ def __init__(self, uri: URI, settings: Settings) -> None:+ self.uri = uri+ self.settings = settings++ def buildProtocol(self, addr) -> H2ClientProtocol:+ return H2ClientProtocol(self.uri, self.settings)diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 8b66d4b85..5017f9cd4 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDictfrom scrapy.http import Requestfrom scrapy.http.headers import Headersfrom scrapy.responsetypes import responsetypes+from scrapy.utils.python import to_unicodeif TYPE_CHECKING:from scrapy.core.http2.protocol import H2ClientProtocol@@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed):class InvalidHostname(Exception):- def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None:+ def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:self.request = requestself.expected_hostname = expected_hostnameself.expected_netloc = expected_netloc@@ -83,10 +84,7 @@ class Stream:self,stream_id: int,request: Request,- protocol: "H2ClientProtocol",- download_maxsize: int = 0,- download_warnsize: int = 0,- fail_on_data_loss: bool = True+ protocol: "H2ClientProtocol") -> None:"""Arguments:@@ -98,9 +96,14 @@ class Stream:self._request: Request = requestself._protocol: "H2ClientProtocol" = protocol- self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)- self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)- self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss)+ self._download_maxsize = self._request.meta.get(+ 'download_maxsize',+ self._protocol.metadata['default_download_maxsize']+ )+ self._download_warnsize = self._request.meta.get(+ 'download_warnsize',+ self._protocol.metadata['default_download_warnsize']+ )self.request_start_time = None@@ -174,9 +177,9 @@ class Stream:# Make sure that we are sending the request to the correct URLurl = urlparse(self._request.url)return (- url.netloc == self._protocol.metadata['hostname']- or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}'- or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'+ url.netloc == to_unicode(self._protocol.metadata['uri'].host)+ or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc)+ or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}')def _get_request_headers(self) -> List[Tuple[str, str]]:@@ -391,8 +394,8 @@ class Stream:elif reason is StreamCloseReason.INVALID_HOSTNAME:self._deferred_response.errback(InvalidHostname(self._request,- self._protocol.metadata['hostname'],- f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'+ to_unicode(self._protocol.metadata['uri'].host),+ f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'))def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None:diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.pyindex dd7b1187b..d2aa1a9d8 100644--- a/scrapy/core/http2/types.py+++ b/scrapy/core/http2/types.py@@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Addressfrom typing import Union, Optionalfrom twisted.internet.ssl import Certificate+from twisted.web.client import URI# for python < 3.8 -- typing.TypedDict is undefinedfrom typing_extensions import TypedDict@@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict):# is updated when HTTP/2 connection is made successfullyip_address: Optional[Union[IPv4Address, IPv6Address]]- # Name of the peer HTTP/2 connection is established- hostname: Optional[str]+ # URI of the peer HTTP/2 connection is made+ uri: URI- port: Optional[int]-- # Both ip_address and hostname are used by the Stream before+ # Both ip_address and uri are used by the Stream before# initiating the request to verify that the base address+ # Variables taken from Project Settings+ default_download_maxsize: int+ default_download_warnsize: int+class H2ResponseDict(TypedDict):# Data received frame by frame from the server is appendeddiff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 9efca5267..05e5f5047 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthErrorfrom twisted.internet import reactorfrom twisted.internet.defer import inlineCallbacks, DeferredList, CancelledErrorfrom twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint-from twisted.internet.protocol import Factoryfrom twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificatefrom twisted.python.failure import Failurefrom twisted.trial.unittest import TestCase+from twisted.web.client import URIfrom twisted.web.http import Request as TxRequestfrom twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.static import File-from scrapy.core.http2.protocol import H2ClientProtocol+from scrapy.core.http2.protocol import H2ClientFactoryfrom scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostnamefrom scrapy.http import Request, Response, JsonRequest+from scrapy.settings import Settingsfrom scrapy.utils.python import to_bytes, to_unicodefrom tests.mockserver import ssl_context_factory, LeafResource, Status@@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase):trustRoot=self.client_certificate,acceptableProtocols=[b'h2'])- h2_client_factory = Factory.forProtocol(H2ClientProtocol)+ uri = URI.fromBytes(to_bytes(self.get_url('/')))+ h2_client_factory = H2ClientFactory(uri, Settings())client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)self.client = yield client_endpoint.connect(h2_client_factory)