Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

feat: H2ClientFactory

Aditya2ea7d82534ca
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 55dbcabec..ee51300cc 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -2,29 +2,38 @@ import ipaddress
import itertools
import logging
from collections import deque
-from typing import Dict, Optional
+from typing import Dict, List, Optional
from h2.config import H2Configuration
from h2.connection import H2Connection
from h2.events import (
- DataReceived, ResponseReceived, SettingsAcknowledged,
+ Event, DataReceived, ResponseReceived, SettingsAcknowledged,
StreamEnded, StreamReset, WindowUpdated
)
from h2.exceptions import ProtocolError
from twisted.internet.defer import Deferred
-from twisted.internet.protocol import connectionDone, Protocol
+from twisted.internet.protocol import connectionDone, Factory, Protocol
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
+from twisted.web.client import URI
from scrapy.core.http2.stream import Stream, StreamCloseReason
from scrapy.core.http2.types import H2ConnectionMetadataDict
from scrapy.http import Request
+from scrapy.settings import Settings
logger = logging.getLogger(__name__)
class H2ClientProtocol(Protocol):
- def __init__(self) -> None:
+ def __init__(self, uri: URI, settings: Settings) -> None:
+ """
+ Arguments:
+ uri -- URI of the base url to which HTTP/2 Connection will be made.
+ uri is used to verify that incoming client requests have correct
+ base URL.
+ settings -- Scrapy project settings
+ """
config = H2Configuration(client_side=True, header_encoding='utf-8')
self.conn = H2Connection(config=config)
@@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol):
self.metadata: H2ConnectionMetadataDict = {
'certificate': None,
'ip_address': None,
- 'hostname': None,
- 'port': None,
+ 'uri': uri,
+ 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
+ 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
}
@property
@@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol):
destination = self.transport.getPeer()
logger.debug('Connection made to {}'.format(destination))
self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
- self.metadata['port'] = destination.port
- self.metadata['hostname'] = self.transport.transport.addr[0]
self.conn.initiate_connection()
self._write_to_transport()
@@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol):
self._pending_request_stream_pool.clear()
self.conn.close_connection()
- def _handle_events(self, events: list) -> None:
+ def _handle_events(self, events: List[Event]) -> None:
"""Private method which acts as a bridge between the events
received from the HTTP/2 data and IH2EventsHandler
@@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol):
# Send leftover data for all the streams
for stream in self.streams.values():
stream.receive_window_update()
+
+
+class H2ClientFactory(Factory):
+ def __init__(self, uri: URI, settings: Settings) -> None:
+ self.uri = uri
+ self.settings = settings
+
+ def buildProtocol(self, addr) -> H2ClientProtocol:
+ return H2ClientProtocol(self.uri, self.settings)
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 8b66d4b85..5017f9cd4 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict
from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
+from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from scrapy.core.http2.protocol import H2ClientProtocol
@@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed):
class InvalidHostname(Exception):
- def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None:
+ def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
self.request = request
self.expected_hostname = expected_hostname
self.expected_netloc = expected_netloc
@@ -83,10 +84,7 @@ class Stream:
self,
stream_id: int,
request: Request,
- protocol: "H2ClientProtocol",
- download_maxsize: int = 0,
- download_warnsize: int = 0,
- fail_on_data_loss: bool = True
+ protocol: "H2ClientProtocol"
) -> None:
"""
Arguments:
@@ -98,9 +96,14 @@ class Stream:
self._request: Request = request
self._protocol: "H2ClientProtocol" = protocol
- self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
- self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
- self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss)
+ self._download_maxsize = self._request.meta.get(
+ 'download_maxsize',
+ self._protocol.metadata['default_download_maxsize']
+ )
+ self._download_warnsize = self._request.meta.get(
+ 'download_warnsize',
+ self._protocol.metadata['default_download_warnsize']
+ )
self.request_start_time = None
@@ -174,9 +177,9 @@ class Stream:
# Make sure that we are sending the request to the correct URL
url = urlparse(self._request.url)
return (
- url.netloc == self._protocol.metadata['hostname']
- or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}'
- or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
+ url.netloc == to_unicode(self._protocol.metadata['uri'].host)
+ or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc)
+ or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
)
def _get_request_headers(self) -> List[Tuple[str, str]]:
@@ -391,8 +394,8 @@ class Stream:
elif reason is StreamCloseReason.INVALID_HOSTNAME:
self._deferred_response.errback(InvalidHostname(
self._request,
- self._protocol.metadata['hostname'],
- f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
+ to_unicode(self._protocol.metadata['uri'].host),
+ f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
))
def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None:
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
index dd7b1187b..d2aa1a9d8 100644
--- a/scrapy/core/http2/types.py
+++ b/scrapy/core/http2/types.py
@@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address
from typing import Union, Optional
from twisted.internet.ssl import Certificate
+from twisted.web.client import URI
# for python < 3.8 -- typing.TypedDict is undefined
from typing_extensions import TypedDict
@@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict):
# is updated when HTTP/2 connection is made successfully
ip_address: Optional[Union[IPv4Address, IPv6Address]]
- # Name of the peer HTTP/2 connection is established
- hostname: Optional[str]
+ # URI of the peer HTTP/2 connection is made
+ uri: URI
- port: Optional[int]
-
- # Both ip_address and hostname are used by the Stream before
+ # Both ip_address and uri are used by the Stream before
# initiating the request to verify that the base address
+ # Variables taken from Project Settings
+ default_download_maxsize: int
+ default_download_warnsize: int
+
class H2ResponseDict(TypedDict):
# Data received frame by frame from the server is appended
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 9efca5267..05e5f5047 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
-from twisted.internet.protocol import Factory
from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
+from twisted.web.client import URI
from twisted.web.http import Request as TxRequest
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.static import File
-from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.core.http2.protocol import H2ClientFactory
from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
from scrapy.http import Request, Response, JsonRequest
+from scrapy.settings import Settings
from scrapy.utils.python import to_bytes, to_unicode
from tests.mockserver import ssl_context_factory, LeafResource, Status
@@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase):
trustRoot=self.client_certificate,
acceptableProtocols=[b'h2']
)
- h2_client_factory = Factory.forProtocol(H2ClientProtocol)
+ uri = URI.fromBytes(to_bytes(self.get_url('/')))
+ h2_client_factory = H2ClientFactory(uri, Settings())
client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
self.client = yield client_endpoint.connect(h2_client_factory)