Scrapy, a fast high-level web crawling & scraping framework for Python.
feat: MethodNotAllowed405, Content-Length header
Aditya92bec38591fc
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex 81ea78e69..e9cc5ebbc 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -3,6 +3,7 @@ from time import timefrom typing import Optional, Tuplefrom urllib.parse import urldefrag+from twisted.internet.defer import Deferredfrom twisted.internet.base import ReactorBasefrom twisted.internet.error import TimeoutErrorfrom twisted.web.client import URI@@ -23,9 +24,6 @@ class H2DownloadHandler:from twisted.internet import reactorself._pool = H2ConnectionPool(reactor, settings)self._context_factory = load_context_factory_from_settings(settings, crawler)- self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')- self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')- self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')@classmethoddef from_crawler(cls, crawler):@@ -35,8 +33,6 @@ class H2DownloadHandler:agent = ScrapyH2Agent(context_factory=self._context_factory,pool=self._pool,- maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),- warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),crawler=self._crawler)return agent.download_request(request, spider)@@ -72,15 +68,12 @@ class ScrapyH2Agent:self, context_factory,connect_timeout=10,bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,- maxsize: int = 0, warnsize: int = 0,crawler=None) -> None:self._context_factory = context_factoryself._connect_timeout = connect_timeoutself._bind_address = bind_addressself._pool = pool- self._maxsize = maxsize- self._warnsize = warnsizeself._crawler = crawlerdef _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:@@ -121,7 +114,7 @@ class ScrapyH2Agent:pool=self._pool)- def download_request(self, request: Request, spider: Spider):+ def download_request(self, request: Request, spider: Spider) -> Deferred:from twisted.internet import reactortimeout = request.meta.get('download_timeout') or self._connect_timeoutagent = self._get_agent(request, timeout)@@ -135,12 +128,12 @@ class ScrapyH2Agent:return d@staticmethod- def _cb_latency(response: Response, request: Request, start_time: float):+ def _cb_latency(response: Response, request: Request, start_time: float) -> Response:request.meta['download_latency'] = time() - start_timereturn response@staticmethod- def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl):+ def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response:if timeout_cl.active():timeout_cl.cancel()return responsediff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.pyindex c7a49fd42..e62eef263 100644--- a/scrapy/core/http2/agent.py+++ b/scrapy/core/http2/agent.py@@ -93,7 +93,7 @@ class H2ConnectionPool:Deferred that fires when all connections have been closed"""for conn in self._connections.values():- conn.transport.loseConnection()+ conn.transport.abortConnection()@implementer(IPolicyForHTTPS)diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex feb034a0c..1ce8b6548 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -13,7 +13,7 @@ from h2.events import (SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,WindowUpdated)-from h2.exceptions import H2Error, ProtocolError+from h2.exceptions import H2Errorfrom twisted.internet.defer import Deferredfrom twisted.internet.error import TimeoutErrorfrom twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory@@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error):self.negotiated_protocol = negotiated_protocoldef __str__(self) -> str:- return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}'+ return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}'class RemoteTerminatedConnection(H2Error):@@ -48,7 +48,15 @@ class RemoteTerminatedConnection(H2Error):self.terminate_event = eventdef __str__(self) -> str:- return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}'+ return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}'+++class MethodNotAllowed405(H2Error):+ def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]):+ self.remote_ip_address = remote_ip_address++ def __str__(self) -> str:+ return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"@implementer(IHandshakeListener)@@ -217,14 +225,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin):# So, no need to send a GOAWAY frame to the remoteself._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])+ def _check_received_data(self, data: bytes) -> None:+ """Checks for edge cases where the connection to remote fails+ without raising an appropriate H2Error++ Arguments:+ data -- Data received from the remote+ """+ if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):+ raise MethodNotAllowed405(self.metadata['ip_address'])+def dataReceived(self, data: bytes) -> None:# Reset the idle timeout as connection is still actively receiving dataself.resetTimeout()try:+ self._check_received_data(data)events = self.conn.receive_data(data)self._handle_events(events)- except ProtocolError as e:+ except H2Error as e:# Save this error as ultimately the connection will be dropped# internally by hyper-h2. Saved error will be passed to all the streams# closed with the connection.@@ -271,9 +290,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):for stream in self.streams.values():if stream.request_sent:- stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True)+ close_reason = StreamCloseReason.CONNECTION_LOSTelse:- stream.close(StreamCloseReason.INACTIVE, from_protocol=True)+ close_reason = StreamCloseReason.INACTIVE+ stream.close(close_reason, self._conn_lost_errors, from_protocol=True)self._active_streams -= len(self.streams)self.streams.clear()diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 133196797..5bffa67e7 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -189,12 +189,15 @@ class Stream:headers = [(':method', self._request.method),(':authority', url.netloc),- (':scheme', 'https'),+ (':scheme', self._protocol.metadata['uri'].scheme),(':path', path),]for name, value in self._request.headers.items():- headers.append((name, value[0]))+ headers.append((str(name, 'utf-8'), str(value[0], 'utf-8')))++ if b'Content-Length' not in self._request.headers.keys():+ headers.append(('Content-Length', str(len(self._request.body))))return headers@@ -337,6 +340,10 @@ class Stream:if not isinstance(reason, StreamCloseReason):raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')+ # Have default value of errors as an empty list as+ # some cases can add a list of exceptions+ errors = errors or []+if not from_protocol:self._protocol.pop_stream(self.stream_id)@@ -387,7 +394,8 @@ class Stream:self._deferred_response.errback(ResponseFailed(errors))elif reason is StreamCloseReason.INACTIVE:- self._deferred_response.errback(InactiveStreamClosed(self._request))+ errors.insert(0, InactiveStreamClosed(self._request))+ self._deferred_response.errback(ResponseFailed(errors))elif reason is StreamCloseReason.INVALID_HOSTNAME:self._deferred_response.errback(InvalidHostname(diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex 746eef4d6..4926ada14 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -11,14 +11,14 @@ from h2.exceptions import InvalidBodyLengthErrorfrom twisted.internet import reactorfrom twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacksfrom twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint+from twisted.internet.error import TimeoutErrorfrom twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificatefrom twisted.python.failure import Failurefrom twisted.trial.unittest import TestCase-from twisted.web.client import URI+from twisted.web.client import ResponseFailed, URIfrom twisted.web.http import Request as TxRequestfrom twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.static import File-from twisted.internet.error import TimeoutErrorfrom scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocolfrom scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname@@ -152,6 +152,19 @@ class QueryParams(LeafResource):return bytes(json.dumps(query_params), 'utf-8')+class RequestHeaders(LeafResource):+ """Sends all the headers received as a response"""++ def render_GET(self, request: TxRequest):+ request.setHeader('Content-Type', 'application/json; charset=UTF-8')+ request.setHeader('Content-Encoding', 'UTF-8')+ headers = {}+ for k, v in request.requestHeaders.getAllRawHeaders():+ headers[str(k, 'utf-8')] = str(v[0], 'utf-8')++ return bytes(json.dumps(headers), 'utf-8')++def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:pem = ''.join(key.readlines()) + ''.join(certificate.readlines())@@ -179,6 +192,7 @@ class Https2ClientProtocolTestCase(TestCase):r.putChild(b'status', Status())r.putChild(b'query-params', QueryParams())r.putChild(b'timeout', TimeoutResponse())+ r.putChild(b'request-headers', RequestHeaders())return r@inlineCallbacks@@ -488,7 +502,11 @@ class Https2ClientProtocolTestCase(TestCase):d_list = []def assert_inactive_stream(failure):- self.assertIsNotNone(failure.check(InactiveStreamClosed))+ self.assertIsNotNone(failure.check(ResponseFailed))+ self.assertTrue(any(+ isinstance(e, InactiveStreamClosed)+ for e in failure.value.reasons+ ))# Send 100 request (we do not check the result)for _ in range(100):@@ -616,3 +634,25 @@ class Https2ClientProtocolTestCase(TestCase):d.addCallback(self.fail)d.addErrback(assert_timeout_error)return d++ def test_request_headers_received(self):+ request = Request(self.get_url('/request-headers'), headers={+ 'header-1': 'header value 1',+ 'header-2': 'header value 2'+ })+ d = self.make_request(request)++ def assert_request_headers(response: Response):+ self.assertEqual(response.status, 200)+ self.assertEqual(response.request, request)++ response_headers = json.loads(str(response.body, 'utf-8'))+ self.assertIsInstance(response_headers, dict)+ for k, v in request.headers.items():+ k, v = str(k, 'utf-8'), str(v[0], 'utf-8')+ self.assertIn(k, response_headers)+ self.assertEqual(v, response_headers[k])++ d.addErrback(self.fail)+ d.addCallback(assert_request_headers)+ return d