Scrapy, a fast high-level web crawling & scraping framework for Python.
Copy resource classes from twisted.web.test.test_webclient.
Andrey Rakhmatullin1e1cfc26dbdd
diff --git a/pytest.ini b/pytest.iniindex fa5d6b34f..ae2ed2029 100644--- a/pytest.ini+++ b/pytest.ini@@ -21,5 +21,3 @@ addopts =markers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passedonly_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed-filterwarnings=- ignore::DeprecationWarning:twisted.web.test.test_webclientdiff --git a/tests/mockserver.py b/tests/mockserver.pyindex ab9aec6a6..72d7e0241 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, sslfrom twisted.internet.task import deferLaterfrom twisted.names import dns, errorfrom twisted.names.server import DNSServerFactory-from twisted.web.resource import EncodingResourceWrapper, Resource+from twisted.web import resource, serverfrom twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Sitefrom twisted.web.static import File-from twisted.web.test.test_webclient import PayloadResourcefrom twisted.web.util import redirectTofrom scrapy.utils.python import to_bytes, to_unicode@@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None):return default-class LeafResource(Resource):+# most of the following resources are copied from twisted.web.test.test_webclient+class ForeverTakingResource(resource.Resource):+ """+ L{ForeverTakingResource} is a resource which never finishes responding+ to requests.+ """++ def __init__(self, write=False):+ resource.Resource.__init__(self)+ self._write = write++ def render(self, request):+ if self._write:+ request.write(b"some bytes")+ return server.NOT_DONE_YET+++class ErrorResource(resource.Resource):+ def render(self, request):+ request.setResponseCode(401)+ if request.args.get(b"showlength"):+ request.setHeader(b"content-length", b"0")+ return b""+++class NoLengthResource(resource.Resource):+ def render(self, request):+ return b"nolength"+++class HostHeaderResource(resource.Resource):+ """+ A testing resource which renders itself as the value of the host header+ from the request.+ """++ def render(self, request):+ return request.requestHeaders.getRawHeaders(b"host")[0]+++class PayloadResource(resource.Resource):+ """+ A testing resource which renders itself as the contents of the request body+ as long as the request body is 100 bytes long, otherwise which renders+ itself as C{"ERROR"}.+ """++ def render(self, request):+ data = request.content.read()+ contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]+ if len(data) != 100 or int(contentLength) != 100:+ return b"ERROR"+ return data+++class BrokenDownloadResource(resource.Resource):+ def render(self, request):+ # only sends 3 bytes even though it claims to send 5+ request.setHeader(b"content-length", b"5")+ request.write(b"abc")+ return b""+++class LeafResource(resource.Resource):isLeaf = True@@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource):return request.content.read()-class Root(Resource):+class Root(resource.Resource):def __init__(self):- Resource.__init__(self)+ resource.Resource.__init__(self)self.putChild(b"status", Status())self.putChild(b"follow", Follow())self.putChild(b"delay", Delay())@@ -187,7 +249,7 @@ class Root(Resource):self.putChild(b"raw", Raw())self.putChild(b"echo", Echo())self.putChild(b"payload", PayloadResource())- self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))+ self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))self.putChild(b"alpayload", ArbitraryLengthPayloadResource())try:from tests import tests_datadirdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex a1ea4c679..2bb53950d 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -15,8 +15,6 @@ from twisted.trial import unittestfrom twisted.web import resource, server, static, utilfrom twisted.web._newclient import ResponseFailedfrom twisted.web.http import _DataLoss-from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource,- NoLengthResource, PayloadResource)from w3lib.url import path_to_file_urifrom scrapy.core.downloader.handlers import DownloadHandlers@@ -34,7 +32,15 @@ from scrapy.spiders import Spiderfrom scrapy.utils.misc import create_instancefrom scrapy.utils.python import to_bytesfrom scrapy.utils.test import get_crawler, skip_if_no_boto-from tests.mockserver import MockServer, ssl_context_factory, Echo+from tests.mockserver import (+ Echo,+ ForeverTakingResource,+ HostHeaderResource,+ MockServer,+ NoLengthResource,+ PayloadResource,+ ssl_context_factory,+)from tests.spiders import SingleRequestSpiderdiff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex 6e4cb9b6e..a6d55cb38 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -21,14 +21,6 @@ except ImportError:from twisted.python.filepath import FilePathfrom twisted.protocols.policies import WrappingFactoryfrom twisted.internet.defer import inlineCallbacks-from twisted.web.test.test_webclient import (- ForeverTakingResource,- ErrorResource,- NoLengthResource,- HostHeaderResource,- PayloadResource,- BrokenDownloadResource,-)from scrapy.core.downloader import webclient as clientfrom scrapy.core.downloader.contextfactory import ScrapyClientContextFactory@@ -36,7 +28,15 @@ from scrapy.http import Request, Headersfrom scrapy.settings import Settingsfrom scrapy.utils.misc import create_instancefrom scrapy.utils.python import to_bytes, to_unicode-from tests.mockserver import ssl_context_factory+from tests.mockserver import (+ BrokenDownloadResource,+ ErrorResource,+ ForeverTakingResource,+ HostHeaderResource,+ NoLengthResource,+ PayloadResource,+ ssl_context_factory,+)def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):