Scrapy, a fast high-level web crawling & scraping framework for Python.
Add "Content-Length: 0" for body-less HTTP/1.1 POST requests
Paul Tremberth617419256425
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex ad3285a32..ca13c609d 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -209,7 +209,21 @@ class ScrapyAgent(object):headers = TxHeaders(request.headers)if isinstance(agent, self._TunnelingAgent):headers.removeHeader(b'Proxy-Authorization')- bodyproducer = _RequestBodyProducer(request.body) if request.body else None+ if request.body:+ bodyproducer = _RequestBodyProducer(request.body)+ else:+ bodyproducer = None+ # Setting Content-Length: 0 even for POST requests is not a+ # MUST per HTTP RFCs, but it's common behavior, and some+ # server require this otherwise returing HTTP 411+ #+ # RFC 7230#section-3.3.2:+ # "a Content-Length header field is normally sent in a POST+ # request even when the value is 0 (indicating an empty payload body)."+ #+ # Twisted Agent will not add "Content-Length: 0" by itself+ if method == b'POST':+ headers.addRawHeader(b'Content-Length', b'0')start_time = time()d = agent.request(diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 1885a53a0..88b60e4ff 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -105,6 +105,15 @@ class FileTestCase(unittest.TestCase):return self.assertFailure(d, IOError)+class ContentLengthHeaderResource(resource.Resource):+ """+ A testing resource which renders itself as the value of the Content-Length+ header from the request.+ """+ def render(self, request):+ return request.requestHeaders.getRawHeaders(b"content-length")[0]++class HttpTestCase(unittest.TestCase):scheme = 'http'@@ -122,6 +131,7 @@ class HttpTestCase(unittest.TestCase):r.putChild(b"host", HostHeaderResource())r.putChild(b"payload", PayloadResource())r.putChild(b"broken", BrokenDownloadResource())+ r.putChild(b"contentlength", ContentLengthHeaderResource())self.site = server.Site(r, timeout=None)self.wrapper = WrappingFactory(self.site)self.host = 'localhost'@@ -209,6 +219,28 @@ class HttpTestCase(unittest.TestCase):d.addCallback(self.assertEquals, b'example.com')return d+ def test_content_length_zero_bodyless_post_request_headers(self):+ """Tests if "Content-Length: 0" is sent for bodyless POST requests.++ This is not strictly required by HTTP RFCs but can cause trouble+ for some web servers.+ See:+ https://github.com/scrapy/scrapy/issues/823+ https://issues.apache.org/jira/browse/TS-2902+ https://github.com/kennethreitz/requests/issues/405+ https://bugs.python.org/issue14721+ """+ def _test(response):+ self.assertEquals(response.body, b'0')++ request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})+ return self.download_request(request, Spider('foo')).addCallback(_test)++ d = self.download_request(request, Spider('foo'))+ d.addCallback(lambda r: r.body)+ d.addCallback(self.assertEquals, b'0')+ return d+def test_payload(self):body = b'1'*100 # PayloadResource requires body length to be 100request = Request(self.getURL('payload'), method='POST', body=body)