Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add "Content-Length: 0" for body-less HTTP/1.1 POST requests

Paul Tremberth617419256425
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index ad3285a32..ca13c609d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -209,7 +209,21 @@ class ScrapyAgent(object):
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
headers.removeHeader(b'Proxy-Authorization')
- bodyproducer = _RequestBodyProducer(request.body) if request.body else None
+ if request.body:
+ bodyproducer = _RequestBodyProducer(request.body)
+ else:
+ bodyproducer = None
+ # Setting Content-Length: 0 even for POST requests is not a
+ # MUST per HTTP RFCs, but it's common behavior, and some
+ # server require this otherwise returing HTTP 411
+ #
+ # RFC 7230#section-3.3.2:
+ # "a Content-Length header field is normally sent in a POST
+ # request even when the value is 0 (indicating an empty payload body)."
+ #
+ # Twisted Agent will not add "Content-Length: 0" by itself
+ if method == b'POST':
+ headers.addRawHeader(b'Content-Length', b'0')
start_time = time()
d = agent.request(
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 1885a53a0..88b60e4ff 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -105,6 +105,15 @@ class FileTestCase(unittest.TestCase):
return self.assertFailure(d, IOError)
+class ContentLengthHeaderResource(resource.Resource):
+ """
+ A testing resource which renders itself as the value of the Content-Length
+ header from the request.
+ """
+ def render(self, request):
+ return request.requestHeaders.getRawHeaders(b"content-length")[0]
+
+
class HttpTestCase(unittest.TestCase):
scheme = 'http'
@@ -122,6 +131,7 @@ class HttpTestCase(unittest.TestCase):
r.putChild(b"host", HostHeaderResource())
r.putChild(b"payload", PayloadResource())
r.putChild(b"broken", BrokenDownloadResource())
+ r.putChild(b"contentlength", ContentLengthHeaderResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
self.host = 'localhost'
@@ -209,6 +219,28 @@ class HttpTestCase(unittest.TestCase):
d.addCallback(self.assertEquals, b'example.com')
return d
+ def test_content_length_zero_bodyless_post_request_headers(self):
+ """Tests if "Content-Length: 0" is sent for bodyless POST requests.
+
+ This is not strictly required by HTTP RFCs but can cause trouble
+ for some web servers.
+ See:
+ https://github.com/scrapy/scrapy/issues/823
+ https://issues.apache.org/jira/browse/TS-2902
+ https://github.com/kennethreitz/requests/issues/405
+ https://bugs.python.org/issue14721
+ """
+ def _test(response):
+ self.assertEquals(response.body, b'0')
+
+ request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
+ return self.download_request(request, Spider('foo')).addCallback(_test)
+
+ d = self.download_request(request, Spider('foo'))
+ d.addCallback(lambda r: r.body)
+ d.addCallback(self.assertEquals, b'0')
+ return d
+
def test_payload(self):
body = b'1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)