Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Set bodyproducer with empty content for POST

Paul Tremberth99e3c0d653e2
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 37e836809..bff4a30c9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -280,19 +280,7 @@ class ScrapyAgent(object):
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
else:
- bodyproducer = None
- # Setting Content-Length: 0 even for POST requests is not a
- # MUST per HTTP RFCs, but it's common behavior, and some
- # servers require this, otherwise returning HTTP 411 Length required
- #
- # RFC 7230#section-3.3.2:
- # "a Content-Length header field is normally sent in a POST
- # request even when the value is 0 (indicating an empty payload body)."
- #
- # Twisted Agent will not add "Content-Length: 0" by itself
- if method == b'POST':
- headers.addRawHeader(b'Content-Length', b'0')
-
+ bodyproducer = _RequestBodyProducer(b'') if method == b'POST' else None
start_time = time()
d = agent.request(
method, to_bytes(url, encoding='ascii'), headers, bodyproducer)