Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Adding support for zstd in HttpCompressionMiddleware

P. Chen892dd9da57c5
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 727c41466..b1abb8b5c 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -14,6 +14,12 @@ try:
except ImportError:
pass
+try:
+ import zstd
+ ACCEPTED_ENCODINGS.append(b'zstd')
+except ImportError:
+ pass
+
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
@@ -67,4 +73,6 @@ class HttpCompressionMiddleware:
body = zlib.decompress(body, -15)
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
+ if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
+ body = zstd.decompress(body)
return body