Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'scrapy/2.11' into 2.11-port

Adrián Chaves6bd45bb6d92e
diff --cc scrapy/downloadermiddlewares/httpcompression.py
index 56a58a750,816be25a1..709333948
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@@ -1,60 -1,83 +1,102 @@@
+from __future__ import annotations
+
- import io
- import zlib
+ import warnings
+ from logging import getLogger
+from typing import TYPE_CHECKING, List, Optional, Union
- from scrapy import Request, Spider
-from scrapy import signals
++from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
- from scrapy.exceptions import NotConfigured
+ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
+from scrapy.statscollectors import StatsCollector
+ from scrapy.utils._compression import (
+ _DecompressionMaxSizeExceeded,
+ _inflate,
+ _unbrotli,
+ _unzstd,
+ )
+ from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
+if TYPE_CHECKING:
+ # typing.Self requires Python 3.11
+ from typing_extensions import Self
+
+ logger = getLogger(__name__)
+
-ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]
+ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
try:
- import brotli
-
- ACCEPTED_ENCODINGS.append(b"br")
+ import brotli # noqa: F401
except ImportError:
pass
+ else:
+ ACCEPTED_ENCODINGS.append(b"br")
try:
- import zstandard
-
- ACCEPTED_ENCODINGS.append(b"zstd")
+ import zstandard # noqa: F401
except ImportError:
pass
+ else:
+ ACCEPTED_ENCODINGS.append(b"zstd")
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
- def __init__(self, stats: Optional[StatsCollector] = None):
- self.stats = stats
- def __init__(self, stats=None, *, crawler=None):
++ def __init__(
++ self,
++ stats: Optional[StatsCollector] = None,
++ *,
++ crawler: Optional[Crawler] = None,
++ ):
+ if not crawler:
+ self.stats = stats
+ self._max_size = 1073741824
+ self._warn_size = 33554432
+ return
+ self.stats = crawler.stats
+ self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+ self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
+ crawler.signals.connect(self.open_spider, signals.spider_opened)
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
raise NotConfigured
- return cls(stats=crawler.stats)
+ try:
+ return cls(crawler=crawler)
+ except TypeError:
+ warnings.warn(
+ "HttpCompressionMiddleware subclasses must either modify "
+ "their '__init__' method to support a 'crawler' parameter or "
+ "reimplement their 'from_crawler' method.",
+ ScrapyDeprecationWarning,
+ )
+ mw = cls()
+ mw.stats = crawler.stats
+ mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+ mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
+ crawler.signals.connect(mw.open_spider, signals.spider_opened)
+ return mw
+
+ def open_spider(self, spider):
+ if hasattr(spider, "download_maxsize"):
+ self._max_size = spider.download_maxsize
+ if hasattr(spider, "download_warnsize"):
+ self._warn_size = spider.download_warnsize
- def process_request(self, request, spider):
+ def process_request(
+ self, request: Request, spider: Spider
+ ) -> Union[Request, Response, None]:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
+ return None
- def process_response(self, request, response, spider):
+ def process_response(
+ self, request: Request, response: Response, spider: Spider
+ ) -> Union[Request, Response]:
if request.method == "HEAD":
return response
if isinstance(response, Response):
@@@ -85,25 -125,13 +144,13 @@@
return response
- def _decode(self, body: bytes, encoding: bytes) -> bytes:
- def _decode(self, body, encoding, max_size):
++ def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
if encoding == b"gzip" or encoding == b"x-gzip":
- body = gunzip(body)
-
+ return gunzip(body, max_size=max_size)
if encoding == b"deflate":
- try:
- body = zlib.decompress(body)
- except zlib.error:
- # ugly hack to work with raw deflate content that may
- # be sent by microsoft servers. For more information, see:
- # http://carsten.codimi.de/gzip.yaws/
- # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
- # http://www.gzip.org/zlib/zlib_faq.html#faq38
- body = zlib.decompress(body, -15)
+ return _inflate(body, max_size=max_size)
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
- body = brotli.decompress(body)
+ return _unbrotli(body, max_size=max_size)
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
- # Using its streaming API since its simple API could handle only cases
- # where there is content size data embedded in the frame
- reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
- body = reader.read()
+ return _unzstd(body, max_size=max_size)
return body
diff --cc scrapy/utils/response.py
index 77d54aff9,fabfb1167..33cd692bf
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@@ -54,6 -55,37 +54,18 @@@ def response_status_message(status: Uni
return f"{status_int} {to_unicode(message)}"
-@deprecated
-def response_httprepr(response: Response) -> bytes:
- """Return raw HTTP representation (as bytes) of the given response. This
- is provided only for reference, since it's not the exact stream of bytes
- that was received (that's not exposed by Twisted).
- """
- values = [
- b"HTTP/1.1 ",
- to_bytes(str(response.status)),
- b" ",
- to_bytes(http.RESPONSES.get(response.status, b"")),
- b"\r\n",
- ]
- if response.headers:
- values.extend([response.headers.to_string(), b"\r\n"])
- values.extend([b"\r\n", response.body])
- return b"".join(values)
-
-
+ def _remove_html_comments(body):
+ start = body.find(b"<!--")
+ while start != -1:
+ end = body.find(b"-->", start + 1)
+ if end == -1:
+ return body[:start]
+ else:
+ body = body[:start] + body[end + 3 :]
+ start = body.find(b"<!--")
+ return body
+
+
def open_in_browser(
response: Union[
"scrapy.http.response.html.HtmlResponse",
diff --cc tests/test_utils_response.py
index 661fb47a3,37ef89e76..7ad86127b
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@@ -1,7 -1,12 +1,10 @@@
import unittest
-import warnings
from pathlib import Path
+ from time import process_time
from urllib.parse import urlparse
+ import pytest
+
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response, TextResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
diff --cc tox.ini
index d9dcacc01,381da9773..359ff0f73
--- a/tox.ini
+++ b/tox.ini
@@@ -124,6 -128,6 +124,8 @@@ deps
robotexclusionrulesparser
Pillow
Twisted[http2]
++ brotli
++ zstandard
[testenv:extra-deps-pinned]
basepython = python3.8