Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'scrapy/2.11' into 2.11-port
Adrián Chaves6bd45bb6d92e
diff --cc scrapy/downloadermiddlewares/httpcompression.pyindex 56a58a750,816be25a1..709333948--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@@ -1,60 -1,83 +1,102 @@@+from __future__ import annotations+- import io- import zlib+ import warnings+ from logging import getLogger+from typing import TYPE_CHECKING, List, Optional, Union- from scrapy import Request, Spider-from scrapy import signals++from scrapy import Request, Spider, signals+from scrapy.crawler import Crawler- from scrapy.exceptions import NotConfigured+ from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Response, TextResponsefrom scrapy.responsetypes import responsetypes+from scrapy.statscollectors import StatsCollector+ from scrapy.utils._compression import (+ _DecompressionMaxSizeExceeded,+ _inflate,+ _unbrotli,+ _unzstd,+ )+ from scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.gz import gunzip+if TYPE_CHECKING:+ # typing.Self requires Python 3.11+ from typing_extensions import Self++ logger = getLogger(__name__)+-ACCEPTED_ENCODINGS = [b"gzip", b"deflate"]+ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]try:- import brotli-- ACCEPTED_ENCODINGS.append(b"br")+ import brotli # noqa: F401except ImportError:pass+ else:+ ACCEPTED_ENCODINGS.append(b"br")try:- import zstandard-- ACCEPTED_ENCODINGS.append(b"zstd")+ import zstandard # noqa: F401except ImportError:pass+ else:+ ACCEPTED_ENCODINGS.append(b"zstd")class HttpCompressionMiddleware:"""This middleware allows compressed (gzip, deflate) traffic to besent/received from web sites"""- def __init__(self, stats: Optional[StatsCollector] = None):- self.stats = stats- def __init__(self, stats=None, *, crawler=None):++ def __init__(++ self,++ stats: Optional[StatsCollector] = None,++ *,++ crawler: Optional[Crawler] = None,++ ):+ if not crawler:+ self.stats = stats+ self._max_size = 1073741824+ self._warn_size = 33554432+ return+ self.stats = crawler.stats+ self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")+ self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")+ crawler.signals.connect(self.open_spider, signals.spider_opened)@classmethod- def from_crawler(cls, crawler):+ def from_crawler(cls, crawler: Crawler) -> Self:if not crawler.settings.getbool("COMPRESSION_ENABLED"):raise NotConfigured- return cls(stats=crawler.stats)+ try:+ return cls(crawler=crawler)+ except TypeError:+ warnings.warn(+ "HttpCompressionMiddleware subclasses must either modify "+ "their '__init__' method to support a 'crawler' parameter or "+ "reimplement their 'from_crawler' method.",+ ScrapyDeprecationWarning,+ )+ mw = cls()+ mw.stats = crawler.stats+ mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")+ mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")+ crawler.signals.connect(mw.open_spider, signals.spider_opened)+ return mw++ def open_spider(self, spider):+ if hasattr(spider, "download_maxsize"):+ self._max_size = spider.download_maxsize+ if hasattr(spider, "download_warnsize"):+ self._warn_size = spider.download_warnsize- def process_request(self, request, spider):+ def process_request(+ self, request: Request, spider: Spider+ ) -> Union[Request, Response, None]:request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))+ return None- def process_response(self, request, response, spider):+ def process_response(+ self, request: Request, response: Response, spider: Spider+ ) -> Union[Request, Response]:if request.method == "HEAD":return responseif isinstance(response, Response):@@@ -85,25 -125,13 +144,13 @@@return response- def _decode(self, body: bytes, encoding: bytes) -> bytes:- def _decode(self, body, encoding, max_size):++ def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:if encoding == b"gzip" or encoding == b"x-gzip":- body = gunzip(body)-+ return gunzip(body, max_size=max_size)if encoding == b"deflate":- try:- body = zlib.decompress(body)- except zlib.error:- # ugly hack to work with raw deflate content that may- # be sent by microsoft servers. For more information, see:- # http://carsten.codimi.de/gzip.yaws/- # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx- # http://www.gzip.org/zlib/zlib_faq.html#faq38- body = zlib.decompress(body, -15)+ return _inflate(body, max_size=max_size)if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:- body = brotli.decompress(body)+ return _unbrotli(body, max_size=max_size)if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:- # Using its streaming API since its simple API could handle only cases- # where there is content size data embedded in the frame- reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))- body = reader.read()+ return _unzstd(body, max_size=max_size)return bodydiff --cc scrapy/utils/response.pyindex 77d54aff9,fabfb1167..33cd692bf--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@@ -54,6 -55,37 +54,18 @@@ def response_status_message(status: Unireturn f"{status_int} {to_unicode(message)}"-@deprecated-def response_httprepr(response: Response) -> bytes:- """Return raw HTTP representation (as bytes) of the given response. This- is provided only for reference, since it's not the exact stream of bytes- that was received (that's not exposed by Twisted).- """- values = [- b"HTTP/1.1 ",- to_bytes(str(response.status)),- b" ",- to_bytes(http.RESPONSES.get(response.status, b"")),- b"\r\n",- ]- if response.headers:- values.extend([response.headers.to_string(), b"\r\n"])- values.extend([b"\r\n", response.body])- return b"".join(values)--+ def _remove_html_comments(body):+ start = body.find(b"<!--")+ while start != -1:+ end = body.find(b"-->", start + 1)+ if end == -1:+ return body[:start]+ else:+ body = body[:start] + body[end + 3 :]+ start = body.find(b"<!--")+ return body++def open_in_browser(response: Union["scrapy.http.response.html.HtmlResponse",diff --cc tests/test_utils_response.pyindex 661fb47a3,37ef89e76..7ad86127b--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@@ -1,7 -1,12 +1,10 @@@import unittest-import warningsfrom pathlib import Path+ from time import process_timefrom urllib.parse import urlparse+ import pytest+-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Response, TextResponsefrom scrapy.utils.python import to_bytesfrom scrapy.utils.response import (diff --cc tox.iniindex d9dcacc01,381da9773..359ff0f73--- a/tox.ini+++ b/tox.ini@@@ -124,6 -128,6 +124,8 @@@ depsrobotexclusionrulesparserPillowTwisted[http2]++ brotli++ zstandard[testenv:extra-deps-pinned]basepython = python3.8