Scrapy, a fast high-level web crawling & scraping framework for Python.
Frostwork integration (#8393)
Adrianae4c0cee1726
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rstindex 788de9d1f..63e536214 100644--- a/docs/topics/addons.rst+++ b/docs/topics/addons.rst@@ -125,7 +125,7 @@ Set some basic configuration:class MyAddon:def update_settings(self, settings):settings.set("DNSCACHE_ENABLED", True, "addon")- settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript")+ settings.remove_from_list("RETRY_HTTP_CODES", 429)settings.setdefault_in_component_priority_dict("ITEM_PIPELINES", MyPipeline, 200)diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 0ddf2bd70..a85b4e455 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -1079,8 +1079,8 @@ The :class:`MetaRefreshMiddleware` can be configured through the followingsettings (see the settings documentation for more info):* :setting:`METAREFRESH_ENABLED`-* :setting:`METAREFRESH_IGNORE_TAGS`* :setting:`METAREFRESH_MAXDELAY`+* :setting:`METAREFRESH_SCRIPTING`This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described@@ -1099,19 +1099,6 @@ Default: ``True``Whether the Meta Refresh middleware will be enabled.-.. setting:: METAREFRESH_IGNORE_TAGS--METAREFRESH_IGNORE_TAGS-^^^^^^^^^^^^^^^^^^^^^^^--Default: ``["noscript"]``--Meta tags within these tags are ignored.--.. versionchanged:: 2.11.2- The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from- ``[]`` to ``["noscript"]``.-.. setting:: METAREFRESH_MAXDELAYMETAREFRESH_MAXDELAY@@ -1123,6 +1110,19 @@ The maximum meta-refresh delay (in seconds) to follow the redirection.Some sites use meta-refresh for redirecting to a session expired page, so werestrict automatic redirection to the maximum delay.+METAREFRESH_SCRIPTING+^^^^^^^^^^^^^^^^^^^^^++.. versionadded:: VERSION++.. setting:: METAREFRESH_SCRIPTING++Default: ``True``++Whether to follow the meta refresh that a web browser would follow with+JavaScript enabled (``True``), ignoring those within ``<noscript>``, or with+JavaScript disabled (``False``).+RetryMiddleware---------------diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex b954e2a73..58a0d8a39 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -1454,10 +1454,12 @@ TextResponse objectsis the same as for the :class:`Response` class and is not documented here.:param encoding: is a string which contains the encoding to use for this- response. If you create a :class:`TextResponse` object with a string as- body, it will be converted to bytes encoded using this encoding. If- *encoding* is ``None`` (default), the encoding will be looked up in the- response headers and body instead.+ response, as a label of the `Encoding Standard`_, which web browsers+ implement, or as the name of the Python codec of one of its encodings,+ e.g. ``cp932`` for Shift_JIS. If you create a :class:`TextResponse`+ object with a string as body, it will be converted to bytes encoded+ using this encoding. If *encoding* is ``None`` (default), the encoding+ will be looked up in the response headers and body instead.:type encoding: str:class:`TextResponse` objects support the following attributes in addition@@ -1467,8 +1469,7 @@ TextResponse objectsResponse body, as a string.- The same as ``response.body.decode(response.encoding)``, but the- result is cached after the first call, so you can access+ The result is cached after the first call, so you can access``response.text`` multiple times without extra overhead... note::@@ -1540,6 +1541,10 @@ TextResponse objectsA :class:`~scrapy.Selector` instance using the response astarget. The selector is lazily instantiated on first access.+ .. autoattribute:: TextResponse.base_url++ .. autoattribute:: TextResponse.meta_refresh+.. autoattribute:: TextResponse.attributes:class:`TextResponse` objects support the following methods in addition to@@ -1581,12 +1586,14 @@ TextResponse objects.. method:: TextResponse.urljoin(url)- Constructs an absolute url by combining the Response's base url with- a possible relative url. The base url shall be extracted from the- ``<base>`` tag, or just :attr:`Response.url` if there is no such- tag.+ Constructs an absolute url by combining :attr:`base_url` with a+ possible relative url.++.. autoclass:: MetaRefresh+ :members:.. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark+.. _Encoding Standard: https://encoding.spec.whatwg.org/.. _encoding sniffing algorithm: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encodingdiff --git a/pyproject.toml b/pyproject.tomlindex 9e61bbf32..f66c54992 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -14,11 +14,11 @@ dependencies = ["aiohttp>=3.13.3",'backports.zstd>=1.3.0; python_version < "3.14"',"brotli>=1.2.0",- "charset-normalizer>=3.4.0","cryptography>=41.0.5","cssselect>=1.2.0","defusedxml>=0.7.1","formerly>=0.1.0",+ "frostwork>=0.2.2","itemadapter>=0.1.0","itemloaders>=1.0.5","lxml>=4.9.4",@@ -30,8 +30,8 @@ dependencies = ["queuelib>=1.8.0","service_identity>=24.2.0","tldextract",- 'typing-extensions>=4.6.0; python_version < "3.13"',- "w3lib>=2.1.1",+ 'typing-extensions>=4.13.0; python_version < "3.13"',+ "w3lib>=2.6.0","zope.interface>=5.1.0",]classifiers = [diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 80fe2da39..271b4de4e 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -1,16 +1,19 @@from __future__ import annotationsimport logging-from typing import TYPE_CHECKING, Any, Self, cast+import warnings+from typing import TYPE_CHECKING, Any, Self++from w3lib.url import canonicalize_urlfrom scrapy import signals-from scrapy.exceptions import IgnoreRequest, NotConfigured+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Responsefrom scrapy.spidermiddlewares.referer import RefererMiddlewarefrom scrapy.utils.decorators import _warn_spider_argfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import global_object_name-from scrapy.utils.response import _body_size, get_meta_refresh+from scrapy.utils.response import _IGNORABLE_TAGS, _body_size, _get_meta_refreshfrom scrapy.utils.url import _redirect_urlif TYPE_CHECKING:@@ -264,6 +267,33 @@ class RedirectMiddleware(BaseRedirectMiddleware):return self._redirect(redirected, request, response.status)+def _get_scripting(settings: BaseSettings) -> bool:+ ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")+ ignore_tags_priority = settings.getpriority("METAREFRESH_IGNORE_TAGS") or 0+ scripting_priority = settings.getpriority("METAREFRESH_SCRIPTING") or 0+ # remove_from_list() keeps the default priority, hence the value check.+ if (not ignore_tags_priority and ignore_tags == ["noscript"]) or (+ scripting_priority and scripting_priority >= ignore_tags_priority+ ):+ return settings.getbool("METAREFRESH_SCRIPTING")+ if not set(ignore_tags) <= _IGNORABLE_TAGS:+ raise ValueError(+ f"METAREFRESH_IGNORE_TAGS is set to {ignore_tags!r}, but only "+ f"{sorted(_IGNORABLE_TAGS)!r} are supported. "+ f"METAREFRESH_IGNORE_TAGS is deprecated, we strongly recommend "+ f"removing it and using METAREFRESH_SCRIPTING instead. As a last "+ f"resort, subclass MetaRefreshMiddleware."+ )+ scripting = "noscript" in ignore_tags+ warnings.warn(+ f"The METAREFRESH_IGNORE_TAGS setting is deprecated, replace it "+ f"with METAREFRESH_SCRIPTING = {scripting}.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return scripting++class MetaRefreshMiddleware(BaseRedirectMiddleware):"""Handle redirection of requests based on meta-refresh html tag."""@@ -271,7 +301,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):def __init__(self, settings: BaseSettings):super().__init__(settings)- self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")+ self._scripting: bool = _get_scripting(settings)self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")@_warn_spider_arg@@ -286,12 +316,17 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):):return response- interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)- if not url:+ meta_refresh = _get_meta_refresh(response, scripting=self._scripting)+ if not meta_refresh:+ return response+ interval, url = meta_refresh+ # A refresh without a URL reloads the page, which the dupefilter would+ # drop.+ if canonicalize_url(url) == canonicalize_url(response.url):return responseredirected = self._redirect_request_using_get(request, response, url)if urlparse_cached(redirected).scheme not in {"http", "https"}:return response- if cast("float", interval) < self._maxdelay:+ if interval < self._maxdelay:return self._redirect(redirected, request, "meta refresh")return responsediff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.pyindex ae6486c4e..fba327afe 100644--- a/scrapy/http/__init__.py+++ b/scrapy/http/__init__.py@@ -14,7 +14,7 @@ from scrapy.http.response import Responsefrom scrapy.http.response.body_file import BodyFileResponsefrom scrapy.http.response.html import HtmlResponsefrom scrapy.http.response.json import JsonResponse-from scrapy.http.response.text import TextResponse+from scrapy.http.response.text import MetaRefresh, TextResponsefrom scrapy.http.response.xml import XmlResponse__all__ = [@@ -24,6 +24,7 @@ __all__ = ["HtmlResponse","JsonRequest","JsonResponse",+ "MetaRefresh","Request","Response","TextResponse",diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 6ba03229a..60c4c52be 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -8,15 +8,18 @@ See documentation in docs/topics/request-response.rstfrom __future__ import annotationsimport json+import warningsimport weakreffrom contextlib import suppress-from typing import TYPE_CHECKING, Any, cast+from enum import Enum, auto+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, castfrom urllib.parse import urljoin+import frostworkimport parsel-from charset_normalizer import from_bytesfrom w3lib.encoding import (html_body_declared_encoding,+ html_encoding,html_to_unicode,http_content_type_encoding,read_bom,@@ -24,9 +27,9 @@ from w3lib.encoding import ()from w3lib.html import strip_html5_whitespace+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.response import Responsefrom scrapy.utils.python import memoizemethod_noargs-from scrapy.utils.response import get_base_urlif TYPE_CHECKING:from collections.abc import Callable, Iterable, Mapping@@ -38,27 +41,66 @@ if TYPE_CHECKING:from scrapy.selector import Selector, SelectorList-_NONE = object()+class _Unset(Enum):+ UNSET = auto()-class TextResponse(Response):- _DEFAULT_ENCODING = "ascii"+_UNSET = _Unset.UNSET+++class MetaRefresh(NamedTuple):+ """Meta refresh of a :class:`TextResponse`, as+ :attr:`TextResponse.meta_refresh` reports it.++ .. versionadded:: VERSION+ """++ delay: float+ """Seconds to wait before the refresh."""++ url: str+ """Absolute URL to load."""++def _is_browser_encoding(encoding: str) -> bool:+ resolved = resolve_encoding(encoding)+ # "replacement" is what response.encoding reports for labels that browsers+ # decode as a single U+FFFD, e.g. a Content-Type charset of hz.+ return resolved is not None and (+ resolved != "replacement" or encoding == "replacement"+ )+++class TextResponse(Response):attributes: tuple[str, ...] = (*Response.attributes, "encoding")__slots__ = (+ "_base_url","_cached_benc","_cached_decoded_json","_cached_selector","_cached_ubody","_encoding",+ "_meta_refresh",)def __init__(self, *args: Any, **kwargs: Any):self._encoding: str | None = kwargs.pop("encoding", None)+ if self._encoding is not None and not _is_browser_encoding(self._encoding):+ warnings.warn(+ f"Passing encoding={self._encoding!r} to {type(self).__name__} "+ f"is deprecated, because web browsers do not support that "+ f"encoding, and it will raise ValueError in a future version "+ f"of Scrapy. To decode the response body with it, use "+ f"response.body.decode({self._encoding!r}) instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )self._cached_benc: str | None = Noneself._cached_ubody: str | None = Noneself._cached_selector: Selector | None = None- self._cached_decoded_json: object = _NONE+ self._cached_decoded_json: object = _UNSET+ self._base_url: str | None = None+ self._meta_refresh: MetaRefresh | Literal[_Unset.UNSET] | None = _UNSETsuper().__init__(*args, **kwargs)def _set_body(self, body: str | bytes | None) -> None:@@ -103,7 +145,7 @@ class TextResponse(Response):The result is cached after the first call."""- if self._cached_decoded_json is _NONE:+ if self._cached_decoded_json is _UNSET:try:self._cached_decoded_json = json.loads(self.body)except UnicodeDecodeError:@@ -113,18 +155,74 @@ class TextResponse(Response):@propertydef text(self) -> str:"""Body as unicode"""- # access self.encoding before _cached_ubody to make sure- # _body_inferred_encoding is called- benc = self.encodingif self._cached_ubody is None:- charset = f"charset={benc}"- self._cached_ubody = html_to_unicode(charset, self.body)[1]+ if (+ self._encoding is not None+ and not _is_browser_encoding(self._encoding)+ and not self._bom_encoding()+ ):+ with suppress(LookupError):+ self._cached_ubody = self.body.decode(self._encoding, "replace")+ if self._cached_ubody is None:+ charset = f"charset={self.encoding}"+ self._cached_ubody = html_to_unicode(charset, self.body)[1]return self._cached_ubodydef urljoin(self, url: str) -> str:"""Join this Response's url with a possible relative url to form anabsolute interpretation of the latter."""- return urljoin(get_base_url(self), url)+ return urljoin(self.base_url, url)++ @property+ def base_url(self) -> str:+ """Absolute URL that relative URLs of the response are relative to.++ .. versionadded:: VERSION++ It is the ``href`` of the first ``<base>`` element that has one,+ resolved against :attr:`~Response.url`, or :attr:`~Response.url`+ itself, serialized like a web browser does, e.g. with ``/`` as empty+ path. Setting it overrides the value read from the body.+ """+ if self._base_url is None:+ self._read_html_metadata()+ assert self._base_url is not None+ return self._base_url++ @base_url.setter+ def base_url(self, value: str) -> None:+ self._base_url = value++ @property+ def meta_refresh(self) -> MetaRefresh | None:+ """:class:`MetaRefresh` that a web browser with JavaScript enabled+ would follow, or ``None``.++ .. versionadded:: VERSION++ Setting it overrides the value read from the body.+ """+ if self._meta_refresh is _UNSET:+ self._read_html_metadata()+ assert self._meta_refresh is not _UNSET+ return self._meta_refresh++ @meta_refresh.setter+ def meta_refresh(self, value: tuple[float, str] | None) -> None:+ self._meta_refresh = None if value is None else MetaRefresh(*value)++ def _read_html_metadata(self) -> None:+ metadata = frostwork.navigation(+ self.body,+ self.url,+ self.encoding,+ base_url=self._base_url is None,+ meta_refresh=self._meta_refresh is _UNSET,+ )+ if self._base_url is None:+ self._base_url = metadata.base_url+ if self._meta_refresh is _UNSET:+ self.meta_refresh = metadata.meta_refresh@memoizemethod_noargsdef _headers_encoding(self) -> str | None:@@ -132,28 +230,13 @@ class TextResponse(Response):return http_content_type_encoding(content_type.decode("latin-1"))def _body_inferred_encoding(self) -> str:- content_type = self.headers.get(b"Content-Type") or b""if self._cached_benc is None:- benc, ubody = html_to_unicode(- content_type.decode("latin-1"),- self.body,- auto_detect_fun=self._auto_detect_fun,- default_encoding=self._DEFAULT_ENCODING,+ content_type = self.headers.get(b"Content-Type") or b""+ self._cached_benc = html_encoding(+ content_type.decode("latin-1"), self.body, url=self.url)- self._cached_benc = benc- self._cached_ubody = ubodyreturn self._cached_benc- def _auto_detect_fun(self, text: bytes) -> str | None:- for enc in (self._DEFAULT_ENCODING, "utf-8"):- try:- text.decode(enc)- except UnicodeError:- continue- return resolve_encoding(enc)- match = from_bytes(text).best()- return resolve_encoding(match.encoding) if match else None-@memoizemethod_noargsdef _body_declared_encoding(self) -> str | None:return html_body_declared_encoding(self.body)diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex beae807e4..7bd761ff1 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -22,7 +22,6 @@ from scrapy.linkextractors import IGNORED_EXTENSIONS, SUPPORTED_SCHEMES, _matchefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.misc import arg_to_iter, rel_has_nofollowfrom scrapy.utils.python import unique as unique_list-from scrapy.utils.response import get_base_urlfrom scrapy.utils.url import url_is_from_any_domainif TYPE_CHECKING:@@ -164,7 +163,7 @@ class LxmlParserLinkExtractor:return self._deduplicate_if_needed(links)def extract_links(self, response: TextResponse) -> list[Link]:- base_url = get_base_url(response)+ base_url = response.base_urlreturn self._extract_links(response.selector, response.url, response.encoding, base_url)@@ -438,7 +437,7 @@ class LxmlLinkExtractor:Duplicate links are omitted if the ``unique`` attribute is set to ``True``,otherwise they are returned."""- base_url = get_base_url(response)+ base_url = response.base_urlif self.restrict_xpaths:docs = [subdoc for x in self.restrict_xpaths for subdoc in response.xpath(x)diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex d351ca84e..1caacc9e4 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -4,11 +4,11 @@ import weakreffrom typing import Any, Literalfrom parsel import Selector as _ParselSelector+from w3lib.encoding import resolve_encodingfrom scrapy.http import HtmlResponse, JsonResponse, TextResponse, XmlResponsefrom scrapy.utils._deps_compat import PARSEL_BODY_ENCODING_FIXEDfrom scrapy.utils.python import to_bytes-from scrapy.utils.response import get_base_urlfrom scrapy.utils.trackref import object_ref__all__ = ["Selector", "SelectorList"]@@ -106,9 +106,12 @@ class Selector(_ParselSelector, object_ref):# "xml", and treats an empty body as absent, unlike empty text.if PARSEL_BODY_ENCODING_FIXED and type in ("html", "xml") and response.body:encoding = response.encoding- # None means the encoding was declared, not inferred from- # the body, so the body hasn't been decoded as a side effect.- if response._cached_ubody is None:+ # parsel decodes non-UTF-8 bodies with Python's codecs, which+ # differ from the browser decoding of response.text.+ if (+ response._cached_ubody is None+ and resolve_encoding(encoding) == "utf-8"+ ):kwargs["body"] = response.bodykwargs["encoding"] = encodingtext = None@@ -116,7 +119,7 @@ class Selector(_ParselSelector, object_ref):text = response.textelse:text = response.text- kwargs.setdefault("base_url", get_base_url(response))+ kwargs.setdefault("base_url", response.base_url)self._response: TextResponse | weakref.ref[TextResponse] | None = responsediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex ede9d09c5..0a86215e5 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -181,6 +181,7 @@ __all__ = ["METAREFRESH_ENABLED","METAREFRESH_IGNORE_TAGS","METAREFRESH_MAXDELAY",+ "METAREFRESH_SCRIPTING","NEWSPIDER_MODULE","PERIODIC_LOG_DELTA","PERIODIC_LOG_STATS",@@ -546,6 +547,7 @@ MEMUSAGE_WARNING_MB = 0METAREFRESH_ENABLED = TrueMETAREFRESH_IGNORE_TAGS = ["noscript"]METAREFRESH_MAXDELAY = 100+METAREFRESH_SCRIPTING = TrueNEWSPIDER_MODULE = ""diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex fa8911aeb..d62d4fcd1 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -9,14 +9,15 @@ import mimetypesimport osimport reimport tempfile+import warningsimport webbrowserfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any-from weakref import WeakKeyDictionary+import frostworkfrom twisted.web import http-from w3lib import html+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.headers import Headersfrom scrapy.http.response.body_file import BodyFileResponsefrom scrapy.utils.misc import load_object@@ -33,35 +34,50 @@ def _body_size(response: Response) -> int:return len(response.body)-_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()+def get_base_url(response: TextResponse) -> str:+ warnings.warn(+ "scrapy.utils.response.get_base_url() is deprecated, use "+ "TextResponse.base_url instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return response.base_url-def get_base_url(response: TextResponse) -> str:- """Return the base url of the given response, joined with the response url"""- if response not in _baseurl_cache:- text = response.text[0:4096]- _baseurl_cache[response] = html.get_base_url(- text, response.url, response.encoding- )- return _baseurl_cache[response]+_IGNORABLE_TAGS = {"script", "noscript", "template"}-_metaref_cache: WeakKeyDictionary[Response, tuple[None, None] | tuple[float, str]] = (- WeakKeyDictionary()-)+def _get_meta_refresh(+ response: TextResponse, *, scripting: bool+) -> tuple[float, str] | None:+ if scripting:+ return response.meta_refresh+ return frostwork.navigation(+ response.body,+ response.url,+ response.encoding,+ base_url=False,+ scripting=False,+ ).meta_refreshdef get_meta_refresh(response: TextResponse,ignore_tags: Iterable[str] = ("script", "noscript"),) -> tuple[None, None] | tuple[float, str]:- """Parse the http-equiv refresh parameter from the given response"""- if response not in _metaref_cache:- text = response.text[0:4096]- _metaref_cache[response] = html.get_meta_refresh(- text, get_base_url(response), response.encoding, ignore_tags=ignore_tags+ warnings.warn(+ "scrapy.utils.response.get_meta_refresh() is deprecated, use "+ "TextResponse.meta_refresh instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ ignore_tags = set(ignore_tags)+ if not ignore_tags <= _IGNORABLE_TAGS:+ raise ValueError(+ f"Unsupported ignore_tags: {sorted(ignore_tags - _IGNORABLE_TAGS)}")- return _metaref_cache[response]+ meta_refresh = _get_meta_refresh(response, scripting="noscript" in ignore_tags)+ return meta_refresh or (None, None)def response_from_dict(d: dict[str, Any]) -> Response:@@ -134,7 +150,7 @@ def open_in_browser(# any base tag of the response, and matches it when there is one.doctype = _DOCTYPE_RE.match(body)index = doctype.end() if doctype else 0- base_tag = f'<base href="{get_base_url(response)}">'.encode()+ base_tag = f'<base href="{response.base_url}">'.encode()body = body[:index] + base_tag + body[index:]ext = ".html"elif isinstance(response, TextResponse):diff --git a/tests/benchmarks/test_encoding.py b/tests/benchmarks/test_encoding.pynew file mode 100644index 000000000..09e402d6c--- /dev/null+++ b/tests/benchmarks/test_encoding.py@@ -0,0 +1,66 @@+from __future__ import annotations++from typing import TYPE_CHECKING++import pytest++from scrapy.http import TextResponse+from tests import get_testdata++if TYPE_CHECKING:+ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]++pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")++# A real page without its encoding declaration.+_PAGE = (+ get_testdata("books", "listing.html")+ .decode()+ .replace("text/html; charset=UTF-8", "text/html")+)+++def _body(text: str, encoding: str) -> bytes:+ """Return :data:`_PAGE` with *text* added, encoded with *encoding*."""+ return _PAGE.replace("</body>", f"<p>{text * 500}</p></body>").encode(encoding)+++_JAPANESE = "この本は在庫があります。"+_FRENCH = "Ce livre à prix réduit est déjà épuisé. "+++_UNDECLARED = pytest.mark.parametrize(+ ("body", "encoding"),+ [+ pytest.param(_body(_JAPANESE, "utf-8"), "utf-8", id="utf-8"),+ pytest.param(_body(_JAPANESE, "shift_jis"), "cp932", id="shift_jis"),+ pytest.param(_body(_FRENCH, "cp1252"), "cp1252", id="windows-1252"),+ ],+)+++@_UNDECLARED+def test_undeclared_encoding(+ benchmark: BenchmarkFixture, body: bytes, encoding: str+) -> None:+ """Decoding a page that declares no encoding."""++ def run() -> None:+ response = TextResponse("https://example.com", body=body)+ assert response.text+ assert response.encoding == encoding++ benchmark(run)+++@_UNDECLARED+def test_undeclared_encoding_only(+ benchmark: BenchmarkFixture, body: bytes, encoding: str+) -> None:+ """Getting the encoding of a page that declares none, without its text."""++ def run() -> None:+ response = TextResponse("https://example.com", body=body)+ assert response.encoding == encoding++ benchmark(run)diff --git a/tests/sample_data/feeds/feed-sample5.csv b/tests/sample_data/feeds/feed-sample5.csvindex 9cff1dfa4..90ad09272 100644--- a/tests/sample_data/feeds/feed-sample5.csv+++ b/tests/sample_data/feeds/feed-sample5.csv@@ -1,3 +1,3 @@id,name,value-1,cp852,test+1,cp866,test2,something,�����ͻdiff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex 9d4a00dc8..504b3d1ed 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -623,9 +623,9 @@ class TestContractsManager:requests = self.conman.from_method(spider.returns_multiple_urls, self.results)assert sorted(request.url for request in requests) == [- "http://example.com",- "http://scrapy.org",- "http://scrapy.org",+ "http://example.com/",+ "http://scrapy.org/",+ "http://scrapy.org/",]for request in requests:diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex a3ab5d1c1..c0564f52f 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -167,7 +167,7 @@ class TestHttpsWithCrawler(TestHttpWithCrawler):class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):- expected_http_proxy_request_body = b"http://example.com/"+ passclass TestHttpsProxy(TestHttpProxy):@@ -186,7 +186,7 @@ class TestHttpsProxy(TestHttpProxy):request = Request("http://example.com", meta={"proxy": http_proxy})async with self.get_dh() as download_handler:response = await download_handler.download_request(request)- assert response.body == self.expected_http_proxy_request_body+ assert response.body == b"http://example.com/"assert keylog_file.read_text()diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex c6b19d464..e6c593712 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -1004,7 +1004,7 @@ class TestConcurrentDownloads(TestBase):policy_class = "scrapy.extensions.httpcache.DummyPolicy"storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"- url = "http://www.example.com"+ url = "http://www.example.com/"@contextmanagerdef _manager(@@ -1204,7 +1204,7 @@ class TestInFlightTracking(TestBase):policy_class = "scrapy.extensions.httpcache.DummyPolicy"storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"- url = "http://www.example.com"+ url = "http://www.example.com/"@coroutine_testasync def test_garbage_collected_request(self):diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 3372813f9..a5f05fad0 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -18,7 +18,9 @@ class TestHttpProxyMiddleware:with pytest.raises(NotConfigured):build_from_crawler(HttpProxyMiddleware, crawler)- @pytest.mark.parametrize("url", ["http://e.com", "https://e.com", "file:///tmp/a"])+ @pytest.mark.parametrize(+ "url", ["http://e.com/", "https://e.com/", "file:///tmp/a"]+ )def test_no_environment_proxies(self, url: str, monkeypatch: pytest.MonkeyPatch) -> None:@@ -35,8 +37,8 @@ class TestHttpProxyMiddleware:@pytest.mark.parametrize(("url", "proxy"),[- ("http://e.com", "https://proxy.for.http:3128"),- ("https://e.com", "http://proxy.for.https:8080"),+ ("http://e.com/", "https://proxy.for.http:3128"),+ ("https://e.com/", "http://proxy.for.https:8080"),("file://tmp/a", None),],)diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex d3cc1be0a..ceb0b09f9 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -234,7 +234,7 @@ class TestRedirectMiddleware(TestRedirectBase):@pytest.mark.parametrize("status", [307, 308])def test_cross_origin_maintain_body(self, status):source_url = "https://example.com"- target_url = "https://attacker.example"+ target_url = "https://attacker.example/"body = b"secret"request = Request(source_url,diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.pyindex d118da005..06988e003 100644--- a/tests/test_downloadermiddleware_redirect_metarefresh.py+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py@@ -2,12 +2,13 @@ from __future__ import annotationsimport loggingfrom itertools import chain+from typing import TYPE_CHECKINGfrom unittest.mock import MagicMockimport pytestfrom scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware-from scrapy.exceptions import NotConfigured+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Request, Responsefrom scrapy.spiders import Spiderfrom scrapy.utils.misc import build_from_crawler@@ -20,6 +21,9 @@ from tests.utils.redirect import (SCHEME_PARAMS,)+if TYPE_CHECKING:+ from scrapy.settings import BaseSettings+def meta_refresh_body(url: str, interval: int = 5) -> bytes:html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""@@ -91,19 +95,90 @@ class TestMetaRefreshMiddleware(TestRedirectBase):response = self.mw.process_response(req, rsp)assert isinstance(response, Response)- def test_ignore_tags_1_x_list(self):- """Test that Scrapy 1.x behavior remains possible"""- settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]}- crawler = get_crawler(Spider, settings)+ def test_scripting_false(self):+ crawler = get_crawler(Spider, {"METAREFRESH_SCRIPTING": False})mw = build_from_crawler(MetaRefreshMiddleware, crawler)req = Request(url="http://example.org")- body = (- """<noscript><meta http-equiv="refresh" """- """content="0;URL='http://example.org/newpage'"></noscript>"""+ body = b"<noscript>" + self._body(interval=0) + b"</noscript>"+ req2 = mw.process_response(req, HtmlResponse(req.url, body=body))+ assert isinstance(req2, Request)+ assert req2.url == "http://example.org/newpage"++ @pytest.mark.parametrize("content", ["0", "0;url=#x", "0;url=http://EXAMPLE.org/"])+ def test_refresh_self(self, content: str) -> None:+ req = Request(url="http://example.org")+ body = f'<meta http-equiv="refresh" content="{content}">'.encode()+ rsp = HtmlResponse(req.url, body=body)+ assert self.mw.process_response(req, rsp) is rsp++ def test_maxdelay_shortest(self):+ req = Request(url="http://example.org")+ body = self._body(interval=1000, url="/slow") + self._body(+ interval=1, url="/fast")- rsp = HtmlResponse(req.url, body=body.encode())- response = mw.process_response(req, rsp)- assert isinstance(response, Response)+ req2 = self.mw.process_response(req, HtmlResponse(req.url, body=body))+ assert isinstance(req2, Request)+ assert req2.url == "http://example.org/fast"+++NOSCRIPT_REFRESH = (+ b"<noscript>" + meta_refresh_body("http://example.org/newpage", 0) + b"</noscript>"+)+++def _process_noscript_refresh(mw: MetaRefreshMiddleware) -> Request | Response:+ req = Request(url="http://example.org")+ return mw.process_response(req, HtmlResponse(req.url, body=NOSCRIPT_REFRESH))+++ ("ignore_tags", "scripting"),+ [+ (["script", "noscript"], True),+ (["noscript", "template"], True),+ (["script"], False),+ ([], False),+ ],+)+def test_ignore_tags(ignore_tags: list[str], scripting: bool) -> None:+ crawler = get_crawler(Spider, {"METAREFRESH_IGNORE_TAGS": ignore_tags})+ with pytest.warns(+ ScrapyDeprecationWarning, match=f"METAREFRESH_SCRIPTING = {scripting}"+ ):+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)+ assert isinstance(_process_noscript_refresh(mw), Request) is not scripting+++class _RemoveNoscriptAddon:+ def update_settings(self, settings: BaseSettings) -> None:+ settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript")+++def test_ignore_tags_removed_from_list() -> None:+ crawler = get_crawler(Spider, {"ADDONS": {_RemoveNoscriptAddon: 0}})+ with pytest.warns(ScrapyDeprecationWarning, match="METAREFRESH_SCRIPTING = False"):+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)+ assert isinstance(_process_noscript_refresh(mw), Request)+++def test_ignore_tags_unsupported() -> None:+ crawler = get_crawler(Spider, {"METAREFRESH_IGNORE_TAGS": ["noscript", "div"]})+ with pytest.raises(ValueError, match="METAREFRESH_SCRIPTING"):+ build_from_crawler(MetaRefreshMiddleware, crawler)+++class _IgnoreNoTagsAddon:+ def update_settings(self, settings: BaseSettings) -> None:+ settings.set("METAREFRESH_IGNORE_TAGS", [], priority="addon")+++def test_ignore_tags_overridden() -> None:+ crawler = get_crawler(+ Spider,+ {"ADDONS": {_IgnoreNoTagsAddon: 0}, "METAREFRESH_SCRIPTING": True},+ )+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)+ assert isinstance(_process_noscript_refresh(mw), Response)@pytest.mark.parametrize(diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.pyindex ceee8d865..bad26bfcb 100644--- a/tests/test_http2_client_protocol.py+++ b/tests/test_http2_client_protocol.py@@ -785,6 +785,22 @@ class TestHttps2ClientProtocol:assert client.transport is not Noneassert client.transport.disconnecting+ @pytest.mark.parametrize(("method", "path"), [("GET", b"/"), ("OPTIONS", b"*")])+ @deferred_f_from_coro_f+ async def test_empty_path(+ self, method: str, path: bytes, server_port: int, client: H2ClientProtocol+ ) -> None:+ request = Request(+ f"{self.scheme}://{self.host}:{server_port}",+ method=method,+ meta={"verbatim_url": True},+ )+ with mock.patch.object(+ client.conn, "send_headers", wraps=client.conn.send_headers+ ) as send_headers:+ await make_request(client, request)+ assert (b":path", path) in send_headers.call_args.args[1]+@deferred_f_from_coro_fasync def test_request_headers_received(self, server_port: int, client: H2ClientProtocoldiff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex ec6a4896f..fd0136642 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -6,9 +6,11 @@ import weakreffrom typing import castfrom unittest import mock+import frostworkimport pytest-from scrapy.http import HtmlResponse, TextResponse, XmlResponse+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.http import HtmlResponse, MetaRefresh, TextResponse, XmlResponsefrom scrapy.selector import Selectorfrom tests.utils.bases.http_response import TestResponseBase@@ -32,21 +34,21 @@ class TestTextResponse(TestResponseBase):def test_replace(self) -> None:super().test_replace()r1 = self.response_class(- "http://www.example.com", body="hello", encoding="cp852"+ "http://www.example.com", body="hello", encoding="cp866")r2 = r1.replace(url="http://www.example.com/other")r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")assert isinstance(r2, self.response_class)assert r2.url == "http://www.example.com/other"- self._assert_response_encoding(r2, "cp852")+ assert r2.encoding == "cp866"assert r3.url == "http://www.example.com/other"assert r3._declared_encoding() == "latin1"def test_unicode_url(self):# instantiate with unicode url without encoding (should set default encoding)resp = self.response_class("http://www.example.com/")- self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)+ self._assert_response_encoding(resp, "utf-8")# make sure urls are converted to strresp = self.response_class(url="http://www.example.com/", encoding="utf-8")@@ -167,8 +169,8 @@ class TestTextResponse(TestResponseBase):self._assert_response_values(r1, "utf-8", "\xa3")self._assert_response_values(r2, "utf-8", "\xa3")self._assert_response_values(r3, "iso-8859-1", "\xa3")- self._assert_response_values(r6, "gb18030", "\u2015")- self._assert_response_values(r7, "gb18030", "\u2015")+ self._assert_response_values(r6, "gbk", "\u2015")+ self._assert_response_values(r7, "gbk", "\u2015")# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodieswith pytest.raises(TypeError):@@ -184,11 +186,17 @@ class TestTextResponse(TestResponseBase):assert r._declared_encoding() is Noneself._assert_response_values(r, "utf-8", "\xa3")- def test_body_inferred_encoding_of_undeclared_legacy_page(self):+ @pytest.mark.parametrize(+ ("url", "encoding"),+ [("http://www.example.com", "cp932"), ("http://www.example.cn", "gbk")],+ )+ def test_body_inferred_encoding_of_undeclared_legacy_page(+ self, url: str, encoding: str+ ) -> None:body = "ICD10 国際疾病分類第10版2013年版 病名マスター".encode("shift-jis")- r = self.response_class("http://www.example.com", body=body)+ r = self.response_class(url, body=body)assert r._declared_encoding() is None- assert r.encoding in {"shift_jis", "cp932"}+ assert r.encoding == encodingassert r.text == body.decode(r.encoding)def test_utf16(self):@@ -199,7 +207,36 @@ class TestTextResponse(TestResponseBase):body=body,encoding="utf-16",)- self._assert_response_values(r, "utf-16", body)+ assert r.encoding == "utf-16"+ assert r.text == "hi"++ @pytest.mark.parametrize(+ ("encoding", "body", "text"),+ [+ ("cp437", "<p>é", "<p>é"),+ ("hz", b"~{Dc:C~}", "你好"),+ ("foo", b"\xc2\xa3", "£"),+ ("cp437", codecs.BOM_UTF8 + b"\xc2\xa3", "£"),+ ],+ )+ def test_non_browser_encoding(+ self, encoding: str, body: str | bytes, text: str+ ) -> None:+ with pytest.warns(ScrapyDeprecationWarning, match=repr(encoding)):+ r = self.response_class(+ "http://www.example.com", body=body, encoding=encoding+ )+ assert r.encoding == encoding+ assert r.text == text++ def test_replace_replacement_encoding(self) -> None:+ r = self.response_class(+ "http://www.example.com",+ headers={"Content-Type": ["text/html; charset=hz"]},+ body=b"~{Dc:C~}",+ )+ assert r.encoding == "replacement"+ assert r.replace(url="http://www.example.com/other").text == "�"def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):r6 = self.response_class(@@ -318,7 +355,6 @@ class TestTextResponse(TestResponseBase):)def test_urljoin_with_base_url(self):- """Test urljoin shortcut which also evaluates base-url through get_base_url()."""body = b'<html><body><base href="https://example.net"></body></html>'joined = self.response_class("http://www.example.com", body=body).urljoin("/test"@@ -352,6 +388,84 @@ class TestTextResponse(TestResponseBase):assert response.urljoin("/test") == "https://example.net/test"assert response.follow_redirect().url == "http://www.example.com/test"+ def test_base_url_serialized(self) -> None:+ response = self.response_class("http://www.EXAMPLE.com", body=b"")+ assert response.base_url == "http://www.example.com/"++ @pytest.mark.parametrize(+ ("body", "base_url"),+ [+ (b"", "http://www.example.com/a/b"),+ (b'<base href="c/">', "http://www.example.com/a/c/"),+ (+ b'<base target="_blank"><base href="/c"><base href="/d">',+ "http://www.example.com/c",+ ),+ ],+ )+ def test_base_url(self, body: bytes, base_url: str) -> None:+ response = self.response_class("http://www.example.com/a/b", body=body)+ assert response.base_url == base_url++ @pytest.mark.parametrize(+ ("body", "meta_refresh"),+ [+ (b"", None),+ (+ b'<base href="/c/"><meta http-equiv="refresh" content="5;url=d">',+ (5.0, "http://www.example.com/c/d"),+ ),+ (+ (+ b'<meta http-equiv="refresh" content="2;url=/a">'+ b'<meta http-equiv="refresh" content="1;url=/b">'+ b'<meta http-equiv="refresh" content="1;url=/c">'+ b'<meta http-equiv="refresh" content="3;url=/d">'+ ),+ (1.0, "http://www.example.com/c"),+ ),+ (+ (+ b'<noscript><meta http-equiv="refresh" content="0;url=/a">'+ b"</noscript><template>"+ b'<meta http-equiv="refresh" content="0;url=/b"></template>'+ b'<script>document.write(\'<meta http-equiv="refresh" '+ b'content="0;url=/c">\')</script>'+ ),+ None,+ ),+ ],+ )+ def test_meta_refresh(+ self, body: bytes, meta_refresh: tuple[float, str] | None+ ) -> None:+ response = self.response_class("http://www.example.com", body=body)+ assert response.meta_refresh == meta_refresh++ def test_html_metadata_once(self) -> None:+ body = b'<base href="/a/"><meta http-equiv="refresh" content="0;url=b">'+ response = self.response_class("http://www.example.com", body=body)+ with mock.patch("frostwork.navigation", wraps=frostwork.navigation) as read:+ assert response.base_url == "http://www.example.com/a/"+ assert response.meta_refresh == (0.0, "http://www.example.com/a/b")+ assert isinstance(response.meta_refresh, MetaRefresh)+ read.assert_called_once()++ def test_html_metadata_set(self) -> None:+ body = b'<base href="/a/"><meta http-equiv="refresh" content="0;url=b">'+ response = self.response_class("http://www.example.com", body=body)+ response.base_url = "http://example.net/"+ assert response.meta_refresh == (0.0, "http://www.example.com/a/b")+ assert response.base_url == "http://example.net/"+ response.meta_refresh = None+ assert response.meta_refresh is None++ response = self.response_class("http://www.example.com", body=body)+ response.meta_refresh = (1.0, "http://example.net/")+ assert response.base_url == "http://www.example.com/a/"+ assert response.meta_refresh == (1.0, "http://example.net/")+ assert isinstance(response.meta_refresh, MetaRefresh)+def test_follow_selector(self):resp = self._links_response()urls = [@@ -596,7 +710,7 @@ class TestXmlResponse(TestTextResponse):def test_xml_encoding(self):body = b"<xml></xml>"r1 = self.response_class("http://www.example.com", body=body)- self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)+ self._assert_response_values(r1, "utf-8", body)body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""r2 = self.response_class("http://www.example.com", body=body)diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 8da842da5..c6fc0328c 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -38,7 +38,7 @@ class TestLogFormatter:res = Response("http://www.example.com")logkws = self.formatter.crawled(req, res, self.spider)logline = logkws["msg"] % logkws["args"]- assert logline == "Crawled (200) <GET http://www.example.com> (referer: None)"+ assert logline == "Crawled (200) <GET http://www.example.com/> (referer: None)"def test_crawled_with_referer(self):req = Request(@@ -49,7 +49,7 @@ class TestLogFormatter:logline = logkws["msg"] % logkws["args"]assert (logline- == "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']"+ == "Crawled (200) <GET http://www.example.com/> (referer: http://example.com) ['cached']")def test_flags_in_request(self):@@ -59,7 +59,7 @@ class TestLogFormatter:logline = logkws["msg"] % logkws["args"]assert (logline- == "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)"+ == "Crawled (200) <GET http://www.example.com/> ['test', 'flag'] (referer: None)")def test_dropped(self):@@ -149,7 +149,7 @@ class TestLogFormatter:logline = logkws["msg"] % logkws["args"]assert (logline- == "Spider error processing <GET http://www.example.com> (referer: http://example.org)"+ == "Spider error processing <GET http://www.example.com/> (referer: http://example.org)")def test_download_error_short(self):@@ -159,7 +159,7 @@ class TestLogFormatter:request = Request("http://www.example.com")logkws = self.formatter.download_error(failure, request, self.spider)logline = logkws["msg"] % logkws["args"]- assert logline == "Error downloading <GET http://www.example.com>"+ assert logline == "Error downloading <GET http://www.example.com/>"def test_download_error_long(self):# In practice, the complete traceback is shown by passing the@@ -170,7 +170,9 @@ class TestLogFormatter:failure, request, self.spider, "Some message")logline = logkws["msg"] % logkws["args"]- assert logline == "Error downloading <GET http://www.example.com>: Some message"+ assert (+ logline == "Error downloading <GET http://www.example.com/>: Some message"+ )def test_scraped(self):item = CustomItem()@@ -208,7 +210,7 @@ class TestLogformatterSubclass(TestLogFormatter):logkws = self.formatter.crawled(req, res, self.spider)logline = logkws["msg"] % logkws["args"]assert (- logline == "Crawled (200) <GET http://www.example.com> (referer: None) []"+ logline == "Crawled (200) <GET http://www.example.com/> (referer: None) []")def test_crawled_with_referer(self):@@ -222,7 +224,7 @@ class TestLogformatterSubclass(TestLogFormatter):logline = logkws["msg"] % logkws["args"]assert (logline- == "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']"+ == "Crawled (200) <GET http://www.example.com/> (referer: http://example.com) ['cached']")def test_flags_in_request(self):@@ -232,7 +234,7 @@ class TestLogformatterSubclass(TestLogFormatter):logline = logkws["msg"] % logkws["args"]assert (logline- == "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']"+ == "Crawled (200) <GET http://www.example.com/> (referer: None) ['test', 'flag']")diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex be8aa1f3d..3884d4730 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -125,7 +125,7 @@ class TestImagesPipeline:)assert (thumb_path(Request("file://foo.png"), name)- == "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg"+ == "thumbs/50/3f7f26a0132d8c3df6b56f0691b89083df00d62a.jpg")assert (thumb_path(Request("file:///tmp/foo"), name)diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex 85e300e40..be46363de 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -14,7 +14,7 @@ from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from tests.mockserver.http import MockServer-OVERRIDDEN_URL = "https://example.org"+OVERRIDDEN_URL = "https://example.org/"class ProcessResponseMiddleware:diff --git a/tests/test_selector.py b/tests/test_selector.pyindex 443331c69..fbccdeacc 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -38,7 +38,7 @@ class TestSelector:def test_root_base_url(self) -> None:body = b'<html><form action="/path"><input name="a" /></form></html>'- url = "http://example.com"+ url = "http://example.com/"response = TextResponse(url=url, body=body, encoding="utf-8")sel = Selector(response)assert url == sel.root.base@@ -124,6 +124,17 @@ class TestSelector:response = HtmlResponse("https://example.com", body=body, headers=headers)assert response.css("p::text").get() == "£€"+ @pytest.mark.parametrize(+ "body",+ [+ b"<meta charset=euc-jp><p>\xa1\xc1 \xad\xa1</p>",+ b"<p>\xa1\xc1 \xad\xa1" + "日本語です。".encode("euc_jp") * 50 + b"</p>",+ ],+ )+ def test_browser_decoding(self, body: bytes) -> None:+ response = HtmlResponse("https://example.com", body=body)+ assert response.css("p::text").get("").startswith("~ ①")+def test_weakref_slots(self) -> None:"""Check that classes are using slots and are weak-referenceable"""x = Selector(text="")diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.pyindex af500aad9..0ac75f754 100644--- a/tests/test_squeues_request.py+++ b/tests/test_squeues_request.py@@ -239,7 +239,7 @@ def test_sqlite_queue_survives_unclean_shutdown() -> None:key = str(tmp_path / "queue")queue = queue_cls.from_crawler(crawler=crawler, key=key)- queue.push(Request("https://toscrape.com"))+ queue.push(Request("https://toscrape.com/"))# No close(), as if the process had been killed. The connection is closed# directly instead, since on Windows the file cannot be removed later while# it is still open.@@ -250,7 +250,7 @@ def test_sqlite_queue_survives_unclean_shutdown(assert len(queue) == 1request = queue.pop()assert request is not None- assert request.url == "https://toscrape.com"+ assert request.url == "https://toscrape.com/"finally:queue.close()diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex eefb562bd..6c15cd596 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -474,10 +474,10 @@ class TestUtilsCsv:{"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"},]- response = TextResponse(url="http://example.com/", body=body2, encoding="cp852")+ response = TextResponse(url="http://example.com/", body=body2, encoding="cp866")csv = csviter(response)assert list(csv) == [- {"id": "1", "name": "cp852", "value": "test"},+ {"id": "1", "name": "cp866", "value": "test"},{"id": "2","name": "something",diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 8fc6e16be..7b6dbdb50 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -423,7 +423,7 @@ class TestRequestToCurl:def test_get(self) -> None:request_object = Request("https://www.example.com")- expected_curl_command = "curl -X GET https://www.example.com"+ expected_curl_command = "curl -X GET https://www.example.com/"self._test_request(request_object, expected_curl_command)def test_post(self) -> None:@@ -464,7 +464,7 @@ class TestRequestToCurl:)def test_header_values(self, headers: dict[str, Any], expected_args: str) -> None:request_object = Request("https://www.example.com", headers=headers)- expected_curl_command = f"curl -X GET https://www.example.com {expected_args}"+ expected_curl_command = f"curl -X GET https://www.example.com/ {expected_args}"self._test_request(request_object, expected_curl_command)def test_headers_from_curl(self) -> None:diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 915a28802..74eec5c58 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -6,6 +6,7 @@ from urllib.parse import urlparseimport pytest+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Response, TextResponsefrom scrapy.utils.response import (get_base_url,@@ -41,72 +42,27 @@ def test_open_in_browser():open_in_browser(resp, _openfunc=browser_open)-def test_get_meta_refresh():- r1 = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>- <body>blahablsdfsal&</body>- </html>""",- )- r2 = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><title>Dummy</title><noScript>- <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>- </noSCRIPT>- <body>blahablsdfsal&</body>- </html>""",- )- r3 = HtmlResponse(- "http://www.example.com",- body=b"""-<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>-<script type="text/javascript">-if(!checkCookies()){- document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');-}-</script>- """,- )- r4 = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><title>Dummy</title>- <base href="http://www.another-domain.com/base/path/">- <meta http-equiv="refresh" content="5;url=target.html"</head>- <body>blahablsdfsal&</body>- </html>""",- )- assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")- assert get_meta_refresh(r2) == (None, None)- assert get_meta_refresh(r3) == (None, None)- assert get_meta_refresh(r4) == (- 5.0,- "http://www.another-domain.com/base/path/target.html",- )-+def test_get_meta_refresh() -> None:+ body = b'<noscript><meta http-equiv="refresh" content="5;url=/a"></noscript>'+ response = HtmlResponse("http://www.example.com", body=body)+ with pytest.warns(ScrapyDeprecationWarning, match="TextResponse.meta_refresh"):+ assert get_meta_refresh(response) == (None, None)+ with pytest.warns(ScrapyDeprecationWarning):+ assert get_meta_refresh(response, ignore_tags=["script"]) == (+ 5.0,+ "http://www.example.com/a",+ )+ with (+ pytest.warns(ScrapyDeprecationWarning),+ pytest.raises(ValueError, match="div"),+ ):+ get_meta_refresh(response, ignore_tags=["div"])-def test_get_base_url():- resp = HtmlResponse(- "http://www.example.com",- body=b"""- <html>- <head><base href="http://www.example.com/img/" target="_blank"></head>- <body>blahablsdfsal&</body>- </html>""",- )- assert get_base_url(resp) == "http://www.example.com/img/"- resp2 = HtmlResponse(- "http://www.example.com",- body=b"""- <html><body>blahablsdfsal&</body></html>""",- )- assert get_base_url(resp2) == "http://www.example.com"+def test_get_base_url() -> None:+ response = HtmlResponse("http://www.example.com", body=b'<base href="/a/">')+ with pytest.warns(ScrapyDeprecationWarning, match="TextResponse.base_url"):+ assert get_base_url(response) == "http://www.example.com/a/"def test_response_status_message():@@ -195,7 +151,7 @@ def test_response_status_message():],)def test_inject_base_url(body: bytes) -> None:- url = "http://www.example.com"+ url = "http://www.example.com/"def check_base_url(burl):bbody = _read_browser_output(burl)@@ -247,7 +203,7 @@ def test_open_in_browser_redos_head():def test_open_in_browser_preserves_html_comments():- url = "http://www.example.com"+ url = "http://www.example.com/"body = (b"<html>"b"<!-- preserved comment -->"@@ -290,7 +246,7 @@ def test_open_in_browser_keeps_base_url_of_response(def test_open_in_browser_injects_base_when_only_in_comment():- url = "http://www.example.com"+ url = "http://www.example.com/"body = (b"<html>"b"<!-- <base href='http://other.com'> -->"@@ -309,7 +265,7 @@ def test_open_in_browser_injects_base_when_only_in_comment():def test_open_in_browser_injects_base_before_head_contents():- url = "http://www.example.com"+ url = "http://www.example.com/"body = (b"<html>"b"<!--<head>comment head</head>-->"diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex 2e1988bfb..de609312b 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -1868,7 +1868,6 @@ class TestHttpProxyBase(ABC):is_secure = False# whether the handler supports HTTPS proxies with HTTPS destinationshandler_supports_tls_in_tls: bool = True- expected_http_proxy_request_body = b"http://example.com"@property@abstractmethod@@ -1901,7 +1900,7 @@ class TestHttpProxyBase(ABC):response = await download_handler.download_request(request)assert response.status == 200assert response.url == request.url- assert response.body == self.expected_http_proxy_request_body+ assert response.body == b"http://example.com/"@coroutine_testasync def test_download_without_proxy(@@ -1962,7 +1961,7 @@ class TestHttpProxyBase(ABC):response = await download_handler.download_request(request)assert response.status == 200assert response.url == request.url- assert response.body == self.expected_http_proxy_request_body+ assert response.body == b"http://example.com/"PROXY_KINDS = ["http", "https", "socks5"]diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.pyindex 541a7ca14..ddbbeac2c 100644--- a/tests/utils/bases/http_request.py+++ b/tests/utils/bases/http_request.py@@ -36,7 +36,7 @@ class TestRequestBase(ABC):r = self.request_class("http://www.example.com")assert isinstance(r.url, str)- assert r.url == "http://www.example.com"+ assert r.url == "http://www.example.com/"assert r.method == self.default_methodassert isinstance(r.headers, Headers)diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.pyindex 9df62a186..2271e8ca2 100644--- a/tests/utils/bases/http_response.py+++ b/tests/utils/bases/http_response.py@@ -246,7 +246,7 @@ class TestResponseBase(ABC):# Response.followdef test_follow_url_absolute(self):- self._assert_followed_url("http://foo.example.com", "http://foo.example.com")+ self._assert_followed_url("http://foo.example.com/", "http://foo.example.com/")def test_follow_url_relative(self):self._assert_followed_url("foo", "http://example.com/foo")@@ -333,10 +333,10 @@ class TestResponseBase(ABC):def test_follow_all_absolute(self):url_list = [- "http://example.org",- "http://www.example.org",- "http://example.com",- "http://www.example.com",+ "http://example.org/",+ "http://www.example.org/",+ "http://example.com/",+ "http://www.example.com/",]self._assert_followed_all_urls(url_list, url_list)diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.pyindex 209bc30f3..4a40a18b5 100644--- a/tests/utils/bases/redirect.py+++ b/tests/utils/bases/redirect.py@@ -289,18 +289,18 @@ class TestRedirectBase(ABC):[pytest.param("http://example.com",- "http://example.com",- "http://example.com",+ "http://example.com/a",+ "http://example.com/b",id="http-absolute",),- pytest.param("http://example.com", "/a", "/a", id="http-relative"),+ pytest.param("http://example.com", "/a", "/b", id="http-relative"),pytest.param("https://example.com",- "https://example.com",- "https://example.com",+ "https://example.com/a",+ "https://example.com/b",id="https-absolute",),- pytest.param("https://example.com", "/a", "/a", id="https-relative"),+ pytest.param("https://example.com", "/a", "/b", id="https-relative"),pytest.param("http://example.com","https://example.com",@@ -335,8 +335,8 @@ class TestRedirectBase(ABC):pytest.param({"http_proxy": PROXY_A},"http://example.com",- "http://example.com",- "http://example.com",+ "http://example.com/a",+ "http://example.com/b",("a", "a", "a", "a", "a"),id="http-absolute",),@@ -344,15 +344,15 @@ class TestRedirectBase(ABC):{"http_proxy": PROXY_A},"http://example.com","/a",- "/a",+ "/b",("a", "a", "a", "a", "a"),id="http-relative",),pytest.param({"https_proxy": PROXY_A},"https://example.com",- "https://example.com",- "https://example.com",+ "https://example.com/a",+ "https://example.com/b",("a", "a", "a", "a", "a"),id="https-absolute",),@@ -360,7 +360,7 @@ class TestRedirectBase(ABC):{"https_proxy": PROXY_A},"https://example.com","/a",- "/a",+ "/b",("a", "a", "a", "a", "a"),id="https-relative",),diff --git a/tox.ini b/tox.iniindex 2cee007ea..5957752f3 100644--- a/tox.ini+++ b/tox.ini@@ -79,6 +79,7 @@ deps =boto3-stubs[s3]==1.43.107botocore-stubs==1.43.67colorlog==6.12.0+ frostwork==0.2.2h2==4.4.1httpx2==2.13.1itemadapter==0.13.1@@ -94,7 +95,7 @@ deps =types-lxml==2026.2.16types-pexpect==4.9.0.20260518uvloop==0.23.0- w3lib==2.5.0+ w3lib==2.6.0commands =mypy {posargs:scrapy tests}@@ -146,10 +147,10 @@ deps =aiohttp==3.13.3backports.zstd==1.3.0brotli==1.2.0- charset-normalizer==3.4.0cryptography==41.0.5cssselect==1.2.0formerly==0.1.0+ frostwork==0.2.2itemadapter==0.1.0itemloaders==1.0.5lxml==4.9.4@@ -160,8 +161,8 @@ deps =pyOpenSSL==24.3.0queuelib==1.8.0service_identity==24.2.0- typing-extensions==4.6.0- w3lib==2.1.1+ typing-extensions==4.13.0+ w3lib==2.6.0zope.interface==5.1.0{[test-requirements]deps}setenv =