Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Frostwork integration (#8393)

Adrianae4c0cee1726
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 788de9d1f..63e536214 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -125,7 +125,7 @@ Set some basic configuration:
class MyAddon:
def update_settings(self, settings):
settings.set("DNSCACHE_ENABLED", True, "addon")
- settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript")
+ settings.remove_from_list("RETRY_HTTP_CODES", 429)
settings.setdefault_in_component_priority_dict(
"ITEM_PIPELINES", MyPipeline, 200
)
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0ddf2bd70..a85b4e455 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1079,8 +1079,8 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
settings (see the settings documentation for more info):
* :setting:`METAREFRESH_ENABLED`
-* :setting:`METAREFRESH_IGNORE_TAGS`
* :setting:`METAREFRESH_MAXDELAY`
+* :setting:`METAREFRESH_SCRIPTING`
This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
@@ -1099,19 +1099,6 @@ Default: ``True``
Whether the Meta Refresh middleware will be enabled.
-.. setting:: METAREFRESH_IGNORE_TAGS
-
-METAREFRESH_IGNORE_TAGS
-^^^^^^^^^^^^^^^^^^^^^^^
-
-Default: ``["noscript"]``
-
-Meta tags within these tags are ignored.
-
-.. versionchanged:: 2.11.2
- The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
- ``[]`` to ``["noscript"]``.
-
.. setting:: METAREFRESH_MAXDELAY
METAREFRESH_MAXDELAY
@@ -1123,6 +1110,19 @@ The maximum meta-refresh delay (in seconds) to follow the redirection.
Some sites use meta-refresh for redirecting to a session expired page, so we
restrict automatic redirection to the maximum delay.
+METAREFRESH_SCRIPTING
+^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: VERSION
+
+.. setting:: METAREFRESH_SCRIPTING
+
+Default: ``True``
+
+Whether to follow the meta refresh that a web browser would follow with
+JavaScript enabled (``True``), ignoring those within ``<noscript>``, or with
+JavaScript disabled (``False``).
+
RetryMiddleware
---------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b954e2a73..58a0d8a39 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1454,10 +1454,12 @@ TextResponse objects
is the same as for the :class:`Response` class and is not documented here.
:param encoding: is a string which contains the encoding to use for this
- response. If you create a :class:`TextResponse` object with a string as
- body, it will be converted to bytes encoded using this encoding. If
- *encoding* is ``None`` (default), the encoding will be looked up in the
- response headers and body instead.
+ response, as a label of the `Encoding Standard`_, which web browsers
+ implement, or as the name of the Python codec of one of its encodings,
+ e.g. ``cp932`` for Shift_JIS. If you create a :class:`TextResponse`
+ object with a string as body, it will be converted to bytes encoded
+ using this encoding. If *encoding* is ``None`` (default), the encoding
+ will be looked up in the response headers and body instead.
:type encoding: str
:class:`TextResponse` objects support the following attributes in addition
@@ -1467,8 +1469,7 @@ TextResponse objects
Response body, as a string.
- The same as ``response.body.decode(response.encoding)``, but the
- result is cached after the first call, so you can access
+ The result is cached after the first call, so you can access
``response.text`` multiple times without extra overhead.
.. note::
@@ -1540,6 +1541,10 @@ TextResponse objects
A :class:`~scrapy.Selector` instance using the response as
target. The selector is lazily instantiated on first access.
+ .. autoattribute:: TextResponse.base_url
+
+ .. autoattribute:: TextResponse.meta_refresh
+
.. autoattribute:: TextResponse.attributes
:class:`TextResponse` objects support the following methods in addition to
@@ -1581,12 +1586,14 @@ TextResponse objects
.. method:: TextResponse.urljoin(url)
- Constructs an absolute url by combining the Response's base url with
- a possible relative url. The base url shall be extracted from the
- ``<base>`` tag, or just :attr:`Response.url` if there is no such
- tag.
+ Constructs an absolute url by combining :attr:`base_url` with a
+ possible relative url.
+
+.. autoclass:: MetaRefresh
+ :members:
.. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
+.. _Encoding Standard: https://encoding.spec.whatwg.org/
.. _encoding sniffing algorithm: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
diff --git a/pyproject.toml b/pyproject.toml
index 9e61bbf32..f66c54992 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,11 +14,11 @@ dependencies = [
"aiohttp>=3.13.3",
'backports.zstd>=1.3.0; python_version < "3.14"',
"brotli>=1.2.0",
- "charset-normalizer>=3.4.0",
"cryptography>=41.0.5",
"cssselect>=1.2.0",
"defusedxml>=0.7.1",
"formerly>=0.1.0",
+ "frostwork>=0.2.2",
"itemadapter>=0.1.0",
"itemloaders>=1.0.5",
"lxml>=4.9.4",
@@ -30,8 +30,8 @@ dependencies = [
"queuelib>=1.8.0",
"service_identity>=24.2.0",
"tldextract",
- 'typing-extensions>=4.6.0; python_version < "3.13"',
- "w3lib>=2.1.1",
+ 'typing-extensions>=4.13.0; python_version < "3.13"',
+ "w3lib>=2.6.0",
"zope.interface>=5.1.0",
]
classifiers = [
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 80fe2da39..271b4de4e 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,16 +1,19 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING, Any, Self, cast
+import warnings
+from typing import TYPE_CHECKING, Any, Self
+
+from w3lib.url import canonicalize_url
from scrapy import signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response
from scrapy.spidermiddlewares.referer import RefererMiddleware
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import global_object_name
-from scrapy.utils.response import _body_size, get_meta_refresh
+from scrapy.utils.response import _IGNORABLE_TAGS, _body_size, _get_meta_refresh
from scrapy.utils.url import _redirect_url
if TYPE_CHECKING:
@@ -264,6 +267,33 @@ class RedirectMiddleware(BaseRedirectMiddleware):
return self._redirect(redirected, request, response.status)
+def _get_scripting(settings: BaseSettings) -> bool:
+ ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
+ ignore_tags_priority = settings.getpriority("METAREFRESH_IGNORE_TAGS") or 0
+ scripting_priority = settings.getpriority("METAREFRESH_SCRIPTING") or 0
+ # remove_from_list() keeps the default priority, hence the value check.
+ if (not ignore_tags_priority and ignore_tags == ["noscript"]) or (
+ scripting_priority and scripting_priority >= ignore_tags_priority
+ ):
+ return settings.getbool("METAREFRESH_SCRIPTING")
+ if not set(ignore_tags) <= _IGNORABLE_TAGS:
+ raise ValueError(
+ f"METAREFRESH_IGNORE_TAGS is set to {ignore_tags!r}, but only "
+ f"{sorted(_IGNORABLE_TAGS)!r} are supported. "
+ f"METAREFRESH_IGNORE_TAGS is deprecated, we strongly recommend "
+ f"removing it and using METAREFRESH_SCRIPTING instead. As a last "
+ f"resort, subclass MetaRefreshMiddleware."
+ )
+ scripting = "noscript" in ignore_tags
+ warnings.warn(
+ f"The METAREFRESH_IGNORE_TAGS setting is deprecated, replace it "
+ f"with METAREFRESH_SCRIPTING = {scripting}.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return scripting
+
+
class MetaRefreshMiddleware(BaseRedirectMiddleware):
"""Handle redirection of requests based on meta-refresh html tag."""
@@ -271,7 +301,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
def __init__(self, settings: BaseSettings):
super().__init__(settings)
- self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
+ self._scripting: bool = _get_scripting(settings)
self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")
@_warn_spider_arg
@@ -286,12 +316,17 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
):
return response
- interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags)
- if not url:
+ meta_refresh = _get_meta_refresh(response, scripting=self._scripting)
+ if not meta_refresh:
+ return response
+ interval, url = meta_refresh
+ # A refresh without a URL reloads the page, which the dupefilter would
+ # drop.
+ if canonicalize_url(url) == canonicalize_url(response.url):
return response
redirected = self._redirect_request_using_get(request, response, url)
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
- if cast("float", interval) < self._maxdelay:
+ if interval < self._maxdelay:
return self._redirect(redirected, request, "meta refresh")
return response
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index ae6486c4e..fba327afe 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -14,7 +14,7 @@ from scrapy.http.response import Response
from scrapy.http.response.body_file import BodyFileResponse
from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.json import JsonResponse
-from scrapy.http.response.text import TextResponse
+from scrapy.http.response.text import MetaRefresh, TextResponse
from scrapy.http.response.xml import XmlResponse
__all__ = [
@@ -24,6 +24,7 @@ __all__ = [
"HtmlResponse",
"JsonRequest",
"JsonResponse",
+ "MetaRefresh",
"Request",
"Response",
"TextResponse",
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6ba03229a..60c4c52be 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,15 +8,18 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
import json
+import warnings
import weakref
from contextlib import suppress
-from typing import TYPE_CHECKING, Any, cast
+from enum import Enum, auto
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast
from urllib.parse import urljoin
+import frostwork
import parsel
-from charset_normalizer import from_bytes
from w3lib.encoding import (
html_body_declared_encoding,
+ html_encoding,
html_to_unicode,
http_content_type_encoding,
read_bom,
@@ -24,9 +27,9 @@ from w3lib.encoding import (
)
from w3lib.html import strip_html5_whitespace
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.response import Response
from scrapy.utils.python import memoizemethod_noargs
-from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Mapping
@@ -38,27 +41,66 @@ if TYPE_CHECKING:
from scrapy.selector import Selector, SelectorList
-_NONE = object()
+class _Unset(Enum):
+ UNSET = auto()
-class TextResponse(Response):
- _DEFAULT_ENCODING = "ascii"
+_UNSET = _Unset.UNSET
+
+
+class MetaRefresh(NamedTuple):
+ """Meta refresh of a :class:`TextResponse`, as
+ :attr:`TextResponse.meta_refresh` reports it.
+
+ .. versionadded:: VERSION
+ """
+
+ delay: float
+ """Seconds to wait before the refresh."""
+
+ url: str
+ """Absolute URL to load."""
+
+def _is_browser_encoding(encoding: str) -> bool:
+ resolved = resolve_encoding(encoding)
+ # "replacement" is what response.encoding reports for labels that browsers
+ # decode as a single U+FFFD, e.g. a Content-Type charset of hz.
+ return resolved is not None and (
+ resolved != "replacement" or encoding == "replacement"
+ )
+
+
+class TextResponse(Response):
attributes: tuple[str, ...] = (*Response.attributes, "encoding")
__slots__ = (
+ "_base_url",
"_cached_benc",
"_cached_decoded_json",
"_cached_selector",
"_cached_ubody",
"_encoding",
+ "_meta_refresh",
)
def __init__(self, *args: Any, **kwargs: Any):
self._encoding: str | None = kwargs.pop("encoding", None)
+ if self._encoding is not None and not _is_browser_encoding(self._encoding):
+ warnings.warn(
+ f"Passing encoding={self._encoding!r} to {type(self).__name__} "
+ f"is deprecated, because web browsers do not support that "
+ f"encoding, and it will raise ValueError in a future version "
+ f"of Scrapy. To decode the response body with it, use "
+ f"response.body.decode({self._encoding!r}) instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
self._cached_benc: str | None = None
self._cached_ubody: str | None = None
self._cached_selector: Selector | None = None
- self._cached_decoded_json: object = _NONE
+ self._cached_decoded_json: object = _UNSET
+ self._base_url: str | None = None
+ self._meta_refresh: MetaRefresh | Literal[_Unset.UNSET] | None = _UNSET
super().__init__(*args, **kwargs)
def _set_body(self, body: str | bytes | None) -> None:
@@ -103,7 +145,7 @@ class TextResponse(Response):
The result is cached after the first call.
"""
- if self._cached_decoded_json is _NONE:
+ if self._cached_decoded_json is _UNSET:
try:
self._cached_decoded_json = json.loads(self.body)
except UnicodeDecodeError:
@@ -113,18 +155,74 @@ class TextResponse(Response):
@property
def text(self) -> str:
"""Body as unicode"""
- # access self.encoding before _cached_ubody to make sure
- # _body_inferred_encoding is called
- benc = self.encoding
if self._cached_ubody is None:
- charset = f"charset={benc}"
- self._cached_ubody = html_to_unicode(charset, self.body)[1]
+ if (
+ self._encoding is not None
+ and not _is_browser_encoding(self._encoding)
+ and not self._bom_encoding()
+ ):
+ with suppress(LookupError):
+ self._cached_ubody = self.body.decode(self._encoding, "replace")
+ if self._cached_ubody is None:
+ charset = f"charset={self.encoding}"
+ self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
def urljoin(self, url: str) -> str:
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""
- return urljoin(get_base_url(self), url)
+ return urljoin(self.base_url, url)
+
+ @property
+ def base_url(self) -> str:
+ """Absolute URL that relative URLs of the response are relative to.
+
+ .. versionadded:: VERSION
+
+ It is the ``href`` of the first ``<base>`` element that has one,
+ resolved against :attr:`~Response.url`, or :attr:`~Response.url`
+ itself, serialized like a web browser does, e.g. with ``/`` as empty
+ path. Setting it overrides the value read from the body.
+ """
+ if self._base_url is None:
+ self._read_html_metadata()
+ assert self._base_url is not None
+ return self._base_url
+
+ @base_url.setter
+ def base_url(self, value: str) -> None:
+ self._base_url = value
+
+ @property
+ def meta_refresh(self) -> MetaRefresh | None:
+ """:class:`MetaRefresh` that a web browser with JavaScript enabled
+ would follow, or ``None``.
+
+ .. versionadded:: VERSION
+
+ Setting it overrides the value read from the body.
+ """
+ if self._meta_refresh is _UNSET:
+ self._read_html_metadata()
+ assert self._meta_refresh is not _UNSET
+ return self._meta_refresh
+
+ @meta_refresh.setter
+ def meta_refresh(self, value: tuple[float, str] | None) -> None:
+ self._meta_refresh = None if value is None else MetaRefresh(*value)
+
+ def _read_html_metadata(self) -> None:
+ metadata = frostwork.navigation(
+ self.body,
+ self.url,
+ self.encoding,
+ base_url=self._base_url is None,
+ meta_refresh=self._meta_refresh is _UNSET,
+ )
+ if self._base_url is None:
+ self._base_url = metadata.base_url
+ if self._meta_refresh is _UNSET:
+ self.meta_refresh = metadata.meta_refresh
@memoizemethod_noargs
def _headers_encoding(self) -> str | None:
@@ -132,28 +230,13 @@ class TextResponse(Response):
return http_content_type_encoding(content_type.decode("latin-1"))
def _body_inferred_encoding(self) -> str:
- content_type = self.headers.get(b"Content-Type") or b""
if self._cached_benc is None:
- benc, ubody = html_to_unicode(
- content_type.decode("latin-1"),
- self.body,
- auto_detect_fun=self._auto_detect_fun,
- default_encoding=self._DEFAULT_ENCODING,
+ content_type = self.headers.get(b"Content-Type") or b""
+ self._cached_benc = html_encoding(
+ content_type.decode("latin-1"), self.body, url=self.url
)
- self._cached_benc = benc
- self._cached_ubody = ubody
return self._cached_benc
- def _auto_detect_fun(self, text: bytes) -> str | None:
- for enc in (self._DEFAULT_ENCODING, "utf-8"):
- try:
- text.decode(enc)
- except UnicodeError:
- continue
- return resolve_encoding(enc)
- match = from_bytes(text).best()
- return resolve_encoding(match.encoding) if match else None
-
@memoizemethod_noargs
def _body_declared_encoding(self) -> str | None:
return html_body_declared_encoding(self.body)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index beae807e4..7bd761ff1 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -22,7 +22,6 @@ from scrapy.linkextractors import IGNORED_EXTENSIONS, SUPPORTED_SCHEMES, _matche
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list
-from scrapy.utils.response import get_base_url
from scrapy.utils.url import url_is_from_any_domain
if TYPE_CHECKING:
@@ -164,7 +163,7 @@ class LxmlParserLinkExtractor:
return self._deduplicate_if_needed(links)
def extract_links(self, response: TextResponse) -> list[Link]:
- base_url = get_base_url(response)
+ base_url = response.base_url
return self._extract_links(
response.selector, response.url, response.encoding, base_url
)
@@ -438,7 +437,7 @@ class LxmlLinkExtractor:
Duplicate links are omitted if the ``unique`` attribute is set to ``True``,
otherwise they are returned.
"""
- base_url = get_base_url(response)
+ base_url = response.base_url
if self.restrict_xpaths:
docs = [
subdoc for x in self.restrict_xpaths for subdoc in response.xpath(x)
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index d351ca84e..1caacc9e4 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -4,11 +4,11 @@ import weakref
from typing import Any, Literal
from parsel import Selector as _ParselSelector
+from w3lib.encoding import resolve_encoding
from scrapy.http import HtmlResponse, JsonResponse, TextResponse, XmlResponse
from scrapy.utils._deps_compat import PARSEL_BODY_ENCODING_FIXED
from scrapy.utils.python import to_bytes
-from scrapy.utils.response import get_base_url
from scrapy.utils.trackref import object_ref
__all__ = ["Selector", "SelectorList"]
@@ -106,9 +106,12 @@ class Selector(_ParselSelector, object_ref):
# "xml", and treats an empty body as absent, unlike empty text.
if PARSEL_BODY_ENCODING_FIXED and type in ("html", "xml") and response.body:
encoding = response.encoding
- # None means the encoding was declared, not inferred from
- # the body, so the body hasn't been decoded as a side effect.
- if response._cached_ubody is None:
+ # parsel decodes non-UTF-8 bodies with Python's codecs, which
+ # differ from the browser decoding of response.text.
+ if (
+ response._cached_ubody is None
+ and resolve_encoding(encoding) == "utf-8"
+ ):
kwargs["body"] = response.body
kwargs["encoding"] = encoding
text = None
@@ -116,7 +119,7 @@ class Selector(_ParselSelector, object_ref):
text = response.text
else:
text = response.text
- kwargs.setdefault("base_url", get_base_url(response))
+ kwargs.setdefault("base_url", response.base_url)
self._response: TextResponse | weakref.ref[TextResponse] | None = response
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ede9d09c5..0a86215e5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -181,6 +181,7 @@ __all__ = [
"METAREFRESH_ENABLED",
"METAREFRESH_IGNORE_TAGS",
"METAREFRESH_MAXDELAY",
+ "METAREFRESH_SCRIPTING",
"NEWSPIDER_MODULE",
"PERIODIC_LOG_DELTA",
"PERIODIC_LOG_STATS",
@@ -546,6 +547,7 @@ MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
METAREFRESH_IGNORE_TAGS = ["noscript"]
METAREFRESH_MAXDELAY = 100
+METAREFRESH_SCRIPTING = True
NEWSPIDER_MODULE = ""
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index fa8911aeb..d62d4fcd1 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -9,14 +9,15 @@ import mimetypes
import os
import re
import tempfile
+import warnings
import webbrowser
from pathlib import Path
from typing import TYPE_CHECKING, Any
-from weakref import WeakKeyDictionary
+import frostwork
from twisted.web import http
-from w3lib import html
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.headers import Headers
from scrapy.http.response.body_file import BodyFileResponse
from scrapy.utils.misc import load_object
@@ -33,35 +34,50 @@ def _body_size(response: Response) -> int:
return len(response.body)
-_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()
+def get_base_url(response: TextResponse) -> str:
+ warnings.warn(
+ "scrapy.utils.response.get_base_url() is deprecated, use "
+ "TextResponse.base_url instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return response.base_url
-def get_base_url(response: TextResponse) -> str:
- """Return the base url of the given response, joined with the response url"""
- if response not in _baseurl_cache:
- text = response.text[0:4096]
- _baseurl_cache[response] = html.get_base_url(
- text, response.url, response.encoding
- )
- return _baseurl_cache[response]
+_IGNORABLE_TAGS = {"script", "noscript", "template"}
-_metaref_cache: WeakKeyDictionary[Response, tuple[None, None] | tuple[float, str]] = (
- WeakKeyDictionary()
-)
+def _get_meta_refresh(
+ response: TextResponse, *, scripting: bool
+) -> tuple[float, str] | None:
+ if scripting:
+ return response.meta_refresh
+ return frostwork.navigation(
+ response.body,
+ response.url,
+ response.encoding,
+ base_url=False,
+ scripting=False,
+ ).meta_refresh
def get_meta_refresh(
response: TextResponse,
ignore_tags: Iterable[str] = ("script", "noscript"),
) -> tuple[None, None] | tuple[float, str]:
- """Parse the http-equiv refresh parameter from the given response"""
- if response not in _metaref_cache:
- text = response.text[0:4096]
- _metaref_cache[response] = html.get_meta_refresh(
- text, get_base_url(response), response.encoding, ignore_tags=ignore_tags
+ warnings.warn(
+ "scrapy.utils.response.get_meta_refresh() is deprecated, use "
+ "TextResponse.meta_refresh instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ ignore_tags = set(ignore_tags)
+ if not ignore_tags <= _IGNORABLE_TAGS:
+ raise ValueError(
+ f"Unsupported ignore_tags: {sorted(ignore_tags - _IGNORABLE_TAGS)}"
)
- return _metaref_cache[response]
+ meta_refresh = _get_meta_refresh(response, scripting="noscript" in ignore_tags)
+ return meta_refresh or (None, None)
def response_from_dict(d: dict[str, Any]) -> Response:
@@ -134,7 +150,7 @@ def open_in_browser(
# any base tag of the response, and matches it when there is one.
doctype = _DOCTYPE_RE.match(body)
index = doctype.end() if doctype else 0
- base_tag = f'<base href="{get_base_url(response)}">'.encode()
+ base_tag = f'<base href="{response.base_url}">'.encode()
body = body[:index] + base_tag + body[index:]
ext = ".html"
elif isinstance(response, TextResponse):
diff --git a/tests/benchmarks/test_encoding.py b/tests/benchmarks/test_encoding.py
new file mode 100644
index 000000000..09e402d6c
--- /dev/null
+++ b/tests/benchmarks/test_encoding.py
@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from scrapy.http import TextResponse
+from tests import get_testdata
+
+if TYPE_CHECKING:
+ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
+
+pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
+
+# A real page without its encoding declaration.
+_PAGE = (
+ get_testdata("books", "listing.html")
+ .decode()
+ .replace("text/html; charset=UTF-8", "text/html")
+)
+
+
+def _body(text: str, encoding: str) -> bytes:
+ """Return :data:`_PAGE` with *text* added, encoded with *encoding*."""
+ return _PAGE.replace("</body>", f"<p>{text * 500}</p></body>").encode(encoding)
+
+
+_JAPANESE = "この本は在庫があります。"
+_FRENCH = "Ce livre à prix réduit est déjà épuisé. "
+
+
+_UNDECLARED = pytest.mark.parametrize(
+ ("body", "encoding"),
+ [
+ pytest.param(_body(_JAPANESE, "utf-8"), "utf-8", id="utf-8"),
+ pytest.param(_body(_JAPANESE, "shift_jis"), "cp932", id="shift_jis"),
+ pytest.param(_body(_FRENCH, "cp1252"), "cp1252", id="windows-1252"),
+ ],
+)
+
+
+@_UNDECLARED
+def test_undeclared_encoding(
+ benchmark: BenchmarkFixture, body: bytes, encoding: str
+) -> None:
+ """Decoding a page that declares no encoding."""
+
+ def run() -> None:
+ response = TextResponse("https://example.com", body=body)
+ assert response.text
+ assert response.encoding == encoding
+
+ benchmark(run)
+
+
+@_UNDECLARED
+def test_undeclared_encoding_only(
+ benchmark: BenchmarkFixture, body: bytes, encoding: str
+) -> None:
+ """Getting the encoding of a page that declares none, without its text."""
+
+ def run() -> None:
+ response = TextResponse("https://example.com", body=body)
+ assert response.encoding == encoding
+
+ benchmark(run)
diff --git a/tests/sample_data/feeds/feed-sample5.csv b/tests/sample_data/feeds/feed-sample5.csv
index 9cff1dfa4..90ad09272 100644
--- a/tests/sample_data/feeds/feed-sample5.csv
+++ b/tests/sample_data/feeds/feed-sample5.csv
@@ -1,3 +1,3 @@
id,name,value
-1,cp852,test
+1,cp866,test
2,something,�����ͻ
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 9d4a00dc8..504b3d1ed 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -623,9 +623,9 @@ class TestContractsManager:
requests = self.conman.from_method(spider.returns_multiple_urls, self.results)
assert sorted(request.url for request in requests) == [
- "http://example.com",
- "http://scrapy.org",
- "http://scrapy.org",
+ "http://example.com/",
+ "http://scrapy.org/",
+ "http://scrapy.org/",
]
for request in requests:
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index a3ab5d1c1..c0564f52f 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -167,7 +167,7 @@ class TestHttpsWithCrawler(TestHttpWithCrawler):
class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
- expected_http_proxy_request_body = b"http://example.com/"
+ pass
class TestHttpsProxy(TestHttpProxy):
@@ -186,7 +186,7 @@ class TestHttpsProxy(TestHttpProxy):
request = Request("http://example.com", meta={"proxy": http_proxy})
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
- assert response.body == self.expected_http_proxy_request_body
+ assert response.body == b"http://example.com/"
assert keylog_file.read_text()
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index c6b19d464..e6c593712 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -1004,7 +1004,7 @@ class TestConcurrentDownloads(TestBase):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
- url = "http://www.example.com"
+ url = "http://www.example.com/"
@contextmanager
def _manager(
@@ -1204,7 +1204,7 @@ class TestInFlightTracking(TestBase):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
- url = "http://www.example.com"
+ url = "http://www.example.com/"
@coroutine_test
async def test_garbage_collected_request(self):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 3372813f9..a5f05fad0 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -18,7 +18,9 @@ class TestHttpProxyMiddleware:
with pytest.raises(NotConfigured):
build_from_crawler(HttpProxyMiddleware, crawler)
- @pytest.mark.parametrize("url", ["http://e.com", "https://e.com", "file:///tmp/a"])
+ @pytest.mark.parametrize(
+ "url", ["http://e.com/", "https://e.com/", "file:///tmp/a"]
+ )
def test_no_environment_proxies(
self, url: str, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -35,8 +37,8 @@ class TestHttpProxyMiddleware:
@pytest.mark.parametrize(
("url", "proxy"),
[
- ("http://e.com", "https://proxy.for.http:3128"),
- ("https://e.com", "http://proxy.for.https:8080"),
+ ("http://e.com/", "https://proxy.for.http:3128"),
+ ("https://e.com/", "http://proxy.for.https:8080"),
("file://tmp/a", None),
],
)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index d3cc1be0a..ceb0b09f9 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -234,7 +234,7 @@ class TestRedirectMiddleware(TestRedirectBase):
@pytest.mark.parametrize("status", [307, 308])
def test_cross_origin_maintain_body(self, status):
source_url = "https://example.com"
- target_url = "https://attacker.example"
+ target_url = "https://attacker.example/"
body = b"secret"
request = Request(
source_url,
diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py
index d118da005..06988e003 100644
--- a/tests/test_downloadermiddleware_redirect_metarefresh.py
+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py
@@ -2,12 +2,13 @@ from __future__ import annotations
import logging
from itertools import chain
+from typing import TYPE_CHECKING
from unittest.mock import MagicMock
import pytest
from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
@@ -20,6 +21,9 @@ from tests.utils.redirect import (
SCHEME_PARAMS,
)
+if TYPE_CHECKING:
+ from scrapy.settings import BaseSettings
+
def meta_refresh_body(url: str, interval: int = 5) -> bytes:
html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
@@ -91,19 +95,90 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
response = self.mw.process_response(req, rsp)
assert isinstance(response, Response)
- def test_ignore_tags_1_x_list(self):
- """Test that Scrapy 1.x behavior remains possible"""
- settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]}
- crawler = get_crawler(Spider, settings)
+ def test_scripting_false(self):
+ crawler = get_crawler(Spider, {"METAREFRESH_SCRIPTING": False})
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
req = Request(url="http://example.org")
- body = (
- """<noscript><meta http-equiv="refresh" """
- """content="0;URL='http://example.org/newpage'"></noscript>"""
+ body = b"<noscript>" + self._body(interval=0) + b"</noscript>"
+ req2 = mw.process_response(req, HtmlResponse(req.url, body=body))
+ assert isinstance(req2, Request)
+ assert req2.url == "http://example.org/newpage"
+
+ @pytest.mark.parametrize("content", ["0", "0;url=#x", "0;url=http://EXAMPLE.org/"])
+ def test_refresh_self(self, content: str) -> None:
+ req = Request(url="http://example.org")
+ body = f'<meta http-equiv="refresh" content="{content}">'.encode()
+ rsp = HtmlResponse(req.url, body=body)
+ assert self.mw.process_response(req, rsp) is rsp
+
+ def test_maxdelay_shortest(self):
+ req = Request(url="http://example.org")
+ body = self._body(interval=1000, url="/slow") + self._body(
+ interval=1, url="/fast"
)
- rsp = HtmlResponse(req.url, body=body.encode())
- response = mw.process_response(req, rsp)
- assert isinstance(response, Response)
+ req2 = self.mw.process_response(req, HtmlResponse(req.url, body=body))
+ assert isinstance(req2, Request)
+ assert req2.url == "http://example.org/fast"
+
+
+NOSCRIPT_REFRESH = (
+ b"<noscript>" + meta_refresh_body("http://example.org/newpage", 0) + b"</noscript>"
+)
+
+
+def _process_noscript_refresh(mw: MetaRefreshMiddleware) -> Request | Response:
+ req = Request(url="http://example.org")
+ return mw.process_response(req, HtmlResponse(req.url, body=NOSCRIPT_REFRESH))
+
+
+ ("ignore_tags", "scripting"),
+ [
+ (["script", "noscript"], True),
+ (["noscript", "template"], True),
+ (["script"], False),
+ ([], False),
+ ],
+)
+def test_ignore_tags(ignore_tags: list[str], scripting: bool) -> None:
+ crawler = get_crawler(Spider, {"METAREFRESH_IGNORE_TAGS": ignore_tags})
+ with pytest.warns(
+ ScrapyDeprecationWarning, match=f"METAREFRESH_SCRIPTING = {scripting}"
+ ):
+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)
+ assert isinstance(_process_noscript_refresh(mw), Request) is not scripting
+
+
+class _RemoveNoscriptAddon:
+ def update_settings(self, settings: BaseSettings) -> None:
+ settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript")
+
+
+def test_ignore_tags_removed_from_list() -> None:
+ crawler = get_crawler(Spider, {"ADDONS": {_RemoveNoscriptAddon: 0}})
+ with pytest.warns(ScrapyDeprecationWarning, match="METAREFRESH_SCRIPTING = False"):
+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)
+ assert isinstance(_process_noscript_refresh(mw), Request)
+
+
+def test_ignore_tags_unsupported() -> None:
+ crawler = get_crawler(Spider, {"METAREFRESH_IGNORE_TAGS": ["noscript", "div"]})
+ with pytest.raises(ValueError, match="METAREFRESH_SCRIPTING"):
+ build_from_crawler(MetaRefreshMiddleware, crawler)
+
+
+class _IgnoreNoTagsAddon:
+ def update_settings(self, settings: BaseSettings) -> None:
+ settings.set("METAREFRESH_IGNORE_TAGS", [], priority="addon")
+
+
+def test_ignore_tags_overridden() -> None:
+ crawler = get_crawler(
+ Spider,
+ {"ADDONS": {_IgnoreNoTagsAddon: 0}, "METAREFRESH_SCRIPTING": True},
+ )
+ mw = build_from_crawler(MetaRefreshMiddleware, crawler)
+ assert isinstance(_process_noscript_refresh(mw), Response)
@pytest.mark.parametrize(
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index ceee8d865..bad26bfcb 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -785,6 +785,22 @@ class TestHttps2ClientProtocol:
assert client.transport is not None
assert client.transport.disconnecting
+ @pytest.mark.parametrize(("method", "path"), [("GET", b"/"), ("OPTIONS", b"*")])
+ @deferred_f_from_coro_f
+ async def test_empty_path(
+ self, method: str, path: bytes, server_port: int, client: H2ClientProtocol
+ ) -> None:
+ request = Request(
+ f"{self.scheme}://{self.host}:{server_port}",
+ method=method,
+ meta={"verbatim_url": True},
+ )
+ with mock.patch.object(
+ client.conn, "send_headers", wraps=client.conn.send_headers
+ ) as send_headers:
+ await make_request(client, request)
+ assert (b":path", path) in send_headers.call_args.args[1]
+
@deferred_f_from_coro_f
async def test_request_headers_received(
self, server_port: int, client: H2ClientProtocol
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index ec6a4896f..fd0136642 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -6,9 +6,11 @@ import weakref
from typing import cast
from unittest import mock
+import frostwork
import pytest
-from scrapy.http import HtmlResponse, TextResponse, XmlResponse
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.http import HtmlResponse, MetaRefresh, TextResponse, XmlResponse
from scrapy.selector import Selector
from tests.utils.bases.http_response import TestResponseBase
@@ -32,21 +34,21 @@ class TestTextResponse(TestResponseBase):
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
- "http://www.example.com", body="hello", encoding="cp852"
+ "http://www.example.com", body="hello", encoding="cp866"
)
r2 = r1.replace(url="http://www.example.com/other")
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
assert isinstance(r2, self.response_class)
assert r2.url == "http://www.example.com/other"
- self._assert_response_encoding(r2, "cp852")
+ assert r2.encoding == "cp866"
assert r3.url == "http://www.example.com/other"
assert r3._declared_encoding() == "latin1"
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
resp = self.response_class("http://www.example.com/")
- self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
+ self._assert_response_encoding(resp, "utf-8")
# make sure urls are converted to str
resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
@@ -167,8 +169,8 @@ class TestTextResponse(TestResponseBase):
self._assert_response_values(r1, "utf-8", "\xa3")
self._assert_response_values(r2, "utf-8", "\xa3")
self._assert_response_values(r3, "iso-8859-1", "\xa3")
- self._assert_response_values(r6, "gb18030", "\u2015")
- self._assert_response_values(r7, "gb18030", "\u2015")
+ self._assert_response_values(r6, "gbk", "\u2015")
+ self._assert_response_values(r7, "gbk", "\u2015")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
with pytest.raises(TypeError):
@@ -184,11 +186,17 @@ class TestTextResponse(TestResponseBase):
assert r._declared_encoding() is None
self._assert_response_values(r, "utf-8", "\xa3")
- def test_body_inferred_encoding_of_undeclared_legacy_page(self):
+ @pytest.mark.parametrize(
+ ("url", "encoding"),
+ [("http://www.example.com", "cp932"), ("http://www.example.cn", "gbk")],
+ )
+ def test_body_inferred_encoding_of_undeclared_legacy_page(
+ self, url: str, encoding: str
+ ) -> None:
body = "ICD10 国際疾病分類第10版2013年版 病名マスター".encode("shift-jis")
- r = self.response_class("http://www.example.com", body=body)
+ r = self.response_class(url, body=body)
assert r._declared_encoding() is None
- assert r.encoding in {"shift_jis", "cp932"}
+ assert r.encoding == encoding
assert r.text == body.decode(r.encoding)
def test_utf16(self):
@@ -199,7 +207,36 @@ class TestTextResponse(TestResponseBase):
body=body,
encoding="utf-16",
)
- self._assert_response_values(r, "utf-16", body)
+ assert r.encoding == "utf-16"
+ assert r.text == "hi"
+
+ @pytest.mark.parametrize(
+ ("encoding", "body", "text"),
+ [
+ ("cp437", "<p>é", "<p>é"),
+ ("hz", b"~{Dc:C~}", "你好"),
+ ("foo", b"\xc2\xa3", "£"),
+ ("cp437", codecs.BOM_UTF8 + b"\xc2\xa3", "£"),
+ ],
+ )
+ def test_non_browser_encoding(
+ self, encoding: str, body: str | bytes, text: str
+ ) -> None:
+ with pytest.warns(ScrapyDeprecationWarning, match=repr(encoding)):
+ r = self.response_class(
+ "http://www.example.com", body=body, encoding=encoding
+ )
+ assert r.encoding == encoding
+ assert r.text == text
+
+ def test_replace_replacement_encoding(self) -> None:
+ r = self.response_class(
+ "http://www.example.com",
+ headers={"Content-Type": ["text/html; charset=hz"]},
+ body=b"~{Dc:C~}",
+ )
+ assert r.encoding == "replacement"
+ assert r.replace(url="http://www.example.com/other").text == "�"
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
r6 = self.response_class(
@@ -318,7 +355,6 @@ class TestTextResponse(TestResponseBase):
)
def test_urljoin_with_base_url(self):
- """Test urljoin shortcut which also evaluates base-url through get_base_url()."""
body = b'<html><body><base href="https://example.net"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"/test"
@@ -352,6 +388,84 @@ class TestTextResponse(TestResponseBase):
assert response.urljoin("/test") == "https://example.net/test"
assert response.follow_redirect().url == "http://www.example.com/test"
+ def test_base_url_serialized(self) -> None:
+ response = self.response_class("http://www.EXAMPLE.com", body=b"")
+ assert response.base_url == "http://www.example.com/"
+
+ @pytest.mark.parametrize(
+ ("body", "base_url"),
+ [
+ (b"", "http://www.example.com/a/b"),
+ (b'<base href="c/">', "http://www.example.com/a/c/"),
+ (
+ b'<base target="_blank"><base href="/c"><base href="/d">',
+ "http://www.example.com/c",
+ ),
+ ],
+ )
+ def test_base_url(self, body: bytes, base_url: str) -> None:
+ response = self.response_class("http://www.example.com/a/b", body=body)
+ assert response.base_url == base_url
+
+ @pytest.mark.parametrize(
+ ("body", "meta_refresh"),
+ [
+ (b"", None),
+ (
+ b'<base href="/c/"><meta http-equiv="refresh" content="5;url=d">',
+ (5.0, "http://www.example.com/c/d"),
+ ),
+ (
+ (
+ b'<meta http-equiv="refresh" content="2;url=/a">'
+ b'<meta http-equiv="refresh" content="1;url=/b">'
+ b'<meta http-equiv="refresh" content="1;url=/c">'
+ b'<meta http-equiv="refresh" content="3;url=/d">'
+ ),
+ (1.0, "http://www.example.com/c"),
+ ),
+ (
+ (
+ b'<noscript><meta http-equiv="refresh" content="0;url=/a">'
+ b"</noscript><template>"
+ b'<meta http-equiv="refresh" content="0;url=/b"></template>'
+ b'<script>document.write(\'<meta http-equiv="refresh" '
+ b'content="0;url=/c">\')</script>'
+ ),
+ None,
+ ),
+ ],
+ )
+ def test_meta_refresh(
+ self, body: bytes, meta_refresh: tuple[float, str] | None
+ ) -> None:
+ response = self.response_class("http://www.example.com", body=body)
+ assert response.meta_refresh == meta_refresh
+
+ def test_html_metadata_once(self) -> None:
+ body = b'<base href="/a/"><meta http-equiv="refresh" content="0;url=b">'
+ response = self.response_class("http://www.example.com", body=body)
+ with mock.patch("frostwork.navigation", wraps=frostwork.navigation) as read:
+ assert response.base_url == "http://www.example.com/a/"
+ assert response.meta_refresh == (0.0, "http://www.example.com/a/b")
+ assert isinstance(response.meta_refresh, MetaRefresh)
+ read.assert_called_once()
+
+ def test_html_metadata_set(self) -> None:
+ body = b'<base href="/a/"><meta http-equiv="refresh" content="0;url=b">'
+ response = self.response_class("http://www.example.com", body=body)
+ response.base_url = "http://example.net/"
+ assert response.meta_refresh == (0.0, "http://www.example.com/a/b")
+ assert response.base_url == "http://example.net/"
+ response.meta_refresh = None
+ assert response.meta_refresh is None
+
+ response = self.response_class("http://www.example.com", body=body)
+ response.meta_refresh = (1.0, "http://example.net/")
+ assert response.base_url == "http://www.example.com/a/"
+ assert response.meta_refresh == (1.0, "http://example.net/")
+ assert isinstance(response.meta_refresh, MetaRefresh)
+
def test_follow_selector(self):
resp = self._links_response()
urls = [
@@ -596,7 +710,7 @@ class TestXmlResponse(TestTextResponse):
def test_xml_encoding(self):
body = b"<xml></xml>"
r1 = self.response_class("http://www.example.com", body=body)
- self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)
+ self._assert_response_values(r1, "utf-8", body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r2 = self.response_class("http://www.example.com", body=body)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 8da842da5..c6fc0328c 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -38,7 +38,7 @@ class TestLogFormatter:
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws["msg"] % logkws["args"]
- assert logline == "Crawled (200) <GET http://www.example.com> (referer: None)"
+ assert logline == "Crawled (200) <GET http://www.example.com/> (referer: None)"
def test_crawled_with_referer(self):
req = Request(
@@ -49,7 +49,7 @@ class TestLogFormatter:
logline = logkws["msg"] % logkws["args"]
assert (
logline
- == "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']"
+ == "Crawled (200) <GET http://www.example.com/> (referer: http://example.com) ['cached']"
)
def test_flags_in_request(self):
@@ -59,7 +59,7 @@ class TestLogFormatter:
logline = logkws["msg"] % logkws["args"]
assert (
logline
- == "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)"
+ == "Crawled (200) <GET http://www.example.com/> ['test', 'flag'] (referer: None)"
)
def test_dropped(self):
@@ -149,7 +149,7 @@ class TestLogFormatter:
logline = logkws["msg"] % logkws["args"]
assert (
logline
- == "Spider error processing <GET http://www.example.com> (referer: http://example.org)"
+ == "Spider error processing <GET http://www.example.com/> (referer: http://example.org)"
)
def test_download_error_short(self):
@@ -159,7 +159,7 @@ class TestLogFormatter:
request = Request("http://www.example.com")
logkws = self.formatter.download_error(failure, request, self.spider)
logline = logkws["msg"] % logkws["args"]
- assert logline == "Error downloading <GET http://www.example.com>"
+ assert logline == "Error downloading <GET http://www.example.com/>"
def test_download_error_long(self):
# In practice, the complete traceback is shown by passing the
@@ -170,7 +170,9 @@ class TestLogFormatter:
failure, request, self.spider, "Some message"
)
logline = logkws["msg"] % logkws["args"]
- assert logline == "Error downloading <GET http://www.example.com>: Some message"
+ assert (
+ logline == "Error downloading <GET http://www.example.com/>: Some message"
+ )
def test_scraped(self):
item = CustomItem()
@@ -208,7 +210,7 @@ class TestLogformatterSubclass(TestLogFormatter):
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws["msg"] % logkws["args"]
assert (
- logline == "Crawled (200) <GET http://www.example.com> (referer: None) []"
+ logline == "Crawled (200) <GET http://www.example.com/> (referer: None) []"
)
def test_crawled_with_referer(self):
@@ -222,7 +224,7 @@ class TestLogformatterSubclass(TestLogFormatter):
logline = logkws["msg"] % logkws["args"]
assert (
logline
- == "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']"
+ == "Crawled (200) <GET http://www.example.com/> (referer: http://example.com) ['cached']"
)
def test_flags_in_request(self):
@@ -232,7 +234,7 @@ class TestLogformatterSubclass(TestLogFormatter):
logline = logkws["msg"] % logkws["args"]
assert (
logline
- == "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']"
+ == "Crawled (200) <GET http://www.example.com/> (referer: None) ['test', 'flag']"
)
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index be8aa1f3d..3884d4730 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -125,7 +125,7 @@ class TestImagesPipeline:
)
assert (
thumb_path(Request("file://foo.png"), name)
- == "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg"
+ == "thumbs/50/3f7f26a0132d8c3df6b56f0691b89083df00d62a.jpg"
)
assert (
thumb_path(Request("file:///tmp/foo"), name)
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 85e300e40..be46363de 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -14,7 +14,7 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
-OVERRIDDEN_URL = "https://example.org"
+OVERRIDDEN_URL = "https://example.org/"
class ProcessResponseMiddleware:
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 443331c69..fbccdeacc 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -38,7 +38,7 @@ class TestSelector:
def test_root_base_url(self) -> None:
body = b'<html><form action="/path"><input name="a" /></form></html>'
- url = "http://example.com"
+ url = "http://example.com/"
response = TextResponse(url=url, body=body, encoding="utf-8")
sel = Selector(response)
assert url == sel.root.base
@@ -124,6 +124,17 @@ class TestSelector:
response = HtmlResponse("https://example.com", body=body, headers=headers)
assert response.css("p::text").get() == "£€"
+ @pytest.mark.parametrize(
+ "body",
+ [
+ b"<meta charset=euc-jp><p>\xa1\xc1 \xad\xa1</p>",
+ b"<p>\xa1\xc1 \xad\xa1" + "日本語です。".encode("euc_jp") * 50 + b"</p>",
+ ],
+ )
+ def test_browser_decoding(self, body: bytes) -> None:
+ response = HtmlResponse("https://example.com", body=body)
+ assert response.css("p::text").get("").startswith("~ ①")
+
def test_weakref_slots(self) -> None:
"""Check that classes are using slots and are weak-referenceable"""
x = Selector(text="")
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index af500aad9..0ac75f754 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -239,7 +239,7 @@ def test_sqlite_queue_survives_unclean_shutdown(
) -> None:
key = str(tmp_path / "queue")
queue = queue_cls.from_crawler(crawler=crawler, key=key)
- queue.push(Request("https://toscrape.com"))
+ queue.push(Request("https://toscrape.com/"))
# No close(), as if the process had been killed. The connection is closed
# directly instead, since on Windows the file cannot be removed later while
# it is still open.
@@ -250,7 +250,7 @@ def test_sqlite_queue_survives_unclean_shutdown(
assert len(queue) == 1
request = queue.pop()
assert request is not None
- assert request.url == "https://toscrape.com"
+ assert request.url == "https://toscrape.com/"
finally:
queue.close()
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index eefb562bd..6c15cd596 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -474,10 +474,10 @@ class TestUtilsCsv:
{"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"},
]
- response = TextResponse(url="http://example.com/", body=body2, encoding="cp852")
+ response = TextResponse(url="http://example.com/", body=body2, encoding="cp866")
csv = csviter(response)
assert list(csv) == [
- {"id": "1", "name": "cp852", "value": "test"},
+ {"id": "1", "name": "cp866", "value": "test"},
{
"id": "2",
"name": "something",
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 8fc6e16be..7b6dbdb50 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -423,7 +423,7 @@ class TestRequestToCurl:
def test_get(self) -> None:
request_object = Request("https://www.example.com")
- expected_curl_command = "curl -X GET https://www.example.com"
+ expected_curl_command = "curl -X GET https://www.example.com/"
self._test_request(request_object, expected_curl_command)
def test_post(self) -> None:
@@ -464,7 +464,7 @@ class TestRequestToCurl:
)
def test_header_values(self, headers: dict[str, Any], expected_args: str) -> None:
request_object = Request("https://www.example.com", headers=headers)
- expected_curl_command = f"curl -X GET https://www.example.com {expected_args}"
+ expected_curl_command = f"curl -X GET https://www.example.com/ {expected_args}"
self._test_request(request_object, expected_curl_command)
def test_headers_from_curl(self) -> None:
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 915a28802..74eec5c58 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -6,6 +6,7 @@ from urllib.parse import urlparse
import pytest
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Response, TextResponse
from scrapy.utils.response import (
get_base_url,
@@ -41,72 +42,27 @@ def test_open_in_browser():
open_in_browser(resp, _openfunc=browser_open)
-def test_get_meta_refresh():
- r1 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- r2 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><title>Dummy</title><noScript>
- <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
- </noSCRIPT>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- r3 = HtmlResponse(
- "http://www.example.com",
- body=b"""
-<noscript><meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage</noscript>
-<script type="text/javascript">
-if(!checkCookies()){
- document.write('<meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage">');
-}
-</script>
- """,
- )
- r4 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><title>Dummy</title>
- <base href="http://www.another-domain.com/base/path/">
- <meta http-equiv="refresh" content="5;url=target.html"</head>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")
- assert get_meta_refresh(r2) == (None, None)
- assert get_meta_refresh(r3) == (None, None)
- assert get_meta_refresh(r4) == (
- 5.0,
- "http://www.another-domain.com/base/path/target.html",
- )
-
+def test_get_meta_refresh() -> None:
+ body = b'<noscript><meta http-equiv="refresh" content="5;url=/a"></noscript>'
+ response = HtmlResponse("http://www.example.com", body=body)
+ with pytest.warns(ScrapyDeprecationWarning, match="TextResponse.meta_refresh"):
+ assert get_meta_refresh(response) == (None, None)
+ with pytest.warns(ScrapyDeprecationWarning):
+ assert get_meta_refresh(response, ignore_tags=["script"]) == (
+ 5.0,
+ "http://www.example.com/a",
+ )
+ with (
+ pytest.warns(ScrapyDeprecationWarning),
+ pytest.raises(ValueError, match="div"),
+ ):
+ get_meta_refresh(response, ignore_tags=["div"])
-def test_get_base_url():
- resp = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html>
- <head><base href="http://www.example.com/img/" target="_blank"></head>
- <body>blahablsdfsal&amp;</body>
- </html>""",
- )
- assert get_base_url(resp) == "http://www.example.com/img/"
- resp2 = HtmlResponse(
- "http://www.example.com",
- body=b"""
- <html><body>blahablsdfsal&amp;</body></html>""",
- )
- assert get_base_url(resp2) == "http://www.example.com"
+def test_get_base_url() -> None:
+ response = HtmlResponse("http://www.example.com", body=b'<base href="/a/">')
+ with pytest.warns(ScrapyDeprecationWarning, match="TextResponse.base_url"):
+ assert get_base_url(response) == "http://www.example.com/a/"
def test_response_status_message():
@@ -195,7 +151,7 @@ def test_response_status_message():
],
)
def test_inject_base_url(body: bytes) -> None:
- url = "http://www.example.com"
+ url = "http://www.example.com/"
def check_base_url(burl):
bbody = _read_browser_output(burl)
@@ -247,7 +203,7 @@ def test_open_in_browser_redos_head():
def test_open_in_browser_preserves_html_comments():
- url = "http://www.example.com"
+ url = "http://www.example.com/"
body = (
b"<html>"
b"<!-- preserved comment -->"
@@ -290,7 +246,7 @@ def test_open_in_browser_keeps_base_url_of_response(
def test_open_in_browser_injects_base_when_only_in_comment():
- url = "http://www.example.com"
+ url = "http://www.example.com/"
body = (
b"<html>"
b"<!-- <base href='http://other.com'> -->"
@@ -309,7 +265,7 @@ def test_open_in_browser_injects_base_when_only_in_comment():
def test_open_in_browser_injects_base_before_head_contents():
- url = "http://www.example.com"
+ url = "http://www.example.com/"
body = (
b"<html>"
b"<!--<head>comment head</head>-->"
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index 2e1988bfb..de609312b 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -1868,7 +1868,6 @@ class TestHttpProxyBase(ABC):
is_secure = False
# whether the handler supports HTTPS proxies with HTTPS destinations
handler_supports_tls_in_tls: bool = True
- expected_http_proxy_request_body = b"http://example.com"
@property
@abstractmethod
@@ -1901,7 +1900,7 @@ class TestHttpProxyBase(ABC):
response = await download_handler.download_request(request)
assert response.status == 200
assert response.url == request.url
- assert response.body == self.expected_http_proxy_request_body
+ assert response.body == b"http://example.com/"
@coroutine_test
async def test_download_without_proxy(
@@ -1962,7 +1961,7 @@ class TestHttpProxyBase(ABC):
response = await download_handler.download_request(request)
assert response.status == 200
assert response.url == request.url
- assert response.body == self.expected_http_proxy_request_body
+ assert response.body == b"http://example.com/"
PROXY_KINDS = ["http", "https", "socks5"]
diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py
index 541a7ca14..ddbbeac2c 100644
--- a/tests/utils/bases/http_request.py
+++ b/tests/utils/bases/http_request.py
@@ -36,7 +36,7 @@ class TestRequestBase(ABC):
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
- assert r.url == "http://www.example.com"
+ assert r.url == "http://www.example.com/"
assert r.method == self.default_method
assert isinstance(r.headers, Headers)
diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py
index 9df62a186..2271e8ca2 100644
--- a/tests/utils/bases/http_response.py
+++ b/tests/utils/bases/http_response.py
@@ -246,7 +246,7 @@ class TestResponseBase(ABC):
# Response.follow
def test_follow_url_absolute(self):
- self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
+ self._assert_followed_url("http://foo.example.com/", "http://foo.example.com/")
def test_follow_url_relative(self):
self._assert_followed_url("foo", "http://example.com/foo")
@@ -333,10 +333,10 @@ class TestResponseBase(ABC):
def test_follow_all_absolute(self):
url_list = [
- "http://example.org",
- "http://www.example.org",
- "http://example.com",
- "http://www.example.com",
+ "http://example.org/",
+ "http://www.example.org/",
+ "http://example.com/",
+ "http://www.example.com/",
]
self._assert_followed_all_urls(url_list, url_list)
diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py
index 209bc30f3..4a40a18b5 100644
--- a/tests/utils/bases/redirect.py
+++ b/tests/utils/bases/redirect.py
@@ -289,18 +289,18 @@ class TestRedirectBase(ABC):
[
pytest.param(
"http://example.com",
- "http://example.com",
- "http://example.com",
+ "http://example.com/a",
+ "http://example.com/b",
id="http-absolute",
),
- pytest.param("http://example.com", "/a", "/a", id="http-relative"),
+ pytest.param("http://example.com", "/a", "/b", id="http-relative"),
pytest.param(
"https://example.com",
- "https://example.com",
- "https://example.com",
+ "https://example.com/a",
+ "https://example.com/b",
id="https-absolute",
),
- pytest.param("https://example.com", "/a", "/a", id="https-relative"),
+ pytest.param("https://example.com", "/a", "/b", id="https-relative"),
pytest.param(
"http://example.com",
"https://example.com",
@@ -335,8 +335,8 @@ class TestRedirectBase(ABC):
pytest.param(
{"http_proxy": PROXY_A},
"http://example.com",
- "http://example.com",
- "http://example.com",
+ "http://example.com/a",
+ "http://example.com/b",
("a", "a", "a", "a", "a"),
id="http-absolute",
),
@@ -344,15 +344,15 @@ class TestRedirectBase(ABC):
{"http_proxy": PROXY_A},
"http://example.com",
"/a",
- "/a",
+ "/b",
("a", "a", "a", "a", "a"),
id="http-relative",
),
pytest.param(
{"https_proxy": PROXY_A},
"https://example.com",
- "https://example.com",
- "https://example.com",
+ "https://example.com/a",
+ "https://example.com/b",
("a", "a", "a", "a", "a"),
id="https-absolute",
),
@@ -360,7 +360,7 @@ class TestRedirectBase(ABC):
{"https_proxy": PROXY_A},
"https://example.com",
"/a",
- "/a",
+ "/b",
("a", "a", "a", "a", "a"),
id="https-relative",
),
diff --git a/tox.ini b/tox.ini
index 2cee007ea..5957752f3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -79,6 +79,7 @@ deps =
boto3-stubs[s3]==1.43.107
botocore-stubs==1.43.67
colorlog==6.12.0
+ frostwork==0.2.2
h2==4.4.1
httpx2==2.13.1
itemadapter==0.13.1
@@ -94,7 +95,7 @@ deps =
types-lxml==2026.2.16
types-pexpect==4.9.0.20260518
uvloop==0.23.0
- w3lib==2.5.0
+ w3lib==2.6.0
commands =
mypy {posargs:scrapy tests}
@@ -146,10 +147,10 @@ deps =
aiohttp==3.13.3
backports.zstd==1.3.0
brotli==1.2.0
- charset-normalizer==3.4.0
cryptography==41.0.5
cssselect==1.2.0
formerly==0.1.0
+ frostwork==0.2.2
itemadapter==0.1.0
itemloaders==1.0.5
lxml==4.9.4
@@ -160,8 +161,8 @@ deps =
pyOpenSSL==24.3.0
queuelib==1.8.0
service_identity==24.2.0
- typing-extensions==4.6.0
- w3lib==2.1.1
+ typing-extensions==4.13.0
+ w3lib==2.6.0
zope.interface==5.1.0
{[test-requirements]deps}
setenv =