Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate remaining planned scrapy.utils (#8146)
Andrey Rakhmatullinf67c87db61e5
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex c32a01395..79c25cd98 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -12,7 +12,7 @@ from scrapy.commands import ScrapyCommandfrom scrapy.commands.edit import _edit_filefrom scrapy.exceptions import UsageErrorfrom scrapy.spiderloader import get_spider_loader-from scrapy.utils.template import render_templatefile, string_camelcase+from scrapy.utils._template import render_templatefile, string_camelcaseif TYPE_CHECKING:import argparsediff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex d50075fb5..4edadfa58 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -13,7 +13,7 @@ from w3lib.url import is_urlfrom scrapy.commands import BaseRunSpiderCommandfrom scrapy.exceptions import UsageErrorfrom scrapy.http import Request, Response-from scrapy.utils import display+from scrapy.utils import _colorizefrom scrapy.utils.asyncgen import collect_asyncgenfrom scrapy.utils.defer import _schedule_coro, aiter_errback, deferred_from_corofrom scrapy.utils.log import failure_to_exc_info@@ -170,7 +170,7 @@ class Command(BaseRunSpiderCommand):items = self.items.get(lvl, [])print("# Scraped Items ", "-" * 60)- display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)+ _colorize.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)def print_requests(self, lvl: int | None = None, colour: bool = True) -> None:if lvl is not None:@@ -181,7 +181,7 @@ class Command(BaseRunSpiderCommand):requests = []print("# Requests ", "-" * 65)- display.pprint(requests, colorize=colour)+ _colorize.pprint(requests, colorize=colour)def print_results(self, opts: argparse.Namespace) -> None:colour = not opts.nocolourdiff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex 5661a0a48..497c24e9e 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, ClassVarimport scrapyfrom scrapy.commands import ScrapyCommandfrom scrapy.exceptions import UsageError-from scrapy.utils.template import render_templatefile, string_camelcase+from scrapy.utils._template import render_templatefile, string_camelcaseif TYPE_CHECKING:import argparsediff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 0c2830c4c..1f40d563b 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -20,6 +20,7 @@ from scrapy.extension import ExtensionManagerfrom scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settingsfrom scrapy.signalmanager import SignalManagerfrom scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader+from scrapy.utils._ossignals import install_shutdown_handlers, signal_namesfrom scrapy.utils._stopmode import _normalize_stop_mode, _StopModefrom scrapy.utils.defer import _DEFER_DELAY, deferred_from_coro, ensure_awaitablefrom scrapy.utils.log import (@@ -30,7 +31,6 @@ from scrapy.utils.log import (log_scrapy_info,)from scrapy.utils.misc import build_from_crawler, load_object-from scrapy.utils.ossignal import install_shutdown_handlers, signal_namesfrom scrapy.utils.reactor import (_asyncio_reactor_path,_is_asyncio_reactor_installed,diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 320106705..d3dc03c53 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -27,11 +27,11 @@ from zope.interface import Interfacefrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarningfrom scrapy.extensions.postprocessing import PostProcessingManager+from scrapy.utils._ftp import ftp_store_filefrom scrapy.utils.asyncio import is_asyncio_available, run_in_threadfrom scrapy.utils.boto import _get_max_pool_connectionsfrom scrapy.utils.conf import feed_complete_default_values_from_settingsfrom scrapy.utils.defer import deferred_from_coro, ensure_awaitable-from scrapy.utils.ftp import ftp_store_filefrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.python import without_none_valuesdiff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 9d67e98bf..b3c44dee9 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -34,11 +34,11 @@ from scrapy.pipelines.media import (MediaPipeline,_MediaRequestFiltered,)+from scrapy.utils._ftp import ftp_store_filefrom scrapy.utils.asyncio import run_in_threadfrom scrapy.utils.boto import _get_max_pool_connections, is_botocore_availablefrom scrapy.utils.datatypes import CaseInsensitiveDictfrom scrapy.utils.defer import deferred_from_coro, ensure_awaitable-from scrapy.utils.ftp import ftp_store_filefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.python import to_bytesdiff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 92acef9d3..4e73d247d 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -14,7 +14,7 @@ from scrapy.utils._compression import (gunzip,gzip_magic_number,)-from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots+from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robotsif TYPE_CHECKING:# typing.Self requires Python 3.11diff --git a/scrapy/utils/_colorize.py b/scrapy/utils/_colorize.pynew file mode 100644index 000000000..53aa685fc--- /dev/null+++ b/scrapy/utils/_colorize.py@@ -0,0 +1,47 @@+import ctypes+import platform+import sys+from pprint import pformat as pformat_+from typing import Any++from packaging.version import Version as parse_version+++def _enable_windows_terminal_processing() -> bool:+ # https://stackoverflow.com/a/36760881+ kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]+ return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))+++def tty_supports_color() -> bool:+ if sys.platform != "win32":+ return True++ if parse_version(platform.version()) < parse_version("10.0.14393"):+ return True++ # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal+ # processing is enabled.+ return _enable_windows_terminal_processing()+++def _colorize(text: str, colorize: bool = True) -> str:+ # pylint: disable=no-name-in-module+ if not colorize or not sys.stdout.isatty() or not tty_supports_color():+ return text+ try:+ from pygments import highlight # noqa: PLC0415+ except ImportError:+ return text+ from pygments.formatters import TerminalFormatter # noqa: PLC0415+ from pygments.lexers import PythonLexer # noqa: PLC0415++ return highlight(text, PythonLexer(), TerminalFormatter())+++def pformat(obj: object, *args: Any, **kwargs: Any) -> str:+ return _colorize(pformat_(obj), kwargs.pop("colorize", True))+++def pprint(obj: object, *args: Any, **kwargs: Any) -> None:+ print(pformat(obj, *args, **kwargs))diff --git a/scrapy/utils/_ftp.py b/scrapy/utils/_ftp.pynew file mode 100644index 000000000..f08e5303d--- /dev/null+++ b/scrapy/utils/_ftp.py@@ -0,0 +1,53 @@+import posixpath+from contextlib import closing+from ftplib import FTP, FTP_TLS, error_perm+from posixpath import dirname+from ssl import create_default_context+from typing import IO+++def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:+ """Set the current directory of the FTP connection given in the ``ftp``+ argument (as a ftplib.FTP object), creating all parent directories if they+ don't exist. The ftplib.FTP object must be already connected and logged in.+ """+ try:+ ftp.cwd(path)+ except error_perm:+ ftp_makedirs_cwd(ftp, dirname(path), False)+ ftp.mkd(path)+ if first_call:+ ftp.cwd(path)+++def ftp_store_file(+ *,+ path: str,+ file: IO[bytes],+ host: str,+ port: int,+ username: str,+ password: str,+ use_active_mode: bool = False,+ overwrite: bool = True,+ tls: bool = False,+) -> None:+ """Opens a FTP connection with passed credentials, sets current directory+ to the directory extracted from given path, then uploads the file to server.++ If *tls* is ``True``, the connection is secured with TLS (FTPS), and the+ certificate of the server is verified.+ """+ ftp = FTP_TLS(context=create_default_context()) if tls else FTP()+ with ftp, closing(file):+ ftp.connect(host, port)+ ftp.login(username, password)+ if isinstance(ftp, FTP_TLS):+ ftp.prot_p()+ if use_active_mode:+ ftp.set_pasv(False)+ file.seek(0)+ dirname, filename = posixpath.split(path)+ ftp_makedirs_cwd(ftp, dirname)+ command = "STOR" if overwrite else "APPE"+ ftp.storbinary(f"{command} {filename}", file)diff --git a/scrapy/utils/_ossignals.py b/scrapy/utils/_ossignals.pynew file mode 100644index 000000000..4ad8e5741--- /dev/null+++ b/scrapy/utils/_ossignals.py@@ -0,0 +1,33 @@+from __future__ import annotations++import signal+from collections.abc import Callable+from types import FrameType+from typing import Any, TypeAlias++# copy of _HANDLER from typeshed/stdlib/signal.pyi+SignalHandlerT: TypeAlias = (+ Callable[[int, FrameType | None], Any] | int | signal.Handlers | None+)++signal_names: dict[int, str] = {member.value: member.name for member in signal.Signals}+++def install_shutdown_handlers(+ function: SignalHandlerT, override_sigint: bool = True+) -> None:+ """Install the given function as a signal handler for all common shutdown+ signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the+ SIGINT handler won't be installed if there is already a handler in place+ (e.g. Pdb)+ """+ signal.signal(signal.SIGTERM, function)+ if (+ signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable+ == signal.default_int_handler+ or override_sigint+ ):+ signal.signal(signal.SIGINT, function)+ # Catch Ctrl-Break in windows+ if hasattr(signal, "SIGBREAK"):+ signal.signal(signal.SIGBREAK, function)diff --git a/scrapy/utils/_sitemap.py b/scrapy/utils/_sitemap.pynew file mode 100644index 000000000..2edf2d5ef--- /dev/null+++ b/scrapy/utils/_sitemap.py@@ -0,0 +1,129 @@+from __future__ import annotations++import warnings+from io import BytesIO, StringIO+from typing import TYPE_CHECKING, Any+from urllib.parse import urljoin++import lxml.etree++from scrapy.exceptions import ScrapyDeprecationWarning++if TYPE_CHECKING:+ from collections.abc import Iterable, Iterator+++class Sitemap:+ """Class to parse Sitemap (type=urlset) and Sitemap Index+ (type=sitemapindex) files"""++ __slots__ = ("type", "xmliter")++ def __init__(self, xmltext: str | bytes):+ if isinstance(xmltext, str):+ warnings.warn(+ "Passing `str` type as `xmltext` is deprecated, use `bytes`",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ xmltext = xmltext.encode()++ self.xmliter = lxml.etree.iterparse(+ BytesIO(xmltext),+ recover=True,+ remove_comments=True,+ resolve_entities=False,+ remove_blank_text=True,+ collect_ids=False,+ remove_pis=True,+ events=("start", "end"),+ )+ _, root = next(self.xmliter)+ self.type = self._get_tag_name(root)++ def __iter__(self) -> Iterator[dict[str, Any]]:+ for event, elem in self.xmliter:+ if event == "start":+ continue++ if self._get_tag_name(elem) not in {"url", "sitemap"}:+ continue++ if d := self._process_sitemap_element(elem):+ yield d++ def _process_sitemap_element(+ self, elem: lxml.etree._Element+ ) -> dict[str, Any] | None:+ d: dict[str, Any] = {}+ alternate: list[str] = []+ has_loc = False++ for el in elem:+ try:+ tag_name = self._get_tag_name(el)+ if not tag_name:+ continue++ if tag_name == "link":+ if href := el.get("href"):+ alternate.append(href)+ else:+ d[tag_name] = el.text.strip() if el.text else ""+ if not has_loc and tag_name == "loc":+ has_loc = True+ finally:+ el.clear()+ elem.clear()+ parent = elem.getparent()+ if parent is not None:+ while elem.getprevious() is not None:+ del parent[0]++ if not has_loc:+ return None++ if alternate:+ d["alternate"] = alternate++ return d++ @staticmethod+ def _get_tag_name(elem: lxml.etree._Element) -> str:+ tag = elem.tag+ if not isinstance(tag, str):+ return ""+ _, _, localname = tag.partition("}")+ return localname or tag+++def sitemap_urls_from_robots(+ robots_text: str | bytes,+ base_url: str | None = None,+) -> Iterable[str]:+ if isinstance(robots_text, bytes):+ for line in BytesIO(robots_text):+ if line.lstrip()[:8].lower() == b"sitemap:":+ try:+ url = line.partition(b":")[2].strip().decode()+ except UnicodeDecodeError:+ continue+ yield urljoin(base_url or "", url)++ else:+ warnings.warn(+ "Passing `str` type as `robots_text` is deprecated, use `bytes`",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ yield from _sitemap_urls_from_robots_str(robots_text, base_url)+++def _sitemap_urls_from_robots_str(+ robots_text: str,+ base_url: str | None = None,+) -> Iterable[str]:+ for line in StringIO(robots_text):+ if line.lstrip()[:8].lower() == "sitemap:":+ url = line.partition(":")[2].strip()+ yield urljoin(base_url or "", url)diff --git a/scrapy/utils/_template.py b/scrapy/utils/_template.pynew file mode 100644index 000000000..55c006b39--- /dev/null+++ b/scrapy/utils/_template.py@@ -0,0 +1,39 @@+from __future__ import annotations++import re+import string+from pathlib import Path+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from os import PathLike+++def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:+ path_obj = Path(path)+ raw = path_obj.read_text("utf8")++ content = string.Template(raw).substitute(**kwargs)++ render_path = path_obj.with_suffix("") if path_obj.suffix == ".tmpl" else path_obj++ if path_obj.suffix == ".tmpl":+ path_obj.rename(render_path)++ render_path.write_text(content, "utf8")+++CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")+++def string_camelcase(string: str) -> str:+ """Convert a word to its CamelCase version and remove invalid chars++ >>> string_camelcase('lost-pound')+ 'LostPound'++ >>> string_camelcase('missing_images')+ 'MissingImages'++ """+ return CAMELCASE_INVALID_CHARS.sub("", string.title())diff --git a/scrapy/utils/display.py b/scrapy/utils/display.pyindex 08e89dc0d..88d219840 100644--- a/scrapy/utils/display.py+++ b/scrapy/utils/display.py@@ -1,51 +1,16 @@-"""-pprint and pformat wrappers with colorization support-"""--import ctypes-import platform-import sys-from pprint import pformat as pformat_-from typing import Any--from packaging.version import Version as parse_version---def _enable_windows_terminal_processing() -> bool:- # https://stackoverflow.com/a/36760881- kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]- return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))---def _tty_supports_color() -> bool:- if sys.platform != "win32":- return True-- if parse_version(platform.version()) < parse_version("10.0.14393"):- return True-- # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal- # processing is enabled.- return _enable_windows_terminal_processing()---def _colorize(text: str, colorize: bool = True) -> str:- # pylint: disable=no-name-in-module- if not colorize or not sys.stdout.isatty() or not _tty_supports_color():- return text- try:- from pygments import highlight # noqa: PLC0415- except ImportError:- return text- from pygments.formatters import TerminalFormatter # noqa: PLC0415- from pygments.lexers import PythonLexer # noqa: PLC0415-- return highlight(text, PythonLexer(), TerminalFormatter())---def pformat(obj: Any, *args: Any, **kwargs: Any) -> str:- return _colorize(pformat_(obj), kwargs.pop("colorize", True))---def pprint(obj: Any, *args: Any, **kwargs: Any) -> None:- print(pformat(obj, *args, **kwargs))+# pragma: no file cover+import warnings++from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils._colorize import pformat, pprint++warnings.warn(+ "The scrapy.utils.display module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)++__all__ = [+ "pformat",+ "pprint",+]diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.pyindex f08e5303d..5bed01a3a 100644--- a/scrapy/utils/ftp.py+++ b/scrapy/utils/ftp.py@@ -1,53 +1,16 @@-import posixpath-from contextlib import closing-from ftplib import FTP, FTP_TLS, error_perm-from posixpath import dirname-from ssl import create_default_context-from typing import IO+# pragma: no file cover+import warnings+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils._ftp import ftp_makedirs_cwd, ftp_store_file-def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:- """Set the current directory of the FTP connection given in the ``ftp``- argument (as a ftplib.FTP object), creating all parent directories if they- don't exist. The ftplib.FTP object must be already connected and logged in.- """- try:- ftp.cwd(path)- except error_perm:- ftp_makedirs_cwd(ftp, dirname(path), False)- ftp.mkd(path)- if first_call:- ftp.cwd(path)+warnings.warn(+ "The scrapy.utils.ftp module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)--def ftp_store_file(- *,- path: str,- file: IO[bytes],- host: str,- port: int,- username: str,- password: str,- use_active_mode: bool = False,- overwrite: bool = True,- tls: bool = False,-) -> None:- """Opens a FTP connection with passed credentials, sets current directory- to the directory extracted from given path, then uploads the file to server.-- If *tls* is ``True``, the connection is secured with TLS (FTPS), and the- certificate of the server is verified.- """- ftp = FTP_TLS(context=create_default_context()) if tls else FTP()- with ftp, closing(file):- ftp.connect(host, port)- ftp.login(username, password)- if isinstance(ftp, FTP_TLS):- ftp.prot_p()- if use_active_mode:- ftp.set_pasv(False)- file.seek(0)- dirname, filename = posixpath.split(path)- ftp_makedirs_cwd(ftp, dirname)- command = "STOR" if overwrite else "APPE"- ftp.storbinary(f"{command} {filename}", file)+__all__ = [+ "ftp_makedirs_cwd",+ "ftp_store_file",+]diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex bb4254f0c..1d56ff5e1 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -16,8 +16,8 @@ from twisted.python.failure import Failureimport scrapyfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.settings import Settings+from scrapy.utils._colorize import tty_supports_colorfrom scrapy.utils._versions import get_versions-from scrapy.utils.display import _tty_supports_colorif TYPE_CHECKING:from types import TracebackType@@ -187,7 +187,7 @@ def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Formand not isinstance(handler, logging.FileHandler)and settings.getbool("LOG_COLOR")and handler.stream.isatty()- and _tty_supports_color()+ and tty_supports_color()):try:from colorlog import ColoredFormatter # noqa: PLC0415diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.pyindex 4ad8e5741..bd9048bd6 100644--- a/scrapy/utils/ossignal.py+++ b/scrapy/utils/ossignal.py@@ -1,33 +1,21 @@-from __future__ import annotations+# pragma: no file cover+import warnings-import signal-from collections.abc import Callable-from types import FrameType-from typing import Any, TypeAlias--# copy of _HANDLER from typeshed/stdlib/signal.pyi-SignalHandlerT: TypeAlias = (- Callable[[int, FrameType | None], Any] | int | signal.Handlers | None+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils._ossignals import (+ SignalHandlerT,+ install_shutdown_handlers,+ signal_names,)-signal_names: dict[int, str] = {member.value: member.name for member in signal.Signals}-+warnings.warn(+ "The scrapy.utils.ossignal module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)-def install_shutdown_handlers(- function: SignalHandlerT, override_sigint: bool = True-) -> None:- """Install the given function as a signal handler for all common shutdown- signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the- SIGINT handler won't be installed if there is already a handler in place- (e.g. Pdb)- """- signal.signal(signal.SIGTERM, function)- if (- signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable- == signal.default_int_handler- or override_sigint- ):- signal.signal(signal.SIGINT, function)- # Catch Ctrl-Break in windows- if hasattr(signal, "SIGBREAK"):- signal.signal(signal.SIGBREAK, function)+__all__ = [+ "SignalHandlerT",+ "install_shutdown_handlers",+ "signal_names",+]diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex 03b7bf3b1..cc66f5d1b 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -1,136 +1,16 @@-"""-Module for processing Sitemaps.--Note: The main purpose of this module is to provide support for the-SitemapSpider, its API is subject to change without notice.-"""--from __future__ import annotations-+# pragma: no file coverimport warnings-from io import BytesIO, StringIO-from typing import TYPE_CHECKING, Any-from urllib.parse import urljoin--import lxml.etreefrom scrapy.exceptions import ScrapyDeprecationWarning--if TYPE_CHECKING:- from collections.abc import Iterable, Iterator---class Sitemap:- """Class to parse Sitemap (type=urlset) and Sitemap Index- (type=sitemapindex) files"""-- __slots__ = ("type", "xmliter")-- def __init__(self, xmltext: str | bytes):- if isinstance(xmltext, str):- warnings.warn(- "Passing `str` type as `xmltext` is deprecated, use `bytes`",- ScrapyDeprecationWarning,- stacklevel=2,- )- xmltext = xmltext.encode()-- self.xmliter = lxml.etree.iterparse(- BytesIO(xmltext),- recover=True,- remove_comments=True,- resolve_entities=False,- remove_blank_text=True,- collect_ids=False,- remove_pis=True,- events=("start", "end"),- )- _, root = next(self.xmliter)- self.type = self._get_tag_name(root)-- def __iter__(self) -> Iterator[dict[str, Any]]:- for event, elem in self.xmliter:- if event == "start":- continue-- if self._get_tag_name(elem) not in {"url", "sitemap"}:- continue-- if d := self._process_sitemap_element(elem):- yield d-- def _process_sitemap_element(- self, elem: lxml.etree._Element- ) -> dict[str, Any] | None:- d: dict[str, Any] = {}- alternate: list[str] = []- has_loc = False-- for el in elem:- try:- tag_name = self._get_tag_name(el)- if not tag_name:- continue-- if tag_name == "link":- if href := el.get("href"):- alternate.append(href)- else:- d[tag_name] = el.text.strip() if el.text else ""- if not has_loc and tag_name == "loc":- has_loc = True- finally:- el.clear()- elem.clear()- parent = elem.getparent()- if parent is not None:- while elem.getprevious() is not None:- del parent[0]-- if not has_loc:- return None-- if alternate:- d["alternate"] = alternate-- return d-- @staticmethod- def _get_tag_name(elem: lxml.etree._Element) -> str:- tag = elem.tag- if not isinstance(tag, str):- return ""- _, _, localname = tag.partition("}")- return localname or tag---def sitemap_urls_from_robots(- robots_text: str | bytes,- base_url: str | None = None,-) -> Iterable[str]:- if isinstance(robots_text, bytes):- for line in BytesIO(robots_text):- if line.lstrip()[:8].lower() == b"sitemap:":- try:- url = line.partition(b":")[2].strip().decode()- except UnicodeDecodeError:- continue- yield urljoin(base_url or "", url)-- else:- warnings.warn(- "Passing `str` type as `robots_text` is deprecated, use `bytes`",- ScrapyDeprecationWarning,- stacklevel=2,- )- yield from _sitemap_urls_from_robots_str(robots_text, base_url)---def _sitemap_urls_from_robots_str(- robots_text: str,- base_url: str | None = None,-) -> Iterable[str]:- for line in StringIO(robots_text):- if line.lstrip()[:8].lower() == "sitemap:":- url = line.partition(":")[2].strip()- yield urljoin(base_url or "", url)+from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robots++warnings.warn(+ "The scrapy.utils.sitemap module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)++__all__ = [+ "Sitemap",+ "sitemap_urls_from_robots",+]diff --git a/scrapy/utils/template.py b/scrapy/utils/template.pyindex 977d5a42a..cccaba9a5 100644--- a/scrapy/utils/template.py+++ b/scrapy/utils/template.py@@ -1,41 +1,21 @@-"""Helper functions for working with templates"""--from __future__ import annotations--import re-import string-from pathlib import Path-from typing import TYPE_CHECKING, Any--if TYPE_CHECKING:- from os import PathLike---def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:- path_obj = Path(path)- raw = path_obj.read_text("utf8")-- content = string.Template(raw).substitute(**kwargs)-- render_path = path_obj.with_suffix("") if path_obj.suffix == ".tmpl" else path_obj-- if path_obj.suffix == ".tmpl":- path_obj.rename(render_path)-- render_path.write_text(content, "utf8")---CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")---def string_camelcase(string: str) -> str:- """Convert a word to its CamelCase version and remove invalid chars-- >>> string_camelcase('lost-pound')- 'LostPound'-- >>> string_camelcase('missing_images')- 'MissingImages'-- """- return CAMELCASE_INVALID_CHARS.sub("", string.title())+# pragma: no file cover+import warnings++from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils._template import (+ CAMELCASE_INVALID_CHARS,+ render_templatefile,+ string_camelcase,+)++warnings.warn(+ "The scrapy.utils.template module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)++__all__ = [+ "CAMELCASE_INVALID_CHARS",+ "render_templatefile",+ "string_camelcase",+]diff --git a/tests/ignores.txt b/tests/ignores.txtindex 75ae2d141..6893acebc 100644--- a/tests/ignores.txt+++ b/tests/ignores.txt@@ -4,7 +4,12 @@ scrapy/interfaces.pyscrapy/mail.pyscrapy/utils/benchserver.pyscrapy/utils/console.py+scrapy/utils/display.py+scrapy/utils/ftp.pyscrapy/utils/gz.pyscrapy/utils/job.py+scrapy/utils/ossignal.py+scrapy/utils/sitemap.pyscrapy/utils/ssl.py+scrapy/utils/template.pyscrapy/utils/versions.pydiff --git a/tests/test_utils_display.py b/tests/test_utils_display.pyindex b8d15bcba..9a545f9c2 100644--- a/tests/test_utils_display.py+++ b/tests/test_utils_display.py@@ -8,7 +8,7 @@ from unittest import mockimport pytest-from scrapy.utils.display import _enable_windows_terminal_processing, pformat, pprint+from scrapy.utils._colorize import _enable_windows_terminal_processing, pformat, pprintif TYPE_CHECKING:from collections.abc import Mapping, Sequence@@ -59,7 +59,7 @@ def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:@mock.patch("sys.platform", "win32")[email protected]("scrapy.utils.display._enable_windows_terminal_processing")[email protected]("scrapy.utils._colorize._enable_windows_terminal_processing")@mock.patch("platform.version")@mock.patch("sys.stdout.isatty")def test_pformat_windows_no_terminal_processing(@@ -72,7 +72,7 @@ def test_pformat_windows_no_terminal_processing(@mock.patch("sys.platform", "win32")[email protected]("scrapy.utils.display._enable_windows_terminal_processing")[email protected]("scrapy.utils._colorize._enable_windows_terminal_processing")@mock.patch("platform.version")@mock.patch("sys.stdout.isatty")def test_pformat_windows(diff --git a/tests/test_utils_log.py b/tests/test_utils_log.pyindex ce8da0b39..825d281d9 100644--- a/tests/test_utils_log.py+++ b/tests/test_utils_log.py@@ -163,7 +163,7 @@ class TestGetFormatter:self, monkeypatch: pytest.MonkeyPatch) -> None:pytest.importorskip("colorlog")- monkeypatch.setattr("scrapy.utils.log._tty_supports_color", lambda: True)+ monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)stream = _TTYStringIO()handler = logging.StreamHandler(stream)handler.setFormatter(_get_formatter(handler, self._settings()))@@ -186,7 +186,7 @@ class TestGetFormatter:def test_plain_when_log_color_disabled(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr("scrapy.utils.log._tty_supports_color", lambda: True)+ monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)handler = logging.StreamHandler(_TTYStringIO())formatter = _get_formatter(handler, self._settings(LOG_COLOR=False))assert type(formatter) is logging.Formatter@@ -202,7 +202,7 @@ class TestGetFormatter:def test_plain_when_colorlog_not_installed(self, monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr("scrapy.utils.log._tty_supports_color", lambda: True)+ monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)monkeypatch.setitem(sys.modules, "colorlog", None)handler = logging.StreamHandler(_TTYStringIO())formatter = _get_formatter(handler, self._settings())diff --git a/tests/test_utils_ossignal.py b/tests/test_utils_ossignal.pyindex 4c8f7d0b4..e38a853b6 100644--- a/tests/test_utils_ossignal.py+++ b/tests/test_utils_ossignal.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Anyimport pytest-from scrapy.utils.ossignal import install_shutdown_handlers+from scrapy.utils._ossignals import install_shutdown_handlersif TYPE_CHECKING:from collections.abc import Generatordiff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.pyindex dda13006d..571ad4075 100644--- a/tests/test_utils_sitemap.py+++ b/tests/test_utils_sitemap.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport pytestfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots+from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robotsdef test_sitemap():diff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex 076edf3db..fed314974 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from scrapy.utils.template import render_templatefile+from scrapy.utils._template import render_templatefileif TYPE_CHECKING:from pathlib import Path