Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate remaining planned scrapy.utils (#8146)

Andrey Rakhmatullinf67c87db61e5
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index c32a01395..79c25cd98 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -12,7 +12,7 @@ from scrapy.commands import ScrapyCommand
from scrapy.commands.edit import _edit_file
from scrapy.exceptions import UsageError
from scrapy.spiderloader import get_spider_loader
-from scrapy.utils.template import render_templatefile, string_camelcase
+from scrapy.utils._template import render_templatefile, string_camelcase
if TYPE_CHECKING:
import argparse
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index d50075fb5..4edadfa58 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -13,7 +13,7 @@ from w3lib.url import is_url
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
-from scrapy.utils import display
+from scrapy.utils import _colorize
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import _schedule_coro, aiter_errback, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
@@ -170,7 +170,7 @@ class Command(BaseRunSpiderCommand):
items = self.items.get(lvl, [])
print("# Scraped Items ", "-" * 60)
- display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
+ _colorize.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
def print_requests(self, lvl: int | None = None, colour: bool = True) -> None:
if lvl is not None:
@@ -181,7 +181,7 @@ class Command(BaseRunSpiderCommand):
requests = []
print("# Requests ", "-" * 65)
- display.pprint(requests, colorize=colour)
+ _colorize.pprint(requests, colorize=colour)
def print_results(self, opts: argparse.Namespace) -> None:
colour = not opts.nocolour
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 5661a0a48..497c24e9e 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
-from scrapy.utils.template import render_templatefile, string_camelcase
+from scrapy.utils._template import render_templatefile, string_camelcase
if TYPE_CHECKING:
import argparse
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 0c2830c4c..1f40d563b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -20,6 +20,7 @@ from scrapy.extension import ExtensionManager
from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
+from scrapy.utils._ossignals import install_shutdown_handlers, signal_names
from scrapy.utils._stopmode import _normalize_stop_mode, _StopMode
from scrapy.utils.defer import _DEFER_DELAY, deferred_from_coro, ensure_awaitable
from scrapy.utils.log import (
@@ -30,7 +31,6 @@ from scrapy.utils.log import (
log_scrapy_info,
)
from scrapy.utils.misc import build_from_crawler, load_object
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import (
_asyncio_reactor_path,
_is_asyncio_reactor_installed,
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 320106705..d3dc03c53 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -27,11 +27,11 @@ from zope.interface import Interface
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
+from scrapy.utils._ftp import ftp_store_file
from scrapy.utils.asyncio import is_asyncio_available, run_in_thread
from scrapy.utils.boto import _get_max_pool_connections
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
-from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 9d67e98bf..b3c44dee9 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -34,11 +34,11 @@ from scrapy.pipelines.media import (
MediaPipeline,
_MediaRequestFiltered,
)
+from scrapy.utils._ftp import ftp_store_file
from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available
from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
-from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import to_bytes
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 92acef9d3..4e73d247d 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -14,7 +14,7 @@ from scrapy.utils._compression import (
gunzip,
gzip_magic_number,
)
-from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
+from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robots
if TYPE_CHECKING:
# typing.Self requires Python 3.11
diff --git a/scrapy/utils/_colorize.py b/scrapy/utils/_colorize.py
new file mode 100644
index 000000000..53aa685fc
--- /dev/null
+++ b/scrapy/utils/_colorize.py
@@ -0,0 +1,47 @@
+import ctypes
+import platform
+import sys
+from pprint import pformat as pformat_
+from typing import Any
+
+from packaging.version import Version as parse_version
+
+
+def _enable_windows_terminal_processing() -> bool:
+ # https://stackoverflow.com/a/36760881
+ kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
+ return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
+
+
+def tty_supports_color() -> bool:
+ if sys.platform != "win32":
+ return True
+
+ if parse_version(platform.version()) < parse_version("10.0.14393"):
+ return True
+
+ # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
+ # processing is enabled.
+ return _enable_windows_terminal_processing()
+
+
+def _colorize(text: str, colorize: bool = True) -> str:
+ # pylint: disable=no-name-in-module
+ if not colorize or not sys.stdout.isatty() or not tty_supports_color():
+ return text
+ try:
+ from pygments import highlight # noqa: PLC0415
+ except ImportError:
+ return text
+ from pygments.formatters import TerminalFormatter # noqa: PLC0415
+ from pygments.lexers import PythonLexer # noqa: PLC0415
+
+ return highlight(text, PythonLexer(), TerminalFormatter())
+
+
+def pformat(obj: object, *args: Any, **kwargs: Any) -> str:
+ return _colorize(pformat_(obj), kwargs.pop("colorize", True))
+
+
+def pprint(obj: object, *args: Any, **kwargs: Any) -> None:
+ print(pformat(obj, *args, **kwargs))
diff --git a/scrapy/utils/_ftp.py b/scrapy/utils/_ftp.py
new file mode 100644
index 000000000..f08e5303d
--- /dev/null
+++ b/scrapy/utils/_ftp.py
@@ -0,0 +1,53 @@
+import posixpath
+from contextlib import closing
+from ftplib import FTP, FTP_TLS, error_perm
+from posixpath import dirname
+from ssl import create_default_context
+from typing import IO
+
+
+def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
+ """Set the current directory of the FTP connection given in the ``ftp``
+ argument (as a ftplib.FTP object), creating all parent directories if they
+ don't exist. The ftplib.FTP object must be already connected and logged in.
+ """
+ try:
+ ftp.cwd(path)
+ except error_perm:
+ ftp_makedirs_cwd(ftp, dirname(path), False)
+ ftp.mkd(path)
+ if first_call:
+ ftp.cwd(path)
+
+
+def ftp_store_file(
+ *,
+ path: str,
+ file: IO[bytes],
+ host: str,
+ port: int,
+ username: str,
+ password: str,
+ use_active_mode: bool = False,
+ overwrite: bool = True,
+ tls: bool = False,
+) -> None:
+ """Opens a FTP connection with passed credentials, sets current directory
+ to the directory extracted from given path, then uploads the file to server.
+
+ If *tls* is ``True``, the connection is secured with TLS (FTPS), and the
+ certificate of the server is verified.
+ """
+ ftp = FTP_TLS(context=create_default_context()) if tls else FTP()
+ with ftp, closing(file):
+ ftp.connect(host, port)
+ ftp.login(username, password)
+ if isinstance(ftp, FTP_TLS):
+ ftp.prot_p()
+ if use_active_mode:
+ ftp.set_pasv(False)
+ file.seek(0)
+ dirname, filename = posixpath.split(path)
+ ftp_makedirs_cwd(ftp, dirname)
+ command = "STOR" if overwrite else "APPE"
+ ftp.storbinary(f"{command} {filename}", file)
diff --git a/scrapy/utils/_ossignals.py b/scrapy/utils/_ossignals.py
new file mode 100644
index 000000000..4ad8e5741
--- /dev/null
+++ b/scrapy/utils/_ossignals.py
@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import signal
+from collections.abc import Callable
+from types import FrameType
+from typing import Any, TypeAlias
+
+# copy of _HANDLER from typeshed/stdlib/signal.pyi
+SignalHandlerT: TypeAlias = (
+ Callable[[int, FrameType | None], Any] | int | signal.Handlers | None
+)
+
+signal_names: dict[int, str] = {member.value: member.name for member in signal.Signals}
+
+
+def install_shutdown_handlers(
+ function: SignalHandlerT, override_sigint: bool = True
+) -> None:
+ """Install the given function as a signal handler for all common shutdown
+ signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the
+ SIGINT handler won't be installed if there is already a handler in place
+ (e.g. Pdb)
+ """
+ signal.signal(signal.SIGTERM, function)
+ if (
+ signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable
+ == signal.default_int_handler
+ or override_sigint
+ ):
+ signal.signal(signal.SIGINT, function)
+ # Catch Ctrl-Break in windows
+ if hasattr(signal, "SIGBREAK"):
+ signal.signal(signal.SIGBREAK, function)
diff --git a/scrapy/utils/_sitemap.py b/scrapy/utils/_sitemap.py
new file mode 100644
index 000000000..2edf2d5ef
--- /dev/null
+++ b/scrapy/utils/_sitemap.py
@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import warnings
+from io import BytesIO, StringIO
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urljoin
+
+import lxml.etree
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Iterator
+
+
+class Sitemap:
+ """Class to parse Sitemap (type=urlset) and Sitemap Index
+ (type=sitemapindex) files"""
+
+ __slots__ = ("type", "xmliter")
+
+ def __init__(self, xmltext: str | bytes):
+ if isinstance(xmltext, str):
+ warnings.warn(
+ "Passing `str` type as `xmltext` is deprecated, use `bytes`",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ xmltext = xmltext.encode()
+
+ self.xmliter = lxml.etree.iterparse(
+ BytesIO(xmltext),
+ recover=True,
+ remove_comments=True,
+ resolve_entities=False,
+ remove_blank_text=True,
+ collect_ids=False,
+ remove_pis=True,
+ events=("start", "end"),
+ )
+ _, root = next(self.xmliter)
+ self.type = self._get_tag_name(root)
+
+ def __iter__(self) -> Iterator[dict[str, Any]]:
+ for event, elem in self.xmliter:
+ if event == "start":
+ continue
+
+ if self._get_tag_name(elem) not in {"url", "sitemap"}:
+ continue
+
+ if d := self._process_sitemap_element(elem):
+ yield d
+
+ def _process_sitemap_element(
+ self, elem: lxml.etree._Element
+ ) -> dict[str, Any] | None:
+ d: dict[str, Any] = {}
+ alternate: list[str] = []
+ has_loc = False
+
+ for el in elem:
+ try:
+ tag_name = self._get_tag_name(el)
+ if not tag_name:
+ continue
+
+ if tag_name == "link":
+ if href := el.get("href"):
+ alternate.append(href)
+ else:
+ d[tag_name] = el.text.strip() if el.text else ""
+ if not has_loc and tag_name == "loc":
+ has_loc = True
+ finally:
+ el.clear()
+ elem.clear()
+ parent = elem.getparent()
+ if parent is not None:
+ while elem.getprevious() is not None:
+ del parent[0]
+
+ if not has_loc:
+ return None
+
+ if alternate:
+ d["alternate"] = alternate
+
+ return d
+
+ @staticmethod
+ def _get_tag_name(elem: lxml.etree._Element) -> str:
+ tag = elem.tag
+ if not isinstance(tag, str):
+ return ""
+ _, _, localname = tag.partition("}")
+ return localname or tag
+
+
+def sitemap_urls_from_robots(
+ robots_text: str | bytes,
+ base_url: str | None = None,
+) -> Iterable[str]:
+ if isinstance(robots_text, bytes):
+ for line in BytesIO(robots_text):
+ if line.lstrip()[:8].lower() == b"sitemap:":
+ try:
+ url = line.partition(b":")[2].strip().decode()
+ except UnicodeDecodeError:
+ continue
+ yield urljoin(base_url or "", url)
+
+ else:
+ warnings.warn(
+ "Passing `str` type as `robots_text` is deprecated, use `bytes`",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ yield from _sitemap_urls_from_robots_str(robots_text, base_url)
+
+
+def _sitemap_urls_from_robots_str(
+ robots_text: str,
+ base_url: str | None = None,
+) -> Iterable[str]:
+ for line in StringIO(robots_text):
+ if line.lstrip()[:8].lower() == "sitemap:":
+ url = line.partition(":")[2].strip()
+ yield urljoin(base_url or "", url)
diff --git a/scrapy/utils/_template.py b/scrapy/utils/_template.py
new file mode 100644
index 000000000..55c006b39
--- /dev/null
+++ b/scrapy/utils/_template.py
@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import re
+import string
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from os import PathLike
+
+
+def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:
+ path_obj = Path(path)
+ raw = path_obj.read_text("utf8")
+
+ content = string.Template(raw).substitute(**kwargs)
+
+ render_path = path_obj.with_suffix("") if path_obj.suffix == ".tmpl" else path_obj
+
+ if path_obj.suffix == ".tmpl":
+ path_obj.rename(render_path)
+
+ render_path.write_text(content, "utf8")
+
+
+CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")
+
+
+def string_camelcase(string: str) -> str:
+ """Convert a word to its CamelCase version and remove invalid chars
+
+ >>> string_camelcase('lost-pound')
+ 'LostPound'
+
+ >>> string_camelcase('missing_images')
+ 'MissingImages'
+
+ """
+ return CAMELCASE_INVALID_CHARS.sub("", string.title())
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 08e89dc0d..88d219840 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -1,51 +1,16 @@
-"""
-pprint and pformat wrappers with colorization support
-"""
-
-import ctypes
-import platform
-import sys
-from pprint import pformat as pformat_
-from typing import Any
-
-from packaging.version import Version as parse_version
-
-
-def _enable_windows_terminal_processing() -> bool:
- # https://stackoverflow.com/a/36760881
- kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
- return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
-
-
-def _tty_supports_color() -> bool:
- if sys.platform != "win32":
- return True
-
- if parse_version(platform.version()) < parse_version("10.0.14393"):
- return True
-
- # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
- # processing is enabled.
- return _enable_windows_terminal_processing()
-
-
-def _colorize(text: str, colorize: bool = True) -> str:
- # pylint: disable=no-name-in-module
- if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
- return text
- try:
- from pygments import highlight # noqa: PLC0415
- except ImportError:
- return text
- from pygments.formatters import TerminalFormatter # noqa: PLC0415
- from pygments.lexers import PythonLexer # noqa: PLC0415
-
- return highlight(text, PythonLexer(), TerminalFormatter())
-
-
-def pformat(obj: Any, *args: Any, **kwargs: Any) -> str:
- return _colorize(pformat_(obj), kwargs.pop("colorize", True))
-
-
-def pprint(obj: Any, *args: Any, **kwargs: Any) -> None:
- print(pformat(obj, *args, **kwargs))
+# pragma: no file cover
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._colorize import pformat, pprint
+
+warnings.warn(
+ "The scrapy.utils.display module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = [
+ "pformat",
+ "pprint",
+]
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index f08e5303d..5bed01a3a 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -1,53 +1,16 @@
-import posixpath
-from contextlib import closing
-from ftplib import FTP, FTP_TLS, error_perm
-from posixpath import dirname
-from ssl import create_default_context
-from typing import IO
+# pragma: no file cover
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._ftp import ftp_makedirs_cwd, ftp_store_file
-def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
- """Set the current directory of the FTP connection given in the ``ftp``
- argument (as a ftplib.FTP object), creating all parent directories if they
- don't exist. The ftplib.FTP object must be already connected and logged in.
- """
- try:
- ftp.cwd(path)
- except error_perm:
- ftp_makedirs_cwd(ftp, dirname(path), False)
- ftp.mkd(path)
- if first_call:
- ftp.cwd(path)
+warnings.warn(
+ "The scrapy.utils.ftp module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
-
-def ftp_store_file(
- *,
- path: str,
- file: IO[bytes],
- host: str,
- port: int,
- username: str,
- password: str,
- use_active_mode: bool = False,
- overwrite: bool = True,
- tls: bool = False,
-) -> None:
- """Opens a FTP connection with passed credentials, sets current directory
- to the directory extracted from given path, then uploads the file to server.
-
- If *tls* is ``True``, the connection is secured with TLS (FTPS), and the
- certificate of the server is verified.
- """
- ftp = FTP_TLS(context=create_default_context()) if tls else FTP()
- with ftp, closing(file):
- ftp.connect(host, port)
- ftp.login(username, password)
- if isinstance(ftp, FTP_TLS):
- ftp.prot_p()
- if use_active_mode:
- ftp.set_pasv(False)
- file.seek(0)
- dirname, filename = posixpath.split(path)
- ftp_makedirs_cwd(ftp, dirname)
- command = "STOR" if overwrite else "APPE"
- ftp.storbinary(f"{command} {filename}", file)
+__all__ = [
+ "ftp_makedirs_cwd",
+ "ftp_store_file",
+]
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index bb4254f0c..1d56ff5e1 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -16,8 +16,8 @@ from twisted.python.failure import Failure
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
+from scrapy.utils._colorize import tty_supports_color
from scrapy.utils._versions import get_versions
-from scrapy.utils.display import _tty_supports_color
if TYPE_CHECKING:
from types import TracebackType
@@ -187,7 +187,7 @@ def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Form
and not isinstance(handler, logging.FileHandler)
and settings.getbool("LOG_COLOR")
and handler.stream.isatty()
- and _tty_supports_color()
+ and tty_supports_color()
):
try:
from colorlog import ColoredFormatter # noqa: PLC0415
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index 4ad8e5741..bd9048bd6 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,33 +1,21 @@
-from __future__ import annotations
+# pragma: no file cover
+import warnings
-import signal
-from collections.abc import Callable
-from types import FrameType
-from typing import Any, TypeAlias
-
-# copy of _HANDLER from typeshed/stdlib/signal.pyi
-SignalHandlerT: TypeAlias = (
- Callable[[int, FrameType | None], Any] | int | signal.Handlers | None
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._ossignals import (
+ SignalHandlerT,
+ install_shutdown_handlers,
+ signal_names,
)
-signal_names: dict[int, str] = {member.value: member.name for member in signal.Signals}
-
+warnings.warn(
+ "The scrapy.utils.ossignal module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
-def install_shutdown_handlers(
- function: SignalHandlerT, override_sigint: bool = True
-) -> None:
- """Install the given function as a signal handler for all common shutdown
- signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the
- SIGINT handler won't be installed if there is already a handler in place
- (e.g. Pdb)
- """
- signal.signal(signal.SIGTERM, function)
- if (
- signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable
- == signal.default_int_handler
- or override_sigint
- ):
- signal.signal(signal.SIGINT, function)
- # Catch Ctrl-Break in windows
- if hasattr(signal, "SIGBREAK"):
- signal.signal(signal.SIGBREAK, function)
+__all__ = [
+ "SignalHandlerT",
+ "install_shutdown_handlers",
+ "signal_names",
+]
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 03b7bf3b1..cc66f5d1b 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -1,136 +1,16 @@
-"""
-Module for processing Sitemaps.
-
-Note: The main purpose of this module is to provide support for the
-SitemapSpider, its API is subject to change without notice.
-"""
-
-from __future__ import annotations
-
+# pragma: no file cover
import warnings
-from io import BytesIO, StringIO
-from typing import TYPE_CHECKING, Any
-from urllib.parse import urljoin
-
-import lxml.etree
from scrapy.exceptions import ScrapyDeprecationWarning
-
-if TYPE_CHECKING:
- from collections.abc import Iterable, Iterator
-
-
-class Sitemap:
- """Class to parse Sitemap (type=urlset) and Sitemap Index
- (type=sitemapindex) files"""
-
- __slots__ = ("type", "xmliter")
-
- def __init__(self, xmltext: str | bytes):
- if isinstance(xmltext, str):
- warnings.warn(
- "Passing `str` type as `xmltext` is deprecated, use `bytes`",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- xmltext = xmltext.encode()
-
- self.xmliter = lxml.etree.iterparse(
- BytesIO(xmltext),
- recover=True,
- remove_comments=True,
- resolve_entities=False,
- remove_blank_text=True,
- collect_ids=False,
- remove_pis=True,
- events=("start", "end"),
- )
- _, root = next(self.xmliter)
- self.type = self._get_tag_name(root)
-
- def __iter__(self) -> Iterator[dict[str, Any]]:
- for event, elem in self.xmliter:
- if event == "start":
- continue
-
- if self._get_tag_name(elem) not in {"url", "sitemap"}:
- continue
-
- if d := self._process_sitemap_element(elem):
- yield d
-
- def _process_sitemap_element(
- self, elem: lxml.etree._Element
- ) -> dict[str, Any] | None:
- d: dict[str, Any] = {}
- alternate: list[str] = []
- has_loc = False
-
- for el in elem:
- try:
- tag_name = self._get_tag_name(el)
- if not tag_name:
- continue
-
- if tag_name == "link":
- if href := el.get("href"):
- alternate.append(href)
- else:
- d[tag_name] = el.text.strip() if el.text else ""
- if not has_loc and tag_name == "loc":
- has_loc = True
- finally:
- el.clear()
- elem.clear()
- parent = elem.getparent()
- if parent is not None:
- while elem.getprevious() is not None:
- del parent[0]
-
- if not has_loc:
- return None
-
- if alternate:
- d["alternate"] = alternate
-
- return d
-
- @staticmethod
- def _get_tag_name(elem: lxml.etree._Element) -> str:
- tag = elem.tag
- if not isinstance(tag, str):
- return ""
- _, _, localname = tag.partition("}")
- return localname or tag
-
-
-def sitemap_urls_from_robots(
- robots_text: str | bytes,
- base_url: str | None = None,
-) -> Iterable[str]:
- if isinstance(robots_text, bytes):
- for line in BytesIO(robots_text):
- if line.lstrip()[:8].lower() == b"sitemap:":
- try:
- url = line.partition(b":")[2].strip().decode()
- except UnicodeDecodeError:
- continue
- yield urljoin(base_url or "", url)
-
- else:
- warnings.warn(
- "Passing `str` type as `robots_text` is deprecated, use `bytes`",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- yield from _sitemap_urls_from_robots_str(robots_text, base_url)
-
-
-def _sitemap_urls_from_robots_str(
- robots_text: str,
- base_url: str | None = None,
-) -> Iterable[str]:
- for line in StringIO(robots_text):
- if line.lstrip()[:8].lower() == "sitemap:":
- url = line.partition(":")[2].strip()
- yield urljoin(base_url or "", url)
+from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robots
+
+warnings.warn(
+ "The scrapy.utils.sitemap module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = [
+ "Sitemap",
+ "sitemap_urls_from_robots",
+]
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 977d5a42a..cccaba9a5 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -1,41 +1,21 @@
-"""Helper functions for working with templates"""
-
-from __future__ import annotations
-
-import re
-import string
-from pathlib import Path
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
- from os import PathLike
-
-
-def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:
- path_obj = Path(path)
- raw = path_obj.read_text("utf8")
-
- content = string.Template(raw).substitute(**kwargs)
-
- render_path = path_obj.with_suffix("") if path_obj.suffix == ".tmpl" else path_obj
-
- if path_obj.suffix == ".tmpl":
- path_obj.rename(render_path)
-
- render_path.write_text(content, "utf8")
-
-
-CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]")
-
-
-def string_camelcase(string: str) -> str:
- """Convert a word to its CamelCase version and remove invalid chars
-
- >>> string_camelcase('lost-pound')
- 'LostPound'
-
- >>> string_camelcase('missing_images')
- 'MissingImages'
-
- """
- return CAMELCASE_INVALID_CHARS.sub("", string.title())
+# pragma: no file cover
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._template import (
+ CAMELCASE_INVALID_CHARS,
+ render_templatefile,
+ string_camelcase,
+)
+
+warnings.warn(
+ "The scrapy.utils.template module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = [
+ "CAMELCASE_INVALID_CHARS",
+ "render_templatefile",
+ "string_camelcase",
+]
diff --git a/tests/ignores.txt b/tests/ignores.txt
index 75ae2d141..6893acebc 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -4,7 +4,12 @@ scrapy/interfaces.py
scrapy/mail.py
scrapy/utils/benchserver.py
scrapy/utils/console.py
+scrapy/utils/display.py
+scrapy/utils/ftp.py
scrapy/utils/gz.py
scrapy/utils/job.py
+scrapy/utils/ossignal.py
+scrapy/utils/sitemap.py
scrapy/utils/ssl.py
+scrapy/utils/template.py
scrapy/utils/versions.py
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index b8d15bcba..9a545f9c2 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -8,7 +8,7 @@ from unittest import mock
import pytest
-from scrapy.utils.display import _enable_windows_terminal_processing, pformat, pprint
+from scrapy.utils._colorize import _enable_windows_terminal_processing, pformat, pprint
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
@@ -59,7 +59,7 @@ def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:
@mock.patch("sys.platform", "win32")
[email protected]("scrapy.utils.display._enable_windows_terminal_processing")
[email protected]("scrapy.utils._colorize._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows_no_terminal_processing(
@@ -72,7 +72,7 @@ def test_pformat_windows_no_terminal_processing(
@mock.patch("sys.platform", "win32")
[email protected]("scrapy.utils.display._enable_windows_terminal_processing")
[email protected]("scrapy.utils._colorize._enable_windows_terminal_processing")
@mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows(
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index ce8da0b39..825d281d9 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -163,7 +163,7 @@ class TestGetFormatter:
self, monkeypatch: pytest.MonkeyPatch
) -> None:
pytest.importorskip("colorlog")
- monkeypatch.setattr("scrapy.utils.log._tty_supports_color", lambda: True)
+ monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)
stream = _TTYStringIO()
handler = logging.StreamHandler(stream)
handler.setFormatter(_get_formatter(handler, self._settings()))
@@ -186,7 +186,7 @@ class TestGetFormatter:
def test_plain_when_log_color_disabled(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- monkeypatch.setattr("scrapy.utils.log._tty_supports_color", lambda: True)
+ monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)
handler = logging.StreamHandler(_TTYStringIO())
formatter = _get_formatter(handler, self._settings(LOG_COLOR=False))
assert type(formatter) is logging.Formatter
@@ -202,7 +202,7 @@ class TestGetFormatter:
def test_plain_when_colorlog_not_installed(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- monkeypatch.setattr("scrapy.utils.log._tty_supports_color", lambda: True)
+ monkeypatch.setattr("scrapy.utils.log.tty_supports_color", lambda: True)
monkeypatch.setitem(sys.modules, "colorlog", None)
handler = logging.StreamHandler(_TTYStringIO())
formatter = _get_formatter(handler, self._settings())
diff --git a/tests/test_utils_ossignal.py b/tests/test_utils_ossignal.py
index 4c8f7d0b4..e38a853b6 100644
--- a/tests/test_utils_ossignal.py
+++ b/tests/test_utils_ossignal.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any
import pytest
-from scrapy.utils.ossignal import install_shutdown_handlers
+from scrapy.utils._ossignals import install_shutdown_handlers
if TYPE_CHECKING:
from collections.abc import Generator
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index dda13006d..571ad4075 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
+from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robots
def test_sitemap():
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 076edf3db..fed314974 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from scrapy.utils.template import render_templatefile
+from scrapy.utils._template import render_templatefile
if TYPE_CHECKING:
from pathlib import Path