Scrapy, a fast high-level web crawling & scraping framework for Python.
Assorted docstring fixes (#7698)
Andrey Rakhmatullin00098cb596d0
diff --git a/scrapy/addons.py b/scrapy/addons.pyindex 2e12f8c8a..470be47c1 100644--- a/scrapy/addons.py+++ b/scrapy/addons.py@@ -64,7 +64,7 @@ class AddonManager::param settings: The :class:`~scrapy.settings.BaseSettings` object from \which to read the early add-on configuration- :type settings: :class:`~scrapy.settings.Settings`+ :type settings: :class:`~scrapy.settings.BaseSettings`"""for clspath in build_component_list(settings["ADDONS"]):addoncls = load_object(clspath)diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 598e8060e..d9c919db9 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -73,7 +73,7 @@ class ScrapyCommand(ABC):def long_desc(self) -> str:"""A long description of the command. Return short description when notavailable. It cannot contain newlines since contents will be formatted- by optparser which removes newlines and wraps text.+ by argparse which removes newlines and wraps text."""return self.short_desc()diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex c1796c723..a934cbbc7 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):instance.The purpose of this custom class is to provide a ``creatorForNetloc()``- method that returns a ``_ScrapyClientTLSOptions`` instance configured based- on TLS settings provided to the factory.+ method that returns:++ - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance+ configured based on TLS settings provided to the factory (when the+ certificate verification is disabled);+ - a result of ``optionsForClientTLS()`` called with those TLS settings+ (when the certificate verification is enabled)."""def __init__(diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 9064aa53a..07ff4a74e 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -2,9 +2,9 @@An asynchronous FTP file download handler for scrapy which somehow emulates an http response.FTP connection parameters are passed using the request meta field:-- ftp_user (required)-- ftp_password (required)-- ftp_passive (by default, enabled) sets FTP connection passive mode+- ftp_user (optional, falls back to FTP_USER)+- ftp_password (optional, falls back to FTP_PASSWORD)+- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode- ftp_local_filename- If not given, file data will come in the response.body, as a normal scrapy Response,which will imply that the entire file will be on memory.diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 1fc504c59..17dca5acb 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):self._disconnect_timeout: int = 1async def download_request(self, request: Request) -> Response:- """Return a deferred for the HTTP download"""if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no coverwarn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover@@ -283,7 +282,7 @@ def _tunnel_request_data(class _TunnelingAgent(Agent):- """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS+ """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPSdownloads. It may look strange that we have chosen to subclass Agent and notProxyAgent but consider that after the tunnel is opened the proxy istransparent to the client; thus the agent should behave like there is nodiff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.pyindex 7137a0f2b..aa55e29a0 100644--- a/scrapy/core/http2/agent.py+++ b/scrapy/core/http2/agent.py@@ -114,11 +114,7 @@ class H2ConnectionPool:d.errback(ResponseFailed(errors))def close_connections(self) -> None:- """Close all the HTTP/2 connections and remove them from pool-- Returns:- Deferred that fires when all connections have been closed- """+ """Close all the HTTP/2 connections and remove them from pool."""for conn in self._connections.values():assert conn.transport is not None # typingconn.transport.abortConnection()diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.pyindex 39703f976..7136e829e 100644--- a/scrapy/core/http2/protocol.py+++ b/scrapy/core/http2/protocol.py@@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):uri is used to verify that incoming client requests have correctbase URL.settings -- Scrapy project settings- conn_lost_deferred -- Deferred fires with the reason: Failure to notify+ conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notifythat connection was losttls_verbose_logging -- Whether to log TLS details"""@@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):def _handle_events(self, events: list[Event]) -> None:"""Private method which acts as a bridge between the events- received from the HTTP/2 data and IH2EventsHandler+ received from the HTTP/2 data and the handlers in this class.Arguments:events -- A list of events that the remote peer triggered by sending datadiff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 7217da942..82e31b90b 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -131,10 +131,10 @@ class Scheduler(BaseScheduler):(:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by:attr:`~scrapy.http.Request.priority`.- By default, a single, memory-based priority queue is used for all requests.- When using :setting:`JOBDIR`, a disk-based priority queue is also created,+ By default, memory-based priority queues are used for all requests.+ When using :setting:`JOBDIR`, disk-based priority queues are also created,and only unserializable requests are stored in the memory-based priority- queue. For a given priority value, requests in memory take precedence over+ queues. For a given priority value, requests in memory take precedence overrequests in disk.Each priority queue stores requests in separate internal queues, one per@@ -209,8 +209,8 @@ class Scheduler(BaseScheduler):-------------------------While pending requests are below the configured values of- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`- or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent+ :setting:`CONCURRENT_REQUESTS` or+ :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sentconcurrently.As a result, the first few requests of a crawl may not follow the desired@@ -342,7 +342,7 @@ class Scheduler(BaseScheduler):def open(self, spider: Spider) -> Deferred[None] | None:"""(1) initialize the memory queue- (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory+ (2) initialize the disk queue if the ``jobdir`` argument wasn't empty(3) return the result of the dupefilter's ``open`` method"""self.spider: Spider = spiderdiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 466ce656d..58e37ce5e 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -441,7 +441,7 @@ class Scraper:self, output: Any, response: Response | Failure) -> Deferred[None]:"""Process each Request/Item (given in the output parameter) returned- from the given spider.+ from the spider.Items are sent to the item pipelines, requests are scheduled."""@@ -451,7 +451,7 @@ class Scraper:self, output: Any, response: Response | Failure) -> None:"""Process each Request/Item (given in the output parameter) returned- from the given spider.+ from the spider.Items are sent to the item pipelines, requests are scheduled."""diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 9828fe6f5..c72752915 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):"""Run a crawler with the provided arguments.- It will call the given Crawler's :meth:`~Crawler.crawl` method, while- keeping track of it so it can be stopped later.+ It will call the given Crawler's :meth:`~Crawler.crawl_async` method,+ while keeping track of it so it can be stopped later.If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`instance, this method will try to create one using this parameter as@@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):"""This method starts a :mod:`~twisted.internet.reactor`, adjusts its poolsize to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS- resolver based on :setting:`DNSCACHE_ENABLED`.+ resolver based on :setting:`TWISTED_DNS_RESOLVER`.If ``stop_after_crawl`` is True, the reactor will be stopped after allcrawlers have finished, using :meth:`join`.@@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):When using a reactor it adjusts its pool size to:setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based- on :setting:`DNSCACHE_ENABLED`.+ on :setting:`TWISTED_DNS_RESOLVER`.- If ``stop_after_crawl`` is True, the reactor will be stopped after all- crawlers have finished, using :meth:`join`.+ If ``stop_after_crawl`` is True, the reactor/event loop will be stopped+ after all crawlers have finished, using :meth:`join`.:param bool stop_after_crawl: stop or not the reactor when allcrawlers have finisheddiff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 2b1721ced..6ca04a50e 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -60,7 +60,7 @@ else:class HttpCompressionMiddleware:- """This middleware allows compressed (gzip, deflate) traffic to be+ """This middleware allows compressed (gzip, deflate etc.) traffic to besent/received from websites"""def __init__(diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 821f41699..45af5c67a 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -196,10 +196,7 @@ class BaseRedirectMiddleware:class RedirectMiddleware(BaseRedirectMiddleware):- """- Handle redirection of requests based on response status- and meta-refresh html tag.- """+ """Handle redirection of requests based on response status."""@_warn_spider_argdef process_response(@@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):class MetaRefreshMiddleware(BaseRedirectMiddleware):+ """Handle redirection of requests based on meta-refresh html tag."""+enabled_setting = "METAREFRESH_ENABLED"def __init__(self, settings: BaseSettings):diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 5f125cae4..a0a0b60a2 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error.You can change the behaviour of this middleware by modifying the scraping settings:RETRY_TIMES - how many times to retry a failed pageRETRY_HTTP_CODES - which HTTP response codes to retry--Failed pages are collected on the scraping process and rescheduled at the end,-once the spider has finished crawling all regular (non-failed) pages."""from __future__ import annotations@@ -70,8 +67,9 @@ def get_retry_request(and :ref:`stats <topics-stats>`, and to provide extra logging context (see:func:`logging.debug`).- *reason* is a string or an :class:`Exception` object that indicates the- reason why the request needs to be retried. It is used to name retry stats.+ *reason* is a string, an :class:`Exception` subclass or an+ :class:`Exception` object that indicates the reason why the request needs+ to be retried. It is used to name retry stats.*max_retry_times* is a number that determines the maximum number of timesthat *request* can be retried. If not specified or ``None``, the number isdiff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 204132973..5330eab48 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -115,7 +115,7 @@ class UsageError(Exception):class ScrapyDeprecationWarning(Warning):"""Warning category for deprecated features, since the default- DeprecationWarning is silenced on Python 2.7++ :exc:`DeprecationWarning` is silenced."""diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.pyindex 3d7674905..6c94d947e 100644--- a/scrapy/extensions/logstats.py+++ b/scrapy/extensions/logstats.py@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)class LogStats:"""Log basic scraping stats periodically like:- * RPM - Requests per Minute+ * RPM - Responses per Minute* IPM - Items per Minute"""diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.pyindex cdb0671ae..542ff1cdc 100644--- a/scrapy/extensions/throttle.py+++ b/scrapy/extensions/throttle.py@@ -116,7 +116,7 @@ class AutoThrottle:# It works better with problematic sites.new_delay = max(target_delay, new_delay)- # Make sure self.mindelay <= new_delay <= self.max_delay+ # Make sure self.mindelay <= new_delay <= self.maxdelaynew_delay = min(max(self.mindelay, new_delay), self.maxdelay)# Dont adjust delay if response status != 200 and new delay is smallerdiff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex 599f20947..8edeae01c 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -136,9 +136,9 @@ class _DummyLock:class WrappedRequest:- """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class-- see http://docs.python.org/library/urllib2.html#urllib2.Request+ """Wraps a :class:`scrapy.Request` class with methods defined by+ the :class:`urllib.request.Request` class to interact with+ the :class:`http.cookiejar.CookieJar` class."""def __init__(self, request: Request):diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.pyindex 70c08c11d..6d3a9ee12 100644--- a/scrapy/http/response/html.py+++ b/scrapy/http/response/html.py@@ -1,6 +1,7 @@"""-This module implements the HtmlResponse class which adds encoding-discovering through HTML encoding declarations to the TextResponse class.+This module implements the :class:`HtmlResponse` class which is used as a+content type marker by :class:`~scrapy.selector.Selector` and can be used in+``isinstance()`` checks.See documentation in docs/topics/request-response.rst"""diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.pyindex 6d9c4cb73..847fb2b3c 100644--- a/scrapy/http/response/xml.py+++ b/scrapy/http/response/xml.py@@ -1,6 +1,7 @@"""-This module implements the XmlResponse class which adds encoding-discovering through XML encoding declarations to the TextResponse class.+This module implements the :class:`XmlResponse` class which is used as a+content type marker by :class:`~scrapy.selector.Selector` and can be used in+``isinstance()`` checks.See documentation in docs/topics/request-response.rst"""diff --git a/scrapy/item.py b/scrapy/item.pyindex 1cc0ae584..d5adc1efb 100644--- a/scrapy/item.py+++ b/scrapy/item.py@@ -1,7 +1,7 @@"""Scrapy Item-See documentation in docs/topics/item.rst+See documentation in docs/topics/items.rst"""from __future__ import annotationsdiff --git a/scrapy/link.py b/scrapy/link.pyindex 046630403..8211faccd 100644--- a/scrapy/link.py+++ b/scrapy/link.py@@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rstclass Link:"""Link objects represent an extracted link by the LinkExtractor.- Using the anchor tag sample below to illustrate the parameters::+ Using the anchor tag sample below to illustrate the parameters:++ .. code-block:: html<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>diff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex a50064e08..bfb2d5dff 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -58,18 +58,20 @@ class LogFormatter:logging an action the method must return ``None``.Here is an example on how to create a custom log formatter to lower the severity level of- the log message when an item is dropped from the pipeline::-- class PoliteLogFormatter(logformatter.LogFormatter):- def dropped(self, item, exception, response, spider):- return {- 'level': logging.INFO, # lowering the level from logging.WARNING- 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",- 'args': {- 'exception': exception,- 'item': item,- }- }+ the log message when an item is dropped from the pipeline:++ .. code-block:: python++ class PoliteLogFormatter(logformatter.LogFormatter):+ def dropped(self, item, exception, response, spider):+ return {+ "level": logging.INFO, # lowering the level from logging.WARNING+ "msg": "Dropped: %(exception)s" + os.linesep + "%(item)s",+ "args": {+ "exception": exception,+ "item": item,+ },+ }"""def crawled(diff --git a/scrapy/mail.py b/scrapy/mail.pyindex fbd11ad1d..97123e63c 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -1,7 +1,5 @@"""Mail sending helpers--See documentation in docs/topics/email.rst"""from __future__ import annotationsdiff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 84fb5f85e..383c461e6 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -1,7 +1,7 @@"""Item pipeline-See documentation in docs/item-pipeline.rst+See documentation in docs/topics/item-pipeline.rst"""from __future__ import annotationsdiff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 0066fd38f..44c422430 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -423,7 +423,7 @@ class FTPFilesStore:class FilesPipeline(MediaPipeline):- """Abstract pipeline that implement the file downloading+ """Pipeline that implements file downloading.This pipeline tries to minimize network transfers and file processing,doing stat of the files and determining if file is new, up-to-date ordiff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 762b0fdf1..79b6c4f27 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -47,7 +47,7 @@ class ImageException(FileException):class ImagesPipeline(FilesPipeline):- """Abstract pipeline that implement the image thumbnail generation logic"""+ """Pipeline that implements the handling logic specific to images."""MEDIA_NAME: str = "image"diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 2efc43d4b..41411ceaf 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -92,9 +92,10 @@ class ScrapyPriorityQueue:- The :data:`~scrapy.Request.priority` of the request.For each combination of the above seen, this class creates an instance of- *downstream_queue_cls* with *key* set to a subdirectory of the persistence- directory, named as the request priority (e.g. ``1``), with an ``s`` suffix- in case of a start request (e.g. ``1s``).+ *downstream_queue_cls* (or *start_queue_cls* for start requests if it was+ passed) with *key* set to a subdirectory of the persistence directory,+ named as the negated request priority (e.g. ``-1``), with an ``s`` suffix+ in case of a start request (e.g. ``-1s``)."""@classmethoddiff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex be298e9b1..932463ba9 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]):"""Store a key/value attribute with a given priority.- Settings should be populated *before* configuring the Crawler object- (through the :meth:`~scrapy.crawler.Crawler.configure` method),- otherwise they won't have any effect.+ Settings should be populated *before* the Crawler object applies them+ (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or+ :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't+ have any effect.:param name: the setting name:type name: str@@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]):"""Make a deep copy of current settings.- This method returns a new instance of the :class:`Settings` class,+ This method returns a new instance of this class,populated with the same values and their priorities.Modifications to the new object won't be reflected on the original@@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]):Make a copy of current settings and convert to a dict.This method returns a new dict populated with the same values- and their priorities as the current settings.+ as the current settings.Modifications to the returned dict won't be reflected on the originalsettings.diff --git a/scrapy/shell.py b/scrapy/shell.pyindex 44dcd880e..dfea00c46 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -75,7 +75,7 @@ if TYPE_CHECKING:# running event loop.## Side note: it should be possible to remove _request_deferred() by using-# engine.download_async() instead of engine.schedule(), losing the usual stuff+# engine.download_async() instead of engine.crawl(), losing the usual stuff# like spider middlewares (none of which should be important).## Other architecture problems:@@ -188,7 +188,8 @@ class Shell:async def _schedule(self, request: Request, spider: Spider | None) -> Response:"""Send the request to the engine, wait for the result.- Runs in the reactor thread.+ Runs in the reactor thread when using the reactor, or in the asyncio+ event loop thread otherwise."""if not self.spider:await self._open_spider(spider)diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pyindex e09f2d10e..6c62dccb8 100644--- a/scrapy/spidermiddlewares/base.py+++ b/scrapy/spidermiddlewares/base.py@@ -75,7 +75,7 @@ class BaseSpiderMiddleware:) -> Request | None:"""Return a processed request from the spider output.- This method is called with a single request from the start seeds or the+ This method is called with a single request from ``start()`` or thespider output. It should return the same or a different request, or``None`` to ignore it.@@ -84,7 +84,7 @@ class BaseSpiderMiddleware::param response: the response being processed:type response: :class:`~scrapy.http.Response` object or ``None`` for- start seeds+ start requests:return: the processed request or ``None``"""@@ -93,7 +93,7 @@ class BaseSpiderMiddleware:def get_processed_item(self, item: Any, response: Response | None) -> Any:"""Return a processed item from the spider output.- This method is called with a single item from the start seeds or the+ This method is called with a single item from ``start()`` or thespider output. It should return the same or a different item, or``None`` to ignore it.@@ -102,7 +102,7 @@ class BaseSpiderMiddleware::param response: the response being processed:type response: :class:`~scrapy.http.Response` object or ``None`` for- start seeds+ start items:return: the processed item or ``None``"""diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.pyindex 94b6dfbb5..156b73e7e 100644--- a/scrapy/spidermiddlewares/httperror.py+++ b/scrapy/spidermiddlewares/httperror.py@@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)class HttpError(IgnoreRequest):- """A non-200 response was filtered"""+ """A non-2xx response was filtered"""def __init__(self, response: Response, *args: Any, **kwargs: Any):self.response = responsediff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex 264f685c1..1305874d5 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -92,7 +92,7 @@ class ReferrerPolicy(ABC):)def origin(self, url: str) -> str | None:- """Return serialized origin (scheme, host, path) for a request or response URL."""+ """Return serialized origin (scheme, host, port) for a request or response URL."""return self.strip_url(url, origin_only=True)def potentially_trustworthy(self, url: str) -> bool:diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.pyindex 395183613..925f31ede 100644--- a/scrapy/spiders/feed.py+++ b/scrapy/spiders/feed.py@@ -54,17 +54,21 @@ class XMLFeedSpider(Spider):return responsedef parse_node(self, response: Response, selector: Selector) -> Any:- """This method must be overridden with your custom spider functionality"""+ """This method is called for the nodes matching the provided tag name+ (itertag). Receives the response and an Selector for each node.++ This method must return either an item, a request, or a list+ containing any of them.++ This method must be overridden with your custom spider functionality.+ """if hasattr(self, "parse_item"): # backward compatibilityreturn self.parse_item(response, selector)raise NotImplementedErrordef parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:"""This method is called for the nodes matching the provided tag name- (itertag). Receives the response and an Selector for each node.- Overriding this method is mandatory. Otherwise, you spider won't work.- This method must return either an item, a request, or a list- containing any of them.+ (itertag). Receives the response and an iterable of Selectors."""for selector in nodes:@@ -113,6 +117,9 @@ class CSVFeedSpider(Spider):It receives a CSV file in a response; iterates through each of its rows,and calls parse_row with a dict containing each field's data.+ This spider also gives the opportunity to override adapt_response and+ process_results methods for pre and post-processing purposes.+You can set some options regarding the CSV file, such as the delimiter, quotecharand the file's headers."""@@ -136,16 +143,14 @@ class CSVFeedSpider(Spider):return responsedef parse_row(self, response: Response, row: dict[str, str]) -> Any:- """This method must be overridden with your custom spider functionality"""- raise NotImplementedError-- def parse_rows(self, response: Response) -> Any:"""Receives a response and a dict (representing each row) with a key for- each provided (or detected) header of the CSV file. This spider also- gives the opportunity to override adapt_response and- process_results methods for pre and post-processing purposes.+ each provided (or detected) header of the CSV file.++ This method must be overridden with your custom spider functionality."""+ raise NotImplementedError+ def parse_rows(self, response: Response) -> Any:for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex dd0e062d0..c020ff4b9 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -1,8 +1,6 @@"""This module contains data types used by Scrapy which are not included in thePython Standard Library.--This module must not depend on any module outside the Standard Library."""from __future__ import annotationsdiff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 29a34d4ef..d0259b634 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None:def defer_result(result: Any) -> Deferred[Any]: # pragma: no coverwarnings.warn("scrapy.utils.defer.defer_result() is deprecated, use"- " twisted.internet.defer.success() and twisted.internet.defer.fail(),"+ " twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"" plus an explicit sleep if needed, or explicit reactor.callLater().",category=ScrapyDeprecationWarning,stacklevel=2,@@ -469,22 +469,22 @@ def _maybeDeferred_coro(def deferred_to_future(d: Deferred[_T]) -> Future[_T]:"""Return an :class:`asyncio.Future` object that wraps *d*.- This function requires- :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be- installed.+ This function requires an installed asyncio reactor or a running asyncio+ event loop, see :ref:`using-asyncio`.- When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await- on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy- callables defined as coroutines <coroutine-support>`, you can only await on- ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects- allows you to wait on them::+ In this state you cannot await on :class:`~twisted.internet.defer.Deferred`+ objects from :ref:`Scrapy callables defined as coroutines+ <coroutine-support>`, you can only await on ``Future`` objects. Wrapping+ ``Deferred`` objects into ``Future`` objects allows you to wait on them:++ .. code-block:: pythonclass MySpider(Spider):...+async def parse(self, response):- additional_request = scrapy.Request('https://example.org/price')- deferred = self.crawler.engine.download(additional_request)- additional_response = await deferred_to_future(deferred)+ deferred = some_dfd_helper()+ result = await deferred_to_future(deferred).. versionchanged:: 2.14This function no longer installs an asyncio loop if called before the@@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:in this case."""if not is_asyncio_available():- raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")+ raise RuntimeError(+ "deferred_to_future() requires an installed asyncio reactor"+ " or a running asyncio event loop."+ )return d.asFuture(asyncio.get_event_loop())@@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:defined as a coroutine <coroutine-support>`.What you can await in Scrapy callables defined as coroutines depends on the- value of :setting:`TWISTED_REACTOR`:+ value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`:- - When :ref:`using the asyncio reactor <install-asyncio>`, you can only- await on :class:`asyncio.Future` objects.+ - When :ref:`using the asyncio reactor <install-asyncio>`, or :ref:`not+ using a reactor at all <asyncio-without-reactor>`, you can only await+ on :class:`asyncio.Future` objects.- - When not using the asyncio reactor, you can only await on- :class:`~twisted.internet.defer.Deferred` objects.+ - When :ref:`using a non-asyncio reactor <disable-asyncio>`, you can only+ await on :class:`~twisted.internet.defer.Deferred` objects.- If you want to write code that uses ``Deferred`` objects but works with any- reactor, use this function on all ``Deferred`` objects::+ If you want to write code that uses ``Deferred`` objects but works in both+ of these states, use this function on all ``Deferred`` objects:++ .. code-block:: pythonclass MySpider(Spider):...+async def parse(self, response):- additional_request = scrapy.Request('https://example.org/price')- deferred = self.crawler.engine.download(additional_request)- additional_response = await maybe_deferred_to_future(deferred)+ deferred = some_dfd_helper()+ result = await maybe_deferred_to_future(deferred)"""if not is_asyncio_available():return ddiff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.pyindex 359f819d7..4fd50fdad 100644--- a/scrapy/utils/deprecate.py+++ b/scrapy/utils/deprecate.py@@ -43,15 +43,18 @@ def create_deprecated_class(It can be used to rename a base class in a library. For example, if wehave- class OldName(SomeClass):- # ...+ .. code-block:: python- and we want to rename it to NewName, we can do the following::+ class OldName(SomeClass): ...- class NewName(SomeClass):- # ...+ and we want to rename it to NewName, we can do the following:- OldName = create_deprecated_class('OldName', NewName)+ .. code-block:: python++ class NewName(SomeClass): ...+++ OldName = create_deprecated_class("OldName", NewName)Then, if user class inherits from OldName, warning is issued. Also, ifsome code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``diff --git a/scrapy/utils/log.py b/scrapy/utils/log.pyindex aa77e692a..7645b235e 100644--- a/scrapy/utils/log.py+++ b/scrapy/utils/log.py@@ -248,8 +248,7 @@ def logformatter_adapter() -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:"""Helper that takes the dictionary output from the methods in LogFormatter- and adapts it into a tuple of positional arguments for logger.log calls,- handling backward compatibility as well.+ and adapts it into a tuple of positional arguments for logger.log calls."""level = logkws.get("level", logging.INFO)diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex c068d9b1e..7747a7b9b 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -88,7 +88,7 @@ def open_in_browser(def parse_details(self, response):- if "item name" not in response.body:+ if "item name" not in response.text:open_in_browser(response)"""# circular importsdiff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex 919f67240..eca95e225 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -39,7 +39,7 @@ def send_catch_log(*arguments: TypingAny,**named: TypingAny,) -> list[tuple[TypingAny, TypingAny]]:- """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns+ """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returnsFailures instead of exceptions."""dont_log = named.pop("dont_log", ())@@ -172,9 +172,8 @@ async def _send_catch_log_asyncio(Returns a coroutine that completes once all signal handlers have finished.- This function requires- :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be- installed.+ This function requires an installed asyncio reactor or a running asyncio+ event loop... versionadded:: 2.14"""diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.pyindex 87df10a02..22f9eadd0 100644--- a/scrapy/utils/trackref.py+++ b/scrapy/utils/trackref.py@@ -4,9 +4,7 @@ references to live object instances.If you want live objects for a particular class to be tracked, you only have tosubclass from object_ref (instead of object).-About performance: This library has a minimal performance impact when enabled,-and no performance penalty at all when disabled (as object_ref becomes just an-alias to object in that case).+This library has a minimal performance impact... note:: PyPy uses a tracing garbage collector, so objects mayremain in the ``live_refs`` longer than expected, even after theydiff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 4d2bbdda2..f67853ece 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -117,8 +117,8 @@ def strip_url(- ``strip_credentials`` removes "user:password@"- ``strip_default_port`` removes ":80" (resp. ":443", ":21")from http:// (resp. https://, ftp://) URLs- - ``origin_only`` replaces path component with "/", also dropping- query and fragment components ; it also strips credentials+ - ``origin_only`` replaces the path component with "/", also dropping+ the query component; it also strips credentials- ``strip_fragment`` drops any #fragment component"""