Scrapy, a fast high-level web crawling & scraping framework for Python.
Use autodoc for SitemapSpider (#8229)
Adriane4715799fd60
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 75b28cf56..728b08d1c 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -816,121 +816,9 @@ Let's see an example similar to the previous one, but using aSitemapSpider--------------.. class:: SitemapSpider-- SitemapSpider allows you to crawl a site by discovering the URLs using- `Sitemaps`_.-- It supports nested sitemaps and discovering sitemap urls from- `robots.txt`_.-- .. attribute:: sitemap_urls-- A list of urls pointing to the sitemaps whose urls you want to crawl.-- You can also point to a `robots.txt`_ and it will be parsed to extract- sitemap urls from it.-- .. attribute:: sitemap_rules-- A list of tuples ``(regex, callback)`` where:-- * ``regex`` is a regular expression to match urls extracted from sitemaps.- ``regex`` can be either a str or a compiled regex object.-- * callback is the callback to use for processing the urls that match- the regular expression. ``callback`` can be a string (indicating the- name of a spider method) or a callable.-- For example:-- .. code-block:: python-- sitemap_rules = [("/product/", "parse_product")]-- Rules are applied in order, and only the first one that matches will be- used.-- If you omit this attribute, all urls found in sitemaps will be- processed with the ``parse`` callback.-- .. attribute:: sitemap_follow-- A list of regexes of sitemap that should be followed. This is only- for sites that use `Sitemap index files`_ that point to other sitemap- files.-- By default, all sitemaps are followed.-- .. attribute:: sitemap_alternate_links-- Specifies if alternate links for one ``url`` should be followed. These- are links for the same website in another language passed within- the same ``url`` block.-- For example:-- .. code-block:: xml-- <url>- <loc>http://example.com/</loc>- <xhtml:link rel="alternate" hreflang="de" href="http://example.com/de"/>- </url>-- With ``sitemap_alternate_links`` set, this would retrieve both URLs. With- ``sitemap_alternate_links`` disabled, only ``http://example.com/`` would be- retrieved.-- Default is ``sitemap_alternate_links`` disabled.-- .. method:: sitemap_filter(entries)-- This is a filter function that could be overridden to select sitemap entries- based on their attributes.-- For example:-- .. code-block:: xml-- <url>- <loc>http://example.com/</loc>- <lastmod>2005-01-01</lastmod>- </url>-- We can define a ``sitemap_filter`` function to filter ``entries`` by date:-- .. code-block:: python-- from datetime import datetime- from scrapy.spiders import SitemapSpider--- class FilteredSitemapSpider(SitemapSpider):- name = "filtered_sitemap_spider"- allowed_domains = ["example.com"]- sitemap_urls = ["http://example.com/sitemap.xml"]-- def sitemap_filter(self, entries):- for entry in entries:- date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")- if date_time.year >= 2005:- yield entry-- This would retrieve only ``entries`` modified on 2005 and the following- years.-- Entries are dict objects extracted from the sitemap document.- Usually, the key is the tag name and the value is the text inside it.-- It's important to notice that:-- - as the loc attribute is required, entries without this tag are discarded- - alternate links are stored in a list with the key ``alternate``- (see ``sitemap_alternate_links``)- - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``-- If you omit this method, all entries found in sitemaps will be- processed, observing other attributes and their settings.+.. autoclass:: SitemapSpider+ :members:+ :exclude-members: startSitemapSpider examples@@ -1019,7 +907,6 @@ Combine SitemapSpider with other sources of urls:.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html.. _Sitemaps: https://www.sitemaps.org/index.html-.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index.. _robots.txt: https://www.robotstxt.org/.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 36608a87b..6382112ae 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -28,12 +28,71 @@ logger = logging.getLogger(__name__)class SitemapSpider(Spider):+ """Spider that crawls a site by discovering its URLs using `sitemaps+ <https://www.sitemaps.org/index.html>`_.++ It supports nested sitemaps and discovering sitemap URLs from `robots.txt+ <https://www.robotstxt.org/>`_.+ """+sitemap_urls: Sequence[str] = ()+ """URLs pointing to the sitemaps whose URLs you want to crawl.++ You can also point to a `robots.txt <https://www.robotstxt.org/>`_ and it+ will be parsed to extract sitemap URLs from it.+ """+sitemap_rules: Sequence[tuple[re.Pattern[str] | str, str | CallbackT]] = [("", "parse")]+ """``(regex, callback)`` tuples where:++ - ``regex`` is a regular expression to match URLs extracted from+ sitemaps. ``regex`` can be either a str or a compiled regex object.++ - ``callback`` is the callback to use for processing the URLs that match+ the regular expression. ``callback`` can be a string (indicating the+ name of a spider method) or a callable.++ For example:++ .. code-block:: python++ sitemap_rules = [("/product/", "parse_product")]++ Rules are applied in order, and only the first one that matches will be+ used.++ The default value makes all URLs found in sitemaps be processed with the+ :meth:`~scrapy.Spider.parse` callback.+ """+sitemap_follow: Sequence[re.Pattern[str] | str] = [""]+ """Regexes of sitemaps that should be followed. This is only for sites that+ use `sitemap index files <https://www.sitemaps.org/protocol.html#index>`_+ that point to other sitemap files.++ By default, all sitemaps are followed.+ """+sitemap_alternate_links: bool = False+ """Specifies if alternate links for one ``url`` should be followed. These are+ links for the same website in another language passed within the same+ ``url`` block.++ For example:++ .. code-block:: xml++ <url>+ <loc>http://example.com/</loc>+ <xhtml:link rel="alternate" hreflang="de" href="http://example.com/de"/>+ </url>++ When enabled, this would retrieve both URLs. When disabled, only+ ``http://example.com/`` would be retrieved.+ """+_max_size: int_warn_size: int@@ -64,9 +123,54 @@ class SitemapSpider(Spider):def sitemap_filter(self, entries: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]:- """This method can be used to filter sitemap entries by their- attributes, for example, you can filter locs with lastmod greater- than a given date (see docs).+ """Yield the sitemap entries from *entries* that should be processed.++ Override it to select sitemap entries based on their attributes. For+ example, given the following sitemap entry:++ .. code-block:: xml++ <url>+ <loc>http://example.com/</loc>+ <lastmod>2005-01-01</lastmod>+ </url>++ You can filter entries by date as follows:++ .. code-block:: python++ from datetime import datetime+ from scrapy.spiders import SitemapSpider+++ class FilteredSitemapSpider(SitemapSpider):+ name = "filtered_sitemap_spider"+ allowed_domains = ["example.com"]+ sitemap_urls = ["http://example.com/sitemap.xml"]++ def sitemap_filter(self, entries):+ for entry in entries:+ date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")+ if date_time.year >= 2005:+ yield entry++ This would retrieve only entries modified on 2005 and the following+ years.++ Entries are dict objects extracted from the sitemap document. Usually,+ the key is the tag name and the value is the text inside it.++ It's important to notice that:++ - as the ``loc`` attribute is required, entries without this tag are+ discarded+ - alternate links are stored in a list with the key ``alternate``+ (see :attr:`sitemap_alternate_links`)+ - namespaces are removed, so lxml tags named as ``{namespace}tagname``+ become only ``tagname``++ The default implementation yields all entries, observing other+ attributes and their settings."""yield from entries