Scrapy, a fast high-level web crawling & scraping framework for Python.
Optimize SitemapSpider memory usage (#7007)
Albert Eduardovich N.9fffcc1b8288
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex cb5779d74..d87ffccaf 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -72,31 +72,53 @@ class SitemapSpider(Spider):def _parse_sitemap(self, response: Response) -> Iterable[Request]:if response.url.endswith("/robots.txt"):- for url in sitemap_urls_from_robots(response.text, base_url=response.url):- yield Request(url, callback=self._parse_sitemap)- else:- body = self._get_sitemap_body(response)- if body is None:- logger.warning(- "Ignoring invalid sitemap: %(response)s",- {"response": response},- extra={"spider": self},- )- return-- s = Sitemap(body)- it = self.sitemap_filter(s)-- if s.type == "sitemapindex":- for loc in iterloc(it, self.sitemap_alternate_links):- if any(x.search(loc) for x in self._follow):- yield Request(loc, callback=self._parse_sitemap)- elif s.type == "urlset":- for loc in iterloc(it, self.sitemap_alternate_links):- for r, c in self._cbs:- if r.search(loc):- yield Request(loc, callback=c)- break+ urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))+ return (Request(url, callback=self._parse_sitemap) for url in urls)++ body = self._get_sitemap_body(response)+ if not body:+ logger.warning(+ "Ignoring invalid sitemap: %(response)s",+ {"response": response},+ extra={"spider": self},+ )+ return ()++ s = Sitemap(body)++ if s.type == "sitemapindex":+ urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s)))+ return (Request(loc, callback=self._parse_sitemap) for loc in urls)++ if s.type == "urlset":+ url_callback_pairs = list(+ self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s))+ )+ return (Request(loc, callback=c) for loc, c in url_callback_pairs)++ logger.warning(+ "Ignoring invalid sitemap: %(response)s",+ {"response": response},+ extra={"spider": self},+ )++ return ()++ def _get_urls_from_sitemapindex(+ self, it: Iterable[dict[str, Any]]+ ) -> Iterable[str]:+ for loc in iterloc(it, self.sitemap_alternate_links):+ if any(x.search(loc) for x in self._follow):+ yield loc++ def _get_urls_and_callbacks_from_urlset(+ self, it: Iterable[dict[str, Any]]+ ) -> Iterable[tuple[str, CallbackT]]:+ for loc in iterloc(it, self.sitemap_alternate_links):+ for r, c in self._cbs:+ if r.search(loc):+ yield loc, c+ breakdef _get_sitemap_body(self, response: Response) -> bytes | None:"""Return the sitemap body contained in the given response,@@ -140,8 +162,9 @@ def regex(x: re.Pattern[str] | str) -> re.Pattern[str]:def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]:for d in it:- yield d["loc"]+ if loc := d["loc"]:+ yield loc# Also consider alternate URLs (xhtml:link rel="alternate")- if alt and "alternate" in d:- yield from d["alternate"]+ if alt and (alt_list := d.get("alternate")):+ yield from alt_listdiff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex c0100ea62..042ed6281 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -7,11 +7,15 @@ SitemapSpider, its API is subject to change without notice.from __future__ import annotations+import warnings+from io import BytesIO, StringIOfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urljoinimport lxml.etree+from scrapy.exceptions import ScrapyDeprecationWarning+if TYPE_CHECKING:from collections.abc import Iterable, Iterator@@ -20,40 +24,112 @@ class Sitemap:"""Class to parse Sitemap (type=urlset) and Sitemap Index(type=sitemapindex) files"""+ __slots__ = ("type", "xmliter")+def __init__(self, xmltext: str | bytes):- xmlp = lxml.etree.XMLParser(- recover=True, remove_comments=True, resolve_entities=False+ if isinstance(xmltext, str):+ warnings.warn(+ "Passing `str` type as `xmltext` is deprecated, use `bytes`",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ xmltext = xmltext.encode()++ self.xmliter = lxml.etree.iterparse(+ BytesIO(xmltext),+ recover=True,+ remove_comments=True,+ resolve_entities=False,+ remove_blank_text=True,+ collect_ids=False,+ remove_pis=True,+ events=("start", "end"),)- self._root = lxml.etree.fromstring(xmltext, parser=xmlp)- rt = self._root.tag- assert isinstance(rt, str)- self.type = rt.split("}", 1)[1] if "}" in rt else rt+ _, root = next(self.xmliter)+ self.type = self._get_tag_name(root)def __iter__(self) -> Iterator[dict[str, Any]]:- for elem in self._root.getchildren():- d: dict[str, Any] = {}- for el in elem.getchildren():- tag = el.tag- assert isinstance(tag, str)- name = tag.split("}", 1)[1] if "}" in tag else tag-- if name == "link":- if "href" in el.attrib:- d.setdefault("alternate", []).append(el.get("href"))- else:- d[name] = el.text.strip() if el.text else ""+ for event, elem in self.xmliter:+ if event == "start":+ continue++ if self._get_tag_name(elem) not in {"url", "sitemap"}:+ continue- if "loc" in d:+ if d := self._process_sitemap_element(elem):yield d+ def _process_sitemap_element(+ self, elem: lxml.etree._Element+ ) -> dict[str, Any] | None:+ d: dict[str, Any] = {}+ alternate: list[str] = []+ has_loc = False++ for el in elem:+ try:+ tag_name = self._get_tag_name(el)+ if not tag_name:+ continue++ if tag_name == "link":+ if href := el.get("href"):+ alternate.append(href)+ else:+ d[tag_name] = el.text.strip() if el.text else ""+ if not has_loc and tag_name == "loc":+ has_loc = True+ finally:+ el.clear()+ elem.clear()+ parent = elem.getparent()+ if parent is not None:+ while elem.getprevious() is not None:+ del parent[0]++ if not has_loc:+ return None++ if alternate:+ d["alternate"] = alternate++ return d++ @staticmethod+ def _get_tag_name(elem: lxml.etree._Element) -> str:+ if TYPE_CHECKING:+ assert isinstance(elem.tag, str)+ _, _, localname = elem.tag.partition("}")+ return localname or elem.tag+def sitemap_urls_from_robots(- robots_text: str, base_url: str | None = None+ robots_text: str | bytes,+ base_url: str | None = None,+) -> Iterable[str]:+ if isinstance(robots_text, bytes):+ for line in BytesIO(robots_text):+ if line.lstrip()[:8].lower() == b"sitemap:":+ try:+ url = line.partition(b":")[2].strip().decode()+ except UnicodeDecodeError:+ continue+ yield urljoin(base_url or "", url)++ else:+ yield from _sitemap_urls_from_robots_str(robots_text, base_url)+++def _sitemap_urls_from_robots_str(+ robots_text: str,+ base_url: str | None = None,) -> Iterable[str]:- """Return an iterator over all sitemap urls contained in the given- robots.txt file- """- for line in robots_text.splitlines():- if line.lstrip().lower().startswith("sitemap:"):- url = line.split(":", 1)[1].strip()+ warnings.warn(+ "Passing `str` type as `robots_text` is deprecated, use `bytes`",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ for line in StringIO(robots_text):+ if line.lstrip()[:8].lower() == "sitemap:":+ url = line.partition(":")[2].strip()yield urljoin(base_url or "", url)diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex d2664b19e..57c209615 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -1,6 +1,7 @@from __future__ import annotationsimport gzip+import reimport warningsfrom datetime import datetimefrom io import BytesIO@@ -43,6 +44,9 @@ class TestSitemapSpider(TestSpider):r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)self.assertSitemapBody(r, None)+ r = XmlResponse(url="http://www.example.com/", body=b"")+ self.assertSitemapBody(r, b"")+def test_get_sitemap_body_gzip_headers(self):r = Response(url="http://www.example.com/sitemap",@@ -85,6 +89,20 @@ Sitemap: /sitemap-relative-url.xml"http://www.example.com/sitemap-relative-url.xml",]+ def test_get_sitemap_urls_from_robotstxt_skips_invalid_utf8_urls(self):+ robots = (+ b"User-agent: *\n"+ b"Sitemap: http://example.com/\xff.xml\n"+ b"Sitemap: http://example.com/ok.xml\n"+ )++ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)+ spider = self.spider_class("example.com")++ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://example.com/ok.xml",+ ]+def test_alternate_url_locs(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"@@ -218,6 +236,79 @@ Sitemap: /sitemap-relative-url.xml"http://www.example.com/sitemap2.xml"]+ @pytest.mark.parametrize(+ ("rule", "result"),+ [(r"english", ["http://www.example.com/english/"]), (r"nonexistent", [])],+ )+ def test_sitemap_filter_with_rule(self, rule: str, result: list[str]):+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url><loc>http://www.example.com/english/</loc></url>+ <url><loc>http://www.example.com/portuguese/</loc></url>+ </urlset>"""+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)++ class _RuleSpider(self.spider_class): # type: ignore[name-defined,misc]+ sitemap_rules = [(rule, "parse")]++ spider = _RuleSpider("example.com")+ urls = [req.url for req in spider._parse_sitemap(r)]+ assert urls == result++ def test_parse_sitemap_empty_body(self):+ r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")+ spider = self.spider_class("example.com")++ with LogCapture() as lc:+ results = list(spider._parse_sitemap(r))++ assert not results++ lc.check(+ (+ "scrapy.spiders.sitemap",+ "WARNING",+ "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>",+ )+ )++ def test_parse_sitemap_not_sitemap(self):+ body = b"""<?xml version="1.0" encoding="UTF-8"?>+ <some attr="string">+ <tag><tag3>sometext</tag3></tag>+ <tag2><tag4>sometext2</tag4></tag2>+ </some>"""+ r = XmlResponse(url="http://www.example.com/random.xml", body=body)+ spider = self.spider_class("example.com")++ results = list(spider._parse_sitemap(r))++ assert not results++ @pytest.mark.parametrize(+ ("follow", "result"),+ [+ (r"1.xml", ["http://www.example.com/sitemap1.xml"]),+ (re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]),+ (r"nonexistent", []),+ ],+ )+ def test_sitemap_follow(self, follow, result):+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>http://www.example.com/sitemap1.xml</loc>+ </sitemap>+ </sitemapindex>"""+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)++ class _FollowSpider(self.spider_class):+ sitemap_follow = [follow]++ spider = _FollowSpider("example.com")+ urls = [req.url for req in spider._parse_sitemap(r)]+ assert urls == result+def test_compression_bomb_setting(self):settings = {"DOWNLOAD_MAXSIZE": 10_000_000}crawler = get_crawler(settings_dict=settings)diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.pyindex 464a31777..f2bcd7541 100644--- a/tests/test_utils_sitemap.py+++ b/tests/test_utils_sitemap.py@@ -1,3 +1,5 @@+import warnings+from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots@@ -156,7 +158,7 @@ def test_sitemap_wrong_ns2():def test_sitemap_urls_from_robots():- robots = """User-agent: *+ robots = b"""User-agent: *Disallow: /aff/Disallow: /wl/@@ -182,6 +184,40 @@ Disallow: /forum/active/]+def test_sitemap_urls_from_robots_str_compat():+ robots = """User-agent: *+Disallow: /aff/+Disallow: /wl/++# Search and shopping refining+Disallow: /s*/*facet+Disallow: /s*/*tags++# Sitemap files+Sitemap: http://example.com/sitemap.xml+Sitemap: http://example.com/sitemap-product-index.xml+Sitemap: HTTP://example.com/sitemap-uppercase.xml+Sitemap: /sitemap-relative-url.xml++# Forums+Disallow: /forum/search/+Disallow: /forum/active/+"""++ with warnings.catch_warnings(record=True) as w:+ assert list(+ sitemap_urls_from_robots(robots, base_url="http://example.com")+ ) == [+ "http://example.com/sitemap.xml",+ "http://example.com/sitemap-product-index.xml",+ "http://example.com/sitemap-uppercase.xml",+ "http://example.com/sitemap-relative-url.xml",+ ]+ assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str(+ w[0].message+ )++def test_sitemap_blanklines():"""Assert we can deal with starting blank lines before <xml> tag"""s = Sitemap(