Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Optimize SitemapSpider memory usage (#7007)

Albert Eduardovich N.9fffcc1b8288
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index cb5779d74..d87ffccaf 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -72,31 +72,53 @@ class SitemapSpider(Spider):
def _parse_sitemap(self, response: Response) -> Iterable[Request]:
if response.url.endswith("/robots.txt"):
- for url in sitemap_urls_from_robots(response.text, base_url=response.url):
- yield Request(url, callback=self._parse_sitemap)
- else:
- body = self._get_sitemap_body(response)
- if body is None:
- logger.warning(
- "Ignoring invalid sitemap: %(response)s",
- {"response": response},
- extra={"spider": self},
- )
- return
-
- s = Sitemap(body)
- it = self.sitemap_filter(s)
-
- if s.type == "sitemapindex":
- for loc in iterloc(it, self.sitemap_alternate_links):
- if any(x.search(loc) for x in self._follow):
- yield Request(loc, callback=self._parse_sitemap)
- elif s.type == "urlset":
- for loc in iterloc(it, self.sitemap_alternate_links):
- for r, c in self._cbs:
- if r.search(loc):
- yield Request(loc, callback=c)
- break
+ urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))
+ return (Request(url, callback=self._parse_sitemap) for url in urls)
+
+ body = self._get_sitemap_body(response)
+ if not body:
+ logger.warning(
+ "Ignoring invalid sitemap: %(response)s",
+ {"response": response},
+ extra={"spider": self},
+ )
+ return ()
+
+ s = Sitemap(body)
+
+ if s.type == "sitemapindex":
+ urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s)))
+ return (Request(loc, callback=self._parse_sitemap) for loc in urls)
+
+ if s.type == "urlset":
+ url_callback_pairs = list(
+ self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s))
+ )
+ return (Request(loc, callback=c) for loc, c in url_callback_pairs)
+
+ logger.warning(
+ "Ignoring invalid sitemap: %(response)s",
+ {"response": response},
+ extra={"spider": self},
+ )
+
+ return ()
+
+ def _get_urls_from_sitemapindex(
+ self, it: Iterable[dict[str, Any]]
+ ) -> Iterable[str]:
+ for loc in iterloc(it, self.sitemap_alternate_links):
+ if any(x.search(loc) for x in self._follow):
+ yield loc
+
+ def _get_urls_and_callbacks_from_urlset(
+ self, it: Iterable[dict[str, Any]]
+ ) -> Iterable[tuple[str, CallbackT]]:
+ for loc in iterloc(it, self.sitemap_alternate_links):
+ for r, c in self._cbs:
+ if r.search(loc):
+ yield loc, c
+ break
def _get_sitemap_body(self, response: Response) -> bytes | None:
"""Return the sitemap body contained in the given response,
@@ -140,8 +162,9 @@ def regex(x: re.Pattern[str] | str) -> re.Pattern[str]:
def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]:
for d in it:
- yield d["loc"]
+ if loc := d["loc"]:
+ yield loc
# Also consider alternate URLs (xhtml:link rel="alternate")
- if alt and "alternate" in d:
- yield from d["alternate"]
+ if alt and (alt_list := d.get("alternate")):
+ yield from alt_list
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index c0100ea62..042ed6281 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -7,11 +7,15 @@ SitemapSpider, its API is subject to change without notice.
from __future__ import annotations
+import warnings
+from io import BytesIO, StringIO
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
import lxml.etree
+from scrapy.exceptions import ScrapyDeprecationWarning
+
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
@@ -20,40 +24,112 @@ class Sitemap:
"""Class to parse Sitemap (type=urlset) and Sitemap Index
(type=sitemapindex) files"""
+ __slots__ = ("type", "xmliter")
+
def __init__(self, xmltext: str | bytes):
- xmlp = lxml.etree.XMLParser(
- recover=True, remove_comments=True, resolve_entities=False
+ if isinstance(xmltext, str):
+ warnings.warn(
+ "Passing `str` type as `xmltext` is deprecated, use `bytes`",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ xmltext = xmltext.encode()
+
+ self.xmliter = lxml.etree.iterparse(
+ BytesIO(xmltext),
+ recover=True,
+ remove_comments=True,
+ resolve_entities=False,
+ remove_blank_text=True,
+ collect_ids=False,
+ remove_pis=True,
+ events=("start", "end"),
)
- self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
- rt = self._root.tag
- assert isinstance(rt, str)
- self.type = rt.split("}", 1)[1] if "}" in rt else rt
+ _, root = next(self.xmliter)
+ self.type = self._get_tag_name(root)
def __iter__(self) -> Iterator[dict[str, Any]]:
- for elem in self._root.getchildren():
- d: dict[str, Any] = {}
- for el in elem.getchildren():
- tag = el.tag
- assert isinstance(tag, str)
- name = tag.split("}", 1)[1] if "}" in tag else tag
-
- if name == "link":
- if "href" in el.attrib:
- d.setdefault("alternate", []).append(el.get("href"))
- else:
- d[name] = el.text.strip() if el.text else ""
+ for event, elem in self.xmliter:
+ if event == "start":
+ continue
+
+ if self._get_tag_name(elem) not in {"url", "sitemap"}:
+ continue
- if "loc" in d:
+ if d := self._process_sitemap_element(elem):
yield d
+ def _process_sitemap_element(
+ self, elem: lxml.etree._Element
+ ) -> dict[str, Any] | None:
+ d: dict[str, Any] = {}
+ alternate: list[str] = []
+ has_loc = False
+
+ for el in elem:
+ try:
+ tag_name = self._get_tag_name(el)
+ if not tag_name:
+ continue
+
+ if tag_name == "link":
+ if href := el.get("href"):
+ alternate.append(href)
+ else:
+ d[tag_name] = el.text.strip() if el.text else ""
+ if not has_loc and tag_name == "loc":
+ has_loc = True
+ finally:
+ el.clear()
+ elem.clear()
+ parent = elem.getparent()
+ if parent is not None:
+ while elem.getprevious() is not None:
+ del parent[0]
+
+ if not has_loc:
+ return None
+
+ if alternate:
+ d["alternate"] = alternate
+
+ return d
+
+ @staticmethod
+ def _get_tag_name(elem: lxml.etree._Element) -> str:
+ if TYPE_CHECKING:
+ assert isinstance(elem.tag, str)
+ _, _, localname = elem.tag.partition("}")
+ return localname or elem.tag
+
def sitemap_urls_from_robots(
- robots_text: str, base_url: str | None = None
+ robots_text: str | bytes,
+ base_url: str | None = None,
+) -> Iterable[str]:
+ if isinstance(robots_text, bytes):
+ for line in BytesIO(robots_text):
+ if line.lstrip()[:8].lower() == b"sitemap:":
+ try:
+ url = line.partition(b":")[2].strip().decode()
+ except UnicodeDecodeError:
+ continue
+ yield urljoin(base_url or "", url)
+
+ else:
+ yield from _sitemap_urls_from_robots_str(robots_text, base_url)
+
+
+def _sitemap_urls_from_robots_str(
+ robots_text: str,
+ base_url: str | None = None,
) -> Iterable[str]:
- """Return an iterator over all sitemap urls contained in the given
- robots.txt file
- """
- for line in robots_text.splitlines():
- if line.lstrip().lower().startswith("sitemap:"):
- url = line.split(":", 1)[1].strip()
+ warnings.warn(
+ "Passing `str` type as `robots_text` is deprecated, use `bytes`",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ for line in StringIO(robots_text):
+ if line.lstrip()[:8].lower() == "sitemap:":
+ url = line.partition(":")[2].strip()
yield urljoin(base_url or "", url)
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index d2664b19e..57c209615 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import gzip
+import re
import warnings
from datetime import datetime
from io import BytesIO
@@ -43,6 +44,9 @@ class TestSitemapSpider(TestSpider):
r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)
self.assertSitemapBody(r, None)
+ r = XmlResponse(url="http://www.example.com/", body=b"")
+ self.assertSitemapBody(r, b"")
+
def test_get_sitemap_body_gzip_headers(self):
r = Response(
url="http://www.example.com/sitemap",
@@ -85,6 +89,20 @@ Sitemap: /sitemap-relative-url.xml
"http://www.example.com/sitemap-relative-url.xml",
]
+ def test_get_sitemap_urls_from_robotstxt_skips_invalid_utf8_urls(self):
+ robots = (
+ b"User-agent: *\n"
+ b"Sitemap: http://example.com/\xff.xml\n"
+ b"Sitemap: http://example.com/ok.xml\n"
+ )
+
+ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
+ spider = self.spider_class("example.com")
+
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://example.com/ok.xml",
+ ]
+
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
@@ -218,6 +236,79 @@ Sitemap: /sitemap-relative-url.xml
"http://www.example.com/sitemap2.xml"
]
+ @pytest.mark.parametrize(
+ ("rule", "result"),
+ [(r"english", ["http://www.example.com/english/"]), (r"nonexistent", [])],
+ )
+ def test_sitemap_filter_with_rule(self, rule: str, result: list[str]):
+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url><loc>http://www.example.com/english/</loc></url>
+ <url><loc>http://www.example.com/portuguese/</loc></url>
+ </urlset>"""
+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+
+ class _RuleSpider(self.spider_class): # type: ignore[name-defined,misc]
+ sitemap_rules = [(rule, "parse")]
+
+ spider = _RuleSpider("example.com")
+ urls = [req.url for req in spider._parse_sitemap(r)]
+ assert urls == result
+
+ def test_parse_sitemap_empty_body(self):
+ r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
+ spider = self.spider_class("example.com")
+
+ with LogCapture() as lc:
+ results = list(spider._parse_sitemap(r))
+
+ assert not results
+
+ lc.check(
+ (
+ "scrapy.spiders.sitemap",
+ "WARNING",
+ "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>",
+ )
+ )
+
+ def test_parse_sitemap_not_sitemap(self):
+ body = b"""<?xml version="1.0" encoding="UTF-8"?>
+ <some attr="string">
+ <tag><tag3>sometext</tag3></tag>
+ <tag2><tag4>sometext2</tag4></tag2>
+ </some>"""
+ r = XmlResponse(url="http://www.example.com/random.xml", body=body)
+ spider = self.spider_class("example.com")
+
+ results = list(spider._parse_sitemap(r))
+
+ assert not results
+
+ @pytest.mark.parametrize(
+ ("follow", "result"),
+ [
+ (r"1.xml", ["http://www.example.com/sitemap1.xml"]),
+ (re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]),
+ (r"nonexistent", []),
+ ],
+ )
+ def test_sitemap_follow(self, follow, result):
+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>http://www.example.com/sitemap1.xml</loc>
+ </sitemap>
+ </sitemapindex>"""
+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+
+ class _FollowSpider(self.spider_class):
+ sitemap_follow = [follow]
+
+ spider = _FollowSpider("example.com")
+ urls = [req.url for req in spider._parse_sitemap(r)]
+ assert urls == result
+
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 464a31777..f2bcd7541 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -1,3 +1,5 @@
+import warnings
+
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
@@ -156,7 +158,7 @@ def test_sitemap_wrong_ns2():
def test_sitemap_urls_from_robots():
- robots = """User-agent: *
+ robots = b"""User-agent: *
Disallow: /aff/
Disallow: /wl/
@@ -182,6 +184,40 @@ Disallow: /forum/active/
]
+def test_sitemap_urls_from_robots_str_compat():
+ robots = """User-agent: *
+Disallow: /aff/
+Disallow: /wl/
+
+# Search and shopping refining
+Disallow: /s*/*facet
+Disallow: /s*/*tags
+
+# Sitemap files
+Sitemap: http://example.com/sitemap.xml
+Sitemap: http://example.com/sitemap-product-index.xml
+Sitemap: HTTP://example.com/sitemap-uppercase.xml
+Sitemap: /sitemap-relative-url.xml
+
+# Forums
+Disallow: /forum/search/
+Disallow: /forum/active/
+"""
+
+ with warnings.catch_warnings(record=True) as w:
+ assert list(
+ sitemap_urls_from_robots(robots, base_url="http://example.com")
+ ) == [
+ "http://example.com/sitemap.xml",
+ "http://example.com/sitemap-product-index.xml",
+ "http://example.com/sitemap-uppercase.xml",
+ "http://example.com/sitemap-relative-url.xml",
+ ]
+ assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str(
+ w[0].message
+ )
+
+
def test_sitemap_blanklines():
"""Assert we can deal with starting blank lines before <xml> tag"""
s = Sitemap(