Scrapy, a fast high-level web crawling & scraping framework for Python.
Bandit: allow-list lxml usages (#6265)
Adrián Chavesbf149356fc6e
diff --git a/.bandit.yml b/.bandit.ymlindex db2fbb84c..b7f1817e0 100644--- a/.bandit.yml+++ b/.bandit.yml@@ -1,9 +1,7 @@skips:- B101 # assert_used, needed for mypy-- B320 # xml_bad_etree- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180-- B410 # import_lxml- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082- B503 # ssl_with_bad_defaultsexclude_dirs: ['tests']diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex 0f80a0ab7..3206d79cd 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -10,21 +10,16 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, castfrom urllib.parse import urlencode, urljoin, urlsplit, urlunsplit-from lxml.html import (- FormElement,- HTMLParser,- InputElement,- MultipleSelectOptions,- SelectElement,- TextareaElement,-)-from parsel.selector import create_root_node+from lxml.html import FormElement # nosec+from lxml.html import InputElement # nosec+from lxml.html import MultipleSelectOptions # nosec+from lxml.html import SelectElement # nosec+from lxml.html import TextareaElement # nosecfrom w3lib.html import strip_html5_whitespacefrom scrapy.http.request import Requestfrom scrapy.http.response.text import TextResponsefrom scrapy.utils.python import is_listlike, to_bytes-from scrapy.utils.response import get_base_urlif TYPE_CHECKING:# typing.Self requires Python 3.11@@ -120,7 +115,7 @@ def _get_form(formxpath: Optional[str],) -> FormElement:"""Find the wanted form element within the given response."""- root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))+ root = response.selector.rootforms = root.xpath("//form")if not forms:raise ValueError(f"No <form> element found in {response}")diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex d76db20ba..55bc0fc43 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -7,7 +7,7 @@ import operatorfrom functools import partialfrom urllib.parse import urljoin, urlparse-from lxml import etree+from lxml import etree # nosecfrom parsel.csstranslator import HTMLTranslatorfrom w3lib.html import strip_html5_whitespacefrom w3lib.url import canonicalize_url, safe_url_stringdiff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex 75d5e9fbd..aa9581fcd 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -8,6 +8,7 @@ from parsel import Selector as _ParselSelectorfrom scrapy.http import HtmlResponse, TextResponse, XmlResponsefrom scrapy.utils.python import to_bytes+from scrapy.utils.response import get_base_urlfrom scrapy.utils.trackref import object_ref__all__ = ["Selector", "SelectorList"]@@ -88,7 +89,7 @@ class Selector(_ParselSelector, object_ref):if response is not None:text = response.text- kwargs.setdefault("base_url", response.url)+ kwargs.setdefault("base_url", get_base_url(response))self.response = responsediff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 14531df3f..7c40d0a02 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -12,7 +12,6 @@ else:try:brotli.Decompressor.processexcept AttributeError:-warn(("You have brotlipy installed, and Scrapy will use it, but "diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex 93a2ba7a1..49493e9c6 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -18,7 +18,7 @@ from typing import ()from warnings import warn-from lxml import etree+from lxml import etree # nosecfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Response, TextResponse@@ -26,7 +26,7 @@ from scrapy.selector import Selectorfrom scrapy.utils.python import re_rsearch, to_unicodeif TYPE_CHECKING:- from lxml._types import SupportsReadClose+ from lxml._types import SupportsReadClose # noseclogger = logging.getLogger(__name__)@@ -101,6 +101,7 @@ def xmliter_lxml(cast("SupportsReadClose[bytes]", reader),encoding=reader.encoding,events=("end", "start-ns"),+ resolve_entities=False,huge_tree=True,)selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.pyindex 8bf941eb2..7dcee3a2f 100644--- a/scrapy/utils/sitemap.py+++ b/scrapy/utils/sitemap.py@@ -8,7 +8,7 @@ SitemapSpider, its API is subject to change without notice.from typing import Any, Dict, Generator, Iterator, Optionalfrom urllib.parse import urljoin-import lxml.etree+import lxml.etree # nosecclass Sitemap:@@ -19,7 +19,7 @@ class Sitemap:xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False)- self._root = lxml.etree.fromstring(xmltext, parser=xmlp)+ self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosecrt = self._root.tagself.type = self._root.tag.split("}", 1)[1] if "}" in rt else rtdiff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.pyindex 9b637bdb0..42e5e9be4 100644--- a/scrapy/utils/versions.py+++ b/scrapy/utils/versions.py@@ -4,7 +4,7 @@ from typing import List, Tupleimport cryptographyimport cssselect-import lxml.etree+import lxml.etree # nosecimport parselimport twistedimport w3lib