Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove the lxml version check for huge_tree on xmliter_lxml

Adrián Chavesce9d290eff8b
diff --git a/docs/news.rst b/docs/news.rst
index 525ddbf40..fab8b6f20 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -19,7 +19,7 @@ Scrapy 2.11.1 (unreleased)
To minimize the impact of this change on existing code,
:func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating
the node namespace with a prefix in the node name, and big files with
- highly nested trees.
+ highly nested trees when using libxml2 2.7+.
- Fixed regular expressions in the implementation of the
:func:`~scrapy.utils.response.open_in_browser` function.
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 8610e9b77..b6abe2e0c 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -19,7 +19,6 @@ from typing import (
from warnings import warn
from lxml import etree
-from packaging.version import Version
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse
@@ -31,12 +30,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-_LXML_VERSION = Version(etree.__version__)
-_LXML_HUGE_TREE_VERSION = Version("4.2")
-_ITERPARSE_KWARGS = {}
-if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION:
- _ITERPARSE_KWARGS["huge_tree"] = True
-
def xmliter(
obj: Union[Response, str, bytes], nodename: str
@@ -108,7 +101,7 @@ def xmliter_lxml(
reader,
encoding=reader.encoding,
events=("end", "start-ns"),
- **_ITERPARSE_KWARGS,
+ huge_tree=True,
)
selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
needs_namespace_resolution = not namespace and ":" in nodename