Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Avoid calling iterparse twice

Adrián Chavesa49c8762dd16
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index f239630f4..8610e9b77 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -12,7 +12,6 @@ from typing import (
List,
Literal,
Optional,
- Tuple,
Union,
cast,
overload,
@@ -97,43 +96,38 @@ def xmliter(
yield Selector(text=nodetext, type="xml")
-def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]:
- if ":" not in element_name:
- return element_name, None, None
- reader: "SupportsReadClose[bytes]" = _StreamReader(data)
- input_prefix, element_name = element_name.split(":", maxsplit=1)
- ns_iterator = etree.iterparse(
- reader,
- encoding=reader.encoding,
- events=("start-ns",),
- **_ITERPARSE_KWARGS,
- )
- for event, (prefix, namespace) in ns_iterator:
- if prefix != input_prefix:
- continue
- return element_name, prefix, namespace
- return f"{input_prefix}:{element_name}", None, None
-
-
def xmliter_lxml(
obj: Union[Response, str, bytes],
nodename: str,
namespace: Optional[str] = None,
prefix: str = "x",
) -> Generator[Selector, Any, None]:
- if not namespace:
- nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj)
-
reader: "SupportsReadClose[bytes]" = _StreamReader(obj)
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
iterable = etree.iterparse(
reader,
- tag=tag,
encoding=reader.encoding,
+ events=("end", "start-ns"),
**_ITERPARSE_KWARGS,
)
selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
- for _, node in iterable:
+ needs_namespace_resolution = not namespace and ":" in nodename
+ if needs_namespace_resolution:
+ prefix, nodename = nodename.split(":", maxsplit=1)
+ for event, data in iterable:
+ if event == "start-ns":
+ if needs_namespace_resolution:
+ _prefix, _namespace = data
+ if _prefix != prefix:
+ continue
+ namespace = _namespace
+ needs_namespace_resolution = False
+ selxpath = f"//{prefix}:{nodename}"
+ tag = f"{{{namespace}}}{nodename}"
+ continue
+ node = data
+ if node.tag != tag:
+ continue
nodetext = etree.tostring(node, encoding="unicode")
node.clear()
xs = Selector(text=nodetext, type="xml")