Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let parsel determine the type of nested selectors (#8040)

Adrianac738e07d573
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 0bd6d49a2..0fa2575b7 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -88,7 +88,7 @@ class Selector(_ParselSelector, object_ref):
# Any other response, e.g. a plain-text one, keeps type unset, so that
# parsel determines it from the body.
- if type is None:
+ if type is None and root is _NOT_SET:
if isinstance(response, XmlResponse):
type = "xml" # noqa: A001
elif isinstance(response, JsonResponse):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index cf8d9686b..aa43ce249 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -167,7 +167,7 @@ class TestJMESPath:
)
assert resp.jmespath("html").xpath("//div/a/text()").getall() == ["a", "b", "d"]
assert resp.jmespath("html").css("div > b").getall() == ["<b>f</b>"]
- assert resp.jmespath("content").jmespath("name.age").get() == "18"
+ assert resp.jmespath("content").jmespath("name.age").get() == 18 # type: ignore[comparison-overlap]
def test_html_has_json(self) -> None:
body = """
@@ -212,7 +212,7 @@ class TestJMESPath:
"C",
"D",
]
- assert resp.xpath("//div/content").jmespath("total").get() == "4"
+ assert resp.xpath("//div/content").jmespath("total").get() == 4 # type: ignore[comparison-overlap]
def test_jmestpath_with_re(self) -> None:
body = """