Scrapy, a fast high-level web crawling & scraping framework for Python.
added tests for jmespath
Jalil SA3d29f20fc202
diff --git a/tests/test_selector.py b/tests/test_selector.pyindex febae46ac..b0deb5c99 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -108,3 +108,141 @@ class SelectorTestCase(unittest.TestCase):def test_selector_bad_args(self):with self.assertRaisesRegex(ValueError, "received both response and text"):Selector(TextResponse(url="http://example.com", body=b""), text="")+++class JMESPathTestCase(unittest.TestCase):+ def test_json_has_html(self) -> None:+ """Sometimes the information is returned in a json wrapper"""+ body = """+ {+ "content": [+ {+ "name": "A",+ "value": "a"+ },+ {+ "name": {+ "age": 18+ },+ "value": "b"+ },+ {+ "name": "C",+ "value": "c"+ },+ {+ "name": "<a>D</a>",+ "value": "<div>d</div>"+ }+ ],+ "html": "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>"+ }+ """+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")+ self.assertEqual(+ resp.jmespath("html").get(),+ "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>",+ )+ self.assertEqual(+ resp.jmespath("html").xpath("//div/a/text()").getall(),+ ["a", "b", "d"],+ )+ self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["<b>f</b>"])+ self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18")++ def test_html_has_json(self) -> None:+ body = """+ <div>+ <h1>Information</h1>+ <content>+ {+ "user": [+ {+ "name": "A",+ "age": 18+ },+ {+ "name": "B",+ "age": 32+ },+ {+ "name": "C",+ "age": 22+ },+ {+ "name": "D",+ "age": 25+ }+ ],+ "total": 4,+ "status": "ok"+ }+ </content>+ </div>+ """+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")+ self.assertEqual(+ resp.xpath("//div/content/text()").jmespath("user[*].name").getall(),+ ["A", "B", "C", "D"],+ )+ self.assertEqual(+ resp.xpath("//div/content").jmespath("user[*].name").getall(),+ ["A", "B", "C", "D"],+ )+ self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4")++ def test_jmestpath_with_re(self) -> None:+ body = """+ <div>+ <h1>Information</h1>+ <content>+ {+ "user": [+ {+ "name": "A",+ "age": 18+ },+ {+ "name": "B",+ "age": 32+ },+ {+ "name": "C",+ "age": 22+ },+ {+ "name": "D",+ "age": 25+ }+ ],+ "total": 4,+ "status": "ok"+ }+ </content>+ </div>+ """+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")+ self.assertEqual(+ resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"),+ ["A", "B", "C", "D"],+ )+ self.assertEqual(+ resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"),+ ["A", "B", "C", "D"],+ )++ self.assertEqual(+ resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), []+ )++ self.assertEqual(+ resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"),+ None,+ )++ self.assertEqual(+ resp.xpath("//div/content")+ .jmespath("user[*].age.to_string(@)")+ .re(r"(\d+)"),+ ["18", "32", "22", "25"],+ )