Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Moving parser tests to a better structure

Karim shoaird632bb14af40
diff --git a/tests/parser/__init__.py b/tests/parser/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/parser/test_automatch.py b/tests/parser/test_automatch.py
new file mode 100644
index 0000000..1e78e87
--- /dev/null
+++ b/tests/parser/test_automatch.py
@@ -0,0 +1,56 @@
+import unittest
+
+from scrapling import Adaptor
+
+
+class TestParserAutoMatch(unittest.TestCase):
+
+ def test_element_relocation(self):
+ """Test relocating element after structure change"""
+ original_html = '''
+ <div class="container">
+ <section class="products">
+ <article class="product" id="p1">
+ <h3>Product 1</h3>
+ <p class="description">Description 1</p>
+ </article>
+ <article class="product" id="p2">
+ <h3>Product 2</h3>
+ <p class="description">Description 2</p>
+ </article>
+ </section>
+ </div>
+ '''
+ changed_html = '''
+ <div class="new-container">
+ <div class="product-wrapper">
+ <section class="products">
+ <article class="product new-class" data-id="p1">
+ <div class="product-info">
+ <h3>Product 1</h3>
+ <p class="new-description">Description 1</p>
+ </div>
+ </article>
+ <article class="product new-class" data-id="p2">
+ <div class="product-info">
+ <h3>Product 2</h3>
+ <p class="new-description">Description 2</p>
+ </div>
+ </article>
+ </section>
+ </div>
+ </div>
+ '''
+
+ old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
+ new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
+
+ # 'p1' was used as ID and now it's not and all the path elements have changes
+ # Also at the same time testing auto-match vs combined selectors
+ _ = old_page.css('#p1, #p2', auto_save=True)[0]
+ relocated = new_page.css('#p1', auto_match=True)
+
+ self.assertIsNotNone(relocated)
+ self.assertEqual(relocated[0].attrib['data-id'], 'p1')
+ self.assertTrue(relocated[0].has_class('new-class'))
+ self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
diff --git a/tests/test_parser_functions.py b/tests/parser/test_general.py
similarity index 82%
rename from tests/test_parser_functions.py
rename to tests/parser/test_general.py
index 88013ad..4e746ec 100644
--- a/tests/test_parser_functions.py
+++ b/tests/parser/test_general.py
@@ -112,7 +112,7 @@ class TestParser(unittest.TestCase):
def test_find_similar_elements(self):
"""Test Finding similar elements of an element"""
- first_product = self.page.css('.product')[0]
+ first_product = self.page.css_first('.product')
similar_products = first_product.find_similar()
self.assertEqual(len(similar_products), 2)
@@ -265,56 +265,6 @@ class TestParser(unittest.TestCase):
self.assertEqual(attr_json, {'jsonable': 'data'})
self.assertEqual(type(self.page.css('#products')[0].attrib.json_string), bytes)
- def test_element_relocation(self):
- """Test relocating element after structure change"""
- original_html = '''
- <div class="container">
- <section class="products">
- <article class="product" id="p1">
- <h3>Product 1</h3>
- <p class="description">Description 1</p>
- </article>
- <article class="product" id="p2">
- <h3>Product 2</h3>
- <p class="description">Description 2</p>
- </article>
- </section>
- </div>
- '''
- changed_html = '''
- <div class="new-container">
- <div class="product-wrapper">
- <section class="products">
- <article class="product new-class" data-id="p1">
- <div class="product-info">
- <h3>Product 1</h3>
- <p class="new-description">Description 1</p>
- </div>
- </article>
- <article class="product new-class" data-id="p2">
- <div class="product-info">
- <h3>Product 2</h3>
- <p class="new-description">Description 2</p>
- </div>
- </article>
- </section>
- </div>
- </div>
- '''
-
- old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
- new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
-
- # 'p1' was used as ID and now it's not and all the path elements have changes
- # Also at the same time testing auto-match vs combined selectors
- _ = old_page.css('#p1, #p2', auto_save=True)[0]
- relocated = new_page.css('#p1', auto_match=True)
-
- self.assertIsNotNone(relocated)
- self.assertEqual(relocated[0].attrib['data-id'], 'p1')
- self.assertTrue(relocated[0].has_class('new-class'))
- self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
-
def test_performance(self):
"""Test parsing and selecting speed"""
import time