π·οΈ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: add edge case tests for filter, iterancestors, and find_similar
awanawonacec5acd68a3c
diff --git a/tests/parser/test_ancestor_navigation.py b/tests/parser/test_ancestor_navigation.pynew file mode 100644index 0000000..1814ec5--- /dev/null+++ b/tests/parser/test_ancestor_navigation.py@@ -0,0 +1,66 @@+"""+Tests for Selector.iterancestors() and Selector.find_ancestor() methods.+Target file: tests/parser/test_general.py (append to TestElementNavigation class)+"""+import pytest+from scrapling import Selector+++def nested_page():+ html = """+ <html><body>+ <div id="level1">+ <section id="level2" class="wrapper">+ <article id="level3" class="card">+ <p id="level4"><span id="target">deep text</span></p>+ </article>+ </section>+ </div>+ </body></html>+ """+ return Selector(html, adaptive=False)+++class TestAncestorNavigation:+ def test_iterancestors_returns_all_ancestors(self, nested_page):+ """iterancestors() should yield every ancestor up to <html>"""+ target = nested_page.css("#target")[0]+ ancestor_tags = [a.tag for a in target.iterancestors()]+ # Expected order: p β article β section β div β body β html+ assert ancestor_tags[:4] == ["p", "article", "section", "div"]+ assert "body" in ancestor_tags+ assert "html" in ancestor_tags++ def test_iterancestors_order_is_bottom_up(self, nested_page):+ """iterancestors() should start from the immediate parent, not the root"""+ target = nested_page.css("#target")[0]+ first_ancestor = next(target.iterancestors())+ assert first_ancestor.attrib.get("id") == "level4"++ def test_find_ancestor_returns_first_match(self, nested_page):+ """find_ancestor() should return the closest ancestor matching the predicate"""+ target = nested_page.css("#target")[0]+ # Looking for the nearest ancestor with class "card"+ result = target.find_ancestor(lambda el: el.has_class("card"))+ assert result is not None+ assert result.attrib.get("id") == "level3"++ def test_find_ancestor_returns_none_when_not_found(self, nested_page):+ """find_ancestor() should return None if no ancestor matches"""+ target = nested_page.css("#target")[0]+ result = target.find_ancestor(lambda el: el.has_class("nonexistent-class"))+ assert result is None++ def test_iterancestors_on_text_node_is_empty(self, nested_page):+ """iterancestors() on a text node should yield nothing (not raise)"""+ text_node = nested_page.css("#target::text")[0]+ ancestors = list(text_node.iterancestors())+ assert ancestors == []++ def test_find_ancestor_on_root_element_returns_none(self, nested_page):+ """find_ancestor() on the root <html> element should return None gracefully"""+ # html element has no ancestors+ html_el = nested_page.css("html")[0]+ result = html_el.find_ancestor(lambda el: True)+ assert result is Nonediff --git a/tests/parser/test_find_similar_advanced.py b/tests/parser/test_find_similar_advanced.pynew file mode 100644index 0000000..480f155--- /dev/null+++ b/tests/parser/test_find_similar_advanced.py@@ -0,0 +1,78 @@+"""+Tests for Selector.find_similar() with non-default parameters.+Target file: tests/parser/test_general.py (append to TestSimilarElements class)+"""+import pytest+from scrapling import Selector+++def product_page():+ html = """+ <html><body>+ <div class="product-list">+ <div class="product" data-category="fruit" data-price="10">+ <span class="name">Apple</span>+ </div>+ <div class="product" data-category="fruit" data-price="5">+ <span class="name">Banana</span>+ </div>+ <div class="product" data-category="veggie" data-price="3">+ <span class="name">Carrot</span>+ </div>+ <!-- Structurally similar but different tag β should NOT be found -->+ <section class="product" data-category="fruit" data-price="8">+ <span class="name">Grape</span>+ </section>+ </div>+ </body></html>+ """+ return Selector(html, adaptive=False)+++class TestFindSimilarAdvanced:+ def test_find_similar_default_finds_same_tag_siblings(self, product_page):+ """find_similar() with defaults should find div.product siblings, not the section"""+ first = product_page.css("div.product")[0]+ similar = first.find_similar()+ tags = [el.tag for el in similar]+ assert all(t == "div" for t in tags), "Should only return <div> elements"+ assert len(similar) == 2 # Banana and Carrot, not Grape (section)++ def test_find_similar_high_threshold_filters_more(self, product_page):+ """A higher similarity_threshold should return fewer (or equal) results"""+ first = product_page.css("div.product")[0]+ low_threshold = first.find_similar(similarity_threshold=0.1)+ high_threshold = first.find_similar(similarity_threshold=0.9)+ assert len(high_threshold) <= len(low_threshold)++ def test_find_similar_match_text_excludes_different_text(self, product_page):+ """match_text=True should factor in text content during similarity scoring"""+ first = product_page.css("div.product")[0] # Apple+ # With match_text=True and a high threshold, "Apple" vs "Banana"/"Carrot" text+ # should reduce similarity scores β result count may drop+ with_text = first.find_similar(similarity_threshold=0.8, match_text=True)+ without_text = first.find_similar(similarity_threshold=0.8, match_text=False)+ # match_text=True is stricter when text differs, so result should be <= without_text+ assert len(with_text) <= len(without_text)++ def test_find_similar_ignore_attributes_affects_matching(self, product_page):+ """Ignoring data-price should make more elements qualify as similar"""+ first = product_page.css("div.product")[0]+ # Ignore both data-price and data-category β only class matters β all 3 divs match+ ignore_all_data = first.find_similar(+ similarity_threshold=0.2,+ ignore_attributes=["data-price", "data-category"]+ )+ # Ignore nothing β data-category difference (fruit vs veggie) may reduce matches+ ignore_nothing = first.find_similar(+ similarity_threshold=0.9,+ ignore_attributes=[]+ )+ assert len(ignore_all_data) >= len(ignore_nothing)++ def test_find_similar_on_text_node_returns_empty(self, product_page):+ """find_similar() on a text node should return empty Selectors without raising"""+ text_node = product_page.css(".name::text")[0]+ result = text_node.find_similar()+ assert len(result) == 0diff --git a/tests/parser/test_selectors_filter.py b/tests/parser/test_selectors_filter.pynew file mode 100644index 0000000..9612abc--- /dev/null+++ b/tests/parser/test_selectors_filter.py@@ -0,0 +1,64 @@+"""+Tests for Selectors.filter() method edge cases.+Target file: tests/parser/test_parser_advanced.py (append to TestAdvancedSelectors class)+"""+import pytest+from scrapling import Selector, Selectors+++def page():+ html = """+ <html><body>+ <ul>+ <li class="item" data-value="10">Apple</li>+ <li class="item" data-value="5">Banana</li>+ <li class="item" data-value="20">Cherry</li>+ <li class="item disabled" data-value="0">Durian</li>+ </ul>+ </body></html>+ """+ return Selector(html, adaptive=False)+++class TestSelectorsFilter:+ def test_filter_basic(self, page):+ """filter() should return only elements matching the predicate"""+ items = page.css("li.item")+ expensive = items.filter(lambda el: int(el.attrib.get("data-value", 0)) >= 10)+ assert len(expensive) == 2+ texts = expensive.getall()+ assert any("Apple" in t for t in texts)+ assert any("Cherry" in t for t in texts)++ def test_filter_returns_empty_selectors_when_no_match(self, page):+ """filter() should return an empty Selectors (not None/exception) when nothing matches"""+ items = page.css("li.item")+ result = items.filter(lambda el: int(el.attrib.get("data-value", 0)) > 9999)+ assert isinstance(result, Selectors)+ assert len(result) == 0+ assert result.first is None++ def test_filter_all_pass(self, page):+ """filter() with always-True predicate should return all elements"""+ items = page.css("li.item")+ result = items.filter(lambda el: True)+ assert len(result) == len(items)++ def test_filter_chained(self, page):+ """filter() should be chainable β apply two filters in sequence"""+ items = page.css("li.item")+ # First: value > 0, then: not disabled+ result = (+ items+ .filter(lambda el: int(el.attrib.get("data-value", 0)) > 0)+ .filter(lambda el: not el.has_class("disabled"))+ )+ assert len(result) == 3 # Apple, Banana, Cherry (Durian is disabled AND value=0)++ def test_filter_on_empty_selectors(self):+ """filter() on an already-empty Selectors should not raise"""+ empty = Selectors()+ result = empty.filter(lambda el: True)+ assert isinstance(result, Selectors)+ assert len(result) == 0