Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: prevent IndexError in adaptive relocation with auto_save

Mubashir Rahimcd4cdc69e622
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 2b88af3..200cb43 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -671,7 +671,7 @@ class Selector(SelectorsGeneration):
element_data = self.retrieve(identifier or selector)
if element_data:
elements = self.relocate(element_data, percentage)
- if elements is not None and auto_save:
+ if elements and auto_save:
self.save(elements[0], identifier or selector)
return self.__handle_elements(elements)
diff --git a/tests/parser/test_adaptive.py b/tests/parser/test_adaptive.py
index 8d40f77..85881d1 100644
--- a/tests/parser/test_adaptive.py
+++ b/tests/parser/test_adaptive.py
@@ -56,6 +56,32 @@ class TestParserAdaptive:
assert relocated[0].has_class("new-class")
assert relocated[0].css(".new-description")[0].text == "Description 1"
+ def test_relocation_auto_save_no_match_above_threshold(self):
+ """Adaptive relocation with `auto_save=True` must not crash when no element
+ clears the `percentage` threshold (relocate() returns an empty list)."""
+ original_html = """
+ <div class="container">
+ <article class="product" id="target">
+ <h3>Widget</h3>
+ <p class="desc">A widget</p>
+ </article>
+ </div>
+ """
+ # Unrelated structure so nothing can match a high threshold
+ changed_html = "<html><body><span>totally unrelated content</span></body></html>"
+
+ old_page = Selector(original_html, url="example.com", adaptive=True)
+ new_page = Selector(changed_html, url="example.com", adaptive=True)
+
+ old_page.css("#target", identifier="target", auto_save=True)
+
+ # Before the fix this raised `IndexError: list index out of range` because the
+ # guard checked `elements is not None` but relocate() returns [] (never None).
+ result = new_page.css(
+ "#target", identifier="target", adaptive=True, auto_save=True, percentage=95
+ )
+ assert list(result) == []
+
@pytest.mark.asyncio
async def test_element_relocation_async(self):
"""Test relocating element after structure change in async mode"""