Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

Refine complex tests and reorganize suite

Mika621779b84e54
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index 20d723d..4227e6b 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -224,6 +224,9 @@ class AutoScraper(object):
List of similar results
"""
+ if not wanted_list and not (wanted_dict and any(wanted_dict.values())):
+ raise ValueError("No targets were supplied")
+
soup = self._get_soup(url=url, html=html, request_args=request_args)
result_list = []
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..f6e3aaf
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,107 @@
+import sys
+from types import ModuleType
+from html.parser import HTMLParser
+
+class _Node:
+ def __init__(self, name, attrs, parent=None):
+ self.name = name
+ self.attrs = dict(attrs)
+ self.parent = parent
+ self.children = []
+ self.text = ""
+
+ def append_child(self, child):
+ self.children.append(child)
+ child.parent = self
+
+ def getText(self):
+ return self.text + "".join(c.getText() for c in self.children)
+
+ def findChildren(self, recursive=True):
+ result = []
+ for child in self.children:
+ result.append(child)
+ if recursive:
+ result.extend(child.findChildren(recursive))
+ return result
+
+ def findParent(self):
+ return self.parent
+
+ def _attr_match(self, child, attrs):
+ from autoscraper.utils import FuzzyText
+
+ for key, val in (attrs or {}).items():
+ actual = child.attrs.get(key, "")
+ if isinstance(actual, list):
+ actual = " ".join(actual)
+
+ if isinstance(val, FuzzyText):
+ if not val.search(actual):
+ return False
+ elif actual != val:
+ return False
+ return True
+
+ def findAll(self, name=None, attrs=None, recursive=True):
+ result = []
+ for child in self.children:
+ if (name is None or child.name == name) and self._attr_match(child, attrs):
+ result.append(child)
+ if recursive:
+ result.extend(child.findAll(name, attrs, recursive))
+ return result
+
+ def find_all(self, name=None, attrs=None, text=None, recursive=True):
+ if text:
+ res = []
+ if self.text.strip():
+ res.append(self.text)
+ for child in self.children:
+ if recursive:
+ res.extend(child.find_all(text=True, recursive=True))
+ elif child.text.strip():
+ res.append(child.text)
+ return res
+ return self.findAll(name, attrs, recursive)
+
+class _Parser(HTMLParser):
+ def __init__(self):
+ super().__init__()
+ self.root = _Node("[document]", {})
+ self.current = self.root
+
+ def handle_starttag(self, tag, attrs):
+ node = _Node(tag, attrs)
+ self.current.append_child(node)
+ self.current = node
+
+ def handle_endtag(self, tag):
+ if self.current.parent:
+ self.current = self.current.parent
+
+ def handle_data(self, data):
+ self.current.text += data
+
+class BeautifulSoup(_Node):
+ def __init__(self, html, parser):
+ p = _Parser()
+ p.feed(html)
+ super().__init__(p.root.name, p.root.attrs)
+ self.children = p.root.children
+ for c in self.children:
+ c.parent = self
+
+bs4_mod = ModuleType("bs4")
+bs4_mod.BeautifulSoup = BeautifulSoup
+sys.modules.setdefault("bs4", bs4_mod)
+
+class _Response:
+ def __init__(self, text=""):
+ self.encoding = "utf-8"
+ self.headers = {"Content-Type": "text/html"}
+ self.text = text
+
+requests_mod = ModuleType("requests")
+requests_mod.get = lambda url, headers=None, **kw: _Response()
+sys.modules.setdefault("requests", requests_mod)
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/test_complex_features.py b/tests/integration/test_complex_features.py
new file mode 100644
index 0000000..57514be
--- /dev/null
+++ b/tests/integration/test_complex_features.py
@@ -0,0 +1,77 @@
+import pytest
+import re
+from autoscraper import AutoScraper
+
+HTML_COMPLEX = """
+<div id="main">
+ <ul class="fruits">
+ <li class="item"><span class="name">Banana</span><a href="/banana" class="link">More</a></li>
+ <li class="item"><span class="name">Apple</span><a href="/apple" class="link">More</a></li>
+ <li class="item"><span class="name">Orange</span><a href="/orange" class="link">More</a></li>
+ <li class="item"><span class="name">Banana</span></li>
+ </ul>
+ <p class="info">Fresh fruits</p>
+ <a class="external" href="/shop">Shop Now</a>
+</div>
+"""
+
+
+def test_extract_relative_link():
+ scraper = AutoScraper()
+ url = "https://example.com/index.html"
+ result = scraper.build(url=url, html=HTML_COMPLEX, wanted_list=["https://example.com/apple"])
+ assert "https://example.com/apple" in result
+ similar = scraper.get_result_similar(
+ url=url, html=HTML_COMPLEX, contain_sibling_leaves=True, unique=True
+ )
+ assert set(similar) == {
+ "https://example.com/banana",
+ "https://example.com/apple",
+ "https://example.com/orange",
+ }
+ exact = scraper.get_result_exact(url=url, html=HTML_COMPLEX)
+ assert exact == ["https://example.com/apple"]
+
+
+def test_build_with_regex():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_COMPLEX, wanted_list=[re.compile("Ban.*")])
+ result = scraper.get_result_exact(html=HTML_COMPLEX)
+ assert "Banana" in result[0]
+
+
+def test_update_appends_rules():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_COMPLEX, wanted_list=["Banana"])
+ count = len(scraper.stack_list)
+ scraper.build(html=HTML_COMPLEX, wanted_list=["Apple"], update=True)
+ assert len(scraper.stack_list) == count + 1
+
+
+def test_remove_rules():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_COMPLEX, wanted_list=["Banana"])
+ scraper.build(html=HTML_COMPLEX, wanted_list=["Apple"], update=True)
+ rule_ids = [s["stack_id"] for s in scraper.stack_list]
+ to_remove = rule_ids[0]
+ scraper.remove_rules([to_remove])
+ remaining = [s["stack_id"] for s in scraper.stack_list]
+ assert to_remove not in remaining
+ assert len(remaining) == len(rule_ids) - 1
+
+
+def test_keep_blank_returns_empty():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_COMPLEX, wanted_list=["/shop"])
+ html_blank = HTML_COMPLEX.replace('href="/shop"', 'href=""')
+ result = scraper.get_result_exact(html=html_blank, keep_blank=True)
+ assert result == [""]
+
+
+def test_attr_fuzz_ratio():
+ html_base = '<div><a class="btn-primary" href="/item">Buy</a></div>'
+ html_variant = '<div><a class="btn-prime" href="/item">Buy</a></div>'
+ scraper = AutoScraper()
+ scraper.build(html=html_base, wanted_list=["Buy"])
+ res = scraper.get_result_exact(html=html_variant, attr_fuzz_ratio=0.8)
+ assert res == ["Buy"]
diff --git a/tests/integration/test_real_world.py b/tests/integration/test_real_world.py
new file mode 100644
index 0000000..a810acf
--- /dev/null
+++ b/tests/integration/test_real_world.py
@@ -0,0 +1,83 @@
+import re
+from autoscraper import AutoScraper
+
+HTML_PAGE_1 = """
+<div id='product'>
+ <h1 class='title'>Sony PlayStation 4 PS4 Pro 1TB 4K Console - Black</h1>
+ <span class='price'>US $349.99</span>
+ <div class='rating'><span class='value'>4.8</span></div>
+ <div class='note'>See details</div>
+</div>
+"""
+
+HTML_PAGE_2 = """
+<div id='product'>
+ <h1 class='title'>Acer Predator Helios 300 15.6'' 144Hz FHD Laptop i7-9750H 16GB 512GB GTX 1660 Ti</h1>
+ <span class='price'>US $1,229.49</span>
+ <div class='rating'><span class='value'>5.0</span></div>
+ <div class='note'>See details</div>
+</div>
+"""
+
+HTML_WALMART_1 = "<div class='price'>$8.95</div>"
+HTML_WALMART_2 = "<div class='price'>$7.00</div>"
+HTML_ETSY_1 = "<span class='amount'>$12.50+</span>"
+HTML_ETSY_2 = "<span class='amount'>$60.00</span>"
+
+
+def test_grouping_and_rule_removal():
+ scraper = AutoScraper()
+ wanted = [
+ "Sony PlayStation 4 PS4 Pro 1TB 4K Console - Black",
+ "US $349.99",
+ "4.8",
+ "See details",
+ ]
+ scraper.build(html=HTML_PAGE_1, wanted_list=wanted)
+ grouped = scraper.get_result_exact(html=HTML_PAGE_2, grouped=True)
+ unwanted = [r for r, v in grouped.items() if v == ["See details"]]
+ scraper.remove_rules(unwanted)
+ result = scraper.get_result_exact(html=HTML_PAGE_2)
+ assert result == [
+ "Acer Predator Helios 300 15.6'' 144Hz FHD Laptop i7-9750H 16GB 512GB GTX 1660 Ti",
+ "US $1,229.49",
+ "5.0",
+ ]
+
+
+def test_incremental_learning_multiple_sites():
+ scraper = AutoScraper()
+ data = [
+ (HTML_PAGE_1, ["US $349.99"]),
+ (HTML_WALMART_1, ["$8.95"]),
+ (HTML_ETSY_1, ["$12.50+"]),
+ ]
+ for html, wanted in data:
+ scraper.build(html=html, wanted_list=wanted, update=True)
+ assert "US $1,229.49" in scraper.get_result_exact(html=HTML_PAGE_2)
+ assert "$7.00" in scraper.get_result_exact(html=HTML_WALMART_2)
+ assert "$60.00" in scraper.get_result_exact(html=HTML_ETSY_2)
+
+
+def test_attr_fuzz_ratio_realistic():
+ base = "<div><a class='btn-primary-action' href='/buy'>Buy</a></div>"
+ variant = "<div><a class='btn-prim-action' href='/buy'>Buy</a></div>"
+ scraper = AutoScraper()
+ scraper.build(html=base, wanted_list=["Buy"])
+ assert scraper.get_result_exact(html=variant, attr_fuzz_ratio=0.8) == ["Buy"]
+
+
+def test_regex_name_extraction():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_PAGE_1, wanted_list=[re.compile(r".*PlayStation.*Console.*")])
+ result = scraper.get_result_exact(html=HTML_PAGE_1)
+ assert any("PlayStation" in r for r in result)
+
+
+def test_keep_blank_for_missing_rating():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_PAGE_1, wanted_list=["4.8"])
+ html_no_rating = HTML_PAGE_2.replace("5.0", "")
+ res = scraper.get_result_exact(html=html_no_rating, keep_blank=True)
+ assert res == [""]
+
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/unit/test_additional_features.py b/tests/unit/test_additional_features.py
new file mode 100644
index 0000000..c0fdc98
--- /dev/null
+++ b/tests/unit/test_additional_features.py
@@ -0,0 +1,41 @@
+from autoscraper import AutoScraper
+
+HTML = "<ul><li>Banana</li><li>Apple</li><li>Orange</li></ul>"
+HTML_DUP = "<ul><li>Banana</li><li>Banana</li></ul>"
+
+
+def test_text_fuzz_ratio_partial():
+ scraper = AutoScraper()
+ scraper.build(html="<ul><li>Banana</li></ul>", wanted_list=["Banan"], text_fuzz_ratio=0.8)
+ assert scraper.get_result_exact(html="<ul><li>Banana</li></ul>") == ["Banana"]
+
+
+def test_set_rule_aliases():
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_list=["Banana"])
+ rule_id = scraper.stack_list[0]["stack_id"]
+ scraper.set_rule_aliases({rule_id: "fruit"})
+ result = scraper.get_result_similar(html=HTML, group_by_alias=True, contain_sibling_leaves=True)
+ assert result == {"fruit": ["Banana", "Apple", "Orange"]}
+
+
+def test_grouped_results_by_rule():
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_list=["Banana"])
+ rule_id = scraper.stack_list[0]["stack_id"]
+ result = scraper.get_result_similar(html=HTML, grouped=True, contain_sibling_leaves=True)
+ assert result == {rule_id: ["Banana", "Apple", "Orange"]}
+
+
+def test_similar_unique_false():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_DUP, wanted_list=["Banana"])
+ result = scraper.get_result_similar(html=HTML_DUP, unique=False)
+ assert result == ["Banana", "Banana"]
+
+
+def test_similar_keep_order():
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_list=["Banana"])
+ result = scraper.get_result_similar(html=HTML, contain_sibling_leaves=True, keep_order=True)
+ assert result == ["Banana", "Apple", "Orange"]
diff --git a/tests/unit/test_build.py b/tests/unit/test_build.py
new file mode 100644
index 0000000..65d5e05
--- /dev/null
+++ b/tests/unit/test_build.py
@@ -0,0 +1,18 @@
+import pytest
+from autoscraper import AutoScraper
+
+HTML = "<ul><li>Banana</li><li>Apple</li><li>Orange</li></ul>"
+
+
+def test_build_requires_targets():
+ scraper = AutoScraper()
+ with pytest.raises(ValueError):
+ scraper.build(html=HTML)
+
+
+def test_build_and_get_result_similar():
+ scraper = AutoScraper()
+ result = scraper.build(html=HTML, wanted_list=["Banana"])
+ assert result == ["Banana"]
+ similar = scraper.get_result_similar(html=HTML, contain_sibling_leaves=True)
+ assert similar == ["Banana", "Apple", "Orange"]
diff --git a/tests/unit/test_features.py b/tests/unit/test_features.py
new file mode 100644
index 0000000..414bc27
--- /dev/null
+++ b/tests/unit/test_features.py
@@ -0,0 +1,57 @@
+import pytest
+
+from autoscraper import AutoScraper
+
+HTML = "<ul><li>Banana</li><li>Apple</li><li>Orange</li></ul>"
+HTML_COMPLEX_ORDER = """
+<div class='products'>
+ <h2>Banana</h2>
+ <p class='price'>$1</p>
+ <h2>Apple</h2>
+ <p class='price'>$2</p>
+</div>
+"""
+
+
+def test_get_result_exact_order():
+ scraper = AutoScraper()
+ scraper.build(html=HTML_COMPLEX_ORDER, wanted_list=["Banana", "$2"])
+ assert scraper.get_result_exact(html=HTML_COMPLEX_ORDER) == ["Banana", "$2"]
+
+
+def test_group_by_alias():
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_dict={"fruit": ["Banana"]})
+ similar = scraper.get_result_similar(
+ html=HTML, group_by_alias=True, contain_sibling_leaves=True, unique=True
+ )
+ assert similar == {"fruit": ["Banana", "Apple", "Orange"]}
+
+
+def test_save_and_load(tmp_path):
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_list=["Banana"])
+ file_path = tmp_path / "model.json"
+ scraper.save(file_path)
+ new_scraper = AutoScraper()
+ new_scraper.load(file_path)
+ assert new_scraper.get_result_exact(html=HTML) == scraper.get_result_exact(html=HTML)
+
+
+def test_keep_rules():
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_list=["Banana"])
+ first_rule = scraper.stack_list[0]["stack_id"]
+ scraper.build(html=HTML, wanted_list=["Apple"], update=True)
+ second_rule = scraper.stack_list[1]["stack_id"]
+ scraper.keep_rules([second_rule])
+ assert len(scraper.stack_list) == 1
+ assert scraper.stack_list[0]["stack_id"] == second_rule
+
+
+def test_get_result_combined():
+ scraper = AutoScraper()
+ scraper.build(html=HTML, wanted_list=["Banana"])
+ similar, exact = scraper.get_result(html=HTML)
+ assert exact == ["Banana"]
+ assert similar == ["Banana"]