A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Refine complex tests and reorganize suite
Mika621779b84e54
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex 20d723d..4227e6b 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -224,6 +224,9 @@ class AutoScraper(object):List of similar results"""+ if not wanted_list and not (wanted_dict and any(wanted_dict.values())):+ raise ValueError("No targets were supplied")+soup = self._get_soup(url=url, html=html, request_args=request_args)result_list = []diff --git a/tests/__init__.py b/tests/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tests/conftest.py b/tests/conftest.pynew file mode 100644index 0000000..f6e3aaf--- /dev/null+++ b/tests/conftest.py@@ -0,0 +1,107 @@+import sys+from types import ModuleType+from html.parser import HTMLParser++class _Node:+ def __init__(self, name, attrs, parent=None):+ self.name = name+ self.attrs = dict(attrs)+ self.parent = parent+ self.children = []+ self.text = ""++ def append_child(self, child):+ self.children.append(child)+ child.parent = self++ def getText(self):+ return self.text + "".join(c.getText() for c in self.children)++ def findChildren(self, recursive=True):+ result = []+ for child in self.children:+ result.append(child)+ if recursive:+ result.extend(child.findChildren(recursive))+ return result++ def findParent(self):+ return self.parent++ def _attr_match(self, child, attrs):+ from autoscraper.utils import FuzzyText++ for key, val in (attrs or {}).items():+ actual = child.attrs.get(key, "")+ if isinstance(actual, list):+ actual = " ".join(actual)++ if isinstance(val, FuzzyText):+ if not val.search(actual):+ return False+ elif actual != val:+ return False+ return True++ def findAll(self, name=None, attrs=None, recursive=True):+ result = []+ for child in self.children:+ if (name is None or child.name == name) and self._attr_match(child, attrs):+ result.append(child)+ if recursive:+ result.extend(child.findAll(name, attrs, recursive))+ return result++ def find_all(self, name=None, attrs=None, text=None, recursive=True):+ if text:+ res = []+ if self.text.strip():+ res.append(self.text)+ for child in self.children:+ if recursive:+ res.extend(child.find_all(text=True, recursive=True))+ elif child.text.strip():+ res.append(child.text)+ return res+ return self.findAll(name, attrs, recursive)++class _Parser(HTMLParser):+ def __init__(self):+ super().__init__()+ self.root = _Node("[document]", {})+ self.current = self.root++ def handle_starttag(self, tag, attrs):+ node = _Node(tag, attrs)+ self.current.append_child(node)+ self.current = node++ def handle_endtag(self, tag):+ if self.current.parent:+ self.current = self.current.parent++ def handle_data(self, data):+ self.current.text += data++class BeautifulSoup(_Node):+ def __init__(self, html, parser):+ p = _Parser()+ p.feed(html)+ super().__init__(p.root.name, p.root.attrs)+ self.children = p.root.children+ for c in self.children:+ c.parent = self++bs4_mod = ModuleType("bs4")+bs4_mod.BeautifulSoup = BeautifulSoup+sys.modules.setdefault("bs4", bs4_mod)++class _Response:+ def __init__(self, text=""):+ self.encoding = "utf-8"+ self.headers = {"Content-Type": "text/html"}+ self.text = text++requests_mod = ModuleType("requests")+requests_mod.get = lambda url, headers=None, **kw: _Response()+sys.modules.setdefault("requests", requests_mod)diff --git a/tests/integration/__init__.py b/tests/integration/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tests/integration/test_complex_features.py b/tests/integration/test_complex_features.pynew file mode 100644index 0000000..57514be--- /dev/null+++ b/tests/integration/test_complex_features.py@@ -0,0 +1,77 @@+import pytest+import re+from autoscraper import AutoScraper++HTML_COMPLEX = """+<div id="main">+ <ul class="fruits">+ <li class="item"><span class="name">Banana</span><a href="/banana" class="link">More</a></li>+ <li class="item"><span class="name">Apple</span><a href="/apple" class="link">More</a></li>+ <li class="item"><span class="name">Orange</span><a href="/orange" class="link">More</a></li>+ <li class="item"><span class="name">Banana</span></li>+ </ul>+ <p class="info">Fresh fruits</p>+ <a class="external" href="/shop">Shop Now</a>+</div>+"""+++def test_extract_relative_link():+ scraper = AutoScraper()+ url = "https://example.com/index.html"+ result = scraper.build(url=url, html=HTML_COMPLEX, wanted_list=["https://example.com/apple"])+ assert "https://example.com/apple" in result+ similar = scraper.get_result_similar(+ url=url, html=HTML_COMPLEX, contain_sibling_leaves=True, unique=True+ )+ assert set(similar) == {+ "https://example.com/banana",+ "https://example.com/apple",+ "https://example.com/orange",+ }+ exact = scraper.get_result_exact(url=url, html=HTML_COMPLEX)+ assert exact == ["https://example.com/apple"]+++def test_build_with_regex():+ scraper = AutoScraper()+ scraper.build(html=HTML_COMPLEX, wanted_list=[re.compile("Ban.*")])+ result = scraper.get_result_exact(html=HTML_COMPLEX)+ assert "Banana" in result[0]+++def test_update_appends_rules():+ scraper = AutoScraper()+ scraper.build(html=HTML_COMPLEX, wanted_list=["Banana"])+ count = len(scraper.stack_list)+ scraper.build(html=HTML_COMPLEX, wanted_list=["Apple"], update=True)+ assert len(scraper.stack_list) == count + 1+++def test_remove_rules():+ scraper = AutoScraper()+ scraper.build(html=HTML_COMPLEX, wanted_list=["Banana"])+ scraper.build(html=HTML_COMPLEX, wanted_list=["Apple"], update=True)+ rule_ids = [s["stack_id"] for s in scraper.stack_list]+ to_remove = rule_ids[0]+ scraper.remove_rules([to_remove])+ remaining = [s["stack_id"] for s in scraper.stack_list]+ assert to_remove not in remaining+ assert len(remaining) == len(rule_ids) - 1+++def test_keep_blank_returns_empty():+ scraper = AutoScraper()+ scraper.build(html=HTML_COMPLEX, wanted_list=["/shop"])+ html_blank = HTML_COMPLEX.replace('href="/shop"', 'href=""')+ result = scraper.get_result_exact(html=html_blank, keep_blank=True)+ assert result == [""]+++def test_attr_fuzz_ratio():+ html_base = '<div><a class="btn-primary" href="/item">Buy</a></div>'+ html_variant = '<div><a class="btn-prime" href="/item">Buy</a></div>'+ scraper = AutoScraper()+ scraper.build(html=html_base, wanted_list=["Buy"])+ res = scraper.get_result_exact(html=html_variant, attr_fuzz_ratio=0.8)+ assert res == ["Buy"]diff --git a/tests/integration/test_real_world.py b/tests/integration/test_real_world.pynew file mode 100644index 0000000..a810acf--- /dev/null+++ b/tests/integration/test_real_world.py@@ -0,0 +1,83 @@+import re+from autoscraper import AutoScraper++HTML_PAGE_1 = """+<div id='product'>+ <h1 class='title'>Sony PlayStation 4 PS4 Pro 1TB 4K Console - Black</h1>+ <span class='price'>US $349.99</span>+ <div class='rating'><span class='value'>4.8</span></div>+ <div class='note'>See details</div>+</div>+"""++HTML_PAGE_2 = """+<div id='product'>+ <h1 class='title'>Acer Predator Helios 300 15.6'' 144Hz FHD Laptop i7-9750H 16GB 512GB GTX 1660 Ti</h1>+ <span class='price'>US $1,229.49</span>+ <div class='rating'><span class='value'>5.0</span></div>+ <div class='note'>See details</div>+</div>+"""++HTML_WALMART_1 = "<div class='price'>$8.95</div>"+HTML_WALMART_2 = "<div class='price'>$7.00</div>"+HTML_ETSY_1 = "<span class='amount'>$12.50+</span>"+HTML_ETSY_2 = "<span class='amount'>$60.00</span>"+++def test_grouping_and_rule_removal():+ scraper = AutoScraper()+ wanted = [+ "Sony PlayStation 4 PS4 Pro 1TB 4K Console - Black",+ "US $349.99",+ "4.8",+ "See details",+ ]+ scraper.build(html=HTML_PAGE_1, wanted_list=wanted)+ grouped = scraper.get_result_exact(html=HTML_PAGE_2, grouped=True)+ unwanted = [r for r, v in grouped.items() if v == ["See details"]]+ scraper.remove_rules(unwanted)+ result = scraper.get_result_exact(html=HTML_PAGE_2)+ assert result == [+ "Acer Predator Helios 300 15.6'' 144Hz FHD Laptop i7-9750H 16GB 512GB GTX 1660 Ti",+ "US $1,229.49",+ "5.0",+ ]+++def test_incremental_learning_multiple_sites():+ scraper = AutoScraper()+ data = [+ (HTML_PAGE_1, ["US $349.99"]),+ (HTML_WALMART_1, ["$8.95"]),+ (HTML_ETSY_1, ["$12.50+"]),+ ]+ for html, wanted in data:+ scraper.build(html=html, wanted_list=wanted, update=True)+ assert "US $1,229.49" in scraper.get_result_exact(html=HTML_PAGE_2)+ assert "$7.00" in scraper.get_result_exact(html=HTML_WALMART_2)+ assert "$60.00" in scraper.get_result_exact(html=HTML_ETSY_2)+++def test_attr_fuzz_ratio_realistic():+ base = "<div><a class='btn-primary-action' href='/buy'>Buy</a></div>"+ variant = "<div><a class='btn-prim-action' href='/buy'>Buy</a></div>"+ scraper = AutoScraper()+ scraper.build(html=base, wanted_list=["Buy"])+ assert scraper.get_result_exact(html=variant, attr_fuzz_ratio=0.8) == ["Buy"]+++def test_regex_name_extraction():+ scraper = AutoScraper()+ scraper.build(html=HTML_PAGE_1, wanted_list=[re.compile(r".*PlayStation.*Console.*")])+ result = scraper.get_result_exact(html=HTML_PAGE_1)+ assert any("PlayStation" in r for r in result)+++def test_keep_blank_for_missing_rating():+ scraper = AutoScraper()+ scraper.build(html=HTML_PAGE_1, wanted_list=["4.8"])+ html_no_rating = HTML_PAGE_2.replace("5.0", "")+ res = scraper.get_result_exact(html=html_no_rating, keep_blank=True)+ assert res == [""]+diff --git a/tests/unit/__init__.py b/tests/unit/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tests/unit/test_additional_features.py b/tests/unit/test_additional_features.pynew file mode 100644index 0000000..c0fdc98--- /dev/null+++ b/tests/unit/test_additional_features.py@@ -0,0 +1,41 @@+from autoscraper import AutoScraper++HTML = "<ul><li>Banana</li><li>Apple</li><li>Orange</li></ul>"+HTML_DUP = "<ul><li>Banana</li><li>Banana</li></ul>"+++def test_text_fuzz_ratio_partial():+ scraper = AutoScraper()+ scraper.build(html="<ul><li>Banana</li></ul>", wanted_list=["Banan"], text_fuzz_ratio=0.8)+ assert scraper.get_result_exact(html="<ul><li>Banana</li></ul>") == ["Banana"]+++def test_set_rule_aliases():+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_list=["Banana"])+ rule_id = scraper.stack_list[0]["stack_id"]+ scraper.set_rule_aliases({rule_id: "fruit"})+ result = scraper.get_result_similar(html=HTML, group_by_alias=True, contain_sibling_leaves=True)+ assert result == {"fruit": ["Banana", "Apple", "Orange"]}+++def test_grouped_results_by_rule():+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_list=["Banana"])+ rule_id = scraper.stack_list[0]["stack_id"]+ result = scraper.get_result_similar(html=HTML, grouped=True, contain_sibling_leaves=True)+ assert result == {rule_id: ["Banana", "Apple", "Orange"]}+++def test_similar_unique_false():+ scraper = AutoScraper()+ scraper.build(html=HTML_DUP, wanted_list=["Banana"])+ result = scraper.get_result_similar(html=HTML_DUP, unique=False)+ assert result == ["Banana", "Banana"]+++def test_similar_keep_order():+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_list=["Banana"])+ result = scraper.get_result_similar(html=HTML, contain_sibling_leaves=True, keep_order=True)+ assert result == ["Banana", "Apple", "Orange"]diff --git a/tests/unit/test_build.py b/tests/unit/test_build.pynew file mode 100644index 0000000..65d5e05--- /dev/null+++ b/tests/unit/test_build.py@@ -0,0 +1,18 @@+import pytest+from autoscraper import AutoScraper++HTML = "<ul><li>Banana</li><li>Apple</li><li>Orange</li></ul>"+++def test_build_requires_targets():+ scraper = AutoScraper()+ with pytest.raises(ValueError):+ scraper.build(html=HTML)+++def test_build_and_get_result_similar():+ scraper = AutoScraper()+ result = scraper.build(html=HTML, wanted_list=["Banana"])+ assert result == ["Banana"]+ similar = scraper.get_result_similar(html=HTML, contain_sibling_leaves=True)+ assert similar == ["Banana", "Apple", "Orange"]diff --git a/tests/unit/test_features.py b/tests/unit/test_features.pynew file mode 100644index 0000000..414bc27--- /dev/null+++ b/tests/unit/test_features.py@@ -0,0 +1,57 @@+import pytest++from autoscraper import AutoScraper++HTML = "<ul><li>Banana</li><li>Apple</li><li>Orange</li></ul>"+HTML_COMPLEX_ORDER = """+<div class='products'>+ <h2>Banana</h2>+ <p class='price'>$1</p>+ <h2>Apple</h2>+ <p class='price'>$2</p>+</div>+"""+++def test_get_result_exact_order():+ scraper = AutoScraper()+ scraper.build(html=HTML_COMPLEX_ORDER, wanted_list=["Banana", "$2"])+ assert scraper.get_result_exact(html=HTML_COMPLEX_ORDER) == ["Banana", "$2"]+++def test_group_by_alias():+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_dict={"fruit": ["Banana"]})+ similar = scraper.get_result_similar(+ html=HTML, group_by_alias=True, contain_sibling_leaves=True, unique=True+ )+ assert similar == {"fruit": ["Banana", "Apple", "Orange"]}+++def test_save_and_load(tmp_path):+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_list=["Banana"])+ file_path = tmp_path / "model.json"+ scraper.save(file_path)+ new_scraper = AutoScraper()+ new_scraper.load(file_path)+ assert new_scraper.get_result_exact(html=HTML) == scraper.get_result_exact(html=HTML)+++def test_keep_rules():+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_list=["Banana"])+ first_rule = scraper.stack_list[0]["stack_id"]+ scraper.build(html=HTML, wanted_list=["Apple"], update=True)+ second_rule = scraper.stack_list[1]["stack_id"]+ scraper.keep_rules([second_rule])+ assert len(scraper.stack_list) == 1+ assert scraper.stack_list[0]["stack_id"] == second_rule+++def test_get_result_combined():+ scraper = AutoScraper()+ scraper.build(html=HTML, wanted_list=["Banana"])+ similar, exact = scraper.get_result(html=HTML)+ assert exact == ["Banana"]+ assert similar == ["Banana"]