A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Add ability to set fuzziness ratio for matching the wanted items
alireza10153e896c11
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex af17616..028ce8d 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -10,7 +10,7 @@ import requestsfrom bs4 import BeautifulSoupfrom autoscraper.utils import get_random_str, unique_hashable, unique_stack_list, \- ResultItem, FuzzyText, get_non_rec_text+ ResultItem, FuzzyText, get_non_rec_text, text_matchclass AutoScraper(object):@@ -119,10 +119,10 @@ class AutoScraper(object):return attrs@staticmethod- def _child_has_text(child, text, url):+ def _child_has_text(child, text, url, text_fuzz_ratio):child_text = child.getText().strip()- if text == child_text:+ if text_match(text, child_text, text_fuzz_ratio):parent_text = child.parent.getText().strip()if child_text == parent_text:return False@@ -130,7 +130,7 @@ class AutoScraper(object):child.wanted_attr = Nonereturn True- if text == get_non_rec_text(child):+ if text_match(text, get_non_rec_text(child), text_fuzz_ratio):child.is_non_rec_text = Truechild.wanted_attr = Nonereturn True@@ -140,7 +140,7 @@ class AutoScraper(object):continuevalue = value.strip()- if text == value:+ if text_match(text, value, text_fuzz_ratio):child.wanted_attr = keyreturn True@@ -153,13 +153,14 @@ class AutoScraper(object):return False- def _get_children(self, soup, text, url):+ def _get_children(self, soup, text, url, text_fuzz_ratio):text = text.strip()children = reversed(soup.findChildren())- children = [x for x in children if self._child_has_text(x, text, url)]+ children = [x for x in children if self._child_has_text(x, text, url, text_fuzz_ratio)]return children- def build(self, url=None, wanted_list=None, wanted_dict=None, html=None, request_args=None, update=False):+ def build(self, url=None, wanted_list=None, wanted_dict=None, html=None, request_args=None,+ update=False, text_fuzz_ratio=1.0):"""Automatically constructs a set of rules to scrape the specified target[s] from a web page.The rules are represented as stack_list.@@ -190,9 +191,12 @@ class AutoScraper(object):If True, new learned rules will be added to the previous ones.If False, all previously learned rules will be removed.+ text_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0+ The fuzziness ratio threshold for matching the wanted contents.+Returns:--------- None+ List of similar results"""soup = self._get_soup(url=url, html=html, request_args=request_args)@@ -212,7 +216,7 @@ class AutoScraper(object):wanted_list += wanted_itemsfor wanted in wanted_items:- children = self._get_children(soup, wanted, url)+ children = self._get_children(soup, wanted, url, text_fuzz_ratio)for child in children:result, stack = self._get_result_for_child(child, soup, url)@@ -223,11 +227,8 @@ class AutoScraper(object):result_list = [item.text for item in result_list]result_list = unique_hashable(result_list)- if all(w in result_list for w in wanted_list):- self.stack_list = unique_stack_list(self.stack_list)- return result_list-- return None+ self.stack_list = unique_stack_list(self.stack_list)+ return result_list@classmethoddef _build_stack(cls, child, url):diff --git a/autoscraper/utils.py b/autoscraper/utils.pyindex f67a95c..8c7c028 100644--- a/autoscraper/utils.py+++ b/autoscraper/utils.py@@ -32,6 +32,12 @@ def get_non_rec_text(element):return ''.join(element.find_all(text=True, recursive=False)).strip()+def text_match(t1, t2, ratio_limit):+ if ratio_limit >= 1:+ return t1 == t2+ return SequenceMatcher(None, t1, t2).ratio() >= ratio_limit++class ResultItem():def __init__(self, text, index):self.text = text