A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Add ability to set fuzziness threshold for matching html tag attributes
alirezadeb8a297d35b
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex c85effd..63d3229 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -9,7 +9,8 @@ from urllib.parse import urljoin, urlparseimport requestsfrom bs4 import BeautifulSoup-from autoscraper.utils import get_random_str, unique_hashable, unique_stack_list, ResultItem+from autoscraper.utils import get_random_str, unique_hashable, unique_stack_list, \+ ResultItem, FuzzyTextclass AutoScraper(object):@@ -256,7 +257,7 @@ class AutoScraper(object):def _get_result_for_child(self, child, soup, url):stack = self._build_stack(child, url)- result = self._get_result_with_stack(stack, soup, url)+ result = self._get_result_with_stack(stack, soup, url, 1.0)return result, stack@staticmethod@@ -272,14 +273,30 @@ class AutoScraper(object):return child.attrs[wanted_attr]- def _get_result_with_stack(self, stack, soup, url, **kwargs):+ @staticmethod+ def _get_fuzzy_attrs(attrs, attr_fuzz_ratio):+ attrs = dict(attrs)+ for key, val in attrs.items():+ if isinstance(val, str) and val:+ val = FuzzyText(val, attr_fuzz_ratio)+ elif isinstance(val, (list, tuple)):+ val = [FuzzyText(x, attr_fuzz_ratio) if x else x for x in val]+ attrs[key] = val+ return attrs++ def _get_result_with_stack(self, stack, soup, url, attr_fuzz_ratio, **kwargs):parents = [soup]stack_content = stack['content']contain_sibling_leaves = kwargs.get('contain_sibling_leaves', False)for index, item in enumerate(stack_content):children = []for parent in parents:- found = parent.findAll(item[0], item[1], recursive=False)++ attrs = item[1]+ if attr_fuzz_ratio < 1.0:+ attrs = self._get_fuzzy_attrs(attrs, attr_fuzz_ratio)++ found = parent.findAll(item[0], attrs, recursive=False)if not found:continue@@ -298,12 +315,15 @@ class AutoScraper(object):result = [x for x in result if x.text]return result- def _get_result_with_stack_index_based(self, stack, soup, url, **kwargs):+ def _get_result_with_stack_index_based(self, stack, soup, url, attr_fuzz_ratio, **kwargs):p = soup.findChildren(recursive=False)[0]stack_content = stack['content']for index, item in enumerate(stack_content[:-1]):content = stack_content[index + 1]- p = p.findAll(content[0], content[1], recursive=False)+ attrs = content[1]+ if attr_fuzz_ratio < 1.0:+ attrs = self._get_fuzzy_attrs(attrs, attr_fuzz_ratio)+ p = p.findAll(content[0], attrs, recursive=False)if not p:return []idx = min(len(p) - 1, item[2])@@ -315,7 +335,7 @@ class AutoScraper(object):return resultdef _get_result_by_func(self, func, url, html, soup, request_args, grouped,- group_by_alias, unique, **kwargs):+ group_by_alias, unique, attr_fuzz_ratio, **kwargs):if not soup:soup = self._get_soup(url=url, html=html, request_args=request_args)@@ -331,7 +351,7 @@ class AutoScraper(object):if not url:url = stack.get('url', '')- result = func(stack, soup, url, **kwargs)+ result = func(stack, soup, url, attr_fuzz_ratio, **kwargs)if not grouped and not group_by_alias:result_list += result@@ -366,8 +386,8 @@ class AutoScraper(object):return dict(grouped_result)def get_result_similar(self, url=None, html=None, soup=None, request_args=None,- grouped=False, group_by_alias=False, unique=None, keep_order=False,- contain_sibling_leaves=False):+ grouped=False, group_by_alias=False, unique=None, attr_fuzz_ratio=1.0,+ keep_order=False, contain_sibling_leaves=False):"""Gets similar results based on the previously learned rules.@@ -396,6 +416,9 @@ class AutoScraper(object):False for grouped results.If set to True, will remove duplicates from returned result list.+ attr_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0+ The fuzziness ratio threshold for matching html tag attributes.+keep_order: bool, optional, defaults to FalseIf set to True, the results will be ordered as they are present on the web page.@@ -410,11 +433,12 @@ class AutoScraper(object):func = self._get_result_with_stackreturn self._get_result_by_func(func, url, html, soup, request_args, grouped,- group_by_alias, unique, keep_order=keep_order,+ group_by_alias, unique, attr_fuzz_ratio,+ keep_order=keep_order,contain_sibling_leaves=contain_sibling_leaves)def get_result_exact(self, url=None, html=None, soup=None, request_args=None,- grouped=False, group_by_alias=False, unique=None):+ grouped=False, group_by_alias=False, unique=None, attr_fuzz_ratio=1.0):"""Gets exact results based on the previously learned rules.@@ -443,6 +467,9 @@ class AutoScraper(object):False for grouped results.If set to True, will remove duplicates from returned result list.+ attr_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0+ The fuzziness ratio threshold for matching html tag attributes.+Returns:--------List of exact results scraped from the web page.@@ -451,9 +478,10 @@ class AutoScraper(object):func = self._get_result_with_stack_index_basedreturn self._get_result_by_func(func, url, html, soup, request_args, grouped,- group_by_alias, unique)+ group_by_alias, unique, attr_fuzz_ratio)- def get_result(self, url=None, html=None, request_args=None, grouped=False, group_by_alias=False):+ def get_result(self, url=None, html=None, request_args=None, grouped=False,+ group_by_alias=False, unique=None, attr_fuzz_ratio=1.0):"""Gets similar and exact results based on the previously learned rules.@@ -478,6 +506,13 @@ class AutoScraper(object):If set to True, the result will be a dictionary with the rule alias as keysand a list of scraped data per alias as values.+ unique: bool, optional, defaults to True for non grouped results and+ False for grouped results.+ If set to True, will remove duplicates from returned result list.++ attr_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0+ The fuzziness ratio threshold for matching html tag attributes.+Returns:--------Pair of (similar, exact) results.@@ -485,8 +520,10 @@ class AutoScraper(object):"""soup = self._get_soup(url=url, html=html, request_args=request_args)- similar = self.get_result_similar(url=url, soup=soup, grouped=grouped, group_by_alias=group_by_alias)- exact = self.get_result_exact(url=url, soup=soup, grouped=grouped, group_by_alias=group_by_alias)+ args = dict(url=url, soup=soup, grouped=grouped, group_by_alias=group_by_alias,+ unique=unique, attr_fuzz_ratio=attr_fuzz_ratio)+ similar = self.get_result_similar(**args)+ exact = self.get_result_exact(**args)return similar, exactdef remove_rules(self, rules):diff --git a/autoscraper/utils.py b/autoscraper/utils.pyindex 6fa54d0..39503e3 100644--- a/autoscraper/utils.py+++ b/autoscraper/utils.py@@ -2,6 +2,11 @@ from collections import OrderedDictimport randomimport string+import warnings++with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ from fuzzywuzzy import fuzzdef unique_stack_list(stack_list):@@ -33,3 +38,13 @@ class ResultItem():def __str__(self):return self.text+++class FuzzyText(object):+ def __init__(self, text, ratio_limit):+ self.text = text+ self.ratio_limit = ratio_limit+ self.match = None++ def search(self, text):+ return fuzz.ratio(self.text, text)/100. >= self.ratio_limitdiff --git a/setup.py b/setup.pyindex 8df6ec7..c0e902a 100644--- a/setup.py+++ b/setup.py@@ -10,7 +10,7 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f:setup(name='autoscraper',- version='1.1.6',+ version='1.1.7',description='A Smart, Automatic, Fast and Lightweight Web Scraper for Python',long_description_content_type="text/markdown",@@ -34,6 +34,6 @@ setup(packages=find_packages(exclude=['contrib', 'docs', 'tests']),python_requires='>=3.6',- install_requires=['requests', 'bs4', 'lxml'],+ install_requires=['requests', 'bs4', 'lxml', 'fuzzywuzzy'],)