A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Remove redundant rules
alireza95fe21e69c11
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex 0e7dcaa..a115029 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -102,19 +102,30 @@ class AutoScraper(object):headers.update(user_headers)html = requests.get(url, headers=headers, **request_args).texthtml = unicodedata.normalize("NFKD", unescape(html))-+return BeautifulSoup(html, 'lxml')@staticmethoddef _get_valid_attrs(item):- return {- k: v if v != [] else '' for k, v in item.attrs.items() if k in {'class', 'style'}+ key_attrs = {'class', 'style'}+ attrs = {+ k: v if v != [] else '' for k, v in item.attrs.items() if k in key_attrs}+ for attr in key_attrs:+ if attr not in attrs:+ attrs[attr] = ''+ return attrs+@staticmethoddef _child_has_text(child, text, url):child_text = child.getText().strip()+if text == child_text:+ parent_text = child.parent.getText().strip()+ if child_text == parent_text:+ return False+child.wanted_attr = Nonereturn True@@ -249,10 +260,19 @@ class AutoScraper(object):def _get_result_with_stack(self, stack, soup, url):parents = [soup]- for _, item in enumerate(stack['content']):+ stack_content = stack['content']+ for index, item in enumerate(stack_content):children = []for parent in parents:- children += parent.findAll(item[0], item[1], recursive=False)+ found = parent.findAll(item[0], item[1], recursive=False)+ if not found:+ continue++ if index == len(stack_content) - 1:+ idx = min(len(found) - 1, stack_content[index - 1][2])+ found = [found[idx]]++ children += foundparents = childrendiff --git a/setup.py b/setup.pyindex fdd76a9..349406a 100644--- a/setup.py+++ b/setup.py@@ -10,7 +10,7 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f:setup(name='autoscraper',- version='1.1.2',+ version='1.1.3',description='A Smart, Automatic, Fast and Lightweight Web Scraper for Python',long_description_content_type="text/markdown",