A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Replace filter with list comprehensions
PickNickChock69dbdc1e1cd0
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex 511946a..c9ec721 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -136,7 +136,7 @@ class AutoScraper(object):def _get_children(self, soup, text):text = text.strip()children = reversed(soup.findChildren())- children = list(filter(lambda x: self._child_has_text(x, text), children))+ children = [x for x in children if self._child_has_text(x, text)]return childrendef build(self, url=None, wanted_list=None, html=None, request_args=None):@@ -245,7 +245,7 @@ class AutoScraper(object):wanted_attr = stack['wanted_attr']is_full_url = stack['is_full_url']result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]- result = list(filter(lambda x: x, result))+ result = [x for x in result if x]return resultdef _get_result_with_stack_index_based(self, stack, soup):@@ -260,7 +260,7 @@ class AutoScraper(object):p = p[idx]result = [self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])]- result = list(filter(lambda x: x, result))+ result = [x for x in result if x]return resultdef _get_result_by_func(self, func, url, html, soup, request_args, grouped):@@ -312,7 +312,7 @@ class AutoScraper(object):None"""- self.stack_list = list(filter(lambda x: x['stack_id'] not in rules, self.stack_list))+ self.stack_list = [x for x in self.stack_list if x['stack_id'] not in rules]def keep_rules(self, rules):"""@@ -324,7 +324,7 @@ class AutoScraper(object):A list of rules to keep in stack_list."""- self.stack_list = list(filter(lambda x: x['stack_id'] in rules, self.stack_list))+ self.stack_list = [x for x in self.stack_list if x['stack_id'] in rules]def generate_python_code(self):# deprecated