Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

Replace filter with list comprehensions

PickNickChock69dbdc1e1cd0
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index 511946a..c9ec721 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -136,7 +136,7 @@ class AutoScraper(object):
def _get_children(self, soup, text):
text = text.strip()
children = reversed(soup.findChildren())
- children = list(filter(lambda x: self._child_has_text(x, text), children))
+ children = [x for x in children if self._child_has_text(x, text)]
return children
def build(self, url=None, wanted_list=None, html=None, request_args=None):
@@ -245,7 +245,7 @@ class AutoScraper(object):
wanted_attr = stack['wanted_attr']
is_full_url = stack['is_full_url']
result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]
- result = list(filter(lambda x: x, result))
+ result = [x for x in result if x]
return result
def _get_result_with_stack_index_based(self, stack, soup):
@@ -260,7 +260,7 @@ class AutoScraper(object):
p = p[idx]
result = [self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])]
- result = list(filter(lambda x: x, result))
+ result = [x for x in result if x]
return result
def _get_result_by_func(self, func, url, html, soup, request_args, grouped):
@@ -312,7 +312,7 @@ class AutoScraper(object):
None
"""
- self.stack_list = list(filter(lambda x: x['stack_id'] not in rules, self.stack_list))
+ self.stack_list = [x for x in self.stack_list if x['stack_id'] not in rules]
def keep_rules(self, rules):
"""
@@ -324,7 +324,7 @@ class AutoScraper(object):
A list of rules to keep in stack_list.
"""
- self.stack_list = list(filter(lambda x: x['stack_id'] in rules, self.stack_list))
+ self.stack_list = [x for x in self.stack_list if x['stack_id'] in rules]
def generate_python_code(self):
# deprecated