Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

refactor: use default value instead of if-else for request_args param

cthulhu-irlc6a5334a2b91
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index cd277d3..a874340 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -25,11 +25,10 @@ class AutoScraper(object):
self.stack_list = json.load(f)
@classmethod
- def _get_soup(cls, url=None, html=None, request_args=None):
+ def _get_soup(cls, url=None, html=None, request_args={}):
if html:
return BeautifulSoup(html, 'lxml')
- request_args = request_args if request_args else {}
headers = dict(cls.request_headers)
if url:
headers['Host'] = urlparse(url).netloc
@@ -93,7 +92,7 @@ class AutoScraper(object):
return unique_list
- def build(self, url=None, wanted_list=None, html=None, request_args=None):
+ def build(self, url=None, wanted_list=None, html=None, request_args={}):
self.url = url
soup = self._get_soup(url=url, html=html, request_args=request_args)
@@ -213,7 +212,7 @@ class AutoScraper(object):
return self.unique(result)
- def get_result_exact(self, url=None, html=None, soup=None, request_args=None):
+ def get_result_exact(self, url=None, html=None, soup=None, request_args={}):
if url:
self.url = url
@@ -232,7 +231,7 @@ class AutoScraper(object):
return self.unique(result)
- def get_result(self, url=None, html=None, request_args=None):
+ def get_result(self, url=None, html=None, request_args={}):
soup = self._get_soup(url=url, html=html, request_args=request_args)
similar = self.get_result_similar(soup=soup)
exact = self.get_result_exact(soup=soup)
diff --git a/autoscraper/code_template.py b/autoscraper/code_template.py
index afa1968..b573eca 100644
--- a/autoscraper/code_template.py
+++ b/autoscraper/code_template.py
@@ -15,11 +15,10 @@ class GeneratedAutoScraper(object):
self.stack_list = "{STACK_LIST}"
@classmethod
- def _get_soup(url=None, html=None, request_args=None):
+ def _get_soup(url=None, html=None, request_args={}):
if html:
return BeautifulSoup(html, 'lxml')
- request_args = request_args if request_args else {}
headers = dict(cls.request_headers)
if url:
headers['Host'] = urlparse(url).netloc
@@ -75,7 +74,7 @@ class GeneratedAutoScraper(object):
result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])
return result
- def get_result_similar(self, url=None, html=None, soup=None, request_args=None):
+ def get_result_similar(self, url=None, html=None, soup=None, request_args={}):
if url:
self.url = url
@@ -88,7 +87,7 @@ class GeneratedAutoScraper(object):
return self.unique(result)
- def get_result_exact(self, url=None, html=None, soup=None, request_args=None):
+ def get_result_exact(self, url=None, html=None, soup=None, request_args={}):
if url:
self.url = url
@@ -105,7 +104,7 @@ class GeneratedAutoScraper(object):
return self.unique(result)
- def get_result(self, url=None, html=None, request_args=None):
+ def get_result(self, url=None, html=None, request_args={}):
soup = self._get_soup(url=url, html=html, request_args=request_args)
similar = self.get_result_similar(soup=soup)
exact = self.get_result_exact(soup=soup)