A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
refactor: use default value instead of if-else for request_args param
cthulhu-irlc6a5334a2b91
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex cd277d3..a874340 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -25,11 +25,10 @@ class AutoScraper(object):self.stack_list = json.load(f)@classmethod- def _get_soup(cls, url=None, html=None, request_args=None):+ def _get_soup(cls, url=None, html=None, request_args={}):if html:return BeautifulSoup(html, 'lxml')- request_args = request_args if request_args else {}headers = dict(cls.request_headers)if url:headers['Host'] = urlparse(url).netloc@@ -93,7 +92,7 @@ class AutoScraper(object):return unique_list- def build(self, url=None, wanted_list=None, html=None, request_args=None):+ def build(self, url=None, wanted_list=None, html=None, request_args={}):self.url = urlsoup = self._get_soup(url=url, html=html, request_args=request_args)@@ -213,7 +212,7 @@ class AutoScraper(object):return self.unique(result)- def get_result_exact(self, url=None, html=None, soup=None, request_args=None):+ def get_result_exact(self, url=None, html=None, soup=None, request_args={}):if url:self.url = url@@ -232,7 +231,7 @@ class AutoScraper(object):return self.unique(result)- def get_result(self, url=None, html=None, request_args=None):+ def get_result(self, url=None, html=None, request_args={}):soup = self._get_soup(url=url, html=html, request_args=request_args)similar = self.get_result_similar(soup=soup)exact = self.get_result_exact(soup=soup)diff --git a/autoscraper/code_template.py b/autoscraper/code_template.pyindex afa1968..b573eca 100644--- a/autoscraper/code_template.py+++ b/autoscraper/code_template.py@@ -15,11 +15,10 @@ class GeneratedAutoScraper(object):self.stack_list = "{STACK_LIST}"@classmethod- def _get_soup(url=None, html=None, request_args=None):+ def _get_soup(url=None, html=None, request_args={}):if html:return BeautifulSoup(html, 'lxml')- request_args = request_args if request_args else {}headers = dict(cls.request_headers)if url:headers['Host'] = urlparse(url).netloc@@ -75,7 +74,7 @@ class GeneratedAutoScraper(object):result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])return result- def get_result_similar(self, url=None, html=None, soup=None, request_args=None):+ def get_result_similar(self, url=None, html=None, soup=None, request_args={}):if url:self.url = url@@ -88,7 +87,7 @@ class GeneratedAutoScraper(object):return self.unique(result)- def get_result_exact(self, url=None, html=None, soup=None, request_args=None):+ def get_result_exact(self, url=None, html=None, soup=None, request_args={}):if url:self.url = url@@ -105,7 +104,7 @@ class GeneratedAutoScraper(object):return self.unique(result)- def get_result(self, url=None, html=None, request_args=None):+ def get_result(self, url=None, html=None, request_args={}):soup = self._get_soup(url=url, html=html, request_args=request_args)similar = self.get_result_similar(soup=soup)exact = self.get_result_exact(soup=soup)