A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
refactor: revert to use None as default arg, rather than dict
cthulhu-irl0b156bf9b194
diff --git a/autoscraper/code_template.py b/autoscraper/code_template.pyindex b573eca..a5ad560 100644--- a/autoscraper/code_template.py+++ b/autoscraper/code_template.py@@ -15,7 +15,9 @@ class GeneratedAutoScraper(object):self.stack_list = "{STACK_LIST}"@classmethod- def _get_soup(url=None, html=None, request_args={}):+ def _get_soup(url=None, html=None, request_args=None):+ request_args = request_args or {}+if html:return BeautifulSoup(html, 'lxml')@@ -74,7 +76,7 @@ class GeneratedAutoScraper(object):result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])return result- def get_result_similar(self, url=None, html=None, soup=None, request_args={}):+ def get_result_similar(self, url=None, html=None, soup=None, request_args=None):if url:self.url = url@@ -87,7 +89,7 @@ class GeneratedAutoScraper(object):return self.unique(result)- def get_result_exact(self, url=None, html=None, soup=None, request_args={}):+ def get_result_exact(self, url=None, html=None, soup=None, request_args=None):if url:self.url = url@@ -104,7 +106,7 @@ class GeneratedAutoScraper(object):return self.unique(result)- def get_result(self, url=None, html=None, request_args={}):+ def get_result(self, url=None, html=None, request_args=None):soup = self._get_soup(url=url, html=html, request_args=request_args)similar = self.get_result_similar(soup=soup)exact = self.get_result_exact(soup=soup)