Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

refactor fetch html method

alirezaa4ab3eda3daa
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index ee2a278..2486501 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -95,6 +95,7 @@ class AutoScraper(object):
@classmethod
def _fetch_html(cls, url, request_args=None):
+ request_args = request_args or {}
headers = dict(cls.request_headers)
if url:
headers["Host"] = urlparse(url).netloc
@@ -106,8 +107,6 @@ class AutoScraper(object):
@classmethod
def _get_soup(cls, url=None, html=None, request_args=None):
- request_args = request_args or {}
-
if html:
html = normalize(unescape(html))
return BeautifulSoup(html, "lxml")