A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
refactor fetch html method
alirezaa4ab3eda3daa
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex ee2a278..2486501 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -95,6 +95,7 @@ class AutoScraper(object):@classmethoddef _fetch_html(cls, url, request_args=None):+ request_args = request_args or {}headers = dict(cls.request_headers)if url:headers["Host"] = urlparse(url).netloc@@ -106,8 +107,6 @@ class AutoScraper(object):@classmethoddef _get_soup(cls, url=None, html=None, request_args=None):- request_args = request_args or {}-if html:html = normalize(unescape(html))return BeautifulSoup(html, "lxml")