A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
update headers with user headers
alireza82c3a2e33a18
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex acb071d..9acf40d 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -46,7 +46,8 @@ class AutoScraper(object):if url:headers['Host'] = urlparse(url).netloc- headers = request_args.get('headers', headers)+ user_headers = request_args.pop('headers', {})+ headers.update(user_headers)html = requests.get(url, headers=headers, **request_args).textreturn BeautifulSoup(html, 'lxml')