Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

update headers with user headers

alireza82c3a2e33a18
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index acb071d..9acf40d 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -46,7 +46,8 @@ class AutoScraper(object):
if url:
headers['Host'] = urlparse(url).netloc
- headers = request_args.get('headers', headers)
+ user_headers = request_args.pop('headers', {})
+ headers.update(user_headers)
html = requests.get(url, headers=headers, **request_args).text
return BeautifulSoup(html, 'lxml')