Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

update default request headers

alirezaf688c90cf522
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index 304ff41..a622b82 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -7,8 +7,8 @@ from bs4 import BeautifulSoup
class AutoScraper(object):
request_headers = {
- 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; \
- Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'
+ 'User-Agent': 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36',
+ 'accept': '*/*'
}
def __init__(self, stack_list=None, url=None):