Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

Fix requests encoding

alireza731f62534a69
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index 2486501..53662f1 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -102,7 +102,12 @@ class AutoScraper(object):
user_headers = request_args.pop("headers", {})
headers.update(user_headers)
- html = requests.get(url, headers=headers, **request_args).text
+ res = requests.get(url, headers=headers, **request_args)
+ if res.encoding == "ISO-8859-1" and not "ISO-8859-1" in res.headers.get(
+ "Content-Type", ""
+ ):
+ res.encoding = res.apparent_encoding
+ html = res.text
return html
@classmethod