A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Fix requests encoding
alireza731f62534a69
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex 2486501..53662f1 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -102,7 +102,12 @@ class AutoScraper(object):user_headers = request_args.pop("headers", {})headers.update(user_headers)- html = requests.get(url, headers=headers, **request_args).text+ res = requests.get(url, headers=headers, **request_args)+ if res.encoding == "ISO-8859-1" and not "ISO-8859-1" in res.headers.get(+ "Content-Type", ""+ ):+ res.encoding = res.apparent_encoding+ html = res.textreturn html@classmethod