A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
normalize the html content
alirezaa61aec9b6b9b
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex 21d6ef3..fc6e5a9 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -1,6 +1,9 @@import hashlibimport json+import unicodedata+from collections import defaultdict+from html import unescapefrom urllib.parse import urljoin, urlparseimport requests@@ -97,7 +100,8 @@ class AutoScraper(object):user_headers = request_args.pop('headers', {})headers.update(user_headers)html = requests.get(url, headers=headers, **request_args).text-+ html = unicodedata.normalize("NFKD", unescape(html))+return BeautifulSoup(html, 'lxml')@staticmethod