Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

normalize the html content

alirezaa61aec9b6b9b
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index 21d6ef3..fc6e5a9 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -1,6 +1,9 @@
import hashlib
import json
+import unicodedata
+
from collections import defaultdict
+from html import unescape
from urllib.parse import urljoin, urlparse
import requests
@@ -97,7 +100,8 @@ class AutoScraper(object):
user_headers = request_args.pop('headers', {})
headers.update(user_headers)
html = requests.get(url, headers=headers, **request_args).text
-
+ html = unicodedata.normalize("NFKD", unescape(html))
+
return BeautifulSoup(html, 'lxml')
@staticmethod