A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
refactor: use reduce to pick unique values in unique method
cthulhu-irlcf6b215e6480
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex ae30c7f..aac7e88 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -1,5 +1,6 @@import osimport json+from functools import reducefrom urllib.parse import urljoin, urlparseimport requests@@ -78,12 +79,7 @@ class AutoScraper(object):@classmethoddef unique(cls, item_list):- unique_list = []- for item in item_list:- if item not in unique_list:- unique_list.append(item)-- return unique_list+ return list(set(item_list))def build(self, url=None, wanted_list=None, html=None, request_args={}):self.url = url