A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
refactor: use utils.unique instead of local definition
cthulhu-irl46d630d8ca8b
diff --git a/autoscraper/code_template.py b/autoscraper/code_template.pyindex a5ad560..04454a2 100644--- a/autoscraper/code_template.py+++ b/autoscraper/code_template.py@@ -3,6 +3,7 @@ from urllib.parse import urljoin, urlparseimport requestsfrom bs4 import BeautifulSoup+from autoscraper.utils import uniqueclass GeneratedAutoScraper(object):request_headers = {@@ -30,15 +31,6 @@ class GeneratedAutoScraper(object):return BeautifulSoup(html, 'lxml')- @classmethod- def unique(cls, item_list):- unique_list = []- for item in item_list:- if item not in unique_list:- unique_list.append(item)-- return unique_list-def _fetch_result_from_child(self, child, wanted_attr, is_full_url):if wanted_attr is None:return child.getText().strip()@@ -87,7 +79,7 @@ class GeneratedAutoScraper(object):for stack in self.stack_list:result += self._get_result_with_stack(stack, soup)- return self.unique(result)+ return unique(result)def get_result_exact(self, url=None, html=None, soup=None, request_args=None):if url:@@ -104,7 +96,7 @@ class GeneratedAutoScraper(object):except IndexError:continue- return self.unique(result)+ return unique(result)def get_result(self, url=None, html=None, request_args=None):soup = self._get_soup(url=url, html=html, request_args=request_args)