A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
remove code template
alireza78407f11a256
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex 1dd546b..9401f0e 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -1,5 +1,4 @@import json-import osfrom urllib.parse import urljoin, urlparseimport requests@@ -75,9 +74,7 @@ class AutoScraper(object):def _get_children(self, soup, text):text = text.strip()children = reversed(soup.findChildren())- children = list(- filter(lambda x: self._child_has_text(x, text), children))-+ children = list(filter(lambda x: self._child_has_text(x, text), children))return childrendef build(self, url=None, wanted_list=None, html=None, request_args=None):@@ -126,15 +123,12 @@ class AutoScraper(object):wanted_attr = getattr(child, 'wanted_attr', None)is_full_url = getattr(child, 'is_full_url', False)- stack = dict(content=content, wanted_attr=wanted_attr,- is_full_url=is_full_url)-+ stack = dict(content=content, wanted_attr=wanted_attr, is_full_url=is_full_url)return stackdef _get_result_for_child(self, child, soup):stack = self._build_stack(child)result = self._get_result_with_stack(stack, soup)-return result, stackdef _fetch_result_from_child(self, child, wanted_attr, is_full_url):@@ -160,22 +154,17 @@ class AutoScraper(object):wanted_attr = stack['wanted_attr']is_full_url = stack['is_full_url']- result = [self._fetch_result_from_child(- i, wanted_attr, is_full_url) for i in parents]+ result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]result = list(filter(lambda x: x, result))-return resultdef _get_result_with_stack_index_based(self, stack, soup):p = soup.findChildren(recursive=False)[0]stack_content = stack['content']for index, item in enumerate(stack_content[:-1]):- p = p.findAll(stack_content[index + 1]- [0], recursive=False)[item[2]]-- result = self._fetch_result_from_child(- p, stack['wanted_attr'], stack['is_full_url'])+ p = p.findAll(stack_content[index + 1][0], recursive=False)[item[2]]+ result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])return resultdef get_result_similar(self, url=None, html=None, soup=None, request_args=None):@@ -183,13 +172,11 @@ class AutoScraper(object):self.url = urlif not soup:- soup = self._get_soup(url=url, html=html,- request_args=request_args)+ soup = self._get_soup(url=url, html=html, request_args=request_args)result = []for stack in self.stack_list:result += self._get_result_with_stack(stack, soup)-return unique(result)def get_result_exact(self, url=None, html=None, soup=None, request_args=None):@@ -197,14 +184,12 @@ class AutoScraper(object):self.url = urlif not soup:- soup = self._get_soup(url=url, html=html,- request_args=request_args)+ soup = self._get_soup(url=url, html=html, request_args=request_args)result = []for stack in self.stack_list:try:- result.append(- self._get_result_with_stack_index_based(stack, soup))+ result.append(self._get_result_with_stack_index_based(stack, soup))except IndexError:continue@@ -215,16 +200,8 @@ class AutoScraper(object):soup = self._get_soup(url=url, html=html, request_args=request_args)similar = self.get_result_similar(soup=soup)exact = self.get_result_exact(soup=soup)-return similar, exactdef generate_python_code(self):# deprecatedprint('This function is deprecated. Please use save() and load() instead.')- file_path = os.path.join(os.path.dirname(__file__), "code_template.py")- with open(file_path, 'r') as f:- code = f.read()-- code = code.replace('"{STACK_LIST}"', str(self.stack_list))-- return codediff --git a/autoscraper/code_template.py b/autoscraper/code_template.pydeleted file mode 100644index bfbb660..0000000--- a/autoscraper/code_template.py+++ /dev/null@@ -1,106 +0,0 @@-from urllib.parse import urljoin, urlparse--import requests-from bs4 import BeautifulSoup--from autoscraper.utils import unique--class GeneratedAutoScraper(object):- request_headers = {- 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; \- Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'- }-- def __init__(self):- self.url = ''- self.stack_list = "{STACK_LIST}"-- @classmethod- def _get_soup(cls, url=None, html=None, request_args=None):- request_args = request_args or {}-- if html:- return BeautifulSoup(html, 'lxml')-- headers = dict(cls.request_headers)- if url:- headers['Host'] = urlparse(url).netloc-- headers = request_args.get('headers', headers)- html = requests.get(url, headers=headers, **request_args).text-- return BeautifulSoup(html, 'lxml')-- def _fetch_result_from_child(self, child, wanted_attr, is_full_url):- if wanted_attr is None:- return child.getText().strip()-- if wanted_attr not in child.attrs:- return None-- if is_full_url:- return urljoin(self.url, child.attrs[wanted_attr])-- return child.attrs[wanted_attr]-- def _get_result_with_stack(self, stack, soup):- parents = [soup]- for _, item in enumerate(stack['content']):- children = []- for parent in parents:- children += parent.findAll(item[0], item[1], recursive=False)-- parents = children-- wanted_attr = stack['wanted_attr']- is_full_url = stack['is_full_url']- result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]- result = list(filter(lambda x: x, result))-- return result-- def _get_result_with_stack_index_based(self, stack, soup):- p = soup.findChildren(recursive=False)[0]- stack_content = stack['content']- for index, item in enumerate(stack_content[:-1]):- p = p.findAll(stack_content[index + 1][0], recursive=False)[item[2]]-- result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])- return result-- def get_result_similar(self, url=None, html=None, soup=None, request_args=None):- if url:- self.url = url-- if not soup:- soup = self._get_soup(url=url, html=html, request_args=request_args)-- result = []- for stack in self.stack_list:- result += self._get_result_with_stack(stack, soup)-- return unique(result)-- def get_result_exact(self, url=None, html=None, soup=None, request_args=None):- if url:- self.url = url-- if not soup:- soup = self._get_soup(url=url, html=html, request_args=request_args)-- result = []- for stack in self.stack_list:- try:- result.append(self._get_result_with_stack_index_based(stack, soup))-- except IndexError:- continue-- return unique(result)-- def get_result(self, url=None, html=None, request_args=None):- soup = self._get_soup(url=url, html=html, request_args=request_args)- similar = self.get_result_similar(soup=soup)- exact = self.get_result_exact(soup=soup)-- return similar, exactdiff --git a/autoscraper/utils.py b/autoscraper/utils.pyindex ff67c4f..ee44a18 100644--- a/autoscraper/utils.py+++ b/autoscraper/utils.py@@ -1,8 +1,6 @@-def unique(item_list):- unique_list = []- for item in item_list:- if item not in unique_list:- unique_list.append(item)+def unique(item_list):+ unique_list = []+ for item in item_list:+ if item not in unique_list:+ unique_list.append(item)return unique_list--