Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

remove code template

alireza78407f11a256
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index 1dd546b..9401f0e 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -1,5 +1,4 @@
import json
-import os
from urllib.parse import urljoin, urlparse
import requests
@@ -75,9 +74,7 @@ class AutoScraper(object):
def _get_children(self, soup, text):
text = text.strip()
children = reversed(soup.findChildren())
- children = list(
- filter(lambda x: self._child_has_text(x, text), children))
-
+ children = list(filter(lambda x: self._child_has_text(x, text), children))
return children
def build(self, url=None, wanted_list=None, html=None, request_args=None):
@@ -126,15 +123,12 @@ class AutoScraper(object):
wanted_attr = getattr(child, 'wanted_attr', None)
is_full_url = getattr(child, 'is_full_url', False)
- stack = dict(content=content, wanted_attr=wanted_attr,
- is_full_url=is_full_url)
-
+ stack = dict(content=content, wanted_attr=wanted_attr, is_full_url=is_full_url)
return stack
def _get_result_for_child(self, child, soup):
stack = self._build_stack(child)
result = self._get_result_with_stack(stack, soup)
-
return result, stack
def _fetch_result_from_child(self, child, wanted_attr, is_full_url):
@@ -160,22 +154,17 @@ class AutoScraper(object):
wanted_attr = stack['wanted_attr']
is_full_url = stack['is_full_url']
- result = [self._fetch_result_from_child(
- i, wanted_attr, is_full_url) for i in parents]
+ result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]
result = list(filter(lambda x: x, result))
-
return result
def _get_result_with_stack_index_based(self, stack, soup):
p = soup.findChildren(recursive=False)[0]
stack_content = stack['content']
for index, item in enumerate(stack_content[:-1]):
- p = p.findAll(stack_content[index + 1]
- [0], recursive=False)[item[2]]
-
- result = self._fetch_result_from_child(
- p, stack['wanted_attr'], stack['is_full_url'])
+ p = p.findAll(stack_content[index + 1][0], recursive=False)[item[2]]
+ result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])
return result
def get_result_similar(self, url=None, html=None, soup=None, request_args=None):
@@ -183,13 +172,11 @@ class AutoScraper(object):
self.url = url
if not soup:
- soup = self._get_soup(url=url, html=html,
- request_args=request_args)
+ soup = self._get_soup(url=url, html=html, request_args=request_args)
result = []
for stack in self.stack_list:
result += self._get_result_with_stack(stack, soup)
-
return unique(result)
def get_result_exact(self, url=None, html=None, soup=None, request_args=None):
@@ -197,14 +184,12 @@ class AutoScraper(object):
self.url = url
if not soup:
- soup = self._get_soup(url=url, html=html,
- request_args=request_args)
+ soup = self._get_soup(url=url, html=html, request_args=request_args)
result = []
for stack in self.stack_list:
try:
- result.append(
- self._get_result_with_stack_index_based(stack, soup))
+ result.append(self._get_result_with_stack_index_based(stack, soup))
except IndexError:
continue
@@ -215,16 +200,8 @@ class AutoScraper(object):
soup = self._get_soup(url=url, html=html, request_args=request_args)
similar = self.get_result_similar(soup=soup)
exact = self.get_result_exact(soup=soup)
-
return similar, exact
def generate_python_code(self):
# deprecated
print('This function is deprecated. Please use save() and load() instead.')
- file_path = os.path.join(os.path.dirname(__file__), "code_template.py")
- with open(file_path, 'r') as f:
- code = f.read()
-
- code = code.replace('"{STACK_LIST}"', str(self.stack_list))
-
- return code
diff --git a/autoscraper/code_template.py b/autoscraper/code_template.py
deleted file mode 100644
index bfbb660..0000000
--- a/autoscraper/code_template.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from urllib.parse import urljoin, urlparse
-
-import requests
-from bs4 import BeautifulSoup
-
-from autoscraper.utils import unique
-
-class GeneratedAutoScraper(object):
- request_headers = {
- 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; \
- Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'
- }
-
- def __init__(self):
- self.url = ''
- self.stack_list = "{STACK_LIST}"
-
- @classmethod
- def _get_soup(cls, url=None, html=None, request_args=None):
- request_args = request_args or {}
-
- if html:
- return BeautifulSoup(html, 'lxml')
-
- headers = dict(cls.request_headers)
- if url:
- headers['Host'] = urlparse(url).netloc
-
- headers = request_args.get('headers', headers)
- html = requests.get(url, headers=headers, **request_args).text
-
- return BeautifulSoup(html, 'lxml')
-
- def _fetch_result_from_child(self, child, wanted_attr, is_full_url):
- if wanted_attr is None:
- return child.getText().strip()
-
- if wanted_attr not in child.attrs:
- return None
-
- if is_full_url:
- return urljoin(self.url, child.attrs[wanted_attr])
-
- return child.attrs[wanted_attr]
-
- def _get_result_with_stack(self, stack, soup):
- parents = [soup]
- for _, item in enumerate(stack['content']):
- children = []
- for parent in parents:
- children += parent.findAll(item[0], item[1], recursive=False)
-
- parents = children
-
- wanted_attr = stack['wanted_attr']
- is_full_url = stack['is_full_url']
- result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]
- result = list(filter(lambda x: x, result))
-
- return result
-
- def _get_result_with_stack_index_based(self, stack, soup):
- p = soup.findChildren(recursive=False)[0]
- stack_content = stack['content']
- for index, item in enumerate(stack_content[:-1]):
- p = p.findAll(stack_content[index + 1][0], recursive=False)[item[2]]
-
- result = self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])
- return result
-
- def get_result_similar(self, url=None, html=None, soup=None, request_args=None):
- if url:
- self.url = url
-
- if not soup:
- soup = self._get_soup(url=url, html=html, request_args=request_args)
-
- result = []
- for stack in self.stack_list:
- result += self._get_result_with_stack(stack, soup)
-
- return unique(result)
-
- def get_result_exact(self, url=None, html=None, soup=None, request_args=None):
- if url:
- self.url = url
-
- if not soup:
- soup = self._get_soup(url=url, html=html, request_args=request_args)
-
- result = []
- for stack in self.stack_list:
- try:
- result.append(self._get_result_with_stack_index_based(stack, soup))
-
- except IndexError:
- continue
-
- return unique(result)
-
- def get_result(self, url=None, html=None, request_args=None):
- soup = self._get_soup(url=url, html=html, request_args=request_args)
- similar = self.get_result_similar(soup=soup)
- exact = self.get_result_exact(soup=soup)
-
- return similar, exact
diff --git a/autoscraper/utils.py b/autoscraper/utils.py
index ff67c4f..ee44a18 100644
--- a/autoscraper/utils.py
+++ b/autoscraper/utils.py
@@ -1,8 +1,6 @@
-def unique(item_list):
- unique_list = []
- for item in item_list:
- if item not in unique_list:
- unique_list.append(item)
+def unique(item_list):
+ unique_list = []
+ for item in item_list:
+ if item not in unique_list:
+ unique_list.append(item)
return unique_list
-
-