A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
Add ability to learn new rules while preserving the previous ones
alireza2a96a0198c4f
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex c9ec721..9c1161b 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -1,3 +1,4 @@+import hashlibimport jsonfrom collections import defaultdictfrom urllib.parse import urljoin, urlparse@@ -5,7 +6,7 @@ from urllib.parse import urljoin, urlparseimport requestsfrom bs4 import BeautifulSoup-from autoscraper.utils import unique, get_random_str+from autoscraper.utils import get_random_str, unique_hashable, unique_stack_listclass AutoScraper(object):@@ -16,8 +17,6 @@ class AutoScraper(object):Attributes----------- url: str- Url of the web page being scrapedstack_list: listList of rules learned by AutoScraper@@ -25,11 +24,11 @@ class AutoScraper(object):-------build() - Learns a set of rules represented as stack_list based on the wanted_list,which can be reused for scraping similar elements from other web pages in the future.- get_result_similar() - Gets similar results based on the previously learned stack_list.- get_result_exact() - Gets exact results based on the previously learned stack_list.- get_results() - Gets exact and similar results based on the previously learned stack_list.- save() - Serializes the stack_list and url as JSON and saves it to disk.- load() - De-serializes the JSON representation of the stack_list and the url and loads it back.+ get_result_similar() - Gets similar results based on the previously learned rules.+ get_result_exact() - Gets exact results based on the previously learned rules.+ get_results() - Gets exact and similar results based on the previously learned rules.+ save() - Serializes the stack_list as JSON and saves it to disk.+ load() - De-serializes the JSON representation of the stack_list and loads it back.remove_rules() - Removes one or more learned rule[s] from the stack_list.keep_rules() - Keeps only the specified learned rules in the stack_list and removes the others."""@@ -39,17 +38,16 @@ class AutoScraper(object):(KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36'}- def __init__(self, stack_list=None, url=None):- self.stack_list = stack_list- self.url = url+ def __init__(self, stack_list=None):+ self.stack_list = stack_list or []def save(self, file_path):"""- Serializes the stack_list and url as JSON and saves it to the disk.+ Serializes the stack_list as JSON and saves it to the disk.Parameters----------- file_path : str+ file_path: strPath of the JSON outputReturns@@ -57,18 +55,18 @@ class AutoScraper(object):None"""- data = dict(url=self.url, stack_list=self.stack_list)+ data = dict(stack_list=self.stack_list)with open(file_path, 'w') as f:json.dump(data, f)def load(self, file_path):"""- De-serializes the JSON representation of the stack_list and the url and loads it back.+ De-serializes the JSON representation of the stack_list and loads it back.Parameters----------- file_path : str- Path of the JSON file to load stack_list and url from.+ file_path: str+ Path of the JSON file to load stack_list from.Returns-------@@ -84,7 +82,6 @@ class AutoScraper(object):returnself.stack_list = data['stack_list']- self.url = data['url']@classmethoddef _get_soup(cls, url=None, html=None, request_args=None):@@ -109,7 +106,8 @@ class AutoScraper(object):k: v if v != [] else '' for k, v in item.attrs.items() if k in {'class', 'style'}}- def _child_has_text(self, child, text):+ @staticmethod+ def _child_has_text(child, text, url):child_text = child.getText().strip()if text == child_text:child.wanted_attr = None@@ -125,7 +123,7 @@ class AutoScraper(object):return Trueif key in {'href', 'src'}:- full_url = urljoin(self.url, value)+ full_url = urljoin(url, value)if text == full_url:child.wanted_attr = keychild.is_full_url = True@@ -133,62 +131,68 @@ class AutoScraper(object):return False- def _get_children(self, soup, text):+ def _get_children(self, soup, text, url):text = text.strip()children = reversed(soup.findChildren())- children = [x for x in children if self._child_has_text(x, text)]+ children = [x for x in children if self._child_has_text(x, text, url)]return children- def build(self, url=None, wanted_list=None, html=None, request_args=None):+ def build(self, url=None, wanted_list=None, html=None, request_args=None, update=True):"""Automatically constructs a set of rules to scrape the specified target[s] from a web page.The rules are represented as stack_list.Parameters:----------- url : str, optional- URL of the target web page. You should either pass url or html.- wanted_list : list, optional+ url: str, optional+ URL of the target web page. You should either pass url or html or both.++ wanted_list: list, optionalA list of needed contents to be scraped.AutoScraper learns a set of rules to scrape these targets.- html : str, optional+ html: str, optionalAn HTML string can also be passed instead of URL.- You should either pass url or html.+ You should either pass url or html or both.- request_args : dict, optional+ request_args: dict, optionalA dictionary used to specify a set of additional request parameters used by requestsmodule. You can specify proxy URLs, custom headers etc.+ update: bool, optional, defaults to True+ If True, new learned rules will be added to the previous ones.+ If False, all previously learned rules will be removed.+Returns:--------None"""- self.url = urlsoup = self._get_soup(url=url, html=html, request_args=request_args)result_list = []- stack_list = []++ if update is False:+ self.stack_list = []for wanted in wanted_list:- children = self._get_children(soup, wanted)+ children = self._get_children(soup, wanted, url)for child in children:- result, stack = self._get_result_for_child(child, soup)+ result, stack = self._get_result_for_child(child, soup, url)result_list += result- stack_list.append(stack)+ self.stack_list.append(stack)- result_list = unique(result_list)+ result_list = unique_hashable(result_list)if all(w in result_list for w in wanted_list):- self.stack_list = unique(stack_list)+ self.stack_list = unique_stack_list(self.stack_list)return result_listreturn None@classmethod- def _build_stack(cls, child):+ def _build_stack(cls, child, url):content = [(child.name, cls._get_valid_attrs(child))]parent = child@@ -213,15 +217,18 @@ class AutoScraper(object):wanted_attr = getattr(child, 'wanted_attr', None)is_full_url = getattr(child, 'is_full_url', False)stack = dict(content=content, wanted_attr=wanted_attr, is_full_url=is_full_url)+ stack['url'] = url if is_full_url else ''+ stack['hash'] = hashlib.sha256(str(stack).encode('utf-8')).hexdigest()stack['stack_id'] = 'rule_' + get_random_str(4)return stack- def _get_result_for_child(self, child, soup):- stack = self._build_stack(child)- result = self._get_result_with_stack(stack, soup)+ def _get_result_for_child(self, child, soup, url):+ stack = self._build_stack(child, url)+ result = self._get_result_with_stack(stack, soup, url)return result, stack- def _fetch_result_from_child(self, child, wanted_attr, is_full_url):+ @staticmethod+ def _fetch_result_from_child(child, wanted_attr, is_full_url, url):if wanted_attr is None:return child.getText().strip()@@ -229,11 +236,11 @@ class AutoScraper(object):return Noneif is_full_url:- return urljoin(self.url, child.attrs[wanted_attr])+ return urljoin(url, child.attrs[wanted_attr])return child.attrs[wanted_attr]- def _get_result_with_stack(self, stack, soup):+ def _get_result_with_stack(self, stack, soup, url):parents = [soup]for _, item in enumerate(stack['content']):children = []@@ -244,11 +251,11 @@ class AutoScraper(object):wanted_attr = stack['wanted_attr']is_full_url = stack['is_full_url']- result = [self._fetch_result_from_child(i, wanted_attr, is_full_url) for i in parents]+ result = [self._fetch_result_from_child(i, wanted_attr, is_full_url, url) for i in parents]result = [x for x in result if x]return result- def _get_result_with_stack_index_based(self, stack, soup):+ def _get_result_with_stack_index_based(self, stack, soup, url):p = soup.findChildren(recursive=False)[0]stack_content = stack['content']for index, item in enumerate(stack_content[:-1]):@@ -259,21 +266,21 @@ class AutoScraper(object):idx = min(len(p) - 1, item[2])p = p[idx]- result = [self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'])]+ result = [self._fetch_result_from_child(p, stack['wanted_attr'], stack['is_full_url'], url)]result = [x for x in result if x]return resultdef _get_result_by_func(self, func, url, html, soup, request_args, grouped):- if url:- self.url = url-if not soup:soup = self._get_soup(url=url, html=html, request_args=request_args)result_list = []grouped_result = defaultdict(list)for stack in self.stack_list:- result = func(stack, soup)+ if not url:+ url = stack.get('url', '')++ result = func(stack, soup, url)if not grouped:result_list += result@@ -282,20 +289,98 @@ class AutoScraper(object):stack_id = stack['stack_id']grouped_result[stack_id] += result- return dict(grouped_result) if grouped else unique(result_list)+ return dict(grouped_result) if grouped else unique_hashable(result_list)def get_result_similar(self, url=None, html=None, soup=None, request_args=None, grouped=False):+ """+ Gets similar results based on the previously learned rules.++ Parameters:+ ----------+ url: str, optional+ URL of the target web page. You should either pass url or html or both.++ html: str, optional+ An HTML string can also be passed instead of URL.+ You should either pass url or html or both.++ request_args: dict, optional+ A dictionary used to specify a set of additional request parameters used by requests+ module. You can specify proxy URLs, custom headers etc.++ grouped: bool, optional, defaults to False+ If set to True, the result will be a dictionary with the rule_ids as keys+ and a list of scraped data per rule as values.++ Returns:+ --------+ List of similar results scraped from the web page.+ Dictionary if grouped=True.+ """+func = self._get_result_with_stackreturn self._get_result_by_func(func, url, html, soup, request_args, grouped)def get_result_exact(self, url=None, html=None, soup=None, request_args=None, grouped=False):+ """+ Gets exact results based on the previously learned rules.++ Parameters:+ ----------+ url: str, optional+ URL of the target web page. You should either pass url or html or both.++ html: str, optional+ An HTML string can also be passed instead of URL.+ You should either pass url or html or both.++ request_args: dict, optional+ A dictionary used to specify a set of additional request parameters used by requests+ module. You can specify proxy URLs, custom headers etc.++ grouped: bool, optional, defaults to False+ If set to True, the result will be a dictionary with the rule_ids as keys+ and a list of scraped data per rule as values.++ Returns:+ --------+ List of exact results scraped from the web page.+ Dictionary if grouped=True.+ """+func = self._get_result_with_stack_index_basedreturn self._get_result_by_func(func, url, html, soup, request_args, grouped)- def get_result(self, url=None, html=None, request_args=None):+ def get_result(self, url=None, html=None, request_args=None, grouped=False):+ """+ Gets similar and exact results based on the previously learned rules.++ Parameters:+ ----------+ url: str, optional+ URL of the target web page. You should either pass url or html or both.++ html: str, optional+ An HTML string can also be passed instead of URL.+ You should either pass url or html or both.++ request_args: dict, optional+ A dictionary used to specify a set of additional request parameters used by requests+ module. You can specify proxy URLs, custom headers etc.++ grouped: bool, optional, defaults to False+ If set to True, the result will be dictionaries with the rule_ids as keys+ and a list of scraped data per rule as values.++ Returns:+ --------+ Pair of (similar, exact) results.+ See get_result_similar and get_result_exact methods.+ """+soup = self._get_soup(url=url, html=html, request_args=request_args)- similar = self.get_result_similar(soup=soup)- exact = self.get_result_exact(soup=soup)+ similar = self.get_result_similar(url=url, soup=soup, grouped=grouped)+ exact = self.get_result_exact(url=url, soup=soup, grouped=grouped)return similar, exactdef remove_rules(self, rules):diff --git a/autoscraper/utils.py b/autoscraper/utils.pyindex 141ced8..33f1e72 100644--- a/autoscraper/utils.py+++ b/autoscraper/utils.py@@ -1,14 +1,24 @@+from collections import OrderedDict+import randomimport string-def unique(item_list):+def unique_stack_list(stack_list):+ seen = set()unique_list = []- for item in item_list:- if item not in unique_list:- unique_list.append(item)+ for stack in stack_list:+ stack_hash = stack['hash']+ if stack_hash in seen:+ continue+ unique_list.append(stack)+ seen.add(stack_hash)return unique_list+def unique_hashable(hashable_items):+ """Removes duplicates from the list. Must preserve the orders."""+ return list(OrderedDict.fromkeys(hashable_items))+def get_random_str(n):chars = string.ascii_lowercase + string.digitsreturn ''.join(random.choice(chars) for i in range(n))