A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
add host to default headers
alireza09ac61d7c7d6
diff --git a/README.md b/README.mdindex 1dc0df6..6b1b419 100644--- a/README.md+++ b/README.md@@ -3,7 +3,7 @@This project is made for automatic web scraping to make scraping easy.-It gets a url or the html content of a web page and a list of sample data which we want to scrape from that page. This data can be text, url or any html tag value of that page. It learns the scraping rules and returns the similar elements. Then you can use this learned object with new urls to get similar content or the exact same element of those new pages.+It gets a url or the html content of a web page and a list of sample data which we want to scrape from that page. **This data can be text, url or any html tag value of that page.** It learns the scraping rules and returns the similar elements. Then you can use this learned object with new urls to get similar content or the exact same element of those new pages.## Installation@@ -93,7 +93,19 @@ Now we can get the price of any symbol:scraper.get_result_exact('https://finance.yahoo.com/quote/MSFT/')```-You may want to get other info as well. For example if you want to get market cap too, you can just append it to the wanted list. By using the `get_result_exact` method, it will retrieve the data as the same exact order in the wanted list.+**You may want to get other info as well.** For example if you want to get market cap too, you can just append it to the wanted list. By using the `get_result_exact` method, it will retrieve the data as the same exact order in the wanted list.++**Another example:** Say we want to scrape the About text, number of stars and the link to pull requests of Github repo pages:++```python+url = 'https://github.com/alirezamika/autoscraper'++wanted_list = ['A Smart, Automatic, Fast and Lightweight Web Scraper for Python', '662', 'https://github.com/alirezamika/autoscraper/issues']++scraper.build(url, wanted_list)+```++Simple, right?### Saving the modeldiff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.pyindex a187aac..380a147 100644--- a/autoscraper/auto_scraper.py+++ b/autoscraper/auto_scraper.py@@ -1,6 +1,6 @@import jsonimport os-from urllib.parse import urljoin+from urllib.parse import urljoin, urlparseimport requestsfrom bs4 import BeautifulSoup@@ -8,8 +8,7 @@ from bs4 import BeautifulSoupclass AutoScraper(object):request_headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 \- (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36',- 'accept': '*/*'+ (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36'}def __init__(self, stack_list=None, url=None):@@ -29,7 +28,10 @@ class AutoScraper(object):if html:return BeautifulSoup(html, 'lxml')request_args = request_args if request_args else {}- headers = request_args.get('headers', AutoScraper.request_headers)+ headers = dict(AutoScraper.request_headers)+ if url:+ headers['Host'] = urlparse(url).netloc+ headers = request_args.get('headers', headers)html = requests.get(url, headers=headers, **request_args).textreturn BeautifulSoup(html, 'lxml')@@ -145,6 +147,8 @@ class AutoScraper(object):def _fetch_result_from_child(self, child, wanted_attr, is_full_url):if wanted_attr is None:return child.getText().strip().rstrip()+ if wanted_attr not in child.attrs:+ return Noneif is_full_url:return urljoin(self.url, child.attrs[wanted_attr])return child.attrs[wanted_attr]@@ -196,8 +200,8 @@ class AutoScraper(object):try:result.append(self._get_result_with_stack_index_based(stack, soup))- except IndexError as e:- print(e)+ except IndexError:+ continuereturn self.unique(result)def get_result(self, url=None, html=None, request_args=None):@@ -207,6 +211,8 @@ class AutoScraper(object):return similar, exactdef generate_python_code(self):+ # deprecated+ print('This function is deprecated. Please use save() and load() instead.')file_path = os.path.join(os.path.dirname(__file__), "code_template.py")with open(file_path, 'r') as f:code = f.read()diff --git a/autoscraper/code_template.py b/autoscraper/code_template.pyindex 956fa8a..1ecd63d 100644--- a/autoscraper/code_template.py+++ b/autoscraper/code_template.py@@ -1,4 +1,4 @@-from urllib.parse import urljoin+from urllib.parse import urljoin, urlparseimport requestsfrom bs4 import BeautifulSoup@@ -19,7 +19,10 @@ class GeneratedAutoScraper(object):if html:return BeautifulSoup(html, 'lxml')request_args = request_args if request_args else {}- headers = request_args.get('headers', GeneratedAutoScraper.request_headers)+ headers = dict(GeneratedAutoScraper.request_headers)+ if url:+ headers['Host'] = urlparse(url).netloc+ headers = request_args.get('headers', headers)html = requests.get(url, headers=headers, **request_args).textreturn BeautifulSoup(html, 'lxml')@@ -34,6 +37,8 @@ class GeneratedAutoScraper(object):def _fetch_result_from_child(self, child, wanted_attr, is_full_url):if wanted_attr is None:return child.getText().strip().rstrip()+ if wanted_attr not in child.attrs:+ return Noneif is_full_url:return urljoin(self.url, child.attrs[wanted_attr])return child.attrs[wanted_attr]@@ -79,8 +84,8 @@ class GeneratedAutoScraper(object):for stack in self.stack_list:try:result.append(self._get_result_with_stack_index_based(stack, soup))- except IndexError as e:- print(e)+ except IndexError:+ continuereturn self.unique(result)def get_result(self, url=None, html=None, request_args=None):