Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

add host to default headers

alireza09ac61d7c7d6
diff --git a/README.md b/README.md
index 1dc0df6..6b1b419 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
![img](https://user-images.githubusercontent.com/17881612/91968083-5ee92080-ed29-11ea-82ec-d99ec85367a5.png)
This project is made for automatic web scraping to make scraping easy.
-It gets a url or the html content of a web page and a list of sample data which we want to scrape from that page. This data can be text, url or any html tag value of that page. It learns the scraping rules and returns the similar elements. Then you can use this learned object with new urls to get similar content or the exact same element of those new pages.
+It gets a url or the html content of a web page and a list of sample data which we want to scrape from that page. **This data can be text, url or any html tag value of that page.** It learns the scraping rules and returns the similar elements. Then you can use this learned object with new urls to get similar content or the exact same element of those new pages.
## Installation
@@ -93,7 +93,19 @@ Now we can get the price of any symbol:
scraper.get_result_exact('https://finance.yahoo.com/quote/MSFT/')
```
-You may want to get other info as well. For example if you want to get market cap too, you can just append it to the wanted list. By using the `get_result_exact` method, it will retrieve the data as the same exact order in the wanted list.
+**You may want to get other info as well.** For example if you want to get market cap too, you can just append it to the wanted list. By using the `get_result_exact` method, it will retrieve the data as the same exact order in the wanted list.
+
+**Another example:** Say we want to scrape the About text, number of stars and the link to pull requests of Github repo pages:
+
+```python
+url = 'https://github.com/alirezamika/autoscraper'
+
+wanted_list = ['A Smart, Automatic, Fast and Lightweight Web Scraper for Python', '662', 'https://github.com/alirezamika/autoscraper/issues']
+
+scraper.build(url, wanted_list)
+```
+
+Simple, right?
### Saving the model
diff --git a/autoscraper/auto_scraper.py b/autoscraper/auto_scraper.py
index a187aac..380a147 100644
--- a/autoscraper/auto_scraper.py
+++ b/autoscraper/auto_scraper.py
@@ -1,6 +1,6 @@
import json
import os
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
@@ -8,8 +8,7 @@ from bs4 import BeautifulSoup
class AutoScraper(object):
request_headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 \
- (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36',
- 'accept': '*/*'
+ (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36'
}
def __init__(self, stack_list=None, url=None):
@@ -29,7 +28,10 @@ class AutoScraper(object):
if html:
return BeautifulSoup(html, 'lxml')
request_args = request_args if request_args else {}
- headers = request_args.get('headers', AutoScraper.request_headers)
+ headers = dict(AutoScraper.request_headers)
+ if url:
+ headers['Host'] = urlparse(url).netloc
+ headers = request_args.get('headers', headers)
html = requests.get(url, headers=headers, **request_args).text
return BeautifulSoup(html, 'lxml')
@@ -145,6 +147,8 @@ class AutoScraper(object):
def _fetch_result_from_child(self, child, wanted_attr, is_full_url):
if wanted_attr is None:
return child.getText().strip().rstrip()
+ if wanted_attr not in child.attrs:
+ return None
if is_full_url:
return urljoin(self.url, child.attrs[wanted_attr])
return child.attrs[wanted_attr]
@@ -196,8 +200,8 @@ class AutoScraper(object):
try:
result.append(
self._get_result_with_stack_index_based(stack, soup))
- except IndexError as e:
- print(e)
+ except IndexError:
+ continue
return self.unique(result)
def get_result(self, url=None, html=None, request_args=None):
@@ -207,6 +211,8 @@ class AutoScraper(object):
return similar, exact
def generate_python_code(self):
+ # deprecated
+ print('This function is deprecated. Please use save() and load() instead.')
file_path = os.path.join(os.path.dirname(__file__), "code_template.py")
with open(file_path, 'r') as f:
code = f.read()
diff --git a/autoscraper/code_template.py b/autoscraper/code_template.py
index 956fa8a..1ecd63d 100644
--- a/autoscraper/code_template.py
+++ b/autoscraper/code_template.py
@@ -1,4 +1,4 @@
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
@@ -19,7 +19,10 @@ class GeneratedAutoScraper(object):
if html:
return BeautifulSoup(html, 'lxml')
request_args = request_args if request_args else {}
- headers = request_args.get('headers', GeneratedAutoScraper.request_headers)
+ headers = dict(GeneratedAutoScraper.request_headers)
+ if url:
+ headers['Host'] = urlparse(url).netloc
+ headers = request_args.get('headers', headers)
html = requests.get(url, headers=headers, **request_args).text
return BeautifulSoup(html, 'lxml')
@@ -34,6 +37,8 @@ class GeneratedAutoScraper(object):
def _fetch_result_from_child(self, child, wanted_attr, is_full_url):
if wanted_attr is None:
return child.getText().strip().rstrip()
+ if wanted_attr not in child.attrs:
+ return None
if is_full_url:
return urljoin(self.url, child.attrs[wanted_attr])
return child.attrs[wanted_attr]
@@ -79,8 +84,8 @@ class GeneratedAutoScraper(object):
for stack in self.stack_list:
try:
result.append(self._get_result_with_stack_index_based(stack, soup))
- except IndexError as e:
- print(e)
+ except IndexError:
+ continue
return self.unique(result)
def get_result(self, url=None, html=None, request_args=None):