Skip to content

A Smart, Automatic, Fast and Lightweight Web Scraper for Python

add autoscraper

alireza9eb0952e39e0
diff --git a/README.rst b/README.rst
index 8f28f02..22fbfa8 100644
--- a/README.rst
+++ b/README.rst
@@ -42,9 +42,12 @@ Say we want to fetch all related post titles in a stackoverflow page:
result = scraper.build(url, wanted_list)
print(result)
+
Here's the output:
-.. code::
+
+.. code:: python
+
[
'How do I merge two dictionaries in a single expression in Python (taking union of dictionaries)?',
'How to call an external command?',
@@ -87,6 +90,7 @@ Say we want to scrape stock live price from nasdaq:
You can also pass any custom `requests` attributes. for example you may want to use proxies:
.. code:: python
+
proxies = {
"http": 'http://127.0.0.1:8001',
"https": 'https://127.0.0.1:8001',
diff --git a/autoscraper/code_template.py b/autoscraper/code_template.py
index 2c4a42a..956fa8a 100644
--- a/autoscraper/code_template.py
+++ b/autoscraper/code_template.py
@@ -4,7 +4,7 @@ import requests
from bs4 import BeautifulSoup
-class AutoScraper(object):
+class GeneratedAutoScraper(object):
request_headers = {
'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; \
Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'
@@ -19,7 +19,7 @@ class AutoScraper(object):
if html:
return BeautifulSoup(html, 'lxml')
request_args = request_args if request_args else {}
- headers = request_args.get('headers', AutoScraper.request_headers)
+ headers = request_args.get('headers', GeneratedAutoScraper.request_headers)
html = requests.get(url, headers=headers, **request_args).text
return BeautifulSoup(html, 'lxml')