A Smart, Automatic, Fast and Lightweight Web Scraper for Python
Python aiartificial-intelligenceautomationcrawlermachine-learningpythonscrapescraperscrapingweb-scrapingwebautomationwebscraping
add autoscraper
alireza9eb0952e39e0
diff --git a/README.rst b/README.rstindex 8f28f02..22fbfa8 100644--- a/README.rst+++ b/README.rst@@ -42,9 +42,12 @@ Say we want to fetch all related post titles in a stackoverflow page:result = scraper.build(url, wanted_list)print(result)+Here's the output:-.. code::++.. code:: python+['How do I merge two dictionaries in a single expression in Python (taking union of dictionaries)?','How to call an external command?',@@ -87,6 +90,7 @@ Say we want to scrape stock live price from nasdaq:You can also pass any custom `requests` attributes. for example you may want to use proxies:.. code:: python+proxies = {"http": 'http://127.0.0.1:8001',"https": 'https://127.0.0.1:8001',diff --git a/autoscraper/code_template.py b/autoscraper/code_template.pyindex 2c4a42a..956fa8a 100644--- a/autoscraper/code_template.py+++ b/autoscraper/code_template.py@@ -4,7 +4,7 @@ import requestsfrom bs4 import BeautifulSoup-class AutoScraper(object):+class GeneratedAutoScraper(object):request_headers = {'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; \Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'@@ -19,7 +19,7 @@ class AutoScraper(object):if html:return BeautifulSoup(html, 'lxml')request_args = request_args if request_args else {}- headers = request_args.get('headers', AutoScraper.request_headers)+ headers = request_args.get('headers', GeneratedAutoScraper.request_headers)html = requests.get(url, headers=headers, **request_args).textreturn BeautifulSoup(html, 'lxml')