Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
yelp class methods added, small enhancments
Madi609f624e389b
diff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pycindex e82db45..f3f9be7 100644Binary files a/__pycache__/webdriver.cpython-38.pyc and b/__pycache__/webdriver.cpython-38.pyc differdiff --git a/driver_retry.py b/driver_retry.pydeleted file mode 100644index f9d550c..0000000--- a/driver_retry.py+++ /dev/null@@ -1,9 +0,0 @@-async def _evaluate(page, retries=0):- if retries > 10:- return await page.evaluate('document.body.innerHTML')- else:- try:- return await page.evaluate('document.body.innerHTML')- except pyppeteer.errors.NetworkError:- await asyncio.sleep(0.5)- return await _evaluate(page, retries+1)diff --git a/google-maps/extractor.py b/google-maps/extractor.pyindex 050e3ca..1a2fbdc 100644--- a/google-maps/extractor.py+++ b/google-maps/extractor.py@@ -8,7 +8,6 @@ from bs4 import BeautifulSoupimport syssys.path.append('..')-from webdriver import Webdriverfrom logger_config import get_logger@@ -87,6 +86,8 @@ class Searcher(Webdriver):# await self._do_retry(self.page.goBack, result_xpath)next_button = (await self.page.xpath(next_xpath))[0]+ if not next_button:+ breakawait next_button.click()sleep(1)diff --git a/locators.py b/locators.pydeleted file mode 100644index b8b736f..0000000--- a/locators.py+++ /dev/null@@ -1,8 +0,0 @@-button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'-search_xpath = '//*[@role="gridcell"]'-next_xpath = '//*[@id="n7lv7yjyC35__section-pagination-button-next"]'-check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'-place_xpath = '//*[@class="section-hero-header-title-title GLOBAL__gm2-headline-5"]'-result_xpath = '//*[@class="section-result-content"]'-back_xpath = '//*[@class="section-back-to-list-button blue-link noprint"]'-diff --git a/main.py b/main.pydeleted file mode 100644index e69de29..0000000diff --git a/webdriver.py b/webdriver.pyindex 59cdd2a..5002a27 100644--- a/webdriver.py+++ b/webdriver.py@@ -1,16 +1,11 @@import asyncioimport pyppeteer-import re--from bs4 import BeautifulSoupfrom fake_useragent import UserAgentfrom pyppeteer import launchfrom time import sleep-from locators import *-from my_config import *from logger_config import get_loggerfrom bufferization import Buffer@@ -36,60 +31,14 @@ class Webdriver:await self.page.setUserAgent(self._ua.random)await self.page.setViewport(self.viewport)await self.page.setExtraHTTPHeaders({'Accept-Language': language})- await self.page.reload()+ # await self.page.reload()async def _shut_browser(self):- await asyncio.sleep(50)+ logger.debug('Shutting down browser')+ sleep(15)await self.page.close()await self.browser.close()- async def search(self, location=None, keyword=None, url=None):- if location and keyword and not url:- logger.debug(- 'Working based on location `%s` and keyword `%s`', location, keyword)- self._location = location- self._keyword = keyword- await self._locate()- elif not (location and keyword) and url:- logger.debug('Working based on given URL `%s`', url)- self._url = url- await self._jump()- else:- await self._shut_browser()- raise ValueError(- f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')-- await self._scrape()- await self._shut_browser()-- async def _locate(self):- async def _enter(word):- await self.page.keyboard.type(word)- await self.page.keyboard.press('Enter')-- # Go to google maps, type the location- await self.page.goto(google_maps)- await self.page.click('input')- await _enter(self._location)- await self.page.waitForXPath(button_xpath, {'visible': True})-- # Click search nearby, type the keyword- await (await self.page.xpath(button_xpath))[2].click()- await self.page.waitForXPath(search_xpath)- await self.page.click('input')- await _enter(self._keyword)-- await self.page.waitForXPath(result_xpath, {'visible': True})-- async def _jump(self):- try:- await self.page.goto(self._url)- logger.debug('Validating given URL')- self.page.waitForXPath(check_xpath, {'visible': True})- except:- await self._shut_browser()- raise ValueError('Got invalid URL for google maps')-async def _do_retry(self, operation, xpath, dest=None, retries=0):if retries == 10:raise SystemError(@@ -106,59 +55,3 @@ class Webdriver:except Exception as e:print(e)raise SystemError('Some shit happened to pyppeteer, fix it')-- async def _scrape(self):- while True:- places = len(await self.page.xpath(result_xpath))- logger.debug('%s places found', places)-- for i in range(places):- logger.debug('Gonna scrape %s out of %s', i + 1, places)-- place = (await self.page.xpath(result_xpath))[i]- await self._do_retry(place.click, place_xpath)-- data = self._extract(await self.page.content())- self._buf.store(data)-- back = (await self.page.xpath(back_xpath))[0]- await self._do_retry(back.click, result_xpath)- # await self._do_retry(self.page.goBack, result_xpath)-- next_button = (await self.page.xpath(next_xpath))[0]- await next_button.click()- sleep(1)-- def _extract(self, html):- soup = BeautifulSoup(html, 'html.parser')-- data = {}-- for field, src in order.items():- try:- if src:- value = soup.find(- attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()- else:- value = soup.find('h1').text.strip()- if 'Add' in value or 'Добавить' in value:- logger.debug('Got empty information %s', value)- value = '-'- except:- value = '-'- data.update({field: value})-- logger.debug('Data: %s', list(data.values()))- return data-- async def _scrape_pages(self):- pass---async def main():- w = Webdriver()- await w.init_browser('ru')- await w.search('New Zealand', 'Pastor')--if __name__ == '__main__':- asyncio.run(main())diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex d07b772..5b95767 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -26,9 +26,6 @@ def validate_args(func):return inner---class Yelp(Webdriver):@validate_args@@ -38,7 +35,16 @@ class Yelp(Webdriver):self._desc = descprint(self._loc, self._desc)-++ async def _jump(self, url):+ pass++ async def scrape(self):+ pass+++ def _extract(self, html):+ soup = BeautifulSoup(html, 'html.parser')y = Yelp('London', 'Pizza')