Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
click retry
Madi28702460fb16
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycindex f9c7dc7..8b4cf68 100644Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/__pycache__/driver_retry.cpython-38.pyc b/__pycache__/driver_retry.cpython-38.pycnew file mode 100644index 0000000..7145c11Binary files /dev/null and b/__pycache__/driver_retry.cpython-38.pyc differdiff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pycindex 2b61383..dc8720b 100644Binary files a/__pycache__/locators.cpython-38.pyc and b/__pycache__/locators.cpython-38.pyc differdiff --git a/locators.py b/locators.pyindex 16a5d09..308f672 100644--- a/locators.py+++ b/locators.py@@ -1,8 +1,8 @@button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'search_xpath = '//*[@role="gridcell"]'-next_xpath = '//*[contains(@src, "chevron_right_black_24dp.png")]'+next_xpath = '//*[@id="n7lv7yjyC35__section-pagination-button-next"]'check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'--result_css = '.section-result-content'+place_xpath = '//*[@class="section-hero-header-title-title GLOBAL__gm2-headline-5"]'+result_xpath = '//*[@class="section-result-content"]'# <button jstcache="897" style="display:none"></button>\ No newline at end of filediff --git a/webdriver.py b/webdriver.pyindex e62fe13..da071fe 100644--- a/webdriver.py+++ b/webdriver.py@@ -1,11 +1,13 @@import asyncio+import pyppeteerimport re+from bs4 import BeautifulSoupfrom fake_useragent import UserAgentfrom pyppeteer import launch-from retry import *+from driver_retry import *from time import sleepfrom locators import *@@ -28,6 +30,7 @@ class Webdriver:headless=False,# slowMo=30,autoclose=True,+ args=['--start-maximized'])self.page = (await self.browser.pages())[0]@@ -43,7 +46,8 @@ class Webdriver:async def search(self, location=None, keyword=None, url=None):if location and keyword and not url:- logger.debug('Working based on location `%s` and keyword `%s`', location, keyword)+ logger.debug(+ 'Working based on location `%s` and keyword `%s`', location, keyword)self._location = locationself._keyword = keywordawait self._locate()@@ -83,32 +87,57 @@ class Webdriver:await self._shut_browser()raise ValueError('Got invalid URL for google maps')- async def _scrape(self):- async def wait_for_result():- await self.page.waitForSelector(result_css, {'visible': True})+ async def _retry_click(self, element, xpath, retries=0):+ if retries == 10:+ raise SystemError(+ 'Max 10 retries exceeded when clicking the place')+ try:+ await element.click()+ await self.page.waitForXPath(xpath, {'visible': True})+ print('Returned here')+ sleep(0.7)+ return+ except pyppeteer.errors.TimeoutError:+ self._retry_click(xpath, element, retries + 1)+ print('Timeout')++ else:+ logger.critical('Some shit happened to pyppeteer, fix it')+ print('Critical')++ async def _scrape(self):next_button = Truewhile next_button:- await wait_for_result()+ await self.page.waitForXPath(result_xpath, {'visible': True})home = self.page.url- places = len(await self.page.querySelectorAll(result_css))+ places = len(await self.page.xpath(result_xpath))logger.debug('%s places found', places)for i in range(places):- logger.debug('Gonna scrape %s out of %s', i, places)- await wait_for_result()- place = (await self.page.querySelectorAll(result_css))[i]- await place.click()- sleep(0.8)+ logger.debug('Gonna scrape %s out of %s', i, places)+ await self.page.waitForXPath(result_xpath, {'visible': True})++ place = (await self.page.xpath(result_xpath))[i]+ self._retry_click(place, place_xpath)+ #try:+ # await place.click()+ # await self.page.waitForXPath(place_xpath)+ # sleep(0.7)+ #except:+ # await place.click()+ # await self.page.waitForXPath(place_xpath)+ # sleep(0.7)data = self._extract(await self.page.content())self._buf.store(data)await self.page.goto(home)+ sleep(0.7)next_button = (await self.page.xpath(next_xpath))[0]await next_button.click()