Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

click retry

Madi28702460fb16
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pyc
index f9c7dc7..8b4cf68 100644
Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differ
diff --git a/__pycache__/driver_retry.cpython-38.pyc b/__pycache__/driver_retry.cpython-38.pyc
new file mode 100644
index 0000000..7145c11
Binary files /dev/null and b/__pycache__/driver_retry.cpython-38.pyc differ
diff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pyc
index 2b61383..dc8720b 100644
Binary files a/__pycache__/locators.cpython-38.pyc and b/__pycache__/locators.cpython-38.pyc differ
diff --git a/locators.py b/locators.py
index 16a5d09..308f672 100644
--- a/locators.py
+++ b/locators.py
@@ -1,8 +1,8 @@
button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'
search_xpath = '//*[@role="gridcell"]'
-next_xpath = '//*[contains(@src, "chevron_right_black_24dp.png")]'
+next_xpath = '//*[@id="n7lv7yjyC35__section-pagination-button-next"]'
check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'
-
-result_css = '.section-result-content'
+place_xpath = '//*[@class="section-hero-header-title-title GLOBAL__gm2-headline-5"]'
+result_xpath = '//*[@class="section-result-content"]'
# <button jstcache="897" style="display:none"></button>
\ No newline at end of file
diff --git a/webdriver.py b/webdriver.py
index e62fe13..da071fe 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -1,11 +1,13 @@
import asyncio
+import pyppeteer
import re
+
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from pyppeteer import launch
-from retry import *
+from driver_retry import *
from time import sleep
from locators import *
@@ -28,6 +30,7 @@ class Webdriver:
headless=False,
# slowMo=30,
autoclose=True,
+ args=['--start-maximized']
)
self.page = (await self.browser.pages())[0]
@@ -43,7 +46,8 @@ class Webdriver:
async def search(self, location=None, keyword=None, url=None):
if location and keyword and not url:
- logger.debug('Working based on location `%s` and keyword `%s`', location, keyword)
+ logger.debug(
+ 'Working based on location `%s` and keyword `%s`', location, keyword)
self._location = location
self._keyword = keyword
await self._locate()
@@ -83,32 +87,57 @@ class Webdriver:
await self._shut_browser()
raise ValueError('Got invalid URL for google maps')
- async def _scrape(self):
- async def wait_for_result():
- await self.page.waitForSelector(result_css, {'visible': True})
+ async def _retry_click(self, element, xpath, retries=0):
+ if retries == 10:
+ raise SystemError(
+ 'Max 10 retries exceeded when clicking the place')
+ try:
+ await element.click()
+ await self.page.waitForXPath(xpath, {'visible': True})
+ print('Returned here')
+ sleep(0.7)
+ return
+ except pyppeteer.errors.TimeoutError:
+ self._retry_click(xpath, element, retries + 1)
+ print('Timeout')
+
+ else:
+ logger.critical('Some shit happened to pyppeteer, fix it')
+ print('Critical')
+
+ async def _scrape(self):
next_button = True
while next_button:
- await wait_for_result()
+ await self.page.waitForXPath(result_xpath, {'visible': True})
home = self.page.url
- places = len(await self.page.querySelectorAll(result_css))
+ places = len(await self.page.xpath(result_xpath))
logger.debug('%s places found', places)
for i in range(places):
- logger.debug('Gonna scrape %s out of %s', i, places)
- await wait_for_result()
- place = (await self.page.querySelectorAll(result_css))[i]
- await place.click()
- sleep(0.8)
+ logger.debug('Gonna scrape %s out of %s', i, places)
+ await self.page.waitForXPath(result_xpath, {'visible': True})
+
+ place = (await self.page.xpath(result_xpath))[i]
+ self._retry_click(place, place_xpath)
+ #try:
+ # await place.click()
+ # await self.page.waitForXPath(place_xpath)
+ # sleep(0.7)
+ #except:
+ # await place.click()
+ # await self.page.waitForXPath(place_xpath)
+ # sleep(0.7)
data = self._extract(await self.page.content())
self._buf.store(data)
await self.page.goto(home)
+ sleep(0.7)
next_button = (await self.page.xpath(next_xpath))[0]
await next_button.click()