Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
retry click fixed
Madif877944b8e65
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycindex 8b4cf68..e5338ba 100644Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/__pycache__/driver_retry.cpython-38.pyc b/__pycache__/driver_retry.cpython-38.pycindex 7145c11..cc0ec1e 100644Binary files a/__pycache__/driver_retry.cpython-38.pyc and b/__pycache__/driver_retry.cpython-38.pyc differdiff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pycindex 087b78c..7bfc5a9 100644Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differdiff --git a/__pycache__/my_config.cpython-38.pyc b/__pycache__/my_config.cpython-38.pycnew file mode 100644index 0000000..c05d077Binary files /dev/null and b/__pycache__/my_config.cpython-38.pyc differdiff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pycnew file mode 100644index 0000000..e82db45Binary files /dev/null and b/__pycache__/webdriver.cpython-38.pyc differdiff --git a/bufferization.py b/bufferization.pyindex 9f9868b..325f9b1 100644--- a/bufferization.py+++ b/bufferization.py@@ -1,14 +1,14 @@import csv-from config import order+from my_config import orderclass Buffer:- lower_limit = 20+ lower_limit = 10upper_limit = 201fn = list(order.keys())- def __init__(self, filename: str = 'leads.csv', buffer_size: int = 20):+ def __init__(self, filename: str = 'leads.csv', buffer_size: int = 15):if not filename.endswith('.csv'):raise ValueError('Incorrect filename specified. Filename should end with `.csv`')diff --git a/driver_retry.py b/driver_retry.pyindex aa0c9ea..f9d550c 100644--- a/driver_retry.py+++ b/driver_retry.py@@ -1,9 +1,9 @@-# async def _evaluate(page, retries=0):-# if retries > 10:-# return await page.evaluate('document.body.innerHTML')-# else:-# try:-# return await page.evaluate('document.body.innerHTML')-# except pyppeteer.errors.NetworkError:-# await asyncio.sleep(0.5)-# return await _evaluate(page, retries+1)\ No newline at end of file+async def _evaluate(page, retries=0):+ if retries > 10:+ return await page.evaluate('document.body.innerHTML')+ else:+ try:+ return await page.evaluate('document.body.innerHTML')+ except pyppeteer.errors.NetworkError:+ await asyncio.sleep(0.5)+ return await _evaluate(page, retries+1)diff --git a/google-maps/searcher.py b/google-maps/searcher.pyindex cf75a5c..8edb576 100644--- a/google-maps/searcher.py+++ b/google-maps/searcher.py@@ -1,8 +1,9 @@-from pyppeteer import launch+import sys+sys.path.append('..')+from webdriver import Webdriver--class Searcher():+class Searcher(Webdriver):def __init__(self):- pass\ No newline at end of file+ passdiff --git a/config.py b/my_config.pysimilarity index 100%rename from config.pyrename to my_config.pydiff --git a/webdriver.py b/webdriver.pyindex da071fe..51f5026 100644--- a/webdriver.py+++ b/webdriver.py@@ -11,7 +11,7 @@ from driver_retry import *from time import sleepfrom locators import *-from config import *+from my_config import *from logger_config import get_loggerfrom bufferization import Buffer@@ -91,20 +91,18 @@ class Webdriver:if retries == 10:raise SystemError('Max 10 retries exceeded when clicking the place')- try:- await element.click()- await self.page.waitForXPath(xpath, {'visible': True})- print('Returned here')- sleep(0.7)- return+ else:+ try:+ await element.click()+ await self.page.waitForXPath(xpath, {'visible': True})+ sleep(0.7)+ return- except pyppeteer.errors.TimeoutError:- self._retry_click(xpath, element, retries + 1)- print('Timeout')+ except pyppeteer.errors.TimeoutError:+ self._retry_click(xpath, element, retries + 1)- else:- logger.critical('Some shit happened to pyppeteer, fix it')- print('Critical')+ else:+ raise SystemError('Some shit happened to pyppeteer, fix it')async def _scrape(self):next_button = True@@ -117,28 +115,21 @@ class Webdriver:places = len(await self.page.xpath(result_xpath))logger.debug('%s places found', places)- for i in range(places):-- logger.debug('Gonna scrape %s out of %s', i, places)- await self.page.waitForXPath(result_xpath, {'visible': True})-- place = (await self.page.xpath(result_xpath))[i]- self._retry_click(place, place_xpath)- #try:- # await place.click()- # await self.page.waitForXPath(place_xpath)- # sleep(0.7)- #except:- # await place.click()- # await self.page.waitForXPath(place_xpath)- # sleep(0.7)-- data = self._extract(await self.page.content())- self._buf.store(data)-- await self.page.goto(home)- sleep(0.7)-+ #for i in range(places):+#+ # logger.debug('Gonna scrape %s out of %s', i + 1, places)+ # await self.page.waitForXPath(result_xpath, {'visible': True})+#+ # place = (await self.page.xpath(result_xpath))[i]+ # await self._retry_click(place, place_xpath)+#+ # data = self._extract(await self.page.content())+ # self._buf.store(data)+#+ # await self.page.goto(home)+ # sleep(0.7)++ logger.debug('Going to the next page')next_button = (await self.page.xpath(next_xpath))[0]await next_button.click()@@ -170,5 +161,5 @@ async def main():await w.init_browser()await w.search('Кокшетау', 'Спортзал')--asyncio.run(main())+if __name__ == '__main__':+ asyncio.run(main())