Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

retry click fixed

Madif877944b8e65
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pyc
index 8b4cf68..e5338ba 100644
Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differ
diff --git a/__pycache__/driver_retry.cpython-38.pyc b/__pycache__/driver_retry.cpython-38.pyc
index 7145c11..cc0ec1e 100644
Binary files a/__pycache__/driver_retry.cpython-38.pyc and b/__pycache__/driver_retry.cpython-38.pyc differ
diff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pyc
index 087b78c..7bfc5a9 100644
Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differ
diff --git a/__pycache__/my_config.cpython-38.pyc b/__pycache__/my_config.cpython-38.pyc
new file mode 100644
index 0000000..c05d077
Binary files /dev/null and b/__pycache__/my_config.cpython-38.pyc differ
diff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pyc
new file mode 100644
index 0000000..e82db45
Binary files /dev/null and b/__pycache__/webdriver.cpython-38.pyc differ
diff --git a/bufferization.py b/bufferization.py
index 9f9868b..325f9b1 100644
--- a/bufferization.py
+++ b/bufferization.py
@@ -1,14 +1,14 @@
import csv
-from config import order
+from my_config import order
class Buffer:
- lower_limit = 20
+ lower_limit = 10
upper_limit = 201
fn = list(order.keys())
- def __init__(self, filename: str = 'leads.csv', buffer_size: int = 20):
+ def __init__(self, filename: str = 'leads.csv', buffer_size: int = 15):
if not filename.endswith('.csv'):
raise ValueError(
'Incorrect filename specified. Filename should end with `.csv`')
diff --git a/driver_retry.py b/driver_retry.py
index aa0c9ea..f9d550c 100644
--- a/driver_retry.py
+++ b/driver_retry.py
@@ -1,9 +1,9 @@
-# async def _evaluate(page, retries=0):
-# if retries > 10:
-# return await page.evaluate('document.body.innerHTML')
-# else:
-# try:
-# return await page.evaluate('document.body.innerHTML')
-# except pyppeteer.errors.NetworkError:
-# await asyncio.sleep(0.5)
-# return await _evaluate(page, retries+1)
\ No newline at end of file
+async def _evaluate(page, retries=0):
+ if retries > 10:
+ return await page.evaluate('document.body.innerHTML')
+ else:
+ try:
+ return await page.evaluate('document.body.innerHTML')
+ except pyppeteer.errors.NetworkError:
+ await asyncio.sleep(0.5)
+ return await _evaluate(page, retries+1)
diff --git a/google-maps/searcher.py b/google-maps/searcher.py
index cf75a5c..8edb576 100644
--- a/google-maps/searcher.py
+++ b/google-maps/searcher.py
@@ -1,8 +1,9 @@
-from pyppeteer import launch
+import sys
+sys.path.append('..')
+from webdriver import Webdriver
-
-class Searcher():
+class Searcher(Webdriver):
def __init__(self):
- pass
\ No newline at end of file
+ pass
diff --git a/config.py b/my_config.py
similarity index 100%
rename from config.py
rename to my_config.py
diff --git a/webdriver.py b/webdriver.py
index da071fe..51f5026 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -11,7 +11,7 @@ from driver_retry import *
from time import sleep
from locators import *
-from config import *
+from my_config import *
from logger_config import get_logger
from bufferization import Buffer
@@ -91,20 +91,18 @@ class Webdriver:
if retries == 10:
raise SystemError(
'Max 10 retries exceeded when clicking the place')
- try:
- await element.click()
- await self.page.waitForXPath(xpath, {'visible': True})
- print('Returned here')
- sleep(0.7)
- return
+ else:
+ try:
+ await element.click()
+ await self.page.waitForXPath(xpath, {'visible': True})
+ sleep(0.7)
+ return
- except pyppeteer.errors.TimeoutError:
- self._retry_click(xpath, element, retries + 1)
- print('Timeout')
+ except pyppeteer.errors.TimeoutError:
+ self._retry_click(xpath, element, retries + 1)
- else:
- logger.critical('Some shit happened to pyppeteer, fix it')
- print('Critical')
+ else:
+ raise SystemError('Some shit happened to pyppeteer, fix it')
async def _scrape(self):
next_button = True
@@ -117,28 +115,21 @@ class Webdriver:
places = len(await self.page.xpath(result_xpath))
logger.debug('%s places found', places)
- for i in range(places):
-
- logger.debug('Gonna scrape %s out of %s', i, places)
- await self.page.waitForXPath(result_xpath, {'visible': True})
-
- place = (await self.page.xpath(result_xpath))[i]
- self._retry_click(place, place_xpath)
- #try:
- # await place.click()
- # await self.page.waitForXPath(place_xpath)
- # sleep(0.7)
- #except:
- # await place.click()
- # await self.page.waitForXPath(place_xpath)
- # sleep(0.7)
-
- data = self._extract(await self.page.content())
- self._buf.store(data)
-
- await self.page.goto(home)
- sleep(0.7)
-
+ #for i in range(places):
+#
+ # logger.debug('Gonna scrape %s out of %s', i + 1, places)
+ # await self.page.waitForXPath(result_xpath, {'visible': True})
+#
+ # place = (await self.page.xpath(result_xpath))[i]
+ # await self._retry_click(place, place_xpath)
+#
+ # data = self._extract(await self.page.content())
+ # self._buf.store(data)
+#
+ # await self.page.goto(home)
+ # sleep(0.7)
+
+ logger.debug('Going to the next page')
next_button = (await self.page.xpath(next_xpath))[0]
await next_button.click()
@@ -170,5 +161,5 @@ async def main():
await w.init_browser()
await w.search('Кокшетау', 'Спортзал')
-
-asyncio.run(main())
+if __name__ == '__main__':
+ asyncio.run(main())