Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

retry script added

Madi258f6d3dd2d2
diff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pyc
index 7ae0a07..087b78c 100644
Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differ
diff --git a/bufferization.py b/bufferization.py
index 0dca5b9..9f9868b 100644
--- a/bufferization.py
+++ b/bufferization.py
@@ -33,8 +33,10 @@ class Buffer:
def store(self, info: dict):
if info.get('Title') and not info in self._data:
+ print('Buffered')
self._data.append(info)
if len(self._data) >= self._buffer_size:
self._dump()
self._data.clear()
- print(f'Got duplicate or empty dictionary {info}')
\ No newline at end of file
+ else:
+ print(f'Got duplicate or empty dictionary {info}')
\ No newline at end of file
diff --git a/driver_retry.py b/driver_retry.py
new file mode 100644
index 0000000..aa0c9ea
--- /dev/null
+++ b/driver_retry.py
@@ -0,0 +1,9 @@
+# async def _evaluate(page, retries=0):
+# if retries > 10:
+# return await page.evaluate('document.body.innerHTML')
+# else:
+# try:
+# return await page.evaluate('document.body.innerHTML')
+# except pyppeteer.errors.NetworkError:
+# await asyncio.sleep(0.5)
+# return await _evaluate(page, retries+1)
\ No newline at end of file
diff --git a/webdriver.py b/webdriver.py
index a898743..e62fe13 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -5,6 +5,7 @@ import re
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from pyppeteer import launch
+from retry import *
from time import sleep
from locators import *
@@ -102,13 +103,12 @@ class Webdriver:
place = (await self.page.querySelectorAll(result_css))[i]
await place.click()
- sleep(0.7)
+ sleep(0.8)
data = self._extract(await self.page.content())
self._buf.store(data)
await self.page.goto(home)
- sleep(0.7)
next_button = (await self.page.xpath(next_xpath))[0]
await next_button.click()
diff --git a/yt_bot.log b/yt_bot.log
deleted file mode 100644
index e69de29..0000000