Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

logger config added

Madi8225ea6dd80c
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pyc
index 1b638d9..f9c7dc7 100644
Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differ
diff --git a/__pycache__/config.cpython-38.pyc b/__pycache__/config.cpython-38.pyc
index c88140f..35ba21c 100644
Binary files a/__pycache__/config.cpython-38.pyc and b/__pycache__/config.cpython-38.pyc differ
diff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pyc
index b7c07c0..2b61383 100644
Binary files a/__pycache__/locators.cpython-38.pyc and b/__pycache__/locators.cpython-38.pyc differ
diff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pyc
new file mode 100644
index 0000000..5024658
Binary files /dev/null and b/__pycache__/logger_config.cpython-38.pyc differ
diff --git a/config.py b/config.py
index d50d210..4ddb2eb 100644
--- a/config.py
+++ b/config.py
@@ -1,5 +1,3 @@
-
-
google_maps = 'https://www.google.ru/maps'
order = {
diff --git a/leads.csv b/leads.csv
index 15248b1..58ce79a 100644
--- a/leads.csv
+++ b/leads.csv
@@ -1,61 +1 @@
Title,Address,WebSite,PhoneNumber
-,,,
-Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526
-Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294
-Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988
-Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636
-Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668
-Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567
-Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477
-Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999
-Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220
-Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154
-Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060
-De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203
-White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172
-Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329
-Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830
-Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966
-Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080
-Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887
-Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328
-,,,
-Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526
-Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294
-Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988
-Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636
-Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668
-Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567
-Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477
-Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999
-Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220
-Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154
-Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060
-De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203
-White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172
-Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329
-Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830
-Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966
-Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080
-Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887
-Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328
-,,,
-Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526
-Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294
-Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988
-Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636
-Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668
-Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567
-Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477
-Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999
-Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220
-Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154
-Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060
-De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203
-White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172
-Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329
-Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830
-Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966
-Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080
-Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887
-Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328
diff --git a/locators.py b/locators.py
index 8394f86..16a5d09 100644
--- a/locators.py
+++ b/locators.py
@@ -1,5 +1,8 @@
button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'
search_xpath = '//*[@role="gridcell"]'
-# wait_xpath = '//*[@jstcache="1098"]'
next_xpath = '//*[contains(@src, "chevron_right_black_24dp.png")]'
-check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'
\ No newline at end of file
+check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'
+
+result_css = '.section-result-content'
+
+# <button jstcache="897" style="display:none"></button>
\ No newline at end of file
diff --git a/logger_config.py b/logger_config.py
index f249b3e..81f0d77 100644
--- a/logger_config.py
+++ b/logger_config.py
@@ -1,2 +1,26 @@
import logging
+
+
+def get_logger(name: str):
+ if isinstance(name, str):
+ formatter = logging.Formatter(
+ fmt='%(asctime)s - %(levelname)s:%(levelno)s | %(filename)s/%(funcName)s/%(lineno)d | %(message)s')
+
+ s_handler = logging.StreamHandler()
+ s_handler.setLevel(logging.DEBUG)
+
+ f_handler = logging.FileHandler('yt_bot.log', 'a')
+ f_handler.setLevel(logging.WARNING)
+
+ handlers = [s_handler, f_handler]
+
+ logger = logging.getLogger(name)
+ logger.setLevel(logging.DEBUG)
+
+ for handler in handlers:
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ return logger
+ raise TypeError('Logger name must be string')
\ No newline at end of file
diff --git a/webdriver.py b/webdriver.py
index ece48f0..4cc17a6 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -10,9 +10,12 @@ from time import sleep
from locators import *
from config import *
+from logger_config import get_logger
from bufferization import Buffer
+
ua = UserAgent()
+logger = get_logger('driver')
class Webdriver:
@@ -36,6 +39,7 @@ class Webdriver:
await self.page.reload()
async def _shut_browser(self):
+ await asyncio.sleep(50)
await self.page.close()
await self.browser.close()
@@ -64,19 +68,20 @@ class Webdriver:
await self.page.keyboard.press('Enter')
await self.page.goto(google_maps)
+ await self.page.click('input')
await _enter(self._location)
- await self.page.waitForXPath(button_xpath)
+ await self.page.waitForXPath(button_xpath, {'visible': True})
await (await self.page.xpath(button_xpath))[2].click()
-
await self.page.waitForXPath(search_xpath)
+ await self.page.click('input')
await _enter(self._keyword)
async def _jump(self):
try:
await self.page.goto(self._url)
print('Checking the given URL')
- self.page.waitForXPath(check_xpath)
+ self.page.waitForXPath(check_xpath, {'visible': True})
except:
await self._shut_browser()
raise ValueError('Got invalid URL for google maps')
@@ -84,42 +89,36 @@ class Webdriver:
async def _scrape(self):
next_button = True
while next_button:
- await self.page.waitForSelector('.section-result-content')
+ await self.page.waitForSelector(result_css, {'visible': True})
home = self.page.url
- overall = len(await self.page.querySelectorAll('.section-result-content'))
+ overall = len(await self.page.querySelectorAll(result_css))
print(f'Overall found {overall}')
for i in range(overall):
print(f'Gonna scrape {i} out of {overall}')
- element = (await self.page.querySelectorAll('.section-result-content'))[i]
-
- # await asyncio.wait([
- # element.click(),
- # self.page.waitForNavigation()
- # ])
+ element = (await self.page.querySelectorAll(result_css))[i]
- await element.click()
- await self.page.waitForNavigation()
- sleep(0.7)
+ await asyncio.wait([
+ element.click(),
+ self.page.waitForNavigation()
+ ])
+ sleep(0.5)
- data = self.extract(await self.page.content())
+ data = self._extract(await self.page.content())
self.buf.store(data)
- # await asyncio.wait([
- # self.page.goto(home),
- # self.page.waitForNavigation()
- # ])
-
- await self.page.goto(home)
- await self.page.waitForNavigation()
- sleep(0.7)
+ await asyncio.wait([
+ self.page.goto(home),
+ self.page.waitForNavigation()
+ ])
+ sleep(0.5)
next_button = (await self.page.xpath(next_xpath))[0]
await next_button.click()
- def extract(self, html):
- soup = BeautifulSoup(html, 'lxml')
+ def _extract(self, html):
+ soup = BeautifulSoup(html, 'html.parser')
data = {}
@@ -129,7 +128,7 @@ class Webdriver:
value = soup.find(
attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()
else:
- value = soup.select('h1')[0].text.strip()
+ value = soup.find('h1').text.strip()
except:
value = None
data.update({field: value})
@@ -137,11 +136,14 @@ class Webdriver:
print(data)
return data
+ async def _scrape_pages(self):
+ pass
+
async def main():
w = Webdriver()
await w.init_browser()
- await w.search('Calgary', 'Thai food')
+ await w.search('Кокшетау', 'Спортзал')
asyncio.run(main())
diff --git a/yt_bot.log b/yt_bot.log
new file mode 100644
index 0000000..e69de29