Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
logger config added
Madi8225ea6dd80c
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycindex 1b638d9..f9c7dc7 100644Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/__pycache__/config.cpython-38.pyc b/__pycache__/config.cpython-38.pycindex c88140f..35ba21c 100644Binary files a/__pycache__/config.cpython-38.pyc and b/__pycache__/config.cpython-38.pyc differdiff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pycindex b7c07c0..2b61383 100644Binary files a/__pycache__/locators.cpython-38.pyc and b/__pycache__/locators.cpython-38.pyc differdiff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pycnew file mode 100644index 0000000..5024658Binary files /dev/null and b/__pycache__/logger_config.cpython-38.pyc differdiff --git a/config.py b/config.pyindex d50d210..4ddb2eb 100644--- a/config.py+++ b/config.py@@ -1,5 +1,3 @@--google_maps = 'https://www.google.ru/maps'order = {diff --git a/leads.csv b/leads.csvindex 15248b1..58ce79a 100644--- a/leads.csv+++ b/leads.csv@@ -1,61 +1 @@Title,Address,WebSite,PhoneNumber-,,,-Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526-Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294-Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988-Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636-Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668-Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567-Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477-Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999-Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220-Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154-Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060-De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203-White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172-Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329-Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830-Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966-Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080-Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887-Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328-,,,-Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526-Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294-Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988-Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636-Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668-Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567-Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477-Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999-Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220-Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154-Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060-De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203-White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172-Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329-Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830-Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966-Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080-Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887-Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328-,,,-Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526-Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294-Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988-Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636-Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668-Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567-Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477-Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999-Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220-Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154-Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060-De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203-White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172-Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329-Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830-Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966-Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080-Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887-Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328diff --git a/locators.py b/locators.pyindex 8394f86..16a5d09 100644--- a/locators.py+++ b/locators.py@@ -1,5 +1,8 @@button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'search_xpath = '//*[@role="gridcell"]'-# wait_xpath = '//*[@jstcache="1098"]'next_xpath = '//*[contains(@src, "chevron_right_black_24dp.png")]'-check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'\ No newline at end of file+check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'++result_css = '.section-result-content'++# <button jstcache="897" style="display:none"></button>\ No newline at end of filediff --git a/logger_config.py b/logger_config.pyindex f249b3e..81f0d77 100644--- a/logger_config.py+++ b/logger_config.py@@ -1,2 +1,26 @@import logging+++def get_logger(name: str):+ if isinstance(name, str):+ formatter = logging.Formatter(+ fmt='%(asctime)s - %(levelname)s:%(levelno)s | %(filename)s/%(funcName)s/%(lineno)d | %(message)s')++ s_handler = logging.StreamHandler()+ s_handler.setLevel(logging.DEBUG)++ f_handler = logging.FileHandler('yt_bot.log', 'a')+ f_handler.setLevel(logging.WARNING)++ handlers = [s_handler, f_handler]++ logger = logging.getLogger(name)+ logger.setLevel(logging.DEBUG)++ for handler in handlers:+ handler.setFormatter(formatter)+ logger.addHandler(handler)++ return logger+ raise TypeError('Logger name must be string')\ No newline at end of filediff --git a/webdriver.py b/webdriver.pyindex ece48f0..4cc17a6 100644--- a/webdriver.py+++ b/webdriver.py@@ -10,9 +10,12 @@ from time import sleepfrom locators import *from config import *+from logger_config import get_loggerfrom bufferization import Buffer+ua = UserAgent()+logger = get_logger('driver')class Webdriver:@@ -36,6 +39,7 @@ class Webdriver:await self.page.reload()async def _shut_browser(self):+ await asyncio.sleep(50)await self.page.close()await self.browser.close()@@ -64,19 +68,20 @@ class Webdriver:await self.page.keyboard.press('Enter')await self.page.goto(google_maps)+ await self.page.click('input')await _enter(self._location)- await self.page.waitForXPath(button_xpath)+ await self.page.waitForXPath(button_xpath, {'visible': True})await (await self.page.xpath(button_xpath))[2].click()-await self.page.waitForXPath(search_xpath)+ await self.page.click('input')await _enter(self._keyword)async def _jump(self):try:await self.page.goto(self._url)print('Checking the given URL')- self.page.waitForXPath(check_xpath)+ self.page.waitForXPath(check_xpath, {'visible': True})except:await self._shut_browser()raise ValueError('Got invalid URL for google maps')@@ -84,42 +89,36 @@ class Webdriver:async def _scrape(self):next_button = Truewhile next_button:- await self.page.waitForSelector('.section-result-content')+ await self.page.waitForSelector(result_css, {'visible': True})home = self.page.url- overall = len(await self.page.querySelectorAll('.section-result-content'))+ overall = len(await self.page.querySelectorAll(result_css))print(f'Overall found {overall}')for i in range(overall):print(f'Gonna scrape {i} out of {overall}')- element = (await self.page.querySelectorAll('.section-result-content'))[i]-- # await asyncio.wait([- # element.click(),- # self.page.waitForNavigation()- # ])+ element = (await self.page.querySelectorAll(result_css))[i]- await element.click()- await self.page.waitForNavigation()- sleep(0.7)+ await asyncio.wait([+ element.click(),+ self.page.waitForNavigation()+ ])+ sleep(0.5)- data = self.extract(await self.page.content())+ data = self._extract(await self.page.content())self.buf.store(data)- # await asyncio.wait([- # self.page.goto(home),- # self.page.waitForNavigation()- # ])-- await self.page.goto(home)- await self.page.waitForNavigation()- sleep(0.7)+ await asyncio.wait([+ self.page.goto(home),+ self.page.waitForNavigation()+ ])+ sleep(0.5)next_button = (await self.page.xpath(next_xpath))[0]await next_button.click()- def extract(self, html):- soup = BeautifulSoup(html, 'lxml')+ def _extract(self, html):+ soup = BeautifulSoup(html, 'html.parser')data = {}@@ -129,7 +128,7 @@ class Webdriver:value = soup.find(attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()else:- value = soup.select('h1')[0].text.strip()+ value = soup.find('h1').text.strip()except:value = Nonedata.update({field: value})@@ -137,11 +136,14 @@ class Webdriver:print(data)return data+ async def _scrape_pages(self):+ pass+async def main():w = Webdriver()await w.init_browser()- await w.search('Calgary', 'Thai food')+ await w.search('Кокшетау', 'Спортзал')asyncio.run(main())diff --git a/yt_bot.log b/yt_bot.lognew file mode 100644index 0000000..e69de29