Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
retry do added small error fixed
Madi316ba3be4163
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycindex e5338ba..2cf077a 100644Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pycindex dc8720b..498206c 100644Binary files a/__pycache__/locators.cpython-38.pyc and b/__pycache__/locators.cpython-38.pyc differdiff --git a/bufferization.py b/bufferization.pyindex 325f9b1..86fa8b9 100644--- a/bufferization.py+++ b/bufferization.py@@ -1,6 +1,9 @@import csvfrom my_config import order+from logger_config import get_logger++logger = get_logger('buffer')class Buffer:@@ -33,7 +36,6 @@ class Buffer:def store(self, info: dict):if info.get('Title') and not info in self._data:- print('Buffered')self._data.append(info)if len(self._data) >= self._buffer_size:self._dump()diff --git a/leads.csv b/leads.csvindex 58ce79a..95d2670 100644--- a/leads.csv+++ b/leads.csv@@ -1 +1,16 @@Title,Address,WebSite,PhoneNumber+Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202+Фитнес Клуб Kapital,Kokshetau 020000,Add website,8 (7162) 50 05 00+"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",Add website,Add phone number+Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10+"Sportkompleks ""Kazhymukan""",Kokshetau 020000,Add website,8 (7162) 77 08 11+Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",Add website,Add phone number+Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",Add website,8 (7162) 42 07 80+Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",Add website,Add phone number+Fok Pri Sel'khozinstitute,Kokshetau 020000,Add website,Add phone number+"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",Add website,Add phone number+DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,Add phone number+"""Domino""","Akan Seri St, Kokshetau 020000",Add website,8 (7162) 25 30 61+Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",Add website,Add phone number+"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03+Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",Add website,Add phone numberdiff --git a/webdriver.py b/webdriver.pyindex 51f5026..95d9c07 100644--- a/webdriver.py+++ b/webdriver.py@@ -7,7 +7,6 @@ import refrom bs4 import BeautifulSoupfrom fake_useragent import UserAgentfrom pyppeteer import launch-from driver_retry import *from time import sleepfrom locators import *@@ -15,7 +14,7 @@ from my_config import *from logger_config import get_loggerfrom bufferization import Buffer-logger = get_logger('driver')+logger = get_logger('webdriver')class Webdriver:@@ -24,17 +23,16 @@ class Webdriver:_ua = UserAgent()_buf = Buffer()- async def init_browser(self, language='ru'):+ async def init_browser(self, language='en-gb'):self.browser = await launch(ignoreHTTPSErrors=True,headless=False,- # slowMo=30,+ viewport=None,autoclose=True,- args=['--start-maximized']+ args=['--start-maximized'],)self.page = (await self.browser.pages())[0]- # await self.page.setViewport(self.viewport)await self.page.setUserAgent(self._ua.random)await self.page.setExtraHTTPHeaders({'Accept-Language': language})await self.page.reload()@@ -68,16 +66,20 @@ class Webdriver:await self.page.keyboard.type(word)await self.page.keyboard.press('Enter')+ # Go to google maps, type the locationawait self.page.goto(google_maps)await self.page.click('input')await _enter(self._location)-await self.page.waitForXPath(button_xpath, {'visible': True})++ # Click search nearby, type the keywordawait (await self.page.xpath(button_xpath))[2].click()await self.page.waitForXPath(search_xpath)await self.page.click('input')await _enter(self._keyword)+ await self.page.waitForXPath(result_xpath, {'visible': True})+async def _jump(self):try:await self.page.goto(self._url)@@ -87,51 +89,49 @@ class Webdriver:await self._shut_browser()raise ValueError('Got invalid URL for google maps')- async def _retry_click(self, element, xpath, retries=0):+ async def _do_retry(self, operation, xpath, dest=None, retries=0):if retries == 10:raise SystemError('Max 10 retries exceeded when clicking the place')- else:- try:- await element.click()- await self.page.waitForXPath(xpath, {'visible': True})- sleep(0.7)- return-- except pyppeteer.errors.TimeoutError:- self._retry_click(xpath, element, retries + 1)-+ try:+ if dest:+ await operation(dest)else:- raise SystemError('Some shit happened to pyppeteer, fix it')+ await operation()+ await self.page.waitForXPath(xpath, {'visible': True})+ sleep(1)+ except pyppeteer.errors.TimeoutError:+ self._do_retry(operation, xpath, dest, retries + 1)+ except Exception as e:+ print(e)+ raise SystemError('Some shit happened to pyppeteer, fix it')async def _scrape(self):- next_button = True-- while next_button:-- await self.page.waitForXPath(result_xpath, {'visible': True})+ while True:home = self.page.urlplaces = len(await self.page.xpath(result_xpath))logger.debug('%s places found', places)- #for i in range(places):-#- # logger.debug('Gonna scrape %s out of %s', i + 1, places)- # await self.page.waitForXPath(result_xpath, {'visible': True})-#- # place = (await self.page.xpath(result_xpath))[i]- # await self._retry_click(place, place_xpath)-#- # data = self._extract(await self.page.content())- # self._buf.store(data)-#- # await self.page.goto(home)- # sleep(0.7)-- logger.debug('Going to the next page')+ for i in range(places):+ logger.debug('Gonna scrape %s out of %s', i + 1, places)++ place = (await self.page.xpath(result_xpath))[i]+ await self._do_retry(place.click, place_xpath)++ data = self._extract(await self.page.content())+ self._buf.store(data)++ await self._do_retry(self.page.goto, result_xpath, home)+next_button = (await self.page.xpath(next_xpath))[0]- await next_button.click()++ if next_button:+ await self._do_retry(next_button.click, result_xpath)+ logger.debug('Going to the next page')+ else:+ logger.debug('Done with Google Maps')+ breakdef _extract(self, html):soup = BeautifulSoup(html, 'html.parser')@@ -145,6 +145,8 @@ class Webdriver:attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()else:value = soup.find('h1').text.strip()+ if 'Add' or 'Добавить' in value:+ value = Noneexcept:value = Nonedata.update({field: value})