Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
go back instead of go home
Madi24807bc84f81
diff --git a/leads.csv b/leads.csvindex 95d2670..58ce79a 100644--- a/leads.csv+++ b/leads.csv@@ -1,16 +1 @@Title,Address,WebSite,PhoneNumber-Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202-Фитнес Клуб Kapital,Kokshetau 020000,Add website,8 (7162) 50 05 00-"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",Add website,Add phone number-Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10-"Sportkompleks ""Kazhymukan""",Kokshetau 020000,Add website,8 (7162) 77 08 11-Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",Add website,Add phone number-Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",Add website,8 (7162) 42 07 80-Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",Add website,Add phone number-Fok Pri Sel'khozinstitute,Kokshetau 020000,Add website,Add phone number-"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",Add website,Add phone number-DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,Add phone number-"""Domino""","Akan Seri St, Kokshetau 020000",Add website,8 (7162) 25 30 61-Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",Add website,Add phone number-"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03-Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",Add website,Add phone numberdiff --git a/webdriver.py b/webdriver.pyindex 95d9c07..9c36a6d 100644--- a/webdriver.py+++ b/webdriver.py@@ -101,7 +101,7 @@ class Webdriver:await self.page.waitForXPath(xpath, {'visible': True})sleep(1)except pyppeteer.errors.TimeoutError:- self._do_retry(operation, xpath, dest, retries + 1)+ await self._do_retry(operation, xpath, dest, retries + 1)except Exception as e:print(e)raise SystemError('Some shit happened to pyppeteer, fix it')@@ -109,7 +109,7 @@ class Webdriver:async def _scrape(self):while True:- home = self.page.url+ # home = self.page.urlplaces = len(await self.page.xpath(result_xpath))logger.debug('%s places found', places)@@ -122,16 +122,19 @@ class Webdriver:data = self._extract(await self.page.content())self._buf.store(data)- await self._do_retry(self.page.goto, result_xpath, home)+ await self._do_retry(self.page.goBack, result_xpath)next_button = (await self.page.xpath(next_xpath))[0]+ await next_button.click()+ print('Clicked')+ sleep(3)- if next_button:- await self._do_retry(next_button.click, result_xpath)- logger.debug('Going to the next page')- else:- logger.debug('Done with Google Maps')- break+ # if next_button:+ # await self._do_retry(next_button.click, result_xpath)+ # logger.debug('Going to the next page')+ # else:+ # logger.debug('Done with Google Maps')+ # breakdef _extract(self, html):soup = BeautifulSoup(html, 'html.parser')@@ -145,10 +148,11 @@ class Webdriver:attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()else:value = soup.find('h1').text.strip()- if 'Add' or 'Добавить' in value:- value = None+ if 'Add' in value or 'Добавить' in value:+ logger.debug('Got empty information %s', value)+ value = '-'except:- value = None+ value = '-'data.update({field: value})logger.debug('Data: %s', list(data.values()))@@ -161,7 +165,7 @@ class Webdriver:async def main():w = Webdriver()await w.init_browser()- await w.search('Кокшетау', 'Спортзал')+ await w.search('Memphis, Tennessee', 'Loan officers')if __name__ == '__main__':asyncio.run(main())