Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

go back instead of go home

Madi24807bc84f81
diff --git a/leads.csv b/leads.csv
index 95d2670..58ce79a 100644
--- a/leads.csv
+++ b/leads.csv
@@ -1,16 +1 @@
Title,Address,WebSite,PhoneNumber
-Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202
-Фитнес Клуб Kapital,Kokshetau 020000,Add website,8 (7162) 50 05 00
-"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",Add website,Add phone number
-Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10
-"Sportkompleks ""Kazhymukan""",Kokshetau 020000,Add website,8 (7162) 77 08 11
-Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",Add website,Add phone number
-Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",Add website,8 (7162) 42 07 80
-Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",Add website,Add phone number
-Fok Pri Sel'khozinstitute,Kokshetau 020000,Add website,Add phone number
-"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",Add website,Add phone number
-DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,Add phone number
-"""Domino""","Akan Seri St, Kokshetau 020000",Add website,8 (7162) 25 30 61
-Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",Add website,Add phone number
-"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03
-Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",Add website,Add phone number
diff --git a/webdriver.py b/webdriver.py
index 95d9c07..9c36a6d 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -101,7 +101,7 @@ class Webdriver:
await self.page.waitForXPath(xpath, {'visible': True})
sleep(1)
except pyppeteer.errors.TimeoutError:
- self._do_retry(operation, xpath, dest, retries + 1)
+ await self._do_retry(operation, xpath, dest, retries + 1)
except Exception as e:
print(e)
raise SystemError('Some shit happened to pyppeteer, fix it')
@@ -109,7 +109,7 @@ class Webdriver:
async def _scrape(self):
while True:
- home = self.page.url
+ # home = self.page.url
places = len(await self.page.xpath(result_xpath))
logger.debug('%s places found', places)
@@ -122,16 +122,19 @@ class Webdriver:
data = self._extract(await self.page.content())
self._buf.store(data)
- await self._do_retry(self.page.goto, result_xpath, home)
+ await self._do_retry(self.page.goBack, result_xpath)
next_button = (await self.page.xpath(next_xpath))[0]
+ await next_button.click()
+ print('Clicked')
+ sleep(3)
- if next_button:
- await self._do_retry(next_button.click, result_xpath)
- logger.debug('Going to the next page')
- else:
- logger.debug('Done with Google Maps')
- break
+ # if next_button:
+ # await self._do_retry(next_button.click, result_xpath)
+ # logger.debug('Going to the next page')
+ # else:
+ # logger.debug('Done with Google Maps')
+ # break
def _extract(self, html):
soup = BeautifulSoup(html, 'html.parser')
@@ -145,10 +148,11 @@ class Webdriver:
attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()
else:
value = soup.find('h1').text.strip()
- if 'Add' or 'Добавить' in value:
- value = None
+ if 'Add' in value or 'Добавить' in value:
+ logger.debug('Got empty information %s', value)
+ value = '-'
except:
- value = None
+ value = '-'
data.update({field: value})
logger.debug('Data: %s', list(data.values()))
@@ -161,7 +165,7 @@ class Webdriver:
async def main():
w = Webdriver()
await w.init_browser()
- await w.search('Кокшетау', 'Спортзал')
+ await w.search('Memphis, Tennessee', 'Loan officers')
if __name__ == '__main__':
asyncio.run(main())