Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

yelp extractor done

Madi1eb8d13d7ba6
diff --git a/yellow-pages/__pycache__/locators.cpython-38.pyc b/yellow-pages/__pycache__/locators.cpython-38.pyc
index b6a5b4c..fa8ad1e 100644
Binary files a/yellow-pages/__pycache__/locators.cpython-38.pyc and b/yellow-pages/__pycache__/locators.cpython-38.pyc differ
diff --git a/yellow-pages/__pycache__/my_config.cpython-38.pyc b/yellow-pages/__pycache__/my_config.cpython-38.pyc
index f879c53..e61acb0 100644
Binary files a/yellow-pages/__pycache__/my_config.cpython-38.pyc and b/yellow-pages/__pycache__/my_config.cpython-38.pyc differ
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index 6093bae..9f48a35 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -73,7 +73,10 @@ class Yelp(Webdriver):
soup = BeautifulSoup(html, 'html.parser')
raw_data = json.loads(soup.find('script', attrs=json_attrs).string)
- website = soup.find(attrs=website_attrs).text
+ try:
+ website = soup.find(attrs=website_attrs).text
+ except:
+ website = '-'
data = {'Title': raw_data['name'],
'Address': ', '.join(list(raw_data['address'].values())), 'WebSite': website,
@@ -81,18 +84,22 @@ class Yelp(Webdriver):
logger.debug('Extracted: %s', list(data.values()))
self._buf.store(data)
+ sleep(3)
while True:
- await self._page.waitForXPath(results_xpath)
+ await self._page.waitForXPath(places_xpath)
places = len(await self._page.xpath(places_xpath))
logger.debug('%s places found', places)
for i in range(places):
- await self._page.waitForXPath(results_xpath)
- await (await self._page.xpath(places_xpath))[i].click()
+
+ place = (await self._page.xpath(places_xpath))[i]
+ await place.click()
await self._page.waitForXPath(inner_xpath)
- sleep(1.5)
+
parse(await self._page.content())
+ # await self._do_retry(self._page.goBack, results_xpath)
await self._page.goBack()
+ await self._page.waitForXPath(results_xpath)
next_button = await self._page.xpath(next_xpath)
@@ -100,8 +107,8 @@ class Yelp(Webdriver):
self._buf.dump()
break
- await next_button[0].click()
- sleep(2)
+ await self._do_retry(next_button[0].click, results_xpath)
+ sleep(1.5)
async def main():
diff --git a/yellow-pages/leads.csv b/yellow-pages/leads.csv
index 58ce79a..401f783 100644
--- a/yellow-pages/leads.csv
+++ b/yellow-pages/leads.csv
@@ -1 +1,22 @@
Title,Address,WebSite,PhoneNumber
+lululemon,"38-42 King's Road SW3 4UD, London, GB, XGL, SW3 4UD",lululemon.co.uk,020 7584 0005
+The Sports Edit,"172 Fulham Road, London, GB, XGL, SW10 9PR",thesportsedit.com,020 3793 0964
+Wigmore Sports,"39 Wigmore Street, London, GB, XGL, W1U 1PD",wigmoresports.co.uk,020 7486 7761
+Solo Sportswear,"401 Mare Street
+Hackney
+London, London, GB, XGL, E8 1HY",-,020 8985 6951
+Hawkesbay Sportswear,"24 Bedford Row, London, GB, ABE, WC1R 4TQ",-,07904 626522
+Lillywhites,"24-36 Regent Street, London, GB, XGL, SW1Y 4QF",lillywhites.com,0844 332 5602
+Sports Direct,"105 Baker Street, London, GB, XGL, W1U 6EZ",sportsdirect.com,020 7224 0049
+Addiscombe Sports,"185 Lower Addiscombe Road
+Addiscombe, Croydon, GB, SRY, CR0 6RA",onestopsports.co.uk,020 8654 9513
+Bosco,"128-130 The Street
+Westfield Stratford City, London, GB, XGL, E20 1EN",boscosport.ru,020 8534 7337
+Cenci,"4 Nettleford Place, London, GB, XGL, SE27 0JW",cenci.co.uk,020 8766 8564
+Sweaty Betty,"133 Upper Street, London, GB, XGL, N1 1QP",sweatybetty.com,020 7226 4051
+JD Sports,"54-55 Chapel Market, London, GB, XGL, N1 9EW",jdsports.co.uk,020 7837 9440
+Social Industries,"184 West End Lane
+West Hampstead
+London, London, GB, XGL, NW6 1SG",-,020 7794 5531
+Sweaty Betty,"5 Kensington Church Street, London, GB, XGL, W8 4LD",sweatybetty.com,020 7937 5523
+Lik + Neon,"106 Sclater Street, London, GB, XGL, E1 6HR",likneon.com,020 7729 4650
diff --git a/yellow-pages/my_config.py b/yellow-pages/my_config.py
index 3632642..7e54410 100644
--- a/yellow-pages/my_config.py
+++ b/yellow-pages/my_config.py
@@ -1,8 +1 @@
-yelp = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'
-
-order = {
- 'Title': None,
- 'Address': 'icon--24-phone-v2',
- 'WebSite': 'icon--24-external-link-v2',
- 'PhoneNumber': 'icon--24-directions-v2'
-}
\ No newline at end of file
+yelp = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'
\ No newline at end of file