Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
yelp extractor done
Madi1eb8d13d7ba6
diff --git a/yellow-pages/__pycache__/locators.cpython-38.pyc b/yellow-pages/__pycache__/locators.cpython-38.pycindex b6a5b4c..fa8ad1e 100644Binary files a/yellow-pages/__pycache__/locators.cpython-38.pyc and b/yellow-pages/__pycache__/locators.cpython-38.pyc differdiff --git a/yellow-pages/__pycache__/my_config.cpython-38.pyc b/yellow-pages/__pycache__/my_config.cpython-38.pycindex f879c53..e61acb0 100644Binary files a/yellow-pages/__pycache__/my_config.cpython-38.pyc and b/yellow-pages/__pycache__/my_config.cpython-38.pyc differdiff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex 6093bae..9f48a35 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -73,7 +73,10 @@ class Yelp(Webdriver):soup = BeautifulSoup(html, 'html.parser')raw_data = json.loads(soup.find('script', attrs=json_attrs).string)- website = soup.find(attrs=website_attrs).text+ try:+ website = soup.find(attrs=website_attrs).text+ except:+ website = '-'data = {'Title': raw_data['name'],'Address': ', '.join(list(raw_data['address'].values())), 'WebSite': website,@@ -81,18 +84,22 @@ class Yelp(Webdriver):logger.debug('Extracted: %s', list(data.values()))self._buf.store(data)+ sleep(3)while True:- await self._page.waitForXPath(results_xpath)+ await self._page.waitForXPath(places_xpath)places = len(await self._page.xpath(places_xpath))logger.debug('%s places found', places)for i in range(places):- await self._page.waitForXPath(results_xpath)- await (await self._page.xpath(places_xpath))[i].click()++ place = (await self._page.xpath(places_xpath))[i]+ await place.click()await self._page.waitForXPath(inner_xpath)- sleep(1.5)+parse(await self._page.content())+ # await self._do_retry(self._page.goBack, results_xpath)await self._page.goBack()+ await self._page.waitForXPath(results_xpath)next_button = await self._page.xpath(next_xpath)@@ -100,8 +107,8 @@ class Yelp(Webdriver):self._buf.dump()break- await next_button[0].click()- sleep(2)+ await self._do_retry(next_button[0].click, results_xpath)+ sleep(1.5)async def main():diff --git a/yellow-pages/leads.csv b/yellow-pages/leads.csvindex 58ce79a..401f783 100644--- a/yellow-pages/leads.csv+++ b/yellow-pages/leads.csv@@ -1 +1,22 @@Title,Address,WebSite,PhoneNumber+lululemon,"38-42 King's Road SW3 4UD, London, GB, XGL, SW3 4UD",lululemon.co.uk,020 7584 0005+The Sports Edit,"172 Fulham Road, London, GB, XGL, SW10 9PR",thesportsedit.com,020 3793 0964+Wigmore Sports,"39 Wigmore Street, London, GB, XGL, W1U 1PD",wigmoresports.co.uk,020 7486 7761+Solo Sportswear,"401 Mare Street+Hackney+London, London, GB, XGL, E8 1HY",-,020 8985 6951+Hawkesbay Sportswear,"24 Bedford Row, London, GB, ABE, WC1R 4TQ",-,07904 626522+Lillywhites,"24-36 Regent Street, London, GB, XGL, SW1Y 4QF",lillywhites.com,0844 332 5602+Sports Direct,"105 Baker Street, London, GB, XGL, W1U 6EZ",sportsdirect.com,020 7224 0049+Addiscombe Sports,"185 Lower Addiscombe Road+Addiscombe, Croydon, GB, SRY, CR0 6RA",onestopsports.co.uk,020 8654 9513+Bosco,"128-130 The Street+Westfield Stratford City, London, GB, XGL, E20 1EN",boscosport.ru,020 8534 7337+Cenci,"4 Nettleford Place, London, GB, XGL, SE27 0JW",cenci.co.uk,020 8766 8564+Sweaty Betty,"133 Upper Street, London, GB, XGL, N1 1QP",sweatybetty.com,020 7226 4051+JD Sports,"54-55 Chapel Market, London, GB, XGL, N1 9EW",jdsports.co.uk,020 7837 9440+Social Industries,"184 West End Lane+West Hampstead+London, London, GB, XGL, NW6 1SG",-,020 7794 5531+Sweaty Betty,"5 Kensington Church Street, London, GB, XGL, W8 4LD",sweatybetty.com,020 7937 5523+Lik + Neon,"106 Sclater Street, London, GB, XGL, E1 6HR",likneon.com,020 7729 4650diff --git a/yellow-pages/my_config.py b/yellow-pages/my_config.pyindex 3632642..7e54410 100644--- a/yellow-pages/my_config.py+++ b/yellow-pages/my_config.py@@ -1,8 +1 @@-yelp = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'--order = {- 'Title': None,- 'Address': 'icon--24-phone-v2',- 'WebSite': 'icon--24-external-link-v2',- 'PhoneNumber': 'icon--24-directions-v2'-}\ No newline at end of file+yelp = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'\ No newline at end of file