Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
small refinements
Madia60812e4b5f9
diff --git a/google-maps/extractor.py b/google-maps/extractor.pyindex 858dc48..c7dff14 100644--- a/google-maps/extractor.py+++ b/google-maps/extractor.py@@ -5,6 +5,7 @@ from locators import *from my_config import *from time import sleep+from random import uniformfrom bs4 import BeautifulSoupfrom pyppeteer.errors import PageError@@ -48,14 +49,13 @@ class GoogleMaps(Webdriver):async def _enter(word):await self._page.keyboard.type(word)await self._page.keyboard.press('Enter')- # Go to google maps, type the location+try:await self._page.goto(google_maps)await self._page.click('input')await _enter(self._location)sleep(2.5)- # Click search nearby, type the keywordawait self._page.waitForXPath(button_xpath, {'visible': True})await (await self._page.xpath(button_xpath))[2].click()await self._page.waitForXPath(search_xpath)@@ -96,23 +96,27 @@ class GoogleMaps(Webdriver):logger.debug('%s places found', places)for i in range(places):- logger.debug('Gonna scrape %s out of %s', i + 1, places)+ try:+ logger.debug('Gonna scrape %s out of %s', i + 1, places)- place = (await self._page.xpath(result_xpath))[i]- await self._do_retry(place.click, place_xpath)+ place = (await self._page.xpath(result_xpath))[i]+ await self._do_retry(place.click, place_xpath)- data = extract(await self._page.content())- self._buf.store(data)+ data = extract(await self._page.content())+ self._buf.store(data)- back = (await self._page.xpath(back_xpath))[0]- await self._do_retry(back.click, result_xpath)+ back = (await self._page.xpath(back_xpath))[0]+ await self._do_retry(back.click, result_xpath)+ except:+ logger.warning('Error occured',exc_info=True)next_button = (await self._page.xpath(next_xpath))[0]if not next_button:- self._buf.dump()breakawait next_button.click()- sleep(0.7)+ sleep(uniform(0.7, 1))++ self._buf.dump()async def main():diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex 322a3fb..2feb57d 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -118,12 +118,13 @@ class Yelp(Webdriver):next_button = await self._page.xpath(next_xpath)if not next_button:- self._buf.dump()breakawait self._do_retry(next_button[0].click, results_xpath)sleep(uniform(1.3, 1.7))+ self._buf.dump()+async def main():y = Yelp()diff --git a/yellow-pages/leads.csv b/yellow-pages/leads.csvindex 58ce79a..3c465e8 100644--- a/yellow-pages/leads.csv+++ b/yellow-pages/leads.csv@@ -1 +1,46 @@Title,Address,WebSite,PhoneNumber+lululemon,"38-42 King's Road SW3 4UD, London, GB, XGL, SW3 4UD",lululemon.co.uk,020 7584 0005+The Sports Edit,"172 Fulham Road, London, GB, XGL, SW10 9PR",thesportsedit.com,020 3793 0964+Wigmore Sports,"39 Wigmore Street, London, GB, XGL, W1U 1PD",wigmoresports.co.uk,020 7486 7761+Solo Sportswear,"401 Mare Street+Hackney+London, London, GB, XGL, E8 1HY",-,020 8985 6951+Lillywhites,"24-36 Regent Street, London, GB, XGL, SW1Y 4QF",lillywhites.com,0844 332 5602+Hawkesbay Sportswear,"24 Bedford Row, London, GB, ABE, WC1R 4TQ",-,07904 626522+Sports Direct,"105 Baker Street, London, GB, XGL, W1U 6EZ",sportsdirect.com,020 7224 0049+Bosco,"128-130 The Street+Westfield Stratford City, London, GB, XGL, E20 1EN",boscosport.ru,020 8534 7337+Addiscombe Sports,"185 Lower Addiscombe Road+Addiscombe, Croydon, GB, SRY, CR0 6RA",onestopsports.co.uk,020 8654 9513+Cenci,"4 Nettleford Place, London, GB, XGL, SE27 0JW",cenci.co.uk,020 8766 8564+Sweaty Betty,"133 Upper Street, London, GB, XGL, N1 1QP",sweatybetty.com,020 7226 4051+JD Sports,"54-55 Chapel Market, London, GB, XGL, N1 9EW",jdsports.co.uk,020 7837 9440+Social Industries,"184 West End Lane+West Hampstead+London, London, GB, XGL, NW6 1SG",-,020 7794 5531+Sweaty Betty,"5 Kensington Church Street, London, GB, XGL, W8 4LD",sweatybetty.com,020 7937 5523+Nike,"236 Oxford Street, London, GB, XGL, W1C 1DE",nike.com,020 7612 0800+Lik + Neon,"106 Sclater Street, London, GB, XGL, E1 6HR",likneon.com,020 7729 4650+John Varvatos,"12-13 Conduit Street+Mayfair, London, GB, XGL, W1S 2XQ",johnvarvatos.com,04420 374 3220+Sports Direct,"150-154 Oxford Street, London, GB, XGL, W1D 1ND",sportsdirect.com,+Ermenegildo Zegna,"37-38 New Bond Street, London, GB, XGL, W1S 2RU",zegna.com,020 7518 2700+Dy Mensionscreen Printers,"54-56 Bensham Grove, Thornton Heath, GB, XGL, CR7 8DB",dy-mensionscreen.com,020 8653 8210+TK Maxx,"57 King Street, London, GB, XGL, W6 9HW",tkmaxx.com,020 8563 9200+John Lewis & Partners,"300 Oxford Street, London, GB, XGL, W1A 1EX",johnlewis.com,020 7629 7711+n y c,"329 Station Road, London, GB, XGL, HA1 2AA",-,020 8427 3161+Aqua Uniforms Ltd,"274 B High Street, Langley, GB, KEN, SL3 8HD",aquauniforms.co.uk,01753 580722+G&M,"232 Cobham Road+Fetcham, Leatherhead, GB, SRY, KT22 9JQ",-,01372 378303+Solo Sportswear,"70 Kingsland High Street, London, GB, XGL, E8 2NS",-,020 7254 9749+Primark,"13 ST Anns Shopping Centre+ST Anns Road, Harrow, GB, XGL, HA1 1AS",primark.co.uk,020 8861 1843+American Sportswear,"2 Manor Parade, London, GB, XGL, NW10 8TR",americansportswear.us.com,020 8961 7444+Core Sportswears,"Sublime chowk wazirabad road 51310 Sialkot Pakistan, London, GB, XGL, ",coresportswears.com,09252 300 0048+Hard Bodies Sportswear,"Pearlgate House 107 Mays Road+Wood Green, London, GB, XGL, N22 6UP",-,020 8881 4040+Hosana Sportswear,"223 Longley Road, London, GB, XGL, SW17 9LG",-,020 8682 2708+Nike,"Ariel Way, London, GB, XGL, W12 7GF",store.nike.com,020 8749 3725+Sports Direct,"427-429 North End Road, Fulham, GB, XGL, SW6 1NY",sportsdirect.com,0344 332 5032+Jab Sportswear,"50 Valley Hill, Loughton, GB, ESS, IG10 3AL",-,020 8502 4335+Marks & Spencer,"27 King st, London, GB, XGL, W6 9XW",marksandspencer.com,020 8741 8311