Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

small refinements

Madia60812e4b5f9
diff --git a/google-maps/extractor.py b/google-maps/extractor.py
index 858dc48..c7dff14 100644
--- a/google-maps/extractor.py
+++ b/google-maps/extractor.py
@@ -5,6 +5,7 @@ from locators import *
from my_config import *
from time import sleep
+from random import uniform
from bs4 import BeautifulSoup
from pyppeteer.errors import PageError
@@ -48,14 +49,13 @@ class GoogleMaps(Webdriver):
async def _enter(word):
await self._page.keyboard.type(word)
await self._page.keyboard.press('Enter')
- # Go to google maps, type the location
+
try:
await self._page.goto(google_maps)
await self._page.click('input')
await _enter(self._location)
sleep(2.5)
- # Click search nearby, type the keyword
await self._page.waitForXPath(button_xpath, {'visible': True})
await (await self._page.xpath(button_xpath))[2].click()
await self._page.waitForXPath(search_xpath)
@@ -96,23 +96,27 @@ class GoogleMaps(Webdriver):
logger.debug('%s places found', places)
for i in range(places):
- logger.debug('Gonna scrape %s out of %s', i + 1, places)
+ try:
+ logger.debug('Gonna scrape %s out of %s', i + 1, places)
- place = (await self._page.xpath(result_xpath))[i]
- await self._do_retry(place.click, place_xpath)
+ place = (await self._page.xpath(result_xpath))[i]
+ await self._do_retry(place.click, place_xpath)
- data = extract(await self._page.content())
- self._buf.store(data)
+ data = extract(await self._page.content())
+ self._buf.store(data)
- back = (await self._page.xpath(back_xpath))[0]
- await self._do_retry(back.click, result_xpath)
+ back = (await self._page.xpath(back_xpath))[0]
+ await self._do_retry(back.click, result_xpath)
+ except:
+ logger.warning('Error occured',exc_info=True)
next_button = (await self._page.xpath(next_xpath))[0]
if not next_button:
- self._buf.dump()
break
await next_button.click()
- sleep(0.7)
+ sleep(uniform(0.7, 1))
+
+ self._buf.dump()
async def main():
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index 322a3fb..2feb57d 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -118,12 +118,13 @@ class Yelp(Webdriver):
next_button = await self._page.xpath(next_xpath)
if not next_button:
- self._buf.dump()
break
await self._do_retry(next_button[0].click, results_xpath)
sleep(uniform(1.3, 1.7))
+ self._buf.dump()
+
async def main():
y = Yelp()
diff --git a/yellow-pages/leads.csv b/yellow-pages/leads.csv
index 58ce79a..3c465e8 100644
--- a/yellow-pages/leads.csv
+++ b/yellow-pages/leads.csv
@@ -1 +1,46 @@
Title,Address,WebSite,PhoneNumber
+lululemon,"38-42 King's Road SW3 4UD, London, GB, XGL, SW3 4UD",lululemon.co.uk,020 7584 0005
+The Sports Edit,"172 Fulham Road, London, GB, XGL, SW10 9PR",thesportsedit.com,020 3793 0964
+Wigmore Sports,"39 Wigmore Street, London, GB, XGL, W1U 1PD",wigmoresports.co.uk,020 7486 7761
+Solo Sportswear,"401 Mare Street
+Hackney
+London, London, GB, XGL, E8 1HY",-,020 8985 6951
+Lillywhites,"24-36 Regent Street, London, GB, XGL, SW1Y 4QF",lillywhites.com,0844 332 5602
+Hawkesbay Sportswear,"24 Bedford Row, London, GB, ABE, WC1R 4TQ",-,07904 626522
+Sports Direct,"105 Baker Street, London, GB, XGL, W1U 6EZ",sportsdirect.com,020 7224 0049
+Bosco,"128-130 The Street
+Westfield Stratford City, London, GB, XGL, E20 1EN",boscosport.ru,020 8534 7337
+Addiscombe Sports,"185 Lower Addiscombe Road
+Addiscombe, Croydon, GB, SRY, CR0 6RA",onestopsports.co.uk,020 8654 9513
+Cenci,"4 Nettleford Place, London, GB, XGL, SE27 0JW",cenci.co.uk,020 8766 8564
+Sweaty Betty,"133 Upper Street, London, GB, XGL, N1 1QP",sweatybetty.com,020 7226 4051
+JD Sports,"54-55 Chapel Market, London, GB, XGL, N1 9EW",jdsports.co.uk,020 7837 9440
+Social Industries,"184 West End Lane
+West Hampstead
+London, London, GB, XGL, NW6 1SG",-,020 7794 5531
+Sweaty Betty,"5 Kensington Church Street, London, GB, XGL, W8 4LD",sweatybetty.com,020 7937 5523
+Nike,"236 Oxford Street, London, GB, XGL, W1C 1DE",nike.com,020 7612 0800
+Lik + Neon,"106 Sclater Street, London, GB, XGL, E1 6HR",likneon.com,020 7729 4650
+John Varvatos,"12-13 Conduit Street
+Mayfair, London, GB, XGL, W1S 2XQ",johnvarvatos.com,04420 374 3220
+Sports Direct,"150-154 Oxford Street, London, GB, XGL, W1D 1ND",sportsdirect.com,
+Ermenegildo Zegna,"37-38 New Bond Street, London, GB, XGL, W1S 2RU",zegna.com,020 7518 2700
+Dy Mensionscreen Printers,"54-56 Bensham Grove, Thornton Heath, GB, XGL, CR7 8DB",dy-mensionscreen.com,020 8653 8210
+TK Maxx,"57 King Street, London, GB, XGL, W6 9HW",tkmaxx.com,020 8563 9200
+John Lewis & Partners,"300 Oxford Street, London, GB, XGL, W1A 1EX",johnlewis.com,020 7629 7711
+n y c,"329 Station Road, London, GB, XGL, HA1 2AA",-,020 8427 3161
+Aqua Uniforms Ltd,"274 B High Street, Langley, GB, KEN, SL3 8HD",aquauniforms.co.uk,01753 580722
+G&M,"232 Cobham Road
+Fetcham, Leatherhead, GB, SRY, KT22 9JQ",-,01372 378303
+Solo Sportswear,"70 Kingsland High Street, London, GB, XGL, E8 2NS",-,020 7254 9749
+Primark,"13 ST Anns Shopping Centre
+ST Anns Road, Harrow, GB, XGL, HA1 1AS",primark.co.uk,020 8861 1843
+American Sportswear,"2 Manor Parade, London, GB, XGL, NW10 8TR",americansportswear.us.com,020 8961 7444
+Core Sportswears,"Sublime chowk wazirabad road 51310 Sialkot Pakistan, London, GB, XGL, ",coresportswears.com,09252 300 0048
+Hard Bodies Sportswear,"Pearlgate House 107 Mays Road
+Wood Green, London, GB, XGL, N22 6UP",-,020 8881 4040
+Hosana Sportswear,"223 Longley Road, London, GB, XGL, SW17 9LG",-,020 8682 2708
+Nike,"Ariel Way, London, GB, XGL, W12 7GF",store.nike.com,020 8749 3725
+Sports Direct,"427-429 North End Road, Fulham, GB, XGL, SW6 1NY",sportsdirect.com,0344 332 5032
+Jab Sportswear,"50 Valley Hill, Loughton, GB, ESS, IG10 3AL",-,020 8502 4335
+Marks & Spencer,"27 King st, London, GB, XGL, W6 9XW",marksandspencer.com,020 8741 8311