Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

yelp extractor finished

Madi591b0083b69d
diff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pyc
index 545d0db..f6e3c78 100644
Binary files a/__pycache__/webdriver.cpython-38.pyc and b/__pycache__/webdriver.cpython-38.pyc differ
diff --git a/google-maps/__pycache__/my_config.cpython-38.pyc b/google-maps/__pycache__/my_config.cpython-38.pyc
new file mode 100644
index 0000000..5ab18e2
Binary files /dev/null and b/google-maps/__pycache__/my_config.cpython-38.pyc differ
diff --git a/google-maps/extractor.py b/google-maps/extractor.py
index d0f3ad6..d9be21c 100644
--- a/google-maps/extractor.py
+++ b/google-maps/extractor.py
@@ -1,17 +1,18 @@
import re
-
import asyncio
from locators import *
+from my_config import *
from time import sleep
from bs4 import BeautifulSoup
+from pyppeteer.errors import PageError
+
import sys
sys.path.append('..')
-from webdriver import Webdriver
from logger_config import get_logger
-from my_config import *
+from webdriver import Webdriver
logger = get_logger('webdriver.google_maps')
@@ -19,44 +20,77 @@ logger = get_logger('webdriver.google_maps')
class GoogleMaps(Webdriver):
-
-
async def search(self, location=None, keyword=None, url=None):
- if not 'page' in self.__dict__:
- raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')
+ if not '_page' in self.__dict__:
+ raise ValueError(
+ 'Initialize the browser before searching by `await *.init_broswser()`')
+
if location and keyword and not url:
logger.debug(
'Working based on location `%s` and keyword `%s`', location, keyword)
self._location = location
self._keyword = keyword
await self._locate()
+
elif not (location and keyword) and url:
- logger.debug('Working based on given URL `%s`', url)
await self._jump(url, search_xpath)
+ logger.debug('Working based on given URL `%s`', url)
+
else:
await self._shut_browser()
raise ValueError(
f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')
+
await self._scrape()
await self._shut_browser()
async def _locate(self):
- # Go to google maps, type the location
- await self._page.goto(google_maps)
- await self._page.click('input')
- await self._enter(self._location)
- await self._page.waitForXPath(button_xpath, {'visible': True})
+ async def _enter(word):
+ await self._page.keyboard.type(word)
+ await self._page.keyboard.press('Enter')
+ # Go to google maps, type the location
+ try:
+ await self._page.goto(google_maps)
+ await self._page.click('input')
+ await _enter(self._location)
+ sleep(2.5)
+
+ # Click search nearby, type the keyword
+ await self._page.waitForXPath(button_xpath, {'visible': True})
+ await (await self._page.xpath(button_xpath))[2].click()
+ await self._page.waitForXPath(search_xpath)
+ await self._page.click('input')
+ await _enter(self._keyword)
+
+ await self._page.waitForXPath(result_xpath, {'visible': True})
+ except PageError:
+ self._page.reload()
+ self._locate()
- # Click search nearby, type the keyword
- await (await self._page.xpath(button_xpath))[2].click()
- await self._page.waitForXPath(search_xpath)
- await self._page.click('input')
- await self._enter(self._keyword)
+ async def _scrape(self):
- await self._page.waitForXPath(result_xpath, {'visible': True})
+ def extract(html):
+ soup = BeautifulSoup(html, 'html.parser')
+
+ data = {}
+
+ for field, src in order.items():
+ try:
+ if src:
+ value = soup.find(attrs={'src': re.compile(
+ src)}).parent.parent.parent.text.strip()
+ else:
+ value = soup.find('h1').text.strip()
+ if 'Add' in value or 'Добавить' in value:
+ logger.debug('Got empty information %s', value)
+ value = '-'
+ except:
+ value = '-'
+ data.update({field: value})
+ logger.debug('Data: %s', list(data.values()))
+ return data
- async def _scrape(self):
while True:
places = len(await self._page.xpath(result_xpath))
logger.debug('%s places found', places)
@@ -67,7 +101,7 @@ class GoogleMaps(Webdriver):
place = (await self._page.xpath(result_xpath))[i]
await self._do_retry(place.click, place_xpath)
- data = self._extract(await self._page.content())
+ data = extract(await self._page.content())
self._buf.store(data)
back = (await self._page.xpath(back_xpath))[0]
@@ -78,37 +112,14 @@ class GoogleMaps(Webdriver):
self._buf.dump()
break
await next_button.click()
- sleep(1)
-
- def _extract(self, html):
- soup = BeautifulSoup(html, 'html.parser')
-
- data = {}
-
- for field, src in order.items():
- try:
- if src:
- value = soup.find(
- attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()
- else:
- value = soup.find('h1').text.strip()
- if 'Add' in value or 'Добавить' in value:
- logger.debug('Got empty information %s', value)
- value = '-'
- except:
- value = '-'
- data.update({field: value})
-
- logger.debug('Data: %s', list(data.values()))
- return data
+ sleep(0.7)
async def main():
g = GoogleMaps()
- await g.init_browser(headless=True)
- await g.search('Кокшетау','Спортзал')
-
+ await g.init_browser()
+ await g.search('London', 'Sportswear')
if __name__ == '__main__':
- asyncio.run(main())
\ No newline at end of file
+ asyncio.run(main())
diff --git a/google-maps/leads.csv b/google-maps/leads.csv
index 3106c8b..a40d3e9 100644
--- a/google-maps/leads.csv
+++ b/google-maps/leads.csv
@@ -1,121 +1,76 @@
Title,Address,WebSite,PhoneNumber
-Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202
-Фитнес Клуб Kapital,Kokshetau 020000,-,8 (7162) 50 05 00
-"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",-,-
-Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10
-"Sportkompleks ""Kazhymukan""",Kokshetau 020000,-,8 (7162) 77 08 11
-Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",-,-
-Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",-,8 (7162) 42 07 80
-Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",-,-
-Fok Pri Sel'khozinstitute,Kokshetau 020000,-,-
-"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",-,-
-DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,-
-"""Domino""","Akan Seri St, Kokshetau 020000",-,8 (7162) 25 30 61
-Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",-,-
-"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03
-Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",-,-
-Lider,"Абай Көшесі, Petropavl 150000",-,-
-"Dvorets Sporta ""Burabay""","Burabay, A-343 85, Kokshetau 020000",dsb.kz,8 (7162) 32 82 84
-Sto,Sergeevka 150000,-,-
-Тренажерный Зал XXL,Nur-Sultan 010000,-,-
-Grand Gym,"020000, Nur-Sultan",-,8 (707) 177 7327
-Fitness room,"Kenzhebek Kumisbekov St 7, Nur-Sultan 020000",-,8 (778) 169 2621
-Argo,"Mira St 41, Stepnogorsk 021501",-,8 (71645) 6 38 78
-Grand Fitness,"Сары Арка 31/2, вход со стороны ул.Жангельдина, проспект Сарыарка, Saryarka Avenue, Nur-Sultan 020000",business.site,8 (702) 132 8294
-T-Tau,"Respublika Ave, Nur-Sultan 010000",vk.com,8 (7172) 40 38 27
-ДЮСШ «Триатлон»,Kokshetau 020000,-,8 (7162) 42 05 91
-Start,"Здание 7,, Tselinogradskaya st. 50, Stepnogorsk",-,8 (71645) 3 60 59
-Sportzal,"Ulitsa Muravlenko, 13, Tyumen, Tyumen Oblast, Russia, 625022",-,-
-Trenazhernyy Zal,Karagandy 100000,-,-
-Flex Gym,"Ulitsa Pushkina, 59, Omsk, Omsk Oblast, Russia, 644010",flexgym55.ru,+7 381 266-66-99
-Rekord,"110000, Al Farabi Ave 124, Kostanay",-,8 (7142) 56 87 08
-Фитнес Клуб World CLASS,Karagandy 100000,worldclass.kz,-
-Okzhetpes Stadium,"Abay Street 116, Kokshetau 020000",okzhetpes.kz,8 (7162) 92 60 16
-Space Fitness,"Amanat St, Nur-Sultan 010000",spacefitness.kz,8 (707) 650 5050
-Gym Studio,"Karla Marksa Prospekt, 32, Omsk, Omsk Oblast, Russia, 644042",-,+7 381 232-05-00
-ENERGY CLUB,"3, Lomov St 164/11, Pavlodar",-,8 (7182) 66 37 85
-Life Fitness,"Туркестан, 2, Nur-Sultan 010000",lifefit.kz,8 (7172) 79 73 73
-Trenazhernyy Zal,"Auezov Street, Aksu 140000",-,-
-Фитнес-клуб Prime,"Ulitsa Kuybysheva, 55, 2 этаж, Kurgan, Kurgan Oblast, Russia, 640020",kurgan.zoon.ru,+7 912 573-56-83
-Sportzal,"Ulitsa Malysheva, 138а, Yekaterinburg, Sverdlovsk Oblast, Russia, 620078",-,-
-Ples,Shoptykol 020000,-,-
-Sportzal,"Yemashi, Republic of Bashkortostan, Russia, 452590",-,-
-"Gostinitsa ""Dostyk""","Abay Street 69, Kokshetau 020000",dostyqhotel.kz,-
-HARD GYM,"Ulitsa Svobody, д.110Б, Chelyabinsk, Chelyabinsk Oblast, Russia, 454005",-,-
-"Ostanovka ""Sportzal""","Altynsarin Street, Kostanay 110000",-,-
-"Fitnes Tsentr ""Arena""",Semey 070000,-,-
-Fokus,"Zhambyl St 4-6, Pavlodar 140000",vk.com,8 (7182) 62 44 24
-Ozon,"Komarova, 11к1, 4 этаж, Omsk, Omsk Oblast, Russia, 644112",ozone-fitness.ru,+7 381 297-24-24
-Agrimer-Astyk LLP,Kokshetau 020000,-,-
-Kapital,"Sabataev Street 82, Kokshetau 020000",-,-
-Sportzal,"Tolmachevskoye Shosse, Novosibirsk, Novosibirsk Oblast, Russia, 633100",-,-
-STEEL BODY,"Solnechnyy Proyezd, 7, Tyumen, Tyumen Oblast, Russia, 625022",-,+7 345 260-06-85
-Winox Fitness club & SPA,"Tauelsizdik Ave 32, Nur-Sultan 020000",winox.kz,8 (701) 110 1010
-Sportzal,"Ulitsa Marshala Zhukova, 2/11, Ufa, Republic of Bashkortostan, Russia, 450073",-,-
-"Strong & Smart, fitness club","Ulitsa Brat'yev Kashirinykh, 54Б, Chelyabinsk, Chelyabinsk Oblast, Russia, 454106",strsm.ru,+7 909 079-86-66
-Sportzal,"Ulitsa Rabkorov, 16/1, Ufa, Republic of Bashkortostan, Russia, 450092",-,+7 347 250-29-02
-"Energiya, Atleticheskiy Klub","Ulitsa Yel'kina, 63А, Chelyabinsk, Chelyabinsk Oblast, Russia, 454092",-,+7 351 263-56-40
-FLEX FITNESS CLUB,"Kabanbay Batyr St 41, Ust'-Kamenogorsk 070000",-,8 (777) 726 2830
-Shkola Olimpiyskogo Rezerva,"б, Erubaev St 43, Karagandy 100000",-,-
-Sportzal,"Ulitsa Yeroshevskogo, 18, Samara, Samara Oblast, Russia, 443056",-,-
-Sportzal,"Zelenyy Prospekt, 20, Moscow, Russia, 111401",-,-
-Sportzal,"Zelenyy Prospekt, 20, Moscow, Russia, 111401",-,-
-Sportzal,"Lesnaya Ulitsa, 6А, Aktiubinskii, Republic of Tatarstan, Russia, 423304",-,-
-Sportzal,"Krasnogorsk, Uzbekistan",-,-
-Neptun,"Makhmet Qayyrbayev St 85, Pavlodar 140000",neptun.org.kz,8 (7182) 32 85 87
-Sportzal,"с, Leningradsky Ave, 31А строение 1, Moscow, Russia, 125284",-,-
-FitnessGym,"Sadovnikov, Almaty 050000",fitnessgym.kz,8 (7272) 49 54 59
-Sportzal,"Ulitsa Severnoye Kol'tso, 16А, Kirov, Kirov Oblast, Russia, 610006",-,+7 833 225-05-12
-Sportzal,"Kogalym, Ханты-Мансийский Автономный округ - Югра АО, Russia, 628483",-,-
-Sportzal,"Naberezhnaya Ulitsa, 37-41, Sol-Iletsk, Orenburg Oblast, Russia, 461505",-,-
-Lider,"Средняя школа №60, Ulitsa Dolgireva, 91, Omsk, Omsk Oblast, Russia, 644034",-,+7 950 781-46-78
-Liga Chempionov,"Ulitsa Sputnikov, 11, Yekaterinburg, Sverdlovsk Oblast, Russia, 620025",m.vk.com,+7 961 775-85-07
-Sportzal,"Гимназия №4, Ulitsa Engel'sa, 1А, 2 этаж, Cheboksary, Chuvashia Republic, Russia, 428003",-,+7 835 262-41-42
-Sportivnyy Zal,Shieli,-,-
-Sportzal,"Shosse Revolyutsii, 15, St Petersburg, Russia, 195176",-,-
-Sportzal,"St Petersburg, Russia, 197022",-,-
-Za Gym,"Naymanbaev St, Semey 070000",-,8 (7222) 44 44 46
-Lider,"Grammar school №159, Barnaul'skaya Ulitsa, 134, Omsk, Omsk Oblast, Russia, 644105",-,+7 904 588-17-10
-Art-fitness,"Krylov Street 70, Ust'-Kamenogorsk 070000",artfitness.kz,8 (7232) 54 44 10
-Sportzal,"Volgograd, Volgograd Oblast, Russia, 400050",-,-
-Hammer,Aktobe,-,-
-Sportzal,"Ulitsa Belinskogo, 61, Nizhny Novgorod, Nizhny Novgorod Oblast, Russia, 603006",-,-
-"Тренажерный зал ""Гранат""","Alchevskyh St, 31, Kharkiv, Kharkiv Oblast, Ukraine, 61000",-,+380 97 867 1661
-Sportzal,"Monastyrskoye, Republic of Tatarstan, Russia, 422392",-,-
-Спорт Зал,"Kyiv, Ukraine, 02000",-,+380 63 897 7297
-Sportzal,"Ulitsa Dzerzhinskogo, 38А, Astrakhan, Astrakhan Oblast, Russia, 414006",-,-
-Sportzal,"Verkhnyaya Unzha, Kostroma Oblast, Russia, 157440",-,-
-Sportzal,"Prospekt Akushinskogo, 114, Makhachkala, Republic of Dagestan, Russia, 367014",-,-
-Sportzal,"Volgograd, Volgograd Oblast, Russia, 400080",-,-
-Sportzal,"Ulitsa Tsiolkovskogo, 28, Makhachkala, Republic of Dagestan, Russia, 367008",-,-
-Sportzal,"Olimpiyskiy Prospekt, 33, Mytishchi, Moscow Oblast, Russia, 141006",-,-
-Спортзал,"14 Bagratuniats St, Yerevan 0039, Armenia",-,+374 94 428502
-Sportzal,"Ulitsa Aviator Mikrorayon, 73, Achinsk, Krasnoyarsk Krai, Russia, 662150",-,-
-Sportzal,"Ulitsa Novoazhimova, 3, Izhevsk, Udmurt Republic, Russia, 426006",-,-
-Sportzal,"вуліца Глінкі 44, Baranovichi, Belarus",спортзал-барановичи.бел,+375 29 124-95-87
-ORGANIZM 23 Августа,"23-ho Serpnya St, 43, Kharkiv, Kharkiv Oblast, Ukraine, 61000",organizm.in.ua,+380 95 917 1212
-Gladiator Gym,"Kostyantyna Dan'kevycha St, 12, Kyiv, Ukraine, 02000",nailspro.kiev.ua,+380 93 379 7949
-Sportzal,"Stavropol Krai, Russia, 356195",-,+7 905 490-25-82
-Sportzal,"Ulitsa Sverdlova, 3, Bogorodsk, Nizhny Novgorod Oblast, Russia, 607600",-,-
-Sportzal,"Ulitsa Leningradskaya, 97, Vologda, Vologda Oblast, Russia, 160034",-,+7 981 503-20-10
-Sportzal,"Nurlat, Republic of Tatarstan, Russia, 423042",-,-
-Sportzal,"2-Y Mikrorayon, 1а, Shushenskoe, Krasnoyarsk Krai, Russia, 662710",-,-
-Sportzal,"3-Ya Karta, Volgograd Oblast, Russia, 404150",-,-
-Gym,"Ulitsa Kirova, 44/1, Novosibirsk, Novosibirsk Oblast, Russia, 630102",-,+7 383 383-02-72
-Фитнес клуб Sport Zone,"Heroiv Pratsi St, 60В, Kharkiv, Kharkiv Oblast, Ukraine, 61000",instagram.com,+380 50 553 0155
-Razdevalka I Dushevaya Zhenskaya,"НТТЖКиГХ, Черноисточинское ш., 58, Nizhny Tagil, Sverdlovsk Oblast, Russia, 622030",nttgk-gh.umi.ru,+7 343 547-62-99
-Sportzal,"Almak, Republic of Dagestan, Russia, 368150",-,-
-Sportzal,"Belgradskaya Ulitsa, 30, St Petersburg, Russia, 192212",-,-
-Sportzal,"Republic of Dagestan, Russia, 368935",-,-
-Sport & Fit,"Sofiivska Borschahivka, Kyiv Oblast, Ukraine",mixsport.pro,+380 98 230 9090
-"Спортзала ""Alex""","1а, Soborna St, Kropyvnytskyi, Kirovohrad Oblast, Ukraine, 25000",-,+380 50 814 6246
-Sportzal,"Ulitsa Tani Baramzinoy, 4, Glazov, Udmurt Republic, Russia, 427622",-,-
-Тренажерный зал,"Kniazia Volodymyra Velykoho Street, 12, Dnipro, Dnipropetrovsk Oblast, Ukraine, 49000",-,+380 67 784 3785
-Sportzal,"Tsurib, Republic of Dagestan, Russia, 368450",-,-
-Sportzal,"Kugesi, Chuvashia Republic, Russia, 429500",-,-
-Sportzal,"Ulitsa Gor'kogo, 162В, Kaliningrad, Kaliningrad Oblast, Russia, 236029",-,+7 401 232-33-06
-Fitness Palace,"Turan Ave 30, Nur-Sultan 010000",fitness-palace.kz,8 (7172) 56 77 77
-Sportzal,"Bavtugay, Republic of Dagestan, Russia, 368101",-,-
-Sportzal,"Kovalia St, 3, Poltava, Poltava Oblast, Ukraine, 36000",-,-
-Sportzal,"а, Ulitsa Antonova-Ovseyenko, 31, Voronezh, Voronezh Oblast, Russia, 394088",-,-
-Sportzal,"Pervomansk, Krasnoyarsk Krai, Russia, 663502",-,-
+Runners Need Victoria,"24 Palace St, Victoria, London SW1E 5JD, United Kingdom",runnersneed.com,+44 20 3780 2062
+Sports Direct,"Piccadilly Circus, 24-36 Regent St, St. James's, London SW1Y 4QF, United Kingdom",sportsdirect.com,+44 344 332 5602
+Striders Edge Ltd,"Elizabeth House, 39 York Rd, Bishop's, London SE1 7NQ, United Kingdom",stridersedge.com,-
+Lillywhites,"24-36 Regent St, St. James's, London SW1Y 4QF, United Kingdom",lillywhites.com,+44 344 332 5602
+lululemon,"125-126 Long Acre, Covent Garden, London WC2E 9LD, United Kingdom",lululemon.co.uk,+44 20 7836 5052
+Ten Edit Boutique - Activewear Shop,"83 Great Titchfield St, Fitzrovia, London W1W 6RH, United Kingdom",ten.co.uk,+44 20 7636 5111
+Sportex,"2ND FLOOR, 67-68 Jermyn St, St. James's, London SW1Y 6NY, United Kingdom",sportex.online,+44 800 634 9304
+Champion Store,"48-50 Brewer St, Soho, London W1F 9TG, United Kingdom",championstore.com,+44 20 7734 3087
+Sports Direct,"150-154 Oxford St, Fitzrovia, London W1D 1ND, United Kingdom",sportsdirect.com,+44 344 332 5666
+lululemon,"38, 42 King's Rd, Chelsea, London SW3 4UD, United Kingdom",lululemon.co.uk,+44 20 7584 0005
+Sports Direct,"16-18 Ludgate Hill, London EC4M 7DR, United Kingdom",sportsdirect.com,+44 344 332 5211
+Columbia Sportswear Carnaby Street,"39 Carnaby St, Soho, London W1F 7DP, United Kingdom",columbiasportswear.co.uk,+44 20 3966 6195
+GapBody,"374 Oxford St, London W1C 1JR, United Kingdom",-,-
+GapBody,"60 Gracechurch St, Bridge, London EC3V 0HR, United Kingdom",gap.com,-
+GapBody,"223-225 Oxford St, West End, London W1D 2LR, United Kingdom",-,-
+GapBody,"124B King's Rd, Chelsea, London SW3 4TR, United Kingdom",-,-
+Runners Need Piccadilly,"23/26 Piccadilly, Mayfair, London W1J 0DJ, United Kingdom",runnersneed.com,+44 20 7437 7399
+Runners Need Waterloo Bridge,"Unit 1 Brettenham House, Lancaster Pl, London WC2E 7EN, United Kingdom",runnersneed.com,+44 20 7212 9936
+Sports Direct,"181-183 Walworth Rd, Walworth, London SE17 1RW, United Kingdom",sportsdirect.com,+44 344 332 5043
+lululemon,"87-135 Brompton Rd, Knightsbridge, London SW1X 7XL, United Kingdom",lululemon.co.uk,+44 20 7893 8377
+JD Sports,"375-383 Oxford St, Mayfair, London W1C 2JS, United Kingdom",jdsports.co.uk,+44 20 3892 9080
+Sports Direct,"89-91 Chapel Market, The Angel, London N1 9EX, United Kingdom",sportsdirect.com,+44 344 332 5337
+GapBody,"8 Brompton Pl, Knightsbridge, London SW3 1QE, United Kingdom",-,-
+Sports Direct,"Unit 11, Surrey Quays Centre, London SE16 7LL, United Kingdom",sportsdirect.com,+44 344 332 5022
+JD Sports,"197-203 Oxford St, Soho, London W1D 2LE, United Kingdom",jdsports.co.uk,+44 20 3370 2266
+Dufeal Your Best Sportswear,"Floor, Suite 23, 63/66 Hatton Garden Fifth, London EC1N 8LE, United Kingdom",dufealyourbestsportswear.com,+44 7763 883186
+Sports Direct,"154-156 Stoke Newington High St, Clapton, London N16 7JL, United Kingdom",sportsdirect.com,+44 344 332 5241
+lululemon,"74-75 Marylebone High St, Marylebone, London W1U 5JR, United Kingdom",lululemon.co.uk,+44 20 7935 3105
+SweatShop,"1 Norton Folgate, Spitalfields, London E1 6DB, United Kingdom",sweatshop.co.uk,+44 344 332 5237
+ROXX Sports,-,roxxsports.com,+44 7574 259300
+GapBody,"5-6 Parkfield St, The Angel, London N1 0PS, United Kingdom",gap.com,-
+Runners Need Covent Garden,"4 Mercer St, Covent Garden, London WC2H 9QA, United Kingdom",runnersneed.com,+44 20 3815 5740
+lululemon,"187-191 Regent St, Mayfair, London W1B 4JP, United Kingdom",lululemon.co.uk,+44 20 3897 8500
+Oakley,"1-4 King St, Covent Garden, London WC2E 8HH, United Kingdom",uk.oakley.com,+44 20 7395 6070
+lululemon,"3rd Floor, 400 Oxford Street Selfridges, London W1A 1AB, United Kingdom",lululemon.co.uk,+44 800 123400
+Sweaty Betty,"87-135 Brompton Rd, Knightsbridge, London SW1X 7XL, United Kingdom",sweatybetty.com,+44 20 7730 1234
+JD Sports,"277-279 Walworth Rd, Walworth, London SE17 2TG, United Kingdom",jdsports.co.uk,+44 20 3758 7877
+Sports Direct,"427, 429 North End Rd, Fulham, London SW6 1NY, United Kingdom",sportsdirect.com,+44 344 332 5032
+Columbia Sportswear Kensington Store,"168-170 Kensington High St, Kensington, London W8 7RG, United Kingdom",columbiasportswear.co.uk,+44 20 7937 2486
+Runners Need London - Great Portland Street,"42-48 Great Portland St, Fitzrovia, London W1W 7LZ, United Kingdom",runnersneed.com,+44 20 3780 2055
+Sports Direct,"2058, Westfield Shopping Centre, Ariel Way, Shepherd's Bush, London W12 7GE, United Kingdom",sportsdirect.com,+44 344 332 5413
+Fashercise,"46 Monmouth St, West End, London WC2H 9LE, United Kingdom",fashercise.com,+44 20 7379 6106
+Runners Need LONDON - MONUMENT,"Monument Station, 47-51 King William St, Candlewick, London EC4R 9AF, United Kingdom",runnersneed.com,+44 20 3780 2059
+Flybery,-,flybery.com,+44 7514 234421
+JD Sports,"213-219 Camden High St, Camden Town, London NW1 7BT, United Kingdom",jdsports.co.uk,+44 20 7482 0522
+JD Sports,"261-263 Whitechapel Rd, Shadwell, London E1 1DB, United Kingdom",jdsports.co.uk,+44 20 7377 9134
+JD Sports,"55 Chapel Street Market N1 9EW, United Kingdom",jdsports.co.uk,+44 20 7837 9440
+Sweaty Betty,"400 Oxford St, Marylebone, London W1A 1AB, United Kingdom",sweatybetty.com,+44 800 123400
+Runners Need Kensington,"188 Kensington High St, Kensington, London W8 7RG, United Kingdom",runnersneed.com,+44 20 3780 2057
+Runners Need Camden Town,"34 Parkway, Camden Town, London NW1 7AH, United Kingdom",runnersneed.com,+44 20 3780 2051
+OHMME,-,-,+44 20 7387 0732
+JD Sports,"Unit 8 Kingsland Shopping Centre, Dalston E8 2LX, United Kingdom",jdsports.co.uk,+44 20 7993 1316
+Runners Need Moorgate,"72 Chiswell St, London EC1Y 4AB, United Kingdom",runnersneed.com,+44 20 3780 2060
+adidas Flagship Store London,"425 Oxford St, Mayfair, London W1C 2PG, United Kingdom",adidas.co.uk,+44 1244 747263
+DAÖMEY LONDON,"93 Kingsland Rd, Hackney, London E2 8AG, United Kingdom",daomeyuk.com,+44 20 3669 8129
+LIVE Clothing UK,"109A King's Rd, Chelsea, London SW3 4PA, United Kingdom",uk.liveclothing.com,+44 20 7352 9878
+Nike Kings Road,"33 King's Rd, Chelsea, London SW3 4LX, United Kingdom",nike.com,+44 20 7259 9960
+Runners Need King's Cross,"Unit 1, 200 Pentonville Rd, London N1 9JP, United Kingdom",runnersneed.com,+44 20 3780 2058
+TSPORT,"22 Abbey Rd, St John's Wood, London NW10 7RE, United Kingdom",-,-
+JD Sports,"451-453 Brixton Rd, Brixton, London SW9 8HH, United Kingdom",jdsports.co.uk,+44 20 7737 3728
+JD Sports,"Unit 21 West 12 Sc, Shepherd's Bush W12 8PP, United Kingdom",jdsports.co.uk,+44 20 8150 2055
+Originals Flagship Store London,"15 Foubert's Pl, Carnaby St, Soho, London W1F 7QB, United Kingdom",adidas.co.uk,+44 1244 747263
+In Corpore Sano,"129 New King's Rd, Fulham, London SW6 4SL, United Kingdom",-,-
+Mr Gymnastic,"131-151 Great Titchfield St, Fitzrovia, London W1W 5BB, United Kingdom",mrgymnastic.co.uk,-
+Adrenna,-,adrenna.co.uk,-
+Players Sportswear,-,-,-
+SOS Activewear,-,sosactivewear.com,-
+Sweaty Betty,"3 Marylebone High St, Marylebone, London W1U 4NG, United Kingdom",sweatybetty.com,+44 20 7486 6204
+Runners Need Clapham Junction,"85 St John's Rd, London SW11 1QY, United Kingdom",runnersneed.com,+44 20 3780 2054
+Capital Golf London,"14 John Prince's St, Marylebone, London W1G 0JS, United Kingdom",capitalgolf.com,+44 20 7493 4414
+The Pentland Group Limited,"8 Manchester Square, Marylebone, London W1U 3PH, United Kingdom",pentlandgroup.com,+44 20 7535 3800
+Mountain Warehouse Islington,"40 Upper St, The Angel, London N1 0PN, United Kingdom",mountainwarehouse.com,+44 20 7354 2444
+KEJO FITNESS,-,kejofitness.com,-
+Focus,"Kemp House, 160 City Rd, London EC1V 2NX, United Kingdom",-,+44 7578 976613
+Maxibillion,"43 Berkeley Square, Mayfair, London W1J 5AP, United Kingdom",maxibillion.co.uk,+44 20 7889 2868
diff --git a/my_config.py b/google-maps/my_config.py
similarity index 100%
rename from my_config.py
rename to google-maps/my_config.py
diff --git a/webdriver.py b/webdriver.py
index d8ee7c9..feaa224 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -18,10 +18,10 @@ class Webdriver:
_ua = UserAgent()
_buf = Buffer()
- async def init_browser(self, headless='False', language='en-gb'):
+ async def init_browser(self, headless_status=False, language='en-gb'):
self.browser = await launch(
ignoreHTTPSErrors=True,
- headless=headless,
+ headless=headless_status,
viewport=None,
autoclose=True,
args=['--start-maximized'],
@@ -33,6 +33,8 @@ class Webdriver:
await self._page.setExtraHTTPHeaders({'Accept-Language': language})
await self._page.reload()
+ logger.debug('Successfull initialization')
+
async def _shut_browser(self):
logger.debug('Shutting down browser in 15 seconds')
sleep(15)
@@ -56,10 +58,6 @@ class Webdriver:
print(e)
raise SystemError('Some shit happened to pyppeteer, fix it')
- async def _enter(self, word):
- await self._page.keyboard.type(word)
- await self._page.keyboard.press('Enter')
-
async def _jump(self, url, xpath):
try:
await self._page.goto(url)
@@ -67,4 +65,4 @@ class Webdriver:
self._page.waitForXPath(xpath, {'visible': True})
except:
await self._shut_browser()
- raise ValueError('Got invalid URL for google maps')
\ No newline at end of file
+ raise ValueError('Got invalid URL for google maps')
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index bbc4a70..2ead7e4 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -8,75 +8,112 @@ from bs4 import BeautifulSoup
from time import sleep
from locators import *
-from config import *
+from my_config import *
import sys
sys.path.append('..')
-from my_config import
+from logger_config import get_logger
from webdriver import Webdriver
-not_found = lambda loc, desc: f'No Results for {desc}' in requests.get(url.format(desc, loc), headers={'User-Agent': ua.random}).text
+logger = get_logger('webdriver.yelp')
+
+
ua = UserAgent()
-def validate_args(func):
-
- def inner(self, loc, desc):
- if not_found(loc, desc):
- raise ValueError(f'No results for `{desc}` in `{loc}`')
- func(self, loc, desc)
+class Yelp(Webdriver):
+ _found = lambda loc, desc: not f'No Results for {desc}' in requests.get(yelp.format(desc, loc), headers={'User-Agent': ua.random}).text
+ _yelp = yelp.split('search')[0]
+
+
+ async def search(self, loc=None, desc=None, url=None):
+ if not '_page' in self.__dict__:
+ raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')
+
+ if loc and desc and not url:
+ if self._found(loc, desc):
+ logger.debug('Working based on location `%s` and keyword `%s`', loc, desc)
+ self._data = [desc, loc]
+ await self._locate()
+ else:
+ raise ValueError(f'No result for `{desc}` and `{loc}`')
+
+ elif not (loc and desc) and url:
+ await self._jump(url, yelp_xpath)
+ logger.debug('Working based on given URL `%s`', url)
+
+ else:
+ await self._shut_browser()
+ raise ValueError('Specify only location and description or only URL for yelp page')
+
+ await self._scrape
+ await self._shut_browser
+
+ async def _locate(self):
+ await self._page.goto(self._yelp)
+ sleep(2.5)
+
+ inputs = await self._page.querySelectorAll(input_locator)
+ for i in range(2):
+ await inputs[i].click()
+ sleep(0.13)
+ await inputs[i].enter(self._data[i])
- return inner
+ sleep(0.21)
+ await self._page.keyboard.press('Enter')
-class Yelp(Webdriver):
- @validate_args
- def __init__(self, loc, desc):
- self._loc = loc
- self._desc = desc
- async def _extract_urls(self, html):
- return ['https://www.yelp.com' + url for url in lxml.html.document_fromstring(html).xpath(urls_xpath)]
+ async def _scrape(self):
+
+ def parse(url):
+ r = requests.get(url)
+ if r.ok:
+ soup = BeautifulSoup(html, 'lxml')
+
+ try:
+ title = soup.select_one('h1').text.strip()
+ except:
+ title = '-'
+
+ try:
+ phone = soup.find(string='Phone number').next_sibling().text().strip()
+ except:
+ phone = '-'
+
+ try:
+ addr = soup.find(string='Get Directions').next_sibling().text.strip()
+ except:
+ addr = '-'
+ try:
+ website = soup.find(attrs={'rel':'noopener'})['href'].split('/biz_redir?url=http%3A%2F%2F')[0].split('&')[0]
+ except:
+ website = '-'
+
+ self._buf.store(dict(zip(order, [title, addr, website, phone])))
+ else:
+ logger.warning(f'Failed request {r} to {url}')
+
+ def extract_urls(html):
+ return ['https://www.yelp.com' + url for url in lxml.html.document_fromstring(html).xpath(urls_xpath)]
+
- async def _locate(self):
- self._page.goto()
-
- def _parse(self, html):
- soup = BeautifulSoup(html, 'lxml')
-
- try:
- title = soup.select_one('h1').text.strip()
- except:
- title = '-'
-
- try:
- phone = soup.find(string='Phone number').next_sibling().text().strip()
- except:
- phone = '-'
-
- try:
- addr = soup.find(string='Get Directions').next_sibling().text.strip()
- except:
- addr = None
- try:
- website = soup.find(attrs={'rel':'noopener'})['href'].split('/biz_redir?url=http%3A%2F%2F')[0].split('&')[0]
- except:
- website = None
-
- return dict(zip(order, [title, addr, website, phone]))
-
- async def scrape(self):
- # 1) Go to the yelp with given description and location
- # 2) Extract urls from each page
- # 3) Scrape each page using Pool
- # 4) Go to next page (if exists)
- # 5) Iterate it over
while True:
- html = await self._locate()
- urls = self._extract_urls(html)
+ urls = extract_urls(await self._page.content())
+
with Pool(10) as p:
- p.map(self._parse, urls)
+ p.map(parse, urls)
+
+ next_button = await self._page.xpath(next_xpath)
+ if not next_button:
+ self._buf.dump()
+ break
+ await next_button.click()
+ sleep(2)
-y = Yelp('London', 'Pizza')
+if __name__ == '__main__':
+ y = Yelp()
+ await y.init_browser()
+ await y.search('London','Sportswear')
diff --git a/yellow-pages/locators.py b/yellow-pages/locators.py
index 8e897ad..161b76f 100644
--- a/yellow-pages/locators.py
+++ b/yellow-pages/locators.py
@@ -1,3 +1,5 @@
urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'
+yelp_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'
+next_xpath = '//*[@class="icon--24-chevron-right-v2 navigation-button-icon__09f24__2FnZH css-12anxc3"]'
-title_locator = 'h1'
+input_locator = '.pseudo-input_field.business-search-form_input-field'
\ No newline at end of file
diff --git a/yellow-pages/config.py b/yellow-pages/my_config.py
similarity index 51%
rename from yellow-pages/config.py
rename to yellow-pages/my_config.py
index dbb66c6..c7e6b3c 100644
--- a/yellow-pages/config.py
+++ b/yellow-pages/my_config.py
@@ -1,4 +1,4 @@
-url = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'
+yelp = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'
order = [
'Title',