Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
yelp extractor finished
Madi591b0083b69d
diff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pycindex 545d0db..f6e3c78 100644Binary files a/__pycache__/webdriver.cpython-38.pyc and b/__pycache__/webdriver.cpython-38.pyc differdiff --git a/google-maps/__pycache__/my_config.cpython-38.pyc b/google-maps/__pycache__/my_config.cpython-38.pycnew file mode 100644index 0000000..5ab18e2Binary files /dev/null and b/google-maps/__pycache__/my_config.cpython-38.pyc differdiff --git a/google-maps/extractor.py b/google-maps/extractor.pyindex d0f3ad6..d9be21c 100644--- a/google-maps/extractor.py+++ b/google-maps/extractor.py@@ -1,17 +1,18 @@import re-import asynciofrom locators import *+from my_config import *from time import sleepfrom bs4 import BeautifulSoup+from pyppeteer.errors import PageError+import syssys.path.append('..')-from webdriver import Webdriverfrom logger_config import get_logger-from my_config import *+from webdriver import Webdriverlogger = get_logger('webdriver.google_maps')@@ -19,44 +20,77 @@ logger = get_logger('webdriver.google_maps')class GoogleMaps(Webdriver):--async def search(self, location=None, keyword=None, url=None):- if not 'page' in self.__dict__:- raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')+ if not '_page' in self.__dict__:+ raise ValueError(+ 'Initialize the browser before searching by `await *.init_broswser()`')+if location and keyword and not url:logger.debug('Working based on location `%s` and keyword `%s`', location, keyword)self._location = locationself._keyword = keywordawait self._locate()+elif not (location and keyword) and url:- logger.debug('Working based on given URL `%s`', url)await self._jump(url, search_xpath)+ logger.debug('Working based on given URL `%s`', url)+else:await self._shut_browser()raise ValueError(f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')+await self._scrape()await self._shut_browser()async def _locate(self):- # Go to google maps, type the location- await self._page.goto(google_maps)- await self._page.click('input')- await self._enter(self._location)- await self._page.waitForXPath(button_xpath, {'visible': True})+ async def _enter(word):+ await self._page.keyboard.type(word)+ await self._page.keyboard.press('Enter')+ # Go to google maps, type the location+ try:+ await self._page.goto(google_maps)+ await self._page.click('input')+ await _enter(self._location)+ sleep(2.5)++ # Click search nearby, type the keyword+ await self._page.waitForXPath(button_xpath, {'visible': True})+ await (await self._page.xpath(button_xpath))[2].click()+ await self._page.waitForXPath(search_xpath)+ await self._page.click('input')+ await _enter(self._keyword)++ await self._page.waitForXPath(result_xpath, {'visible': True})+ except PageError:+ self._page.reload()+ self._locate()- # Click search nearby, type the keyword- await (await self._page.xpath(button_xpath))[2].click()- await self._page.waitForXPath(search_xpath)- await self._page.click('input')- await self._enter(self._keyword)+ async def _scrape(self):- await self._page.waitForXPath(result_xpath, {'visible': True})+ def extract(html):+ soup = BeautifulSoup(html, 'html.parser')++ data = {}++ for field, src in order.items():+ try:+ if src:+ value = soup.find(attrs={'src': re.compile(+ src)}).parent.parent.parent.text.strip()+ else:+ value = soup.find('h1').text.strip()+ if 'Add' in value or 'Добавить' in value:+ logger.debug('Got empty information %s', value)+ value = '-'+ except:+ value = '-'+ data.update({field: value})+ logger.debug('Data: %s', list(data.values()))+ return data- async def _scrape(self):while True:places = len(await self._page.xpath(result_xpath))logger.debug('%s places found', places)@@ -67,7 +101,7 @@ class GoogleMaps(Webdriver):place = (await self._page.xpath(result_xpath))[i]await self._do_retry(place.click, place_xpath)- data = self._extract(await self._page.content())+ data = extract(await self._page.content())self._buf.store(data)back = (await self._page.xpath(back_xpath))[0]@@ -78,37 +112,14 @@ class GoogleMaps(Webdriver):self._buf.dump()breakawait next_button.click()- sleep(1)-- def _extract(self, html):- soup = BeautifulSoup(html, 'html.parser')-- data = {}-- for field, src in order.items():- try:- if src:- value = soup.find(- attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()- else:- value = soup.find('h1').text.strip()- if 'Add' in value or 'Добавить' in value:- logger.debug('Got empty information %s', value)- value = '-'- except:- value = '-'- data.update({field: value})-- logger.debug('Data: %s', list(data.values()))- return data+ sleep(0.7)async def main():g = GoogleMaps()- await g.init_browser(headless=True)- await g.search('Кокшетау','Спортзал')-+ await g.init_browser()+ await g.search('London', 'Sportswear')if __name__ == '__main__':- asyncio.run(main())\ No newline at end of file+ asyncio.run(main())diff --git a/google-maps/leads.csv b/google-maps/leads.csvindex 3106c8b..a40d3e9 100644--- a/google-maps/leads.csv+++ b/google-maps/leads.csv@@ -1,121 +1,76 @@Title,Address,WebSite,PhoneNumber-Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202-Фитнес Клуб Kapital,Kokshetau 020000,-,8 (7162) 50 05 00-"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",-,--Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10-"Sportkompleks ""Kazhymukan""",Kokshetau 020000,-,8 (7162) 77 08 11-Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",-,--Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",-,8 (7162) 42 07 80-Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",-,--Fok Pri Sel'khozinstitute,Kokshetau 020000,-,--"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",-,--DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,--"""Domino""","Akan Seri St, Kokshetau 020000",-,8 (7162) 25 30 61-Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",-,--"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03-Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",-,--Lider,"Абай Көшесі, Petropavl 150000",-,--"Dvorets Sporta ""Burabay""","Burabay, A-343 85, Kokshetau 020000",dsb.kz,8 (7162) 32 82 84-Sto,Sergeevka 150000,-,--Тренажерный Зал XXL,Nur-Sultan 010000,-,--Grand Gym,"020000, Nur-Sultan",-,8 (707) 177 7327-Fitness room,"Kenzhebek Kumisbekov St 7, Nur-Sultan 020000",-,8 (778) 169 2621-Argo,"Mira St 41, Stepnogorsk 021501",-,8 (71645) 6 38 78-Grand Fitness,"Сары Арка 31/2, вход со стороны ул.Жангельдина, проспект Сарыарка, Saryarka Avenue, Nur-Sultan 020000",business.site,8 (702) 132 8294-T-Tau,"Respublika Ave, Nur-Sultan 010000",vk.com,8 (7172) 40 38 27-ДЮСШ «Триатлон»,Kokshetau 020000,-,8 (7162) 42 05 91-Start,"Здание 7,, Tselinogradskaya st. 50, Stepnogorsk",-,8 (71645) 3 60 59-Sportzal,"Ulitsa Muravlenko, 13, Tyumen, Tyumen Oblast, Russia, 625022",-,--Trenazhernyy Zal,Karagandy 100000,-,--Flex Gym,"Ulitsa Pushkina, 59, Omsk, Omsk Oblast, Russia, 644010",flexgym55.ru,+7 381 266-66-99-Rekord,"110000, Al Farabi Ave 124, Kostanay",-,8 (7142) 56 87 08-Фитнес Клуб World CLASS,Karagandy 100000,worldclass.kz,--Okzhetpes Stadium,"Abay Street 116, Kokshetau 020000",okzhetpes.kz,8 (7162) 92 60 16-Space Fitness,"Amanat St, Nur-Sultan 010000",spacefitness.kz,8 (707) 650 5050-Gym Studio,"Karla Marksa Prospekt, 32, Omsk, Omsk Oblast, Russia, 644042",-,+7 381 232-05-00-ENERGY CLUB,"3, Lomov St 164/11, Pavlodar",-,8 (7182) 66 37 85-Life Fitness,"Туркестан, 2, Nur-Sultan 010000",lifefit.kz,8 (7172) 79 73 73-Trenazhernyy Zal,"Auezov Street, Aksu 140000",-,--Фитнес-клуб Prime,"Ulitsa Kuybysheva, 55, 2 этаж, Kurgan, Kurgan Oblast, Russia, 640020",kurgan.zoon.ru,+7 912 573-56-83-Sportzal,"Ulitsa Malysheva, 138а, Yekaterinburg, Sverdlovsk Oblast, Russia, 620078",-,--Ples,Shoptykol 020000,-,--Sportzal,"Yemashi, Republic of Bashkortostan, Russia, 452590",-,--"Gostinitsa ""Dostyk""","Abay Street 69, Kokshetau 020000",dostyqhotel.kz,--HARD GYM,"Ulitsa Svobody, д.110Б, Chelyabinsk, Chelyabinsk Oblast, Russia, 454005",-,--"Ostanovka ""Sportzal""","Altynsarin Street, Kostanay 110000",-,--"Fitnes Tsentr ""Arena""",Semey 070000,-,--Fokus,"Zhambyl St 4-6, Pavlodar 140000",vk.com,8 (7182) 62 44 24-Ozon,"Komarova, 11к1, 4 этаж, Omsk, Omsk Oblast, Russia, 644112",ozone-fitness.ru,+7 381 297-24-24-Agrimer-Astyk LLP,Kokshetau 020000,-,--Kapital,"Sabataev Street 82, Kokshetau 020000",-,--Sportzal,"Tolmachevskoye Shosse, Novosibirsk, Novosibirsk Oblast, Russia, 633100",-,--STEEL BODY,"Solnechnyy Proyezd, 7, Tyumen, Tyumen Oblast, Russia, 625022",-,+7 345 260-06-85-Winox Fitness club & SPA,"Tauelsizdik Ave 32, Nur-Sultan 020000",winox.kz,8 (701) 110 1010-Sportzal,"Ulitsa Marshala Zhukova, 2/11, Ufa, Republic of Bashkortostan, Russia, 450073",-,--"Strong & Smart, fitness club","Ulitsa Brat'yev Kashirinykh, 54Б, Chelyabinsk, Chelyabinsk Oblast, Russia, 454106",strsm.ru,+7 909 079-86-66-Sportzal,"Ulitsa Rabkorov, 16/1, Ufa, Republic of Bashkortostan, Russia, 450092",-,+7 347 250-29-02-"Energiya, Atleticheskiy Klub","Ulitsa Yel'kina, 63А, Chelyabinsk, Chelyabinsk Oblast, Russia, 454092",-,+7 351 263-56-40-FLEX FITNESS CLUB,"Kabanbay Batyr St 41, Ust'-Kamenogorsk 070000",-,8 (777) 726 2830-Shkola Olimpiyskogo Rezerva,"б, Erubaev St 43, Karagandy 100000",-,--Sportzal,"Ulitsa Yeroshevskogo, 18, Samara, Samara Oblast, Russia, 443056",-,--Sportzal,"Zelenyy Prospekt, 20, Moscow, Russia, 111401",-,--Sportzal,"Zelenyy Prospekt, 20, Moscow, Russia, 111401",-,--Sportzal,"Lesnaya Ulitsa, 6А, Aktiubinskii, Republic of Tatarstan, Russia, 423304",-,--Sportzal,"Krasnogorsk, Uzbekistan",-,--Neptun,"Makhmet Qayyrbayev St 85, Pavlodar 140000",neptun.org.kz,8 (7182) 32 85 87-Sportzal,"с, Leningradsky Ave, 31А строение 1, Moscow, Russia, 125284",-,--FitnessGym,"Sadovnikov, Almaty 050000",fitnessgym.kz,8 (7272) 49 54 59-Sportzal,"Ulitsa Severnoye Kol'tso, 16А, Kirov, Kirov Oblast, Russia, 610006",-,+7 833 225-05-12-Sportzal,"Kogalym, Ханты-Мансийский Автономный округ - Югра АО, Russia, 628483",-,--Sportzal,"Naberezhnaya Ulitsa, 37-41, Sol-Iletsk, Orenburg Oblast, Russia, 461505",-,--Lider,"Средняя школа №60, Ulitsa Dolgireva, 91, Omsk, Omsk Oblast, Russia, 644034",-,+7 950 781-46-78-Liga Chempionov,"Ulitsa Sputnikov, 11, Yekaterinburg, Sverdlovsk Oblast, Russia, 620025",m.vk.com,+7 961 775-85-07-Sportzal,"Гимназия №4, Ulitsa Engel'sa, 1А, 2 этаж, Cheboksary, Chuvashia Republic, Russia, 428003",-,+7 835 262-41-42-Sportivnyy Zal,Shieli,-,--Sportzal,"Shosse Revolyutsii, 15, St Petersburg, Russia, 195176",-,--Sportzal,"St Petersburg, Russia, 197022",-,--Za Gym,"Naymanbaev St, Semey 070000",-,8 (7222) 44 44 46-Lider,"Grammar school №159, Barnaul'skaya Ulitsa, 134, Omsk, Omsk Oblast, Russia, 644105",-,+7 904 588-17-10-Art-fitness,"Krylov Street 70, Ust'-Kamenogorsk 070000",artfitness.kz,8 (7232) 54 44 10-Sportzal,"Volgograd, Volgograd Oblast, Russia, 400050",-,--Hammer,Aktobe,-,--Sportzal,"Ulitsa Belinskogo, 61, Nizhny Novgorod, Nizhny Novgorod Oblast, Russia, 603006",-,--"Тренажерный зал ""Гранат""","Alchevskyh St, 31, Kharkiv, Kharkiv Oblast, Ukraine, 61000",-,+380 97 867 1661-Sportzal,"Monastyrskoye, Republic of Tatarstan, Russia, 422392",-,--Спорт Зал,"Kyiv, Ukraine, 02000",-,+380 63 897 7297-Sportzal,"Ulitsa Dzerzhinskogo, 38А, Astrakhan, Astrakhan Oblast, Russia, 414006",-,--Sportzal,"Verkhnyaya Unzha, Kostroma Oblast, Russia, 157440",-,--Sportzal,"Prospekt Akushinskogo, 114, Makhachkala, Republic of Dagestan, Russia, 367014",-,--Sportzal,"Volgograd, Volgograd Oblast, Russia, 400080",-,--Sportzal,"Ulitsa Tsiolkovskogo, 28, Makhachkala, Republic of Dagestan, Russia, 367008",-,--Sportzal,"Olimpiyskiy Prospekt, 33, Mytishchi, Moscow Oblast, Russia, 141006",-,--Спортзал,"14 Bagratuniats St, Yerevan 0039, Armenia",-,+374 94 428502-Sportzal,"Ulitsa Aviator Mikrorayon, 73, Achinsk, Krasnoyarsk Krai, Russia, 662150",-,--Sportzal,"Ulitsa Novoazhimova, 3, Izhevsk, Udmurt Republic, Russia, 426006",-,--Sportzal,"вуліца Глінкі 44, Baranovichi, Belarus",спортзал-барановичи.бел,+375 29 124-95-87-ORGANIZM 23 Августа,"23-ho Serpnya St, 43, Kharkiv, Kharkiv Oblast, Ukraine, 61000",organizm.in.ua,+380 95 917 1212-Gladiator Gym,"Kostyantyna Dan'kevycha St, 12, Kyiv, Ukraine, 02000",nailspro.kiev.ua,+380 93 379 7949-Sportzal,"Stavropol Krai, Russia, 356195",-,+7 905 490-25-82-Sportzal,"Ulitsa Sverdlova, 3, Bogorodsk, Nizhny Novgorod Oblast, Russia, 607600",-,--Sportzal,"Ulitsa Leningradskaya, 97, Vologda, Vologda Oblast, Russia, 160034",-,+7 981 503-20-10-Sportzal,"Nurlat, Republic of Tatarstan, Russia, 423042",-,--Sportzal,"2-Y Mikrorayon, 1а, Shushenskoe, Krasnoyarsk Krai, Russia, 662710",-,--Sportzal,"3-Ya Karta, Volgograd Oblast, Russia, 404150",-,--Gym,"Ulitsa Kirova, 44/1, Novosibirsk, Novosibirsk Oblast, Russia, 630102",-,+7 383 383-02-72-Фитнес клуб Sport Zone,"Heroiv Pratsi St, 60В, Kharkiv, Kharkiv Oblast, Ukraine, 61000",instagram.com,+380 50 553 0155-Razdevalka I Dushevaya Zhenskaya,"НТТЖКиГХ, Черноисточинское ш., 58, Nizhny Tagil, Sverdlovsk Oblast, Russia, 622030",nttgk-gh.umi.ru,+7 343 547-62-99-Sportzal,"Almak, Republic of Dagestan, Russia, 368150",-,--Sportzal,"Belgradskaya Ulitsa, 30, St Petersburg, Russia, 192212",-,--Sportzal,"Republic of Dagestan, Russia, 368935",-,--Sport & Fit,"Sofiivska Borschahivka, Kyiv Oblast, Ukraine",mixsport.pro,+380 98 230 9090-"Спортзала ""Alex""","1а, Soborna St, Kropyvnytskyi, Kirovohrad Oblast, Ukraine, 25000",-,+380 50 814 6246-Sportzal,"Ulitsa Tani Baramzinoy, 4, Glazov, Udmurt Republic, Russia, 427622",-,--Тренажерный зал,"Kniazia Volodymyra Velykoho Street, 12, Dnipro, Dnipropetrovsk Oblast, Ukraine, 49000",-,+380 67 784 3785-Sportzal,"Tsurib, Republic of Dagestan, Russia, 368450",-,--Sportzal,"Kugesi, Chuvashia Republic, Russia, 429500",-,--Sportzal,"Ulitsa Gor'kogo, 162В, Kaliningrad, Kaliningrad Oblast, Russia, 236029",-,+7 401 232-33-06-Fitness Palace,"Turan Ave 30, Nur-Sultan 010000",fitness-palace.kz,8 (7172) 56 77 77-Sportzal,"Bavtugay, Republic of Dagestan, Russia, 368101",-,--Sportzal,"Kovalia St, 3, Poltava, Poltava Oblast, Ukraine, 36000",-,--Sportzal,"а, Ulitsa Antonova-Ovseyenko, 31, Voronezh, Voronezh Oblast, Russia, 394088",-,--Sportzal,"Pervomansk, Krasnoyarsk Krai, Russia, 663502",-,-+Runners Need Victoria,"24 Palace St, Victoria, London SW1E 5JD, United Kingdom",runnersneed.com,+44 20 3780 2062+Sports Direct,"Piccadilly Circus, 24-36 Regent St, St. James's, London SW1Y 4QF, United Kingdom",sportsdirect.com,+44 344 332 5602+Striders Edge Ltd,"Elizabeth House, 39 York Rd, Bishop's, London SE1 7NQ, United Kingdom",stridersedge.com,-+Lillywhites,"24-36 Regent St, St. James's, London SW1Y 4QF, United Kingdom",lillywhites.com,+44 344 332 5602+lululemon,"125-126 Long Acre, Covent Garden, London WC2E 9LD, United Kingdom",lululemon.co.uk,+44 20 7836 5052+Ten Edit Boutique - Activewear Shop,"83 Great Titchfield St, Fitzrovia, London W1W 6RH, United Kingdom",ten.co.uk,+44 20 7636 5111+Sportex,"2ND FLOOR, 67-68 Jermyn St, St. James's, London SW1Y 6NY, United Kingdom",sportex.online,+44 800 634 9304+Champion Store,"48-50 Brewer St, Soho, London W1F 9TG, United Kingdom",championstore.com,+44 20 7734 3087+Sports Direct,"150-154 Oxford St, Fitzrovia, London W1D 1ND, United Kingdom",sportsdirect.com,+44 344 332 5666+lululemon,"38, 42 King's Rd, Chelsea, London SW3 4UD, United Kingdom",lululemon.co.uk,+44 20 7584 0005+Sports Direct,"16-18 Ludgate Hill, London EC4M 7DR, United Kingdom",sportsdirect.com,+44 344 332 5211+Columbia Sportswear Carnaby Street,"39 Carnaby St, Soho, London W1F 7DP, United Kingdom",columbiasportswear.co.uk,+44 20 3966 6195+GapBody,"374 Oxford St, London W1C 1JR, United Kingdom",-,-+GapBody,"60 Gracechurch St, Bridge, London EC3V 0HR, United Kingdom",gap.com,-+GapBody,"223-225 Oxford St, West End, London W1D 2LR, United Kingdom",-,-+GapBody,"124B King's Rd, Chelsea, London SW3 4TR, United Kingdom",-,-+Runners Need Piccadilly,"23/26 Piccadilly, Mayfair, London W1J 0DJ, United Kingdom",runnersneed.com,+44 20 7437 7399+Runners Need Waterloo Bridge,"Unit 1 Brettenham House, Lancaster Pl, London WC2E 7EN, United Kingdom",runnersneed.com,+44 20 7212 9936+Sports Direct,"181-183 Walworth Rd, Walworth, London SE17 1RW, United Kingdom",sportsdirect.com,+44 344 332 5043+lululemon,"87-135 Brompton Rd, Knightsbridge, London SW1X 7XL, United Kingdom",lululemon.co.uk,+44 20 7893 8377+JD Sports,"375-383 Oxford St, Mayfair, London W1C 2JS, United Kingdom",jdsports.co.uk,+44 20 3892 9080+Sports Direct,"89-91 Chapel Market, The Angel, London N1 9EX, United Kingdom",sportsdirect.com,+44 344 332 5337+GapBody,"8 Brompton Pl, Knightsbridge, London SW3 1QE, United Kingdom",-,-+Sports Direct,"Unit 11, Surrey Quays Centre, London SE16 7LL, United Kingdom",sportsdirect.com,+44 344 332 5022+JD Sports,"197-203 Oxford St, Soho, London W1D 2LE, United Kingdom",jdsports.co.uk,+44 20 3370 2266+Dufeal Your Best Sportswear,"Floor, Suite 23, 63/66 Hatton Garden Fifth, London EC1N 8LE, United Kingdom",dufealyourbestsportswear.com,+44 7763 883186+Sports Direct,"154-156 Stoke Newington High St, Clapton, London N16 7JL, United Kingdom",sportsdirect.com,+44 344 332 5241+lululemon,"74-75 Marylebone High St, Marylebone, London W1U 5JR, United Kingdom",lululemon.co.uk,+44 20 7935 3105+SweatShop,"1 Norton Folgate, Spitalfields, London E1 6DB, United Kingdom",sweatshop.co.uk,+44 344 332 5237+ROXX Sports,-,roxxsports.com,+44 7574 259300+GapBody,"5-6 Parkfield St, The Angel, London N1 0PS, United Kingdom",gap.com,-+Runners Need Covent Garden,"4 Mercer St, Covent Garden, London WC2H 9QA, United Kingdom",runnersneed.com,+44 20 3815 5740+lululemon,"187-191 Regent St, Mayfair, London W1B 4JP, United Kingdom",lululemon.co.uk,+44 20 3897 8500+Oakley,"1-4 King St, Covent Garden, London WC2E 8HH, United Kingdom",uk.oakley.com,+44 20 7395 6070+lululemon,"3rd Floor, 400 Oxford Street Selfridges, London W1A 1AB, United Kingdom",lululemon.co.uk,+44 800 123400+Sweaty Betty,"87-135 Brompton Rd, Knightsbridge, London SW1X 7XL, United Kingdom",sweatybetty.com,+44 20 7730 1234+JD Sports,"277-279 Walworth Rd, Walworth, London SE17 2TG, United Kingdom",jdsports.co.uk,+44 20 3758 7877+Sports Direct,"427, 429 North End Rd, Fulham, London SW6 1NY, United Kingdom",sportsdirect.com,+44 344 332 5032+Columbia Sportswear Kensington Store,"168-170 Kensington High St, Kensington, London W8 7RG, United Kingdom",columbiasportswear.co.uk,+44 20 7937 2486+Runners Need London - Great Portland Street,"42-48 Great Portland St, Fitzrovia, London W1W 7LZ, United Kingdom",runnersneed.com,+44 20 3780 2055+Sports Direct,"2058, Westfield Shopping Centre, Ariel Way, Shepherd's Bush, London W12 7GE, United Kingdom",sportsdirect.com,+44 344 332 5413+Fashercise,"46 Monmouth St, West End, London WC2H 9LE, United Kingdom",fashercise.com,+44 20 7379 6106+Runners Need LONDON - MONUMENT,"Monument Station, 47-51 King William St, Candlewick, London EC4R 9AF, United Kingdom",runnersneed.com,+44 20 3780 2059+Flybery,-,flybery.com,+44 7514 234421+JD Sports,"213-219 Camden High St, Camden Town, London NW1 7BT, United Kingdom",jdsports.co.uk,+44 20 7482 0522+JD Sports,"261-263 Whitechapel Rd, Shadwell, London E1 1DB, United Kingdom",jdsports.co.uk,+44 20 7377 9134+JD Sports,"55 Chapel Street Market N1 9EW, United Kingdom",jdsports.co.uk,+44 20 7837 9440+Sweaty Betty,"400 Oxford St, Marylebone, London W1A 1AB, United Kingdom",sweatybetty.com,+44 800 123400+Runners Need Kensington,"188 Kensington High St, Kensington, London W8 7RG, United Kingdom",runnersneed.com,+44 20 3780 2057+Runners Need Camden Town,"34 Parkway, Camden Town, London NW1 7AH, United Kingdom",runnersneed.com,+44 20 3780 2051+OHMME,-,-,+44 20 7387 0732+JD Sports,"Unit 8 Kingsland Shopping Centre, Dalston E8 2LX, United Kingdom",jdsports.co.uk,+44 20 7993 1316+Runners Need Moorgate,"72 Chiswell St, London EC1Y 4AB, United Kingdom",runnersneed.com,+44 20 3780 2060+adidas Flagship Store London,"425 Oxford St, Mayfair, London W1C 2PG, United Kingdom",adidas.co.uk,+44 1244 747263+DAÖMEY LONDON,"93 Kingsland Rd, Hackney, London E2 8AG, United Kingdom",daomeyuk.com,+44 20 3669 8129+LIVE Clothing UK,"109A King's Rd, Chelsea, London SW3 4PA, United Kingdom",uk.liveclothing.com,+44 20 7352 9878+Nike Kings Road,"33 King's Rd, Chelsea, London SW3 4LX, United Kingdom",nike.com,+44 20 7259 9960+Runners Need King's Cross,"Unit 1, 200 Pentonville Rd, London N1 9JP, United Kingdom",runnersneed.com,+44 20 3780 2058+TSPORT,"22 Abbey Rd, St John's Wood, London NW10 7RE, United Kingdom",-,-+JD Sports,"451-453 Brixton Rd, Brixton, London SW9 8HH, United Kingdom",jdsports.co.uk,+44 20 7737 3728+JD Sports,"Unit 21 West 12 Sc, Shepherd's Bush W12 8PP, United Kingdom",jdsports.co.uk,+44 20 8150 2055+Originals Flagship Store London,"15 Foubert's Pl, Carnaby St, Soho, London W1F 7QB, United Kingdom",adidas.co.uk,+44 1244 747263+In Corpore Sano,"129 New King's Rd, Fulham, London SW6 4SL, United Kingdom",-,-+Mr Gymnastic,"131-151 Great Titchfield St, Fitzrovia, London W1W 5BB, United Kingdom",mrgymnastic.co.uk,-+Adrenna,-,adrenna.co.uk,-+Players Sportswear,-,-,-+SOS Activewear,-,sosactivewear.com,-+Sweaty Betty,"3 Marylebone High St, Marylebone, London W1U 4NG, United Kingdom",sweatybetty.com,+44 20 7486 6204+Runners Need Clapham Junction,"85 St John's Rd, London SW11 1QY, United Kingdom",runnersneed.com,+44 20 3780 2054+Capital Golf London,"14 John Prince's St, Marylebone, London W1G 0JS, United Kingdom",capitalgolf.com,+44 20 7493 4414+The Pentland Group Limited,"8 Manchester Square, Marylebone, London W1U 3PH, United Kingdom",pentlandgroup.com,+44 20 7535 3800+Mountain Warehouse Islington,"40 Upper St, The Angel, London N1 0PN, United Kingdom",mountainwarehouse.com,+44 20 7354 2444+KEJO FITNESS,-,kejofitness.com,-+Focus,"Kemp House, 160 City Rd, London EC1V 2NX, United Kingdom",-,+44 7578 976613+Maxibillion,"43 Berkeley Square, Mayfair, London W1J 5AP, United Kingdom",maxibillion.co.uk,+44 20 7889 2868diff --git a/my_config.py b/google-maps/my_config.pysimilarity index 100%rename from my_config.pyrename to google-maps/my_config.pydiff --git a/webdriver.py b/webdriver.pyindex d8ee7c9..feaa224 100644--- a/webdriver.py+++ b/webdriver.py@@ -18,10 +18,10 @@ class Webdriver:_ua = UserAgent()_buf = Buffer()- async def init_browser(self, headless='False', language='en-gb'):+ async def init_browser(self, headless_status=False, language='en-gb'):self.browser = await launch(ignoreHTTPSErrors=True,- headless=headless,+ headless=headless_status,viewport=None,autoclose=True,args=['--start-maximized'],@@ -33,6 +33,8 @@ class Webdriver:await self._page.setExtraHTTPHeaders({'Accept-Language': language})await self._page.reload()+ logger.debug('Successfull initialization')+async def _shut_browser(self):logger.debug('Shutting down browser in 15 seconds')sleep(15)@@ -56,10 +58,6 @@ class Webdriver:print(e)raise SystemError('Some shit happened to pyppeteer, fix it')- async def _enter(self, word):- await self._page.keyboard.type(word)- await self._page.keyboard.press('Enter')-async def _jump(self, url, xpath):try:await self._page.goto(url)@@ -67,4 +65,4 @@ class Webdriver:self._page.waitForXPath(xpath, {'visible': True})except:await self._shut_browser()- raise ValueError('Got invalid URL for google maps')\ No newline at end of file+ raise ValueError('Got invalid URL for google maps')diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex bbc4a70..2ead7e4 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -8,75 +8,112 @@ from bs4 import BeautifulSoupfrom time import sleepfrom locators import *-from config import *+from my_config import *import syssys.path.append('..')-from my_config import+from logger_config import get_loggerfrom webdriver import Webdriver-not_found = lambda loc, desc: f'No Results for {desc}' in requests.get(url.format(desc, loc), headers={'User-Agent': ua.random}).text+logger = get_logger('webdriver.yelp')++ua = UserAgent()-def validate_args(func):-- def inner(self, loc, desc):- if not_found(loc, desc):- raise ValueError(f'No results for `{desc}` in `{loc}`')- func(self, loc, desc)+class Yelp(Webdriver):+ _found = lambda loc, desc: not f'No Results for {desc}' in requests.get(yelp.format(desc, loc), headers={'User-Agent': ua.random}).text+ _yelp = yelp.split('search')[0]+++ async def search(self, loc=None, desc=None, url=None):+ if not '_page' in self.__dict__:+ raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')++ if loc and desc and not url:+ if self._found(loc, desc):+ logger.debug('Working based on location `%s` and keyword `%s`', loc, desc)+ self._data = [desc, loc]+ await self._locate()+ else:+ raise ValueError(f'No result for `{desc}` and `{loc}`')++ elif not (loc and desc) and url:+ await self._jump(url, yelp_xpath)+ logger.debug('Working based on given URL `%s`', url)++ else:+ await self._shut_browser()+ raise ValueError('Specify only location and description or only URL for yelp page')++ await self._scrape+ await self._shut_browser++ async def _locate(self):+ await self._page.goto(self._yelp)+ sleep(2.5)++ inputs = await self._page.querySelectorAll(input_locator)+ for i in range(2):+ await inputs[i].click()+ sleep(0.13)+ await inputs[i].enter(self._data[i])- return inner+ sleep(0.21)+ await self._page.keyboard.press('Enter')-class Yelp(Webdriver):- @validate_args- def __init__(self, loc, desc):- self._loc = loc- self._desc = desc- async def _extract_urls(self, html):- return ['https://www.yelp.com' + url for url in lxml.html.document_fromstring(html).xpath(urls_xpath)]+ async def _scrape(self):++ def parse(url):+ r = requests.get(url)+ if r.ok:+ soup = BeautifulSoup(html, 'lxml')++ try:+ title = soup.select_one('h1').text.strip()+ except:+ title = '-'++ try:+ phone = soup.find(string='Phone number').next_sibling().text().strip()+ except:+ phone = '-'++ try:+ addr = soup.find(string='Get Directions').next_sibling().text.strip()+ except:+ addr = '-'+ try:+ website = soup.find(attrs={'rel':'noopener'})['href'].split('/biz_redir?url=http%3A%2F%2F')[0].split('&')[0]+ except:+ website = '-'++ self._buf.store(dict(zip(order, [title, addr, website, phone])))+ else:+ logger.warning(f'Failed request {r} to {url}')++ def extract_urls(html):+ return ['https://www.yelp.com' + url for url in lxml.html.document_fromstring(html).xpath(urls_xpath)]+- async def _locate(self):- self._page.goto()-- def _parse(self, html):- soup = BeautifulSoup(html, 'lxml')-- try:- title = soup.select_one('h1').text.strip()- except:- title = '-'-- try:- phone = soup.find(string='Phone number').next_sibling().text().strip()- except:- phone = '-'-- try:- addr = soup.find(string='Get Directions').next_sibling().text.strip()- except:- addr = None- try:- website = soup.find(attrs={'rel':'noopener'})['href'].split('/biz_redir?url=http%3A%2F%2F')[0].split('&')[0]- except:- website = None-- return dict(zip(order, [title, addr, website, phone]))-- async def scrape(self):- # 1) Go to the yelp with given description and location- # 2) Extract urls from each page- # 3) Scrape each page using Pool- # 4) Go to next page (if exists)- # 5) Iterate it overwhile True:- html = await self._locate()- urls = self._extract_urls(html)+ urls = extract_urls(await self._page.content())+with Pool(10) as p:- p.map(self._parse, urls)+ p.map(parse, urls)++ next_button = await self._page.xpath(next_xpath)+ if not next_button:+ self._buf.dump()+ break+ await next_button.click()+ sleep(2)-y = Yelp('London', 'Pizza')+if __name__ == '__main__':+ y = Yelp()+ await y.init_browser()+ await y.search('London','Sportswear')diff --git a/yellow-pages/locators.py b/yellow-pages/locators.pyindex 8e897ad..161b76f 100644--- a/yellow-pages/locators.py+++ b/yellow-pages/locators.py@@ -1,3 +1,5 @@urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'+yelp_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'+next_xpath = '//*[@class="icon--24-chevron-right-v2 navigation-button-icon__09f24__2FnZH css-12anxc3"]'-title_locator = 'h1'+input_locator = '.pseudo-input_field.business-search-form_input-field'\ No newline at end of filediff --git a/yellow-pages/config.py b/yellow-pages/my_config.pysimilarity index 51%rename from yellow-pages/config.pyrename to yellow-pages/my_config.pyindex dbb66c6..c7e6b3c 100644--- a/yellow-pages/config.py+++ b/yellow-pages/my_config.py@@ -1,4 +1,4 @@-url = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'+yelp = 'https://www.yelp.com/search?find_desc={}&find_loc={}&ns=1'order = ['Title',