Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
refined method to locate information for yelp
Madi94d16685ae8d
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex 43aa8be..6093bae 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -1,5 +1,5 @@import re-import lxml.html+import jsonimport requestsimport asyncio@@ -12,44 +12,48 @@ from my_config import *import syssys.path.append('..')-from logger_config import get_loggerfrom webdriver import Webdriver-+from logger_config import get_loggerlogger = get_logger('webdriver.yelp')ua = UserAgent()-_found = lambda loc, desc: not f'No Results for {desc}' in requests.get(yelp.format(desc, loc), headers={'User-Agent': ua.random}).text-class Yelp(Webdriver):- _yelp = yelp.split('search')[0]+def _found(loc, desc): return not f'No Results for {desc}' in requests.get(+ yelp.format(desc, loc), headers={'User-Agent': ua.random}).text++class Yelp(Webdriver):+ _yelp = yelp.split('search')[0]async def search(self, loc=None, desc=None, url=None):if not '_page' in self.__dict__:- raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')-+ raise ValueError(+ 'Initialize the browser before searching by `await *.init_broswser()`')+if loc and desc and not url:if _found(loc, desc):- logger.debug('Working based on location `%s` and keyword `%s`', loc, desc)+ logger.debug(+ 'Working based on location `%s` and keyword `%s`', loc, desc)self._data = [desc, loc]await self._locate()else:raise ValueError(f'No result for `{desc}` and `{loc}`')-+elif not (loc and desc) and url:await self._jump(url, results_xpath)logger.debug('Working based on given URL `%s`', url)-+else:await self._shut_browser()- raise ValueError('Specify only location and description or only URL for yelp page')-+ raise ValueError(+ 'Specify only location and description or only URL for yelp page')+await self._scrape()await self._shut_browser()-+async def _locate(self):await self._page.goto(self._yelp)sleep(2.5)@@ -64,30 +68,24 @@ class Yelp(Webdriver):await self._page.click(search_button_selector)async def _scrape(self):-+def parse(html):- data = {}- soup = BeautifulSoup(html, 'lxml')-- for field, cls_ in order.items():- try:- if cls_:- value = soup.find(class_=cls_,).parent.previous_sibling().text.replace('Business website','').replace('Phone number','').strip()- else:- value = soup.find('h1').text.strip()- except Exception as e:- print(e)- value = '-'- data.update({field: value})+ soup = BeautifulSoup(html, 'html.parser')++ raw_data = json.loads(soup.find('script', attrs=json_attrs).string)+ website = soup.find(attrs=website_attrs).text++ data = {'Title': raw_data['name'],+ 'Address': ', '.join(list(raw_data['address'].values())), 'WebSite': website,+ 'PhoneNumber': raw_data['telephone']}logger.debug('Extracted: %s', list(data.values()))self._buf.store(data)-while True:await self._page.waitForXPath(results_xpath)places = len(await self._page.xpath(places_xpath))logger.debug('%s places found', places)-+for i in range(places):await self._page.waitForXPath(results_xpath)await (await self._page.xpath(places_xpath))[i].click()@@ -105,12 +103,12 @@ class Yelp(Webdriver):await next_button[0].click()sleep(2)+async def main():y = Yelp()await y.init_browser()- await y.search('London','Sportswear')+ await y.search('London', 'Sportswear')if __name__ == '__main__':asyncio.run(main())-diff --git a/yellow-pages/locators.py b/yellow-pages/locators.pyindex ec0e90b..289f5ba 100644--- a/yellow-pages/locators.py+++ b/yellow-pages/locators.py@@ -4,4 +4,7 @@ places_xpath = '//h4'inner_xpath = '//*[@class="lemon--h1__373c0__2ZHSL heading--h1__373c0__dvYgw undefined heading--inline__373c0__10ozy"]'search_button_selector = '.ybtn.ybtn--primary.ybtn--small.business-search-form_button'-input_selector = '.pseudo-input_field.business-search-form_input-field'\ No newline at end of file+input_selector = '.pseudo-input_field.business-search-form_input-field'++json_attrs = {'type': 'application/ld+json'}+website_attrs = {'rel': 'noopener nofollow'}diff --git a/yellow-pages/test.py b/yellow-pages/test.pyindex 25a607c..c130d71 100644--- a/yellow-pages/test.py+++ b/yellow-pages/test.py@@ -1,18 +1,10 @@+import jsonfrom bs4 import BeautifulSoupf = open('html.html')html = f.read()soup = BeautifulSoup(html, 'html.parser')-order = {- #'Title': None,- 'Address': 'icon--24-directions-v2',- 'WebSite': 'Business website',- 'PhoneNumber': 'icon--24-phone-v2'-}+data = json.loads(soup.find('script', attrs={'type':'application/ld+json'}).string)-data = []-for title, str_ in order.items():- data.append(soup.find(string=str_).next_element.text)--print(data)\ No newline at end of file+print(soup.find(attrs={'rel': 'noopener nofollow'}).text)\ No newline at end of file