Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

refined method to locate information for yelp

Madi94d16685ae8d
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index 43aa8be..6093bae 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -1,5 +1,5 @@
import re
-import lxml.html
+import json
import requests
import asyncio
@@ -12,44 +12,48 @@ from my_config import *
import sys
sys.path.append('..')
-from logger_config import get_logger
from webdriver import Webdriver
-
+from logger_config import get_logger
logger = get_logger('webdriver.yelp')
ua = UserAgent()
-_found = lambda loc, desc: not f'No Results for {desc}' in requests.get(yelp.format(desc, loc), headers={'User-Agent': ua.random}).text
-class Yelp(Webdriver):
- _yelp = yelp.split('search')[0]
+def _found(loc, desc): return not f'No Results for {desc}' in requests.get(
+ yelp.format(desc, loc), headers={'User-Agent': ua.random}).text
+
+class Yelp(Webdriver):
+ _yelp = yelp.split('search')[0]
async def search(self, loc=None, desc=None, url=None):
if not '_page' in self.__dict__:
- raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')
-
+ raise ValueError(
+ 'Initialize the browser before searching by `await *.init_broswser()`')
+
if loc and desc and not url:
if _found(loc, desc):
- logger.debug('Working based on location `%s` and keyword `%s`', loc, desc)
+ logger.debug(
+ 'Working based on location `%s` and keyword `%s`', loc, desc)
self._data = [desc, loc]
await self._locate()
else:
raise ValueError(f'No result for `{desc}` and `{loc}`')
-
+
elif not (loc and desc) and url:
await self._jump(url, results_xpath)
logger.debug('Working based on given URL `%s`', url)
-
+
else:
await self._shut_browser()
- raise ValueError('Specify only location and description or only URL for yelp page')
-
+ raise ValueError(
+ 'Specify only location and description or only URL for yelp page')
+
await self._scrape()
await self._shut_browser()
-
+
async def _locate(self):
await self._page.goto(self._yelp)
sleep(2.5)
@@ -64,30 +68,24 @@ class Yelp(Webdriver):
await self._page.click(search_button_selector)
async def _scrape(self):
-
+
def parse(html):
- data = {}
- soup = BeautifulSoup(html, 'lxml')
-
- for field, cls_ in order.items():
- try:
- if cls_:
- value = soup.find(class_=cls_,).parent.previous_sibling().text.replace('Business website','').replace('Phone number','').strip()
- else:
- value = soup.find('h1').text.strip()
- except Exception as e:
- print(e)
- value = '-'
- data.update({field: value})
+ soup = BeautifulSoup(html, 'html.parser')
+
+ raw_data = json.loads(soup.find('script', attrs=json_attrs).string)
+ website = soup.find(attrs=website_attrs).text
+
+ data = {'Title': raw_data['name'],
+ 'Address': ', '.join(list(raw_data['address'].values())), 'WebSite': website,
+ 'PhoneNumber': raw_data['telephone']}
logger.debug('Extracted: %s', list(data.values()))
self._buf.store(data)
-
while True:
await self._page.waitForXPath(results_xpath)
places = len(await self._page.xpath(places_xpath))
logger.debug('%s places found', places)
-
+
for i in range(places):
await self._page.waitForXPath(results_xpath)
await (await self._page.xpath(places_xpath))[i].click()
@@ -105,12 +103,12 @@ class Yelp(Webdriver):
await next_button[0].click()
sleep(2)
+
async def main():
y = Yelp()
await y.init_browser()
- await y.search('London','Sportswear')
+ await y.search('London', 'Sportswear')
if __name__ == '__main__':
asyncio.run(main())
-
diff --git a/yellow-pages/locators.py b/yellow-pages/locators.py
index ec0e90b..289f5ba 100644
--- a/yellow-pages/locators.py
+++ b/yellow-pages/locators.py
@@ -4,4 +4,7 @@ places_xpath = '//h4'
inner_xpath = '//*[@class="lemon--h1__373c0__2ZHSL heading--h1__373c0__dvYgw undefined heading--inline__373c0__10ozy"]'
search_button_selector = '.ybtn.ybtn--primary.ybtn--small.business-search-form_button'
-input_selector = '.pseudo-input_field.business-search-form_input-field'
\ No newline at end of file
+input_selector = '.pseudo-input_field.business-search-form_input-field'
+
+json_attrs = {'type': 'application/ld+json'}
+website_attrs = {'rel': 'noopener nofollow'}
diff --git a/yellow-pages/test.py b/yellow-pages/test.py
index 25a607c..c130d71 100644
--- a/yellow-pages/test.py
+++ b/yellow-pages/test.py
@@ -1,18 +1,10 @@
+import json
from bs4 import BeautifulSoup
f = open('html.html')
html = f.read()
soup = BeautifulSoup(html, 'html.parser')
-order = {
- #'Title': None,
- 'Address': 'icon--24-directions-v2',
- 'WebSite': 'Business website',
- 'PhoneNumber': 'icon--24-phone-v2'
-}
+data = json.loads(soup.find('script', attrs={'type':'application/ld+json'}).string)
-data = []
-for title, str_ in order.items():
- data.append(soup.find(string=str_).next_element.text)
-
-print(data)
\ No newline at end of file
+print(soup.find(attrs={'rel': 'noopener nofollow'}).text)
\ No newline at end of file