Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

locator issue fixed

Madi0c8c4c4cd55f
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index 8691249..ec532f8 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -17,9 +17,8 @@ sys.path.append('..')
from webdriver import Webdriver
from logger_config import get_logger
-logger = get_logger('webdriver.yelp')
-
+logger = get_logger('webdriver.yelp')
ua = UserAgent()
@@ -77,15 +76,24 @@ class Yelp(Webdriver):
def parse(html):
soup = BeautifulSoup(html, 'html.parser')
- raw_data = json.loads(soup.find('script', attrs=json_attrs).string)
+ try:
+ raw_data = json.loads(soup.find('script', attrs=json_attrs).string)
+ except AttributeError:
+ logger.warning('Could not locate `raw_data` skipping current page')
+ return
+
try:
website = soup.find(attrs=website_attrs).text
except:
website = '-'
+
try:
- data = {'Title': raw_data['name'],
- 'Address': ', '.join(list(raw_data['address'].values())), 'WebSite': website,
- 'PhoneNumber': raw_data['telephone']}
+ data = {
+ 'WebSite': website,
+ 'Title': raw_data['name'],
+ 'PhoneNumber': raw_data['telephone'],
+ 'Address': ', '.join(list(raw_data['address'].values())),
+ }
logger.debug('Extracted: %s', list(data.values()))
self._buf.store(data)
except Exception:
diff --git a/yellow-pages/locators.py b/yellow-pages/locators.py
index 5b3691d..1ffcd4b 100644
--- a/yellow-pages/locators.py
+++ b/yellow-pages/locators.py
@@ -1,7 +1,7 @@
results_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'
next_xpath = '//*[@class="icon--24-chevron-right-v2 navigation-button-icon__09f24__2FnZH css-12anxc3"]'
places_xpath = '//h4'
-inner_xpath = '//*[@class="lemon--h1__373c0__2ZHSL heading--h1__373c0__dvYgw undefined heading--inline__373c0__10ozy"]'
+inner_xpath = '//section[@class=" margin-b3__373c0__q1DuY border-color--default__373c0__3-ifU"]'
search_button_selector = '.ybtn.ybtn--primary.ybtn--small.business-search-form_button'
input_selector = '.pseudo-input_field.business-search-form_input-field'