Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

enter press issue resolved

Madi68cb3aad3438
diff --git a/yellow-pages/__pycache__/locators.cpython-38.pyc b/yellow-pages/__pycache__/locators.cpython-38.pyc
index 9d36f07..e6f7af8 100644
Binary files a/yellow-pages/__pycache__/locators.cpython-38.pyc and b/yellow-pages/__pycache__/locators.cpython-38.pyc differ
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index b6a5e2d..9221883 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -41,7 +41,7 @@ class Yelp(Webdriver):
raise ValueError(f'No result for `{desc}` and `{loc}`')
elif not (loc and desc) and url:
- await self._jump(url, yelp_xpath)
+ await self._jump(url, results_xpath)
logger.debug('Working based on given URL `%s`', url)
else:
@@ -62,7 +62,9 @@ class Yelp(Webdriver):
await self._page.keyboard.type(self._data[i])
sleep(0.21)
- await self._page.keyboard.press('Enter')
+ # await self._page.keyboard.press('Enter')
+ sleep(3)
+ print(len(await self._page.content()))
async def _scrape(self):
@@ -98,19 +100,25 @@ class Yelp(Webdriver):
def extract_urls(html):
return ['https://www.yelp.com' + url for url in lxml.html.document_fromstring(html).xpath(urls_xpath)]
-
+ # await self._page.goto('https://www.yelp.com/search?find_desc=Pizza&find_loc=London&ns=1')
while True:
+ sleep(5)
+ await self._page.waitForXPath(results_xpath)
urls = extract_urls(await self._page.content())
- with Pool(10) as p:
- p.map(parse, urls)
-
+ # with Pool(10) as p:
+ # print('Etnered pool')
+ # p.map(parse, urls)
+ # print('Exited pool')
+ for url in urls:
+ logger.debug('Parsing %s', url)
+ parse(url)
next_button = await self._page.xpath(next_xpath)
if not next_button:
self._buf.dump()
break
- await next_button.click()
+ await next_button[0].click()
sleep(2)
async def main():
diff --git a/yellow-pages/locators.py b/yellow-pages/locators.py
index 161b76f..63432a2 100644
--- a/yellow-pages/locators.py
+++ b/yellow-pages/locators.py
@@ -1,5 +1,5 @@
urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'
-yelp_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'
+results_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'
next_xpath = '//*[@class="icon--24-chevron-right-v2 navigation-button-icon__09f24__2FnZH css-12anxc3"]'
input_locator = '.pseudo-input_field.business-search-form_input-field'
\ No newline at end of file
diff --git a/yellow-pages/pylg.log b/yellow-pages/pylg.log
index e69de29..d70c406 100644
--- a/yellow-pages/pylg.log
+++ b/yellow-pages/pylg.log
@@ -0,0 +1,9 @@
+2020-12-22 18:46:32,074 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:33,359 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:34,790 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:36,292 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:37,827 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:39,783 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:41,092 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:42,481 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-22 18:46:43,892 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}