Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
enter press issue resolved
Madi68cb3aad3438
diff --git a/yellow-pages/__pycache__/locators.cpython-38.pyc b/yellow-pages/__pycache__/locators.cpython-38.pycindex 9d36f07..e6f7af8 100644Binary files a/yellow-pages/__pycache__/locators.cpython-38.pyc and b/yellow-pages/__pycache__/locators.cpython-38.pyc differdiff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex b6a5e2d..9221883 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -41,7 +41,7 @@ class Yelp(Webdriver):raise ValueError(f'No result for `{desc}` and `{loc}`')elif not (loc and desc) and url:- await self._jump(url, yelp_xpath)+ await self._jump(url, results_xpath)logger.debug('Working based on given URL `%s`', url)else:@@ -62,7 +62,9 @@ class Yelp(Webdriver):await self._page.keyboard.type(self._data[i])sleep(0.21)- await self._page.keyboard.press('Enter')+ # await self._page.keyboard.press('Enter')+ sleep(3)+ print(len(await self._page.content()))async def _scrape(self):@@ -98,19 +100,25 @@ class Yelp(Webdriver):def extract_urls(html):return ['https://www.yelp.com' + url for url in lxml.html.document_fromstring(html).xpath(urls_xpath)]-+ # await self._page.goto('https://www.yelp.com/search?find_desc=Pizza&find_loc=London&ns=1')while True:+ sleep(5)+ await self._page.waitForXPath(results_xpath)urls = extract_urls(await self._page.content())- with Pool(10) as p:- p.map(parse, urls)-+ # with Pool(10) as p:+ # print('Etnered pool')+ # p.map(parse, urls)+ # print('Exited pool')+ for url in urls:+ logger.debug('Parsing %s', url)+ parse(url)next_button = await self._page.xpath(next_xpath)if not next_button:self._buf.dump()break- await next_button.click()+ await next_button[0].click()sleep(2)async def main():diff --git a/yellow-pages/locators.py b/yellow-pages/locators.pyindex 161b76f..63432a2 100644--- a/yellow-pages/locators.py+++ b/yellow-pages/locators.py@@ -1,5 +1,5 @@urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'-yelp_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'+results_xpath = '//*[@class=" leftRailSearchResultsContainer__09f24__3vlwA border-color--default__09f24__R1nRO"]'next_xpath = '//*[@class="icon--24-chevron-right-v2 navigation-button-icon__09f24__2FnZH css-12anxc3"]'input_locator = '.pseudo-input_field.business-search-form_input-field'\ No newline at end of filediff --git a/yellow-pages/pylg.log b/yellow-pages/pylg.logindex e69de29..d70c406 100644--- a/yellow-pages/pylg.log+++ b/yellow-pages/pylg.log@@ -0,0 +1,9 @@+2020-12-22 18:46:32,074 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:33,359 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:34,790 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:36,292 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:37,827 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:39,783 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:41,092 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:42,481 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}+2020-12-22 18:46:43,892 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}