Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

yelp class methods added, small enhancments

Madi609f624e389b
diff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pyc
index e82db45..f3f9be7 100644
Binary files a/__pycache__/webdriver.cpython-38.pyc and b/__pycache__/webdriver.cpython-38.pyc differ
diff --git a/driver_retry.py b/driver_retry.py
deleted file mode 100644
index f9d550c..0000000
--- a/driver_retry.py
+++ /dev/null
@@ -1,9 +0,0 @@
-async def _evaluate(page, retries=0):
- if retries > 10:
- return await page.evaluate('document.body.innerHTML')
- else:
- try:
- return await page.evaluate('document.body.innerHTML')
- except pyppeteer.errors.NetworkError:
- await asyncio.sleep(0.5)
- return await _evaluate(page, retries+1)
diff --git a/google-maps/extractor.py b/google-maps/extractor.py
index 050e3ca..1a2fbdc 100644
--- a/google-maps/extractor.py
+++ b/google-maps/extractor.py
@@ -8,7 +8,6 @@ from bs4 import BeautifulSoup
import sys
sys.path.append('..')
-
from webdriver import Webdriver
from logger_config import get_logger
@@ -87,6 +86,8 @@ class Searcher(Webdriver):
# await self._do_retry(self.page.goBack, result_xpath)
next_button = (await self.page.xpath(next_xpath))[0]
+ if not next_button:
+ break
await next_button.click()
sleep(1)
diff --git a/locators.py b/locators.py
deleted file mode 100644
index b8b736f..0000000
--- a/locators.py
+++ /dev/null
@@ -1,8 +0,0 @@
-button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'
-search_xpath = '//*[@role="gridcell"]'
-next_xpath = '//*[@id="n7lv7yjyC35__section-pagination-button-next"]'
-check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'
-place_xpath = '//*[@class="section-hero-header-title-title GLOBAL__gm2-headline-5"]'
-result_xpath = '//*[@class="section-result-content"]'
-back_xpath = '//*[@class="section-back-to-list-button blue-link noprint"]'
-
diff --git a/main.py b/main.py
deleted file mode 100644
index e69de29..0000000
diff --git a/webdriver.py b/webdriver.py
index 59cdd2a..5002a27 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -1,16 +1,11 @@
import asyncio
import pyppeteer
-import re
-
-from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from pyppeteer import launch
from time import sleep
-from locators import *
-from my_config import *
from logger_config import get_logger
from bufferization import Buffer
@@ -36,60 +31,14 @@ class Webdriver:
await self.page.setUserAgent(self._ua.random)
await self.page.setViewport(self.viewport)
await self.page.setExtraHTTPHeaders({'Accept-Language': language})
- await self.page.reload()
+ # await self.page.reload()
async def _shut_browser(self):
- await asyncio.sleep(50)
+ logger.debug('Shutting down browser')
+ sleep(15)
await self.page.close()
await self.browser.close()
- async def search(self, location=None, keyword=None, url=None):
- if location and keyword and not url:
- logger.debug(
- 'Working based on location `%s` and keyword `%s`', location, keyword)
- self._location = location
- self._keyword = keyword
- await self._locate()
- elif not (location and keyword) and url:
- logger.debug('Working based on given URL `%s`', url)
- self._url = url
- await self._jump()
- else:
- await self._shut_browser()
- raise ValueError(
- f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')
-
- await self._scrape()
- await self._shut_browser()
-
- async def _locate(self):
- async def _enter(word):
- await self.page.keyboard.type(word)
- await self.page.keyboard.press('Enter')
-
- # Go to google maps, type the location
- await self.page.goto(google_maps)
- await self.page.click('input')
- await _enter(self._location)
- await self.page.waitForXPath(button_xpath, {'visible': True})
-
- # Click search nearby, type the keyword
- await (await self.page.xpath(button_xpath))[2].click()
- await self.page.waitForXPath(search_xpath)
- await self.page.click('input')
- await _enter(self._keyword)
-
- await self.page.waitForXPath(result_xpath, {'visible': True})
-
- async def _jump(self):
- try:
- await self.page.goto(self._url)
- logger.debug('Validating given URL')
- self.page.waitForXPath(check_xpath, {'visible': True})
- except:
- await self._shut_browser()
- raise ValueError('Got invalid URL for google maps')
-
async def _do_retry(self, operation, xpath, dest=None, retries=0):
if retries == 10:
raise SystemError(
@@ -106,59 +55,3 @@ class Webdriver:
except Exception as e:
print(e)
raise SystemError('Some shit happened to pyppeteer, fix it')
-
- async def _scrape(self):
- while True:
- places = len(await self.page.xpath(result_xpath))
- logger.debug('%s places found', places)
-
- for i in range(places):
- logger.debug('Gonna scrape %s out of %s', i + 1, places)
-
- place = (await self.page.xpath(result_xpath))[i]
- await self._do_retry(place.click, place_xpath)
-
- data = self._extract(await self.page.content())
- self._buf.store(data)
-
- back = (await self.page.xpath(back_xpath))[0]
- await self._do_retry(back.click, result_xpath)
- # await self._do_retry(self.page.goBack, result_xpath)
-
- next_button = (await self.page.xpath(next_xpath))[0]
- await next_button.click()
- sleep(1)
-
- def _extract(self, html):
- soup = BeautifulSoup(html, 'html.parser')
-
- data = {}
-
- for field, src in order.items():
- try:
- if src:
- value = soup.find(
- attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()
- else:
- value = soup.find('h1').text.strip()
- if 'Add' in value or 'Добавить' in value:
- logger.debug('Got empty information %s', value)
- value = '-'
- except:
- value = '-'
- data.update({field: value})
-
- logger.debug('Data: %s', list(data.values()))
- return data
-
- async def _scrape_pages(self):
- pass
-
-
-async def main():
- w = Webdriver()
- await w.init_browser('ru')
- await w.search('New Zealand', 'Pastor')
-
-if __name__ == '__main__':
- asyncio.run(main())
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index d07b772..5b95767 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -26,9 +26,6 @@ def validate_args(func):
return inner
-
-
-
class Yelp(Webdriver):
@validate_args
@@ -38,7 +35,16 @@ class Yelp(Webdriver):
self._desc = desc
print(self._loc, self._desc)
-
+
+ async def _jump(self, url):
+ pass
+
+ async def scrape(self):
+ pass
+
+
+ def _extract(self, html):
+ soup = BeautifulSoup(html, 'html.parser')
y = Yelp('London', 'Pizza')