Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
logger config improved
Madi901d4e37e2f3
diff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pycindex 5024658..7ae0a07 100644Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differdiff --git a/logger_config.py b/logger_config.pyindex e85ea16..ae769a0 100644--- a/logger_config.py+++ b/logger_config.py@@ -1,26 +1,28 @@import logging-def get_logger(name: str):+if isinstance(name, str):- formatter = logging.Formatter(- fmt='%(asctime)s - %(levelname)s:%(levelno)s | %(filename)s/%(funcName)s/%(lineno)d | %(message)s')++ f_formatter = logging.Formatter(+ fmt='%(asctime)s - %(levelname)s | %(filename)s/%(funcName)s/%(lineno)d | %(message)s')+ s_formatter = logging.Formatter(+ fmt='%(levelname)s|%(name)s: %(message)s')s_handler = logging.StreamHandler()s_handler.setLevel(logging.DEBUG)+ s_handler.setFormatter(s_formatter)f_handler = logging.FileHandler('pylg.log', 'a')f_handler.setLevel(logging.WARNING)-- handlers = [s_handler, f_handler]+ f_handler.setFormatter(f_formatter)logger = logging.getLogger(name)logger.setLevel(logging.DEBUG)-- for handler in handlers:- handler.setFormatter(formatter)- logger.addHandler(handler)+ logger.addHandler(s_handler)+ logger.addHandler(f_handler)return logger- raise TypeError('Logger name must be string')\ No newline at end of file++ raise TypeError('Logger name must be string')diff --git a/pylg.log b/pylg.lognew file mode 100644index 0000000..e69de29diff --git a/webdriver.py b/webdriver.pyindex 4cc17a6..a898743 100644--- a/webdriver.py+++ b/webdriver.py@@ -1,6 +1,5 @@import asyncio-import csvimport refrom bs4 import BeautifulSoup@@ -13,16 +12,14 @@ from config import *from logger_config import get_loggerfrom bufferization import Buffer--ua = UserAgent()logger = get_logger('driver')class Webdriver:viewport = {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1.0,'isMobile': False, 'hasTouch': False, 'isLandscape': False}-- buf = Buffer()+ _ua = UserAgent()+ _buf = Buffer()async def init_browser(self, language='ru'):self.browser = await launch(@@ -34,7 +31,7 @@ class Webdriver:self.page = (await self.browser.pages())[0]# await self.page.setViewport(self.viewport)- await self.page.setUserAgent(ua.random)+ await self.page.setUserAgent(self._ua.random)await self.page.setExtraHTTPHeaders({'Accept-Language': language})await self.page.reload()@@ -45,13 +42,12 @@ class Webdriver:async def search(self, location=None, keyword=None, url=None):if location and keyword and not url:- print(- f'Working based on location `{location}` and keyword `{keyword}`')+ logger.debug('Working based on location `%s` and keyword `%s`', location, keyword)self._location = locationself._keyword = keywordawait self._locate()elif not (location and keyword) and url:- print(f'Working based on given URL `{url}`')+ logger.debug('Working based on given URL `%s`', url)self._url = urlawait self._jump()else:@@ -80,39 +76,39 @@ class Webdriver:async def _jump(self):try:await self.page.goto(self._url)- print('Checking the given URL')+ logger.debug('Validating given URL')self.page.waitForXPath(check_xpath, {'visible': True})except:await self._shut_browser()raise ValueError('Got invalid URL for google maps')async def _scrape(self):+ async def wait_for_result():+ await self.page.waitForSelector(result_css, {'visible': True})+next_button = True+while next_button:- await self.page.waitForSelector(result_css, {'visible': True})++ await wait_for_result()home = self.page.url- overall = len(await self.page.querySelectorAll(result_css))- print(f'Overall found {overall}')+ places = len(await self.page.querySelectorAll(result_css))+ logger.debug('%s places found', places)- for i in range(overall):- print(f'Gonna scrape {i} out of {overall}')- element = (await self.page.querySelectorAll(result_css))[i]+ for i in range(places):+ logger.debug('Gonna scrape %s out of %s', i, places)+ await wait_for_result()+ place = (await self.page.querySelectorAll(result_css))[i]- await asyncio.wait([- element.click(),- self.page.waitForNavigation()- ])- sleep(0.5)+ await place.click()+ sleep(0.7)data = self._extract(await self.page.content())- self.buf.store(data)+ self._buf.store(data)- await asyncio.wait([- self.page.goto(home),- self.page.waitForNavigation()- ])- sleep(0.5)+ await self.page.goto(home)+ sleep(0.7)next_button = (await self.page.xpath(next_xpath))[0]await next_button.click()@@ -133,7 +129,7 @@ class Webdriver:value = Nonedata.update({field: value})- print(data)+ logger.debug('Data: %s', list(data.values()))return dataasync def _scrape_pages(self):