Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

logger config improved

Madi901d4e37e2f3
diff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pyc
index 5024658..7ae0a07 100644
Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differ
diff --git a/logger_config.py b/logger_config.py
index e85ea16..ae769a0 100644
--- a/logger_config.py
+++ b/logger_config.py
@@ -1,26 +1,28 @@
import logging
-
def get_logger(name: str):
+
if isinstance(name, str):
- formatter = logging.Formatter(
- fmt='%(asctime)s - %(levelname)s:%(levelno)s | %(filename)s/%(funcName)s/%(lineno)d | %(message)s')
+
+ f_formatter = logging.Formatter(
+ fmt='%(asctime)s - %(levelname)s | %(filename)s/%(funcName)s/%(lineno)d | %(message)s')
+ s_formatter = logging.Formatter(
+ fmt='%(levelname)s|%(name)s: %(message)s')
s_handler = logging.StreamHandler()
s_handler.setLevel(logging.DEBUG)
+ s_handler.setFormatter(s_formatter)
f_handler = logging.FileHandler('pylg.log', 'a')
f_handler.setLevel(logging.WARNING)
-
- handlers = [s_handler, f_handler]
+ f_handler.setFormatter(f_formatter)
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
-
- for handler in handlers:
- handler.setFormatter(formatter)
- logger.addHandler(handler)
+ logger.addHandler(s_handler)
+ logger.addHandler(f_handler)
return logger
- raise TypeError('Logger name must be string')
\ No newline at end of file
+
+ raise TypeError('Logger name must be string')
diff --git a/pylg.log b/pylg.log
new file mode 100644
index 0000000..e69de29
diff --git a/webdriver.py b/webdriver.py
index 4cc17a6..a898743 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -1,6 +1,5 @@
import asyncio
-import csv
import re
from bs4 import BeautifulSoup
@@ -13,16 +12,14 @@ from config import *
from logger_config import get_logger
from bufferization import Buffer
-
-ua = UserAgent()
logger = get_logger('driver')
class Webdriver:
viewport = {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1.0,
'isMobile': False, 'hasTouch': False, 'isLandscape': False}
-
- buf = Buffer()
+ _ua = UserAgent()
+ _buf = Buffer()
async def init_browser(self, language='ru'):
self.browser = await launch(
@@ -34,7 +31,7 @@ class Webdriver:
self.page = (await self.browser.pages())[0]
# await self.page.setViewport(self.viewport)
- await self.page.setUserAgent(ua.random)
+ await self.page.setUserAgent(self._ua.random)
await self.page.setExtraHTTPHeaders({'Accept-Language': language})
await self.page.reload()
@@ -45,13 +42,12 @@ class Webdriver:
async def search(self, location=None, keyword=None, url=None):
if location and keyword and not url:
- print(
- f'Working based on location `{location}` and keyword `{keyword}`')
+ logger.debug('Working based on location `%s` and keyword `%s`', location, keyword)
self._location = location
self._keyword = keyword
await self._locate()
elif not (location and keyword) and url:
- print(f'Working based on given URL `{url}`')
+ logger.debug('Working based on given URL `%s`', url)
self._url = url
await self._jump()
else:
@@ -80,39 +76,39 @@ class Webdriver:
async def _jump(self):
try:
await self.page.goto(self._url)
- print('Checking the given URL')
+ logger.debug('Validating given URL')
self.page.waitForXPath(check_xpath, {'visible': True})
except:
await self._shut_browser()
raise ValueError('Got invalid URL for google maps')
async def _scrape(self):
+ async def wait_for_result():
+ await self.page.waitForSelector(result_css, {'visible': True})
+
next_button = True
+
while next_button:
- await self.page.waitForSelector(result_css, {'visible': True})
+
+ await wait_for_result()
home = self.page.url
- overall = len(await self.page.querySelectorAll(result_css))
- print(f'Overall found {overall}')
+ places = len(await self.page.querySelectorAll(result_css))
+ logger.debug('%s places found', places)
- for i in range(overall):
- print(f'Gonna scrape {i} out of {overall}')
- element = (await self.page.querySelectorAll(result_css))[i]
+ for i in range(places):
+ logger.debug('Gonna scrape %s out of %s', i, places)
+ await wait_for_result()
+ place = (await self.page.querySelectorAll(result_css))[i]
- await asyncio.wait([
- element.click(),
- self.page.waitForNavigation()
- ])
- sleep(0.5)
+ await place.click()
+ sleep(0.7)
data = self._extract(await self.page.content())
- self.buf.store(data)
+ self._buf.store(data)
- await asyncio.wait([
- self.page.goto(home),
- self.page.waitForNavigation()
- ])
- sleep(0.5)
+ await self.page.goto(home)
+ sleep(0.7)
next_button = (await self.page.xpath(next_xpath))[0]
await next_button.click()
@@ -133,7 +129,7 @@ class Webdriver:
value = None
data.update({field: value})
- print(data)
+ logger.debug('Data: %s', list(data.values()))
return data
async def _scrape_pages(self):