Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

logger issue fixed, google-maps working

Madic4e435e420f0
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pyc
index 2cf077a..9ceebdb 100644
Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differ
diff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pyc
index 7bfc5a9..d24809c 100644
Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differ
diff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pyc
index f3f9be7..545d0db 100644
Binary files a/__pycache__/webdriver.cpython-38.pyc and b/__pycache__/webdriver.cpython-38.pyc differ
diff --git a/bufferization.py b/bufferization.py
index 86fa8b9..302a802 100644
--- a/bufferization.py
+++ b/bufferization.py
@@ -27,18 +27,17 @@ class Buffer:
writer = csv.DictWriter(f, fieldnames=self.fn)
writer.writeheader()
- def _dump(self):
+ def dump(self):
with open('leads.csv', 'a', encoding='utf-8', newline='') as f:
writer = csv.DictWriter(f, fieldnames=self.fn)
for d in self._data:
writer.writerow(d)
-
+ self._data.clear()
def store(self, info: dict):
if info.get('Title') and not info in self._data:
self._data.append(info)
if len(self._data) >= self._buffer_size:
- self._dump()
- self._data.clear()
+ self.dump()
else:
print(f'Got duplicate or empty dictionary {info}')
\ No newline at end of file
diff --git a/google-maps/__pycache__/config.cpython-38.pyc b/google-maps/__pycache__/config.cpython-38.pyc
new file mode 100644
index 0000000..e190f10
Binary files /dev/null and b/google-maps/__pycache__/config.cpython-38.pyc differ
diff --git a/google-maps/__pycache__/locators.cpython-38.pyc b/google-maps/__pycache__/locators.cpython-38.pyc
index f1b5d98..81d7a9a 100644
Binary files a/google-maps/__pycache__/locators.cpython-38.pyc and b/google-maps/__pycache__/locators.cpython-38.pyc differ
diff --git a/google-maps/extractor.py b/google-maps/extractor.py
index 1a2fbdc..631ee8f 100644
--- a/google-maps/extractor.py
+++ b/google-maps/extractor.py
@@ -1,5 +1,7 @@
import re
+import asyncio
+
from locators import *
from config import *
@@ -15,12 +17,13 @@ from logger_config import get_logger
logger = get_logger('webdriver.google_maps')
-class Searcher(Webdriver):
+class GoogleMaps(Webdriver):
+
- def __init__(self):
- pass
async def search(self, location=None, keyword=None, url=None):
+ if not 'page' in self.__dict__:
+ raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')
if location and keyword and not url:
logger.debug(
'Working based on location `%s` and keyword `%s`', location, keyword)
@@ -35,7 +38,6 @@ class Searcher(Webdriver):
await self._shut_browser()
raise ValueError(
f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')
-
await self._scrape()
await self._shut_browser()
@@ -83,10 +85,10 @@ class Searcher(Webdriver):
back = (await self.page.xpath(back_xpath))[0]
await self._do_retry(back.click, result_xpath)
- # await self._do_retry(self.page.goBack, result_xpath)
next_button = (await self.page.xpath(next_xpath))[0]
if not next_button:
+ self._buf.dump()
break
await next_button.click()
sleep(1)
@@ -112,3 +114,14 @@ class Searcher(Webdriver):
logger.debug('Data: %s', list(data.values()))
return data
+
+
+async def main():
+ g = GoogleMaps()
+ await g.init_browser(headless=True)
+ await g.search('Кокшетау','Спортзал')
+
+
+
+if __name__ == '__main__':
+ asyncio.run(main())
\ No newline at end of file
diff --git a/google-maps/leads.csv b/google-maps/leads.csv
index 58ce79a..a899b9a 100644
--- a/google-maps/leads.csv
+++ b/google-maps/leads.csv
@@ -1 +1,16 @@
Title,Address,WebSite,PhoneNumber
+Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202
+Фитнес Клуб Kapital,Kokshetau 020000,-,8 (7162) 50 05 00
+"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",-,-
+Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10
+"Sportkompleks ""Kazhymukan""",Kokshetau 020000,-,8 (7162) 77 08 11
+Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",-,-
+Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",-,8 (7162) 42 07 80
+Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",-,-
+Fok Pri Sel'khozinstitute,Kokshetau 020000,-,-
+"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",-,-
+DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,-
+"""Domino""","Akan Seri St, Kokshetau 020000",-,8 (7162) 25 30 61
+Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",-,-
+"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03
+Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",-,-
diff --git a/google-maps/pylg.log b/google-maps/pylg.log
new file mode 100644
index 0000000..e69de29
diff --git a/logger_config.py b/logger_config.py
index ae769a0..05fc971 100644
--- a/logger_config.py
+++ b/logger_config.py
@@ -20,6 +20,7 @@ def get_logger(name: str):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
+ logger.propagate = False
logger.addHandler(s_handler)
logger.addHandler(f_handler)
diff --git a/webdriver.py b/webdriver.py
index 5002a27..d16e31c 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -14,14 +14,14 @@ logger = get_logger('webdriver')
class Webdriver:
viewport = {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1.0,
- 'isMobile': False, 'hasTouch': False, 'isLandscape': False, 'isMobile': True}
+ 'isMobile': False, 'hasTouch': False, 'isLandscape': False}
_ua = UserAgent()
_buf = Buffer()
- async def init_browser(self, language='en-gb'):
+ async def init_browser(self, headless='False', language='en-gb'):
self.browser = await launch(
ignoreHTTPSErrors=True,
- headless=False,
+ headless=headless,
viewport=None,
autoclose=True,
args=['--start-maximized'],
@@ -48,8 +48,8 @@ class Webdriver:
await operation(dest)
else:
await operation()
- await self.page.waitForXPath(xpath, {'visible': True})
sleep(1)
+ await self.page.waitForXPath(xpath, {'visible': True})
except pyppeteer.errors.TimeoutError:
await self._do_retry(operation, xpath, dest, retries + 1)
except Exception as e:
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index 5b95767..6a54225 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -1,5 +1,6 @@
import re
import requests
+import lxml.html
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
@@ -18,10 +19,10 @@ ua = UserAgent()
def validate_args(func):
- def inner(self, loc, desc, *args, **kwargs):
+ def inner(self, loc, desc):
if not_found(loc, desc):
raise ValueError(f'No results for `{desc}` in `{loc}`')
- func(self, loc, desc, *args, **kwargs)
+ func(self, loc, desc)
return inner
@@ -30,21 +31,30 @@ class Yelp(Webdriver):
@validate_args
def __init__(self, loc, desc):
-
self._loc = loc
self._desc = desc
- print(self._loc, self._desc)
+ async def _collect_urls(self, html):
+ tree = lxml.html.document_fromstring(html)
+ urls = ['https://www.yelp.com' + url for url in tree.xpath(urls_xpath)]
- async def _jump(self, url):
+ async def _locate(self):
pass
- async def scrape(self):
- pass
+ def _extract_urls(self, html):
+ soup = BeautifulSoup(html, 'lxml')
- def _extract(self, html):
- soup = BeautifulSoup(html, 'html.parser')
+ async def scrape(self):
+ # 1) Go to the yelp with given description and location
+ # 2) Extract urls from each page
+ # 3) Scrape each page using Pool
+ # 4) Go to next page (if exists)
+ # 5) Iterate it over
+ while True:
+ html = await self._locate()
+ urls = self._extract_urls(html)
+ self.page
y = Yelp('London', 'Pizza')
diff --git a/yellow-pages/locators.py b/yellow-pages/locators.py
index e69de29..8e897ad 100644
--- a/yellow-pages/locators.py
+++ b/yellow-pages/locators.py
@@ -0,0 +1,3 @@
+urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'
+
+title_locator = 'h1'