Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
logger issue fixed, google-maps working
Madic4e435e420f0
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycindex 2cf077a..9ceebdb 100644Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/__pycache__/logger_config.cpython-38.pyc b/__pycache__/logger_config.cpython-38.pycindex 7bfc5a9..d24809c 100644Binary files a/__pycache__/logger_config.cpython-38.pyc and b/__pycache__/logger_config.cpython-38.pyc differdiff --git a/__pycache__/webdriver.cpython-38.pyc b/__pycache__/webdriver.cpython-38.pycindex f3f9be7..545d0db 100644Binary files a/__pycache__/webdriver.cpython-38.pyc and b/__pycache__/webdriver.cpython-38.pyc differdiff --git a/bufferization.py b/bufferization.pyindex 86fa8b9..302a802 100644--- a/bufferization.py+++ b/bufferization.py@@ -27,18 +27,17 @@ class Buffer:writer = csv.DictWriter(f, fieldnames=self.fn)writer.writeheader()- def _dump(self):+ def dump(self):with open('leads.csv', 'a', encoding='utf-8', newline='') as f:writer = csv.DictWriter(f, fieldnames=self.fn)for d in self._data:writer.writerow(d)-+ self._data.clear()def store(self, info: dict):if info.get('Title') and not info in self._data:self._data.append(info)if len(self._data) >= self._buffer_size:- self._dump()- self._data.clear()+ self.dump()else:print(f'Got duplicate or empty dictionary {info}')\ No newline at end of filediff --git a/google-maps/__pycache__/config.cpython-38.pyc b/google-maps/__pycache__/config.cpython-38.pycnew file mode 100644index 0000000..e190f10Binary files /dev/null and b/google-maps/__pycache__/config.cpython-38.pyc differdiff --git a/google-maps/__pycache__/locators.cpython-38.pyc b/google-maps/__pycache__/locators.cpython-38.pycindex f1b5d98..81d7a9a 100644Binary files a/google-maps/__pycache__/locators.cpython-38.pyc and b/google-maps/__pycache__/locators.cpython-38.pyc differdiff --git a/google-maps/extractor.py b/google-maps/extractor.pyindex 1a2fbdc..631ee8f 100644--- a/google-maps/extractor.py+++ b/google-maps/extractor.py@@ -1,5 +1,7 @@import re+import asyncio+from locators import *from config import *@@ -15,12 +17,13 @@ from logger_config import get_loggerlogger = get_logger('webdriver.google_maps')-class Searcher(Webdriver):+class GoogleMaps(Webdriver):+- def __init__(self):- passasync def search(self, location=None, keyword=None, url=None):+ if not 'page' in self.__dict__:+ raise ValueError('Initialize the browser before searching by `await *.init_broswser()`')if location and keyword and not url:logger.debug('Working based on location `%s` and keyword `%s`', location, keyword)@@ -35,7 +38,6 @@ class Searcher(Webdriver):await self._shut_browser()raise ValueError(f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')-await self._scrape()await self._shut_browser()@@ -83,10 +85,10 @@ class Searcher(Webdriver):back = (await self.page.xpath(back_xpath))[0]await self._do_retry(back.click, result_xpath)- # await self._do_retry(self.page.goBack, result_xpath)next_button = (await self.page.xpath(next_xpath))[0]if not next_button:+ self._buf.dump()breakawait next_button.click()sleep(1)@@ -112,3 +114,14 @@ class Searcher(Webdriver):logger.debug('Data: %s', list(data.values()))return data+++async def main():+ g = GoogleMaps()+ await g.init_browser(headless=True)+ await g.search('Кокшетау','Спортзал')++++if __name__ == '__main__':+ asyncio.run(main())\ No newline at end of filediff --git a/google-maps/leads.csv b/google-maps/leads.csvindex 58ce79a..a899b9a 100644--- a/google-maps/leads.csv+++ b/google-maps/leads.csv@@ -1 +1,16 @@Title,Address,WebSite,PhoneNumber+Family Fitness Фитнес Клуб,"Сулейменова 18, Kokshetau 020000",familyfitness.kz,8 (707) 152 0202+Фитнес Клуб Kapital,Kokshetau 020000,-,8 (7162) 50 05 00+"Sportivnyy Kompleks ""Arena Fitnes""","Ulitsa Mal. Zhumabayeva 52, Kokshetau 020000",-,-+Lok Bodrost',"Ulitsa Kanaybi 33, Kokshetau 020000",bax.kz,8 (7162) 44 29 10+"Sportkompleks ""Kazhymukan""",Kokshetau 020000,-,8 (7162) 77 08 11+Legkoatleticheskiy Manezh,"Abay Street 122, Kokshetau 020000",-,-+Tennisnyy Tsentr,"Maly Gorkogo St, Kokshetau 020000",-,8 (7162) 42 07 80+Gu Upravleniye Turizma I Sporta Akmolinskoy Oblasti,"Abay Street 11, Kokshetau 020000",-,-+Fok Pri Sel'khozinstitute,Kokshetau 020000,-,-+"Dyufsh,, Okzhetpes""","Ulitsa Krasnaya 150, Kokshetau 020000",-,-+DERBY academy,"Ulitsa Kanaybi 24, Kokshetau 020000",derbyacademy.kz,-+"""Domino""","Akan Seri St, Kokshetau 020000",-,8 (7162) 25 30 61+Sportivnyy Klub Po Igrovym Vidam Sporta,"Abay Street 116-122, Kokshetau 020000",-,-+"Fitnes-Klub ""Yaguar""","Ulitsa Abylay-Khana 18/2, Shchuchinsk 020000",instagram.com,8 (71636) 3 81 03+Trenazhernyy Zal,"Ulitsa Konstitutsii Kazakhstana 54, Petropavl 150000",-,-diff --git a/google-maps/pylg.log b/google-maps/pylg.lognew file mode 100644index 0000000..e69de29diff --git a/logger_config.py b/logger_config.pyindex ae769a0..05fc971 100644--- a/logger_config.py+++ b/logger_config.py@@ -20,6 +20,7 @@ def get_logger(name: str):logger = logging.getLogger(name)logger.setLevel(logging.DEBUG)+ logger.propagate = Falselogger.addHandler(s_handler)logger.addHandler(f_handler)diff --git a/webdriver.py b/webdriver.pyindex 5002a27..d16e31c 100644--- a/webdriver.py+++ b/webdriver.py@@ -14,14 +14,14 @@ logger = get_logger('webdriver')class Webdriver:viewport = {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1.0,- 'isMobile': False, 'hasTouch': False, 'isLandscape': False, 'isMobile': True}+ 'isMobile': False, 'hasTouch': False, 'isLandscape': False}_ua = UserAgent()_buf = Buffer()- async def init_browser(self, language='en-gb'):+ async def init_browser(self, headless='False', language='en-gb'):self.browser = await launch(ignoreHTTPSErrors=True,- headless=False,+ headless=headless,viewport=None,autoclose=True,args=['--start-maximized'],@@ -48,8 +48,8 @@ class Webdriver:await operation(dest)else:await operation()- await self.page.waitForXPath(xpath, {'visible': True})sleep(1)+ await self.page.waitForXPath(xpath, {'visible': True})except pyppeteer.errors.TimeoutError:await self._do_retry(operation, xpath, dest, retries + 1)except Exception as e:diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.pyindex 5b95767..6a54225 100644--- a/yellow-pages/extractor.py+++ b/yellow-pages/extractor.py@@ -1,5 +1,6 @@import reimport requests+import lxml.htmlfrom fake_useragent import UserAgentfrom bs4 import BeautifulSoup@@ -18,10 +19,10 @@ ua = UserAgent()def validate_args(func):- def inner(self, loc, desc, *args, **kwargs):+ def inner(self, loc, desc):if not_found(loc, desc):raise ValueError(f'No results for `{desc}` in `{loc}`')- func(self, loc, desc, *args, **kwargs)+ func(self, loc, desc)return inner@@ -30,21 +31,30 @@ class Yelp(Webdriver):@validate_argsdef __init__(self, loc, desc):-self._loc = locself._desc = desc- print(self._loc, self._desc)+ async def _collect_urls(self, html):+ tree = lxml.html.document_fromstring(html)+ urls = ['https://www.yelp.com' + url for url in tree.xpath(urls_xpath)]- async def _jump(self, url):+ async def _locate(self):pass- async def scrape(self):- pass+ def _extract_urls(self, html):+ soup = BeautifulSoup(html, 'lxml')- def _extract(self, html):- soup = BeautifulSoup(html, 'html.parser')+ async def scrape(self):+ # 1) Go to the yelp with given description and location+ # 2) Extract urls from each page+ # 3) Scrape each page using Pool+ # 4) Go to next page (if exists)+ # 5) Iterate it over+ while True:+ html = await self._locate()+ urls = self._extract_urls(html)+ self.pagey = Yelp('London', 'Pizza')diff --git a/yellow-pages/locators.py b/yellow-pages/locators.pyindex e69de29..8e897ad 100644--- a/yellow-pages/locators.py+++ b/yellow-pages/locators.py@@ -0,0 +1,3 @@+urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'++title_locator = 'h1'