Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
minor bugs fixed
Madid174d23bea0c
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycnew file mode 100644index 0000000..c8522a4Binary files /dev/null and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/__pycache__/config.cpython-38.pyc b/__pycache__/config.cpython-38.pycnew file mode 100644index 0000000..c88140fBinary files /dev/null and b/__pycache__/config.cpython-38.pyc differdiff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pycnew file mode 100644index 0000000..b7c07c0Binary files /dev/null and b/__pycache__/locators.cpython-38.pyc differdiff --git a/bufferization.py b/bufferization.pyindex 7f75793..a911d9c 100644--- a/bufferization.py+++ b/bufferization.py@@ -29,8 +29,11 @@ class Buffer:writer = csv.DictWriter(f, fieldnames=self.fn)for d in self._data:writer.writerow(d)+def __iadd__(self, other: dict):+ print('In here!')self._data.append(dict)- if self._data >= self._buffer_size:- self._dump()\ No newline at end of file+ if len(self._data) >= self._buffer_size:+ self._dump()+ self._data.clear()diff --git a/leads.csv b/leads.csvnew file mode 100644index 0000000..58ce79a--- /dev/null+++ b/leads.csv@@ -0,0 +1 @@+Title,Address,WebSite,PhoneNumberdiff --git a/logger_config.py b/logger_config.pynew file mode 100644index 0000000..f249b3e--- /dev/null+++ b/logger_config.py@@ -0,0 +1,2 @@+import logging+diff --git a/webdriver.py b/webdriver.pyindex 01fb2c6..71dd7d5 100644--- a/webdriver.py+++ b/webdriver.py@@ -15,21 +15,25 @@ from bufferization import Bufferua = UserAgent()--class Webdriver:viewport = {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1.0,'isMobile': False, 'hasTouch': False, 'isLandscape': False}- buffer = Buffer()+ buf = Buffer()- async def init_browser(self):+ async def init_browser(self, language):self.browser = await launch(ignoreHTTPSErrors=True,headless=False,# slowMo=30,autoclose=True,)+ self.page = (await self.browser.pages())[0]++ # await self.page.setViewport(self.viewport)+ await self.page.setUserAgent(ua.random)+ await self.page.setExtraHTTPHeaders({'Accept-Language': 'en-us'})+ await self.page.reload()async def _shut_browser(self):await self.page.close()@@ -38,18 +42,18 @@ class Webdriver:async def search(self, location=None, keyword=None, url=None):if location and keyword and not url:print(- f'Working based on location {location} and keyword {keyword}')+ f'Working based on location `{location}` and keyword `{keyword}`')self._location = locationself._keyword = keywordawait self._locate()elif not (location and keyword) and url:- print(f'Working based on given URL {url}')+ print(f'Working based on given URL `{url}`')self._url = urlawait self._jump()else:await self._shut_browser()raise ValueError(- f'Specify location and keyword. Otherwise, specify URL only. Got location {location}, keyword {keyword} and URL {url}')+ f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')await self.scrape()await self._shut_browser()@@ -59,11 +63,6 @@ class Webdriver:await self.page.keyboard.press('Enter')async def _locate(self):- self.page = (await self.browser.pages())[0]-- # await self.page.setViewport(self.viewport)- await self.page.setUserAgent(ua.random)- await self.page.reload()await self.page.goto(google_maps)await self._enter(self._location)@@ -104,9 +103,11 @@ class Webdriver:await element.click()await self.page.waitForNavigation()+ sleep(1.5)data = self.extract(await self.page.content())- buffer += data+ print(self.buf._data)+ self.buf += data# await asyncio.wait([# self.page.goto(home),