Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

minor bugs fixed

Madid174d23bea0c
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pyc
new file mode 100644
index 0000000..c8522a4
Binary files /dev/null and b/__pycache__/bufferization.cpython-38.pyc differ
diff --git a/__pycache__/config.cpython-38.pyc b/__pycache__/config.cpython-38.pyc
new file mode 100644
index 0000000..c88140f
Binary files /dev/null and b/__pycache__/config.cpython-38.pyc differ
diff --git a/__pycache__/locators.cpython-38.pyc b/__pycache__/locators.cpython-38.pyc
new file mode 100644
index 0000000..b7c07c0
Binary files /dev/null and b/__pycache__/locators.cpython-38.pyc differ
diff --git a/bufferization.py b/bufferization.py
index 7f75793..a911d9c 100644
--- a/bufferization.py
+++ b/bufferization.py
@@ -29,8 +29,11 @@ class Buffer:
writer = csv.DictWriter(f, fieldnames=self.fn)
for d in self._data:
writer.writerow(d)
+
def __iadd__(self, other: dict):
+ print('In here!')
self._data.append(dict)
- if self._data >= self._buffer_size:
- self._dump()
\ No newline at end of file
+ if len(self._data) >= self._buffer_size:
+ self._dump()
+ self._data.clear()
diff --git a/leads.csv b/leads.csv
new file mode 100644
index 0000000..58ce79a
--- /dev/null
+++ b/leads.csv
@@ -0,0 +1 @@
+Title,Address,WebSite,PhoneNumber
diff --git a/logger_config.py b/logger_config.py
new file mode 100644
index 0000000..f249b3e
--- /dev/null
+++ b/logger_config.py
@@ -0,0 +1,2 @@
+import logging
+
diff --git a/webdriver.py b/webdriver.py
index 01fb2c6..71dd7d5 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -15,21 +15,25 @@ from bufferization import Buffer
ua = UserAgent()
-
-
class Webdriver:
viewport = {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1.0,
'isMobile': False, 'hasTouch': False, 'isLandscape': False}
- buffer = Buffer()
+ buf = Buffer()
- async def init_browser(self):
+ async def init_browser(self, language):
self.browser = await launch(
ignoreHTTPSErrors=True,
headless=False,
# slowMo=30,
autoclose=True,
)
+ self.page = (await self.browser.pages())[0]
+
+ # await self.page.setViewport(self.viewport)
+ await self.page.setUserAgent(ua.random)
+ await self.page.setExtraHTTPHeaders({'Accept-Language': 'en-us'})
+ await self.page.reload()
async def _shut_browser(self):
await self.page.close()
@@ -38,18 +42,18 @@ class Webdriver:
async def search(self, location=None, keyword=None, url=None):
if location and keyword and not url:
print(
- f'Working based on location {location} and keyword {keyword}')
+ f'Working based on location `{location}` and keyword `{keyword}`')
self._location = location
self._keyword = keyword
await self._locate()
elif not (location and keyword) and url:
- print(f'Working based on given URL {url}')
+ print(f'Working based on given URL `{url}`')
self._url = url
await self._jump()
else:
await self._shut_browser()
raise ValueError(
- f'Specify location and keyword. Otherwise, specify URL only. Got location {location}, keyword {keyword} and URL {url}')
+ f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')
await self.scrape()
await self._shut_browser()
@@ -59,11 +63,6 @@ class Webdriver:
await self.page.keyboard.press('Enter')
async def _locate(self):
- self.page = (await self.browser.pages())[0]
-
- # await self.page.setViewport(self.viewport)
- await self.page.setUserAgent(ua.random)
- await self.page.reload()
await self.page.goto(google_maps)
await self._enter(self._location)
@@ -104,9 +103,11 @@ class Webdriver:
await element.click()
await self.page.waitForNavigation()
+ sleep(1.5)
data = self.extract(await self.page.content())
- buffer += data
+ print(self.buf._data)
+ self.buf += data
# await asyncio.wait([
# self.page.goto(home),