Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

bufferization error fixed

Madid67f7ac85788
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pyc
index c8522a4..1b638d9 100644
Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differ
diff --git a/bufferization.py b/bufferization.py
index a911d9c..0dca5b9 100644
--- a/bufferization.py
+++ b/bufferization.py
@@ -31,9 +31,10 @@ class Buffer:
writer.writerow(d)
- def __iadd__(self, other: dict):
- print('In here!')
- self._data.append(dict)
- if len(self._data) >= self._buffer_size:
- self._dump()
- self._data.clear()
+ def store(self, info: dict):
+ if info.get('Title') and not info in self._data:
+ self._data.append(info)
+ if len(self._data) >= self._buffer_size:
+ self._dump()
+ self._data.clear()
+ print(f'Got duplicate or empty dictionary {info}')
\ No newline at end of file
diff --git a/leads.csv b/leads.csv
index 58ce79a..15248b1 100644
--- a/leads.csv
+++ b/leads.csv
@@ -1 +1,61 @@
Title,Address,WebSite,PhoneNumber
+,,,
+Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526
+Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294
+Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988
+Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636
+Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668
+Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567
+Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477
+Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999
+Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220
+Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154
+Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060
+De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203
+White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172
+Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329
+Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830
+Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966
+Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080
+Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887
+Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328
+,,,
+Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526
+Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294
+Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988
+Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636
+Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668
+Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567
+Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477
+Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999
+Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220
+Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154
+Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060
+De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203
+White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172
+Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329
+Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830
+Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966
+Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080
+Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887
+Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328
+,,,
+Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526
+Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294
+Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988
+Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636
+Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668
+Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567
+Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477
+Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999
+Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220
+Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154
+Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060
+De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203
+White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172
+Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329
+Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830
+Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966
+Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080
+Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887
+Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328
diff --git a/webdriver.py b/webdriver.py
index 71dd7d5..ece48f0 100644
--- a/webdriver.py
+++ b/webdriver.py
@@ -21,7 +21,7 @@ class Webdriver:
buf = Buffer()
- async def init_browser(self, language):
+ async def init_browser(self, language='ru'):
self.browser = await launch(
ignoreHTTPSErrors=True,
headless=False,
@@ -32,7 +32,7 @@ class Webdriver:
# await self.page.setViewport(self.viewport)
await self.page.setUserAgent(ua.random)
- await self.page.setExtraHTTPHeaders({'Accept-Language': 'en-us'})
+ await self.page.setExtraHTTPHeaders({'Accept-Language': language})
await self.page.reload()
async def _shut_browser(self):
@@ -55,24 +55,22 @@ class Webdriver:
raise ValueError(
f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')
- await self.scrape()
+ await self._scrape()
await self._shut_browser()
- async def _enter(self, word):
- await self.page.keyboard.type(word)
- await self.page.keyboard.press('Enter')
-
async def _locate(self):
- await self.page.goto(google_maps)
+ async def _enter(word):
+ await self.page.keyboard.type(word)
+ await self.page.keyboard.press('Enter')
- await self._enter(self._location)
+ await self.page.goto(google_maps)
+ await _enter(self._location)
await self.page.waitForXPath(button_xpath)
- button = (await self.page.xpath(button_xpath))[2]
- await button.click()
+ await (await self.page.xpath(button_xpath))[2].click()
await self.page.waitForXPath(search_xpath)
- await self._enter(self._keyword)
+ await _enter(self._keyword)
async def _jump(self):
try:
@@ -83,7 +81,7 @@ class Webdriver:
await self._shut_browser()
raise ValueError('Got invalid URL for google maps')
- async def scrape(self):
+ async def _scrape(self):
next_button = True
while next_button:
await self.page.waitForSelector('.section-result-content')
@@ -103,11 +101,10 @@ class Webdriver:
await element.click()
await self.page.waitForNavigation()
- sleep(1.5)
+ sleep(0.7)
data = self.extract(await self.page.content())
- print(self.buf._data)
- self.buf += data
+ self.buf.store(data)
# await asyncio.wait([
# self.page.goto(home),
@@ -116,7 +113,7 @@ class Webdriver:
await self.page.goto(home)
await self.page.waitForNavigation()
- sleep(1.5)
+ sleep(0.7)
next_button = (await self.page.xpath(next_xpath))[0]
await next_button.click()