Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
bufferization error fixed
Madid67f7ac85788
diff --git a/__pycache__/bufferization.cpython-38.pyc b/__pycache__/bufferization.cpython-38.pycindex c8522a4..1b638d9 100644Binary files a/__pycache__/bufferization.cpython-38.pyc and b/__pycache__/bufferization.cpython-38.pyc differdiff --git a/bufferization.py b/bufferization.pyindex a911d9c..0dca5b9 100644--- a/bufferization.py+++ b/bufferization.py@@ -31,9 +31,10 @@ class Buffer:writer.writerow(d)- def __iadd__(self, other: dict):- print('In here!')- self._data.append(dict)- if len(self._data) >= self._buffer_size:- self._dump()- self._data.clear()+ def store(self, info: dict):+ if info.get('Title') and not info in self._data:+ self._data.append(info)+ if len(self._data) >= self._buffer_size:+ self._dump()+ self._data.clear()+ print(f'Got duplicate or empty dictionary {info}')\ No newline at end of filediff --git a/leads.csv b/leads.csvindex 58ce79a..15248b1 100644--- a/leads.csv+++ b/leads.csv@@ -1 +1,61 @@Title,Address,WebSite,PhoneNumber+,,,+Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526+Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294+Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988+Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636+Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668+Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567+Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477+Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999+Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220+Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154+Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060+De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203+White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172+Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329+Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830+Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966+Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080+Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887+Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328+,,,+Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526+Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294+Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988+Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636+Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668+Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567+Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477+Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999+Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220+Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154+Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060+De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203+White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172+Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329+Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830+Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966+Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080+Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887+Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328+,,,+Thai Sa-On Restaurant,"351 10 Ave SW, Calgary, AB T2R 0A5, Канада",thai-sa-on.com,+1 403-264-3526+Jackie's Thai,"200 Barclay Parade Southwest, Downtown, Division No. 6, Calgary, AB T2P 4V9, Канада",jackiesthai.ca,+1 403-999-6294+Rose Garden Thai Restaurant,"112 8 Ave SW, Calgary, AB T2P 1B4, Канада",rosegardenthaicalgary.com,+1 403-264-1988+Tuk Tuk Thai - Fifth Avenue Place,"420 2 St SW, Calgary, AB T2P 0H6, Канада",tuktukthai.com,+1 403-453-5636+Khao San Thai Kitchen,"1314 17 Ave SW, Calgary, AB T2T 0C3, Канада",khaosanthaikitchen.ca,+1 587-353-2668+Pad Thai Restaurant,"3106B 4 St NW, Calgary, AB T2M 3A4, Канада",padthaicalgary.com,+1 403-800-0567+Jurees Thai Place Restaurant,"2055 16 Ave NW, Calgary, AB T2M 0M3, Канада",jureesthaiplace.com,+1 403-264-6477+Tuk Tuk Thai,"636 17 Ave SW, Calgary, AB T2S 0B4, Канада",tuktukthai.com,+1 403-455-0999+Tuk Tuk Thai - Kensington,"268 10 St NW, Calgary, AB T2N 1V8, Канада",tuktukthai.com,+1 403-453-0220+Thai Express,"Centennial Place West Tower, 520 3 Ave SW, Calgary, AB T2P 0R3, Канада",thaiexpress.ca,+1 403-288-1154+Thai Tai,"615 17 Ave SW, Calgary, AB T2S 0B3, Канада",thaitaivietnamese.com,+1 403-244-3060+De Thai Cuisine,"101 - 2215 33 Ave SW, Calgary, AB T2T 1Z9, Канада",dethaicuisine.com,+1 403-705-2203+White Elephant Thai Cuisine,"1808 19 St NE, Calgary, AB T2E 4Y3, Канада",whiteelephantcuisinecalgary.com,+1 403-457-1172+Thai Nongkhai Restaurant,"7400 Macleod Trail #10, Calgary, AB T2H 0L9, Канада",thai-nk.com,+1 403-705-3329+Spicy Hut,"203 4 St NE, Calgary, AB T2E 3S1, Канада",telus.net,+1 403-237-0830+Thai Express,"The Core Shopping Centre, 751 3 St SW, Calgary, AB T2P 4K8, Канада",thaiexpress.ca,+1 403-237-9966+Thai Thien Sub,"811a 1 St SW, Calgary, AB T2P 7N2, Канада",skipthedishes.com,+1 403-263-4080+Thai Boat Restaurant,"2323 32 Ave NE #108, Calgary, AB T2E 6Z3, Канада",thaiboat.ca,+1 403-291-9887+Thai Express,"Suncor Energy Centre, 150 6 Ave SW, Calgary, AB T2P 3Y7, Канада",thaiexpress.ca,+1 403-210-3328diff --git a/webdriver.py b/webdriver.pyindex 71dd7d5..ece48f0 100644--- a/webdriver.py+++ b/webdriver.py@@ -21,7 +21,7 @@ class Webdriver:buf = Buffer()- async def init_browser(self, language):+ async def init_browser(self, language='ru'):self.browser = await launch(ignoreHTTPSErrors=True,headless=False,@@ -32,7 +32,7 @@ class Webdriver:# await self.page.setViewport(self.viewport)await self.page.setUserAgent(ua.random)- await self.page.setExtraHTTPHeaders({'Accept-Language': 'en-us'})+ await self.page.setExtraHTTPHeaders({'Accept-Language': language})await self.page.reload()async def _shut_browser(self):@@ -55,24 +55,22 @@ class Webdriver:raise ValueError(f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')- await self.scrape()+ await self._scrape()await self._shut_browser()- async def _enter(self, word):- await self.page.keyboard.type(word)- await self.page.keyboard.press('Enter')-async def _locate(self):- await self.page.goto(google_maps)+ async def _enter(word):+ await self.page.keyboard.type(word)+ await self.page.keyboard.press('Enter')- await self._enter(self._location)+ await self.page.goto(google_maps)+ await _enter(self._location)await self.page.waitForXPath(button_xpath)- button = (await self.page.xpath(button_xpath))[2]- await button.click()+ await (await self.page.xpath(button_xpath))[2].click()await self.page.waitForXPath(search_xpath)- await self._enter(self._keyword)+ await _enter(self._keyword)async def _jump(self):try:@@ -83,7 +81,7 @@ class Webdriver:await self._shut_browser()raise ValueError('Got invalid URL for google maps')- async def scrape(self):+ async def _scrape(self):next_button = Truewhile next_button:await self.page.waitForSelector('.section-result-content')@@ -103,11 +101,10 @@ class Webdriver:await element.click()await self.page.waitForNavigation()- sleep(1.5)+ sleep(0.7)data = self.extract(await self.page.content())- print(self.buf._data)- self.buf += data+ self.buf.store(data)# await asyncio.wait([# self.page.goto(home),@@ -116,7 +113,7 @@ class Webdriver:await self.page.goto(home)await self.page.waitForNavigation()- sleep(1.5)+ sleep(0.7)next_button = (await self.page.xpath(next_xpath))[0]await next_button.click()