Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

import packages error resolved

Madi5bebaba1d3a8
diff --git a/google-maps/__pycache__/extractor.cpython-38.pyc b/google-maps/__pycache__/extractor.cpython-38.pyc
new file mode 100644
index 0000000..7cdfbca
Binary files /dev/null and b/google-maps/__pycache__/extractor.cpython-38.pyc differ
diff --git a/google-maps/__pycache__/locators.cpython-38.pyc b/google-maps/__pycache__/locators.cpython-38.pyc
new file mode 100644
index 0000000..f1b5d98
Binary files /dev/null and b/google-maps/__pycache__/locators.cpython-38.pyc differ
diff --git a/google-maps/config.py b/google-maps/config.py
new file mode 100644
index 0000000..4ddb2eb
--- /dev/null
+++ b/google-maps/config.py
@@ -0,0 +1,8 @@
+google_maps = 'https://www.google.ru/maps'
+
+order = {
+ 'Title': None,
+ 'Address': 'place_gm_blue_24dp.png',
+ 'WebSite': 'public_gm_blue_24dp.png',
+ 'PhoneNumber': 'phone_gm_blue_24dp.png'
+}
diff --git a/google-maps/extractor.py b/google-maps/extractor.py
index 8edb576..5032da7 100644
--- a/google-maps/extractor.py
+++ b/google-maps/extractor.py
@@ -1,9 +1,112 @@
+import re
+
+from locators import *
+from config import *
+
+from bs4 import BeautifulSoup
+
import sys
+
sys.path.append('..')
from webdriver import Webdriver
+from logger_config import get_logger
+
+
+logger = get_logger('webdriver.google_maps')
class Searcher(Webdriver):
def __init__(self):
pass
+
+ async def search(self, location=None, keyword=None, url=None):
+ if location and keyword and not url:
+ logger.debug(
+ 'Working based on location `%s` and keyword `%s`', location, keyword)
+ self._location = location
+ self._keyword = keyword
+ await self._locate()
+ elif not (location and keyword) and url:
+ logger.debug('Working based on given URL `%s`', url)
+ self._url = url
+ await self._jump()
+ else:
+ await self._shut_browser()
+ raise ValueError(
+ f'Specify location and keyword. Otherwise, specify URL only. Got location `{location}`, keyword `{keyword}` and URL `{url}`')
+
+ await self._scrape()
+ await self._shut_browser()
+
+ async def _locate(self):
+ async def _enter(word):
+ await self.page.keyboard.type(word)
+ await self.page.keyboard.press('Enter')
+
+ # Go to google maps, type the location
+ await self.page.goto(google_maps)
+ await self.page.click('input')
+ await _enter(self._location)
+ await self.page.waitForXPath(button_xpath, {'visible': True})
+
+ # Click search nearby, type the keyword
+ await (await self.page.xpath(button_xpath))[2].click()
+ await self.page.waitForXPath(search_xpath)
+ await self.page.click('input')
+ await _enter(self._keyword)
+
+ await self.page.waitForXPath(result_xpath, {'visible': True})
+
+ async def _jump(self):
+ try:
+ await self.page.goto(self._url)
+ logger.debug('Validating given URL')
+ self.page.waitForXPath(check_xpath, {'visible': True})
+ except:
+ await self._shut_browser()
+ raise ValueError('Got invalid URL for google maps')
+
+ async def _scrape(self):
+ while True:
+ places = len(await self.page.xpath(result_xpath))
+ logger.debug('%s places found', places)
+
+ for i in range(places):
+ logger.debug('Gonna scrape %s out of %s', i + 1, places)
+
+ place = (await self.page.xpath(result_xpath))[i]
+ await self._do_retry(place.click, place_xpath)
+
+ data = self._extract(await self.page.content())
+ self._buf.store(data)
+
+ back = (await self.page.xpath(back_xpath))[0]
+ await self._do_retry(back.click, result_xpath)
+ # await self._do_retry(self.page.goBack, result_xpath)
+
+ next_button = (await self.page.xpath(next_xpath))[0]
+ await next_button.click()
+ sleep(1)
+
+ def _extract(self, html):
+ soup = BeautifulSoup(html, 'html.parser')
+
+ data = {}
+
+ for field, src in order.items():
+ try:
+ if src:
+ value = soup.find(
+ attrs={'src': re.compile(src)}).parent.parent.parent.text.strip()
+ else:
+ value = soup.find('h1').text.strip()
+ if 'Add' in value or 'Добавить' in value:
+ logger.debug('Got empty information %s', value)
+ value = '-'
+ except:
+ value = '-'
+ data.update({field: value})
+
+ logger.debug('Data: %s', list(data.values()))
+ return data
diff --git a/google-maps/leads.csv b/google-maps/leads.csv
new file mode 100644
index 0000000..58ce79a
--- /dev/null
+++ b/google-maps/leads.csv
@@ -0,0 +1 @@
+Title,Address,WebSite,PhoneNumber
diff --git a/google-maps/locators.py b/google-maps/locators.py
new file mode 100644
index 0000000..b8b736f
--- /dev/null
+++ b/google-maps/locators.py
@@ -0,0 +1,8 @@
+button_xpath = '//*[@class="iRxY3GoUYUY__button gm2-hairline-border section-action-chip-button"]'
+search_xpath = '//*[@role="gridcell"]'
+next_xpath = '//*[@id="n7lv7yjyC35__section-pagination-button-next"]'
+check_xpath = '//*[@src="//www.gstatic.com/images/icons/material/system_gm/2x/share_gm_blue_18dp.png"]'
+place_xpath = '//*[@class="section-hero-header-title-title GLOBAL__gm2-headline-5"]'
+result_xpath = '//*[@class="section-result-content"]'
+back_xpath = '//*[@class="section-back-to-list-button blue-link noprint"]'
+