Skip to content

Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.

purify data in yelp

Madi69846bbdf008
diff --git a/yellow-pages/extractor.py b/yellow-pages/extractor.py
index 556f8e9..43aa8be 100644
--- a/yellow-pages/extractor.py
+++ b/yellow-pages/extractor.py
@@ -72,23 +72,25 @@ class Yelp(Webdriver):
for field, cls_ in order.items():
try:
if cls_:
- value = soup.find(class_=cls_,).parent.previous_sibling().text.strip()
+ value = soup.find(class_=cls_,).parent.previous_sibling().text.replace('Business website','').replace('Phone number','').strip()
else:
value = soup.find('h1').text.strip()
- except:
+ except Exception as e:
+ print(e)
value = '-'
data.update({field: value})
-
+ logger.debug('Extracted: %s', list(data.values()))
self._buf.store(data)
while True:
await self._page.waitForXPath(results_xpath)
- places = await self._page.xpath(places_xpath)
+ places = len(await self._page.xpath(places_xpath))
+ logger.debug('%s places found', places)
- for place in places:
+ for i in range(places):
await self._page.waitForXPath(results_xpath)
- await place.click()
+ await (await self._page.xpath(places_xpath))[i].click()
await self._page.waitForXPath(inner_xpath)
sleep(1.5)
parse(await self._page.content())
diff --git a/yellow-pages/leads.csv b/yellow-pages/leads.csv
index 58ce79a..6e6f3b4 100644
--- a/yellow-pages/leads.csv
+++ b/yellow-pages/leads.csv
@@ -1 +1,36 @@
Title,Address,WebSite,PhoneNumber
+lululemon,-,-,-
+The Sports Edit,-,-,-
+Wigmore Sports,-,-,-
+Solo Sportswear,-,-,-
+Hawkesbay Sportswear,-,-,-
+Lillywhites,-,-,-
+Sports Direct,-,-,-
+Addiscombe Sports,-,-,-
+Bosco,-,-,-
+Cenci,-,-,-
+Sweaty Betty,-,-,-
+JD Sports,-,-,-
+Social Industries,-,-,-
+Lik + Neon,-,-,-
+Nike,-,-,-
+John Varvatos,-,-,-
+Sports Direct,-,-,-
+Ermenegildo Zegna,-,-,-
+Dy Mensionscreen Printers,-,-,-
+TK Maxx,-,-,-
+John Lewis & Partners,-,-,-
+n y c,-,-,-
+Aqua Uniforms Ltd,-,-,-
+G&M,-,-,-
+Solo Sportswear,-,-,-
+Primark,-,-,-
+American Sportswear,-,-,-
+Core Sportswears,-,-,-
+Hard Bodies Sportswear,-,-,-
+Hosana Sportswear,-,-,-
+Nike,-,-,-
+Jab Sportswear,-,-,-
+R & J Usa Sportswear,-,-,-
+Sports Direct,-,-,-
+Marks & Spencer,-,-,-
diff --git a/yellow-pages/pylg.log b/yellow-pages/pylg.log
index d11e51a..95620bd 100644
--- a/yellow-pages/pylg.log
+++ b/yellow-pages/pylg.log
@@ -34,3 +34,4 @@
2020-12-23 09:12:10,116 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
2020-12-23 09:12:11,587 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
2020-12-23 09:12:12,800 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': '-', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}
+2020-12-23 09:32:17,812 - WARNING | bufferization.py/store/42 | Got duplicate or empty dictionary {'Title': 'Sweaty Betty', 'Address': '-', 'WebSite': '-', 'PhoneNumber': '-'}