Python script, which empowers people with no programming background to generate robust leads on a mass scale. This repo will be compiled of various versatile techniques used in lead generation.
lxml instead of bs4 used
Madi9c810d1b0304
diff --git a/raw_requests.py b/raw_requests.pyindex 0716350..d80ade3 100644--- a/raw_requests.py+++ b/raw_requests.py@@ -1,6 +1,10 @@import requestsfrom fake_useragent import UserAgent-from bs4 import BeautifulSoup+import lxml.html+from multiprocessing import Pool++urls_xpath = '//a[@class=" link__09f24__1kwXV link-color--inherit__09f24__3PYlA link-size--inherit__09f24__2Uj95"]/@href'+title_xpath = 'h1'ua = UserAgent()@@ -11,17 +15,17 @@ status = 'ok'def extract_urls(html):- soup = BeautifulSoup(html, 'lxml')- urls = soup.select('')+ tree = lxml.html.document_fromstring(html)+ urls = ['https://www.yelp.com' + url for url in tree.xpath(urls_xpath)]print(urls)def parse(html):- pass+ tree = lxml.html.document_fromstring(html)while status:r = requests.get(URL, headers={'User-Agent': ua.random})print(r, r.ok)- print(extract_urls(r.text))+ extract_urls(r.text)status = r.ok