Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
"""Optional Playwright rendering for business website email and social-link enrichment.
Each job owns one browser on a dedicated asyncio thread, with at most two isolated pages at once.
The browser's HTTP proxy pins public DNS addresses and checks connected peers, including for
redirects, scripts and iframes. It preserves the existing website fetcher's private-network guard.
"""
import asyncio
import concurrent.futures
import select
import socket
import threading
import time
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
_SLOTS = threading.BoundedSemaphore(2)
_MIN_START = 3 # starting the driver with milliseconds left wastes time and trips Node timers
_CLOSE_SECONDS = 5 # each of browser.close() and pw.stop(); close took 1.6-2.3 s live, and hung 24-300 s at times
SETUP = "Install optional rendering in a virtual environment: see SETUP.md, Optional website rendering (Playwright)."
def _quiet_target_closed(loop, context):
"""Playwright settles the protocol calls of a cancelled or closed page with TargetClosedError once
the browser is gone. Nothing can await those internal futures, so asyncio would otherwise print
"Future exception was never retrieved" when the run ends. Everything else still gets reported."""
if type(context.get("exception")).__name__ != "TargetClosedError":
loop.default_exception_handler(context)
def merge_status(current, incoming):
"""A successful fetch survives later failed enrichment attempts."""
if current == "rendered" or (current == "http" and incoming != "rendered"):
return current
return incoming
class Budget:
"""One job's lazy browser, shared deadline and two-page concurrency limit.
Playwright objects stay on their owning event-loop thread. Call close() after enrichment.
"""
def __init__(self, seconds=30):
self.seconds, self.deadline, self.lock = seconds, None, threading.Lock()
self.loop = self.thread = self.session = None
self.closed = False
def fetch(self, url, public_ip):
with self.lock:
if self.closed:
return {"html": "", "status": "budget_exhausted"}
if self.deadline is None:
self.deadline = time.monotonic() + self.seconds
if self.deadline - time.monotonic() < (_MIN_START if self.loop is None else 0.5):
return {"html": "", "status": "budget_exhausted"}
if self.loop is None:
self.loop = asyncio.new_event_loop()
self.loop.set_exception_handler(_quiet_target_closed)
self.thread = threading.Thread(target=self.loop.run_forever, daemon=True)
self.thread.start()
future = asyncio.run_coroutine_threadsafe(self._fetch(url, public_ip), self.loop)
try:
return future.result()
except concurrent.futures.CancelledError: # close() ran while this page was still rendering
return {"html": "", "status": "budget_exhausted"}
async def _fetch(self, url, public_ip):
if self.session is None:
self.session = _Session(public_ip, self.deadline)
return await self.session.render(url)
def close(self):
with self.lock:
if self.closed:
return
self.closed = True
if self.loop is None:
return
future = asyncio.run_coroutine_threadsafe(self._close(), self.loop)
try:
future.result()
finally:
self.loop.call_soon_threadsafe(self.loop.stop)
self.thread.join()
self.loop.close()
async def _close(self):
# On the loop, after any fetch scheduled before close() has created the session.
if self.session is not None:
await self.session.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def check_available():
try:
from playwright.sync_api import sync_playwright # noqa: F401
except ImportError as e:
raise RuntimeError(SETUP) from e
from browser import find_chrome
path = find_chrome()
if not path:
raise RuntimeError("Website rendering needs Chrome/Chromium. Install Chrome or set CHROME to its path.")
return path
def _connect_public(host, port, public_ip, deadline):
"""Resolve once, reject mixed/private answers, and connect to a validated numeric address."""
infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
if not infos or not all(public_ip(i[4][0]) for i in infos):
raise OSError("non-public destination")
for family, kind, proto, _, address in infos:
peer = socket.socket(family, kind, proto)
try:
peer.settimeout(min(5, max(0.1, deadline - time.monotonic())))
peer.connect(address)
if not public_ip(peer.getpeername()[0]):
raise OSError("non-public peer")
return peer
except OSError:
peer.close()
raise OSError("destination unavailable")
class _ProxyHandler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def _relay(self, peer):
with peer:
while not self.server.stopped.is_set() and time.monotonic() < self.server.deadline:
ready, _, _ = select.select([self.connection, peer], [], [], 0.2)
for source in ready:
data = source.recv(65536)
if not data:
return
(peer if source is self.connection else self.connection).sendall(data)
def do_CONNECT(self):
try:
dest = urllib.parse.urlsplit("//" + self.path)
if not dest.hostname or dest.username or dest.password or dest.path:
raise ValueError("invalid destination")
peer = _connect_public(dest.hostname, dest.port or 443, self.server.public_ip, self.server.deadline)
except (OSError, ValueError, UnicodeError):
self.send_error(403, "Destination unavailable")
return
try:
self.send_response(200, "Connection established")
self.end_headers()
self._relay(peer)
except OSError:
pass
finally:
peer.close()
def do_GET(self):
try:
dest = urllib.parse.urlsplit(self.path)
if dest.scheme != "http" or not dest.hostname or dest.username or dest.password:
raise ValueError("invalid destination")
peer = _connect_public(dest.hostname, dest.port or 80, self.server.public_ip, self.server.deadline)
except (OSError, ValueError, UnicodeError):
self.send_error(403, "Destination unavailable")
return
try:
path = urllib.parse.urlunsplit(("", "", dest.path or "/", dest.query, ""))
lines = [f"{self.command} {path} HTTP/1.1", f"Host: {dest.netloc}", "Connection: close"]
lines += [f"{k}: {v}" for k, v in self.headers.items()
if k.lower() not in ("host", "connection", "proxy-connection", "proxy-authorization",
"content-length", "transfer-encoding")]
peer.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
self._relay(peer)
except OSError:
peer.close()
do_HEAD = do_GET
class _PublicProxy(ThreadingHTTPServer):
daemon_threads = True
def __init__(self, public_ip, deadline):
super().__init__(("127.0.0.1", 0), _ProxyHandler)
self.public_ip, self.deadline, self.stopped = public_ip, deadline, threading.Event()
def __enter__(self):
self.thread = threading.Thread(target=self.serve_forever, daemon=True)
self.thread.start()
return f"http://127.0.0.1:{self.server_port}"
def __exit__(self, *args):
self.stopped.set()
self.shutdown()
self.server_close()
self.thread.join(2)
class _Session:
def __init__(self, public_ip, deadline):
self.public_ip, self.deadline = public_ip, deadline
self.pages, self.start_lock = asyncio.Semaphore(2), asyncio.Lock()
self.browser = self.pw = self.proxy = None
self.tasks = set()
self.has_slot = False
self.start_error = None
async def start(self):
async with self.start_lock:
if self.browser is not None:
return
if self.start_error is not None: # one launch attempt per job; close() frees what it took
raise self.start_error
try:
if self.deadline - time.monotonic() < _MIN_START:
raise TimeoutError("browser budget exhausted")
path = check_available()
self.has_slot = _SLOTS.acquire(blocking=False)
if not self.has_slot:
raise TimeoutError("browser capacity unavailable")
from playwright.async_api import async_playwright
self.proxy = _PublicProxy(self.public_ip, self.deadline)
proxy = self.proxy.__enter__()
self.pw = await async_playwright().start()
self.browser = await self.pw.chromium.launch(
executable_path=path, headless=True, timeout=self.left(self.deadline),
chromium_sandbox=True, proxy={"server": proxy},
args=["--proxy-bypass-list=<-loopback>",
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp"])
except Exception as e:
self.start_error = e
raise
@staticmethod
def left(deadline):
remaining = deadline - time.monotonic()
if remaining < 0.5:
raise TimeoutError("browser budget exhausted")
return int((remaining - 0.2) * 1000)
async def render(self, url):
stage = "validate"
failure = lambda status: {"html": "", "status": status, "stage": stage}
task = asyncio.current_task()
self.tasks.add(task)
acquired = False
try:
parsed = urllib.parse.urlsplit(url if "://" in url else "https://" + url)
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
return failure("blocked")
url = parsed.geturl()
stage = "capacity"
remaining = self.deadline - time.monotonic()
if remaining < 0.5:
return failure("budget_exhausted")
await asyncio.wait_for(self.pages.acquire(), remaining)
acquired = True
stage = "browser_start"
await self.start()
deadline = min(self.deadline, time.monotonic() + 20)
left = lambda: self.left(deadline)
from playwright.async_api import TimeoutError as PlaywrightTimeout
context = await self.browser.new_context(service_workers="block", accept_downloads=False)
try:
async def route_request(route):
req = route.request
if (req.resource_type in ("image", "media", "font") or req.method not in ("GET", "HEAD")
or urllib.parse.urlsplit(req.url).scheme not in ("http", "https")):
await route.abort()
else:
await route.continue_()
await context.route("**/*", route_request)
page = await context.new_page()
stage = "navigation"
response = await page.goto(url, wait_until="commit", timeout=min(10000, left()))
if response and response.status >= 400:
return failure("blocked" if response.status in (401, 403, 429) else "error")
stage = "rendering"
try:
await page.wait_for_load_state("domcontentloaded", timeout=min(3000, left()))
except PlaywrightTimeout:
pass # slow third-party scripts must not discard an already usable page
try:
await page.wait_for_function(r"""() => document.body && (
document.querySelector(`a[href^="mailto:"], a[href*="instagram.com/"],
a[href*="facebook.com/"], a[href*="linkedin.com/"], a[href*="tiktok.com/"],
a[href*="youtube.com/"], a[href*="twitter.com/"], a[href*="x.com/"]`) ||
/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i.test(document.body.innerText))""",
timeout=min(2000, left()))
except PlaywrightTimeout:
pass # not every business publishes contact details
stage = "read_dom"
title = (await page.title()).lower()
if any(t in title for t in ("just a moment", "access denied", "verify you are human", "captcha")):
return failure("blocked")
return {"html": (await page.content())[:400_000], "status": "rendered", "stage": "complete"}
finally:
await context.close()
except ValueError:
return failure("blocked")
except Exception as e:
# Never log exception text: it may contain URLs, credentials or page content.
return failure("timeout" if "timeout" in type(e).__name__.lower() else "error")
finally:
if acquired:
self.pages.release()
self.tasks.discard(task)
async def close(self):
tasks = list(self.tasks)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if self.proxy is not None:
# Nothing is served while the browser closes; its relays would otherwise run on
# until the deadline.
self.proxy.__exit__()
try:
if self.browser is not None:
# Chrome's graceful close can hang for minutes after many pages; pw.stop() below
# ends the driver, which ends the browser it launched.
await asyncio.wait_for(self.browser.close(), _CLOSE_SECONDS)
except Exception:
pass
finally:
try:
if self.pw is not None:
await asyncio.wait_for(self.pw.stop(), _CLOSE_SECONDS)
except Exception:
pass
finally:
if self.has_slot:
_SLOTS.release()
def render(url, public_ip, timeout=20):
"""Standalone diagnostic render; enrichment uses one Budget across the whole job."""
with Budget(timeout) as budget:
return budget.fetch(url, public_ip)