Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
"""Find, fetch and run the Chrome the engine drives: headless, loopback DevTools port, private profile.
with Browser() as b:
page = cdp.Page(b.cdp)
An installed Google Chrome, Chromium, Edge or Brave is used first (set CHROME=/path/to/binary to
pick one). On a machine with none, `python3 scripts/engine.py start` downloads Chrome for Testing's
headless shell into .engine/, pinned below by version and SHA-256. Standard library only.
"""
import hashlib, http.client, os, platform, queue, shutil, subprocess, tempfile, threading, time, urllib.parse, urllib.request
import zipfile
from cdp import CDP, CDPError
KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HOME = os.path.join(KIT, ".engine") # everything the engine keeps: jobs, results, logs, its Chrome
CANDIDATES = {
"Darwin": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
"Linux": ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser",
"microsoft-edge", "brave-browser"],
"Windows": [r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
r"%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe",
r"%LocalAppData%\Google\Chrome\Application\chrome.exe",
r"%ProgramFiles%\Chromium\Application\chrome.exe",
r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
r"%ProgramFiles%\BraveSoftware\Brave-Browser\Application\brave.exe"],
}
# Chrome for Testing's headless shell (https://googlechromelabs.github.io/chrome-for-testing/),
# Stable channel. Each zip's SHA-256 was computed from the download on 2026-10-31. A download that
# does not match is deleted and never run. Windows on Arm runs the x64 build under emulation.
CFT_VERSION = "156.0.8078.80"
CFT_URL = "https://storage.googleapis.com/chrome-for-testing-public/{v}/{p}/chrome-headless-shell-{p}.zip"
CFT_SHA256 = {
"mac-arm64": "ea615baa2e9caf0d02d1898a6562b5f8fa9c35239707dae0f592178df5b986ac",
"mac-x64": "535d417d9034cbd11e7dacd1a71aa3d6e1b525e6d5dcd2671eb2532f01481228",
"linux64": "547dcd374e02961bf998f6046a267644f8502225c36bb78ca307dab32a0b3eb4",
"linux-arm64": "5bb6a00a4285c13303d3f72c5afdb70f144ea7a93eed462365a52b69631c9607",
"win64": "10a77466005e7f64e00f1a75fd1ea81f4a38a60093c65fea6895843b26e60a9a",
}
# What Chrome needs on a bare Debian or Ubuntu (a server, a container). apt picks the t64 variants
# (libasound2t64 and friends) on Ubuntu 24.04 by itself.
APT_LINE = ("sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 "
"libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 "
"libcairo2 libasound2")
PY = "py" if os.name == "nt" else "python3" # what the user types in the messages below
class BrowserError(RuntimeError):
pass
def make_home():
""".engine/, private to this user on macOS and Linux (results hold scraped phone numbers)."""
os.makedirs(HOME, mode=0o700, exist_ok=True)
if os.name != "nt":
os.chmod(HOME, 0o700)
def cft_platform(system=None, machine=None):
"""Chrome for Testing's name for this machine (mac-arm64, linux64, ...), or None."""
system = system or platform.system()
arch = {"x86_64": "x64", "amd64": "x64", "arm64": "arm64", "aarch64": "arm64"}.get(
(machine or platform.machine()).lower())
return {("Darwin", "arm64"): "mac-arm64", ("Darwin", "x64"): "mac-x64",
("Linux", "x64"): "linux64", ("Linux", "arm64"): "linux-arm64",
("Windows", "x64"): "win64", ("Windows", "arm64"): "win64"}.get((system, arch))
def shell_path(plat=None):
"""Where the downloaded headless shell lives (it may not exist yet)."""
plat = plat or cft_platform() or "none"
exe = "chrome-headless-shell.exe" if plat.startswith("win") else "chrome-headless-shell"
return os.path.join(HOME, "chrome", CFT_VERSION, f"chrome-headless-shell-{plat}", exe)
def find_chrome():
"""Path of a usable Chromium-based browser: CHROME, an installed one, then the downloaded shell."""
if os.environ.get("CHROME"):
return os.environ["CHROME"]
for c in CANDIDATES.get(platform.system(), []):
c = os.path.expandvars(c)
if os.sep not in c:
c = shutil.which(c) or ""
if is_snap(c): # Ubuntu's chromium: the snap's private /tmp hides DevToolsActivePort
continue
if os.path.isfile(c):
return c
return shell_path() if os.path.isfile(shell_path()) else None
def always_headless(path):
"""True for Chrome for Testing's headless shell, which cannot open a window (SCRAPER_HEADLESS=0)."""
return "headless-shell" in os.path.basename(path or "")
def is_snap(path):
"""True for a snap browser: /snap/bin/chromium, or Ubuntu's /usr/bin/chromium-browser script
that runs it."""
if path.startswith("/snap/") or os.path.realpath(path).startswith("/snap/"):
return True
try:
with open(path, "rb") as f:
head = f.read(4096)
except OSError:
return False
return head.startswith(b"#!") and b"/snap/" in head
def sweep_profiles(days=2):
"""Remove temporary Chrome profiles older than `days` left by an engine that was killed
(taskkill /F on Windows skips the cleanup). No job runs that long."""
root, cutoff = tempfile.gettempdir(), time.time() - days * 86400
try:
names = [n for n in os.listdir(root) if n.startswith("gmskit-chrome-")]
except OSError:
return
for n in names:
path = os.path.join(root, n)
try:
if os.path.getmtime(path) < cutoff:
shutil.rmtree(path, ignore_errors=True)
except OSError:
pass
def download(log=print, plat=None, opener=urllib.request.urlopen):
"""Fetch, verify and unpack the pinned headless shell into .engine/chrome/; its path."""
plat = plat or cft_platform()
if plat not in CFT_SHA256:
raise BrowserError(f"No Chrome for Testing build exists for {platform.system()} {platform.machine()}. "
"Install Chromium, then set CHROME to its path.")
url = CFT_URL.format(v=CFT_VERSION, p=plat)
root = os.path.dirname(os.path.dirname(shell_path(plat)))
make_home()
os.makedirs(root, exist_ok=True)
tmp = os.path.join(root, f"{plat}.zip.part")
log(f"▶ No Chrome found, so downloading Chrome for Testing {CFT_VERSION} (headless shell, {plat}, "
f"about 100 MB) once into .engine/ from {url}")
h = hashlib.sha256()
try:
with opener(url, timeout=60) as r, open(tmp, "wb") as f:
for chunk in iter(lambda: r.read(1 << 20), b""):
h.update(chunk)
f.write(chunk)
except (OSError, http.client.HTTPException) as e: # a cut connection raises IncompleteRead, not OSError
_remove(tmp)
raise BrowserError(f"Could not download Chrome for Testing from {url}: {e}") from e
if h.hexdigest() != CFT_SHA256[plat]:
_remove(tmp)
raise BrowserError("The Chrome download does not match the SHA-256 pinned in engine/browser.py, "
"so it was deleted and not run. Try again later, or install Google Chrome.")
staging = tempfile.mkdtemp(prefix="unpack-", dir=root)
try:
with zipfile.ZipFile(tmp) as z:
for m in z.infolist():
path = z.extract(m, staging) # zipfile drops absolute paths and ".." parts
mode = m.external_attr >> 16
if mode & 0o111 and not m.is_dir():
os.chmod(path, 0o755)
final = os.path.dirname(shell_path(plat))
shutil.rmtree(final, ignore_errors=True)
os.replace(os.path.join(staging, os.path.basename(final)), final) # a half-unpacked shell never shows
except (OSError, zipfile.BadZipFile) as e:
raise BrowserError(f"Could not unpack the Chrome download: {e}") from e
finally:
shutil.rmtree(staging, ignore_errors=True)
_remove(tmp)
log(" ✓ SHA-256 verified, unpacked")
return shell_path(plat)
def _remove(path):
try:
os.remove(path)
except OSError:
pass
def start_error(path, code, stderr):
"""The message for a Chrome that exited at once, with the apt line when Linux libraries are missing."""
missing = [l.strip() for l in stderr.splitlines() if "error while loading shared libraries" in l]
if missing:
return (f"Chrome cannot start: {missing[0]}\n Install the libraries it needs "
f"(Debian or Ubuntu):\n {APT_LINE}")
lines = [l.strip() for l in stderr.strip().splitlines() if l.strip()]
fatal = [l for l in lines if "FATAL" in l] # Chrome prints a stack trace after it
last = fatal[0] if fatal else "; ".join(lines[-3:])
return f"Chrome exited at once (code {code}): {path}" + (f": {last}" if last else "")
def split_proxy(url):
"""(the proxy as Chrome takes it, username, password): Chrome takes no user:pass in a proxy URL
and knows socks5:// only (it resolves names through a SOCKS5 proxy anyway)."""
u = urllib.parse.urlsplit(url)
host = f"[{u.hostname}]" if ":" in (u.hostname or "") else u.hostname
scheme = "socks5" if u.scheme == "socks5h" else u.scheme
return (f"{scheme}://{host}" + (f":{u.port}" if u.port else ""),
urllib.parse.unquote(u.username or ""), urllib.parse.unquote(u.password or ""))
class ProxyAuth:
"""Logs one tab in to its proxy: Chrome asks over CDP (Fetch.authRequired) when the proxy answers
407, and gets the username and password here. With Fetch on, every request of the tab waits until
it is continued, so a thread continues them. The credentials go to Chrome only, never to a log."""
def __init__(self, page, user, password):
self.page, self.user, self.password = page, user, password
self.q = page.cdp.subscribe("Fetch.requestPaused", page.sid)
page.cdp.subscribe("Fetch.authRequired", page.sid, q=self.q)
self.done, self.asked, self.refused = threading.Event(), set(), False
page.send("Fetch.enable", {"handleAuthRequests": True, "patterns": [{"urlPattern": "*"}]})
threading.Thread(target=self._run, daemon=True, name="proxy-auth").start()
def _run(self):
while not self.done.is_set():
try:
ev = self.q.get(timeout=0.5)
except queue.Empty:
continue
if not isinstance(ev, dict): # the connection died
return
rid = ev.get("requestId")
try:
if "authChallenge" not in ev:
self.page.send("Fetch.continueRequest", {"requestId": rid})
elif ev["authChallenge"].get("source") == "Proxy" and rid not in self.asked:
self.asked.add(rid) # asked again for the same request: the proxy refused them
self.page.send("Fetch.continueWithAuth", {"requestId": rid, "authChallengeResponse": {
"response": "ProvideCredentials", "username": self.user, "password": self.password}})
else:
self.refused = self.refused or rid in self.asked
self.page.send("Fetch.continueWithAuth", {"requestId": rid, "authChallengeResponse": {
"response": "CancelAuth" if rid in self.asked else "Default"}})
except CDPError: # the tab closed or the request finished meanwhile
if self.page.cdp.closed:
return
def close(self):
self.done.set()
for m in ("Fetch.requestPaused", "Fetch.authRequired"):
self.page.cdp.unsubscribe(m, self.q, self.page.sid)
class Browser:
"""A Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP. Headless by
default; headless=False runs it as a normal window off-screen."""
def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):
self.path = path or find_chrome()
if not self.path:
raise BrowserError(f"No Chrome, Chromium, Edge or Brave found. Run `{PY} scripts/engine.py start`, "
"which downloads one, or install Google Chrome.")
self._own_profile = profile is None
self.profile = profile or tempfile.mkdtemp(prefix="gmskit-chrome-")
args = [self.path, "--remote-debugging-port=0", f"--user-data-dir={self.profile}",
"--no-first-run", "--no-default-browser-check", "--disable-extensions",
"--disable-background-networking", "--disable-sync", "--mute-audio",
"--disable-component-update", "--disable-domain-reliability", "--disable-breakpad",
"--metrics-recording-only", "--no-pings", "--disable-client-side-phishing-detection",
"--disable-blink-features=AutomationControlled", "--window-size=1280,900",
f"--lang={lang}", "--disable-dev-shm-usage"]
if headless and not always_headless(self.path): # the shell is always headless
args.append("--headless=new")
elif not headless:
args.append("--window-position=-3000,-3000") # a real window, kept out of sight
if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:
args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)
self.proc, self.cdp = None, None
self._err = open(os.path.join(self.profile, "chrome-stderr.log"), "w+b")
try:
try:
port, path = self._launch(args, timeout)
except BrowserError:
# Ubuntu 23.10 and newer (AppArmor) block Chrome's sandbox for a browser that is not
# installed under /opt, such as the downloaded headless shell. Like Playwright, run it
# without; it only ever loads Google Maps.
self._err.seek(0)
if b"No usable sandbox" not in self._err.read() or "--no-sandbox" in args:
raise
port, path = self._launch(args + ["--no-sandbox"], timeout)
self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)
# Google gives a browser whose user agent says HeadlessChrome its "limited view": no review
# counts, today's hours only, no reviews. So a headless Chrome sends its own user agent with
# HeadlessChrome/ replaced by Chrome/ (same OS, same version, nothing else changed), which
# gets the full view. Page sets it in every tab before the first request. A window's
# agent has no HeadlessChrome and is left alone.
native_agent = self.cdp.send("Browser.getVersion")["userAgent"]
if "HeadlessChrome/" in native_agent:
self.cdp.user_agent = native_agent.replace("HeadlessChrome/", "Chrome/")
except BrowserError:
self.close()
raise
except Exception as e: # the DevTools websocket refused or broke: never leave Chrome running
self.close()
raise BrowserError(f"Could not connect to Chrome's DevTools port: {e}") from e
def _launch(self, args, timeout):
"""Start Chrome; (DevTools port, path) once it listens, else BrowserError (Chrome stopped)."""
self._err.seek(0)
self._err.truncate()
_remove(os.path.join(self.profile, "DevToolsActivePort"))
kw = {"stdout": subprocess.DEVNULL, "stderr": self._err, "stdin": subprocess.DEVNULL}
if os.name == "nt":
kw["creationflags"] = subprocess.CREATE_NO_WINDOW
try:
self.proc = subprocess.Popen(args + ["about:blank"], **kw)
except OSError as e:
raise BrowserError(f"Could not start Chrome ({self.path}): {e}") from e
active = os.path.join(self.profile, "DevToolsActivePort")
end = time.time() + timeout
while time.time() < end:
if self.proc.poll() is not None:
self._err.seek(0)
raise BrowserError(start_error(self.path, self.proc.returncode,
self._err.read().decode("utf-8", "replace")))
try:
with open(active, encoding="utf-8") as f:
port, path = f.read().split()
return port, path
except (OSError, ValueError):
time.sleep(0.1)
raise BrowserError(f"Chrome did not open its DevTools port within {timeout} s: {self.path}")
def version(self):
return self.cdp.send("Browser.getVersion").get("product", "")
def context(self, proxy=None):
"""A new browser context (its own cookies, and `proxy` for all its traffic); its id. Each
keyword gets one, so proxies rotate without restarting Chrome."""
params = {"disposeOnDetach": True}
if proxy:
params["proxyServer"] = split_proxy(proxy)[0]
return self.cdp.send("Target.createBrowserContext", params)["browserContextId"]
def dispose(self, context):
try:
self.cdp.send("Target.disposeBrowserContext", {"browserContextId": context})
except (CDPError, OSError):
pass
def kill(self):
"""Stop Chrome now (the job watchdog): every CDP call still waiting fails at once."""
try:
self.proc.kill()
except (OSError, AttributeError):
pass
def close(self):
if self.cdp:
try:
self.cdp.close()
except Exception: # noqa: BLE001 - closing is best effort
pass
if self.proc and self.proc.poll() is None:
self.proc.terminate()
try:
self.proc.wait(5)
except subprocess.TimeoutExpired:
self.proc.kill()
self.proc.wait(5)
self._cleanup()
def _cleanup(self):
self._err.close()
if self._own_profile:
for _ in range(10): # Windows keeps the profile locked for a moment after Chrome exits
shutil.rmtree(self.profile, ignore_errors=True)
if not os.path.exists(self.profile):
break
time.sleep(0.3)
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()