Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
#!/usr/bin/env python3
"""Run the kit's own engine on this machine, without Docker.
python3 scripts/engine.py start # first run on a machine without Chrome downloads one (~100 MB)
python3 scripts/engine.py status
python3 scripts/engine.py restart
python3 scripts/engine.py stop
python3 scripts/engine.py jobs # id, status, date, name, newest first
python3 scripts/engine.py delete <id>
Standard library only. The engine (engine/serve.py) serves http://localhost:8080 (SCRAPER_PORT in
.env), the same engine and API that `docker compose up -d` runs in a container. It drives an
installed Chrome, Chromium, Edge or Brave; on a machine with none, `start` downloads Chrome for
Testing's headless shell, pinned by version and SHA-256. Everything lives in .engine/ in the kit
folder; delete it to remove the engine and its results.
"""
import json, os, re, signal, socket, subprocess, sys, time, urllib.error, urllib.parse, urllib.request
for _stream in (sys.stdout, sys.stderr): # ▶ ✓ ✗ on a Windows cp1252 console
try:
_stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, ValueError):
pass
KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(KIT, "engine"))
import browser # noqa: E402
HOME = browser.HOME
SERVE = os.path.join(KIT, "engine", "serve.py")
STATE = os.path.join(HOME, "state.json")
LOCK = os.path.join(HOME, "start.lock")
LOG = os.path.join(HOME, "engine.log")
LOG_KEEP = 2_000_000 # bytes of engine.log kept across starts
MARK = "gmskit-engine/"
_NO_PROXY = urllib.request.build_opener(urllib.request.ProxyHandler({})) # HTTP_PROXY must not catch localhost
PY = "py" if os.name == "nt" else "python3" # what the user types; the README says `py` on Windows
# `status` tells the user when a newer kit is released: one request a day to GitHub's public releases
# endpoint (nothing about the user in it), cached here. It only prints; scripts/update.sh updates.
RELEASES = "https://api.github.com/repos/Mahanaicoach/google-maps-scraper-kit/releases/latest"
RELEASE_PAGE = "https://github.com/Mahanaicoach/google-maps-scraper-kit/releases/latest"
UPDATE_CACHE = os.path.join(HOME, "update-check.json")
def public_port():
"""The port scrape.py talks to (SCRAPER_PORT / SCRAPER_BASE_URL, environment or .env), read by
scrape.py's own loader so both always agree."""
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import scrape
try:
u = urllib.parse.urlparse(scrape.BASE)
port = u.port
except ValueError:
sys.exit(f"✗ SCRAPER_PORT (or SCRAPER_BASE_URL) is not a valid port: {scrape.BASE}. Set a number from 1 to 65535 in .env.")
if u.hostname not in ("localhost", "127.0.0.1", "::1"):
sys.exit(f"✗ SCRAPER_BASE_URL points at {scrape.BASE}; engine.py runs the engine on this machine only.")
return port or (443 if u.scheme == "https" else 80)
def probe(port, timeout=2):
""""ours" (this kit's engine answers, here or in Docker), "other" (something else, such as the
kit's 1.x Docker engine) or None."""
try:
with _NO_PROXY.open(f"http://127.0.0.1:{port}/api/v1/jobs", timeout=timeout) as r:
return "ours" if (r.headers.get("X-Engine") or "").startswith(MARK) else "other"
except urllib.error.HTTPError as e: # 401 without the key is still ours
e.close()
return "ours" if (e.headers.get("X-Engine") or "").startswith(MARK) else "other"
except Exception: # noqa: BLE001 - nothing listening, or not HTTP
return None
def port_free(port):
"""Nothing listens on the port and it can be bound. The connect test matters on macOS, where
SO_REUSEADDR lets 127.0.0.1:port bind while another server listens on *:port."""
try:
socket.create_connection(("127.0.0.1", port), timeout=1).close()
return False
except OSError:
pass
with socket.socket() as s:
if os.name != "nt": # like the engine's own server: connections in TIME_WAIT do not block
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.bind(("127.0.0.1", port))
return True
except OSError:
return False
def read_state():
try:
with open(STATE, encoding="utf-8") as f:
return json.load(f)
except (OSError, ValueError):
return None
def write_state(st):
tmp = STATE + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(st, f)
os.replace(tmp, STATE) # a crash mid-write never leaves half a state file
def command_line(pid):
"""The command line of `pid`; "" if the process is gone; None if it cannot be read (PowerShell
blocked or too slow), which stop must not mistake for "gone"."""
try:
if os.name == "nt":
ps = ["powershell", "-NoProfile", "-Command", # UTF-8 out: a kit path with é would not match SERVE
"[Console]::OutputEncoding=[Text.Encoding]::UTF8; "
f"(Get-CimInstance Win32_Process -Filter 'ProcessId={int(pid)}').CommandLine"]
out = subprocess.run(ps, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=20)
return None if out.returncode else out.stdout
if os.path.exists(f"/proc/{int(pid)}/cmdline"):
with open(f"/proc/{int(pid)}/cmdline", "rb") as f:
return f.read().replace(b"\0", b" ").decode("utf-8", "replace")
return subprocess.run(["ps", "-p", str(int(pid)), "-o", "command="], capture_output=True,
text=True, errors="replace", timeout=10).stdout
except (OSError, ValueError, subprocess.SubprocessError):
return None
def alive_nt(pid):
"""Windows: True while `pid` runs (OpenProcess + GetExitCodeProcess; 259 is STILL_ACTIVE)."""
import ctypes
k32 = ctypes.windll.kernel32
h = k32.OpenProcess(0x1000, False, int(pid)) # PROCESS_QUERY_LIMITED_INFORMATION
if not h:
return False
try:
code = ctypes.c_ulong()
return bool(k32.GetExitCodeProcess(h, ctypes.byref(code))) and code.value == 259
finally:
k32.CloseHandle(h)
def kill(pid):
"""Stop the engine and its Chrome. False if it could not be told whether `pid` is the engine:
after a reboot or crash the pid in state.json may belong to an unrelated process."""
cmd = command_line(pid)
if cmd is None:
return False
if os.path.normcase(SERVE) not in os.path.normcase(cmd): # Windows: c:\ and C:\ are one path
if not cmd and os.name != "nt": # the engine is gone (crash, OOM) but its Chrome may live on in its group
try:
os.killpg(pid, signal.SIGTERM)
except (ProcessLookupError, PermissionError):
pass
return True # gone, or someone else's process now: nothing of ours to stop
if os.name == "nt":
try: # Ctrl-Break to the engine's own group: it pauses its job, so the next start resumes it
os.kill(pid, signal.CTRL_BREAK_EVENT)
for _ in range(100):
if not alive_nt(pid):
return True
time.sleep(0.1)
except OSError:
pass
subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True)
return True
try:
# The engine alone: it pauses its job (which resumes on the next start) and closes its own
# Chrome. A Chrome that got the signal too would end the job before the engine could pause it.
os.kill(pid, signal.SIGTERM)
except (ProcessLookupError, PermissionError):
return True
for _ in range(150): # the engine leads its own process group, Chrome included
try:
os.waitpid(pid, os.WNOHANG) # our own child (start that timed out): reap it, or it lingers as a zombie
except ChildProcessError:
pass
try:
os.killpg(pid, 0)
except (ProcessLookupError, PermissionError):
return True
time.sleep(0.1)
try:
os.killpg(pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
return True
def tail(path, lines=15):
try:
with open(path, "rb") as f:
f.seek(max(0, os.path.getsize(path) - 8000))
return "\n".join(f.read().decode("utf-8", "replace").splitlines()[-lines:])
except OSError:
return ""
def trim_log():
"""Keep the last LOG_KEEP bytes of engine.log, so it cannot grow without bound across starts."""
try:
if os.path.getsize(LOG) > LOG_KEEP:
with open(LOG, "rb") as f:
f.seek(-LOG_KEEP, os.SEEK_END)
keep = f.read()
with open(LOG, "wb") as f:
f.write(keep)
except OSError:
pass
def busy(port):
hint = f"netstat -ano | findstr :{port}" if os.name == "nt" else f"lsof -nP -iTCP:{port} -sTCP:LISTEN"
sys.exit(f"✗ Port {port} is in use. Is the kit's Docker engine or an older copy running? Stop it with\n"
f" `docker compose stop`. Or use another port: `cp .env.example .env` (if there is no .env yet), set\n"
f" SCRAPER_PORT=8081 in .env, start again, and use that port wherever the docs say 8080.\n"
f" To see what holds {port}: {hint}")
def chrome(download):
"""A browser for the engine; downloads the pinned one only when allowed (start, never restart:
scrape.py may restart the engine unattended, and fetching code is for an explicit start)."""
path = browser.find_chrome()
if path:
return path
if not download:
sys.exit(f"✗ No Chrome found. Run: {PY} scripts/engine.py start (it downloads one)")
try:
return browser.download()
except browser.BrowserError as e:
sys.exit(f"✗ {e}")
def start(download=True):
port = public_port()
browser.make_home()
try: # one start at a time, from the first look: a second start during the first's 30 s boot
lock = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY) # would otherwise see its state file and stop it
except FileExistsError:
if probe(port) == "ours": # the other start finished, or the lock is stale and the engine is up
print(f"✓ The engine is already running: http://localhost:{port}")
return
if time.time() - os.path.getmtime(LOCK) < 600:
sys.exit("✗ Another `engine.py start` is running. Wait for it, or delete .engine/start.lock.")
os.remove(LOCK) # left by a start that crashed
lock = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
try:
seen = probe(port)
if seen == "ours":
print(f"✓ The engine is already running: http://localhost:{port}")
return
if seen == "other":
busy(port)
if read_state():
stop(quiet=True) # stale state from a crash or reboot: clean up whatever is still ours
if not port_free(port):
busy(port)
_start(port, chrome(download))
finally:
os.close(lock)
os.remove(LOCK)
def _start(port, chrome_path):
trim_log()
env = dict(os.environ, CHROME=chrome_path, SCRAPER_PORT=str(port))
with open(LOG, "ab") as out: # the engine keeps its own handle
kw = {"stdout": out, "stderr": out, "stdin": subprocess.DEVNULL, "cwd": HOME, "env": env}
if os.name == "nt":
# A hidden console, not DETACHED_PROCESS: Chrome started from a process with no console
# at all closed at its first page on Windows.
kw["creationflags"] = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP
else:
kw["start_new_session"] = True
try:
proc = subprocess.Popen([sys.executable, SERVE, "--port", str(port)], **kw)
except OSError as e:
sys.exit(f"✗ Could not start the engine: {e}")
write_state({"pid": proc.pid, "port": port, "chrome": chrome_path, "log": LOG})
deadline = time.time() + 30
while time.time() < deadline:
if probe(port) == "ours":
print(f"✓ Engine running: http://localhost:{port} (web UI and API)\n"
f" Browser: {chrome_path}\n"
f" Stop it with: {PY} scripts/engine.py stop")
return
if proc.poll() is not None:
stop(quiet=True)
sys.exit(f"✗ The engine exited at once (code {proc.returncode}). The end of {LOG}:\n{tail(LOG)}")
time.sleep(0.5)
stop(quiet=True)
sys.exit(f"✗ The engine did not answer within 30 seconds. The end of {LOG}:\n{tail(LOG)}")
def stop(quiet=False):
st = read_state()
if not st:
if not quiet:
port = public_port()
if probe(port) == "ours": # started by docker compose, not by this script
print(f"ℹ The engine on port {port} was not started by engine.py; it runs in Docker, probably.\n"
" Stop that one with: docker compose stop")
else:
print("The engine is not running.")
return
if st.get("pid") and not kill(st["pid"]):
how = "taskkill /PID {} /T /F" if os.name == "nt" else "kill {}"
sys.exit(f"✗ Could not check whether process {st['pid']} is the engine, so it was left running.\n"
f" Stop it yourself: {how.format(st['pid'])}")
try:
os.remove(STATE)
except OSError:
pass
if not quiet:
print("✓ Engine stopped. Its jobs and results stay in .engine/data.")
def restart():
if not read_state() and probe(public_port()) == "ours": # docker compose started it, not this script
sys.exit("ℹ This engine runs in Docker, not under engine.py. Restart it with: docker compose restart\n"
" (after an update: docker compose up -d --build)")
stop(quiet=True)
start(download=False)
def status():
port = public_port()
seen = probe(port)
if seen == "ours":
st = read_state() or {}
print(f"✓ Running on http://localhost:{port}" + (f" (browser: {st['chrome']})" if st.get("chrome") else ""))
update_hint()
elif seen == "other":
print(f"ℹ Port {port} answers, but not as this kit's engine (the kit's older Docker engine, probably).")
sys.exit(1)
else:
found = browser.find_chrome()
print(f"✗ Not running. Start it with: {PY} scripts/engine.py start\n Browser: " + (found or
"none found, so start would download Chrome for Testing (about 100 MB, pinned by SHA-256)"))
update_hint()
sys.exit(1)
def local_version():
"""This copy's engine version, read from engine/serve.py without importing it."""
path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "engine", "serve.py")
try:
with open(path, encoding="utf-8") as f:
m = re.search(r'^VERSION = "([^"]+)"', f.read(), re.M)
return m.group(1) if m else ""
except OSError:
return ""
def newer(latest, current):
"""True when `latest` (a tag like v2.3.0) is above `current` on its first three numbers."""
def num(v):
out = []
for x in re.sub(r"^v", "", str(v)).split(".")[:3]:
m = re.match(r"\d+", x)
out.append(int(m.group()) if m else 0)
return out + [0] * (3 - len(out))
return num(latest) > num(current)
def latest_release(now=None):
"""The latest release tag, from the day's cache or one 3 s request to GitHub; "" when unknown."""
now = time.time() if now is None else now
try:
with open(UPDATE_CACHE, encoding="utf-8") as f:
c = json.load(f)
if now - float(c.get("t", 0)) < 86400:
return str(c.get("tag", ""))
except (OSError, ValueError, TypeError):
pass
try:
req = urllib.request.Request(RELEASES, headers={"Accept": "application/vnd.github+json",
"User-Agent": "google-maps-scraper-kit"})
with urllib.request.urlopen(req, timeout=3) as r:
tag = str(json.load(r).get("tag_name", ""))
except Exception: # noqa: BLE001 - offline, rate-limited, odd answer: no notice this time
return ""
try:
os.makedirs(HOME, exist_ok=True)
with open(UPDATE_CACHE, "w", encoding="utf-8") as f:
json.dump({"t": now, "tag": tag}, f)
except OSError:
pass
return tag
def update_hint():
"""Print one line when a newer kit is released. Off with SCRAPER_NO_UPDATE_CHECK=1 (environment or .env)."""
if os.environ.get("SCRAPER_NO_UPDATE_CHECK", "").strip().lower() in ("1", "true", "yes"):
return
mine, tag = local_version(), latest_release()
if mine and re.match(r"^v?\d+\.\d+\.\d+", tag) and newer(tag, mine):
print(f"↑ Kit {tag} is out (this copy is v{mine}). Update: bash scripts/update.sh · {RELEASE_PAGE}")
JOB_ID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
def api(method, path):
"""One call to the running engine through scrape.py's opener (its port, key and no-proxy rule)."""
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import scrape
try:
return scrape.req(method, path)
except urllib.error.HTTPError as e:
body = e.read().decode("utf-8", "replace")[:200]
sys.exit(f"✗ The engine answered HTTP {e.code}: {body}")
except Exception as e: # noqa: BLE001 - not running, wrong port
sys.exit(f"✗ The engine is not reachable at {scrape.BASE} ({e}). Check: {PY} scripts/engine.py status")
def jobs():
"""The engine's job list, newest first: id, status, when, name. Names are scraped data."""
_, raw = api("GET", "/api/v1/jobs")
rows = json.loads(raw) or []
if not rows:
print("No jobs yet.")
return
print(f"{'ID':36} {'STATUS':8} {'CREATED (UTC)':16} NAME")
for j in rows:
name = "".join(c if c.isprintable() else "?" for c in str(j.get("Name", "")))[:60]
print(f"{j['ID']:36} {j.get('Status', ''):8} {str(j.get('Date', ''))[:16]:16} {name}")
def delete(jid):
if not JOB_ID.match(jid):
sys.exit(f"✗ That is not a job id (8-4-4-4-12 hex). List them with: {PY} scripts/engine.py jobs")
api("DELETE", f"/api/v1/jobs/{jid}")
print(f"✓ Deleted job {jid} and its results.")
def main():
cmds = {"start": start, "stop": stop, "status": status, "restart": restart, "jobs": jobs}
if sys.version_info < (3, 9):
sys.exit(f"✗ The engine needs Python 3.9 or newer; this is {sys.version.split()[0]}.")
if len(sys.argv) == 3 and sys.argv[1] == "delete":
return delete(sys.argv[2])
if len(sys.argv) != 2 or sys.argv[1] not in cmds:
sys.exit(f"usage: {PY} scripts/engine.py start|stop|status|restart|jobs|delete <job-id>")
cmds[sys.argv[1]]()
if __name__ == "__main__":
main()