Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
#!/usr/bin/env python3
"""Google Maps scraper CLI for lead generation: turns a keyword and a place into a CSV of the 8 lead fields.
Uses only the Python standard library (no pip installs) and drives the kit's local engine.
Single keyword:
python3 scripts/scrape.py "gyms in Miami FL" 25.7617 -80.1918 --depth 5
Auto-geocode (no coordinates needed):
python3 scripts/scrape.py "cafes in Austin TX" --city "Austin, TX" --depth 5
Batch (one job, many keywords) from a file (one keyword per line):
python3 scripts/scrape.py --keywords-file examples/queries.example.txt --city "Denver, CO"
Whole city past Google's ~120 results per search (3 x 3 map tiles, run one at a time, deduped on place_id):
python3 scripts/scrape.py "plumbers in Phoenix AZ" --city "Phoenix, AZ" --grid 3
Check homepages only for emails, not contact and about pages:
python3 scripts/scrape.py "gyms in Miami FL" --city "Miami, FL" --no-deep-emails
Up to 5 reviews per place and its star histogram (one more page load per place):
python3 scripts/scrape.py "dentists in Austin TX" --city "Austin, TX" --reviews 3
Notes:
- Required by the API: keywords, lat/lon (strings), max_time (SECONDS). This script fills them in.
- Geocoding uses OpenStreetMap Nominatim (free, no key). It allows ~1 request/sec and requires a
descriptive User-Agent (set below). Do not loop it aggressively.
"""
import argparse, csv, functools, io, ipaddress, json, math, os, re, socket, sys, threading, time
import html as _html, http.client, urllib.request, urllib.parse, urllib.error
csv.field_size_limit(2 ** 31 - 1) # a long cell (a description) can pass csv's 128 KB default
from concurrent.futures import ThreadPoolExecutor
# The status lines below use ▶ ✓ ℹ •, and every file this script writes can hold accented business
# names. Windows picks cp1252 for both, so without this the first print raises UnicodeEncodeError
# before any work happens, and a CSV of scraped names lands in cp1252 instead of UTF-8.
# errors="replace" keeps a terminal that still cannot render a glyph from killing a finished run.
for _stream in (sys.stdout, sys.stderr):
try:
_stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, ValueError): # not a reconfigurable stream (pytest capture, a pipe wrapper)
pass
ENV_KEYS = ("SCRAPER_BASE_URL", "SCRAPER_PORT", "SCRAPER_API_KEY", "SCRAPER_NO_UPDATE_CHECK")
def _env_value(v):
"""A .env value: quotes stripped, a trailing " # comment" dropped unless it sits inside the quotes
(then docker compose keeps it too). Same rule as engine/serve.py."""
v = v.strip()
if v[:1] in ("\"", "'") and v[0] in v[1:]:
return v[1:v.index(v[0], 1)]
return v.split(" #", 1)[0].strip()
def load_env_file():
"""Read SCRAPER_* from the kit's .env, the same file docker compose reads, so a remapped
port needs one edit. The last line for a key wins, as in compose, so a second SCRAPER_PORT
line lower in the file (the second-copy block) is the one both sides use. Variables already
set in the environment win over the file."""
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, ".env")
found = {}
try:
with open(path, encoding="utf-8-sig", errors="replace") as f:
for line in f:
line = line.strip()
if line.startswith("export "): # shell-style .env files
line = line[7:]
k, sep, v = line.partition("=")
if sep and k.strip() in ENV_KEYS:
found[k.strip()] = _env_value(v)
except OSError:
return
for k, v in found.items():
os.environ.setdefault(k, v)
load_env_file()
BASE = (os.environ.get("SCRAPER_BASE_URL") or f"http://localhost:{os.environ.get('SCRAPER_PORT') or '8080'}").rstrip("/")
KEY = os.environ.get("SCRAPER_API_KEY", "")
# The 8 lead fields: what you use to contact and qualify a lead.
# Everything else (geo coordinates, IDs, hours, images, review blobs) is dropped by default.
LEAD = ["title", "phone", "emails", "website", "category", "address", "review_rating", "review_count"]
UA = "google-maps-scraper-kit/2.0 (https://github.com/Mahanaicoach/google-maps-scraper-kit)"
# Business websites fetched at once by the email crawl, --socials and --tech. They are many
# different small sites, not Google, so this does not raise the rate-limit risk.
SITE_WORKERS = 16
POLL_EVERY = 3 # seconds between status checks (the engine is local, a check is cheap)
POLL_GRACE = 120 # past max_time: the engine's watchdog stops Chrome 90 s after it at the latest
POLL_CEILING = 7200 # never wait longer than 2h, whatever --max-time asked for
PY = "py" if os.name == "nt" else "python3" # what the user types; the README says `py` on Windows
RESTART = f"{PY} scripts/engine.py restart (Docker: docker compose restart)"
# Messages of a job whose Chrome never started (engine/browser.py). Retrying fails the same way, and
# Google has nothing to do with it.
CHROME_START = ("No Chrome", "Chrome cannot start", "Chrome exited at once", "Could not start Chrome",
"Chrome did not open its DevTools port", "Could not connect to Chrome's DevTools port")
BLOCKED = "unusual-traffic page" # the engine's Error when Google refused this IP
# Notes of an "ok" job whose rows are complete: only its reviews or website pass hit max_time
WARN_NOTE = re.compile(r"(?:reviews read for \d+ of \d+ places|\d+ website\(s\) not checked): ran out of time")
# Chrome's network errors for a proxy that failed. Matched as written: the engine's message also
# holds the search URL, so a keyword such as "tunnel car wash" must not read as a proxy error.
PROXY_ERROR = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_|proxy login refused")
def poll_attempts(max_time):
"""How many status checks to make before giving up on a job.
Derived from the job's own max_time, because that is how long the engine is allowed to
work: waiting any less abandons a job that is still running normally. POLL_CEILING stops an
absurd value from parking the script for a day."""
budget = min(POLL_CEILING, max(1, int(max_time or 0)) + POLL_GRACE)
return max(1, budget // POLL_EVERY)
# HTTP_PROXY from the environment must not catch the engine on this machine (the key would go to the proxy)
_API = urllib.request.build_opener(urllib.request.ProxyHandler({})) \
if urllib.parse.urlparse(BASE).hostname in ("localhost", "127.0.0.1", "::1") else urllib.request.build_opener()
def req(method, path, body=None):
headers = {"Content-Type": "application/json", "User-Agent": UA}
if KEY:
headers["X-API-Key"] = KEY
data = json.dumps(body).encode() if body is not None else None
r = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
with _API.open(r, timeout=60) as resp:
return resp.status, resp.read()
def geocode(place, with_bbox=False):
"""City/place name -> ('lat','lon') strings via Nominatim, or None.
with_bbox=True returns ('lat','lon', [south, north, west, east]) instead."""
q = urllib.parse.urlencode({"format": "json", "limit": 1, "q": place})
url = f"https://nominatim.openstreetmap.org/search?{q}"
r = urllib.request.Request(url, headers={"User-Agent": UA})
try:
with urllib.request.urlopen(r, timeout=30) as resp:
hits = json.loads(resp.read())
time.sleep(1) # respect Nominatim's ~1 req/sec policy
if hits:
ll = (str(hits[0]["lat"]), str(hits[0]["lon"]))
return ll + (hits[0].get("boundingbox"),) if with_bbox else ll
except Exception as e:
print(f" (geocoding failed: {e})", file=sys.stderr)
return None
def grid_cells(bbox, n):
"""Split a Nominatim bbox [south, north, west, east] into n x n cells -> [{lat, lon, radius, zoom}].
radius (m) = 1.15 x half the cell diagonal (equirectangular), at least 1000.
zoom = the Maps zoom whose ~1000px viewport is about as wide as the cell (11..17), so each
search is launched looking at its own tile instead of a city-wide or block-wide view."""
s, nn, w, e = (float(x) for x in bbox)
dlat, dlon = (nn - s) / n, (e - w) / n
m_per_deg = 6371000 * math.pi / 180
cells = []
for i in range(n):
for j in range(n):
lat, lon = s + (i + 0.5) * dlat, w + (j + 0.5) * dlon
cos = math.cos(math.radians(lat))
width = max(1.0, dlon * m_per_deg * cos)
diag = math.hypot(dlat * m_per_deg, width)
zoom = round(math.log2(156543.03 * cos * 1000 / width)) # 156543 m/px at zoom 0
cells.append({"lat": f"{lat:.6f}", "lon": f"{lon:.6f}",
"radius": max(1000, math.ceil(1.15 * diag / 2)),
"zoom": min(17, max(11, zoom)),
"_bbox": [s + i * dlat, s + (i + 1) * dlat, w + j * dlon, w + (j + 1) * dlon]})
return cells
# A tile that returns this many rows for one keyword hit Google's cap (about 120 per search), so
# the tile holds more than one search can show. --deep-scan splits such a tile into 2 x 2 sub-tiles.
SATURATED = 100
def engine_cell(cell):
"""The cell without the script's private `_` keys, which never reach the engine."""
return {k: v for k, v in cell.items() if not k.startswith("_")}
def in_bbox(row, bbox, margin=0.10):
"""True if the row's latitude/longitude sit inside bbox [south, north, west, east] grown by
`margin` of its size on each side. Rows with no usable coordinates are kept (True)."""
try:
lat, lon = float(row.get("latitude")), float(row.get("longitude"))
except (TypeError, ValueError):
return True
s, n, w, e = (float(x) for x in bbox)
my, mx = (n - s) * margin, (e - w) * margin
return s - my <= lat <= n + my and w - mx <= lon <= e + mx
def _keys(r, weak=False):
"""The identities a row carries: place_id, cid, link, name+address. With weak=True also
name+phone and name+website, which a --fields results file may be the only thing to match on.
They are weak because two branches of a chain share a website and sometimes a phone, so they
never dedupe rows within a run, only match an exclusion list."""
ks = [f"{k}:{r[k]}" for k in ("place_id", "cid", "link") if r.get(k)]
t = r.get("title")
if t:
ks.append(f"ta:{t}|{r.get('address', '')}")
if weak:
ks += [f"t{c[0]}:{t}|{r[c]}" for c in ("phone", "website") if r.get(c)]
return ks
def dedupe(rows, exclude=()):
"""Keep the first row per place; drop rows already present in `exclude` (an earlier run). A name
key (the t… ones) never joins two rows whose place_ids differ: branches and service-area listings
share a name and often a blank address."""
seen, out = {}, []
def add(r, weak):
for k in _keys(r, weak):
seen.setdefault(k, set()).add(r.get("place_id", ""))
def known(r, k):
ids, pid = seen.get(k), r.get("place_id", "")
return ids is not None and (not k.startswith("t") or not pid or "" in ids or pid in ids)
for r in exclude:
add(r, True)
for r in rows:
if not any(known(r, k) for k in _keys(r, weak=True)):
add(r, False); out.append(r)
return out
# What Excel or Sheets would run as a formula (a phone number is not), also behind quotes the guard
# added, so "'=x" scraped as is gets one more and unguard() stays exact. engine/jobs.py has the same.
FORMULA = re.compile(r"'*(?:[=@\t\r]|[+-](?![\d\s().-]*(?:(?i:ext\.?|x)\s*\d+)?$))") # "+1 … ext. 12" is a phone too
def csv_safe(v):
"""A cell with a leading ' when a spreadsheet would run it as a formula ("=HYPERLINK(...)" in a
business name or review); "+1 512-555-0100" is left alone. The engine's CSV download does the same."""
s = "" if v is None else str(v)
return "'" + s if FORMULA.match(s) else s
def unguard(rows):
"""Undo csv_safe on rows read from a CSV (the engine's download, an earlier file), so filters,
--exclude and --json see the scraped text."""
for r in rows:
for k, v in r.items():
if isinstance(v, str) and len(v) > 1 and v[0] == "'" and FORMULA.match(v[1:]):
r[k] = v[1:]
return rows
def load_rows(path):
"""Rows from an earlier results file, .csv or .json."""
with open(path, encoding="utf-8-sig", newline="") as f: # -sig: also reads the kit's own CSVs
rows = json.load(f) if path.lower().endswith(".json") else unguard(list(csv.DictReader(f)))
if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):
raise ValueError(f"{path} is not a results file (expected a list of rows)")
return rows
FAILED_RETRY_WAIT = 60 # seconds before the one retry of a "failed" job (usually a passing error)
def shown(v, n=60):
"""Third-party text (a scraped value, the engine's Error) as one printable line of at most n
characters, for what the script prints into an agent's context."""
return "".join(c if c.isprintable() else " " for c in str(v))[:n]
class JobFailed(RuntimeError):
pass
class Partial(RuntimeError):
"""A job that ended early with some rows (job_id, rows): the caller keeps them and counts the
tile or place as incomplete."""
def __init__(self, message, job_id=None, rows=()):
super().__init__(message)
self.job_id, self.rows = job_id, list(rows)
class Blocked(Partial):
"""Google answered with its unusual-traffic page. More searches from this IP only extend the
block, so the grid and city loops stop."""
CURRENT_JOB = None # the job being polled, for the Ctrl-C message
SALVAGE = {"rows": [], "job": None} # what Ctrl-C can still save: the rows of the tiles and cities done so far
WARNINGS = [] # engine notes on "ok" jobs (reviews cut short): printed once more in the final summary
def run_job(body):
"""Create one engine job, poll it, download it -> (job_id, raw CSV rows). A job the engine marks
"failed" is retried once after a pause, unless its Error says a retry cannot help (Chrome would
not start, the proxy failed, Google refused this IP). Raises RuntimeError."""
for attempt in (1, 2):
try:
return _run_job_once(body)
except JobFailed as e:
reason = e.args[1].strip().rstrip(".")
said = f" The engine says: {reason}." if reason else " The engine gave no reason."
if reason.startswith(CHROME_START):
raise RuntimeError(f"\n✗ Job failed: the engine could not start Chrome.{said}\n"
f" Fix that and scrape again. If it fails the same way, restart the "
f"engine: {RESTART}")
if PROXY_ERROR.search(reason):
raise RuntimeError(f"\n✗ Job failed: the proxy did not work.{said}\n"
" Check its host, port, username and password (or that it allows your IP "
"address).")
if BLOCKED in reason:
raise Blocked(f"\n✗ Job failed.{said}")
if attempt == 2:
raise RuntimeError(f"\n✗ Job failed twice.{said}\n The engine's log has more: "
".engine/engine.log (Docker: docker compose logs)")
print(f"\n⚠ Job failed.{said} Waiting {FAILED_RETRY_WAIT} s, then retrying once…")
time.sleep(FAILED_RETRY_WAIT)
def _get(path, tries=3):
"""GET with retries 3 s apart: a poll or a download must survive a passing 502 or timeout."""
for attempt in range(tries):
try:
return req("GET", path)
except urllib.error.HTTPError as e:
if e.code < 500 or attempt == tries - 1:
raise
except OSError:
if attempt == tries - 1:
raise
time.sleep(3)
def progress_text(job):
"""", keyword 2/3, 41 places, 50%" from the job's Progress (what this engine reports), or ""."""
p = job.get("Progress")
if not isinstance(p, dict) or not p.get("keywords_total"):
return ""
total = p["keywords_total"]
reviews = f"reading reviews {p['reviews_done']}/{p['reviews_total']}" if p.get("reviews_total") else "reading reviews"
where = {"reviews": reviews, "websites": "checking websites"}.get(
p.get("stage"), f"keyword {min(p.get('keywords_done', 0) + 1, total)}/{total}")
return f", {where}, {p.get('places', 0)} places, {p.get('percent', 0)}%"
ENGINE_RESTART_TRIES = 20 # ~1 min of polls: a restarted engine resumes the job, so wait for it
def _run_job_once(body):
global CURRENT_JOB
try:
_, raw = req("POST", "/api/v1/jobs", body)
except urllib.error.HTTPError as e:
raise RuntimeError(f"✗ Create failed: HTTP {e.code} — {e.read().decode()[:200]}")
except OSError as e:
raise RuntimeError(f"✗ Could not reach the engine at {BASE} to create the job ({e}).")
job_id = json.loads(raw).get("id")
if not isinstance(job_id, str) or not re.fullmatch(r"[A-Za-z0-9-]{1,64}", job_id): # it goes into URLs and file names
raise RuntimeError("✗ No job id returned.")
print(f" job id: {job_id}")
CURRENT_JOB = job_id
try:
result = _poll(body, job_id)
except OSError as e: # the engine went away (restart, crash) or the job was deleted mid-poll
CURRENT_JOB = None
raise RuntimeError(f"\n✗ Lost the engine while job {job_id} ran ({e}). When it starts again it resumes "
f"the job from its last finished keyword; its rows: {BASE}/api/v1/jobs/{job_id}/download")
except Exception:
CURRENT_JOB = None
raise
# Not a finally: on Ctrl-C (not an Exception) the id must survive for the message in __main__.
CURRENT_JOB = None
return result
def _poll(body, job_id):
attempts = poll_attempts(body.get("max_time"))
print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")
status = None
for i in range(attempts):
_, raw = _get(f"/api/v1/jobs/{job_id}", ENGINE_RESTART_TRIES)
job = json.loads(raw)
# Error: why the job failed, or why an "ok" job stopped early (max_time, a Google block).
status, error = job.get("Status"), shown(job.get("Error") or "", 600)
progress = progress_text(job) if status in ("working", "pending") else ""
print(f"\r status: {str(status):<10} (attempt {i + 1}{progress})", end="", flush=True)
if status == "ok":
print()
if BLOCKED in error: # the rows before the block are kept; searching on would extend it
raise Blocked(f"⚠ The engine says: {error}", job_id, download_rows(job_id))
if WARN_NOTE.sub("", error).strip(" ;,."): # stopped at max_time, or keywords failed: incomplete
raise Partial(f"⚠ The engine says: {error}", job_id, download_rows(job_id))
if error: # only the reviews pass ran out of time: the rows are complete, the reviews of the rest are not
WARNINGS.append(f"The engine says: {error}. The rows are complete, but the places after that "
f"have no reviews; raise --max-time for the rest.")
return job_id, download_rows(job_id)
if status == "failed":
print()
try: # the engine keeps the rows of a job it was stopped in the middle of
rows = download_rows(job_id)
except Exception:
rows = []
if rows:
raise Partial(f"⚠ The job failed part way ({error or 'no reason given'}); keeping its "
f"{len(rows)} rows.", job_id, rows)
raise JobFailed(job_id, error)
time.sleep(POLL_EVERY)
# Still pending or working: the engine may have run an earlier job first. It keeps this one, so
# take whatever it has written.
rows = download_rows_or_none(job_id)
if not rows:
raise RuntimeError(
f"\n✗ Gave up waiting after ~{attempts * POLL_EVERY // 60} min with no rows yet (status "
f"\"{status}\"). The engine keeps the job, so check it later with\n"
f" python3 scripts/engine.py jobs\n"
f" and once it is \"ok\", download its CSV from the web UI at {BASE}")
raise Partial(f"\n⚠ Job still \"{status}\" after ~{attempts * POLL_EVERY // 60} min; keeping the {len(rows)} "
f"rows it has written. It keeps running in the engine: {BASE}/api/v1/jobs/{job_id}", job_id, rows)
def download_rows_or_none(job_id):
"""The rows so far, or None when the engine has written none (404 until its first places)."""
try:
return download_rows(job_id)
except urllib.error.HTTPError as e:
if e.code == 404:
return None
raise
def download_rows(job_id):
_, raw = _get(f"/api/v1/jobs/{job_id}/download")
return unguard(list(csv.DictReader(io.StringIO(raw.decode("utf-8", "replace"), newline=""))))
def collect_keywords(a):
kws = []
if a.keyword:
kws.append(a.keyword)
kws.extend(a.also or [])
if a.keywords_file:
try:
with open(a.keywords_file, encoding="utf-8-sig") as f:
lines = [line.strip() for line in f]
kws.extend(line for line in lines if line and not line.startswith("#"))
except (OSError, UnicodeDecodeError) as e:
sys.exit(f"✗ Cannot read --keywords-file {a.keywords_file}: {getattr(e, 'strerror', e)}")
return list(dict.fromkeys(kws)) # de-dupe, keep order
# ── Optional social enrichment (Instagram / Facebook / LinkedIn) ──────────────
# Pure code (HTTP + regex) → ZERO LLM tokens. Visits each business website once.
SOCIAL_RE = {
"instagram": re.compile(r"https?://(?:www\.)?instagram\.com/([A-Za-z0-9_.]+)", re.I),
"facebook": re.compile(r"https?://(?:www\.|m\.|web\.)?facebook\.com/([A-Za-z0-9_.\-]+)", re.I),
"linkedin": re.compile(r"https?://(?:[a-z]{2,3}\.)?linkedin\.com/(?:company|in|school)/([A-Za-z0-9_.\-%]+)", re.I),
"tiktok": re.compile(r"https?://(?:www\.)?tiktok\.com/@([A-Za-z0-9_.]+)", re.I),
"youtube": re.compile(r"https?://(?:www\.)?youtube\.com/(?:@|channel/|c/|user/)([A-Za-z0-9_.\-]+)", re.I),
"x": re.compile(r"https?://(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})\b", re.I),
}
SOCIAL_FIELDS = list(SOCIAL_RE)
_SKIP_HANDLE = {"", "home", "pages", "people", "help", "about", "policies", "policy",
"legal", "tos", "privacy", "settings", "sharer", "tr", "profile.php",
"plugins", "dialog", "intent", "login", "share.php", "permalink.php",
"p", "reel", "reels", "explore", "stories", "tv", "watch", "events",
"groups", "marketplace", "gaming", "photo", "hashtag", "search", "share", "i"}
# Website builders and booking or delivery platforms link their own profiles from the sites they host:
# those are not the business's. Kept the same in engine/jobs.py and scripts/scrape.py (a test checks).
_SKIP_HANDLE |= {"squarespace", "wix", "wixsite", "shopify", "wordpress", "wordpressdotcom", "godaddy", "weebly",
"webflow", "hubspot", "mailchimp", "yelp", "tripadvisor", "opentable", "doordash", "ubereats", "grubhub",
"toasttab", "squareup"}
_NAT64 = ipaddress.ip_network("64:ff9b::/96")
def _is_public_ip(ip):
"""Public, and not an IPv6 form that wraps a private IPv4 (::ffff:10.0.0.1, 64:ff9b::7f00:1
through a NAT64, ::127.0.0.1). Same rule as engine/jobs.py."""
try:
a = ipaddress.ip_address(ip.split("%")[0])
except ValueError:
return False
if a.version == 6:
inner = a.ipv4_mapped or a.sixtofour
if inner is None and (a in _NAT64 or 1 < int(a) < 1 << 32):
inner = ipaddress.ip_address(int(a) & 0xFFFFFFFF)
if inner is not None:
a = inner
return a.is_global and not (a.is_multicast or getattr(a, "is_site_local", False))
@functools.lru_cache(maxsize=1024)
def _is_public_host(host):
"""True only if every address the host resolves to is public. Websites come from third-party
listings, so one could point a fetch at this machine or its network instead (the engine API on
localhost:8080, a router admin page, a cloud metadata endpoint)."""
try:
infos = socket.getaddrinfo(host, None)
except (OSError, UnicodeError):
return False
return bool(infos) and all(_is_public_ip(i[4][0]) for i in infos)
SITE_HARD = 30 # seconds a website request may take in all, then its socket is closed
_FETCH = threading.local() # per thread: the hard-stop timer of the connection in progress
def _public_connection(address, *args, **kwargs):
"""socket.create_connection that refuses a private peer. The connect resolves the host again,
so a DNS answer that changed after _is_public_host (rebinding) is caught here, before a byte
of the request is sent. The socket also gets a hard deadline: its timeout is per read, so a
website that drips a byte at a time would otherwise hold a crawl thread for days."""
sock = socket.create_connection(address, *args, **kwargs)
peer = sock.getpeername()[0]
if not _is_public_ip(peer):
sock.close()
raise OSError(f"refused non-public address {peer}")
return sock
def _hard_stop(conn):
"""Arm the fetch's deadline (SITE_HARD s from _fetch_window's start, so redirect hops share it);
at the deadline the socket is shut down from a timer thread, which wakes a read blocked on a
dripping website. _fetch_window() disarms it."""
sock = conn.sock # taken now: urllib sends Connection: close, so the connection drops its reference
# once the response starts; the response's file keeps this same socket object
def kill():
try:
sock.shutdown(socket.SHUT_RDWR) # wakes a read blocked in the fetch thread with EOF; the fetch
except OSError: # thread closes the socket itself (close() here would wait on its buffer lock)
pass
_cancel_hard_stop()
left = getattr(_FETCH, "deadline", None)
t = threading.Timer(max(0.1, left - time.monotonic()) if left else SITE_HARD, kill)
t.daemon = True
t.start()
_FETCH.timer = t
class _fetch_window:
"""`with _fetch_window():` around one website or DNS request: one SITE_HARD deadline for all its
connections (redirect hops too), and the timer disarmed on every exit."""
def __enter__(self):
_FETCH.deadline = time.monotonic() + SITE_HARD
def __exit__(self, *exc):
_cancel_hard_stop()
_FETCH.deadline = None
def _cancel_hard_stop():
t = getattr(_FETCH, "timer", None)
if t is not None:
t.cancel()
_FETCH.timer = None
def _read_capped(resp, cap, seconds):
"""Up to cap bytes of a response, for at most about `seconds`: read1 returns what has arrived
(read(n) would block for all n bytes), and a connection the hard stop cut keeps what came."""
out, end = bytearray(), time.monotonic() + seconds
while len(out) < cap and time.monotonic() < end:
try:
chunk = resp.read1(min(1 << 14, cap - len(out)))
except (OSError, ValueError, http.client.HTTPException):
break
if not chunk:
break
out += chunk
return bytes(out)
def _ascii_url(url):
"""The URL with a non-ASCII path or query percent-encoded (http.client refuses to send them raw):
a site at /контакты was silently skipped."""
try:
p = urllib.parse.urlsplit(url)
if p.path.isascii() and p.query.isascii():
return url
return urllib.parse.urlunsplit((p.scheme, p.netloc, urllib.parse.quote(p.path, safe="/%:@&=+$,;~!*'()"),
urllib.parse.quote(p.query, safe="=&%+/?:@;,!*'()$"), ""))
except ValueError:
return url
class _PublicPeer:
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if not _PROXIED: # behind a proxy the socket peer is the proxy, often on a private address
self._create_connection = _public_connection
def connect(self):
super().connect()
_hard_stop(self)
class _PublicHTTPConnection(_PublicPeer, http.client.HTTPConnection):
pass
class _PublicHTTPSConnection(_PublicPeer, http.client.HTTPSConnection):
pass
class _PublicHTTPHandler(urllib.request.HTTPHandler):
def http_open(self, req):
return self.do_open(_PublicHTTPConnection, req)
class _PublicHTTPSHandler(urllib.request.HTTPSHandler):
def https_open(self, req):
return self.do_open(_PublicHTTPSConnection, req, context=self._context)
class _PublicRedirectsOnly(urllib.request.HTTPRedirectHandler):
"""A public site may not redirect the fetch to a private address either."""
def redirect_request(self, req, fp, code, msg, headers, newurl):
if urllib.parse.urlparse(newurl).scheme not in ("http", "https"):
return None # urllib's ftp handler would connect past the peer check
if not _is_public_host(urllib.parse.urlparse(newurl).hostname or ""):
return None # urllib then raises HTTPError, which _fetch_html turns into ""
return super().redirect_request(req, fp, code, msg, headers, newurl)
# Behind a proxy from the environment only the resolve-time check applies (see _PublicPeer); the
# hard stop applies either way.
def env_proxied():
return any(k != "no" for k in urllib.request.getproxies()) # NO_PROXY alone is no proxy
_PROXIED = env_proxied()
_OPENER = urllib.request.build_opener(_PublicRedirectsOnly, _PublicHTTPHandler, _PublicHTTPSHandler)
def _fetch_html(url, timeout=10):
if not url:
return ""
if not url.startswith(("http://", "https://")):
url = "http://" + url
try:
if not _is_public_host(urllib.parse.urlparse(url).hostname or ""):
return ""
r = urllib.request.Request(_ascii_url(url), headers={
"User-Agent": "Mozilla/5.0 (compatible; " + UA + ")", "Accept": "text/html"})
with _fetch_window(), _OPENER.open(r, timeout=timeout) as resp:
return _read_capped(resp, 400_000, 2 * timeout).decode("utf-8", "replace")
except Exception:
return ""
@functools.lru_cache(maxsize=256) # ponytail: homepages only (up to 400 KB each); evicted pages just refetch
def _fetch_cached(url):
"""Homepage fetch shared by the email crawl, --socials and --tech (one fetch per site)."""
return _fetch_html(url, timeout=8)
_BROWSER_BUDGET = None
_BROWSER_LOCK = threading.Lock()
def _web():
"""The engine's optional website_browser module (standard library until it renders)."""
engine_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "engine")
if engine_dir not in sys.path:
sys.path.insert(0, engine_dir)
import website_browser
return website_browser
@functools.lru_cache(maxsize=128)
def _render_site(url):
"""Rendered homepages shared by email and social extraction during this CLI run."""
global _BROWSER_BUDGET
with _BROWSER_LOCK:
if _BROWSER_BUDGET is None:
_BROWSER_BUDGET = _web().Budget()
budget = _BROWSER_BUDGET
return budget.fetch(url, _is_public_ip)
def _set_fetch_status(r, status):
r["website_fetch_status"] = _web().merge_status(r.get("website_fetch_status"), status)
def _find_socials(html):
out = {k: "" for k in SOCIAL_FIELDS}
for plat, rx in SOCIAL_RE.items():
for m in rx.finditer(html or ""):
h = m.group(1).lower()
if h in _SKIP_HANDLE:
continue
if plat == "facebook" and (h.isdigit() or len(h) < 3): # skip junk like /2008
continue
out[plat] = m.group(0).rstrip('"\'/').replace("\\", "")
break
return out
def _each_site(todo, work, label, workers=None):
"""Run work(row) over the rows, SITE_WORKERS at a time, with one progress line. One row that
blows up (a malformed website, an odd page) is skipped; it never aborts the run."""
done, failed = 0, 0
def safe(r):
try:
work(r)
except Exception:
return 1
return 0
with ThreadPoolExecutor(max_workers=workers or SITE_WORKERS) as ex:
for f in ex.map(safe, todo):
done, failed = done + 1, failed + f
print(f"\r {label}: {done}/{len(todo)} checked", end="", flush=True)
print(f" ({failed} skipped after an error)" if failed else "")
def enrich_sites(results, find, fields, label, workers=None, browser_sites=False):
"""Finish the HTTP pass before browser retries, so retries never occupy HTTP worker slots."""
for r in results:
for k in fields:
r.setdefault(k, "")
def work(r):
page = _fetch_cached(r["website"])
found = find(page)
if browser_sites:
r.setdefault("website_fetch_status", "http" if page else "error")
r.update(found)
todo = [r for r in results if r.get("website")]
_each_site(todo, work, label, workers)
if browser_sites:
def retry(r):
rendered = _render_site(r["website"])
_set_fetch_status(r, rendered["status"])
r.update({k: v for k, v in find(rendered["html"]).items() if v})
_each_site([r for r in todo if not any(r.get(k) for k in fields)],
retry, "browser " + label, workers=2)
# ── Optional ad and tech signals (--tech): who already spends on marketing ──────
# Regexes from OpenLeadFinder's website layer. They run on the homepage already fetched above.
TECH_RE = {
"fb_pixel": re.compile(r"connect\.facebook\.net/[^\"']{0,300}fbevents\.js|fbq\(", re.I),
"google_ads_tag": re.compile(r"googleadservices\.com|AW-\d{6,}|googleads\.g\.doubleclick\.net", re.I),
"analytics": re.compile(r"googletagmanager\.com/gtag|gtag\(|UA-\d{4,}|G-[A-Z0-9]{6,}", re.I),
"bing_ads": re.compile(r"bat\.bing\.com|window\.uetq", re.I),
"linkedin_insight": re.compile(r"snap\.licdn\.com|_linkedin_partner_id", re.I),
"tiktok_pixel": re.compile(r"analytics\.tiktok\.com|ttq\.load\(", re.I),
}
CMS_RE = [
("wordpress", re.compile(r"/wp-content/|/wp-includes/", re.I)),
("shopify", re.compile(r"cdn\.shopify\.com|shopify\.theme", re.I)),
("wix", re.compile(r"static\.wixstatic\.com|_wixcss|wix\.com", re.I)),
("squarespace", re.compile(r"static1\.squarespace|squarespace\.com", re.I)),
("webflow", re.compile(r"webflow\.com", re.I)),
]
VIEWPORT_RE = re.compile(r"<meta[^>]{1,500}name=[\"']viewport[\"']", re.I)
TECH_FIELDS = list(TECH_RE) + ["cms", "mobile_friendly"]
def _find_tech(html):
"""yes/no per signal, the site platform, and mobile support. Blank when the site did not load,
so "unknown" never reads as "no pixel"."""
if not html:
return {k: "" for k in TECH_FIELDS}
yn = lambda hit: "yes" if hit else "no"
out = {k: yn(rx.search(html)) for k, rx in TECH_RE.items()}
out["cms"] = next((name for name, rx in CMS_RE if rx.search(html)), "")
out["mobile_friendly"] = yn(VIEWPORT_RE.search(html))
return out
def _num(v, kind=float):
try:
return kind(str(v).replace(",", ""))
except ValueError:
return 0
def intent_score(r):
"""0-100 "already spends on marketing" score: ad pixels dominate, an established, reachable
business adds a little. A heuristic, adapted from OpenLeadFinder without its live ad checks."""
s = 35 * (r.get("fb_pixel") == "yes") + 25 * (r.get("google_ads_tag") == "yes")
s += 10 * (r.get("analytics") == "yes")
reviews = _num(r.get("review_count"), int)
s += 15 if reviews >= 50 else 8 if reviews >= 10 else 0
s += 10 * (_num(r.get("review_rating")) >= 4.0)
s += 5 * bool(r.get("emails"))
return s
# ── Email crawl: each business website's homepage, then its contact/about pages ──
# The lookbehind pins each match attempt to the start of a run of local-part characters: without
# it the scan is quadratic, and a 60 KB run of "a" (a minified blob) took a minute per page.
_LOCAL = r"[A-Za-z0-9._%+-]"
EMAIL_RE = re.compile(rf"(?<!{_LOCAL}){_LOCAL}+@[A-Za-z0-9.-]+\.[A-Za-z]{{2,}}")
MAILTO_RE = re.compile(r"mailto:([^\"'>\s?]+)", re.I)
# Bounded quantifiers: a page of repeated "<a " made the unbounded form quadratic (12 s per 100 KB)
HREF_RE = re.compile(r"<a\s[^>]{0,1000}?href\s*=\s*[\"']([^\"']{1,2000})[\"'][^>]{0,1000}>(.{0,5000}?)</a>", re.I | re.S)
_JUNK_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".avif", ".css", ".js")
_JUNK_DOMAIN = ("example", "sentry", "wixpress", "wix.com", "godaddy", "squarespace", "wordpress",
"domain.com", "email.com", "yourdomain", "mysite.com", "schema.org", "w3.org", "sentry.io",
# link-in-bio pages and website agencies: their support address is not the business's
"linktr.ee", "linkin.bio", "beacons.ai", "officite")
_JUNK_LOCAL = {"name", "email", "user", "you", "someone", "example", "username",
"youremail", "yourname", "firstname", "lastname", "sample"}
# A JSON-escaped <, >, ", ' or & glued to the address (">[email protected]" in page source),
# which the regex reads as part of the local part: [email protected].
_ESCAPE_PREFIX = re.compile(r"^(?:u00(?:3c|3e|22|27|26))+")
def _host(url):
if not url.startswith(("http://", "https://")):
url = "http://" + url
try:
h = (urllib.parse.urlparse(url).hostname or "").lower()
except ValueError: # "http://[bad": a malformed website value from a listing
return ""
return h[4:] if h.startswith("www.") else h
# Google Business Profile links often carry campaign tags (?utm_source=gbp&utm_medium=organic),
# about one website in four. They bloat the column and hide that two rows share one site.
_TRACKING_PARAMS = ("utm_", "gclid", "fbclid", "msclkid", "mc_cid", "mc_eid")
def clean_website(url):
"""Drop tracking parameters and the fragment. Other parameters stay: some store pages need
them (store?storeid=...)."""
if not url:
return ""
try:
p = urllib.parse.urlsplit(url)
except ValueError: # malformed: leave it as the listing had it
return url
q = urllib.parse.parse_qsl(p.query, keep_blank_values=True)
kept = [(k, v) for k, v in q if not k.lower().startswith(_TRACKING_PARAMS)]
frag = p.fragment if p.fragment.startswith(("/", "!")) else "" # keep hash-routed store pages
if len(kept) == len(q) and frag == p.fragment:
return url # nothing to drop: never re-encode a query the site chose
query = p.query if len(kept) == len(q) else urllib.parse.urlencode(kept)
return urllib.parse.urlunsplit(p._replace(query=query, fragment=frag))
def is_real_email(e):
"""False for anything not worth sending to: a template placeholder ([email protected],
[email protected]), a vendor/boilerplate domain, or an image path that parsed as an address."""
local, _, dom = e.rpartition("@")
if not EMAIL_RE.fullmatch(e) or e.endswith(_JUNK_EXT):
return False
return local not in _JUNK_LOCAL and not junk_domain(dom)
def junk_domain(dom, junk=_JUNK_DOMAIN):
"""An entry with a dot is a domain and its subdomains, one without is a whole label: sentry.io
and [email protected] are junk, sentryinsurance.com and myemail.com are not. engine/jobs.py has the same."""
labels = dom.split(".")
return any(dom == j or dom.endswith("." + j) if "." in j else j in labels for j in junk)
# "name [at] domain [dot] com", "name (at) domain dot com" and friends, adapted from OpenLeadFinder.
# The "at" must be bracketed and the domain must end in a known TLD: a bare " at " also matched
# prose such as "meet at noon dot com".
_AT = r"(?:\[\s*at\s*\]|\(\s*at\s*\)|\{\s*at\s*\})"
_DOT = r"(?:\[\s*dot\s*\]|\(\s*dot\s*\)|\{\s*dot\s*\}|\s+dot\s+)"
_TLD = (r"(?:com|net|org|co|io|biz|info|us|uk|ca|au|de|fr|es|it|nl|eu|edu|gov|me|app|dev|"
r"shop|online|store|xyz|ai)")
_DEOBF_RE = re.compile(rf"(?<!{_LOCAL})({_LOCAL}+)\s*{_AT}\s*([A-Za-z0-9.\-]+?)\s*{_DOT}\s*({_TLD})\b", re.I)
def extract_emails(html, site_host):
"""Emails from mailto: links, plain text and obfuscated spans, junk dropped, site-domain first,
max 3. HTML entities are decoded first, so info@site.com counts."""
site = _host(site_host or "")
text = _html.unescape(html or "")[:400_000]
found = []
deobf = [f"{m[0]}@{m[1]}.{m[2]}" for m in _DEOBF_RE.findall(text)]
for e in MAILTO_RE.findall(text) + EMAIL_RE.findall(text) + deobf:
e = _ESCAPE_PREFIX.sub("", urllib.parse.unquote(e).strip().lower())
if not is_real_email(e) or e in found:
continue
found.append(e)
on_site = lambda e: site and (e.endswith("@" + site) or e.endswith("." + site))
return sorted(found, key=lambda e: not on_site(e))[:3]
def contact_links(html, base_url):
"""Up to 3 same-site contact/about URLs: discovered links first, then fixed fallbacks."""
if not base_url.startswith(("http://", "https://")):
base_url = "http://" + base_url
host, out = _host(base_url), []
if not host: # malformed or empty website: nothing to join the paths onto
return out
for href, text in HREF_RE.findall(html or ""):
if not re.search(r"contact|about|team|staff|support", href + " " + text, re.I):
continue
try:
url = urllib.parse.urldefrag(urllib.parse.urljoin(base_url, href.strip()))[0]
except ValueError: # a malformed href on the page
continue
if url.startswith(("http://", "https://")) and _host(url) == host and url not in out:
out.append(url)
for path in ("/contact", "/contact-us", "/about", "/about-us"):
url = urllib.parse.urljoin(base_url, path)
if len(out) < 3 and url not in out:
out.append(url)
return out[:3]
def enrich_emails(results, deep=True, workers=None, browser_sites=False):
"""Finish all HTTP email lookups before bounded browser retries for still-empty rows."""
def work(r):
site = r["website"]
home = _fetch_cached(site)
found = extract_emails(home, site)
if deep and home and not found:
pages = "\n".join(_fetch_html(u, timeout=8) for u in contact_links(home, site))
found = extract_emails(pages, site)
if browser_sites:
r.setdefault("website_fetch_status", "http" if home else "error")
r["emails"] = ", ".join(found)
todo = [r for r in results if r.get("website") and not r.get("emails")]
_each_site(todo, work, "emails", workers)
if browser_sites:
def retry(r):
site = r["website"]
rendered = _render_site(site)
_set_fetch_status(r, rendered["status"])
home = rendered["html"]
found = extract_emails(home, site)
if deep and home and not found:
for url in contact_links(home, site):
contact = _render_site(url)
if contact["status"] == "budget_exhausted":
_set_fetch_status(r, "budget_exhausted")
break
found = extract_emails(contact["html"], site)
if found:
break
r["emails"] = ", ".join(found)
_each_site([r for r in todo if not r.get("emails")], retry, "browser emails", workers=2)
# ── Optional email domain check (--verify-emails): can the domain receive mail, and who hosts it ──
def _doh_mx(domain):
"""MX lookup over DNS-over-HTTPS (Google's public resolver, no key) -> the JSON answer."""
url = "https://dns.google/resolve?" + urllib.parse.urlencode({"name": domain, "type": "MX"})
with _fetch_window(), _OPENER.open(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=8) as r:
return json.load(r)
@functools.lru_cache(maxsize=1024)
def mx_host(domain):
"""'nxdomain' when the domain does not exist, 'none' when it has no MX record, else who runs
its mail: 'google_workspace', 'microsoft_365' or 'other'. '' when the lookup itself failed."""
try:
d = _doh_mx(domain)
except Exception:
return ""
if d.get("Status") == 3:
return "nxdomain"
mx = " ".join(a.get("data", "") for a in d.get("Answer") or [] if a.get("type") == 15).lower()
if not mx:
return "none" if d.get("Status") == 0 else ""
if "google.com" in mx or "googlemail.com" in mx:
return "google_workspace"
return "microsoft_365" if "outlook.com" in mx else "other"
def verify_emails(rows):
"""Drop addresses on domains that do not exist; label the rest with email_host."""
def work(r):
kept = [(e, mx_host(e.rpartition("@")[2])) for e in r["emails"].split(", ") if e]
kept = [(e, h) for e, h in kept if h != "nxdomain"]
r["emails"] = ", ".join(e for e, _ in kept)
r["email_host"] = kept[0][1] if kept else ""
_each_site([r for r in rows if r.get("emails")], work, "email domains", workers=8)
def keep_row(r, a):
"""The row filters, minus --has-email (applied after the crawl). A missing rating or review
count reads as 0, so --min-rating and --min-reviews drop unrated listings."""
return not ((a.min_rating and _num(r.get("review_rating")) < a.min_rating)
or (a.min_reviews and _num(r.get("review_count"), int) < a.min_reviews)
or (a.no_website and r.get("website"))
or (a.has_phone and not r.get("phone")))
def scrape_place(a, body, city, keywords):
"""All the engine jobs for one place: a single job, or --grid tiles one at a time clipped to the
city box -> (first job id or None, rows, failed tile count, tile count)."""
lat, lon, bbox, cells = a.lat, a.lon, None, None
if a.grid: # always geocodes, because it needs the bounding box
print(f"▶ Geocoding \"{city}\"…")
geo = geocode(city, with_bbox=True)
if not geo or not geo[2]:
print("✗ Could not resolve a bounding box. Try a clearer --city.")
return None, [], 0, 0
lat, lon, bbox = geo
cells = grid_cells(bbox, a.grid)
print(f" → bbox S{bbox[0]} N{bbox[1]} W{bbox[2]} E{bbox[3]}, {len(cells)} tiles")
elif city or not (lat and lon): # explicit lat/lon, else geocode --city, else the first keyword
place = city or keywords[0]
print(f"▶ Geocoding \"{place}\"…")
coords = geocode(place)
if not coords:
print("✗ Could not resolve coordinates. Pass lat/lon or a clearer --city.")
return None, [], 0, 0
lat, lon = coords
print(f" → {lat}, {lon}")
# The job name is what the web UI lists, so make it say what the job was.
body = dict(body, lat=str(lat), lon=str(lon), name=(f"{keywords[0]} @ {city}" if city else keywords[0])[:80])
for k in keywords:
print(f" • {k}")
if not cells:
print(f"▶ Creating job: {len(keywords)} keyword(s) @ {lat},{lon} depth={a.depth}")
try:
job_id, rows = run_job(body)
except Blocked:
raise
except Partial as e: # its rows count, but the place is incomplete
print(str(e))
return e.job_id, e.rows, 1, 1
except RuntimeError as e:
print(str(e))
return None, [], 0, 0
return job_id, rows, 0, 0
print(f"▶ Grid {a.grid}x{a.grid}: {len(cells)} jobs, one at a time, {len(keywords)} keyword(s) depth={a.depth}"
+ (" (! deep scan: a tile at Google's cap is split into 4 sub-tiles)" if a.deep_scan else ""))
job_id, rows, skipped, i = None, [], 0, 0
while i < len(cells): # sequential on purpose: never parallel jobs against Google
i, cell = i + 1, cells[i]
print(f"▶ Cell {i}/{len(cells)} @ {cell['lat']},{cell['lon']} radius={cell['radius']}m"
+ (" (sub-tile)" if cell.get("_sub") else ""))
try:
jid, cell_rows = run_job(dict(body, **engine_cell(cell)))
except Blocked as e: # keep what the tiles so far found, inside the city box, and stop
e.job_id = job_id or e.job_id
e.rows = [r for r in dedupe(rows + e.rows) if in_bbox(r, bbox)]
print(f"\n ⚠ Stopped at cell {i}/{len(cells)}: the cells after it were not searched.")
raise
except Partial as e: # its rows count, but the tile is incomplete
print(f"\n ⚠ cell {i}/{len(cells)} incomplete: {str(e).strip()}")
skipped += 1
job_id, rows = job_id or e.job_id, dedupe(rows + e.rows)
SALVAGE["rows"], SALVAGE["job"] = dedupe(SALVAGE["rows"] + rows), SALVAGE["job"] or job_id
continue
except Exception as e:
print(f"\n ⚠ cell {i}/{len(cells)} skipped: {str(e).strip()}")
skipped += 1
continue
job_id = job_id or jid
rows = dedupe(rows + cell_rows)
SALVAGE["rows"], SALVAGE["job"] = dedupe(SALVAGE["rows"] + rows), SALVAGE["job"] or job_id # Ctrl-C keeps these
print(f" cell {i}/{len(cells)}: {len(cell_rows)} rows, {len(rows)} unique so far")
if a.deep_scan and not cell.get("_sub") and len(cell_rows) >= SATURATED * len(keywords):
subs = [dict(c, _sub=True) for c in grid_cells(cell["_bbox"], 2)]
cells.extend(subs) # one level only: sub-tiles never split again
print(f" ! cell {i} is at Google's cap: {len(subs)} sub-tiles added, {len(cells)} cells now")
# the engine does not clip results to the tile, so tiles leak outside the city
kept = [r for r in rows if in_bbox(r, bbox)]
print(f" dropped {len(rows) - len(kept)} rows outside the city box")
return job_id, kept, skipped, len(cells)
def main():
global _BROWSER_BUDGET
ap = argparse.ArgumentParser(description="Scrape Google Maps business listings.")
ap.add_argument("keyword", nargs="?", help='e.g. "coffee shops in Austin TX"')
ap.add_argument("lat", nargs="?", help="latitude (optional if --city given)")
ap.add_argument("lon", nargs="?", help="longitude (optional if --city given)")
ap.add_argument("--keyword", dest="also", action="append", help="extra keyword (repeatable)")
ap.add_argument("--keywords-file", help="file with one keyword per line (batch)")
ap.add_argument("--city", action="append",
help="city/place to auto-geocode for lat/lon; repeatable, one run per city into one file")
ap.add_argument("--proxy", action="append", metavar="URL",
help="proxy for the engine: http://[user:pass@]host:port, https://, socks5:// or "
"socks5h://host:port (no user:pass on SOCKS5); repeatable: the keywords take turns, "
"and a keyword Google refuses is retried through the next one")
ap.add_argument("--depth", type=int, default=5)
# The engine passes this to Google as the interface language, which decides the language of
# the category names it returns (and nudges which listings come back). It has to match the
# market you are scraping: "en" on a Brazilian city labels every row "Bakery" instead of
# "Padaria". The engine validates it as exactly 2 characters.
ap.add_argument("--lang", default="en", metavar="XX",
help="interface language the engine asks Google for, 2 letters "
"(default: en; use the local language, e.g. pt, es, de)")
# Emails are ON by default (they're the most valuable lead field): the script crawls each
# business website after the Maps job. --no-email skips the crawl.
ap.add_argument("--country", metavar="XX",
help="country Google ranks and formats results for, 2 letters (default: the country "
"of your IP address; e.g. us, de, br)")
ap.add_argument("--no-email", dest="email", action="store_false", default=True,
help="skip the website crawl for emails (emails are on by default)")
ap.add_argument("--no-deep-emails", dest="deep_emails", action="store_false",
help="check homepages only, not contact and about pages")
ap.add_argument("--grid", type=int, metavar="N",
help="split the --city area into N x N map tiles, one job each, deduped "
"(breaks Google's ~120 per search cap; max 10)")
ap.add_argument("--deep-scan", action="store_true",
help="! slower: a whole-city run past Google's cap. Runs --grid (3 x 3 unless --grid "
"says otherwise), then splits every tile that came back at the cap (about 120 "
"places per keyword) into 4 sub-tiles one zoom level in, deduped; up to 5x the "
"tiles, about 25 s each per keyword")
ap.add_argument("--max-time", type=int, default=600,
help="job time limit in SECONDS; the script waits that long for the "
"engine, plus a short grace (default: 600)")
ap.add_argument("--reviews", type=int, default=0, metavar="N",
help="also read up to N reviews per place (1-5, no reviewer names) and its star "
"histogram, from each place's panel: one more page load per place, about 2-4 s "
"(raise --max-time for many places); adds user_reviews and reviews_per_rating")
ap.add_argument("--out", default=None, help="output file path (default: results-<id>.csv). "
"The extension picks the format: .csv (default) or .json")
ap.add_argument("--json", action="store_true", help="write JSON instead of the default CSV")
ap.add_argument("--full", action="store_true", help="keep ALL raw columns (default: lean lead fields)")
ap.add_argument("--fields", help="comma-separated columns to keep (overrides the default lead set)")
ap.add_argument("--no-socials", dest="socials", action="store_false", default=True,
help="skip the social profiles (Instagram, Facebook, LinkedIn, TikTok, YouTube and X links on "
"each website), which are on by default and cost 0 LLM tokens")
ap.add_argument("--browser-sites", action="store_true",
help="retry websites with Playwright when HTTP finds no emails or social links; "
"requires requirements-browser.txt and Chrome, adds website_fetch_status")
ap.add_argument("--socials", dest="socials", action="store_true", help=argparse.SUPPRESS) # the old opt-in
ap.add_argument("--tech", action="store_true",
help="also detect Meta Pixel, Google Ads tag, analytics, Bing, LinkedIn and TikTok "
"pixels, site platform and mobile support on each website, and sort by an "
"intent_score (0 LLM tokens; slower)")
ap.add_argument("--verify-emails", action="store_true",
help="check each email domain's MX record over DNS (dns.google, no key): drops "
"addresses on domains that do not exist and adds an email_host column "
"(google_workspace, microsoft_365, other, none)")
f = ap.add_argument_group("filters", "applied before the email crawl, except --has-email")
f.add_argument("--min-rating", type=float, metavar="X", help="keep rows rated X or higher")
f.add_argument("--min-reviews", type=int, metavar="N",
help="keep rows with N or more reviews (ignored with a warning if all review counts are missing)")
f.add_argument("--no-website", action="store_true", help="keep only businesses without a website")
f.add_argument("--has-phone", action="store_true", help="keep only rows with a phone number")
f.add_argument("--has-email", action="store_true", help="keep only rows where an email was found")
f.add_argument("--exclude", action="append", metavar="FILE",
help="drop places already in this earlier results file (.csv or .json); repeatable")
f.add_argument("--limit", type=int, metavar="N", help="stop at N rows after filtering")
a = ap.parse_args()
if a.browser_sites:
try:
_web().check_available()
except RuntimeError as e:
sys.exit(f"✗ {e}")
_render_site.cache_clear()
_BROWSER_BUDGET = None
keywords = collect_keywords(a)
if not keywords:
sys.exit("✗ No keywords. Pass a keyword, --keyword, or --keywords-file.")
if len(a.lang) != 2 or not a.lang.isalpha():
sys.exit(f"✗ --lang must be a 2-letter code (got {a.lang!r}); the engine rejects anything else.")
if a.country is not None and (len(a.country) != 2 or not a.country.isalpha()):
sys.exit(f"✗ --country must be a 2-letter code (got {a.country!r}), e.g. us or de.")
if a.depth < 1:
sys.exit("✗ --depth must be 1 or more.")
if a.max_time < 1:
sys.exit("✗ --max-time must be 1 or more seconds.")
if not 0 <= a.reviews <= 5:
sys.exit("✗ --reviews must be from 0 to 5 (the reviews a place's panel shows first).")
if a.limit is not None and a.limit < 1:
sys.exit("✗ --limit must be 1 or more.")
if a.has_email and not a.email:
sys.exit("✗ --has-email needs the email crawl: drop --no-email.")
if a.has_email and a.fields and not a.full and "emails" not in [c.strip() for c in a.fields.split(",")]:
sys.exit("✗ --has-email needs the emails column: add emails to --fields.")
if a.city and (a.lat or a.lon):
print("ℹ --city given, so the lat/lon arguments are ignored.")
if a.out and os.path.splitext(a.out)[1].lower() not in (".csv", ".json"): # never a .zshrc or a key file
sys.exit("✗ --out must name a .csv or .json file.")
if a.out and os.path.dirname(a.out): # fail now, not after the scrape, if the folder is unusable
try:
os.makedirs(os.path.dirname(a.out), exist_ok=True)
except OSError as e:
sys.exit(f"✗ Cannot create the folder for --out: {e}")
if a.deep_scan:
if not a.city:
sys.exit("✗ --deep-scan needs --city: the city's bounding box is what gets split into tiles.")
a.grid = a.grid or 3
if a.grid is not None:
if not a.city:
sys.exit("✗ --grid needs --city: the city's bounding box is what gets split into tiles.")
if not 1 <= a.grid <= 10:
sys.exit("✗ --grid must be between 1 and 10 (10 x 10 = 100 jobs is the ceiling).")
cities = a.city or [None]
try:
excluded = [r for p in a.exclude or [] for r in load_rows(p)]
except (OSError, ValueError) as e:
sys.exit(f"✗ Cannot read --exclude file: {e}")
# --- WARN, don't block: flag high-volume jobs (proceed anyway) ---
if a.deep_scan:
n = a.grid ** 2
print(f"! Deep scan: {n} tiles, and up to {4 * n} more where a tile is at Google's cap, one at a "
f"time, about 25 s each per keyword: expect {n * 25 * len(keywords) // 60}-{5 * n * 25 * len(keywords) // 60} min per city.")
if a.depth >= 15 or len(keywords) >= 10 or (a.grid or 1) ** 2 >= 4 or len(cities) >= 3:
print("⚠️ Heads-up: this is a large job (high depth, many keywords or a grid).")
print(" Running it back-to-back without proxies can get your IP temporarily rate-limited by")
print(" Google (clears in minutes–hours; returns empty/failed jobs meanwhile). For big or")
print(" repeated runs, add proxies. Proceeding…\n")
# Health check. Right after `docker compose up -d` Docker accepts connections and drops them while
# the engine starts (a few seconds), and the README's next line runs at once, so wait for that.
# Nothing listening at all (connection refused) means no engine: say so after 10 s, not a minute.
refused, err = 0, None
for i in range(30):
try:
req("GET", "/api/v1/jobs")
err = None
break
except urllib.error.HTTPError as e:
if e.code < 500: # the engine is up but refuses us (wrong API key, wrong URL): say so at once
sys.exit(f"✗ The engine at {BASE} answered HTTP {e.code} to the health check: {e.read().decode()[:200]}")
err = e
except Exception as e:
err = e
refused += isinstance(getattr(e, "reason", e), ConnectionRefusedError)
if refused >= 5:
break
print("\r waiting for the engine to start" + "." * (i + 1), end="", flush=True)
time.sleep(2)
if err is not None:
sys.exit(f"\n✗ Scraper not reachable at {BASE}. Start the engine in the kit folder with "
f"'{PY} scripts/engine.py start' (or 'docker compose up -d' with Docker Desktop open), "
f"then try again.\n ({err})")
if i:
print()
body = {"keywords": keywords, "lang": a.lang, "zoom": 15,
"fast_mode": False, "radius": 10000,
# The engine's own email pass reads homepages only; the crawl below also reads contact and
# about pages and decodes obfuscated addresses, so the engine only reads Google Maps.
"depth": a.depth, "email": False, "max_time": a.max_time}
if a.reviews:
body["reviews_max"] = a.reviews
if a.proxy:
body["proxies"] = a.proxy
if a.country:
body["country"] = a.country.lower()
job_id, rows, skipped, ncells, failed_places, blocked = None, [], 0, 0, 0, False
for city in cities: # one place at a time, like the grid tiles
try:
jid, place_rows, s, n = scrape_place(a, body, city, keywords)
except Blocked as e:
print(str(e))
job_id, rows, blocked = job_id or e.job_id, dedupe(rows + e.rows), True
break
job_id, skipped, ncells = job_id or jid, skipped + s, ncells + n
failed_places += jid is None
rows = dedupe(rows + place_rows)
SALVAGE["rows"], SALVAGE["job"] = rows, job_id
if job_id is None:
sys.exit("✗ No job finished; the reason is in the ✗ line above.")
if failed_places:
print(f"⚠ Incomplete: {failed_places} of {len(cities)} places returned nothing (see the ✗ lines above).")
for r in rows:
r["website"] = clean_website(r.get("website", ""))
if a.min_reviews and rows and not any(r.get("review_count") for r in rows):
# Missing counts read as 0 and would otherwise drop every row.
print("⚠ --min-reviews ignored: Google returned no review_count values, so the filter would drop "
"every row. --min-rating works.")
a.min_reviews = None
before = len(rows)
rows = [r for r in dedupe(rows, exclude=excluded) if keep_row(r, a)]
if len(rows) != before:
print(f" filters and --exclude kept {len(rows)} of {before} rows")
if a.limit and not a.has_email:
rows = rows[:a.limit]
SALVAGE["rows"] = rows # Ctrl-C in the crawl below keeps the filtered rows, not the raw ones
if a.full:
fields = list(rows[0].keys()) if rows else LEAD
elif a.fields:
fields = [c.strip() for c in a.fields.split(",") if c.strip()]
else:
fields = LEAD
if a.reviews and not a.full: # the page loads were paid for: keep what they read
fields = fields + [c for c in ("reviews_per_rating", "user_reviews") if c not in fields]
if before and not rows:
print(f"⚠ Nothing left: the filters and --exclude dropped all {before} rows. Loosen them and rerun; "
f"the file below has only a header.")
else:
extra = f", then {', '.join(SOCIAL_FIELDS)}" if a.socials and not a.full else "" # added after the crawl
print(f"✓ Done — {len(rows)} businesses (fields: {', '.join(fields)}{extra}).")
if skipped:
print(f"⚠ Incomplete: {skipped} of {ncells} tiles or places failed or stopped early, so parts of the "
f"area are missing. Rerun later.")
if a.email and "emails" in fields and not a.no_website: # no website, nothing to crawl
pages = "homepage, then contact and about pages" if a.deep_emails else "homepage only"
print(f"▶ Emails: checking each business website ({pages}, {SITE_WORKERS} at a time)…")
enrich_emails(rows, deep=a.deep_emails, browser_sites=a.browser_sites)
total, n = sum(1 for r in rows if r.get("emails")), len(rows)
print(f" emails found for {total}/{n} businesses ({round(100 * total / n) if n else 0}%)")
if a.verify_emails:
print("▶ Checking each email domain's MX record over DNS (dns.google)…")
verify_emails(rows)
fields = fields + ["email_host"]
dead = total - sum(1 for r in rows if r.get("emails"))
print(f" dropped {dead} addresses on domains that do not exist")
if a.has_email:
rows = [r for r in rows if r.get("emails")][:a.limit or None]
print(f" --has-email kept {len(rows)} rows")
# Enrichment runs on the raw rows, which always carry the website, whatever --fields keeps.
if a.socials:
print("▶ Finding socials (Instagram / Facebook / LinkedIn / TikTok / YouTube / X) on each website…")
enrich_sites(rows, _find_socials, SOCIAL_FIELDS, "socials", browser_sites=a.browser_sites)
fields = fields + SOCIAL_FIELDS
found = sum(1 for r in rows if any(r.get(k) for k in SOCIAL_FIELDS))
print(f" socials found for {found}/{len(rows)} businesses")
if a.browser_sites:
fields = fields + ["website_fetch_status"]
if _BROWSER_BUDGET is not None: # the last pass that renders: close the job's one browser
_BROWSER_BUDGET.close()
if a.tech:
print("▶ Checking ad pixels, analytics and site platform on each website (0 LLM tokens)…")
enrich_sites(rows, _find_tech, TECH_FIELDS, "tech")
for r in rows:
r["intent_score"] = intent_score(r)
rows.sort(key=lambda r: r["intent_score"], reverse=True)
fields = fields + TECH_FIELDS + ["intent_score"]
ads = sum(1 for r in rows if "yes" in (r.get("fb_pixel"), r.get("google_ads_tag")))
print(f" Meta Pixel or Google Ads tag on {ads}/{len(rows)} websites; "
f"rows sorted by intent_score (highest first)")
results = [{k: r.get(k, "") for k in fields} for r in rows]
# Default output is a CSV file (opens in Excel / Google Sheets). Use --json (or a .json --out path) for JSON.
as_json = a.json or (a.out and a.out.lower().endswith(".json"))
out = a.out or f"results-{job_id[:8]}.{'json' if as_json else 'csv'}"
def write(path):
if as_json:
with open(path, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2, ensure_ascii=False)
else:
# utf-8-sig: the BOM makes Excel on Windows read accented names as UTF-8 instead of cp1252.
with open(path, "w", newline="", encoding="utf-8-sig") as f:
w = csv.DictWriter(f, fieldnames=fields)
w.writeheader()
w.writerows([{k: csv_safe(v) for k, v in r.items()} for r in results])
try:
write(out)
except OSError as e: # the target is locked (open in Excel) or unwritable: never lose a finished scrape
fallback = f"results-{job_id[:8]}.{'json' if as_json else 'csv'}"
print(f"⚠ Could not write {out} ({e.strerror}); writing {fallback} instead.")
out = fallback
write(out)
for w in dict.fromkeys(WARNINGS):
print(f"⚠ {w}")
print(f" saved → {os.path.abspath(out)} (opens in Excel or Google Sheets)")
for r in results[:5]: # scraped text: one short printable line per field, never a line of its own
tail = f" | IG:{shown(r.get('instagram','')) or '—'}" if a.socials else f" | {shown(r.get('website',''))}"
print(f" • {shown(r.get('title',''))} | {shown(r.get('phone',''))} | {shown(r.get('emails',''))}{tail}")
if blocked: # a non-zero exit, so an agent or a scheduled run knows the scrape was cut short
sys.exit("⚠ Incomplete: Google is rate-limiting this IP, so the scrape stopped; the file has the "
"rows found before that. Wait (minutes to hours) or add proxies, then rerun with "
"--exclude on this file to get only the rest.")
if skipped or failed_places: # the ⚠ Incomplete line above says what is missing; exit 0 means complete
sys.exit(1)
def interrupted():
"""The Ctrl-C message. A grid or multi-city run stopped between tiles keeps what the finished
ones found, in a -partial.csv with the lead fields (the websites were not crawled yet)."""
saved = "nothing was saved."
if SALVAGE["rows"]:
path = f"results-{(SALVAGE['job'] or 'partial')[:8]}-partial.csv"
with open(path, "w", newline="", encoding="utf-8-sig") as f:
w = csv.DictWriter(f, fieldnames=LEAD)
w.writeheader()
w.writerows([{k: csv_safe(r.get(k, "")) for k in LEAD} for r in SALVAGE["rows"]])
saved = f"the {len(SALVAGE['rows'])} rows found so far are in {path} (lead fields, websites not crawled)."
if CURRENT_JOB:
return (f"\n✗ Stopped; {saved} Job {CURRENT_JOB} keeps running in the engine. "
f"Cancel it in the web UI at {BASE}, or with:\n python3 scripts/engine.py delete {CURRENT_JOB}")
return f"\n✗ Stopped; {saved} Jobs: python3 scripts/engine.py jobs"
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
sys.exit(interrupted())