Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix: survive hostile input and transient errors: linear email regexes, guarded URL parsing, retrying polls, safer recovery, argument checks

Mahanaicoach92780f5494a4
diff --git a/.claude/settings.json b/.claude/settings.json
index d0f0719..14a507a 100644
--- a/.claude/settings.json
+++ b/.claude/settings.json
@@ -8,18 +8,19 @@
"Bash(docker compose start:*)",
"Bash(docker compose ps:*)",
"Bash(docker ps:*)",
- "Bash(curl http://localhost:8080:*)",
- "Bash(curl -s http://localhost:8080:*)",
- "Bash(curl http://127.0.0.1:8080:*)",
- "Bash(curl -s -X POST http://localhost:8080:*)",
- "Bash(curl -s -X DELETE http://localhost:8080:*)",
- "Bash(curl -X DELETE http://localhost:8080:*)",
+ "Bash(curl http://localhost:8080/:*)",
+ "Bash(curl -s http://localhost:8080/:*)",
+ "Bash(curl http://127.0.0.1:8080/:*)",
+ "Bash(curl -s -X POST http://localhost:8080/:*)",
+ "Bash(curl -s -X DELETE http://localhost:8080/:*)",
+ "Bash(curl -X DELETE http://localhost:8080/:*)",
"Bash(docker restart gmaps-scraper)",
"Bash(python3 scripts/test_scrape.py)",
"Bash(./scripts/scrape.sh:*)",
"Bash(bash scripts/scrape.sh:*)",
"Bash(python3 scripts/scrape.py:*)",
- "Bash(python scripts/scrape.py:*)"
+ "Bash(python scripts/scrape.py:*)",
+ "Bash(py scripts/scrape.py:*)"
],
"deny": []
}
diff --git a/scripts/scrape.py b/scripts/scrape.py
index 4d184e8..5f91c91 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -45,17 +45,21 @@ def load_env_file():
port needs one edit. Variables already set in the environment win over the file."""
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, ".env")
try:
- with open(path) as f:
+ with open(path, encoding="utf-8-sig", errors="replace") as f:
for line in f:
- k, sep, v = line.strip().partition("=")
+ line = line.strip()
+ if line.startswith("export "): # shell-style .env files
+ line = line[7:]
+ k, sep, v = line.partition("=")
+ k, v = k.strip(), v.split(" #", 1)[0].strip().strip("\"'") # drop a trailing comment
if sep and k in ENV_KEYS and k not in os.environ:
- os.environ[k] = v.strip().strip("\"'")
+ os.environ[k] = v
except OSError:
pass
load_env_file()
-BASE = os.environ.get("SCRAPER_BASE_URL") or f"http://localhost:{os.environ.get('SCRAPER_PORT') or '8080'}"
+BASE = (os.environ.get("SCRAPER_BASE_URL") or f"http://localhost:{os.environ.get('SCRAPER_PORT') or '8080'}").rstrip("/")
KEY = os.environ.get("SCRAPER_API_KEY", "")
# The 8 lead fields: what you use to contact and qualify a lead.
# Everything else (geo coordinates, IDs, hours, images, review blobs) is dropped by default.
@@ -148,31 +152,36 @@ def in_bbox(row, bbox, margin=0.10):
return s - my <= lat <= n + my and w - mx <= lon <= e + mx
-def _keys(r):
- """Every identity a row carries: place_id, cid, link and title+address. A lean 8-column CSV
- from an earlier run has only the last one, so matching on any key lets --exclude use it."""
+def _keys(r, weak=False):
+ """The identities a row carries: place_id, cid, link, name+address. With weak=True also
+ name+phone and name+website, which a --fields results file may be the only thing to match on.
+ They are weak because two branches of a chain share a website and sometimes a phone, so they
+ never dedupe rows within a run, only match an exclusion list."""
ks = [f"{k}:{r[k]}" for k in ("place_id", "cid", "link") if r.get(k)]
t = r.get("title")
- if t: # name plus address, and name plus phone or website: a --fields file may lack the address
+ if t:
ks.append(f"ta:{t}|{r.get('address', '')}")
- ks += [f"t{c[0]}:{t}|{r[c]}" for c in ("phone", "website") if r.get(c)]
+ if weak:
+ ks += [f"t{c[0]}:{t}|{r[c]}" for c in ("phone", "website") if r.get(c)]
return ks
def dedupe(rows, exclude=()):
"""Keep the first row per place; drop rows already present in `exclude` (an earlier run)."""
- seen, out = {k for r in exclude for k in _keys(r)}, []
+ seen, out = {k for r in exclude for k in _keys(r, weak=True)}, []
for r in rows:
- ks = _keys(r)
- if not any(k in seen for k in ks):
- seen.update(ks); out.append(r)
+ if not any(k in seen for k in _keys(r, weak=True)):
+ seen.update(_keys(r)); out.append(r)
return out
def load_rows(path):
"""Rows from an earlier results file, .csv or .json."""
with open(path, encoding="utf-8-sig", newline="") as f: # -sig: also reads the kit's own CSVs
- return json.load(f) if path.lower().endswith(".json") else list(csv.DictReader(f))
+ rows = json.load(f) if path.lower().endswith(".json") else list(csv.DictReader(f))
+ if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):
+ raise ValueError(f"{path} is not a results file (expected a list of rows)")
+ return rows
def _local_docker():
@@ -186,10 +195,10 @@ def engine_run_ended(since):
return False
try:
out = subprocess.run(["docker", "logs", "--since", since, ENGINE_CONTAINER],
- capture_output=True, text=True, timeout=15)
+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)
except (OSError, subprocess.SubprocessError):
return False
- return "scrapemate exited" in out.stdout + out.stderr
+ return out.returncode == 0 and "scrapemate exited" in out.stdout + out.stderr
def restart_engine():
"""Restart the local engine and wait until its API answers again. Without the wait, the next
@@ -240,10 +249,11 @@ def engine_idle(seconds=45):
return False
try:
out = subprocess.run(["docker", "logs", "--since", f"{seconds}s", ENGINE_CONTAINER],
- capture_output=True, text=True, timeout=15)
+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)
except (OSError, subprocess.SubprocessError):
return False
- return "scrapemate" not in out.stdout + out.stderr
+ # A failing `docker logs` (wrong container name, daemon down) must not read as "idle".
+ return out.returncode == 0 and "scrapemate" not in out.stdout + out.stderr
class JobFailed(RuntimeError):
@@ -263,6 +273,20 @@ def run_job(body):
time.sleep(FAILED_RETRY_WAIT)
+def _get(path):
+ """GET with three tries: a poll or a download must survive a passing 502 or timeout."""
+ for attempt in range(3):
+ try:
+ return req("GET", path)
+ except urllib.error.HTTPError as e:
+ if e.code < 500 or attempt == 2:
+ raise
+ except OSError:
+ if attempt == 2:
+ raise
+ time.sleep(3)
+
+
def _run_job_once(body, restarted=False):
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
try:
@@ -276,14 +300,15 @@ def _run_job_once(body, restarted=False):
attempts = poll_attempts(body.get("max_time"))
print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")
- status, ended, pending = None, 0, 0
+ status, ended, pending, n_rows, last_rows = None, 0, 0, None, None
for i in range(attempts):
- _, raw = req("GET", f"/api/v1/jobs/{job_id}")
+ _, raw = _get(f"/api/v1/jobs/{job_id}")
status = json.loads(raw).get("Status")
progress = ""
if status == "working": # the engine writes rows as it goes, so the download shows progress
try:
- progress = f", {len(download_rows(job_id))} rows so far"
+ n_rows = len(download_rows(job_id))
+ progress = f", {n_rows} rows so far"
except Exception:
pass
print(f"\r status: {str(status):<10} (attempt {i + 1}{progress})", end="", flush=True)
@@ -291,13 +316,16 @@ def _run_job_once(body, restarted=False):
print(); break
if status == "failed":
raise JobFailed("job failed")
- if status == "working" and engine_run_ended(since):
+ # Stuck (gosom#346): the engine log says the run ended and the row count has stopped moving.
+ # The second condition guards against a Docker VM clock that runs ahead of the host.
+ if status == "working" and n_rows == last_rows and engine_run_ended(since):
ended += 1
if ended > 2: # the status normally flips within seconds of the run ending
return job_id, recover_stuck_job(job_id)
+ last_rows = n_rows
if status == "pending":
pending += 1
- if pending == PENDING_RESTART_AFTER and not restarted and engine_idle():
+ if pending % PENDING_RESTART_AFTER == 0 and not restarted and engine_idle():
# Delete the pending job first, or the restarted engine runs it as well as the new one.
print("\n⚠ The engine has not started this job and shows no activity: an earlier job left it "
"stuck (gosom/google-maps-scraper#346). Restarting the engine and re-creating the job…")
@@ -305,8 +333,10 @@ def _run_job_once(body, restarted=False):
req("DELETE", f"/api/v1/jobs/{job_id}")
except Exception:
pass
- if restart_engine():
- return _run_job_once(body, restarted=True)
+ if not restart_engine():
+ raise RuntimeError(f"\n✗ The engine never started this job and could not be restarted. "
+ f"Run: docker restart {ENGINE_CONTAINER}")
+ return _run_job_once(body, restarted=True)
time.sleep(POLL_EVERY)
else:
# The engine writes rows as it goes, and v1.18.x can leave a finished job stuck on
@@ -328,7 +358,7 @@ def _run_job_once(body, restarted=False):
def download_rows(job_id):
- _, raw = req("GET", f"/api/v1/jobs/{job_id}/download")
+ _, raw = _get(f"/api/v1/jobs/{job_id}/download")
return list(csv.DictReader(io.StringIO(raw.decode("utf-8", "replace"))))
@@ -339,10 +369,11 @@ def collect_keywords(a):
kws.extend(a.also or [])
if a.keywords_file:
try:
- with open(a.keywords_file) as f:
- kws.extend(line.strip() for line in f if line.strip() and not line.startswith("#"))
- except OSError as e:
- sys.exit(f"✗ Cannot read --keywords-file {a.keywords_file}: {e.strerror}")
+ with open(a.keywords_file, encoding="utf-8-sig") as f:
+ lines = [line.strip() for line in f]
+ kws.extend(line for line in lines if line and not line.startswith("#"))
+ except (OSError, UnicodeDecodeError) as e:
+ sys.exit(f"✗ Cannot read --keywords-file {a.keywords_file}: {getattr(e, 'strerror', e)}")
return list(dict.fromkeys(kws)) # de-dupe, keep order
@@ -429,9 +460,9 @@ def _fetch_html(url, timeout=10):
return ""
if not url.startswith(("http://", "https://")):
url = "http://" + url
- if not _is_public_host(urllib.parse.urlparse(url).hostname or ""):
- return ""
try:
+ if not _is_public_host(urllib.parse.urlparse(url).hostname or ""):
+ return ""
r = urllib.request.Request(url, headers={
"User-Agent": "Mozilla/5.0 (compatible; " + UA + ")", "Accept": "text/html"})
with _OPENER.open(r, timeout=timeout) as resp:
@@ -439,7 +470,7 @@ def _fetch_html(url, timeout=10):
except Exception:
return ""
[email protected]_cache(maxsize=512) # ponytail: homepages only; evicted pages just refetch
[email protected]_cache(maxsize=256) # ponytail: homepages only (up to 400 KB each); evicted pages just refetch
def _fetch_cached(url):
"""Homepage fetch shared by the email crawl, --socials and --tech (one fetch per site)."""
return _fetch_html(url, timeout=8)
@@ -458,13 +489,20 @@ def _find_socials(html):
return out
def _each_site(todo, work, label, workers=None):
- """Run work(row) over the rows, SITE_WORKERS at a time, with one progress line."""
- done = 0
+ """Run work(row) over the rows, SITE_WORKERS at a time, with one progress line. One row that
+ blows up (a malformed website, an odd page) is skipped; it never aborts the run."""
+ done, failed = 0, 0
+ def safe(r):
+ try:
+ work(r)
+ except Exception:
+ return 1
+ return 0
with ThreadPoolExecutor(max_workers=workers or SITE_WORKERS) as ex:
- for _ in ex.map(work, todo):
- done += 1
+ for f in ex.map(safe, todo):
+ done, failed = done + 1, failed + f
print(f"\r {label}: {done}/{len(todo)} checked", end="", flush=True)
- print()
+ print(f" ({failed} skipped after an error)" if failed else "")
def enrich_sites(results, find, fields, label, workers=None):
"""Merge find(homepage html) into every lead that has a website. Zero LLM tokens."""
@@ -525,7 +563,10 @@ def intent_score(r):
# ── Email crawl: each business website's homepage, then its contact/about pages ──
-EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
+# The lookbehind pins each match attempt to the start of a run of local-part characters: without
+# it the scan is quadratic, and a 60 KB run of "a" (a minified blob) took a minute per page.
+_LOCAL = r"[A-Za-z0-9._%+-]"
+EMAIL_RE = re.compile(rf"(?<!{_LOCAL}){_LOCAL}+@[A-Za-z0-9.-]+\.[A-Za-z]{{2,}}")
MAILTO_RE = re.compile(r"mailto:([^\"'>\s?]+)", re.I)
HREF_RE = re.compile(r"<a\s[^>]*?href\s*=\s*[\"']([^\"']+)[\"'][^>]*>(.*?)</a>", re.I | re.S)
_JUNK_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".avif", ".css", ".js")
@@ -542,7 +583,10 @@ _ESCAPE_PREFIX = re.compile(r"^(?:u00(?:3c|3e|22|27|26))+")
def _host(url):
if not url.startswith(("http://", "https://")):
url = "http://" + url
- h = (urllib.parse.urlparse(url).hostname or "").lower()
+ try:
+ h = (urllib.parse.urlparse(url).hostname or "").lower()
+ except ValueError: # "http://[bad": a malformed website value from a listing
+ return ""
return h[4:] if h.startswith("www.") else h
# Google Business Profile links often carry campaign tags (?utm_source=gbp&utm_medium=organic),
@@ -554,10 +598,17 @@ def clean_website(url):
them (store?storeid=...)."""
if not url:
return ""
- p = urllib.parse.urlsplit(url)
- q = [(k, v) for k, v in urllib.parse.parse_qsl(p.query, keep_blank_values=True)
- if not k.lower().startswith(_TRACKING_PARAMS)]
- return urllib.parse.urlunsplit(p._replace(query=urllib.parse.urlencode(q), fragment=""))
+ try:
+ p = urllib.parse.urlsplit(url)
+ except ValueError: # malformed: leave it as the listing had it
+ return url
+ q = urllib.parse.parse_qsl(p.query, keep_blank_values=True)
+ kept = [(k, v) for k, v in q if not k.lower().startswith(_TRACKING_PARAMS)]
+ frag = p.fragment if p.fragment.startswith(("/", "!")) else "" # keep hash-routed store pages
+ if len(kept) == len(q) and frag == p.fragment:
+ return url # nothing to drop: never re-encode a query the site chose
+ query = p.query if len(kept) == len(q) else urllib.parse.urlencode(kept)
+ return urllib.parse.urlunsplit(p._replace(query=query, fragment=frag))
def is_real_email(e):
"""False for anything not worth sending to: a template placeholder ([email protected],
@@ -575,13 +626,13 @@ _AT = r"(?:\[\s*at\s*\]|\(\s*at\s*\)|\{\s*at\s*\})"
_DOT = r"(?:\[\s*dot\s*\]|\(\s*dot\s*\)|\{\s*dot\s*\}|\s+dot\s+)"
_TLD = (r"(?:com|net|org|co|io|biz|info|us|uk|ca|au|de|fr|es|it|nl|eu|edu|gov|me|app|dev|"
r"shop|online|store|xyz|ai)")
-_DEOBF_RE = re.compile(rf"([A-Za-z0-9._%+\-]+)\s*{_AT}\s*([A-Za-z0-9.\-]+?)\s*{_DOT}\s*({_TLD})\b", re.I)
+_DEOBF_RE = re.compile(rf"(?<!{_LOCAL})({_LOCAL}+)\s*{_AT}\s*([A-Za-z0-9.\-]+?)\s*{_DOT}\s*({_TLD})\b", re.I)
def extract_emails(html, site_host):
"""Emails from mailto: links, plain text and obfuscated spans, junk dropped, site-domain first,
max 3. HTML entities are decoded first, so info&#64;site.com counts."""
site = _host(site_host or "")
- text = _html.unescape(html or "")
+ text = _html.unescape(html or "")[:400_000]
found = []
deobf = [f"{m[0]}@{m[1]}.{m[2]}" for m in _DEOBF_RE.findall(text)]
for e in MAILTO_RE.findall(text) + EMAIL_RE.findall(text) + deobf:
@@ -597,10 +648,15 @@ def contact_links(html, base_url):
if not base_url.startswith(("http://", "https://")):
base_url = "http://" + base_url
host, out = _host(base_url), []
+ if not host: # malformed or empty website: nothing to join the paths onto
+ return out
for href, text in HREF_RE.findall(html or ""):
if not re.search(r"contact|about|team|staff|support", href + " " + text, re.I):
continue
- url = urllib.parse.urldefrag(urllib.parse.urljoin(base_url, href.strip()))[0]
+ try:
+ url = urllib.parse.urldefrag(urllib.parse.urljoin(base_url, href.strip()))[0]
+ except ValueError: # a malformed href on the page
+ continue
if url.startswith(("http://", "https://")) and _host(url) == host and url not in out:
out.append(url)
for path in ("/contact", "/contact-us", "/about", "/about-us"):
@@ -781,6 +837,21 @@ def main():
if len(a.lang) != 2 or not a.lang.isalpha():
sys.exit(f"✗ --lang must be a 2-letter code (got {a.lang!r}); the engine rejects anything else.")
+ if a.depth < 1:
+ sys.exit("✗ --depth must be 1 or more.")
+ if a.max_time < 1:
+ sys.exit("✗ --max-time must be 1 or more seconds (the engine runs a job for at least 180).")
+ if a.limit is not None and a.limit < 1:
+ sys.exit("✗ --limit must be 1 or more.")
+ if a.has_email and not a.email:
+ sys.exit("✗ --has-email needs the email crawl: drop --no-email.")
+ if a.city and (a.lat or a.lon):
+ print("ℹ --city given, so the lat/lon arguments are ignored.")
+ if a.out and os.path.dirname(a.out): # fail now, not after the scrape, if the folder is unusable
+ try:
+ os.makedirs(os.path.dirname(a.out), exist_ok=True)
+ except OSError as e:
+ sys.exit(f"✗ Cannot create the folder for --out: {e}")
if a.grid is not None:
if not a.city:
@@ -807,6 +878,10 @@ def main():
try:
req("GET", "/api/v1/jobs")
break
+ except urllib.error.HTTPError as e:
+ if e.code < 500: # the engine is up but refuses us (wrong API key, wrong URL): say so at once
+ sys.exit(f"✗ The engine at {BASE} answered HTTP {e.code} to the health check: {e.read().decode()[:200]}")
+ err = e
except Exception as e:
err = e
print("\r waiting for the engine to start" + "." * (i + 1), end="", flush=True)
@@ -825,13 +900,16 @@ def main():
"depth": a.depth, "email": False, "max_time": a.max_time}
if a.proxy:
body["proxies"] = a.proxy
- job_id, rows, skipped, ncells = None, [], 0, 0
+ job_id, rows, skipped, ncells, failed_places = None, [], 0, 0, 0
for city in cities: # one place at a time, like the grid tiles
jid, place_rows, s, n = scrape_place(a, body, city, keywords)
job_id, skipped, ncells = job_id or jid, skipped + s, ncells + n
+ failed_places += jid is None
rows = dedupe(rows + place_rows)
if job_id is None:
sys.exit("✗ No job finished. You may be rate-limited — wait or add proxies.")
+ if failed_places:
+ print(f"⚠ Incomplete: {failed_places} of {len(cities)} places returned nothing (see the ✗ lines above).")
for r in rows:
r["website"] = clean_website(r.get("website", ""))
before = len(rows)
@@ -846,6 +924,8 @@ def main():
fields = [c.strip() for c in a.fields.split(",") if c.strip()]
else:
fields = LEAD
+ if a.has_email and "emails" not in fields:
+ sys.exit("✗ --has-email needs the emails column: add emails to --fields.")
print(f"✓ Done — {len(rows)} businesses (fields: {', '.join(fields)}).")
if skipped:
print(f"⚠ Incomplete: {skipped} of {ncells} tiles failed, so parts of the area are missing. Rerun later.")
@@ -865,40 +945,49 @@ def main():
if a.has_email:
rows = [r for r in rows if r.get("emails")][:a.limit or None]
print(f" --has-email kept {len(rows)} rows")
- results = [{k: r.get(k, "") for k in fields} for r in rows]
+ # Enrichment runs on the raw rows, which always carry the website, whatever --fields keeps.
if a.socials:
print("▶ Finding socials (Instagram / Facebook / LinkedIn / TikTok / YouTube / X) on each website…")
print(" ℹ Cost: this runs in CODE = 0 LLM tokens. It only adds ~40–50 tokens per business")
print(" IF you load the results into an AI's context (negligible if you keep the file on disk).")
- enrich_sites(results, _find_socials, SOCIAL_FIELDS, "socials")
+ enrich_sites(rows, _find_socials, SOCIAL_FIELDS, "socials")
fields = fields + SOCIAL_FIELDS
- found = sum(1 for r in results if any(r.get(k) for k in SOCIAL_FIELDS))
- print(f" socials found for {found}/{len(results)} businesses")
+ found = sum(1 for r in rows if any(r.get(k) for k in SOCIAL_FIELDS))
+ print(f" socials found for {found}/{len(rows)} businesses")
if a.tech:
print("▶ Checking ad pixels, analytics and site platform on each website (0 LLM tokens)…")
- enrich_sites(results, _find_tech, TECH_FIELDS, "tech")
- for r in results:
+ enrich_sites(rows, _find_tech, TECH_FIELDS, "tech")
+ for r in rows:
r["intent_score"] = intent_score(r)
- results.sort(key=lambda r: r["intent_score"], reverse=True)
+ rows.sort(key=lambda r: r["intent_score"], reverse=True)
fields = fields + TECH_FIELDS + ["intent_score"]
- ads = sum(1 for r in results if "yes" in (r.get("fb_pixel"), r.get("google_ads_tag")))
- print(f" Meta Pixel or Google Ads tag on {ads}/{len(results)} websites; "
+ ads = sum(1 for r in rows if "yes" in (r.get("fb_pixel"), r.get("google_ads_tag")))
+ print(f" Meta Pixel or Google Ads tag on {ads}/{len(rows)} websites; "
f"rows sorted by intent_score (highest first)")
+ results = [{k: r.get(k, "") for k in fields} for r in rows]
# Default output is a CSV file (opens in Excel / Google Sheets). Use --json (or a .json --out path) for JSON.
as_json = a.json or (a.out and a.out.lower().endswith(".json"))
out = a.out or f"results-{job_id[:8]}.{'json' if as_json else 'csv'}"
- if as_json:
- with open(out, "w", encoding="utf-8") as f:
- json.dump(results, f, indent=2, ensure_ascii=False)
- else:
- # utf-8-sig: the BOM makes Excel on Windows read accented names as UTF-8 instead of cp1252.
- with open(out, "w", newline="", encoding="utf-8-sig") as f:
- w = csv.DictWriter(f, fieldnames=fields)
- w.writeheader()
- w.writerows(results)
+ def write(path):
+ if as_json:
+ with open(path, "w", encoding="utf-8") as f:
+ json.dump(results, f, indent=2, ensure_ascii=False)
+ else:
+ # utf-8-sig: the BOM makes Excel on Windows read accented names as UTF-8 instead of cp1252.
+ with open(path, "w", newline="", encoding="utf-8-sig") as f:
+ w = csv.DictWriter(f, fieldnames=fields)
+ w.writeheader()
+ w.writerows(results)
+ try:
+ write(out)
+ except OSError as e: # the target is locked (open in Excel) or unwritable: never lose a finished scrape
+ fallback = f"results-{job_id[:8]}.{'json' if as_json else 'csv'}"
+ print(f"⚠ Could not write {out} ({e.strerror}); writing {fallback} instead.")
+ out = fallback
+ write(out)
print(f" saved → {os.path.abspath(out)} (opens in Excel or Google Sheets)")
for r in results[:5]:
tail = f" | IG:{r.get('instagram','') or '—'}" if a.socials else f" | {r.get('website','')}"
@@ -906,4 +995,7 @@ def main():
if __name__ == "__main__":
- main()
+ try:
+ main()
+ except KeyboardInterrupt:
+ sys.exit(f"\n✗ Stopped. A job already created keeps running in the engine; list jobs at {BASE}/api/v1/jobs")
diff --git a/scripts/scrape.sh b/scripts/scrape.sh
index f175e6c..a4ae611 100755
--- a/scripts/scrape.sh
+++ b/scripts/scrape.sh
@@ -14,9 +14,17 @@ if [ -z "$KEYWORD" ] || [ -z "$LAT" ] || [ -z "$LON" ]; then
exit 1
fi
-# Git Bash on Windows often has only `python`.
-PY="$(command -v python3 || command -v python || true)"
+# Git Bash on Windows often has only `python`, and `python3` there can be the Microsoft Store stub
+# that only prints a message, so take the first name that really runs.
+PY=""
+for p in python3 python py; do
+ command -v "$p" >/dev/null 2>&1 && "$p" -c 'import sys' >/dev/null 2>&1 && { PY="$p"; break; }
+done
[ -n "$PY" ] || { echo "✗ Python 3 is required (standard library only, no pip installs)." >&2; exit 1; }
-OUT="results-$(echo "$KEYWORD" | tr ' /' '__' | tr -cd 'A-Za-z0-9_-').csv"
+# File name from the keyword; a keyword with no ASCII letters (寿司 in 東京) falls back to a timestamp
+# so reruns never overwrite each other.
+SLUG="$(printf '%s' "$KEYWORD" | tr ' /' '__' | tr -cd 'A-Za-z0-9_-')"
+[ -n "$SLUG" ] || SLUG="$(date +%Y%m%d-%H%M%S)"
+OUT="results-$SLUG.csv"
exec "$PY" "$(dirname "$0")/scrape.py" "$KEYWORD" "$LAT" "$LON" --depth "$DEPTH" --out "$OUT"
diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py
index 1552da7..347691d 100644
--- a/scripts/test_scrape.py
+++ b/scripts/test_scrape.py
@@ -123,11 +123,11 @@ class StuckJob(unittest.TestCase):
self.run_stuck(b"")
def test_ended_run_recovered_in_seconds_and_engine_restarted(self):
- # The engine log says the run ended but the status never flips: stop after 3 polls
- # (about 24 s) instead of the whole budget, keep the rows, restart the engine.
+ # The engine log says the run ended and the row count has stopped moving for 3 polls: stop
+ # after 4 polls (about 32 s) instead of the whole budget, keep the rows, restart the engine.
rows, polls, restarted, deleted = self.run_stuck(b"title\nA\n", run_ended=True, attempts=90)
self.assertEqual([r["title"] for r in rows], ["A"])
- self.assertEqual(polls, 3)
+ self.assertEqual(polls, 4)
self.assertTrue(restarted)
self.assertEqual(deleted, ["/api/v1/jobs/stuck"]) # no longer a "working" ghost in the job list
@@ -349,6 +349,8 @@ class CleanWebsite(unittest.TestCase):
"https://www.hpb.com/store?storeid=HPB-005")
self.assertEqual(scrape.clean_website("http://plain.com/"), "http://plain.com/")
self.assertEqual(scrape.clean_website(""), "")
+ self.assertEqual(scrape.clean_website("http://x.com/?a=1;b=2#top"), "http://x.com/?a=1;b=2") # never re-encoded
+ self.assertEqual(scrape.clean_website("http://x.com/#/store/5"), "http://x.com/#/store/5") # hash routing kept
class KeywordsFile(unittest.TestCase):
@@ -384,7 +386,7 @@ FIELDS = ["input_id", "link", "title", "category", "address", "website", "phone"
"review_rating", "latitude", "longitude", "cid", "place_id", "emails"]
def _row(pid, lat="30.2"):
- return {"title": f"Biz {pid}", "place_id": pid, "cid": "c" + pid, "latitude": lat,
+ return {"title": f"Biz {pid}", "place_id": pid, "cid": "c" + pid, "latitude": lat, "website": f"http://{pid.lower()}.biz",
"longitude": "-97.7", "phone": "+1 555", "emails": f"{pid.lower()}@biz.net"}
CELL_ROWS = {"job-1": [_row("A"), _row("B"), _row("C")],
@@ -466,11 +468,12 @@ class EndToEnd(unittest.TestCase):
w = csv.DictWriter(f, fieldnames=scrape.LEAD); w.writeheader()
w.writerow({"title": "Biz A", "address": ""})
argv = ["scrape.py", "plumbers", "--city", "Austin, TX", "--city", "Round Rock, TX", "--no-email",
- "--exclude", earlier, "--limit", "2", "--out", out]
+ "--exclude", earlier, "--limit", "2", "--out", out, "--socials", "--fields", "title,phone"]
log = io.StringIO()
try:
with mock.patch.object(scrape, "BASE", f"http://127.0.0.1:{srv.server_port}"), \
mock.patch.object(scrape, "geocode", return_value=("30.25", "-97.75")) as geo, \
+ mock.patch.object(scrape, "_fetch_cached", return_value='<a href="https://instagram.com/biz_b">ig</a>'), \
mock.patch.object(scrape.time, "sleep"), \
mock.patch.object(sys, "argv", argv), redirect_stdout(log):
scrape.main()
@@ -481,6 +484,8 @@ class EndToEnd(unittest.TestCase):
with open(out, newline="", encoding="utf-8-sig") as f: # the kit writes a BOM for Excel
rows = list(csv.DictReader(f))
self.assertEqual([r["title"] for r in rows], ["Biz B", "Biz C"]) # --limit 2
+ # --socials works even though --fields dropped the website column: enrichment reads the raw rows
+ self.assertEqual(rows[0]["instagram"], "https://instagram.com/biz_b")
class Filters(unittest.TestCase):
@@ -495,6 +500,14 @@ class Filters(unittest.TestCase):
self.assertTrue(scrape.keep_row({"website": ""}, nw))
self.assertFalse(scrape.keep_row({"website": "http://x.com"}, nw))
+ def test_chain_branches_are_not_merged(self):
+ # Two branches of a chain share the website (and sometimes the phone): distinct place_ids
+ # must both survive, and only an exclusion list may match on name+phone or name+website.
+ branches = [{"place_id": "P1", "title": "Anytime Fitness", "address": "1 A St", "website": "http://af.com", "phone": "+1 1"},
+ {"place_id": "P2", "title": "Anytime Fitness", "address": "9 B St", "website": "http://af.com", "phone": "+1 1"}]
+ self.assertEqual(len(scrape.dedupe(branches)), 2)
+ self.assertEqual(len(scrape.dedupe(branches, exclude=[{"title": "Anytime Fitness", "address": "9 B St"}])), 1)
+
def test_exclude_matches_lean_rows(self):
new = [{"place_id": "P1", "title": "Joe's", "address": "1 Main"}, {"place_id": "P2", "title": "Ann's"}]
self.assertEqual(scrape.dedupe(new, exclude=[{"title": "Joe's", "address": "1 Main"}]), new[1:])
@@ -513,6 +526,56 @@ class Filters(unittest.TestCase):
self.assertEqual(scrape.load_rows(c), [{"title": "B", "phone": "1"}]) # BOM stripped, key is clean
+class RegexSpeed(unittest.TestCase):
+ def test_long_runs_are_linear(self):
+ import time as _t
+ blob = "a" * 100_000 + " contact: [email protected] " + "b" * 100_000
+ t0 = _t.perf_counter()
+ self.assertEqual(scrape.extract_emails(blob, "acme.com"), ["[email protected]"])
+ self.assertLess(_t.perf_counter() - t0, 1.0) # was about a minute per 60 KB run
+
+
+class Resilience(unittest.TestCase):
+ def test_get_retries_transient_errors(self):
+ answers = iter([scrape.urllib.error.HTTPError("u", 502, "bad gateway", {}, None),
+ OSError("timed out"), (200, b"null")])
+ def fake_req(method, path, body=None):
+ a = next(answers)
+ if isinstance(a, Exception):
+ raise a
+ return a
+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep") as slept:
+ self.assertEqual(scrape._get("/api/v1/jobs"), (200, b"null"))
+ self.assertEqual(slept.call_count, 2)
+ with mock.patch.object(scrape, "req", side_effect=scrape.urllib.error.HTTPError("u", 404, "nf", {}, None)), \
+ self.assertRaises(scrape.urllib.error.HTTPError):
+ scrape._get("/api/v1/jobs/x") # a 4xx is final, no retry
+
+ def test_pending_restart_failure_raises_instead_of_polling_a_deleted_job(self):
+ statuses = iter(["pending"] * 20)
+ def fake_req(method, path, body=None):
+ if method == "POST":
+ return 201, b'{"id":"j1"}'
+ if method == "DELETE":
+ return 200, b""
+ return 200, json.dumps({"Status": next(statuses)}).encode()
+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \
+ mock.patch.object(scrape, "poll_attempts", return_value=50), \
+ mock.patch.object(scrape, "engine_run_ended", return_value=False), \
+ mock.patch.object(scrape, "engine_idle", return_value=True), \
+ mock.patch.object(scrape, "restart_engine", return_value=False), \
+ redirect_stdout(io.StringIO()), self.assertRaises(RuntimeError) as cm:
+ scrape.run_job({"max_time": 1})
+ self.assertIn("docker restart", str(cm.exception))
+
+ def test_failing_docker_logs_is_not_idle(self):
+ failed = mock.Mock(returncode=1, stdout="", stderr="Error: No such container")
+ with mock.patch.object(scrape, "_local_docker", return_value=True), \
+ mock.patch.object(scrape.subprocess, "run", return_value=failed):
+ self.assertFalse(scrape.engine_idle())
+ self.assertFalse(scrape.engine_run_ended("2026-01-01T00:00:00Z"))
+
+
class VerifyEmails(unittest.TestCase):
def test_mx_host_labels_and_drops_dead_domains(self):
answers = {
@@ -536,5 +599,82 @@ class VerifyEmails(unittest.TestCase):
self.assertEqual(scrape.mx_host("x.com"), "") # lookup failed: keep the address, no label
+class HostileInput(unittest.TestCase):
+ """A listing can carry a malformed website; nothing in the crawl may raise on it."""
+ BAD = "http://[bad"
+
+ def test_url_helpers_never_raise(self):
+ self.assertEqual(scrape.clean_website(self.BAD), self.BAD)
+ self.assertEqual(scrape._host(self.BAD), "")
+ self.assertEqual(scrape.extract_emails("[email protected]", self.BAD), ["[email protected]"])
+ self.assertEqual(scrape._fetch_html(self.BAD), "")
+ links = scrape.contact_links('<a href="http://[x">contact</a><a href="/contact">c</a>', self.BAD)
+ self.assertTrue(all(isinstance(u, str) for u in links))
+
+ def test_one_bad_row_does_not_abort_the_run(self):
+ rows = [{"website": "boom"}, {"website": "fine"}]
+ def work(r):
+ if r["website"] == "boom":
+ raise ValueError("parser blew up")
+ r["seen"] = "yes"
+ log = io.StringIO()
+ with redirect_stdout(log):
+ scrape._each_site(rows, work, "x", workers=2)
+ self.assertEqual(rows[1].get("seen"), "yes")
+ self.assertIn("1 skipped after an error", log.getvalue())
+
+ def test_exclude_file_must_be_a_list_of_rows(self):
+ d = tempfile.mkdtemp()
+ j = os.path.join(d, "job.json"); open(j, "w").write('{"job": {"keywords": ["x"]}}')
+ with self.assertRaises(ValueError):
+ scrape.load_rows(j)
+
+
+class EnvFile(unittest.TestCase):
+ def test_export_prefix_comment_quotes_and_trailing_slash(self):
+ d = tempfile.mkdtemp(); os.mkdir(os.path.join(d, "scripts"))
+ open(os.path.join(d, ".env"), "w").write(
+ "export SCRAPER_PORT=8081 # my port\nSCRAPER_BASE_URL='http://localhost:8081/'\n# SCRAPER_API_KEY=nope\n")
+ saved = {k: os.environ.pop(k, None) for k in scrape.ENV_KEYS}
+ try:
+ with mock.patch.object(scrape, "__file__", os.path.join(d, "scripts", "scrape.py")):
+ scrape.load_env_file()
+ self.assertEqual(os.environ.get("SCRAPER_PORT"), "8081")
+ self.assertEqual(os.environ.get("SCRAPER_BASE_URL"), "http://localhost:8081/")
+ self.assertIsNone(os.environ.get("SCRAPER_API_KEY"))
+ finally:
+ for k, v in saved.items():
+ os.environ.pop(k, None)
+ if v is not None:
+ os.environ[k] = v
+
+ def test_base_url_has_no_trailing_slash(self):
+ # "http://host/" + "/api/..." would be "//api/...", which the proxy answers with a 307 that
+ # urllib will not follow for a POST.
+ with mock.patch.dict(os.environ, {"SCRAPER_BASE_URL": "http://localhost:8090/"}):
+ self.assertEqual((os.environ.get("SCRAPER_BASE_URL") or "").rstrip("/"), "http://localhost:8090")
+ self.assertFalse(scrape.BASE.endswith("/"))
+
+
+class ArgumentBounds(unittest.TestCase):
+ def run_main(self, *extra):
+ argv = ["scrape.py", "x in y", "1", "2", *extra]
+ with mock.patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()), \
+ self.assertRaises(SystemExit) as cm:
+ scrape.main()
+ return str(cm.exception)
+
+ def test_has_email_needs_the_crawl(self):
+ self.assertIn("--has-email", self.run_main("--has-email", "--no-email"))
+
+ def test_rejects_values_the_engine_would_choke_on(self):
+ self.assertIn("--depth", self.run_main("--depth", "0"))
+ self.assertIn("--max-time", self.run_main("--max-time", "0")) # the engine says "missing max time"
+ self.assertIn("--limit", self.run_main("--limit", "-1"))
+ bad_dir = os.path.join(tempfile.mkdtemp(), "a-file")
+ open(bad_dir, "w").close() # a file where the folder should be
+ self.assertIn("--out", self.run_main("--out", os.path.join(bad_dir, "x", "leads.csv")))
+
+
if __name__ == "__main__":
unittest.main(verbosity=2)