Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
fix: survive hostile input and transient errors: linear email regexes, guarded URL parsing, retrying polls, safer recovery, argument checks
Mahanaicoach92780f5494a4
diff --git a/.claude/settings.json b/.claude/settings.jsonindex d0f0719..14a507a 100644--- a/.claude/settings.json+++ b/.claude/settings.json@@ -8,18 +8,19 @@"Bash(docker compose start:*)","Bash(docker compose ps:*)","Bash(docker ps:*)",- "Bash(curl http://localhost:8080:*)",- "Bash(curl -s http://localhost:8080:*)",- "Bash(curl http://127.0.0.1:8080:*)",- "Bash(curl -s -X POST http://localhost:8080:*)",- "Bash(curl -s -X DELETE http://localhost:8080:*)",- "Bash(curl -X DELETE http://localhost:8080:*)",+ "Bash(curl http://localhost:8080/:*)",+ "Bash(curl -s http://localhost:8080/:*)",+ "Bash(curl http://127.0.0.1:8080/:*)",+ "Bash(curl -s -X POST http://localhost:8080/:*)",+ "Bash(curl -s -X DELETE http://localhost:8080/:*)",+ "Bash(curl -X DELETE http://localhost:8080/:*)","Bash(docker restart gmaps-scraper)","Bash(python3 scripts/test_scrape.py)","Bash(./scripts/scrape.sh:*)","Bash(bash scripts/scrape.sh:*)","Bash(python3 scripts/scrape.py:*)",- "Bash(python scripts/scrape.py:*)"+ "Bash(python scripts/scrape.py:*)",+ "Bash(py scripts/scrape.py:*)"],"deny": []}diff --git a/scripts/scrape.py b/scripts/scrape.pyindex 4d184e8..5f91c91 100755--- a/scripts/scrape.py+++ b/scripts/scrape.py@@ -45,17 +45,21 @@ def load_env_file():port needs one edit. Variables already set in the environment win over the file."""path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, ".env")try:- with open(path) as f:+ with open(path, encoding="utf-8-sig", errors="replace") as f:for line in f:- k, sep, v = line.strip().partition("=")+ line = line.strip()+ if line.startswith("export "): # shell-style .env files+ line = line[7:]+ k, sep, v = line.partition("=")+ k, v = k.strip(), v.split(" #", 1)[0].strip().strip("\"'") # drop a trailing commentif sep and k in ENV_KEYS and k not in os.environ:- os.environ[k] = v.strip().strip("\"'")+ os.environ[k] = vexcept OSError:passload_env_file()-BASE = os.environ.get("SCRAPER_BASE_URL") or f"http://localhost:{os.environ.get('SCRAPER_PORT') or '8080'}"+BASE = (os.environ.get("SCRAPER_BASE_URL") or f"http://localhost:{os.environ.get('SCRAPER_PORT') or '8080'}").rstrip("/")KEY = os.environ.get("SCRAPER_API_KEY", "")# The 8 lead fields: what you use to contact and qualify a lead.# Everything else (geo coordinates, IDs, hours, images, review blobs) is dropped by default.@@ -148,31 +152,36 @@ def in_bbox(row, bbox, margin=0.10):return s - my <= lat <= n + my and w - mx <= lon <= e + mx-def _keys(r):- """Every identity a row carries: place_id, cid, link and title+address. A lean 8-column CSV- from an earlier run has only the last one, so matching on any key lets --exclude use it."""+def _keys(r, weak=False):+ """The identities a row carries: place_id, cid, link, name+address. With weak=True also+ name+phone and name+website, which a --fields results file may be the only thing to match on.+ They are weak because two branches of a chain share a website and sometimes a phone, so they+ never dedupe rows within a run, only match an exclusion list."""ks = [f"{k}:{r[k]}" for k in ("place_id", "cid", "link") if r.get(k)]t = r.get("title")- if t: # name plus address, and name plus phone or website: a --fields file may lack the address+ if t:ks.append(f"ta:{t}|{r.get('address', '')}")- ks += [f"t{c[0]}:{t}|{r[c]}" for c in ("phone", "website") if r.get(c)]+ if weak:+ ks += [f"t{c[0]}:{t}|{r[c]}" for c in ("phone", "website") if r.get(c)]return ksdef dedupe(rows, exclude=()):"""Keep the first row per place; drop rows already present in `exclude` (an earlier run)."""- seen, out = {k for r in exclude for k in _keys(r)}, []+ seen, out = {k for r in exclude for k in _keys(r, weak=True)}, []for r in rows:- ks = _keys(r)- if not any(k in seen for k in ks):- seen.update(ks); out.append(r)+ if not any(k in seen for k in _keys(r, weak=True)):+ seen.update(_keys(r)); out.append(r)return outdef load_rows(path):"""Rows from an earlier results file, .csv or .json."""with open(path, encoding="utf-8-sig", newline="") as f: # -sig: also reads the kit's own CSVs- return json.load(f) if path.lower().endswith(".json") else list(csv.DictReader(f))+ rows = json.load(f) if path.lower().endswith(".json") else list(csv.DictReader(f))+ if not isinstance(rows, list) or not all(isinstance(r, dict) for r in rows):+ raise ValueError(f"{path} is not a results file (expected a list of rows)")+ return rowsdef _local_docker():@@ -186,10 +195,10 @@ def engine_run_ended(since):return Falsetry:out = subprocess.run(["docker", "logs", "--since", since, ENGINE_CONTAINER],- capture_output=True, text=True, timeout=15)+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)except (OSError, subprocess.SubprocessError):return False- return "scrapemate exited" in out.stdout + out.stderr+ return out.returncode == 0 and "scrapemate exited" in out.stdout + out.stderrdef restart_engine():"""Restart the local engine and wait until its API answers again. Without the wait, the next@@ -240,10 +249,11 @@ def engine_idle(seconds=45):return Falsetry:out = subprocess.run(["docker", "logs", "--since", f"{seconds}s", ENGINE_CONTAINER],- capture_output=True, text=True, timeout=15)+ capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)except (OSError, subprocess.SubprocessError):return False- return "scrapemate" not in out.stdout + out.stderr+ # A failing `docker logs` (wrong container name, daemon down) must not read as "idle".+ return out.returncode == 0 and "scrapemate" not in out.stdout + out.stderrclass JobFailed(RuntimeError):@@ -263,6 +273,20 @@ def run_job(body):time.sleep(FAILED_RETRY_WAIT)+def _get(path):+ """GET with three tries: a poll or a download must survive a passing 502 or timeout."""+ for attempt in range(3):+ try:+ return req("GET", path)+ except urllib.error.HTTPError as e:+ if e.code < 500 or attempt == 2:+ raise+ except OSError:+ if attempt == 2:+ raise+ time.sleep(3)++def _run_job_once(body, restarted=False):since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())try:@@ -276,14 +300,15 @@ def _run_job_once(body, restarted=False):attempts = poll_attempts(body.get("max_time"))print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")- status, ended, pending = None, 0, 0+ status, ended, pending, n_rows, last_rows = None, 0, 0, None, Nonefor i in range(attempts):- _, raw = req("GET", f"/api/v1/jobs/{job_id}")+ _, raw = _get(f"/api/v1/jobs/{job_id}")status = json.loads(raw).get("Status")progress = ""if status == "working": # the engine writes rows as it goes, so the download shows progresstry:- progress = f", {len(download_rows(job_id))} rows so far"+ n_rows = len(download_rows(job_id))+ progress = f", {n_rows} rows so far"except Exception:passprint(f"\r status: {str(status):<10} (attempt {i + 1}{progress})", end="", flush=True)@@ -291,13 +316,16 @@ def _run_job_once(body, restarted=False):print(); breakif status == "failed":raise JobFailed("job failed")- if status == "working" and engine_run_ended(since):+ # Stuck (gosom#346): the engine log says the run ended and the row count has stopped moving.+ # The second condition guards against a Docker VM clock that runs ahead of the host.+ if status == "working" and n_rows == last_rows and engine_run_ended(since):ended += 1if ended > 2: # the status normally flips within seconds of the run endingreturn job_id, recover_stuck_job(job_id)+ last_rows = n_rowsif status == "pending":pending += 1- if pending == PENDING_RESTART_AFTER and not restarted and engine_idle():+ if pending % PENDING_RESTART_AFTER == 0 and not restarted and engine_idle():# Delete the pending job first, or the restarted engine runs it as well as the new one.print("\n⚠ The engine has not started this job and shows no activity: an earlier job left it ""stuck (gosom/google-maps-scraper#346). Restarting the engine and re-creating the job…")@@ -305,8 +333,10 @@ def _run_job_once(body, restarted=False):req("DELETE", f"/api/v1/jobs/{job_id}")except Exception:pass- if restart_engine():- return _run_job_once(body, restarted=True)+ if not restart_engine():+ raise RuntimeError(f"\n✗ The engine never started this job and could not be restarted. "+ f"Run: docker restart {ENGINE_CONTAINER}")+ return _run_job_once(body, restarted=True)time.sleep(POLL_EVERY)else:# The engine writes rows as it goes, and v1.18.x can leave a finished job stuck on@@ -328,7 +358,7 @@ def _run_job_once(body, restarted=False):def download_rows(job_id):- _, raw = req("GET", f"/api/v1/jobs/{job_id}/download")+ _, raw = _get(f"/api/v1/jobs/{job_id}/download")return list(csv.DictReader(io.StringIO(raw.decode("utf-8", "replace"))))@@ -339,10 +369,11 @@ def collect_keywords(a):kws.extend(a.also or [])if a.keywords_file:try:- with open(a.keywords_file) as f:- kws.extend(line.strip() for line in f if line.strip() and not line.startswith("#"))- except OSError as e:- sys.exit(f"✗ Cannot read --keywords-file {a.keywords_file}: {e.strerror}")+ with open(a.keywords_file, encoding="utf-8-sig") as f:+ lines = [line.strip() for line in f]+ kws.extend(line for line in lines if line and not line.startswith("#"))+ except (OSError, UnicodeDecodeError) as e:+ sys.exit(f"✗ Cannot read --keywords-file {a.keywords_file}: {getattr(e, 'strerror', e)}")return list(dict.fromkeys(kws)) # de-dupe, keep order@@ -429,9 +460,9 @@ def _fetch_html(url, timeout=10):return ""if not url.startswith(("http://", "https://")):url = "http://" + url- if not _is_public_host(urllib.parse.urlparse(url).hostname or ""):- return ""try:+ if not _is_public_host(urllib.parse.urlparse(url).hostname or ""):+ return ""r = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; " + UA + ")", "Accept": "text/html"})with _OPENER.open(r, timeout=timeout) as resp:@@ -439,7 +470,7 @@ def _fetch_html(url, timeout=10):except Exception:return ""[email protected]_cache(maxsize=512) # ponytail: homepages only; evicted pages just refetch[email protected]_cache(maxsize=256) # ponytail: homepages only (up to 400 KB each); evicted pages just refetchdef _fetch_cached(url):"""Homepage fetch shared by the email crawl, --socials and --tech (one fetch per site)."""return _fetch_html(url, timeout=8)@@ -458,13 +489,20 @@ def _find_socials(html):return outdef _each_site(todo, work, label, workers=None):- """Run work(row) over the rows, SITE_WORKERS at a time, with one progress line."""- done = 0+ """Run work(row) over the rows, SITE_WORKERS at a time, with one progress line. One row that+ blows up (a malformed website, an odd page) is skipped; it never aborts the run."""+ done, failed = 0, 0+ def safe(r):+ try:+ work(r)+ except Exception:+ return 1+ return 0with ThreadPoolExecutor(max_workers=workers or SITE_WORKERS) as ex:- for _ in ex.map(work, todo):- done += 1+ for f in ex.map(safe, todo):+ done, failed = done + 1, failed + fprint(f"\r {label}: {done}/{len(todo)} checked", end="", flush=True)- print()+ print(f" ({failed} skipped after an error)" if failed else "")def enrich_sites(results, find, fields, label, workers=None):"""Merge find(homepage html) into every lead that has a website. Zero LLM tokens."""@@ -525,7 +563,10 @@ def intent_score(r):# ── Email crawl: each business website's homepage, then its contact/about pages ──-EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")+# The lookbehind pins each match attempt to the start of a run of local-part characters: without+# it the scan is quadratic, and a 60 KB run of "a" (a minified blob) took a minute per page.+_LOCAL = r"[A-Za-z0-9._%+-]"+EMAIL_RE = re.compile(rf"(?<!{_LOCAL}){_LOCAL}+@[A-Za-z0-9.-]+\.[A-Za-z]{{2,}}")MAILTO_RE = re.compile(r"mailto:([^\"'>\s?]+)", re.I)HREF_RE = re.compile(r"<a\s[^>]*?href\s*=\s*[\"']([^\"']+)[\"'][^>]*>(.*?)</a>", re.I | re.S)_JUNK_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".avif", ".css", ".js")@@ -542,7 +583,10 @@ _ESCAPE_PREFIX = re.compile(r"^(?:u00(?:3c|3e|22|27|26))+")def _host(url):if not url.startswith(("http://", "https://")):url = "http://" + url- h = (urllib.parse.urlparse(url).hostname or "").lower()+ try:+ h = (urllib.parse.urlparse(url).hostname or "").lower()+ except ValueError: # "http://[bad": a malformed website value from a listing+ return ""return h[4:] if h.startswith("www.") else h# Google Business Profile links often carry campaign tags (?utm_source=gbp&utm_medium=organic),@@ -554,10 +598,17 @@ def clean_website(url):them (store?storeid=...)."""if not url:return ""- p = urllib.parse.urlsplit(url)- q = [(k, v) for k, v in urllib.parse.parse_qsl(p.query, keep_blank_values=True)- if not k.lower().startswith(_TRACKING_PARAMS)]- return urllib.parse.urlunsplit(p._replace(query=urllib.parse.urlencode(q), fragment=""))+ try:+ p = urllib.parse.urlsplit(url)+ except ValueError: # malformed: leave it as the listing had it+ return url+ q = urllib.parse.parse_qsl(p.query, keep_blank_values=True)+ kept = [(k, v) for k, v in q if not k.lower().startswith(_TRACKING_PARAMS)]+ frag = p.fragment if p.fragment.startswith(("/", "!")) else "" # keep hash-routed store pages+ if len(kept) == len(q) and frag == p.fragment:+ return url # nothing to drop: never re-encode a query the site chose+ query = p.query if len(kept) == len(q) else urllib.parse.urlencode(kept)+ return urllib.parse.urlunsplit(p._replace(query=query, fragment=frag))def is_real_email(e):"""False for anything not worth sending to: a template placeholder ([email protected],@@ -575,13 +626,13 @@ _AT = r"(?:\[\s*at\s*\]|\(\s*at\s*\)|\{\s*at\s*\})"_DOT = r"(?:\[\s*dot\s*\]|\(\s*dot\s*\)|\{\s*dot\s*\}|\s+dot\s+)"_TLD = (r"(?:com|net|org|co|io|biz|info|us|uk|ca|au|de|fr|es|it|nl|eu|edu|gov|me|app|dev|"r"shop|online|store|xyz|ai)")-_DEOBF_RE = re.compile(rf"([A-Za-z0-9._%+\-]+)\s*{_AT}\s*([A-Za-z0-9.\-]+?)\s*{_DOT}\s*({_TLD})\b", re.I)+_DEOBF_RE = re.compile(rf"(?<!{_LOCAL})({_LOCAL}+)\s*{_AT}\s*([A-Za-z0-9.\-]+?)\s*{_DOT}\s*({_TLD})\b", re.I)def extract_emails(html, site_host):"""Emails from mailto: links, plain text and obfuscated spans, junk dropped, site-domain first,max 3. HTML entities are decoded first, so info@site.com counts."""site = _host(site_host or "")- text = _html.unescape(html or "")+ text = _html.unescape(html or "")[:400_000]found = []deobf = [f"{m[0]}@{m[1]}.{m[2]}" for m in _DEOBF_RE.findall(text)]for e in MAILTO_RE.findall(text) + EMAIL_RE.findall(text) + deobf:@@ -597,10 +648,15 @@ def contact_links(html, base_url):if not base_url.startswith(("http://", "https://")):base_url = "http://" + base_urlhost, out = _host(base_url), []+ if not host: # malformed or empty website: nothing to join the paths onto+ return outfor href, text in HREF_RE.findall(html or ""):if not re.search(r"contact|about|team|staff|support", href + " " + text, re.I):continue- url = urllib.parse.urldefrag(urllib.parse.urljoin(base_url, href.strip()))[0]+ try:+ url = urllib.parse.urldefrag(urllib.parse.urljoin(base_url, href.strip()))[0]+ except ValueError: # a malformed href on the page+ continueif url.startswith(("http://", "https://")) and _host(url) == host and url not in out:out.append(url)for path in ("/contact", "/contact-us", "/about", "/about-us"):@@ -781,6 +837,21 @@ def main():if len(a.lang) != 2 or not a.lang.isalpha():sys.exit(f"✗ --lang must be a 2-letter code (got {a.lang!r}); the engine rejects anything else.")+ if a.depth < 1:+ sys.exit("✗ --depth must be 1 or more.")+ if a.max_time < 1:+ sys.exit("✗ --max-time must be 1 or more seconds (the engine runs a job for at least 180).")+ if a.limit is not None and a.limit < 1:+ sys.exit("✗ --limit must be 1 or more.")+ if a.has_email and not a.email:+ sys.exit("✗ --has-email needs the email crawl: drop --no-email.")+ if a.city and (a.lat or a.lon):+ print("ℹ --city given, so the lat/lon arguments are ignored.")+ if a.out and os.path.dirname(a.out): # fail now, not after the scrape, if the folder is unusable+ try:+ os.makedirs(os.path.dirname(a.out), exist_ok=True)+ except OSError as e:+ sys.exit(f"✗ Cannot create the folder for --out: {e}")if a.grid is not None:if not a.city:@@ -807,6 +878,10 @@ def main():try:req("GET", "/api/v1/jobs")break+ except urllib.error.HTTPError as e:+ if e.code < 500: # the engine is up but refuses us (wrong API key, wrong URL): say so at once+ sys.exit(f"✗ The engine at {BASE} answered HTTP {e.code} to the health check: {e.read().decode()[:200]}")+ err = eexcept Exception as e:err = eprint("\r waiting for the engine to start" + "." * (i + 1), end="", flush=True)@@ -825,13 +900,16 @@ def main():"depth": a.depth, "email": False, "max_time": a.max_time}if a.proxy:body["proxies"] = a.proxy- job_id, rows, skipped, ncells = None, [], 0, 0+ job_id, rows, skipped, ncells, failed_places = None, [], 0, 0, 0for city in cities: # one place at a time, like the grid tilesjid, place_rows, s, n = scrape_place(a, body, city, keywords)job_id, skipped, ncells = job_id or jid, skipped + s, ncells + n+ failed_places += jid is Nonerows = dedupe(rows + place_rows)if job_id is None:sys.exit("✗ No job finished. You may be rate-limited — wait or add proxies.")+ if failed_places:+ print(f"⚠ Incomplete: {failed_places} of {len(cities)} places returned nothing (see the ✗ lines above).")for r in rows:r["website"] = clean_website(r.get("website", ""))before = len(rows)@@ -846,6 +924,8 @@ def main():fields = [c.strip() for c in a.fields.split(",") if c.strip()]else:fields = LEAD+ if a.has_email and "emails" not in fields:+ sys.exit("✗ --has-email needs the emails column: add emails to --fields.")print(f"✓ Done — {len(rows)} businesses (fields: {', '.join(fields)}).")if skipped:print(f"⚠ Incomplete: {skipped} of {ncells} tiles failed, so parts of the area are missing. Rerun later.")@@ -865,40 +945,49 @@ def main():if a.has_email:rows = [r for r in rows if r.get("emails")][:a.limit or None]print(f" --has-email kept {len(rows)} rows")- results = [{k: r.get(k, "") for k in fields} for r in rows]+ # Enrichment runs on the raw rows, which always carry the website, whatever --fields keeps.if a.socials:print("▶ Finding socials (Instagram / Facebook / LinkedIn / TikTok / YouTube / X) on each website…")print(" ℹ Cost: this runs in CODE = 0 LLM tokens. It only adds ~40–50 tokens per business")print(" IF you load the results into an AI's context (negligible if you keep the file on disk).")- enrich_sites(results, _find_socials, SOCIAL_FIELDS, "socials")+ enrich_sites(rows, _find_socials, SOCIAL_FIELDS, "socials")fields = fields + SOCIAL_FIELDS- found = sum(1 for r in results if any(r.get(k) for k in SOCIAL_FIELDS))- print(f" socials found for {found}/{len(results)} businesses")+ found = sum(1 for r in rows if any(r.get(k) for k in SOCIAL_FIELDS))+ print(f" socials found for {found}/{len(rows)} businesses")if a.tech:print("▶ Checking ad pixels, analytics and site platform on each website (0 LLM tokens)…")- enrich_sites(results, _find_tech, TECH_FIELDS, "tech")- for r in results:+ enrich_sites(rows, _find_tech, TECH_FIELDS, "tech")+ for r in rows:r["intent_score"] = intent_score(r)- results.sort(key=lambda r: r["intent_score"], reverse=True)+ rows.sort(key=lambda r: r["intent_score"], reverse=True)fields = fields + TECH_FIELDS + ["intent_score"]- ads = sum(1 for r in results if "yes" in (r.get("fb_pixel"), r.get("google_ads_tag")))- print(f" Meta Pixel or Google Ads tag on {ads}/{len(results)} websites; "+ ads = sum(1 for r in rows if "yes" in (r.get("fb_pixel"), r.get("google_ads_tag")))+ print(f" Meta Pixel or Google Ads tag on {ads}/{len(rows)} websites; "f"rows sorted by intent_score (highest first)")+ results = [{k: r.get(k, "") for k in fields} for r in rows]# Default output is a CSV file (opens in Excel / Google Sheets). Use --json (or a .json --out path) for JSON.as_json = a.json or (a.out and a.out.lower().endswith(".json"))out = a.out or f"results-{job_id[:8]}.{'json' if as_json else 'csv'}"- if as_json:- with open(out, "w", encoding="utf-8") as f:- json.dump(results, f, indent=2, ensure_ascii=False)- else:- # utf-8-sig: the BOM makes Excel on Windows read accented names as UTF-8 instead of cp1252.- with open(out, "w", newline="", encoding="utf-8-sig") as f:- w = csv.DictWriter(f, fieldnames=fields)- w.writeheader()- w.writerows(results)+ def write(path):+ if as_json:+ with open(path, "w", encoding="utf-8") as f:+ json.dump(results, f, indent=2, ensure_ascii=False)+ else:+ # utf-8-sig: the BOM makes Excel on Windows read accented names as UTF-8 instead of cp1252.+ with open(path, "w", newline="", encoding="utf-8-sig") as f:+ w = csv.DictWriter(f, fieldnames=fields)+ w.writeheader()+ w.writerows(results)+ try:+ write(out)+ except OSError as e: # the target is locked (open in Excel) or unwritable: never lose a finished scrape+ fallback = f"results-{job_id[:8]}.{'json' if as_json else 'csv'}"+ print(f"⚠ Could not write {out} ({e.strerror}); writing {fallback} instead.")+ out = fallback+ write(out)print(f" saved → {os.path.abspath(out)} (opens in Excel or Google Sheets)")for r in results[:5]:tail = f" | IG:{r.get('instagram','') or '—'}" if a.socials else f" | {r.get('website','')}"@@ -906,4 +995,7 @@ def main():if __name__ == "__main__":- main()+ try:+ main()+ except KeyboardInterrupt:+ sys.exit(f"\n✗ Stopped. A job already created keeps running in the engine; list jobs at {BASE}/api/v1/jobs")diff --git a/scripts/scrape.sh b/scripts/scrape.shindex f175e6c..a4ae611 100755--- a/scripts/scrape.sh+++ b/scripts/scrape.sh@@ -14,9 +14,17 @@ if [ -z "$KEYWORD" ] || [ -z "$LAT" ] || [ -z "$LON" ]; thenexit 1fi-# Git Bash on Windows often has only `python`.-PY="$(command -v python3 || command -v python || true)"+# Git Bash on Windows often has only `python`, and `python3` there can be the Microsoft Store stub+# that only prints a message, so take the first name that really runs.+PY=""+for p in python3 python py; do+ command -v "$p" >/dev/null 2>&1 && "$p" -c 'import sys' >/dev/null 2>&1 && { PY="$p"; break; }+done[ -n "$PY" ] || { echo "✗ Python 3 is required (standard library only, no pip installs)." >&2; exit 1; }-OUT="results-$(echo "$KEYWORD" | tr ' /' '__' | tr -cd 'A-Za-z0-9_-').csv"+# File name from the keyword; a keyword with no ASCII letters (寿司 in 東京) falls back to a timestamp+# so reruns never overwrite each other.+SLUG="$(printf '%s' "$KEYWORD" | tr ' /' '__' | tr -cd 'A-Za-z0-9_-')"+[ -n "$SLUG" ] || SLUG="$(date +%Y%m%d-%H%M%S)"+OUT="results-$SLUG.csv"exec "$PY" "$(dirname "$0")/scrape.py" "$KEYWORD" "$LAT" "$LON" --depth "$DEPTH" --out "$OUT"diff --git a/scripts/test_scrape.py b/scripts/test_scrape.pyindex 1552da7..347691d 100644--- a/scripts/test_scrape.py+++ b/scripts/test_scrape.py@@ -123,11 +123,11 @@ class StuckJob(unittest.TestCase):self.run_stuck(b"")def test_ended_run_recovered_in_seconds_and_engine_restarted(self):- # The engine log says the run ended but the status never flips: stop after 3 polls- # (about 24 s) instead of the whole budget, keep the rows, restart the engine.+ # The engine log says the run ended and the row count has stopped moving for 3 polls: stop+ # after 4 polls (about 32 s) instead of the whole budget, keep the rows, restart the engine.rows, polls, restarted, deleted = self.run_stuck(b"title\nA\n", run_ended=True, attempts=90)self.assertEqual([r["title"] for r in rows], ["A"])- self.assertEqual(polls, 3)+ self.assertEqual(polls, 4)self.assertTrue(restarted)self.assertEqual(deleted, ["/api/v1/jobs/stuck"]) # no longer a "working" ghost in the job list@@ -349,6 +349,8 @@ class CleanWebsite(unittest.TestCase):"https://www.hpb.com/store?storeid=HPB-005")self.assertEqual(scrape.clean_website("http://plain.com/"), "http://plain.com/")self.assertEqual(scrape.clean_website(""), "")+ self.assertEqual(scrape.clean_website("http://x.com/?a=1;b=2#top"), "http://x.com/?a=1;b=2") # never re-encoded+ self.assertEqual(scrape.clean_website("http://x.com/#/store/5"), "http://x.com/#/store/5") # hash routing keptclass KeywordsFile(unittest.TestCase):@@ -384,7 +386,7 @@ FIELDS = ["input_id", "link", "title", "category", "address", "website", "phone""review_rating", "latitude", "longitude", "cid", "place_id", "emails"]def _row(pid, lat="30.2"):- return {"title": f"Biz {pid}", "place_id": pid, "cid": "c" + pid, "latitude": lat,+ return {"title": f"Biz {pid}", "place_id": pid, "cid": "c" + pid, "latitude": lat, "website": f"http://{pid.lower()}.biz","longitude": "-97.7", "phone": "+1 555", "emails": f"{pid.lower()}@biz.net"}CELL_ROWS = {"job-1": [_row("A"), _row("B"), _row("C")],@@ -466,11 +468,12 @@ class EndToEnd(unittest.TestCase):w = csv.DictWriter(f, fieldnames=scrape.LEAD); w.writeheader()w.writerow({"title": "Biz A", "address": ""})argv = ["scrape.py", "plumbers", "--city", "Austin, TX", "--city", "Round Rock, TX", "--no-email",- "--exclude", earlier, "--limit", "2", "--out", out]+ "--exclude", earlier, "--limit", "2", "--out", out, "--socials", "--fields", "title,phone"]log = io.StringIO()try:with mock.patch.object(scrape, "BASE", f"http://127.0.0.1:{srv.server_port}"), \mock.patch.object(scrape, "geocode", return_value=("30.25", "-97.75")) as geo, \+ mock.patch.object(scrape, "_fetch_cached", return_value='<a href="https://instagram.com/biz_b">ig</a>'), \mock.patch.object(scrape.time, "sleep"), \mock.patch.object(sys, "argv", argv), redirect_stdout(log):scrape.main()@@ -481,6 +484,8 @@ class EndToEnd(unittest.TestCase):with open(out, newline="", encoding="utf-8-sig") as f: # the kit writes a BOM for Excelrows = list(csv.DictReader(f))self.assertEqual([r["title"] for r in rows], ["Biz B", "Biz C"]) # --limit 2+ # --socials works even though --fields dropped the website column: enrichment reads the raw rows+ self.assertEqual(rows[0]["instagram"], "https://instagram.com/biz_b")class Filters(unittest.TestCase):@@ -495,6 +500,14 @@ class Filters(unittest.TestCase):self.assertTrue(scrape.keep_row({"website": ""}, nw))self.assertFalse(scrape.keep_row({"website": "http://x.com"}, nw))+ def test_chain_branches_are_not_merged(self):+ # Two branches of a chain share the website (and sometimes the phone): distinct place_ids+ # must both survive, and only an exclusion list may match on name+phone or name+website.+ branches = [{"place_id": "P1", "title": "Anytime Fitness", "address": "1 A St", "website": "http://af.com", "phone": "+1 1"},+ {"place_id": "P2", "title": "Anytime Fitness", "address": "9 B St", "website": "http://af.com", "phone": "+1 1"}]+ self.assertEqual(len(scrape.dedupe(branches)), 2)+ self.assertEqual(len(scrape.dedupe(branches, exclude=[{"title": "Anytime Fitness", "address": "9 B St"}])), 1)+def test_exclude_matches_lean_rows(self):new = [{"place_id": "P1", "title": "Joe's", "address": "1 Main"}, {"place_id": "P2", "title": "Ann's"}]self.assertEqual(scrape.dedupe(new, exclude=[{"title": "Joe's", "address": "1 Main"}]), new[1:])@@ -513,6 +526,56 @@ class Filters(unittest.TestCase):self.assertEqual(scrape.load_rows(c), [{"title": "B", "phone": "1"}]) # BOM stripped, key is clean+class RegexSpeed(unittest.TestCase):+ def test_long_runs_are_linear(self):+ import time as _t+ blob = "a" * 100_000 + " contact: [email protected] " + "b" * 100_000+ t0 = _t.perf_counter()+ self.assertEqual(scrape.extract_emails(blob, "acme.com"), ["[email protected]"])+ self.assertLess(_t.perf_counter() - t0, 1.0) # was about a minute per 60 KB run+++class Resilience(unittest.TestCase):+ def test_get_retries_transient_errors(self):+ answers = iter([scrape.urllib.error.HTTPError("u", 502, "bad gateway", {}, None),+ OSError("timed out"), (200, b"null")])+ def fake_req(method, path, body=None):+ a = next(answers)+ if isinstance(a, Exception):+ raise a+ return a+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep") as slept:+ self.assertEqual(scrape._get("/api/v1/jobs"), (200, b"null"))+ self.assertEqual(slept.call_count, 2)+ with mock.patch.object(scrape, "req", side_effect=scrape.urllib.error.HTTPError("u", 404, "nf", {}, None)), \+ self.assertRaises(scrape.urllib.error.HTTPError):+ scrape._get("/api/v1/jobs/x") # a 4xx is final, no retry++ def test_pending_restart_failure_raises_instead_of_polling_a_deleted_job(self):+ statuses = iter(["pending"] * 20)+ def fake_req(method, path, body=None):+ if method == "POST":+ return 201, b'{"id":"j1"}'+ if method == "DELETE":+ return 200, b""+ return 200, json.dumps({"Status": next(statuses)}).encode()+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \+ mock.patch.object(scrape, "poll_attempts", return_value=50), \+ mock.patch.object(scrape, "engine_run_ended", return_value=False), \+ mock.patch.object(scrape, "engine_idle", return_value=True), \+ mock.patch.object(scrape, "restart_engine", return_value=False), \+ redirect_stdout(io.StringIO()), self.assertRaises(RuntimeError) as cm:+ scrape.run_job({"max_time": 1})+ self.assertIn("docker restart", str(cm.exception))++ def test_failing_docker_logs_is_not_idle(self):+ failed = mock.Mock(returncode=1, stdout="", stderr="Error: No such container")+ with mock.patch.object(scrape, "_local_docker", return_value=True), \+ mock.patch.object(scrape.subprocess, "run", return_value=failed):+ self.assertFalse(scrape.engine_idle())+ self.assertFalse(scrape.engine_run_ended("2026-01-01T00:00:00Z"))++class VerifyEmails(unittest.TestCase):def test_mx_host_labels_and_drops_dead_domains(self):answers = {@@ -536,5 +599,82 @@ class VerifyEmails(unittest.TestCase):self.assertEqual(scrape.mx_host("x.com"), "") # lookup failed: keep the address, no label+class HostileInput(unittest.TestCase):+ """A listing can carry a malformed website; nothing in the crawl may raise on it."""+ BAD = "http://[bad"++ def test_url_helpers_never_raise(self):+ self.assertEqual(scrape.clean_website(self.BAD), self.BAD)+ self.assertEqual(scrape._host(self.BAD), "")+ self.assertEqual(scrape.extract_emails("[email protected]", self.BAD), ["[email protected]"])+ self.assertEqual(scrape._fetch_html(self.BAD), "")+ links = scrape.contact_links('<a href="http://[x">contact</a><a href="/contact">c</a>', self.BAD)+ self.assertTrue(all(isinstance(u, str) for u in links))++ def test_one_bad_row_does_not_abort_the_run(self):+ rows = [{"website": "boom"}, {"website": "fine"}]+ def work(r):+ if r["website"] == "boom":+ raise ValueError("parser blew up")+ r["seen"] = "yes"+ log = io.StringIO()+ with redirect_stdout(log):+ scrape._each_site(rows, work, "x", workers=2)+ self.assertEqual(rows[1].get("seen"), "yes")+ self.assertIn("1 skipped after an error", log.getvalue())++ def test_exclude_file_must_be_a_list_of_rows(self):+ d = tempfile.mkdtemp()+ j = os.path.join(d, "job.json"); open(j, "w").write('{"job": {"keywords": ["x"]}}')+ with self.assertRaises(ValueError):+ scrape.load_rows(j)+++class EnvFile(unittest.TestCase):+ def test_export_prefix_comment_quotes_and_trailing_slash(self):+ d = tempfile.mkdtemp(); os.mkdir(os.path.join(d, "scripts"))+ open(os.path.join(d, ".env"), "w").write(+ "export SCRAPER_PORT=8081 # my port\nSCRAPER_BASE_URL='http://localhost:8081/'\n# SCRAPER_API_KEY=nope\n")+ saved = {k: os.environ.pop(k, None) for k in scrape.ENV_KEYS}+ try:+ with mock.patch.object(scrape, "__file__", os.path.join(d, "scripts", "scrape.py")):+ scrape.load_env_file()+ self.assertEqual(os.environ.get("SCRAPER_PORT"), "8081")+ self.assertEqual(os.environ.get("SCRAPER_BASE_URL"), "http://localhost:8081/")+ self.assertIsNone(os.environ.get("SCRAPER_API_KEY"))+ finally:+ for k, v in saved.items():+ os.environ.pop(k, None)+ if v is not None:+ os.environ[k] = v++ def test_base_url_has_no_trailing_slash(self):+ # "http://host/" + "/api/..." would be "//api/...", which the proxy answers with a 307 that+ # urllib will not follow for a POST.+ with mock.patch.dict(os.environ, {"SCRAPER_BASE_URL": "http://localhost:8090/"}):+ self.assertEqual((os.environ.get("SCRAPER_BASE_URL") or "").rstrip("/"), "http://localhost:8090")+ self.assertFalse(scrape.BASE.endswith("/"))+++class ArgumentBounds(unittest.TestCase):+ def run_main(self, *extra):+ argv = ["scrape.py", "x in y", "1", "2", *extra]+ with mock.patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()), \+ self.assertRaises(SystemExit) as cm:+ scrape.main()+ return str(cm.exception)++ def test_has_email_needs_the_crawl(self):+ self.assertIn("--has-email", self.run_main("--has-email", "--no-email"))++ def test_rejects_values_the_engine_would_choke_on(self):+ self.assertIn("--depth", self.run_main("--depth", "0"))+ self.assertIn("--max-time", self.run_main("--max-time", "0")) # the engine says "missing max time"+ self.assertIn("--limit", self.run_main("--limit", "-1"))+ bad_dir = os.path.join(tempfile.mkdtemp(), "a-file")+ open(bad_dir, "w").close() # a file where the folder should be+ self.assertIn("--out", self.run_main("--out", os.path.join(bad_dir, "x", "leads.csv")))++if __name__ == "__main__":unittest.main(verbosity=2)