Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(kit): ECC review small bugs: bad 1.x proxy, huge coordinates, junk-domain substrings, early --has-email, filtered Ctrl-C file; CI on every Python 3.9 to 3.14

Mahan Jafari3b93598f8dcc
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index fdeb5f8..8ea481c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -15,10 +15,13 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
- python: ["3.12"]
- include: # the oldest Python the docs promise (macOS's own python3 is 3.9)
- - os: ubuntu-latest
- python: "3.9"
+ python: ["3.12", "3.14"]
+ include: # every version the docs promise, 3.9 up, once each; macOS's own python3 3.9 is run by hand
+ - {os: ubuntu-latest, python: "3.9"}
+ - {os: ubuntu-latest, python: "3.10"}
+ - {os: ubuntu-latest, python: "3.11"}
+ - {os: ubuntu-latest, python: "3.13"}
+ - {os: windows-latest, python: "3.9"}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
diff --git a/engine/jobs.py b/engine/jobs.py
index 078f330..a47f0b2 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -145,7 +145,7 @@ def _coords(lat, lon):
if isinstance(lat, bool) or isinstance(lon, bool):
raise ValueError
flat, flon = float(lat), float(lon)
- except (TypeError, ValueError):
+ except (TypeError, ValueError, OverflowError): # OverflowError: a JSON integer of 400 digits
raise ValueError("invalid geo coordinates") from None
if not (math.isfinite(flat) and math.isfinite(flon) and -90 <= flat <= 90 and -180 <= flon <= 180):
raise ValueError("invalid geo coordinates")
@@ -179,7 +179,10 @@ def _proxies(v):
def mask(proxy):
"""A proxy URL with its password replaced by *** (or its username, when that is the only
secret, as some providers do): what the API and the web UI show."""
- u = urllib.parse.urlsplit(proxy)
+ try:
+ u = urllib.parse.urlsplit(proxy)
+ except ValueError: # "http://[abc" from a 1.x job: validate() never saw it; it must not break the job list
+ return "***"
if not (u.username or u.password):
return proxy
user, host = u.netloc.rpartition("@")[0], u.netloc.rpartition("@")[2]
@@ -863,7 +866,14 @@ class Worker(threading.Thread):
# ── Website pass (email, socials, verify_emails): each business homepage over plain HTTP ──
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")
JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")
-JUNK_DOMAIN = ("example.", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")
+JUNK_DOMAIN = ("example", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")
+
+
+def junk_domain(dom, junk=JUNK_DOMAIN):
+ """An entry with a dot is a domain and its subdomains, one without is a whole label: sentry.io
+ and [email protected] are junk, sentryinsurance.com and myemail.com are not. scripts/scrape.py has the same."""
+ labels = dom.split(".")
+ return any(dom == j or dom.endswith("." + j) if "." in j else j in labels for j in junk)
UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"
# The same patterns as scripts/scrape.py --socials (a test keeps them in step).
SOCIAL_RE = {
@@ -1107,7 +1117,7 @@ def find_emails(text, site=""):
for e in EMAIL_RE.findall(html.unescape(text or "")):
e = urllib.parse.unquote(e).lower()
dom = e.rpartition("@")[2]
- if not EMAIL_RE.fullmatch(e) or e.endswith(JUNK_SUFFIX) or any(j in dom for j in JUNK_DOMAIN) or e in found:
+ if not EMAIL_RE.fullmatch(e) or e.endswith(JUNK_SUFFIX) or junk_domain(dom) or e in found:
continue # "a%[email protected]" unquotes to a newline: not an address
found.append(e)
own = lambda e: site and (e.endswith("@" + site) or e.endswith("." + site))
diff --git a/engine/test_engine.py b/engine/test_engine.py
index bf93c4d..39e2215 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -175,7 +175,10 @@ class Validate(unittest.TestCase):
self.assertEqual((d["country"], jobs.locale(d), d["proxies"]), ("de", "en-DE", ["http://u%40x:p%3A@h:1"]))
self.assertEqual(jobs.mask("http://u%40x:p%3A@h:1"), "http://u%40x:***@h:1")
self.assertEqual(jobs.mask("socks5://h:1"), "socks5://h:1")
+ self.assertEqual(jobs.mask("http://[abc"), "***") # a 1.x job's proxy validate() never saw
self.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)
+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en", max_time=60, lat=10 ** 400, lon=1),
+ "invalid geo coordinates") # float() overflows: a 422, not a dropped connection
class Rows(unittest.TestCase):
@@ -392,6 +395,8 @@ class Emails(unittest.TestCase):
def test_find_emails(self):
self.assertEqual(jobs.find_emails(text, "shop.com"), ["[email protected]", "[email protected]"])
+ ["[email protected]", "[email protected]"]) # whole labels, not substrings
def test_public_only(self):
self.assertFalse(jobs._public_ip("127.0.0.1"))
diff --git a/scripts/scrape.py b/scripts/scrape.py
index 444b36e..afdc37f 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -769,7 +769,14 @@ def is_real_email(e):
local, _, dom = e.rpartition("@")
if not EMAIL_RE.fullmatch(e) or e.endswith(_JUNK_EXT):
return False
- return local not in _JUNK_LOCAL and not any(j in dom for j in _JUNK_DOMAIN)
+ return local not in _JUNK_LOCAL and not junk_domain(dom)
+
+
+def junk_domain(dom, junk=_JUNK_DOMAIN):
+ """An entry with a dot is a domain and its subdomains, one without is a whole label: sentry.io
+ and [email protected] are junk, sentryinsurance.com and myemail.com are not. engine/jobs.py has the same."""
+ labels = dom.split(".")
+ return any(dom == j or dom.endswith("." + j) if "." in j else j in labels for j in junk)
# "name [at] domain [dot] com", "name (at) domain dot com" and friends, adapted from OpenLeadFinder.
@@ -928,6 +935,7 @@ def scrape_place(a, body, city, keywords):
print(f"\n ⚠ cell {i}/{len(cells)} incomplete: {str(e).strip()}")
skipped += 1
job_id, rows = job_id or e.job_id, dedupe(rows + e.rows)
+ SALVAGE["rows"], SALVAGE["job"] = dedupe(SALVAGE["rows"] + rows), SALVAGE["job"] or job_id
continue
except Exception as e:
print(f"\n ⚠ cell {i}/{len(cells)} skipped: {str(e).strip()}")
@@ -1030,6 +1038,8 @@ def main():
sys.exit("✗ --limit must be 1 or more.")
if a.has_email and not a.email:
sys.exit("✗ --has-email needs the email crawl: drop --no-email.")
+ if a.has_email and a.fields and not a.full and "emails" not in [c.strip() for c in a.fields.split(",")]:
+ sys.exit("✗ --has-email needs the emails column: add emails to --fields.")
if a.city and (a.lat or a.lon):
print("ℹ --city given, so the lat/lon arguments are ignored.")
if a.out and os.path.splitext(a.out)[1].lower() not in (".csv", ".json"): # never a .zshrc or a key file
@@ -1126,6 +1136,7 @@ def main():
print(f" filters and --exclude kept {len(rows)} of {before} rows")
if a.limit and not a.has_email:
rows = rows[:a.limit]
+ SALVAGE["rows"] = rows # Ctrl-C in the crawl below keeps the filtered rows, not the raw ones
if a.full:
fields = list(rows[0].keys()) if rows else LEAD
elif a.fields:
@@ -1134,8 +1145,6 @@ def main():
fields = LEAD
if a.reviews and not a.full: # the page loads were paid for: keep what they read
fields = fields + [c for c in ("reviews_per_rating", "user_reviews") if c not in fields]
- if a.has_email and "emails" not in fields:
- sys.exit("✗ --has-email needs the emails column: add emails to --fields.")
if before and not rows:
print(f"⚠ Nothing left: the filters and --exclude dropped all {before} rows. Loosen them and rerun; "
f"the file below has only a header.")
diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py
index 97f685f..00f2396 100644
--- a/scripts/test_scrape.py
+++ b/scripts/test_scrape.py
@@ -889,6 +889,23 @@ class ArgumentBounds(unittest.TestCase):
def test_has_email_needs_the_crawl(self):
self.assertIn("--has-email", self.run_main("--has-email", "--no-email"))
+ def test_has_email_needs_the_column_before_the_scrape(self): # no engine here: it must exit first
+ self.assertIn("needs the emails column", self.run_main("--has-email", "--fields", "title,phone"))
+
+ def test_ctrl_c_in_the_crawl_keeps_only_the_filtered_rows(self):
+ good, bad = dict(_row("A"), review_rating="4.8"), dict(_row("B"), review_rating="3.0")
+ argv = ["scrape.py", "x", "1", "2", "--min-rating", "4.5", "--no-socials", "--out",
+ os.path.join(tempfile.mkdtemp(), "leads.csv")]
+ with mock.patch.object(scrape, "run_job", return_value=("job-1", [good, bad])), \
+ mock.patch.object(scrape, "enrich_emails", side_effect=KeyboardInterrupt), \
+ mock.patch.dict(scrape.SALVAGE, {"rows": [], "job": None}), \
+ mock.patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()), self.assertRaises(KeyboardInterrupt):
+ try:
+ scrape.main()
+ finally:
+ titles = [r["title"] for r in scrape.SALVAGE["rows"]]
+ self.assertEqual(titles, ["Biz A"])
+
def test_rejects_values_the_engine_would_choke_on(self):
self.assertIn("--country", self.run_main("--country", "usa"))
self.assertIn("--depth", self.run_main("--depth", "0"))
@@ -904,6 +921,14 @@ class ArgumentBounds(unittest.TestCase):
self.assertIn("--out must name a .csv or .json file", self.run_main("--out", path), path)
+class JunkDomains(unittest.TestCase):
+ def test_whole_labels_and_domains_only(self):
+ self.assertTrue(scrape.is_real_email(e), e)
+ self.assertFalse(scrape.is_real_email(e), e)
+
+
class ScrapedTextInOutput(unittest.TestCase):
"""What the script prints from the engine and the results reaches an agent: one line, no tricks."""