Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
fix(kit): ECC review small bugs: bad 1.x proxy, huge coordinates, junk-domain substrings, early --has-email, filtered Ctrl-C file; CI on every Python 3.9 to 3.14
Mahan Jafari3b93598f8dcc
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex fdeb5f8..8ea481c 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -15,10 +15,13 @@ jobs:fail-fast: falsematrix:os: [ubuntu-latest, macos-latest, windows-latest]- python: ["3.12"]- include: # the oldest Python the docs promise (macOS's own python3 is 3.9)- - os: ubuntu-latest- python: "3.9"+ python: ["3.12", "3.14"]+ include: # every version the docs promise, 3.9 up, once each; macOS's own python3 3.9 is run by hand+ - {os: ubuntu-latest, python: "3.9"}+ - {os: ubuntu-latest, python: "3.10"}+ - {os: ubuntu-latest, python: "3.11"}+ - {os: ubuntu-latest, python: "3.13"}+ - {os: windows-latest, python: "3.9"}runs-on: ${{ matrix.os }}steps:- uses: actions/checkout@v4diff --git a/engine/jobs.py b/engine/jobs.pyindex 078f330..a47f0b2 100644--- a/engine/jobs.py+++ b/engine/jobs.py@@ -145,7 +145,7 @@ def _coords(lat, lon):if isinstance(lat, bool) or isinstance(lon, bool):raise ValueErrorflat, flon = float(lat), float(lon)- except (TypeError, ValueError):+ except (TypeError, ValueError, OverflowError): # OverflowError: a JSON integer of 400 digitsraise ValueError("invalid geo coordinates") from Noneif not (math.isfinite(flat) and math.isfinite(flon) and -90 <= flat <= 90 and -180 <= flon <= 180):raise ValueError("invalid geo coordinates")@@ -179,7 +179,10 @@ def _proxies(v):def mask(proxy):"""A proxy URL with its password replaced by *** (or its username, when that is the onlysecret, as some providers do): what the API and the web UI show."""- u = urllib.parse.urlsplit(proxy)+ try:+ u = urllib.parse.urlsplit(proxy)+ except ValueError: # "http://[abc" from a 1.x job: validate() never saw it; it must not break the job list+ return "***"if not (u.username or u.password):return proxyuser, host = u.netloc.rpartition("@")[0], u.netloc.rpartition("@")[2]@@ -863,7 +866,14 @@ class Worker(threading.Thread):# ── Website pass (email, socials, verify_emails): each business homepage over plain HTTP ──EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")-JUNK_DOMAIN = ("example.", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")+JUNK_DOMAIN = ("example", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")+++def junk_domain(dom, junk=JUNK_DOMAIN):+ """An entry with a dot is a domain and its subdomains, one without is a whole label: sentry.io+ and [email protected] are junk, sentryinsurance.com and myemail.com are not. scripts/scrape.py has the same."""+ labels = dom.split(".")+ return any(dom == j or dom.endswith("." + j) if "." in j else j in labels for j in junk)UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"# The same patterns as scripts/scrape.py --socials (a test keeps them in step).SOCIAL_RE = {@@ -1107,7 +1117,7 @@ def find_emails(text, site=""):for e in EMAIL_RE.findall(html.unescape(text or "")):e = urllib.parse.unquote(e).lower()dom = e.rpartition("@")[2]- if not EMAIL_RE.fullmatch(e) or e.endswith(JUNK_SUFFIX) or any(j in dom for j in JUNK_DOMAIN) or e in found:+ if not EMAIL_RE.fullmatch(e) or e.endswith(JUNK_SUFFIX) or junk_domain(dom) or e in found:continue # "a%[email protected]" unquotes to a newline: not an addressfound.append(e)own = lambda e: site and (e.endswith("@" + site) or e.endswith("." + site))diff --git a/engine/test_engine.py b/engine/test_engine.pyindex bf93c4d..39e2215 100644--- a/engine/test_engine.py+++ b/engine/test_engine.py@@ -175,7 +175,10 @@ class Validate(unittest.TestCase):self.assertEqual((d["country"], jobs.locale(d), d["proxies"]), ("de", "en-DE", ["http://u%40x:p%3A@h:1"]))self.assertEqual(jobs.mask("http://u%40x:p%3A@h:1"), "http://u%40x:***@h:1")self.assertEqual(jobs.mask("socks5://h:1"), "socks5://h:1")+ self.assertEqual(jobs.mask("http://[abc"), "***") # a 1.x job's proxy validate() never sawself.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en", max_time=60, lat=10 ** 400, lon=1),+ "invalid geo coordinates") # float() overflows: a 422, not a dropped connectionclass Rows(unittest.TestCase):@@ -392,6 +395,8 @@ class Emails(unittest.TestCase):def test_find_emails(self):text = "[email protected] x [email protected] [email protected] [email protected] [email protected] [email protected]"self.assertEqual(jobs.find_emails(text, "shop.com"), ["[email protected]", "[email protected]"])+ self.assertEqual(jobs.find_emails("[email protected] [email protected] [email protected] [email protected]"),+ ["[email protected]", "[email protected]"]) # whole labels, not substringsdef test_public_only(self):self.assertFalse(jobs._public_ip("127.0.0.1"))diff --git a/scripts/scrape.py b/scripts/scrape.pyindex 444b36e..afdc37f 100755--- a/scripts/scrape.py+++ b/scripts/scrape.py@@ -769,7 +769,14 @@ def is_real_email(e):local, _, dom = e.rpartition("@")if not EMAIL_RE.fullmatch(e) or e.endswith(_JUNK_EXT):return False- return local not in _JUNK_LOCAL and not any(j in dom for j in _JUNK_DOMAIN)+ return local not in _JUNK_LOCAL and not junk_domain(dom)+++def junk_domain(dom, junk=_JUNK_DOMAIN):+ """An entry with a dot is a domain and its subdomains, one without is a whole label: sentry.io+ and [email protected] are junk, sentryinsurance.com and myemail.com are not. engine/jobs.py has the same."""+ labels = dom.split(".")+ return any(dom == j or dom.endswith("." + j) if "." in j else j in labels for j in junk)# "name [at] domain [dot] com", "name (at) domain dot com" and friends, adapted from OpenLeadFinder.@@ -928,6 +935,7 @@ def scrape_place(a, body, city, keywords):print(f"\n ⚠ cell {i}/{len(cells)} incomplete: {str(e).strip()}")skipped += 1job_id, rows = job_id or e.job_id, dedupe(rows + e.rows)+ SALVAGE["rows"], SALVAGE["job"] = dedupe(SALVAGE["rows"] + rows), SALVAGE["job"] or job_idcontinueexcept Exception as e:print(f"\n ⚠ cell {i}/{len(cells)} skipped: {str(e).strip()}")@@ -1030,6 +1038,8 @@ def main():sys.exit("✗ --limit must be 1 or more.")if a.has_email and not a.email:sys.exit("✗ --has-email needs the email crawl: drop --no-email.")+ if a.has_email and a.fields and not a.full and "emails" not in [c.strip() for c in a.fields.split(",")]:+ sys.exit("✗ --has-email needs the emails column: add emails to --fields.")if a.city and (a.lat or a.lon):print("ℹ --city given, so the lat/lon arguments are ignored.")if a.out and os.path.splitext(a.out)[1].lower() not in (".csv", ".json"): # never a .zshrc or a key file@@ -1126,6 +1136,7 @@ def main():print(f" filters and --exclude kept {len(rows)} of {before} rows")if a.limit and not a.has_email:rows = rows[:a.limit]+ SALVAGE["rows"] = rows # Ctrl-C in the crawl below keeps the filtered rows, not the raw onesif a.full:fields = list(rows[0].keys()) if rows else LEADelif a.fields:@@ -1134,8 +1145,6 @@ def main():fields = LEADif a.reviews and not a.full: # the page loads were paid for: keep what they readfields = fields + [c for c in ("reviews_per_rating", "user_reviews") if c not in fields]- if a.has_email and "emails" not in fields:- sys.exit("✗ --has-email needs the emails column: add emails to --fields.")if before and not rows:print(f"⚠ Nothing left: the filters and --exclude dropped all {before} rows. Loosen them and rerun; "f"the file below has only a header.")diff --git a/scripts/test_scrape.py b/scripts/test_scrape.pyindex 97f685f..00f2396 100644--- a/scripts/test_scrape.py+++ b/scripts/test_scrape.py@@ -889,6 +889,23 @@ class ArgumentBounds(unittest.TestCase):def test_has_email_needs_the_crawl(self):self.assertIn("--has-email", self.run_main("--has-email", "--no-email"))+ def test_has_email_needs_the_column_before_the_scrape(self): # no engine here: it must exit first+ self.assertIn("needs the emails column", self.run_main("--has-email", "--fields", "title,phone"))++ def test_ctrl_c_in_the_crawl_keeps_only_the_filtered_rows(self):+ good, bad = dict(_row("A"), review_rating="4.8"), dict(_row("B"), review_rating="3.0")+ argv = ["scrape.py", "x", "1", "2", "--min-rating", "4.5", "--no-socials", "--out",+ os.path.join(tempfile.mkdtemp(), "leads.csv")]+ with mock.patch.object(scrape, "run_job", return_value=("job-1", [good, bad])), \+ mock.patch.object(scrape, "enrich_emails", side_effect=KeyboardInterrupt), \+ mock.patch.dict(scrape.SALVAGE, {"rows": [], "job": None}), \+ mock.patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()), self.assertRaises(KeyboardInterrupt):+ try:+ scrape.main()+ finally:+ titles = [r["title"] for r in scrape.SALVAGE["rows"]]+ self.assertEqual(titles, ["Biz A"])+def test_rejects_values_the_engine_would_choke_on(self):self.assertIn("--country", self.run_main("--country", "usa"))self.assertIn("--depth", self.run_main("--depth", "0"))@@ -904,6 +921,14 @@ class ArgumentBounds(unittest.TestCase):self.assertIn("--out must name a .csv or .json file", self.run_main("--out", path), path)+class JunkDomains(unittest.TestCase):+ def test_whole_labels_and_domains_only(self):+ self.assertTrue(scrape.is_real_email(e), e)+ self.assertFalse(scrape.is_real_email(e), e)++class ScrapedTextInOutput(unittest.TestCase):"""What the script prints from the engine and the results reaches an agent: one line, no tricks."""