Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
fix(engine): audit loop 1: leaks, Google blocks, port shadowing, races
Mahan Jafarif938acb1ac82
diff --git a/engine/browser.py b/engine/browser.pyindex 3b789f6..951d495 100644--- a/engine/browser.py+++ b/engine/browser.py@@ -83,13 +83,45 @@ def find_chrome():return os.environ["CHROME"]for c in CANDIDATES.get(platform.system(), []):c = os.path.expandvars(c)+ if os.sep not in c:+ c = shutil.which(c) or ""+ if is_snap(c): # Ubuntu's chromium: the snap's private /tmp hides DevToolsActivePort+ continueif os.path.isfile(c):return c- if os.sep not in c and shutil.which(c):- return shutil.which(c)return shell_path() if os.path.isfile(shell_path()) else None+def is_snap(path):+ """True for a snap browser: /snap/bin/chromium, or Ubuntu's /usr/bin/chromium-browser script+ that runs it."""+ if path.startswith("/snap/") or os.path.realpath(path).startswith("/snap/"):+ return True+ try:+ with open(path, "rb") as f:+ head = f.read(4096)+ except OSError:+ return False+ return head.startswith(b"#!") and b"/snap/" in head+++def sweep_profiles(days=2):+ """Remove temporary Chrome profiles older than `days` left by an engine that was killed+ (taskkill /F on Windows skips the cleanup). No job runs that long."""+ root, cutoff = tempfile.gettempdir(), time.time() - days * 86400+ try:+ names = [n for n in os.listdir(root) if n.startswith("gmskit-chrome-")]+ except OSError:+ return+ for n in names:+ path = os.path.join(root, n)+ try:+ if os.path.getmtime(path) < cutoff:+ shutil.rmtree(path, ignore_errors=True)+ except OSError:+ pass++def download(log=print, plat=None, opener=urllib.request.urlopen):"""Fetch, verify and unpack the pinned headless shell into .engine/chrome/; its path."""plat = plat or cft_platform()@@ -174,8 +206,8 @@ class Browser:f"--lang={lang}", "--disable-dev-shm-usage"]if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headlessargs.append("--headless=new")- if proxy:- args.append(f"--proxy-server={proxy}")+ if proxy: # Chrome knows socks5:// only; it resolves names through a SOCKS5 proxy anyway+ args.append("--proxy-server=" + ("socks5://" + proxy[10:] if proxy.startswith("socks5h://") else proxy))if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)self.proc, self.cdp = None, None@@ -191,10 +223,13 @@ class Browser:if b"No usable sandbox" not in self._err.read() or "--no-sandbox" in args:raiseport, path = self._launch(args + ["--no-sandbox"], timeout)+ self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)except BrowserError:self.close()raise- self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)+ except Exception as e: # the DevTools websocket refused or broke: never leave Chrome running+ self.close()+ raise BrowserError(f"Could not connect to Chrome's DevTools port: {e}") from edef _launch(self, args, timeout):"""Start Chrome; (DevTools port, path) once it listens, else BrowserError (Chrome stopped)."""diff --git a/engine/jobs.py b/engine/jobs.pyindex 07d0f82..72ab46c 100644--- a/engine/jobs.py+++ b/engine/jobs.py@@ -24,7 +24,10 @@ COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "poEXTRA = ["categories"]RENAME = {"open_state": "status", "description": "descriptions"} # details.FIELDS name -> CSV column-WATCHDOG_GRACE = 30 # seconds past max_time before the watchdog kills Chrome+# Seconds past max_time before the watchdog kills Chrome. maps.collect's page load and first waits+# (navigate 45 s, consent 15 s, feed 15 s, first batch 10 s) sit outside its max_seconds, so a+# shorter grace could kill a keyword that was about to return its rows.+WATCHDOG_GRACE = 90SITE_WORKERS = 8 # business websites fetched at once by the email passSITE_TIMEOUT = 10 # seconds per website requestEMAIL_MIN_SECONDS = 60 # the email pass gets at least this long, even after a job used its max_time@@ -84,7 +87,7 @@ def validate(body):data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "lat": str(lat).strip(),"lon": str(lon).strip(),"zoom": _int(body, "zoom", 15, 1, 21, "zoom"),- "radius": _int(body, "radius", 10000, 0, 1_000_000, "radius"),+ "radius": _int(body, "radius", 10000, 0, 10**9, "radius"), # accepted, not used"depth": _int(body, "depth", 1, 1, 1000, "depth"),"max_time": _int(body, "max_time", 600, 1, 86400, "max time"),"fast_mode": bool(body.get("fast_mode")), "email": bool(body.get("email")),@@ -240,16 +243,22 @@ class Worker(threading.Thread):def run(self):while not self.stopped.is_set():- job = self.store.claim()- if not job:- self.wake.wait(5)- self.wake.clear()- continue+ job = Nonetry:+ job = self.store.claim()+ if not job:+ self.wake.wait(5)+ self.wake.clear()+ continueself.run_job(job)- except Exception as e: # noqa: BLE001 - one bad job must never stop the worker- self.log(f"job {job['ID']} failed: {type(e).__name__}: {e}")- self.store.update(job["ID"], "failed", f"{type(e).__name__}: {e}")+ except Exception as e: # noqa: BLE001 - one bad job (or a full disk) must never stop the worker+ self.log(f"job {job['ID'] if job else '-'} failed: {type(e).__name__}: {e}")+ try:+ if job:+ self.store.update(job["ID"], "failed", f"{type(e).__name__}: {e}")+ except Exception: # noqa: BLE001+ pass+ self.stopped.wait(2)def cancel(self, jid):"""Stop `jid` if it is the job in progress (it is being deleted)."""@@ -273,6 +282,9 @@ class Worker(threading.Thread):deadline = time.monotonic() + d["max_time"]stop = threading.Event()self._running = (jid, stop)+ if not self.store.get(jid): # deleted between claim() and now, before cancel() could see it+ self._running = (None, None)+ returnrows, seen, errors, blocked = [], set(), [], Falsekws = d["keywords"]self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")@@ -308,7 +320,10 @@ class Worker(threading.Thread):finally:if page and not self._browser.cdp.closed:page.close()- blocked |= reason == "blocked"+ if reason == "blocked": # more searches from a rate-limited IP only extend the block+ blocked = True+ self.log(f"job {jid} keyword {i}/{len(kws)}: Google's unusual-traffic page; stopping")+ breaknew = 0for p in places:key = p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")@@ -329,22 +344,25 @@ class Worker(threading.Thread):self._browser.close()self._browser = Noneif d["email"] and rows and not stop.is_set():- n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS))+ n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), stop=stop.is_set)self.log(f"job {jid}: emails found for {n}/{sum(1 for r in rows if r['website'])} websites")self._running = (None, None)if not self.store.write_csv(jid, rows):self.log(f"job {jid} deleted while it ran")return+ block = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "+ "Wait, or add proxies")if rows or not (errors or blocked):note = "stopped at max_time; the rows found so far are kept" \if time.monotonic() >= deadline or (stop.is_set() and not errors) else ""if errors:note = f"{len(errors)} keyword(s) failed: {errors[-1]}"+ if blocked:+ note = f"{block}. The rows found before that are kept"self.store.update(jid, "ok", note)self.log(f"job {jid} ok: {len(rows)} places")else:- reason = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "- "Wait, or add proxies") if blocked else errors[-1]+ reason = block if blocked else errors[-1]self.store.update(jid, "failed", reason)self.log(f"job {jid} failed: {reason}")@@ -450,14 +468,17 @@ def find_emails(text, site=""):return sorted(found, key=lambda e: not own(e))[:3]-def add_emails(rows, seconds, get=None):+def add_emails(rows, seconds, get=None, stop=lambda: False):"""Fill `emails` (joined by ", ", like the old engine) from each row's website homepage, for at- most `seconds`; the number of rows that got one."""+ most `seconds` or until stop() is true; the number of rows that got one."""get = get or fetchtodo = [r for r in rows if r.get("website") and not r.get("emails")]pool = concurrent.futures.ThreadPoolExecutor(SITE_WORKERS)futs = {pool.submit(lambda r=r: find_emails(get(r["website"]), site_host(r["website"]))): r for r in todo}- done, _ = concurrent.futures.wait(futs, timeout=seconds)+ end, pending = time.monotonic() + seconds, set(futs)+ while pending and not stop() and time.monotonic() < end:+ _, pending = concurrent.futures.wait(pending, timeout=min(1, max(0, end - time.monotonic())))+ done = set(futs) - pendingpool.shutdown(wait=False, cancel_futures=True)n = 0for f in done:diff --git a/engine/serve.py b/engine/serve.pyindex 8aadb6b..292970b 100644--- a/engine/serve.py+++ b/engine/serve.py@@ -323,6 +323,16 @@ class Server(ThreadingHTTPServer):self.server_name, self.server_port = "localhost", self.server_address[1]+def port_taken(port):+ """True if something answers on 127.0.0.1:port. With SO_REUSEADDR, macOS lets a new server bind+ 127.0.0.1:port while another listens on *:port (the Docker engine), and takes its traffic."""+ try:+ socket.create_connection(("127.0.0.1", port), timeout=1).close()+ return True+ except OSError:+ return False++class Server6(Server):address_family = socket.AF_INET6@@ -330,6 +340,8 @@ class Server6(Server):def make_servers(port, store, worker, api_key=""):"""The server on 127.0.0.1:port, plus one on [::1]:port when this machine has IPv6 loopback(Windows tries ::1 first for "localhost" and waits ~2 s before falling back to 127.0.0.1)."""+ if port and port_taken(port):+ raise OSError(f"something already listens on port {port}")servers = [Server(("127.0.0.1", port), Handler)]try:servers.append(Server6(("::1", servers[0].server_address[1]), Handler))@@ -351,6 +363,7 @@ def main():except ValueError:sys.exit(f"✗ SCRAPER_PORT is not a number: {env['SCRAPER_PORT']!r}")browser.make_home()+ browser.sweep_profiles()store = jobs.Store(a.data)if n := store.recover():log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")diff --git a/engine/test_engine.py b/engine/test_engine.pyindex 4aa66fc..9355edc 100644--- a/engine/test_engine.py+++ b/engine/test_engine.py@@ -6,7 +6,8 @@ No network, no Chrome, no Docker.python3 engine/test_engine.py"""-import csv, hashlib, http.client, io, json, os, re, shutil, sys, tempfile, threading, time, unittest, zipfile+import csv, hashlib, http.client, importlib.util, io, json, os, re, shutil, socket, subprocess, sys, tempfile+import threading, time, unittest, zipfileHERE = os.path.dirname(os.path.abspath(__file__))sys.path.insert(0, HERE)@@ -131,6 +132,7 @@ class Validate(unittest.TestCase):name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])+ self.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)class Rows(unittest.TestCase):@@ -202,6 +204,45 @@ class Worker(Fakes):self.assertEqual(job["Status"], "failed")self.assertIn("unusual-traffic", job["Error"])+ def test_block_stops_the_job(self):+ RESULTS["cafes"] = ([place(1)], "end-of-list")+ RESULTS["bakeries"] = ([], "blocked")+ RESULTS["florists"] = ([place(9)], "end-of-list")+ jid, job = self.run_one(keywords=["cafes", "bakeries", "florists"])+ self.assertEqual(job["Status"], "ok") # the rows from before the block are kept+ self.assertIn("unusual-traffic", job["Error"])+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1"]) # florists never searched++ def test_deleted_before_it_starts(self):+ name, data = jobs.validate(GOOD)+ jid = self.store.create(name, data)+ job = self.store.claim()+ self.store.delete(jid)+ self.worker.run_job(job)+ self.assertEqual(FakeBrowser.made, []) # no Chrome for a job that is gone++ def test_worker_survives_a_store_error(self):+ calls = []+ def claim():+ calls.append(1)+ if len(calls) == 1:+ raise OSError("disk full")+ return None+ self.store.claim = claim+ self.worker.wake.set()+ self.worker.start()+ deadline = time.time() + 5+ while len(calls) < 2 and time.time() < deadline:+ time.sleep(0.05)+ self.assertGreaterEqual(len(calls), 2)+ self.assertTrue(self.worker.is_alive())++ def test_email_pass_stops_when_asked(self):+ rows = [{"website": f"https://s{i}.com", "emails": ""} for i in range(3)]+ t = time.monotonic()+ jobs.add_emails(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)+ self.assertLess(time.monotonic() - t, 3)+def test_watchdog_kills_a_hung_chrome(self):RESULTS["cafes in Austin"] = ([], "hang")saved, jobs.WATCHDOG_GRACE = jobs.WATCHDOG_GRACE, 0.2@@ -257,6 +298,18 @@ class Emails(unittest.TestCase):self.assertEqual(jobs.fetch("http://127.0.0.1:9/"), "")self.assertEqual(jobs.site_host("https://www.Shop.com/x"), "shop.com")+ def test_private_peer_and_redirect_refused(self):+ srv = socket.socket()+ srv.bind(("127.0.0.1", 0))+ srv.listen(1)+ with self.assertRaisesRegex(OSError, "non-public"):+ jobs._public_connection(srv.getsockname())+ srv.close()+ import urllib.request+ req = urllib.request.Request("https://example.com/")+ self.assertIsNone(jobs._Redirects().redirect_request(req, None, 302, "Found", {}, "http://127.0.0.1:8080/api/v1/jobs"))+ self.assertIsNone(jobs._Redirects().redirect_request(req, None, 302, "Found", {}, "http://169.254.169.254/"))+class Server(Fakes):key = ""@@ -439,5 +492,115 @@ class Download(unittest.TestCase):self.assertIn("No usable sandbox", browser.start_error("/x/chrome", -5, trace))+FAKE_CHROME = """#!{py}+import os, sys, time+args = sys.argv[1:]+with open(os.environ["FAKE_LOG"], "a") as f:+ f.write(" ".join(args) + "\\n")+if os.environ.get("FAKE_SANDBOX") and "--no-sandbox" not in args:+ sys.stderr.write("[1:2:FATAL:zygote_host_impl_linux.cc(130)] No usable sandbox!\\n#0 0x1 trace\\n")+ sys.exit(1)+profile = [a.split("=", 1)[1] for a in args if a.startswith("--user-data-dir=")][0]+with open(os.path.join(profile, "DevToolsActivePort"), "w") as f:+ f.write("{port}\\n/devtools/browser/x\\n") # nothing listens there+time.sleep(60)+"""++[email protected](os.name == "nt", "the fake Chrome is a script with a shebang")+class LaunchChrome(unittest.TestCase):+ """browser.Browser against a fake Chrome: cleanup on a dead DevTools port, the sandbox relaunch."""++ def setUp(self):+ self.dir = tempfile.mkdtemp()+ s = socket.socket()+ s.bind(("127.0.0.1", 0))+ port = s.getsockname()[1]+ s.close()+ self.exe = os.path.join(self.dir, "fake-chrome")+ with open(self.exe, "w") as f:+ f.write(FAKE_CHROME.format(py=sys.executable, port=port))+ os.chmod(self.exe, 0o755)+ os.environ["FAKE_LOG"] = os.path.join(self.dir, "argv.log")++ def tearDown(self):+ os.environ.pop("FAKE_LOG", None)+ os.environ.pop("FAKE_SANDBOX", None)+ shutil.rmtree(self.dir, ignore_errors=True)++ def launches(self):+ with open(os.environ["FAKE_LOG"]) as f:+ return f.read().splitlines()++ def test_dead_devtools_port_leaves_nothing(self):+ profile = os.path.join(self.dir, "profile")+ os.mkdir(profile)+ b = None+ with self.assertRaisesRegex(browser.BrowserError, "DevTools"):+ b = browser.Browser(path=self.exe, profile=profile, proxy="socks5h://10.0.0.1:1080", timeout=5)+ self.assertIsNone(b)+ self.assertIn("--proxy-server=socks5://10.0.0.1:1080", self.launches()[0])+ out = subprocess.run(["ps", "-axo", "command"], capture_output=True, text=True).stdout+ self.assertNotIn(self.exe, out) # the fake Chrome was stopped++ def test_temp_profile_removed(self):+ before = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}+ with self.assertRaises(browser.BrowserError):+ browser.Browser(path=self.exe, timeout=5)+ after = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}+ self.assertEqual(after - before, set())++ def test_no_usable_sandbox_relaunches_once_without_it(self):+ os.environ["FAKE_SANDBOX"] = "1"+ with self.assertRaisesRegex(browser.BrowserError, "DevTools"): # second launch got past the sandbox+ browser.Browser(path=self.exe, timeout=5)+ runs = self.launches()+ self.assertEqual(len(runs), 2)+ self.assertNotIn("--no-sandbox", runs[0])+ self.assertIn("--no-sandbox", runs[1])++ def test_snap_wrapper_skipped(self):+ wrapper = os.path.join(self.dir, "chromium-browser")+ with open(wrapper, "w") as f:+ f.write("#!/bin/sh\nexec /snap/bin/chromium \"$@\"\n")+ self.assertTrue(browser.is_snap(wrapper))+ self.assertTrue(browser.is_snap("/snap/bin/chromium"))+ self.assertFalse(browser.is_snap(self.exe))+++def load_engine_cli():+ path = os.path.join(os.path.dirname(HERE), "scripts", "engine.py")+ spec = importlib.util.spec_from_file_location("engine_cli", path)+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ return mod+++class EngineCli(unittest.TestCase):+ def test_port_with_a_wildcard_listener_is_not_free(self):+ cli = load_engine_cli()+ s = socket.socket()+ s.bind(("0.0.0.0", 0))+ s.listen(8) # both checks connect; a full backlog would refuse the second+ port = s.getsockname()[1]+ try:+ self.assertFalse(cli.port_free(port))+ self.assertTrue(serve.port_taken(port))+ finally:+ s.close()+ self.assertFalse(serve.port_taken(port))++ def test_stop_never_kills_a_stranger(self):+ cli = load_engine_cli()+ p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])+ try:+ self.assertTrue(cli.kill(p.pid)) # "nothing of ours to stop"+ time.sleep(0.3)+ self.assertIsNone(p.poll())+ finally:+ p.kill()+ p.wait()++if __name__ == "__main__":unittest.main(verbosity=1)diff --git a/scripts/engine.py b/scripts/engine.pyindex 2e58294..a3f235b 100644--- a/scripts/engine.py+++ b/scripts/engine.py@@ -59,6 +59,13 @@ def probe(port, timeout=2):def port_free(port):+ """Nothing listens on the port and it can be bound. The connect test matters on macOS, where+ SO_REUSEADDR lets 127.0.0.1:port bind while another server listens on *:port."""+ try:+ socket.create_connection(("127.0.0.1", port), timeout=1).close()+ return False+ except OSError:+ passwith socket.socket() as s:if os.name != "nt": # like the engine's own server: connections in TIME_WAIT do not blocks.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)