Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(engine): audit loop 1: leaks, Google blocks, port shadowing, races

Mahan Jafarif938acb1ac82
diff --git a/engine/browser.py b/engine/browser.py
index 3b789f6..951d495 100644
--- a/engine/browser.py
+++ b/engine/browser.py
@@ -83,13 +83,45 @@ def find_chrome():
return os.environ["CHROME"]
for c in CANDIDATES.get(platform.system(), []):
c = os.path.expandvars(c)
+ if os.sep not in c:
+ c = shutil.which(c) or ""
+ if is_snap(c): # Ubuntu's chromium: the snap's private /tmp hides DevToolsActivePort
+ continue
if os.path.isfile(c):
return c
- if os.sep not in c and shutil.which(c):
- return shutil.which(c)
return shell_path() if os.path.isfile(shell_path()) else None
+def is_snap(path):
+ """True for a snap browser: /snap/bin/chromium, or Ubuntu's /usr/bin/chromium-browser script
+ that runs it."""
+ if path.startswith("/snap/") or os.path.realpath(path).startswith("/snap/"):
+ return True
+ try:
+ with open(path, "rb") as f:
+ head = f.read(4096)
+ except OSError:
+ return False
+ return head.startswith(b"#!") and b"/snap/" in head
+
+
+def sweep_profiles(days=2):
+ """Remove temporary Chrome profiles older than `days` left by an engine that was killed
+ (taskkill /F on Windows skips the cleanup). No job runs that long."""
+ root, cutoff = tempfile.gettempdir(), time.time() - days * 86400
+ try:
+ names = [n for n in os.listdir(root) if n.startswith("gmskit-chrome-")]
+ except OSError:
+ return
+ for n in names:
+ path = os.path.join(root, n)
+ try:
+ if os.path.getmtime(path) < cutoff:
+ shutil.rmtree(path, ignore_errors=True)
+ except OSError:
+ pass
+
+
def download(log=print, plat=None, opener=urllib.request.urlopen):
"""Fetch, verify and unpack the pinned headless shell into .engine/chrome/; its path."""
plat = plat or cft_platform()
@@ -174,8 +206,8 @@ class Browser:
f"--lang={lang}", "--disable-dev-shm-usage"]
if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headless
args.append("--headless=new")
- if proxy:
- args.append(f"--proxy-server={proxy}")
+ if proxy: # Chrome knows socks5:// only; it resolves names through a SOCKS5 proxy anyway
+ args.append("--proxy-server=" + ("socks5://" + proxy[10:] if proxy.startswith("socks5h://") else proxy))
if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:
args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)
self.proc, self.cdp = None, None
@@ -191,10 +223,13 @@ class Browser:
if b"No usable sandbox" not in self._err.read() or "--no-sandbox" in args:
raise
port, path = self._launch(args + ["--no-sandbox"], timeout)
+ self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)
except BrowserError:
self.close()
raise
- self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)
+ except Exception as e: # the DevTools websocket refused or broke: never leave Chrome running
+ self.close()
+ raise BrowserError(f"Could not connect to Chrome's DevTools port: {e}") from e
def _launch(self, args, timeout):
"""Start Chrome; (DevTools port, path) once it listens, else BrowserError (Chrome stopped)."""
diff --git a/engine/jobs.py b/engine/jobs.py
index 07d0f82..72ab46c 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -24,7 +24,10 @@ COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "po
EXTRA = ["categories"]
RENAME = {"open_state": "status", "description": "descriptions"} # details.FIELDS name -> CSV column
-WATCHDOG_GRACE = 30 # seconds past max_time before the watchdog kills Chrome
+# Seconds past max_time before the watchdog kills Chrome. maps.collect's page load and first waits
+# (navigate 45 s, consent 15 s, feed 15 s, first batch 10 s) sit outside its max_seconds, so a
+# shorter grace could kill a keyword that was about to return its rows.
+WATCHDOG_GRACE = 90
SITE_WORKERS = 8 # business websites fetched at once by the email pass
SITE_TIMEOUT = 10 # seconds per website request
EMAIL_MIN_SECONDS = 60 # the email pass gets at least this long, even after a job used its max_time
@@ -84,7 +87,7 @@ def validate(body):
data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "lat": str(lat).strip(),
"lon": str(lon).strip(),
"zoom": _int(body, "zoom", 15, 1, 21, "zoom"),
- "radius": _int(body, "radius", 10000, 0, 1_000_000, "radius"),
+ "radius": _int(body, "radius", 10000, 0, 10**9, "radius"), # accepted, not used
"depth": _int(body, "depth", 1, 1, 1000, "depth"),
"max_time": _int(body, "max_time", 600, 1, 86400, "max time"),
"fast_mode": bool(body.get("fast_mode")), "email": bool(body.get("email")),
@@ -240,16 +243,22 @@ class Worker(threading.Thread):
def run(self):
while not self.stopped.is_set():
- job = self.store.claim()
- if not job:
- self.wake.wait(5)
- self.wake.clear()
- continue
+ job = None
try:
+ job = self.store.claim()
+ if not job:
+ self.wake.wait(5)
+ self.wake.clear()
+ continue
self.run_job(job)
- except Exception as e: # noqa: BLE001 - one bad job must never stop the worker
- self.log(f"job {job['ID']} failed: {type(e).__name__}: {e}")
- self.store.update(job["ID"], "failed", f"{type(e).__name__}: {e}")
+ except Exception as e: # noqa: BLE001 - one bad job (or a full disk) must never stop the worker
+ self.log(f"job {job['ID'] if job else '-'} failed: {type(e).__name__}: {e}")
+ try:
+ if job:
+ self.store.update(job["ID"], "failed", f"{type(e).__name__}: {e}")
+ except Exception: # noqa: BLE001
+ pass
+ self.stopped.wait(2)
def cancel(self, jid):
"""Stop `jid` if it is the job in progress (it is being deleted)."""
@@ -273,6 +282,9 @@ class Worker(threading.Thread):
deadline = time.monotonic() + d["max_time"]
stop = threading.Event()
self._running = (jid, stop)
+ if not self.store.get(jid): # deleted between claim() and now, before cancel() could see it
+ self._running = (None, None)
+ return
rows, seen, errors, blocked = [], set(), [], False
kws = d["keywords"]
self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")
@@ -308,7 +320,10 @@ class Worker(threading.Thread):
finally:
if page and not self._browser.cdp.closed:
page.close()
- blocked |= reason == "blocked"
+ if reason == "blocked": # more searches from a rate-limited IP only extend the block
+ blocked = True
+ self.log(f"job {jid} keyword {i}/{len(kws)}: Google's unusual-traffic page; stopping")
+ break
new = 0
for p in places:
key = p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")
@@ -329,22 +344,25 @@ class Worker(threading.Thread):
self._browser.close()
self._browser = None
if d["email"] and rows and not stop.is_set():
- n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS))
+ n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), stop=stop.is_set)
self.log(f"job {jid}: emails found for {n}/{sum(1 for r in rows if r['website'])} websites")
self._running = (None, None)
if not self.store.write_csv(jid, rows):
self.log(f"job {jid} deleted while it ran")
return
+ block = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
+ "Wait, or add proxies")
if rows or not (errors or blocked):
note = "stopped at max_time; the rows found so far are kept" \
if time.monotonic() >= deadline or (stop.is_set() and not errors) else ""
if errors:
note = f"{len(errors)} keyword(s) failed: {errors[-1]}"
+ if blocked:
+ note = f"{block}. The rows found before that are kept"
self.store.update(jid, "ok", note)
self.log(f"job {jid} ok: {len(rows)} places")
else:
- reason = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
- "Wait, or add proxies") if blocked else errors[-1]
+ reason = block if blocked else errors[-1]
self.store.update(jid, "failed", reason)
self.log(f"job {jid} failed: {reason}")
@@ -450,14 +468,17 @@ def find_emails(text, site=""):
return sorted(found, key=lambda e: not own(e))[:3]
-def add_emails(rows, seconds, get=None):
+def add_emails(rows, seconds, get=None, stop=lambda: False):
"""Fill `emails` (joined by ", ", like the old engine) from each row's website homepage, for at
- most `seconds`; the number of rows that got one."""
+ most `seconds` or until stop() is true; the number of rows that got one."""
get = get or fetch
todo = [r for r in rows if r.get("website") and not r.get("emails")]
pool = concurrent.futures.ThreadPoolExecutor(SITE_WORKERS)
futs = {pool.submit(lambda r=r: find_emails(get(r["website"]), site_host(r["website"]))): r for r in todo}
- done, _ = concurrent.futures.wait(futs, timeout=seconds)
+ end, pending = time.monotonic() + seconds, set(futs)
+ while pending and not stop() and time.monotonic() < end:
+ _, pending = concurrent.futures.wait(pending, timeout=min(1, max(0, end - time.monotonic())))
+ done = set(futs) - pending
pool.shutdown(wait=False, cancel_futures=True)
n = 0
for f in done:
diff --git a/engine/serve.py b/engine/serve.py
index 8aadb6b..292970b 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -323,6 +323,16 @@ class Server(ThreadingHTTPServer):
self.server_name, self.server_port = "localhost", self.server_address[1]
+def port_taken(port):
+ """True if something answers on 127.0.0.1:port. With SO_REUSEADDR, macOS lets a new server bind
+ 127.0.0.1:port while another listens on *:port (the Docker engine), and takes its traffic."""
+ try:
+ socket.create_connection(("127.0.0.1", port), timeout=1).close()
+ return True
+ except OSError:
+ return False
+
+
class Server6(Server):
address_family = socket.AF_INET6
@@ -330,6 +340,8 @@ class Server6(Server):
def make_servers(port, store, worker, api_key=""):
"""The server on 127.0.0.1:port, plus one on [::1]:port when this machine has IPv6 loopback
(Windows tries ::1 first for "localhost" and waits ~2 s before falling back to 127.0.0.1)."""
+ if port and port_taken(port):
+ raise OSError(f"something already listens on port {port}")
servers = [Server(("127.0.0.1", port), Handler)]
try:
servers.append(Server6(("::1", servers[0].server_address[1]), Handler))
@@ -351,6 +363,7 @@ def main():
except ValueError:
sys.exit(f"✗ SCRAPER_PORT is not a number: {env['SCRAPER_PORT']!r}")
browser.make_home()
+ browser.sweep_profiles()
store = jobs.Store(a.data)
if n := store.recover():
log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 4aa66fc..9355edc 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -6,7 +6,8 @@ No network, no Chrome, no Docker.
python3 engine/test_engine.py
"""
-import csv, hashlib, http.client, io, json, os, re, shutil, sys, tempfile, threading, time, unittest, zipfile
+import csv, hashlib, http.client, importlib.util, io, json, os, re, shutil, socket, subprocess, sys, tempfile
+import threading, time, unittest, zipfile
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
@@ -131,6 +132,7 @@ class Validate(unittest.TestCase):
name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))
self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))
self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])
+ self.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)
class Rows(unittest.TestCase):
@@ -202,6 +204,45 @@ class Worker(Fakes):
self.assertEqual(job["Status"], "failed")
self.assertIn("unusual-traffic", job["Error"])
+ def test_block_stops_the_job(self):
+ RESULTS["cafes"] = ([place(1)], "end-of-list")
+ RESULTS["bakeries"] = ([], "blocked")
+ RESULTS["florists"] = ([place(9)], "end-of-list")
+ jid, job = self.run_one(keywords=["cafes", "bakeries", "florists"])
+ self.assertEqual(job["Status"], "ok") # the rows from before the block are kept
+ self.assertIn("unusual-traffic", job["Error"])
+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1"]) # florists never searched
+
+ def test_deleted_before_it_starts(self):
+ name, data = jobs.validate(GOOD)
+ jid = self.store.create(name, data)
+ job = self.store.claim()
+ self.store.delete(jid)
+ self.worker.run_job(job)
+ self.assertEqual(FakeBrowser.made, []) # no Chrome for a job that is gone
+
+ def test_worker_survives_a_store_error(self):
+ calls = []
+ def claim():
+ calls.append(1)
+ if len(calls) == 1:
+ raise OSError("disk full")
+ return None
+ self.store.claim = claim
+ self.worker.wake.set()
+ self.worker.start()
+ deadline = time.time() + 5
+ while len(calls) < 2 and time.time() < deadline:
+ time.sleep(0.05)
+ self.assertGreaterEqual(len(calls), 2)
+ self.assertTrue(self.worker.is_alive())
+
+ def test_email_pass_stops_when_asked(self):
+ rows = [{"website": f"https://s{i}.com", "emails": ""} for i in range(3)]
+ t = time.monotonic()
+ jobs.add_emails(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)
+ self.assertLess(time.monotonic() - t, 3)
+
def test_watchdog_kills_a_hung_chrome(self):
RESULTS["cafes in Austin"] = ([], "hang")
saved, jobs.WATCHDOG_GRACE = jobs.WATCHDOG_GRACE, 0.2
@@ -257,6 +298,18 @@ class Emails(unittest.TestCase):
self.assertEqual(jobs.fetch("http://127.0.0.1:9/"), "")
self.assertEqual(jobs.site_host("https://www.Shop.com/x"), "shop.com")
+ def test_private_peer_and_redirect_refused(self):
+ srv = socket.socket()
+ srv.bind(("127.0.0.1", 0))
+ srv.listen(1)
+ with self.assertRaisesRegex(OSError, "non-public"):
+ jobs._public_connection(srv.getsockname())
+ srv.close()
+ import urllib.request
+ req = urllib.request.Request("https://example.com/")
+ self.assertIsNone(jobs._Redirects().redirect_request(req, None, 302, "Found", {}, "http://127.0.0.1:8080/api/v1/jobs"))
+ self.assertIsNone(jobs._Redirects().redirect_request(req, None, 302, "Found", {}, "http://169.254.169.254/"))
+
class Server(Fakes):
key = ""
@@ -439,5 +492,115 @@ class Download(unittest.TestCase):
self.assertIn("No usable sandbox", browser.start_error("/x/chrome", -5, trace))
+FAKE_CHROME = """#!{py}
+import os, sys, time
+args = sys.argv[1:]
+with open(os.environ["FAKE_LOG"], "a") as f:
+ f.write(" ".join(args) + "\\n")
+if os.environ.get("FAKE_SANDBOX") and "--no-sandbox" not in args:
+ sys.stderr.write("[1:2:FATAL:zygote_host_impl_linux.cc(130)] No usable sandbox!\\n#0 0x1 trace\\n")
+ sys.exit(1)
+profile = [a.split("=", 1)[1] for a in args if a.startswith("--user-data-dir=")][0]
+with open(os.path.join(profile, "DevToolsActivePort"), "w") as f:
+ f.write("{port}\\n/devtools/browser/x\\n") # nothing listens there
+time.sleep(60)
+"""
+
+
[email protected](os.name == "nt", "the fake Chrome is a script with a shebang")
+class LaunchChrome(unittest.TestCase):
+ """browser.Browser against a fake Chrome: cleanup on a dead DevTools port, the sandbox relaunch."""
+
+ def setUp(self):
+ self.dir = tempfile.mkdtemp()
+ s = socket.socket()
+ s.bind(("127.0.0.1", 0))
+ port = s.getsockname()[1]
+ s.close()
+ self.exe = os.path.join(self.dir, "fake-chrome")
+ with open(self.exe, "w") as f:
+ f.write(FAKE_CHROME.format(py=sys.executable, port=port))
+ os.chmod(self.exe, 0o755)
+ os.environ["FAKE_LOG"] = os.path.join(self.dir, "argv.log")
+
+ def tearDown(self):
+ os.environ.pop("FAKE_LOG", None)
+ os.environ.pop("FAKE_SANDBOX", None)
+ shutil.rmtree(self.dir, ignore_errors=True)
+
+ def launches(self):
+ with open(os.environ["FAKE_LOG"]) as f:
+ return f.read().splitlines()
+
+ def test_dead_devtools_port_leaves_nothing(self):
+ profile = os.path.join(self.dir, "profile")
+ os.mkdir(profile)
+ b = None
+ with self.assertRaisesRegex(browser.BrowserError, "DevTools"):
+ b = browser.Browser(path=self.exe, profile=profile, proxy="socks5h://10.0.0.1:1080", timeout=5)
+ self.assertIsNone(b)
+ self.assertIn("--proxy-server=socks5://10.0.0.1:1080", self.launches()[0])
+ out = subprocess.run(["ps", "-axo", "command"], capture_output=True, text=True).stdout
+ self.assertNotIn(self.exe, out) # the fake Chrome was stopped
+
+ def test_temp_profile_removed(self):
+ before = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}
+ with self.assertRaises(browser.BrowserError):
+ browser.Browser(path=self.exe, timeout=5)
+ after = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}
+ self.assertEqual(after - before, set())
+
+ def test_no_usable_sandbox_relaunches_once_without_it(self):
+ os.environ["FAKE_SANDBOX"] = "1"
+ with self.assertRaisesRegex(browser.BrowserError, "DevTools"): # second launch got past the sandbox
+ browser.Browser(path=self.exe, timeout=5)
+ runs = self.launches()
+ self.assertEqual(len(runs), 2)
+ self.assertNotIn("--no-sandbox", runs[0])
+ self.assertIn("--no-sandbox", runs[1])
+
+ def test_snap_wrapper_skipped(self):
+ wrapper = os.path.join(self.dir, "chromium-browser")
+ with open(wrapper, "w") as f:
+ f.write("#!/bin/sh\nexec /snap/bin/chromium \"$@\"\n")
+ self.assertTrue(browser.is_snap(wrapper))
+ self.assertTrue(browser.is_snap("/snap/bin/chromium"))
+ self.assertFalse(browser.is_snap(self.exe))
+
+
+def load_engine_cli():
+ path = os.path.join(os.path.dirname(HERE), "scripts", "engine.py")
+ spec = importlib.util.spec_from_file_location("engine_cli", path)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod
+
+
+class EngineCli(unittest.TestCase):
+ def test_port_with_a_wildcard_listener_is_not_free(self):
+ cli = load_engine_cli()
+ s = socket.socket()
+ s.bind(("0.0.0.0", 0))
+ s.listen(8) # both checks connect; a full backlog would refuse the second
+ port = s.getsockname()[1]
+ try:
+ self.assertFalse(cli.port_free(port))
+ self.assertTrue(serve.port_taken(port))
+ finally:
+ s.close()
+ self.assertFalse(serve.port_taken(port))
+
+ def test_stop_never_kills_a_stranger(self):
+ cli = load_engine_cli()
+ p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])
+ try:
+ self.assertTrue(cli.kill(p.pid)) # "nothing of ours to stop"
+ time.sleep(0.3)
+ self.assertIsNone(p.poll())
+ finally:
+ p.kill()
+ p.wait()
+
+
if __name__ == "__main__":
unittest.main(verbosity=1)
diff --git a/scripts/engine.py b/scripts/engine.py
index 2e58294..a3f235b 100644
--- a/scripts/engine.py
+++ b/scripts/engine.py
@@ -59,6 +59,13 @@ def probe(port, timeout=2):
def port_free(port):
+ """Nothing listens on the port and it can be bound. The connect test matters on macOS, where
+ SO_REUSEADDR lets 127.0.0.1:port bind while another server listens on *:port."""
+ try:
+ socket.create_connection(("127.0.0.1", port), timeout=1).close()
+ return False
+ except OSError:
+ pass
with socket.socket() as s:
if os.name != "nt": # like the engine's own server: connections in TIME_WAIT do not block
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)