Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(engine): macOS start hang, Linux sandbox under AppArmor, Windows test reset

Mahan Jafari6e17cba3bd53
diff --git a/engine/browser.py b/engine/browser.py
index 6e7f3ca..3b789f6 100644
--- a/engine/browser.py
+++ b/engine/browser.py
@@ -149,7 +149,9 @@ def start_error(path, code, stderr):
if missing:
return (f"Chrome cannot start: {missing[0]}\n Install the libraries it needs "
f"(Debian or Ubuntu):\n {APT_LINE}")
- last = "; ".join(l.strip() for l in stderr.strip().splitlines()[-3:])
+ lines = [l.strip() for l in stderr.strip().splitlines() if l.strip()]
+ fatal = [l for l in lines if "FATAL" in l] # Chrome prints a stack trace after it
+ last = fatal[0] if fatal else "; ".join(lines[-3:])
return f"Chrome exited at once (code {code}): {path}" + (f": {last}" if last else "")
@@ -176,35 +178,50 @@ class Browser:
args.append(f"--proxy-server={proxy}")
if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:
args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)
- args.append("about:blank")
+ self.proc, self.cdp = None, None
self._err = open(os.path.join(self.profile, "chrome-stderr.log"), "w+b")
+ try:
+ try:
+ port, path = self._launch(args, timeout)
+ except BrowserError:
+ # Ubuntu 23.10 and newer (AppArmor) block Chrome's sandbox for a browser that is not
+ # installed under /opt, such as the downloaded headless shell. Like Playwright, run it
+ # without; it only ever loads Google Maps.
+ self._err.seek(0)
+ if b"No usable sandbox" not in self._err.read() or "--no-sandbox" in args:
+ raise
+ port, path = self._launch(args + ["--no-sandbox"], timeout)
+ except BrowserError:
+ self.close()
+ raise
+ self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)
+
+ def _launch(self, args, timeout):
+ """Start Chrome; (DevTools port, path) once it listens, else BrowserError (Chrome stopped)."""
+ self._err.seek(0)
+ self._err.truncate()
+ _remove(os.path.join(self.profile, "DevToolsActivePort"))
kw = {"stdout": subprocess.DEVNULL, "stderr": self._err, "stdin": subprocess.DEVNULL}
if os.name == "nt":
kw["creationflags"] = subprocess.CREATE_NO_WINDOW
try:
- self.proc = subprocess.Popen(args, **kw)
+ self.proc = subprocess.Popen(args + ["about:blank"], **kw)
except OSError as e:
- self._cleanup()
raise BrowserError(f"Could not start Chrome ({self.path}): {e}") from e
- self.cdp = None
active = os.path.join(self.profile, "DevToolsActivePort")
end = time.time() + timeout
while time.time() < end:
if self.proc.poll() is not None:
self._err.seek(0)
- msg = start_error(self.path, self.proc.returncode, self._err.read().decode("utf-8", "replace"))
- self.close()
- raise BrowserError(msg)
+ raise BrowserError(start_error(self.path, self.proc.returncode,
+ self._err.read().decode("utf-8", "replace")))
try:
with open(active, encoding="utf-8") as f:
port, path = f.read().split()
- break
+ return port, path
except (OSError, ValueError):
time.sleep(0.1)
- else:
- self.close()
- raise BrowserError(f"Chrome did not open its DevTools port within {timeout} s: {self.path}")
- self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)
+ raise BrowserError(f"Chrome did not open its DevTools port within {timeout} s: {self.path}")
def version(self):
return self.cdp.send("Browser.getVersion").get("product", "")
@@ -213,7 +230,7 @@ class Browser:
"""Stop Chrome now (the job watchdog): every CDP call still waiting fails at once."""
try:
self.proc.kill()
- except OSError:
+ except (OSError, AttributeError):
pass
def close(self):
@@ -222,7 +239,7 @@ class Browser:
self.cdp.close()
except Exception: # noqa: BLE001 - closing is best effort
pass
- if self.proc.poll() is None:
+ if self.proc and self.proc.poll() is None:
self.proc.terminate()
try:
self.proc.wait(5)
diff --git a/engine/serve.py b/engine/serve.py
index cc6c228..8aadb6b 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -12,7 +12,7 @@ posts, 403); with SCRAPER_API_KEY set it also wants that X-API-Key (401). No tel
Google Maps, and for email: true jobs to the businesses' own websites, nothing else.
Standard library only.
"""
-import argparse, hmac, json, os, re, secrets, signal, socket, sys, threading, urllib.parse
+import argparse, hmac, json, os, re, secrets, signal, socket, socketserver, sys, threading, urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -316,6 +316,12 @@ class Server(ThreadingHTTPServer):
allow_reuse_address = os.name != "nt" # on Windows SO_REUSEADDR would let a second server share the port
daemon_threads = True
+ def server_bind(self):
+ # HTTPServer.server_bind also calls socket.getfqdn(), a reverse DNS lookup that took over
+ # 30 s on a macOS runner before the engine could answer. The name is never used here.
+ socketserver.TCPServer.server_bind(self)
+ self.server_name, self.server_port = "localhost", self.server_address[1]
+
class Server6(Server):
address_family = socket.AF_INET6
diff --git a/engine/test_cdp.py b/engine/test_cdp.py
index 4571cb5..325df58 100644
--- a/engine/test_cdp.py
+++ b/engine/test_cdp.py
@@ -124,6 +124,7 @@ class WebSocketAndCDP(unittest.TestCase):
s.listen(1)
def refuse():
with s, s.accept()[0] as c:
+ c.recv(4096) # closing with the request unread makes Windows reset the connection
c.sendall(b"HTTP/1.1 404 No\r\n\r\n")
threading.Thread(target=refuse, daemon=True).start()
with self.assertRaises(cdp.WebSocketError):
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 7de576d..4aa66fc 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -435,6 +435,8 @@ class Download(unittest.TestCase):
self.assertIn("libnss3.so", msg)
self.assertIn("sudo apt-get install -y libnss3", msg)
self.assertIn("code 3", browser.start_error("/x/chrome", 3, ""))
+ trace = "[1:2:FATAL:zygote_host_impl_linux.cc(130)] No usable sandbox!\n#0 0x1 base::debug\n#1 0x2 x\n#2 0x3 y\n"
+ self.assertIn("No usable sandbox", browser.start_error("/x/chrome", -5, trace))
if __name__ == "__main__":