Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(engine): audit loop 2: memory, partial rows, dead connections

Mahan Jafari51afb7b9332f
diff --git a/.claude/settings.json b/.claude/settings.json
index c494561..7f4bbb6 100644
--- a/.claude/settings.json
+++ b/.claude/settings.json
@@ -1,5 +1,5 @@
{
- "$comment": "Pre-approves the safe, local commands this kit uses so Claude can scrape on autopilot without a permission prompt each time. All entries are local-only (localhost / this folder's scripts / docker). Nothing here can reach the public internet except the scraper's own outbound calls.",
+ "$comment": "Pre-approves the safe, local commands this kit uses so Claude can scrape on autopilot without a permission prompt each time. All entries are local-only (localhost / this folder's scripts / docker). Nothing here can reach the public internet except the scraper's own outbound calls, and `scripts/engine.py start` on a machine with no Chrome, which downloads Chrome for Testing once, pinned by SHA-256.",
"permissions": {
"allow": [
"Bash(docker compose up:*)",
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.yml
index 207e4ca..f218846 100644
--- a/.github/workflows/engine-live.yml
+++ b/.github/workflows/engine-live.yml
@@ -10,6 +10,8 @@ on:
- "engine/**"
- "scripts/engine.py"
- ".github/workflows/engine-live.yml"
+permissions:
+ contents: read
jobs:
live:
strategy:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index ff006a5..0ddec20 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -6,6 +6,8 @@ on:
push:
branches: [master]
pull_request:
+permissions:
+ contents: read
jobs:
offline:
strategy:
diff --git a/engine/cdp.py b/engine/cdp.py
index 81685c5..e6c6d82 100644
--- a/engine/cdp.py
+++ b/engine/cdp.py
@@ -48,6 +48,9 @@ class WebSocket:
if k.strip().lower() == "sec-websocket-accept"]
if accept != [want]:
raise WebSocketError("websocket handshake: bad Sec-WebSocket-Accept")
+ # The reader blocks on recv() for as long as the browser is quiet (a slow page, a proxy that
+ # takes its time); each command has its own timeout in CDP.send instead.
+ self.sock.settimeout(None)
def _read(self, n):
while len(self._buf) < n:
@@ -122,8 +125,12 @@ class CDPError(Exception):
pass
+_DEAD = object() # put on every event queue when the connection dies
+
+
class CDP:
- """One DevTools connection. send() blocks for the reply; events queue up per (session, name)."""
+ """One DevTools connection. send() blocks for the reply; events reach the queues subscribed to
+ (session, name) and are dropped otherwise, so a long job holds no unread events."""
def __init__(self, ws_url, timeout=30):
self.ws = WebSocket(ws_url, timeout=timeout)
@@ -131,7 +138,6 @@ class CDP:
self._id = 0
self._lock = threading.Lock()
self._pending = {} # id -> queue of one reply
- self.events = queue.Queue() # (session_id or None, method, params), for callers who want all
self._subs = {} # (session_id, method) -> list of queues
self.closed = None
self._reader = threading.Thread(target=self._loop, daemon=True)
@@ -149,11 +155,13 @@ class CDP:
key = (msg.get("sessionId"), msg.get("method"))
for q in self._subs.get(key, ()):
q.put(msg.get("params", {}))
- self.events.put((msg.get("sessionId"), msg.get("method"), msg.get("params", {})))
except Exception as e: # noqa: BLE001 - a dead connection ends every waiter with the cause
self.closed = e
for q in list(self._pending.values()):
q.put({"error": {"message": f"connection lost: {e}"}})
+ for qs in list(self._subs.values()):
+ for q in list(qs):
+ q.put(_DEAD)
def send(self, method, params=None, session_id=None, timeout=None):
if self.closed:
@@ -163,6 +171,9 @@ class CDP:
mid = self._id
q = queue.Queue(1)
self._pending[mid] = q
+ if self.closed: # the reader died after the check above and will never answer
+ self._pending.pop(mid, None)
+ raise CDPError(f"connection lost: {self.closed}")
msg = {"id": mid, "method": method, "params": params or {}}
if session_id:
msg["sessionId"] = session_id
@@ -191,13 +202,18 @@ class CDP:
"""Block until `method` fires on `session_id` (subscribe first with q= to not miss a fast one)."""
own = q is None
q = q or self.subscribe(method, session_id)
+ if self.closed and q.empty():
+ q.put(_DEAD)
try:
- return q.get(timeout=timeout or self.timeout)
+ got = q.get(timeout=timeout or self.timeout)
except queue.Empty:
- raise CDPError(f"no {method} event within {timeout or self.timeout} s")
+ raise CDPError(f"no {method} event within {timeout or self.timeout} s") from None
finally:
if own:
self.unsubscribe(method, q, session_id)
+ if got is _DEAD:
+ raise CDPError(f"connection lost: {self.closed}")
+ return got
def close(self):
self.ws.close()
@@ -243,10 +259,16 @@ class Page:
return self.eval("location.href")
def wait_for(self, expression, timeout=15, every=0.25):
- """Poll a JavaScript expression until it is truthy; None if it never is."""
+ """Poll a JavaScript expression until it is truthy; None if it never is. An error while the
+ page navigates ("Execution context was destroyed") is retried; a dead connection is not."""
end = time.time() + timeout
while time.time() < end:
- v = self.eval(expression)
+ try:
+ v = self.eval(expression)
+ except CDPError:
+ if self.cdp.closed:
+ raise
+ v = None
if v:
return v
time.sleep(every)
@@ -255,5 +277,5 @@ class Page:
def close(self):
try:
self.cdp.send("Target.closeTarget", {"targetId": self.target})
- except CDPError:
+ except (CDPError, OSError): # a dead socket fails in send() before the reader notices
pass
diff --git a/engine/jobs.py b/engine/jobs.py
index 72ab46c..c86c6d0 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -38,7 +38,7 @@ def utc_now():
# ── Job body ──
-LANG = re.compile(r"^[A-Za-z]{2}$")
+LANG = re.compile(r"[A-Za-z]{2}") # fullmatch: "$" would let "en\n" through
PROXY_SCHEMES = ("http", "https", "socks5", "socks5h")
@@ -69,7 +69,7 @@ def validate(body):
lang = body.get("lang")
if lang in (None, ""):
raise ValueError("missing lang")
- if not isinstance(lang, str) or not LANG.match(lang):
+ if not isinstance(lang, str) or not LANG.fullmatch(lang):
raise ValueError("invalid lang")
lat, lon = body.get("lat"), body.get("lon")
if lat in (None, "") or lon in (None, ""):
@@ -335,6 +335,10 @@ class Worker(threading.Thread):
break # deleted
self.log(f"job {jid} keyword {i}/{len(kws)}: {len(places)} places, {new} new, "
f"{stats['batches']} batches, {time.monotonic() - started:.1f} s ({reason})")
+ if reason.startswith("error: "): # Chrome failed mid-scroll; the places before it are kept
+ errors.append(reason[7:])
+ if self._browser.cdp.closed:
+ break
except browser.BrowserError as e:
errors.append(str(e))
self.log(f"job {jid}: {e}")
diff --git a/engine/maps.py b/engine/maps.py
index aead9cc..817574e 100644
--- a/engine/maps.py
+++ b/engine/maps.py
@@ -63,7 +63,7 @@ class Batches:
def __init__(self, page):
self.page = page
self.q = page.cdp.subscribe("Network.responseReceived", page.sid)
- self.pending = [] # requestIds seen but not yet readable
+ self.pending = {} # requestId -> tries, for batches seen but not yet readable
self.count = 0
def drain(self):
@@ -73,19 +73,25 @@ class Batches:
p = self.q.get_nowait()
except queue.Empty:
break
+ if not isinstance(p, dict): # the connection died (cdp puts a marker on every queue)
+ raise CDPError(f"connection lost: {self.page.cdp.closed}")
r = p.get("response", {})
if BATCH_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):
- self.pending.append(p["requestId"])
- rows, still = [], []
- for rid in self.pending:
+ self.pending[p["requestId"]] = 0
+ rows = []
+ for rid in list(self.pending):
try:
body = self.page.send("Network.getResponseBody", {"requestId": rid})["body"]
except CDPError:
- still.append(rid) # not finished yet; try again next time
+ if self.page.cdp.closed:
+ raise
+ self.pending[rid] += 1 # not finished yet; try again next time, for about 10 s
+ if self.pending[rid] > 25: # failed or evicted: it never will be
+ del self.pending[rid]
continue
+ del self.pending[rid]
self.count += 1
rows += details.parse_batch(body)
- self.pending = still
return rows
def close(self):
@@ -123,38 +129,43 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
new += 1
return new
- end = time.time() + 10 # the first batch arrives with the page
- while time.time() < end and batches.count == 0:
- take()
- time.sleep(0.4)
- log(f" batch {batches.count}: {len(seen)} places")
- while batches.count < depth and time.time() - started < max_seconds and not stop():
- if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):
- reason = "end-of-list"
- break
- before = batches.count
- page.eval(JS_SCROLL)
- end = time.time() + 10
- while time.time() < end and batches.count == before:
- time.sleep(0.4)
+ try: # a Chrome error mid-scroll keeps the places already read
+ end = time.time() + 10 # the first batch arrives with the page
+ while time.time() < end and batches.count == 0:
take()
- if batches.count == before:
- page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging
- if batches.count == before:
- stale += 1
- if stale >= 2:
- reason = "no-growth"
+ time.sleep(0.4)
+ log(f" batch {batches.count}: {len(seen)} places")
+ while batches.count < depth and time.time() - started < max_seconds and not stop():
+ if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):
+ reason = "end-of-list"
break
+ before = batches.count
+ page.eval(JS_SCROLL)
+ end = time.time() + 10
+ while time.time() < end and batches.count == before:
+ time.sleep(0.4)
+ take()
+ if batches.count == before:
+ page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging
+ if batches.count == before:
+ stale += 1
+ if stale >= 2:
+ reason = "no-growth"
+ break
+ else:
+ stale = 0
+ log(f" batch {batches.count}: {len(seen)} places")
else:
- stale = 0
- log(f" batch {batches.count}: {len(seen)} places")
- else:
- if stop():
- reason = "stopped"
- elif time.time() - started >= max_seconds:
- reason = "timeout"
- take()
- links = page.eval(JS_LINKS) or []
+ if stop():
+ reason = "stopped"
+ elif time.time() - started >= max_seconds:
+ reason = "timeout"
+ take()
+ links = page.eval(JS_LINKS) or []
+ except CDPError as e:
+ if not seen:
+ raise
+ return list(seen.values()), f"error: {e}", {"batches": batches.count, "dom_links": 0}
if not seen: # the batch format changed under us: fall back to what the page shows
for a in links:
p = parse_link(a["href"], a["label"])
diff --git a/engine/serve.py b/engine/serve.py
index 292970b..99077e5 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -204,12 +204,16 @@ class Handler(BaseHTTPRequestHandler):
self.send_header("Cache-Control", "no-store")
for k, v in headers:
self.send_header(k, v)
+ if self.close_connection: # tell the client too, so it never reuses the socket
+ self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(body)
if code >= 400:
log(f"{self.command} {self.path.split('?')[0]} -> {code}")
def fail(self, code, message):
+ if self.command != "GET": # a refused POST or DELETE may leave its body unread on the socket
+ self.close_connection = True
self.reply(code, {"code": code, "message": message})
def guard(self):
diff --git a/engine/test_cdp.py b/engine/test_cdp.py
index 325df58..3074db8 100644
--- a/engine/test_cdp.py
+++ b/engine/test_cdp.py
@@ -4,7 +4,7 @@ place-link parsing and the APP_INITIALIZATION_STATE parser on a synthetic page.
python3 engine/test_cdp.py
"""
-import base64, hashlib, json, os, socket, struct, sys, threading, unittest
+import base64, hashlib, json, os, queue, socket, struct, sys, threading, time, unittest
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import cdp, details, maps # noqa: E402
@@ -83,6 +83,8 @@ class FakeDevTools(threading.Thread):
_send_frame(conn, 0x1, json.dumps({"method": "Page.loadEventFired", "sessionId": "S1",
"params": {"timestamp": 1}}).encode())
_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {}}).encode())
+ elif msg["method"] == "Test.die": # Chrome crashed: the socket closes without a reply
+ return
elif msg["method"] == "Test.error":
_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "error": {"message": "nope"}}).encode())
else:
@@ -114,6 +116,24 @@ class WebSocketAndCDP(unittest.TestCase):
self.c.send("Test.event")
self.assertEqual(self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q), {"timestamp": 1})
+ def test_dead_connection_wakes_every_waiter(self):
+ q = self.c.subscribe("Page.loadEventFired", "S1")
+ with self.assertRaises(cdp.CDPError):
+ self.c.send("Test.die")
+ t = time.time()
+ with self.assertRaisesRegex(cdp.CDPError, "connection lost"):
+ self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q)
+ with self.assertRaisesRegex(cdp.CDPError, "connection lost"):
+ self.c.wait_event("Page.frameNavigated", "S1", timeout=5) # subscribed after the death
+ self.assertLess(time.time() - t, 2)
+
+ def test_quiet_browser_keeps_the_reader(self):
+ c = cdp.CDP(f"ws://127.0.0.1:{FakeDevTools().port}/devtools/browser/y", timeout=0.5)
+ time.sleep(1) # longer than the command timeout with no frame at all
+ self.assertIsNone(c.closed)
+ self.assertEqual(c.send("X.y")["echo"]["method"], "X.y")
+ c.close()
+
def test_error_reply_raises(self):
with self.assertRaises(cdp.CDPError):
self.c.send("Test.error")
@@ -201,5 +221,75 @@ class Parser(unittest.TestCase):
self.assertFalse(maps.BATCH_URL.match("https://www.google.com/search?q=plain"))
+class FakeCDP:
+ def __init__(self):
+ self.closed, self.q = None, queue.Queue()
+
+ def subscribe(self, method, session_id=None):
+ return self.q
+
+ def unsubscribe(self, *a):
+ pass
+
+
+class FakeMapsPage:
+ """Just enough of cdp.Page for maps.collect: batches arrive as responseReceived events and
+ getResponseBody replies; `fail_eval` makes Runtime.evaluate of that script raise."""
+
+ def __init__(self, bodies, fail_eval=None):
+ self.cdp, self.sid, self.bodies, self.fail_eval, self.body_calls = FakeCDP(), "S", bodies, fail_eval, {}
+ for rid in bodies:
+ self.cdp.q.put({"requestId": rid, "response": {
+ "url": "https://www.google.com/search?tbm=map&pb=x", "mimeType": "application/json"}})
+
+ def set_cookie(self, *a):
+ pass
+
+ def navigate(self, url):
+ pass
+
+ def url(self):
+ return "https://www.google.com/maps/search/x"
+
+ def eval(self, expr):
+ if expr == self.fail_eval:
+ raise cdp.CDPError("Runtime.evaluate: no reply within 30 s")
+ return [] if expr == maps.JS_LINKS else 1
+
+ def wait_for(self, expr, timeout=15):
+ return True
+
+ def send(self, method, params=None):
+ rid = params["requestId"]
+ self.body_calls[rid] = self.body_calls.get(rid, 0) + 1
+ if self.bodies[rid] is None:
+ raise cdp.CDPError("No resource with given identifier found")
+ return {"body": self.bodies[rid]}
+
+
+def batch(*titles):
+ return ")]}'\n" + json.dumps([None] * 64 + [[[None, entity(i11=t, i10=f"0x1:0x{i:x}")] for i, t in enumerate(titles, 1)]])
+
+
+class Collect(unittest.TestCase):
+ def test_error_mid_scroll_keeps_places(self):
+ page = FakeMapsPage({"r1": batch("A", "B")}, fail_eval=maps.JS_END)
+ places, reason, _ = maps.collect(page, "u", depth=3, log=lambda m: None)
+ self.assertEqual([p["title"] for p in places], ["A", "B"])
+ self.assertTrue(reason.startswith("error: "))
+
+ def test_error_before_any_place_raises(self):
+ page = FakeMapsPage({"r1": None}, fail_eval=maps.JS_END)
+ with self.assertRaises(cdp.CDPError):
+ maps.collect(page, "u", depth=3, log=lambda m: None, max_seconds=20)
+
+ def test_unreadable_batch_is_dropped(self):
+ b = maps.Batches(FakeMapsPage({"good": batch("A"), "gone": None}))
+ rows = [r for _ in range(40) for r in b.drain()]
+ self.assertEqual([r["title"] for r in rows], ["A"])
+ self.assertEqual(b.pending, {})
+ self.assertLessEqual(b.page.body_calls["gone"], 27)
+
+
if __name__ == "__main__":
unittest.main(verbosity=1)
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 9355edc..9da4120 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -110,6 +110,7 @@ class Validate(unittest.TestCase):
self.assertEqual(self.err(name="x"), "missing keywords")
self.assertEqual(self.err(name="x", keywords=["a"]), "missing lang")
self.assertEqual(self.err(name="x", keywords=["a"], lang="eng"), "invalid lang")
+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en\n"), "invalid lang")
self.assertEqual(self.err(name="x", keywords=["a"], lang="en"), "missing geo coordinates")
self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lat="1", lon="2"), "missing max time")
@@ -377,6 +378,14 @@ class Api(Server):
self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"0\r\n\r\n", headers={"Transfer-Encoding": "chunked"})[0], 411)
self.assertEqual(self.call("POST", "/elsewhere", GOOD)[0], 404)
+ def test_refused_post_closes_the_connection(self):
+ c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
+ c.request("POST", "/elsewhere", json.dumps(GOOD), {"Host": "localhost"})
+ r = c.getresponse()
+ r.read()
+ self.assertEqual((r.status, r.getheader("Connection")), (404, "close"))
+ c.close()
+
def test_page_and_docs(self):
code, body, r = self.call("GET", "/")
csp = r.getheader("Content-Security-Policy")
@@ -544,11 +553,15 @@ class LaunchChrome(unittest.TestCase):
self.assertNotIn(self.exe, out) # the fake Chrome was stopped
def test_temp_profile_removed(self):
- before = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}
- with self.assertRaises(browser.BrowserError):
- browser.Browser(path=self.exe, timeout=5)
- after = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}
- self.assertEqual(after - before, set())
+ tmp = os.path.join(self.dir, "tmp")
+ os.mkdir(tmp)
+ saved, tempfile.tempdir = tempfile.tempdir, tmp # Browser's mkdtemp lands here, not in the shared temp
+ try:
+ with self.assertRaises(browser.BrowserError):
+ browser.Browser(path=self.exe, timeout=5)
+ finally:
+ tempfile.tempdir = saved
+ self.assertEqual(os.listdir(tmp), [])
def test_no_usable_sandbox_relaunches_once_without_it(self):
os.environ["FAKE_SANDBOX"] = "1"
diff --git a/scripts/engine.py b/scripts/engine.py
index a3f235b..00d5bf8 100644
--- a/scripts/engine.py
+++ b/scripts/engine.py
@@ -115,7 +115,7 @@ def kill(pid):
cmd = command_line(pid)
if cmd is None:
return False
- if SERVE not in cmd:
+ if os.path.normcase(SERVE) not in os.path.normcase(cmd): # Windows: c:\ and C:\ are one path
return True # gone, or someone else's process now: nothing of ours to stop
if os.name == "nt":
subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True)