Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
fix(engine): audit loop 2: memory, partial rows, dead connections
Mahan Jafari51afb7b9332f
diff --git a/.claude/settings.json b/.claude/settings.jsonindex c494561..7f4bbb6 100644--- a/.claude/settings.json+++ b/.claude/settings.json@@ -1,5 +1,5 @@{- "$comment": "Pre-approves the safe, local commands this kit uses so Claude can scrape on autopilot without a permission prompt each time. All entries are local-only (localhost / this folder's scripts / docker). Nothing here can reach the public internet except the scraper's own outbound calls.",+ "$comment": "Pre-approves the safe, local commands this kit uses so Claude can scrape on autopilot without a permission prompt each time. All entries are local-only (localhost / this folder's scripts / docker). Nothing here can reach the public internet except the scraper's own outbound calls, and `scripts/engine.py start` on a machine with no Chrome, which downloads Chrome for Testing once, pinned by SHA-256.","permissions": {"allow": ["Bash(docker compose up:*)",diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.ymlindex 207e4ca..f218846 100644--- a/.github/workflows/engine-live.yml+++ b/.github/workflows/engine-live.yml@@ -10,6 +10,8 @@ on:- "engine/**"- "scripts/engine.py"- ".github/workflows/engine-live.yml"+permissions:+ contents: readjobs:live:strategy:diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex ff006a5..0ddec20 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -6,6 +6,8 @@ on:push:branches: [master]pull_request:+permissions:+ contents: readjobs:offline:strategy:diff --git a/engine/cdp.py b/engine/cdp.pyindex 81685c5..e6c6d82 100644--- a/engine/cdp.py+++ b/engine/cdp.py@@ -48,6 +48,9 @@ class WebSocket:if k.strip().lower() == "sec-websocket-accept"]if accept != [want]:raise WebSocketError("websocket handshake: bad Sec-WebSocket-Accept")+ # The reader blocks on recv() for as long as the browser is quiet (a slow page, a proxy that+ # takes its time); each command has its own timeout in CDP.send instead.+ self.sock.settimeout(None)def _read(self, n):while len(self._buf) < n:@@ -122,8 +125,12 @@ class CDPError(Exception):pass+_DEAD = object() # put on every event queue when the connection dies++class CDP:- """One DevTools connection. send() blocks for the reply; events queue up per (session, name)."""+ """One DevTools connection. send() blocks for the reply; events reach the queues subscribed to+ (session, name) and are dropped otherwise, so a long job holds no unread events."""def __init__(self, ws_url, timeout=30):self.ws = WebSocket(ws_url, timeout=timeout)@@ -131,7 +138,6 @@ class CDP:self._id = 0self._lock = threading.Lock()self._pending = {} # id -> queue of one reply- self.events = queue.Queue() # (session_id or None, method, params), for callers who want allself._subs = {} # (session_id, method) -> list of queuesself.closed = Noneself._reader = threading.Thread(target=self._loop, daemon=True)@@ -149,11 +155,13 @@ class CDP:key = (msg.get("sessionId"), msg.get("method"))for q in self._subs.get(key, ()):q.put(msg.get("params", {}))- self.events.put((msg.get("sessionId"), msg.get("method"), msg.get("params", {})))except Exception as e: # noqa: BLE001 - a dead connection ends every waiter with the causeself.closed = efor q in list(self._pending.values()):q.put({"error": {"message": f"connection lost: {e}"}})+ for qs in list(self._subs.values()):+ for q in list(qs):+ q.put(_DEAD)def send(self, method, params=None, session_id=None, timeout=None):if self.closed:@@ -163,6 +171,9 @@ class CDP:mid = self._idq = queue.Queue(1)self._pending[mid] = q+ if self.closed: # the reader died after the check above and will never answer+ self._pending.pop(mid, None)+ raise CDPError(f"connection lost: {self.closed}")msg = {"id": mid, "method": method, "params": params or {}}if session_id:msg["sessionId"] = session_id@@ -191,13 +202,18 @@ class CDP:"""Block until `method` fires on `session_id` (subscribe first with q= to not miss a fast one)."""own = q is Noneq = q or self.subscribe(method, session_id)+ if self.closed and q.empty():+ q.put(_DEAD)try:- return q.get(timeout=timeout or self.timeout)+ got = q.get(timeout=timeout or self.timeout)except queue.Empty:- raise CDPError(f"no {method} event within {timeout or self.timeout} s")+ raise CDPError(f"no {method} event within {timeout or self.timeout} s") from Nonefinally:if own:self.unsubscribe(method, q, session_id)+ if got is _DEAD:+ raise CDPError(f"connection lost: {self.closed}")+ return gotdef close(self):self.ws.close()@@ -243,10 +259,16 @@ class Page:return self.eval("location.href")def wait_for(self, expression, timeout=15, every=0.25):- """Poll a JavaScript expression until it is truthy; None if it never is."""+ """Poll a JavaScript expression until it is truthy; None if it never is. An error while the+ page navigates ("Execution context was destroyed") is retried; a dead connection is not."""end = time.time() + timeoutwhile time.time() < end:- v = self.eval(expression)+ try:+ v = self.eval(expression)+ except CDPError:+ if self.cdp.closed:+ raise+ v = Noneif v:return vtime.sleep(every)@@ -255,5 +277,5 @@ class Page:def close(self):try:self.cdp.send("Target.closeTarget", {"targetId": self.target})- except CDPError:+ except (CDPError, OSError): # a dead socket fails in send() before the reader noticespassdiff --git a/engine/jobs.py b/engine/jobs.pyindex 72ab46c..c86c6d0 100644--- a/engine/jobs.py+++ b/engine/jobs.py@@ -38,7 +38,7 @@ def utc_now():# ── Job body ──-LANG = re.compile(r"^[A-Za-z]{2}$")+LANG = re.compile(r"[A-Za-z]{2}") # fullmatch: "$" would let "en\n" throughPROXY_SCHEMES = ("http", "https", "socks5", "socks5h")@@ -69,7 +69,7 @@ def validate(body):lang = body.get("lang")if lang in (None, ""):raise ValueError("missing lang")- if not isinstance(lang, str) or not LANG.match(lang):+ if not isinstance(lang, str) or not LANG.fullmatch(lang):raise ValueError("invalid lang")lat, lon = body.get("lat"), body.get("lon")if lat in (None, "") or lon in (None, ""):@@ -335,6 +335,10 @@ class Worker(threading.Thread):break # deletedself.log(f"job {jid} keyword {i}/{len(kws)}: {len(places)} places, {new} new, "f"{stats['batches']} batches, {time.monotonic() - started:.1f} s ({reason})")+ if reason.startswith("error: "): # Chrome failed mid-scroll; the places before it are kept+ errors.append(reason[7:])+ if self._browser.cdp.closed:+ breakexcept browser.BrowserError as e:errors.append(str(e))self.log(f"job {jid}: {e}")diff --git a/engine/maps.py b/engine/maps.pyindex aead9cc..817574e 100644--- a/engine/maps.py+++ b/engine/maps.py@@ -63,7 +63,7 @@ class Batches:def __init__(self, page):self.page = pageself.q = page.cdp.subscribe("Network.responseReceived", page.sid)- self.pending = [] # requestIds seen but not yet readable+ self.pending = {} # requestId -> tries, for batches seen but not yet readableself.count = 0def drain(self):@@ -73,19 +73,25 @@ class Batches:p = self.q.get_nowait()except queue.Empty:break+ if not isinstance(p, dict): # the connection died (cdp puts a marker on every queue)+ raise CDPError(f"connection lost: {self.page.cdp.closed}")r = p.get("response", {})if BATCH_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):- self.pending.append(p["requestId"])- rows, still = [], []- for rid in self.pending:+ self.pending[p["requestId"]] = 0+ rows = []+ for rid in list(self.pending):try:body = self.page.send("Network.getResponseBody", {"requestId": rid})["body"]except CDPError:- still.append(rid) # not finished yet; try again next time+ if self.page.cdp.closed:+ raise+ self.pending[rid] += 1 # not finished yet; try again next time, for about 10 s+ if self.pending[rid] > 25: # failed or evicted: it never will be+ del self.pending[rid]continue+ del self.pending[rid]self.count += 1rows += details.parse_batch(body)- self.pending = stillreturn rowsdef close(self):@@ -123,38 +129,43 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):new += 1return new- end = time.time() + 10 # the first batch arrives with the page- while time.time() < end and batches.count == 0:- take()- time.sleep(0.4)- log(f" batch {batches.count}: {len(seen)} places")- while batches.count < depth and time.time() - started < max_seconds and not stop():- if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):- reason = "end-of-list"- break- before = batches.count- page.eval(JS_SCROLL)- end = time.time() + 10- while time.time() < end and batches.count == before:- time.sleep(0.4)+ try: # a Chrome error mid-scroll keeps the places already read+ end = time.time() + 10 # the first batch arrives with the page+ while time.time() < end and batches.count == 0:take()- if batches.count == before:- page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging- if batches.count == before:- stale += 1- if stale >= 2:- reason = "no-growth"+ time.sleep(0.4)+ log(f" batch {batches.count}: {len(seen)} places")+ while batches.count < depth and time.time() - started < max_seconds and not stop():+ if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):+ reason = "end-of-list"break+ before = batches.count+ page.eval(JS_SCROLL)+ end = time.time() + 10+ while time.time() < end and batches.count == before:+ time.sleep(0.4)+ take()+ if batches.count == before:+ page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging+ if batches.count == before:+ stale += 1+ if stale >= 2:+ reason = "no-growth"+ break+ else:+ stale = 0+ log(f" batch {batches.count}: {len(seen)} places")else:- stale = 0- log(f" batch {batches.count}: {len(seen)} places")- else:- if stop():- reason = "stopped"- elif time.time() - started >= max_seconds:- reason = "timeout"- take()- links = page.eval(JS_LINKS) or []+ if stop():+ reason = "stopped"+ elif time.time() - started >= max_seconds:+ reason = "timeout"+ take()+ links = page.eval(JS_LINKS) or []+ except CDPError as e:+ if not seen:+ raise+ return list(seen.values()), f"error: {e}", {"batches": batches.count, "dom_links": 0}if not seen: # the batch format changed under us: fall back to what the page showsfor a in links:p = parse_link(a["href"], a["label"])diff --git a/engine/serve.py b/engine/serve.pyindex 292970b..99077e5 100644--- a/engine/serve.py+++ b/engine/serve.py@@ -204,12 +204,16 @@ class Handler(BaseHTTPRequestHandler):self.send_header("Cache-Control", "no-store")for k, v in headers:self.send_header(k, v)+ if self.close_connection: # tell the client too, so it never reuses the socket+ self.send_header("Connection", "close")self.end_headers()self.wfile.write(body)if code >= 400:log(f"{self.command} {self.path.split('?')[0]} -> {code}")def fail(self, code, message):+ if self.command != "GET": # a refused POST or DELETE may leave its body unread on the socket+ self.close_connection = Trueself.reply(code, {"code": code, "message": message})def guard(self):diff --git a/engine/test_cdp.py b/engine/test_cdp.pyindex 325df58..3074db8 100644--- a/engine/test_cdp.py+++ b/engine/test_cdp.py@@ -4,7 +4,7 @@ place-link parsing and the APP_INITIALIZATION_STATE parser on a synthetic page.python3 engine/test_cdp.py"""-import base64, hashlib, json, os, socket, struct, sys, threading, unittest+import base64, hashlib, json, os, queue, socket, struct, sys, threading, time, unittestsys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))import cdp, details, maps # noqa: E402@@ -83,6 +83,8 @@ class FakeDevTools(threading.Thread):_send_frame(conn, 0x1, json.dumps({"method": "Page.loadEventFired", "sessionId": "S1","params": {"timestamp": 1}}).encode())_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {}}).encode())+ elif msg["method"] == "Test.die": # Chrome crashed: the socket closes without a reply+ returnelif msg["method"] == "Test.error":_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "error": {"message": "nope"}}).encode())else:@@ -114,6 +116,24 @@ class WebSocketAndCDP(unittest.TestCase):self.c.send("Test.event")self.assertEqual(self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q), {"timestamp": 1})+ def test_dead_connection_wakes_every_waiter(self):+ q = self.c.subscribe("Page.loadEventFired", "S1")+ with self.assertRaises(cdp.CDPError):+ self.c.send("Test.die")+ t = time.time()+ with self.assertRaisesRegex(cdp.CDPError, "connection lost"):+ self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q)+ with self.assertRaisesRegex(cdp.CDPError, "connection lost"):+ self.c.wait_event("Page.frameNavigated", "S1", timeout=5) # subscribed after the death+ self.assertLess(time.time() - t, 2)++ def test_quiet_browser_keeps_the_reader(self):+ c = cdp.CDP(f"ws://127.0.0.1:{FakeDevTools().port}/devtools/browser/y", timeout=0.5)+ time.sleep(1) # longer than the command timeout with no frame at all+ self.assertIsNone(c.closed)+ self.assertEqual(c.send("X.y")["echo"]["method"], "X.y")+ c.close()+def test_error_reply_raises(self):with self.assertRaises(cdp.CDPError):self.c.send("Test.error")@@ -201,5 +221,75 @@ class Parser(unittest.TestCase):self.assertFalse(maps.BATCH_URL.match("https://www.google.com/search?q=plain"))+class FakeCDP:+ def __init__(self):+ self.closed, self.q = None, queue.Queue()++ def subscribe(self, method, session_id=None):+ return self.q++ def unsubscribe(self, *a):+ pass+++class FakeMapsPage:+ """Just enough of cdp.Page for maps.collect: batches arrive as responseReceived events and+ getResponseBody replies; `fail_eval` makes Runtime.evaluate of that script raise."""++ def __init__(self, bodies, fail_eval=None):+ self.cdp, self.sid, self.bodies, self.fail_eval, self.body_calls = FakeCDP(), "S", bodies, fail_eval, {}+ for rid in bodies:+ self.cdp.q.put({"requestId": rid, "response": {+ "url": "https://www.google.com/search?tbm=map&pb=x", "mimeType": "application/json"}})++ def set_cookie(self, *a):+ pass++ def navigate(self, url):+ pass++ def url(self):+ return "https://www.google.com/maps/search/x"++ def eval(self, expr):+ if expr == self.fail_eval:+ raise cdp.CDPError("Runtime.evaluate: no reply within 30 s")+ return [] if expr == maps.JS_LINKS else 1++ def wait_for(self, expr, timeout=15):+ return True++ def send(self, method, params=None):+ rid = params["requestId"]+ self.body_calls[rid] = self.body_calls.get(rid, 0) + 1+ if self.bodies[rid] is None:+ raise cdp.CDPError("No resource with given identifier found")+ return {"body": self.bodies[rid]}+++def batch(*titles):+ return ")]}'\n" + json.dumps([None] * 64 + [[[None, entity(i11=t, i10=f"0x1:0x{i:x}")] for i, t in enumerate(titles, 1)]])+++class Collect(unittest.TestCase):+ def test_error_mid_scroll_keeps_places(self):+ page = FakeMapsPage({"r1": batch("A", "B")}, fail_eval=maps.JS_END)+ places, reason, _ = maps.collect(page, "u", depth=3, log=lambda m: None)+ self.assertEqual([p["title"] for p in places], ["A", "B"])+ self.assertTrue(reason.startswith("error: "))++ def test_error_before_any_place_raises(self):+ page = FakeMapsPage({"r1": None}, fail_eval=maps.JS_END)+ with self.assertRaises(cdp.CDPError):+ maps.collect(page, "u", depth=3, log=lambda m: None, max_seconds=20)++ def test_unreadable_batch_is_dropped(self):+ b = maps.Batches(FakeMapsPage({"good": batch("A"), "gone": None}))+ rows = [r for _ in range(40) for r in b.drain()]+ self.assertEqual([r["title"] for r in rows], ["A"])+ self.assertEqual(b.pending, {})+ self.assertLessEqual(b.page.body_calls["gone"], 27)++if __name__ == "__main__":unittest.main(verbosity=1)diff --git a/engine/test_engine.py b/engine/test_engine.pyindex 9355edc..9da4120 100644--- a/engine/test_engine.py+++ b/engine/test_engine.py@@ -110,6 +110,7 @@ class Validate(unittest.TestCase):self.assertEqual(self.err(name="x"), "missing keywords")self.assertEqual(self.err(name="x", keywords=["a"]), "missing lang")self.assertEqual(self.err(name="x", keywords=["a"], lang="eng"), "invalid lang")+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en\n"), "invalid lang")self.assertEqual(self.err(name="x", keywords=["a"], lang="en"), "missing geo coordinates")self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lat="1", lon="2"), "missing max time")@@ -377,6 +378,14 @@ class Api(Server):self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"0\r\n\r\n", headers={"Transfer-Encoding": "chunked"})[0], 411)self.assertEqual(self.call("POST", "/elsewhere", GOOD)[0], 404)+ def test_refused_post_closes_the_connection(self):+ c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)+ c.request("POST", "/elsewhere", json.dumps(GOOD), {"Host": "localhost"})+ r = c.getresponse()+ r.read()+ self.assertEqual((r.status, r.getheader("Connection")), (404, "close"))+ c.close()+def test_page_and_docs(self):code, body, r = self.call("GET", "/")csp = r.getheader("Content-Security-Policy")@@ -544,11 +553,15 @@ class LaunchChrome(unittest.TestCase):self.assertNotIn(self.exe, out) # the fake Chrome was stoppeddef test_temp_profile_removed(self):- before = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}- with self.assertRaises(browser.BrowserError):- browser.Browser(path=self.exe, timeout=5)- after = {n for n in os.listdir(tempfile.gettempdir()) if n.startswith("gmskit-chrome-")}- self.assertEqual(after - before, set())+ tmp = os.path.join(self.dir, "tmp")+ os.mkdir(tmp)+ saved, tempfile.tempdir = tempfile.tempdir, tmp # Browser's mkdtemp lands here, not in the shared temp+ try:+ with self.assertRaises(browser.BrowserError):+ browser.Browser(path=self.exe, timeout=5)+ finally:+ tempfile.tempdir = saved+ self.assertEqual(os.listdir(tmp), [])def test_no_usable_sandbox_relaunches_once_without_it(self):os.environ["FAKE_SANDBOX"] = "1"diff --git a/scripts/engine.py b/scripts/engine.pyindex a3f235b..00d5bf8 100644--- a/scripts/engine.py+++ b/scripts/engine.py@@ -115,7 +115,7 @@ def kill(pid):cmd = command_line(pid)if cmd is None:return False- if SERVE not in cmd:+ if os.path.normcase(SERVE) not in os.path.normcase(cmd): # Windows: c:\ and C:\ are one pathreturn True # gone, or someone else's process now: nothing of ours to stopif os.name == "nt":subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True)