Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
#!/usr/bin/env python3
"""Offline checks for the engine spike: websocket framing against a fake server, CDP routing,
place-link parsing and the APP_INITIALIZATION_STATE parser on a synthetic page. No network, no Chrome.
python3 engine/test_cdp.py
"""
import base64, hashlib, json, os, queue, socket, struct, sys, threading, time, unittest
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import cdp, details, maps # noqa: E402
GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
def _recv_frame(conn):
"""(opcode, payload) of one client frame, unmasked."""
b0, b1 = conn.recv(2)
n = b1 & 0x7F
if n == 126:
n = struct.unpack(">H", conn.recv(2))[0]
elif n == 127:
n = struct.unpack(">Q", conn.recv(8))[0]
mask = conn.recv(4)
data = b""
while len(data) < n:
data += conn.recv(n - len(data))
return b0 & 0x0F, bytes(b ^ mask[i & 3] for i, b in enumerate(data))
def _send_frame(conn, opcode, payload, fin=True):
head = bytes([(0x80 if fin else 0) | opcode])
n = len(payload)
if n < 126:
head += bytes([n])
elif n < 65536:
head += bytes([126]) + struct.pack(">H", n)
else:
head += bytes([127]) + struct.pack(">Q", n)
conn.sendall(head + payload)
class FakeDevTools(threading.Thread):
"""Answers the handshake, then: echoes CDP requests as results, emits one event, pings once,
and sends one big and one fragmented message when asked."""
def __init__(self):
super().__init__(daemon=True)
self.srv = socket.socket()
self.srv.bind(("127.0.0.1", 0))
self.srv.listen(1)
self.port = self.srv.getsockname()[1]
self.start()
def run(self):
conn, _ = self.srv.accept()
self.srv.close()
with conn:
self.serve(conn)
def serve(self, conn):
head = b""
while b"\r\n\r\n" not in head:
head += conn.recv(4096)
key = [l.split(":", 1)[1].strip() for l in head.decode().split("\r\n") if l.lower().startswith("sec-websocket-key")][0]
accept = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode()
conn.sendall(f"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
f"Sec-WebSocket-Accept: {accept}\r\n\r\n".encode())
while True:
op, data = _recv_frame(conn)
if op == 0x8:
return
if op == 0xA:
continue
msg = json.loads(data)
if msg["method"] == "Test.big":
_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {"s": "x" * 70000}}).encode())
elif msg["method"] == "Test.fragments":
body = json.dumps({"id": msg["id"], "result": {"ok": True}}).encode()
_send_frame(conn, 0x1, body[:5], fin=False)
_send_frame(conn, 0x0, body[5:], fin=True)
elif msg["method"] == "Test.event":
_send_frame(conn, 0x9, b"ping")
_send_frame(conn, 0x1, json.dumps({"method": "Page.loadEventFired", "sessionId": "S1",
"params": {"timestamp": 1}}).encode())
_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {}}).encode())
elif msg["method"] == "Test.die": # Chrome crashed: the socket closes without a reply
return
elif msg["method"] == "Test.error":
_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "error": {"message": "nope"}}).encode())
else:
_send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {"echo": msg}}).encode())
class WebSocketAndCDP(unittest.TestCase):
def setUp(self):
self.srv = FakeDevTools()
self.c = cdp.CDP(f"ws://127.0.0.1:{self.srv.port}/devtools/browser/x", timeout=5)
def tearDown(self):
self.c.close()
def test_roundtrip_masked_and_routed(self):
r = self.c.send("Target.createTarget", {"url": "about:blank"}, session_id="S1")
self.assertEqual(r["echo"]["method"], "Target.createTarget")
self.assertEqual(r["echo"]["sessionId"], "S1")
self.assertEqual(r["echo"]["params"], {"url": "about:blank"})
def test_large_frame_64bit_length(self):
self.assertEqual(len(self.c.send("Test.big")["s"]), 70000)
def test_fragments_joined(self):
self.assertTrue(self.c.send("Test.fragments")["ok"])
def test_event_and_ping(self):
q = self.c.subscribe("Page.loadEventFired", "S1")
self.c.send("Test.event")
self.assertEqual(self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q), {"timestamp": 1})
def test_dead_connection_wakes_every_waiter(self):
q = self.c.subscribe("Page.loadEventFired", "S1")
with self.assertRaises(cdp.CDPError):
self.c.send("Test.die")
t = time.time()
with self.assertRaisesRegex(cdp.CDPError, "connection lost"):
self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q)
with self.assertRaisesRegex(cdp.CDPError, "connection lost"):
self.c.wait_event("Page.frameNavigated", "S1", timeout=5) # subscribed after the death
self.assertLess(time.time() - t, 2)
def test_broken_socket_is_a_cdp_error(self):
self.c.ws.sock.close() # as if Chrome died a moment before the reader noticed
with self.assertRaisesRegex(cdp.CDPError, "connection lost"):
self.c.send("X.y")
self.assertEqual(self.c._pending, {})
def test_wait_for(self):
class Tab(cdp.Page):
def __init__(self, answers):
self.cdp, self.answers = self.c, list(answers)
def eval(self, expression, timeout=None):
a = self.answers.pop(0) if self.answers else None
if isinstance(a, Exception):
raise a
return a
Tab.c = self.c
gone = cdp.CDPError("Execution context was destroyed")
self.assertEqual(Tab([gone, None, 7]).wait_for("x", timeout=2, every=0.01), 7) # through a navigation
with self.assertRaisesRegex(cdp.CDPError, "Session with given id not found"):
Tab([cdp.CDPError("Session with given id not found")] * 500).wait_for("x", timeout=0.2, every=0.01)
self.assertIsNone(Tab([None] * 500).wait_for("x", timeout=0.2, every=0.01))
def test_quiet_browser_keeps_the_reader(self):
c = cdp.CDP(f"ws://127.0.0.1:{FakeDevTools().port}/devtools/browser/y", timeout=0.5)
time.sleep(1) # longer than the command timeout with no frame at all
self.assertIsNone(c.closed)
self.assertEqual(c.send("X.y")["echo"]["method"], "X.y")
c.close()
def test_error_reply_raises(self):
with self.assertRaises(cdp.CDPError):
self.c.send("Test.error")
def test_bad_handshake(self):
s = socket.socket()
s.bind(("127.0.0.1", 0))
s.listen(1)
def refuse():
with s, s.accept()[0] as c:
c.recv(4096) # closing with the request unread makes Windows reset the connection
c.sendall(b"HTTP/1.1 404 No\r\n\r\n")
threading.Thread(target=refuse, daemon=True).start()
with self.assertRaises(cdp.WebSocketError):
cdp.WebSocket(f"ws://127.0.0.1:{s.getsockname()[1]}/x", timeout=5)
class Links(unittest.TestCase):
def test_search_url(self):
self.assertEqual(maps.search_url("coffee shops in Austin TX", 30.2672, -97.7431),
"https://www.google.com/maps/search/coffee%20shops%20in%20Austin%20TX/@30.2672,-97.7431,15z?hl=en")
self.assertEqual(maps.search_url("x", lang="de"), "https://www.google.com/maps/search/x?hl=de")
def test_parse_link(self):
href = ("https://www.google.com/maps/place/Example+Caf%C3%A9/data=!4m7!3m6!1s0x8644b5a3c1234567:0x9abcdef012345678"
"!8m2!3d30.2671!4d-97.7430!16s%2Fg%2F11abc!19sChIJabcDEF123456789-_")
p = maps.parse_link(href)
self.assertEqual(p["title"], "Example Café")
self.assertEqual(p["data_id"], "0x8644b5a3c1234567:0x9abcdef012345678")
self.assertEqual((p["latitude"], p["longitude"]), ("30.2671", "-97.7430"))
self.assertEqual(p["place_id"], "ChIJabcDEF123456789-_")
self.assertEqual(maps.parse_link(href, "Label wins")["title"], "Label wins")
def entity(**over):
"""A synthetic place entity with the fields at the positions details.FIELDS expects."""
e = [None] * 210
e[11] = "Example Café"
e[13] = ["Coffee shop", "Cafe"]
e[2] = ["1 Main St", "Austin, TX 78701"]
e[18] = "1 Main St, Austin, TX 78701"
e[7] = ["https://example.com/"]
e[178] = [["+1 512-555-0100"]]
e[4] = [None, None, "$$"] + [None] * 4 + [4.6]
e[37] = [None, 321]
e[9] = [None, None, 30.2671, -97.743]
e[78] = "ChIJabc"
e[10] = "0x1:0xff"
e[30] = "America/Chicago"
for k, v in over.items():
e[int(k[1:])] = v
return e
class Parser(unittest.TestCase):
def test_fields_resolve(self):
got = details.parse_entity(entity())
self.assertEqual(got["title"], "Example Café")
self.assertEqual(got["category"], "Coffee shop")
self.assertEqual(got["categories"], "Coffee shop | Cafe")
self.assertEqual(got["address_lines"], "1 Main St | Austin, TX 78701")
self.assertEqual(got["phone"], "+1 512-555-0100")
self.assertEqual(got["website"], "https://example.com/")
self.assertEqual(got["review_rating"], 4.6)
self.assertNotIn("review_count", got) # (37, 1) is not the count: it comes from the card labels
self.assertEqual((got["latitude"], got["longitude"]), (30.2671, -97.743))
self.assertEqual(got["place_id"], "ChIJabc")
self.assertEqual(got["cid"], "255")
self.assertEqual(got["price_range"], "$$")
self.assertNotIn("plus_code", got)
self.assertNotIn("phone", details.parse_entity(entity(i178=None)))
def test_batch(self):
data = [None] * 64 + [[[None, entity()], [None, entity(i11="Other")], ["not an entry"], [None, [None] * 20]]]
rows = details.parse_batch(")]}'\n" + json.dumps(data))
self.assertEqual([r["title"] for r in rows], ["Example Café", "Other"])
self.assertEqual(details.parse_batch(json.dumps(data)), rows) # prefix optional
self.assertEqual(details.parse_batch("not json"), [])
self.assertEqual(details.parse_batch("[1,2,3]"), [])
def test_batch_url(self):
self.assertTrue(maps.BATCH_URL.match("https://www.google.com/search?tbm=map&authuser=0&hl=en&pb=!4m12"))
self.assertTrue(maps.BATCH_URL.match("https://www.google.co.uk/search?q=x&tbm=map"))
self.assertFalse(maps.BATCH_URL.match("https://www.google.com/maps/vt/pb=!1m4"))
self.assertFalse(maps.BATCH_URL.match("https://www.google.com/search?q=plain"))
class PageUserAgent(unittest.TestCase):
def test_user_agent_applied_to_every_context_before_navigation(self):
class Connection:
user_agent = "Mozilla/5.0 Chrome/156.0.0.0"
def __init__(self):
self.calls = []
def send(self, method, params=None, session_id=None, timeout=None):
self.calls.append((method, params, session_id))
return {"targetId": "T", "sessionId": "S"}
conn = Connection()
for context in (None, "proxy-context", "reviews-context"):
conn.calls.clear()
cdp.Page(conn, context=context)
self.assertEqual(conn.calls[-1], ("Emulation.setUserAgentOverride",
{"userAgent": conn.user_agent}, "S"))
self.assertEqual(conn.calls[0][1].get("browserContextId"), context)
conn.calls.clear()
conn.user_agent = None
cdp.Page(conn)
self.assertNotIn("Emulation.setUserAgentOverride", [c[0] for c in conn.calls])
def test_a_failed_setup_closes_the_tab(self):
from unittest.mock import Mock
conn = Mock(user_agent="Mozilla/5.0 Chrome/156.0.0.0", closed=None)
def send(method, params=None, session_id=None, timeout=None):
if method == "Emulation.setUserAgentOverride":
raise cdp.CDPError("refused")
return {"targetId": "T", "sessionId": "S"}
conn.send.side_effect = send
with self.assertRaises(cdp.CDPError):
cdp.Page(conn)
self.assertEqual(conn.send.call_args.args, ("Target.closeTarget", {"targetId": "T"}))
def test_initial_url_loads_after_tab_configuration(self):
from unittest.mock import Mock
conn = Mock(user_agent="Mozilla/5.0 Chrome/156.0.0.0")
conn.send.return_value = {"targetId": "T", "sessionId": "S"}
conn.wait_event.return_value = {}
cdp.Page(conn, url="https://example.com", context="proxy-context")
calls = conn.send.call_args_list
self.assertEqual(calls[0].args, ("Target.createTarget",
{"url": "about:blank", "browserContextId": "proxy-context"}))
methods = [c.args[0] for c in calls]
self.assertLess(methods.index("Emulation.setUserAgentOverride"), methods.index("Page.navigate"))
self.assertEqual(calls[-1].args[:2], ("Page.navigate", {"url": "https://example.com"}))
class FakeCDP:
def __init__(self):
self.closed, self.q = None, queue.Queue()
def subscribe(self, method, session_id=None, q=None):
return self.q
def unsubscribe(self, *a):
pass
class FakeMapsPage:
"""Just enough of cdp.Page for maps.collect: batches arrive as responseReceived events and
getResponseBody replies; `fail_eval` makes Runtime.evaluate of that script raise."""
def __init__(self, bodies, fail_eval=None):
self.cdp, self.sid, self.bodies, self.fail_eval, self.body_calls = FakeCDP(), "S", bodies, fail_eval, {}
for rid in bodies:
self.cdp.q.put({"requestId": rid, "response": {
"url": "https://www.google.com/search?tbm=map&pb=x", "mimeType": "application/json"}})
def set_cookie(self, *a):
pass
def navigate(self, url):
pass
def url(self):
return "https://www.google.com/maps/search/x"
def eval(self, expr):
if expr == self.fail_eval:
raise cdp.CDPError("Runtime.evaluate: no reply within 30 s")
return [] if expr == maps.JS_LINKS else 1
def wait_for(self, expr, timeout=15):
return True
def send(self, method, params=None):
rid = params["requestId"]
self.body_calls[rid] = self.body_calls.get(rid, 0) + 1
if self.bodies[rid] is None:
raise cdp.CDPError("No resource with given identifier found")
return {"body": self.bodies[rid]}
def batch(*titles, start=1):
"""One result batch body; `start` numbers the place ids so later batches hold new places."""
return ")]}'\n" + json.dumps([None] * 64 + [[[None, entity(i11=t, i10=f"0x1:0x{i:x}")] for i, t in enumerate(titles, start)]])
def batch_events(rid, body, q, delay=0.0, fail=False):
"""Replay one result batch the way Chrome reports it: the request now, its answer after `delay`."""
q.put({"requestId": rid, "request": {"url": "https://www.google.com/search?tbm=map&pb=x"}})
def answer():
if fail:
q.put({"requestId": rid, "errorText": "net::ERR_FAILED"})
else:
q.put({"requestId": rid, "response": {"url": "https://www.google.com/search?tbm=map&pb=x",
"mimeType": "application/json"}})
if delay:
threading.Timer(delay, answer).start()
else:
answer()
class ScrollingPage(FakeMapsPage):
"""A page whose scrolls fetch more batches: `later` is a list of (body, delay, fail) that each
JS_SCROLL after the first batch triggers in turn; an empty list means Google asks for nothing."""
def __init__(self, first, later, render_nudges=0):
super().__init__({})
self.later, self.scrolls, self.render_nudges = list(later), 0, render_nudges
self.bodies["r0"] = first
batch_events("r0", first, self.cdp.q)
def eval(self, expr):
if expr == maps.JS_SCROLL:
self.scrolls += 1
if self.render_nudges and self.scrolls % (self.render_nudges + 1):
return self.scrolls # the feed is still drawing: its height grows, nothing is asked yet
if self.later:
body, delay, fail = self.later.pop(0)
rid = f"r{self.scrolls}"
self.bodies[rid] = body
batch_events(rid, body, self.cdp.q, delay, fail)
return super().eval(expr)
class Collect(unittest.TestCase):
def setUp(self):
self.saved = maps.SETTLE, maps.SLOW_BATCH
maps.SETTLE, maps.SLOW_BATCH = 0.6, 4
def tearDown(self):
maps.SETTLE, maps.SLOW_BATCH = self.saved
def test_silent_end_is_end_of_list_and_quick(self):
# Google often stops without drawing its end line: the page then asks for nothing on a scroll.
page = ScrollingPage(batch("A", "B"), [])
t0 = time.time()
places, reason, stats = maps.collect(page, "u", depth=10, log=lambda m: None)
self.assertEqual(reason, "end-of-list")
self.assertEqual(len(places), 2)
self.assertLess(time.time() - t0, 3) # two SETTLE windows, not two 10 s stalls
def test_slow_batch_is_awaited(self):
# A batch Google answers after SETTLE but before SLOW_BATCH still counts.
page = ScrollingPage(batch("A"), [(batch("B", start=2), 1.5, False), (batch("C", start=3), 1.5, False)])
places, reason, stats = maps.collect(page, "u", depth=3, log=lambda m: None)
self.assertEqual([p["title"] for p in places], ["A", "B", "C"])
self.assertEqual(reason, "depth")
self.assertEqual(stats["batches"], 3)
def test_slow_rendering_is_not_the_end(self):
# The feed takes longer than SETTLE to draw the last batch's cards (its height keeps growing),
# so the bottom, and the next request, come late. That is not a complete list.
page = ScrollingPage(batch("A"), [(batch("B", start=2), 0, False), (batch("C", start=3), 0, False)],
render_nudges=8) # 8 nudges x 0.4 s, well past SETTLE, before each request
places, reason, stats = maps.collect(page, "u", depth=3, log=lambda m: None)
self.assertEqual([p["title"] for p in places], ["A", "B", "C"])
self.assertEqual(reason, "depth")
def test_failed_batches_end_as_no_growth(self):
page = ScrollingPage(batch("A"), [(batch("B", start=2), 0.2, True), (batch("C", start=3), 0.2, True)])
places, reason, _ = maps.collect(page, "u", depth=5, log=lambda m: None)
self.assertEqual([p["title"] for p in places], ["A"])
self.assertEqual(reason, "no-growth")
def test_too_slow_batch_is_no_growth_not_a_hang(self):
page = ScrollingPage(batch("A"), [(batch("B", start=2), 30, False), (batch("C", start=3), 30, False)])
t0 = time.time()
places, reason, _ = maps.collect(page, "u", depth=5, log=lambda m: None)
self.assertEqual(reason, "no-growth")
self.assertLess(time.time() - t0, 2 * maps.SLOW_BATCH + 2)
def test_error_mid_scroll_keeps_places(self):
page = FakeMapsPage({"r1": batch("A", "B")}, fail_eval=maps.JS_END)
places, reason, _ = maps.collect(page, "u", depth=3, log=lambda m: None)
self.assertEqual([p["title"] for p in places], ["A", "B"])
self.assertTrue(reason.startswith("error: "))
def test_error_before_any_place_raises(self):
page = FakeMapsPage({"r1": None}, fail_eval=maps.JS_END)
with self.assertRaises(cdp.CDPError):
maps.collect(page, "u", depth=3, log=lambda m: None, max_seconds=20)
def test_unreadable_batch_is_dropped(self):
b = maps.Batches(FakeMapsPage({"good": batch("A"), "gone": None}))
rows = [r for _ in range(40) for r in b.drain()]
self.assertEqual([r["title"] for r in rows], ["A"])
self.assertEqual(b.pending, {})
self.assertLessEqual(b.page.body_calls["gone"], 27)
if __name__ == "__main__":
unittest.main(verbosity=1)