Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
spike: own Google Maps engine over Chrome DevTools (stdlib)
Mahanaicoache101a7d894e3
diff --git a/engine/browser.py b/engine/browser.pynew file mode 100644index 0000000..f161d8d--- /dev/null+++ b/engine/browser.py@@ -0,0 +1,102 @@+"""Find and run a local Chrome for the engine: headless, loopback DevTools port, private profile.++ with Browser() as b:+ page = cdp.Page(b.cdp)++Uses an installed Google Chrome, Chromium, Edge or Brave (set CHROME=/path/to/binary to pick one).+Standard library only.+"""+import os, platform, shutil, subprocess, sys, tempfile, time++from cdp import CDP++CANDIDATES = {+ "Darwin": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",+ "/Applications/Chromium.app/Contents/MacOS/Chromium",+ "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",+ "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],+ "Linux": ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser",+ "microsoft-edge", "brave-browser"],+ "Windows": [r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",+ r"%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe",+ r"%LocalAppData%\Google\Chrome\Application\chrome.exe",+ r"%ProgramFiles%\Chromium\Application\chrome.exe",+ r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",+ r"%ProgramFiles%\BraveSoftware\Brave-Browser\Application\brave.exe"],+}+++def find_chrome():+ """Path of a usable Chromium-based browser, or None."""+ if os.environ.get("CHROME"):+ return os.environ["CHROME"]+ for c in CANDIDATES.get(platform.system(), []):+ c = os.path.expandvars(c)+ if os.path.isfile(c):+ return c+ if os.sep not in c and shutil.which(c):+ return shutil.which(c)+ return None+++class Browser:+ """A headless Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP."""++ def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):+ self.path = path or find_chrome()+ if not self.path:+ sys.exit("✗ No Chrome, Chromium, Edge or Brave found. Install Google Chrome, or set CHROME to "+ "the browser's path.")+ self._own_profile = profile is None+ self.profile = profile or tempfile.mkdtemp(prefix="gmskit-chrome-")+ args = [self.path, "--remote-debugging-port=0", f"--user-data-dir={self.profile}",+ "--no-first-run", "--no-default-browser-check", "--disable-extensions",+ "--disable-background-networking", "--disable-sync", "--mute-audio",+ "--disable-blink-features=AutomationControlled", "--window-size=1280,900",+ f"--lang={lang}", "--disable-dev-shm-usage", "about:blank"]+ if headless:+ args.insert(1, "--headless=new")+ kw = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL, "stdin": subprocess.DEVNULL}+ if os.name == "nt":+ kw["creationflags"] = subprocess.CREATE_NO_WINDOW+ self.proc = subprocess.Popen(args, **kw)+ active = os.path.join(self.profile, "DevToolsActivePort")+ end = time.time() + timeout+ while time.time() < end:+ if self.proc.poll() is not None:+ sys.exit(f"✗ Chrome exited at once (code {self.proc.returncode}): {self.path}")+ try:+ with open(active, encoding="utf-8") as f:+ port, path = f.read().split()+ break+ except (OSError, ValueError):+ time.sleep(0.1)+ else:+ self.close()+ sys.exit(f"✗ Chrome did not open its DevTools port within {timeout} s: {self.path}")+ self.ws_url = f"ws://127.0.0.1:{port}{path}"+ self.cdp = CDP(self.ws_url, timeout=timeout)++ def version(self):+ return self.cdp.send("Browser.getVersion").get("product", "")++ def close(self):+ try:+ self.cdp.close()+ except Exception: # noqa: BLE001 - closing is best effort+ pass+ if self.proc.poll() is None:+ self.proc.terminate()+ try:+ self.proc.wait(5)+ except subprocess.TimeoutExpired:+ self.proc.kill()+ self.proc.wait(5)+ if self._own_profile:+ shutil.rmtree(self.profile, ignore_errors=True)++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ self.close()diff --git a/engine/cdp.py b/engine/cdp.pynew file mode 100644index 0000000..81685c5--- /dev/null+++ b/engine/cdp.py@@ -0,0 +1,259 @@+"""Chrome DevTools Protocol over a standard-library websocket client.++ cdp = CDP("ws://127.0.0.1:9222/devtools/browser/<id>")+ target = cdp.send("Target.createTarget", {"url": "about:blank"})["targetId"]+ sid = cdp.send("Target.attachToTarget", {"targetId": target, "flatten": True})["sessionId"]+ cdp.send("Page.navigate", {"url": "https://example.com"}, sid)+ cdp.wait_event("Page.loadEventFired", sid)++One reader thread parses frames and routes replies to the caller that sent them (by id) and events+to a queue. Nothing here is Google-specific. Standard library only.+"""+import base64, hashlib, json, os, queue, socket, struct, threading, time, urllib.parse++_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"+++class WebSocketError(OSError):+ pass+++class WebSocket:+ """RFC 6455 client for one ws:// URL: text frames in and out, fragments joined, pings answered."""++ def __init__(self, url, timeout=30):+ u = urllib.parse.urlparse(url)+ if u.scheme != "ws":+ raise WebSocketError(f"only ws:// is supported, not {u.scheme}://")+ self.sock = socket.create_connection((u.hostname, u.port or 80), timeout=timeout)+ self.sock.settimeout(timeout)+ self._lock = threading.Lock()+ key = base64.b64encode(os.urandom(16)).decode()+ path = (u.path or "/") + (f"?{u.query}" if u.query else "")+ self.sock.sendall((f"GET {path} HTTP/1.1\r\nHost: {u.hostname}:{u.port or 80}\r\n"+ "Upgrade: websocket\r\nConnection: Upgrade\r\n"+ f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n").encode())+ head = b""+ while b"\r\n\r\n" not in head:+ chunk = self.sock.recv(4096)+ if not chunk:+ raise WebSocketError("connection closed during the websocket handshake")+ head += chunk+ head, self._buf = head.split(b"\r\n\r\n", 1)+ status, _, rest = head.decode("latin-1").partition("\r\n")+ if " 101 " not in status:+ raise WebSocketError(f"websocket handshake refused: {status}")+ want = base64.b64encode(hashlib.sha1((key + _GUID).encode()).digest()).decode()+ accept = [v.strip() for k, _, v in (l.partition(":") for l in rest.split("\r\n"))+ if k.strip().lower() == "sec-websocket-accept"]+ if accept != [want]:+ raise WebSocketError("websocket handshake: bad Sec-WebSocket-Accept")++ def _read(self, n):+ while len(self._buf) < n:+ chunk = self.sock.recv(max(65536, n - len(self._buf)))+ if not chunk:+ raise WebSocketError("websocket closed")+ self._buf += chunk+ out, self._buf = self._buf[:n], self._buf[n:]+ return out++ def _frame(self, opcode, payload):+ mask = os.urandom(4) # clients must mask (RFC 6455 §5.3)+ n = len(payload)+ head = bytes([0x80 | opcode])+ if n < 126:+ head += bytes([0x80 | n])+ elif n < 65536:+ head += bytes([0x80 | 126]) + struct.pack(">H", n)+ else:+ head += bytes([0x80 | 127]) + struct.pack(">Q", n)+ masked = bytes(b ^ mask[i & 3] for i, b in enumerate(payload))+ with self._lock:+ self.sock.sendall(head + mask + masked)++ def send(self, text):+ self._frame(0x1, text.encode("utf-8"))++ def recv(self):+ """The next complete text message (fragments joined); pings answered, close raised."""+ parts, kind = [], None+ while True:+ b0, b1 = self._read(2)+ fin, opcode, n = b0 & 0x80, b0 & 0x0F, b1 & 0x7F+ if n == 126:+ n = struct.unpack(">H", self._read(2))[0]+ elif n == 127:+ n = struct.unpack(">Q", self._read(8))[0]+ mask = self._read(4) if b1 & 0x80 else None # servers normally send unmasked+ data = self._read(n)+ if mask:+ data = bytes(b ^ mask[i & 3] for i, b in enumerate(data))+ if opcode == 0x8:+ raise WebSocketError("websocket closed by the server")+ if opcode == 0x9:+ self._frame(0xA, data)+ continue+ if opcode == 0xA:+ continue+ if opcode in (0x1, 0x2):+ kind, parts = opcode, [data]+ elif opcode == 0x0 and kind is not None:+ parts.append(data)+ else:+ raise WebSocketError(f"unexpected websocket opcode {opcode}")+ if fin:+ msg = b"".join(parts)+ kind, parts = None, []+ return msg.decode("utf-8")++ def close(self):+ try:+ self._frame(0x8, struct.pack(">H", 1000))+ except OSError:+ pass+ try:+ self.sock.close()+ except OSError:+ pass+++class CDPError(Exception):+ pass+++class CDP:+ """One DevTools connection. send() blocks for the reply; events queue up per (session, name)."""++ def __init__(self, ws_url, timeout=30):+ self.ws = WebSocket(ws_url, timeout=timeout)+ self.timeout = timeout+ self._id = 0+ self._lock = threading.Lock()+ self._pending = {} # id -> queue of one reply+ self.events = queue.Queue() # (session_id or None, method, params), for callers who want all+ self._subs = {} # (session_id, method) -> list of queues+ self.closed = None+ self._reader = threading.Thread(target=self._loop, daemon=True)+ self._reader.start()++ def _loop(self):+ try:+ while True:+ msg = json.loads(self.ws.recv())+ if "id" in msg:+ q = self._pending.pop(msg["id"], None)+ if q:+ q.put(msg)+ else:+ key = (msg.get("sessionId"), msg.get("method"))+ for q in self._subs.get(key, ()):+ q.put(msg.get("params", {}))+ self.events.put((msg.get("sessionId"), msg.get("method"), msg.get("params", {})))+ except Exception as e: # noqa: BLE001 - a dead connection ends every waiter with the cause+ self.closed = e+ for q in list(self._pending.values()):+ q.put({"error": {"message": f"connection lost: {e}"}})++ def send(self, method, params=None, session_id=None, timeout=None):+ if self.closed:+ raise CDPError(f"connection lost: {self.closed}")+ with self._lock:+ self._id += 1+ mid = self._id+ q = queue.Queue(1)+ self._pending[mid] = q+ msg = {"id": mid, "method": method, "params": params or {}}+ if session_id:+ msg["sessionId"] = session_id+ self.ws.send(json.dumps(msg))+ try:+ reply = q.get(timeout=timeout or self.timeout)+ except queue.Empty:+ self._pending.pop(mid, None)+ raise CDPError(f"{method}: no reply within {timeout or self.timeout} s")+ if "error" in reply:+ raise CDPError(f"{method}: {reply['error'].get('message')}")+ return reply.get("result", {})++ def subscribe(self, method, session_id=None):+ """A queue that receives the params of every `method` event on `session_id` from now on."""+ q = queue.Queue()+ self._subs.setdefault((session_id, method), []).append(q)+ return q++ def unsubscribe(self, method, q, session_id=None):+ subs = self._subs.get((session_id, method), [])+ if q in subs:+ subs.remove(q)++ def wait_event(self, method, session_id=None, timeout=None, q=None):+ """Block until `method` fires on `session_id` (subscribe first with q= to not miss a fast one)."""+ own = q is None+ q = q or self.subscribe(method, session_id)+ try:+ return q.get(timeout=timeout or self.timeout)+ except queue.Empty:+ raise CDPError(f"no {method} event within {timeout or self.timeout} s")+ finally:+ if own:+ self.unsubscribe(method, q, session_id)++ def close(self):+ self.ws.close()+++class Page:+ """One attached tab: navigate, evaluate JavaScript, wait."""++ def __init__(self, cdp, url="about:blank"):+ self.cdp = cdp+ self.target = cdp.send("Target.createTarget", {"url": url})["targetId"]+ self.sid = cdp.send("Target.attachToTarget", {"targetId": self.target, "flatten": True})["sessionId"]+ for domain in ("Page", "Runtime", "Network"):+ cdp.send(f"{domain}.enable", session_id=self.sid)++ def send(self, method, params=None, timeout=None):+ return self.cdp.send(method, params, self.sid, timeout)++ def set_cookie(self, name, value, domain):+ self.send("Network.setCookie", {"name": name, "value": value, "domain": domain, "path": "/",+ "secure": True})++ def navigate(self, url, timeout=45):+ loaded = self.cdp.subscribe("Page.loadEventFired", self.sid)+ try:+ r = self.send("Page.navigate", {"url": url}, timeout=timeout)+ if r.get("errorText"):+ raise CDPError(f"navigate {url}: {r['errorText']}")+ self.cdp.wait_event("Page.loadEventFired", self.sid, timeout, q=loaded)+ finally:+ self.cdp.unsubscribe("Page.loadEventFired", loaded, self.sid)++ def eval(self, expression, timeout=None):+ """The JSON value of a JavaScript expression (awaited if it is a promise)."""+ r = self.send("Runtime.evaluate", {"expression": expression, "returnByValue": True,+ "awaitPromise": True}, timeout)+ if "exceptionDetails" in r:+ d = r["exceptionDetails"]+ raise CDPError(f"JavaScript: {d.get('exception', {}).get('description') or d.get('text')}")+ return r.get("result", {}).get("value")++ def url(self):+ return self.eval("location.href")++ def wait_for(self, expression, timeout=15, every=0.25):+ """Poll a JavaScript expression until it is truthy; None if it never is."""+ end = time.time() + timeout+ while time.time() < end:+ v = self.eval(expression)+ if v:+ return v+ time.sleep(every)+ return None++ def close(self):+ try:+ self.cdp.send("Target.closeTarget", {"targetId": self.target})+ except CDPError:+ passdiff --git a/engine/details.py b/engine/details.pynew file mode 100644index 0000000..60fda9f--- /dev/null+++ b/engine/details.py@@ -0,0 +1,89 @@+"""Place fields from Google Maps' own result data.++While the result list scrolls, the page fetches batches of about 20 places from+`https://www.google.com/search?tbm=map…` as `)]}'\\n[...]` JSON; each batch carries the full place+entity (name, address, phone, website, rating, coordinates, ids…). The engine captures those+responses in the browser (cdp Network events) and parses them here, so no place page is ever+opened: the spike showed the place page itself, fetched by curl or by Chrome, holds no entity data+any more. Standard library only.+"""+import json++# Index path of each field inside a place entity. Checked on live batches in the spike+# (2026-10-31): title, category, address, rating, lat/lon, place_id, data_id, timezone, thumbnail+# filled on 60/60 entities, website 55/60, phone 46/60 (listings without one).+# review_count is the one unverified path: (37, 1) is an integer on every entity in the range a+# review count has (23 to 1998 on 40 coffee shops), but signed-out Google Maps shows no counts in+# the list or the place panel, so it could not be compared against the page. Verify it in P1+# against a few listings whose counts are known (a signed-in browser shows them).+FIELDS = {+ "title": (11,),+ "category": (13, 0),+ "address": (18,),+ "website": (7, 0),+ "phone": (178, 0, 0),+ "review_rating": (4, 7),+ "review_count": (37, 1),+ "latitude": (9, 2),+ "longitude": (9, 3),+ "plus_code": (183, 2, 2, 0),+ "place_id": (78,),+ "data_id": (10,),+ "open_state": (203, 1, 4, 0),+ "thumbnail": (72, 0, 0, 6, 0),+ "timezone": (30,),+ "price_range": (4, 2),+ "description": (32, 1, 1),+}+LISTS = {"categories": (13,), "address_lines": (2,)}+ENTRIES = 64 # position of the result list inside a batch+++def dig(node, path):+ for i in path:+ if not isinstance(node, list) or i >= len(node) or node[i] is None:+ return None+ node = node[i]+ return node+++def parse_entity(entity):+ """{field: value} for every field that resolves to a string or number; lists joined with ' | '."""+ out = {}+ for name, path in FIELDS.items():+ v = dig(entity, path)+ if isinstance(v, (str, int, float)) and not isinstance(v, bool) and v != "":+ out[name] = v+ for name, path in LISTS.items():+ v = dig(entity, path)+ if isinstance(v, list):+ items = [x for x in v if isinstance(x, str) and x]+ if items:+ out[name] = " | ".join(items)+ if "data_id" in out and ":" in out["data_id"]:+ try:+ out["cid"] = str(int(out["data_id"].split(":")[1], 16))+ except ValueError:+ pass+ return out+++def parse_batch(body):+ """The places in one `/search?tbm=map` response body (with or without the `)]}'` prefix)."""+ if body.startswith(")]}'"):+ body = body.split("\n", 1)[-1]+ try:+ data = json.loads(body)+ except ValueError:+ return []+ entries = dig(data, (ENTRIES,))+ if not isinstance(entries, list):+ return []+ out = []+ for e in entries:+ entity = dig(e, (1,))+ if isinstance(entity, list):+ row = parse_entity(entity)+ if row.get("title"):+ out.append(row)+ return outdiff --git a/engine/maps.py b/engine/maps.pynew file mode 100644index 0000000..20bca15--- /dev/null+++ b/engine/maps.py@@ -0,0 +1,159 @@+"""Google Maps search in a tab: open the search, pass the consent page, scroll the result feed,+and collect the places from the result batches the page itself downloads. Standard library only.+"""+import queue, re, time, urllib.parse++import details+from cdp import CDPError++# Google's consent interstitial (EU and some other regions) is skipped when these cookies exist.+CONSENT_COOKIES = {"CONSENT": "YES+cb.20210720-07-p0.en+FX+410",+ "SOCS": "CAISHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmVuIAEaBgiAo_CmBg"}+END_MARKERS = ("You've reached the end of the list", "reached the end")+BATCH_URL = re.compile(r"^https://www\.google\.[a-z.]+/search\?.*tbm=map")+_CID = re.compile(r"!1s(0x[0-9a-f]+:0x[0-9a-f]+)")+_LAT_LON = re.compile(r"!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)")+_PLACE_ID = re.compile(r"!19s(ChIJ[\w-]+)")++# JavaScript run in the page. The feed is the scrollable results column.+JS_FEED = "!!document.querySelector('div[role=\"feed\"]')"+JS_LINKS = """(() => {+ const out = [];+ for (const a of document.querySelectorAll('div[role="feed"] a[href*="/maps/place/"]'))+ out.push({href: a.href, label: a.getAttribute('aria-label') || ''});+ return out;+})()"""+JS_SCROLL = """(() => {+ const f = document.querySelector('div[role="feed"]');+ if (!f) return null;+ f.scrollTo(0, f.scrollHeight);+ return f.scrollHeight;+})()"""+JS_END = "(() => { const f = document.querySelector('div[role=\"feed\"]'); return f ? f.innerText : ''; })()"+JS_CONSENT = """(() => {+ const b = [...document.querySelectorAll('button')].find(b => /accept all|reject all|alle akzeptieren|alle ablehnen/i.test(b.innerText));+ if (b) { b.click(); return true; } return false;+})()"""+++def search_url(query, lat=None, lon=None, zoom=15, lang="en"):+ q = urllib.parse.quote(query, safe="")+ at = f"/@{lat},{lon},{zoom}z" if lat is not None and lon is not None else ""+ return f"https://www.google.com/maps/search/{q}{at}?hl={lang}"+++def parse_link(href, label=""):+ """{link, title, data_id, latitude, longitude, place_id} from a /maps/place/ URL (the fallback+ when no result batch could be read)."""+ path = urllib.parse.urlparse(href).path+ name = ""+ if "/maps/place/" in path:+ name = urllib.parse.unquote_plus(path.split("/maps/place/", 1)[1].split("/", 1)[0])+ m = _CID.search(href)+ ll = _LAT_LON.search(href)+ pid = _PLACE_ID.search(href)+ return {"link": href, "title": label or name, "data_id": m.group(1) if m else "",+ "latitude": ll.group(1) if ll else "", "longitude": ll.group(2) if ll else "",+ "place_id": pid.group(1) if pid else ""}+++class Batches:+ """Collects the result batches the page downloads: subscribe before navigating, then drain()."""++ def __init__(self, page):+ self.page = page+ self.q = page.cdp.subscribe("Network.responseReceived", page.sid)+ self.pending = [] # requestIds seen but not yet readable+ self.count = 0++ def drain(self):+ """New places from batches that finished since the last call."""+ while True:+ try:+ p = self.q.get_nowait()+ except queue.Empty:+ break+ r = p.get("response", {})+ if BATCH_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):+ self.pending.append(p["requestId"])+ rows, still = [], []+ for rid in self.pending:+ try:+ body = self.page.send("Network.getResponseBody", {"requestId": rid})["body"]+ except CDPError:+ still.append(rid) # not finished yet; try again next time+ continue+ self.count += 1+ rows += details.parse_batch(body)+ self.pending = still+ return rows++ def close(self):+ self.page.cdp.unsubscribe("Network.responseReceived", self.q, self.page.sid)+++def collect(page, url, depth=1, max_seconds=180, log=print):+ """Open `url`, scroll the result feed, and return (places, stop reason, stats).+ `depth` is the number of result batches to read (about 20 places each, like the old engine's+ depth); the list may end sooner. Places are de-duplicated on data_id."""+ for k, v in CONSENT_COOKIES.items():+ page.set_cookie(k, v, ".google.com")+ batches = Batches(page)+ seen, started, reason, stale = {}, time.time(), "depth", 0+ try:+ page.navigate(url)+ if "consent.google" in page.url():+ page.eval(JS_CONSENT)+ page.wait_for("!location.href.includes('consent.google')", timeout=15)+ if not page.wait_for(JS_FEED, timeout=15):+ if "/maps/place/" in page.url(): # the query resolved to a single place+ return [parse_link(page.url())], "single", {"batches": 0, "dom_links": 1}+ return [], "no-feed", {"batches": 0, "dom_links": 0}++ def take():+ new = 0+ for row in batches.drain():+ key = row.get("data_id") or row.get("place_id") or row["title"]+ if key not in seen:+ seen[key] = row+ new += 1+ return new++ end = time.time() + 10 # the first batch arrives with the page+ while time.time() < end and batches.count == 0:+ take()+ time.sleep(0.4)+ log(f" batch {batches.count}: {len(seen)} places")+ while batches.count < depth and time.time() - started < max_seconds:+ if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):+ reason = "end-of-list"+ break+ before = batches.count+ page.eval(JS_SCROLL)+ end = time.time() + 10+ while time.time() < end and batches.count == before:+ time.sleep(0.4)+ take()+ if batches.count == before:+ page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging+ if batches.count == before:+ stale += 1+ if stale >= 2:+ reason = "no-growth"+ break+ else:+ stale = 0+ log(f" batch {batches.count}: {len(seen)} places")+ else:+ if time.time() - started >= max_seconds:+ reason = "timeout"+ take()+ links = page.eval(JS_LINKS) or []+ if not seen: # the batch format changed under us: fall back to what the page shows+ for a in links:+ p = parse_link(a["href"], a["label"])+ seen.setdefault(p["data_id"] or p["link"], p)+ reason = reason + "+dom-fallback"+ return list(seen.values()), reason, {"batches": batches.count, "dom_links": len(links)}+ finally:+ batches.close()diff --git a/engine/spike.py b/engine/spike.pynew file mode 100644index 0000000..3cc70e3--- /dev/null+++ b/engine/spike.py@@ -0,0 +1,53 @@+#!/usr/bin/env python3+"""Feasibility spike for the kit's own engine: one Google Maps search in a local headless Chrome,+driven over the DevTools protocol with the standard library; places come from the result batches+the page downloads while the list scrolls, so no place page is opened.++ python3 engine/spike.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 3++Prints counts and fill rates only, never names, phones or emails. Exit 0 when places came back.+"""+import argparse, collections, json, os, sys, time++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+import details, maps # noqa: E402+from browser import Browser # noqa: E402+from cdp import Page # noqa: E402+++def main():+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])+ ap.add_argument("query")+ ap.add_argument("lat", nargs="?", type=float)+ ap.add_argument("lon", nargs="?", type=float)+ ap.add_argument("--depth", type=int, default=3, help="result batches to read, about 20 places each (default 3)")+ ap.add_argument("--lang", default="en")+ ap.add_argument("--headful", action="store_true", help="show the browser window")+ ap.add_argument("--json", help="also write the counts to this file")+ a = ap.parse_args()++ t0 = time.time()+ url = maps.search_url(a.query, a.lat, a.lon, lang=a.lang)+ print(f"▶ {url}")+ with Browser(headless=not a.headful) as b:+ print(f" browser: {b.version()} ({b.path})")+ page = Page(b.cdp)+ places, reason, stats = maps.collect(page, url, depth=a.depth)+ page.close()+ took = time.time() - t0+ fill = collections.Counter(k for p in places for k in p)+ print(f"✓ {len(places)} places in {took:.1f} s from {stats['batches']} batches "+ f"(stopped: {reason}; the page showed {stats['dom_links']} result links)")+ for k in list(details.FIELDS) + list(details.LISTS) + ["cid"]:+ print(f" {k:14} {fill[k]:>3}/{len(places)}")+ summary = {"query_words": len(a.query.split()), "depth": a.depth, "places": len(places),+ "batches": stats["batches"], "dom_links": stats["dom_links"], "stop": reason,+ "seconds": round(took, 1), "filled": {k: fill[k] for k in fill}}+ if a.json:+ with open(a.json, "w", encoding="utf-8") as f:+ json.dump(summary, f, indent=1)+ sys.exit(0 if places else 1)+++if __name__ == "__main__":+ main()diff --git a/engine/test_cdp.py b/engine/test_cdp.pynew file mode 100644index 0000000..840bb2f--- /dev/null+++ b/engine/test_cdp.py@@ -0,0 +1,204 @@+#!/usr/bin/env python3+"""Offline checks for the engine spike: websocket framing against a fake server, CDP routing,+place-link parsing and the APP_INITIALIZATION_STATE parser on a synthetic page. No network, no Chrome.++ python3 engine/test_cdp.py+"""+import base64, hashlib, json, os, socket, struct, sys, threading, unittest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+import cdp, details, maps # noqa: E402++GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"+++def _recv_frame(conn):+ """(opcode, payload) of one client frame, unmasked."""+ b0, b1 = conn.recv(2)+ n = b1 & 0x7F+ if n == 126:+ n = struct.unpack(">H", conn.recv(2))[0]+ elif n == 127:+ n = struct.unpack(">Q", conn.recv(8))[0]+ mask = conn.recv(4)+ data = b""+ while len(data) < n:+ data += conn.recv(n - len(data))+ return b0 & 0x0F, bytes(b ^ mask[i & 3] for i, b in enumerate(data))+++def _send_frame(conn, opcode, payload, fin=True):+ head = bytes([(0x80 if fin else 0) | opcode])+ n = len(payload)+ if n < 126:+ head += bytes([n])+ elif n < 65536:+ head += bytes([126]) + struct.pack(">H", n)+ else:+ head += bytes([127]) + struct.pack(">Q", n)+ conn.sendall(head + payload)+++class FakeDevTools(threading.Thread):+ """Answers the handshake, then: echoes CDP requests as results, emits one event, pings once,+ and sends one big and one fragmented message when asked."""++ def __init__(self):+ super().__init__(daemon=True)+ self.srv = socket.socket()+ self.srv.bind(("127.0.0.1", 0))+ self.srv.listen(1)+ self.port = self.srv.getsockname()[1]+ self.start()++ def run(self):+ conn, _ = self.srv.accept()+ self.srv.close()+ with conn:+ self.serve(conn)++ def serve(self, conn):+ head = b""+ while b"\r\n\r\n" not in head:+ head += conn.recv(4096)+ key = [l.split(":", 1)[1].strip() for l in head.decode().split("\r\n") if l.lower().startswith("sec-websocket-key")][0]+ accept = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode()+ conn.sendall(f"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"+ f"Sec-WebSocket-Accept: {accept}\r\n\r\n".encode())+ while True:+ op, data = _recv_frame(conn)+ if op == 0x8:+ return+ if op == 0xA:+ continue+ msg = json.loads(data)+ if msg["method"] == "Test.big":+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {"s": "x" * 70000}}).encode())+ elif msg["method"] == "Test.fragments":+ body = json.dumps({"id": msg["id"], "result": {"ok": True}}).encode()+ _send_frame(conn, 0x1, body[:5], fin=False)+ _send_frame(conn, 0x0, body[5:], fin=True)+ elif msg["method"] == "Test.event":+ _send_frame(conn, 0x9, b"ping")+ _send_frame(conn, 0x1, json.dumps({"method": "Page.loadEventFired", "sessionId": "S1",+ "params": {"timestamp": 1}}).encode())+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {}}).encode())+ elif msg["method"] == "Test.error":+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "error": {"message": "nope"}}).encode())+ else:+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {"echo": msg}}).encode())+++class WebSocketAndCDP(unittest.TestCase):+ def setUp(self):+ self.srv = FakeDevTools()+ self.c = cdp.CDP(f"ws://127.0.0.1:{self.srv.port}/devtools/browser/x", timeout=5)++ def tearDown(self):+ self.c.close()++ def test_roundtrip_masked_and_routed(self):+ r = self.c.send("Target.createTarget", {"url": "about:blank"}, session_id="S1")+ self.assertEqual(r["echo"]["method"], "Target.createTarget")+ self.assertEqual(r["echo"]["sessionId"], "S1")+ self.assertEqual(r["echo"]["params"], {"url": "about:blank"})++ def test_large_frame_64bit_length(self):+ self.assertEqual(len(self.c.send("Test.big")["s"]), 70000)++ def test_fragments_joined(self):+ self.assertTrue(self.c.send("Test.fragments")["ok"])++ def test_event_and_ping(self):+ q = self.c.subscribe("Page.loadEventFired", "S1")+ self.c.send("Test.event")+ self.assertEqual(self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q), {"timestamp": 1})++ def test_error_reply_raises(self):+ with self.assertRaises(cdp.CDPError):+ self.c.send("Test.error")++ def test_bad_handshake(self):+ s = socket.socket()+ s.bind(("127.0.0.1", 0))+ s.listen(1)+ def refuse():+ with s, s.accept()[0] as c:+ c.sendall(b"HTTP/1.1 404 No\r\n\r\n")+ threading.Thread(target=refuse, daemon=True).start()+ with self.assertRaises(cdp.WebSocketError):+ cdp.WebSocket(f"ws://127.0.0.1:{s.getsockname()[1]}/x", timeout=5)+++class Links(unittest.TestCase):+ def test_search_url(self):+ self.assertEqual(maps.search_url("coffee shops in Austin TX", 30.2672, -97.7431),+ "https://www.google.com/maps/search/coffee%20shops%20in%20Austin%20TX/@30.2672,-97.7431,15z?hl=en")+ self.assertEqual(maps.search_url("x", lang="de"), "https://www.google.com/maps/search/x?hl=de")++ def test_parse_link(self):+ href = ("https://www.google.com/maps/place/Example+Caf%C3%A9/data=!4m7!3m6!1s0x8644b5a3c1234567:0x9abcdef012345678"+ "!8m2!3d30.2671!4d-97.7430!16s%2Fg%2F11abc!19sChIJabcDEF123456789-_")+ p = maps.parse_link(href)+ self.assertEqual(p["title"], "Example Café")+ self.assertEqual(p["data_id"], "0x8644b5a3c1234567:0x9abcdef012345678")+ self.assertEqual((p["latitude"], p["longitude"]), ("30.2671", "-97.7430"))+ self.assertEqual(p["place_id"], "ChIJabcDEF123456789-_")+ self.assertEqual(maps.parse_link(href, "Label wins")["title"], "Label wins")+++def entity(**over):+ """A synthetic place entity with the fields at the positions details.FIELDS expects."""+ e = [None] * 210+ e[11] = "Example Café"+ e[13] = ["Coffee shop", "Cafe"]+ e[2] = ["1 Main St", "Austin, TX 78701"]+ e[18] = "1 Main St, Austin, TX 78701"+ e[7] = ["https://example.com/"]+ e[178] = [["+1 512-555-0100"]]+ e[4] = [None, None, "$$"] + [None] * 4 + [4.6]+ e[37] = [None, 321]+ e[9] = [None, None, 30.2671, -97.743]+ e[78] = "ChIJabc"+ e[10] = "0x1:0xff"+ e[30] = "America/Chicago"+ for k, v in over.items():+ e[int(k[1:])] = v+ return e+++class Parser(unittest.TestCase):+ def test_fields_resolve(self):+ got = details.parse_entity(entity())+ self.assertEqual(got["title"], "Example Café")+ self.assertEqual(got["category"], "Coffee shop")+ self.assertEqual(got["categories"], "Coffee shop | Cafe")+ self.assertEqual(got["address_lines"], "1 Main St | Austin, TX 78701")+ self.assertEqual(got["phone"], "+1 512-555-0100")+ self.assertEqual(got["website"], "https://example.com/")+ self.assertEqual(got["review_rating"], 4.6)+ self.assertEqual(got["review_count"], 321)+ self.assertEqual((got["latitude"], got["longitude"]), (30.2671, -97.743))+ self.assertEqual(got["place_id"], "ChIJabc")+ self.assertEqual(got["cid"], "255")+ self.assertEqual(got["price_range"], "$$")+ self.assertNotIn("plus_code", got)+ self.assertNotIn("phone", details.parse_entity(entity(i178=None)))++ def test_batch(self):+ data = [None] * 64 + [[[None, entity()], [None, entity(i11="Other")], ["not an entry"], [None, [None] * 20]]]+ rows = details.parse_batch(")]}'\n" + json.dumps(data))+ self.assertEqual([r["title"] for r in rows], ["Example Café", "Other"])+ self.assertEqual(details.parse_batch(json.dumps(data)), rows) # prefix optional+ self.assertEqual(details.parse_batch("not json"), [])+ self.assertEqual(details.parse_batch("[1,2,3]"), [])++ def test_batch_url(self):+ self.assertTrue(maps.BATCH_URL.match("https://www.google.com/search?tbm=map&authuser=0&hl=en&pb=!4m12"))+ self.assertTrue(maps.BATCH_URL.match("https://www.google.co.uk/search?q=x&tbm=map"))+ self.assertFalse(maps.BATCH_URL.match("https://www.google.com/maps/vt/pb=!1m4"))+ self.assertFalse(maps.BATCH_URL.match("https://www.google.com/search?q=plain"))+++if __name__ == "__main__":+ unittest.main(verbosity=1)