Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

spike: own Google Maps engine over Chrome DevTools (stdlib)

Mahanaicoache101a7d894e3
diff --git a/engine/browser.py b/engine/browser.py
new file mode 100644
index 0000000..f161d8d
--- /dev/null
+++ b/engine/browser.py
@@ -0,0 +1,102 @@
+"""Find and run a local Chrome for the engine: headless, loopback DevTools port, private profile.
+
+ with Browser() as b:
+ page = cdp.Page(b.cdp)
+
+Uses an installed Google Chrome, Chromium, Edge or Brave (set CHROME=/path/to/binary to pick one).
+Standard library only.
+"""
+import os, platform, shutil, subprocess, sys, tempfile, time
+
+from cdp import CDP
+
+CANDIDATES = {
+ "Darwin": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
+ "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+ "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
+ "Linux": ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser",
+ "microsoft-edge", "brave-browser"],
+ "Windows": [r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
+ r"%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe",
+ r"%LocalAppData%\Google\Chrome\Application\chrome.exe",
+ r"%ProgramFiles%\Chromium\Application\chrome.exe",
+ r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
+ r"%ProgramFiles%\BraveSoftware\Brave-Browser\Application\brave.exe"],
+}
+
+
+def find_chrome():
+ """Path of a usable Chromium-based browser, or None."""
+ if os.environ.get("CHROME"):
+ return os.environ["CHROME"]
+ for c in CANDIDATES.get(platform.system(), []):
+ c = os.path.expandvars(c)
+ if os.path.isfile(c):
+ return c
+ if os.sep not in c and shutil.which(c):
+ return shutil.which(c)
+ return None
+
+
+class Browser:
+ """A headless Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP."""
+
+ def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):
+ self.path = path or find_chrome()
+ if not self.path:
+ sys.exit("✗ No Chrome, Chromium, Edge or Brave found. Install Google Chrome, or set CHROME to "
+ "the browser's path.")
+ self._own_profile = profile is None
+ self.profile = profile or tempfile.mkdtemp(prefix="gmskit-chrome-")
+ args = [self.path, "--remote-debugging-port=0", f"--user-data-dir={self.profile}",
+ "--no-first-run", "--no-default-browser-check", "--disable-extensions",
+ "--disable-background-networking", "--disable-sync", "--mute-audio",
+ "--disable-blink-features=AutomationControlled", "--window-size=1280,900",
+ f"--lang={lang}", "--disable-dev-shm-usage", "about:blank"]
+ if headless:
+ args.insert(1, "--headless=new")
+ kw = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL, "stdin": subprocess.DEVNULL}
+ if os.name == "nt":
+ kw["creationflags"] = subprocess.CREATE_NO_WINDOW
+ self.proc = subprocess.Popen(args, **kw)
+ active = os.path.join(self.profile, "DevToolsActivePort")
+ end = time.time() + timeout
+ while time.time() < end:
+ if self.proc.poll() is not None:
+ sys.exit(f"✗ Chrome exited at once (code {self.proc.returncode}): {self.path}")
+ try:
+ with open(active, encoding="utf-8") as f:
+ port, path = f.read().split()
+ break
+ except (OSError, ValueError):
+ time.sleep(0.1)
+ else:
+ self.close()
+ sys.exit(f"✗ Chrome did not open its DevTools port within {timeout} s: {self.path}")
+ self.ws_url = f"ws://127.0.0.1:{port}{path}"
+ self.cdp = CDP(self.ws_url, timeout=timeout)
+
+ def version(self):
+ return self.cdp.send("Browser.getVersion").get("product", "")
+
+ def close(self):
+ try:
+ self.cdp.close()
+ except Exception: # noqa: BLE001 - closing is best effort
+ pass
+ if self.proc.poll() is None:
+ self.proc.terminate()
+ try:
+ self.proc.wait(5)
+ except subprocess.TimeoutExpired:
+ self.proc.kill()
+ self.proc.wait(5)
+ if self._own_profile:
+ shutil.rmtree(self.profile, ignore_errors=True)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ self.close()
diff --git a/engine/cdp.py b/engine/cdp.py
new file mode 100644
index 0000000..81685c5
--- /dev/null
+++ b/engine/cdp.py
@@ -0,0 +1,259 @@
+"""Chrome DevTools Protocol over a standard-library websocket client.
+
+ cdp = CDP("ws://127.0.0.1:9222/devtools/browser/<id>")
+ target = cdp.send("Target.createTarget", {"url": "about:blank"})["targetId"]
+ sid = cdp.send("Target.attachToTarget", {"targetId": target, "flatten": True})["sessionId"]
+ cdp.send("Page.navigate", {"url": "https://example.com"}, sid)
+ cdp.wait_event("Page.loadEventFired", sid)
+
+One reader thread parses frames and routes replies to the caller that sent them (by id) and events
+to a queue. Nothing here is Google-specific. Standard library only.
+"""
+import base64, hashlib, json, os, queue, socket, struct, threading, time, urllib.parse
+
+_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+
+
+class WebSocketError(OSError):
+ pass
+
+
+class WebSocket:
+ """RFC 6455 client for one ws:// URL: text frames in and out, fragments joined, pings answered."""
+
+ def __init__(self, url, timeout=30):
+ u = urllib.parse.urlparse(url)
+ if u.scheme != "ws":
+ raise WebSocketError(f"only ws:// is supported, not {u.scheme}://")
+ self.sock = socket.create_connection((u.hostname, u.port or 80), timeout=timeout)
+ self.sock.settimeout(timeout)
+ self._lock = threading.Lock()
+ key = base64.b64encode(os.urandom(16)).decode()
+ path = (u.path or "/") + (f"?{u.query}" if u.query else "")
+ self.sock.sendall((f"GET {path} HTTP/1.1\r\nHost: {u.hostname}:{u.port or 80}\r\n"
+ "Upgrade: websocket\r\nConnection: Upgrade\r\n"
+ f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n").encode())
+ head = b""
+ while b"\r\n\r\n" not in head:
+ chunk = self.sock.recv(4096)
+ if not chunk:
+ raise WebSocketError("connection closed during the websocket handshake")
+ head += chunk
+ head, self._buf = head.split(b"\r\n\r\n", 1)
+ status, _, rest = head.decode("latin-1").partition("\r\n")
+ if " 101 " not in status:
+ raise WebSocketError(f"websocket handshake refused: {status}")
+ want = base64.b64encode(hashlib.sha1((key + _GUID).encode()).digest()).decode()
+ accept = [v.strip() for k, _, v in (l.partition(":") for l in rest.split("\r\n"))
+ if k.strip().lower() == "sec-websocket-accept"]
+ if accept != [want]:
+ raise WebSocketError("websocket handshake: bad Sec-WebSocket-Accept")
+
+ def _read(self, n):
+ while len(self._buf) < n:
+ chunk = self.sock.recv(max(65536, n - len(self._buf)))
+ if not chunk:
+ raise WebSocketError("websocket closed")
+ self._buf += chunk
+ out, self._buf = self._buf[:n], self._buf[n:]
+ return out
+
+ def _frame(self, opcode, payload):
+ mask = os.urandom(4) # clients must mask (RFC 6455 §5.3)
+ n = len(payload)
+ head = bytes([0x80 | opcode])
+ if n < 126:
+ head += bytes([0x80 | n])
+ elif n < 65536:
+ head += bytes([0x80 | 126]) + struct.pack(">H", n)
+ else:
+ head += bytes([0x80 | 127]) + struct.pack(">Q", n)
+ masked = bytes(b ^ mask[i & 3] for i, b in enumerate(payload))
+ with self._lock:
+ self.sock.sendall(head + mask + masked)
+
+ def send(self, text):
+ self._frame(0x1, text.encode("utf-8"))
+
+ def recv(self):
+ """The next complete text message (fragments joined); pings answered, close raised."""
+ parts, kind = [], None
+ while True:
+ b0, b1 = self._read(2)
+ fin, opcode, n = b0 & 0x80, b0 & 0x0F, b1 & 0x7F
+ if n == 126:
+ n = struct.unpack(">H", self._read(2))[0]
+ elif n == 127:
+ n = struct.unpack(">Q", self._read(8))[0]
+ mask = self._read(4) if b1 & 0x80 else None # servers normally send unmasked
+ data = self._read(n)
+ if mask:
+ data = bytes(b ^ mask[i & 3] for i, b in enumerate(data))
+ if opcode == 0x8:
+ raise WebSocketError("websocket closed by the server")
+ if opcode == 0x9:
+ self._frame(0xA, data)
+ continue
+ if opcode == 0xA:
+ continue
+ if opcode in (0x1, 0x2):
+ kind, parts = opcode, [data]
+ elif opcode == 0x0 and kind is not None:
+ parts.append(data)
+ else:
+ raise WebSocketError(f"unexpected websocket opcode {opcode}")
+ if fin:
+ msg = b"".join(parts)
+ kind, parts = None, []
+ return msg.decode("utf-8")
+
+ def close(self):
+ try:
+ self._frame(0x8, struct.pack(">H", 1000))
+ except OSError:
+ pass
+ try:
+ self.sock.close()
+ except OSError:
+ pass
+
+
+class CDPError(Exception):
+ pass
+
+
+class CDP:
+ """One DevTools connection. send() blocks for the reply; events queue up per (session, name)."""
+
+ def __init__(self, ws_url, timeout=30):
+ self.ws = WebSocket(ws_url, timeout=timeout)
+ self.timeout = timeout
+ self._id = 0
+ self._lock = threading.Lock()
+ self._pending = {} # id -> queue of one reply
+ self.events = queue.Queue() # (session_id or None, method, params), for callers who want all
+ self._subs = {} # (session_id, method) -> list of queues
+ self.closed = None
+ self._reader = threading.Thread(target=self._loop, daemon=True)
+ self._reader.start()
+
+ def _loop(self):
+ try:
+ while True:
+ msg = json.loads(self.ws.recv())
+ if "id" in msg:
+ q = self._pending.pop(msg["id"], None)
+ if q:
+ q.put(msg)
+ else:
+ key = (msg.get("sessionId"), msg.get("method"))
+ for q in self._subs.get(key, ()):
+ q.put(msg.get("params", {}))
+ self.events.put((msg.get("sessionId"), msg.get("method"), msg.get("params", {})))
+ except Exception as e: # noqa: BLE001 - a dead connection ends every waiter with the cause
+ self.closed = e
+ for q in list(self._pending.values()):
+ q.put({"error": {"message": f"connection lost: {e}"}})
+
+ def send(self, method, params=None, session_id=None, timeout=None):
+ if self.closed:
+ raise CDPError(f"connection lost: {self.closed}")
+ with self._lock:
+ self._id += 1
+ mid = self._id
+ q = queue.Queue(1)
+ self._pending[mid] = q
+ msg = {"id": mid, "method": method, "params": params or {}}
+ if session_id:
+ msg["sessionId"] = session_id
+ self.ws.send(json.dumps(msg))
+ try:
+ reply = q.get(timeout=timeout or self.timeout)
+ except queue.Empty:
+ self._pending.pop(mid, None)
+ raise CDPError(f"{method}: no reply within {timeout or self.timeout} s")
+ if "error" in reply:
+ raise CDPError(f"{method}: {reply['error'].get('message')}")
+ return reply.get("result", {})
+
+ def subscribe(self, method, session_id=None):
+ """A queue that receives the params of every `method` event on `session_id` from now on."""
+ q = queue.Queue()
+ self._subs.setdefault((session_id, method), []).append(q)
+ return q
+
+ def unsubscribe(self, method, q, session_id=None):
+ subs = self._subs.get((session_id, method), [])
+ if q in subs:
+ subs.remove(q)
+
+ def wait_event(self, method, session_id=None, timeout=None, q=None):
+ """Block until `method` fires on `session_id` (subscribe first with q= to not miss a fast one)."""
+ own = q is None
+ q = q or self.subscribe(method, session_id)
+ try:
+ return q.get(timeout=timeout or self.timeout)
+ except queue.Empty:
+ raise CDPError(f"no {method} event within {timeout or self.timeout} s")
+ finally:
+ if own:
+ self.unsubscribe(method, q, session_id)
+
+ def close(self):
+ self.ws.close()
+
+
+class Page:
+ """One attached tab: navigate, evaluate JavaScript, wait."""
+
+ def __init__(self, cdp, url="about:blank"):
+ self.cdp = cdp
+ self.target = cdp.send("Target.createTarget", {"url": url})["targetId"]
+ self.sid = cdp.send("Target.attachToTarget", {"targetId": self.target, "flatten": True})["sessionId"]
+ for domain in ("Page", "Runtime", "Network"):
+ cdp.send(f"{domain}.enable", session_id=self.sid)
+
+ def send(self, method, params=None, timeout=None):
+ return self.cdp.send(method, params, self.sid, timeout)
+
+ def set_cookie(self, name, value, domain):
+ self.send("Network.setCookie", {"name": name, "value": value, "domain": domain, "path": "/",
+ "secure": True})
+
+ def navigate(self, url, timeout=45):
+ loaded = self.cdp.subscribe("Page.loadEventFired", self.sid)
+ try:
+ r = self.send("Page.navigate", {"url": url}, timeout=timeout)
+ if r.get("errorText"):
+ raise CDPError(f"navigate {url}: {r['errorText']}")
+ self.cdp.wait_event("Page.loadEventFired", self.sid, timeout, q=loaded)
+ finally:
+ self.cdp.unsubscribe("Page.loadEventFired", loaded, self.sid)
+
+ def eval(self, expression, timeout=None):
+ """The JSON value of a JavaScript expression (awaited if it is a promise)."""
+ r = self.send("Runtime.evaluate", {"expression": expression, "returnByValue": True,
+ "awaitPromise": True}, timeout)
+ if "exceptionDetails" in r:
+ d = r["exceptionDetails"]
+ raise CDPError(f"JavaScript: {d.get('exception', {}).get('description') or d.get('text')}")
+ return r.get("result", {}).get("value")
+
+ def url(self):
+ return self.eval("location.href")
+
+ def wait_for(self, expression, timeout=15, every=0.25):
+ """Poll a JavaScript expression until it is truthy; None if it never is."""
+ end = time.time() + timeout
+ while time.time() < end:
+ v = self.eval(expression)
+ if v:
+ return v
+ time.sleep(every)
+ return None
+
+ def close(self):
+ try:
+ self.cdp.send("Target.closeTarget", {"targetId": self.target})
+ except CDPError:
+ pass
diff --git a/engine/details.py b/engine/details.py
new file mode 100644
index 0000000..60fda9f
--- /dev/null
+++ b/engine/details.py
@@ -0,0 +1,89 @@
+"""Place fields from Google Maps' own result data.
+
+While the result list scrolls, the page fetches batches of about 20 places from
+`https://www.google.com/search?tbm=map…` as `)]}'\\n[...]` JSON; each batch carries the full place
+entity (name, address, phone, website, rating, coordinates, ids…). The engine captures those
+responses in the browser (cdp Network events) and parses them here, so no place page is ever
+opened: the spike showed the place page itself, fetched by curl or by Chrome, holds no entity data
+any more. Standard library only.
+"""
+import json
+
+# Index path of each field inside a place entity. Checked on live batches in the spike
+# (2026-10-31): title, category, address, rating, lat/lon, place_id, data_id, timezone, thumbnail
+# filled on 60/60 entities, website 55/60, phone 46/60 (listings without one).
+# review_count is the one unverified path: (37, 1) is an integer on every entity in the range a
+# review count has (23 to 1998 on 40 coffee shops), but signed-out Google Maps shows no counts in
+# the list or the place panel, so it could not be compared against the page. Verify it in P1
+# against a few listings whose counts are known (a signed-in browser shows them).
+FIELDS = {
+ "title": (11,),
+ "category": (13, 0),
+ "address": (18,),
+ "website": (7, 0),
+ "phone": (178, 0, 0),
+ "review_rating": (4, 7),
+ "review_count": (37, 1),
+ "latitude": (9, 2),
+ "longitude": (9, 3),
+ "plus_code": (183, 2, 2, 0),
+ "place_id": (78,),
+ "data_id": (10,),
+ "open_state": (203, 1, 4, 0),
+ "thumbnail": (72, 0, 0, 6, 0),
+ "timezone": (30,),
+ "price_range": (4, 2),
+ "description": (32, 1, 1),
+}
+LISTS = {"categories": (13,), "address_lines": (2,)}
+ENTRIES = 64 # position of the result list inside a batch
+
+
+def dig(node, path):
+ for i in path:
+ if not isinstance(node, list) or i >= len(node) or node[i] is None:
+ return None
+ node = node[i]
+ return node
+
+
+def parse_entity(entity):
+ """{field: value} for every field that resolves to a string or number; lists joined with ' | '."""
+ out = {}
+ for name, path in FIELDS.items():
+ v = dig(entity, path)
+ if isinstance(v, (str, int, float)) and not isinstance(v, bool) and v != "":
+ out[name] = v
+ for name, path in LISTS.items():
+ v = dig(entity, path)
+ if isinstance(v, list):
+ items = [x for x in v if isinstance(x, str) and x]
+ if items:
+ out[name] = " | ".join(items)
+ if "data_id" in out and ":" in out["data_id"]:
+ try:
+ out["cid"] = str(int(out["data_id"].split(":")[1], 16))
+ except ValueError:
+ pass
+ return out
+
+
+def parse_batch(body):
+ """The places in one `/search?tbm=map` response body (with or without the `)]}'` prefix)."""
+ if body.startswith(")]}'"):
+ body = body.split("\n", 1)[-1]
+ try:
+ data = json.loads(body)
+ except ValueError:
+ return []
+ entries = dig(data, (ENTRIES,))
+ if not isinstance(entries, list):
+ return []
+ out = []
+ for e in entries:
+ entity = dig(e, (1,))
+ if isinstance(entity, list):
+ row = parse_entity(entity)
+ if row.get("title"):
+ out.append(row)
+ return out
diff --git a/engine/maps.py b/engine/maps.py
new file mode 100644
index 0000000..20bca15
--- /dev/null
+++ b/engine/maps.py
@@ -0,0 +1,159 @@
+"""Google Maps search in a tab: open the search, pass the consent page, scroll the result feed,
+and collect the places from the result batches the page itself downloads. Standard library only.
+"""
+import queue, re, time, urllib.parse
+
+import details
+from cdp import CDPError
+
+# Google's consent interstitial (EU and some other regions) is skipped when these cookies exist.
+CONSENT_COOKIES = {"CONSENT": "YES+cb.20210720-07-p0.en+FX+410",
+ "SOCS": "CAISHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmVuIAEaBgiAo_CmBg"}
+END_MARKERS = ("You've reached the end of the list", "reached the end")
+BATCH_URL = re.compile(r"^https://www\.google\.[a-z.]+/search\?.*tbm=map")
+_CID = re.compile(r"!1s(0x[0-9a-f]+:0x[0-9a-f]+)")
+_LAT_LON = re.compile(r"!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)")
+_PLACE_ID = re.compile(r"!19s(ChIJ[\w-]+)")
+
+# JavaScript run in the page. The feed is the scrollable results column.
+JS_FEED = "!!document.querySelector('div[role=\"feed\"]')"
+JS_LINKS = """(() => {
+ const out = [];
+ for (const a of document.querySelectorAll('div[role="feed"] a[href*="/maps/place/"]'))
+ out.push({href: a.href, label: a.getAttribute('aria-label') || ''});
+ return out;
+})()"""
+JS_SCROLL = """(() => {
+ const f = document.querySelector('div[role="feed"]');
+ if (!f) return null;
+ f.scrollTo(0, f.scrollHeight);
+ return f.scrollHeight;
+})()"""
+JS_END = "(() => { const f = document.querySelector('div[role=\"feed\"]'); return f ? f.innerText : ''; })()"
+JS_CONSENT = """(() => {
+ const b = [...document.querySelectorAll('button')].find(b => /accept all|reject all|alle akzeptieren|alle ablehnen/i.test(b.innerText));
+ if (b) { b.click(); return true; } return false;
+})()"""
+
+
+def search_url(query, lat=None, lon=None, zoom=15, lang="en"):
+ q = urllib.parse.quote(query, safe="")
+ at = f"/@{lat},{lon},{zoom}z" if lat is not None and lon is not None else ""
+ return f"https://www.google.com/maps/search/{q}{at}?hl={lang}"
+
+
+def parse_link(href, label=""):
+ """{link, title, data_id, latitude, longitude, place_id} from a /maps/place/ URL (the fallback
+ when no result batch could be read)."""
+ path = urllib.parse.urlparse(href).path
+ name = ""
+ if "/maps/place/" in path:
+ name = urllib.parse.unquote_plus(path.split("/maps/place/", 1)[1].split("/", 1)[0])
+ m = _CID.search(href)
+ ll = _LAT_LON.search(href)
+ pid = _PLACE_ID.search(href)
+ return {"link": href, "title": label or name, "data_id": m.group(1) if m else "",
+ "latitude": ll.group(1) if ll else "", "longitude": ll.group(2) if ll else "",
+ "place_id": pid.group(1) if pid else ""}
+
+
+class Batches:
+ """Collects the result batches the page downloads: subscribe before navigating, then drain()."""
+
+ def __init__(self, page):
+ self.page = page
+ self.q = page.cdp.subscribe("Network.responseReceived", page.sid)
+ self.pending = [] # requestIds seen but not yet readable
+ self.count = 0
+
+ def drain(self):
+ """New places from batches that finished since the last call."""
+ while True:
+ try:
+ p = self.q.get_nowait()
+ except queue.Empty:
+ break
+ r = p.get("response", {})
+ if BATCH_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):
+ self.pending.append(p["requestId"])
+ rows, still = [], []
+ for rid in self.pending:
+ try:
+ body = self.page.send("Network.getResponseBody", {"requestId": rid})["body"]
+ except CDPError:
+ still.append(rid) # not finished yet; try again next time
+ continue
+ self.count += 1
+ rows += details.parse_batch(body)
+ self.pending = still
+ return rows
+
+ def close(self):
+ self.page.cdp.unsubscribe("Network.responseReceived", self.q, self.page.sid)
+
+
+def collect(page, url, depth=1, max_seconds=180, log=print):
+ """Open `url`, scroll the result feed, and return (places, stop reason, stats).
+ `depth` is the number of result batches to read (about 20 places each, like the old engine's
+ depth); the list may end sooner. Places are de-duplicated on data_id."""
+ for k, v in CONSENT_COOKIES.items():
+ page.set_cookie(k, v, ".google.com")
+ batches = Batches(page)
+ seen, started, reason, stale = {}, time.time(), "depth", 0
+ try:
+ page.navigate(url)
+ if "consent.google" in page.url():
+ page.eval(JS_CONSENT)
+ page.wait_for("!location.href.includes('consent.google')", timeout=15)
+ if not page.wait_for(JS_FEED, timeout=15):
+ if "/maps/place/" in page.url(): # the query resolved to a single place
+ return [parse_link(page.url())], "single", {"batches": 0, "dom_links": 1}
+ return [], "no-feed", {"batches": 0, "dom_links": 0}
+
+ def take():
+ new = 0
+ for row in batches.drain():
+ key = row.get("data_id") or row.get("place_id") or row["title"]
+ if key not in seen:
+ seen[key] = row
+ new += 1
+ return new
+
+ end = time.time() + 10 # the first batch arrives with the page
+ while time.time() < end and batches.count == 0:
+ take()
+ time.sleep(0.4)
+ log(f" batch {batches.count}: {len(seen)} places")
+ while batches.count < depth and time.time() - started < max_seconds:
+ if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):
+ reason = "end-of-list"
+ break
+ before = batches.count
+ page.eval(JS_SCROLL)
+ end = time.time() + 10
+ while time.time() < end and batches.count == before:
+ time.sleep(0.4)
+ take()
+ if batches.count == before:
+ page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging
+ if batches.count == before:
+ stale += 1
+ if stale >= 2:
+ reason = "no-growth"
+ break
+ else:
+ stale = 0
+ log(f" batch {batches.count}: {len(seen)} places")
+ else:
+ if time.time() - started >= max_seconds:
+ reason = "timeout"
+ take()
+ links = page.eval(JS_LINKS) or []
+ if not seen: # the batch format changed under us: fall back to what the page shows
+ for a in links:
+ p = parse_link(a["href"], a["label"])
+ seen.setdefault(p["data_id"] or p["link"], p)
+ reason = reason + "+dom-fallback"
+ return list(seen.values()), reason, {"batches": batches.count, "dom_links": len(links)}
+ finally:
+ batches.close()
diff --git a/engine/spike.py b/engine/spike.py
new file mode 100644
index 0000000..3cc70e3
--- /dev/null
+++ b/engine/spike.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Feasibility spike for the kit's own engine: one Google Maps search in a local headless Chrome,
+driven over the DevTools protocol with the standard library; places come from the result batches
+the page downloads while the list scrolls, so no place page is opened.
+
+ python3 engine/spike.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 3
+
+Prints counts and fill rates only, never names, phones or emails. Exit 0 when places came back.
+"""
+import argparse, collections, json, os, sys, time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import details, maps # noqa: E402
+from browser import Browser # noqa: E402
+from cdp import Page # noqa: E402
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ ap.add_argument("query")
+ ap.add_argument("lat", nargs="?", type=float)
+ ap.add_argument("lon", nargs="?", type=float)
+ ap.add_argument("--depth", type=int, default=3, help="result batches to read, about 20 places each (default 3)")
+ ap.add_argument("--lang", default="en")
+ ap.add_argument("--headful", action="store_true", help="show the browser window")
+ ap.add_argument("--json", help="also write the counts to this file")
+ a = ap.parse_args()
+
+ t0 = time.time()
+ url = maps.search_url(a.query, a.lat, a.lon, lang=a.lang)
+ print(f"▶ {url}")
+ with Browser(headless=not a.headful) as b:
+ print(f" browser: {b.version()} ({b.path})")
+ page = Page(b.cdp)
+ places, reason, stats = maps.collect(page, url, depth=a.depth)
+ page.close()
+ took = time.time() - t0
+ fill = collections.Counter(k for p in places for k in p)
+ print(f"✓ {len(places)} places in {took:.1f} s from {stats['batches']} batches "
+ f"(stopped: {reason}; the page showed {stats['dom_links']} result links)")
+ for k in list(details.FIELDS) + list(details.LISTS) + ["cid"]:
+ print(f" {k:14} {fill[k]:>3}/{len(places)}")
+ summary = {"query_words": len(a.query.split()), "depth": a.depth, "places": len(places),
+ "batches": stats["batches"], "dom_links": stats["dom_links"], "stop": reason,
+ "seconds": round(took, 1), "filled": {k: fill[k] for k in fill}}
+ if a.json:
+ with open(a.json, "w", encoding="utf-8") as f:
+ json.dump(summary, f, indent=1)
+ sys.exit(0 if places else 1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/engine/test_cdp.py b/engine/test_cdp.py
new file mode 100644
index 0000000..840bb2f
--- /dev/null
+++ b/engine/test_cdp.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Offline checks for the engine spike: websocket framing against a fake server, CDP routing,
+place-link parsing and the APP_INITIALIZATION_STATE parser on a synthetic page. No network, no Chrome.
+
+ python3 engine/test_cdp.py
+"""
+import base64, hashlib, json, os, socket, struct, sys, threading, unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import cdp, details, maps # noqa: E402
+
+GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+
+
+def _recv_frame(conn):
+ """(opcode, payload) of one client frame, unmasked."""
+ b0, b1 = conn.recv(2)
+ n = b1 & 0x7F
+ if n == 126:
+ n = struct.unpack(">H", conn.recv(2))[0]
+ elif n == 127:
+ n = struct.unpack(">Q", conn.recv(8))[0]
+ mask = conn.recv(4)
+ data = b""
+ while len(data) < n:
+ data += conn.recv(n - len(data))
+ return b0 & 0x0F, bytes(b ^ mask[i & 3] for i, b in enumerate(data))
+
+
+def _send_frame(conn, opcode, payload, fin=True):
+ head = bytes([(0x80 if fin else 0) | opcode])
+ n = len(payload)
+ if n < 126:
+ head += bytes([n])
+ elif n < 65536:
+ head += bytes([126]) + struct.pack(">H", n)
+ else:
+ head += bytes([127]) + struct.pack(">Q", n)
+ conn.sendall(head + payload)
+
+
+class FakeDevTools(threading.Thread):
+ """Answers the handshake, then: echoes CDP requests as results, emits one event, pings once,
+ and sends one big and one fragmented message when asked."""
+
+ def __init__(self):
+ super().__init__(daemon=True)
+ self.srv = socket.socket()
+ self.srv.bind(("127.0.0.1", 0))
+ self.srv.listen(1)
+ self.port = self.srv.getsockname()[1]
+ self.start()
+
+ def run(self):
+ conn, _ = self.srv.accept()
+ self.srv.close()
+ with conn:
+ self.serve(conn)
+
+ def serve(self, conn):
+ head = b""
+ while b"\r\n\r\n" not in head:
+ head += conn.recv(4096)
+ key = [l.split(":", 1)[1].strip() for l in head.decode().split("\r\n") if l.lower().startswith("sec-websocket-key")][0]
+ accept = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode()
+ conn.sendall(f"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
+ f"Sec-WebSocket-Accept: {accept}\r\n\r\n".encode())
+ while True:
+ op, data = _recv_frame(conn)
+ if op == 0x8:
+ return
+ if op == 0xA:
+ continue
+ msg = json.loads(data)
+ if msg["method"] == "Test.big":
+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {"s": "x" * 70000}}).encode())
+ elif msg["method"] == "Test.fragments":
+ body = json.dumps({"id": msg["id"], "result": {"ok": True}}).encode()
+ _send_frame(conn, 0x1, body[:5], fin=False)
+ _send_frame(conn, 0x0, body[5:], fin=True)
+ elif msg["method"] == "Test.event":
+ _send_frame(conn, 0x9, b"ping")
+ _send_frame(conn, 0x1, json.dumps({"method": "Page.loadEventFired", "sessionId": "S1",
+ "params": {"timestamp": 1}}).encode())
+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {}}).encode())
+ elif msg["method"] == "Test.error":
+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "error": {"message": "nope"}}).encode())
+ else:
+ _send_frame(conn, 0x1, json.dumps({"id": msg["id"], "result": {"echo": msg}}).encode())
+
+
+class WebSocketAndCDP(unittest.TestCase):
+ def setUp(self):
+ self.srv = FakeDevTools()
+ self.c = cdp.CDP(f"ws://127.0.0.1:{self.srv.port}/devtools/browser/x", timeout=5)
+
+ def tearDown(self):
+ self.c.close()
+
+ def test_roundtrip_masked_and_routed(self):
+ r = self.c.send("Target.createTarget", {"url": "about:blank"}, session_id="S1")
+ self.assertEqual(r["echo"]["method"], "Target.createTarget")
+ self.assertEqual(r["echo"]["sessionId"], "S1")
+ self.assertEqual(r["echo"]["params"], {"url": "about:blank"})
+
+ def test_large_frame_64bit_length(self):
+ self.assertEqual(len(self.c.send("Test.big")["s"]), 70000)
+
+ def test_fragments_joined(self):
+ self.assertTrue(self.c.send("Test.fragments")["ok"])
+
+ def test_event_and_ping(self):
+ q = self.c.subscribe("Page.loadEventFired", "S1")
+ self.c.send("Test.event")
+ self.assertEqual(self.c.wait_event("Page.loadEventFired", "S1", timeout=5, q=q), {"timestamp": 1})
+
+ def test_error_reply_raises(self):
+ with self.assertRaises(cdp.CDPError):
+ self.c.send("Test.error")
+
+ def test_bad_handshake(self):
+ s = socket.socket()
+ s.bind(("127.0.0.1", 0))
+ s.listen(1)
+ def refuse():
+ with s, s.accept()[0] as c:
+ c.sendall(b"HTTP/1.1 404 No\r\n\r\n")
+ threading.Thread(target=refuse, daemon=True).start()
+ with self.assertRaises(cdp.WebSocketError):
+ cdp.WebSocket(f"ws://127.0.0.1:{s.getsockname()[1]}/x", timeout=5)
+
+
+class Links(unittest.TestCase):
+ def test_search_url(self):
+ self.assertEqual(maps.search_url("coffee shops in Austin TX", 30.2672, -97.7431),
+ "https://www.google.com/maps/search/coffee%20shops%20in%20Austin%20TX/@30.2672,-97.7431,15z?hl=en")
+ self.assertEqual(maps.search_url("x", lang="de"), "https://www.google.com/maps/search/x?hl=de")
+
+ def test_parse_link(self):
+ href = ("https://www.google.com/maps/place/Example+Caf%C3%A9/data=!4m7!3m6!1s0x8644b5a3c1234567:0x9abcdef012345678"
+ "!8m2!3d30.2671!4d-97.7430!16s%2Fg%2F11abc!19sChIJabcDEF123456789-_")
+ p = maps.parse_link(href)
+ self.assertEqual(p["title"], "Example Café")
+ self.assertEqual(p["data_id"], "0x8644b5a3c1234567:0x9abcdef012345678")
+ self.assertEqual((p["latitude"], p["longitude"]), ("30.2671", "-97.7430"))
+ self.assertEqual(p["place_id"], "ChIJabcDEF123456789-_")
+ self.assertEqual(maps.parse_link(href, "Label wins")["title"], "Label wins")
+
+
+def entity(**over):
+ """A synthetic place entity with the fields at the positions details.FIELDS expects."""
+ e = [None] * 210
+ e[11] = "Example Café"
+ e[13] = ["Coffee shop", "Cafe"]
+ e[2] = ["1 Main St", "Austin, TX 78701"]
+ e[18] = "1 Main St, Austin, TX 78701"
+ e[7] = ["https://example.com/"]
+ e[178] = [["+1 512-555-0100"]]
+ e[4] = [None, None, "$$"] + [None] * 4 + [4.6]
+ e[37] = [None, 321]
+ e[9] = [None, None, 30.2671, -97.743]
+ e[78] = "ChIJabc"
+ e[10] = "0x1:0xff"
+ e[30] = "America/Chicago"
+ for k, v in over.items():
+ e[int(k[1:])] = v
+ return e
+
+
+class Parser(unittest.TestCase):
+ def test_fields_resolve(self):
+ got = details.parse_entity(entity())
+ self.assertEqual(got["title"], "Example Café")
+ self.assertEqual(got["category"], "Coffee shop")
+ self.assertEqual(got["categories"], "Coffee shop | Cafe")
+ self.assertEqual(got["address_lines"], "1 Main St | Austin, TX 78701")
+ self.assertEqual(got["phone"], "+1 512-555-0100")
+ self.assertEqual(got["website"], "https://example.com/")
+ self.assertEqual(got["review_rating"], 4.6)
+ self.assertEqual(got["review_count"], 321)
+ self.assertEqual((got["latitude"], got["longitude"]), (30.2671, -97.743))
+ self.assertEqual(got["place_id"], "ChIJabc")
+ self.assertEqual(got["cid"], "255")
+ self.assertEqual(got["price_range"], "$$")
+ self.assertNotIn("plus_code", got)
+ self.assertNotIn("phone", details.parse_entity(entity(i178=None)))
+
+ def test_batch(self):
+ data = [None] * 64 + [[[None, entity()], [None, entity(i11="Other")], ["not an entry"], [None, [None] * 20]]]
+ rows = details.parse_batch(")]}'\n" + json.dumps(data))
+ self.assertEqual([r["title"] for r in rows], ["Example Café", "Other"])
+ self.assertEqual(details.parse_batch(json.dumps(data)), rows) # prefix optional
+ self.assertEqual(details.parse_batch("not json"), [])
+ self.assertEqual(details.parse_batch("[1,2,3]"), [])
+
+ def test_batch_url(self):
+ self.assertTrue(maps.BATCH_URL.match("https://www.google.com/search?tbm=map&authuser=0&hl=en&pb=!4m12"))
+ self.assertTrue(maps.BATCH_URL.match("https://www.google.co.uk/search?q=x&tbm=map"))
+ self.assertFalse(maps.BATCH_URL.match("https://www.google.com/maps/vt/pb=!1m4"))
+ self.assertFalse(maps.BATCH_URL.match("https://www.google.com/search?q=plain"))
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=1)