Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(website_browser): quiet Playwright shutdown and bound the driver stop

Mahan0e5ae8448292
diff --git a/engine/test_website_browser.py b/engine/test_website_browser.py
index 00f03da..4d9561a 100644
--- a/engine/test_website_browser.py
+++ b/engine/test_website_browser.py
@@ -93,6 +93,56 @@ class BrowserBudget(unittest.TestCase):
self.assertLess(time.monotonic() - started, 5)
self.assertEqual(stopped, [True])
+ def test_close_during_a_render_returns_budget_exhausted_not_an_exception(self):
+ started = threading.Event()
+ async def render(session, url):
+ session.tasks.add(asyncio.current_task()) # as the real render registers itself
+ started.set()
+ await asyncio.sleep(60)
+ with patch.object(web._Session, "render", render):
+ budget = web.Budget(30)
+ out = {}
+ t = threading.Thread(target=lambda: out.update(budget.fetch("https://fixture-business.test", jobs._public_ip)), daemon=True)
+ t.start()
+ self.assertTrue(started.wait(5))
+ budget.close()
+ t.join(5)
+ self.assertEqual(out.get("status"), "budget_exhausted")
+
+ def test_close_stops_the_proxy_before_the_browser_and_bounds_the_driver_stop(self):
+ order = []
+ async def hang():
+ await asyncio.sleep(60)
+ def browser_close():
+ order.append("browser")
+ return hang()
+ async def render(session, url):
+ session.browser = Mock(close=browser_close)
+ session.pw = Mock(stop=hang)
+ session.proxy = Mock(__exit__=Mock(side_effect=lambda *a: order.append("proxy")))
+ return {"html": "", "status": "rendered"}
+ with patch.object(web._Session, "render", render), patch.object(web, "_CLOSE_SECONDS", 0.2):
+ budget = web.Budget(30)
+ budget.fetch("https://fixture-business.test", jobs._public_ip)
+ started = time.monotonic()
+ budget.close()
+ self.assertLess(time.monotonic() - started, 2) # two bounded waits, not a hung driver
+ self.assertEqual(order, ["proxy", "browser"])
+
+ def test_target_closed_futures_do_not_print_at_exit_but_other_errors_do(self):
+ TargetClosedError = type("TargetClosedError", (Exception,), {})
+ loop = asyncio.new_event_loop()
+ loop.set_exception_handler(web._quiet_target_closed)
+ with patch.object(asyncio.BaseEventLoop, "default_exception_handler") as default:
+ loop.call_exception_handler({"message": "Future exception was never retrieved", "exception": TargetClosedError("gone")})
+ default.assert_not_called()
+ loop.call_exception_handler({"message": "Future exception was never retrieved", "exception": RuntimeError("real")})
+ default.assert_called_once()
+ loop.close()
+ with patch.object(web._Session, "render", return_value={"html": "", "status": "rendered"}), web.Budget(30) as budget:
+ budget.fetch("https://fixture-business.test", jobs._public_ip)
+ self.assertIs(budget.loop.get_exception_handler(), web._quiet_target_closed)
+
def test_status_keeps_the_best_fetch(self):
self.assertEqual(web.merge_status("http", "budget_exhausted"), "http")
self.assertEqual(web.merge_status("http", "rendered"), "rendered")
diff --git a/engine/website_browser.py b/engine/website_browser.py
index 2764d7d..4992650 100644
--- a/engine/website_browser.py
+++ b/engine/website_browser.py
@@ -5,6 +5,7 @@ The browser's HTTP proxy pins public DNS addresses and checks connected peers, i
redirects, scripts and iframes. It preserves the existing website fetcher's private-network guard.
"""
import asyncio
+import concurrent.futures
import select
import socket
import threading
@@ -14,10 +15,18 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
_SLOTS = threading.BoundedSemaphore(2)
_MIN_START = 3 # starting the driver with milliseconds left wastes time and trips Node timers
-_CLOSE_SECONDS = 5 # graceful browser close; it took 1.6-2.3 s live, and hung 24-300 s at times
+_CLOSE_SECONDS = 5 # each of browser.close() and pw.stop(); close took 1.6-2.3 s live, and hung 24-300 s at times
SETUP = "Install optional rendering in a virtual environment: see SETUP.md, Optional website rendering (Playwright)."
+def _quiet_target_closed(loop, context):
+ """Playwright settles the protocol calls of a cancelled or closed page with TargetClosedError once
+ the browser is gone. Nothing can await those internal futures, so asyncio would otherwise print
+ "Future exception was never retrieved" when the run ends. Everything else still gets reported."""
+ if type(context.get("exception")).__name__ != "TargetClosedError":
+ loop.default_exception_handler(context)
+
+
def merge_status(current, incoming):
"""A successful fetch survives later failed enrichment attempts."""
if current == "rendered" or (current == "http" and incoming != "rendered"):
@@ -45,10 +54,14 @@ class Budget:
return {"html": "", "status": "budget_exhausted"}
if self.loop is None:
self.loop = asyncio.new_event_loop()
+ self.loop.set_exception_handler(_quiet_target_closed)
self.thread = threading.Thread(target=self.loop.run_forever, daemon=True)
self.thread.start()
future = asyncio.run_coroutine_threadsafe(self._fetch(url, public_ip), self.loop)
- return future.result()
+ try:
+ return future.result()
+ except concurrent.futures.CancelledError: # close() ran while this page was still rendering
+ return {"html": "", "status": "budget_exhausted"}
async def _fetch(self, url, public_ip):
if self.session is None:
@@ -301,6 +314,10 @@ class _Session:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
+ if self.proxy is not None:
+ # Nothing is served while the browser closes; its relays would otherwise run on
+ # until the deadline.
+ self.proxy.__exit__()
try:
if self.browser is not None:
# Chrome's graceful close can hang for minutes after many pages; pw.stop() below
@@ -311,10 +328,10 @@ class _Session:
finally:
try:
if self.pw is not None:
- await self.pw.stop()
+ await asyncio.wait_for(self.pw.stop(), _CLOSE_SECONDS)
+ except Exception:
+ pass
finally:
- if self.proxy is not None:
- self.proxy.__exit__()
if self.has_slot:
_SLOTS.release()