Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
fix(website_browser): quiet Playwright shutdown and bound the driver stop
Mahan0e5ae8448292
diff --git a/engine/test_website_browser.py b/engine/test_website_browser.pyindex 00f03da..4d9561a 100644--- a/engine/test_website_browser.py+++ b/engine/test_website_browser.py@@ -93,6 +93,56 @@ class BrowserBudget(unittest.TestCase):self.assertLess(time.monotonic() - started, 5)self.assertEqual(stopped, [True])+ def test_close_during_a_render_returns_budget_exhausted_not_an_exception(self):+ started = threading.Event()+ async def render(session, url):+ session.tasks.add(asyncio.current_task()) # as the real render registers itself+ started.set()+ await asyncio.sleep(60)+ with patch.object(web._Session, "render", render):+ budget = web.Budget(30)+ out = {}+ t = threading.Thread(target=lambda: out.update(budget.fetch("https://fixture-business.test", jobs._public_ip)), daemon=True)+ t.start()+ self.assertTrue(started.wait(5))+ budget.close()+ t.join(5)+ self.assertEqual(out.get("status"), "budget_exhausted")++ def test_close_stops_the_proxy_before_the_browser_and_bounds_the_driver_stop(self):+ order = []+ async def hang():+ await asyncio.sleep(60)+ def browser_close():+ order.append("browser")+ return hang()+ async def render(session, url):+ session.browser = Mock(close=browser_close)+ session.pw = Mock(stop=hang)+ session.proxy = Mock(__exit__=Mock(side_effect=lambda *a: order.append("proxy")))+ return {"html": "", "status": "rendered"}+ with patch.object(web._Session, "render", render), patch.object(web, "_CLOSE_SECONDS", 0.2):+ budget = web.Budget(30)+ budget.fetch("https://fixture-business.test", jobs._public_ip)+ started = time.monotonic()+ budget.close()+ self.assertLess(time.monotonic() - started, 2) # two bounded waits, not a hung driver+ self.assertEqual(order, ["proxy", "browser"])++ def test_target_closed_futures_do_not_print_at_exit_but_other_errors_do(self):+ TargetClosedError = type("TargetClosedError", (Exception,), {})+ loop = asyncio.new_event_loop()+ loop.set_exception_handler(web._quiet_target_closed)+ with patch.object(asyncio.BaseEventLoop, "default_exception_handler") as default:+ loop.call_exception_handler({"message": "Future exception was never retrieved", "exception": TargetClosedError("gone")})+ default.assert_not_called()+ loop.call_exception_handler({"message": "Future exception was never retrieved", "exception": RuntimeError("real")})+ default.assert_called_once()+ loop.close()+ with patch.object(web._Session, "render", return_value={"html": "", "status": "rendered"}), web.Budget(30) as budget:+ budget.fetch("https://fixture-business.test", jobs._public_ip)+ self.assertIs(budget.loop.get_exception_handler(), web._quiet_target_closed)+def test_status_keeps_the_best_fetch(self):self.assertEqual(web.merge_status("http", "budget_exhausted"), "http")self.assertEqual(web.merge_status("http", "rendered"), "rendered")diff --git a/engine/website_browser.py b/engine/website_browser.pyindex 2764d7d..4992650 100644--- a/engine/website_browser.py+++ b/engine/website_browser.py@@ -5,6 +5,7 @@ The browser's HTTP proxy pins public DNS addresses and checks connected peers, iredirects, scripts and iframes. It preserves the existing website fetcher's private-network guard."""import asyncio+import concurrent.futuresimport selectimport socketimport threading@@ -14,10 +15,18 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer_SLOTS = threading.BoundedSemaphore(2)_MIN_START = 3 # starting the driver with milliseconds left wastes time and trips Node timers-_CLOSE_SECONDS = 5 # graceful browser close; it took 1.6-2.3 s live, and hung 24-300 s at times+_CLOSE_SECONDS = 5 # each of browser.close() and pw.stop(); close took 1.6-2.3 s live, and hung 24-300 s at timesSETUP = "Install optional rendering in a virtual environment: see SETUP.md, Optional website rendering (Playwright)."+def _quiet_target_closed(loop, context):+ """Playwright settles the protocol calls of a cancelled or closed page with TargetClosedError once+ the browser is gone. Nothing can await those internal futures, so asyncio would otherwise print+ "Future exception was never retrieved" when the run ends. Everything else still gets reported."""+ if type(context.get("exception")).__name__ != "TargetClosedError":+ loop.default_exception_handler(context)++def merge_status(current, incoming):"""A successful fetch survives later failed enrichment attempts."""if current == "rendered" or (current == "http" and incoming != "rendered"):@@ -45,10 +54,14 @@ class Budget:return {"html": "", "status": "budget_exhausted"}if self.loop is None:self.loop = asyncio.new_event_loop()+ self.loop.set_exception_handler(_quiet_target_closed)self.thread = threading.Thread(target=self.loop.run_forever, daemon=True)self.thread.start()future = asyncio.run_coroutine_threadsafe(self._fetch(url, public_ip), self.loop)- return future.result()+ try:+ return future.result()+ except concurrent.futures.CancelledError: # close() ran while this page was still rendering+ return {"html": "", "status": "budget_exhausted"}async def _fetch(self, url, public_ip):if self.session is None:@@ -301,6 +314,10 @@ class _Session:for task in tasks:task.cancel()await asyncio.gather(*tasks, return_exceptions=True)+ if self.proxy is not None:+ # Nothing is served while the browser closes; its relays would otherwise run on+ # until the deadline.+ self.proxy.__exit__()try:if self.browser is not None:# Chrome's graceful close can hang for minutes after many pages; pw.stop() below@@ -311,10 +328,10 @@ class _Session:finally:try:if self.pw is not None:- await self.pw.stop()+ await asyncio.wait_for(self.pw.stop(), _CLOSE_SECONDS)+ except Exception:+ passfinally:- if self.proxy is not None:- self.proxy.__exit__()if self.has_slot:_SLOTS.release()