Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
Fix website rendering waits and expired browser budgets
Mahan Jafari4e5608762f0a
diff --git a/SETUP.md b/SETUP.mdindex 13e35b5..8bbaaaf 100644--- a/SETUP.md+++ b/SETUP.md@@ -235,7 +235,9 @@ On Windows, use `py -m venv .engine/playwright-venv`, activate witha 30-second browser budget between emails, contact pages and socials. The API also caps retriesat 30 seconds or its remaining website-pass time, whichever is less. Browser cleanup can add alittle overhead. Once exhausted, remaining retries report `budget_exhausted` rather than adding- more page waits. Browsers use fresh contexts, block image/media/font downloads, and close after use.+ more page waits. Retries also skip when less than three seconds remain for browser startup.+ Slow third-party scripts do not discard an already usable page: rendering uses bounded DOM and+ contact-content waits. Browsers use fresh contexts, block image/media/font downloads, and close after use.- `website_fetch_status` records `http`, `rendered`, `blocked`, `timeout`, `budget_exhausted` or `error`. A rendered pagemay contain no published contact information. Authentication, CAPTCHA and access denials canremain unavailable; the renderer does not solve them.diff --git a/engine/serve.py b/engine/serve.pyindex f996ed6..1da3f36 100644--- a/engine/serve.py+++ b/engine/serve.py@@ -28,7 +28,7 @@ for _stream in (sys.stdout, sys.stderr): # ✓ and accented names on a Windowsexcept (AttributeError, ValueError):pass-VERSION = "2.0.1-dev.3"+VERSION = "2.0.1-dev.4"MARK = f"gmskit-engine/{VERSION}" # X-Engine header: tells scripts/engine.py this engine from othersREPO = "https://github.com/Mahanaicoach/google-maps-scraper-kit"# The web UI's sponsor box: the browser reads this static file from GitHub (cached a day, nodiff --git a/engine/test_website_browser.py b/engine/test_website_browser.pyindex a32bb2d..952621f 100644--- a/engine/test_website_browser.py+++ b/engine/test_website_browser.py@@ -39,6 +39,12 @@ class PublicProxy(unittest.TestCase):self.assertEqual(web.render(url, jobs._public_ip)['status'], 'blocked')available.assert_not_called()+ def test_nearly_expired_budget_never_starts_the_driver(self):+ with patch.object(web, "check_available") as available:+ result = web.render("https://fixture-business.test", jobs._public_ip, timeout=0.001)+ self.assertEqual(result["status"], "budget_exhausted")+ available.assert_not_called()+def test_missing_dependency_is_actionable(self):with patch.dict(sys.modules, {'playwright': None, 'playwright.sync_api': None}):with self.assertRaisesRegex(RuntimeError, 'requirements-browser.txt'):@@ -52,9 +58,9 @@ class BrowserBudget(unittest.TestCase):budget = web.Budget(30)budget.fetch("https://fixture-business.test", jobs._public_ip)self.assertEqual(render.call_args.kwargs["timeout"], 20)- clock.return_value = 128+ clock.return_value = 126budget.fetch("https://fixture-business.test/contact", jobs._public_ip)- self.assertEqual(render.call_args.kwargs["timeout"], 2)+ self.assertEqual(render.call_args.kwargs["timeout"], 4)clock.return_value = 131self.assertEqual(budget.fetch("https://fixture-business.test/about", jobs._public_ip)["status"], "budget_exhausted")self.assertEqual(render.call_count, 2)@@ -111,6 +117,11 @@ class Fixture(BaseHTTPRequestHandler):passdef do_GET(self):+ if self.path == '/slow-script':+ self.server.release_script.wait(25)+ self.send_response(200)+ self.end_headers()+ returnif self.path == '/blocked':self.send_error(403)return@@ -119,7 +130,11 @@ class Fixture(BaseHTTPRequestHandler):self.send_header('Location', f'http://127.0.0.2:{self.server.server_port}/')self.end_headers()return- if self.path == '/data':+ if self.path == '/slow-page':+ body = b'''<!doctype html><title>Business</title>+ <a href="mailto:[email protected]">Contact</a>+ <script src="/slow-script"></script>'''+ elif self.path == '/data':body = json.dumps({'mail': ['hello', 'fixture-business.test'], 'profile': 'fixture_business'}).encode()else:body = b'''<!doctype html><title>Local business fixture</title><main>Loading</main>@@ -134,6 +149,21 @@ class Fixture(BaseHTTPRequestHandler):@unittest.skipUnless(os.environ.get('GMSKIT_BROWSER_LIVE') == '1', 'optional real-browser fixture')class LiveBrowser(unittest.TestCase):+ def test_usable_contacts_survive_slow_script(self):+ with ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as server:+ server.release_script = threading.Event()+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ result = web.render(f'http://127.0.0.1:{server.server_port}/slow-page',+ lambda ip: ip == '127.0.0.1', timeout=15)+ self.assertEqual(result['status'], 'rendered', result.get('stage'))+ self.assertEqual(jobs.find_emails(result['html']), ['[email protected]'])+ finally:+ server.release_script.set()+ server.shutdown()+ thread.join(2)+def test_javascript_links_and_private_destinations(self):with ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as server:thread = threading.Thread(target=server.serve_forever, daemon=True)diff --git a/engine/website_browser.py b/engine/website_browser.pyindex aff14cf..82105d7 100644--- a/engine/website_browser.py+++ b/engine/website_browser.py@@ -12,6 +12,7 @@ import urllib.parsefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer_SLOTS = threading.BoundedSemaphore(2)+_MIN_START = 3 # starting the driver with milliseconds left wastes time and trips Node timersSETUP = "Install optional rendering with: python3 -m pip install -r requirements-browser.txt"@@ -25,7 +26,7 @@ class Budget:if self.deadline is None:self.deadline = time.monotonic() + self.secondsleft = self.deadline - time.monotonic()- if left <= 0:+ if left < _MIN_START:return {"html": "", "status": "budget_exhausted"}return render(url, public_ip, timeout=min(20, left))@@ -136,7 +137,8 @@ class _PublicProxy(ThreadingHTTPServer):def render(url, public_ip, timeout=20):"""{html, status}: render public content, with bounded waits and no persistent browser state."""- failure = lambda status: {"html": "", "status": status}+ stage = "validate"+ failure = lambda status: {"html": "", "status": status, "stage": stage}try:parsed = urllib.parse.urlsplit(url if "://" in url else "https://" + url)if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:@@ -144,14 +146,24 @@ def render(url, public_ip, timeout=20):url = parsed.geturl()except ValueError:return failure("blocked")+ if timeout < _MIN_START:+ return failure("budget_exhausted")deadline = time.monotonic() + timeout+ stage = "capacity"if not _SLOTS.acquire(timeout=max(0, timeout)):return failure("timeout")try:+ if deadline - time.monotonic() < _MIN_START:+ return failure("budget_exhausted")+ stage = "setup"path = check_available()from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutdef left():- return max(1, int((deadline - time.monotonic()) * 1000))+ remaining = deadline - time.monotonic()+ if remaining < 0.5:+ raise TimeoutError("browser budget exhausted")+ return int((remaining - 0.2) * 1000)+ stage = "browser_start"with _PublicProxy(public_ip, deadline) as proxy, sync_playwright() as pw:with pw.chromium.launch(executable_path=path, headless=True, timeout=left(),chromium_sandbox=True, proxy={"server": proxy},@@ -168,17 +180,29 @@ def render(url, public_ip, timeout=20):route.continue_()context.route("**/*", route_request)page = context.new_page()- response = page.goto(url, wait_until="domcontentloaded", timeout=left())+ stage = "navigation"+ response = page.goto(url, wait_until="commit", timeout=min(10000, left()))if response and response.status >= 400:return failure("blocked" if response.status in (401, 403, 429) else "error")+ stage = "rendering"try:- page.wait_for_load_state("networkidle", timeout=min(3000, left()))+ page.wait_for_load_state("domcontentloaded", timeout=min(3000, left()))except PlaywrightTimeout:- pass # analytics may never settle; keep the rendered DOM+ pass # slow third-party scripts must not discard an already usable page+ try:+ page.wait_for_function(r"""() => document.body && (+ document.querySelector(`a[href^="mailto:"], a[href*="instagram.com/"],+ a[href*="facebook.com/"], a[href*="linkedin.com/"], a[href*="tiktok.com/"],+ a[href*="youtube.com/"], a[href*="twitter.com/"], a[href*="x.com/"]`) ||+ /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i.test(document.body.innerText))""",+ timeout=min(2000, left()))+ except PlaywrightTimeout:+ pass # not every business publishes contact details+ stage = "read_dom"title = page.title().lower()if any(t in title for t in ("just a moment", "access denied", "verify you are human", "captcha")):return failure("blocked")- return {"html": page.content()[:400_000], "status": "rendered"}+ return {"html": page.content()[:400_000], "status": "rendered", "stage": "complete"}finally:context.close()except Exception as e: