Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

Fix website rendering waits and expired browser budgets

Mahan Jafari4e5608762f0a
diff --git a/SETUP.md b/SETUP.md
index 13e35b5..8bbaaaf 100644
--- a/SETUP.md
+++ b/SETUP.md
@@ -235,7 +235,9 @@ On Windows, use `py -m venv .engine/playwright-venv`, activate with
a 30-second browser budget between emails, contact pages and socials. The API also caps retries
at 30 seconds or its remaining website-pass time, whichever is less. Browser cleanup can add a
little overhead. Once exhausted, remaining retries report `budget_exhausted` rather than adding
- more page waits. Browsers use fresh contexts, block image/media/font downloads, and close after use.
+ more page waits. Retries also skip when less than three seconds remain for browser startup.
+ Slow third-party scripts do not discard an already usable page: rendering uses bounded DOM and
+ contact-content waits. Browsers use fresh contexts, block image/media/font downloads, and close after use.
- `website_fetch_status` records `http`, `rendered`, `blocked`, `timeout`, `budget_exhausted` or `error`. A rendered page
may contain no published contact information. Authentication, CAPTCHA and access denials can
remain unavailable; the renderer does not solve them.
diff --git a/engine/serve.py b/engine/serve.py
index f996ed6..1da3f36 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -28,7 +28,7 @@ for _stream in (sys.stdout, sys.stderr): # ✓ and accented names on a Windows
except (AttributeError, ValueError):
pass
-VERSION = "2.0.1-dev.3"
+VERSION = "2.0.1-dev.4"
MARK = f"gmskit-engine/{VERSION}" # X-Engine header: tells scripts/engine.py this engine from others
REPO = "https://github.com/Mahanaicoach/google-maps-scraper-kit"
# The web UI's sponsor box: the browser reads this static file from GitHub (cached a day, no
diff --git a/engine/test_website_browser.py b/engine/test_website_browser.py
index a32bb2d..952621f 100644
--- a/engine/test_website_browser.py
+++ b/engine/test_website_browser.py
@@ -39,6 +39,12 @@ class PublicProxy(unittest.TestCase):
self.assertEqual(web.render(url, jobs._public_ip)['status'], 'blocked')
available.assert_not_called()
+ def test_nearly_expired_budget_never_starts_the_driver(self):
+ with patch.object(web, "check_available") as available:
+ result = web.render("https://fixture-business.test", jobs._public_ip, timeout=0.001)
+ self.assertEqual(result["status"], "budget_exhausted")
+ available.assert_not_called()
+
def test_missing_dependency_is_actionable(self):
with patch.dict(sys.modules, {'playwright': None, 'playwright.sync_api': None}):
with self.assertRaisesRegex(RuntimeError, 'requirements-browser.txt'):
@@ -52,9 +58,9 @@ class BrowserBudget(unittest.TestCase):
budget = web.Budget(30)
budget.fetch("https://fixture-business.test", jobs._public_ip)
self.assertEqual(render.call_args.kwargs["timeout"], 20)
- clock.return_value = 128
+ clock.return_value = 126
budget.fetch("https://fixture-business.test/contact", jobs._public_ip)
- self.assertEqual(render.call_args.kwargs["timeout"], 2)
+ self.assertEqual(render.call_args.kwargs["timeout"], 4)
clock.return_value = 131
self.assertEqual(budget.fetch("https://fixture-business.test/about", jobs._public_ip)["status"], "budget_exhausted")
self.assertEqual(render.call_count, 2)
@@ -111,6 +117,11 @@ class Fixture(BaseHTTPRequestHandler):
pass
def do_GET(self):
+ if self.path == '/slow-script':
+ self.server.release_script.wait(25)
+ self.send_response(200)
+ self.end_headers()
+ return
if self.path == '/blocked':
self.send_error(403)
return
@@ -119,7 +130,11 @@ class Fixture(BaseHTTPRequestHandler):
self.send_header('Location', f'http://127.0.0.2:{self.server.server_port}/')
self.end_headers()
return
- if self.path == '/data':
+ if self.path == '/slow-page':
+ body = b'''<!doctype html><title>Business</title>
+ <a href="mailto:[email protected]">Contact</a>
+ <script src="/slow-script"></script>'''
+ elif self.path == '/data':
body = json.dumps({'mail': ['hello', 'fixture-business.test'], 'profile': 'fixture_business'}).encode()
else:
body = b'''<!doctype html><title>Local business fixture</title><main>Loading</main>
@@ -134,6 +149,21 @@ class Fixture(BaseHTTPRequestHandler):
@unittest.skipUnless(os.environ.get('GMSKIT_BROWSER_LIVE') == '1', 'optional real-browser fixture')
class LiveBrowser(unittest.TestCase):
+ def test_usable_contacts_survive_slow_script(self):
+ with ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as server:
+ server.release_script = threading.Event()
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ result = web.render(f'http://127.0.0.1:{server.server_port}/slow-page',
+ lambda ip: ip == '127.0.0.1', timeout=15)
+ self.assertEqual(result['status'], 'rendered', result.get('stage'))
+ self.assertEqual(jobs.find_emails(result['html']), ['[email protected]'])
+ finally:
+ server.release_script.set()
+ server.shutdown()
+ thread.join(2)
+
def test_javascript_links_and_private_destinations(self):
with ThreadingHTTPServer(('127.0.0.1', 0), Fixture) as server:
thread = threading.Thread(target=server.serve_forever, daemon=True)
diff --git a/engine/website_browser.py b/engine/website_browser.py
index aff14cf..82105d7 100644
--- a/engine/website_browser.py
+++ b/engine/website_browser.py
@@ -12,6 +12,7 @@ import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
_SLOTS = threading.BoundedSemaphore(2)
+_MIN_START = 3 # starting the driver with milliseconds left wastes time and trips Node timers
SETUP = "Install optional rendering with: python3 -m pip install -r requirements-browser.txt"
@@ -25,7 +26,7 @@ class Budget:
if self.deadline is None:
self.deadline = time.monotonic() + self.seconds
left = self.deadline - time.monotonic()
- if left <= 0:
+ if left < _MIN_START:
return {"html": "", "status": "budget_exhausted"}
return render(url, public_ip, timeout=min(20, left))
@@ -136,7 +137,8 @@ class _PublicProxy(ThreadingHTTPServer):
def render(url, public_ip, timeout=20):
"""{html, status}: render public content, with bounded waits and no persistent browser state."""
- failure = lambda status: {"html": "", "status": status}
+ stage = "validate"
+ failure = lambda status: {"html": "", "status": status, "stage": stage}
try:
parsed = urllib.parse.urlsplit(url if "://" in url else "https://" + url)
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
@@ -144,14 +146,24 @@ def render(url, public_ip, timeout=20):
url = parsed.geturl()
except ValueError:
return failure("blocked")
+ if timeout < _MIN_START:
+ return failure("budget_exhausted")
deadline = time.monotonic() + timeout
+ stage = "capacity"
if not _SLOTS.acquire(timeout=max(0, timeout)):
return failure("timeout")
try:
+ if deadline - time.monotonic() < _MIN_START:
+ return failure("budget_exhausted")
+ stage = "setup"
path = check_available()
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeout
def left():
- return max(1, int((deadline - time.monotonic()) * 1000))
+ remaining = deadline - time.monotonic()
+ if remaining < 0.5:
+ raise TimeoutError("browser budget exhausted")
+ return int((remaining - 0.2) * 1000)
+ stage = "browser_start"
with _PublicProxy(public_ip, deadline) as proxy, sync_playwright() as pw:
with pw.chromium.launch(executable_path=path, headless=True, timeout=left(),
chromium_sandbox=True, proxy={"server": proxy},
@@ -168,17 +180,29 @@ def render(url, public_ip, timeout=20):
route.continue_()
context.route("**/*", route_request)
page = context.new_page()
- response = page.goto(url, wait_until="domcontentloaded", timeout=left())
+ stage = "navigation"
+ response = page.goto(url, wait_until="commit", timeout=min(10000, left()))
if response and response.status >= 400:
return failure("blocked" if response.status in (401, 403, 429) else "error")
+ stage = "rendering"
try:
- page.wait_for_load_state("networkidle", timeout=min(3000, left()))
+ page.wait_for_load_state("domcontentloaded", timeout=min(3000, left()))
except PlaywrightTimeout:
- pass # analytics may never settle; keep the rendered DOM
+ pass # slow third-party scripts must not discard an already usable page
+ try:
+ page.wait_for_function(r"""() => document.body && (
+ document.querySelector(`a[href^="mailto:"], a[href*="instagram.com/"],
+ a[href*="facebook.com/"], a[href*="linkedin.com/"], a[href*="tiktok.com/"],
+ a[href*="youtube.com/"], a[href*="twitter.com/"], a[href*="x.com/"]`) ||
+ /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i.test(document.body.innerText))""",
+ timeout=min(2000, left()))
+ except PlaywrightTimeout:
+ pass # not every business publishes contact details
+ stage = "read_dom"
title = page.title().lower()
if any(t in title for t in ("just a moment", "access denied", "verify you are human", "captcha")):
return failure("blocked")
- return {"html": page.content()[:400_000], "status": "rendered"}
+ return {"html": page.content()[:400_000], "status": "rendered", "stage": "complete"}
finally:
context.close()
except Exception as e: