Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(engine): window mode with only the headless shell says so; live CI skips it on ubuntu-arm

Mahan Jafaride89b304bab9
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.yml
index 51f5d91..fb47aef 100644
--- a/.github/workflows/engine-live.yml
+++ b/.github/workflows/engine-live.yml
@@ -76,6 +76,8 @@ jobs:
sys.exit(0 if rows and counts["tunnels"] and counts["refused"] else 1)
PY
- name: window mode (SCRAPER_HEADLESS=0) with the reviews pass
+ # ubuntu-24.04-arm has no Chrome, only the downloaded headless shell, which cannot open a window
+ if: matrix.os != 'ubuntu-24.04-arm'
env:
SCRAPER_HEADLESS: "0"
run: |
diff --git a/engine/browser.py b/engine/browser.py
index fc85c07..3694e97 100644
--- a/engine/browser.py
+++ b/engine/browser.py
@@ -96,6 +96,11 @@ def find_chrome():
return shell_path() if os.path.isfile(shell_path()) else None
+def always_headless(path):
+ """True for Chrome for Testing's headless shell, which cannot open a window (SCRAPER_HEADLESS=0)."""
+ return "headless-shell" in os.path.basename(path or "")
+
+
def is_snap(path):
"""True for a snap browser: /snap/bin/chromium, or Ubuntu's /usr/bin/chromium-browser script
that runs it."""
@@ -262,7 +267,7 @@ class Browser:
"--metrics-recording-only", "--no-pings", "--disable-client-side-phishing-detection",
"--disable-blink-features=AutomationControlled", "--window-size=1280,900",
f"--lang={lang}", "--disable-dev-shm-usage"]
- if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headless
+ if headless and not always_headless(self.path): # the shell is always headless
args.append("--headless=new")
elif not headless:
args.append("--window-position=-3000,-3000") # a real window, kept out of sight
diff --git a/engine/jobs.py b/engine/jobs.py
index b1f4ee8..bdd5e00 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -537,9 +537,9 @@ class Worker(threading.Thread):
n = self.add_reviews(rows, d, deadline, stop, proxies, save)
self.store.rewrite(jid, rows, cols) # keep the reviews if the engine stops during the next pass
self.log(f"job {jid}: reviews for {n}/{sum(1 for r in rows if r.get('place_id'))} places")
- if not n and self.headless:
+ if not n and (self.headless or browser.always_headless(self._browser.path)):
notes.append("no reviews: Google's limited view, which a headless Chrome gets, shows none; "
- "set SCRAPER_HEADLESS=0 to run Chrome as a window")
+ "set SCRAPER_HEADLESS=0, with an installed Chrome, to run it as a window")
if stage == "reviews" and not stop.is_set():
stage = "websites"
save(stage=stage)
diff --git a/engine/serve.py b/engine/serve.py
index 2e190ea..9eb02e8 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -537,6 +537,9 @@ def main():
f"{a.host}:{port} in this container, published on the host's 127.0.0.1"
log(f"engine {VERSION} on {where} (browser: {chrome or 'none found'}{'' if headless else ', as a window'}"
f"{', API key required' if env['SCRAPER_API_KEY'] else ''})")
+ if not headless and browser.always_headless(chrome):
+ log("SCRAPER_HEADLESS=0 is set, but the only browser is the downloaded headless shell, which cannot "
+ "open a window: install Google Chrome (or Chromium, Edge, Brave) for window mode")
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
for s in servers[1:]:
threading.Thread(target=s.serve_forever, daemon=True).start()
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 8133690..8624825 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -628,6 +628,8 @@ class LaunchChrome(unittest.TestCase):
b = browser.Browser(path=self.exe, profile=profile, headless=False, timeout=5)
self.assertIsNone(b)
self.assertIn("--window-position=-3000,-3000", self.launches()[0]) # a window, out of sight
+ self.assertTrue(browser.always_headless(browser.shell_path("linux64")))
+ self.assertFalse(browser.always_headless("/usr/bin/google-chrome"))
self.assertNotIn("--headless", self.launches()[0])
out = subprocess.run(["ps", "-axo", "command"], capture_output=True, text=True).stdout
self.assertNotIn(self.exe, out) # the fake Chrome was stopped