Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix: wait for the job as long as --max-time allows the engine

nathanpissinatti12-cellcaa7d83863cf
diff --git a/scripts/scrape.py b/scripts/scrape.py
index a669000..ff9d5ba 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -51,6 +51,23 @@ KEY = os.environ.get("SCRAPER_API_KEY", "")
LEAD = ["title", "phone", "emails", "website", "category", "address", "review_rating", "review_count"]
UA = "google-maps-scraper-kit/1.1 (https://github.com/Mahanaicoach/google-maps-scraper-kit)"
+POLL_EVERY = 8 # seconds between status checks
+POLL_GRACE = 120 # the engine still has to flush results and flip the status
+POLL_CEILING = 7200 # never wait longer than 2h, whatever --max-time asked for
+ENGINE_MIN_RUNTIME = 180 # the engine floors anything below this (runner/webrunner)
+
+
+def poll_attempts(max_time):
+ """How many status checks to make before giving up on a job.
+
+ Derived from the job's own max_time, because that is how long the engine is allowed to
+ work: waiting any less abandons a job that is still running normally. The engine floors
+ max_time at ENGINE_MIN_RUNTIME, so even a tiny --max-time needs that much patience, and
+ POLL_CEILING stops an absurd value from parking the script for a day."""
+ engine_runtime = max(ENGINE_MIN_RUNTIME, int(max_time or 0))
+ budget = min(POLL_CEILING, engine_runtime + POLL_GRACE)
+ return max(1, budget // POLL_EVERY)
+
def req(method, path, body=None):
headers = {"Content-Type": "application/json", "User-Agent": UA}
@@ -136,9 +153,10 @@ def run_job(body):
raise RuntimeError("✗ No job id returned.")
print(f" job id: {job_id}")
- print("▶ Polling…")
+ attempts = poll_attempts(body.get("max_time"))
+ print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")
status = None
- for i in range(120):
+ for i in range(attempts):
_, raw = req("GET", f"/api/v1/jobs/{job_id}")
status = json.loads(raw).get("Status")
print(f"\r status: {str(status):<10} (attempt {i + 1})", end="", flush=True)
@@ -146,9 +164,14 @@ def run_job(body):
print(); break
if status == "failed":
raise RuntimeError("\n✗ Job failed. If this keeps happening you may be rate-limited — wait or add proxies.")
- time.sleep(8)
+ time.sleep(POLL_EVERY)
else:
- raise RuntimeError("\n✗ Timed out.")
+ raise RuntimeError(
+ f"\n✗ Gave up waiting after ~{attempts * POLL_EVERY // 60} min. The job may still be "
+ f"running, and the engine keeps the results either way, so nothing is lost:\n"
+ f" curl {BASE}/api/v1/jobs/{job_id}\n"
+ f" and once its Status is \"ok\", download the CSV with\n"
+ f" curl -o results.csv {BASE}/api/v1/jobs/{job_id}/download")
_, raw = req("GET", f"/api/v1/jobs/{job_id}/download")
return job_id, list(csv.DictReader(io.StringIO(raw.decode("utf-8", "replace"))))
@@ -313,7 +336,9 @@ def main():
ap.add_argument("--grid", type=int, metavar="N",
help="split the --city area into N x N map tiles, one job each, deduped "
"(breaks Google's ~120 per search cap; max 10)")
- ap.add_argument("--max-time", type=int, default=600, help="job time limit in SECONDS")
+ ap.add_argument("--max-time", type=int, default=600,
+ help="job time limit in SECONDS; the script waits that long for the "
+ "engine, plus a short grace (default: 600)")
ap.add_argument("--out", default=None, help="output file path (default: results-<id>.csv). "
"The extension picks the format: .csv (default) or .json")
ap.add_argument("--json", action="store_true", help="write JSON instead of the default CSV")
diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py
index 0588f1d..860f352 100644
--- a/scripts/test_scrape.py
+++ b/scripts/test_scrape.py
@@ -68,6 +68,27 @@ class InBbox(unittest.TestCase):
self.assertTrue(scrape.in_bbox({}, BBOX))
+class PollAttempts(unittest.TestCase):
+ def test_follows_max_time(self):
+ # The budget tracks max_time instead of the old fixed 120 attempts (960s), which
+ # gave up on a batch the engine was still working through.
+ self.assertEqual(scrape.poll_attempts(600), (600 + 120) // 8) # 90
+ self.assertEqual(scrape.poll_attempts(3600), (3600 + 120) // 8) # 465, was 120
+ self.assertGreater(scrape.poll_attempts(3600), 120)
+
+ def test_floors_at_engine_minimum(self):
+ # The engine runs a job for at least 180s however small max_time is, so waiting
+ # only max_time would abandon it.
+ for tiny in (1, 60, 179, 180):
+ self.assertEqual(scrape.poll_attempts(tiny), (180 + 120) // 8)
+ self.assertEqual(scrape.poll_attempts(0), (180 + 120) // 8)
+ self.assertEqual(scrape.poll_attempts(None), (180 + 120) // 8)
+
+ def test_ceiling_and_never_zero(self):
+ self.assertEqual(scrape.poll_attempts(10 ** 9), 7200 // 8) # clamped at 2h
+ self.assertGreaterEqual(scrape.poll_attempts(-5), 1)
+
+
class ExtractEmails(unittest.TestCase):
def test_mailto_regex_junk_preference_cap(self):
html = """