Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

fix(engine): P3 audit loop 1: blocks before the first place, resumes after the search, refused logins

Mahan Jafaric55406455097
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.yml
index 197f26f..51f5d91 100644
--- a/.github/workflows/engine-live.yml
+++ b/.github/workflows/engine-live.yml
@@ -22,7 +22,7 @@ jobs:
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
- timeout-minutes: 15
+ timeout-minutes: 25 # four live steps: scrape, two proxy checks, window mode with reviews
defaults:
run:
shell: bash
diff --git a/engine/jobs.py b/engine/jobs.py
index 6feb43c..b1f4ee8 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -311,16 +311,24 @@ class Store:
except OSError:
pass
- def finish(self, jid, rows, cols, status, error, progress):
- """Replace the job's CSV in one step and record how the job ended, under one lock; False
+ def rewrite(self, jid, rows, cols):
+ """Replace the job's CSV in one step (after a pass that changed rows); its new size, or None
(nothing written) if the job was deleted."""
with self.lock:
if not self._exists(jid):
- return False
+ return None
tmp = self.csv_path(jid) + ".tmp"
with open(tmp, "wb") as f:
f.write(to_csv(rows, cols))
os.replace(tmp, self.csv_path(jid))
+ return os.path.getsize(self.csv_path(jid))
+
+ def finish(self, jid, rows, cols, status, error, progress):
+ """Replace the job's CSV in one step and record how the job ended, under one lock; False
+ (nothing written) if the job was deleted."""
+ with self.lock:
+ if self.rewrite(jid, rows, cols) is None:
+ return False
with self.db:
self.db.execute("UPDATE jobs SET status = ?, error = ?, progress = ? WHERE id = ?",
(status, error[:500], json.dumps(progress), jid))
@@ -414,6 +422,8 @@ class Worker(threading.Thread):
deadline = started + budget
stop = threading.Event()
self._running = (jid, stop)
+ if self.stopped.is_set(): # shutdown() came between claim() and now
+ stop.set()
if not self.store.get(jid): # deleted between claim() and now, before cancel() could see it
self._running = (None, None)
return
@@ -421,9 +431,11 @@ class Worker(threading.Thread):
reviews = d.get("reviews_max") and stage in ("search", "reviews")
if stage == "search":
self.store.cut(jid, p.get("bytes", 0)) # the rows of a keyword cut short are searched again
- rows = self.store.read_rows(jid) if p.get("bytes") else []
+ rows = self.store.read_rows(jid) # [] when the cut removed the file
seen = {place_key(r) for r in rows}
- errors, notes, blocked, retries, rot = [], [], False, 0, 0
+ # What an earlier run of this job saw, for its final note (a job resumed after its search).
+ errors = [p["last_error"]] * p.get("failed", 0) if p.get("last_error") else []
+ notes, blocked, retries, rot, refused = [], p.get("blocked", False), p.get("retried", 0), 0, set()
cur = {"i": done, "kw": ""}
if done or p.get("resumes"):
self.log(f"job {jid} resumed at keyword {done + 1}/{len(kws)} with {len(rows)} places, {budget:.0f} s left")
@@ -457,10 +469,10 @@ class Worker(threading.Thread):
timer.start()
try:
save(stage=stage, done=done, total=len(kws))
- if (done < len(kws) or reviews) and budget > 0:
+ if ((stage == "search" and done < len(kws)) or reviews) and budget > 0 and not stop.is_set():
self._browser = browser.Browser(lang=locale(d), headless=self.headless)
i = done
- while i < len(kws) and self._browser:
+ while stage == "search" and i < len(kws) and self._browser:
left = deadline - time.monotonic()
if stop.is_set() or left <= 0:
break
@@ -471,17 +483,21 @@ class Worker(threading.Thread):
try:
places, reason, stats = self._collect(proxy, url, d["depth"], left, stop, on_batch)
except (CDPError, OSError) as e:
+ if str(e) == LOGIN_REFUSED:
+ refused.add(proxy)
if (PROXY_FAILED.search(str(e)) and len(proxies) > 1 and retries < len(BACKOFF)
- and not self._browser.cdp.closed and not stop.is_set()):
+ and len(refused) < len(set(proxies)) and not self._browser.cdp.closed
+ and not stop.is_set()):
retries, rot = retries + 1, rot + 1
self.log(f"job {jid} keyword {i + 1}/{len(kws)}: the proxy failed; retry {retries}/"
f"{len(BACKOFF)} through the next proxy")
continue
errors.append(str(e))
self.log(f"job {jid} keyword {i + 1}/{len(kws)}: error: {e}")
- if self._browser.cdp.closed: # Chrome is gone: the other keywords would fail too
+ # Chrome is gone, or no proxy takes the job's login: the other keywords would fail too
+ if self._browser.cdp.closed or (refused and len(refused) >= len(set(proxies))):
break
- i += 1
+ i, retries = i + 1, 0
continue
on_batch(places, stats["batches"]) # places no batch carried (one place, the DOM fallback)
if stop.is_set(): # deleted, or the engine is exiting: this keyword did not finish
@@ -494,6 +510,7 @@ class Worker(threading.Thread):
self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; "
f"retry {retries}/{len(BACKOFF)} in {wait} s"
+ (" through the next proxy" if len(proxies) > 1 else ""))
+ save()
if stop.wait(wait):
break
continue
@@ -512,13 +529,20 @@ class Worker(threading.Thread):
if self._browser.cdp.closed:
break
i += 1
- if reviews and rows and self._browser and not (stop.is_set() or blocked):
- save(stage="reviews")
- n = self.add_reviews(rows, d, deadline, stop, proxies)
+ if stage == "search" and not stop.is_set(): # the search is over, for good: never search again
+ stage = "reviews" if reviews else "websites"
+ save(stage=stage, bytes=self.store.append(jid, [], cols), blocked=blocked, retried=retries,
+ failed=len(errors), last_error=errors[-1] if errors else "")
+ if stage == "reviews" and rows and self._browser and not (stop.is_set() or blocked):
+ n = self.add_reviews(rows, d, deadline, stop, proxies, save)
+ self.store.rewrite(jid, rows, cols) # keep the reviews if the engine stops during the next pass
self.log(f"job {jid}: reviews for {n}/{sum(1 for r in rows if r.get('place_id'))} places")
if not n and self.headless:
notes.append("no reviews: Google's limited view, which a headless Chrome gets, shows none; "
"set SCRAPER_HEADLESS=0 to run Chrome as a window")
+ if stage == "reviews" and not stop.is_set():
+ stage = "websites"
+ save(stage=stage)
except browser.BrowserError as e:
errors.append(str(e))
self.log(f"job {jid}: {e}")
@@ -527,16 +551,18 @@ class Worker(threading.Thread):
if self._browser:
self._browser.close()
self._browser = None
- if (d["email"] or d.get("socials")) and rows and not stop.is_set():
- save(stage="websites")
+ if stage == "websites" and (d["email"] or d.get("socials")) and rows and not stop.is_set():
n = add_site_data(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), email=d["email"],
socials=d.get("socials"), verify=d.get("verify_emails"), stop=stop.is_set)
self.log(f"job {jid}: website data for {n}/{sum(1 for r in rows if r['website'])} websites")
self._running = (None, None)
if self.stopped.is_set() and self.store.get(jid): # the engine is exiting: resume on the next start
+ if stage != "search": # rows a pass changed; while searching the CSV is the record already
+ self.store.rewrite(jid, rows, cols)
save()
self.store.update(jid, "pending")
- self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)}: resumes when the engine starts again")
+ self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)} ({stage}): resumes when the "
+ "engine starts again")
return
block = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
+ (f"Waited and retried {retries} time(s). " if retries else "") + "Wait, or add proxies")
@@ -585,7 +611,7 @@ class Worker(threading.Thread):
if ctx and not self._browser.cdp.closed:
self._browser.dispose(ctx)
- def add_reviews(self, rows, d, deadline, stop, proxies):
+ def add_reviews(self, rows, d, deadline, stop, proxies, save=lambda: None):
"""Review count, stars histogram and up to reviews_max reviews per place, from each place's
panel (the result batches carry no review texts): one page load per place, until the
deadline. The number of places that got reviews."""
@@ -613,6 +639,7 @@ class Worker(threading.Thread):
if k in ("user_reviews", "reviews_per_rating") or not r.get(k):
r[k] = str(v)
n += "user_reviews" in got
+ save() # elapsed time, for a resume after a crash
except (CDPError, OSError):
pass # Chrome went away: the reviews so far are kept
finally:
diff --git a/engine/maps.py b/engine/maps.py
index 9d29bdc..38f6a3e 100644
--- a/engine/maps.py
+++ b/engine/maps.py
@@ -170,7 +170,7 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False,
try: # a Chrome error mid-scroll keeps the places already read
end = time.time() + 10 # the first batch arrives with the page
- while time.time() < end and batches.count == 0:
+ while time.time() < end and batches.count == 0 and not batches.limited:
take()
time.sleep(0.4)
log(f" batch {batches.count}: {len(seen)} places")
@@ -213,7 +213,7 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False,
if not seen:
raise
return list(seen.values()), f"error: {e}", {"batches": batches.count, "dom_links": 0}
- if not seen: # the batch format changed under us: fall back to what the page shows
+ if not seen and reason != "blocked": # the batch format changed under us: fall back to the page
for a in links:
p = parse_link(a["href"], a["label"])
seen.setdefault(p["data_id"] or p["link"], p)
diff --git a/engine/serve.py b/engine/serve.py
index 252e876..2e190ea 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -518,18 +518,19 @@ def main():
browser.make_home()
browser.sweep_profiles()
store = jobs.Store(a.data)
- resumed, failed = store.recover()
- if resumed:
- log(f"{resumed} job(s) were still working when the engine last stopped: they resume from their last "
- "finished keyword")
- if failed:
- log(f"{failed} job(s) stopped mid-run {jobs.MAX_RESUMES + 1} times: marked failed, rows kept")
headless = env["SCRAPER_HEADLESS"].strip().lower() not in ("0", "false", "no", "off")
worker = jobs.Worker(store, log, headless=headless)
try:
servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"], a.host)
except OSError as e:
sys.exit(f"✗ Cannot listen on {a.host}:{port}: {e}")
+ # Only now: an engine that cannot bind (another one runs on this data) must not touch its jobs.
+ resumed, failed = store.recover()
+ if resumed:
+ log(f"{resumed} job(s) were still working when the engine last stopped: they resume from their last "
+ "finished keyword")
+ if failed:
+ log(f"{failed} job(s) stopped mid-run {jobs.MAX_RESUMES + 1} times: marked failed, rows kept")
worker.start()
chrome = browser.find_chrome()
where = f"http://localhost:{port}" if a.host == "127.0.0.1" else \
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 1f3f32a..8133690 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -1035,5 +1035,118 @@ class ApiKeyDocs(Server):
self.assertEqual(self.call("GET", "/api/v1/jobs/00000000-0000-0000-0000-000000000000/download")[0], 401)
+
+class LoopOne(Fakes):
+ """Audit loop 1: blocks with no places yet, resumes after the search, refused logins, races."""
+
+ def setUp(self):
+ super().setUp()
+ self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)
+ jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01
+
+ def tearDown(self):
+ jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits
+ super().tearDown()
+
+ def test_real_collect_calls_a_429_blocked(self):
+ q = queue.Queue()
+ class C:
+ closed = None
+ def subscribe(self, *a):
+ return q
+ def unsubscribe(self, *a):
+ pass
+ class P:
+ sid, cdp = "s", C()
+ def set_cookie(self, *a):
+ pass
+ def navigate(self, url):
+ q.put({"requestId": "1", "response": {"url": "https://www.google.com/search?tbm=map&pb=x",
+ "status": 429, "mimeType": "text/html"}})
+ def url(self):
+ return "https://www.google.com/maps/search/x"
+ def wait_for(self, expr, timeout=15):
+ return True
+ def eval(self, expr):
+ return {"cls": False, "text": ""} if expr == maps.JS_END else []
+ def send(self, *a, **k):
+ raise CDPError("no body")
+ t = time.time()
+ real_collect = self.saved[2] # Fakes swapped maps.collect for the fake one
+ places, reason, _ = real_collect(P(), "https://www.google.com/maps/search/x", depth=3, log=lambda m: None)
+ self.assertEqual((places, reason), ([], "blocked")) # not "blocked+dom-fallback"
+ self.assertLess(time.time() - t, 5) # no 10 s wait for a first batch that Google refused
+
+ def test_resume_after_the_reviews_keeps_them(self):
+ RESULTS["cafes in Austin"] = ([place(1, website="https://a.com"), place(2)], "end-of-list")
+ def reviews(rows, d, deadline, stop, proxies, save=None):
+ for r in rows:
+ r["user_reviews"] = '[{"Rating": 5}]'
+ return len(rows)
+ self.worker.add_reviews = reviews
+ stopping = lambda *a, **k: self.worker.shutdown() or 0 # the engine stops during the website pass
+ with unittest.mock.patch.object(jobs, "add_site_data", side_effect=stopping):
+ jid, job = self.run_one(reviews_max=1, email=True)
+ self.assertEqual((job["Status"], job["Progress"]["stage"]), ("pending", "websites"))
+ w2 = jobs.Worker(self.store, self.lines.append)
+ w2.add_reviews = lambda *a, **k: self.fail("the reviews were done already")
+ with unittest.mock.patch.object(jobs, "fetch", return_value="[email protected]"):
+ w2.run_job(self.store.claim())
+ rows = self.rows(jid)
+ self.assertEqual(SEARCHED, ["cafes in Austin"]) # searched once, across the restart
+ self.assertEqual([r["user_reviews"] for r in rows], ['[{"Rating": 5}]'] * 2)
+ self.assertEqual((rows[0]["emails"], self.store.get(jid)["Status"]), ("[email protected]", "ok"))
+
+ def test_blocked_job_resumed_in_its_website_pass_keeps_rows_and_its_note(self):
+ RESULTS["cafes in Austin"] = ([place(1, website="https://a.com")], ["slow-blocked"])
+ def collect(page, url, on_batch=None, **kw):
+ SEARCHED.append("cafes in Austin")
+ on_batch([place(1, website="https://a.com")], 1) # streamed, then Google refuses the rest
+ return [], "blocked", {"batches": 1, "dom_links": 0}
+ maps.collect = collect
+ stopping = lambda *a, **k: self.worker.shutdown() or 0
+ with unittest.mock.patch.object(jobs, "add_site_data", side_effect=stopping):
+ jid, job = self.run_one(email=True)
+ self.assertEqual(job["Status"], "pending")
+ SEARCHED.clear()
+ with unittest.mock.patch.object(jobs, "fetch", return_value=""):
+ jobs.Worker(self.store, self.lines.append).run_job(self.store.claim())
+ job = self.store.get(jid)
+ self.assertEqual(SEARCHED, []) # never back to the block
+ self.assertEqual((job["Status"], [r["title"] for r in self.rows(jid)]), ("ok", ["Biz 1"]))
+ self.assertIn("unusual-traffic page", job["Error"])
+ self.assertIn("Waited and retried 2 time(s)", job["Error"])
+
+ def test_a_refused_login_stops_the_job(self):
+ def collect(page, url, **kw):
+ SEARCHED.append(url)
+ raise CDPError(jobs.LOGIN_REFUSED)
+ maps.collect = collect
+ _, job = self.run_one(keywords=["a", "b", "c"], proxies=["http://u:bad@h:1"])
+ self.assertEqual((job["Status"], job["Error"], len(SEARCHED)), ("failed", jobs.LOGIN_REFUSED, 1))
+
+ def test_shutdown_before_the_job_starts(self):
+ name, data = jobs.validate(GOOD)
+ jid = self.store.create(name, data)
+ job = self.store.claim()
+ self.worker.stopped.set()
+ self.worker.run_job(job)
+ self.assertEqual((self.store.get(jid)["Status"], FakeBrowser.made), ("pending", []))
+
+ def test_an_engine_that_cannot_bind_leaves_the_jobs_alone(self):
+ name, data = jobs.validate(GOOD)
+ jid = self.store.create(name, data)
+ self.store.claim() # another engine on this data folder runs it
+ s = socket.socket()
+ s.bind(("127.0.0.1", 0))
+ s.listen(8)
+ argv = ["serve.py", "--port", str(s.getsockname()[1]), "--data", os.path.join(self.dir, "data")]
+ try:
+ with unittest.mock.patch.object(sys, "argv", argv), self.assertRaises(SystemExit):
+ serve.main()
+ finally:
+ s.close()
+ self.assertEqual(self.store.get(jid)["Status"], "working")
+
if __name__ == "__main__":
unittest.main(verbosity=1)