Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
fix(engine): P3 audit loop 1: blocks before the first place, resumes after the search, refused logins
Mahan Jafaric55406455097
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.ymlindex 197f26f..51f5d91 100644--- a/.github/workflows/engine-live.yml+++ b/.github/workflows/engine-live.yml@@ -22,7 +22,7 @@ jobs:matrix:os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]runs-on: ${{ matrix.os }}- timeout-minutes: 15+ timeout-minutes: 25 # four live steps: scrape, two proxy checks, window mode with reviewsdefaults:run:shell: bashdiff --git a/engine/jobs.py b/engine/jobs.pyindex 6feb43c..b1f4ee8 100644--- a/engine/jobs.py+++ b/engine/jobs.py@@ -311,16 +311,24 @@ class Store:except OSError:pass- def finish(self, jid, rows, cols, status, error, progress):- """Replace the job's CSV in one step and record how the job ended, under one lock; False+ def rewrite(self, jid, rows, cols):+ """Replace the job's CSV in one step (after a pass that changed rows); its new size, or None(nothing written) if the job was deleted."""with self.lock:if not self._exists(jid):- return False+ return Nonetmp = self.csv_path(jid) + ".tmp"with open(tmp, "wb") as f:f.write(to_csv(rows, cols))os.replace(tmp, self.csv_path(jid))+ return os.path.getsize(self.csv_path(jid))++ def finish(self, jid, rows, cols, status, error, progress):+ """Replace the job's CSV in one step and record how the job ended, under one lock; False+ (nothing written) if the job was deleted."""+ with self.lock:+ if self.rewrite(jid, rows, cols) is None:+ return Falsewith self.db:self.db.execute("UPDATE jobs SET status = ?, error = ?, progress = ? WHERE id = ?",(status, error[:500], json.dumps(progress), jid))@@ -414,6 +422,8 @@ class Worker(threading.Thread):deadline = started + budgetstop = threading.Event()self._running = (jid, stop)+ if self.stopped.is_set(): # shutdown() came between claim() and now+ stop.set()if not self.store.get(jid): # deleted between claim() and now, before cancel() could see itself._running = (None, None)return@@ -421,9 +431,11 @@ class Worker(threading.Thread):reviews = d.get("reviews_max") and stage in ("search", "reviews")if stage == "search":self.store.cut(jid, p.get("bytes", 0)) # the rows of a keyword cut short are searched again- rows = self.store.read_rows(jid) if p.get("bytes") else []+ rows = self.store.read_rows(jid) # [] when the cut removed the fileseen = {place_key(r) for r in rows}- errors, notes, blocked, retries, rot = [], [], False, 0, 0+ # What an earlier run of this job saw, for its final note (a job resumed after its search).+ errors = [p["last_error"]] * p.get("failed", 0) if p.get("last_error") else []+ notes, blocked, retries, rot, refused = [], p.get("blocked", False), p.get("retried", 0), 0, set()cur = {"i": done, "kw": ""}if done or p.get("resumes"):self.log(f"job {jid} resumed at keyword {done + 1}/{len(kws)} with {len(rows)} places, {budget:.0f} s left")@@ -457,10 +469,10 @@ class Worker(threading.Thread):timer.start()try:save(stage=stage, done=done, total=len(kws))- if (done < len(kws) or reviews) and budget > 0:+ if ((stage == "search" and done < len(kws)) or reviews) and budget > 0 and not stop.is_set():self._browser = browser.Browser(lang=locale(d), headless=self.headless)i = done- while i < len(kws) and self._browser:+ while stage == "search" and i < len(kws) and self._browser:left = deadline - time.monotonic()if stop.is_set() or left <= 0:break@@ -471,17 +483,21 @@ class Worker(threading.Thread):try:places, reason, stats = self._collect(proxy, url, d["depth"], left, stop, on_batch)except (CDPError, OSError) as e:+ if str(e) == LOGIN_REFUSED:+ refused.add(proxy)if (PROXY_FAILED.search(str(e)) and len(proxies) > 1 and retries < len(BACKOFF)- and not self._browser.cdp.closed and not stop.is_set()):+ and len(refused) < len(set(proxies)) and not self._browser.cdp.closed+ and not stop.is_set()):retries, rot = retries + 1, rot + 1self.log(f"job {jid} keyword {i + 1}/{len(kws)}: the proxy failed; retry {retries}/"f"{len(BACKOFF)} through the next proxy")continueerrors.append(str(e))self.log(f"job {jid} keyword {i + 1}/{len(kws)}: error: {e}")- if self._browser.cdp.closed: # Chrome is gone: the other keywords would fail too+ # Chrome is gone, or no proxy takes the job's login: the other keywords would fail too+ if self._browser.cdp.closed or (refused and len(refused) >= len(set(proxies))):break- i += 1+ i, retries = i + 1, 0continueon_batch(places, stats["batches"]) # places no batch carried (one place, the DOM fallback)if stop.is_set(): # deleted, or the engine is exiting: this keyword did not finish@@ -494,6 +510,7 @@ class Worker(threading.Thread):self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; "f"retry {retries}/{len(BACKOFF)} in {wait} s"+ (" through the next proxy" if len(proxies) > 1 else ""))+ save()if stop.wait(wait):breakcontinue@@ -512,13 +529,20 @@ class Worker(threading.Thread):if self._browser.cdp.closed:breaki += 1- if reviews and rows and self._browser and not (stop.is_set() or blocked):- save(stage="reviews")- n = self.add_reviews(rows, d, deadline, stop, proxies)+ if stage == "search" and not stop.is_set(): # the search is over, for good: never search again+ stage = "reviews" if reviews else "websites"+ save(stage=stage, bytes=self.store.append(jid, [], cols), blocked=blocked, retried=retries,+ failed=len(errors), last_error=errors[-1] if errors else "")+ if stage == "reviews" and rows and self._browser and not (stop.is_set() or blocked):+ n = self.add_reviews(rows, d, deadline, stop, proxies, save)+ self.store.rewrite(jid, rows, cols) # keep the reviews if the engine stops during the next passself.log(f"job {jid}: reviews for {n}/{sum(1 for r in rows if r.get('place_id'))} places")if not n and self.headless:notes.append("no reviews: Google's limited view, which a headless Chrome gets, shows none; ""set SCRAPER_HEADLESS=0 to run Chrome as a window")+ if stage == "reviews" and not stop.is_set():+ stage = "websites"+ save(stage=stage)except browser.BrowserError as e:errors.append(str(e))self.log(f"job {jid}: {e}")@@ -527,16 +551,18 @@ class Worker(threading.Thread):if self._browser:self._browser.close()self._browser = None- if (d["email"] or d.get("socials")) and rows and not stop.is_set():- save(stage="websites")+ if stage == "websites" and (d["email"] or d.get("socials")) and rows and not stop.is_set():n = add_site_data(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), email=d["email"],socials=d.get("socials"), verify=d.get("verify_emails"), stop=stop.is_set)self.log(f"job {jid}: website data for {n}/{sum(1 for r in rows if r['website'])} websites")self._running = (None, None)if self.stopped.is_set() and self.store.get(jid): # the engine is exiting: resume on the next start+ if stage != "search": # rows a pass changed; while searching the CSV is the record already+ self.store.rewrite(jid, rows, cols)save()self.store.update(jid, "pending")- self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)}: resumes when the engine starts again")+ self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)} ({stage}): resumes when the "+ "engine starts again")returnblock = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "+ (f"Waited and retried {retries} time(s). " if retries else "") + "Wait, or add proxies")@@ -585,7 +611,7 @@ class Worker(threading.Thread):if ctx and not self._browser.cdp.closed:self._browser.dispose(ctx)- def add_reviews(self, rows, d, deadline, stop, proxies):+ def add_reviews(self, rows, d, deadline, stop, proxies, save=lambda: None):"""Review count, stars histogram and up to reviews_max reviews per place, from each place'spanel (the result batches carry no review texts): one page load per place, until thedeadline. The number of places that got reviews."""@@ -613,6 +639,7 @@ class Worker(threading.Thread):if k in ("user_reviews", "reviews_per_rating") or not r.get(k):r[k] = str(v)n += "user_reviews" in got+ save() # elapsed time, for a resume after a crashexcept (CDPError, OSError):pass # Chrome went away: the reviews so far are keptfinally:diff --git a/engine/maps.py b/engine/maps.pyindex 9d29bdc..38f6a3e 100644--- a/engine/maps.py+++ b/engine/maps.py@@ -170,7 +170,7 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False,try: # a Chrome error mid-scroll keeps the places already readend = time.time() + 10 # the first batch arrives with the page- while time.time() < end and batches.count == 0:+ while time.time() < end and batches.count == 0 and not batches.limited:take()time.sleep(0.4)log(f" batch {batches.count}: {len(seen)} places")@@ -213,7 +213,7 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False,if not seen:raisereturn list(seen.values()), f"error: {e}", {"batches": batches.count, "dom_links": 0}- if not seen: # the batch format changed under us: fall back to what the page shows+ if not seen and reason != "blocked": # the batch format changed under us: fall back to the pagefor a in links:p = parse_link(a["href"], a["label"])seen.setdefault(p["data_id"] or p["link"], p)diff --git a/engine/serve.py b/engine/serve.pyindex 252e876..2e190ea 100644--- a/engine/serve.py+++ b/engine/serve.py@@ -518,18 +518,19 @@ def main():browser.make_home()browser.sweep_profiles()store = jobs.Store(a.data)- resumed, failed = store.recover()- if resumed:- log(f"{resumed} job(s) were still working when the engine last stopped: they resume from their last "- "finished keyword")- if failed:- log(f"{failed} job(s) stopped mid-run {jobs.MAX_RESUMES + 1} times: marked failed, rows kept")headless = env["SCRAPER_HEADLESS"].strip().lower() not in ("0", "false", "no", "off")worker = jobs.Worker(store, log, headless=headless)try:servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"], a.host)except OSError as e:sys.exit(f"✗ Cannot listen on {a.host}:{port}: {e}")+ # Only now: an engine that cannot bind (another one runs on this data) must not touch its jobs.+ resumed, failed = store.recover()+ if resumed:+ log(f"{resumed} job(s) were still working when the engine last stopped: they resume from their last "+ "finished keyword")+ if failed:+ log(f"{failed} job(s) stopped mid-run {jobs.MAX_RESUMES + 1} times: marked failed, rows kept")worker.start()chrome = browser.find_chrome()where = f"http://localhost:{port}" if a.host == "127.0.0.1" else \diff --git a/engine/test_engine.py b/engine/test_engine.pyindex 1f3f32a..8133690 100644--- a/engine/test_engine.py+++ b/engine/test_engine.py@@ -1035,5 +1035,118 @@ class ApiKeyDocs(Server):self.assertEqual(self.call("GET", "/api/v1/jobs/00000000-0000-0000-0000-000000000000/download")[0], 401)++class LoopOne(Fakes):+ """Audit loop 1: blocks with no places yet, resumes after the search, refused logins, races."""++ def setUp(self):+ super().setUp()+ self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)+ jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01++ def tearDown(self):+ jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits+ super().tearDown()++ def test_real_collect_calls_a_429_blocked(self):+ q = queue.Queue()+ class C:+ closed = None+ def subscribe(self, *a):+ return q+ def unsubscribe(self, *a):+ pass+ class P:+ sid, cdp = "s", C()+ def set_cookie(self, *a):+ pass+ def navigate(self, url):+ q.put({"requestId": "1", "response": {"url": "https://www.google.com/search?tbm=map&pb=x",+ "status": 429, "mimeType": "text/html"}})+ def url(self):+ return "https://www.google.com/maps/search/x"+ def wait_for(self, expr, timeout=15):+ return True+ def eval(self, expr):+ return {"cls": False, "text": ""} if expr == maps.JS_END else []+ def send(self, *a, **k):+ raise CDPError("no body")+ t = time.time()+ real_collect = self.saved[2] # Fakes swapped maps.collect for the fake one+ places, reason, _ = real_collect(P(), "https://www.google.com/maps/search/x", depth=3, log=lambda m: None)+ self.assertEqual((places, reason), ([], "blocked")) # not "blocked+dom-fallback"+ self.assertLess(time.time() - t, 5) # no 10 s wait for a first batch that Google refused++ def test_resume_after_the_reviews_keeps_them(self):+ RESULTS["cafes in Austin"] = ([place(1, website="https://a.com"), place(2)], "end-of-list")+ def reviews(rows, d, deadline, stop, proxies, save=None):+ for r in rows:+ r["user_reviews"] = '[{"Rating": 5}]'+ return len(rows)+ self.worker.add_reviews = reviews+ stopping = lambda *a, **k: self.worker.shutdown() or 0 # the engine stops during the website pass+ with unittest.mock.patch.object(jobs, "add_site_data", side_effect=stopping):+ jid, job = self.run_one(reviews_max=1, email=True)+ self.assertEqual((job["Status"], job["Progress"]["stage"]), ("pending", "websites"))+ w2 = jobs.Worker(self.store, self.lines.append)+ w2.add_reviews = lambda *a, **k: self.fail("the reviews were done already")+ with unittest.mock.patch.object(jobs, "fetch", return_value="[email protected]"):+ w2.run_job(self.store.claim())+ rows = self.rows(jid)+ self.assertEqual(SEARCHED, ["cafes in Austin"]) # searched once, across the restart+ self.assertEqual([r["user_reviews"] for r in rows], ['[{"Rating": 5}]'] * 2)+ self.assertEqual((rows[0]["emails"], self.store.get(jid)["Status"]), ("[email protected]", "ok"))++ def test_blocked_job_resumed_in_its_website_pass_keeps_rows_and_its_note(self):+ RESULTS["cafes in Austin"] = ([place(1, website="https://a.com")], ["slow-blocked"])+ def collect(page, url, on_batch=None, **kw):+ SEARCHED.append("cafes in Austin")+ on_batch([place(1, website="https://a.com")], 1) # streamed, then Google refuses the rest+ return [], "blocked", {"batches": 1, "dom_links": 0}+ maps.collect = collect+ stopping = lambda *a, **k: self.worker.shutdown() or 0+ with unittest.mock.patch.object(jobs, "add_site_data", side_effect=stopping):+ jid, job = self.run_one(email=True)+ self.assertEqual(job["Status"], "pending")+ SEARCHED.clear()+ with unittest.mock.patch.object(jobs, "fetch", return_value=""):+ jobs.Worker(self.store, self.lines.append).run_job(self.store.claim())+ job = self.store.get(jid)+ self.assertEqual(SEARCHED, []) # never back to the block+ self.assertEqual((job["Status"], [r["title"] for r in self.rows(jid)]), ("ok", ["Biz 1"]))+ self.assertIn("unusual-traffic page", job["Error"])+ self.assertIn("Waited and retried 2 time(s)", job["Error"])++ def test_a_refused_login_stops_the_job(self):+ def collect(page, url, **kw):+ SEARCHED.append(url)+ raise CDPError(jobs.LOGIN_REFUSED)+ maps.collect = collect+ _, job = self.run_one(keywords=["a", "b", "c"], proxies=["http://u:bad@h:1"])+ self.assertEqual((job["Status"], job["Error"], len(SEARCHED)), ("failed", jobs.LOGIN_REFUSED, 1))++ def test_shutdown_before_the_job_starts(self):+ name, data = jobs.validate(GOOD)+ jid = self.store.create(name, data)+ job = self.store.claim()+ self.worker.stopped.set()+ self.worker.run_job(job)+ self.assertEqual((self.store.get(jid)["Status"], FakeBrowser.made), ("pending", []))++ def test_an_engine_that_cannot_bind_leaves_the_jobs_alone(self):+ name, data = jobs.validate(GOOD)+ jid = self.store.create(name, data)+ self.store.claim() # another engine on this data folder runs it+ s = socket.socket()+ s.bind(("127.0.0.1", 0))+ s.listen(8)+ argv = ["serve.py", "--port", str(s.getsockname()[1]), "--data", os.path.join(self.dir, "data")]+ try:+ with unittest.mock.patch.object(sys, "argv", argv), self.assertRaises(SystemExit):+ serve.main()+ finally:+ s.close()+ self.assertEqual(self.store.get(jid)["Status"], "working")+if __name__ == "__main__":unittest.main(verbosity=1)