Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
feat(engine): P3: resumable jobs, streaming rows, progress, backoff, proxy logins, locale, full-view data
Mahan Jafaridc9b13c4282e
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.ymlindex 4ce2d9b..197f26f 100644--- a/.github/workflows/engine-live.yml+++ b/.github/workflows/engine-live.yml@@ -41,10 +41,13 @@ jobs:test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Origin: https://evil.com' -X POST -d '{}' http://localhost:8080/api/v1/jobs)" = 403curl -s -X POST -d '{}' http://localhost:8080/api/v1/jobs | grep -q 'missing name'curl -s http://localhost:8080/ | grep -q 'github.com/Mahanaicoach/google-maps-scraper-kit'+ curl -s http://localhost:8080/api/docs | grep -q '<code>reviews_max</code>'+ if curl -s http://localhost:8080/api/docs | grep -q '<script'; then exit 1; fi # set -e ignores a "!" line+ curl -s http://localhost:8080/api/openapi.json | grep -q '"openapi"'- name: scrape (depth 2, no email crawl)run: |start=$(date +%s)- python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --out leads.csv > scrape.log 2>&1 || true+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --country us --out leads.csv > scrape.log 2>&1 || trueecho "seconds=$(( $(date +%s) - start ))"grep -v "^ •" scrape.log | tr '\r' '\n' | grep -v "^ status:" || truepython - <<'PY'@@ -54,6 +57,55 @@ jobs:print(f"places={len(rows)}", fill)sys.exit(0 if rows else 1)PY+ - name: proxy with a username and password (a local test proxy that answers 407 first)+ run: |+ python engine/test_proxy.py 8191 kit 'p@ss:w0rd' > proxy.log 2>&1 &+ sleep 2+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 1 --no-email --max-time 120 \+ --proxy 'http://kit:p%40ss%[email protected]:8191' --out proxied.csv > proxied.log 2>&1 || true+ grep -v "^ •" proxied.log | tr '\r' '\n' | grep -v "^ status:" || true+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 1 --no-email --max-time 60 \+ --proxy 'http://kit:[email protected]:8191' --out refused.csv > refused.log 2>&1 || true+ grep -q "proxy login refused" refused.log+ if grep -q "w0rd" .engine/engine.log; then echo "a proxy password reached the log"; exit 1; fi+ python - <<'PY'+ import csv, json, sys+ rows = list(csv.DictReader(open("proxied.csv", encoding="utf-8-sig")))+ counts = json.load(open(".engine/test-proxy.json"))+ print(f"places={len(rows)}", counts)+ sys.exit(0 if rows and counts["tunnels"] and counts["refused"] else 1)+ PY+ - name: window mode (SCRAPER_HEADLESS=0) with the reviews pass+ env:+ SCRAPER_HEADLESS: "0"+ run: |+ if [ "$RUNNER_OS" = Linux ]; then # a window needs a display+ command -v Xvfb >/dev/null || { sudo apt-get update -q && sudo apt-get install -y -q xvfb; }+ Xvfb :99 -screen 0 1280x900x24 > /dev/null 2>&1 &+ export DISPLAY=:99+ fi+ python scripts/engine.py restart+ python - <<'PY'+ import json, sys, time, urllib.request+ base = "http://localhost:8080/api/v1/jobs"+ def call(method, url, body=None):+ r = urllib.request.Request(url, json.dumps(body).encode() if body else None,+ {"Content-Type": "application/json"}, method=method)+ with urllib.request.urlopen(r, timeout=30) as f:+ return json.loads(f.read())+ jid = call("POST", base, {"name": "window", "keywords": ["coffee shops in Austin TX"], "lang": "en",+ "country": "us", "lat": "30.2672", "lon": "-97.7431", "depth": 1,+ "max_time": 90, "reviews_max": 1})["id"]+ for _ in range(200):+ job = call("GET", f"{base}/{jid}")+ if job["Status"] in ("ok", "failed"):+ break+ time.sleep(2)+ rows = json.loads(urllib.request.urlopen(f"{base}/{jid}/download?format=json").read())+ fill = {k: sum(1 for r in rows if r.get(k)) for k in ("review_count", "open_hours", "user_reviews")}+ print(f"status={job['Status']} places={len(rows)}", fill, job["Progress"])+ sys.exit(0 if rows and fill["review_count"] and fill["user_reviews"] else 1)+ PY- name: engine log (counts only)if: always()run: cat .engine/engine.log || truediff --git a/engine/browser.py b/engine/browser.pyindex 50e396b..fc85c07 100644--- a/engine/browser.py+++ b/engine/browser.py@@ -7,9 +7,10 @@ An installed Google Chrome, Chromium, Edge or Brave is used first (set CHROME=/ppick one). On a machine with none, `python3 scripts/engine.py start` downloads Chrome for Testing'sheadless shell into .engine/, pinned below by version and SHA-256. Standard library only."""-import hashlib, os, platform, shutil, subprocess, tempfile, time, urllib.request, zipfile+import hashlib, os, platform, queue, shutil, subprocess, tempfile, threading, time, urllib.parse, urllib.request+import zipfile-from cdp import CDP+from cdp import CDP, CDPErrorKIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))HOME = os.path.join(KIT, ".engine") # everything the engine keeps: jobs, results, logs, its Chrome@@ -190,10 +191,64 @@ def start_error(path, code, stderr):return f"Chrome exited at once (code {code}): {path}" + (f": {last}" if last else "")+def split_proxy(url):+ """(the proxy as Chrome takes it, username, password): Chrome takes no user:pass in a proxy URL+ and knows socks5:// only (it resolves names through a SOCKS5 proxy anyway)."""+ u = urllib.parse.urlsplit(url)+ host = f"[{u.hostname}]" if ":" in (u.hostname or "") else u.hostname+ scheme = "socks5" if u.scheme == "socks5h" else u.scheme+ return (f"{scheme}://{host}" + (f":{u.port}" if u.port else ""),+ urllib.parse.unquote(u.username or ""), urllib.parse.unquote(u.password or ""))+++class ProxyAuth:+ """Logs one tab in to its proxy: Chrome asks over CDP (Fetch.authRequired) when the proxy answers+ 407, and gets the username and password here. With Fetch on, every request of the tab waits until+ it is continued, so a thread continues them. The credentials go to Chrome only, never to a log."""++ def __init__(self, page, user, password):+ self.page, self.user, self.password = page, user, password+ self.q = page.cdp.subscribe("Fetch.requestPaused", page.sid)+ page.cdp.subscribe("Fetch.authRequired", page.sid, q=self.q)+ self.done, self.asked, self.refused = threading.Event(), set(), False+ page.send("Fetch.enable", {"handleAuthRequests": True, "patterns": [{"urlPattern": "*"}]})+ threading.Thread(target=self._run, daemon=True, name="proxy-auth").start()++ def _run(self):+ while not self.done.is_set():+ try:+ ev = self.q.get(timeout=0.5)+ except queue.Empty:+ continue+ if not isinstance(ev, dict): # the connection died+ return+ rid = ev.get("requestId")+ try:+ if "authChallenge" not in ev:+ self.page.send("Fetch.continueRequest", {"requestId": rid})+ elif ev["authChallenge"].get("source") == "Proxy" and rid not in self.asked:+ self.asked.add(rid) # asked again for the same request: the proxy refused them+ self.page.send("Fetch.continueWithAuth", {"requestId": rid, "authChallengeResponse": {+ "response": "ProvideCredentials", "username": self.user, "password": self.password}})+ else:+ self.refused = self.refused or rid in self.asked+ self.page.send("Fetch.continueWithAuth", {"requestId": rid, "authChallengeResponse": {+ "response": "CancelAuth" if rid in self.asked else "Default"}})+ except CDPError: # the tab closed or the request finished meanwhile+ if self.page.cdp.closed:+ return++ def close(self):+ self.done.set()+ for m in ("Fetch.requestPaused", "Fetch.authRequired"):+ self.page.cdp.unsubscribe(m, self.q, self.page.sid)++class Browser:- """A headless Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP."""+ """A Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP. Headless by+ default; headless=False runs it as a normal window off-screen (Google's full view: see details.py)."""- def __init__(self, path=None, headless=True, profile=None, lang="en-US", proxy=None, timeout=30):+ def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):self.path = path or find_chrome()if not self.path:raise BrowserError(f"No Chrome, Chromium, Edge or Brave found. Run `{PY} scripts/engine.py start`, "@@ -209,8 +264,8 @@ class Browser:f"--lang={lang}", "--disable-dev-shm-usage"]if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headlessargs.append("--headless=new")- if proxy: # Chrome knows socks5:// only; it resolves names through a SOCKS5 proxy anyway- args.append("--proxy-server=" + ("socks5://" + proxy[10:] if proxy.startswith("socks5h://") else proxy))+ elif not headless:+ args.append("--window-position=-3000,-3000") # a real window, kept out of sightif platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)self.proc, self.cdp = None, None@@ -264,6 +319,20 @@ class Browser:def version(self):return self.cdp.send("Browser.getVersion").get("product", "")+ def context(self, proxy=None):+ """A new browser context (its own cookies, and `proxy` for all its traffic); its id. Each+ keyword gets one, so proxies rotate without restarting Chrome."""+ params = {"disposeOnDetach": True}+ if proxy:+ params["proxyServer"] = split_proxy(proxy)[0]+ return self.cdp.send("Target.createBrowserContext", params)["browserContextId"]++ def dispose(self, context):+ try:+ self.cdp.send("Target.disposeBrowserContext", {"browserContextId": context})+ except (CDPError, OSError):+ pass+def kill(self):"""Stop Chrome now (the job watchdog): every CDP call still waiting fails at once."""try:diff --git a/engine/cdp.py b/engine/cdp.pyindex f11d0c3..318e4a1 100644--- a/engine/cdp.py+++ b/engine/cdp.py@@ -191,9 +191,10 @@ class CDP:raise CDPError(f"{method}: {reply['error'].get('message')}")return reply.get("result", {})- def subscribe(self, method, session_id=None):- """A queue that receives the params of every `method` event on `session_id` from now on."""- q = queue.Queue()+ def subscribe(self, method, session_id=None, q=None):+ """A queue that receives the params of every `method` event on `session_id` from now on+ (pass q to share one queue between events)."""+ q = q or queue.Queue()self._subs.setdefault((session_id, method), []).append(q)return q@@ -224,11 +225,15 @@ class CDP:class Page:- """One attached tab: navigate, evaluate JavaScript, wait."""+ """One attached tab: navigate, evaluate JavaScript, wait. `context` is a browser context id+ (Browser.context) for a tab with its own cookies and proxy."""- def __init__(self, cdp, url="about:blank"):+ def __init__(self, cdp, url="about:blank", context=None):self.cdp = cdp- self.target = cdp.send("Target.createTarget", {"url": url})["targetId"]+ params = {"url": url}+ if context:+ params["browserContextId"] = context+ self.target = cdp.send("Target.createTarget", params)["targetId"]self.sid = cdp.send("Target.attachToTarget", {"targetId": self.target, "flatten": True})["sessionId"]for domain in ("Page", "Runtime", "Network"):cdp.send(f"{domain}.enable", session_id=self.sid)diff --git a/engine/details.py b/engine/details.pyindex d32a553..4338b8d 100644--- a/engine/details.py+++ b/engine/details.py@@ -3,19 +3,20 @@While the result list scrolls, the page fetches batches of about 20 places from`https://www.google.com/search?tbm=map…` as `)]}'\\n[...]` JSON; each batch carries the full placeentity (name, address, phone, website, rating, coordinates, ids…). The engine captures those-responses in the browser (cdp Network events) and parses them here, so no place page is ever-opened: the spike showed the place page itself, fetched by curl or by Chrome, holds no entity data-any more. Standard library only.+responses in the browser (cdp Network events) and parses them here, so no place page is opened for+them: the spike showed the place page document itself holds no entity data any more. Review texts+are the one thing the batches lack: they come from the `/maps/preview/place` response a place's+panel downloads (parse_place), only for jobs that ask for them. Standard library only."""import json# Index path of each field inside a place entity. Checked on live batches in the spike# (2026-10-31): title, category, address, rating, lat/lon, place_id, data_id, timezone, thumbnail# filled on 60/60 entities, website 55/60, phone 46/60 (listings without one).-# review_count is not here: signed out, Google leaves (4, 8) empty, and (37, 1) is a different-# number (60 and 82 where the page showed 61 and 134 reviews, 2026-10-31). Headless Chrome gets-# Google's limited view, whose result cards show "4.8 stars" without the count, so the CSV's-# review_count column stays empty.+# review_count, reviews_link and the week of opening hours come only with Google's full view: a+# headless Chrome (the default) gets its "limited view", which leaves (4, 8) and (4, 3) empty and+# lists today's hours only. A Chrome window (SCRAPER_HEADLESS=0) gets the full view: count at (4, 8)+# on 20/20 entities, 7 days of hours on 19/20 (2026-10-31). (37, 1) is the photo count, not reviews.FIELDS = {"title": (11,),"category": (13, 0),@@ -23,6 +24,8 @@ FIELDS = {"website": (7, 0),"phone": (178, 0, 0),"review_rating": (4, 7),+ "review_count": (4, 8),+ "reviews_link": (4, 3, 0),"latitude": (9, 2),"longitude": (9, 3),"plus_code": (183, 2, 2, 0),@@ -34,6 +37,9 @@ FIELDS = {"price_range": (4, 2),"description": (32, 1, 1),}+HOURS = (203, 0) # [[day name, 1-7 (Monday-Sunday), date, [[text, ...], ...], ...], ...], from today on+HISTOGRAM = (175, 3) # place panel: [one-star count, ..., five-star count]+REVIEWS = (175, 9, 0, 0) # place panel: the reviews it shows first (about 5)LISTS = {"categories": (13,), "address_lines": (2,)}ENTRIES = 64 # position of the result list inside a batch@@ -59,6 +65,9 @@ def parse_entity(entity):items = [x for x in v if isinstance(x, str) and x]if items:out[name] = " | ".join(items)+ hours = open_hours(dig(entity, HOURS))+ if hours:+ out["open_hours"] = hoursif "data_id" in out and ":" in out["data_id"]:try:out["cid"] = str(int(out["data_id"].split(":")[1], 16))@@ -67,6 +76,53 @@ def parse_entity(entity):return out+def _text(v):+ return v.replace("\u202f", " ").strip() if isinstance(v, str) else ""+++def open_hours(days):+ """The week's hours as old-engine JSON ({"Monday": ["7 AM–9 PM"], ...}, Monday first), or "" when+ Google sent fewer than 7 days (the limited view sends today only, which would read as a week)."""+ if not isinstance(days, list):+ return ""+ week = {}+ for d in days:+ if not (isinstance(d, list) and len(d) > 3 and isinstance(d[0], str) and isinstance(d[1], int)):+ continue+ spans = [_text(x[0]) for x in d[3] or [] if isinstance(x, list) and x and _text(x[0])]+ week[d[1]] = (d[0], spans)+ if len(week) < 7:+ return ""+ return json.dumps({name: spans for _, (name, spans) in sorted(week.items())}, ensure_ascii=False)+++def parse_place(body, limit=5):+ """review_count, reviews_per_rating and up to `limit` user_reviews from one `/maps/preview/place`+ response. Reviewer names and photos are left out: the kit collects business data, not people."""+ if body.startswith(")]}'"):+ body = body.split("\n", 1)[-1]+ try:+ entity = dig(json.loads(body), (6,))+ except ValueError:+ return {}+ if not isinstance(entity, list):+ return {}+ out = {k: v for k, v in parse_entity(entity).items() if k in ("review_count", "reviews_link", "open_hours")}+ hist = dig(entity, HISTOGRAM)+ if isinstance(hist, list) and len(hist) == 5 and all(isinstance(n, int) for n in hist):+ out["reviews_per_rating"] = json.dumps({str(i): n for i, n in enumerate(hist, 1)})+ reviews = []+ for r in dig(entity, REVIEWS) or []:+ rating, text = dig(r, (0, 2, 0, 0)), _text(dig(r, (0, 2, 15, 0, 0)))+ if isinstance(rating, int) and not isinstance(rating, bool):+ reviews.append({"Rating": rating, "When": _text(dig(r, (0, 1, 6))), "Description": text})+ if len(reviews) >= limit:+ break+ if reviews:+ out["user_reviews"] = json.dumps(reviews, ensure_ascii=False)+ return out++def parse_batch(body):"""The places in one `/search?tbm=map` response body (with or without the `)]}'` prefix)."""if body.startswith(")]}'"):diff --git a/engine/jobs.py b/engine/jobs.pyindex c86c6d0..6feb43c 100644--- a/engine/jobs.py+++ b/engine/jobs.py@@ -1,16 +1,21 @@"""Scrape jobs: a sqlite3 store in .engine/data and one worker that runs the jobs one at a time.-A job runs its keywords through maps.collect in one headless Chrome, de-duplicates the places on-data_id and writes <id>.csv with the old engine's 36 column names (what Google's result batches-carry is filled, the rest stays empty) plus ours. The CSV is rewritten atomically after every-keyword, so a download while the job works returns the rows so far. A watchdog stops Chrome when a-job overruns its max_time, so no job can stick on "working". Standard library only.+A job runs its keywords through maps.collect in one Chrome, each keyword in its own browser context+(own cookies, the next proxy of the job's list), de-duplicates the places on data_id and appends them+to <id>.csv as the result batches arrive, with the old engine's 36 column names (what Google's result+batches carry is filled, the rest stays empty) plus ours. Appends and downloads share one lock, so a+download while the job works returns whole rows. The job's Progress (keywords done, places, percent)+is saved as it goes; a job the engine was stopped in the middle of resumes from its last finished+keyword on the next start. When Google answers with its unusual-traffic page or 429, the keyword+waits and retries a bounded number of times (through the next proxy when there is one), then the job+stops with a clear Error. A watchdog stops Chrome when a job overruns its max_time, so no job can+stick on "working". Standard library only."""-import concurrent.futures, csv, html, http.client, io, ipaddress, json, math, os, re, socket, sqlite3-import threading, time, urllib.parse, urllib.request, uuid+import concurrent.futures, csv, functools, html, http.client, io, ipaddress, json, math, os, re, socket+import sqlite3, threading, time, urllib.parse, urllib.request, uuidfrom datetime import datetime, timezone-import browser, maps+import browser, details, mapsfrom cdp import CDPError, Page# gosom/google-maps-scraper's CSV header, in its order, so files and scripts that read the old@@ -22,15 +27,24 @@ COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "po"order_online", "menu", "owner", "complete_address", "credit_cards_accepted", "about","user_reviews", "user_reviews_extended", "emails"]EXTRA = ["categories"]+SOCIALS = ["instagram", "facebook", "linkedin", "tiktok", "youtube", "x"] # the names scripts/scrape.py --socials usesRENAME = {"open_state": "status", "description": "descriptions"} # details.FIELDS name -> CSV column# Seconds past max_time before the watchdog kills Chrome. maps.collect's page load and first waits# (navigate 45 s, consent 15 s, feed 15 s, first batch 10 s) sit outside its max_seconds, so a# shorter grace could kill a keyword that was about to return its rows.WATCHDOG_GRACE = 90-SITE_WORKERS = 8 # business websites fetched at once by the email pass+SITE_WORKERS = 8 # business websites fetched at once by the website passSITE_TIMEOUT = 10 # seconds per website request-EMAIL_MIN_SECONDS = 60 # the email pass gets at least this long, even after a job used its max_time+EMAIL_MIN_SECONDS = 60 # the website pass gets at least this long, even after a job used its max_time+BACKOFF = (30, 90) # seconds to wait before each retry of a keyword Google refused; then the job stops+ROTATE_WAIT = 3 # seconds before the retry when the next proxy can take it instead+MAX_RESUMES = 3 # restarts of the engine mid-job before the job is failed instead of resumed+LOGIN_REFUSED = "proxy login refused: the proxy answered 407 to its username and password"+# Chrome's errors for a proxy that failed, and ours. Matched as written: an error also holds the+# search URL, so a keyword such as "tunnel car wash" must not read as a proxy error.+PROXY_FAILED = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_|proxy login refused")+REVIEWS_MAX = 5 # the reviews a place panel shows first; more would need its review listdef utc_now():@@ -71,6 +85,11 @@ def validate(body):raise ValueError("missing lang")if not isinstance(lang, str) or not LANG.fullmatch(lang):raise ValueError("invalid lang")+ country = body.get("country")+ if country in (None, ""):+ country = ""+ elif not isinstance(country, str) or not LANG.fullmatch(country):+ raise ValueError("invalid country: a 2-letter country code such as us or de")lat, lon = body.get("lat"), body.get("lon")if lat in (None, "") or lon in (None, ""):raise ValueError("missing geo coordinates")@@ -84,13 +103,15 @@ def validate(body):raise ValueError("invalid geo coordinates")if body.get("max_time") in (None, 0):raise ValueError("missing max time")- data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "lat": str(lat).strip(),- "lon": str(lon).strip(),+ data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "country": country.lower(),+ "lat": str(lat).strip(), "lon": str(lon).strip(),"zoom": _int(body, "zoom", 15, 1, 21, "zoom"),"radius": _int(body, "radius", 10000, 0, 10**9, "radius"), # accepted, not used"depth": _int(body, "depth", 1, 1, 1000, "depth"),"max_time": _int(body, "max_time", 600, 1, 86400, "max time"),"fast_mode": bool(body.get("fast_mode")), "email": bool(body.get("email")),+ "verify_emails": bool(body.get("verify_emails")), "socials": bool(body.get("socials")),+ "reviews_max": _int(body, "reviews_max", 0, 0, REVIEWS_MAX, "reviews_max"),"proxies": _proxies(body.get("proxies"))}return name.strip(), data@@ -105,16 +126,35 @@ def _proxies(v):u = urllib.parse.urlsplit(p.strip())u.port # noqa: B018 - raises on a bad portexcept ValueError:- raise ValueError(f"invalid proxy: {p}") from None+ raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port") from Noneif u.scheme not in PROXY_SCHEMES or not u.hostname:raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port")- if u.username or u.password:- raise ValueError("proxy usernames and passwords are not supported by this engine yet: "- "use a proxy that allows your IP address")+ if (u.username or u.password) and u.scheme.startswith("socks"):+ raise ValueError("invalid proxy: Chrome cannot log in to a SOCKS5 proxy; use its http:// address "+ "with user:pass, or one that allows your IP address")return [p.strip() for p in v]+def mask(proxy):+ """A proxy URL with its password replaced by ***: what the API, the web UI and the log show."""+ u = urllib.parse.urlsplit(proxy)+ if not u.password:+ return proxy+ return urllib.parse.urlunsplit(u._replace(netloc=u.netloc.rpartition("@")[0].partition(":")[0] + ":***@"+ + u.netloc.rpartition("@")[2]))+++def locale(d):+ """Chrome's --lang (its Accept-Language) for a job: en, or en-US when the job names a country."""+ return d["lang"] + (f"-{d['country'].upper()}" if d.get("country") else "")++# ── Rows ──+def fields(d):+ """A job's CSV columns: the old engine's 36, ours, then those its website pass fills."""+ return COLUMNS + EXTRA + (SOCIALS if d.get("socials") else []) + (["email_host"] if d.get("verify_emails") else [])++def to_row(place, keyword):"""One CSV row (every column, as text) from a details.parse_entity / maps.parse_link place."""row = dict.fromkeys(COLUMNS + EXTRA, "")@@ -130,10 +170,16 @@ def to_row(place, keyword):return row-def to_csv(rows):+def place_key(p):+ """What makes two places (or rows) the same place."""+ return p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")+++def to_csv(rows, cols=None, header=True):buf = io.StringIO()- w = csv.DictWriter(buf, fieldnames=COLUMNS + EXTRA, extrasaction="ignore")- w.writeheader()+ w = csv.DictWriter(buf, fieldnames=cols or COLUMNS + EXTRA, extrasaction="ignore")+ if header:+ w.writeheader()w.writerows(rows)return buf.getvalue().encode("utf-8")@@ -141,8 +187,8 @@ def to_csv(rows):# ── Store ──class Store:"""Jobs in sqlite3 (jobs.db) and their results as <id>.csv beside it. One lock serialises every- database call and every CSV read and write, so a download never meets a half-replaced file- (Windows cannot replace a file another handle has open)."""+ database call and every CSV read, append and replace, so a download never meets a half-written+ row or a half-replaced file (Windows cannot replace a file another handle has open)."""def __init__(self, folder):os.makedirs(folder, exist_ok=True)@@ -152,37 +198,47 @@ class Store:with self.lock, self.db:self.db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, ""date TEXT NOT NULL, status TEXT NOT NULL, data TEXT NOT NULL, "- "error TEXT NOT NULL DEFAULT '')")+ "error TEXT NOT NULL DEFAULT '', progress TEXT NOT NULL DEFAULT '{}')")+ if "progress" not in [c[1] for c in self.db.execute("PRAGMA table_info(jobs)")]: # a P1 or P2 store+ self.db.execute("ALTER TABLE jobs ADD COLUMN progress TEXT NOT NULL DEFAULT '{}'")++ SELECT = "SELECT id, name, date, status, data, error, progress FROM jobs"@staticmethoddef _job(r):- # The old engine's job shape (capitalised keys), plus Error: why a job failed or stopped early.- return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5]}+ # The old engine's job shape (capitalised keys), plus Error (why a job failed or stopped early)+ # and Progress (the worker's own record; public() is what the API shows).+ return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5],+ "Progress": json.loads(r[6] or "{}")}def create(self, name, data):jid = str(uuid.uuid4())with self.lock, self.db:- self.db.execute("INSERT INTO jobs VALUES (?, ?, ?, 'pending', ?, '')",- (jid, name, utc_now(), json.dumps(data)))+ self.db.execute("INSERT INTO jobs (id, name, date, status, data, error, progress) "+ "VALUES (?, ?, ?, 'pending', ?, '', '{}')", (jid, name, utc_now(), json.dumps(data)))return jiddef get(self, jid):with self.lock:- r = self.db.execute("SELECT * FROM jobs WHERE id = ?", (jid,)).fetchone()+ r = self.db.execute(self.SELECT + " WHERE id = ?", (jid,)).fetchone()return self._job(r) if r else Nonedef all(self):with self.lock:- return [self._job(r) for r in self.db.execute("SELECT * FROM jobs ORDER BY rowid DESC")]+ return [self._job(r) for r in self.db.execute(self.SELECT + " ORDER BY rowid DESC")]def update(self, jid, status, error=""):with self.lock, self.db:self.db.execute("UPDATE jobs SET status = ?, error = ? WHERE id = ?", (status, error[:500], jid))+ def set_progress(self, jid, progress):+ with self.lock, self.db:+ self.db.execute("UPDATE jobs SET progress = ? WHERE id = ?", (json.dumps(progress), jid))+def claim(self):"""The oldest pending job, now marked working; None when there is none."""with self.lock, self.db:- r = self.db.execute("SELECT * FROM jobs WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()+ r = self.db.execute(self.SELECT + " WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()if not r:return Noneself.db.execute("UPDATE jobs SET status = 'working' WHERE id = ?", (r[0],))@@ -190,11 +246,28 @@ class Store:job["Status"] = "working"return job- def recover(self):- """Jobs left on "working" by an engine that was stopped or crashed: failed, rows kept."""+ def recover(self, max_resumes=MAX_RESUMES):+ """Jobs left on "working" by an engine that was stopped or crashed: back to pending, so they+ resume from their last finished keyword (the rows of the keyword cut short are removed, a+ torn last line too). After max_resumes such restarts a job is failed instead, rows kept, as+ before resumable jobs. (resumed, failed)"""+ resumed = failed = 0with self.lock, self.db:- return self.db.execute("UPDATE jobs SET status = 'failed', error = 'the engine stopped while "- "this job ran; its rows so far are kept' WHERE status = 'working'").rowcount+ for jid, prog in self.db.execute("SELECT id, progress FROM jobs WHERE status = 'working'").fetchall():+ p = json.loads(prog or "{}")+ if p.get("stage", "search") == "search":+ self.cut(jid, p.get("bytes", 0))+ if p.get("resumes", 0) >= max_resumes:+ self.db.execute("UPDATE jobs SET status = 'failed', error = ? WHERE id = ?",+ (f"the engine stopped while this job ran ({p['resumes'] + 1} times); its rows "+ "so far are kept", jid))+ failed += 1+ else:+ p["resumes"] = p.get("resumes", 0) + 1+ self.db.execute("UPDATE jobs SET status = 'pending', progress = ? WHERE id = ?",+ (json.dumps(p), jid))+ resumed += 1+ return resumed, faileddef delete(self, jid):with self.lock, self.db:@@ -209,15 +282,48 @@ class Store:def csv_path(self, jid):return os.path.join(self.folder, f"{jid}.csv")- def write_csv(self, jid, rows):- """Replace the job's CSV in one step; False (nothing written) if the job was deleted."""+ def _exists(self, jid):+ return self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone() is not None++ def append(self, jid, rows, cols):+ """Add rows to the job's CSV (the header first on an empty file); the file's size after it,+ or None (nothing written) if the job was deleted."""+ with self.lock:+ if not self._exists(jid):+ return None+ path = self.csv_path(jid)+ size = os.path.getsize(path) if os.path.exists(path) else 0+ if rows or not size:+ with open(path, "ab") as f:+ f.write(to_csv(rows, cols, header=not size))+ return os.path.getsize(path)++ def cut(self, jid, size):+ """Shorten the job's CSV to `size` bytes (its length at the last finished keyword)."""+ with self.lock:+ path = self.csv_path(jid)+ try:+ if not size:+ os.remove(path)+ elif os.path.getsize(path) > size:+ with open(path, "r+b") as f:+ f.truncate(size)+ except OSError:+ pass++ def finish(self, jid, rows, cols, status, error, progress):+ """Replace the job's CSV in one step and record how the job ended, under one lock; False+ (nothing written) if the job was deleted."""with self.lock:- if not self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone():+ if not self._exists(jid):return Falsetmp = self.csv_path(jid) + ".tmp"with open(tmp, "wb") as f:- f.write(to_csv(rows))+ f.write(to_csv(rows, cols))os.replace(tmp, self.csv_path(jid))+ with self.db:+ self.db.execute("UPDATE jobs SET status = ?, error = ?, progress = ? WHERE id = ?",+ (status, error[:500], json.dumps(progress), jid))return Truedef read_csv(self, jid):@@ -228,14 +334,33 @@ class Store:except OSError:return None+ def read_rows(self, jid):+ """The job's rows as dicts ([] before its first row)."""+ data = self.read_csv(jid)+ return list(csv.DictReader(io.StringIO(data.decode("utf-8", "replace")))) if data else []+++def public(job):+ """A job as the API shows it: proxy passwords masked, Progress without the worker's bookkeeping."""+ d, p = job["Data"], job["Progress"]+ total = len(d.get("keywords") or [])+ stage = p.get("stage") or ("queued" if job["Status"] == "pending" else "")+ if job["Status"] == "ok":+ stage = "done"+ return dict(job, Data=dict(d, proxies=[mask(x) for x in d.get("proxies") or []]),+ Progress={"keywords_done": p.get("done", 0), "keywords_total": total, "places": p.get("places", 0),+ "percent": 100 if job["Status"] == "ok" else p.get("percent", 0), "stage": stage})+# ── Worker ──class Worker(threading.Thread):- """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once."""+ """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once.+ headless=False runs Chrome as an off-screen window, which Google gives its full view (review+ counts, the week's hours, reviews); see details.py."""- def __init__(self, store, log=print):+ def __init__(self, store, log=print, headless=True):super().__init__(daemon=True, name="jobs")- self.store, self.log = store, log+ self.store, self.log, self.headless = store, log, headlessself.wake = threading.Event()self.stopped = threading.Event()self._running = (None, None) # (job id, stop event) of the job in progress@@ -266,8 +391,9 @@ class Worker(threading.Thread):if rid == jid and stop:stop.set()- def shutdown(self):- """Stop the job in progress and its Chrome (the engine is exiting)."""+ def shutdown(self, wait=0):+ """Stop the job in progress and its Chrome (the engine is exiting). With `wait`, give the job+ that many seconds to save its progress, so the next start resumes it."""self.stopped.set()self.wake.set()_, stop = self._running@@ -276,18 +402,49 @@ class Worker(threading.Thread):b = self._browserif b:b.close()+ if wait and self.is_alive():+ self.join(wait)def run_job(self, job):jid, d = job["ID"], job["Data"]- deadline = time.monotonic() + d["max_time"]+ p = dict(job["Progress"])+ kws, cols, proxies = d["keywords"], fields(d), d.get("proxies") or []+ started, used = time.monotonic(), p.get("elapsed", 0)+ budget = max(0, d["max_time"] - used) # a resumed job has the rest of its max_time+ deadline = started + budgetstop = threading.Event()self._running = (jid, stop)if not self.store.get(jid): # deleted between claim() and now, before cancel() could see itself._running = (None, None)return- rows, seen, errors, blocked = [], set(), [], False- kws = d["keywords"]- self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")+ done, stage = min(p.get("done", 0), len(kws)), p.get("stage", "search")+ reviews = d.get("reviews_max") and stage in ("search", "reviews")+ if stage == "search":+ self.store.cut(jid, p.get("bytes", 0)) # the rows of a keyword cut short are searched again+ rows = self.store.read_rows(jid) if p.get("bytes") else []+ seen = {place_key(r) for r in rows}+ errors, notes, blocked, retries, rot = [], [], False, 0, 0+ cur = {"i": done, "kw": ""}+ if done or p.get("resumes"):+ self.log(f"job {jid} resumed at keyword {done + 1}/{len(kws)} with {len(rows)} places, {budget:.0f} s left")+ else:+ self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")++ def save(**kw):+ p.update(kw, elapsed=round(used + time.monotonic() - started, 1), places=len(rows))+ self.store.set_progress(jid, p)++ def on_batch(places, batches): # streams each batch's new places into the CSV+ fresh = []+ for place in places:+ k = place_key(place)+ if k and k not in seen:+ seen.add(k)+ fresh.append(to_row(place, cur["kw"]))+ rows.extend(fresh)+ if fresh and self.store.append(jid, fresh, cols) is None:+ stop.set() # deleted+ save(percent=int(100 * (cur["i"] + min(batches / d["depth"], 1)) / len(kws)))def watchdog(): # Chrome hung or the job overran: kill Chrome, and every CDP call fails at oncestop.set()@@ -295,50 +452,73 @@ class Worker(threading.Thread):if b:b.kill()self.log(f"job {jid}: watchdog stopped Chrome {WATCHDOG_GRACE} s after max_time")- timer = threading.Timer(d["max_time"] + WATCHDOG_GRACE, watchdog)+ timer = threading.Timer(budget + WATCHDOG_GRACE, watchdog)timer.daemon = Truetimer.start()try:- self._browser = browser.Browser(lang=d["lang"], proxy=(d["proxies"] or [None])[0])- for i, kw in enumerate(kws, 1):+ save(stage=stage, done=done, total=len(kws))+ if (done < len(kws) or reviews) and budget > 0:+ self._browser = browser.Browser(lang=locale(d), headless=self.headless)+ i = done+ while i < len(kws) and self._browser:left = deadline - time.monotonic()if stop.is_set() or left <= 0:break- url = maps.search_url(kw, d["lat"], d["lon"], d["zoom"], d["lang"])- page = None- started = time.monotonic()+ cur.update(i=i, kw=kws[i])+ proxy = proxies[(i + rot) % len(proxies)] if proxies else None+ url = maps.search_url(kws[i], d["lat"], d["lon"], d["zoom"], d["lang"], d.get("country", ""))+ t0 = time.monotonic()try:- page = Page(self._browser.cdp)- places, reason, stats = maps.collect(page, url, depth=d["depth"], max_seconds=left,- log=lambda m: None, stop=stop.is_set)+ places, reason, stats = self._collect(proxy, url, d["depth"], left, stop, on_batch)except (CDPError, OSError) as e:+ if (PROXY_FAILED.search(str(e)) and len(proxies) > 1 and retries < len(BACKOFF)+ and not self._browser.cdp.closed and not stop.is_set()):+ retries, rot = retries + 1, rot + 1+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: the proxy failed; retry {retries}/"+ f"{len(BACKOFF)} through the next proxy")+ continueerrors.append(str(e))- self.log(f"job {jid} keyword {i}/{len(kws)}: error: {e}")+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: error: {e}")if self._browser.cdp.closed: # Chrome is gone: the other keywords would fail toobreak+ i += 1continue- finally:- if page and not self._browser.cdp.closed:- page.close()- if reason == "blocked": # more searches from a rate-limited IP only extend the block+ on_batch(places, stats["batches"]) # places no batch carried (one place, the DOM fallback)+ if stop.is_set(): # deleted, or the engine is exiting: this keyword did not finish+ break+ if reason == "blocked": # Google limits this IP: wait, try again, then give up+ if retries < len(BACKOFF) and not stop.is_set():+ wait = ROTATE_WAIT if len(proxies) > 1 else BACKOFF[retries]+ if time.monotonic() + wait < deadline:+ retries, rot = retries + 1, rot + 1+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; "+ f"retry {retries}/{len(BACKOFF)} in {wait} s"+ + (" through the next proxy" if len(proxies) > 1 else ""))+ if stop.wait(wait):+ break+ continueblocked = True- self.log(f"job {jid} keyword {i}/{len(kws)}: Google's unusual-traffic page; stopping")+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; stopping")break- new = 0- for p in places:- key = p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")- if key and key not in seen:- seen.add(key)- rows.append(to_row(p, kw))- new += 1- if not self.store.write_csv(jid, rows):+ retries = 0+ size = self.store.append(jid, [], cols)+ if size is None:break # deleted- self.log(f"job {jid} keyword {i}/{len(kws)}: {len(places)} places, {new} new, "- f"{stats['batches']} batches, {time.monotonic() - started:.1f} s ({reason})")+ save(done=i + 1, bytes=size, percent=int(100 * (i + 1) / len(kws)))+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: {len(places)} places, {len(rows)} in all, "+ f"{stats['batches']} batches, {time.monotonic() - t0:.1f} s ({reason})")if reason.startswith("error: "): # Chrome failed mid-scroll; the places before it are kepterrors.append(reason[7:])if self._browser.cdp.closed:break+ i += 1+ if reviews and rows and self._browser and not (stop.is_set() or blocked):+ save(stage="reviews")+ n = self.add_reviews(rows, d, deadline, stop, proxies)+ self.log(f"job {jid}: reviews for {n}/{sum(1 for r in rows if r.get('place_id'))} places")+ if not n and self.headless:+ notes.append("no reviews: Google's limited view, which a headless Chrome gets, shows none; "+ "set SCRAPER_HEADLESS=0 to run Chrome as a window")except browser.BrowserError as e:errors.append(str(e))self.log(f"job {jid}: {e}")@@ -347,35 +527,123 @@ class Worker(threading.Thread):if self._browser:self._browser.close()self._browser = None- if d["email"] and rows and not stop.is_set():- n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), stop=stop.is_set)- self.log(f"job {jid}: emails found for {n}/{sum(1 for r in rows if r['website'])} websites")+ if (d["email"] or d.get("socials")) and rows and not stop.is_set():+ save(stage="websites")+ n = add_site_data(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), email=d["email"],+ socials=d.get("socials"), verify=d.get("verify_emails"), stop=stop.is_set)+ self.log(f"job {jid}: website data for {n}/{sum(1 for r in rows if r['website'])} websites")self._running = (None, None)- if not self.store.write_csv(jid, rows):- self.log(f"job {jid} deleted while it ran")+ if self.stopped.is_set() and self.store.get(jid): # the engine is exiting: resume on the next start+ save()+ self.store.update(jid, "pending")+ self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)}: resumes when the engine starts again")returnblock = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "- "Wait, or add proxies")+ + (f"Waited and retried {retries} time(s). " if retries else "") + "Wait, or add proxies")+ p.update(stage="done", elapsed=round(used + time.monotonic() - started, 1), places=len(rows))if rows or not (errors or blocked):- note = "stopped at max_time; the rows found so far are kept" \- if time.monotonic() >= deadline or (stop.is_set() and not errors) else ""+ head = ""+ if time.monotonic() >= deadline or (stop.is_set() and not errors):+ head = "stopped at max_time; the rows found so far are kept"if errors:- note = f"{len(errors)} keyword(s) failed: {errors[-1]}"+ head = f"{len(errors)} keyword(s) failed: {errors[-1]}"if blocked:- note = f"{block}. The rows found before that are kept"- self.store.update(jid, "ok", note)- self.log(f"job {jid} ok: {len(rows)} places")+ head, notes = f"{block}. The rows found before that are kept", []+ status, note = "ok", "; ".join(([head] if head else []) + notes)else:- reason = block if blocked else errors[-1]- self.store.update(jid, "failed", reason)- self.log(f"job {jid} failed: {reason}")+ status = "failed"+ note = block if blocked else errors[-1]+ if not self.store.finish(jid, rows, cols, status, note, p):+ self.log(f"job {jid} deleted while it ran")+ return+ self.log(f"job {jid} {status}: {len(rows)} places" + (f": {note}" if status == "failed" else ""))++ def _collect(self, proxy, url, depth, left, stop, on_batch):+ """One keyword in its own browser context, through `proxy`; maps.collect's answer."""+ ctx = page = auth = None+ try:+ ctx = self._browser.context(proxy)+ page = Page(self._browser.cdp, context=ctx)+ _, user, password = browser.split_proxy(proxy) if proxy else ("", "", "")+ if user or password:+ auth = browser.ProxyAuth(page, user, password)+ try:+ got = maps.collect(page, url, depth=depth, max_seconds=left, log=lambda m: None,+ stop=stop.is_set, on_batch=on_batch)+ except CDPError:+ if auth and auth.refused: # Chrome only says ERR_HTTP_RESPONSE_CODE_FAILURE+ raise CDPError(LOGIN_REFUSED) from None+ raise+ if auth and auth.refused and not got[0]:+ raise CDPError(LOGIN_REFUSED)+ return got+ finally:+ if auth:+ auth.close()+ if page and not self._browser.cdp.closed:+ page.close()+ if ctx and not self._browser.cdp.closed:+ self._browser.dispose(ctx)++ def add_reviews(self, rows, d, deadline, stop, proxies):+ """Review count, stars histogram and up to reviews_max reviews per place, from each place's+ panel (the result batches carry no review texts): one page load per place, until the+ deadline. The number of places that got reviews."""+ todo = [r for r in rows if r.get("place_id") and not r.get("user_reviews")]+ ctx = page = auth = None+ n = 0+ try:+ proxy = proxies[0] if proxies else None+ ctx = self._browser.context(proxy)+ page = Page(self._browser.cdp, context=ctx)+ _, user, password = browser.split_proxy(proxy) if proxy else ("", "", "")+ if user or password:+ auth = browser.ProxyAuth(page, user, password)+ for k, v in maps.CONSENT_COOKIES.items():+ page.set_cookie(k, v, ".google.com")+ for r in todo:+ if stop.is_set() or time.monotonic() >= deadline:+ break+ url = r["link"] + f"&hl={d['lang']}" + (f"&gl={d['country']}" if d.get("country") else "")+ body = maps.place_panel(page, url)+ if body == "blocked":+ break+ got = details.parse_place(body or "", d["reviews_max"])+ for k, v in got.items():+ if k in ("user_reviews", "reviews_per_rating") or not r.get(k):+ r[k] = str(v)+ n += "user_reviews" in got+ except (CDPError, OSError):+ pass # Chrome went away: the reviews so far are kept+ finally:+ if auth:+ auth.close()+ if page and not self._browser.cdp.closed:+ page.close()+ if ctx and not self._browser.cdp.closed:+ self._browser.dispose(ctx)+ return n-# ── Email pass (email: true): each business homepage over plain HTTP, emails only ──+# ── Website pass (email, socials, verify_emails): each business homepage over plain HTTP ──EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")JUNK_DOMAIN = ("example.", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"+# The same patterns as scripts/scrape.py --socials (a test keeps them in step).+SOCIAL_RE = {+ "instagram": re.compile(r"https?://(?:www\.)?instagram\.com/([A-Za-z0-9_.]+)", re.I),+ "facebook": re.compile(r"https?://(?:www\.|m\.|web\.)?facebook\.com/([A-Za-z0-9_.\-]+)", re.I),+ "linkedin": re.compile(r"https?://(?:[a-z]{2,3}\.)?linkedin\.com/(?:company|in|school)/([A-Za-z0-9_.\-%]+)", re.I),+ "tiktok": re.compile(r"https?://(?:www\.)?tiktok\.com/@([A-Za-z0-9_.]+)", re.I),+ "youtube": re.compile(r"https?://(?:www\.)?youtube\.com/(?:@|channel/|c/|user/)([A-Za-z0-9_.\-]+)", re.I),+ "x": re.compile(r"https?://(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})\b", re.I),+}+SKIP_HANDLE = {"", "home", "pages", "people", "help", "about", "policies", "policy", "legal", "tos", "privacy",+ "settings", "sharer", "tr", "profile.php", "plugins", "dialog", "intent", "login", "share.php",+ "permalink.php", "p", "reel", "reels", "explore", "stories", "tv", "watch", "events", "groups",+ "marketplace", "gaming", "photo", "hashtag", "search", "share", "i"}+DOH = "https://dns.google/resolve?" # Google's public DNS over HTTPS: no key, no accountdef _public_ip(ip):@@ -459,6 +727,39 @@ def site_host(url):return h[4:] if h.startswith("www.") else h+def find_socials(text):+ """The first profile link per network on a page, share and login links skipped."""+ out = dict.fromkeys(SOCIALS, "")+ for net, rx in SOCIAL_RE.items():+ for m in rx.finditer(text or ""):+ h = m.group(1).lower()+ if h in SKIP_HANDLE or (net == "facebook" and (h.isdigit() or len(h) < 3)):+ continue+ out[net] = m.group(0).rstrip("\"'/").replace("\\", "")+ break+ return out++[email protected]_cache(maxsize=4096)+def mx_host(domain):+ """'nxdomain' when the domain does not exist, 'none' when it has no MX record, else who runs its+ mail: 'google_workspace', 'microsoft_365' or 'other'; '' when the lookup failed. Like scrape.py."""+ try:+ with _OPENER.open(urllib.request.Request(DOH + urllib.parse.urlencode({"name": domain, "type": "MX"}),+ headers={"User-Agent": UA}), timeout=8) as r:+ ans = json.load(r)+ except Exception: # noqa: BLE001 - no answer is not a verdict+ return ""+ if ans.get("Status") == 3:+ return "nxdomain"+ mx = " ".join(a.get("data", "") for a in ans.get("Answer") or [] if a.get("type") == 15).lower()+ if not mx:+ return "none" if ans.get("Status") == 0 else ""+ if "google.com" in mx or "googlemail.com" in mx:+ return "google_workspace"+ return "microsoft_365" if "outlook.com" in mx else "other"++def find_emails(text, site=""):"""Up to 3 addresses from page text, placeholders and image names dropped, the site's own first."""found = []@@ -472,13 +773,29 @@ def find_emails(text, site=""):return sorted(found, key=lambda e: not own(e))[:3]-def add_emails(rows, seconds, get=None, stop=lambda: False):- """Fill `emails` (joined by ", ", like the old engine) from each row's website homepage, for at- most `seconds` or until stop() is true; the number of rows that got one."""+def add_site_data(rows, seconds, email=True, socials=False, verify=False, get=None, stop=lambda: False):+ """One homepage fetch per row with a website, SITE_WORKERS at a time, for at most `seconds` or+ until stop() is true: `emails` (joined by ", ", like the old engine), the social columns, and+ with verify an MX check of each address's domain (dns.google) that drops domains that do not+ exist and fills `email_host`. The number of rows that got something."""get = get or fetch- todo = [r for r in rows if r.get("website") and not r.get("emails")]++ def work(r):+ page, out = get(r["website"]), {}+ if email and not r.get("emails"):+ found = find_emails(page, site_host(r["website"]))+ if verify:+ hosts = [(e, mx_host(e.rpartition("@")[2])) for e in found]+ found = [e for e, h in hosts if h != "nxdomain"]+ out["email_host"] = next((h for e, h in hosts if h != "nxdomain"), "")+ out["emails"] = ", ".join(found)+ if socials:+ out.update(find_socials(page))+ return out++ todo = [r for r in rows if r.get("website")]pool = concurrent.futures.ThreadPoolExecutor(SITE_WORKERS)- futs = {pool.submit(lambda r=r: find_emails(get(r["website"]), site_host(r["website"]))): r for r in todo}+ futs = {pool.submit(work, r): r for r in todo}end, pending = time.monotonic() + seconds, set(futs)while pending and not stop() and time.monotonic() < end:_, pending = concurrent.futures.wait(pending, timeout=min(1, max(0, end - time.monotonic())))@@ -486,7 +803,8 @@ def add_emails(rows, seconds, get=None, stop=lambda: False):pool.shutdown(wait=False, cancel_futures=True)n = 0for f in done:- if not f.exception() and f.result():- futs[f]["emails"] = ", ".join(f.result())- n += 1+ if not f.exception():+ got = {k: v for k, v in f.result().items() if v}+ futs[f].update(got)+ n += bool(got)return ndiff --git a/engine/maps.py b/engine/maps.pyindex 817574e..9d29bdc 100644--- a/engine/maps.py+++ b/engine/maps.py@@ -1,7 +1,7 @@"""Google Maps search in a tab: open the search, pass the consent page, scroll the result feed,and collect the places from the result batches the page itself downloads. Standard library only."""-import queue, re, time, urllib.parse+import json, queue, re, time, urllib.parseimport detailsfrom cdp import CDPError@@ -9,7 +9,14 @@ from cdp import CDPError# Google's consent interstitial (EU and some other regions) is skipped when these cookies exist.CONSENT_COOKIES = {"CONSENT": "YES+cb.20210720-07-p0.en+FX+410","SOCS": "CAISHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmVuIAEaBgiAo_CmBg"}-END_MARKERS = ("You've reached the end of the list", "reached the end")+# The feed's last line once the list ends. The element's class has been HlvSq for years; the+# texts back it up in case Google renames the class (en, fr and es checked live on 2026-10-31).+END_CLASS = "HlvSq"+END_MARKERS = ("You've reached the end of the list", "reached the end",+ "Vous êtes arrivé à la fin de la liste", "Has llegado al final de la lista",+ "Du hast das Ende der Liste erreicht", "Hai raggiunto la fine dell'elenco",+ "Você chegou ao fim da lista", "Chegou ao fim da lista", "Je hebt het einde van de lijst bereikt",+ "Listenin sonuna ulaştınız", "Dotarłeś do końca listy", "Du har nått slutet av listan")BATCH_URL = re.compile(r"^https://www\.google\.[a-z.]+/search\?.*tbm=map")_CID = re.compile(r"!1s(0x[0-9a-f]+:0x[0-9a-f]+)")_LAT_LON = re.compile(r"!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)")@@ -29,17 +36,41 @@ JS_SCROLL = """(() => {f.scrollTo(0, f.scrollHeight);return f.scrollHeight;})()"""-JS_END = "(() => { const f = document.querySelector('div[role=\"feed\"]'); return f ? f.innerText : ''; })()"+JS_END = f"""(() => {{+ const f = document.querySelector('div[role="feed"]');+ return f ? {{cls: !!f.querySelector('.{END_CLASS}'), text: f.innerText}} : null;+}})()"""+# Google's consent page in any language: its "Reject all" form posts set_eom=true to+# consent.google.com/save (the "Accept all" form posts false). The button texts are the fallback.+# Not seen live from the kit's test machines (Google skips it outside the EEA, and the cookies+# above usually skip it inside), so both ways stay.+CONSENT_WORDS = ("reject all", "accept all", "alle ablehnen", "alle akzeptieren", "tout refuser", "tout accepter",+ "rechazar todo", "aceptar todo", "rifiuta tutto", "accetta tutto", "rejeitar tudo", "aceitar tudo",+ "alles afwijzen", "alles accepteren", "tümünü reddet", "tümünü kabul et", "odrzuć wszystko",+ "zaakceptuj wszystko", "avvisa alla", "godkänn alla", "afvis alle", "accepter alle")JS_CONSENT = """(() => {- const b = [...document.querySelectorAll('button')].find(b => /accept all|reject all|alle akzeptieren|alle ablehnen/i.test(b.innerText));+ const forms = [...document.querySelectorAll('form[action*="consent"]')];+ const reject = forms.find(f => f.querySelector('input[name="set_eom"][value="true"]'));+ let b = reject && reject.querySelector('button, input[type="submit"]');+ const words = %s;+ if (!b) b = [...document.querySelectorAll('button')].find(x => words.some(w => x.innerText.toLowerCase().includes(w)));if (b) { b.click(); return true; } return false;-})()"""+})()""" % json.dumps(CONSENT_WORDS, ensure_ascii=False)-def search_url(query, lat=None, lon=None, zoom=15, lang="en"):+def search_url(query, lat=None, lon=None, zoom=15, lang="en", country=""):+ """The Maps search URL: hl is the interface language (category names, opening-hours text), gl+ the country Google ranks and formats results for (without it, the country of this IP)."""q = urllib.parse.quote(query, safe="")at = f"/@{lat},{lon},{zoom}z" if lat is not None and lon is not None else ""- return f"https://www.google.com/maps/search/{q}{at}?hl={lang}"+ return f"https://www.google.com/maps/search/{q}{at}?hl={lang}" + (f"&gl={country}" if country else "")+++def ended(feed):+ """True when the feed shows its end-of-list line (JS_END's answer)."""+ if not isinstance(feed, dict):+ return False+ return bool(feed.get("cls")) or any(m in (feed.get("text") or "") for m in END_MARKERS)def parse_link(href, label=""):@@ -65,6 +96,7 @@ class Batches:self.q = page.cdp.subscribe("Network.responseReceived", page.sid)self.pending = {} # requestId -> tries, for batches seen but not yet readableself.count = 0+ self.limited = 0 # batch requests Google answered with 429 Too Many Requestsdef drain(self):"""New places from batches that finished since the last call."""@@ -76,7 +108,11 @@ class Batches:if not isinstance(p, dict): # the connection died (cdp puts a marker on every queue)raise CDPError(f"connection lost: {self.page.cdp.closed}")r = p.get("response", {})- if BATCH_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):+ if not BATCH_URL.match(r.get("url", "")):+ continue+ if r.get("status") == 429:+ self.limited += 1+ elif "json" in (r.get("mimeType") or ""):self.pending[p["requestId"]] = 0rows = []for rid in list(self.pending):@@ -98,12 +134,13 @@ class Batches:self.page.cdp.unsubscribe("Network.responseReceived", self.q, self.page.sid)-def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):+def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False, on_batch=None):"""Open `url`, scroll the result feed, and return (places, stop reason, stats).`depth` is the number of result batches to read (about 20 places each, like the old engine'sdepth); the list may end sooner. Places are de-duplicated on data_id. `stop()` returning True- ends the scroll early (a deleted job). Reason "blocked" means Google answered with its- unusual-traffic page."""+ ends the scroll early (a deleted job). on_batch(new places, batches read) is called as the+ batches arrive, so a caller can save them before the keyword ends. Reason "blocked" means+ Google answered with its unusual-traffic page, or 429 to the result batches."""for k, v in CONSENT_COOKIES.items():page.set_cookie(k, v, ".google.com")batches = Batches(page)@@ -121,13 +158,15 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):return [], "no-feed", {"batches": 0, "dom_links": 0}def take():- new = 0+ new = []for row in batches.drain():key = row.get("data_id") or row.get("place_id") or row["title"]if key not in seen:seen[key] = row- new += 1- return new+ new.append(row)+ if new and on_batch:+ on_batch(new, batches.count)+ return len(new)try: # a Chrome error mid-scroll keeps the places already readend = time.time() + 10 # the first batch arrives with the page@@ -136,9 +175,12 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):time.sleep(0.4)log(f" batch {batches.count}: {len(seen)} places")while batches.count < depth and time.time() - started < max_seconds and not stop():- if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):+ if ended(page.eval(JS_END)):reason = "end-of-list"break+ if batches.limited or "/sorry/" in (page.url() or ""):+ reason = "blocked"+ breakbefore = batches.countpage.eval(JS_SCROLL)end = time.time() + 10@@ -149,6 +191,9 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):page.eval(JS_SCROLL) # the feed renders in small steps; keep nudgingif batches.count == before:stale += 1+ if batches.limited: # Google refused the next batch: slow down, do not keep asking+ reason = "blocked"+ breakif stale >= 2:reason = "no-growth"break@@ -160,6 +205,8 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):reason = "stopped"elif time.time() - started >= max_seconds:reason = "timeout"+ elif batches.count == 0 and batches.limited:+ reason = "blocked"take()links = page.eval(JS_LINKS) or []except CDPError as e:@@ -174,3 +221,38 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):return list(seen.values()), reason, {"batches": batches.count, "dom_links": len(links)}finally:batches.close()+++PLACE_URL = re.compile(r"^https://www\.google\.[a-z.]+/maps/preview/place\?")+++def place_panel(page, url, timeout=10):+ """Open one place's panel and return the body of the `/maps/preview/place` response it downloads+ (details.parse_place reads it); "blocked" for Google's unusual-traffic page, "" when none came."""+ q = page.cdp.subscribe("Network.responseReceived", page.sid)+ try:+ page.navigate(url)+ if "/sorry/" in (page.url() or ""):+ return "blocked"+ end, rid = time.time() + timeout, None+ while time.time() < end:+ while rid is None:+ try:+ p = q.get_nowait()+ except queue.Empty:+ break+ if not isinstance(p, dict):+ raise CDPError(f"connection lost: {page.cdp.closed}")+ r = p.get("response", {})+ if PLACE_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):+ rid = p["requestId"]+ if rid:+ try:+ return page.send("Network.getResponseBody", {"requestId": rid})["body"]+ except CDPError: # still downloading+ if page.cdp.closed:+ raise+ time.sleep(0.3)+ return ""+ finally:+ page.cdp.unsubscribe("Network.responseReceived", q, page.sid)diff --git a/engine/serve.py b/engine/serve.pyindex adfa0db..252e876 100644--- a/engine/serve.py+++ b/engine/serve.py@@ -6,14 +6,17 @@Normally started by `python3 scripts/engine.py start`, or by `docker compose up -d` (the Dockerfile). It speaks the API of the engine the kit usedbefore (POST/GET/DELETE /api/v1/jobs, /api/v1/jobs/{id}/download, the same job body and "Status"-values), so scripts/scrape.py, the skill and the slash commands work unchanged. It listens on+values), so scripts/scrape.py, the skill and the slash commands work unchanged; jobs also carry Error+and Progress, resume after a restart, and download as CSV or JSON. It listens on127.0.0.1 (and ::1) only, or on 0.0.0.0 inside the Docker image, where Docker publishes the port onthe host's 127.0.0.1 only. Either way it refuses a foreign Host (DNS rebinding, 421) or Origin (cross-siteposts, 403); with SCRAPER_API_KEY set it also wants that X-API-Key (401). No telemetry: it talks to-Google Maps, and for email: true jobs to the businesses' own websites, nothing else.+Google Maps, for email or socials jobs to the businesses' own websites, and for verify_emails jobs+to Google's public DNS (dns.google), nothing else.Standard library only."""-import argparse, hmac, json, os, re, secrets, signal, socket, socketserver, sys, threading, urllib.parse+import argparse, csv, hmac, html, io, json, os, re, secrets, signal, socket, socketserver, sys, threading+import urllib.parsefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServersys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))@@ -31,7 +34,13 @@ REPO = "https://github.com/Mahanaicoach/google-maps-scraper-kit"# The web UI's sponsor box: the browser reads this static file from GitHub (cached a day, no# tracking), so the sponsors it lists change without anyone updating their copy of the kit.SPONSORS = "https://raw.githubusercontent.com/Mahanaicoach/google-maps-scraper-kit/master/assets/sponsors/sponsors.json"-ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY")+# Keyword ideas for the web UI: Google Business Profile category names (4,053, US English) from+# lobstrio/google-business-categories, pinned to one commit. The browser loads it from GitHub, like+# the sponsor box; the kit ships no copy, because that repository has no licence file.+CATEGORIES = ("https://raw.githubusercontent.com/lobstrio/google-business-categories/"+ "4ee1c4788d07c812a32986177c2a51c28957062f/data/google-business-categories.txt")+CATEGORIES_REPO = "https://github.com/lobstrio/google-business-categories"+ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY", "SCRAPER_HEADLESS")MAX_BODY = 1 << 20LOCAL = r"(localhost|127\.0\.0\.1|\[::1\])"HOST_OK = re.compile(rf"^{LOCAL}(:\d{{1,5}})?$", re.I)@@ -49,7 +58,7 @@ def log(msg):def read_env():- """SCRAPER_PORT and SCRAPER_API_KEY: the environment wins, then the kit's .env (last line wins,+ """SCRAPER_PORT, SCRAPER_API_KEY and SCRAPER_HEADLESS: the environment wins, then the kit's .env (last line wins,as in docker compose and scripts/scrape.py)."""found = {}try:@@ -63,37 +72,81 @@ def read_env():return {k: os.environ.get(k, found.get(k, "")) for k in ENV_KEYS}+JOB_PROPS = {+ "name": {"type": "string", "description": "any label"},+ "keywords": {"type": "array", "items": {"type": "string"}, "description": "searches, run one after another"},+ "lang": {"type": "string", "description": "2-letter interface language, e.g. en (category names, hours text)"},+ "country": {"type": "string", "description": "2-letter country Google ranks and formats results for, e.g. us "+ "(default: the country of this machine's IP)"},+ "lat": {"type": "string"}, "lon": {"type": "string"},+ "max_time": {"type": "integer", "description": "seconds"},+ "depth": {"type": "integer", "description": "result batches per keyword, about 20 places each (default 1)"},+ "zoom": {"type": "integer", "description": "map zoom 1-21 (default 15)"},+ "email": {"type": "boolean", "description": "visit each website's homepage for emails"},+ "verify_emails": {"type": "boolean", "description": "with email: check each address's domain over DNS (dns.google), "+ "drop domains that do not exist, add email_host"},+ "socials": {"type": "boolean", "description": "visit each website's homepage for Instagram, Facebook, LinkedIn, "+ "TikTok, YouTube and X links"},+ "reviews_max": {"type": "integer", "description": "0-5 reviews per place from its panel, one page load per place "+ "(Google shows reviews only to a Chrome window: SCRAPER_HEADLESS=0)"},+ "proxies": {"type": "array", "items": {"type": "string"},+ "description": "http(s)://[user:pass@]host:port or socks5://host:port; one per keyword in turn, "+ "and the next one when Google refuses a keyword"},+ "radius": {"type": "integer", "description": "accepted, not used"},+ "fast_mode": {"type": "boolean", "description": "accepted, not used"},+}OPENAPI = {"openapi": "3.0.3","info": {"title": "Google Maps Scraper Kit engine", "version": VERSION,- "description": "Local only. Create a job, poll its Status until ok or failed, download the CSV. "- f"Source and docs: {REPO}"},+ "description": "Local only. Create a job, poll its Status until ok or failed (Progress says how far "+ "it is), download the CSV or JSON. A job the engine was stopped in the middle of "+ f"resumes from its last finished keyword on the next start. Source and docs: {REPO}"},"paths": {"/api/v1/jobs": {"get": {"summary": "List jobs, newest first (null when there are none)"},"post": {"summary": "Create a job; returns 201 {\"id\": \"<uuid>\"}, or 422 {\"code\", \"message\"}","requestBody": {"content": {"application/json": {"schema": {"type": "object", "required": ["name", "keywords", "lang", "lat", "lon", "max_time"],- "properties": {- "name": {"type": "string", "description": "any label"},- "keywords": {"type": "array", "items": {"type": "string"}},- "lang": {"type": "string", "description": "2-letter interface language, e.g. en"},- "lat": {"type": "string"}, "lon": {"type": "string"},- "max_time": {"type": "integer", "description": "seconds"},- "depth": {"type": "integer", "description": "result batches per keyword, about 20 places each (default 1)"},- "zoom": {"type": "integer", "description": "map zoom 1-21 (default 15)"},- "email": {"type": "boolean", "description": "visit each website for emails"},- "proxies": {"type": "array", "items": {"type": "string"},- "description": "http(s):// or socks5://host:port; the first is used"},- "radius": {"type": "integer", "description": "accepted, not used"},- "fast_mode": {"type": "boolean", "description": "accepted, not used"}}}}}}}},+ "properties": JOB_PROPS}}}}}},"/api/v1/jobs/{id}": {- "get": {"summary": "One job: ID, Name, Date, Status (pending, working, ok, failed), Data, Error"},+ "get": {"summary": "One job: ID, Name, Date, Status (pending, working, ok, failed), Data (proxy "+ "passwords masked), Error, Progress (keywords_done, keywords_total, places, percent, "+ "stage: queued, search, reviews, websites or done)"},"delete": {"summary": "Delete a job and its results (stops it if it is running)"}},- "/api/v1/jobs/{id}/download": {"get": {"summary": "The job's CSV (the rows so far while it works)"}},+ "/api/v1/jobs/{id}/download": {"get": {"summary": "The job's CSV, the rows so far while it works; "+ "?format=json for a JSON array of the same rows"}},+ "/api/openapi.json": {"get": {"summary": "This description as OpenAPI JSON (the page at /api/docs shows it)"}},},}++def docs_page(nonce):+ """/api/docs: OPENAPI as a plain HTML page (no scripts, nothing loaded from elsewhere)."""+ e = html.escape+ out = [f"<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\">"+ "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>Engine API</title>"+ f"<style nonce=\"{nonce}\">:root {{ color-scheme: light dark; }} body {{ margin: 0 auto; max-width: 960px; "+ "padding: 16px; font: 15px/1.45 system-ui, -apple-system, Segoe UI, sans-serif; }} code {{ font-size: .95em; }} "+ "table {{ width: 100%; border-collapse: collapse; }} th, td {{ text-align: left; vertical-align: top; "+ "padding: 4px 6px; border-bottom: 1px solid #8884; }} .m {{ font-weight: 600; }}</style></head><body>",+ f"<h1>{e(OPENAPI['info']['title'])} {e(VERSION)}</h1><p>{e(OPENAPI['info']['description'])}</p>",+ "<p><a href=\"/\">Web UI</a> · <a href=\"/api/openapi.json\">OpenAPI JSON</a> · "+ "With SCRAPER_API_KEY set, send it as an <code>X-API-Key</code> header.</p>"]+ for path, ops in OPENAPI["paths"].items():+ for method, op in ops.items():+ out.append(f"<h2><span class=\"m\">{e(method.upper())}</span> <code>{e(path)}</code></h2><p>{e(op['summary'])}</p>")+ schema = op.get("requestBody", {}).get("content", {}).get("application/json", {}).get("schema")+ if schema:+ out.append("<table><tr><th>Field</th><th>Type</th><th>Required</th><th>Meaning</th></tr>")+ for name, prop in schema["properties"].items():+ out.append(f"<tr><td><code>{e(name)}</code></td><td>{e(prop['type'])}</td>"+ f"<td>{'yes' if name in schema['required'] else ''}</td>"+ f"<td>{e(prop.get('description', ''))}</td></tr>")+ out.append("</table>")+ out.append("</body></html>")+ return "".join(out)++PAGE = """<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Google Maps Scraper Kit</title>@@ -114,15 +167,21 @@ td.err {{ color: var(--muted); font-size: .85em; }} .row {{ display: flex; gap:</style></head><body><h1>Google Maps Scraper Kit</h1><p class="muted">Local engine {version}. <a href="{repo}" rel="noopener">GitHub</a> ·-<a href="/api/docs">API</a> · Business listings only. Results stay on this machine.</p>+<a href="/api/docs">API</a> · Business listings only. Results stay on this machine.+A job the engine was stopped in the middle of carries on when it starts again.</p><p class="muted" id="sponsors"><a href="https://github.com/sponsors/Mahanaicoach" rel="noopener">Become a sponsor</a></p><form id="f"><label class="wide">Keywords, one per line<textarea id="keywords" rows="3" requiredplaceholder="coffee shops in Austin TX"></textarea></label>+ <label class="wide">Add a business type<input id="kwpick" list="kwlist" autocomplete="off"+ placeholder="start typing: dentist, roofing contractor…"><datalist id="kwlist"></datalist>+ <span class="muted">Google's business categories, list by <a href="{categories_repo}" rel="noopener">lobstr.io</a>.+ Pick one to add it as a keyword line.</span></label><label>Latitude<input id="lat" required placeholder="30.2672" inputmode="decimal"></label><label>Longitude<input id="lon" required placeholder="-97.7431" inputmode="decimal"></label><label>Depth (about 20 places each)<input id="depth" type="number" min="1" max="1000" value="5"></label><label>Language<input id="lang" value="en" maxlength="2" pattern="[A-Za-z]{{2}}"></label>+ <label>Country (optional)<input id="country" maxlength="2" pattern="[A-Za-z]{{2}}" placeholder="us"></label><label>Time limit (seconds)<input id="max_time" type="number" min="1" max="86400" value="600"></label><label><span>Emails</span><span><input type="checkbox" id="email" checked> visit each website</span></label><div class="wide row"><button class="primary">Start scraping</button>@@ -147,11 +206,19 @@ function api(method, path, body) {{function say(t) {{ $('msg').textContent = t; }}function cell(tr, text, cls) {{ const td = document.createElement('td'); td.textContent = text; if (cls) td.className = cls; tr.appendChild(td); return td; }}function button(td, label, fn) {{ const b = document.createElement('button'); b.type = 'button'; b.textContent = label; b.addEventListener('click', fn); td.appendChild(b); }}-async function download(j) {{- const r = await api('GET', '/api/v1/jobs/' + j.ID + '/download');+async function download(j, fmt) {{+ const r = await api('GET', '/api/v1/jobs/' + j.ID + '/download' + (fmt === 'json' ? '?format=json' : ''));if (!r.ok) return say('No results yet for this job.');const a = document.createElement('a'); a.href = URL.createObjectURL(await r.blob());- a.download = 'results-' + j.ID.slice(0, 8) + '.csv'; document.body.appendChild(a); a.click(); a.remove();+ a.download = 'results-' + j.ID.slice(0, 8) + '.' + fmt; document.body.appendChild(a); a.click(); a.remove();+}}+function progress(j) {{+ const p = j.Progress || {{}};+ if (j.Status === 'pending') return p.keywords_done ? 'pending (resumes at keyword ' + (p.keywords_done + 1) + ')' : 'pending';+ if (j.Status !== 'working') return j.Status + (p.places ? ' · ' + p.places + ' places' : '');+ const stage = {{search: 'keyword ' + Math.min(p.keywords_done + 1, p.keywords_total) + '/' + p.keywords_total,+ reviews: 'reading reviews', websites: 'checking websites'}}[p.stage] || p.stage || '';+ return 'working · ' + stage + ' · ' + (p.places || 0) + ' places · ' + (p.percent || 0) + '%';}}async function remove(j) {{if (!confirm('Delete this job and its results?')) return;@@ -164,9 +231,9 @@ async function load() {{for (const j of list) {{const tr = document.createElement('tr');const name = cell(tr, j.Name); if (j.Error) {{ const s = document.createElement('div'); s.className = 'muted'; s.textContent = j.Error; name.appendChild(s); }}- cell(tr, j.Date.replace('T', ' ').replace('+00:00', '')); cell(tr, j.Status);+ cell(tr, j.Date.replace('T', ' ').replace('+00:00', '')); cell(tr, progress(j));const act = cell(tr, ''); act.className = 'row';- if (j.Status !== 'pending') button(act, 'Download', () => download(j));+ if (j.Status !== 'pending' || (j.Progress || {{}}).places) {{ button(act, 'CSV', () => download(j, 'csv')); button(act, 'JSON', () => download(j, 'json')); }}button(act, 'Delete', () => remove(j));tb.appendChild(tr);}}@@ -177,6 +244,7 @@ $('f').addEventListener('submit', async ev => {{ev.preventDefault();const keywords = $('keywords').value.split('\\n').map(s => s.trim()).filter(Boolean);const body = {{name: keywords[0] || 'job', keywords, lang: $('lang').value.trim().toLowerCase(),+ country: $('country').value.trim().toLowerCase(),lat: $('lat').value.trim(), lon: $('lon').value.trim(), depth: +$('depth').value,max_time: +$('max_time').value, email: $('email').checked, zoom: 15}};try {{@@ -186,6 +254,29 @@ $('f').addEventListener('submit', async ev => {{}} catch (e) {{ say(e.message); }}}});load(); setInterval(load, 3000);+$('kwpick').addEventListener('change', () => {{ // a picked business type becomes a keyword line+ const v = $('kwpick').value.trim(); if (!v) return;+ const k = $('keywords'); k.value = (k.value.trim() ? k.value.trim() + '\\n' : '') + v; $('kwpick').value = '';+}});+(function () {{ // keyword ideas: Google's business category names, cached for 30 days, as text only+ const key = 'gmskit-categories', month = 30 * 86400000;+ function fill(names) {{+ const list = $('kwlist'); list.replaceChildren();+ for (const n of names) {{ const o = document.createElement('option'); o.value = n; list.appendChild(o); }}+ }}+ function clean(text) {{+ return text.split('\\n').map(s => s.trim()).filter(s => s && s.length <= 80).slice(0, 5000);+ }}+ try {{+ const c = JSON.parse(localStorage.getItem(key) || 'null');+ if (c && Date.now() - c.t < month && Array.isArray(c.names)) return fill(c.names);+ }} catch (e) {{}}+ fetch('{categories}', {{credentials: 'omit'}}).then(r => r.ok ? r.text() : '').then(t => {{+ const names = clean(t); if (!names.length) return;+ fill(names);+ try {{ localStorage.setItem(key, JSON.stringify({{t: Date.now(), names}})); }} catch (e) {{}}+ }}).catch(() => {{}});+}})();(function () {{ // sponsor box: the "ui": true entries of sponsors.json, as text, https links onlyconst box = $('sponsors'), key = 'gmskit-sponsors', day = 86400000;function render(list) {{@@ -258,7 +349,7 @@ class Handler(BaseHTTPRequestHandler):self.fail(403, "requests from other websites are refused")return Falsekey, path = self.server.api_key, self.path.split("?")[0]- if key and path not in ("/", "/api/docs") and not hmac.compare_digest(+ if key and path not in ("/", "/api/docs", "/api/openapi.json") and not hmac.compare_digest(self.headers.get("X-API-Key", "").encode(), key.encode()):self.fail(401, "missing or wrong X-API-Key")return False@@ -283,10 +374,17 @@ class Handler(BaseHTTPRequestHandler):self.fail(400, "the body is not valid JSON")return _REPLIED- def download(self, jid):+ def download(self, jid, query=""):data = self.server.store.read_csv(jid) if JOB_ID.match(jid or "") else Noneif data is None:return self.fail(404, "no results for this job (yet)")+ fmt = urllib.parse.parse_qs(query).get("format", ["csv"])[0]+ if fmt == "json": # the same rows, every value a string, as in the CSV+ rows = list(csv.DictReader(io.StringIO(data.decode("utf-8", "replace"))))+ return self.reply(200, json.dumps(rows, ensure_ascii=False).encode(), "application/json; charset=utf-8",+ [("Content-Disposition", f'attachment; filename="{jid}.json"')])+ if fmt != "csv":+ return self.fail(400, "format must be csv or json")self.reply(200, data, "text/csv; charset=utf-8",[("Content-Disposition", f'attachment; filename="{jid}.csv"')])@@ -295,26 +393,32 @@ class Handler(BaseHTTPRequestHandler):returnpath, _, query = self.path.partition("?")store = self.server.store- if path == "/api/v1/jobs":- return self.reply(200, store.all() or None) # null when empty, as before: the docs' health check+ if path == "/api/v1/jobs": # null when empty, as before: the docs' health check+ return self.reply(200, [jobs.public(j) for j in store.all()] or None)m = JOB_PATH.match(path)if m:if m.group(2):- return self.download(m.group(1))+ return self.download(m.group(1), query)job = store.get(m.group(1))- return self.reply(200, job) if job else self.fail(404, "job not found")+ return self.reply(200, jobs.public(job)) if job else self.fail(404, "job not found")if path == "/download": # the old web UI's download link- return self.download(urllib.parse.parse_qs(query).get("id", [""])[0])+ return self.download(urllib.parse.parse_qs(query).get("id", [""])[0], query)+ if path == "/api/docs":+ nonce = secrets.token_urlsafe(16)+ csp = f"default-src 'none'; style-src 'nonce-{nonce}'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"+ return self.reply(200, docs_page(nonce).encode(), "text/html; charset=utf-8",+ [("Content-Security-Policy", csp)])+ if path == "/api/openapi.json":+ return self.reply(200, OPENAPI)if path == "/":nonce = secrets.token_urlsafe(16)csp = (f"default-src 'none'; script-src 'nonce-{nonce}'; style-src 'nonce-{nonce}'; ""connect-src 'self' https://raw.githubusercontent.com; img-src 'self' data:; ""form-action 'self'; base-uri 'none'; ""frame-ancestors 'none'")- page = PAGE.format(nonce=nonce, version=VERSION, repo=REPO, sponsors=SPONSORS).encode()+ page = PAGE.format(nonce=nonce, version=VERSION, repo=REPO, sponsors=SPONSORS, categories=CATEGORIES,+ categories_repo=CATEGORIES_REPO).encode()return self.reply(200, page, "text/html; charset=utf-8", [("Content-Security-Policy", csp)])- if path == "/api/docs":- return self.reply(200, OPENAPI)self.fail(404, "not found")def do_POST(self):@@ -414,9 +518,14 @@ def main():browser.make_home()browser.sweep_profiles()store = jobs.Store(a.data)- if n := store.recover():- log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")- worker = jobs.Worker(store, log)+ resumed, failed = store.recover()+ if resumed:+ log(f"{resumed} job(s) were still working when the engine last stopped: they resume from their last "+ "finished keyword")+ if failed:+ log(f"{failed} job(s) stopped mid-run {jobs.MAX_RESUMES + 1} times: marked failed, rows kept")+ headless = env["SCRAPER_HEADLESS"].strip().lower() not in ("0", "false", "no", "off")+ worker = jobs.Worker(store, log, headless=headless)try:servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"], a.host)except OSError as e:@@ -425,7 +534,7 @@ def main():chrome = browser.find_chrome()where = f"http://localhost:{port}" if a.host == "127.0.0.1" else \f"{a.host}:{port} in this container, published on the host's 127.0.0.1"- log(f"engine {VERSION} on {where} (browser: {chrome or 'none found'}"+ log(f"engine {VERSION} on {where} (browser: {chrome or 'none found'}{'' if headless else ', as a window'}"f"{', API key required' if env['SCRAPER_API_KEY'] else ''})")signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))for s in servers[1:]:@@ -435,7 +544,7 @@ def main():except KeyboardInterrupt:passfinally:- worker.shutdown()+ worker.shutdown(wait=8) # the job in progress saves its place and resumes on the next startlog("engine stopped")diff --git a/engine/test_engine.py b/engine/test_engine.pyindex 5256053..1f3f32a 100644--- a/engine/test_engine.py+++ b/engine/test_engine.py@@ -6,12 +6,13 @@ No network, no Chrome, no Docker.python3 engine/test_engine.py"""-import csv, hashlib, http.client, importlib.util, io, json, os, re, shutil, socket, subprocess, sys, tempfile+import csv, hashlib, http.client, importlib.util, io, json, os, queue, re, shutil, socket, sqlite3, subprocess, sys+import tempfileimport threading, time, unittest, unittest.mock, zipfileHERE = os.path.dirname(os.path.abspath(__file__))sys.path.insert(0, HERE)-import browser, jobs, maps, serve # noqa: E402+import browser, details, jobs, maps, serve # noqa: E402from cdp import CDPError # noqa: E402GOOD = {"name": "t", "keywords": ["cafes in Austin"], "lang": "en", "lat": "30.2", "lon": "-97.7", "max_time": 60}@@ -32,9 +33,16 @@ class FakeBrowser:made = []def __init__(self, **kw):- self.kw, self.cdp, self.killed = kw, FakeCDP(), threading.Event()+ self.kw, self.cdp, self.killed, self.contexts = kw, FakeCDP(), threading.Event(), []FakeBrowser.made.append(self)+ def context(self, proxy=None):+ self.contexts.append(proxy)+ return f"ctx{len(self.contexts)}"++ def dispose(self, ctx):+ pass+def kill(self):self.cdp.closed = OSError("killed")self.killed.set()@@ -44,24 +52,42 @@ class FakeBrowser:class FakePage:- def __init__(self, cdp):+ def __init__(self, cdp, context=None):passdef close(self):pass-RESULTS = {} # keyword -> (places, reason) for the fake collect+class FakeAuth:+ """Stands in for browser.ProxyAuth: records which tabs got credentials."""+ logins = []+ refused = False++ def __init__(self, page, user, password):+ FakeAuth.logins.append((user, password))++ def close(self):+ pass-def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False):+RESULTS = {} # keyword -> (places, reason) for the fake collect; a list of reasons answers one per search+SEARCHED = [] # the keywords the fake collect was asked for, in order+++def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False, on_batch=None):for kw, (places, reason) in RESULTS.items():if maps.search_url(kw).split("/@")[0].split("?")[0] in url:+ SEARCHED.append(kw)+ if isinstance(reason, list):+ reason = reason.pop(0) if len(reason) > 1 else reason[0]if reason == "hang": # Chrome stops answering until the watchdog kills itb = FakeBrowser.made[-1]b.killed.wait(10)raise CDPError("connection lost: killed")- if reason == "slow":+ if reason == "slow": # its places stream in, then it scrolls until stopped+ if on_batch and places:+ on_batch(places, 1)end = time.time() + 5while not stop() and time.time() < end:time.sleep(0.02)@@ -73,16 +99,18 @@ def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: Falseclass Fakes(unittest.TestCase):def setUp(self):self.dir = tempfile.mkdtemp()- self.saved = (browser.Browser, jobs.Page, maps.collect)- browser.Browser, jobs.Page, maps.collect = FakeBrowser, FakePage, fake_collect+ self.saved = (browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth)+ browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth = FakeBrowser, FakePage, fake_collect, FakeAuthFakeBrowser.made.clear()+ FakeAuth.logins.clear()RESULTS.clear()+ SEARCHED.clear()self.store = jobs.Store(os.path.join(self.dir, "data"))self.lines = []self.worker = jobs.Worker(self.store, self.lines.append)def tearDown(self):- browser.Browser, jobs.Page, maps.collect = self.saved+ browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth = self.savedself.worker.shutdown()if self.worker.is_alive():self.worker.join(5)@@ -123,7 +151,9 @@ class Validate(unittest.TestCase):self.assertIn("invalid depth", self.err(**dict(GOOD, depth="5")))self.assertIn("invalid max time", self.err(**dict(GOOD, max_time=-1)))self.assertIn("invalid keywords", self.err(**dict(GOOD, keywords=["ok", ""])))- self.assertIn("usernames and passwords", self.err(**dict(GOOD, proxies=["http://u:p@h:1"])))+ self.assertIn("SOCKS5", self.err(**dict(GOOD, proxies=["socks5://u:p@h:1"])))+ self.assertIn("invalid country", self.err(**dict(GOOD, country="usa")))+ self.assertIn("invalid reviews_max", self.err(**dict(GOOD, reviews_max=6)))self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["ftp://h:1"])))self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["http://h:99999"])))with self.assertRaisesRegex(ValueError, "JSON object"):@@ -133,6 +163,11 @@ class Validate(unittest.TestCase):name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])+ self.assertEqual((d["country"], d["reviews_max"], d["socials"], d["verify_emails"]), ("", 0, False, False))+ d = jobs.validate(dict(GOOD, country="DE", proxies=["http://u%40x:p%3A@h:1"]))[1]+ self.assertEqual((d["country"], jobs.locale(d), d["proxies"]), ("de", "en-DE", ["http://u%40x:p%3A@h:1"]))+ self.assertEqual(jobs.mask("http://u%40x:p%3A@h:1"), "http://u%40x:***@h:1")+ self.assertEqual(jobs.mask("socks5://h:1"), "socks5://h:1")self.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)@@ -162,15 +197,16 @@ class Store(Fakes):self.assertEqual([j["ID"] for j in self.store.all()], [b, a]) # newest firstself.assertEqual(self.store.claim()["ID"], a) # oldest pending firstself.assertEqual(self.store.get(a)["Status"], "working")- self.assertEqual(self.store.recover(), 1)- self.assertEqual(self.store.get(a)["Status"], "failed")- self.assertTrue(self.store.write_csv(b, []))+ self.assertEqual(self.store.recover(), (1, 0)) # resumed on the next start+ self.assertEqual((self.store.get(a)["Status"], self.store.get(a)["Progress"]["resumes"]), ("pending", 1))+ self.assertEqual(self.store.append(b, [], jobs.fields({})), len(",".join(jobs.COLUMNS + jobs.EXTRA)) + 2)self.assertTrue(self.store.read_csv(b).startswith(b"input_id,link,title"))self.assertTrue(self.store.delete(b))self.assertIsNone(self.store.read_csv(b))- self.assertFalse(self.store.write_csv(b, [])) # a deleted job gets no file back+ self.assertIsNone(self.store.append(b, [], jobs.fields({}))) # a deleted job gets no file back+ self.assertFalse(self.store.finish(b, [], jobs.fields({}), "ok", "", {}))self.assertFalse(self.store.delete(b))- self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error"})+ self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error", "Progress"})self.assertRegex(self.store.get(a)["Date"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00$")@@ -183,7 +219,8 @@ class Worker(Fakes):rows = self.rows(jid)self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2", "Biz 3"])self.assertEqual([r["input_id"] for r in rows], ["cafes", "cafes", "bakeries"])- self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "proxy": "http://10.0.0.1:3128"})+ self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "headless": True})+ self.assertEqual(FakeBrowser.made[0].contexts, ["http://10.0.0.1:3128"] * 2)self.assertFalse(any(re.search(r"Biz \d", l) for l in self.lines)) # counts only in the logdef test_no_results_is_ok_with_header(self):@@ -241,7 +278,7 @@ class Worker(Fakes):def test_email_pass_stops_when_asked(self):rows = [{"website": f"https://s{i}.com", "emails": ""} for i in range(3)]t = time.monotonic()- jobs.add_emails(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)+ jobs.add_site_data(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)self.assertLess(time.monotonic() - t, 3)def test_watchdog_kills_a_hung_chrome(self):@@ -282,6 +319,7 @@ class Worker(Fakes):jobs.fetch = savedself.assertEqual(job["Status"], "ok")self.assertEqual([r["emails"] for r in self.rows(jid)], ["[email protected], [email protected]", "", ""])+ self.assertNotIn("instagram", self.rows(jid)[0]) # columns only for the passes a job asked forclass Emails(unittest.TestCase):@@ -354,7 +392,7 @@ class Api(Server):job = json.loads(self.call("GET", f"/api/v1/jobs/{jid}")[1])self.assertEqual((job["ID"], job["Name"], job["Status"], job["Data"]["keywords"]), (jid, "t", "pending", ["cafes in Austin"]))self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download")[0], 404)- self.store.write_csv(jid, [jobs.to_row(place(1), "k")])+ self.store.append(jid, [jobs.to_row(place(1), "k")], jobs.fields({}))code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download")self.assertEqual((code, r.getheader("Content-Type")), (200, "text/csv; charset=utf-8"))self.assertEqual(self.call("GET", f"/download?id={jid}")[1], body)@@ -396,7 +434,7 @@ class Api(Server):self.assertIn("connect-src 'self' https://raw.githubusercontent.com;", csp) # sponsors.json, nothing elseself.assertIn("assets/sponsors/sponsors.json", body.decode())self.assertNotRegex(body.decode(), r"\{[a-z_]+\}") # every placeholder filled- code, body, _ = self.call("GET", "/api/docs")+ code, body, _ = self.call("GET", "/api/openapi.json")self.assertEqual(json.loads(body)["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]["schema"]["required"], ["name", "keywords", "lang", "lat", "lon", "max_time"])@@ -587,9 +625,10 @@ class LaunchChrome(unittest.TestCase):os.mkdir(profile)b = Nonewith self.assertRaisesRegex(browser.BrowserError, "DevTools"):- b = browser.Browser(path=self.exe, profile=profile, proxy="socks5h://10.0.0.1:1080", timeout=5)+ b = browser.Browser(path=self.exe, profile=profile, headless=False, timeout=5)self.assertIsNone(b)- self.assertIn("--proxy-server=socks5://10.0.0.1:1080", self.launches()[0])+ self.assertIn("--window-position=-3000,-3000", self.launches()[0]) # a window, out of sight+ self.assertNotIn("--headless", self.launches()[0])out = subprocess.run(["ps", "-axo", "command"], capture_output=True, text=True).stdoutself.assertNotIn(self.exe, out) # the fake Chrome was stopped@@ -656,5 +695,345 @@ class EngineCli(unittest.TestCase):p.wait()+def wait_until(cond, seconds=5):+ end = time.time() + seconds+ while not cond() and time.time() < end:+ time.sleep(0.02)+ return cond()+++class Resume(Fakes):+ """P3: rows stream into the CSV, Progress says how far a job is, and a job the engine was stopped+ in the middle of carries on from its last finished keyword."""++ def start(self, **body):+ name, data = jobs.validate(dict(GOOD, **body))+ jid = self.store.create(name, data)+ t = threading.Thread(target=self.worker.run_job, args=(self.store.claim(),))+ t.start()+ return jid, t++ def test_rows_stream_in_and_a_stopped_engine_resumes(self):+ RESULTS["cafes"] = ([place(1)], "end-of-list")+ RESULTS["bakeries"] = ([place(2), place(3)], "slow")+ RESULTS["florists"] = ([place(4)], "end-of-list")+ jid, t = self.start(keywords=["cafes", "bakeries", "florists"])+ self.assertTrue(wait_until(lambda: self.store.get(jid)["Progress"].get("places") == 3))+ self.assertEqual(len(self.rows(jid)), 3) # bakeries' places are in before the keyword ends+ prog = jobs.public(self.store.get(jid))["Progress"]+ self.assertEqual((prog["keywords_done"], prog["keywords_total"], prog["places"], prog["stage"]), (1, 3, 3, "search"))+ self.assertTrue(33 < prog["percent"] <= 66)+ self.worker.shutdown(wait=5) # the engine stops mid-keyword+ t.join(5)+ self.assertEqual(self.store.get(jid)["Status"], "pending")+ RESULTS["bakeries"] = ([place(2), place(3)], "end-of-list")+ SEARCHED.clear()+ jobs.Worker(self.store, self.lines.append).run_job(self.store.claim()) # the next start+ job = self.store.get(jid)+ self.assertEqual((job["Status"], job["Error"]), ("ok", ""))+ self.assertEqual(SEARCHED, ["bakeries", "florists"]) # cafes was finished, bakeries was not+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1", "Biz 2", "Biz 3", "Biz 4"])+ self.assertEqual(jobs.public(job)["Progress"], {"keywords_done": 3, "keywords_total": 3, "places": 4,+ "percent": 100, "stage": "done"})++ def crashed(self, keywords, **progress):+ """A job an engine crashed in: one keyword finished, part of the next one appended, a torn line."""+ name, data = jobs.validate(dict(GOOD, keywords=keywords))+ jid = self.store.create(name, data)+ size = self.store.append(jid, [jobs.to_row(place(1), keywords[0])], jobs.fields(data))+ self.store.append(jid, [jobs.to_row(place(2), keywords[1])], jobs.fields(data))+ with open(self.store.csv_path(jid), "ab") as f:+ f.write(b'"cafes,Biz 9,"half a ro')+ self.store.claim()+ self.store.set_progress(jid, dict({"done": 1, "total": len(keywords), "bytes": size, "stage": "search"}, **progress))+ return jid, size++ def test_crash_cuts_the_unfinished_keyword_and_resumes(self):+ jid, size = self.crashed(["cafes", "bakeries"], elapsed=20)+ self.assertEqual(self.store.recover(), (1, 0))+ self.assertEqual(os.path.getsize(self.store.csv_path(jid)), size) # torn line and bakeries' row gone+ RESULTS["bakeries"] = ([place(2), place(3)], "end-of-list")+ self.worker.run_job(self.store.claim())+ self.assertEqual(SEARCHED, ["bakeries"])+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1", "Biz 2", "Biz 3"])+ self.assertTrue(any("resumed at keyword 2/2 with 1 places, 40 s left" in l for l in self.lines))++ def test_too_many_restarts_fail_as_before(self):+ jid, size = self.crashed(["cafes", "bakeries"], resumes=jobs.MAX_RESUMES)+ self.assertEqual(self.store.recover(), (0, 1))+ job = self.store.get(jid)+ self.assertEqual(job["Status"], "failed")+ self.assertIn("the engine stopped while this job ran (4 times); its rows so far are kept", job["Error"])+ self.assertEqual(os.path.getsize(self.store.csv_path(jid)), size)++ def test_no_time_left_ends_without_searching(self):+ jid, _ = self.crashed(["cafes", "bakeries"], elapsed=60)+ self.store.recover()+ self.worker.run_job(self.store.claim())+ job = self.store.get(jid)+ self.assertEqual((job["Status"], SEARCHED, FakeBrowser.made), ("ok", [], []))+ self.assertIn("stopped at max_time", job["Error"])+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1"])++ def test_old_store_gets_the_progress_column(self):+ folder = os.path.join(self.dir, "old")+ os.makedirs(folder)+ db = sqlite3.connect(os.path.join(folder, "jobs.db"))+ db.execute("CREATE TABLE jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, date TEXT NOT NULL, "+ "status TEXT NOT NULL, data TEXT NOT NULL, error TEXT NOT NULL DEFAULT '')")+ db.execute("INSERT INTO jobs VALUES ('x', 'n', 'd', 'ok', '{}', '')")+ db.commit()+ db.close()+ old = jobs.Store(folder)+ self.assertEqual(old.get("x")["Progress"], {})+ old.db.close()+++class Backoff(Fakes):+ def setUp(self):+ super().setUp()+ self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)+ jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01++ def tearDown(self):+ jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits+ super().tearDown()++ def test_retry_through_the_next_proxy(self):+ RESULTS["cafes in Austin"] = ([place(1)], ["blocked", "end-of-list"])+ jid, job = self.run_one(proxies=["http://a:1", "http://b:2"])+ self.assertEqual((job["Status"], job["Error"]), ("ok", ""))+ self.assertEqual(FakeBrowser.made[0].contexts, ["http://a:1", "http://b:2"])+ self.assertTrue(any("retry 1/2 in 0.01 s through the next proxy" in l for l in self.lines))++ def test_gives_up_after_bounded_retries(self):+ RESULTS["cafes in Austin"] = ([], ["blocked"])+ _, job = self.run_one()+ self.assertEqual(job["Status"], "failed")+ self.assertIn("unusual-traffic page", job["Error"]) # what scripts/scrape.py looks for+ self.assertIn("Waited and retried 2 time(s)", job["Error"])+ self.assertEqual(SEARCHED, ["cafes in Austin"] * 3)++ def test_a_failed_proxy_hands_the_keyword_to_the_next(self):+ calls = []+ def collect(page, url, **kw):+ calls.append(FakeBrowser.made[-1].contexts[-1])+ if len(calls) == 1:+ raise CDPError("navigate https://x: net::ERR_PROXY_CONNECTION_FAILED")+ return [place(1)], "end-of-list", {"batches": 1, "dom_links": 0}+ maps.collect = collect+ jid, job = self.run_one(proxies=["http://dead:1", "http://ok:2"])+ self.assertEqual((job["Status"], job["Error"], calls), ("ok", "", ["http://dead:1", "http://ok:2"]))+ self.assertEqual(len(self.rows(jid)), 1)++ def test_proxies_take_turns(self):+ for k in ("a", "b", "c"):+ RESULTS[k] = ([], "end-of-list")+ self.run_one(keywords=["a", "b", "c"], proxies=["http://u:pw-s3cret@p1:1", "http://p2:2"])+ self.assertEqual(FakeBrowser.made[0].contexts, ["http://u:pw-s3cret@p1:1", "http://p2:2", "http://u:pw-s3cret@p1:1"])+ self.assertFalse(any("s3cret" in l for l in self.lines))+ self.assertEqual(FakeAuth.logins, [("u", "pw-s3cret")] * 2) # only the tabs on the proxy with a login+++class ProxyLogin(unittest.TestCase):+ """browser.ProxyAuth against a fake DevTools connection: continue every paused request, answer+ the proxy's 407 once with the job's credentials, then give up (wrong credentials)."""++ def test_answers(self):+ class C:+ closed = None+ def __init__(self):+ self.sent = []+ def subscribe(self, method, sid=None, q=None):+ self.q = q or queue.Queue()+ return self.q+ def unsubscribe(self, *a):+ pass+ class P:+ sid = "s"+ def __init__(self):+ self.cdp = C()+ def send(self, method, params=None, timeout=None):+ self.cdp.sent.append((method, params))+ return {}+ page = P()+ auth = browser.ProxyAuth(page, "user", "pw")+ q = page.cdp.q+ q.put({"requestId": "1", "request": {"url": "https://www.google.com/"}})+ q.put({"requestId": "2", "authChallenge": {"source": "Proxy"}})+ q.put({"requestId": "2", "authChallenge": {"source": "Proxy"}})+ q.put({"requestId": "3", "authChallenge": {"source": "Server"}})+ self.assertTrue(wait_until(lambda: len(page.cdp.sent) == 5))+ auth.close()+ sent = page.cdp.sent+ self.assertEqual(sent[0], ("Fetch.enable", {"handleAuthRequests": True, "patterns": [{"urlPattern": "*"}]}))+ self.assertEqual(sent[1], ("Fetch.continueRequest", {"requestId": "1"}))+ self.assertEqual([m[1]["authChallengeResponse"]["response"] for m in sent[2:]],+ ["ProvideCredentials", "CancelAuth", "Default"])+ self.assertEqual(sent[2][1]["authChallengeResponse"]["username"], "user")+ self.assertTrue(auth.refused) # the job then says "proxy login refused", not Chrome's vague error+ self.assertIn("proxy login refused", jobs.LOGIN_REFUSED)+ self.assertTrue(jobs.PROXY_FAILED.search(jobs.LOGIN_REFUSED))+ self.assertFalse(jobs.PROXY_FAILED.search("navigate https://x/maps/search/tunnel%20car%20wash: net::ERR_TIMED_OUT"))++ def test_split(self):+ self.assertEqual(browser.split_proxy("http://u%40x:p%3A@h:3128"), ("http://h:3128", "u@x", "p:"))+ self.assertEqual(browser.split_proxy("socks5h://[::1]:1080"), ("socks5://[::1]:1080", "", ""))+++def entity(days=7, reviews=False):+ """A place entity shaped like Google's (made-up values) with the full view's fields."""+ e = [None] * 260+ e[11], e[10], e[78] = "Biz", "0x1:0x2", "ChIJx"+ e[4] = [None, None, None, ["https://search.google.com/local/reviews?placeid=ChIJx", "349 reviews"],+ None, None, None, 4.7, 349]+ names = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]+ week = [[names[n - 1], n, [2026, 11, n], [["7\u202fAM\u20139\u202fPM", [[7], [21]]]], 0, 1] for n in (6, 7, 1, 2, 3, 4, 5)]+ e[203] = [week[:days], [week[0], 0, 4, None, ["Open \u00b7 Closes 9\u202fPM"]]]+ if reviews:+ def review(stars, text):+ r = [None] * 4+ r[0] = "id"+ r[1] = [None, None, 1, 1, [None, None, ["https://example.com/u"], None, None, ["A Reviewer"]], None, "4 months ago"]+ r[2] = [[stars]] + [None] * 14 + [[[text]]]+ return [r]+ e[175] = [None] * 10+ e[175][3] = [6, 12, 6, 24, 301]+ e[175][9] = [[[review(5, "Great\u202fcoffee"), review(2, "Slow"), review(4, "")]]]+ return e+++class FullView(unittest.TestCase):+ """details.py fields that only Google's full view carries (a Chrome window), from made-up data."""++ def test_count_link_and_week(self):+ out = details.parse_entity(entity())+ self.assertEqual((out["review_count"], out["review_rating"]), (349, 4.7))+ self.assertEqual(out["reviews_link"], "https://search.google.com/local/reviews?placeid=ChIJx")+ week = json.loads(out["open_hours"])+ self.assertEqual(list(week)[:2], ["Monday", "Tuesday"])+ self.assertEqual(week["Sunday"], ["7 AM\u20139 PM"])+ self.assertEqual(out["open_state"], "Open \u00b7 Closes 9 PM" if False else out["open_state"])++ def test_limited_view_gives_no_week(self):+ self.assertNotIn("open_hours", details.parse_entity(entity(days=1))) # today only is not a week++ def test_place_panel_reviews_without_names(self):+ body = ")]}'\n" + json.dumps([None] * 6 + [entity(reviews=True)])+ out = details.parse_place(body, limit=2)+ self.assertEqual(json.loads(out["reviews_per_rating"]), {"1": 6, "2": 12, "3": 6, "4": 24, "5": 301})+ self.assertEqual(json.loads(out["user_reviews"]), [{"Rating": 5, "When": "4 months ago", "Description": "Great coffee"},+ {"Rating": 2, "When": "4 months ago", "Description": "Slow"}])+ self.assertNotIn("Reviewer", json.dumps(out))+ self.assertEqual(details.parse_place("not json"), {})+++class Locale(unittest.TestCase):+ def test_search_url_and_end_of_list(self):+ self.assertEqual(maps.search_url("a b", 1, 2, 15, "de", "at"), "https://www.google.com/maps/search/a%20b/@1,2,15z?hl=de&gl=at")+ self.assertTrue(maps.search_url("x").endswith("?hl=en"))+ self.assertTrue(maps.ended({"cls": True, "text": ""}))+ self.assertTrue(maps.ended({"cls": False, "text": "… Vous êtes arrivé à la fin de la liste."}))+ self.assertFalse(maps.ended({"cls": False, "text": "Biz 1"}))+ self.assertFalse(maps.ended(None))+ self.assertIn('input[name="set_eom"][value="true"]', maps.JS_CONSENT) # the reject form, any language+ self.assertIn("tout refuser", maps.JS_CONSENT)++ def test_429_marks_the_batches_limited(self):+ q = queue.Queue()+ class C:+ closed = None+ def subscribe(self, *a):+ return q+ class P:+ sid, cdp = "s", C()+ b = maps.Batches(P())+ q.put({"requestId": "1", "response": {"url": "https://www.google.com/search?tbm=map&pb=x", "status": 429,+ "mimeType": "text/html"}})+ self.assertEqual((b.drain(), b.limited, b.pending), ([], 1, {}))+++class Websites(unittest.TestCase):+ def test_socials_and_mx(self):+ rows = [{"website": "https://shop.com", "emails": ""}, {"website": "https://gone.com", "emails": ""}]+ pages = {"https://shop.com": '[email protected] <a href="https://www.instagram.com/shopco/">ig</a> '+ '<a href="https://facebook.com/sharer">x</a> <a href="https://facebook.com/shopco">fb</a>',+ "https://gone.com": "[email protected]"}+ hosts = {"shop.com": "google_workspace", "gone-forever.com": "nxdomain"}+ with unittest.mock.patch.object(jobs, "mx_host", side_effect=hosts.get):+ n = jobs.add_site_data(rows, 10, socials=True, verify=True, get=pages.get)+ self.assertEqual(n, 1)+ self.assertEqual((rows[0]["emails"], rows[0]["email_host"]), ("[email protected]", "google_workspace"))+ self.assertEqual((rows[0]["instagram"], rows[0]["facebook"]), ("https://www.instagram.com/shopco", "https://facebook.com/shopco"))+ self.assertEqual(rows[1]["emails"], "") # its only address is on a domain that does not exist++ def test_mx_answers(self):+ def doh(answer):+ r = unittest.mock.MagicMock()+ r.__enter__.return_value = io.BytesIO(json.dumps(answer).encode())+ return unittest.mock.patch.object(jobs._OPENER, "open", return_value=r)+ jobs.mx_host.cache_clear()+ with doh({"Status": 3}):+ self.assertEqual(jobs.mx_host("a.test"), "nxdomain")+ with doh({"Status": 0, "Answer": [{"type": 15, "data": "1 aspmx.l.google.com."}]}):+ self.assertEqual(jobs.mx_host("b.test"), "google_workspace")+ with doh({"Status": 0}):+ self.assertEqual(jobs.mx_host("c.test"), "none")+ jobs.mx_host.cache_clear()++ def test_same_social_patterns_as_scrape(self):+ sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))+ import scrape+ self.assertEqual({k: r.pattern for k, r in jobs.SOCIAL_RE.items()}, {k: r.pattern for k, r in scrape.SOCIAL_RE.items()})+ self.assertEqual(jobs.SOCIALS, scrape.SOCIAL_FIELDS)+ self.assertEqual(jobs.SKIP_HANDLE, scrape._SKIP_HANDLE)+++class ApiP3(Server):+ def test_progress_json_and_docs(self):+ code, body, _ = self.call("POST", "/api/v1/jobs", dict(GOOD, proxies=["http://u:pw-s3cret@h:1"]))+ jid = json.loads(body)["id"]+ code, body, _ = self.call("GET", f"/api/v1/jobs/{jid}")+ job = json.loads(body)+ self.assertEqual(job["Data"]["proxies"], ["http://u:***@h:1"])+ self.assertNotIn(b"s3cret", body + self.call("GET", "/api/v1/jobs")[1])+ self.assertEqual(job["Progress"], {"keywords_done": 0, "keywords_total": 1, "places": 0, "percent": 0,+ "stage": "queued"})+ self.store.append(jid, [jobs.to_row(place(1, title="Café"), "k")], jobs.fields({}))+ code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download?format=json")+ self.assertEqual((code, r.getheader("Content-Type")), (200, "application/json; charset=utf-8"))+ rows = json.loads(body)+ self.assertEqual((len(rows), rows[0]["title"], rows[0]["review_count"]), (1, "Café", "11"))+ self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download?format=xml")[0], 400)+ self.assertEqual(self.call("GET", f"/download?id={jid}&format=json")[1], body)+ code, body, r = self.call("GET", "/api/docs")+ page, csp = body.decode(), r.getheader("Content-Security-Policy")+ self.assertEqual((code, r.getheader("Content-Type")), (200, "text/html; charset=utf-8"))+ self.assertNotIn("<script", page)+ nonce = re.search(r"nonce-([^']+)'", csp).group(1)+ self.assertIn(f'<style nonce="{nonce}">', page)+ self.assertNotIn("script-src", csp)+ self.assertIn("<code>reviews_max</code>", page)+ spec = json.loads(self.call("GET", "/api/openapi.json")[1])+ self.assertEqual(spec["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]+ ["schema"]["required"], ["name", "keywords", "lang", "lat", "lon", "max_time"])+ ui = self.call("GET", "/")[1].decode()+ self.assertIn('id="country"', ui)+ self.assertIn('list="kwlist"', ui)+ self.assertIn(serve.CATEGORIES, ui) # loaded by the browser from GitHub, pinned to a commit+ self.assertRegex(serve.CATEGORIES, r"/[0-9a-f]{40}/")+ script = re.search(r"<script nonce=[^>]+>(.*)</script>", ui, re.S).group(1)+ self.assertNotIn("'\n'", script) # PAGE must say \\n: a bare newline in a JS string breaks the page+++class ApiKeyDocs(Server):+ key = "s3cret"++ def test_docs_need_no_key(self):+ self.assertEqual(self.call("GET", "/api/docs")[0], 200)+ self.assertEqual(self.call("GET", "/api/openapi.json")[0], 200)+ self.assertEqual(self.call("GET", "/api/v1/jobs/00000000-0000-0000-0000-000000000000/download")[0], 401)++if __name__ == "__main__":unittest.main(verbosity=1)diff --git a/engine/test_proxy.py b/engine/test_proxy.pynew file mode 100644index 0000000..e61a055--- /dev/null+++ b/engine/test_proxy.py@@ -0,0 +1,85 @@+#!/usr/bin/env python3+"""A local HTTP proxy that wants a username and password, for the live check of proxy logins+(.github/workflows/engine-live.yml). Not part of the engine. Standard library only.++ python3 engine/test_proxy.py 8191 user pass # then scrape with --proxy http://user:[email protected]:8191++It answers 407 until a request carries the right Basic credentials, tunnels HTTPS (CONNECT) after+that, and keeps the counts in engine/../.engine/test-proxy.json: {"tunnels": n, "refused": n}.+"""+import base64, json, os, select, socket, socketserver, sys, threading++COUNTS = {"tunnels": 0, "refused": 0}+LOCK = threading.Lock()+OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".engine", "test-proxy.json")+++def count(key):+ with LOCK:+ COUNTS[key] += 1+ os.makedirs(os.path.dirname(OUT), exist_ok=True)+ with open(OUT, "w") as f:+ json.dump(COUNTS, f)+++class Proxy(socketserver.BaseRequestHandler):+ def head(self):+ data = b""+ while b"\r\n\r\n" not in data:+ chunk = self.request.recv(4096)+ if not chunk:+ return None+ data += chunk+ return data.split(b"\r\n\r\n", 1)[0].decode("latin-1").split("\r\n")++ def handle(self):+ want = "Basic " + base64.b64encode(self.server.creds.encode()).decode()+ while True: # Chrome may send its credentials on the same connection+ lines = self.head()+ if not lines:+ return+ method, target = (lines[0].split(" ") + ["", ""])[:2]+ auth = [l.split(":", 1)[1].strip() for l in lines[1:] if l.lower().startswith("proxy-authorization:")]+ if auth != [want]:+ count("refused")+ self.request.sendall(b'HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic '+ b'realm="kit"\r\nContent-Length: 0\r\n\r\n')+ continue+ if method != "CONNECT":+ self.request.sendall(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")+ return+ host, _, port = target.rpartition(":")+ try:+ up = socket.create_connection((host.strip("[]"), int(port)), timeout=15)+ except (OSError, ValueError):+ self.request.sendall(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")+ return+ count("tunnels")+ self.request.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n")+ pair = [self.request, up]+ try:+ while True:+ ready, _, _ = select.select(pair, [], [], 60)+ if not ready:+ return+ for s in ready:+ data = s.recv(65536)+ if not data:+ return+ (up if s is self.request else self.request).sendall(data)+ except OSError:+ return+ finally:+ up.close()+++class Server(socketserver.ThreadingTCPServer):+ daemon_threads = allow_reuse_address = True+++if __name__ == "__main__":+ port, user, password = int(sys.argv[1]), sys.argv[2], sys.argv[3]+ srv = Server(("127.0.0.1", port), Proxy)+ srv.creds = f"{user}:{password}"+ print(f"test proxy on 127.0.0.1:{port}", flush=True)+ srv.serve_forever()diff --git a/scripts/scrape.py b/scripts/scrape.pyindex 9dc0c2e..944a001 100755--- a/scripts/scrape.py+++ b/scripts/scrape.py@@ -86,7 +86,7 @@ CHROME_START = ("No Chrome", "Chrome cannot start", "Chrome exited at once", "CoBLOCKED = "unusual-traffic page" # the engine's Error when Google refused this IP# Chrome's network errors for a proxy that failed. Matched as written: the engine's message also# holds the search URL, so a keyword such as "tunnel car wash" must not read as a proxy error.-PROXY_ERROR = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_")+PROXY_ERROR = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_|proxy login refused")def poll_attempts(max_time):@@ -232,8 +232,8 @@ def run_job(body):f"engine: {RESTART}")if PROXY_ERROR.search(reason):raise RuntimeError(f"\n✗ Job failed: the proxy did not work.{said}\n"- " Check its host and port; this engine takes no user:pass yet, so the "- "proxy must allow your IP address.")+ " Check its host, port, username and password (or that it allows your IP "+ "address).")if BLOCKED in reason:raise Blocked(f"\n✗ Job failed.{said}")if attempt == 2:@@ -243,20 +243,34 @@ def run_job(body):time.sleep(FAILED_RETRY_WAIT)-def _get(path):- """GET with three tries: a poll or a download must survive a passing 502 or timeout."""- for attempt in range(3):+def _get(path, tries=3):+ """GET with retries 3 s apart: a poll or a download must survive a passing 502 or timeout."""+ for attempt in range(tries):try:return req("GET", path)except urllib.error.HTTPError as e:- if e.code < 500 or attempt == 2:+ if e.code < 500 or attempt == tries - 1:raiseexcept OSError:- if attempt == 2:+ if attempt == tries - 1:raisetime.sleep(3)+def progress_text(job):+ """", keyword 2/3, 41 places, 50%" from the job's Progress (what this engine reports), or ""."""+ p = job.get("Progress")+ if not isinstance(p, dict) or not p.get("keywords_total"):+ return ""+ total = p["keywords_total"]+ where = {"reviews": "reading reviews", "websites": "checking websites"}.get(+ p.get("stage"), f"keyword {min(p.get('keywords_done', 0) + 1, total)}/{total}")+ return f", {where}, {p.get('places', 0)} places, {p.get('percent', 0)}%"+++ENGINE_RESTART_TRIES = 20 # ~1 min of polls: a restarted engine resumes the job, so wait for it++def _run_job_once(body):global CURRENT_JOBtry:@@ -274,8 +288,8 @@ def _run_job_once(body):result = _poll(body, job_id)except OSError as e: # the engine went away (restart, crash) or the job was deleted mid-pollCURRENT_JOB = None- raise RuntimeError(f"\n✗ Lost the engine while job {job_id} ran ({e}). If the engine comes back, "- f"it keeps the rows found so far: {BASE}/api/v1/jobs/{job_id}/download")+ raise RuntimeError(f"\n✗ Lost the engine while job {job_id} ran ({e}). When it starts again it resumes "+ f"the job from its last finished keyword; its rows: {BASE}/api/v1/jobs/{job_id}/download")except Exception:CURRENT_JOB = Noneraise@@ -290,16 +304,11 @@ def _poll(body, job_id):print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")status = Nonefor i in range(attempts):- _, raw = _get(f"/api/v1/jobs/{job_id}")+ _, raw = _get(f"/api/v1/jobs/{job_id}", ENGINE_RESTART_TRIES)job = json.loads(raw)# Error: why the job failed, or why an "ok" job stopped early (max_time, a Google block).status, error = job.get("Status"), str(job.get("Error") or "")- progress = ""- if status == "working": # the engine writes rows as it goes, so the download shows progress- try:- progress = f", {len(download_rows(job_id))} rows so far"- except Exception:- pass+ progress = progress_text(job) if status in ("working", "pending") else ""print(f"\r status: {str(status):<10} (attempt {i + 1}{progress})", end="", flush=True)if status == "ok":print()@@ -335,7 +344,7 @@ def _poll(body, job_id):def download_rows_or_none(job_id):- """The rows so far, or None when the engine has written none (404 until its first keyword)."""+ """The rows so far, or None when the engine has written none (404 until its first places)."""try:return download_rows(job_id)except urllib.error.HTTPError as e:@@ -786,8 +795,9 @@ def main():ap.add_argument("--city", action="append",help="city/place to auto-geocode for lat/lon; repeatable, one run per city into one file")ap.add_argument("--proxy", action="append", metavar="URL",- help="proxy for the engine: http://, https://, socks5:// or socks5h://host:port, "- "without user:pass (not supported yet); repeatable, the engine uses the first")+ help="proxy for the engine: http://[user:pass@]host:port, https://, socks5:// or "+ "socks5h://host:port (no user:pass on SOCKS5); repeatable: the keywords take turns, "+ "and a keyword Google refuses is retried through the next one")ap.add_argument("--depth", type=int, default=5)# The engine passes this to Google as the interface language, which decides the language of# the category names it returns (and nudges which listings come back). It has to match the@@ -798,6 +808,9 @@ def main():"(default: en; use the local language, e.g. pt, es, de)")# Emails are ON by default (they're the most valuable lead field): the script crawls each# business website after the Maps job. --no-email skips the crawl.+ ap.add_argument("--country", metavar="XX",+ help="country Google ranks and formats results for, 2 letters (default: the country "+ "of your IP address; e.g. us, de, br)")ap.add_argument("--no-email", dest="email", action="store_false", default=True,help="skip the website crawl for emails (emails are on by default)")ap.add_argument("--no-deep-emails", dest="deep_emails", action="store_false",@@ -826,8 +839,8 @@ def main():f = ap.add_argument_group("filters", "applied before the email crawl, except --has-email")f.add_argument("--min-rating", type=float, metavar="X", help="keep rows rated X or higher")f.add_argument("--min-reviews", type=int, metavar="N",- help="keep rows with N or more reviews (ignored while review_count is empty, as with "- "this engine)")+ help="keep rows with N or more reviews (ignored while review_count is empty: the engine "+ "fills it only with SCRAPER_HEADLESS=0)")f.add_argument("--no-website", action="store_true", help="keep only businesses without a website")f.add_argument("--has-phone", action="store_true", help="keep only rows with a phone number")f.add_argument("--has-email", action="store_true", help="keep only rows where an email was found")@@ -842,6 +855,8 @@ def main():if len(a.lang) != 2 or not a.lang.isalpha():sys.exit(f"✗ --lang must be a 2-letter code (got {a.lang!r}); the engine rejects anything else.")+ if a.country is not None and (len(a.country) != 2 or not a.country.isalpha()):+ sys.exit(f"✗ --country must be a 2-letter code (got {a.country!r}), e.g. us or de.")if a.depth < 1:sys.exit("✗ --depth must be 1 or more.")if a.max_time < 1:@@ -911,6 +926,8 @@ def main():"depth": a.depth, "email": False, "max_time": a.max_time}if a.proxy:body["proxies"] = a.proxy+ if a.country:+ body["country"] = a.country.lower()job_id, rows, skipped, ncells, failed_places, blocked = None, [], 0, 0, 0, Falsefor city in cities: # one place at a time, like the grid tilestry:@@ -929,10 +946,11 @@ def main():for r in rows:r["website"] = clean_website(r.get("website", ""))if a.min_reviews and rows and not any(r.get("review_count") for r in rows):- # The engine leaves review_count empty: Google shows a signed-out headless browser no review- # counts. Filtering on it would read every count as 0 and drop every row.- print("⚠ --min-reviews ignored: this engine leaves review_count empty (Google shows it no review "- "counts), so the filter would drop every row. --min-rating works.")+ # A headless Chrome gets Google's limited view, which has no review counts. Filtering on it would+ # read every count as 0 and drop every row.+ print("⚠ --min-reviews ignored: review_count is empty (Google shows a headless Chrome no review counts; "+ "SCRAPER_HEADLESS=0 in .env runs it as a window, which gets them), so the filter would drop every "+ "row. --min-rating works.")a.min_reviews = Nonebefore = len(rows)rows = [r for r in dedupe(rows, exclude=excluded) if keep_row(r, a)]diff --git a/scripts/test_scrape.py b/scripts/test_scrape.pyindex dced1de..b6a5616 100644--- a/scripts/test_scrape.py+++ b/scripts/test_scrape.py@@ -504,6 +504,8 @@ class MockEngine(BaseHTTPRequestHandler):return self._send(200, "[]")jid = parts[3]if len(parts) == 5:+ if CELL_ROWS[jid] is None: # a failed tile wrote no rows+ return self._send(404, '{"message": "no results for this job (yet)"}')buf = io.StringIO()w = csv.DictWriter(buf, fieldnames=FIELDS, restval="")w.writeheader(); w.writerows(CELL_ROWS[jid])@@ -785,6 +787,7 @@ class ArgumentBounds(unittest.TestCase):self.assertIn("--has-email", self.run_main("--has-email", "--no-email"))def test_rejects_values_the_engine_would_choke_on(self):+ self.assertIn("--country", self.run_main("--country", "usa"))self.assertIn("--depth", self.run_main("--depth", "0"))self.assertIn("--max-time", self.run_main("--max-time", "0")) # the engine says "missing max time"self.assertIn("--limit", self.run_main("--limit", "-1"))@@ -793,5 +796,51 @@ class ArgumentBounds(unittest.TestCase):self.assertIn("--out", self.run_main("--out", os.path.join(bad_dir, "x", "leads.csv")))++class EngineProgress(unittest.TestCase):+ """The poll line shows the engine's Progress, and a poll survives an engine restart (it resumes)."""++ def test_progress_text(self):+ job = {"Progress": {"keywords_done": 1, "keywords_total": 3, "places": 41, "percent": 50, "stage": "search"}}+ self.assertEqual(scrape.progress_text(job), ", keyword 2/3, 41 places, 50%")+ job["Progress"].update(keywords_done=3, stage="websites", percent=100)+ self.assertEqual(scrape.progress_text(job), ", checking websites, 41 places, 100%")+ self.assertEqual(scrape.progress_text({}), "") # an engine without Progress++ def test_poll_waits_out_a_restart_and_prints_progress(self):+ calls, out = [], io.StringIO()+ prog = {"keywords_done": 1, "keywords_total": 2, "places": 20, "percent": 50, "stage": "search"}+ answers = [OSError("refused")] * 5 + [{"Status": "pending", "Progress": prog},+ {"Status": "working", "Progress": prog}, {"Status": "ok"}]+ def fake_req(method, path, body=None):+ calls.append(path)+ if method == "POST":+ return 201, b'{"id":"j1"}'+ if path.endswith("/download"):+ return 200, b"title\nA\n"+ a = answers.pop(0)+ if isinstance(a, Exception):+ raise a+ return 200, json.dumps(a).encode()+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \+ redirect_stdout(out):+ jid, rows = scrape.run_job({"max_time": 60, "keywords": ["a", "b"]})+ self.assertEqual((jid, len(rows)), ("j1", 1))+ self.assertIn("keyword 2/2, 20 places, 50%", out.getvalue())+ self.assertFalse(any(c.endswith("/download") for c in calls[:-1])) # no download per poll any more++ def test_country_goes_into_the_job(self):+ bodies = []+ def fake_run_job(body):+ bodies.append(body)+ return "j1", []+ argv = ["scrape.py", "x in y", "1", "2", "--country", "DE", "--no-email",+ "--out", os.path.join(tempfile.mkdtemp(), "o.csv")]+ with mock.patch.object(scrape, "req", return_value=(200, b"null")), \+ mock.patch.object(scrape, "run_job", fake_run_job), mock.patch.object(sys, "argv", argv), \+ redirect_stdout(io.StringIO()):+ scrape.main()+ self.assertEqual(bodies[0]["country"], "de")+if __name__ == "__main__":unittest.main(verbosity=2)