Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

feat(engine): P3: resumable jobs, streaming rows, progress, backoff, proxy logins, locale, full-view data

Mahan Jafaridc9b13c4282e
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.yml
index 4ce2d9b..197f26f 100644
--- a/.github/workflows/engine-live.yml
+++ b/.github/workflows/engine-live.yml
@@ -41,10 +41,13 @@ jobs:
test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Origin: https://evil.com' -X POST -d '{}' http://localhost:8080/api/v1/jobs)" = 403
curl -s -X POST -d '{}' http://localhost:8080/api/v1/jobs | grep -q 'missing name'
curl -s http://localhost:8080/ | grep -q 'github.com/Mahanaicoach/google-maps-scraper-kit'
+ curl -s http://localhost:8080/api/docs | grep -q '<code>reviews_max</code>'
+ if curl -s http://localhost:8080/api/docs | grep -q '<script'; then exit 1; fi # set -e ignores a "!" line
+ curl -s http://localhost:8080/api/openapi.json | grep -q '"openapi"'
- name: scrape (depth 2, no email crawl)
run: |
start=$(date +%s)
- python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --out leads.csv > scrape.log 2>&1 || true
+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --country us --out leads.csv > scrape.log 2>&1 || true
echo "seconds=$(( $(date +%s) - start ))"
grep -v "^ •" scrape.log | tr '\r' '\n' | grep -v "^ status:" || true
python - <<'PY'
@@ -54,6 +57,55 @@ jobs:
print(f"places={len(rows)}", fill)
sys.exit(0 if rows else 1)
PY
+ - name: proxy with a username and password (a local test proxy that answers 407 first)
+ run: |
+ python engine/test_proxy.py 8191 kit 'p@ss:w0rd' > proxy.log 2>&1 &
+ sleep 2
+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 1 --no-email --max-time 120 \
+ --proxy 'http://kit:p%40ss%[email protected]:8191' --out proxied.csv > proxied.log 2>&1 || true
+ grep -v "^ •" proxied.log | tr '\r' '\n' | grep -v "^ status:" || true
+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 1 --no-email --max-time 60 \
+ --proxy 'http://kit:[email protected]:8191' --out refused.csv > refused.log 2>&1 || true
+ grep -q "proxy login refused" refused.log
+ if grep -q "w0rd" .engine/engine.log; then echo "a proxy password reached the log"; exit 1; fi
+ python - <<'PY'
+ import csv, json, sys
+ rows = list(csv.DictReader(open("proxied.csv", encoding="utf-8-sig")))
+ counts = json.load(open(".engine/test-proxy.json"))
+ print(f"places={len(rows)}", counts)
+ sys.exit(0 if rows and counts["tunnels"] and counts["refused"] else 1)
+ PY
+ - name: window mode (SCRAPER_HEADLESS=0) with the reviews pass
+ env:
+ SCRAPER_HEADLESS: "0"
+ run: |
+ if [ "$RUNNER_OS" = Linux ]; then # a window needs a display
+ command -v Xvfb >/dev/null || { sudo apt-get update -q && sudo apt-get install -y -q xvfb; }
+ Xvfb :99 -screen 0 1280x900x24 > /dev/null 2>&1 &
+ export DISPLAY=:99
+ fi
+ python scripts/engine.py restart
+ python - <<'PY'
+ import json, sys, time, urllib.request
+ base = "http://localhost:8080/api/v1/jobs"
+ def call(method, url, body=None):
+ r = urllib.request.Request(url, json.dumps(body).encode() if body else None,
+ {"Content-Type": "application/json"}, method=method)
+ with urllib.request.urlopen(r, timeout=30) as f:
+ return json.loads(f.read())
+ jid = call("POST", base, {"name": "window", "keywords": ["coffee shops in Austin TX"], "lang": "en",
+ "country": "us", "lat": "30.2672", "lon": "-97.7431", "depth": 1,
+ "max_time": 90, "reviews_max": 1})["id"]
+ for _ in range(200):
+ job = call("GET", f"{base}/{jid}")
+ if job["Status"] in ("ok", "failed"):
+ break
+ time.sleep(2)
+ rows = json.loads(urllib.request.urlopen(f"{base}/{jid}/download?format=json").read())
+ fill = {k: sum(1 for r in rows if r.get(k)) for k in ("review_count", "open_hours", "user_reviews")}
+ print(f"status={job['Status']} places={len(rows)}", fill, job["Progress"])
+ sys.exit(0 if rows and fill["review_count"] and fill["user_reviews"] else 1)
+ PY
- name: engine log (counts only)
if: always()
run: cat .engine/engine.log || true
diff --git a/engine/browser.py b/engine/browser.py
index 50e396b..fc85c07 100644
--- a/engine/browser.py
+++ b/engine/browser.py
@@ -7,9 +7,10 @@ An installed Google Chrome, Chromium, Edge or Brave is used first (set CHROME=/p
pick one). On a machine with none, `python3 scripts/engine.py start` downloads Chrome for Testing's
headless shell into .engine/, pinned below by version and SHA-256. Standard library only.
"""
-import hashlib, os, platform, shutil, subprocess, tempfile, time, urllib.request, zipfile
+import hashlib, os, platform, queue, shutil, subprocess, tempfile, threading, time, urllib.parse, urllib.request
+import zipfile
-from cdp import CDP
+from cdp import CDP, CDPError
KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HOME = os.path.join(KIT, ".engine") # everything the engine keeps: jobs, results, logs, its Chrome
@@ -190,10 +191,64 @@ def start_error(path, code, stderr):
return f"Chrome exited at once (code {code}): {path}" + (f": {last}" if last else "")
+def split_proxy(url):
+ """(the proxy as Chrome takes it, username, password): Chrome takes no user:pass in a proxy URL
+ and knows socks5:// only (it resolves names through a SOCKS5 proxy anyway)."""
+ u = urllib.parse.urlsplit(url)
+ host = f"[{u.hostname}]" if ":" in (u.hostname or "") else u.hostname
+ scheme = "socks5" if u.scheme == "socks5h" else u.scheme
+ return (f"{scheme}://{host}" + (f":{u.port}" if u.port else ""),
+ urllib.parse.unquote(u.username or ""), urllib.parse.unquote(u.password or ""))
+
+
+class ProxyAuth:
+ """Logs one tab in to its proxy: Chrome asks over CDP (Fetch.authRequired) when the proxy answers
+ 407, and gets the username and password here. With Fetch on, every request of the tab waits until
+ it is continued, so a thread continues them. The credentials go to Chrome only, never to a log."""
+
+ def __init__(self, page, user, password):
+ self.page, self.user, self.password = page, user, password
+ self.q = page.cdp.subscribe("Fetch.requestPaused", page.sid)
+ page.cdp.subscribe("Fetch.authRequired", page.sid, q=self.q)
+ self.done, self.asked, self.refused = threading.Event(), set(), False
+ page.send("Fetch.enable", {"handleAuthRequests": True, "patterns": [{"urlPattern": "*"}]})
+ threading.Thread(target=self._run, daemon=True, name="proxy-auth").start()
+
+ def _run(self):
+ while not self.done.is_set():
+ try:
+ ev = self.q.get(timeout=0.5)
+ except queue.Empty:
+ continue
+ if not isinstance(ev, dict): # the connection died
+ return
+ rid = ev.get("requestId")
+ try:
+ if "authChallenge" not in ev:
+ self.page.send("Fetch.continueRequest", {"requestId": rid})
+ elif ev["authChallenge"].get("source") == "Proxy" and rid not in self.asked:
+ self.asked.add(rid) # asked again for the same request: the proxy refused them
+ self.page.send("Fetch.continueWithAuth", {"requestId": rid, "authChallengeResponse": {
+ "response": "ProvideCredentials", "username": self.user, "password": self.password}})
+ else:
+ self.refused = self.refused or rid in self.asked
+ self.page.send("Fetch.continueWithAuth", {"requestId": rid, "authChallengeResponse": {
+ "response": "CancelAuth" if rid in self.asked else "Default"}})
+ except CDPError: # the tab closed or the request finished meanwhile
+ if self.page.cdp.closed:
+ return
+
+ def close(self):
+ self.done.set()
+ for m in ("Fetch.requestPaused", "Fetch.authRequired"):
+ self.page.cdp.unsubscribe(m, self.q, self.page.sid)
+
+
class Browser:
- """A headless Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP."""
+ """A Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP. Headless by
+ default; headless=False runs it as a normal window off-screen (Google's full view: see details.py)."""
- def __init__(self, path=None, headless=True, profile=None, lang="en-US", proxy=None, timeout=30):
+ def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):
self.path = path or find_chrome()
if not self.path:
raise BrowserError(f"No Chrome, Chromium, Edge or Brave found. Run `{PY} scripts/engine.py start`, "
@@ -209,8 +264,8 @@ class Browser:
f"--lang={lang}", "--disable-dev-shm-usage"]
if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headless
args.append("--headless=new")
- if proxy: # Chrome knows socks5:// only; it resolves names through a SOCKS5 proxy anyway
- args.append("--proxy-server=" + ("socks5://" + proxy[10:] if proxy.startswith("socks5h://") else proxy))
+ elif not headless:
+ args.append("--window-position=-3000,-3000") # a real window, kept out of sight
if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:
args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)
self.proc, self.cdp = None, None
@@ -264,6 +319,20 @@ class Browser:
def version(self):
return self.cdp.send("Browser.getVersion").get("product", "")
+ def context(self, proxy=None):
+ """A new browser context (its own cookies, and `proxy` for all its traffic); its id. Each
+ keyword gets one, so proxies rotate without restarting Chrome."""
+ params = {"disposeOnDetach": True}
+ if proxy:
+ params["proxyServer"] = split_proxy(proxy)[0]
+ return self.cdp.send("Target.createBrowserContext", params)["browserContextId"]
+
+ def dispose(self, context):
+ try:
+ self.cdp.send("Target.disposeBrowserContext", {"browserContextId": context})
+ except (CDPError, OSError):
+ pass
+
def kill(self):
"""Stop Chrome now (the job watchdog): every CDP call still waiting fails at once."""
try:
diff --git a/engine/cdp.py b/engine/cdp.py
index f11d0c3..318e4a1 100644
--- a/engine/cdp.py
+++ b/engine/cdp.py
@@ -191,9 +191,10 @@ class CDP:
raise CDPError(f"{method}: {reply['error'].get('message')}")
return reply.get("result", {})
- def subscribe(self, method, session_id=None):
- """A queue that receives the params of every `method` event on `session_id` from now on."""
- q = queue.Queue()
+ def subscribe(self, method, session_id=None, q=None):
+ """A queue that receives the params of every `method` event on `session_id` from now on
+ (pass q to share one queue between events)."""
+ q = q or queue.Queue()
self._subs.setdefault((session_id, method), []).append(q)
return q
@@ -224,11 +225,15 @@ class CDP:
class Page:
- """One attached tab: navigate, evaluate JavaScript, wait."""
+ """One attached tab: navigate, evaluate JavaScript, wait. `context` is a browser context id
+ (Browser.context) for a tab with its own cookies and proxy."""
- def __init__(self, cdp, url="about:blank"):
+ def __init__(self, cdp, url="about:blank", context=None):
self.cdp = cdp
- self.target = cdp.send("Target.createTarget", {"url": url})["targetId"]
+ params = {"url": url}
+ if context:
+ params["browserContextId"] = context
+ self.target = cdp.send("Target.createTarget", params)["targetId"]
self.sid = cdp.send("Target.attachToTarget", {"targetId": self.target, "flatten": True})["sessionId"]
for domain in ("Page", "Runtime", "Network"):
cdp.send(f"{domain}.enable", session_id=self.sid)
diff --git a/engine/details.py b/engine/details.py
index d32a553..4338b8d 100644
--- a/engine/details.py
+++ b/engine/details.py
@@ -3,19 +3,20 @@
While the result list scrolls, the page fetches batches of about 20 places from
`https://www.google.com/search?tbm=map…` as `)]}'\\n[...]` JSON; each batch carries the full place
entity (name, address, phone, website, rating, coordinates, ids…). The engine captures those
-responses in the browser (cdp Network events) and parses them here, so no place page is ever
-opened: the spike showed the place page itself, fetched by curl or by Chrome, holds no entity data
-any more. Standard library only.
+responses in the browser (cdp Network events) and parses them here, so no place page is opened for
+them: the spike showed the place page document itself holds no entity data any more. Review texts
+are the one thing the batches lack: they come from the `/maps/preview/place` response a place's
+panel downloads (parse_place), only for jobs that ask for them. Standard library only.
"""
import json
# Index path of each field inside a place entity. Checked on live batches in the spike
# (2026-10-31): title, category, address, rating, lat/lon, place_id, data_id, timezone, thumbnail
# filled on 60/60 entities, website 55/60, phone 46/60 (listings without one).
-# review_count is not here: signed out, Google leaves (4, 8) empty, and (37, 1) is a different
-# number (60 and 82 where the page showed 61 and 134 reviews, 2026-10-31). Headless Chrome gets
-# Google's limited view, whose result cards show "4.8 stars" without the count, so the CSV's
-# review_count column stays empty.
+# review_count, reviews_link and the week of opening hours come only with Google's full view: a
+# headless Chrome (the default) gets its "limited view", which leaves (4, 8) and (4, 3) empty and
+# lists today's hours only. A Chrome window (SCRAPER_HEADLESS=0) gets the full view: count at (4, 8)
+# on 20/20 entities, 7 days of hours on 19/20 (2026-10-31). (37, 1) is the photo count, not reviews.
FIELDS = {
"title": (11,),
"category": (13, 0),
@@ -23,6 +24,8 @@ FIELDS = {
"website": (7, 0),
"phone": (178, 0, 0),
"review_rating": (4, 7),
+ "review_count": (4, 8),
+ "reviews_link": (4, 3, 0),
"latitude": (9, 2),
"longitude": (9, 3),
"plus_code": (183, 2, 2, 0),
@@ -34,6 +37,9 @@ FIELDS = {
"price_range": (4, 2),
"description": (32, 1, 1),
}
+HOURS = (203, 0) # [[day name, 1-7 (Monday-Sunday), date, [[text, ...], ...], ...], ...], from today on
+HISTOGRAM = (175, 3) # place panel: [one-star count, ..., five-star count]
+REVIEWS = (175, 9, 0, 0) # place panel: the reviews it shows first (about 5)
LISTS = {"categories": (13,), "address_lines": (2,)}
ENTRIES = 64 # position of the result list inside a batch
@@ -59,6 +65,9 @@ def parse_entity(entity):
items = [x for x in v if isinstance(x, str) and x]
if items:
out[name] = " | ".join(items)
+ hours = open_hours(dig(entity, HOURS))
+ if hours:
+ out["open_hours"] = hours
if "data_id" in out and ":" in out["data_id"]:
try:
out["cid"] = str(int(out["data_id"].split(":")[1], 16))
@@ -67,6 +76,53 @@ def parse_entity(entity):
return out
+def _text(v):
+ return v.replace("\u202f", " ").strip() if isinstance(v, str) else ""
+
+
+def open_hours(days):
+ """The week's hours as old-engine JSON ({"Monday": ["7 AM–9 PM"], ...}, Monday first), or "" when
+ Google sent fewer than 7 days (the limited view sends today only, which would read as a week)."""
+ if not isinstance(days, list):
+ return ""
+ week = {}
+ for d in days:
+ if not (isinstance(d, list) and len(d) > 3 and isinstance(d[0], str) and isinstance(d[1], int)):
+ continue
+ spans = [_text(x[0]) for x in d[3] or [] if isinstance(x, list) and x and _text(x[0])]
+ week[d[1]] = (d[0], spans)
+ if len(week) < 7:
+ return ""
+ return json.dumps({name: spans for _, (name, spans) in sorted(week.items())}, ensure_ascii=False)
+
+
+def parse_place(body, limit=5):
+ """review_count, reviews_per_rating and up to `limit` user_reviews from one `/maps/preview/place`
+ response. Reviewer names and photos are left out: the kit collects business data, not people."""
+ if body.startswith(")]}'"):
+ body = body.split("\n", 1)[-1]
+ try:
+ entity = dig(json.loads(body), (6,))
+ except ValueError:
+ return {}
+ if not isinstance(entity, list):
+ return {}
+ out = {k: v for k, v in parse_entity(entity).items() if k in ("review_count", "reviews_link", "open_hours")}
+ hist = dig(entity, HISTOGRAM)
+ if isinstance(hist, list) and len(hist) == 5 and all(isinstance(n, int) for n in hist):
+ out["reviews_per_rating"] = json.dumps({str(i): n for i, n in enumerate(hist, 1)})
+ reviews = []
+ for r in dig(entity, REVIEWS) or []:
+ rating, text = dig(r, (0, 2, 0, 0)), _text(dig(r, (0, 2, 15, 0, 0)))
+ if isinstance(rating, int) and not isinstance(rating, bool):
+ reviews.append({"Rating": rating, "When": _text(dig(r, (0, 1, 6))), "Description": text})
+ if len(reviews) >= limit:
+ break
+ if reviews:
+ out["user_reviews"] = json.dumps(reviews, ensure_ascii=False)
+ return out
+
+
def parse_batch(body):
"""The places in one `/search?tbm=map` response body (with or without the `)]}'` prefix)."""
if body.startswith(")]}'"):
diff --git a/engine/jobs.py b/engine/jobs.py
index c86c6d0..6feb43c 100644
--- a/engine/jobs.py
+++ b/engine/jobs.py
@@ -1,16 +1,21 @@
"""Scrape jobs: a sqlite3 store in .engine/data and one worker that runs the jobs one at a time.
-A job runs its keywords through maps.collect in one headless Chrome, de-duplicates the places on
-data_id and writes <id>.csv with the old engine's 36 column names (what Google's result batches
-carry is filled, the rest stays empty) plus ours. The CSV is rewritten atomically after every
-keyword, so a download while the job works returns the rows so far. A watchdog stops Chrome when a
-job overruns its max_time, so no job can stick on "working". Standard library only.
+A job runs its keywords through maps.collect in one Chrome, each keyword in its own browser context
+(own cookies, the next proxy of the job's list), de-duplicates the places on data_id and appends them
+to <id>.csv as the result batches arrive, with the old engine's 36 column names (what Google's result
+batches carry is filled, the rest stays empty) plus ours. Appends and downloads share one lock, so a
+download while the job works returns whole rows. The job's Progress (keywords done, places, percent)
+is saved as it goes; a job the engine was stopped in the middle of resumes from its last finished
+keyword on the next start. When Google answers with its unusual-traffic page or 429, the keyword
+waits and retries a bounded number of times (through the next proxy when there is one), then the job
+stops with a clear Error. A watchdog stops Chrome when a job overruns its max_time, so no job can
+stick on "working". Standard library only.
"""
-import concurrent.futures, csv, html, http.client, io, ipaddress, json, math, os, re, socket, sqlite3
-import threading, time, urllib.parse, urllib.request, uuid
+import concurrent.futures, csv, functools, html, http.client, io, ipaddress, json, math, os, re, socket
+import sqlite3, threading, time, urllib.parse, urllib.request, uuid
from datetime import datetime, timezone
-import browser, maps
+import browser, details, maps
from cdp import CDPError, Page
# gosom/google-maps-scraper's CSV header, in its order, so files and scripts that read the old
@@ -22,15 +27,24 @@ COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "po
"order_online", "menu", "owner", "complete_address", "credit_cards_accepted", "about",
"user_reviews", "user_reviews_extended", "emails"]
EXTRA = ["categories"]
+SOCIALS = ["instagram", "facebook", "linkedin", "tiktok", "youtube", "x"] # the names scripts/scrape.py --socials uses
RENAME = {"open_state": "status", "description": "descriptions"} # details.FIELDS name -> CSV column
# Seconds past max_time before the watchdog kills Chrome. maps.collect's page load and first waits
# (navigate 45 s, consent 15 s, feed 15 s, first batch 10 s) sit outside its max_seconds, so a
# shorter grace could kill a keyword that was about to return its rows.
WATCHDOG_GRACE = 90
-SITE_WORKERS = 8 # business websites fetched at once by the email pass
+SITE_WORKERS = 8 # business websites fetched at once by the website pass
SITE_TIMEOUT = 10 # seconds per website request
-EMAIL_MIN_SECONDS = 60 # the email pass gets at least this long, even after a job used its max_time
+EMAIL_MIN_SECONDS = 60 # the website pass gets at least this long, even after a job used its max_time
+BACKOFF = (30, 90) # seconds to wait before each retry of a keyword Google refused; then the job stops
+ROTATE_WAIT = 3 # seconds before the retry when the next proxy can take it instead
+MAX_RESUMES = 3 # restarts of the engine mid-job before the job is failed instead of resumed
+LOGIN_REFUSED = "proxy login refused: the proxy answered 407 to its username and password"
+# Chrome's errors for a proxy that failed, and ours. Matched as written: an error also holds the
+# search URL, so a keyword such as "tunnel car wash" must not read as a proxy error.
+PROXY_FAILED = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_|proxy login refused")
+REVIEWS_MAX = 5 # the reviews a place panel shows first; more would need its review list
def utc_now():
@@ -71,6 +85,11 @@ def validate(body):
raise ValueError("missing lang")
if not isinstance(lang, str) or not LANG.fullmatch(lang):
raise ValueError("invalid lang")
+ country = body.get("country")
+ if country in (None, ""):
+ country = ""
+ elif not isinstance(country, str) or not LANG.fullmatch(country):
+ raise ValueError("invalid country: a 2-letter country code such as us or de")
lat, lon = body.get("lat"), body.get("lon")
if lat in (None, "") or lon in (None, ""):
raise ValueError("missing geo coordinates")
@@ -84,13 +103,15 @@ def validate(body):
raise ValueError("invalid geo coordinates")
if body.get("max_time") in (None, 0):
raise ValueError("missing max time")
- data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "lat": str(lat).strip(),
- "lon": str(lon).strip(),
+ data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "country": country.lower(),
+ "lat": str(lat).strip(), "lon": str(lon).strip(),
"zoom": _int(body, "zoom", 15, 1, 21, "zoom"),
"radius": _int(body, "radius", 10000, 0, 10**9, "radius"), # accepted, not used
"depth": _int(body, "depth", 1, 1, 1000, "depth"),
"max_time": _int(body, "max_time", 600, 1, 86400, "max time"),
"fast_mode": bool(body.get("fast_mode")), "email": bool(body.get("email")),
+ "verify_emails": bool(body.get("verify_emails")), "socials": bool(body.get("socials")),
+ "reviews_max": _int(body, "reviews_max", 0, 0, REVIEWS_MAX, "reviews_max"),
"proxies": _proxies(body.get("proxies"))}
return name.strip(), data
@@ -105,16 +126,35 @@ def _proxies(v):
u = urllib.parse.urlsplit(p.strip())
u.port # noqa: B018 - raises on a bad port
except ValueError:
- raise ValueError(f"invalid proxy: {p}") from None
+ raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port") from None
if u.scheme not in PROXY_SCHEMES or not u.hostname:
raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port")
- if u.username or u.password:
- raise ValueError("proxy usernames and passwords are not supported by this engine yet: "
- "use a proxy that allows your IP address")
+ if (u.username or u.password) and u.scheme.startswith("socks"):
+ raise ValueError("invalid proxy: Chrome cannot log in to a SOCKS5 proxy; use its http:// address "
+ "with user:pass, or one that allows your IP address")
return [p.strip() for p in v]
+def mask(proxy):
+ """A proxy URL with its password replaced by ***: what the API, the web UI and the log show."""
+ u = urllib.parse.urlsplit(proxy)
+ if not u.password:
+ return proxy
+ return urllib.parse.urlunsplit(u._replace(netloc=u.netloc.rpartition("@")[0].partition(":")[0] + ":***@"
+ + u.netloc.rpartition("@")[2]))
+
+
+def locale(d):
+ """Chrome's --lang (its Accept-Language) for a job: en, or en-US when the job names a country."""
+ return d["lang"] + (f"-{d['country'].upper()}" if d.get("country") else "")
+
+
# ── Rows ──
+def fields(d):
+ """A job's CSV columns: the old engine's 36, ours, then those its website pass fills."""
+ return COLUMNS + EXTRA + (SOCIALS if d.get("socials") else []) + (["email_host"] if d.get("verify_emails") else [])
+
+
def to_row(place, keyword):
"""One CSV row (every column, as text) from a details.parse_entity / maps.parse_link place."""
row = dict.fromkeys(COLUMNS + EXTRA, "")
@@ -130,10 +170,16 @@ def to_row(place, keyword):
return row
-def to_csv(rows):
+def place_key(p):
+ """What makes two places (or rows) the same place."""
+ return p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")
+
+
+def to_csv(rows, cols=None, header=True):
buf = io.StringIO()
- w = csv.DictWriter(buf, fieldnames=COLUMNS + EXTRA, extrasaction="ignore")
- w.writeheader()
+ w = csv.DictWriter(buf, fieldnames=cols or COLUMNS + EXTRA, extrasaction="ignore")
+ if header:
+ w.writeheader()
w.writerows(rows)
return buf.getvalue().encode("utf-8")
@@ -141,8 +187,8 @@ def to_csv(rows):
# ── Store ──
class Store:
"""Jobs in sqlite3 (jobs.db) and their results as <id>.csv beside it. One lock serialises every
- database call and every CSV read and write, so a download never meets a half-replaced file
- (Windows cannot replace a file another handle has open)."""
+ database call and every CSV read, append and replace, so a download never meets a half-written
+ row or a half-replaced file (Windows cannot replace a file another handle has open)."""
def __init__(self, folder):
os.makedirs(folder, exist_ok=True)
@@ -152,37 +198,47 @@ class Store:
with self.lock, self.db:
self.db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, "
"date TEXT NOT NULL, status TEXT NOT NULL, data TEXT NOT NULL, "
- "error TEXT NOT NULL DEFAULT '')")
+ "error TEXT NOT NULL DEFAULT '', progress TEXT NOT NULL DEFAULT '{}')")
+ if "progress" not in [c[1] for c in self.db.execute("PRAGMA table_info(jobs)")]: # a P1 or P2 store
+ self.db.execute("ALTER TABLE jobs ADD COLUMN progress TEXT NOT NULL DEFAULT '{}'")
+
+ SELECT = "SELECT id, name, date, status, data, error, progress FROM jobs"
@staticmethod
def _job(r):
- # The old engine's job shape (capitalised keys), plus Error: why a job failed or stopped early.
- return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5]}
+ # The old engine's job shape (capitalised keys), plus Error (why a job failed or stopped early)
+ # and Progress (the worker's own record; public() is what the API shows).
+ return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5],
+ "Progress": json.loads(r[6] or "{}")}
def create(self, name, data):
jid = str(uuid.uuid4())
with self.lock, self.db:
- self.db.execute("INSERT INTO jobs VALUES (?, ?, ?, 'pending', ?, '')",
- (jid, name, utc_now(), json.dumps(data)))
+ self.db.execute("INSERT INTO jobs (id, name, date, status, data, error, progress) "
+ "VALUES (?, ?, ?, 'pending', ?, '', '{}')", (jid, name, utc_now(), json.dumps(data)))
return jid
def get(self, jid):
with self.lock:
- r = self.db.execute("SELECT * FROM jobs WHERE id = ?", (jid,)).fetchone()
+ r = self.db.execute(self.SELECT + " WHERE id = ?", (jid,)).fetchone()
return self._job(r) if r else None
def all(self):
with self.lock:
- return [self._job(r) for r in self.db.execute("SELECT * FROM jobs ORDER BY rowid DESC")]
+ return [self._job(r) for r in self.db.execute(self.SELECT + " ORDER BY rowid DESC")]
def update(self, jid, status, error=""):
with self.lock, self.db:
self.db.execute("UPDATE jobs SET status = ?, error = ? WHERE id = ?", (status, error[:500], jid))
+ def set_progress(self, jid, progress):
+ with self.lock, self.db:
+ self.db.execute("UPDATE jobs SET progress = ? WHERE id = ?", (json.dumps(progress), jid))
+
def claim(self):
"""The oldest pending job, now marked working; None when there is none."""
with self.lock, self.db:
- r = self.db.execute("SELECT * FROM jobs WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()
+ r = self.db.execute(self.SELECT + " WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()
if not r:
return None
self.db.execute("UPDATE jobs SET status = 'working' WHERE id = ?", (r[0],))
@@ -190,11 +246,28 @@ class Store:
job["Status"] = "working"
return job
- def recover(self):
- """Jobs left on "working" by an engine that was stopped or crashed: failed, rows kept."""
+ def recover(self, max_resumes=MAX_RESUMES):
+ """Jobs left on "working" by an engine that was stopped or crashed: back to pending, so they
+ resume from their last finished keyword (the rows of the keyword cut short are removed, a
+ torn last line too). After max_resumes such restarts a job is failed instead, rows kept, as
+ before resumable jobs. (resumed, failed)"""
+ resumed = failed = 0
with self.lock, self.db:
- return self.db.execute("UPDATE jobs SET status = 'failed', error = 'the engine stopped while "
- "this job ran; its rows so far are kept' WHERE status = 'working'").rowcount
+ for jid, prog in self.db.execute("SELECT id, progress FROM jobs WHERE status = 'working'").fetchall():
+ p = json.loads(prog or "{}")
+ if p.get("stage", "search") == "search":
+ self.cut(jid, p.get("bytes", 0))
+ if p.get("resumes", 0) >= max_resumes:
+ self.db.execute("UPDATE jobs SET status = 'failed', error = ? WHERE id = ?",
+ (f"the engine stopped while this job ran ({p['resumes'] + 1} times); its rows "
+ "so far are kept", jid))
+ failed += 1
+ else:
+ p["resumes"] = p.get("resumes", 0) + 1
+ self.db.execute("UPDATE jobs SET status = 'pending', progress = ? WHERE id = ?",
+ (json.dumps(p), jid))
+ resumed += 1
+ return resumed, failed
def delete(self, jid):
with self.lock, self.db:
@@ -209,15 +282,48 @@ class Store:
def csv_path(self, jid):
return os.path.join(self.folder, f"{jid}.csv")
- def write_csv(self, jid, rows):
- """Replace the job's CSV in one step; False (nothing written) if the job was deleted."""
+ def _exists(self, jid):
+ return self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone() is not None
+
+ def append(self, jid, rows, cols):
+ """Add rows to the job's CSV (the header first on an empty file); the file's size after it,
+ or None (nothing written) if the job was deleted."""
+ with self.lock:
+ if not self._exists(jid):
+ return None
+ path = self.csv_path(jid)
+ size = os.path.getsize(path) if os.path.exists(path) else 0
+ if rows or not size:
+ with open(path, "ab") as f:
+ f.write(to_csv(rows, cols, header=not size))
+ return os.path.getsize(path)
+
+ def cut(self, jid, size):
+ """Shorten the job's CSV to `size` bytes (its length at the last finished keyword)."""
+ with self.lock:
+ path = self.csv_path(jid)
+ try:
+ if not size:
+ os.remove(path)
+ elif os.path.getsize(path) > size:
+ with open(path, "r+b") as f:
+ f.truncate(size)
+ except OSError:
+ pass
+
+ def finish(self, jid, rows, cols, status, error, progress):
+ """Replace the job's CSV in one step and record how the job ended, under one lock; False
+ (nothing written) if the job was deleted."""
with self.lock:
- if not self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone():
+ if not self._exists(jid):
return False
tmp = self.csv_path(jid) + ".tmp"
with open(tmp, "wb") as f:
- f.write(to_csv(rows))
+ f.write(to_csv(rows, cols))
os.replace(tmp, self.csv_path(jid))
+ with self.db:
+ self.db.execute("UPDATE jobs SET status = ?, error = ?, progress = ? WHERE id = ?",
+ (status, error[:500], json.dumps(progress), jid))
return True
def read_csv(self, jid):
@@ -228,14 +334,33 @@ class Store:
except OSError:
return None
+ def read_rows(self, jid):
+ """The job's rows as dicts ([] before its first row)."""
+ data = self.read_csv(jid)
+ return list(csv.DictReader(io.StringIO(data.decode("utf-8", "replace")))) if data else []
+
+
+def public(job):
+ """A job as the API shows it: proxy passwords masked, Progress without the worker's bookkeeping."""
+ d, p = job["Data"], job["Progress"]
+ total = len(d.get("keywords") or [])
+ stage = p.get("stage") or ("queued" if job["Status"] == "pending" else "")
+ if job["Status"] == "ok":
+ stage = "done"
+ return dict(job, Data=dict(d, proxies=[mask(x) for x in d.get("proxies") or []]),
+ Progress={"keywords_done": p.get("done", 0), "keywords_total": total, "places": p.get("places", 0),
+ "percent": 100 if job["Status"] == "ok" else p.get("percent", 0), "stage": stage})
+
# ── Worker ──
class Worker(threading.Thread):
- """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once."""
+ """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once.
+ headless=False runs Chrome as an off-screen window, which Google gives its full view (review
+ counts, the week's hours, reviews); see details.py."""
- def __init__(self, store, log=print):
+ def __init__(self, store, log=print, headless=True):
super().__init__(daemon=True, name="jobs")
- self.store, self.log = store, log
+ self.store, self.log, self.headless = store, log, headless
self.wake = threading.Event()
self.stopped = threading.Event()
self._running = (None, None) # (job id, stop event) of the job in progress
@@ -266,8 +391,9 @@ class Worker(threading.Thread):
if rid == jid and stop:
stop.set()
- def shutdown(self):
- """Stop the job in progress and its Chrome (the engine is exiting)."""
+ def shutdown(self, wait=0):
+ """Stop the job in progress and its Chrome (the engine is exiting). With `wait`, give the job
+ that many seconds to save its progress, so the next start resumes it."""
self.stopped.set()
self.wake.set()
_, stop = self._running
@@ -276,18 +402,49 @@ class Worker(threading.Thread):
b = self._browser
if b:
b.close()
+ if wait and self.is_alive():
+ self.join(wait)
def run_job(self, job):
jid, d = job["ID"], job["Data"]
- deadline = time.monotonic() + d["max_time"]
+ p = dict(job["Progress"])
+ kws, cols, proxies = d["keywords"], fields(d), d.get("proxies") or []
+ started, used = time.monotonic(), p.get("elapsed", 0)
+ budget = max(0, d["max_time"] - used) # a resumed job has the rest of its max_time
+ deadline = started + budget
stop = threading.Event()
self._running = (jid, stop)
if not self.store.get(jid): # deleted between claim() and now, before cancel() could see it
self._running = (None, None)
return
- rows, seen, errors, blocked = [], set(), [], False
- kws = d["keywords"]
- self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")
+ done, stage = min(p.get("done", 0), len(kws)), p.get("stage", "search")
+ reviews = d.get("reviews_max") and stage in ("search", "reviews")
+ if stage == "search":
+ self.store.cut(jid, p.get("bytes", 0)) # the rows of a keyword cut short are searched again
+ rows = self.store.read_rows(jid) if p.get("bytes") else []
+ seen = {place_key(r) for r in rows}
+ errors, notes, blocked, retries, rot = [], [], False, 0, 0
+ cur = {"i": done, "kw": ""}
+ if done or p.get("resumes"):
+ self.log(f"job {jid} resumed at keyword {done + 1}/{len(kws)} with {len(rows)} places, {budget:.0f} s left")
+ else:
+ self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")
+
+ def save(**kw):
+ p.update(kw, elapsed=round(used + time.monotonic() - started, 1), places=len(rows))
+ self.store.set_progress(jid, p)
+
+ def on_batch(places, batches): # streams each batch's new places into the CSV
+ fresh = []
+ for place in places:
+ k = place_key(place)
+ if k and k not in seen:
+ seen.add(k)
+ fresh.append(to_row(place, cur["kw"]))
+ rows.extend(fresh)
+ if fresh and self.store.append(jid, fresh, cols) is None:
+ stop.set() # deleted
+ save(percent=int(100 * (cur["i"] + min(batches / d["depth"], 1)) / len(kws)))
def watchdog(): # Chrome hung or the job overran: kill Chrome, and every CDP call fails at once
stop.set()
@@ -295,50 +452,73 @@ class Worker(threading.Thread):
if b:
b.kill()
self.log(f"job {jid}: watchdog stopped Chrome {WATCHDOG_GRACE} s after max_time")
- timer = threading.Timer(d["max_time"] + WATCHDOG_GRACE, watchdog)
+ timer = threading.Timer(budget + WATCHDOG_GRACE, watchdog)
timer.daemon = True
timer.start()
try:
- self._browser = browser.Browser(lang=d["lang"], proxy=(d["proxies"] or [None])[0])
- for i, kw in enumerate(kws, 1):
+ save(stage=stage, done=done, total=len(kws))
+ if (done < len(kws) or reviews) and budget > 0:
+ self._browser = browser.Browser(lang=locale(d), headless=self.headless)
+ i = done
+ while i < len(kws) and self._browser:
left = deadline - time.monotonic()
if stop.is_set() or left <= 0:
break
- url = maps.search_url(kw, d["lat"], d["lon"], d["zoom"], d["lang"])
- page = None
- started = time.monotonic()
+ cur.update(i=i, kw=kws[i])
+ proxy = proxies[(i + rot) % len(proxies)] if proxies else None
+ url = maps.search_url(kws[i], d["lat"], d["lon"], d["zoom"], d["lang"], d.get("country", ""))
+ t0 = time.monotonic()
try:
- page = Page(self._browser.cdp)
- places, reason, stats = maps.collect(page, url, depth=d["depth"], max_seconds=left,
- log=lambda m: None, stop=stop.is_set)
+ places, reason, stats = self._collect(proxy, url, d["depth"], left, stop, on_batch)
except (CDPError, OSError) as e:
+ if (PROXY_FAILED.search(str(e)) and len(proxies) > 1 and retries < len(BACKOFF)
+ and not self._browser.cdp.closed and not stop.is_set()):
+ retries, rot = retries + 1, rot + 1
+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: the proxy failed; retry {retries}/"
+ f"{len(BACKOFF)} through the next proxy")
+ continue
errors.append(str(e))
- self.log(f"job {jid} keyword {i}/{len(kws)}: error: {e}")
+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: error: {e}")
if self._browser.cdp.closed: # Chrome is gone: the other keywords would fail too
break
+ i += 1
continue
- finally:
- if page and not self._browser.cdp.closed:
- page.close()
- if reason == "blocked": # more searches from a rate-limited IP only extend the block
+ on_batch(places, stats["batches"]) # places no batch carried (one place, the DOM fallback)
+ if stop.is_set(): # deleted, or the engine is exiting: this keyword did not finish
+ break
+ if reason == "blocked": # Google limits this IP: wait, try again, then give up
+ if retries < len(BACKOFF) and not stop.is_set():
+ wait = ROTATE_WAIT if len(proxies) > 1 else BACKOFF[retries]
+ if time.monotonic() + wait < deadline:
+ retries, rot = retries + 1, rot + 1
+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; "
+ f"retry {retries}/{len(BACKOFF)} in {wait} s"
+ + (" through the next proxy" if len(proxies) > 1 else ""))
+ if stop.wait(wait):
+ break
+ continue
blocked = True
- self.log(f"job {jid} keyword {i}/{len(kws)}: Google's unusual-traffic page; stopping")
+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; stopping")
break
- new = 0
- for p in places:
- key = p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")
- if key and key not in seen:
- seen.add(key)
- rows.append(to_row(p, kw))
- new += 1
- if not self.store.write_csv(jid, rows):
+ retries = 0
+ size = self.store.append(jid, [], cols)
+ if size is None:
break # deleted
- self.log(f"job {jid} keyword {i}/{len(kws)}: {len(places)} places, {new} new, "
- f"{stats['batches']} batches, {time.monotonic() - started:.1f} s ({reason})")
+ save(done=i + 1, bytes=size, percent=int(100 * (i + 1) / len(kws)))
+ self.log(f"job {jid} keyword {i + 1}/{len(kws)}: {len(places)} places, {len(rows)} in all, "
+ f"{stats['batches']} batches, {time.monotonic() - t0:.1f} s ({reason})")
if reason.startswith("error: "): # Chrome failed mid-scroll; the places before it are kept
errors.append(reason[7:])
if self._browser.cdp.closed:
break
+ i += 1
+ if reviews and rows and self._browser and not (stop.is_set() or blocked):
+ save(stage="reviews")
+ n = self.add_reviews(rows, d, deadline, stop, proxies)
+ self.log(f"job {jid}: reviews for {n}/{sum(1 for r in rows if r.get('place_id'))} places")
+ if not n and self.headless:
+ notes.append("no reviews: Google's limited view, which a headless Chrome gets, shows none; "
+ "set SCRAPER_HEADLESS=0 to run Chrome as a window")
except browser.BrowserError as e:
errors.append(str(e))
self.log(f"job {jid}: {e}")
@@ -347,35 +527,123 @@ class Worker(threading.Thread):
if self._browser:
self._browser.close()
self._browser = None
- if d["email"] and rows and not stop.is_set():
- n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), stop=stop.is_set)
- self.log(f"job {jid}: emails found for {n}/{sum(1 for r in rows if r['website'])} websites")
+ if (d["email"] or d.get("socials")) and rows and not stop.is_set():
+ save(stage="websites")
+ n = add_site_data(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), email=d["email"],
+ socials=d.get("socials"), verify=d.get("verify_emails"), stop=stop.is_set)
+ self.log(f"job {jid}: website data for {n}/{sum(1 for r in rows if r['website'])} websites")
self._running = (None, None)
- if not self.store.write_csv(jid, rows):
- self.log(f"job {jid} deleted while it ran")
+ if self.stopped.is_set() and self.store.get(jid): # the engine is exiting: resume on the next start
+ save()
+ self.store.update(jid, "pending")
+ self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)}: resumes when the engine starts again")
return
block = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
- "Wait, or add proxies")
+ + (f"Waited and retried {retries} time(s). " if retries else "") + "Wait, or add proxies")
+ p.update(stage="done", elapsed=round(used + time.monotonic() - started, 1), places=len(rows))
if rows or not (errors or blocked):
- note = "stopped at max_time; the rows found so far are kept" \
- if time.monotonic() >= deadline or (stop.is_set() and not errors) else ""
+ head = ""
+ if time.monotonic() >= deadline or (stop.is_set() and not errors):
+ head = "stopped at max_time; the rows found so far are kept"
if errors:
- note = f"{len(errors)} keyword(s) failed: {errors[-1]}"
+ head = f"{len(errors)} keyword(s) failed: {errors[-1]}"
if blocked:
- note = f"{block}. The rows found before that are kept"
- self.store.update(jid, "ok", note)
- self.log(f"job {jid} ok: {len(rows)} places")
+ head, notes = f"{block}. The rows found before that are kept", []
+ status, note = "ok", "; ".join(([head] if head else []) + notes)
else:
- reason = block if blocked else errors[-1]
- self.store.update(jid, "failed", reason)
- self.log(f"job {jid} failed: {reason}")
+ status = "failed"
+ note = block if blocked else errors[-1]
+ if not self.store.finish(jid, rows, cols, status, note, p):
+ self.log(f"job {jid} deleted while it ran")
+ return
+ self.log(f"job {jid} {status}: {len(rows)} places" + (f": {note}" if status == "failed" else ""))
+
+ def _collect(self, proxy, url, depth, left, stop, on_batch):
+ """One keyword in its own browser context, through `proxy`; maps.collect's answer."""
+ ctx = page = auth = None
+ try:
+ ctx = self._browser.context(proxy)
+ page = Page(self._browser.cdp, context=ctx)
+ _, user, password = browser.split_proxy(proxy) if proxy else ("", "", "")
+ if user or password:
+ auth = browser.ProxyAuth(page, user, password)
+ try:
+ got = maps.collect(page, url, depth=depth, max_seconds=left, log=lambda m: None,
+ stop=stop.is_set, on_batch=on_batch)
+ except CDPError:
+ if auth and auth.refused: # Chrome only says ERR_HTTP_RESPONSE_CODE_FAILURE
+ raise CDPError(LOGIN_REFUSED) from None
+ raise
+ if auth and auth.refused and not got[0]:
+ raise CDPError(LOGIN_REFUSED)
+ return got
+ finally:
+ if auth:
+ auth.close()
+ if page and not self._browser.cdp.closed:
+ page.close()
+ if ctx and not self._browser.cdp.closed:
+ self._browser.dispose(ctx)
+
+ def add_reviews(self, rows, d, deadline, stop, proxies):
+ """Review count, stars histogram and up to reviews_max reviews per place, from each place's
+ panel (the result batches carry no review texts): one page load per place, until the
+ deadline. The number of places that got reviews."""
+ todo = [r for r in rows if r.get("place_id") and not r.get("user_reviews")]
+ ctx = page = auth = None
+ n = 0
+ try:
+ proxy = proxies[0] if proxies else None
+ ctx = self._browser.context(proxy)
+ page = Page(self._browser.cdp, context=ctx)
+ _, user, password = browser.split_proxy(proxy) if proxy else ("", "", "")
+ if user or password:
+ auth = browser.ProxyAuth(page, user, password)
+ for k, v in maps.CONSENT_COOKIES.items():
+ page.set_cookie(k, v, ".google.com")
+ for r in todo:
+ if stop.is_set() or time.monotonic() >= deadline:
+ break
+ url = r["link"] + f"&hl={d['lang']}" + (f"&gl={d['country']}" if d.get("country") else "")
+ body = maps.place_panel(page, url)
+ if body == "blocked":
+ break
+ got = details.parse_place(body or "", d["reviews_max"])
+ for k, v in got.items():
+ if k in ("user_reviews", "reviews_per_rating") or not r.get(k):
+ r[k] = str(v)
+ n += "user_reviews" in got
+ except (CDPError, OSError):
+ pass # Chrome went away: the reviews so far are kept
+ finally:
+ if auth:
+ auth.close()
+ if page and not self._browser.cdp.closed:
+ page.close()
+ if ctx and not self._browser.cdp.closed:
+ self._browser.dispose(ctx)
+ return n
-# ── Email pass (email: true): each business homepage over plain HTTP, emails only ──
+# ── Website pass (email, socials, verify_emails): each business homepage over plain HTTP ──
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")
JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")
JUNK_DOMAIN = ("example.", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")
UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"
+# The same patterns as scripts/scrape.py --socials (a test keeps them in step).
+SOCIAL_RE = {
+ "instagram": re.compile(r"https?://(?:www\.)?instagram\.com/([A-Za-z0-9_.]+)", re.I),
+ "facebook": re.compile(r"https?://(?:www\.|m\.|web\.)?facebook\.com/([A-Za-z0-9_.\-]+)", re.I),
+ "linkedin": re.compile(r"https?://(?:[a-z]{2,3}\.)?linkedin\.com/(?:company|in|school)/([A-Za-z0-9_.\-%]+)", re.I),
+ "tiktok": re.compile(r"https?://(?:www\.)?tiktok\.com/@([A-Za-z0-9_.]+)", re.I),
+ "youtube": re.compile(r"https?://(?:www\.)?youtube\.com/(?:@|channel/|c/|user/)([A-Za-z0-9_.\-]+)", re.I),
+ "x": re.compile(r"https?://(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})\b", re.I),
+}
+SKIP_HANDLE = {"", "home", "pages", "people", "help", "about", "policies", "policy", "legal", "tos", "privacy",
+ "settings", "sharer", "tr", "profile.php", "plugins", "dialog", "intent", "login", "share.php",
+ "permalink.php", "p", "reel", "reels", "explore", "stories", "tv", "watch", "events", "groups",
+ "marketplace", "gaming", "photo", "hashtag", "search", "share", "i"}
+DOH = "https://dns.google/resolve?" # Google's public DNS over HTTPS: no key, no account
def _public_ip(ip):
@@ -459,6 +727,39 @@ def site_host(url):
return h[4:] if h.startswith("www.") else h
+def find_socials(text):
+ """The first profile link per network on a page, share and login links skipped."""
+ out = dict.fromkeys(SOCIALS, "")
+ for net, rx in SOCIAL_RE.items():
+ for m in rx.finditer(text or ""):
+ h = m.group(1).lower()
+ if h in SKIP_HANDLE or (net == "facebook" and (h.isdigit() or len(h) < 3)):
+ continue
+ out[net] = m.group(0).rstrip("\"'/").replace("\\", "")
+ break
+ return out
+
+
[email protected]_cache(maxsize=4096)
+def mx_host(domain):
+ """'nxdomain' when the domain does not exist, 'none' when it has no MX record, else who runs its
+ mail: 'google_workspace', 'microsoft_365' or 'other'; '' when the lookup failed. Like scrape.py."""
+ try:
+ with _OPENER.open(urllib.request.Request(DOH + urllib.parse.urlencode({"name": domain, "type": "MX"}),
+ headers={"User-Agent": UA}), timeout=8) as r:
+ ans = json.load(r)
+ except Exception: # noqa: BLE001 - no answer is not a verdict
+ return ""
+ if ans.get("Status") == 3:
+ return "nxdomain"
+ mx = " ".join(a.get("data", "") for a in ans.get("Answer") or [] if a.get("type") == 15).lower()
+ if not mx:
+ return "none" if ans.get("Status") == 0 else ""
+ if "google.com" in mx or "googlemail.com" in mx:
+ return "google_workspace"
+ return "microsoft_365" if "outlook.com" in mx else "other"
+
+
def find_emails(text, site=""):
"""Up to 3 addresses from page text, placeholders and image names dropped, the site's own first."""
found = []
@@ -472,13 +773,29 @@ def find_emails(text, site=""):
return sorted(found, key=lambda e: not own(e))[:3]
-def add_emails(rows, seconds, get=None, stop=lambda: False):
- """Fill `emails` (joined by ", ", like the old engine) from each row's website homepage, for at
- most `seconds` or until stop() is true; the number of rows that got one."""
+def add_site_data(rows, seconds, email=True, socials=False, verify=False, get=None, stop=lambda: False):
+ """One homepage fetch per row with a website, SITE_WORKERS at a time, for at most `seconds` or
+ until stop() is true: `emails` (joined by ", ", like the old engine), the social columns, and
+ with verify an MX check of each address's domain (dns.google) that drops domains that do not
+ exist and fills `email_host`. The number of rows that got something."""
get = get or fetch
- todo = [r for r in rows if r.get("website") and not r.get("emails")]
+
+ def work(r):
+ page, out = get(r["website"]), {}
+ if email and not r.get("emails"):
+ found = find_emails(page, site_host(r["website"]))
+ if verify:
+ hosts = [(e, mx_host(e.rpartition("@")[2])) for e in found]
+ found = [e for e, h in hosts if h != "nxdomain"]
+ out["email_host"] = next((h for e, h in hosts if h != "nxdomain"), "")
+ out["emails"] = ", ".join(found)
+ if socials:
+ out.update(find_socials(page))
+ return out
+
+ todo = [r for r in rows if r.get("website")]
pool = concurrent.futures.ThreadPoolExecutor(SITE_WORKERS)
- futs = {pool.submit(lambda r=r: find_emails(get(r["website"]), site_host(r["website"]))): r for r in todo}
+ futs = {pool.submit(work, r): r for r in todo}
end, pending = time.monotonic() + seconds, set(futs)
while pending and not stop() and time.monotonic() < end:
_, pending = concurrent.futures.wait(pending, timeout=min(1, max(0, end - time.monotonic())))
@@ -486,7 +803,8 @@ def add_emails(rows, seconds, get=None, stop=lambda: False):
pool.shutdown(wait=False, cancel_futures=True)
n = 0
for f in done:
- if not f.exception() and f.result():
- futs[f]["emails"] = ", ".join(f.result())
- n += 1
+ if not f.exception():
+ got = {k: v for k, v in f.result().items() if v}
+ futs[f].update(got)
+ n += bool(got)
return n
diff --git a/engine/maps.py b/engine/maps.py
index 817574e..9d29bdc 100644
--- a/engine/maps.py
+++ b/engine/maps.py
@@ -1,7 +1,7 @@
"""Google Maps search in a tab: open the search, pass the consent page, scroll the result feed,
and collect the places from the result batches the page itself downloads. Standard library only.
"""
-import queue, re, time, urllib.parse
+import json, queue, re, time, urllib.parse
import details
from cdp import CDPError
@@ -9,7 +9,14 @@ from cdp import CDPError
# Google's consent interstitial (EU and some other regions) is skipped when these cookies exist.
CONSENT_COOKIES = {"CONSENT": "YES+cb.20210720-07-p0.en+FX+410",
"SOCS": "CAISHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmVuIAEaBgiAo_CmBg"}
-END_MARKERS = ("You've reached the end of the list", "reached the end")
+# The feed's last line once the list ends. The element's class has been HlvSq for years; the
+# texts back it up in case Google renames the class (en, fr and es checked live on 2026-10-31).
+END_CLASS = "HlvSq"
+END_MARKERS = ("You've reached the end of the list", "reached the end",
+ "Vous êtes arrivé à la fin de la liste", "Has llegado al final de la lista",
+ "Du hast das Ende der Liste erreicht", "Hai raggiunto la fine dell'elenco",
+ "Você chegou ao fim da lista", "Chegou ao fim da lista", "Je hebt het einde van de lijst bereikt",
+ "Listenin sonuna ulaştınız", "Dotarłeś do końca listy", "Du har nått slutet av listan")
BATCH_URL = re.compile(r"^https://www\.google\.[a-z.]+/search\?.*tbm=map")
_CID = re.compile(r"!1s(0x[0-9a-f]+:0x[0-9a-f]+)")
_LAT_LON = re.compile(r"!3d(-?\d+\.\d+)!4d(-?\d+\.\d+)")
@@ -29,17 +36,41 @@ JS_SCROLL = """(() => {
f.scrollTo(0, f.scrollHeight);
return f.scrollHeight;
})()"""
-JS_END = "(() => { const f = document.querySelector('div[role=\"feed\"]'); return f ? f.innerText : ''; })()"
+JS_END = f"""(() => {{
+ const f = document.querySelector('div[role="feed"]');
+ return f ? {{cls: !!f.querySelector('.{END_CLASS}'), text: f.innerText}} : null;
+}})()"""
+# Google's consent page in any language: its "Reject all" form posts set_eom=true to
+# consent.google.com/save (the "Accept all" form posts false). The button texts are the fallback.
+# Not seen live from the kit's test machines (Google skips it outside the EEA, and the cookies
+# above usually skip it inside), so both ways stay.
+CONSENT_WORDS = ("reject all", "accept all", "alle ablehnen", "alle akzeptieren", "tout refuser", "tout accepter",
+ "rechazar todo", "aceptar todo", "rifiuta tutto", "accetta tutto", "rejeitar tudo", "aceitar tudo",
+ "alles afwijzen", "alles accepteren", "tümünü reddet", "tümünü kabul et", "odrzuć wszystko",
+ "zaakceptuj wszystko", "avvisa alla", "godkänn alla", "afvis alle", "accepter alle")
JS_CONSENT = """(() => {
- const b = [...document.querySelectorAll('button')].find(b => /accept all|reject all|alle akzeptieren|alle ablehnen/i.test(b.innerText));
+ const forms = [...document.querySelectorAll('form[action*="consent"]')];
+ const reject = forms.find(f => f.querySelector('input[name="set_eom"][value="true"]'));
+ let b = reject && reject.querySelector('button, input[type="submit"]');
+ const words = %s;
+ if (!b) b = [...document.querySelectorAll('button')].find(x => words.some(w => x.innerText.toLowerCase().includes(w)));
if (b) { b.click(); return true; } return false;
-})()"""
+})()""" % json.dumps(CONSENT_WORDS, ensure_ascii=False)
-def search_url(query, lat=None, lon=None, zoom=15, lang="en"):
+def search_url(query, lat=None, lon=None, zoom=15, lang="en", country=""):
+ """The Maps search URL: hl is the interface language (category names, opening-hours text), gl
+ the country Google ranks and formats results for (without it, the country of this IP)."""
q = urllib.parse.quote(query, safe="")
at = f"/@{lat},{lon},{zoom}z" if lat is not None and lon is not None else ""
- return f"https://www.google.com/maps/search/{q}{at}?hl={lang}"
+ return f"https://www.google.com/maps/search/{q}{at}?hl={lang}" + (f"&gl={country}" if country else "")
+
+
+def ended(feed):
+ """True when the feed shows its end-of-list line (JS_END's answer)."""
+ if not isinstance(feed, dict):
+ return False
+ return bool(feed.get("cls")) or any(m in (feed.get("text") or "") for m in END_MARKERS)
def parse_link(href, label=""):
@@ -65,6 +96,7 @@ class Batches:
self.q = page.cdp.subscribe("Network.responseReceived", page.sid)
self.pending = {} # requestId -> tries, for batches seen but not yet readable
self.count = 0
+ self.limited = 0 # batch requests Google answered with 429 Too Many Requests
def drain(self):
"""New places from batches that finished since the last call."""
@@ -76,7 +108,11 @@ class Batches:
if not isinstance(p, dict): # the connection died (cdp puts a marker on every queue)
raise CDPError(f"connection lost: {self.page.cdp.closed}")
r = p.get("response", {})
- if BATCH_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):
+ if not BATCH_URL.match(r.get("url", "")):
+ continue
+ if r.get("status") == 429:
+ self.limited += 1
+ elif "json" in (r.get("mimeType") or ""):
self.pending[p["requestId"]] = 0
rows = []
for rid in list(self.pending):
@@ -98,12 +134,13 @@ class Batches:
self.page.cdp.unsubscribe("Network.responseReceived", self.q, self.page.sid)
-def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
+def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False, on_batch=None):
"""Open `url`, scroll the result feed, and return (places, stop reason, stats).
`depth` is the number of result batches to read (about 20 places each, like the old engine's
depth); the list may end sooner. Places are de-duplicated on data_id. `stop()` returning True
- ends the scroll early (a deleted job). Reason "blocked" means Google answered with its
- unusual-traffic page."""
+ ends the scroll early (a deleted job). on_batch(new places, batches read) is called as the
+ batches arrive, so a caller can save them before the keyword ends. Reason "blocked" means
+ Google answered with its unusual-traffic page, or 429 to the result batches."""
for k, v in CONSENT_COOKIES.items():
page.set_cookie(k, v, ".google.com")
batches = Batches(page)
@@ -121,13 +158,15 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
return [], "no-feed", {"batches": 0, "dom_links": 0}
def take():
- new = 0
+ new = []
for row in batches.drain():
key = row.get("data_id") or row.get("place_id") or row["title"]
if key not in seen:
seen[key] = row
- new += 1
- return new
+ new.append(row)
+ if new and on_batch:
+ on_batch(new, batches.count)
+ return len(new)
try: # a Chrome error mid-scroll keeps the places already read
end = time.time() + 10 # the first batch arrives with the page
@@ -136,9 +175,12 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
time.sleep(0.4)
log(f" batch {batches.count}: {len(seen)} places")
while batches.count < depth and time.time() - started < max_seconds and not stop():
- if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):
+ if ended(page.eval(JS_END)):
reason = "end-of-list"
break
+ if batches.limited or "/sorry/" in (page.url() or ""):
+ reason = "blocked"
+ break
before = batches.count
page.eval(JS_SCROLL)
end = time.time() + 10
@@ -149,6 +191,9 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
page.eval(JS_SCROLL) # the feed renders in small steps; keep nudging
if batches.count == before:
stale += 1
+ if batches.limited: # Google refused the next batch: slow down, do not keep asking
+ reason = "blocked"
+ break
if stale >= 2:
reason = "no-growth"
break
@@ -160,6 +205,8 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
reason = "stopped"
elif time.time() - started >= max_seconds:
reason = "timeout"
+ elif batches.count == 0 and batches.limited:
+ reason = "blocked"
take()
links = page.eval(JS_LINKS) or []
except CDPError as e:
@@ -174,3 +221,38 @@ def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
return list(seen.values()), reason, {"batches": batches.count, "dom_links": len(links)}
finally:
batches.close()
+
+
+PLACE_URL = re.compile(r"^https://www\.google\.[a-z.]+/maps/preview/place\?")
+
+
+def place_panel(page, url, timeout=10):
+ """Open one place's panel and return the body of the `/maps/preview/place` response it downloads
+ (details.parse_place reads it); "blocked" for Google's unusual-traffic page, "" when none came."""
+ q = page.cdp.subscribe("Network.responseReceived", page.sid)
+ try:
+ page.navigate(url)
+ if "/sorry/" in (page.url() or ""):
+ return "blocked"
+ end, rid = time.time() + timeout, None
+ while time.time() < end:
+ while rid is None:
+ try:
+ p = q.get_nowait()
+ except queue.Empty:
+ break
+ if not isinstance(p, dict):
+ raise CDPError(f"connection lost: {page.cdp.closed}")
+ r = p.get("response", {})
+ if PLACE_URL.match(r.get("url", "")) and "json" in (r.get("mimeType") or ""):
+ rid = p["requestId"]
+ if rid:
+ try:
+ return page.send("Network.getResponseBody", {"requestId": rid})["body"]
+ except CDPError: # still downloading
+ if page.cdp.closed:
+ raise
+ time.sleep(0.3)
+ return ""
+ finally:
+ page.cdp.unsubscribe("Network.responseReceived", q, page.sid)
diff --git a/engine/serve.py b/engine/serve.py
index adfa0db..252e876 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -6,14 +6,17 @@
Normally started by `python3 scripts/engine.py start`, or by `docker compose up -d` (the Dockerfile). It speaks the API of the engine the kit used
before (POST/GET/DELETE /api/v1/jobs, /api/v1/jobs/{id}/download, the same job body and "Status"
-values), so scripts/scrape.py, the skill and the slash commands work unchanged. It listens on
+values), so scripts/scrape.py, the skill and the slash commands work unchanged; jobs also carry Error
+and Progress, resume after a restart, and download as CSV or JSON. It listens on
127.0.0.1 (and ::1) only, or on 0.0.0.0 inside the Docker image, where Docker publishes the port on
the host's 127.0.0.1 only. Either way it refuses a foreign Host (DNS rebinding, 421) or Origin (cross-site
posts, 403); with SCRAPER_API_KEY set it also wants that X-API-Key (401). No telemetry: it talks to
-Google Maps, and for email: true jobs to the businesses' own websites, nothing else.
+Google Maps, for email or socials jobs to the businesses' own websites, and for verify_emails jobs
+to Google's public DNS (dns.google), nothing else.
Standard library only.
"""
-import argparse, hmac, json, os, re, secrets, signal, socket, socketserver, sys, threading, urllib.parse
+import argparse, csv, hmac, html, io, json, os, re, secrets, signal, socket, socketserver, sys, threading
+import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -31,7 +34,13 @@ REPO = "https://github.com/Mahanaicoach/google-maps-scraper-kit"
# The web UI's sponsor box: the browser reads this static file from GitHub (cached a day, no
# tracking), so the sponsors it lists change without anyone updating their copy of the kit.
SPONSORS = "https://raw.githubusercontent.com/Mahanaicoach/google-maps-scraper-kit/master/assets/sponsors/sponsors.json"
-ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY")
+# Keyword ideas for the web UI: Google Business Profile category names (4,053, US English) from
+# lobstrio/google-business-categories, pinned to one commit. The browser loads it from GitHub, like
+# the sponsor box; the kit ships no copy, because that repository has no licence file.
+CATEGORIES = ("https://raw.githubusercontent.com/lobstrio/google-business-categories/"
+ "4ee1c4788d07c812a32986177c2a51c28957062f/data/google-business-categories.txt")
+CATEGORIES_REPO = "https://github.com/lobstrio/google-business-categories"
+ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY", "SCRAPER_HEADLESS")
MAX_BODY = 1 << 20
LOCAL = r"(localhost|127\.0\.0\.1|\[::1\])"
HOST_OK = re.compile(rf"^{LOCAL}(:\d{{1,5}})?$", re.I)
@@ -49,7 +58,7 @@ def log(msg):
def read_env():
- """SCRAPER_PORT and SCRAPER_API_KEY: the environment wins, then the kit's .env (last line wins,
+ """SCRAPER_PORT, SCRAPER_API_KEY and SCRAPER_HEADLESS: the environment wins, then the kit's .env (last line wins,
as in docker compose and scripts/scrape.py)."""
found = {}
try:
@@ -63,37 +72,81 @@ def read_env():
return {k: os.environ.get(k, found.get(k, "")) for k in ENV_KEYS}
+JOB_PROPS = {
+ "name": {"type": "string", "description": "any label"},
+ "keywords": {"type": "array", "items": {"type": "string"}, "description": "searches, run one after another"},
+ "lang": {"type": "string", "description": "2-letter interface language, e.g. en (category names, hours text)"},
+ "country": {"type": "string", "description": "2-letter country Google ranks and formats results for, e.g. us "
+ "(default: the country of this machine's IP)"},
+ "lat": {"type": "string"}, "lon": {"type": "string"},
+ "max_time": {"type": "integer", "description": "seconds"},
+ "depth": {"type": "integer", "description": "result batches per keyword, about 20 places each (default 1)"},
+ "zoom": {"type": "integer", "description": "map zoom 1-21 (default 15)"},
+ "email": {"type": "boolean", "description": "visit each website's homepage for emails"},
+ "verify_emails": {"type": "boolean", "description": "with email: check each address's domain over DNS (dns.google), "
+ "drop domains that do not exist, add email_host"},
+ "socials": {"type": "boolean", "description": "visit each website's homepage for Instagram, Facebook, LinkedIn, "
+ "TikTok, YouTube and X links"},
+ "reviews_max": {"type": "integer", "description": "0-5 reviews per place from its panel, one page load per place "
+ "(Google shows reviews only to a Chrome window: SCRAPER_HEADLESS=0)"},
+ "proxies": {"type": "array", "items": {"type": "string"},
+ "description": "http(s)://[user:pass@]host:port or socks5://host:port; one per keyword in turn, "
+ "and the next one when Google refuses a keyword"},
+ "radius": {"type": "integer", "description": "accepted, not used"},
+ "fast_mode": {"type": "boolean", "description": "accepted, not used"},
+}
OPENAPI = {
"openapi": "3.0.3",
"info": {"title": "Google Maps Scraper Kit engine", "version": VERSION,
- "description": "Local only. Create a job, poll its Status until ok or failed, download the CSV. "
- f"Source and docs: {REPO}"},
+ "description": "Local only. Create a job, poll its Status until ok or failed (Progress says how far "
+ "it is), download the CSV or JSON. A job the engine was stopped in the middle of "
+ f"resumes from its last finished keyword on the next start. Source and docs: {REPO}"},
"paths": {
"/api/v1/jobs": {
"get": {"summary": "List jobs, newest first (null when there are none)"},
"post": {"summary": "Create a job; returns 201 {\"id\": \"<uuid>\"}, or 422 {\"code\", \"message\"}",
"requestBody": {"content": {"application/json": {"schema": {
"type": "object", "required": ["name", "keywords", "lang", "lat", "lon", "max_time"],
- "properties": {
- "name": {"type": "string", "description": "any label"},
- "keywords": {"type": "array", "items": {"type": "string"}},
- "lang": {"type": "string", "description": "2-letter interface language, e.g. en"},
- "lat": {"type": "string"}, "lon": {"type": "string"},
- "max_time": {"type": "integer", "description": "seconds"},
- "depth": {"type": "integer", "description": "result batches per keyword, about 20 places each (default 1)"},
- "zoom": {"type": "integer", "description": "map zoom 1-21 (default 15)"},
- "email": {"type": "boolean", "description": "visit each website for emails"},
- "proxies": {"type": "array", "items": {"type": "string"},
- "description": "http(s):// or socks5://host:port; the first is used"},
- "radius": {"type": "integer", "description": "accepted, not used"},
- "fast_mode": {"type": "boolean", "description": "accepted, not used"}}}}}}}},
+ "properties": JOB_PROPS}}}}}},
"/api/v1/jobs/{id}": {
- "get": {"summary": "One job: ID, Name, Date, Status (pending, working, ok, failed), Data, Error"},
+ "get": {"summary": "One job: ID, Name, Date, Status (pending, working, ok, failed), Data (proxy "
+ "passwords masked), Error, Progress (keywords_done, keywords_total, places, percent, "
+ "stage: queued, search, reviews, websites or done)"},
"delete": {"summary": "Delete a job and its results (stops it if it is running)"}},
- "/api/v1/jobs/{id}/download": {"get": {"summary": "The job's CSV (the rows so far while it works)"}},
+ "/api/v1/jobs/{id}/download": {"get": {"summary": "The job's CSV, the rows so far while it works; "
+ "?format=json for a JSON array of the same rows"}},
+ "/api/openapi.json": {"get": {"summary": "This description as OpenAPI JSON (the page at /api/docs shows it)"}},
},
}
+
+def docs_page(nonce):
+ """/api/docs: OPENAPI as a plain HTML page (no scripts, nothing loaded from elsewhere)."""
+ e = html.escape
+ out = [f"<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\">"
+ "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>Engine API</title>"
+ f"<style nonce=\"{nonce}\">:root {{ color-scheme: light dark; }} body {{ margin: 0 auto; max-width: 960px; "
+ "padding: 16px; font: 15px/1.45 system-ui, -apple-system, Segoe UI, sans-serif; }} code {{ font-size: .95em; }} "
+ "table {{ width: 100%; border-collapse: collapse; }} th, td {{ text-align: left; vertical-align: top; "
+ "padding: 4px 6px; border-bottom: 1px solid #8884; }} .m {{ font-weight: 600; }}</style></head><body>",
+ f"<h1>{e(OPENAPI['info']['title'])} {e(VERSION)}</h1><p>{e(OPENAPI['info']['description'])}</p>",
+ "<p><a href=\"/\">Web UI</a> · <a href=\"/api/openapi.json\">OpenAPI JSON</a> · "
+ "With SCRAPER_API_KEY set, send it as an <code>X-API-Key</code> header.</p>"]
+ for path, ops in OPENAPI["paths"].items():
+ for method, op in ops.items():
+ out.append(f"<h2><span class=\"m\">{e(method.upper())}</span> <code>{e(path)}</code></h2><p>{e(op['summary'])}</p>")
+ schema = op.get("requestBody", {}).get("content", {}).get("application/json", {}).get("schema")
+ if schema:
+ out.append("<table><tr><th>Field</th><th>Type</th><th>Required</th><th>Meaning</th></tr>")
+ for name, prop in schema["properties"].items():
+ out.append(f"<tr><td><code>{e(name)}</code></td><td>{e(prop['type'])}</td>"
+ f"<td>{'yes' if name in schema['required'] else ''}</td>"
+ f"<td>{e(prop.get('description', ''))}</td></tr>")
+ out.append("</table>")
+ out.append("</body></html>")
+ return "".join(out)
+
+
PAGE = """<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>Google Maps Scraper Kit</title>
@@ -114,15 +167,21 @@ td.err {{ color: var(--muted); font-size: .85em; }} .row {{ display: flex; gap:
</style></head><body>
<h1>Google Maps Scraper Kit</h1>
<p class="muted">Local engine {version}. <a href="{repo}" rel="noopener">GitHub</a> ·
-<a href="/api/docs">API</a> · Business listings only. Results stay on this machine.</p>
+<a href="/api/docs">API</a> · Business listings only. Results stay on this machine.
+A job the engine was stopped in the middle of carries on when it starts again.</p>
<p class="muted" id="sponsors"><a href="https://github.com/sponsors/Mahanaicoach" rel="noopener">Become a sponsor</a></p>
<form id="f">
<label class="wide">Keywords, one per line<textarea id="keywords" rows="3" required
placeholder="coffee shops in Austin TX"></textarea></label>
+ <label class="wide">Add a business type<input id="kwpick" list="kwlist" autocomplete="off"
+ placeholder="start typing: dentist, roofing contractor…"><datalist id="kwlist"></datalist>
+ <span class="muted">Google's business categories, list by <a href="{categories_repo}" rel="noopener">lobstr.io</a>.
+ Pick one to add it as a keyword line.</span></label>
<label>Latitude<input id="lat" required placeholder="30.2672" inputmode="decimal"></label>
<label>Longitude<input id="lon" required placeholder="-97.7431" inputmode="decimal"></label>
<label>Depth (about 20 places each)<input id="depth" type="number" min="1" max="1000" value="5"></label>
<label>Language<input id="lang" value="en" maxlength="2" pattern="[A-Za-z]{{2}}"></label>
+ <label>Country (optional)<input id="country" maxlength="2" pattern="[A-Za-z]{{2}}" placeholder="us"></label>
<label>Time limit (seconds)<input id="max_time" type="number" min="1" max="86400" value="600"></label>
<label><span>Emails</span><span><input type="checkbox" id="email" checked> visit each website</span></label>
<div class="wide row"><button class="primary">Start scraping</button>
@@ -147,11 +206,19 @@ function api(method, path, body) {{
function say(t) {{ $('msg').textContent = t; }}
function cell(tr, text, cls) {{ const td = document.createElement('td'); td.textContent = text; if (cls) td.className = cls; tr.appendChild(td); return td; }}
function button(td, label, fn) {{ const b = document.createElement('button'); b.type = 'button'; b.textContent = label; b.addEventListener('click', fn); td.appendChild(b); }}
-async function download(j) {{
- const r = await api('GET', '/api/v1/jobs/' + j.ID + '/download');
+async function download(j, fmt) {{
+ const r = await api('GET', '/api/v1/jobs/' + j.ID + '/download' + (fmt === 'json' ? '?format=json' : ''));
if (!r.ok) return say('No results yet for this job.');
const a = document.createElement('a'); a.href = URL.createObjectURL(await r.blob());
- a.download = 'results-' + j.ID.slice(0, 8) + '.csv'; document.body.appendChild(a); a.click(); a.remove();
+ a.download = 'results-' + j.ID.slice(0, 8) + '.' + fmt; document.body.appendChild(a); a.click(); a.remove();
+}}
+function progress(j) {{
+ const p = j.Progress || {{}};
+ if (j.Status === 'pending') return p.keywords_done ? 'pending (resumes at keyword ' + (p.keywords_done + 1) + ')' : 'pending';
+ if (j.Status !== 'working') return j.Status + (p.places ? ' · ' + p.places + ' places' : '');
+ const stage = {{search: 'keyword ' + Math.min(p.keywords_done + 1, p.keywords_total) + '/' + p.keywords_total,
+ reviews: 'reading reviews', websites: 'checking websites'}}[p.stage] || p.stage || '';
+ return 'working · ' + stage + ' · ' + (p.places || 0) + ' places · ' + (p.percent || 0) + '%';
}}
async function remove(j) {{
if (!confirm('Delete this job and its results?')) return;
@@ -164,9 +231,9 @@ async function load() {{
for (const j of list) {{
const tr = document.createElement('tr');
const name = cell(tr, j.Name); if (j.Error) {{ const s = document.createElement('div'); s.className = 'muted'; s.textContent = j.Error; name.appendChild(s); }}
- cell(tr, j.Date.replace('T', ' ').replace('+00:00', '')); cell(tr, j.Status);
+ cell(tr, j.Date.replace('T', ' ').replace('+00:00', '')); cell(tr, progress(j));
const act = cell(tr, ''); act.className = 'row';
- if (j.Status !== 'pending') button(act, 'Download', () => download(j));
+ if (j.Status !== 'pending' || (j.Progress || {{}}).places) {{ button(act, 'CSV', () => download(j, 'csv')); button(act, 'JSON', () => download(j, 'json')); }}
button(act, 'Delete', () => remove(j));
tb.appendChild(tr);
}}
@@ -177,6 +244,7 @@ $('f').addEventListener('submit', async ev => {{
ev.preventDefault();
const keywords = $('keywords').value.split('\\n').map(s => s.trim()).filter(Boolean);
const body = {{name: keywords[0] || 'job', keywords, lang: $('lang').value.trim().toLowerCase(),
+ country: $('country').value.trim().toLowerCase(),
lat: $('lat').value.trim(), lon: $('lon').value.trim(), depth: +$('depth').value,
max_time: +$('max_time').value, email: $('email').checked, zoom: 15}};
try {{
@@ -186,6 +254,29 @@ $('f').addEventListener('submit', async ev => {{
}} catch (e) {{ say(e.message); }}
}});
load(); setInterval(load, 3000);
+$('kwpick').addEventListener('change', () => {{ // a picked business type becomes a keyword line
+ const v = $('kwpick').value.trim(); if (!v) return;
+ const k = $('keywords'); k.value = (k.value.trim() ? k.value.trim() + '\\n' : '') + v; $('kwpick').value = '';
+}});
+(function () {{ // keyword ideas: Google's business category names, cached for 30 days, as text only
+ const key = 'gmskit-categories', month = 30 * 86400000;
+ function fill(names) {{
+ const list = $('kwlist'); list.replaceChildren();
+ for (const n of names) {{ const o = document.createElement('option'); o.value = n; list.appendChild(o); }}
+ }}
+ function clean(text) {{
+ return text.split('\\n').map(s => s.trim()).filter(s => s && s.length <= 80).slice(0, 5000);
+ }}
+ try {{
+ const c = JSON.parse(localStorage.getItem(key) || 'null');
+ if (c && Date.now() - c.t < month && Array.isArray(c.names)) return fill(c.names);
+ }} catch (e) {{}}
+ fetch('{categories}', {{credentials: 'omit'}}).then(r => r.ok ? r.text() : '').then(t => {{
+ const names = clean(t); if (!names.length) return;
+ fill(names);
+ try {{ localStorage.setItem(key, JSON.stringify({{t: Date.now(), names}})); }} catch (e) {{}}
+ }}).catch(() => {{}});
+}})();
(function () {{ // sponsor box: the "ui": true entries of sponsors.json, as text, https links only
const box = $('sponsors'), key = 'gmskit-sponsors', day = 86400000;
function render(list) {{
@@ -258,7 +349,7 @@ class Handler(BaseHTTPRequestHandler):
self.fail(403, "requests from other websites are refused")
return False
key, path = self.server.api_key, self.path.split("?")[0]
- if key and path not in ("/", "/api/docs") and not hmac.compare_digest(
+ if key and path not in ("/", "/api/docs", "/api/openapi.json") and not hmac.compare_digest(
self.headers.get("X-API-Key", "").encode(), key.encode()):
self.fail(401, "missing or wrong X-API-Key")
return False
@@ -283,10 +374,17 @@ class Handler(BaseHTTPRequestHandler):
self.fail(400, "the body is not valid JSON")
return _REPLIED
- def download(self, jid):
+ def download(self, jid, query=""):
data = self.server.store.read_csv(jid) if JOB_ID.match(jid or "") else None
if data is None:
return self.fail(404, "no results for this job (yet)")
+ fmt = urllib.parse.parse_qs(query).get("format", ["csv"])[0]
+ if fmt == "json": # the same rows, every value a string, as in the CSV
+ rows = list(csv.DictReader(io.StringIO(data.decode("utf-8", "replace"))))
+ return self.reply(200, json.dumps(rows, ensure_ascii=False).encode(), "application/json; charset=utf-8",
+ [("Content-Disposition", f'attachment; filename="{jid}.json"')])
+ if fmt != "csv":
+ return self.fail(400, "format must be csv or json")
self.reply(200, data, "text/csv; charset=utf-8",
[("Content-Disposition", f'attachment; filename="{jid}.csv"')])
@@ -295,26 +393,32 @@ class Handler(BaseHTTPRequestHandler):
return
path, _, query = self.path.partition("?")
store = self.server.store
- if path == "/api/v1/jobs":
- return self.reply(200, store.all() or None) # null when empty, as before: the docs' health check
+ if path == "/api/v1/jobs": # null when empty, as before: the docs' health check
+ return self.reply(200, [jobs.public(j) for j in store.all()] or None)
m = JOB_PATH.match(path)
if m:
if m.group(2):
- return self.download(m.group(1))
+ return self.download(m.group(1), query)
job = store.get(m.group(1))
- return self.reply(200, job) if job else self.fail(404, "job not found")
+ return self.reply(200, jobs.public(job)) if job else self.fail(404, "job not found")
if path == "/download": # the old web UI's download link
- return self.download(urllib.parse.parse_qs(query).get("id", [""])[0])
+ return self.download(urllib.parse.parse_qs(query).get("id", [""])[0], query)
+ if path == "/api/docs":
+ nonce = secrets.token_urlsafe(16)
+ csp = f"default-src 'none'; style-src 'nonce-{nonce}'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"
+ return self.reply(200, docs_page(nonce).encode(), "text/html; charset=utf-8",
+ [("Content-Security-Policy", csp)])
+ if path == "/api/openapi.json":
+ return self.reply(200, OPENAPI)
if path == "/":
nonce = secrets.token_urlsafe(16)
csp = (f"default-src 'none'; script-src 'nonce-{nonce}'; style-src 'nonce-{nonce}'; "
"connect-src 'self' https://raw.githubusercontent.com; img-src 'self' data:; "
"form-action 'self'; base-uri 'none'; "
"frame-ancestors 'none'")
- page = PAGE.format(nonce=nonce, version=VERSION, repo=REPO, sponsors=SPONSORS).encode()
+ page = PAGE.format(nonce=nonce, version=VERSION, repo=REPO, sponsors=SPONSORS, categories=CATEGORIES,
+ categories_repo=CATEGORIES_REPO).encode()
return self.reply(200, page, "text/html; charset=utf-8", [("Content-Security-Policy", csp)])
- if path == "/api/docs":
- return self.reply(200, OPENAPI)
self.fail(404, "not found")
def do_POST(self):
@@ -414,9 +518,14 @@ def main():
browser.make_home()
browser.sweep_profiles()
store = jobs.Store(a.data)
- if n := store.recover():
- log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")
- worker = jobs.Worker(store, log)
+ resumed, failed = store.recover()
+ if resumed:
+ log(f"{resumed} job(s) were still working when the engine last stopped: they resume from their last "
+ "finished keyword")
+ if failed:
+ log(f"{failed} job(s) stopped mid-run {jobs.MAX_RESUMES + 1} times: marked failed, rows kept")
+ headless = env["SCRAPER_HEADLESS"].strip().lower() not in ("0", "false", "no", "off")
+ worker = jobs.Worker(store, log, headless=headless)
try:
servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"], a.host)
except OSError as e:
@@ -425,7 +534,7 @@ def main():
chrome = browser.find_chrome()
where = f"http://localhost:{port}" if a.host == "127.0.0.1" else \
f"{a.host}:{port} in this container, published on the host's 127.0.0.1"
- log(f"engine {VERSION} on {where} (browser: {chrome or 'none found'}"
+ log(f"engine {VERSION} on {where} (browser: {chrome or 'none found'}{'' if headless else ', as a window'}"
f"{', API key required' if env['SCRAPER_API_KEY'] else ''})")
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
for s in servers[1:]:
@@ -435,7 +544,7 @@ def main():
except KeyboardInterrupt:
pass
finally:
- worker.shutdown()
+ worker.shutdown(wait=8) # the job in progress saves its place and resumes on the next start
log("engine stopped")
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 5256053..1f3f32a 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -6,12 +6,13 @@ No network, no Chrome, no Docker.
python3 engine/test_engine.py
"""
-import csv, hashlib, http.client, importlib.util, io, json, os, re, shutil, socket, subprocess, sys, tempfile
+import csv, hashlib, http.client, importlib.util, io, json, os, queue, re, shutil, socket, sqlite3, subprocess, sys
+import tempfile
import threading, time, unittest, unittest.mock, zipfile
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
-import browser, jobs, maps, serve # noqa: E402
+import browser, details, jobs, maps, serve # noqa: E402
from cdp import CDPError # noqa: E402
GOOD = {"name": "t", "keywords": ["cafes in Austin"], "lang": "en", "lat": "30.2", "lon": "-97.7", "max_time": 60}
@@ -32,9 +33,16 @@ class FakeBrowser:
made = []
def __init__(self, **kw):
- self.kw, self.cdp, self.killed = kw, FakeCDP(), threading.Event()
+ self.kw, self.cdp, self.killed, self.contexts = kw, FakeCDP(), threading.Event(), []
FakeBrowser.made.append(self)
+ def context(self, proxy=None):
+ self.contexts.append(proxy)
+ return f"ctx{len(self.contexts)}"
+
+ def dispose(self, ctx):
+ pass
+
def kill(self):
self.cdp.closed = OSError("killed")
self.killed.set()
@@ -44,24 +52,42 @@ class FakeBrowser:
class FakePage:
- def __init__(self, cdp):
+ def __init__(self, cdp, context=None):
pass
def close(self):
pass
-RESULTS = {} # keyword -> (places, reason) for the fake collect
+class FakeAuth:
+ """Stands in for browser.ProxyAuth: records which tabs got credentials."""
+ logins = []
+ refused = False
+
+ def __init__(self, page, user, password):
+ FakeAuth.logins.append((user, password))
+
+ def close(self):
+ pass
-def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False):
+RESULTS = {} # keyword -> (places, reason) for the fake collect; a list of reasons answers one per search
+SEARCHED = [] # the keywords the fake collect was asked for, in order
+
+
+def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False, on_batch=None):
for kw, (places, reason) in RESULTS.items():
if maps.search_url(kw).split("/@")[0].split("?")[0] in url:
+ SEARCHED.append(kw)
+ if isinstance(reason, list):
+ reason = reason.pop(0) if len(reason) > 1 else reason[0]
if reason == "hang": # Chrome stops answering until the watchdog kills it
b = FakeBrowser.made[-1]
b.killed.wait(10)
raise CDPError("connection lost: killed")
- if reason == "slow":
+ if reason == "slow": # its places stream in, then it scrolls until stopped
+ if on_batch and places:
+ on_batch(places, 1)
end = time.time() + 5
while not stop() and time.time() < end:
time.sleep(0.02)
@@ -73,16 +99,18 @@ def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False
class Fakes(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
- self.saved = (browser.Browser, jobs.Page, maps.collect)
- browser.Browser, jobs.Page, maps.collect = FakeBrowser, FakePage, fake_collect
+ self.saved = (browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth)
+ browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth = FakeBrowser, FakePage, fake_collect, FakeAuth
FakeBrowser.made.clear()
+ FakeAuth.logins.clear()
RESULTS.clear()
+ SEARCHED.clear()
self.store = jobs.Store(os.path.join(self.dir, "data"))
self.lines = []
self.worker = jobs.Worker(self.store, self.lines.append)
def tearDown(self):
- browser.Browser, jobs.Page, maps.collect = self.saved
+ browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth = self.saved
self.worker.shutdown()
if self.worker.is_alive():
self.worker.join(5)
@@ -123,7 +151,9 @@ class Validate(unittest.TestCase):
self.assertIn("invalid depth", self.err(**dict(GOOD, depth="5")))
self.assertIn("invalid max time", self.err(**dict(GOOD, max_time=-1)))
self.assertIn("invalid keywords", self.err(**dict(GOOD, keywords=["ok", ""])))
- self.assertIn("usernames and passwords", self.err(**dict(GOOD, proxies=["http://u:p@h:1"])))
+ self.assertIn("SOCKS5", self.err(**dict(GOOD, proxies=["socks5://u:p@h:1"])))
+ self.assertIn("invalid country", self.err(**dict(GOOD, country="usa")))
+ self.assertIn("invalid reviews_max", self.err(**dict(GOOD, reviews_max=6)))
self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["ftp://h:1"])))
self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["http://h:99999"])))
with self.assertRaisesRegex(ValueError, "JSON object"):
@@ -133,6 +163,11 @@ class Validate(unittest.TestCase):
name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))
self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))
self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])
+ self.assertEqual((d["country"], d["reviews_max"], d["socials"], d["verify_emails"]), ("", 0, False, False))
+ d = jobs.validate(dict(GOOD, country="DE", proxies=["http://u%40x:p%3A@h:1"]))[1]
+ self.assertEqual((d["country"], jobs.locale(d), d["proxies"]), ("de", "en-DE", ["http://u%40x:p%3A@h:1"]))
+ self.assertEqual(jobs.mask("http://u%40x:p%3A@h:1"), "http://u%40x:***@h:1")
+ self.assertEqual(jobs.mask("socks5://h:1"), "socks5://h:1")
self.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)
@@ -162,15 +197,16 @@ class Store(Fakes):
self.assertEqual([j["ID"] for j in self.store.all()], [b, a]) # newest first
self.assertEqual(self.store.claim()["ID"], a) # oldest pending first
self.assertEqual(self.store.get(a)["Status"], "working")
- self.assertEqual(self.store.recover(), 1)
- self.assertEqual(self.store.get(a)["Status"], "failed")
- self.assertTrue(self.store.write_csv(b, []))
+ self.assertEqual(self.store.recover(), (1, 0)) # resumed on the next start
+ self.assertEqual((self.store.get(a)["Status"], self.store.get(a)["Progress"]["resumes"]), ("pending", 1))
+ self.assertEqual(self.store.append(b, [], jobs.fields({})), len(",".join(jobs.COLUMNS + jobs.EXTRA)) + 2)
self.assertTrue(self.store.read_csv(b).startswith(b"input_id,link,title"))
self.assertTrue(self.store.delete(b))
self.assertIsNone(self.store.read_csv(b))
- self.assertFalse(self.store.write_csv(b, [])) # a deleted job gets no file back
+ self.assertIsNone(self.store.append(b, [], jobs.fields({}))) # a deleted job gets no file back
+ self.assertFalse(self.store.finish(b, [], jobs.fields({}), "ok", "", {}))
self.assertFalse(self.store.delete(b))
- self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error"})
+ self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error", "Progress"})
self.assertRegex(self.store.get(a)["Date"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00$")
@@ -183,7 +219,8 @@ class Worker(Fakes):
rows = self.rows(jid)
self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2", "Biz 3"])
self.assertEqual([r["input_id"] for r in rows], ["cafes", "cafes", "bakeries"])
- self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "proxy": "http://10.0.0.1:3128"})
+ self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "headless": True})
+ self.assertEqual(FakeBrowser.made[0].contexts, ["http://10.0.0.1:3128"] * 2)
self.assertFalse(any(re.search(r"Biz \d", l) for l in self.lines)) # counts only in the log
def test_no_results_is_ok_with_header(self):
@@ -241,7 +278,7 @@ class Worker(Fakes):
def test_email_pass_stops_when_asked(self):
rows = [{"website": f"https://s{i}.com", "emails": ""} for i in range(3)]
t = time.monotonic()
- jobs.add_emails(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)
+ jobs.add_site_data(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)
self.assertLess(time.monotonic() - t, 3)
def test_watchdog_kills_a_hung_chrome(self):
@@ -282,6 +319,7 @@ class Worker(Fakes):
jobs.fetch = saved
self.assertEqual(job["Status"], "ok")
self.assertEqual([r["emails"] for r in self.rows(jid)], ["[email protected], [email protected]", "", ""])
+ self.assertNotIn("instagram", self.rows(jid)[0]) # columns only for the passes a job asked for
class Emails(unittest.TestCase):
@@ -354,7 +392,7 @@ class Api(Server):
job = json.loads(self.call("GET", f"/api/v1/jobs/{jid}")[1])
self.assertEqual((job["ID"], job["Name"], job["Status"], job["Data"]["keywords"]), (jid, "t", "pending", ["cafes in Austin"]))
self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download")[0], 404)
- self.store.write_csv(jid, [jobs.to_row(place(1), "k")])
+ self.store.append(jid, [jobs.to_row(place(1), "k")], jobs.fields({}))
code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download")
self.assertEqual((code, r.getheader("Content-Type")), (200, "text/csv; charset=utf-8"))
self.assertEqual(self.call("GET", f"/download?id={jid}")[1], body)
@@ -396,7 +434,7 @@ class Api(Server):
self.assertIn("connect-src 'self' https://raw.githubusercontent.com;", csp) # sponsors.json, nothing else
self.assertIn("assets/sponsors/sponsors.json", body.decode())
self.assertNotRegex(body.decode(), r"\{[a-z_]+\}") # every placeholder filled
- code, body, _ = self.call("GET", "/api/docs")
+ code, body, _ = self.call("GET", "/api/openapi.json")
self.assertEqual(json.loads(body)["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]
["schema"]["required"], ["name", "keywords", "lang", "lat", "lon", "max_time"])
@@ -587,9 +625,10 @@ class LaunchChrome(unittest.TestCase):
os.mkdir(profile)
b = None
with self.assertRaisesRegex(browser.BrowserError, "DevTools"):
- b = browser.Browser(path=self.exe, profile=profile, proxy="socks5h://10.0.0.1:1080", timeout=5)
+ b = browser.Browser(path=self.exe, profile=profile, headless=False, timeout=5)
self.assertIsNone(b)
- self.assertIn("--proxy-server=socks5://10.0.0.1:1080", self.launches()[0])
+ self.assertIn("--window-position=-3000,-3000", self.launches()[0]) # a window, out of sight
+ self.assertNotIn("--headless", self.launches()[0])
out = subprocess.run(["ps", "-axo", "command"], capture_output=True, text=True).stdout
self.assertNotIn(self.exe, out) # the fake Chrome was stopped
@@ -656,5 +695,345 @@ class EngineCli(unittest.TestCase):
p.wait()
+def wait_until(cond, seconds=5):
+ end = time.time() + seconds
+ while not cond() and time.time() < end:
+ time.sleep(0.02)
+ return cond()
+
+
+class Resume(Fakes):
+ """P3: rows stream into the CSV, Progress says how far a job is, and a job the engine was stopped
+ in the middle of carries on from its last finished keyword."""
+
+ def start(self, **body):
+ name, data = jobs.validate(dict(GOOD, **body))
+ jid = self.store.create(name, data)
+ t = threading.Thread(target=self.worker.run_job, args=(self.store.claim(),))
+ t.start()
+ return jid, t
+
+ def test_rows_stream_in_and_a_stopped_engine_resumes(self):
+ RESULTS["cafes"] = ([place(1)], "end-of-list")
+ RESULTS["bakeries"] = ([place(2), place(3)], "slow")
+ RESULTS["florists"] = ([place(4)], "end-of-list")
+ jid, t = self.start(keywords=["cafes", "bakeries", "florists"])
+ self.assertTrue(wait_until(lambda: self.store.get(jid)["Progress"].get("places") == 3))
+ self.assertEqual(len(self.rows(jid)), 3) # bakeries' places are in before the keyword ends
+ prog = jobs.public(self.store.get(jid))["Progress"]
+ self.assertEqual((prog["keywords_done"], prog["keywords_total"], prog["places"], prog["stage"]), (1, 3, 3, "search"))
+ self.assertTrue(33 < prog["percent"] <= 66)
+ self.worker.shutdown(wait=5) # the engine stops mid-keyword
+ t.join(5)
+ self.assertEqual(self.store.get(jid)["Status"], "pending")
+ RESULTS["bakeries"] = ([place(2), place(3)], "end-of-list")
+ SEARCHED.clear()
+ jobs.Worker(self.store, self.lines.append).run_job(self.store.claim()) # the next start
+ job = self.store.get(jid)
+ self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
+ self.assertEqual(SEARCHED, ["bakeries", "florists"]) # cafes was finished, bakeries was not
+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1", "Biz 2", "Biz 3", "Biz 4"])
+ self.assertEqual(jobs.public(job)["Progress"], {"keywords_done": 3, "keywords_total": 3, "places": 4,
+ "percent": 100, "stage": "done"})
+
+ def crashed(self, keywords, **progress):
+ """A job an engine crashed in: one keyword finished, part of the next one appended, a torn line."""
+ name, data = jobs.validate(dict(GOOD, keywords=keywords))
+ jid = self.store.create(name, data)
+ size = self.store.append(jid, [jobs.to_row(place(1), keywords[0])], jobs.fields(data))
+ self.store.append(jid, [jobs.to_row(place(2), keywords[1])], jobs.fields(data))
+ with open(self.store.csv_path(jid), "ab") as f:
+ f.write(b'"cafes,Biz 9,"half a ro')
+ self.store.claim()
+ self.store.set_progress(jid, dict({"done": 1, "total": len(keywords), "bytes": size, "stage": "search"}, **progress))
+ return jid, size
+
+ def test_crash_cuts_the_unfinished_keyword_and_resumes(self):
+ jid, size = self.crashed(["cafes", "bakeries"], elapsed=20)
+ self.assertEqual(self.store.recover(), (1, 0))
+ self.assertEqual(os.path.getsize(self.store.csv_path(jid)), size) # torn line and bakeries' row gone
+ RESULTS["bakeries"] = ([place(2), place(3)], "end-of-list")
+ self.worker.run_job(self.store.claim())
+ self.assertEqual(SEARCHED, ["bakeries"])
+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1", "Biz 2", "Biz 3"])
+ self.assertTrue(any("resumed at keyword 2/2 with 1 places, 40 s left" in l for l in self.lines))
+
+ def test_too_many_restarts_fail_as_before(self):
+ jid, size = self.crashed(["cafes", "bakeries"], resumes=jobs.MAX_RESUMES)
+ self.assertEqual(self.store.recover(), (0, 1))
+ job = self.store.get(jid)
+ self.assertEqual(job["Status"], "failed")
+ self.assertIn("the engine stopped while this job ran (4 times); its rows so far are kept", job["Error"])
+ self.assertEqual(os.path.getsize(self.store.csv_path(jid)), size)
+
+ def test_no_time_left_ends_without_searching(self):
+ jid, _ = self.crashed(["cafes", "bakeries"], elapsed=60)
+ self.store.recover()
+ self.worker.run_job(self.store.claim())
+ job = self.store.get(jid)
+ self.assertEqual((job["Status"], SEARCHED, FakeBrowser.made), ("ok", [], []))
+ self.assertIn("stopped at max_time", job["Error"])
+ self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1"])
+
+ def test_old_store_gets_the_progress_column(self):
+ folder = os.path.join(self.dir, "old")
+ os.makedirs(folder)
+ db = sqlite3.connect(os.path.join(folder, "jobs.db"))
+ db.execute("CREATE TABLE jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, date TEXT NOT NULL, "
+ "status TEXT NOT NULL, data TEXT NOT NULL, error TEXT NOT NULL DEFAULT '')")
+ db.execute("INSERT INTO jobs VALUES ('x', 'n', 'd', 'ok', '{}', '')")
+ db.commit()
+ db.close()
+ old = jobs.Store(folder)
+ self.assertEqual(old.get("x")["Progress"], {})
+ old.db.close()
+
+
+class Backoff(Fakes):
+ def setUp(self):
+ super().setUp()
+ self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)
+ jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01
+
+ def tearDown(self):
+ jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits
+ super().tearDown()
+
+ def test_retry_through_the_next_proxy(self):
+ RESULTS["cafes in Austin"] = ([place(1)], ["blocked", "end-of-list"])
+ jid, job = self.run_one(proxies=["http://a:1", "http://b:2"])
+ self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
+ self.assertEqual(FakeBrowser.made[0].contexts, ["http://a:1", "http://b:2"])
+ self.assertTrue(any("retry 1/2 in 0.01 s through the next proxy" in l for l in self.lines))
+
+ def test_gives_up_after_bounded_retries(self):
+ RESULTS["cafes in Austin"] = ([], ["blocked"])
+ _, job = self.run_one()
+ self.assertEqual(job["Status"], "failed")
+ self.assertIn("unusual-traffic page", job["Error"]) # what scripts/scrape.py looks for
+ self.assertIn("Waited and retried 2 time(s)", job["Error"])
+ self.assertEqual(SEARCHED, ["cafes in Austin"] * 3)
+
+ def test_a_failed_proxy_hands_the_keyword_to_the_next(self):
+ calls = []
+ def collect(page, url, **kw):
+ calls.append(FakeBrowser.made[-1].contexts[-1])
+ if len(calls) == 1:
+ raise CDPError("navigate https://x: net::ERR_PROXY_CONNECTION_FAILED")
+ return [place(1)], "end-of-list", {"batches": 1, "dom_links": 0}
+ maps.collect = collect
+ jid, job = self.run_one(proxies=["http://dead:1", "http://ok:2"])
+ self.assertEqual((job["Status"], job["Error"], calls), ("ok", "", ["http://dead:1", "http://ok:2"]))
+ self.assertEqual(len(self.rows(jid)), 1)
+
+ def test_proxies_take_turns(self):
+ for k in ("a", "b", "c"):
+ RESULTS[k] = ([], "end-of-list")
+ self.run_one(keywords=["a", "b", "c"], proxies=["http://u:pw-s3cret@p1:1", "http://p2:2"])
+ self.assertEqual(FakeBrowser.made[0].contexts, ["http://u:pw-s3cret@p1:1", "http://p2:2", "http://u:pw-s3cret@p1:1"])
+ self.assertFalse(any("s3cret" in l for l in self.lines))
+ self.assertEqual(FakeAuth.logins, [("u", "pw-s3cret")] * 2) # only the tabs on the proxy with a login
+
+
+class ProxyLogin(unittest.TestCase):
+ """browser.ProxyAuth against a fake DevTools connection: continue every paused request, answer
+ the proxy's 407 once with the job's credentials, then give up (wrong credentials)."""
+
+ def test_answers(self):
+ class C:
+ closed = None
+ def __init__(self):
+ self.sent = []
+ def subscribe(self, method, sid=None, q=None):
+ self.q = q or queue.Queue()
+ return self.q
+ def unsubscribe(self, *a):
+ pass
+ class P:
+ sid = "s"
+ def __init__(self):
+ self.cdp = C()
+ def send(self, method, params=None, timeout=None):
+ self.cdp.sent.append((method, params))
+ return {}
+ page = P()
+ auth = browser.ProxyAuth(page, "user", "pw")
+ q = page.cdp.q
+ q.put({"requestId": "1", "request": {"url": "https://www.google.com/"}})
+ q.put({"requestId": "2", "authChallenge": {"source": "Proxy"}})
+ q.put({"requestId": "2", "authChallenge": {"source": "Proxy"}})
+ q.put({"requestId": "3", "authChallenge": {"source": "Server"}})
+ self.assertTrue(wait_until(lambda: len(page.cdp.sent) == 5))
+ auth.close()
+ sent = page.cdp.sent
+ self.assertEqual(sent[0], ("Fetch.enable", {"handleAuthRequests": True, "patterns": [{"urlPattern": "*"}]}))
+ self.assertEqual(sent[1], ("Fetch.continueRequest", {"requestId": "1"}))
+ self.assertEqual([m[1]["authChallengeResponse"]["response"] for m in sent[2:]],
+ ["ProvideCredentials", "CancelAuth", "Default"])
+ self.assertEqual(sent[2][1]["authChallengeResponse"]["username"], "user")
+ self.assertTrue(auth.refused) # the job then says "proxy login refused", not Chrome's vague error
+ self.assertIn("proxy login refused", jobs.LOGIN_REFUSED)
+ self.assertTrue(jobs.PROXY_FAILED.search(jobs.LOGIN_REFUSED))
+ self.assertFalse(jobs.PROXY_FAILED.search("navigate https://x/maps/search/tunnel%20car%20wash: net::ERR_TIMED_OUT"))
+
+ def test_split(self):
+ self.assertEqual(browser.split_proxy("http://u%40x:p%3A@h:3128"), ("http://h:3128", "u@x", "p:"))
+ self.assertEqual(browser.split_proxy("socks5h://[::1]:1080"), ("socks5://[::1]:1080", "", ""))
+
+
+def entity(days=7, reviews=False):
+ """A place entity shaped like Google's (made-up values) with the full view's fields."""
+ e = [None] * 260
+ e[11], e[10], e[78] = "Biz", "0x1:0x2", "ChIJx"
+ e[4] = [None, None, None, ["https://search.google.com/local/reviews?placeid=ChIJx", "349 reviews"],
+ None, None, None, 4.7, 349]
+ names = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
+ week = [[names[n - 1], n, [2026, 11, n], [["7\u202fAM\u20139\u202fPM", [[7], [21]]]], 0, 1] for n in (6, 7, 1, 2, 3, 4, 5)]
+ e[203] = [week[:days], [week[0], 0, 4, None, ["Open \u00b7 Closes 9\u202fPM"]]]
+ if reviews:
+ def review(stars, text):
+ r = [None] * 4
+ r[0] = "id"
+ r[1] = [None, None, 1, 1, [None, None, ["https://example.com/u"], None, None, ["A Reviewer"]], None, "4 months ago"]
+ r[2] = [[stars]] + [None] * 14 + [[[text]]]
+ return [r]
+ e[175] = [None] * 10
+ e[175][3] = [6, 12, 6, 24, 301]
+ e[175][9] = [[[review(5, "Great\u202fcoffee"), review(2, "Slow"), review(4, "")]]]
+ return e
+
+
+class FullView(unittest.TestCase):
+ """details.py fields that only Google's full view carries (a Chrome window), from made-up data."""
+
+ def test_count_link_and_week(self):
+ out = details.parse_entity(entity())
+ self.assertEqual((out["review_count"], out["review_rating"]), (349, 4.7))
+ self.assertEqual(out["reviews_link"], "https://search.google.com/local/reviews?placeid=ChIJx")
+ week = json.loads(out["open_hours"])
+ self.assertEqual(list(week)[:2], ["Monday", "Tuesday"])
+ self.assertEqual(week["Sunday"], ["7 AM\u20139 PM"])
+ self.assertEqual(out["open_state"], "Open \u00b7 Closes 9 PM" if False else out["open_state"])
+
+ def test_limited_view_gives_no_week(self):
+ self.assertNotIn("open_hours", details.parse_entity(entity(days=1))) # today only is not a week
+
+ def test_place_panel_reviews_without_names(self):
+ body = ")]}'\n" + json.dumps([None] * 6 + [entity(reviews=True)])
+ out = details.parse_place(body, limit=2)
+ self.assertEqual(json.loads(out["reviews_per_rating"]), {"1": 6, "2": 12, "3": 6, "4": 24, "5": 301})
+ self.assertEqual(json.loads(out["user_reviews"]), [{"Rating": 5, "When": "4 months ago", "Description": "Great coffee"},
+ {"Rating": 2, "When": "4 months ago", "Description": "Slow"}])
+ self.assertNotIn("Reviewer", json.dumps(out))
+ self.assertEqual(details.parse_place("not json"), {})
+
+
+class Locale(unittest.TestCase):
+ def test_search_url_and_end_of_list(self):
+ self.assertEqual(maps.search_url("a b", 1, 2, 15, "de", "at"), "https://www.google.com/maps/search/a%20b/@1,2,15z?hl=de&gl=at")
+ self.assertTrue(maps.search_url("x").endswith("?hl=en"))
+ self.assertTrue(maps.ended({"cls": True, "text": ""}))
+ self.assertTrue(maps.ended({"cls": False, "text": "… Vous êtes arrivé à la fin de la liste."}))
+ self.assertFalse(maps.ended({"cls": False, "text": "Biz 1"}))
+ self.assertFalse(maps.ended(None))
+ self.assertIn('input[name="set_eom"][value="true"]', maps.JS_CONSENT) # the reject form, any language
+ self.assertIn("tout refuser", maps.JS_CONSENT)
+
+ def test_429_marks_the_batches_limited(self):
+ q = queue.Queue()
+ class C:
+ closed = None
+ def subscribe(self, *a):
+ return q
+ class P:
+ sid, cdp = "s", C()
+ b = maps.Batches(P())
+ q.put({"requestId": "1", "response": {"url": "https://www.google.com/search?tbm=map&pb=x", "status": 429,
+ "mimeType": "text/html"}})
+ self.assertEqual((b.drain(), b.limited, b.pending), ([], 1, {}))
+
+
+class Websites(unittest.TestCase):
+ def test_socials_and_mx(self):
+ rows = [{"website": "https://shop.com", "emails": ""}, {"website": "https://gone.com", "emails": ""}]
+ pages = {"https://shop.com": '[email protected] <a href="https://www.instagram.com/shopco/">ig</a> '
+ '<a href="https://facebook.com/sharer">x</a> <a href="https://facebook.com/shopco">fb</a>',
+ "https://gone.com": "[email protected]"}
+ hosts = {"shop.com": "google_workspace", "gone-forever.com": "nxdomain"}
+ with unittest.mock.patch.object(jobs, "mx_host", side_effect=hosts.get):
+ n = jobs.add_site_data(rows, 10, socials=True, verify=True, get=pages.get)
+ self.assertEqual(n, 1)
+ self.assertEqual((rows[0]["emails"], rows[0]["email_host"]), ("[email protected]", "google_workspace"))
+ self.assertEqual((rows[0]["instagram"], rows[0]["facebook"]), ("https://www.instagram.com/shopco", "https://facebook.com/shopco"))
+ self.assertEqual(rows[1]["emails"], "") # its only address is on a domain that does not exist
+
+ def test_mx_answers(self):
+ def doh(answer):
+ r = unittest.mock.MagicMock()
+ r.__enter__.return_value = io.BytesIO(json.dumps(answer).encode())
+ return unittest.mock.patch.object(jobs._OPENER, "open", return_value=r)
+ jobs.mx_host.cache_clear()
+ with doh({"Status": 3}):
+ self.assertEqual(jobs.mx_host("a.test"), "nxdomain")
+ with doh({"Status": 0, "Answer": [{"type": 15, "data": "1 aspmx.l.google.com."}]}):
+ self.assertEqual(jobs.mx_host("b.test"), "google_workspace")
+ with doh({"Status": 0}):
+ self.assertEqual(jobs.mx_host("c.test"), "none")
+ jobs.mx_host.cache_clear()
+
+ def test_same_social_patterns_as_scrape(self):
+ sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))
+ import scrape
+ self.assertEqual({k: r.pattern for k, r in jobs.SOCIAL_RE.items()}, {k: r.pattern for k, r in scrape.SOCIAL_RE.items()})
+ self.assertEqual(jobs.SOCIALS, scrape.SOCIAL_FIELDS)
+ self.assertEqual(jobs.SKIP_HANDLE, scrape._SKIP_HANDLE)
+
+
+class ApiP3(Server):
+ def test_progress_json_and_docs(self):
+ code, body, _ = self.call("POST", "/api/v1/jobs", dict(GOOD, proxies=["http://u:pw-s3cret@h:1"]))
+ jid = json.loads(body)["id"]
+ code, body, _ = self.call("GET", f"/api/v1/jobs/{jid}")
+ job = json.loads(body)
+ self.assertEqual(job["Data"]["proxies"], ["http://u:***@h:1"])
+ self.assertNotIn(b"s3cret", body + self.call("GET", "/api/v1/jobs")[1])
+ self.assertEqual(job["Progress"], {"keywords_done": 0, "keywords_total": 1, "places": 0, "percent": 0,
+ "stage": "queued"})
+ self.store.append(jid, [jobs.to_row(place(1, title="Café"), "k")], jobs.fields({}))
+ code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download?format=json")
+ self.assertEqual((code, r.getheader("Content-Type")), (200, "application/json; charset=utf-8"))
+ rows = json.loads(body)
+ self.assertEqual((len(rows), rows[0]["title"], rows[0]["review_count"]), (1, "Café", "11"))
+ self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download?format=xml")[0], 400)
+ self.assertEqual(self.call("GET", f"/download?id={jid}&format=json")[1], body)
+ code, body, r = self.call("GET", "/api/docs")
+ page, csp = body.decode(), r.getheader("Content-Security-Policy")
+ self.assertEqual((code, r.getheader("Content-Type")), (200, "text/html; charset=utf-8"))
+ self.assertNotIn("<script", page)
+ nonce = re.search(r"nonce-([^']+)'", csp).group(1)
+ self.assertIn(f'<style nonce="{nonce}">', page)
+ self.assertNotIn("script-src", csp)
+ self.assertIn("<code>reviews_max</code>", page)
+ spec = json.loads(self.call("GET", "/api/openapi.json")[1])
+ self.assertEqual(spec["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]
+ ["schema"]["required"], ["name", "keywords", "lang", "lat", "lon", "max_time"])
+ ui = self.call("GET", "/")[1].decode()
+ self.assertIn('id="country"', ui)
+ self.assertIn('list="kwlist"', ui)
+ self.assertIn(serve.CATEGORIES, ui) # loaded by the browser from GitHub, pinned to a commit
+ self.assertRegex(serve.CATEGORIES, r"/[0-9a-f]{40}/")
+ script = re.search(r"<script nonce=[^>]+>(.*)</script>", ui, re.S).group(1)
+ self.assertNotIn("'\n'", script) # PAGE must say \\n: a bare newline in a JS string breaks the page
+
+
+class ApiKeyDocs(Server):
+ key = "s3cret"
+
+ def test_docs_need_no_key(self):
+ self.assertEqual(self.call("GET", "/api/docs")[0], 200)
+ self.assertEqual(self.call("GET", "/api/openapi.json")[0], 200)
+ self.assertEqual(self.call("GET", "/api/v1/jobs/00000000-0000-0000-0000-000000000000/download")[0], 401)
+
+
if __name__ == "__main__":
unittest.main(verbosity=1)
diff --git a/engine/test_proxy.py b/engine/test_proxy.py
new file mode 100644
index 0000000..e61a055
--- /dev/null
+++ b/engine/test_proxy.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""A local HTTP proxy that wants a username and password, for the live check of proxy logins
+(.github/workflows/engine-live.yml). Not part of the engine. Standard library only.
+
+ python3 engine/test_proxy.py 8191 user pass # then scrape with --proxy http://user:[email protected]:8191
+
+It answers 407 until a request carries the right Basic credentials, tunnels HTTPS (CONNECT) after
+that, and keeps the counts in engine/../.engine/test-proxy.json: {"tunnels": n, "refused": n}.
+"""
+import base64, json, os, select, socket, socketserver, sys, threading
+
+COUNTS = {"tunnels": 0, "refused": 0}
+LOCK = threading.Lock()
+OUT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".engine", "test-proxy.json")
+
+
+def count(key):
+ with LOCK:
+ COUNTS[key] += 1
+ os.makedirs(os.path.dirname(OUT), exist_ok=True)
+ with open(OUT, "w") as f:
+ json.dump(COUNTS, f)
+
+
+class Proxy(socketserver.BaseRequestHandler):
+ def head(self):
+ data = b""
+ while b"\r\n\r\n" not in data:
+ chunk = self.request.recv(4096)
+ if not chunk:
+ return None
+ data += chunk
+ return data.split(b"\r\n\r\n", 1)[0].decode("latin-1").split("\r\n")
+
+ def handle(self):
+ want = "Basic " + base64.b64encode(self.server.creds.encode()).decode()
+ while True: # Chrome may send its credentials on the same connection
+ lines = self.head()
+ if not lines:
+ return
+ method, target = (lines[0].split(" ") + ["", ""])[:2]
+ auth = [l.split(":", 1)[1].strip() for l in lines[1:] if l.lower().startswith("proxy-authorization:")]
+ if auth != [want]:
+ count("refused")
+ self.request.sendall(b'HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic '
+ b'realm="kit"\r\nContent-Length: 0\r\n\r\n')
+ continue
+ if method != "CONNECT":
+ self.request.sendall(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+ return
+ host, _, port = target.rpartition(":")
+ try:
+ up = socket.create_connection((host.strip("[]"), int(port)), timeout=15)
+ except (OSError, ValueError):
+ self.request.sendall(b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+ return
+ count("tunnels")
+ self.request.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n")
+ pair = [self.request, up]
+ try:
+ while True:
+ ready, _, _ = select.select(pair, [], [], 60)
+ if not ready:
+ return
+ for s in ready:
+ data = s.recv(65536)
+ if not data:
+ return
+ (up if s is self.request else self.request).sendall(data)
+ except OSError:
+ return
+ finally:
+ up.close()
+
+
+class Server(socketserver.ThreadingTCPServer):
+ daemon_threads = allow_reuse_address = True
+
+
+if __name__ == "__main__":
+ port, user, password = int(sys.argv[1]), sys.argv[2], sys.argv[3]
+ srv = Server(("127.0.0.1", port), Proxy)
+ srv.creds = f"{user}:{password}"
+ print(f"test proxy on 127.0.0.1:{port}", flush=True)
+ srv.serve_forever()
diff --git a/scripts/scrape.py b/scripts/scrape.py
index 9dc0c2e..944a001 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -86,7 +86,7 @@ CHROME_START = ("No Chrome", "Chrome cannot start", "Chrome exited at once", "Co
BLOCKED = "unusual-traffic page" # the engine's Error when Google refused this IP
# Chrome's network errors for a proxy that failed. Matched as written: the engine's message also
# holds the search URL, so a keyword such as "tunnel car wash" must not read as a proxy error.
-PROXY_ERROR = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_")
+PROXY_ERROR = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_|proxy login refused")
def poll_attempts(max_time):
@@ -232,8 +232,8 @@ def run_job(body):
f"engine: {RESTART}")
if PROXY_ERROR.search(reason):
raise RuntimeError(f"\n✗ Job failed: the proxy did not work.{said}\n"
- " Check its host and port; this engine takes no user:pass yet, so the "
- "proxy must allow your IP address.")
+ " Check its host, port, username and password (or that it allows your IP "
+ "address).")
if BLOCKED in reason:
raise Blocked(f"\n✗ Job failed.{said}")
if attempt == 2:
@@ -243,20 +243,34 @@ def run_job(body):
time.sleep(FAILED_RETRY_WAIT)
-def _get(path):
- """GET with three tries: a poll or a download must survive a passing 502 or timeout."""
- for attempt in range(3):
+def _get(path, tries=3):
+ """GET with retries 3 s apart: a poll or a download must survive a passing 502 or timeout."""
+ for attempt in range(tries):
try:
return req("GET", path)
except urllib.error.HTTPError as e:
- if e.code < 500 or attempt == 2:
+ if e.code < 500 or attempt == tries - 1:
raise
except OSError:
- if attempt == 2:
+ if attempt == tries - 1:
raise
time.sleep(3)
+def progress_text(job):
+ """", keyword 2/3, 41 places, 50%" from the job's Progress (what this engine reports), or ""."""
+ p = job.get("Progress")
+ if not isinstance(p, dict) or not p.get("keywords_total"):
+ return ""
+ total = p["keywords_total"]
+ where = {"reviews": "reading reviews", "websites": "checking websites"}.get(
+ p.get("stage"), f"keyword {min(p.get('keywords_done', 0) + 1, total)}/{total}")
+ return f", {where}, {p.get('places', 0)} places, {p.get('percent', 0)}%"
+
+
+ENGINE_RESTART_TRIES = 20 # ~1 min of polls: a restarted engine resumes the job, so wait for it
+
+
def _run_job_once(body):
global CURRENT_JOB
try:
@@ -274,8 +288,8 @@ def _run_job_once(body):
result = _poll(body, job_id)
except OSError as e: # the engine went away (restart, crash) or the job was deleted mid-poll
CURRENT_JOB = None
- raise RuntimeError(f"\n✗ Lost the engine while job {job_id} ran ({e}). If the engine comes back, "
- f"it keeps the rows found so far: {BASE}/api/v1/jobs/{job_id}/download")
+ raise RuntimeError(f"\n✗ Lost the engine while job {job_id} ran ({e}). When it starts again it resumes "
+ f"the job from its last finished keyword; its rows: {BASE}/api/v1/jobs/{job_id}/download")
except Exception:
CURRENT_JOB = None
raise
@@ -290,16 +304,11 @@ def _poll(body, job_id):
print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")
status = None
for i in range(attempts):
- _, raw = _get(f"/api/v1/jobs/{job_id}")
+ _, raw = _get(f"/api/v1/jobs/{job_id}", ENGINE_RESTART_TRIES)
job = json.loads(raw)
# Error: why the job failed, or why an "ok" job stopped early (max_time, a Google block).
status, error = job.get("Status"), str(job.get("Error") or "")
- progress = ""
- if status == "working": # the engine writes rows as it goes, so the download shows progress
- try:
- progress = f", {len(download_rows(job_id))} rows so far"
- except Exception:
- pass
+ progress = progress_text(job) if status in ("working", "pending") else ""
print(f"\r status: {str(status):<10} (attempt {i + 1}{progress})", end="", flush=True)
if status == "ok":
print()
@@ -335,7 +344,7 @@ def _poll(body, job_id):
def download_rows_or_none(job_id):
- """The rows so far, or None when the engine has written none (404 until its first keyword)."""
+ """The rows so far, or None when the engine has written none (404 until its first places)."""
try:
return download_rows(job_id)
except urllib.error.HTTPError as e:
@@ -786,8 +795,9 @@ def main():
ap.add_argument("--city", action="append",
help="city/place to auto-geocode for lat/lon; repeatable, one run per city into one file")
ap.add_argument("--proxy", action="append", metavar="URL",
- help="proxy for the engine: http://, https://, socks5:// or socks5h://host:port, "
- "without user:pass (not supported yet); repeatable, the engine uses the first")
+ help="proxy for the engine: http://[user:pass@]host:port, https://, socks5:// or "
+ "socks5h://host:port (no user:pass on SOCKS5); repeatable: the keywords take turns, "
+ "and a keyword Google refuses is retried through the next one")
ap.add_argument("--depth", type=int, default=5)
# The engine passes this to Google as the interface language, which decides the language of
# the category names it returns (and nudges which listings come back). It has to match the
@@ -798,6 +808,9 @@ def main():
"(default: en; use the local language, e.g. pt, es, de)")
# Emails are ON by default (they're the most valuable lead field): the script crawls each
# business website after the Maps job. --no-email skips the crawl.
+ ap.add_argument("--country", metavar="XX",
+ help="country Google ranks and formats results for, 2 letters (default: the country "
+ "of your IP address; e.g. us, de, br)")
ap.add_argument("--no-email", dest="email", action="store_false", default=True,
help="skip the website crawl for emails (emails are on by default)")
ap.add_argument("--no-deep-emails", dest="deep_emails", action="store_false",
@@ -826,8 +839,8 @@ def main():
f = ap.add_argument_group("filters", "applied before the email crawl, except --has-email")
f.add_argument("--min-rating", type=float, metavar="X", help="keep rows rated X or higher")
f.add_argument("--min-reviews", type=int, metavar="N",
- help="keep rows with N or more reviews (ignored while review_count is empty, as with "
- "this engine)")
+ help="keep rows with N or more reviews (ignored while review_count is empty: the engine "
+ "fills it only with SCRAPER_HEADLESS=0)")
f.add_argument("--no-website", action="store_true", help="keep only businesses without a website")
f.add_argument("--has-phone", action="store_true", help="keep only rows with a phone number")
f.add_argument("--has-email", action="store_true", help="keep only rows where an email was found")
@@ -842,6 +855,8 @@ def main():
if len(a.lang) != 2 or not a.lang.isalpha():
sys.exit(f"✗ --lang must be a 2-letter code (got {a.lang!r}); the engine rejects anything else.")
+ if a.country is not None and (len(a.country) != 2 or not a.country.isalpha()):
+ sys.exit(f"✗ --country must be a 2-letter code (got {a.country!r}), e.g. us or de.")
if a.depth < 1:
sys.exit("✗ --depth must be 1 or more.")
if a.max_time < 1:
@@ -911,6 +926,8 @@ def main():
"depth": a.depth, "email": False, "max_time": a.max_time}
if a.proxy:
body["proxies"] = a.proxy
+ if a.country:
+ body["country"] = a.country.lower()
job_id, rows, skipped, ncells, failed_places, blocked = None, [], 0, 0, 0, False
for city in cities: # one place at a time, like the grid tiles
try:
@@ -929,10 +946,11 @@ def main():
for r in rows:
r["website"] = clean_website(r.get("website", ""))
if a.min_reviews and rows and not any(r.get("review_count") for r in rows):
- # The engine leaves review_count empty: Google shows a signed-out headless browser no review
- # counts. Filtering on it would read every count as 0 and drop every row.
- print("⚠ --min-reviews ignored: this engine leaves review_count empty (Google shows it no review "
- "counts), so the filter would drop every row. --min-rating works.")
+ # A headless Chrome gets Google's limited view, which has no review counts. Filtering on it would
+ # read every count as 0 and drop every row.
+ print("⚠ --min-reviews ignored: review_count is empty (Google shows a headless Chrome no review counts; "
+ "SCRAPER_HEADLESS=0 in .env runs it as a window, which gets them), so the filter would drop every "
+ "row. --min-rating works.")
a.min_reviews = None
before = len(rows)
rows = [r for r in dedupe(rows, exclude=excluded) if keep_row(r, a)]
diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py
index dced1de..b6a5616 100644
--- a/scripts/test_scrape.py
+++ b/scripts/test_scrape.py
@@ -504,6 +504,8 @@ class MockEngine(BaseHTTPRequestHandler):
return self._send(200, "[]")
jid = parts[3]
if len(parts) == 5:
+ if CELL_ROWS[jid] is None: # a failed tile wrote no rows
+ return self._send(404, '{"message": "no results for this job (yet)"}')
buf = io.StringIO()
w = csv.DictWriter(buf, fieldnames=FIELDS, restval="")
w.writeheader(); w.writerows(CELL_ROWS[jid])
@@ -785,6 +787,7 @@ class ArgumentBounds(unittest.TestCase):
self.assertIn("--has-email", self.run_main("--has-email", "--no-email"))
def test_rejects_values_the_engine_would_choke_on(self):
+ self.assertIn("--country", self.run_main("--country", "usa"))
self.assertIn("--depth", self.run_main("--depth", "0"))
self.assertIn("--max-time", self.run_main("--max-time", "0")) # the engine says "missing max time"
self.assertIn("--limit", self.run_main("--limit", "-1"))
@@ -793,5 +796,51 @@ class ArgumentBounds(unittest.TestCase):
self.assertIn("--out", self.run_main("--out", os.path.join(bad_dir, "x", "leads.csv")))
+
+class EngineProgress(unittest.TestCase):
+ """The poll line shows the engine's Progress, and a poll survives an engine restart (it resumes)."""
+
+ def test_progress_text(self):
+ job = {"Progress": {"keywords_done": 1, "keywords_total": 3, "places": 41, "percent": 50, "stage": "search"}}
+ self.assertEqual(scrape.progress_text(job), ", keyword 2/3, 41 places, 50%")
+ job["Progress"].update(keywords_done=3, stage="websites", percent=100)
+ self.assertEqual(scrape.progress_text(job), ", checking websites, 41 places, 100%")
+ self.assertEqual(scrape.progress_text({}), "") # an engine without Progress
+
+ def test_poll_waits_out_a_restart_and_prints_progress(self):
+ calls, out = [], io.StringIO()
+ prog = {"keywords_done": 1, "keywords_total": 2, "places": 20, "percent": 50, "stage": "search"}
+ answers = [OSError("refused")] * 5 + [{"Status": "pending", "Progress": prog},
+ {"Status": "working", "Progress": prog}, {"Status": "ok"}]
+ def fake_req(method, path, body=None):
+ calls.append(path)
+ if method == "POST":
+ return 201, b'{"id":"j1"}'
+ if path.endswith("/download"):
+ return 200, b"title\nA\n"
+ a = answers.pop(0)
+ if isinstance(a, Exception):
+ raise a
+ return 200, json.dumps(a).encode()
+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \
+ redirect_stdout(out):
+ jid, rows = scrape.run_job({"max_time": 60, "keywords": ["a", "b"]})
+ self.assertEqual((jid, len(rows)), ("j1", 1))
+ self.assertIn("keyword 2/2, 20 places, 50%", out.getvalue())
+ self.assertFalse(any(c.endswith("/download") for c in calls[:-1])) # no download per poll any more
+
+ def test_country_goes_into_the_job(self):
+ bodies = []
+ def fake_run_job(body):
+ bodies.append(body)
+ return "j1", []
+ argv = ["scrape.py", "x in y", "1", "2", "--country", "DE", "--no-email",
+ "--out", os.path.join(tempfile.mkdtemp(), "o.csv")]
+ with mock.patch.object(scrape, "req", return_value=(200, b"null")), \
+ mock.patch.object(scrape, "run_job", fake_run_job), mock.patch.object(sys, "argv", argv), \
+ redirect_stdout(io.StringIO()):
+ scrape.main()
+ self.assertEqual(bodies[0]["country"], "de")
+
if __name__ == "__main__":
unittest.main(verbosity=2)