Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.
Python agent-skillsb2b-leadsclaude-codeclaude-skillscodex-skillsemail-extractoremail-scrapergoogle-mapsgoogle-maps-extractorgoogle-maps-leadsgoogle-maps-scrapergoogle-maps-scraper-pythongoogle-maps-scrapinggooglemaps-scraperheadless-chromelead-generationlead-scraperno-api-keypythonweb-scraping
feat(engine): P1 engine core: jobs, compatible API server, pinned Chrome, engine.py
Mahan Jafari0cebf888024d
diff --git a/.claude/settings.json b/.claude/settings.jsonindex 16fd343..c494561 100644--- a/.claude/settings.json+++ b/.claude/settings.json@@ -15,6 +15,20 @@"Bash(curl -s -X DELETE http://localhost:8080/:*)","Bash(curl -X DELETE http://localhost:8080/:*)","Bash(python3 scripts/test_scrape.py)",+ "Bash(python3 scripts/engine.py start)",+ "Bash(python3 scripts/engine.py stop)",+ "Bash(python3 scripts/engine.py status)",+ "Bash(python3 scripts/engine.py restart)",+ "Bash(python scripts/engine.py start)",+ "Bash(python scripts/engine.py stop)",+ "Bash(python scripts/engine.py status)",+ "Bash(python scripts/engine.py restart)",+ "Bash(py scripts/engine.py start)",+ "Bash(py scripts/engine.py stop)",+ "Bash(py scripts/engine.py status)",+ "Bash(py scripts/engine.py restart)",+ "Bash(python3 engine/test_engine.py)",+ "Bash(python3 engine/test_cdp.py)","Bash(./scripts/scrape.sh:*)","Bash(bash scripts/scrape.sh:*)","Bash(python3 scripts/scrape.py:*)",diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.ymlnew file mode 100644index 0000000..207e4ca--- /dev/null+++ b/.github/workflows/engine-live.yml@@ -0,0 +1,57 @@+# The kit's own engine against real Google Maps, once per platform: start it the way a user does,+# run one small scrape through scripts/scrape.py, and require places back. Runs when the engine+# changes, or by hand (Actions > engine-live > Run workflow). The logs print counts only: scraped+# names, phones and emails never reach them.+name: engine-live+on:+ workflow_dispatch:+ pull_request:+ paths:+ - "engine/**"+ - "scripts/engine.py"+ - ".github/workflows/engine-live.yml"+jobs:+ live:+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]+ runs-on: ${{ matrix.os }}+ timeout-minutes: 15+ defaults:+ run:+ shell: bash+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-python@v5+ with:+ python-version: "3.12"+ - name: start the engine (downloads the pinned Chrome only when none is installed)+ run: python scripts/engine.py start+ - name: status, guards and web UI+ run: |+ python scripts/engine.py status+ test "$(curl -s http://localhost:8080/api/v1/jobs)" = null+ test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Host: evil.com' http://localhost:8080/api/v1/jobs)" = 421+ test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Origin: https://evil.com' -X POST -d '{}' http://localhost:8080/api/v1/jobs)" = 403+ curl -s -X POST -d '{}' http://localhost:8080/api/v1/jobs | grep -q 'missing name'+ curl -s http://localhost:8080/ | grep -q 'github.com/Mahanaicoach/google-maps-scraper-kit'+ - name: scrape (depth 2, no email crawl)+ run: |+ start=$(date +%s)+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --out leads.csv > scrape.log 2>&1 || true+ echo "seconds=$(( $(date +%s) - start ))"+ grep -v "^ •" scrape.log | tr '\r' '\n' | grep -v "^ status:" || true+ python - <<'PY'+ import csv, sys+ rows = list(csv.DictReader(open("leads.csv", encoding="utf-8-sig")))+ fill = {k: sum(1 for r in rows if r.get(k)) for k in ("title", "phone", "website", "address", "review_rating")}+ print(f"places={len(rows)}", fill)+ sys.exit(0 if rows else 1)+ PY+ - name: engine log (counts only)+ if: always()+ run: cat .engine/engine.log || true+ - name: stop+ if: always()+ run: python scripts/engine.py stopdiff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 83ada37..ff006a5 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -1,5 +1,5 @@-# Two jobs, no Google traffic. "offline" is the Python client on the three desktop platforms a new-# user installs on. "fresh-install" is a new user's first ten minutes on Linux minus the scrape:+# Two jobs, no Google traffic. "offline" is the Python client and the kit's own engine on the three+# desktop platforms a new user installs on. "fresh-install" is a new user's first ten minutes on Linux minus the scrape:# clone, docker compose up, the health check, the patched web UI and API docs, the proxy's checks.name: testson:@@ -19,6 +19,8 @@ jobs:with:python-version: "3.12"- run: python scripts/test_scrape.py+ - run: python engine/test_cdp.py+ - run: python engine/test_engine.py- run: bash -n scripts/scrape.sh- run: python -c "import json; json.load(open('.claude/settings.json'))"fresh-install:diff --git a/.gitignore b/.gitignoreindex 0ede7c0..d4d9262 100644--- a/.gitignore+++ b/.gitignore@@ -17,3 +17,6 @@ __pycache__/# OS / editor noise.DS_StoreThumbs.db++# The engine's jobs, results, logs and downloaded Chrome+.engine/diff --git a/engine/browser.py b/engine/browser.pyindex f161d8d..6e7f3ca 100644--- a/engine/browser.py+++ b/engine/browser.py@@ -1,15 +1,19 @@-"""Find and run a local Chrome for the engine: headless, loopback DevTools port, private profile.+"""Find, fetch and run the Chrome the engine drives: headless, loopback DevTools port, private profile.with Browser() as b:page = cdp.Page(b.cdp)-Uses an installed Google Chrome, Chromium, Edge or Brave (set CHROME=/path/to/binary to pick one).-Standard library only.+An installed Google Chrome, Chromium, Edge or Brave is used first (set CHROME=/path/to/binary to+pick one). On a machine with none, `python3 scripts/engine.py start` downloads Chrome for Testing's+headless shell into .engine/, pinned below by version and SHA-256. Standard library only."""-import os, platform, shutil, subprocess, sys, tempfile, time+import hashlib, os, platform, shutil, subprocess, tempfile, time, urllib.request, zipfilefrom cdp import CDP+KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+HOME = os.path.join(KIT, ".engine") # everything the engine keeps: jobs, results, logs, its Chrome+CANDIDATES = {"Darwin": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome","/Applications/Chromium.app/Contents/MacOS/Chromium",@@ -25,9 +29,56 @@ CANDIDATES = {r"%ProgramFiles%\BraveSoftware\Brave-Browser\Application\brave.exe"],}+# Chrome for Testing's headless shell (https://googlechromelabs.github.io/chrome-for-testing/),+# Stable channel. Each zip's SHA-256 was computed from the download on 2026-10-31. A download that+# does not match is deleted and never run. Windows on Arm runs the x64 build under emulation.+CFT_VERSION = "156.0.8078.80"+CFT_URL = "https://storage.googleapis.com/chrome-for-testing-public/{v}/{p}/chrome-headless-shell-{p}.zip"+CFT_SHA256 = {+ "mac-arm64": "ea615baa2e9caf0d02d1898a6562b5f8fa9c35239707dae0f592178df5b986ac",+ "mac-x64": "535d417d9034cbd11e7dacd1a71aa3d6e1b525e6d5dcd2671eb2532f01481228",+ "linux64": "547dcd374e02961bf998f6046a267644f8502225c36bb78ca307dab32a0b3eb4",+ "linux-arm64": "5bb6a00a4285c13303d3f72c5afdb70f144ea7a93eed462365a52b69631c9607",+ "win64": "10a77466005e7f64e00f1a75fd1ea81f4a38a60093c65fea6895843b26e60a9a",+}++# What Chrome needs on a bare Debian or Ubuntu (a server, a container). apt picks the t64 variants+# (libasound2t64 and friends) on Ubuntu 24.04 by itself.+APT_LINE = ("sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 "+ "libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 "+ "libcairo2 libasound2")+++class BrowserError(RuntimeError):+ pass+++def make_home():+ """.engine/, private to this user on macOS and Linux (results hold scraped phone numbers)."""+ os.makedirs(HOME, mode=0o700, exist_ok=True)+ if os.name != "nt":+ os.chmod(HOME, 0o700)+++def cft_platform(system=None, machine=None):+ """Chrome for Testing's name for this machine (mac-arm64, linux64, ...), or None."""+ system = system or platform.system()+ arch = {"x86_64": "x64", "amd64": "x64", "arm64": "arm64", "aarch64": "arm64"}.get(+ (machine or platform.machine()).lower())+ return {("Darwin", "arm64"): "mac-arm64", ("Darwin", "x64"): "mac-x64",+ ("Linux", "x64"): "linux64", ("Linux", "arm64"): "linux-arm64",+ ("Windows", "x64"): "win64", ("Windows", "arm64"): "win64"}.get((system, arch))+++def shell_path(plat=None):+ """Where the downloaded headless shell lives (it may not exist yet)."""+ plat = plat or cft_platform() or "none"+ exe = "chrome-headless-shell.exe" if plat.startswith("win") else "chrome-headless-shell"+ return os.path.join(HOME, "chrome", CFT_VERSION, f"chrome-headless-shell-{plat}", exe)+def find_chrome():- """Path of a usable Chromium-based browser, or None."""+ """Path of a usable Chromium-based browser: CHROME, an installed one, then the downloaded shell."""if os.environ.get("CHROME"):return os.environ["CHROME"]for c in CANDIDATES.get(platform.system(), []):@@ -36,35 +87,114 @@ def find_chrome():return cif os.sep not in c and shutil.which(c):return shutil.which(c)- return None+ return shell_path() if os.path.isfile(shell_path()) else None+++def download(log=print, plat=None, opener=urllib.request.urlopen):+ """Fetch, verify and unpack the pinned headless shell into .engine/chrome/; its path."""+ plat = plat or cft_platform()+ if plat not in CFT_SHA256:+ raise BrowserError(f"No Chrome for Testing build exists for {platform.system()} {platform.machine()}. "+ "Install Chromium, then set CHROME to its path.")+ url = CFT_URL.format(v=CFT_VERSION, p=plat)+ root = os.path.dirname(os.path.dirname(shell_path(plat)))+ make_home()+ os.makedirs(root, exist_ok=True)+ tmp = os.path.join(root, f"{plat}.zip.part")+ log(f"▶ No Chrome found, so downloading Chrome for Testing {CFT_VERSION} (headless shell, {plat}, "+ f"about 100 MB) once into .engine/ from {url}")+ h = hashlib.sha256()+ try:+ with opener(url, timeout=60) as r, open(tmp, "wb") as f:+ for chunk in iter(lambda: r.read(1 << 20), b""):+ h.update(chunk)+ f.write(chunk)+ except OSError as e:+ _remove(tmp)+ raise BrowserError(f"Could not download Chrome for Testing from {url}: {e}") from e+ if h.hexdigest() != CFT_SHA256[plat]:+ _remove(tmp)+ raise BrowserError("The Chrome download does not match the SHA-256 pinned in engine/browser.py, "+ "so it was deleted and not run. Try again later, or install Google Chrome.")+ staging = tempfile.mkdtemp(prefix="unpack-", dir=root)+ try:+ with zipfile.ZipFile(tmp) as z:+ for m in z.infolist():+ path = z.extract(m, staging) # zipfile drops absolute paths and ".." parts+ mode = m.external_attr >> 16+ if mode & 0o111 and not m.is_dir():+ os.chmod(path, 0o755)+ final = os.path.dirname(shell_path(plat))+ shutil.rmtree(final, ignore_errors=True)+ os.replace(os.path.join(staging, os.path.basename(final)), final) # a half-unpacked shell never shows+ except (OSError, zipfile.BadZipFile) as e:+ raise BrowserError(f"Could not unpack the Chrome download: {e}") from e+ finally:+ shutil.rmtree(staging, ignore_errors=True)+ _remove(tmp)+ log(" ✓ SHA-256 verified, unpacked")+ return shell_path(plat)+++def _remove(path):+ try:+ os.remove(path)+ except OSError:+ pass+++def start_error(path, code, stderr):+ """The message for a Chrome that exited at once, with the apt line when Linux libraries are missing."""+ missing = [l.strip() for l in stderr.splitlines() if "error while loading shared libraries" in l]+ if missing:+ return (f"Chrome cannot start: {missing[0]}\n Install the libraries it needs "+ f"(Debian or Ubuntu):\n {APT_LINE}")+ last = "; ".join(l.strip() for l in stderr.strip().splitlines()[-3:])+ return f"Chrome exited at once (code {code}): {path}" + (f": {last}" if last else "")class Browser:"""A headless Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP."""- def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):+ def __init__(self, path=None, headless=True, profile=None, lang="en-US", proxy=None, timeout=30):self.path = path or find_chrome()if not self.path:- sys.exit("✗ No Chrome, Chromium, Edge or Brave found. Install Google Chrome, or set CHROME to "- "the browser's path.")+ raise BrowserError("No Chrome, Chromium, Edge or Brave found. Run `python3 scripts/engine.py start`, "+ "which downloads one, or install Google Chrome.")self._own_profile = profile is Noneself.profile = profile or tempfile.mkdtemp(prefix="gmskit-chrome-")args = [self.path, "--remote-debugging-port=0", f"--user-data-dir={self.profile}","--no-first-run", "--no-default-browser-check", "--disable-extensions","--disable-background-networking", "--disable-sync", "--mute-audio",+ "--disable-component-update", "--disable-domain-reliability", "--disable-breakpad",+ "--metrics-recording-only", "--no-pings", "--disable-client-side-phishing-detection","--disable-blink-features=AutomationControlled", "--window-size=1280,900",- f"--lang={lang}", "--disable-dev-shm-usage", "about:blank"]- if headless:- args.insert(1, "--headless=new")- kw = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL, "stdin": subprocess.DEVNULL}+ f"--lang={lang}", "--disable-dev-shm-usage"]+ if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headless+ args.append("--headless=new")+ if proxy:+ args.append(f"--proxy-server={proxy}")+ if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:+ args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)+ args.append("about:blank")+ self._err = open(os.path.join(self.profile, "chrome-stderr.log"), "w+b")+ kw = {"stdout": subprocess.DEVNULL, "stderr": self._err, "stdin": subprocess.DEVNULL}if os.name == "nt":kw["creationflags"] = subprocess.CREATE_NO_WINDOW- self.proc = subprocess.Popen(args, **kw)+ try:+ self.proc = subprocess.Popen(args, **kw)+ except OSError as e:+ self._cleanup()+ raise BrowserError(f"Could not start Chrome ({self.path}): {e}") from e+ self.cdp = Noneactive = os.path.join(self.profile, "DevToolsActivePort")end = time.time() + timeoutwhile time.time() < end:if self.proc.poll() is not None:- sys.exit(f"✗ Chrome exited at once (code {self.proc.returncode}): {self.path}")+ self._err.seek(0)+ msg = start_error(self.path, self.proc.returncode, self._err.read().decode("utf-8", "replace"))+ self.close()+ raise BrowserError(msg)try:with open(active, encoding="utf-8") as f:port, path = f.read().split()@@ -73,18 +203,25 @@ class Browser:time.sleep(0.1)else:self.close()- sys.exit(f"✗ Chrome did not open its DevTools port within {timeout} s: {self.path}")- self.ws_url = f"ws://127.0.0.1:{port}{path}"- self.cdp = CDP(self.ws_url, timeout=timeout)+ raise BrowserError(f"Chrome did not open its DevTools port within {timeout} s: {self.path}")+ self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)def version(self):return self.cdp.send("Browser.getVersion").get("product", "")- def close(self):+ def kill(self):+ """Stop Chrome now (the job watchdog): every CDP call still waiting fails at once."""try:- self.cdp.close()- except Exception: # noqa: BLE001 - closing is best effort+ self.proc.kill()+ except OSError:pass++ def close(self):+ if self.cdp:+ try:+ self.cdp.close()+ except Exception: # noqa: BLE001 - closing is best effort+ passif self.proc.poll() is None:self.proc.terminate()try:@@ -92,8 +229,16 @@ class Browser:except subprocess.TimeoutExpired:self.proc.kill()self.proc.wait(5)+ self._cleanup()++ def _cleanup(self):+ self._err.close()if self._own_profile:- shutil.rmtree(self.profile, ignore_errors=True)+ for _ in range(10): # Windows keeps the profile locked for a moment after Chrome exits+ shutil.rmtree(self.profile, ignore_errors=True)+ if not os.path.exists(self.profile):+ break+ time.sleep(0.3)def __enter__(self):return selfdiff --git a/engine/details.py b/engine/details.pyindex 60fda9f..d32a553 100644--- a/engine/details.py+++ b/engine/details.py@@ -12,10 +12,10 @@ import json# Index path of each field inside a place entity. Checked on live batches in the spike# (2026-10-31): title, category, address, rating, lat/lon, place_id, data_id, timezone, thumbnail# filled on 60/60 entities, website 55/60, phone 46/60 (listings without one).-# review_count is the one unverified path: (37, 1) is an integer on every entity in the range a-# review count has (23 to 1998 on 40 coffee shops), but signed-out Google Maps shows no counts in-# the list or the place panel, so it could not be compared against the page. Verify it in P1-# against a few listings whose counts are known (a signed-in browser shows them).+# review_count is not here: signed out, Google leaves (4, 8) empty, and (37, 1) is a different+# number (60 and 82 where the page showed 61 and 134 reviews, 2026-10-31). Headless Chrome gets+# Google's limited view, whose result cards show "4.8 stars" without the count, so the CSV's+# review_count column stays empty.FIELDS = {"title": (11,),"category": (13, 0),@@ -23,7 +23,6 @@ FIELDS = {"website": (7, 0),"phone": (178, 0, 0),"review_rating": (4, 7),- "review_count": (37, 1),"latitude": (9, 2),"longitude": (9, 3),"plus_code": (183, 2, 2, 0),diff --git a/engine/jobs.py b/engine/jobs.pynew file mode 100644index 0000000..07d0f82--- /dev/null+++ b/engine/jobs.py@@ -0,0 +1,467 @@+"""Scrape jobs: a sqlite3 store in .engine/data and one worker that runs the jobs one at a time.++A job runs its keywords through maps.collect in one headless Chrome, de-duplicates the places on+data_id and writes <id>.csv with the old engine's 36 column names (what Google's result batches+carry is filled, the rest stays empty) plus ours. The CSV is rewritten atomically after every+keyword, so a download while the job works returns the rows so far. A watchdog stops Chrome when a+job overruns its max_time, so no job can stick on "working". Standard library only.+"""+import concurrent.futures, csv, html, http.client, io, ipaddress, json, math, os, re, socket, sqlite3+import threading, time, urllib.parse, urllib.request, uuid+from datetime import datetime, timezone++import browser, maps+from cdp import CDPError, Page++# gosom/google-maps-scraper's CSV header, in its order, so files and scripts that read the old+# engine's output keep working. Our own columns follow.+COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "popular_times", "website",+ "phone", "plus_code", "review_count", "review_rating", "reviews_per_rating", "latitude",+ "longitude", "cid", "status", "descriptions", "reviews_link", "thumbnail", "timezone",+ "price_range", "data_id", "street_view_url", "place_id", "images", "reservations",+ "order_online", "menu", "owner", "complete_address", "credit_cards_accepted", "about",+ "user_reviews", "user_reviews_extended", "emails"]+EXTRA = ["categories"]+RENAME = {"open_state": "status", "description": "descriptions"} # details.FIELDS name -> CSV column++WATCHDOG_GRACE = 30 # seconds past max_time before the watchdog kills Chrome+SITE_WORKERS = 8 # business websites fetched at once by the email pass+SITE_TIMEOUT = 10 # seconds per website request+EMAIL_MIN_SECONDS = 60 # the email pass gets at least this long, even after a job used its max_time+++def utc_now():+ return datetime.now(timezone.utc).isoformat(timespec="seconds")+++# ── Job body ──+LANG = re.compile(r"^[A-Za-z]{2}$")+PROXY_SCHEMES = ("http", "https", "socks5", "socks5h")+++def _int(body, key, default, lo, hi, label):+ v = body.get(key, default)+ if v is None:+ v = default+ if isinstance(v, bool) or not isinstance(v, int) or not lo <= v <= hi:+ raise ValueError(f"invalid {label}: a whole number from {lo} to {hi}")+ return v+++def validate(body):+ """(name, data) from a create-job body, or ValueError with the old engine's message+ ("missing name", "invalid lang", ...), which the kit's docs quote."""+ if not isinstance(body, dict):+ raise ValueError("the body must be a JSON object")+ name = body.get("name")+ if not isinstance(name, str) or not name.strip():+ raise ValueError("missing name")+ if len(name) > 200:+ raise ValueError("invalid name: 200 characters at most")+ kws = body.get("keywords")+ if not isinstance(kws, list) or not kws:+ raise ValueError("missing keywords")+ if len(kws) > 500 or not all(isinstance(k, str) and k.strip() and len(k) <= 500 for k in kws):+ raise ValueError("invalid keywords: 1 to 500 non-empty strings of up to 500 characters")+ lang = body.get("lang")+ if lang in (None, ""):+ raise ValueError("missing lang")+ if not isinstance(lang, str) or not LANG.match(lang):+ raise ValueError("invalid lang")+ lat, lon = body.get("lat"), body.get("lon")+ if lat in (None, "") or lon in (None, ""):+ raise ValueError("missing geo coordinates")+ try:+ if isinstance(lat, bool) or isinstance(lon, bool):+ raise ValueError+ flat, flon = float(lat), float(lon)+ except (TypeError, ValueError):+ raise ValueError("invalid geo coordinates") from None+ if not (math.isfinite(flat) and math.isfinite(flon) and -90 <= flat <= 90 and -180 <= flon <= 180):+ raise ValueError("invalid geo coordinates")+ if body.get("max_time") in (None, 0):+ raise ValueError("missing max time")+ data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "lat": str(lat).strip(),+ "lon": str(lon).strip(),+ "zoom": _int(body, "zoom", 15, 1, 21, "zoom"),+ "radius": _int(body, "radius", 10000, 0, 1_000_000, "radius"),+ "depth": _int(body, "depth", 1, 1, 1000, "depth"),+ "max_time": _int(body, "max_time", 600, 1, 86400, "max time"),+ "fast_mode": bool(body.get("fast_mode")), "email": bool(body.get("email")),+ "proxies": _proxies(body.get("proxies"))}+ return name.strip(), data+++def _proxies(v):+ if v in (None, ""):+ return []+ if not isinstance(v, list) or len(v) > 100 or not all(isinstance(p, str) for p in v):+ raise ValueError("invalid proxies: a list of proxy URLs")+ for p in v:+ try:+ u = urllib.parse.urlsplit(p.strip())+ u.port # noqa: B018 - raises on a bad port+ except ValueError:+ raise ValueError(f"invalid proxy: {p}") from None+ if u.scheme not in PROXY_SCHEMES or not u.hostname:+ raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port")+ if u.username or u.password:+ raise ValueError("proxy usernames and passwords are not supported by this engine yet: "+ "use a proxy that allows your IP address")+ return [p.strip() for p in v]+++# ── Rows ──+def to_row(place, keyword):+ """One CSV row (every column, as text) from a details.parse_entity / maps.parse_link place."""+ row = dict.fromkeys(COLUMNS + EXTRA, "")+ for k, v in place.items():+ k = RENAME.get(k, k)+ if k in row:+ row[k] = str(v)+ row["input_id"] = keyword+ if place.get("place_id"): # Google's documented Maps URL for one place+ row["link"] = ("https://www.google.com/maps/search/?api=1&query="+ + urllib.parse.quote(place.get("title", ""), safe="")+ + "&query_place_id=" + urllib.parse.quote(place["place_id"], safe=""))+ return row+++def to_csv(rows):+ buf = io.StringIO()+ w = csv.DictWriter(buf, fieldnames=COLUMNS + EXTRA, extrasaction="ignore")+ w.writeheader()+ w.writerows(rows)+ return buf.getvalue().encode("utf-8")+++# ── Store ──+class Store:+ """Jobs in sqlite3 (jobs.db) and their results as <id>.csv beside it. One lock serialises every+ database call and every CSV read and write, so a download never meets a half-replaced file+ (Windows cannot replace a file another handle has open)."""++ def __init__(self, folder):+ os.makedirs(folder, exist_ok=True)+ self.folder = folder+ self.lock = threading.RLock()+ self.db = sqlite3.connect(os.path.join(folder, "jobs.db"), check_same_thread=False)+ with self.lock, self.db:+ self.db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, "+ "date TEXT NOT NULL, status TEXT NOT NULL, data TEXT NOT NULL, "+ "error TEXT NOT NULL DEFAULT '')")++ @staticmethod+ def _job(r):+ # The old engine's job shape (capitalised keys), plus Error: why a job failed or stopped early.+ return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5]}++ def create(self, name, data):+ jid = str(uuid.uuid4())+ with self.lock, self.db:+ self.db.execute("INSERT INTO jobs VALUES (?, ?, ?, 'pending', ?, '')",+ (jid, name, utc_now(), json.dumps(data)))+ return jid++ def get(self, jid):+ with self.lock:+ r = self.db.execute("SELECT * FROM jobs WHERE id = ?", (jid,)).fetchone()+ return self._job(r) if r else None++ def all(self):+ with self.lock:+ return [self._job(r) for r in self.db.execute("SELECT * FROM jobs ORDER BY rowid DESC")]++ def update(self, jid, status, error=""):+ with self.lock, self.db:+ self.db.execute("UPDATE jobs SET status = ?, error = ? WHERE id = ?", (status, error[:500], jid))++ def claim(self):+ """The oldest pending job, now marked working; None when there is none."""+ with self.lock, self.db:+ r = self.db.execute("SELECT * FROM jobs WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()+ if not r:+ return None+ self.db.execute("UPDATE jobs SET status = 'working' WHERE id = ?", (r[0],))+ job = self._job(r)+ job["Status"] = "working"+ return job++ def recover(self):+ """Jobs left on "working" by an engine that was stopped or crashed: failed, rows kept."""+ with self.lock, self.db:+ return self.db.execute("UPDATE jobs SET status = 'failed', error = 'the engine stopped while "+ "this job ran; its rows so far are kept' WHERE status = 'working'").rowcount++ def delete(self, jid):+ with self.lock, self.db:+ gone = self.db.execute("DELETE FROM jobs WHERE id = ?", (jid,)).rowcount+ for path in (self.csv_path(jid), self.csv_path(jid) + ".tmp"):+ try:+ os.remove(path)+ except OSError:+ pass+ return bool(gone)++ def csv_path(self, jid):+ return os.path.join(self.folder, f"{jid}.csv")++ def write_csv(self, jid, rows):+ """Replace the job's CSV in one step; False (nothing written) if the job was deleted."""+ with self.lock:+ if not self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone():+ return False+ tmp = self.csv_path(jid) + ".tmp"+ with open(tmp, "wb") as f:+ f.write(to_csv(rows))+ os.replace(tmp, self.csv_path(jid))+ return True++ def read_csv(self, jid):+ with self.lock:+ try:+ with open(self.csv_path(jid), "rb") as f:+ return f.read()+ except OSError:+ return None+++# ── Worker ──+class Worker(threading.Thread):+ """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once."""++ def __init__(self, store, log=print):+ super().__init__(daemon=True, name="jobs")+ self.store, self.log = store, log+ self.wake = threading.Event()+ self.stopped = threading.Event()+ self._running = (None, None) # (job id, stop event) of the job in progress+ self._browser = None++ def run(self):+ while not self.stopped.is_set():+ job = self.store.claim()+ if not job:+ self.wake.wait(5)+ self.wake.clear()+ continue+ try:+ self.run_job(job)+ except Exception as e: # noqa: BLE001 - one bad job must never stop the worker+ self.log(f"job {job['ID']} failed: {type(e).__name__}: {e}")+ self.store.update(job["ID"], "failed", f"{type(e).__name__}: {e}")++ def cancel(self, jid):+ """Stop `jid` if it is the job in progress (it is being deleted)."""+ rid, stop = self._running+ if rid == jid and stop:+ stop.set()++ def shutdown(self):+ """Stop the job in progress and its Chrome (the engine is exiting)."""+ self.stopped.set()+ self.wake.set()+ _, stop = self._running+ if stop:+ stop.set()+ b = self._browser+ if b:+ b.close()++ def run_job(self, job):+ jid, d = job["ID"], job["Data"]+ deadline = time.monotonic() + d["max_time"]+ stop = threading.Event()+ self._running = (jid, stop)+ rows, seen, errors, blocked = [], set(), [], False+ kws = d["keywords"]+ self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")++ def watchdog(): # Chrome hung or the job overran: kill Chrome, and every CDP call fails at once+ stop.set()+ b = self._browser+ if b:+ b.kill()+ self.log(f"job {jid}: watchdog stopped Chrome {WATCHDOG_GRACE} s after max_time")+ timer = threading.Timer(d["max_time"] + WATCHDOG_GRACE, watchdog)+ timer.daemon = True+ timer.start()+ try:+ self._browser = browser.Browser(lang=d["lang"], proxy=(d["proxies"] or [None])[0])+ for i, kw in enumerate(kws, 1):+ left = deadline - time.monotonic()+ if stop.is_set() or left <= 0:+ break+ url = maps.search_url(kw, d["lat"], d["lon"], d["zoom"], d["lang"])+ page = None+ started = time.monotonic()+ try:+ page = Page(self._browser.cdp)+ places, reason, stats = maps.collect(page, url, depth=d["depth"], max_seconds=left,+ log=lambda m: None, stop=stop.is_set)+ except (CDPError, OSError) as e:+ errors.append(str(e))+ self.log(f"job {jid} keyword {i}/{len(kws)}: error: {e}")+ if self._browser.cdp.closed: # Chrome is gone: the other keywords would fail too+ break+ continue+ finally:+ if page and not self._browser.cdp.closed:+ page.close()+ blocked |= reason == "blocked"+ new = 0+ for p in places:+ key = p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")+ if key and key not in seen:+ seen.add(key)+ rows.append(to_row(p, kw))+ new += 1+ if not self.store.write_csv(jid, rows):+ break # deleted+ self.log(f"job {jid} keyword {i}/{len(kws)}: {len(places)} places, {new} new, "+ f"{stats['batches']} batches, {time.monotonic() - started:.1f} s ({reason})")+ except browser.BrowserError as e:+ errors.append(str(e))+ self.log(f"job {jid}: {e}")+ finally:+ timer.cancel()+ if self._browser:+ self._browser.close()+ self._browser = None+ if d["email"] and rows and not stop.is_set():+ n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS))+ self.log(f"job {jid}: emails found for {n}/{sum(1 for r in rows if r['website'])} websites")+ self._running = (None, None)+ if not self.store.write_csv(jid, rows):+ self.log(f"job {jid} deleted while it ran")+ return+ if rows or not (errors or blocked):+ note = "stopped at max_time; the rows found so far are kept" \+ if time.monotonic() >= deadline or (stop.is_set() and not errors) else ""+ if errors:+ note = f"{len(errors)} keyword(s) failed: {errors[-1]}"+ self.store.update(jid, "ok", note)+ self.log(f"job {jid} ok: {len(rows)} places")+ else:+ reason = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "+ "Wait, or add proxies") if blocked else errors[-1]+ self.store.update(jid, "failed", reason)+ self.log(f"job {jid} failed: {reason}")+++# ── Email pass (email: true): each business homepage over plain HTTP, emails only ──+EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")+JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")+JUNK_DOMAIN = ("example.", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")+UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"+++def _public_ip(ip):+ try:+ return ipaddress.ip_address(ip.split("%")[0]).is_global+ except ValueError:+ return False+++def _public_host(host):+ """True only if every address of `host` is public. Websites come from third-party listings, so+ one could point the fetch at this machine (the engine itself) or its network."""+ try:+ infos = socket.getaddrinfo(host, None)+ except (OSError, UnicodeError):+ return False+ return bool(infos) and all(_public_ip(i[4][0]) for i in infos)+++def _public_connection(address, *args, **kwargs):+ """create_connection that refuses a private peer: catches a DNS answer that changed (rebinding)."""+ sock = socket.create_connection(address, *args, **kwargs)+ if not _public_ip(sock.getpeername()[0]):+ sock.close()+ raise OSError("refused a non-public address")+ return sock+++class _HTTP(http.client.HTTPConnection):+ def __init__(self, *a, **kw):+ super().__init__(*a, **kw)+ self._create_connection = _public_connection+++class _HTTPS(http.client.HTTPSConnection):+ def __init__(self, *a, **kw):+ super().__init__(*a, **kw)+ self._create_connection = _public_connection+++class _HTTPHandler(urllib.request.HTTPHandler):+ def http_open(self, req):+ return self.do_open(_HTTP, req)+++class _HTTPSHandler(urllib.request.HTTPSHandler):+ def https_open(self, req):+ return self.do_open(_HTTPS, req, context=self._context)+++class _Redirects(urllib.request.HTTPRedirectHandler):+ def redirect_request(self, req, fp, code, msg, headers, newurl):+ if not _public_host(urllib.parse.urlsplit(newurl).hostname or ""):+ return None+ return super().redirect_request(req, fp, code, msg, headers, newurl)+++# No proxy from the environment: these go straight to the business websites.+_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}), _HTTPHandler, _HTTPSHandler, _Redirects)+++def fetch(url, timeout=SITE_TIMEOUT):+ """Up to 400 KB of a public web page as text; "" on any failure."""+ if not url.startswith(("http://", "https://")):+ url = "http://" + url+ try:+ if not _public_host(urllib.parse.urlsplit(url).hostname or ""):+ return ""+ r = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html"})+ with _OPENER.open(r, timeout=timeout) as resp:+ return resp.read(400_000).decode("utf-8", "replace")+ except Exception: # noqa: BLE001 - a broken website is just a website without emails+ return ""+++def site_host(url):+ try:+ h = (urllib.parse.urlsplit(url if "://" in url else "http://" + url).hostname or "").lower()+ except ValueError:+ return ""+ return h[4:] if h.startswith("www.") else h+++def find_emails(text, site=""):+ """Up to 3 addresses from page text, placeholders and image names dropped, the site's own first."""+ found = []+ for e in EMAIL_RE.findall(html.unescape(text or "")):+ e = urllib.parse.unquote(e).lower()+ dom = e.rpartition("@")[2]+ if e.endswith(JUNK_SUFFIX) or any(j in dom for j in JUNK_DOMAIN) or e in found:+ continue+ found.append(e)+ own = lambda e: site and (e.endswith("@" + site) or e.endswith("." + site))+ return sorted(found, key=lambda e: not own(e))[:3]+++def add_emails(rows, seconds, get=None):+ """Fill `emails` (joined by ", ", like the old engine) from each row's website homepage, for at+ most `seconds`; the number of rows that got one."""+ get = get or fetch+ todo = [r for r in rows if r.get("website") and not r.get("emails")]+ pool = concurrent.futures.ThreadPoolExecutor(SITE_WORKERS)+ futs = {pool.submit(lambda r=r: find_emails(get(r["website"]), site_host(r["website"]))): r for r in todo}+ done, _ = concurrent.futures.wait(futs, timeout=seconds)+ pool.shutdown(wait=False, cancel_futures=True)+ n = 0+ for f in done:+ if not f.exception() and f.result():+ futs[f]["emails"] = ", ".join(f.result())+ n += 1+ return ndiff --git a/engine/maps.py b/engine/maps.pyindex 20bca15..aead9cc 100644--- a/engine/maps.py+++ b/engine/maps.py@@ -92,16 +92,20 @@ class Batches:self.page.cdp.unsubscribe("Network.responseReceived", self.q, self.page.sid)-def collect(page, url, depth=1, max_seconds=180, log=print):+def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):"""Open `url`, scroll the result feed, and return (places, stop reason, stats).`depth` is the number of result batches to read (about 20 places each, like the old engine's- depth); the list may end sooner. Places are de-duplicated on data_id."""+ depth); the list may end sooner. Places are de-duplicated on data_id. `stop()` returning True+ ends the scroll early (a deleted job). Reason "blocked" means Google answered with its+ unusual-traffic page."""for k, v in CONSENT_COOKIES.items():page.set_cookie(k, v, ".google.com")batches = Batches(page)seen, started, reason, stale = {}, time.time(), "depth", 0try:page.navigate(url)+ if "/sorry/" in page.url():+ return [], "blocked", {"batches": 0, "dom_links": 0}if "consent.google" in page.url():page.eval(JS_CONSENT)page.wait_for("!location.href.includes('consent.google')", timeout=15)@@ -124,7 +128,7 @@ def collect(page, url, depth=1, max_seconds=180, log=print):take()time.sleep(0.4)log(f" batch {batches.count}: {len(seen)} places")- while batches.count < depth and time.time() - started < max_seconds:+ while batches.count < depth and time.time() - started < max_seconds and not stop():if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):reason = "end-of-list"break@@ -145,7 +149,9 @@ def collect(page, url, depth=1, max_seconds=180, log=print):stale = 0log(f" batch {batches.count}: {len(seen)} places")else:- if time.time() - started >= max_seconds:+ if stop():+ reason = "stopped"+ elif time.time() - started >= max_seconds:reason = "timeout"take()links = page.eval(JS_LINKS) or []diff --git a/engine/serve.py b/engine/serve.pynew file mode 100644index 0000000..cc6c228--- /dev/null+++ b/engine/serve.py@@ -0,0 +1,373 @@+#!/usr/bin/env python3+"""The kit's own engine: the Google Maps scraper's HTTP API and web UI, on this machine only.++ python3 engine/serve.py # port from SCRAPER_PORT (environment, then .env), default 8080+ python3 engine/serve.py --port 8190++Normally started by `python3 scripts/engine.py start`. It speaks the API of the engine the kit used+before (POST/GET/DELETE /api/v1/jobs, /api/v1/jobs/{id}/download, the same job body and "Status"+values), so scripts/scrape.py, the skill and the slash commands work unchanged. It listens on+127.0.0.1 (and ::1) only and refuses a foreign Host (DNS rebinding, 421) or Origin (cross-site+posts, 403); with SCRAPER_API_KEY set it also wants that X-API-Key (401). No telemetry: it talks to+Google Maps, and for email: true jobs to the businesses' own websites, nothing else.+Standard library only.+"""+import argparse, hmac, json, os, re, secrets, signal, socket, sys, threading, urllib.parse+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+import browser, jobs # noqa: E402++for _stream in (sys.stdout, sys.stderr): # ✓ and accented names on a Windows cp1252 console+ try:+ _stream.reconfigure(encoding="utf-8", errors="replace")+ except (AttributeError, ValueError):+ pass++VERSION = "2.0.0-dev"+MARK = f"gmskit-engine/{VERSION}" # X-Engine header: tells scripts/engine.py this engine from others+REPO = "https://github.com/Mahanaicoach/google-maps-scraper-kit"+ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY")+MAX_BODY = 1 << 20+LOCAL = r"(localhost|127\.0\.0\.1|\[::1\])"+HOST_OK = re.compile(rf"^{LOCAL}(:\d{{1,5}})?$", re.I)+ORIGIN_OK = re.compile(rf"^https?://{LOCAL}(:\d{{1,5}})?$", re.I)+JOB_PATH = re.compile(r"^/api/v1/jobs/([0-9a-f-]{36})(/download)?$")+JOB_ID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")+_log_lock = threading.Lock()+_REPLIED = object() # body() answered the request itself (a bad body)+++def log(msg):+ """One line on stderr with a UTC timestamp and its offset. Counts only, never scraped values."""+ with _log_lock:+ print(f"{jobs.utc_now()} {msg}", file=sys.stderr, flush=True)+++def read_env():+ """SCRAPER_PORT and SCRAPER_API_KEY: the environment wins, then the kit's .env (last line wins,+ as in docker compose and scripts/scrape.py)."""+ found = {}+ try:+ with open(os.path.join(browser.KIT, ".env"), encoding="utf-8-sig", errors="replace") as f:+ for line in f:+ k, sep, v = line.strip().removeprefix("export ").partition("=")+ if sep and k.strip() in ENV_KEYS:+ found[k.strip()] = v.split(" #", 1)[0].strip().strip("\"'")+ except OSError:+ pass+ return {k: os.environ.get(k, found.get(k, "")) for k in ENV_KEYS}+++OPENAPI = {+ "openapi": "3.0.3",+ "info": {"title": "Google Maps Scraper Kit engine", "version": VERSION,+ "description": "Local only. Create a job, poll its Status until ok or failed, download the CSV. "+ f"Source and docs: {REPO}"},+ "paths": {+ "/api/v1/jobs": {+ "get": {"summary": "List jobs, newest first (null when there are none)"},+ "post": {"summary": "Create a job; returns 201 {\"id\": \"<uuid>\"}, or 422 {\"code\", \"message\"}",+ "requestBody": {"content": {"application/json": {"schema": {+ "type": "object", "required": ["name", "keywords", "lang", "lat", "lon", "max_time"],+ "properties": {+ "name": {"type": "string", "description": "any label"},+ "keywords": {"type": "array", "items": {"type": "string"}},+ "lang": {"type": "string", "description": "2-letter interface language, e.g. en"},+ "lat": {"type": "string"}, "lon": {"type": "string"},+ "max_time": {"type": "integer", "description": "seconds"},+ "depth": {"type": "integer", "description": "result batches per keyword, about 20 places each (default 1)"},+ "zoom": {"type": "integer", "description": "map zoom 1-21 (default 15)"},+ "email": {"type": "boolean", "description": "visit each website for emails"},+ "proxies": {"type": "array", "items": {"type": "string"},+ "description": "http(s):// or socks5://host:port; the first is used"},+ "radius": {"type": "integer", "description": "accepted, not used"},+ "fast_mode": {"type": "boolean", "description": "accepted, not used"}}}}}}}},+ "/api/v1/jobs/{id}": {+ "get": {"summary": "One job: ID, Name, Date, Status (pending, working, ok, failed), Data, Error"},+ "delete": {"summary": "Delete a job and its results (stops it if it is running)"}},+ "/api/v1/jobs/{id}/download": {"get": {"summary": "The job's CSV (the rows so far while it works)"}},+ },+}++PAGE = """<!doctype html>+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">+<title>Google Maps Scraper Kit</title>+<style nonce="{nonce}">+:root {{ color-scheme: light dark; --bg: #fff; --fg: #1d1d1f; --muted: #6e6e73; --line: #d2d2d7; --accent: #0a66c2; }}+@media (prefers-color-scheme: dark) {{ :root {{ --bg: #111; --fg: #f2f2f2; --muted: #a1a1a6; --line: #333; --accent: #4ea1ff; }} }}+body {{ margin: 0 auto; max-width: 960px; padding: 16px; background: var(--bg); color: var(--fg);+ font: 15px/1.45 system-ui, -apple-system, Segoe UI, sans-serif; }}+h1 {{ font-size: 1.4em; margin: 0 0 4px; }} a {{ color: var(--accent); }} .muted {{ color: var(--muted); }}+form {{ display: grid; gap: 10px; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); margin: 16px 0; }}+label {{ display: grid; gap: 4px; font-size: .9em; }} .wide {{ grid-column: 1 / -1; }}+input, textarea, button {{ font: inherit; padding: 6px 8px; border: 1px solid var(--line); border-radius: 6px;+ background: var(--bg); color: var(--fg); }}+button {{ cursor: pointer; }} button.primary {{ background: var(--accent); color: #fff; border-color: var(--accent); }}+table {{ width: 100%; border-collapse: collapse; }} th, td {{ text-align: left; padding: 6px; border-bottom: 1px solid var(--line); }}+td.err {{ color: var(--muted); font-size: .85em; }} .row {{ display: flex; gap: 6px; flex-wrap: wrap; }}+#msg {{ min-height: 1.4em; }} .scroll {{ overflow-x: auto; }} [hidden] {{ display: none !important; }}+</style></head><body>+<h1>Google Maps Scraper Kit</h1>+<p class="muted">Local engine {version}. <a href="{repo}" rel="noopener">GitHub</a> ·+<a href="/api/docs">API</a> · Business listings only. Results stay on this machine.</p>+<form id="f">+ <label class="wide">Keywords, one per line<textarea id="keywords" rows="3" required+ placeholder="coffee shops in Austin TX"></textarea></label>+ <label>Latitude<input id="lat" required placeholder="30.2672" inputmode="decimal"></label>+ <label>Longitude<input id="lon" required placeholder="-97.7431" inputmode="decimal"></label>+ <label>Depth (about 20 places each)<input id="depth" type="number" min="1" max="1000" value="5"></label>+ <label>Language<input id="lang" value="en" maxlength="2" pattern="[A-Za-z]{{2}}"></label>+ <label>Time limit (seconds)<input id="max_time" type="number" min="1" max="86400" value="600"></label>+ <label><span>Emails</span><span><input type="checkbox" id="email" checked> visit each website</span></label>+ <div class="wide row"><button class="primary">Start scraping</button>+ <span class="muted">Coordinates: right-click a spot in Google Maps and click the numbers.</span></div>+ <label class="wide" id="keybox" hidden>API key (SCRAPER_API_KEY)<input id="key" type="password" autocomplete="off"></label>+</form>+<p id="msg" role="status"></p>+<div class="scroll"><table><thead><tr><th>Job</th><th>Created (UTC)</th><th>Status</th><th></th></tr></thead>+<tbody id="jobs"></tbody></table></div>+<script nonce="{nonce}">+const $ = id => document.getElementById(id);+let key = ''; try {{ key = localStorage.getItem('gmskit-key') || ''; }} catch (e) {{}}+$('key').value = key;+$('key').addEventListener('change', () => {{ key = $('key').value; try {{ localStorage.setItem('gmskit-key', key); }} catch (e) {{}} load(); }});+function api(method, path, body) {{+ const h = {{}}; if (key) h['X-API-Key'] = key; if (body) h['Content-Type'] = 'application/json';+ return fetch(path, {{method, headers: h, body: body ? JSON.stringify(body) : undefined}}).then(r => {{+ if (r.status === 401) {{ $('keybox').hidden = false; throw new Error('this engine needs its API key'); }}+ return r;+ }});+}}+function say(t) {{ $('msg').textContent = t; }}+function cell(tr, text, cls) {{ const td = document.createElement('td'); td.textContent = text; if (cls) td.className = cls; tr.appendChild(td); return td; }}+function button(td, label, fn) {{ const b = document.createElement('button'); b.type = 'button'; b.textContent = label; b.addEventListener('click', fn); td.appendChild(b); }}+async function download(j) {{+ const r = await api('GET', '/api/v1/jobs/' + j.ID + '/download');+ if (!r.ok) return say('No results yet for this job.');+ const a = document.createElement('a'); a.href = URL.createObjectURL(await r.blob());+ a.download = 'results-' + j.ID.slice(0, 8) + '.csv'; document.body.appendChild(a); a.click(); a.remove();+}}+async function remove(j) {{+ if (!confirm('Delete this job and its results?')) return;+ const r = await api('DELETE', '/api/v1/jobs/' + j.ID); say(r.ok ? 'Deleted.' : 'Could not delete it.'); load();+}}+async function load() {{+ try {{+ const list = await (await api('GET', '/api/v1/jobs')).json() || [];+ const tb = $('jobs'); tb.replaceChildren();+ for (const j of list) {{+ const tr = document.createElement('tr');+ const name = cell(tr, j.Name); if (j.Error) {{ const s = document.createElement('div'); s.className = 'muted'; s.textContent = j.Error; name.appendChild(s); }}+ cell(tr, j.Date.replace('T', ' ').replace('+00:00', '')); cell(tr, j.Status);+ const act = cell(tr, ''); act.className = 'row';+ if (j.Status !== 'pending') button(act, 'Download', () => download(j));+ button(act, 'Delete', () => remove(j));+ tb.appendChild(tr);+ }}+ if (!list.length) {{ const tr = document.createElement('tr'); cell(tr, 'No jobs yet.', 'err').colSpan = 4; tb.appendChild(tr); }}+ }} catch (e) {{ say(e.message); }}+}}+$('f').addEventListener('submit', async ev => {{+ ev.preventDefault();+ const keywords = $('keywords').value.split('\\n').map(s => s.trim()).filter(Boolean);+ const body = {{name: keywords[0] || 'job', keywords, lang: $('lang').value.trim().toLowerCase(),+ lat: $('lat').value.trim(), lon: $('lon').value.trim(), depth: +$('depth').value,+ max_time: +$('max_time').value, email: $('email').checked, zoom: 15}};+ try {{+ const r = await api('POST', '/api/v1/jobs', body), d = await r.json();+ say(r.ok ? 'Job created. It runs in the background; this list refreshes by itself.' : 'Not started: ' + d.message);+ load();+ }} catch (e) {{ say(e.message); }}+}});+load(); setInterval(load, 3000);+</script></body></html>"""+++class Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"+ server_version = "gmskit-engine"+ sys_version = ""++ def log_message(self, fmt, *args): # errors are logged by reply(); polls would flood the log+ pass++ def reply(self, code, body=b"", ctype="application/json", headers=()):+ if ctype == "application/json":+ body = json.dumps(body).encode()+ self.send_response(code)+ self.send_header("Content-Type", ctype)+ self.send_header("Content-Length", str(len(body)))+ self.send_header("X-Engine", MARK)+ self.send_header("X-Content-Type-Options", "nosniff")+ self.send_header("Referrer-Policy", "no-referrer")+ self.send_header("Cache-Control", "no-store")+ for k, v in headers:+ self.send_header(k, v)+ self.end_headers()+ self.wfile.write(body)+ if code >= 400:+ log(f"{self.command} {self.path.split('?')[0]} -> {code}")++ def fail(self, code, message):+ self.reply(code, {"code": code, "message": message})++ def guard(self):+ """The checks ui-branding.conf made in front of the old engine, plus the optional key."""+ if not HOST_OK.match(self.headers.get("Host", "")):+ self.close_connection = True+ self.fail(421, "this engine answers to localhost only")+ return False+ origin = self.headers.get("Origin")+ if origin is not None and not ORIGIN_OK.match(origin):+ self.close_connection = True+ self.fail(403, "requests from other websites are refused")+ return False+ key, path = self.server.api_key, self.path.split("?")[0]+ if key and path not in ("/", "/api/docs") and not hmac.compare_digest(+ self.headers.get("X-API-Key", "").encode(), key.encode()):+ self.fail(401, "missing or wrong X-API-Key")+ return False+ return True++ def body(self):+ if "chunked" in (self.headers.get("Transfer-Encoding") or "").lower():+ self.close_connection = True+ self.fail(411, "send a Content-Length")+ return _REPLIED+ try:+ n = int(self.headers.get("Content-Length") or 0)+ except ValueError:+ n = -1+ if not 0 <= n <= MAX_BODY:+ self.close_connection = True+ self.fail(413 if n > MAX_BODY else 400, "the body must be 1 MB or less")+ return _REPLIED+ try:+ return json.loads(self.rfile.read(n) or b"null")+ except ValueError: # bad JSON or bad UTF-8+ self.fail(400, "the body is not valid JSON")+ return _REPLIED++ def download(self, jid):+ data = self.server.store.read_csv(jid) if JOB_ID.match(jid or "") else None+ if data is None:+ return self.fail(404, "no results for this job (yet)")+ self.reply(200, data, "text/csv; charset=utf-8",+ [("Content-Disposition", f'attachment; filename="{jid}.csv"')])++ def do_GET(self):+ if not self.guard():+ return+ path, _, query = self.path.partition("?")+ store = self.server.store+ if path == "/api/v1/jobs":+ return self.reply(200, store.all() or None) # null when empty, as before: the docs' health check+ m = JOB_PATH.match(path)+ if m:+ if m.group(2):+ return self.download(m.group(1))+ job = store.get(m.group(1))+ return self.reply(200, job) if job else self.fail(404, "job not found")+ if path == "/download": # the old web UI's download link+ return self.download(urllib.parse.parse_qs(query).get("id", [""])[0])+ if path == "/":+ nonce = secrets.token_urlsafe(16)+ csp = (f"default-src 'none'; script-src 'nonce-{nonce}'; style-src 'nonce-{nonce}'; "+ "connect-src 'self'; img-src 'self' data:; form-action 'self'; base-uri 'none'; "+ "frame-ancestors 'none'")+ page = PAGE.format(nonce=nonce, version=VERSION, repo=REPO).encode()+ return self.reply(200, page, "text/html; charset=utf-8", [("Content-Security-Policy", csp)])+ if path == "/api/docs":+ return self.reply(200, OPENAPI)+ self.fail(404, "not found")++ def do_POST(self):+ if not self.guard():+ return+ if self.path != "/api/v1/jobs":+ return self.fail(404, "not found")+ body = self.body()+ if body is _REPLIED:+ return+ try:+ name, data = jobs.validate(body)+ except ValueError as e:+ return self.fail(422, str(e))+ jid = self.server.store.create(name, data)+ self.server.worker.wake.set()+ log(f"job {jid} created: {len(data['keywords'])} keyword(s)")+ self.reply(201, {"id": jid})++ def do_DELETE(self):+ if not self.guard():+ return+ m = JOB_PATH.match(self.path)+ if not m or m.group(2):+ return self.fail(404, "not found")+ self.server.worker.cancel(m.group(1))+ if not self.server.store.delete(m.group(1)):+ return self.fail(404, "job not found")+ log(f"job {m.group(1)} deleted")+ self.reply(200, {"message": "deleted"})+++class Server(ThreadingHTTPServer):+ allow_reuse_address = os.name != "nt" # on Windows SO_REUSEADDR would let a second server share the port+ daemon_threads = True+++class Server6(Server):+ address_family = socket.AF_INET6+++def make_servers(port, store, worker, api_key=""):+ """The server on 127.0.0.1:port, plus one on [::1]:port when this machine has IPv6 loopback+ (Windows tries ::1 first for "localhost" and waits ~2 s before falling back to 127.0.0.1)."""+ servers = [Server(("127.0.0.1", port), Handler)]+ try:+ servers.append(Server6(("::1", servers[0].server_address[1]), Handler))+ except OSError:+ pass+ for s in servers:+ s.store, s.worker, s.api_key = store, worker, api_key+ return servers+++def main():+ ap = argparse.ArgumentParser(description="The kit's Google Maps engine (local HTTP API and web UI).")+ ap.add_argument("--port", type=int, help="default: SCRAPER_PORT from the environment or .env, else 8080")+ ap.add_argument("--data", default=os.path.join(browser.HOME, "data"), help="jobs and results folder")+ a = ap.parse_args()+ env = read_env()+ try:+ port = a.port or int(env["SCRAPER_PORT"] or 8080)+ except ValueError:+ sys.exit(f"✗ SCRAPER_PORT is not a number: {env['SCRAPER_PORT']!r}")+ browser.make_home()+ store = jobs.Store(a.data)+ if n := store.recover():+ log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")+ worker = jobs.Worker(store, log)+ try:+ servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"])+ except OSError as e:+ sys.exit(f"✗ Cannot listen on 127.0.0.1:{port}: {e}")+ worker.start()+ chrome = browser.find_chrome()+ log(f"engine {VERSION} on http://localhost:{port} (browser: {chrome or 'none found'}"+ f"{', API key required' if env['SCRAPER_API_KEY'] else ''})")+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))+ for s in servers[1:]:+ threading.Thread(target=s.serve_forever, daemon=True).start()+ try:+ servers[0].serve_forever()+ except KeyboardInterrupt:+ pass+ finally:+ worker.shutdown()+ log("engine stopped")+++if __name__ == "__main__":+ main()diff --git a/engine/test_cdp.py b/engine/test_cdp.pyindex 840bb2f..4571cb5 100644--- a/engine/test_cdp.py+++ b/engine/test_cdp.py@@ -177,7 +177,7 @@ class Parser(unittest.TestCase):self.assertEqual(got["phone"], "+1 512-555-0100")self.assertEqual(got["website"], "https://example.com/")self.assertEqual(got["review_rating"], 4.6)- self.assertEqual(got["review_count"], 321)+ self.assertNotIn("review_count", got) # (37, 1) is not the count: it comes from the card labelsself.assertEqual((got["latitude"], got["longitude"]), (30.2671, -97.743))self.assertEqual(got["place_id"], "ChIJabc")self.assertEqual(got["cid"], "255")diff --git a/engine/test_engine.py b/engine/test_engine.pynew file mode 100644index 0000000..7de576d--- /dev/null+++ b/engine/test_engine.py@@ -0,0 +1,441 @@+#!/usr/bin/env python3+"""Offline checks for the engine: job body validation, CSV rows, the sqlite store, the worker+(with a fake Chrome and fake Google results), the HTTP server and its guards, scripts/scrape.py's+own client against that server, the Chrome download's SHA-256 check and the email pass.+No network, no Chrome, no Docker.++ python3 engine/test_engine.py+"""+import csv, hashlib, http.client, io, json, os, re, shutil, sys, tempfile, threading, time, unittest, zipfile++HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, HERE)+import browser, jobs, maps, serve # noqa: E402+from cdp import CDPError # noqa: E402++GOOD = {"name": "t", "keywords": ["cafes in Austin"], "lang": "en", "lat": "30.2", "lon": "-97.7", "max_time": 60}+++def place(n, **kw):+ return dict({"title": f"Biz {n}", "data_id": f"0x1:0x{n:x}", "place_id": f"ChIJ{n}", "cid": str(n),+ "category": "Cafe", "review_rating": 4.5, "review_count": 10 + n,+ "latitude": 30.2, "longitude": -97.7}, **kw)+++class FakeCDP:+ closed = None+++class FakeBrowser:+ """Stands in for browser.Browser: no process; kill() makes the next collect fail like a dead Chrome."""+ made = []++ def __init__(self, **kw):+ self.kw, self.cdp, self.killed = kw, FakeCDP(), threading.Event()+ FakeBrowser.made.append(self)++ def kill(self):+ self.cdp.closed = OSError("killed")+ self.killed.set()++ def close(self):+ pass+++class FakePage:+ def __init__(self, cdp):+ pass++ def close(self):+ pass+++RESULTS = {} # keyword -> (places, reason) for the fake collect+++def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False):+ for kw, (places, reason) in RESULTS.items():+ if maps.search_url(kw).split("/@")[0].split("?")[0] in url:+ if reason == "hang": # Chrome stops answering until the watchdog kills it+ b = FakeBrowser.made[-1]+ b.killed.wait(10)+ raise CDPError("connection lost: killed")+ if reason == "slow":+ end = time.time() + 5+ while not stop() and time.time() < end:+ time.sleep(0.02)+ return places, "stopped", {"batches": 1, "dom_links": 0}+ return places, reason, {"batches": 1, "dom_links": 0}+ return [], "no-feed", {"batches": 0, "dom_links": 0}+++class Fakes(unittest.TestCase):+ def setUp(self):+ self.dir = tempfile.mkdtemp()+ self.saved = (browser.Browser, jobs.Page, maps.collect)+ browser.Browser, jobs.Page, maps.collect = FakeBrowser, FakePage, fake_collect+ FakeBrowser.made.clear()+ RESULTS.clear()+ self.store = jobs.Store(os.path.join(self.dir, "data"))+ self.lines = []+ self.worker = jobs.Worker(self.store, self.lines.append)++ def tearDown(self):+ browser.Browser, jobs.Page, maps.collect = self.saved+ self.worker.shutdown()+ if self.worker.is_alive():+ self.worker.join(5)+ self.store.db.close()+ shutil.rmtree(self.dir, ignore_errors=True)++ def run_one(self, **body):+ name, data = jobs.validate(dict(GOOD, **body))+ jid = self.store.create(name, data)+ self.worker.run_job(self.store.claim())+ return jid, self.store.get(jid)++ def rows(self, jid):+ return list(csv.DictReader(io.StringIO(self.store.read_csv(jid).decode("utf-8"))))+++class Validate(unittest.TestCase):+ def err(self, **body):+ with self.assertRaises(ValueError) as c:+ jobs.validate(body)+ return str(c.exception)++ def test_old_engine_messages(self):+ self.assertEqual(self.err(), "missing name")+ self.assertEqual(self.err(name="x"), "missing keywords")+ self.assertEqual(self.err(name="x", keywords=["a"]), "missing lang")+ self.assertEqual(self.err(name="x", keywords=["a"], lang="eng"), "invalid lang")+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en"), "missing geo coordinates")+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lat="1", lon="2"), "missing max time")++ def test_bad_values(self):+ self.assertIn("invalid geo", self.err(**dict(GOOD, lat="north")))+ self.assertIn("invalid geo", self.err(**dict(GOOD, lat="91")))+ self.assertIn("invalid geo", self.err(**dict(GOOD, lon="nan")))+ self.assertIn("invalid geo", self.err(**dict(GOOD, lat=True)))+ self.assertIn("invalid depth", self.err(**dict(GOOD, depth=0)))+ self.assertIn("invalid depth", self.err(**dict(GOOD, depth="5")))+ self.assertIn("invalid max time", self.err(**dict(GOOD, max_time=-1)))+ self.assertIn("invalid keywords", self.err(**dict(GOOD, keywords=["ok", ""])))+ self.assertIn("usernames and passwords", self.err(**dict(GOOD, proxies=["http://u:p@h:1"])))+ self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["ftp://h:1"])))+ self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["http://h:99999"])))+ with self.assertRaisesRegex(ValueError, "JSON object"):+ jobs.validate([1])++ def test_defaults_and_numbers(self):+ name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))+ self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))+ self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])+++class Rows(unittest.TestCase):+ def test_columns_and_mapping(self):+ r = jobs.to_row(dict(place(1), open_state="Open 24 hours", description="Good coffee",+ categories="Cafe | Bakery", junk="x"), "cafes")+ self.assertEqual(len(jobs.COLUMNS), 36)+ self.assertEqual(list(r), jobs.COLUMNS + jobs.EXTRA)+ self.assertEqual((r["input_id"], r["status"], r["descriptions"], r["categories"]),+ ("cafes", "Open 24 hours", "Good coffee", "Cafe | Bakery"))+ self.assertEqual((r["review_rating"], r["review_count"], r["emails"]), ("4.5", "11", ""))+ self.assertTrue(r["link"].startswith("https://www.google.com/maps/search/?api=1&query=Biz%201&query_place_id=ChIJ1"))+ dom = jobs.to_row({"title": "A", "link": "https://www.google.com/maps/place/A/data=!1s0x1:0x2"}, "k")+ self.assertEqual(dom["link"], "https://www.google.com/maps/place/A/data=!1s0x1:0x2")++ def test_csv_utf8_no_bom(self):+ raw = jobs.to_csv([jobs.to_row(place(1, title="Café ☕"), "k")])+ self.assertFalse(raw.startswith(b"\xef\xbb\xbf"))+ self.assertEqual(next(csv.DictReader(io.StringIO(raw.decode())))["title"], "Café ☕")+++class Store(Fakes):+ def test_lifecycle(self):+ a = self.store.create("a", {"x": 1})+ b = self.store.create("b", {"x": 2})+ self.assertEqual([j["ID"] for j in self.store.all()], [b, a]) # newest first+ self.assertEqual(self.store.claim()["ID"], a) # oldest pending first+ self.assertEqual(self.store.get(a)["Status"], "working")+ self.assertEqual(self.store.recover(), 1)+ self.assertEqual(self.store.get(a)["Status"], "failed")+ self.assertTrue(self.store.write_csv(b, []))+ self.assertTrue(self.store.read_csv(b).startswith(b"input_id,link,title"))+ self.assertTrue(self.store.delete(b))+ self.assertIsNone(self.store.read_csv(b))+ self.assertFalse(self.store.write_csv(b, [])) # a deleted job gets no file back+ self.assertFalse(self.store.delete(b))+ self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error"})+ self.assertRegex(self.store.get(a)["Date"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00$")+++class Worker(Fakes):+ def test_keywords_dedupe_into_one_csv(self):+ RESULTS["cafes"] = ([place(1), place(2)], "end-of-list")+ RESULTS["bakeries"] = ([place(2), place(3)], "depth")+ jid, job = self.run_one(keywords=["cafes", "bakeries"], depth=2, lang="de", proxies=["http://10.0.0.1:3128"])+ self.assertEqual((job["Status"], job["Error"]), ("ok", ""))+ rows = self.rows(jid)+ self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2", "Biz 3"])+ self.assertEqual([r["input_id"] for r in rows], ["cafes", "cafes", "bakeries"])+ self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "proxy": "http://10.0.0.1:3128"})+ self.assertFalse(any(re.search(r"Biz \d", l) for l in self.lines)) # counts only in the log++ def test_no_results_is_ok_with_header(self):+ jid, job = self.run_one(keywords=["nothing here"])+ self.assertEqual(job["Status"], "ok")+ self.assertEqual(self.rows(jid), [])++ def test_no_chrome_fails_with_reason(self):+ def no_chrome(**kw):+ raise browser.BrowserError("No Chrome, Chromium, Edge or Brave found.")+ browser.Browser = no_chrome+ _, job = self.run_one()+ self.assertEqual(job["Status"], "failed")+ self.assertIn("No Chrome", job["Error"])++ def test_blocked_fails(self):+ RESULTS["cafes in Austin"] = ([], "blocked")+ _, job = self.run_one()+ self.assertEqual(job["Status"], "failed")+ self.assertIn("unusual-traffic", job["Error"])++ def test_watchdog_kills_a_hung_chrome(self):+ RESULTS["cafes in Austin"] = ([], "hang")+ saved, jobs.WATCHDOG_GRACE = jobs.WATCHDOG_GRACE, 0.2+ try:+ t = time.monotonic()+ _, job = self.run_one(max_time=1)+ finally:+ jobs.WATCHDOG_GRACE = saved+ self.assertLess(time.monotonic() - t, 5)+ self.assertTrue(FakeBrowser.made[0].killed.is_set())+ self.assertEqual(job["Status"], "failed")++ def test_delete_while_running(self):+ RESULTS["cafes in Austin"] = ([place(1)], "slow")+ name, data = jobs.validate(GOOD)+ jid = self.store.create(name, data)+ t = threading.Thread(target=self.worker.run_job, args=(self.store.claim(),))+ t.start()+ time.sleep(0.2)+ self.worker.cancel(jid)+ self.store.delete(jid)+ t.join(5)+ self.assertFalse(t.is_alive())+ self.assertIsNone(self.store.get(jid))+ self.assertFalse(os.path.exists(self.store.csv_path(jid)))++ def test_email_pass(self):+ RESULTS["cafes in Austin"] = ([place(1, website="https://www.biz1.com"), place(2), place(3, website="x.org")], "end-of-list")+ pages = {"https://www.biz1.com": 'mail <a href="mailto:[email protected]">us</a> or sales@biz1.com',+ "x.org": "[email protected] nothing"}+ saved = jobs.fetch+ jobs.fetch = lambda url, timeout=0: pages.get(url, "")+ try:+ jid, job = self.run_one(email=True)+ finally:+ jobs.fetch = saved+ self.assertEqual(job["Status"], "ok")+ self.assertEqual([r["emails"] for r in self.rows(jid)], ["[email protected], [email protected]", "", ""])+++class Emails(unittest.TestCase):+ def test_find_emails(self):+ text = "[email protected] x [email protected] [email protected] [email protected] [email protected] [email protected]"+ self.assertEqual(jobs.find_emails(text, "shop.com"), ["[email protected]", "[email protected]"])++ def test_public_only(self):+ self.assertFalse(jobs._public_ip("127.0.0.1"))+ self.assertFalse(jobs._public_ip("192.168.1.1"))+ self.assertFalse(jobs._public_ip("169.254.169.254"))+ self.assertFalse(jobs._public_ip("::1"))+ self.assertTrue(jobs._public_ip("8.8.8.8"))+ self.assertFalse(jobs._public_host("localhost"))+ self.assertEqual(jobs.fetch("http://127.0.0.1:9/"), "")+ self.assertEqual(jobs.site_host("https://www.Shop.com/x"), "shop.com")+++class Server(Fakes):+ key = ""++ def setUp(self):+ super().setUp()+ self.servers = serve.make_servers(0, self.store, self.worker, self.key)+ self.port = self.servers[0].server_address[1]+ for s in self.servers:+ threading.Thread(target=s.serve_forever, daemon=True).start()++ def tearDown(self):+ for s in self.servers:+ s.shutdown()+ s.server_close()+ super().tearDown()++ def call(self, method, path, body=None, headers=None, raw=None):+ c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)+ h = {"Host": f"localhost:{self.port}"}+ h.update(headers or {})+ data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)+ c.request(method, path, data, h)+ r = c.getresponse()+ out = (r.status, r.read(), r)+ c.close()+ return out+++class Api(Server):+ def test_flow(self):+ code, body, r = self.call("GET", "/api/v1/jobs")+ self.assertEqual((code, body), (200, b"null")) # the docs' health check prints null+ self.assertTrue(r.getheader("X-Engine").startswith("gmskit-engine/"))+ code, body, _ = self.call("POST", "/api/v1/jobs", {})+ self.assertEqual((code, json.loads(body)), (422, {"code": 422, "message": "missing name"}))+ code, body, _ = self.call("POST", "/api/v1/jobs", GOOD)+ self.assertEqual(code, 201)+ jid = json.loads(body)["id"]+ self.assertTrue(self.worker.wake.is_set())+ job = json.loads(self.call("GET", f"/api/v1/jobs/{jid}")[1])+ self.assertEqual((job["ID"], job["Name"], job["Status"], job["Data"]["keywords"]), (jid, "t", "pending", ["cafes in Austin"]))+ self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download")[0], 404)+ self.store.write_csv(jid, [jobs.to_row(place(1), "k")])+ code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download")+ self.assertEqual((code, r.getheader("Content-Type")), (200, "text/csv; charset=utf-8"))+ self.assertEqual(self.call("GET", f"/download?id={jid}")[1], body)+ self.assertEqual(len(json.loads(self.call("GET", "/api/v1/jobs")[1])), 1)+ self.assertEqual(self.call("DELETE", f"/api/v1/jobs/{jid}")[0], 200)+ self.assertEqual(self.call("DELETE", f"/api/v1/jobs/{jid}")[0], 404)+ self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}")[0], 404)+ self.assertEqual(self.call("GET", "/download?id=../../etc/passwd")[0], 404)++ def test_guards(self):+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": "evil.com"})[0], 421)+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": "localhost.evil.com"})[0], 421)+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "https://evil.com"})[0], 403)+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "null"})[0], 403)+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "http://localhost.evil.com"})[0], 403)+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": f"http://localhost:{self.port}"})[0], 201)+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": f"127.0.0.1:{self.port}"})[0], 200)+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"{nope")[0], 400)+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"\xff\xfe")[0], 400)+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"", headers={"Content-Length": "99999999"})[0], 413)+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"0\r\n\r\n", headers={"Transfer-Encoding": "chunked"})[0], 411)+ self.assertEqual(self.call("POST", "/elsewhere", GOOD)[0], 404)++ def test_page_and_docs(self):+ code, body, r = self.call("GET", "/")+ csp = r.getheader("Content-Security-Policy")+ nonce = re.search(r"'nonce-([^']+)'", csp).group(1)+ self.assertIn(f'<script nonce="{nonce}">', body.decode())+ self.assertIn("github.com/Mahanaicoach/google-maps-scraper-kit", body.decode())+ self.assertIn("frame-ancestors 'none'", csp)+ self.assertNotRegex(body.decode(), r"\{[a-z_]+\}") # every placeholder filled+ code, body, _ = self.call("GET", "/api/docs")+ self.assertEqual(json.loads(body)["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]+ ["schema"]["required"], ["name", "keywords", "lang", "lat", "lon", "max_time"])+++class ApiKey(Server):+ key = "s3cret"++ def test_key(self):+ self.assertEqual(self.call("GET", "/api/v1/jobs")[0], 401)+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"X-API-Key": "wrong"})[0], 401)+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"X-API-Key": "s3cret"})[0], 200)+ self.assertEqual(self.call("GET", "/")[0], 200) # the page holds no data; it asks for the key+++class ScrapeClient(Server):+ """scripts/scrape.py's own create, poll and download against this engine, with the worker running."""++ def test_run_job(self):+ RESULTS["cafes in Austin"] = ([place(1), place(2)], "end-of-list")+ os.environ["SCRAPER_BASE_URL"] = f"http://localhost:{self.port}"+ sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))+ try:+ import scrape+ scrape.BASE = os.environ["SCRAPER_BASE_URL"]+ scrape.POLL_EVERY = 1+ scrape._local_docker = lambda: False # never touch a real Docker engine from a test+ self.worker.start()+ out = io.StringIO()+ saved, sys.stdout = sys.stdout, out+ try:+ jid, rows = scrape.run_job({"name": "t", "keywords": ["cafes in Austin"], "lang": "en",+ "lat": "30.2", "lon": "-97.7", "depth": 1, "email": False,+ "max_time": 60, "zoom": 15, "fast_mode": False, "radius": 10000})+ finally:+ sys.stdout = saved+ finally:+ del os.environ["SCRAPER_BASE_URL"]+ self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2"])+ self.assertEqual(self.store.get(jid)["Status"], "ok")+ for k in scrape.LEAD:+ self.assertIn(k, rows[0])+++class Download(unittest.TestCase):+ def setUp(self):+ self.dir = tempfile.mkdtemp()+ self.saved = (browser.HOME, dict(browser.CFT_SHA256))+ browser.HOME = self.dir++ def tearDown(self):+ browser.HOME = self.saved[0]+ browser.CFT_SHA256.clear()+ browser.CFT_SHA256.update(self.saved[1])+ shutil.rmtree(self.dir, ignore_errors=True)++ @staticmethod+ def zipped():+ buf = io.BytesIO()+ with zipfile.ZipFile(buf, "w") as z:+ info = zipfile.ZipInfo("chrome-headless-shell-linux64/chrome-headless-shell")+ info.external_attr = 0o100755 << 16+ z.writestr(info, b"#!/bin/sh\n")+ z.writestr("chrome-headless-shell-linux64/libfoo.so", b"x")+ return buf.getvalue()++ def opener(self, data):+ return lambda url, timeout=0: io.BytesIO(data)++ def test_platforms(self):+ self.assertEqual(browser.cft_platform("Darwin", "arm64"), "mac-arm64")+ self.assertEqual(browser.cft_platform("Darwin", "x86_64"), "mac-x64")+ self.assertEqual(browser.cft_platform("Linux", "aarch64"), "linux-arm64")+ self.assertEqual(browser.cft_platform("Linux", "x86_64"), "linux64")+ self.assertEqual(browser.cft_platform("Windows", "AMD64"), "win64")+ self.assertEqual(browser.cft_platform("Windows", "ARM64"), "win64")+ self.assertIsNone(browser.cft_platform("Linux", "riscv64"))+ self.assertTrue(all(re.fullmatch(r"[0-9a-f]{64}", h) for h in self.saved[1].values()))++ def test_mismatch_is_deleted_and_never_run(self):+ with self.assertRaises(browser.BrowserError) as c:+ browser.download(lambda m: None, "linux64", self.opener(self.zipped()))+ self.assertIn("does not match the SHA-256", str(c.exception))+ leftovers = [f for _, _, fs in os.walk(self.dir) for f in fs]+ self.assertEqual(leftovers, [])++ def test_verified_download_unpacks(self):+ data = self.zipped()+ browser.CFT_SHA256["linux64"] = hashlib.sha256(data).hexdigest()+ path = browser.download(lambda m: None, "linux64", self.opener(data))+ self.assertEqual(path, browser.shell_path("linux64"))+ self.assertTrue(os.path.isfile(path))+ if os.name != "nt":+ self.assertTrue(os.access(path, os.X_OK))+ self.assertEqual(os.stat(self.dir).st_mode & 0o777, 0o700)+ self.assertFalse(any(f.endswith(".part") for _, _, fs in os.walk(self.dir) for f in fs))++ def test_missing_libraries_hint(self):+ err = "chrome: error while loading shared libraries: libnss3.so: cannot open shared object file\n"+ msg = browser.start_error("/x/chrome", 127, err)+ self.assertIn("libnss3.so", msg)+ self.assertIn("sudo apt-get install -y libnss3", msg)+ self.assertIn("code 3", browser.start_error("/x/chrome", 3, ""))+++if __name__ == "__main__":+ unittest.main(verbosity=1)diff --git a/scripts/engine.py b/scripts/engine.pynew file mode 100644index 0000000..2e58294--- /dev/null+++ b/scripts/engine.py@@ -0,0 +1,274 @@+#!/usr/bin/env python3+"""Run the kit's own engine on this machine, without Docker.++ python3 scripts/engine.py start # first run on a machine without Chrome downloads one (~100 MB)+ python3 scripts/engine.py status+ python3 scripts/engine.py restart+ python3 scripts/engine.py stop++Standard library only. The engine (engine/serve.py) serves http://localhost:8080 (SCRAPER_PORT in+.env) with the same API the scripts, slash commands and web UI use with Docker. It drives an+installed Chrome, Chromium, Edge or Brave; on a machine with none, `start` downloads Chrome for+Testing's headless shell, pinned by version and SHA-256. Everything lives in .engine/ in the kit+folder; delete it to remove the engine and its results.+"""+import json, os, signal, socket, subprocess, sys, time, urllib.error, urllib.parse, urllib.request++for _stream in (sys.stdout, sys.stderr): # ▶ ✓ ✗ on a Windows cp1252 console+ try:+ _stream.reconfigure(encoding="utf-8", errors="replace")+ except (AttributeError, ValueError):+ pass++KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, os.path.join(KIT, "engine"))+import browser # noqa: E402++HOME = browser.HOME+SERVE = os.path.join(KIT, "engine", "serve.py")+STATE = os.path.join(HOME, "state.json")+LOCK = os.path.join(HOME, "start.lock")+LOG = os.path.join(HOME, "engine.log")+LOG_KEEP = 2_000_000 # bytes of engine.log kept across starts+MARK = "gmskit-engine/"+_NO_PROXY = urllib.request.build_opener(urllib.request.ProxyHandler({})) # HTTP_PROXY must not catch localhost+PY = "py" if os.name == "nt" else "python3" # what the user types; the README says `py` on Windows+++def public_port():+ """The port scrape.py talks to (SCRAPER_PORT / SCRAPER_BASE_URL, environment or .env), read by+ scrape.py's own loader so both always agree."""+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+ import scrape+ u = urllib.parse.urlparse(scrape.BASE)+ if u.hostname not in ("localhost", "127.0.0.1", "::1"):+ sys.exit(f"✗ SCRAPER_BASE_URL points at {scrape.BASE}; engine.py runs the engine on this machine only.")+ return u.port or 80+++def probe(port, timeout=2):+ """"ours" (this kit's engine answers), "other" (something else, e.g. the Docker engine) or None."""+ try:+ with _NO_PROXY.open(f"http://127.0.0.1:{port}/api/v1/jobs", timeout=timeout) as r:+ return "ours" if (r.headers.get("X-Engine") or "").startswith(MARK) else "other"+ except urllib.error.HTTPError as e: # 401 without the key is still ours+ e.close()+ return "ours" if (e.headers.get("X-Engine") or "").startswith(MARK) else "other"+ except Exception: # noqa: BLE001 - nothing listening, or not HTTP+ return None+++def port_free(port):+ with socket.socket() as s:+ if os.name != "nt": # like the engine's own server: connections in TIME_WAIT do not block+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)+ try:+ s.bind(("127.0.0.1", port))+ return True+ except OSError:+ return False+++def read_state():+ try:+ with open(STATE, encoding="utf-8") as f:+ return json.load(f)+ except (OSError, ValueError):+ return None+++def write_state(st):+ tmp = STATE + ".tmp"+ with open(tmp, "w", encoding="utf-8") as f:+ json.dump(st, f)+ os.replace(tmp, STATE) # a crash mid-write never leaves half a state file+++def command_line(pid):+ """The command line of `pid`; "" if the process is gone; None if it cannot be read (PowerShell+ blocked or too slow), which stop must not mistake for "gone"."""+ try:+ if os.name == "nt":+ ps = ["powershell", "-NoProfile", "-Command",+ f"(Get-CimInstance Win32_Process -Filter 'ProcessId={int(pid)}').CommandLine"]+ out = subprocess.run(ps, capture_output=True, text=True, errors="replace", timeout=20)+ return None if out.returncode else out.stdout+ if os.path.exists(f"/proc/{int(pid)}/cmdline"):+ with open(f"/proc/{int(pid)}/cmdline", "rb") as f:+ return f.read().replace(b"\0", b" ").decode("utf-8", "replace")+ return subprocess.run(["ps", "-p", str(int(pid)), "-o", "command="], capture_output=True,+ text=True, errors="replace", timeout=10).stdout+ except (OSError, ValueError, subprocess.SubprocessError):+ return None+++def kill(pid):+ """Stop the engine and its Chrome. False if it could not be told whether `pid` is the engine:+ after a reboot or crash the pid in state.json may belong to an unrelated process."""+ cmd = command_line(pid)+ if cmd is None:+ return False+ if SERVE not in cmd:+ return True # gone, or someone else's process now: nothing of ours to stop+ if os.name == "nt":+ subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True)+ return True+ try:+ os.killpg(pid, signal.SIGTERM) # the engine leads its own process group, Chrome included+ except (ProcessLookupError, PermissionError):+ return True+ for _ in range(50):+ try:+ os.killpg(pid, 0)+ except (ProcessLookupError, PermissionError):+ return True+ time.sleep(0.1)+ try:+ os.killpg(pid, signal.SIGKILL)+ except (ProcessLookupError, PermissionError):+ pass+ return True+++def tail(path, lines=15):+ try:+ with open(path, "rb") as f:+ f.seek(max(0, os.path.getsize(path) - 8000))+ return "\n".join(f.read().decode("utf-8", "replace").splitlines()[-lines:])+ except OSError:+ return ""+++def trim_log():+ """Keep the last LOG_KEEP bytes of engine.log, so it cannot grow without bound across starts."""+ try:+ if os.path.getsize(LOG) > LOG_KEEP:+ with open(LOG, "rb") as f:+ f.seek(-LOG_KEEP, os.SEEK_END)+ keep = f.read()+ with open(LOG, "wb") as f:+ f.write(keep)+ except OSError:+ pass+++def busy(port):+ hint = f"netstat -ano | findstr :{port}" if os.name == "nt" else f"lsof -nP -iTCP:{port} -sTCP:LISTEN"+ sys.exit(f"✗ Port {port} is in use. Is the Docker engine running? Stop it with `docker compose stop`,\n"+ f" or set SCRAPER_PORT in .env to a free port. To see what holds it: {hint}")+++def chrome(download):+ """A browser for the engine; downloads the pinned one only when allowed (start, never restart:+ scrape.py may restart the engine unattended, and fetching code is for an explicit start)."""+ path = browser.find_chrome()+ if path:+ return path+ if not download:+ sys.exit(f"✗ No Chrome found. Run: {PY} scripts/engine.py start (it downloads one)")+ try:+ return browser.download()+ except browser.BrowserError as e:+ sys.exit(f"✗ {e}")+++def start(download=True):+ port = public_port()+ seen = probe(port)+ if seen == "ours":+ print(f"✓ The engine is already running: http://localhost:{port}")+ return+ if seen == "other":+ busy(port)+ if read_state():+ stop(quiet=True) # stale state from a crash or reboot: clean up whatever is still ours+ if not port_free(port):+ busy(port)+ browser.make_home()+ try: # one start at a time+ lock = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)+ except FileExistsError:+ if time.time() - os.path.getmtime(LOCK) < 600:+ sys.exit("✗ Another `engine.py start` is running. Wait for it, or delete .engine/start.lock.")+ os.remove(LOCK) # left by a start that crashed+ lock = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)+ try:+ _start(port, chrome(download))+ finally:+ os.close(lock)+ os.remove(LOCK)+++def _start(port, chrome_path):+ trim_log()+ env = dict(os.environ, CHROME=chrome_path, SCRAPER_PORT=str(port))+ with open(LOG, "ab") as out: # the engine keeps its own handle+ kw = {"stdout": out, "stderr": out, "stdin": subprocess.DEVNULL, "cwd": HOME, "env": env}+ if os.name == "nt":+ # A hidden console, not DETACHED_PROCESS: Chrome started from a process with no console+ # at all closed at its first page on Windows.+ kw["creationflags"] = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP+ else:+ kw["start_new_session"] = True+ try:+ proc = subprocess.Popen([sys.executable, SERVE, "--port", str(port)], **kw)+ except OSError as e:+ sys.exit(f"✗ Could not start the engine: {e}")+ write_state({"pid": proc.pid, "port": port, "chrome": chrome_path, "log": LOG})+ deadline = time.time() + 30+ while time.time() < deadline:+ if probe(port) == "ours":+ print(f"✓ Engine running: http://localhost:{port} (web UI and API)\n"+ f" Browser: {chrome_path}\n"+ f" Stop it with: {PY} scripts/engine.py stop")+ return+ if proc.poll() is not None:+ stop(quiet=True)+ sys.exit(f"✗ The engine exited at once (code {proc.returncode}). The end of {LOG}:\n{tail(LOG)}")+ time.sleep(0.5)+ stop(quiet=True)+ sys.exit(f"✗ The engine did not answer within 30 seconds. The end of {LOG}:\n{tail(LOG)}")+++def stop(quiet=False):+ st = read_state()+ if not st:+ if not quiet:+ print("The engine is not running.")+ return+ if st.get("pid") and not kill(st["pid"]):+ how = "taskkill /PID {} /T /F" if os.name == "nt" else "kill {}"+ sys.exit(f"✗ Could not check whether process {st['pid']} is the engine, so it was left running.\n"+ f" Stop it yourself: {how.format(st['pid'])}")+ try:+ os.remove(STATE)+ except OSError:+ pass+ if not quiet:+ print("✓ Engine stopped. Its jobs and results stay in .engine/data.")+++def status():+ port = public_port()+ seen = probe(port)+ if seen == "ours":+ st = read_state() or {}+ print(f"✓ Running on http://localhost:{port}" + (f" (browser: {st['chrome']})" if st.get("chrome") else ""))+ elif seen == "other":+ print(f"ℹ Port {port} answers, but not as this kit's engine (the Docker engine, probably).")+ sys.exit(1)+ else:+ print(f"✗ Not running. Start it with: {PY} scripts/engine.py start")+ sys.exit(1)+++def main():+ cmds = {"start": start, "stop": stop, "status": status,+ "restart": lambda: (stop(quiet=True), start(download=False))}+ if len(sys.argv) != 2 or sys.argv[1] not in cmds:+ sys.exit(f"usage: {PY} scripts/engine.py start|stop|status|restart")+ cmds[sys.argv[1]]()+++if __name__ == "__main__":+ main()