Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

feat(engine): P1 engine core: jobs, compatible API server, pinned Chrome, engine.py

Mahan Jafari0cebf888024d
diff --git a/.claude/settings.json b/.claude/settings.json
index 16fd343..c494561 100644
--- a/.claude/settings.json
+++ b/.claude/settings.json
@@ -15,6 +15,20 @@
"Bash(curl -s -X DELETE http://localhost:8080/:*)",
"Bash(curl -X DELETE http://localhost:8080/:*)",
"Bash(python3 scripts/test_scrape.py)",
+ "Bash(python3 scripts/engine.py start)",
+ "Bash(python3 scripts/engine.py stop)",
+ "Bash(python3 scripts/engine.py status)",
+ "Bash(python3 scripts/engine.py restart)",
+ "Bash(python scripts/engine.py start)",
+ "Bash(python scripts/engine.py stop)",
+ "Bash(python scripts/engine.py status)",
+ "Bash(python scripts/engine.py restart)",
+ "Bash(py scripts/engine.py start)",
+ "Bash(py scripts/engine.py stop)",
+ "Bash(py scripts/engine.py status)",
+ "Bash(py scripts/engine.py restart)",
+ "Bash(python3 engine/test_engine.py)",
+ "Bash(python3 engine/test_cdp.py)",
"Bash(./scripts/scrape.sh:*)",
"Bash(bash scripts/scrape.sh:*)",
"Bash(python3 scripts/scrape.py:*)",
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.yml
new file mode 100644
index 0000000..207e4ca
--- /dev/null
+++ b/.github/workflows/engine-live.yml
@@ -0,0 +1,57 @@
+# The kit's own engine against real Google Maps, once per platform: start it the way a user does,
+# run one small scrape through scripts/scrape.py, and require places back. Runs when the engine
+# changes, or by hand (Actions > engine-live > Run workflow). The logs print counts only: scraped
+# names, phones and emails never reach them.
+name: engine-live
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - "engine/**"
+ - "scripts/engine.py"
+ - ".github/workflows/engine-live.yml"
+jobs:
+ live:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 15
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - name: start the engine (downloads the pinned Chrome only when none is installed)
+ run: python scripts/engine.py start
+ - name: status, guards and web UI
+ run: |
+ python scripts/engine.py status
+ test "$(curl -s http://localhost:8080/api/v1/jobs)" = null
+ test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Host: evil.com' http://localhost:8080/api/v1/jobs)" = 421
+ test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Origin: https://evil.com' -X POST -d '{}' http://localhost:8080/api/v1/jobs)" = 403
+ curl -s -X POST -d '{}' http://localhost:8080/api/v1/jobs | grep -q 'missing name'
+ curl -s http://localhost:8080/ | grep -q 'github.com/Mahanaicoach/google-maps-scraper-kit'
+ - name: scrape (depth 2, no email crawl)
+ run: |
+ start=$(date +%s)
+ python scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --out leads.csv > scrape.log 2>&1 || true
+ echo "seconds=$(( $(date +%s) - start ))"
+ grep -v "^ •" scrape.log | tr '\r' '\n' | grep -v "^ status:" || true
+ python - <<'PY'
+ import csv, sys
+ rows = list(csv.DictReader(open("leads.csv", encoding="utf-8-sig")))
+ fill = {k: sum(1 for r in rows if r.get(k)) for k in ("title", "phone", "website", "address", "review_rating")}
+ print(f"places={len(rows)}", fill)
+ sys.exit(0 if rows else 1)
+ PY
+ - name: engine log (counts only)
+ if: always()
+ run: cat .engine/engine.log || true
+ - name: stop
+ if: always()
+ run: python scripts/engine.py stop
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 83ada37..ff006a5 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,5 +1,5 @@
-# Two jobs, no Google traffic. "offline" is the Python client on the three desktop platforms a new
-# user installs on. "fresh-install" is a new user's first ten minutes on Linux minus the scrape:
+# Two jobs, no Google traffic. "offline" is the Python client and the kit's own engine on the three
+# desktop platforms a new user installs on. "fresh-install" is a new user's first ten minutes on Linux minus the scrape:
# clone, docker compose up, the health check, the patched web UI and API docs, the proxy's checks.
name: tests
on:
@@ -19,6 +19,8 @@ jobs:
with:
python-version: "3.12"
- run: python scripts/test_scrape.py
+ - run: python engine/test_cdp.py
+ - run: python engine/test_engine.py
- run: bash -n scripts/scrape.sh
- run: python -c "import json; json.load(open('.claude/settings.json'))"
fresh-install:
diff --git a/.gitignore b/.gitignore
index 0ede7c0..d4d9262 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,6 @@ __pycache__/
# OS / editor noise
.DS_Store
Thumbs.db
+
+# The engine's jobs, results, logs and downloaded Chrome
+.engine/
diff --git a/engine/browser.py b/engine/browser.py
index f161d8d..6e7f3ca 100644
--- a/engine/browser.py
+++ b/engine/browser.py
@@ -1,15 +1,19 @@
-"""Find and run a local Chrome for the engine: headless, loopback DevTools port, private profile.
+"""Find, fetch and run the Chrome the engine drives: headless, loopback DevTools port, private profile.
with Browser() as b:
page = cdp.Page(b.cdp)
-Uses an installed Google Chrome, Chromium, Edge or Brave (set CHROME=/path/to/binary to pick one).
-Standard library only.
+An installed Google Chrome, Chromium, Edge or Brave is used first (set CHROME=/path/to/binary to
+pick one). On a machine with none, `python3 scripts/engine.py start` downloads Chrome for Testing's
+headless shell into .engine/, pinned below by version and SHA-256. Standard library only.
"""
-import os, platform, shutil, subprocess, sys, tempfile, time
+import hashlib, os, platform, shutil, subprocess, tempfile, time, urllib.request, zipfile
from cdp import CDP
+KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+HOME = os.path.join(KIT, ".engine") # everything the engine keeps: jobs, results, logs, its Chrome
+
CANDIDATES = {
"Darwin": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
@@ -25,9 +29,56 @@ CANDIDATES = {
r"%ProgramFiles%\BraveSoftware\Brave-Browser\Application\brave.exe"],
}
+# Chrome for Testing's headless shell (https://googlechromelabs.github.io/chrome-for-testing/),
+# Stable channel. Each zip's SHA-256 was computed from the download on 2026-10-31. A download that
+# does not match is deleted and never run. Windows on Arm runs the x64 build under emulation.
+CFT_VERSION = "156.0.8078.80"
+CFT_URL = "https://storage.googleapis.com/chrome-for-testing-public/{v}/{p}/chrome-headless-shell-{p}.zip"
+CFT_SHA256 = {
+ "mac-arm64": "ea615baa2e9caf0d02d1898a6562b5f8fa9c35239707dae0f592178df5b986ac",
+ "mac-x64": "535d417d9034cbd11e7dacd1a71aa3d6e1b525e6d5dcd2671eb2532f01481228",
+ "linux64": "547dcd374e02961bf998f6046a267644f8502225c36bb78ca307dab32a0b3eb4",
+ "linux-arm64": "5bb6a00a4285c13303d3f72c5afdb70f144ea7a93eed462365a52b69631c9607",
+ "win64": "10a77466005e7f64e00f1a75fd1ea81f4a38a60093c65fea6895843b26e60a9a",
+}
+
+# What Chrome needs on a bare Debian or Ubuntu (a server, a container). apt picks the t64 variants
+# (libasound2t64 and friends) on Ubuntu 24.04 by itself.
+APT_LINE = ("sudo apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 "
+ "libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 "
+ "libcairo2 libasound2")
+
+
+class BrowserError(RuntimeError):
+ pass
+
+
+def make_home():
+ """.engine/, private to this user on macOS and Linux (results hold scraped phone numbers)."""
+ os.makedirs(HOME, mode=0o700, exist_ok=True)
+ if os.name != "nt":
+ os.chmod(HOME, 0o700)
+
+
+def cft_platform(system=None, machine=None):
+ """Chrome for Testing's name for this machine (mac-arm64, linux64, ...), or None."""
+ system = system or platform.system()
+ arch = {"x86_64": "x64", "amd64": "x64", "arm64": "arm64", "aarch64": "arm64"}.get(
+ (machine or platform.machine()).lower())
+ return {("Darwin", "arm64"): "mac-arm64", ("Darwin", "x64"): "mac-x64",
+ ("Linux", "x64"): "linux64", ("Linux", "arm64"): "linux-arm64",
+ ("Windows", "x64"): "win64", ("Windows", "arm64"): "win64"}.get((system, arch))
+
+
+def shell_path(plat=None):
+ """Where the downloaded headless shell lives (it may not exist yet)."""
+ plat = plat or cft_platform() or "none"
+ exe = "chrome-headless-shell.exe" if plat.startswith("win") else "chrome-headless-shell"
+ return os.path.join(HOME, "chrome", CFT_VERSION, f"chrome-headless-shell-{plat}", exe)
+
def find_chrome():
- """Path of a usable Chromium-based browser, or None."""
+ """Path of a usable Chromium-based browser: CHROME, an installed one, then the downloaded shell."""
if os.environ.get("CHROME"):
return os.environ["CHROME"]
for c in CANDIDATES.get(platform.system(), []):
@@ -36,35 +87,114 @@ def find_chrome():
return c
if os.sep not in c and shutil.which(c):
return shutil.which(c)
- return None
+ return shell_path() if os.path.isfile(shell_path()) else None
+
+
+def download(log=print, plat=None, opener=urllib.request.urlopen):
+ """Fetch, verify and unpack the pinned headless shell into .engine/chrome/; its path."""
+ plat = plat or cft_platform()
+ if plat not in CFT_SHA256:
+ raise BrowserError(f"No Chrome for Testing build exists for {platform.system()} {platform.machine()}. "
+ "Install Chromium, then set CHROME to its path.")
+ url = CFT_URL.format(v=CFT_VERSION, p=plat)
+ root = os.path.dirname(os.path.dirname(shell_path(plat)))
+ make_home()
+ os.makedirs(root, exist_ok=True)
+ tmp = os.path.join(root, f"{plat}.zip.part")
+ log(f"▶ No Chrome found, so downloading Chrome for Testing {CFT_VERSION} (headless shell, {plat}, "
+ f"about 100 MB) once into .engine/ from {url}")
+ h = hashlib.sha256()
+ try:
+ with opener(url, timeout=60) as r, open(tmp, "wb") as f:
+ for chunk in iter(lambda: r.read(1 << 20), b""):
+ h.update(chunk)
+ f.write(chunk)
+ except OSError as e:
+ _remove(tmp)
+ raise BrowserError(f"Could not download Chrome for Testing from {url}: {e}") from e
+ if h.hexdigest() != CFT_SHA256[plat]:
+ _remove(tmp)
+ raise BrowserError("The Chrome download does not match the SHA-256 pinned in engine/browser.py, "
+ "so it was deleted and not run. Try again later, or install Google Chrome.")
+ staging = tempfile.mkdtemp(prefix="unpack-", dir=root)
+ try:
+ with zipfile.ZipFile(tmp) as z:
+ for m in z.infolist():
+ path = z.extract(m, staging) # zipfile drops absolute paths and ".." parts
+ mode = m.external_attr >> 16
+ if mode & 0o111 and not m.is_dir():
+ os.chmod(path, 0o755)
+ final = os.path.dirname(shell_path(plat))
+ shutil.rmtree(final, ignore_errors=True)
+ os.replace(os.path.join(staging, os.path.basename(final)), final) # a half-unpacked shell never shows
+ except (OSError, zipfile.BadZipFile) as e:
+ raise BrowserError(f"Could not unpack the Chrome download: {e}") from e
+ finally:
+ shutil.rmtree(staging, ignore_errors=True)
+ _remove(tmp)
+ log(" ✓ SHA-256 verified, unpacked")
+ return shell_path(plat)
+
+
+def _remove(path):
+ try:
+ os.remove(path)
+ except OSError:
+ pass
+
+
+def start_error(path, code, stderr):
+ """The message for a Chrome that exited at once, with the apt line when Linux libraries are missing."""
+ missing = [l.strip() for l in stderr.splitlines() if "error while loading shared libraries" in l]
+ if missing:
+ return (f"Chrome cannot start: {missing[0]}\n Install the libraries it needs "
+ f"(Debian or Ubuntu):\n {APT_LINE}")
+ last = "; ".join(l.strip() for l in stderr.strip().splitlines()[-3:])
+ return f"Chrome exited at once (code {code}): {path}" + (f": {last}" if last else "")
class Browser:
"""A headless Chrome whose DevTools socket is on 127.0.0.1; `cdp` is the browser-level CDP."""
- def __init__(self, path=None, headless=True, profile=None, lang="en-US", timeout=30):
+ def __init__(self, path=None, headless=True, profile=None, lang="en-US", proxy=None, timeout=30):
self.path = path or find_chrome()
if not self.path:
- sys.exit("✗ No Chrome, Chromium, Edge or Brave found. Install Google Chrome, or set CHROME to "
- "the browser's path.")
+ raise BrowserError("No Chrome, Chromium, Edge or Brave found. Run `python3 scripts/engine.py start`, "
+ "which downloads one, or install Google Chrome.")
self._own_profile = profile is None
self.profile = profile or tempfile.mkdtemp(prefix="gmskit-chrome-")
args = [self.path, "--remote-debugging-port=0", f"--user-data-dir={self.profile}",
"--no-first-run", "--no-default-browser-check", "--disable-extensions",
"--disable-background-networking", "--disable-sync", "--mute-audio",
+ "--disable-component-update", "--disable-domain-reliability", "--disable-breakpad",
+ "--metrics-recording-only", "--no-pings", "--disable-client-side-phishing-detection",
"--disable-blink-features=AutomationControlled", "--window-size=1280,900",
- f"--lang={lang}", "--disable-dev-shm-usage", "about:blank"]
- if headless:
- args.insert(1, "--headless=new")
- kw = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL, "stdin": subprocess.DEVNULL}
+ f"--lang={lang}", "--disable-dev-shm-usage"]
+ if headless and "headless-shell" not in os.path.basename(self.path): # the shell is always headless
+ args.append("--headless=new")
+ if proxy:
+ args.append(f"--proxy-server={proxy}")
+ if platform.system() == "Linux" and hasattr(os, "geteuid") and os.geteuid() == 0:
+ args.append("--no-sandbox") # Chrome refuses to run as root with its sandbox (containers)
+ args.append("about:blank")
+ self._err = open(os.path.join(self.profile, "chrome-stderr.log"), "w+b")
+ kw = {"stdout": subprocess.DEVNULL, "stderr": self._err, "stdin": subprocess.DEVNULL}
if os.name == "nt":
kw["creationflags"] = subprocess.CREATE_NO_WINDOW
- self.proc = subprocess.Popen(args, **kw)
+ try:
+ self.proc = subprocess.Popen(args, **kw)
+ except OSError as e:
+ self._cleanup()
+ raise BrowserError(f"Could not start Chrome ({self.path}): {e}") from e
+ self.cdp = None
active = os.path.join(self.profile, "DevToolsActivePort")
end = time.time() + timeout
while time.time() < end:
if self.proc.poll() is not None:
- sys.exit(f"✗ Chrome exited at once (code {self.proc.returncode}): {self.path}")
+ self._err.seek(0)
+ msg = start_error(self.path, self.proc.returncode, self._err.read().decode("utf-8", "replace"))
+ self.close()
+ raise BrowserError(msg)
try:
with open(active, encoding="utf-8") as f:
port, path = f.read().split()
@@ -73,18 +203,25 @@ class Browser:
time.sleep(0.1)
else:
self.close()
- sys.exit(f"✗ Chrome did not open its DevTools port within {timeout} s: {self.path}")
- self.ws_url = f"ws://127.0.0.1:{port}{path}"
- self.cdp = CDP(self.ws_url, timeout=timeout)
+ raise BrowserError(f"Chrome did not open its DevTools port within {timeout} s: {self.path}")
+ self.cdp = CDP(f"ws://127.0.0.1:{port}{path}", timeout=timeout)
def version(self):
return self.cdp.send("Browser.getVersion").get("product", "")
- def close(self):
+ def kill(self):
+ """Stop Chrome now (the job watchdog): every CDP call still waiting fails at once."""
try:
- self.cdp.close()
- except Exception: # noqa: BLE001 - closing is best effort
+ self.proc.kill()
+ except OSError:
pass
+
+ def close(self):
+ if self.cdp:
+ try:
+ self.cdp.close()
+ except Exception: # noqa: BLE001 - closing is best effort
+ pass
if self.proc.poll() is None:
self.proc.terminate()
try:
@@ -92,8 +229,16 @@ class Browser:
except subprocess.TimeoutExpired:
self.proc.kill()
self.proc.wait(5)
+ self._cleanup()
+
+ def _cleanup(self):
+ self._err.close()
if self._own_profile:
- shutil.rmtree(self.profile, ignore_errors=True)
+ for _ in range(10): # Windows keeps the profile locked for a moment after Chrome exits
+ shutil.rmtree(self.profile, ignore_errors=True)
+ if not os.path.exists(self.profile):
+ break
+ time.sleep(0.3)
def __enter__(self):
return self
diff --git a/engine/details.py b/engine/details.py
index 60fda9f..d32a553 100644
--- a/engine/details.py
+++ b/engine/details.py
@@ -12,10 +12,10 @@ import json
# Index path of each field inside a place entity. Checked on live batches in the spike
# (2026-10-31): title, category, address, rating, lat/lon, place_id, data_id, timezone, thumbnail
# filled on 60/60 entities, website 55/60, phone 46/60 (listings without one).
-# review_count is the one unverified path: (37, 1) is an integer on every entity in the range a
-# review count has (23 to 1998 on 40 coffee shops), but signed-out Google Maps shows no counts in
-# the list or the place panel, so it could not be compared against the page. Verify it in P1
-# against a few listings whose counts are known (a signed-in browser shows them).
+# review_count is not here: signed out, Google leaves (4, 8) empty, and (37, 1) is a different
+# number (60 and 82 where the page showed 61 and 134 reviews, 2026-10-31). Headless Chrome gets
+# Google's limited view, whose result cards show "4.8 stars" without the count, so the CSV's
+# review_count column stays empty.
FIELDS = {
"title": (11,),
"category": (13, 0),
@@ -23,7 +23,6 @@ FIELDS = {
"website": (7, 0),
"phone": (178, 0, 0),
"review_rating": (4, 7),
- "review_count": (37, 1),
"latitude": (9, 2),
"longitude": (9, 3),
"plus_code": (183, 2, 2, 0),
diff --git a/engine/jobs.py b/engine/jobs.py
new file mode 100644
index 0000000..07d0f82
--- /dev/null
+++ b/engine/jobs.py
@@ -0,0 +1,467 @@
+"""Scrape jobs: a sqlite3 store in .engine/data and one worker that runs the jobs one at a time.
+
+A job runs its keywords through maps.collect in one headless Chrome, de-duplicates the places on
+data_id and writes <id>.csv with the old engine's 36 column names (what Google's result batches
+carry is filled, the rest stays empty) plus ours. The CSV is rewritten atomically after every
+keyword, so a download while the job works returns the rows so far. A watchdog stops Chrome when a
+job overruns its max_time, so no job can stick on "working". Standard library only.
+"""
+import concurrent.futures, csv, html, http.client, io, ipaddress, json, math, os, re, socket, sqlite3
+import threading, time, urllib.parse, urllib.request, uuid
+from datetime import datetime, timezone
+
+import browser, maps
+from cdp import CDPError, Page
+
+# gosom/google-maps-scraper's CSV header, in its order, so files and scripts that read the old
+# engine's output keep working. Our own columns follow.
+COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "popular_times", "website",
+ "phone", "plus_code", "review_count", "review_rating", "reviews_per_rating", "latitude",
+ "longitude", "cid", "status", "descriptions", "reviews_link", "thumbnail", "timezone",
+ "price_range", "data_id", "street_view_url", "place_id", "images", "reservations",
+ "order_online", "menu", "owner", "complete_address", "credit_cards_accepted", "about",
+ "user_reviews", "user_reviews_extended", "emails"]
+EXTRA = ["categories"]
+RENAME = {"open_state": "status", "description": "descriptions"} # details.FIELDS name -> CSV column
+
+WATCHDOG_GRACE = 30 # seconds past max_time before the watchdog kills Chrome
+SITE_WORKERS = 8 # business websites fetched at once by the email pass
+SITE_TIMEOUT = 10 # seconds per website request
+EMAIL_MIN_SECONDS = 60 # the email pass gets at least this long, even after a job used its max_time
+
+
+def utc_now():
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+# ── Job body ──
+LANG = re.compile(r"^[A-Za-z]{2}$")
+PROXY_SCHEMES = ("http", "https", "socks5", "socks5h")
+
+
+def _int(body, key, default, lo, hi, label):
+ v = body.get(key, default)
+ if v is None:
+ v = default
+ if isinstance(v, bool) or not isinstance(v, int) or not lo <= v <= hi:
+ raise ValueError(f"invalid {label}: a whole number from {lo} to {hi}")
+ return v
+
+
+def validate(body):
+ """(name, data) from a create-job body, or ValueError with the old engine's message
+ ("missing name", "invalid lang", ...), which the kit's docs quote."""
+ if not isinstance(body, dict):
+ raise ValueError("the body must be a JSON object")
+ name = body.get("name")
+ if not isinstance(name, str) or not name.strip():
+ raise ValueError("missing name")
+ if len(name) > 200:
+ raise ValueError("invalid name: 200 characters at most")
+ kws = body.get("keywords")
+ if not isinstance(kws, list) or not kws:
+ raise ValueError("missing keywords")
+ if len(kws) > 500 or not all(isinstance(k, str) and k.strip() and len(k) <= 500 for k in kws):
+ raise ValueError("invalid keywords: 1 to 500 non-empty strings of up to 500 characters")
+ lang = body.get("lang")
+ if lang in (None, ""):
+ raise ValueError("missing lang")
+ if not isinstance(lang, str) or not LANG.match(lang):
+ raise ValueError("invalid lang")
+ lat, lon = body.get("lat"), body.get("lon")
+ if lat in (None, "") or lon in (None, ""):
+ raise ValueError("missing geo coordinates")
+ try:
+ if isinstance(lat, bool) or isinstance(lon, bool):
+ raise ValueError
+ flat, flon = float(lat), float(lon)
+ except (TypeError, ValueError):
+ raise ValueError("invalid geo coordinates") from None
+ if not (math.isfinite(flat) and math.isfinite(flon) and -90 <= flat <= 90 and -180 <= flon <= 180):
+ raise ValueError("invalid geo coordinates")
+ if body.get("max_time") in (None, 0):
+ raise ValueError("missing max time")
+ data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "lat": str(lat).strip(),
+ "lon": str(lon).strip(),
+ "zoom": _int(body, "zoom", 15, 1, 21, "zoom"),
+ "radius": _int(body, "radius", 10000, 0, 1_000_000, "radius"),
+ "depth": _int(body, "depth", 1, 1, 1000, "depth"),
+ "max_time": _int(body, "max_time", 600, 1, 86400, "max time"),
+ "fast_mode": bool(body.get("fast_mode")), "email": bool(body.get("email")),
+ "proxies": _proxies(body.get("proxies"))}
+ return name.strip(), data
+
+
+def _proxies(v):
+ if v in (None, ""):
+ return []
+ if not isinstance(v, list) or len(v) > 100 or not all(isinstance(p, str) for p in v):
+ raise ValueError("invalid proxies: a list of proxy URLs")
+ for p in v:
+ try:
+ u = urllib.parse.urlsplit(p.strip())
+ u.port # noqa: B018 - raises on a bad port
+ except ValueError:
+ raise ValueError(f"invalid proxy: {p}") from None
+ if u.scheme not in PROXY_SCHEMES or not u.hostname:
+ raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port")
+ if u.username or u.password:
+ raise ValueError("proxy usernames and passwords are not supported by this engine yet: "
+ "use a proxy that allows your IP address")
+ return [p.strip() for p in v]
+
+
+# ── Rows ──
+def to_row(place, keyword):
+ """One CSV row (every column, as text) from a details.parse_entity / maps.parse_link place."""
+ row = dict.fromkeys(COLUMNS + EXTRA, "")
+ for k, v in place.items():
+ k = RENAME.get(k, k)
+ if k in row:
+ row[k] = str(v)
+ row["input_id"] = keyword
+ if place.get("place_id"): # Google's documented Maps URL for one place
+ row["link"] = ("https://www.google.com/maps/search/?api=1&query="
+ + urllib.parse.quote(place.get("title", ""), safe="")
+ + "&query_place_id=" + urllib.parse.quote(place["place_id"], safe=""))
+ return row
+
+
+def to_csv(rows):
+ buf = io.StringIO()
+ w = csv.DictWriter(buf, fieldnames=COLUMNS + EXTRA, extrasaction="ignore")
+ w.writeheader()
+ w.writerows(rows)
+ return buf.getvalue().encode("utf-8")
+
+
+# ── Store ──
+class Store:
+ """Jobs in sqlite3 (jobs.db) and their results as <id>.csv beside it. One lock serialises every
+ database call and every CSV read and write, so a download never meets a half-replaced file
+ (Windows cannot replace a file another handle has open)."""
+
+ def __init__(self, folder):
+ os.makedirs(folder, exist_ok=True)
+ self.folder = folder
+ self.lock = threading.RLock()
+ self.db = sqlite3.connect(os.path.join(folder, "jobs.db"), check_same_thread=False)
+ with self.lock, self.db:
+ self.db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, "
+ "date TEXT NOT NULL, status TEXT NOT NULL, data TEXT NOT NULL, "
+ "error TEXT NOT NULL DEFAULT '')")
+
+ @staticmethod
+ def _job(r):
+ # The old engine's job shape (capitalised keys), plus Error: why a job failed or stopped early.
+ return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5]}
+
+ def create(self, name, data):
+ jid = str(uuid.uuid4())
+ with self.lock, self.db:
+ self.db.execute("INSERT INTO jobs VALUES (?, ?, ?, 'pending', ?, '')",
+ (jid, name, utc_now(), json.dumps(data)))
+ return jid
+
+ def get(self, jid):
+ with self.lock:
+ r = self.db.execute("SELECT * FROM jobs WHERE id = ?", (jid,)).fetchone()
+ return self._job(r) if r else None
+
+ def all(self):
+ with self.lock:
+ return [self._job(r) for r in self.db.execute("SELECT * FROM jobs ORDER BY rowid DESC")]
+
+ def update(self, jid, status, error=""):
+ with self.lock, self.db:
+ self.db.execute("UPDATE jobs SET status = ?, error = ? WHERE id = ?", (status, error[:500], jid))
+
+ def claim(self):
+ """The oldest pending job, now marked working; None when there is none."""
+ with self.lock, self.db:
+ r = self.db.execute("SELECT * FROM jobs WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()
+ if not r:
+ return None
+ self.db.execute("UPDATE jobs SET status = 'working' WHERE id = ?", (r[0],))
+ job = self._job(r)
+ job["Status"] = "working"
+ return job
+
+ def recover(self):
+ """Jobs left on "working" by an engine that was stopped or crashed: failed, rows kept."""
+ with self.lock, self.db:
+ return self.db.execute("UPDATE jobs SET status = 'failed', error = 'the engine stopped while "
+ "this job ran; its rows so far are kept' WHERE status = 'working'").rowcount
+
+ def delete(self, jid):
+ with self.lock, self.db:
+ gone = self.db.execute("DELETE FROM jobs WHERE id = ?", (jid,)).rowcount
+ for path in (self.csv_path(jid), self.csv_path(jid) + ".tmp"):
+ try:
+ os.remove(path)
+ except OSError:
+ pass
+ return bool(gone)
+
+ def csv_path(self, jid):
+ return os.path.join(self.folder, f"{jid}.csv")
+
+ def write_csv(self, jid, rows):
+ """Replace the job's CSV in one step; False (nothing written) if the job was deleted."""
+ with self.lock:
+ if not self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone():
+ return False
+ tmp = self.csv_path(jid) + ".tmp"
+ with open(tmp, "wb") as f:
+ f.write(to_csv(rows))
+ os.replace(tmp, self.csv_path(jid))
+ return True
+
+ def read_csv(self, jid):
+ with self.lock:
+ try:
+ with open(self.csv_path(jid), "rb") as f:
+ return f.read()
+ except OSError:
+ return None
+
+
+# ── Worker ──
+class Worker(threading.Thread):
+ """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once."""
+
+ def __init__(self, store, log=print):
+ super().__init__(daemon=True, name="jobs")
+ self.store, self.log = store, log
+ self.wake = threading.Event()
+ self.stopped = threading.Event()
+ self._running = (None, None) # (job id, stop event) of the job in progress
+ self._browser = None
+
+ def run(self):
+ while not self.stopped.is_set():
+ job = self.store.claim()
+ if not job:
+ self.wake.wait(5)
+ self.wake.clear()
+ continue
+ try:
+ self.run_job(job)
+ except Exception as e: # noqa: BLE001 - one bad job must never stop the worker
+ self.log(f"job {job['ID']} failed: {type(e).__name__}: {e}")
+ self.store.update(job["ID"], "failed", f"{type(e).__name__}: {e}")
+
+ def cancel(self, jid):
+ """Stop `jid` if it is the job in progress (it is being deleted)."""
+ rid, stop = self._running
+ if rid == jid and stop:
+ stop.set()
+
+ def shutdown(self):
+ """Stop the job in progress and its Chrome (the engine is exiting)."""
+ self.stopped.set()
+ self.wake.set()
+ _, stop = self._running
+ if stop:
+ stop.set()
+ b = self._browser
+ if b:
+ b.close()
+
+ def run_job(self, job):
+ jid, d = job["ID"], job["Data"]
+ deadline = time.monotonic() + d["max_time"]
+ stop = threading.Event()
+ self._running = (jid, stop)
+ rows, seen, errors, blocked = [], set(), [], False
+ kws = d["keywords"]
+ self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")
+
+ def watchdog(): # Chrome hung or the job overran: kill Chrome, and every CDP call fails at once
+ stop.set()
+ b = self._browser
+ if b:
+ b.kill()
+ self.log(f"job {jid}: watchdog stopped Chrome {WATCHDOG_GRACE} s after max_time")
+ timer = threading.Timer(d["max_time"] + WATCHDOG_GRACE, watchdog)
+ timer.daemon = True
+ timer.start()
+ try:
+ self._browser = browser.Browser(lang=d["lang"], proxy=(d["proxies"] or [None])[0])
+ for i, kw in enumerate(kws, 1):
+ left = deadline - time.monotonic()
+ if stop.is_set() or left <= 0:
+ break
+ url = maps.search_url(kw, d["lat"], d["lon"], d["zoom"], d["lang"])
+ page = None
+ started = time.monotonic()
+ try:
+ page = Page(self._browser.cdp)
+ places, reason, stats = maps.collect(page, url, depth=d["depth"], max_seconds=left,
+ log=lambda m: None, stop=stop.is_set)
+ except (CDPError, OSError) as e:
+ errors.append(str(e))
+ self.log(f"job {jid} keyword {i}/{len(kws)}: error: {e}")
+ if self._browser.cdp.closed: # Chrome is gone: the other keywords would fail too
+ break
+ continue
+ finally:
+ if page and not self._browser.cdp.closed:
+ page.close()
+ blocked |= reason == "blocked"
+ new = 0
+ for p in places:
+ key = p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")
+ if key and key not in seen:
+ seen.add(key)
+ rows.append(to_row(p, kw))
+ new += 1
+ if not self.store.write_csv(jid, rows):
+ break # deleted
+ self.log(f"job {jid} keyword {i}/{len(kws)}: {len(places)} places, {new} new, "
+ f"{stats['batches']} batches, {time.monotonic() - started:.1f} s ({reason})")
+ except browser.BrowserError as e:
+ errors.append(str(e))
+ self.log(f"job {jid}: {e}")
+ finally:
+ timer.cancel()
+ if self._browser:
+ self._browser.close()
+ self._browser = None
+ if d["email"] and rows and not stop.is_set():
+ n = add_emails(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS))
+ self.log(f"job {jid}: emails found for {n}/{sum(1 for r in rows if r['website'])} websites")
+ self._running = (None, None)
+ if not self.store.write_csv(jid, rows):
+ self.log(f"job {jid} deleted while it ran")
+ return
+ if rows or not (errors or blocked):
+ note = "stopped at max_time; the rows found so far are kept" \
+ if time.monotonic() >= deadline or (stop.is_set() and not errors) else ""
+ if errors:
+ note = f"{len(errors)} keyword(s) failed: {errors[-1]}"
+ self.store.update(jid, "ok", note)
+ self.log(f"job {jid} ok: {len(rows)} places")
+ else:
+ reason = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
+ "Wait, or add proxies") if blocked else errors[-1]
+ self.store.update(jid, "failed", reason)
+ self.log(f"job {jid} failed: {reason}")
+
+
+# ── Email pass (email: true): each business homepage over plain HTTP, emails only ──
+EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")
+JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")
+JUNK_DOMAIN = ("example.", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")
+UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"
+
+
+def _public_ip(ip):
+ try:
+ return ipaddress.ip_address(ip.split("%")[0]).is_global
+ except ValueError:
+ return False
+
+
+def _public_host(host):
+ """True only if every address of `host` is public. Websites come from third-party listings, so
+ one could point the fetch at this machine (the engine itself) or its network."""
+ try:
+ infos = socket.getaddrinfo(host, None)
+ except (OSError, UnicodeError):
+ return False
+ return bool(infos) and all(_public_ip(i[4][0]) for i in infos)
+
+
+def _public_connection(address, *args, **kwargs):
+ """create_connection that refuses a private peer: catches a DNS answer that changed (rebinding)."""
+ sock = socket.create_connection(address, *args, **kwargs)
+ if not _public_ip(sock.getpeername()[0]):
+ sock.close()
+ raise OSError("refused a non-public address")
+ return sock
+
+
+class _HTTP(http.client.HTTPConnection):
+ def __init__(self, *a, **kw):
+ super().__init__(*a, **kw)
+ self._create_connection = _public_connection
+
+
+class _HTTPS(http.client.HTTPSConnection):
+ def __init__(self, *a, **kw):
+ super().__init__(*a, **kw)
+ self._create_connection = _public_connection
+
+
+class _HTTPHandler(urllib.request.HTTPHandler):
+ def http_open(self, req):
+ return self.do_open(_HTTP, req)
+
+
+class _HTTPSHandler(urllib.request.HTTPSHandler):
+ def https_open(self, req):
+ return self.do_open(_HTTPS, req, context=self._context)
+
+
+class _Redirects(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ if not _public_host(urllib.parse.urlsplit(newurl).hostname or ""):
+ return None
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+# No proxy from the environment: these go straight to the business websites.
+_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}), _HTTPHandler, _HTTPSHandler, _Redirects)
+
+
+def fetch(url, timeout=SITE_TIMEOUT):
+ """Up to 400 KB of a public web page as text; "" on any failure."""
+ if not url.startswith(("http://", "https://")):
+ url = "http://" + url
+ try:
+ if not _public_host(urllib.parse.urlsplit(url).hostname or ""):
+ return ""
+ r = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html"})
+ with _OPENER.open(r, timeout=timeout) as resp:
+ return resp.read(400_000).decode("utf-8", "replace")
+ except Exception: # noqa: BLE001 - a broken website is just a website without emails
+ return ""
+
+
+def site_host(url):
+ try:
+ h = (urllib.parse.urlsplit(url if "://" in url else "http://" + url).hostname or "").lower()
+ except ValueError:
+ return ""
+ return h[4:] if h.startswith("www.") else h
+
+
+def find_emails(text, site=""):
+ """Up to 3 addresses from page text, placeholders and image names dropped, the site's own first."""
+ found = []
+ for e in EMAIL_RE.findall(html.unescape(text or "")):
+ e = urllib.parse.unquote(e).lower()
+ dom = e.rpartition("@")[2]
+ if e.endswith(JUNK_SUFFIX) or any(j in dom for j in JUNK_DOMAIN) or e in found:
+ continue
+ found.append(e)
+ own = lambda e: site and (e.endswith("@" + site) or e.endswith("." + site))
+ return sorted(found, key=lambda e: not own(e))[:3]
+
+
+def add_emails(rows, seconds, get=None):
+ """Fill `emails` (joined by ", ", like the old engine) from each row's website homepage, for at
+ most `seconds`; the number of rows that got one."""
+ get = get or fetch
+ todo = [r for r in rows if r.get("website") and not r.get("emails")]
+ pool = concurrent.futures.ThreadPoolExecutor(SITE_WORKERS)
+ futs = {pool.submit(lambda r=r: find_emails(get(r["website"]), site_host(r["website"]))): r for r in todo}
+ done, _ = concurrent.futures.wait(futs, timeout=seconds)
+ pool.shutdown(wait=False, cancel_futures=True)
+ n = 0
+ for f in done:
+ if not f.exception() and f.result():
+ futs[f]["emails"] = ", ".join(f.result())
+ n += 1
+ return n
diff --git a/engine/maps.py b/engine/maps.py
index 20bca15..aead9cc 100644
--- a/engine/maps.py
+++ b/engine/maps.py
@@ -92,16 +92,20 @@ class Batches:
self.page.cdp.unsubscribe("Network.responseReceived", self.q, self.page.sid)
-def collect(page, url, depth=1, max_seconds=180, log=print):
+def collect(page, url, depth=1, max_seconds=180, log=print, stop=lambda: False):
"""Open `url`, scroll the result feed, and return (places, stop reason, stats).
`depth` is the number of result batches to read (about 20 places each, like the old engine's
- depth); the list may end sooner. Places are de-duplicated on data_id."""
+ depth); the list may end sooner. Places are de-duplicated on data_id. `stop()` returning True
+ ends the scroll early (a deleted job). Reason "blocked" means Google answered with its
+ unusual-traffic page."""
for k, v in CONSENT_COOKIES.items():
page.set_cookie(k, v, ".google.com")
batches = Batches(page)
seen, started, reason, stale = {}, time.time(), "depth", 0
try:
page.navigate(url)
+ if "/sorry/" in page.url():
+ return [], "blocked", {"batches": 0, "dom_links": 0}
if "consent.google" in page.url():
page.eval(JS_CONSENT)
page.wait_for("!location.href.includes('consent.google')", timeout=15)
@@ -124,7 +128,7 @@ def collect(page, url, depth=1, max_seconds=180, log=print):
take()
time.sleep(0.4)
log(f" batch {batches.count}: {len(seen)} places")
- while batches.count < depth and time.time() - started < max_seconds:
+ while batches.count < depth and time.time() - started < max_seconds and not stop():
if any(m in (page.eval(JS_END) or "") for m in END_MARKERS):
reason = "end-of-list"
break
@@ -145,7 +149,9 @@ def collect(page, url, depth=1, max_seconds=180, log=print):
stale = 0
log(f" batch {batches.count}: {len(seen)} places")
else:
- if time.time() - started >= max_seconds:
+ if stop():
+ reason = "stopped"
+ elif time.time() - started >= max_seconds:
reason = "timeout"
take()
links = page.eval(JS_LINKS) or []
diff --git a/engine/serve.py b/engine/serve.py
new file mode 100644
index 0000000..cc6c228
--- /dev/null
+++ b/engine/serve.py
@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+"""The kit's own engine: the Google Maps scraper's HTTP API and web UI, on this machine only.
+
+ python3 engine/serve.py # port from SCRAPER_PORT (environment, then .env), default 8080
+ python3 engine/serve.py --port 8190
+
+Normally started by `python3 scripts/engine.py start`. It speaks the API of the engine the kit used
+before (POST/GET/DELETE /api/v1/jobs, /api/v1/jobs/{id}/download, the same job body and "Status"
+values), so scripts/scrape.py, the skill and the slash commands work unchanged. It listens on
+127.0.0.1 (and ::1) only and refuses a foreign Host (DNS rebinding, 421) or Origin (cross-site
+posts, 403); with SCRAPER_API_KEY set it also wants that X-API-Key (401). No telemetry: it talks to
+Google Maps, and for email: true jobs to the businesses' own websites, nothing else.
+Standard library only.
+"""
+import argparse, hmac, json, os, re, secrets, signal, socket, sys, threading, urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import browser, jobs # noqa: E402
+
+for _stream in (sys.stdout, sys.stderr): # ✓ and accented names on a Windows cp1252 console
+ try:
+ _stream.reconfigure(encoding="utf-8", errors="replace")
+ except (AttributeError, ValueError):
+ pass
+
+VERSION = "2.0.0-dev"
+MARK = f"gmskit-engine/{VERSION}" # X-Engine header: tells scripts/engine.py this engine from others
+REPO = "https://github.com/Mahanaicoach/google-maps-scraper-kit"
+ENV_KEYS = ("SCRAPER_PORT", "SCRAPER_API_KEY")
+MAX_BODY = 1 << 20
+LOCAL = r"(localhost|127\.0\.0\.1|\[::1\])"
+HOST_OK = re.compile(rf"^{LOCAL}(:\d{{1,5}})?$", re.I)
+ORIGIN_OK = re.compile(rf"^https?://{LOCAL}(:\d{{1,5}})?$", re.I)
+JOB_PATH = re.compile(r"^/api/v1/jobs/([0-9a-f-]{36})(/download)?$")
+JOB_ID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
+_log_lock = threading.Lock()
+_REPLIED = object() # body() answered the request itself (a bad body)
+
+
+def log(msg):
+ """One line on stderr with a UTC timestamp and its offset. Counts only, never scraped values."""
+ with _log_lock:
+ print(f"{jobs.utc_now()} {msg}", file=sys.stderr, flush=True)
+
+
+def read_env():
+ """SCRAPER_PORT and SCRAPER_API_KEY: the environment wins, then the kit's .env (last line wins,
+ as in docker compose and scripts/scrape.py)."""
+ found = {}
+ try:
+ with open(os.path.join(browser.KIT, ".env"), encoding="utf-8-sig", errors="replace") as f:
+ for line in f:
+ k, sep, v = line.strip().removeprefix("export ").partition("=")
+ if sep and k.strip() in ENV_KEYS:
+ found[k.strip()] = v.split(" #", 1)[0].strip().strip("\"'")
+ except OSError:
+ pass
+ return {k: os.environ.get(k, found.get(k, "")) for k in ENV_KEYS}
+
+
+OPENAPI = {
+ "openapi": "3.0.3",
+ "info": {"title": "Google Maps Scraper Kit engine", "version": VERSION,
+ "description": "Local only. Create a job, poll its Status until ok or failed, download the CSV. "
+ f"Source and docs: {REPO}"},
+ "paths": {
+ "/api/v1/jobs": {
+ "get": {"summary": "List jobs, newest first (null when there are none)"},
+ "post": {"summary": "Create a job; returns 201 {\"id\": \"<uuid>\"}, or 422 {\"code\", \"message\"}",
+ "requestBody": {"content": {"application/json": {"schema": {
+ "type": "object", "required": ["name", "keywords", "lang", "lat", "lon", "max_time"],
+ "properties": {
+ "name": {"type": "string", "description": "any label"},
+ "keywords": {"type": "array", "items": {"type": "string"}},
+ "lang": {"type": "string", "description": "2-letter interface language, e.g. en"},
+ "lat": {"type": "string"}, "lon": {"type": "string"},
+ "max_time": {"type": "integer", "description": "seconds"},
+ "depth": {"type": "integer", "description": "result batches per keyword, about 20 places each (default 1)"},
+ "zoom": {"type": "integer", "description": "map zoom 1-21 (default 15)"},
+ "email": {"type": "boolean", "description": "visit each website for emails"},
+ "proxies": {"type": "array", "items": {"type": "string"},
+ "description": "http(s):// or socks5://host:port; the first is used"},
+ "radius": {"type": "integer", "description": "accepted, not used"},
+ "fast_mode": {"type": "boolean", "description": "accepted, not used"}}}}}}}},
+ "/api/v1/jobs/{id}": {
+ "get": {"summary": "One job: ID, Name, Date, Status (pending, working, ok, failed), Data, Error"},
+ "delete": {"summary": "Delete a job and its results (stops it if it is running)"}},
+ "/api/v1/jobs/{id}/download": {"get": {"summary": "The job's CSV (the rows so far while it works)"}},
+ },
+}
+
+PAGE = """<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Google Maps Scraper Kit</title>
+<style nonce="{nonce}">
+:root {{ color-scheme: light dark; --bg: #fff; --fg: #1d1d1f; --muted: #6e6e73; --line: #d2d2d7; --accent: #0a66c2; }}
+@media (prefers-color-scheme: dark) {{ :root {{ --bg: #111; --fg: #f2f2f2; --muted: #a1a1a6; --line: #333; --accent: #4ea1ff; }} }}
+body {{ margin: 0 auto; max-width: 960px; padding: 16px; background: var(--bg); color: var(--fg);
+ font: 15px/1.45 system-ui, -apple-system, Segoe UI, sans-serif; }}
+h1 {{ font-size: 1.4em; margin: 0 0 4px; }} a {{ color: var(--accent); }} .muted {{ color: var(--muted); }}
+form {{ display: grid; gap: 10px; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); margin: 16px 0; }}
+label {{ display: grid; gap: 4px; font-size: .9em; }} .wide {{ grid-column: 1 / -1; }}
+input, textarea, button {{ font: inherit; padding: 6px 8px; border: 1px solid var(--line); border-radius: 6px;
+ background: var(--bg); color: var(--fg); }}
+button {{ cursor: pointer; }} button.primary {{ background: var(--accent); color: #fff; border-color: var(--accent); }}
+table {{ width: 100%; border-collapse: collapse; }} th, td {{ text-align: left; padding: 6px; border-bottom: 1px solid var(--line); }}
+td.err {{ color: var(--muted); font-size: .85em; }} .row {{ display: flex; gap: 6px; flex-wrap: wrap; }}
+#msg {{ min-height: 1.4em; }} .scroll {{ overflow-x: auto; }} [hidden] {{ display: none !important; }}
+</style></head><body>
+<h1>Google Maps Scraper Kit</h1>
+<p class="muted">Local engine {version}. <a href="{repo}" rel="noopener">GitHub</a> ·
+<a href="/api/docs">API</a> · Business listings only. Results stay on this machine.</p>
+<form id="f">
+ <label class="wide">Keywords, one per line<textarea id="keywords" rows="3" required
+ placeholder="coffee shops in Austin TX"></textarea></label>
+ <label>Latitude<input id="lat" required placeholder="30.2672" inputmode="decimal"></label>
+ <label>Longitude<input id="lon" required placeholder="-97.7431" inputmode="decimal"></label>
+ <label>Depth (about 20 places each)<input id="depth" type="number" min="1" max="1000" value="5"></label>
+ <label>Language<input id="lang" value="en" maxlength="2" pattern="[A-Za-z]{{2}}"></label>
+ <label>Time limit (seconds)<input id="max_time" type="number" min="1" max="86400" value="600"></label>
+ <label><span>Emails</span><span><input type="checkbox" id="email" checked> visit each website</span></label>
+ <div class="wide row"><button class="primary">Start scraping</button>
+ <span class="muted">Coordinates: right-click a spot in Google Maps and click the numbers.</span></div>
+ <label class="wide" id="keybox" hidden>API key (SCRAPER_API_KEY)<input id="key" type="password" autocomplete="off"></label>
+</form>
+<p id="msg" role="status"></p>
+<div class="scroll"><table><thead><tr><th>Job</th><th>Created (UTC)</th><th>Status</th><th></th></tr></thead>
+<tbody id="jobs"></tbody></table></div>
+<script nonce="{nonce}">
+const $ = id => document.getElementById(id);
+let key = ''; try {{ key = localStorage.getItem('gmskit-key') || ''; }} catch (e) {{}}
+$('key').value = key;
+$('key').addEventListener('change', () => {{ key = $('key').value; try {{ localStorage.setItem('gmskit-key', key); }} catch (e) {{}} load(); }});
+function api(method, path, body) {{
+ const h = {{}}; if (key) h['X-API-Key'] = key; if (body) h['Content-Type'] = 'application/json';
+ return fetch(path, {{method, headers: h, body: body ? JSON.stringify(body) : undefined}}).then(r => {{
+ if (r.status === 401) {{ $('keybox').hidden = false; throw new Error('this engine needs its API key'); }}
+ return r;
+ }});
+}}
+function say(t) {{ $('msg').textContent = t; }}
+function cell(tr, text, cls) {{ const td = document.createElement('td'); td.textContent = text; if (cls) td.className = cls; tr.appendChild(td); return td; }}
+function button(td, label, fn) {{ const b = document.createElement('button'); b.type = 'button'; b.textContent = label; b.addEventListener('click', fn); td.appendChild(b); }}
+async function download(j) {{
+ const r = await api('GET', '/api/v1/jobs/' + j.ID + '/download');
+ if (!r.ok) return say('No results yet for this job.');
+ const a = document.createElement('a'); a.href = URL.createObjectURL(await r.blob());
+ a.download = 'results-' + j.ID.slice(0, 8) + '.csv'; document.body.appendChild(a); a.click(); a.remove();
+}}
+async function remove(j) {{
+ if (!confirm('Delete this job and its results?')) return;
+ const r = await api('DELETE', '/api/v1/jobs/' + j.ID); say(r.ok ? 'Deleted.' : 'Could not delete it.'); load();
+}}
+async function load() {{
+ try {{
+ const list = await (await api('GET', '/api/v1/jobs')).json() || [];
+ const tb = $('jobs'); tb.replaceChildren();
+ for (const j of list) {{
+ const tr = document.createElement('tr');
+ const name = cell(tr, j.Name); if (j.Error) {{ const s = document.createElement('div'); s.className = 'muted'; s.textContent = j.Error; name.appendChild(s); }}
+ cell(tr, j.Date.replace('T', ' ').replace('+00:00', '')); cell(tr, j.Status);
+ const act = cell(tr, ''); act.className = 'row';
+ if (j.Status !== 'pending') button(act, 'Download', () => download(j));
+ button(act, 'Delete', () => remove(j));
+ tb.appendChild(tr);
+ }}
+ if (!list.length) {{ const tr = document.createElement('tr'); cell(tr, 'No jobs yet.', 'err').colSpan = 4; tb.appendChild(tr); }}
+ }} catch (e) {{ say(e.message); }}
+}}
+$('f').addEventListener('submit', async ev => {{
+ ev.preventDefault();
+ const keywords = $('keywords').value.split('\\n').map(s => s.trim()).filter(Boolean);
+ const body = {{name: keywords[0] || 'job', keywords, lang: $('lang').value.trim().toLowerCase(),
+ lat: $('lat').value.trim(), lon: $('lon').value.trim(), depth: +$('depth').value,
+ max_time: +$('max_time').value, email: $('email').checked, zoom: 15}};
+ try {{
+ const r = await api('POST', '/api/v1/jobs', body), d = await r.json();
+ say(r.ok ? 'Job created. It runs in the background; this list refreshes by itself.' : 'Not started: ' + d.message);
+ load();
+ }} catch (e) {{ say(e.message); }}
+}});
+load(); setInterval(load, 3000);
+</script></body></html>"""
+
+
+class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+ server_version = "gmskit-engine"
+ sys_version = ""
+
+ def log_message(self, fmt, *args): # errors are logged by reply(); polls would flood the log
+ pass
+
+ def reply(self, code, body=b"", ctype="application/json", headers=()):
+ if ctype == "application/json":
+ body = json.dumps(body).encode()
+ self.send_response(code)
+ self.send_header("Content-Type", ctype)
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("X-Engine", MARK)
+ self.send_header("X-Content-Type-Options", "nosniff")
+ self.send_header("Referrer-Policy", "no-referrer")
+ self.send_header("Cache-Control", "no-store")
+ for k, v in headers:
+ self.send_header(k, v)
+ self.end_headers()
+ self.wfile.write(body)
+ if code >= 400:
+ log(f"{self.command} {self.path.split('?')[0]} -> {code}")
+
+ def fail(self, code, message):
+ self.reply(code, {"code": code, "message": message})
+
+ def guard(self):
+ """The checks ui-branding.conf made in front of the old engine, plus the optional key."""
+ if not HOST_OK.match(self.headers.get("Host", "")):
+ self.close_connection = True
+ self.fail(421, "this engine answers to localhost only")
+ return False
+ origin = self.headers.get("Origin")
+ if origin is not None and not ORIGIN_OK.match(origin):
+ self.close_connection = True
+ self.fail(403, "requests from other websites are refused")
+ return False
+ key, path = self.server.api_key, self.path.split("?")[0]
+ if key and path not in ("/", "/api/docs") and not hmac.compare_digest(
+ self.headers.get("X-API-Key", "").encode(), key.encode()):
+ self.fail(401, "missing or wrong X-API-Key")
+ return False
+ return True
+
+ def body(self):
+ if "chunked" in (self.headers.get("Transfer-Encoding") or "").lower():
+ self.close_connection = True
+ self.fail(411, "send a Content-Length")
+ return _REPLIED
+ try:
+ n = int(self.headers.get("Content-Length") or 0)
+ except ValueError:
+ n = -1
+ if not 0 <= n <= MAX_BODY:
+ self.close_connection = True
+ self.fail(413 if n > MAX_BODY else 400, "the body must be 1 MB or less")
+ return _REPLIED
+ try:
+ return json.loads(self.rfile.read(n) or b"null")
+ except ValueError: # bad JSON or bad UTF-8
+ self.fail(400, "the body is not valid JSON")
+ return _REPLIED
+
+ def download(self, jid):
+ data = self.server.store.read_csv(jid) if JOB_ID.match(jid or "") else None
+ if data is None:
+ return self.fail(404, "no results for this job (yet)")
+ self.reply(200, data, "text/csv; charset=utf-8",
+ [("Content-Disposition", f'attachment; filename="{jid}.csv"')])
+
+ def do_GET(self):
+ if not self.guard():
+ return
+ path, _, query = self.path.partition("?")
+ store = self.server.store
+ if path == "/api/v1/jobs":
+ return self.reply(200, store.all() or None) # null when empty, as before: the docs' health check
+ m = JOB_PATH.match(path)
+ if m:
+ if m.group(2):
+ return self.download(m.group(1))
+ job = store.get(m.group(1))
+ return self.reply(200, job) if job else self.fail(404, "job not found")
+ if path == "/download": # the old web UI's download link
+ return self.download(urllib.parse.parse_qs(query).get("id", [""])[0])
+ if path == "/":
+ nonce = secrets.token_urlsafe(16)
+ csp = (f"default-src 'none'; script-src 'nonce-{nonce}'; style-src 'nonce-{nonce}'; "
+ "connect-src 'self'; img-src 'self' data:; form-action 'self'; base-uri 'none'; "
+ "frame-ancestors 'none'")
+ page = PAGE.format(nonce=nonce, version=VERSION, repo=REPO).encode()
+ return self.reply(200, page, "text/html; charset=utf-8", [("Content-Security-Policy", csp)])
+ if path == "/api/docs":
+ return self.reply(200, OPENAPI)
+ self.fail(404, "not found")
+
+ def do_POST(self):
+ if not self.guard():
+ return
+ if self.path != "/api/v1/jobs":
+ return self.fail(404, "not found")
+ body = self.body()
+ if body is _REPLIED:
+ return
+ try:
+ name, data = jobs.validate(body)
+ except ValueError as e:
+ return self.fail(422, str(e))
+ jid = self.server.store.create(name, data)
+ self.server.worker.wake.set()
+ log(f"job {jid} created: {len(data['keywords'])} keyword(s)")
+ self.reply(201, {"id": jid})
+
+ def do_DELETE(self):
+ if not self.guard():
+ return
+ m = JOB_PATH.match(self.path)
+ if not m or m.group(2):
+ return self.fail(404, "not found")
+ self.server.worker.cancel(m.group(1))
+ if not self.server.store.delete(m.group(1)):
+ return self.fail(404, "job not found")
+ log(f"job {m.group(1)} deleted")
+ self.reply(200, {"message": "deleted"})
+
+
+class Server(ThreadingHTTPServer):
+ allow_reuse_address = os.name != "nt" # on Windows SO_REUSEADDR would let a second server share the port
+ daemon_threads = True
+
+
+class Server6(Server):
+ address_family = socket.AF_INET6
+
+
+def make_servers(port, store, worker, api_key=""):
+ """The server on 127.0.0.1:port, plus one on [::1]:port when this machine has IPv6 loopback
+ (Windows tries ::1 first for "localhost" and waits ~2 s before falling back to 127.0.0.1)."""
+ servers = [Server(("127.0.0.1", port), Handler)]
+ try:
+ servers.append(Server6(("::1", servers[0].server_address[1]), Handler))
+ except OSError:
+ pass
+ for s in servers:
+ s.store, s.worker, s.api_key = store, worker, api_key
+ return servers
+
+
+def main():
+ ap = argparse.ArgumentParser(description="The kit's Google Maps engine (local HTTP API and web UI).")
+ ap.add_argument("--port", type=int, help="default: SCRAPER_PORT from the environment or .env, else 8080")
+ ap.add_argument("--data", default=os.path.join(browser.HOME, "data"), help="jobs and results folder")
+ a = ap.parse_args()
+ env = read_env()
+ try:
+ port = a.port or int(env["SCRAPER_PORT"] or 8080)
+ except ValueError:
+ sys.exit(f"✗ SCRAPER_PORT is not a number: {env['SCRAPER_PORT']!r}")
+ browser.make_home()
+ store = jobs.Store(a.data)
+ if n := store.recover():
+ log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")
+ worker = jobs.Worker(store, log)
+ try:
+ servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"])
+ except OSError as e:
+ sys.exit(f"✗ Cannot listen on 127.0.0.1:{port}: {e}")
+ worker.start()
+ chrome = browser.find_chrome()
+ log(f"engine {VERSION} on http://localhost:{port} (browser: {chrome or 'none found'}"
+ f"{', API key required' if env['SCRAPER_API_KEY'] else ''})")
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
+ for s in servers[1:]:
+ threading.Thread(target=s.serve_forever, daemon=True).start()
+ try:
+ servers[0].serve_forever()
+ except KeyboardInterrupt:
+ pass
+ finally:
+ worker.shutdown()
+ log("engine stopped")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/engine/test_cdp.py b/engine/test_cdp.py
index 840bb2f..4571cb5 100644
--- a/engine/test_cdp.py
+++ b/engine/test_cdp.py
@@ -177,7 +177,7 @@ class Parser(unittest.TestCase):
self.assertEqual(got["phone"], "+1 512-555-0100")
self.assertEqual(got["website"], "https://example.com/")
self.assertEqual(got["review_rating"], 4.6)
- self.assertEqual(got["review_count"], 321)
+ self.assertNotIn("review_count", got) # (37, 1) is not the count: it comes from the card labels
self.assertEqual((got["latitude"], got["longitude"]), (30.2671, -97.743))
self.assertEqual(got["place_id"], "ChIJabc")
self.assertEqual(got["cid"], "255")
diff --git a/engine/test_engine.py b/engine/test_engine.py
new file mode 100644
index 0000000..7de576d
--- /dev/null
+++ b/engine/test_engine.py
@@ -0,0 +1,441 @@
+#!/usr/bin/env python3
+"""Offline checks for the engine: job body validation, CSV rows, the sqlite store, the worker
+(with a fake Chrome and fake Google results), the HTTP server and its guards, scripts/scrape.py's
+own client against that server, the Chrome download's SHA-256 check and the email pass.
+No network, no Chrome, no Docker.
+
+ python3 engine/test_engine.py
+"""
+import csv, hashlib, http.client, io, json, os, re, shutil, sys, tempfile, threading, time, unittest, zipfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import browser, jobs, maps, serve # noqa: E402
+from cdp import CDPError # noqa: E402
+
+GOOD = {"name": "t", "keywords": ["cafes in Austin"], "lang": "en", "lat": "30.2", "lon": "-97.7", "max_time": 60}
+
+
+def place(n, **kw):
+ return dict({"title": f"Biz {n}", "data_id": f"0x1:0x{n:x}", "place_id": f"ChIJ{n}", "cid": str(n),
+ "category": "Cafe", "review_rating": 4.5, "review_count": 10 + n,
+ "latitude": 30.2, "longitude": -97.7}, **kw)
+
+
+class FakeCDP:
+ closed = None
+
+
+class FakeBrowser:
+ """Stands in for browser.Browser: no process; kill() makes the next collect fail like a dead Chrome."""
+ made = []
+
+ def __init__(self, **kw):
+ self.kw, self.cdp, self.killed = kw, FakeCDP(), threading.Event()
+ FakeBrowser.made.append(self)
+
+ def kill(self):
+ self.cdp.closed = OSError("killed")
+ self.killed.set()
+
+ def close(self):
+ pass
+
+
+class FakePage:
+ def __init__(self, cdp):
+ pass
+
+ def close(self):
+ pass
+
+
+RESULTS = {} # keyword -> (places, reason) for the fake collect
+
+
+def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False):
+ for kw, (places, reason) in RESULTS.items():
+ if maps.search_url(kw).split("/@")[0].split("?")[0] in url:
+ if reason == "hang": # Chrome stops answering until the watchdog kills it
+ b = FakeBrowser.made[-1]
+ b.killed.wait(10)
+ raise CDPError("connection lost: killed")
+ if reason == "slow":
+ end = time.time() + 5
+ while not stop() and time.time() < end:
+ time.sleep(0.02)
+ return places, "stopped", {"batches": 1, "dom_links": 0}
+ return places, reason, {"batches": 1, "dom_links": 0}
+ return [], "no-feed", {"batches": 0, "dom_links": 0}
+
+
+class Fakes(unittest.TestCase):
+ def setUp(self):
+ self.dir = tempfile.mkdtemp()
+ self.saved = (browser.Browser, jobs.Page, maps.collect)
+ browser.Browser, jobs.Page, maps.collect = FakeBrowser, FakePage, fake_collect
+ FakeBrowser.made.clear()
+ RESULTS.clear()
+ self.store = jobs.Store(os.path.join(self.dir, "data"))
+ self.lines = []
+ self.worker = jobs.Worker(self.store, self.lines.append)
+
+ def tearDown(self):
+ browser.Browser, jobs.Page, maps.collect = self.saved
+ self.worker.shutdown()
+ if self.worker.is_alive():
+ self.worker.join(5)
+ self.store.db.close()
+ shutil.rmtree(self.dir, ignore_errors=True)
+
+ def run_one(self, **body):
+ name, data = jobs.validate(dict(GOOD, **body))
+ jid = self.store.create(name, data)
+ self.worker.run_job(self.store.claim())
+ return jid, self.store.get(jid)
+
+ def rows(self, jid):
+ return list(csv.DictReader(io.StringIO(self.store.read_csv(jid).decode("utf-8"))))
+
+
+class Validate(unittest.TestCase):
+ def err(self, **body):
+ with self.assertRaises(ValueError) as c:
+ jobs.validate(body)
+ return str(c.exception)
+
+ def test_old_engine_messages(self):
+ self.assertEqual(self.err(), "missing name")
+ self.assertEqual(self.err(name="x"), "missing keywords")
+ self.assertEqual(self.err(name="x", keywords=["a"]), "missing lang")
+ self.assertEqual(self.err(name="x", keywords=["a"], lang="eng"), "invalid lang")
+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en"), "missing geo coordinates")
+ self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lat="1", lon="2"), "missing max time")
+
+ def test_bad_values(self):
+ self.assertIn("invalid geo", self.err(**dict(GOOD, lat="north")))
+ self.assertIn("invalid geo", self.err(**dict(GOOD, lat="91")))
+ self.assertIn("invalid geo", self.err(**dict(GOOD, lon="nan")))
+ self.assertIn("invalid geo", self.err(**dict(GOOD, lat=True)))
+ self.assertIn("invalid depth", self.err(**dict(GOOD, depth=0)))
+ self.assertIn("invalid depth", self.err(**dict(GOOD, depth="5")))
+ self.assertIn("invalid max time", self.err(**dict(GOOD, max_time=-1)))
+ self.assertIn("invalid keywords", self.err(**dict(GOOD, keywords=["ok", ""])))
+ self.assertIn("usernames and passwords", self.err(**dict(GOOD, proxies=["http://u:p@h:1"])))
+ self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["ftp://h:1"])))
+ self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["http://h:99999"])))
+ with self.assertRaisesRegex(ValueError, "JSON object"):
+ jobs.validate([1])
+
+ def test_defaults_and_numbers(self):
+ name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))
+ self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))
+ self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])
+
+
+class Rows(unittest.TestCase):
+ def test_columns_and_mapping(self):
+ r = jobs.to_row(dict(place(1), open_state="Open 24 hours", description="Good coffee",
+ categories="Cafe | Bakery", junk="x"), "cafes")
+ self.assertEqual(len(jobs.COLUMNS), 36)
+ self.assertEqual(list(r), jobs.COLUMNS + jobs.EXTRA)
+ self.assertEqual((r["input_id"], r["status"], r["descriptions"], r["categories"]),
+ ("cafes", "Open 24 hours", "Good coffee", "Cafe | Bakery"))
+ self.assertEqual((r["review_rating"], r["review_count"], r["emails"]), ("4.5", "11", ""))
+ self.assertTrue(r["link"].startswith("https://www.google.com/maps/search/?api=1&query=Biz%201&query_place_id=ChIJ1"))
+ dom = jobs.to_row({"title": "A", "link": "https://www.google.com/maps/place/A/data=!1s0x1:0x2"}, "k")
+ self.assertEqual(dom["link"], "https://www.google.com/maps/place/A/data=!1s0x1:0x2")
+
+ def test_csv_utf8_no_bom(self):
+ raw = jobs.to_csv([jobs.to_row(place(1, title="Café ☕"), "k")])
+ self.assertFalse(raw.startswith(b"\xef\xbb\xbf"))
+ self.assertEqual(next(csv.DictReader(io.StringIO(raw.decode())))["title"], "Café ☕")
+
+
+class Store(Fakes):
+ def test_lifecycle(self):
+ a = self.store.create("a", {"x": 1})
+ b = self.store.create("b", {"x": 2})
+ self.assertEqual([j["ID"] for j in self.store.all()], [b, a]) # newest first
+ self.assertEqual(self.store.claim()["ID"], a) # oldest pending first
+ self.assertEqual(self.store.get(a)["Status"], "working")
+ self.assertEqual(self.store.recover(), 1)
+ self.assertEqual(self.store.get(a)["Status"], "failed")
+ self.assertTrue(self.store.write_csv(b, []))
+ self.assertTrue(self.store.read_csv(b).startswith(b"input_id,link,title"))
+ self.assertTrue(self.store.delete(b))
+ self.assertIsNone(self.store.read_csv(b))
+ self.assertFalse(self.store.write_csv(b, [])) # a deleted job gets no file back
+ self.assertFalse(self.store.delete(b))
+ self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error"})
+ self.assertRegex(self.store.get(a)["Date"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00$")
+
+
+class Worker(Fakes):
+ def test_keywords_dedupe_into_one_csv(self):
+ RESULTS["cafes"] = ([place(1), place(2)], "end-of-list")
+ RESULTS["bakeries"] = ([place(2), place(3)], "depth")
+ jid, job = self.run_one(keywords=["cafes", "bakeries"], depth=2, lang="de", proxies=["http://10.0.0.1:3128"])
+ self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
+ rows = self.rows(jid)
+ self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2", "Biz 3"])
+ self.assertEqual([r["input_id"] for r in rows], ["cafes", "cafes", "bakeries"])
+ self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "proxy": "http://10.0.0.1:3128"})
+ self.assertFalse(any(re.search(r"Biz \d", l) for l in self.lines)) # counts only in the log
+
+ def test_no_results_is_ok_with_header(self):
+ jid, job = self.run_one(keywords=["nothing here"])
+ self.assertEqual(job["Status"], "ok")
+ self.assertEqual(self.rows(jid), [])
+
+ def test_no_chrome_fails_with_reason(self):
+ def no_chrome(**kw):
+ raise browser.BrowserError("No Chrome, Chromium, Edge or Brave found.")
+ browser.Browser = no_chrome
+ _, job = self.run_one()
+ self.assertEqual(job["Status"], "failed")
+ self.assertIn("No Chrome", job["Error"])
+
+ def test_blocked_fails(self):
+ RESULTS["cafes in Austin"] = ([], "blocked")
+ _, job = self.run_one()
+ self.assertEqual(job["Status"], "failed")
+ self.assertIn("unusual-traffic", job["Error"])
+
+ def test_watchdog_kills_a_hung_chrome(self):
+ RESULTS["cafes in Austin"] = ([], "hang")
+ saved, jobs.WATCHDOG_GRACE = jobs.WATCHDOG_GRACE, 0.2
+ try:
+ t = time.monotonic()
+ _, job = self.run_one(max_time=1)
+ finally:
+ jobs.WATCHDOG_GRACE = saved
+ self.assertLess(time.monotonic() - t, 5)
+ self.assertTrue(FakeBrowser.made[0].killed.is_set())
+ self.assertEqual(job["Status"], "failed")
+
+ def test_delete_while_running(self):
+ RESULTS["cafes in Austin"] = ([place(1)], "slow")
+ name, data = jobs.validate(GOOD)
+ jid = self.store.create(name, data)
+ t = threading.Thread(target=self.worker.run_job, args=(self.store.claim(),))
+ t.start()
+ time.sleep(0.2)
+ self.worker.cancel(jid)
+ self.store.delete(jid)
+ t.join(5)
+ self.assertFalse(t.is_alive())
+ self.assertIsNone(self.store.get(jid))
+ self.assertFalse(os.path.exists(self.store.csv_path(jid)))
+
+ def test_email_pass(self):
+ RESULTS["cafes in Austin"] = ([place(1, website="https://www.biz1.com"), place(2), place(3, website="x.org")], "end-of-list")
+ pages = {"https://www.biz1.com": 'mail <a href="mailto:[email protected]">us</a> or sales&#64;biz1.com',
+ "x.org": "[email protected] nothing"}
+ saved = jobs.fetch
+ jobs.fetch = lambda url, timeout=0: pages.get(url, "")
+ try:
+ jid, job = self.run_one(email=True)
+ finally:
+ jobs.fetch = saved
+ self.assertEqual(job["Status"], "ok")
+ self.assertEqual([r["emails"] for r in self.rows(jid)], ["[email protected], [email protected]", "", ""])
+
+
+class Emails(unittest.TestCase):
+ def test_find_emails(self):
+ self.assertEqual(jobs.find_emails(text, "shop.com"), ["[email protected]", "[email protected]"])
+
+ def test_public_only(self):
+ self.assertFalse(jobs._public_ip("127.0.0.1"))
+ self.assertFalse(jobs._public_ip("192.168.1.1"))
+ self.assertFalse(jobs._public_ip("169.254.169.254"))
+ self.assertFalse(jobs._public_ip("::1"))
+ self.assertTrue(jobs._public_ip("8.8.8.8"))
+ self.assertFalse(jobs._public_host("localhost"))
+ self.assertEqual(jobs.fetch("http://127.0.0.1:9/"), "")
+ self.assertEqual(jobs.site_host("https://www.Shop.com/x"), "shop.com")
+
+
+class Server(Fakes):
+ key = ""
+
+ def setUp(self):
+ super().setUp()
+ self.servers = serve.make_servers(0, self.store, self.worker, self.key)
+ self.port = self.servers[0].server_address[1]
+ for s in self.servers:
+ threading.Thread(target=s.serve_forever, daemon=True).start()
+
+ def tearDown(self):
+ for s in self.servers:
+ s.shutdown()
+ s.server_close()
+ super().tearDown()
+
+ def call(self, method, path, body=None, headers=None, raw=None):
+ c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
+ h = {"Host": f"localhost:{self.port}"}
+ h.update(headers or {})
+ data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
+ c.request(method, path, data, h)
+ r = c.getresponse()
+ out = (r.status, r.read(), r)
+ c.close()
+ return out
+
+
+class Api(Server):
+ def test_flow(self):
+ code, body, r = self.call("GET", "/api/v1/jobs")
+ self.assertEqual((code, body), (200, b"null")) # the docs' health check prints null
+ self.assertTrue(r.getheader("X-Engine").startswith("gmskit-engine/"))
+ code, body, _ = self.call("POST", "/api/v1/jobs", {})
+ self.assertEqual((code, json.loads(body)), (422, {"code": 422, "message": "missing name"}))
+ code, body, _ = self.call("POST", "/api/v1/jobs", GOOD)
+ self.assertEqual(code, 201)
+ jid = json.loads(body)["id"]
+ self.assertTrue(self.worker.wake.is_set())
+ job = json.loads(self.call("GET", f"/api/v1/jobs/{jid}")[1])
+ self.assertEqual((job["ID"], job["Name"], job["Status"], job["Data"]["keywords"]), (jid, "t", "pending", ["cafes in Austin"]))
+ self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download")[0], 404)
+ self.store.write_csv(jid, [jobs.to_row(place(1), "k")])
+ code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download")
+ self.assertEqual((code, r.getheader("Content-Type")), (200, "text/csv; charset=utf-8"))
+ self.assertEqual(self.call("GET", f"/download?id={jid}")[1], body)
+ self.assertEqual(len(json.loads(self.call("GET", "/api/v1/jobs")[1])), 1)
+ self.assertEqual(self.call("DELETE", f"/api/v1/jobs/{jid}")[0], 200)
+ self.assertEqual(self.call("DELETE", f"/api/v1/jobs/{jid}")[0], 404)
+ self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}")[0], 404)
+ self.assertEqual(self.call("GET", "/download?id=../../etc/passwd")[0], 404)
+
+ def test_guards(self):
+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": "evil.com"})[0], 421)
+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": "localhost.evil.com"})[0], 421)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "https://evil.com"})[0], 403)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "null"})[0], 403)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "http://localhost.evil.com"})[0], 403)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": f"http://localhost:{self.port}"})[0], 201)
+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": f"127.0.0.1:{self.port}"})[0], 200)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"{nope")[0], 400)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"\xff\xfe")[0], 400)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"", headers={"Content-Length": "99999999"})[0], 413)
+ self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"0\r\n\r\n", headers={"Transfer-Encoding": "chunked"})[0], 411)
+ self.assertEqual(self.call("POST", "/elsewhere", GOOD)[0], 404)
+
+ def test_page_and_docs(self):
+ code, body, r = self.call("GET", "/")
+ csp = r.getheader("Content-Security-Policy")
+ nonce = re.search(r"'nonce-([^']+)'", csp).group(1)
+ self.assertIn(f'<script nonce="{nonce}">', body.decode())
+ self.assertIn("github.com/Mahanaicoach/google-maps-scraper-kit", body.decode())
+ self.assertIn("frame-ancestors 'none'", csp)
+ self.assertNotRegex(body.decode(), r"\{[a-z_]+\}") # every placeholder filled
+ code, body, _ = self.call("GET", "/api/docs")
+ self.assertEqual(json.loads(body)["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]
+ ["schema"]["required"], ["name", "keywords", "lang", "lat", "lon", "max_time"])
+
+
+class ApiKey(Server):
+ key = "s3cret"
+
+ def test_key(self):
+ self.assertEqual(self.call("GET", "/api/v1/jobs")[0], 401)
+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"X-API-Key": "wrong"})[0], 401)
+ self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"X-API-Key": "s3cret"})[0], 200)
+ self.assertEqual(self.call("GET", "/")[0], 200) # the page holds no data; it asks for the key
+
+
+class ScrapeClient(Server):
+ """scripts/scrape.py's own create, poll and download against this engine, with the worker running."""
+
+ def test_run_job(self):
+ RESULTS["cafes in Austin"] = ([place(1), place(2)], "end-of-list")
+ os.environ["SCRAPER_BASE_URL"] = f"http://localhost:{self.port}"
+ sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))
+ try:
+ import scrape
+ scrape.BASE = os.environ["SCRAPER_BASE_URL"]
+ scrape.POLL_EVERY = 1
+ scrape._local_docker = lambda: False # never touch a real Docker engine from a test
+ self.worker.start()
+ out = io.StringIO()
+ saved, sys.stdout = sys.stdout, out
+ try:
+ jid, rows = scrape.run_job({"name": "t", "keywords": ["cafes in Austin"], "lang": "en",
+ "lat": "30.2", "lon": "-97.7", "depth": 1, "email": False,
+ "max_time": 60, "zoom": 15, "fast_mode": False, "radius": 10000})
+ finally:
+ sys.stdout = saved
+ finally:
+ del os.environ["SCRAPER_BASE_URL"]
+ self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2"])
+ self.assertEqual(self.store.get(jid)["Status"], "ok")
+ for k in scrape.LEAD:
+ self.assertIn(k, rows[0])
+
+
+class Download(unittest.TestCase):
+ def setUp(self):
+ self.dir = tempfile.mkdtemp()
+ self.saved = (browser.HOME, dict(browser.CFT_SHA256))
+ browser.HOME = self.dir
+
+ def tearDown(self):
+ browser.HOME = self.saved[0]
+ browser.CFT_SHA256.clear()
+ browser.CFT_SHA256.update(self.saved[1])
+ shutil.rmtree(self.dir, ignore_errors=True)
+
+ @staticmethod
+ def zipped():
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as z:
+ info = zipfile.ZipInfo("chrome-headless-shell-linux64/chrome-headless-shell")
+ info.external_attr = 0o100755 << 16
+ z.writestr(info, b"#!/bin/sh\n")
+ z.writestr("chrome-headless-shell-linux64/libfoo.so", b"x")
+ return buf.getvalue()
+
+ def opener(self, data):
+ return lambda url, timeout=0: io.BytesIO(data)
+
+ def test_platforms(self):
+ self.assertEqual(browser.cft_platform("Darwin", "arm64"), "mac-arm64")
+ self.assertEqual(browser.cft_platform("Darwin", "x86_64"), "mac-x64")
+ self.assertEqual(browser.cft_platform("Linux", "aarch64"), "linux-arm64")
+ self.assertEqual(browser.cft_platform("Linux", "x86_64"), "linux64")
+ self.assertEqual(browser.cft_platform("Windows", "AMD64"), "win64")
+ self.assertEqual(browser.cft_platform("Windows", "ARM64"), "win64")
+ self.assertIsNone(browser.cft_platform("Linux", "riscv64"))
+ self.assertTrue(all(re.fullmatch(r"[0-9a-f]{64}", h) for h in self.saved[1].values()))
+
+ def test_mismatch_is_deleted_and_never_run(self):
+ with self.assertRaises(browser.BrowserError) as c:
+ browser.download(lambda m: None, "linux64", self.opener(self.zipped()))
+ self.assertIn("does not match the SHA-256", str(c.exception))
+ leftovers = [f for _, _, fs in os.walk(self.dir) for f in fs]
+ self.assertEqual(leftovers, [])
+
+ def test_verified_download_unpacks(self):
+ data = self.zipped()
+ browser.CFT_SHA256["linux64"] = hashlib.sha256(data).hexdigest()
+ path = browser.download(lambda m: None, "linux64", self.opener(data))
+ self.assertEqual(path, browser.shell_path("linux64"))
+ self.assertTrue(os.path.isfile(path))
+ if os.name != "nt":
+ self.assertTrue(os.access(path, os.X_OK))
+ self.assertEqual(os.stat(self.dir).st_mode & 0o777, 0o700)
+ self.assertFalse(any(f.endswith(".part") for _, _, fs in os.walk(self.dir) for f in fs))
+
+ def test_missing_libraries_hint(self):
+ err = "chrome: error while loading shared libraries: libnss3.so: cannot open shared object file\n"
+ msg = browser.start_error("/x/chrome", 127, err)
+ self.assertIn("libnss3.so", msg)
+ self.assertIn("sudo apt-get install -y libnss3", msg)
+ self.assertIn("code 3", browser.start_error("/x/chrome", 3, ""))
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=1)
diff --git a/scripts/engine.py b/scripts/engine.py
new file mode 100644
index 0000000..2e58294
--- /dev/null
+++ b/scripts/engine.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""Run the kit's own engine on this machine, without Docker.
+
+ python3 scripts/engine.py start # first run on a machine without Chrome downloads one (~100 MB)
+ python3 scripts/engine.py status
+ python3 scripts/engine.py restart
+ python3 scripts/engine.py stop
+
+Standard library only. The engine (engine/serve.py) serves http://localhost:8080 (SCRAPER_PORT in
+.env) with the same API the scripts, slash commands and web UI use with Docker. It drives an
+installed Chrome, Chromium, Edge or Brave; on a machine with none, `start` downloads Chrome for
+Testing's headless shell, pinned by version and SHA-256. Everything lives in .engine/ in the kit
+folder; delete it to remove the engine and its results.
+"""
+import json, os, signal, socket, subprocess, sys, time, urllib.error, urllib.parse, urllib.request
+
+for _stream in (sys.stdout, sys.stderr): # ▶ ✓ ✗ on a Windows cp1252 console
+ try:
+ _stream.reconfigure(encoding="utf-8", errors="replace")
+ except (AttributeError, ValueError):
+ pass
+
+KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(KIT, "engine"))
+import browser # noqa: E402
+
+HOME = browser.HOME
+SERVE = os.path.join(KIT, "engine", "serve.py")
+STATE = os.path.join(HOME, "state.json")
+LOCK = os.path.join(HOME, "start.lock")
+LOG = os.path.join(HOME, "engine.log")
+LOG_KEEP = 2_000_000 # bytes of engine.log kept across starts
+MARK = "gmskit-engine/"
+_NO_PROXY = urllib.request.build_opener(urllib.request.ProxyHandler({})) # HTTP_PROXY must not catch localhost
+PY = "py" if os.name == "nt" else "python3" # what the user types; the README says `py` on Windows
+
+
+def public_port():
+ """The port scrape.py talks to (SCRAPER_PORT / SCRAPER_BASE_URL, environment or .env), read by
+ scrape.py's own loader so both always agree."""
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+ import scrape
+ u = urllib.parse.urlparse(scrape.BASE)
+ if u.hostname not in ("localhost", "127.0.0.1", "::1"):
+ sys.exit(f"✗ SCRAPER_BASE_URL points at {scrape.BASE}; engine.py runs the engine on this machine only.")
+ return u.port or 80
+
+
+def probe(port, timeout=2):
+ """"ours" (this kit's engine answers), "other" (something else, e.g. the Docker engine) or None."""
+ try:
+ with _NO_PROXY.open(f"http://127.0.0.1:{port}/api/v1/jobs", timeout=timeout) as r:
+ return "ours" if (r.headers.get("X-Engine") or "").startswith(MARK) else "other"
+ except urllib.error.HTTPError as e: # 401 without the key is still ours
+ e.close()
+ return "ours" if (e.headers.get("X-Engine") or "").startswith(MARK) else "other"
+ except Exception: # noqa: BLE001 - nothing listening, or not HTTP
+ return None
+
+
+def port_free(port):
+ with socket.socket() as s:
+ if os.name != "nt": # like the engine's own server: connections in TIME_WAIT do not block
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ try:
+ s.bind(("127.0.0.1", port))
+ return True
+ except OSError:
+ return False
+
+
+def read_state():
+ try:
+ with open(STATE, encoding="utf-8") as f:
+ return json.load(f)
+ except (OSError, ValueError):
+ return None
+
+
+def write_state(st):
+ tmp = STATE + ".tmp"
+ with open(tmp, "w", encoding="utf-8") as f:
+ json.dump(st, f)
+ os.replace(tmp, STATE) # a crash mid-write never leaves half a state file
+
+
+def command_line(pid):
+ """The command line of `pid`; "" if the process is gone; None if it cannot be read (PowerShell
+ blocked or too slow), which stop must not mistake for "gone"."""
+ try:
+ if os.name == "nt":
+ ps = ["powershell", "-NoProfile", "-Command",
+ f"(Get-CimInstance Win32_Process -Filter 'ProcessId={int(pid)}').CommandLine"]
+ out = subprocess.run(ps, capture_output=True, text=True, errors="replace", timeout=20)
+ return None if out.returncode else out.stdout
+ if os.path.exists(f"/proc/{int(pid)}/cmdline"):
+ with open(f"/proc/{int(pid)}/cmdline", "rb") as f:
+ return f.read().replace(b"\0", b" ").decode("utf-8", "replace")
+ return subprocess.run(["ps", "-p", str(int(pid)), "-o", "command="], capture_output=True,
+ text=True, errors="replace", timeout=10).stdout
+ except (OSError, ValueError, subprocess.SubprocessError):
+ return None
+
+
+def kill(pid):
+ """Stop the engine and its Chrome. False if it could not be told whether `pid` is the engine:
+ after a reboot or crash the pid in state.json may belong to an unrelated process."""
+ cmd = command_line(pid)
+ if cmd is None:
+ return False
+ if SERVE not in cmd:
+ return True # gone, or someone else's process now: nothing of ours to stop
+ if os.name == "nt":
+ subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True)
+ return True
+ try:
+ os.killpg(pid, signal.SIGTERM) # the engine leads its own process group, Chrome included
+ except (ProcessLookupError, PermissionError):
+ return True
+ for _ in range(50):
+ try:
+ os.killpg(pid, 0)
+ except (ProcessLookupError, PermissionError):
+ return True
+ time.sleep(0.1)
+ try:
+ os.killpg(pid, signal.SIGKILL)
+ except (ProcessLookupError, PermissionError):
+ pass
+ return True
+
+
+def tail(path, lines=15):
+ try:
+ with open(path, "rb") as f:
+ f.seek(max(0, os.path.getsize(path) - 8000))
+ return "\n".join(f.read().decode("utf-8", "replace").splitlines()[-lines:])
+ except OSError:
+ return ""
+
+
+def trim_log():
+ """Keep the last LOG_KEEP bytes of engine.log, so it cannot grow without bound across starts."""
+ try:
+ if os.path.getsize(LOG) > LOG_KEEP:
+ with open(LOG, "rb") as f:
+ f.seek(-LOG_KEEP, os.SEEK_END)
+ keep = f.read()
+ with open(LOG, "wb") as f:
+ f.write(keep)
+ except OSError:
+ pass
+
+
+def busy(port):
+ hint = f"netstat -ano | findstr :{port}" if os.name == "nt" else f"lsof -nP -iTCP:{port} -sTCP:LISTEN"
+ sys.exit(f"✗ Port {port} is in use. Is the Docker engine running? Stop it with `docker compose stop`,\n"
+ f" or set SCRAPER_PORT in .env to a free port. To see what holds it: {hint}")
+
+
+def chrome(download):
+ """A browser for the engine; downloads the pinned one only when allowed (start, never restart:
+ scrape.py may restart the engine unattended, and fetching code is for an explicit start)."""
+ path = browser.find_chrome()
+ if path:
+ return path
+ if not download:
+ sys.exit(f"✗ No Chrome found. Run: {PY} scripts/engine.py start (it downloads one)")
+ try:
+ return browser.download()
+ except browser.BrowserError as e:
+ sys.exit(f"✗ {e}")
+
+
+def start(download=True):
+ port = public_port()
+ seen = probe(port)
+ if seen == "ours":
+ print(f"✓ The engine is already running: http://localhost:{port}")
+ return
+ if seen == "other":
+ busy(port)
+ if read_state():
+ stop(quiet=True) # stale state from a crash or reboot: clean up whatever is still ours
+ if not port_free(port):
+ busy(port)
+ browser.make_home()
+ try: # one start at a time
+ lock = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+ except FileExistsError:
+ if time.time() - os.path.getmtime(LOCK) < 600:
+ sys.exit("✗ Another `engine.py start` is running. Wait for it, or delete .engine/start.lock.")
+ os.remove(LOCK) # left by a start that crashed
+ lock = os.open(LOCK, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+ try:
+ _start(port, chrome(download))
+ finally:
+ os.close(lock)
+ os.remove(LOCK)
+
+
+def _start(port, chrome_path):
+ trim_log()
+ env = dict(os.environ, CHROME=chrome_path, SCRAPER_PORT=str(port))
+ with open(LOG, "ab") as out: # the engine keeps its own handle
+ kw = {"stdout": out, "stderr": out, "stdin": subprocess.DEVNULL, "cwd": HOME, "env": env}
+ if os.name == "nt":
+ # A hidden console, not DETACHED_PROCESS: Chrome started from a process with no console
+ # at all closed at its first page on Windows.
+ kw["creationflags"] = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP
+ else:
+ kw["start_new_session"] = True
+ try:
+ proc = subprocess.Popen([sys.executable, SERVE, "--port", str(port)], **kw)
+ except OSError as e:
+ sys.exit(f"✗ Could not start the engine: {e}")
+ write_state({"pid": proc.pid, "port": port, "chrome": chrome_path, "log": LOG})
+ deadline = time.time() + 30
+ while time.time() < deadline:
+ if probe(port) == "ours":
+ print(f"✓ Engine running: http://localhost:{port} (web UI and API)\n"
+ f" Browser: {chrome_path}\n"
+ f" Stop it with: {PY} scripts/engine.py stop")
+ return
+ if proc.poll() is not None:
+ stop(quiet=True)
+ sys.exit(f"✗ The engine exited at once (code {proc.returncode}). The end of {LOG}:\n{tail(LOG)}")
+ time.sleep(0.5)
+ stop(quiet=True)
+ sys.exit(f"✗ The engine did not answer within 30 seconds. The end of {LOG}:\n{tail(LOG)}")
+
+
+def stop(quiet=False):
+ st = read_state()
+ if not st:
+ if not quiet:
+ print("The engine is not running.")
+ return
+ if st.get("pid") and not kill(st["pid"]):
+ how = "taskkill /PID {} /T /F" if os.name == "nt" else "kill {}"
+ sys.exit(f"✗ Could not check whether process {st['pid']} is the engine, so it was left running.\n"
+ f" Stop it yourself: {how.format(st['pid'])}")
+ try:
+ os.remove(STATE)
+ except OSError:
+ pass
+ if not quiet:
+ print("✓ Engine stopped. Its jobs and results stay in .engine/data.")
+
+
+def status():
+ port = public_port()
+ seen = probe(port)
+ if seen == "ours":
+ st = read_state() or {}
+ print(f"✓ Running on http://localhost:{port}" + (f" (browser: {st['chrome']})" if st.get("chrome") else ""))
+ elif seen == "other":
+ print(f"ℹ Port {port} answers, but not as this kit's engine (the Docker engine, probably).")
+ sys.exit(1)
+ else:
+ print(f"✗ Not running. Start it with: {PY} scripts/engine.py start")
+ sys.exit(1)
+
+
+def main():
+ cmds = {"start": start, "stop": stop, "status": status,
+ "restart": lambda: (stop(quiet=True), start(download=False))}
+ if len(sys.argv) != 2 or sys.argv[1] not in cmds:
+ sys.exit(f"usage: {PY} scripts/engine.py start|stop|status|restart")
+ cmds[sys.argv[1]]()
+
+
+if __name__ == "__main__":
+ main()