Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

feat(kit): P2 Docker image on the own engine, scrape.py without gosom workarounds

Mahan Jafari044af0a39334
diff --git a/.claude/settings.json b/.claude/settings.json
index 7f4bbb6..5955d25 100644
--- a/.claude/settings.json
+++ b/.claude/settings.json
@@ -1,5 +1,5 @@
{
- "$comment": "Pre-approves the safe, local commands this kit uses so Claude can scrape on autopilot without a permission prompt each time. All entries are local-only (localhost / this folder's scripts / docker). Nothing here can reach the public internet except the scraper's own outbound calls, and `scripts/engine.py start` on a machine with no Chrome, which downloads Chrome for Testing once, pinned by SHA-256.",
+ "$comment": "Pre-approves the safe, local commands this kit uses so Claude can scrape on autopilot without a permission prompt each time. All entries are local-only (localhost / this folder's scripts / docker compose in this folder). Nothing here can reach the public internet except the scraper's own outbound calls, `docker compose up` building the engine image (python:3.12-slim and Debian's chromium), and `scripts/engine.py start` on a machine with no Chrome, which downloads Chrome for Testing once, pinned by SHA-256.",
"permissions": {
"allow": [
"Bash(docker compose up:*)",
@@ -8,6 +8,8 @@
"Bash(docker compose start:*)",
"Bash(docker compose ps:*)",
"Bash(docker ps:*)",
+ "Bash(docker compose restart)",
+ "Bash(docker compose logs)",
"Bash(curl http://localhost:8080/:*)",
"Bash(curl -s http://localhost:8080/:*)",
"Bash(curl http://127.0.0.1:8080/:*)",
@@ -33,8 +35,7 @@
"Bash(bash scripts/scrape.sh:*)",
"Bash(python3 scripts/scrape.py:*)",
"Bash(python scripts/scrape.py:*)",
- "Bash(py scripts/scrape.py:*)",
- "Bash(docker restart:*)"
+ "Bash(py scripts/scrape.py:*)"
],
"deny": []
}
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..a2f01ef
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,4 @@
+# Only the engine goes into the image: never .env, scraped results or .engine/ (jobs, a downloaded Chrome).
+*
+!engine
+**/__pycache__
diff --git a/.github/workflows/engine-live.yml b/.github/workflows/engine-live.yml
index f218846..4ce2d9b 100644
--- a/.github/workflows/engine-live.yml
+++ b/.github/workflows/engine-live.yml
@@ -1,7 +1,7 @@
-# The kit's own engine against real Google Maps, once per platform: start it the way a user does,
-# run one small scrape through scripts/scrape.py, and require places back. Runs when the engine
-# changes, or by hand (Actions > engine-live > Run workflow). The logs print counts only: scraped
-# names, phones and emails never reach them.
+# The kit's own engine against real Google Maps, once per platform and once in Docker: start it the
+# way a user does, run one small scrape through scripts/scrape.py, and require places back. Runs when
+# the engine or the scripts change, or by hand (Actions > engine-live > Run workflow). The logs print
+# counts only: scraped names, phones and emails never reach them.
name: engine-live
on:
workflow_dispatch:
@@ -9,6 +9,9 @@ on:
paths:
- "engine/**"
- "scripts/engine.py"
+ - "scripts/scrape.py"
+ - "Dockerfile"
+ - "docker-compose.yml"
- ".github/workflows/engine-live.yml"
permissions:
contents: read
@@ -57,3 +60,28 @@ jobs:
- name: stop
if: always()
run: python scripts/engine.py stop
+ docker:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v4
+ - name: docker compose up (builds the engine image), as SETUP.md says
+ run: docker compose up -d --build
+ - name: scrape (depth 2, no email crawl)
+ run: |
+ start=$(date +%s)
+ python3 scripts/scrape.py "coffee shops in Austin TX" 30.2672 -97.7431 --depth 2 --no-email --out leads.csv > scrape.log 2>&1 || true
+ echo "seconds=$(( $(date +%s) - start ))"
+ grep -v "^ •" scrape.log | tr '\r' '\n' | grep -v "^ status:" || true
+ python3 - <<'PY'
+ import csv, sys
+ rows = list(csv.DictReader(open("leads.csv", encoding="utf-8-sig")))
+ fill = {k: sum(1 for r in rows if r.get(k)) for k in ("title", "phone", "website", "address", "review_rating")}
+ print(f"places={len(rows)}", fill)
+ sys.exit(0 if rows else 1)
+ PY
+ - name: engine log (counts only)
+ if: always()
+ run: docker compose logs --no-log-prefix || true
+ - if: always()
+ run: docker compose down -v
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 0ddec20..5870ece 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,6 +1,7 @@
# Two jobs, no Google traffic. "offline" is the Python client and the kit's own engine on the three
-# desktop platforms a new user installs on. "fresh-install" is a new user's first ten minutes on Linux minus the scrape:
-# clone, docker compose up, the health check, the patched web UI and API docs, the proxy's checks.
+# desktop platforms a new user installs on. "fresh-install" is a new user's first ten minutes on Linux
+# minus the scrape: clone, docker compose up (builds the engine image), the health check, the web UI,
+# the engine's own checks. engine-live.yml runs the scrapes.
name: tests
on:
push:
@@ -29,23 +30,29 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
+ - name: a .env and a results file the image must never contain
+ run: |
+ echo "SCRAPER_PORT=8080" > .env
+ echo "title,phone" > results-ci.csv
- run: docker compose config -q
- - run: docker compose up -d
+ - run: docker compose up -d --build
- name: health check prints null, as SETUP.md says
run: |
for i in $(seq 1 45); do curl -sf http://localhost:8080/api/v1/jobs >/dev/null && break; sleep 2; done
test "$(curl -s http://localhost:8080/api/v1/jobs)" = null
- - name: web UI is patched (kit links, emails on, Redoc pinned)
+ - name: the kit's engine, published on 127.0.0.1 only, not root, nothing but the engine in the image
run: |
- curl -s http://localhost:8080/ | grep -q 'github.com/Mahanaicoach/google-maps-scraper-kit'
- curl -s http://localhost:8080/ | grep -q 'id="email" name="email" checked'
- curl -s http://localhost:8080/api/docs | grep -q 'redoc/v2.5.0/'
- curl -s http://localhost:8080/ | grep -q 'assets/sponsors/sponsors.json'
- - name: proxy refuses other hosts and other sites
+ curl -s -D - -o /dev/null http://localhost:8080/api/v1/jobs | grep -qi '^x-engine: gmskit-engine/'
+ test "$(docker compose port google-maps-scraper 8080)" = 127.0.0.1:8080
+ test "$(docker compose exec -T google-maps-scraper id -u)" != 0
+ test "$(docker compose exec -T google-maps-scraper ls -A /kit | tr '\n' ' ')" = ".engine engine "
+ - name: web UI links the kit
+ run: curl -s http://localhost:8080/ | grep -q 'github.com/Mahanaicoach/google-maps-scraper-kit'
+ - name: engine refuses other hosts and other sites
run: |
test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Host: evil.com' http://localhost:8080/api/v1/jobs)" = 421
test "$(curl -s -o /dev/null -w '%{http_code}' -H 'Origin: https://evil.com' http://localhost:8080/api/v1/jobs)" = 403
- - name: engine validates a job body through the proxy
+ - name: engine validates a job body
run: |
curl -s -X POST -H 'Content-Type: application/json' -d '{}' http://localhost:8080/api/v1/jobs | grep -q 'missing name'
- if: always()
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..6753e9c
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,23 @@
+# The kit's own Google Maps engine (engine/serve.py) with Debian's Chromium, for `docker compose up -d`.
+# Python standard library only: nothing is installed with pip. Without Docker the same engine runs
+# with `python3 scripts/engine.py start`.
+FROM python:3.12-slim
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends chromium \
+ && rm -rf /var/lib/apt/lists/* \
+ && useradd --create-home --uid 10001 gmskit
+
+WORKDIR /kit
+COPY engine/ engine/
+# The volume in docker-compose.yml mounts on .engine/data and takes this folder's owner.
+RUN mkdir -p .engine/data && chown -R gmskit:gmskit .engine
+
+# Not root, so Chromium keeps its sandbox where the container allows one; where Docker's seccomp
+# profile blocks it, the engine relaunches Chromium without (engine/browser.py).
+USER gmskit
+ENV CHROME=/usr/bin/chromium PYTHONUNBUFFERED=1
+EXPOSE 8080
+# 0.0.0.0 inside the container, so Docker can forward the port; docker-compose.yml publishes it on
+# the host's 127.0.0.1 only, and the engine still refuses a Host or Origin that is not localhost.
+CMD ["python", "engine/serve.py", "--host", "0.0.0.0", "--port", "8080"]
diff --git a/docker-compose.yml b/docker-compose.yml
index a52a56d..cfb88db 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,48 +1,24 @@
-# Runs the kit's engine, the open-source Google Maps scraper (gosom/google-maps-scraper), locally.
+# Runs the kit's own Google Maps engine (engine/serve.py with Debian's Chromium, see Dockerfile).
# Serves the web UI and API on http://localhost:8080. Host port: SCRAPER_PORT in .env, default 8080.
-# The image has no built-in auth, so the port is bound to 127.0.0.1 only. That localhost binding is
-# the security boundary. Do not change the mapping to 0.0.0.0 or a public interface without an auth
-# proxy in front.
+# The port is published on 127.0.0.1 only, so other machines cannot reach it: that localhost binding
+# is the security boundary. Inside the container the engine listens on 0.0.0.0 so Docker can forward
+# to it, and refuses any request whose Host or Origin is not localhost. Do not change the mapping to
+# 0.0.0.0 or a public interface.
+# No Docker? `python3 scripts/engine.py start` runs the same engine directly on this machine.
services:
google-maps-scraper:
- # Engine image, pinned to a known-good release so runs are reproducible. Bump it deliberately.
- # Bumped from v1.15.0 on 2026-09-23: that build's embedded Playwright driver (1.57.0)
- # returns 404 from Microsoft's CDN, which purges old driver builds. v1.18.1 ships a driver
- # version that is still hosted.
- # v1.18.1 publishes both amd64 and arm64, so Apple Silicon (M1/M2/M3) runs it natively.
- # Only older tags are amd64 and need the 'platform' line below. Default: commented out.
- image: gosom/google-maps-scraper:v1.18.1
- # platform: linux/amd64
- # SCRAPER_CONTAINER in .env renames both containers, for a second copy of the kit on one machine.
- # The scripts read the same variable, so the stuck-job recovery restarts the right engine.
- container_name: ${SCRAPER_CONTAINER:-gmaps-scraper}
+ build: .
restart: unless-stopped
- # -c: parallel workers. The engine's own default resolves to 1 in this image, so a depth 5 job
- # with emails visits every listing and website one at a time (~8 min). Most of that is business
- # websites, not Google. Lower it if Google starts throttling you; raise it with proxies.
- command: ["-web", "-data-folder", "/gmapsdata", "-c", "4"]
- # No published port: the engine is reached only through the ui service below.
- volumes:
- # Only job results. Do not mount a named volume over /opt: the image already ships Chromium at
- # /opt/browsers and the Playwright driver at /opt/ms-playwright-go, so a volume there shadows
- # both, and after an image bump it keeps serving the OLD browsers from the volume — which is
- # how the driver-mismatch failures mentioned above come back. Restarts download nothing
- # either way: the browsers are baked into the image.
- - gmaps_data:/gmapsdata
-
- # Serves the engine's UI and API with the kit's own GitHub and sponsor links (ui-branding.conf).
- ui:
- image: nginx:1.27-alpine
- container_name: ${SCRAPER_CONTAINER:-gmaps-scraper}-ui
- restart: unless-stopped
- depends_on:
- - google-maps-scraper
+ init: true # reaps Chromium's helper processes, which the engine would leave as zombies as PID 1
+ environment:
+ # Blank by default. When set in .env, the engine wants it as an X-API-Key header.
+ SCRAPER_API_KEY: ${SCRAPER_API_KEY:-}
ports:
# Host port comes from SCRAPER_PORT in .env (default 8080). The container side stays 8080.
- "127.0.0.1:${SCRAPER_PORT:-8080}:8080" # localhost only, see the warning above
volumes:
- - ./ui-branding.conf:/etc/nginx/conf.d/default.conf:ro
+ - engine_data:/kit/.engine/data # jobs and results, kept across restarts and rebuilds
volumes:
- gmaps_data:
+ engine_data:
diff --git a/engine/serve.py b/engine/serve.py
index 99077e5..4faed9d 100644
--- a/engine/serve.py
+++ b/engine/serve.py
@@ -4,10 +4,11 @@
python3 engine/serve.py # port from SCRAPER_PORT (environment, then .env), default 8080
python3 engine/serve.py --port 8190
-Normally started by `python3 scripts/engine.py start`. It speaks the API of the engine the kit used
+Normally started by `python3 scripts/engine.py start`, or by `docker compose up -d` (the Dockerfile). It speaks the API of the engine the kit used
before (POST/GET/DELETE /api/v1/jobs, /api/v1/jobs/{id}/download, the same job body and "Status"
values), so scripts/scrape.py, the skill and the slash commands work unchanged. It listens on
-127.0.0.1 (and ::1) only and refuses a foreign Host (DNS rebinding, 421) or Origin (cross-site
+127.0.0.1 (and ::1) only, or on 0.0.0.0 inside the Docker image, where Docker publishes the port on
+the host's 127.0.0.1 only. Either way it refuses a foreign Host (DNS rebinding, 421) or Origin (cross-site
posts, 403); with SCRAPER_API_KEY set it also wants that X-API-Key (401). No telemetry: it talks to
Google Maps, and for email: true jobs to the businesses' own websites, nothing else.
Standard library only.
@@ -217,7 +218,7 @@ class Handler(BaseHTTPRequestHandler):
self.reply(code, {"code": code, "message": message})
def guard(self):
- """The checks ui-branding.conf made in front of the old engine, plus the optional key."""
+ """Host (DNS rebinding) and Origin (cross-site posts) checks, plus the optional key."""
if not HOST_OK.match(self.headers.get("Host", "")):
self.close_connection = True
self.fail(421, "this engine answers to localhost only")
@@ -341,16 +342,22 @@ class Server6(Server):
address_family = socket.AF_INET6
-def make_servers(port, store, worker, api_key=""):
- """The server on 127.0.0.1:port, plus one on [::1]:port when this machine has IPv6 loopback
- (Windows tries ::1 first for "localhost" and waits ~2 s before falling back to 127.0.0.1)."""
+def in_container():
+ """True inside a Docker or Podman container (each leaves its marker file at the root)."""
+ return os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv")
+
+
+def make_servers(port, store, worker, api_key="", host="127.0.0.1"):
+ """The server on host:port. On 127.0.0.1 also one on [::1]:port when this machine has IPv6
+ loopback (Windows tries ::1 first for "localhost" and waits ~2 s before falling back)."""
if port and port_taken(port):
raise OSError(f"something already listens on port {port}")
- servers = [Server(("127.0.0.1", port), Handler)]
- try:
- servers.append(Server6(("::1", servers[0].server_address[1]), Handler))
- except OSError:
- pass
+ servers = [Server((host, port), Handler)]
+ if host == "127.0.0.1":
+ try:
+ servers.append(Server6(("::1", servers[0].server_address[1]), Handler))
+ except OSError:
+ pass
for s in servers:
s.store, s.worker, s.api_key = store, worker, api_key
return servers
@@ -360,7 +367,15 @@ def main():
ap = argparse.ArgumentParser(description="The kit's Google Maps engine (local HTTP API and web UI).")
ap.add_argument("--port", type=int, help="default: SCRAPER_PORT from the environment or .env, else 8080")
ap.add_argument("--data", default=os.path.join(browser.HOME, "data"), help="jobs and results folder")
+ ap.add_argument("--host", default="127.0.0.1", choices=("127.0.0.1", "0.0.0.0"),
+ help="0.0.0.0 is for the Docker image only, where Docker publishes the port on the "
+ "host's 127.0.0.1 (default: 127.0.0.1)")
a = ap.parse_args()
+ if a.host != "127.0.0.1" and not in_container():
+ # On a laptop or server 0.0.0.0 opens the engine to the network: the Host check stops DNS
+ # rebinding from a browser, not a machine that sends "Host: localhost" itself.
+ sys.exit("✗ --host 0.0.0.0 is for the Docker image only. On this machine the engine listens on "
+ "127.0.0.1, which keeps other machines out.")
env = read_env()
try:
port = a.port or int(env["SCRAPER_PORT"] or 8080)
@@ -373,12 +388,14 @@ def main():
log(f"{n} job(s) were still working when the engine last stopped: marked failed, rows kept")
worker = jobs.Worker(store, log)
try:
- servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"])
+ servers = make_servers(port, store, worker, env["SCRAPER_API_KEY"], a.host)
except OSError as e:
- sys.exit(f"✗ Cannot listen on 127.0.0.1:{port}: {e}")
+ sys.exit(f"✗ Cannot listen on {a.host}:{port}: {e}")
worker.start()
chrome = browser.find_chrome()
- log(f"engine {VERSION} on http://localhost:{port} (browser: {chrome or 'none found'}"
+ where = f"http://localhost:{port}" if a.host == "127.0.0.1" else \
+ f"{a.host}:{port} in this container, published on the host's 127.0.0.1"
+ log(f"engine {VERSION} on {where} (browser: {chrome or 'none found'}"
f"{', API key required' if env['SCRAPER_API_KEY'] else ''})")
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
for s in servers[1:]:
diff --git a/engine/test_engine.py b/engine/test_engine.py
index 9da4120..38ff9b0 100644
--- a/engine/test_engine.py
+++ b/engine/test_engine.py
@@ -7,7 +7,7 @@ No network, no Chrome, no Docker.
python3 engine/test_engine.py
"""
import csv, hashlib, http.client, importlib.util, io, json, os, re, shutil, socket, subprocess, sys, tempfile
-import threading, time, unittest, zipfile
+import threading, time, unittest, unittest.mock, zipfile
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
@@ -409,6 +409,24 @@ class ApiKey(Server):
self.assertEqual(self.call("GET", "/")[0], 200) # the page holds no data; it asks for the key
+class Bind(Fakes):
+ """--host 0.0.0.0 is for the Docker image; elsewhere the engine keeps to loopback."""
+
+ def test_any_address_has_no_ipv6_twin(self):
+ # Mocked: a real 0.0.0.0 bind would open a port to the network on the machine running the tests.
+ with unittest.mock.patch.object(serve, "Server") as v4, unittest.mock.patch.object(serve, "Server6") as v6:
+ serve.make_servers(0, self.store, self.worker, host="0.0.0.0")
+ v4.assert_called_once_with(("0.0.0.0", 0), serve.Handler)
+ v6.assert_not_called()
+
+ def test_any_address_refused_outside_a_container(self):
+ argv = ["serve.py", "--host", "0.0.0.0", "--port", "0", "--data", self.dir]
+ with unittest.mock.patch.object(serve, "in_container", return_value=False), \
+ unittest.mock.patch.object(sys, "argv", argv), self.assertRaises(SystemExit) as cm:
+ serve.main()
+ self.assertIn("Docker image only", str(cm.exception))
+
+
class ScrapeClient(Server):
"""scripts/scrape.py's own create, poll and download against this engine, with the worker running."""
@@ -420,7 +438,6 @@ class ScrapeClient(Server):
import scrape
scrape.BASE = os.environ["SCRAPER_BASE_URL"]
scrape.POLL_EVERY = 1
- scrape._local_docker = lambda: False # never touch a real Docker engine from a test
self.worker.start()
out = io.StringIO()
saved, sys.stdout = sys.stdout, out
diff --git a/scripts/engine.py b/scripts/engine.py
index 00d5bf8..ed987db 100644
--- a/scripts/engine.py
+++ b/scripts/engine.py
@@ -7,7 +7,7 @@
python3 scripts/engine.py stop
Standard library only. The engine (engine/serve.py) serves http://localhost:8080 (SCRAPER_PORT in
-.env) with the same API the scripts, slash commands and web UI use with Docker. It drives an
+.env), the same engine and API that `docker compose up -d` runs in a container. It drives an
installed Chrome, Chromium, Edge or Brave; on a machine with none, `start` downloads Chrome for
Testing's headless shell, pinned by version and SHA-256. Everything lives in .engine/ in the kit
folder; delete it to remove the engine and its results.
@@ -47,7 +47,8 @@ def public_port():
def probe(port, timeout=2):
- """"ours" (this kit's engine answers), "other" (something else, e.g. the Docker engine) or None."""
+ """"ours" (this kit's engine answers, here or in Docker), "other" (something else, such as the
+ kit's 1.x Docker engine) or None."""
try:
with _NO_PROXY.open(f"http://127.0.0.1:{port}/api/v1/jobs", timeout=timeout) as r:
return "ours" if (r.headers.get("X-Engine") or "").startswith(MARK) else "other"
@@ -161,7 +162,7 @@ def trim_log():
def busy(port):
hint = f"netstat -ano | findstr :{port}" if os.name == "nt" else f"lsof -nP -iTCP:{port} -sTCP:LISTEN"
- sys.exit(f"✗ Port {port} is in use. Is the Docker engine running? Stop it with `docker compose stop`,\n"
+ sys.exit(f"✗ Port {port} is in use. Is the kit's older Docker engine running? Stop it with `docker compose stop`,\n"
f" or set SCRAPER_PORT in .env to a free port. To see what holds it: {hint}")
@@ -241,7 +242,12 @@ def stop(quiet=False):
st = read_state()
if not st:
if not quiet:
- print("The engine is not running.")
+ port = public_port()
+ if probe(port) == "ours": # started by docker compose, not by this script
+ print(f"ℹ The engine on port {port} was not started by engine.py; it runs in Docker, probably.\n"
+ " Stop that one with: docker compose stop")
+ else:
+ print("The engine is not running.")
return
if st.get("pid") and not kill(st["pid"]):
how = "taskkill /PID {} /T /F" if os.name == "nt" else "kill {}"
@@ -262,7 +268,7 @@ def status():
st = read_state() or {}
print(f"✓ Running on http://localhost:{port}" + (f" (browser: {st['chrome']})" if st.get("chrome") else ""))
elif seen == "other":
- print(f"ℹ Port {port} answers, but not as this kit's engine (the Docker engine, probably).")
+ print(f"ℹ Port {port} answers, but not as this kit's engine (the kit's older Docker engine, probably).")
sys.exit(1)
else:
print(f"✗ Not running. Start it with: {PY} scripts/engine.py start")
diff --git a/scripts/scrape.py b/scripts/scrape.py
index dbb01f2..dc7f77d 100755
--- a/scripts/scrape.py
+++ b/scripts/scrape.py
@@ -23,7 +23,7 @@ Notes:
- Geocoding uses OpenStreetMap Nominatim (free, no key). It allows ~1 request/sec and requires a
descriptive User-Agent (set below). Do not loop it aggressively.
"""
-import argparse, csv, functools, io, ipaddress, json, math, os, re, shutil, socket, subprocess, sys, time
+import argparse, csv, functools, io, ipaddress, json, math, os, re, socket, sys, time
import html as _html, http.client, urllib.request, urllib.parse, urllib.error
from concurrent.futures import ThreadPoolExecutor
@@ -37,7 +37,7 @@ for _stream in (sys.stdout, sys.stderr):
except (AttributeError, ValueError): # not a reconfigurable stream (pytest capture, a pipe wrapper)
pass
-ENV_KEYS = ("SCRAPER_BASE_URL", "SCRAPER_PORT", "SCRAPER_API_KEY", "SCRAPER_CONTAINER")
+ENV_KEYS = ("SCRAPER_BASE_URL", "SCRAPER_PORT", "SCRAPER_API_KEY")
def load_env_file():
@@ -74,24 +74,25 @@ UA = "google-maps-scraper-kit/1.6 (https://github.com/Mahanaicoach/google-maps-s
# different small sites, not Google, so this does not raise the rate-limit risk.
SITE_WORKERS = 16
-POLL_EVERY = 8 # seconds between status checks
-POLL_GRACE = 120 # the engine still has to flush results and flip the status
+POLL_EVERY = 3 # seconds between status checks (the engine is local, a check is cheap)
+POLL_GRACE = 120 # past max_time: the engine's watchdog stops Chrome 90 s after it at the latest
POLL_CEILING = 7200 # never wait longer than 2h, whatever --max-time asked for
-ENGINE_MIN_RUNTIME = 180 # the engine floors anything below this (runner/webrunner)
-# container_name in docker-compose.yml and SETUP's docker run; the one the script reads the log of
-# and restarts. SCRAPER_CONTAINER in .env for a renamed container or a second stack.
-ENGINE_CONTAINER = os.environ.get("SCRAPER_CONTAINER") or "gmaps-scraper"
+PY = "py" if os.name == "nt" else "python3" # what the user types; the README says `py` on Windows
+RESTART = f"{PY} scripts/engine.py restart (Docker: docker compose restart)"
+# Messages of a job whose Chrome never started (engine/browser.py). Retrying fails the same way, and
+# Google has nothing to do with it.
+CHROME_START = ("No Chrome", "Chrome cannot start", "Chrome exited at once", "Could not start Chrome",
+ "Chrome did not open its DevTools port", "Could not connect to Chrome's DevTools port")
+BLOCKED = "unusual-traffic page" # the engine's Error when Google refused this IP
def poll_attempts(max_time):
"""How many status checks to make before giving up on a job.
Derived from the job's own max_time, because that is how long the engine is allowed to
- work: waiting any less abandons a job that is still running normally. The engine floors
- max_time at ENGINE_MIN_RUNTIME, so even a tiny --max-time needs that much patience, and
- POLL_CEILING stops an absurd value from parking the script for a day."""
- engine_runtime = max(ENGINE_MIN_RUNTIME, int(max_time or 0))
- budget = min(POLL_CEILING, engine_runtime + POLL_GRACE)
+ work: waiting any less abandons a job that is still running normally. POLL_CEILING stops an
+ absurd value from parking the script for a day."""
+ budget = min(POLL_CEILING, max(1, int(max_time or 0)) + POLL_GRACE)
return max(1, budget // POLL_EVERY)
@@ -189,93 +190,7 @@ def load_rows(path):
return rows
-def _local_docker():
- return urllib.parse.urlparse(BASE).hostname in ("localhost", "127.0.0.1", "::1") \
- and shutil.which("docker") is not None
-
-def engine_run_ended(since):
- """True if the local engine's log shows a scraper run ending after `since` (UTC, ISO 8601).
- The scripts run one job at a time, so that is this job's run."""
- if not _local_docker():
- return False
- try:
- out = subprocess.run(["docker", "logs", "--since", since, ENGINE_CONTAINER],
- capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)
- except (OSError, subprocess.SubprocessError):
- return False
- return out.returncode == 0 and "scrapemate exited" in out.stdout + out.stderr
-
-def engine_error(job_id):
- """The engine's own reason for a failed job, from the local log: "error scraping job <id>: ...".
- A proxy typo, for example, otherwise surfaces only as a generic "failed"."""
- if not _local_docker():
- return ""
- try:
- out = subprocess.run(["docker", "logs", "--since", "15m", ENGINE_CONTAINER],
- capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)
- except (OSError, subprocess.SubprocessError):
- return ""
- if out.returncode != 0:
- return ""
- for line in reversed((out.stdout + out.stderr).splitlines()):
- if f"error scraping job {job_id}" in line:
- return line.split(f"job {job_id}:", 1)[-1].strip()[:200]
- return ""
-
-def restart_engine():
- """Restart the local engine and wait until its API answers again. Without the wait, the next
- grid tile hit the proxy while the engine was still starting and failed with HTTP 502."""
- if not _local_docker():
- return False
- try:
- if subprocess.run(["docker", "restart", ENGINE_CONTAINER],
- capture_output=True, timeout=90).returncode != 0:
- return False
- except (OSError, subprocess.SubprocessError):
- return False
- for _ in range(30): # up to a minute; it usually answers within a few seconds
- try:
- req("GET", "/api/v1/jobs")
- return True
- except Exception:
- time.sleep(2)
- return False
-
-def recover_stuck_job(job_id):
- """Engine v1.18.x can finish a job, rows written, without marking it done, and then holds every
- later job on "pending" until it restarts (gosom/google-maps-scraper#346). Keep the rows and
- restart the local engine so the next scrape runs."""
- rows = download_rows(job_id)
- print(f"\n⚠ The engine finished this job but did not mark it done (gosom/google-maps-scraper#346); "
- f"keeping its {len(rows)} rows.")
- try: # otherwise it sits in the job list as "working" forever, with no Download button in the web UI
- req("DELETE", f"/api/v1/jobs/{job_id}")
- print(" Deleted the stuck job from the engine's list; its rows are in your file.")
- except Exception:
- print(f" Remove it from the job list yourself: curl -X DELETE {BASE}/api/v1/jobs/{job_id}")
- if restart_engine():
- print(" Restarted the engine, which would otherwise hold every later job on \"pending\".")
- else:
- print(" If the next scrape stays \"pending\", run: docker restart " + ENGINE_CONTAINER)
- return rows
-
-PENDING_RESTART_AFTER = 6 # polls (~48 s) a new job may sit on "pending" before an idle engine is judged stuck
-FAILED_RETRY_WAIT = 60 # seconds before the one retry of a "failed" job (usually a passing Google block)
-
-
-def engine_idle(seconds=45):
- """True if the local engine logged no scraper activity in the last `seconds`. A job that stays
- "pending" on an idle engine means an earlier job (web UI, raw API) left it wedged
- (gosom/google-maps-scraper#346); a busy engine is just finishing another job first."""
- if not _local_docker():
- return False
- try:
- out = subprocess.run(["docker", "logs", "--since", f"{seconds}s", ENGINE_CONTAINER],
- capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=15)
- except (OSError, subprocess.SubprocessError):
- return False
- # A failing `docker logs` (wrong container name, daemon down) must not read as "idle".
- return out.returncode == 0 and "scrapemate" not in out.stdout + out.stderr
+FAILED_RETRY_WAIT = 60 # seconds before the one retry of a "failed" job (usually a passing error)
class JobFailed(RuntimeError):
@@ -284,20 +199,28 @@ class JobFailed(RuntimeError):
def run_job(body):
"""Create one engine job, poll it, download it -> (job_id, raw CSV rows). A job the engine marks
- "failed" (usually a passing Google block) is retried once after a pause. Raises RuntimeError."""
+ "failed" is retried once after a pause, unless its Error says a retry cannot help (Chrome would
+ not start, the proxy failed, Google refused this IP). Raises RuntimeError."""
for attempt in (1, 2):
try:
return _run_job_once(body)
except JobFailed as e:
- reason = e.args[1] if len(e.args) > 1 else engine_error(e.args[0])
- said = f" The engine says: {reason}." if reason else ""
- if "proxy" in reason.lower(): # a bad proxy fails the same way every time: do not retry
+ reason = e.args[1].strip().rstrip(".")
+ said = f" The engine says: {reason}." if reason else " The engine gave no reason."
+ if reason.startswith(CHROME_START):
+ raise RuntimeError(f"\n✗ Job failed: the engine could not start Chrome.{said}\n"
+ f" Fix that and scrape again. If it fails the same way, restart the "
+ f"engine: {RESTART}")
+ if any(k in reason.upper() for k in ("PROXY", "SOCKS", "TUNNEL")):
raise RuntimeError(f"\n✗ Job failed: the proxy did not work.{said}\n"
- " This engine needs user:pass for socks5 proxies; an http:// proxy works without.")
+ " Check its host and port; this engine takes no user:pass yet, so the "
+ "proxy must allow your IP address.")
+ if BLOCKED in reason: # more searches from a rate-limited IP only extend the block
+ raise RuntimeError(f"\n✗ Job failed.{said}")
if attempt == 2:
- raise RuntimeError(f"\n✗ Job failed twice.{said} You may be rate-limited — wait or add proxies.")
- print(f"\n⚠ Job failed.{said or ' Google may be throttling this IP.'} "
- f"Waiting {FAILED_RETRY_WAIT} s, then retrying once…")
+ raise RuntimeError(f"\n✗ Job failed twice.{said}\n The engine's log has more: "
+ ".engine/engine.log (Docker: docker compose logs)")
+ print(f"\n⚠ Job failed.{said} Waiting {FAILED_RETRY_WAIT} s, then retrying once…")
time.sleep(FAILED_RETRY_WAIT)
@@ -315,8 +238,7 @@ def _get(path):
time.sleep(3)
-def _run_job_once(body, restarted=False):
- since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+def _run_job_once(body):
try:
_, raw = req("POST", "/api/v1/jobs", body)
except urllib.error.HTTPError as e:
@@ -328,73 +250,58 @@ def _run_job_once(body, restarted=False):
attempts = poll_attempts(body.get("max_time"))
print(f"▶ Polling (up to ~{attempts * POLL_EVERY // 60} min)…")
- status, ended, pending, n_rows, last_rows = None, 0, 0, None, None
+ status = None
for i in range(attempts):
_, raw = _get(f"/api/v1/jobs/{job_id}")
- status = json.loads(raw).get("Status")
+ job = json.loads(raw)
+ # Error: why the job failed, or why an "ok" job stopped early (max_time, a Google block).
+ status, error = job.get("Status"), str(job.get("Error") or "")
progress = ""
if status == "working": # the engine writes rows as it goes, so the download shows progress
try:
- n_rows = len(download_rows(job_id))
- progress = f", {n_rows} rows so far"
+ progress = f", {len(download_rows(job_id))} rows so far"
except Exception:
pass
print(f"\r status: {str(status):<10} (attempt {i + 1}{progress})", end="", flush=True)
if status == "ok":
- print(); break
+ print()
+ if error:
+ print(f"ℹ The engine says: {error}")
+ return job_id, download_rows(job_id)
if status == "failed":
- raise JobFailed(job_id)
- if status == "working" and not n_rows:
- # A job the engine could not even start (a bad proxy, a driver error) stays "working"
- # forever with no rows, and holds every later job on "pending". Read the engine's reason,
- # drop the dead job, restart the engine and report.
- reason = engine_error(job_id)
- if reason:
- try:
- req("DELETE", f"/api/v1/jobs/{job_id}")
- except Exception:
- pass
- restart_engine()
- raise JobFailed(job_id, reason)
- # Stuck (gosom#346): the engine log says the run ended and the row count has stopped moving.
- # The second condition guards against a Docker VM clock that runs ahead of the host.
- if status == "working" and n_rows == last_rows and engine_run_ended(since):
- ended += 1
- if ended > 2: # the status normally flips within seconds of the run ending
- return job_id, recover_stuck_job(job_id)
- last_rows = n_rows
- if status == "pending":
- pending += 1
- if pending % PENDING_RESTART_AFTER == 0 and not restarted and engine_idle():
- # Delete the pending job first, or the restarted engine runs it as well as the new one.
- print("\n⚠ The engine has not started this job and shows no activity: an earlier job left it "
- "stuck (gosom/google-maps-scraper#346). Restarting the engine and re-creating the job…")
- try:
- req("DELETE", f"/api/v1/jobs/{job_id}")
- except Exception:
- pass
- if not restart_engine():
- raise RuntimeError(f"\n✗ The engine never started this job and could not be restarted. "
- f"Run: docker restart {ENGINE_CONTAINER}")
- return _run_job_once(body, restarted=True)
+ print()
+ try: # the engine keeps the rows of a job it was stopped in the middle of
+ rows = download_rows(job_id)
+ except Exception:
+ rows = []
+ if rows:
+ print(f"⚠ The job failed part way ({error or 'no reason given'}); keeping its {len(rows)} rows.")
+ return job_id, rows
+ raise JobFailed(job_id, error)
time.sleep(POLL_EVERY)
- else:
- # The engine writes rows as it goes, and v1.18.x can leave a finished job stuck on
- # "working" (gosom/google-maps-scraper#346), so take whatever is on disk.
- rows = download_rows(job_id)
- if not rows:
- raise RuntimeError(
- f"\n✗ Gave up waiting after ~{attempts * POLL_EVERY // 60} min with no rows yet. The "
- f"engine keeps the job, so check it later:\n"
- f" curl {BASE}/api/v1/jobs/{job_id}\n"
- f" and once its Status is \"ok\", download the CSV with\n"
- f" curl -o results.csv {BASE}/api/v1/jobs/{job_id}/download")
- print(f"\n⚠ Job still \"{status}\" after ~{attempts * POLL_EVERY // 60} min; "
- f"keeping the {len(rows)} rows it has written. If the next scrape stays \"pending\", "
- f"restart the engine: docker restart {ENGINE_CONTAINER}")
- return job_id, rows
-
- return job_id, download_rows(job_id)
+ # Still pending or working: the engine may have run an earlier job first. It keeps this one, so
+ # take whatever it has written.
+ rows = download_rows_or_none(job_id)
+ if not rows:
+ raise RuntimeError(
+ f"\n✗ Gave up waiting after ~{attempts * POLL_EVERY // 60} min with no rows yet (status "
+ f"\"{status}\"). The engine keeps the job, so check it later:\n"
+ f" curl {BASE}/api/v1/jobs/{job_id}\n"
+ f" and once its Status is \"ok\", download the CSV with\n"
+ f" curl -o results.csv {BASE}/api/v1/jobs/{job_id}/download")
+ print(f"\n⚠ Job still \"{status}\" after ~{attempts * POLL_EVERY // 60} min; keeping the {len(rows)} "
+ f"rows it has written. It keeps running in the engine: {BASE}/api/v1/jobs/{job_id}")
+ return job_id, rows
+
+
+def download_rows_or_none(job_id):
+ """The rows so far, or None when the engine has written none (404 until its first keyword)."""
+ try:
+ return download_rows(job_id)
+ except urllib.error.HTTPError as e:
+ if e.code == 404:
+ return None
+ raise
def download_rows(job_id):
@@ -808,7 +715,7 @@ def scrape_place(a, body, city, keywords):
job_id = job_id or jid
rows = dedupe(rows + cell_rows)
print(f" cell {i}/{len(cells)}: {len(cell_rows)} rows, {len(rows)} unique so far")
- # radius does not clip results in normal mode, so tiles leak outside the city
+ # the engine does not clip results to the tile, so tiles leak outside the city
kept = [r for r in rows if in_bbox(r, bbox)]
print(f" dropped {len(rows) - len(kept)} rows outside the city box")
return job_id, kept, skipped, len(cells)
@@ -824,8 +731,8 @@ def main():
ap.add_argument("--city", action="append",
help="city/place to auto-geocode for lat/lon; repeatable, one run per city into one file")
ap.add_argument("--proxy", action="append", metavar="URL",
- help="proxy for the engine, repeatable: http://, https://, socks5:// or socks5h://; "
- "this engine needs user:pass@ for socks5; the engine rotates them")
+ help="proxy for the engine: http://, https://, socks5:// or socks5h://host:port, "
+ "without user:pass (not supported yet); repeatable, the engine uses the first")
ap.add_argument("--depth", type=int, default=5)
# The engine passes this to Google as the interface language, which decides the language of
# the category names it returns (and nudges which listings come back). It has to match the
@@ -863,7 +770,9 @@ def main():
"(google_workspace, microsoft_365, other, none)")
f = ap.add_argument_group("filters", "applied before the email crawl, except --has-email")
f.add_argument("--min-rating", type=float, metavar="X", help="keep rows rated X or higher")
- f.add_argument("--min-reviews", type=int, metavar="N", help="keep rows with N or more reviews")
+ f.add_argument("--min-reviews", type=int, metavar="N",
+ help="keep rows with N or more reviews (ignored while review_count is empty, as with "
+ "this engine)")
f.add_argument("--no-website", action="store_true", help="keep only businesses without a website")
f.add_argument("--has-phone", action="store_true", help="keep only rows with a phone number")
f.add_argument("--has-email", action="store_true", help="keep only rows where an email was found")
@@ -881,7 +790,7 @@ def main():
if a.depth < 1:
sys.exit("✗ --depth must be 1 or more.")
if a.max_time < 1:
- sys.exit("✗ --max-time must be 1 or more seconds (the engine runs a job for at least 180).")
+ sys.exit("✗ --max-time must be 1 or more seconds.")
if a.limit is not None and a.limit < 1:
sys.exit("✗ --limit must be 1 or more.")
if a.has_email and not a.email:
@@ -913,9 +822,9 @@ def main():
print(" Google (clears in minutes–hours; returns empty/failed jobs meanwhile). For big or")
print(" repeated runs, add proxies. Proceeding…\n")
- # Health check. Right after `docker compose up -d` the proxy answers 502 while the engine starts
- # (up to a minute), and the README's next line runs at once, so wait for that. Nothing listening at
- # all (connection refused) means the stack is down: say so after 10 s, not after a minute of dots.
+ # Health check. Right after `docker compose up -d` Docker accepts connections and drops them while
+ # the engine starts (a few seconds), and the README's next line runs at once, so wait for that.
+ # Nothing listening at all (connection refused) means no engine: say so after 10 s, not a minute.
refused, err = 0, None
for i in range(30):
try:
@@ -934,16 +843,16 @@ def main():
print("\r waiting for the engine to start" + "." * (i + 1), end="", flush=True)
time.sleep(2)
if err is not None:
- sys.exit(f"\n✗ Scraper not reachable at {BASE}. Is Docker Desktop open? Start the engine with "
- f"'docker compose up -d' in the kit folder, wait 10 seconds, then try again.\n ({err})")
+ sys.exit(f"\n✗ Scraper not reachable at {BASE}. Start the engine in the kit folder with "
+ f"'{PY} scripts/engine.py start' (or 'docker compose up -d' with Docker Desktop open), "
+ f"then try again.\n ({err})")
if i:
print()
body = {"keywords": keywords, "lang": a.lang, "zoom": 15,
"fast_mode": False, "radius": 10000,
- # The engine's own email pass visits every website in a browser, one per worker: it took
- # most of each job and found fewer emails than the crawl below (OpenLeadFinder's
- # approach), so the engine only reads Google Maps.
+ # The engine's own email pass reads homepages only; the crawl below also reads contact and
+ # about pages and decodes obfuscated addresses, so the engine only reads Google Maps.
"depth": a.depth, "email": False, "max_time": a.max_time}
if a.proxy:
body["proxies"] = a.proxy
@@ -959,6 +868,12 @@ def main():
print(f"⚠ Incomplete: {failed_places} of {len(cities)} places returned nothing (see the ✗ lines above).")
for r in rows:
r["website"] = clean_website(r.get("website", ""))
+ if a.min_reviews and rows and not any(r.get("review_count") for r in rows):
+ # The engine leaves review_count empty: Google shows a signed-out headless browser no review
+ # counts. Filtering on it would read every count as 0 and drop every row.
+ print("⚠ --min-reviews ignored: this engine leaves review_count empty (Google shows it no review "
+ "counts), so the filter would drop every row. --min-rating works.")
+ a.min_reviews = None
before = len(rows)
rows = [r for r in dedupe(rows, exclude=excluded) if keep_row(r, a)]
if len(rows) != before:
diff --git a/scripts/test_scrape.py b/scripts/test_scrape.py
index 70c4ab1..356b995 100644
--- a/scripts/test_scrape.py
+++ b/scripts/test_scrape.py
@@ -10,7 +10,6 @@ _spec = importlib.util.spec_from_file_location(
"scrape", os.path.join(os.path.dirname(os.path.abspath(__file__)), "scrape.py"))
scrape = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(scrape)
-scrape.ENGINE_CONTAINER = "kit-tests-no-such-container" # never read or restart a real engine
BBOX = ["30.0", "30.5", "-98.0", "-97.5"] # Nominatim order: south, north, west, east
@@ -71,172 +70,131 @@ class InBbox(unittest.TestCase):
class PollAttempts(unittest.TestCase):
def test_follows_max_time(self):
- # The budget tracks max_time instead of the old fixed 120 attempts (960s), which
- # gave up on a batch the engine was still working through.
- self.assertEqual(scrape.poll_attempts(600), (600 + 120) // 8) # 90
- self.assertEqual(scrape.poll_attempts(3600), (3600 + 120) // 8) # 465, was 120
- self.assertGreater(scrape.poll_attempts(3600), 120)
-
- def test_floors_at_engine_minimum(self):
- # The engine runs a job for at least 180s however small max_time is, so waiting
- # only max_time would abandon it.
- for tiny in (1, 60, 179, 180):
- self.assertEqual(scrape.poll_attempts(tiny), (180 + 120) // 8)
- self.assertEqual(scrape.poll_attempts(0), (180 + 120) // 8)
- self.assertEqual(scrape.poll_attempts(None), (180 + 120) // 8)
+ # The budget tracks max_time instead of a fixed number of attempts, which gave up on a
+ # batch the engine was still working through.
+ self.assertEqual(scrape.poll_attempts(600), (600 + 120) // scrape.POLL_EVERY)
+ self.assertEqual(scrape.poll_attempts(3600), (3600 + 120) // scrape.POLL_EVERY)
+ self.assertEqual(scrape.poll_attempts(1), (1 + 120) // scrape.POLL_EVERY) # no 180 s floor any more
+ self.assertEqual(scrape.poll_attempts(None), scrape.poll_attempts(1))
def test_ceiling_and_never_zero(self):
- self.assertEqual(scrape.poll_attempts(10 ** 9), 7200 // 8) # clamped at 2h
+ self.assertEqual(scrape.poll_attempts(10 ** 9), 7200 // scrape.POLL_EVERY) # clamped at 2h
self.assertGreaterEqual(scrape.poll_attempts(-5), 1)
-class StuckJob(unittest.TestCase):
- """A job the engine never flips to "ok" (gosom/google-maps-scraper#346) keeps its written rows."""
- def run_stuck(self, csv_body, run_ended=False, attempts=2):
- polls, deleted = [], []
- def fake_req(method, path, body=None):
- if method == "POST":
- return 201, b'{"id":"stuck"}'
- if method == "DELETE":
- deleted.append(path)
- return 200, b""
- if path.endswith("/download"):
- return 200, csv_body
- polls.append(path)
- return 200, b'{"Status":"working"}'
- with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \
- mock.patch.object(scrape, "poll_attempts", return_value=attempts), \
- mock.patch.object(scrape, "engine_run_ended", return_value=run_ended), \
- mock.patch.object(scrape, "restart_engine", return_value=True) as restart, \
- redirect_stdout(io.StringIO()):
- job_id, rows = scrape.run_job({"max_time": 1})
- return rows, len(polls), restart.called, deleted
-
- def test_keeps_written_rows(self):
- rows, _, restarted, deleted = self.run_stuck(b"title,phone\nA,1\nB,2\n")
- self.assertEqual([r["title"] for r in rows], ["A", "B"])
- self.assertFalse(restarted)
- self.assertEqual(deleted, []) # the budget ran out, but the job may still be running
-
- def test_no_rows_still_errors(self):
- with self.assertRaises(RuntimeError):
- self.run_stuck(b"")
-
- def test_ended_run_recovered_in_seconds_and_engine_restarted(self):
- # The engine log says the run ended and the row count has stopped moving for 3 polls: stop
- # after 4 polls (about 32 s) instead of the whole budget, keep the rows, restart the engine.
- rows, polls, restarted, deleted = self.run_stuck(b"title\nA\n", run_ended=True, attempts=90)
- self.assertEqual([r["title"] for r in rows], ["A"])
- self.assertEqual(polls, 4)
- self.assertTrue(restarted)
- self.assertEqual(deleted, ["/api/v1/jobs/stuck"]) # no longer a "working" ghost in the job list
-
- def test_remote_engine_is_never_restarted(self):
- with mock.patch.object(scrape, "BASE", "http://scraper.example:8080"):
- self.assertFalse(scrape.engine_run_ended("2026-01-01T00:00:00Z"))
- self.assertFalse(scrape.engine_idle())
- self.assertFalse(scrape.restart_engine())
-
-
-class PendingAndFailed(unittest.TestCase):
- def run_statuses(self, statuses, idle=True):
- """Drive run_job through a status sequence; count job creations and engine restarts."""
- posts, deleted, it = [], [], iter(statuses)
- def fake_req(method, path, body=None):
- if method == "POST":
- posts.append(body)
- return 201, b'{"id":"j%d"}' % len(posts)
- if method == "DELETE":
- deleted.append(path)
- return 200, b""
- if path.endswith("/download"):
- return 200, b"title\nA\n"
- return 200, json.dumps({"Status": next(it)}).encode()
- with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep") as slept, \
- mock.patch.object(scrape, "poll_attempts", return_value=50), \
- mock.patch.object(scrape, "engine_run_ended", return_value=False), \
- mock.patch.object(scrape, "engine_idle", return_value=idle), \
- mock.patch.object(scrape, "restart_engine", return_value=True) as restart, \
- redirect_stdout(io.StringIO()):
- job_id, rows = scrape.run_job({"max_time": 1})
- return job_id, rows, len(posts), restart.call_count, slept, deleted
-
- def test_engine_error_reads_the_failed_job_line(self):
- log = mock.Mock(returncode=0, stdout="x\n2026/10/30 22:38:59 error scraping job j9: failed to start auth proxy for socks5://h:1: username and password are required\n", stderr="")
- with mock.patch.object(scrape, "_local_docker", return_value=True), \
- mock.patch.object(scrape.subprocess, "run", return_value=log):
- self.assertTrue(scrape.engine_error("j9").startswith("failed to start auth proxy"))
- self.assertEqual(scrape.engine_error("other"), "")
-
- def test_pending_on_idle_engine_restarts_once(self):
- # An earlier web UI job wedged the engine: our job never leaves "pending". Delete it, restart,
- # create it again, and never loop on a second pending streak.
- statuses = ["pending"] * scrape.PENDING_RESTART_AFTER + ["pending"] * (scrape.PENDING_RESTART_AFTER + 2) + ["working", "ok"]
- job_id, rows, posts, restarts, _, deleted = self.run_statuses(statuses)
- self.assertEqual((job_id, posts, restarts, deleted), ("j2", 2, 1, ["/api/v1/jobs/j1"]))
- self.assertEqual([r["title"] for r in rows], ["A"])
-
- def test_pending_on_busy_engine_waits(self):
- # The engine is still running someone else's job: pending is normal, never restart.
- statuses = ["pending"] * (scrape.PENDING_RESTART_AFTER + 3) + ["ok"]
- _, _, posts, restarts, _, _ = self.run_statuses(statuses, idle=False)
- self.assertEqual((posts, restarts), (1, 0))
-
- def test_bad_proxy_fails_at_once_with_the_engine_reason(self):
- reason = "failed to start auth proxy for socks5://127.0.0.1:1: username and password are required"
- with mock.patch.object(scrape, "engine_error", return_value=reason), \
- self.assertRaises(RuntimeError) as cm:
- self.run_statuses(["failed"])
- self.assertIn("username and password", str(cm.exception))
- self.assertIn("user:pass", str(cm.exception))
-
- def test_dead_working_job_is_reported_from_the_engine_log(self):
- # The engine logged "error scraping job", left the job on "working" with no rows and never
- # flipped it: delete it, restart the engine, tell the user the engine's reason.
- reason = "failed to start auth proxy for socks5://h:1: username and password are required"
- deletes = []
- def fake_req(method, path, body=None):
- if method == "POST":
- return 201, b'{"id":"j1"}'
- if method == "DELETE":
- deletes.append(path)
- return 200, b""
- if path.endswith("/download"):
- return 200, b"title\n" # header only: zero rows
- return 200, b'{"Status":"working"}'
- with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \
- mock.patch.object(scrape, "poll_attempts", return_value=50), \
- mock.patch.object(scrape, "engine_run_ended", return_value=False), \
- mock.patch.object(scrape, "engine_error", return_value=reason), \
- mock.patch.object(scrape, "restart_engine", return_value=True) as restart, \
- redirect_stdout(io.StringIO()), self.assertRaises(RuntimeError) as cm:
- scrape.run_job({"max_time": 1})
- self.assertIn("username and password", str(cm.exception))
- self.assertEqual((deletes, restart.call_count), (["/api/v1/jobs/j1"], 1))
+def run_jobs(jobs, attempts=50, downloads=None):
+ """Drive scrape.run_job against a fake engine. `jobs` holds one list of (Status, Error) polls per
+ job the script creates; `downloads` maps a job id to its CSV (404 when missing).
+ -> (result or the RuntimeError, job count, the sleeps, the printed text)."""
+ posts, it, out = [], None, io.StringIO()
+ downloads = downloads or {}
+ def fake_req(method, path, body=None):
+ nonlocal it
+ if method == "POST":
+ posts.append(body)
+ it = iter(jobs[len(posts) - 1])
+ return 201, b'{"id":"j%d"}' % len(posts)
+ jid = path.split("/")[4]
+ if path.endswith("/download"):
+ if jid not in downloads:
+ raise scrape.urllib.error.HTTPError(path, 404, "no results", {}, io.BytesIO(b""))
+ return 200, downloads[jid]
+ status, error = next(it)
+ return 200, json.dumps({"ID": jid, "Status": status, "Error": error}).encode()
+ with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep") as slept, \
+ mock.patch.object(scrape, "poll_attempts", return_value=attempts), redirect_stdout(out):
+ try:
+ result = scrape.run_job({"max_time": 1})
+ except RuntimeError as e:
+ result = e
+ return result, len(posts), slept, out.getvalue()
+
+
+class JobOutcomes(unittest.TestCase):
+ """What the script does with each Status and Error the engine reports."""
+ CSV = b"title,phone\nA,1\nB,2\n"
+
+ def test_ok(self):
+ (jid, rows), posts, _, text = run_jobs([[("pending", ""), ("working", ""), ("ok", "")]],
+ downloads={"j1": self.CSV})
+ self.assertEqual((jid, [r["title"] for r in rows], posts), ("j1", ["A", "B"], 1))
+ self.assertNotIn("engine says", text)
+
+ def test_ok_with_a_note_says_why_it_stopped_early(self):
+ note = "stopped at max_time; the rows found so far are kept"
+ (_, rows), _, _, text = run_jobs([[("ok", note)]], downloads={"j1": self.CSV})
+ self.assertEqual(len(rows), 2)
+ self.assertIn(f"The engine says: {note}", text)
def test_failed_job_is_retried_once(self):
- job_id, rows, posts, _, slept, _ = self.run_statuses(["failed", "working", "ok"])
- self.assertEqual((job_id, posts), ("j2", 2))
+ reason = "Page.navigate: no reply within 30 s"
+ (jid, rows), posts, slept, text = run_jobs([[("failed", reason)], [("working", ""), ("ok", "")]],
+ downloads={"j2": self.CSV})
+ self.assertEqual((jid, posts), ("j2", 2))
slept.assert_any_call(scrape.FAILED_RETRY_WAIT)
- self.assertEqual([r["title"] for r in rows], ["A"])
- with self.assertRaises(RuntimeError):
- self.run_statuses(["failed", "failed"])
-
- def test_restart_waits_until_engine_answers(self):
- # Right after `docker restart` the proxy answers 502 for a few seconds; the next grid tile
- # used to hit that and fail. restart_engine must only return once the API answers.
- answers = iter([OSError("502"), OSError("502"), (200, b"null")])
- def fake_req(method, path, body=None):
- a = next(answers)
- if isinstance(a, Exception):
- raise a
- return a
- done = mock.Mock(returncode=0)
- with mock.patch.object(scrape, "_local_docker", return_value=True), \
- mock.patch.object(scrape.subprocess, "run", return_value=done), \
- mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep") as slept:
- self.assertTrue(scrape.restart_engine())
- self.assertEqual(slept.call_count, 2)
+ self.assertIn(reason, text)
+ err, posts, _, _ = run_jobs([[("failed", reason)], [("failed", reason)]])
+ self.assertIsInstance(err, RuntimeError)
+ self.assertEqual(posts, 2)
+ self.assertIn("failed twice", str(err))
+ self.assertIn(reason, str(err))
+
+ def test_chrome_start_error_is_not_retried_or_blamed_on_google(self):
+ for reason in ("Chrome exited at once (code 1): /usr/bin/chromium: Trace/breakpoint trap",
+ "Chrome cannot start: libnss3.so: cannot open shared object file\n Install the libraries",
+ "No Chrome, Chromium, Edge or Brave found. Run `python3 scripts/engine.py start`"):
+ err, posts, slept, text = run_jobs([[("failed", reason)]])
+ self.assertIsInstance(err, RuntimeError, reason)
+ self.assertEqual(posts, 1)
+ slept.assert_not_called()
+ self.assertIn("could not start Chrome", str(err))
+ self.assertIn("scripts/engine.py restart", str(err))
+ self.assertNotIn("throttl", (str(err) + text).lower())
+ self.assertNotIn("rate-limit", (str(err) + text).lower())
+
+ def test_proxy_error_is_not_retried(self):
+ reason = "1 keyword(s) failed: navigate https://www.google.com/maps/search/x: net::ERR_PROXY_CONNECTION_FAILED"
+ err, posts, _, _ = run_jobs([[("failed", reason)]])
+ self.assertEqual(posts, 1)
+ self.assertIn("proxy did not work", str(err))
+ self.assertIn("ERR_PROXY_CONNECTION_FAILED", str(err))
+
+ def test_google_block_is_not_retried(self):
+ reason = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
+ "Wait, or add proxies")
+ err, posts, slept, _ = run_jobs([[("failed", reason)]])
+ self.assertEqual(posts, 1)
+ slept.assert_not_called()
+ self.assertIn("unusual-traffic page", str(err))
+
+ def test_failed_part_way_keeps_its_rows(self):
+ # The engine was stopped while the job ran: it marks the job failed and keeps the rows.
+ reason = "the engine stopped while this job ran; its rows so far are kept"
+ (jid, rows), posts, _, text = run_jobs([[("working", ""), ("failed", reason)]],
+ downloads={"j1": self.CSV})
+ self.assertEqual((jid, len(rows), posts), ("j1", 2, 1))
+ self.assertIn(reason, text)
+
+ def test_budget_runs_out_keeps_written_rows(self):
+ (jid, rows), posts, _, text = run_jobs([[("working", "")] * 3], attempts=3, downloads={"j1": self.CSV})
+ self.assertEqual((jid, len(rows), posts), ("j1", 2, 1))
+ self.assertIn("keeps running in the engine", text)
+
+ def test_budget_runs_out_with_no_rows_is_a_clean_error(self):
+ for jobs in ([[("pending", "")] * 3], [[("working", "")] * 3]):
+ err, _, _, _ = run_jobs(jobs, attempts=3) # the download answers 404: no rows yet
+ self.assertIsInstance(err, RuntimeError)
+ self.assertIn("Gave up waiting", str(err))
+
+
+class NoDockerPaths(unittest.TestCase):
+ def test_script_never_reads_or_restarts_a_container(self):
+ # The old engine's workarounds read `docker logs` and ran `docker restart`, which could restart
+ # another Docker engine on this machine while the script talked to an engine on another port.
+ with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "scrape.py"), encoding="utf-8") as f:
+ src = f.read()
+ for banned in ("subprocess", "docker logs", "docker restart", '"docker"', "SCRAPER_CONTAINER"):
+ self.assertNotIn(banned, src)
class ExtractEmails(unittest.TestCase):
@@ -534,7 +492,7 @@ class NothingLeft(unittest.TestCase):
srv = ThreadingHTTPServer(("127.0.0.1", 0), MockEngine)
threading.Thread(target=srv.serve_forever, daemon=True).start()
out = os.path.join(tempfile.mkdtemp(), "leads.csv")
- argv = ["scrape.py", "plumbers in Austin TX", "30.2", "-97.7", "--no-email", "--min-reviews", "999", "--out", out]
+ argv = ["scrape.py", "plumbers in Austin TX", "30.2", "-97.7", "--no-email", "--min-rating", "5", "--out", out]
log = io.StringIO()
try:
with mock.patch.object(scrape, "BASE", f"http://127.0.0.1:{srv.server_port}"), \
@@ -545,6 +503,37 @@ class NothingLeft(unittest.TestCase):
self.assertIn("Nothing left: the filters and --exclude dropped all 3 rows", log.getvalue())
self.assertNotIn("✓ Done", log.getvalue())
+ def run_min_reviews(self, counts):
+ """--min-reviews 10 over the mock engine's three rows, with these review_count values."""
+ MockEngine.created, MockEngine.polls = 0, {}
+ saved = CELL_ROWS["job-1"]
+ CELL_ROWS["job-1"] = [dict(_row(p), review_count=c) for p, c in zip("ABC", counts)]
+ srv = ThreadingHTTPServer(("127.0.0.1", 0), MockEngine)
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ out = os.path.join(tempfile.mkdtemp(), "leads.csv")
+ argv = ["scrape.py", "plumbers in Austin TX", "30.2", "-97.7", "--no-email", "--min-reviews", "10", "--out", out]
+ log = io.StringIO()
+ try:
+ with mock.patch.object(scrape, "BASE", f"http://127.0.0.1:{srv.server_port}"), \
+ mock.patch.object(scrape.time, "sleep"), mock.patch.object(sys, "argv", argv), redirect_stdout(log):
+ scrape.main()
+ finally:
+ srv.shutdown(); srv.server_close()
+ CELL_ROWS["job-1"] = saved
+ with open(out, newline="", encoding="utf-8-sig") as f:
+ return [r["title"] for r in csv.DictReader(f)], log.getvalue()
+
+ def test_min_reviews_warns_instead_of_dropping_every_row(self):
+ # The engine leaves review_count empty, which would read as 0 and drop every row.
+ titles, text = self.run_min_reviews(["", "", ""])
+ self.assertEqual(titles, ["Biz A", "Biz B", "Biz C"])
+ self.assertIn("--min-reviews ignored", text)
+
+ def test_min_reviews_still_filters_real_counts(self):
+ titles, text = self.run_min_reviews(["12", "3", "1,204"])
+ self.assertEqual(titles, ["Biz A", "Biz C"])
+ self.assertNotIn("--min-reviews ignored", text)
+
class Filters(unittest.TestCase):
def test_keep_row(self):
@@ -611,30 +600,6 @@ class Resilience(unittest.TestCase):
with mock.patch.object(scrape, "req", side_effect=e404), self.assertRaises(scrape.urllib.error.HTTPError):
scrape._get("/api/v1/jobs/x") # a 4xx is final, no retry
- def test_pending_restart_failure_raises_instead_of_polling_a_deleted_job(self):
- statuses = iter(["pending"] * 20)
- def fake_req(method, path, body=None):
- if method == "POST":
- return 201, b'{"id":"j1"}'
- if method == "DELETE":
- return 200, b""
- return 200, json.dumps({"Status": next(statuses)}).encode()
- with mock.patch.object(scrape, "req", fake_req), mock.patch.object(scrape.time, "sleep"), \
- mock.patch.object(scrape, "poll_attempts", return_value=50), \
- mock.patch.object(scrape, "engine_run_ended", return_value=False), \
- mock.patch.object(scrape, "engine_idle", return_value=True), \
- mock.patch.object(scrape, "restart_engine", return_value=False), \
- redirect_stdout(io.StringIO()), self.assertRaises(RuntimeError) as cm:
- scrape.run_job({"max_time": 1})
- self.assertIn("docker restart", str(cm.exception))
-
- def test_failing_docker_logs_is_not_idle(self):
- failed = mock.Mock(returncode=1, stdout="", stderr="Error: No such container")
- with mock.patch.object(scrape, "_local_docker", return_value=True), \
- mock.patch.object(scrape.subprocess, "run", return_value=failed):
- self.assertFalse(scrape.engine_idle())
- self.assertFalse(scrape.engine_run_ended("2026-01-01T00:00:00Z"))
-
class VerifyEmails(unittest.TestCase):
def test_mx_host_labels_and_drops_dead_domains(self):
@@ -727,7 +692,7 @@ class ArgumentBounds(unittest.TestCase):
def test_nothing_listening_fails_in_ten_seconds_not_sixty(self):
refused = scrape.urllib.error.URLError(ConnectionRefusedError(61, "Connection refused"))
with mock.patch.object(scrape, "req", side_effect=refused), mock.patch.object(scrape.time, "sleep") as slept:
- self.assertIn("Docker Desktop", self.run_main())
+ self.assertIn("scripts/engine.py start", self.run_main())
self.assertEqual(slept.call_count, 4) # five tries, two seconds apart
def test_has_email_needs_the_crawl(self):
diff --git a/scripts/update.sh b/scripts/update.sh
index f99ee07..1c524fe 100755
--- a/scripts/update.sh
+++ b/scripts/update.sh
@@ -54,8 +54,12 @@ else
fi
fi
-# A new kit version can pin a new engine image. Restart a running engine so it takes effect.
+# A new kit version can change the engine. Rebuild and restart a running one so it takes effect.
+# --remove-orphans drops the nginx container of kit 1.x, which would otherwise keep the port.
if [ "$BEFORE" != "$(git rev-parse HEAD)" ] && command -v docker >/dev/null 2>&1 \
&& [ -n "$(docker compose ps -q 2>/dev/null)" ]; then
- docker compose up -d
+ docker compose up -d --build --remove-orphans
+fi
+if [ "$BEFORE" != "$(git rev-parse HEAD)" ] && [ -f .engine/state.json ]; then
+ echo "Restart the engine so the update takes effect: python3 scripts/engine.py restart"
fi
diff --git a/ui-branding.conf b/ui-branding.conf
deleted file mode 100644
index 1d2cd5e..0000000
--- a/ui-branding.conf
+++ /dev/null
@@ -1,100 +0,0 @@
-# Fronts the engine's web UI so its "Star on GitHub" button and sponsor box point at this kit
-# instead of upstream. The engine embeds its HTML in the binary, so the links are rewritten in
-# flight. sub_filter only touches text/html: API JSON and CSV downloads pass through unchanged.
-server {
- listen 8080;
-
- # Docker's DNS, re-resolved so a recreated engine container is still found.
- resolver 127.0.0.11 valid=10s;
- set $engine http://google-maps-scraper:8080;
-
- # The engine has no auth, and binding to 127.0.0.1 does not stop the user's own browser. A web
- # page can rebind its domain to 127.0.0.1 and read every job and lead (blocked by the Host
- # check), or post a job from another site without reading the answer (blocked by the Origin
- # check: browsers send Origin on every cross-site POST). Scripts and curl send neither header
- # value, so they pass. Fronting the kit with your own proxy on another hostname? Add it here.
- if ($host !~* ^(localhost|127\.0\.0\.1|\[::1\])$) {
- return 421;
- }
- if ($http_origin !~* ^(https?://(localhost|127\.0\.0\.1|\[::1\])(:\d+)?)?$) {
- return 403;
- }
-
- location / {
- proxy_pass $engine;
- proxy_set_header Host $http_host;
- proxy_set_header Accept-Encoding ""; # sub_filter cannot rewrite a gzipped body
-
- # The engine's own Content-Security-Policy, with one addition: connect-src also allows
- # raw.githubusercontent.com, where the sponsor script below reads sponsors.json.
- proxy_hide_header Content-Security-Policy;
- add_header Content-Security-Policy "default-src 'self'; script-src 'self' cdn.redoc.ly cdnjs.cloudflare.com 'unsafe-inline' 'unsafe-eval'; worker-src 'self' blob:; style-src 'self' 'unsafe-inline' fonts.googleapis.com cdnjs.cloudflare.com; img-src 'self' data: cdn.redoc.ly cdnjs.cloudflare.com *.tile.openstreetmap.org; font-src 'self' fonts.gstatic.com; connect-src 'self' raw.githubusercontent.com" always;
-
- sub_filter_once off;
- sub_filter 'github.com/gosom/google-maps-scraper' 'github.com/Mahanaicoach/google-maps-scraper-kit';
- sub_filter 'github.com/sponsors/gosom' 'github.com/sponsors/Mahanaicoach';
- sub_filter 'Sponsor on GitHub' 'Become a sponsor';
- # Same defaults as the scripts: emails on, depth 5. The engine form ships with emails off and
- # depth 10, so a first scrape from the page came back with no emails.
- sub_filter '<input type="checkbox" id="email" name="email" >' '<input type="checkbox" id="email" name="email" checked>';
- sub_filter 'id="depth" name="depth" value="10">' 'id="depth" name="depth" value="5">';
- # The API docs page loads Redoc from an unpinned "latest" URL, which now serves Redoc 3: it
- # throws "Cannot use 'import.meta' outside a module" and the page stays blank. Pin 2.x.
- sub_filter 'cdn.redoc.ly/redoc/latest/' 'cdn.redoc.ly/redoc/v2.5.0/';
-
- # Fixes the empty job list and dead "Start Scraping" button on engine v1.18.x
- # (gosom/google-maps-scraper#337). Every jobs reply is a <tbody> followed by an out-of-band
- # pagination <div>. htmx parses a <tbody> reply inside a <table>, where that <div> breaks the
- # parse, so #job-tbody disappears and later swaps have no target. The script splits the <div>
- # off before the swap and applies it itself. Remove this once the engine ships the fix.
- # Second script: the sponsor box lists the premium sponsors marked "ui": true in
- # assets/sponsors/sponsors.json on GitHub (a static file, fetched by the browser, cached a day,
- # no tracking). Names and taglines are set as text, never as HTML, and only https links are used.
- # Both scripts avoid double quotes and dollar signs, which this nginx string would mangle.
- sub_filter '</body>' "<script>
-document.addEventListener('htmx:beforeSwap', function (e) {
- var r = e.detail.serverResponse || '', i = r.search(/<div id=.pagination-container/);
- if (i < 0 || !/^\s*<tbody/.test(r)) return;
- var oob = r.slice(i).replace(/ hx-swap-oob=.true./, '');
- e.detail.serverResponse = r.slice(0, i);
- setTimeout(function () {
- var p = document.getElementById('pagination-container');
- if (!p) return;
- p.outerHTML = oob;
- htmx.process(document.getElementById('pagination-container'));
- });
-});
-</script>
-<script>
-(function () {
- var p = document.querySelector('#sponsor-section .sponsor-text');
- if (!p || !window.fetch) return;
- var url = 'https://raw.githubusercontent.com/Mahanaicoach/google-maps-scraper-kit/master/assets/sponsors/sponsors.json';
- var key = 'gmskit-sponsors', day = 86400000;
- function render(list) {
- if (!Array.isArray(list) || !list.some(function (s) { return s && s.ui === true; })) return;
- p.textContent = 'Sponsored by ';
- var shown = 0;
- list.filter(function (s) { return s && s.ui === true; }).slice(0, 4).forEach(function (s) {
- if (typeof s.name !== 'string' || typeof s.url !== 'string' || s.url.indexOf('https://') !== 0) return;
- if (shown++) p.appendChild(document.createTextNode(' | '));
- var a = document.createElement('a');
- a.href = s.url; a.target = '_blank'; a.rel = 'noopener noreferrer sponsored';
- a.textContent = s.name.slice(0, 40);
- p.appendChild(a);
- if (typeof s.tagline === 'string' && s.tagline) p.appendChild(document.createTextNode(': ' + s.tagline.slice(0, 120)));
- });
- }
- try {
- var c = JSON.parse(localStorage.getItem(key) || 'null');
- if (c && Date.now() - c.t < day) { render(c.list); return; }
- } catch (e) {}
- fetch(url, {cache: 'no-store'}).then(function (r) { return r.ok ? r.json() : null; }).then(function (d) {
- if (!d || !Array.isArray(d.sponsors)) return;
- render(d.sponsors);
- try { localStorage.setItem(key, JSON.stringify({t: Date.now(), list: d.sponsors})); } catch (e) {}
- }).catch(function () {});
-})();
-</script></body>";
- }
-}