Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

59.4 KB
Raw Blame
"""Scrape jobs: a sqlite3 store in .engine/data and one worker that runs the jobs one at a time.

A job runs its keywords through maps.collect in one Chrome, each keyword in its own browser context
(own cookies, the next proxy of the job's list), de-duplicates the places on data_id and appends them
to <id>.csv as the result batches arrive, with the old engine's 36 column names (what Google's result
batches carry is filled, the rest stays empty) plus ours. Appends and downloads share one lock, so a
download while the job works returns whole rows. The job's Progress (keywords done, places, percent)
is saved as it goes; a job the engine was stopped in the middle of resumes from its last finished
keyword on the next start. When Google answers with its unusual-traffic page or 429, the keyword
waits and retries a bounded number of times (through the next proxy when there is one), then the job
stops with a clear Error. A watchdog stops Chrome when a job overruns its max_time, so no job can
stick on "working". Standard library only.
"""
import concurrent.futures, csv, html, http.client, io, ipaddress, json, math, os, re, socket
import shutil, sqlite3, tempfile, threading, time, urllib.parse, urllib.request, uuid
from datetime import datetime, timezone

import browser, details, maps
from cdp import CDPError, Page

csv.field_size_limit(2 ** 31 - 1)  # a long cell (a description, a 1.x file) can pass csv's 128 KB default

# Kit 1.x's CSV header, in its order, so files and scripts that read the old engine's output keep
# working. Our own columns follow.
COLUMNS = ["input_id", "link", "title", "category", "address", "open_hours", "popular_times", "website",
           "phone", "plus_code", "review_count", "review_rating", "reviews_per_rating", "latitude",
           "longitude", "cid", "status", "descriptions", "reviews_link", "thumbnail", "timezone",
           "price_range", "data_id", "street_view_url", "place_id", "images", "reservations",
           "order_online", "menu", "owner", "complete_address", "credit_cards_accepted", "about",
           "user_reviews", "user_reviews_extended", "emails"]
EXTRA = ["categories"]
SOCIALS = ["instagram", "facebook", "linkedin", "tiktok", "youtube", "x"]  # the names scripts/scrape.py --socials uses
RENAME = {"open_state": "status", "description": "descriptions"}  # details.FIELDS name -> CSV column

# Seconds past max_time before the watchdog kills Chrome. maps.collect's page load and first waits
# (navigate 45 s, consent 15 s, feed 15 s, first batch 10 s) sit outside its max_seconds, so a
# shorter grace could kill a keyword that was about to return its rows.
WATCHDOG_GRACE = 90
SITE_WORKERS = 8      # business websites fetched at once by the website pass
SITE_TIMEOUT = 10     # seconds per website request (each socket read)
SITE_HARD = 30        # seconds a website request may take in all, then its socket is closed
EMAIL_MIN_SECONDS = 60  # the website pass gets at least this long, even after a job used its max_time
BACKOFF = (30, 90)    # seconds to wait before each retry of a keyword Google refused; then the job stops
ROTATE_WAIT = 3       # seconds before the retry when the next proxy can take it instead
MAX_RESUMES = 3       # restarts of the engine mid-job before the job is failed instead of resumed
LOGIN_REFUSED = "proxy login refused: the proxy answered 407 to its username and password"
# Chrome's errors for a proxy that failed, and ours. Matched as written: an error also holds the
# search URL, so a keyword such as "tunnel car wash" must not read as a proxy error.
PROXY_FAILED = re.compile(r"\bERR_(?:PROXY|SOCKS|TUNNEL)_|proxy login refused")
TIMED_OUT = re.compile(r"\bERR_(?:CONNECTION_)?TIMED_OUT\b")  # through a proxy: maybe the proxy, so try the next
RELAUNCHES = 2        # times a job starts Chrome again after it died on its own (a crash, out of memory)
REVIEWS_MAX = 5       # the reviews a place panel shows first; more would need its review list


def utc_now():
    return datetime.now(timezone.utc).isoformat(timespec="seconds")


# ── Job body ──
LANG = re.compile(r"[A-Za-z]{2}")  # fullmatch: "$" would let "en\n" through
PROXY_SCHEMES = ("http", "https", "socks5", "socks5h")
CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]")  # NUL breaks csv on Python 3.9; the rest never belongs in a search


def _text(v):
    """True for a string the engine can store, search and write: valid UTF-8 (no lone surrogates,
    which sqlite and urllib.parse.quote refuse) and no control characters."""
    if not isinstance(v, str) or CONTROL.search(v):
        return False
    try:
        v.encode("utf-8")
    except UnicodeEncodeError:
        return False
    return True


def _bool(body, key):
    v = body.get(key)
    if v is None:
        return False
    if not isinstance(v, bool):  # "false" is not false
        raise ValueError(f"invalid {key}: true or false")
    return v


def _int(body, key, default, lo, hi, label):
    v = body.get(key, default)
    if v is None:
        v = default
    if isinstance(v, bool) or not isinstance(v, int) or not lo <= v <= hi:
        raise ValueError(f"invalid {label}: a whole number from {lo} to {hi}")
    return v


def validate(body):
    """(name, data) from a create-job body, or ValueError with the old engine's message
    ("missing name", "invalid lang", ...), which the kit's docs quote."""
    if not isinstance(body, dict):
        raise ValueError("the body must be a JSON object")
    name = body.get("name")
    if not isinstance(name, str) or not name.strip():
        raise ValueError("missing name")
    if len(name) > 200 or not _text(name):
        raise ValueError("invalid name: 200 characters at most, no control characters")
    kws = body.get("keywords")
    if not isinstance(kws, list) or not kws:
        raise ValueError("missing keywords")
    if len(kws) > 500 or not all(_text(k) and k.strip() and len(k) <= 500 for k in kws):
        raise ValueError("invalid keywords: 1 to 500 non-empty strings of up to 500 characters")
    lang = body.get("lang")
    if lang in (None, ""):
        raise ValueError("missing lang")
    if not isinstance(lang, str) or not LANG.fullmatch(lang):
        raise ValueError("invalid lang")
    country = body.get("country")
    if country in (None, ""):
        country = ""
    elif not isinstance(country, str) or not LANG.fullmatch(country):
        raise ValueError("invalid country: a 2-letter country code such as us or de")
    lat, lon = body.get("lat"), body.get("lon")
    if (lat in (None, "")) != (lon in (None, "")):
        raise ValueError("missing geo coordinates")
    if lat in (None, ""):  # neither, as the old engine allowed: Google places the search by the keyword's words
        lat = lon = ""
    else:
        lat, lon = _coords(lat, lon)  # canonical text: "1e1", "+30.5" or "٣٠" would go into the URL as typed
    if body.get("max_time") in (None, 0):
        raise ValueError("missing max time")
    data = {"keywords": [k.strip() for k in kws], "lang": lang.lower(), "country": country.lower(),
            "lat": str(lat).strip(), "lon": str(lon).strip(),
            "zoom": _int(body, "zoom", 15, 1, 21, "zoom"),
            "radius": _int(body, "radius", 10000, 0, 10**9, "radius"),  # accepted, not used
            "depth": _int(body, "depth", 1, 1, 1000, "depth"),
            "max_time": _int(body, "max_time", 600, 1, 86400, "max time"),
            "fast_mode": _bool(body, "fast_mode"), "email": _bool(body, "email"),
            "verify_emails": _bool(body, "verify_emails"), "socials": _bool(body, "socials"),
            "browser_sites": _bool(body, "browser_sites"),
            "reviews_max": _int(body, "reviews_max", 0, 0, REVIEWS_MAX, "reviews_max"),
            "proxies": _proxies(body.get("proxies"))}
    if data["browser_sites"]:
        from website_browser import check_available
        try:
            check_available()
        except RuntimeError as e:
            raise ValueError(str(e)) from e
    return name.strip(), data


def _coords(lat, lon):
    """ValueError("invalid geo coordinates") unless lat/lon are a point on Earth."""
    try:
        if isinstance(lat, bool) or isinstance(lon, bool):
            raise ValueError
        flat, flon = float(lat), float(lon)
    except (TypeError, ValueError, OverflowError):  # OverflowError: a JSON integer of 400 digits
        raise ValueError("invalid geo coordinates") from None
    if not (math.isfinite(flat) and math.isfinite(flon) and -90 <= flat <= 90 and -180 <= flon <= 180):
        raise ValueError("invalid geo coordinates")
    return _plain(flat), _plain(flon)


def _plain(v):
    """A float as plain decimal text (no exponent), 7 decimals at most: what the Maps URL takes."""
    return f"{v:.7f}".rstrip("0").rstrip(".") or "0"


def _proxies(v):
    if v in (None, ""):
        return []
    if not isinstance(v, list) or len(v) > 100 or not all(isinstance(p, str) for p in v):
        raise ValueError("invalid proxies: a list of proxy URLs")
    for p in v:
        try:
            u = urllib.parse.urlsplit(p.strip())
            u.port  # noqa: B018 - raises on a bad port
        except ValueError:
            raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port") from None
        if u.scheme not in PROXY_SCHEMES or not u.hostname:
            raise ValueError("invalid proxy: use http://, https://, socks5:// or socks5h://host:port")
        if (u.username or u.password) and u.scheme.startswith("socks"):
            raise ValueError("invalid proxy: Chrome cannot log in to a SOCKS5 proxy; use its http:// address "
                             "with user:pass, or one that allows your IP address")
    return [p.strip() for p in v]


def mask(proxy):
    """A proxy URL with its password replaced by *** (or its username, when that is the only
    secret, as some providers do): what the API and the web UI show."""
    try:
        u = urllib.parse.urlsplit(proxy)
    except ValueError:  # "http://[abc" from a 1.x job: validate() never saw it; it must not break the job list
        return "***"
    if not (u.username or u.password):
        return proxy
    user, host = u.netloc.rpartition("@")[0], u.netloc.rpartition("@")[2]
    shown = user.partition(":")[0] + ":***" if u.password else "***"
    return urllib.parse.urlunsplit(u._replace(netloc=f"{shown}@{host}"))


def locale(d):
    """Chrome's --lang (its Accept-Language) for a job: en, or en-US when the job names a country."""
    return d["lang"] + (f"-{d['country'].upper()}" if d.get("country") else "")


# ── Rows ──
def fields(d):
    """A job's CSV columns: the old engine's 36, ours, then those its website pass fills."""
    return COLUMNS + EXTRA + (SOCIALS if d.get("socials") else []) + (["email_host"] if d.get("verify_emails") else []) + (["website_fetch_status"] if d.get("browser_sites") else [])


def to_row(place, keyword):
    """One CSV row (every column, as text) from a details.parse_entity / maps.parse_link place."""
    row = dict.fromkeys(COLUMNS + EXTRA, "")
    for k, v in place.items():
        k = RENAME.get(k, k)
        if k in row:  # a name cut mid-emoji holds a lone surrogate, which UTF-8 cannot encode: "?"
            row[k] = str(v).replace("\x00", "").encode("utf-8", "replace").decode("utf-8")
    row["input_id"] = keyword
    if not row["plus_code"]:
        row["plus_code"] = details.plus_code(row["latitude"], row["longitude"])
    if place.get("place_id"):  # Google's documented Maps URL for one place
        row["link"] = ("https://www.google.com/maps/search/?api=1&query="
                       + urllib.parse.quote(row["title"], safe="")
                       + "&query_place_id=" + urllib.parse.quote(place["place_id"], safe=""))
    return row


def place_key(p):
    """What makes two places (or rows) the same place."""
    return p.get("data_id") or p.get("place_id") or p.get("link") or p.get("title")


# What Excel or Sheets would run as a formula (a phone number is not), also behind quotes the guard
# added, so "'=x" scraped as is gets one more and unguarding stays exact. scripts/scrape.py has the same.
FORMULA = re.compile(r"'*(?:[=@\t\r]|[+-](?![\d\s().-]*(?:(?i:ext\.?|x)\s*\d+)?$))")  # "+1 … ext. 12" is a phone too


def csv_safe(v):
    """A cell with a leading ' when a spreadsheet would run it as a formula ("=HYPERLINK(...)" in a
    business name or review); "+1 512-555-0100" is left alone. Same rule as scripts/scrape.py."""
    s = "" if v is None else str(v)
    return "'" + s if FORMULA.match(s) else s


def guard_csv(data):
    """A stored CSV with csv_safe on every cell: what /download serves. The file on disk stays as
    scraped, because the engine reads it back (resume, JSON download)."""
    buf = io.StringIO()
    w = csv.writer(buf)
    for row in csv.reader(io.StringIO(data.decode("utf-8", "replace"), newline="")):
        w.writerow([csv_safe(v) for v in row])
    return buf.getvalue().encode("utf-8")


def to_csv(rows, cols=None, header=True):
    buf = io.StringIO()
    w = csv.DictWriter(buf, fieldnames=cols or COLUMNS + EXTRA, extrasaction="ignore")
    if header:
        w.writeheader()
    w.writerows(rows)
    return buf.getvalue().encode("utf-8", "replace")  # reviews are set on rows after to_row


# ── Store ──
class Store:
    """Jobs in sqlite3 (jobs.db) and their results as <id>.csv beside it. One lock serialises every
    database call and every CSV read, append and replace, so a download never meets a half-written
    row or a half-replaced file (Windows cannot replace a file another handle has open)."""

    def __init__(self, folder):
        os.makedirs(folder, exist_ok=True)
        self.folder = folder
        self.lock = threading.RLock()
        self.db = sqlite3.connect(os.path.join(folder, "jobs.db"), check_same_thread=False)
        with self.lock, self.db:
            self.db.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, "
                            "date TEXT NOT NULL, status TEXT NOT NULL, data TEXT NOT NULL, "
                            "error TEXT NOT NULL DEFAULT '', progress TEXT NOT NULL DEFAULT '{}')")
            if "progress" not in [c[1] for c in self.db.execute("PRAGMA table_info(jobs)")]:  # a P1 or P2 store
                self.db.execute("ALTER TABLE jobs ADD COLUMN progress TEXT NOT NULL DEFAULT '{}'")

    SELECT = "SELECT id, name, date, status, data, error, progress FROM jobs"

    @staticmethod
    def _job(r):
        # The old engine's job shape (capitalised keys), plus Error (why a job failed or stopped early)
        # and Progress (the worker's own record; public() is what the API shows).
        return {"ID": r[0], "Name": r[1], "Date": r[2], "Status": r[3], "Data": json.loads(r[4]), "Error": r[5],
                "Progress": json.loads(r[6] or "{}")}

    def create(self, name, data):
        jid = str(uuid.uuid4())
        with self.lock, self.db:
            self.db.execute("INSERT INTO jobs (id, name, date, status, data, error, progress) "
                            "VALUES (?, ?, ?, 'pending', ?, '', '{}')", (jid, name, utc_now(), json.dumps(data)))
        return jid

    def get(self, jid):
        with self.lock:
            r = self.db.execute(self.SELECT + " WHERE id = ?", (jid,)).fetchone()
        return self._job(r) if r else None

    def all(self):
        with self.lock:
            return [self._job(r) for r in self.db.execute(self.SELECT + " ORDER BY rowid DESC")]

    def update(self, jid, status, error=""):
        with self.lock, self.db:
            self.db.execute("UPDATE jobs SET status = ?, error = ? WHERE id = ?", (status, error[:500], jid))

    def set_progress(self, jid, progress):
        with self.lock, self.db:
            self.db.execute("UPDATE jobs SET progress = ? WHERE id = ?", (json.dumps(progress), jid))

    def claim(self):
        """The oldest pending job, now marked working; None when there is none."""
        with self.lock, self.db:
            r = self.db.execute(self.SELECT + " WHERE status = 'pending' ORDER BY rowid LIMIT 1").fetchone()
            if not r:
                return None
            self.db.execute("UPDATE jobs SET status = 'working' WHERE id = ?", (r[0],))
        job = self._job(r)
        job["Status"] = "working"
        return job

    def recover(self, max_resumes=MAX_RESUMES):
        """Jobs left on "working" by an engine that was stopped or crashed: back to pending, so they
        resume from their last finished keyword (the rows of the keyword cut short are removed, a
        torn last line too). After max_resumes such restarts a job is failed instead, rows kept, as
        before resumable jobs. (resumed, failed)"""
        resumed = failed = 0
        with self.lock, self.db:
            for jid, prog in self.db.execute("SELECT id, progress FROM jobs WHERE status = 'working'").fetchall():
                p = json.loads(prog or "{}")
                if p.get("stage", "search") == "search":
                    self.cut(jid, p.get("bytes", 0))
                if p.get("resumes", 0) >= max_resumes:
                    self.db.execute("UPDATE jobs SET status = 'failed', error = ? WHERE id = ?",
                                    (f"the engine stopped while this job ran ({p['resumes'] + 1} times); its rows "
                                     "so far are kept", jid))
                    failed += 1
                else:
                    p["resumes"] = p.get("resumes", 0) + 1
                    self.db.execute("UPDATE jobs SET status = 'pending', progress = ? WHERE id = ?",
                                    (json.dumps(p), jid))
                    resumed += 1
        return resumed, failed

    def delete(self, jid):
        with self.lock, self.db:
            gone = self.db.execute("DELETE FROM jobs WHERE id = ?", (jid,)).rowcount
            for path in (self.csv_path(jid), self.csv_path(jid) + ".tmp"):
                try:
                    os.remove(path)
                except OSError:
                    pass
        return bool(gone)

    def import_kit1(self, folder, log=print):
        """Kit 1.x's jobs and their CSVs, from the old engine's data folder (docker-compose.yml mounts
        its volume read-only on .engine/kit1), so an update from 1.x keeps them in the job list. Once
        only: a marker file stops it next time, so a job deleted here stays deleted. Jobs with no CSV
        (never run) are left out, and one that cannot be read is skipped and logged by id. Jobs
        imported; an unreadable old database raises sqlite3.Error and is tried again on the next start."""
        marker = os.path.join(self.folder, "kit1-imported")
        if os.path.exists(marker) or not os.path.isfile(os.path.join(folder, "jobs.db")):
            return 0
        # The old database is in WAL mode on a read-only volume: read a copy, its WAL included.
        tmp = tempfile.mkdtemp(dir=self.folder)
        try:
            for f in ("jobs.db", "jobs.db-wal"):
                if os.path.isfile(os.path.join(folder, f)):
                    shutil.copyfile(os.path.join(folder, f), os.path.join(tmp, f))
            old = sqlite3.connect(os.path.join(tmp, "jobs.db"))
            try:
                rows = old.execute("SELECT id, name, status, data, created_at FROM jobs ORDER BY created_at").fetchall()
            finally:
                old.close()
        finally:
            shutil.rmtree(tmp, ignore_errors=True)
        n = 0
        for jid, name, status, data, created in rows:
            try:
                plain = str(uuid.UUID(str(jid))) == jid  # a job id, never a path, goes into a file name
            except ValueError:
                plain = False
            src = os.path.join(folder, f"{jid}.csv")
            if not plain or not os.path.isfile(src) or self.get(jid):
                continue
            try:  # one odd old job must never stop the engine from starting
                try:
                    d = json.loads(data or "{}")
                except ValueError:
                    d = {}
                if not isinstance(d, dict):
                    d = {}
                if not isinstance(d.get("keywords"), list):
                    d["keywords"] = []
                d["proxies"] = [p for p in d.get("proxies") or [] if isinstance(p, str)] \
                    if isinstance(d.get("proxies"), list) else []
                if isinstance(d.get("max_time"), int):
                    d["max_time"] //= 10 ** 9  # nanoseconds in 1.x, seconds here
                with open(src, newline="", encoding="utf-8", errors="replace") as f:
                    places = max(sum(1 for _ in csv.reader(f)) - 1, 0)
                date = datetime.fromtimestamp(int(created), timezone.utc).isoformat(timespec="seconds")
                ok = status == "ok"
                with self.lock, self.db:
                    shutil.copyfile(src, self.csv_path(jid))
                    self.db.execute("INSERT INTO jobs (id, name, date, status, data, error, progress) "
                                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
                                    (jid, str(name), date, "ok" if ok else "failed", json.dumps(d),
                                     "" if ok else f"kit 1.x job that was {status} there; its rows are kept",
                                     json.dumps({"done": len(d.get("keywords") or []) if ok else 0, "places": places,
                                                 "stage": "done" if ok else ""})))
            except Exception as e:
                try:
                    os.remove(self.csv_path(jid))
                except OSError:
                    pass
                log(f"kit 1.x job {jid} not imported: {type(e).__name__}")
                continue
            n += 1
        with open(marker, "w"):
            pass
        return n

    def csv_path(self, jid):
        return os.path.join(self.folder, f"{jid}.csv")

    def _exists(self, jid):
        return self.db.execute("SELECT 1 FROM jobs WHERE id = ?", (jid,)).fetchone() is not None

    def append(self, jid, rows, cols):
        """Add rows to the job's CSV (the header first on an empty file); the file's size after it,
        or None (nothing written) if the job was deleted."""
        with self.lock:
            if not self._exists(jid):
                return None
            path = self.csv_path(jid)
            size = os.path.getsize(path) if os.path.exists(path) else 0
            if rows or not size:
                try:
                    with open(path, "ab") as f:
                        f.write(to_csv(rows, cols, header=not size))
                except OSError:  # full disk: cut the torn row, or the next append follows it
                    with open(path, "r+b") as f:
                        f.truncate(size)
                    raise
            return os.path.getsize(path)

    def cut(self, jid, size):
        """Shorten the job's CSV to `size` bytes (its length at the last finished keyword)."""
        with self.lock:
            path = self.csv_path(jid)
            try:
                if not size:
                    os.remove(path)
                elif os.path.getsize(path) > size:
                    with open(path, "r+b") as f:
                        f.truncate(size)
            except OSError:
                pass

    def rewrite(self, jid, rows, cols):
        """Replace the job's CSV in one step (after a pass that changed rows); its new size, or None
        (nothing written) if the job was deleted."""
        with self.lock:
            if not self._exists(jid):
                return None
            tmp = self.csv_path(jid) + ".tmp"
            with open(tmp, "wb") as f:
                f.write(to_csv(rows, cols))
            os.replace(tmp, self.csv_path(jid))
            return os.path.getsize(self.csv_path(jid))

    def finish(self, jid, rows, cols, status, error, progress):
        """Replace the job's CSV in one step and record how the job ended, under one lock; False
        (nothing written) if the job was deleted."""
        with self.lock:
            if self.rewrite(jid, rows, cols) is None:
                return False
            with self.db:
                self.db.execute("UPDATE jobs SET status = ?, error = ?, progress = ? WHERE id = ?",
                                (status, error[:500], json.dumps(progress), jid))
            return True

    def read_csv(self, jid):
        with self.lock:
            try:
                with open(self.csv_path(jid), "rb") as f:
                    return f.read()
            except OSError:
                return None

    def read_rows(self, jid):
        """The job's rows as dicts ([] before its first row)."""
        data = self.read_csv(jid)
        return list(csv.DictReader(io.StringIO(data.decode("utf-8", "replace")))) if data else []


def public(job):
    """A job as the API shows it: proxy passwords masked, Progress without the worker's bookkeeping."""
    d, p = job["Data"], job["Progress"]
    total = len(d.get("keywords") or [])
    stage = p.get("stage") or ("queued" if job["Status"] == "pending" else "")
    if job["Status"] == "ok":
        stage = "done"
    proxies = d.get("proxies") if isinstance(d.get("proxies"), list) else []  # a 1.x job may hold anything
    return dict(job, Data=dict(d, proxies=[mask(x) for x in proxies if isinstance(x, str)]),
                Progress={"keywords_done": p.get("done", 0), "keywords_total": total, "places": p.get("places", 0),
                          "percent": 100 if job["Status"] == "ok" else p.get("percent", 0), "stage": stage,
                          **({"reviews_done": p.get("reviews_done", 0), "reviews_total": p["reviews_total"]}
                             if p.get("reviews_total") else {})})


# ── Worker ──
class Worker(threading.Thread):
    """Runs pending jobs one at a time, oldest first: never two scrapes against Google at once.
    headless=False runs Chrome as an off-screen window. Both get Google's full view (review counts,
    the week's hours, reviews): see browser.py."""

    def __init__(self, store, log=print, headless=True):
        super().__init__(daemon=True, name="jobs")
        self.store, self.log, self.headless = store, log, headless
        self.wake = threading.Event()
        self.stopped = threading.Event()
        self._running = (None, None)  # (job id, stop event) of the job in progress
        self._browser = None

    def run(self):
        while not self.stopped.is_set():
            job = None
            try:
                job = self.store.claim()
                if not job:
                    self.wake.wait(5)
                    self.wake.clear()
                    continue
                self.run_job(job)
            except Exception as e:  # noqa: BLE001 - one bad job (or a full disk) must never stop the worker
                reason = f"{type(e).__name__}: {e}"
                for attempt in (1, 2):  # the second try after a pause: a full disk may have room again
                    try:
                        if job:
                            self.store.update(job["ID"], "failed", reason)
                        break
                    except Exception:  # noqa: BLE001
                        self.stopped.wait(2)
                try:
                    self.log(f"job {job['ID'] if job else '-'} failed: {reason}")
                except Exception:  # noqa: BLE001 - a log that cannot be written must not kill the worker
                    pass
                self.stopped.wait(2)

    def cancel(self, jid):
        """Stop `jid` if it is the job in progress (it is being deleted)."""
        rid, stop = self._running
        if rid == jid and stop:
            stop.set()

    def shutdown(self, wait=0):
        """Stop the job in progress and its Chrome (the engine is exiting). With `wait`, give the job
        that many seconds to save its progress, so the next start resumes it."""
        self.stopped.set()
        self.wake.set()
        _, stop = self._running
        if stop:
            stop.set()
        b = self._browser
        if b:
            b.close()
        if wait and self.is_alive():
            self.join(wait)

    def run_job(self, job):
        jid, d = job["ID"], job["Data"]
        p = dict(job["Progress"])
        kws, cols, proxies = d["keywords"], fields(d), d.get("proxies") or []
        started, used = time.monotonic(), p.get("elapsed", 0)
        budget = max(0, d["max_time"] - used)  # a resumed job has the rest of its max_time
        deadline = started + budget
        stop = threading.Event()
        self._running = (jid, stop)
        if self.stopped.is_set():  # shutdown() came between claim() and now
            stop.set()
        if not self.store.get(jid):  # deleted between claim() and now, before cancel() could see it
            self._running = (None, None)
            return
        done, stage = min(p.get("done", 0), len(kws)), p.get("stage", "search")
        reviews = d.get("reviews_max") and stage in ("search", "reviews")
        if stage == "search":
            self.store.cut(jid, p.get("bytes", 0))  # the rows of a keyword cut short are searched again
        rows = self.store.read_rows(jid)  # [] when the cut removed the file
        seen = {place_key(r) for r in rows}
        # What an earlier run of this job saw, for its final note (a job resumed after its search).
        errors = [p["last_error"]] * p.get("failed", 0) if p.get("last_error") else []
        notes, blocked, retries, rot, refused = [], p.get("blocked", False), p.get("retried", 0), 0, set()
        bad, relaunched, timed_out = set(), 0, {"yes": p.get("timed_out", False)}  # bad: proxies that failed

        def pick(i):  # the keyword's proxy in turn, past the ones that failed (all failed: in turn anyway)
            if not proxies:
                return None
            turn = [proxies[(i + rot + k) % len(proxies)] for k in range(len(proxies))]
            return next((x for x in turn if x not in bad), turn[0])
        cur = {"i": done, "kw": ""}
        if done or p.get("resumes"):
            self.log(f"job {jid} resumed at keyword {done + 1}/{len(kws)} with {len(rows)} places, {budget:.0f} s left")
        else:
            self.log(f"job {jid} started: {len(kws)} keyword(s), depth {d['depth']}, max_time {d['max_time']} s")

        def save(**kw):
            p.update(kw, elapsed=round(used + time.monotonic() - started, 1), places=len(rows))
            self.store.set_progress(jid, p)

        def on_batch(places, batches):  # streams each batch's new places into the CSV
            fresh = []
            for place in places:
                k = place_key(place)
                if k and k not in seen:
                    seen.add(k)
                    fresh.append(to_row(place, cur["kw"]))
            rows.extend(fresh)
            if fresh and self.store.append(jid, fresh, cols) is None:
                stop.set()  # deleted
            save(percent=int(100 * (cur["i"] + min(batches / d["depth"], 1)) / len(kws)))

        def watchdog():  # Chrome hung or the job overran: kill Chrome, and every CDP call fails at once
            timed_out["yes"] = True
            stop.set()
            b = self._browser
            if b:
                b.kill()
            self.log(f"job {jid}: watchdog stopped Chrome {WATCHDOG_GRACE} s after max_time")
        timer = threading.Timer(budget + WATCHDOG_GRACE, watchdog)
        timer.daemon = True
        timer.start()
        try:
            save(stage=stage, done=done, total=len(kws))
            if stage == "search" and done < len(kws) and budget <= 0:
                timed_out["yes"] = True  # resumed with no time left: the other keywords stay unsearched
            if ((stage == "search" and done < len(kws)) or reviews) and budget > 0 and not stop.is_set():
                self._browser = browser.Browser(lang=locale(d), headless=self.headless)
            i, again = done, False
            while stage == "search" and i < len(kws) and self._browser:
                left = deadline - time.monotonic()
                if left <= 0:
                    timed_out["yes"] = True
                if stop.is_set() or left <= 0:
                    break
                cur.update(i=i, kw=kws[i])
                proxy = pick(i)
                url = maps.search_url(kws[i], d["lat"] or None, d["lon"] or None, d["zoom"], d["lang"], d.get("country", ""))
                t0 = time.monotonic()
                try:
                    places, reason, stats = self._collect(proxy, url, d["depth"], left, stop, on_batch)
                except (CDPError, OSError) as e:
                    if str(e) == LOGIN_REFUSED:
                        refused.add(proxy)
                    if PROXY_FAILED.search(str(e)):
                        bad.add(proxy)
                    if (self._browser.cdp.closed and not stop.is_set() and relaunched < RELAUNCHES
                            and deadline - time.monotonic() > 0):
                        relaunched += 1  # Chrome died on its own: a new one takes the same keyword
                        self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Chrome stopped ({e}); starting it again")
                        self._browser.close()
                        self._browser = browser.Browser(lang=locale(d), headless=self.headless)
                        continue
                    if ((PROXY_FAILED.search(str(e)) or TIMED_OUT.search(str(e))) and len(proxies) > 1
                            and retries < len(BACKOFF) and len(refused) < len(set(proxies))
                            and not self._browser.cdp.closed and not stop.is_set()):
                        retries, rot = retries + 1, rot + 1
                        self.log(f"job {jid} keyword {i + 1}/{len(kws)}: the proxy failed; retry {retries}/"
                                 f"{len(BACKOFF)} through the next proxy")
                        continue
                    errors.append(str(e))
                    self.log(f"job {jid} keyword {i + 1}/{len(kws)}: error: {e}")
                    # Chrome is gone, or no proxy takes the job's login: the other keywords would fail too
                    if self._browser.cdp.closed or (refused and len(refused) >= len(set(proxies))):
                        if not stop.is_set() and len(kws) - i > 1:
                            errors[-1] += f"; {len(kws) - i - 1} keyword(s) after it were not searched"
                        break
                    if stop.is_set():  # the engine is stopping: leave the keyword for the resume
                        break
                    size = self.store.append(jid, [], cols)
                    if size is None:
                        break  # deleted
                    i, retries, again = i + 1, 0, False
                    save(done=i, bytes=size, percent=int(100 * i / len(kws)), failed=len(errors),
                         last_error=errors[-1])
                    continue
                on_batch(places, stats["batches"])  # places no batch carried (one place, the DOM fallback)
                if stop.is_set():  # deleted, or the engine is exiting: this keyword did not finish
                    break
                if reason == "blocked":  # Google limits this IP: wait, try again, then give up
                    if retries < len(BACKOFF) and not stop.is_set():
                        wait = ROTATE_WAIT if len(proxies) > 1 else BACKOFF[retries]
                        if time.monotonic() + wait < deadline:
                            retries, rot = retries + 1, rot + 1
                            self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; "
                                     f"retry {retries}/{len(BACKOFF)} in {wait} s"
                                     + (" through the next proxy" if len(proxies) > 1 else ""))
                            save()
                            if stop.wait(wait):
                                break
                            continue
                    blocked = True
                    self.log(f"job {jid} keyword {i + 1}/{len(kws)}: Google's unusual-traffic page; stopping")
                    break
                if reason == "no-feed" and not again and time.monotonic() + 30 < deadline:
                    again = True  # Google sometimes shows no result list once (seen in CI): look once more
                    if len(proxies) > 1:
                        rot += 1  # through the next proxy
                    self.log(f"job {jid} keyword {i + 1}/{len(kws)}: no result list; one more try")
                    continue
                if reason == "no-feed":
                    notes.append(f"keyword {i + 1}: Google showed no result list{' twice' if again else ''} "
                                 "(no results for it, or a page the engine cannot read)")
                if reason.endswith("+dom-fallback"):  # no result batch parsed: Google changed its format
                    notes.append(f"keyword {i + 1}: Google's result data could not be read, so "
                                 + (f"only names, links and coordinates were kept for {len(places)} places"
                                    if places else "no places were read") + " (the engine may need an update)")
                retries, again = 0, False
                if reason.startswith("timeout"):  # "timeout+dom-fallback" too
                    timed_out["yes"] = True
                size = self.store.append(jid, [], cols)
                if size is None:
                    break  # deleted
                if reason.startswith("error: "):
                    errors.append(reason[7:])
                save(done=i + 1, bytes=size, percent=int(100 * (i + 1) / len(kws)), failed=len(errors),
                     last_error=errors[-1] if errors else "")
                self.log(f"job {jid} keyword {i + 1}/{len(kws)}: {len(places)} places, {len(rows)} in all, "
                         f"{stats['batches']} batches, {time.monotonic() - t0:.1f} s ({reason})")
                if reason.startswith("error: ") and self._browser.cdp.closed:  # Chrome failed mid-scroll;
                    if stop.is_set() or relaunched >= RELAUNCHES or deadline <= time.monotonic():  # rows kept
                        if not stop.is_set() and len(kws) - i > 1:
                            errors[-1] += f"; {len(kws) - i - 1} keyword(s) after it were not searched"
                        break
                    relaunched += 1
                    self._browser.close()
                    self._browser = browser.Browser(lang=locale(d), headless=self.headless)
                i += 1
            if stage == "search" and not stop.is_set():  # the search is over, for good: never search again
                stage = "reviews" if reviews else "websites"
                save(stage=stage, bytes=self.store.append(jid, [], cols), blocked=blocked, retried=retries,
                     failed=len(errors), last_error=errors[-1] if errors else "", timed_out=timed_out["yes"])
            if stage == "reviews" and rows and self._browser and not (stop.is_set() or blocked):
                n = self.add_reviews(rows, d, deadline, stop, [x for x in proxies if x not in bad] or proxies,
                                     save, notes)
                self.store.rewrite(jid, rows, cols)  # keep the reviews if the engine stops during the next pass
                self.log(f"job {jid}: reviews for {n}/{sum(1 for r in rows if r.get('place_id'))} places")
            if stage == "reviews" and not stop.is_set():
                stage = "websites"
                save(stage=stage)
        except browser.BrowserError as e:
            errors.append(str(e))
            self.log(f"job {jid}: {e}")
        finally:
            timer.cancel()
            if self._browser:
                self._browser.close()
                self._browser = None
        if stage == "websites" and (d["email"] or d.get("socials")) and rows and not stop.is_set():
            n, unchecked = add_site_data(rows, max(deadline - time.monotonic(), EMAIL_MIN_SECONDS), email=d["email"],
                                         socials=d.get("socials"), verify=d.get("verify_emails"), stop=stop.is_set,
                                         browser_sites=d.get("browser_sites", False))
            self.log(f"job {jid}: website data for {n}/{sum(1 for r in rows if r['website'])} websites")
            if unchecked and not stop.is_set():
                notes.append(f"{unchecked} website(s) not checked: ran out of time")
        self._running = (None, None)
        if self.stopped.is_set() and self.store.get(jid):  # the engine is exiting: resume on the next start
            if stage != "search":  # rows a pass changed; while searching the CSV is the record already
                self.store.rewrite(jid, rows, cols)
            save()
            self.store.update(jid, "pending")
            self.log(f"job {jid} paused at keyword {p.get('done', 0) + 1}/{len(kws)} ({stage}): resumes when the "
                     "engine starts again")
            return
        block = ("Google answered with its unusual-traffic page: this IP is rate-limited for now. "
                 + (f"Waited and retried {retries} time(s). " if retries else "") + "Wait, or add proxies")
        p.update(stage="done", elapsed=round(used + time.monotonic() - started, 1), places=len(rows))
        if rows or not (errors or blocked):
            heads = []
            if timed_out["yes"]:  # the search ran out of time (the passes after it note their own)
                heads.append("stopped at max_time; the rows found so far are kept")
            if errors:
                heads.append(f"{len(errors)} keyword(s) failed: {errors[-1]}")
            if blocked:
                heads, notes = [f"{block}. The rows found before that are kept"], []
            status, note = "ok", "; ".join(heads + notes)
        else:
            status = "failed"
            note = block if blocked else errors[-1]
        if not self.store.finish(jid, rows, cols, status, note, p):
            self.log(f"job {jid} deleted while it ran")
            return
        self.log(f"job {jid} {status}: {len(rows)} places" + (f": {note}" if status == "failed" else ""))

    def _collect(self, proxy, url, depth, left, stop, on_batch):
        """One keyword in its own browser context, through `proxy`; maps.collect's answer."""
        ctx = page = auth = None
        try:
            ctx = self._browser.context(proxy)
            page = Page(self._browser.cdp, context=ctx)
            _, user, password = browser.split_proxy(proxy) if proxy else ("", "", "")
            if user or password:
                auth = browser.ProxyAuth(page, user, password)
            try:
                got = maps.collect(page, url, depth=depth, max_seconds=left, log=lambda m: None,
                                   stop=stop.is_set, on_batch=on_batch)
            except CDPError:
                if auth and auth.refused:  # Chrome only says ERR_HTTP_RESPONSE_CODE_FAILURE
                    raise CDPError(LOGIN_REFUSED) from None
                raise
            if auth and auth.refused and not got[0]:
                raise CDPError(LOGIN_REFUSED)
            return got
        finally:
            if auth:
                auth.close()
            if page and not self._browser.cdp.closed:
                page.close()
            if ctx and not self._browser.cdp.closed:
                self._browser.dispose(ctx)

    def add_reviews(self, rows, d, deadline, stop, proxies, save=lambda: None, notes=None):
        """Review count, stars histogram and up to reviews_max reviews per place, from each place's
        panel (the result batches carry no review texts): one page load per place, until the
        deadline. The number of places that got reviews."""
        todo = [r for r in rows if r.get("place_id") and not r.get("user_reviews")]
        ctx = page = auth = None
        n = 0
        try:
            proxy = proxies[0] if proxies else None
            ctx = self._browser.context(proxy)
            page = Page(self._browser.cdp, context=ctx)
            _, user, password = browser.split_proxy(proxy) if proxy else ("", "", "")
            if user or password:
                auth = browser.ProxyAuth(page, user, password)
            for k, v in maps.CONSENT_COOKIES.items():
                page.set_cookie(k, v, ".google.com")
            for k, r in enumerate(todo):
                if stop.is_set():
                    break
                if time.monotonic() >= deadline:
                    if notes is not None:
                        notes.append(f"reviews read for {k} of {len(todo)} places: ran out of time")
                    break
                url = r["link"] + f"&hl={d['lang']}" + (f"&gl={d['country']}" if d.get("country") else "")
                body = maps.place_panel(page, url)
                if body == "blocked":
                    if notes is not None:
                        notes.append(f"reviews read for {k} of {len(todo)} places: Google's unusual-traffic page")
                    break
                got = details.parse_place(body or "", d["reviews_max"])
                for key, v in got.items():
                    if key in ("user_reviews", "reviews_per_rating") or not r.get(key):
                        r[key] = str(v)
                n += "user_reviews" in got
                save(reviews_done=k + 1, reviews_total=len(todo))  # progress, and elapsed time for a resume
        except (CDPError, OSError) as e:  # Chrome or the proxy went away: the reviews so far are kept
            if notes is not None:
                notes.append(f"the reviews pass stopped early: {e}")
        finally:
            if auth:
                auth.close()
            if page and not self._browser.cdp.closed:
                page.close()
            if ctx and not self._browser.cdp.closed:
                self._browser.dispose(ctx)
        return n


# ── Website pass (email, socials, verify_emails): each business homepage over plain HTTP ──
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}")
JUNK_SUFFIX = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".css", ".js")
JUNK_DOMAIN = ("example", "sentry", "wixpress.com", "domain.com", "yourdomain", "email.com")


def junk_domain(dom, junk=JUNK_DOMAIN):
    """An entry with a dot is a domain and its subdomains, one without is a whole label: sentry.io
    and [email protected] are junk, sentryinsurance.com and myemail.com are not. scripts/scrape.py has the same."""
    labels = dom.split(".")
    return any(dom == j or dom.endswith("." + j) if "." in j else j in labels for j in junk)
UA = "Mozilla/5.0 (compatible; google-maps-scraper-kit; +https://github.com/Mahanaicoach/google-maps-scraper-kit)"
# The same patterns as scripts/scrape.py --socials (a test keeps them in step).
SOCIAL_RE = {
    "instagram": re.compile(r"https?://(?:www\.)?instagram\.com/([A-Za-z0-9_.]+)", re.I),
    "facebook":  re.compile(r"https?://(?:www\.|m\.|web\.)?facebook\.com/([A-Za-z0-9_.\-]+)", re.I),
    "linkedin":  re.compile(r"https?://(?:[a-z]{2,3}\.)?linkedin\.com/(?:company|in|school)/([A-Za-z0-9_.\-%]+)", re.I),
    "tiktok":    re.compile(r"https?://(?:www\.)?tiktok\.com/@([A-Za-z0-9_.]+)", re.I),
    "youtube":   re.compile(r"https?://(?:www\.)?youtube\.com/(?:@|channel/|c/|user/)([A-Za-z0-9_.\-]+)", re.I),
    "x":         re.compile(r"https?://(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})\b", re.I),
}
SKIP_HANDLE = {"", "home", "pages", "people", "help", "about", "policies", "policy", "legal", "tos", "privacy",
               "settings", "sharer", "tr", "profile.php", "plugins", "dialog", "intent", "login", "share.php",
               "permalink.php", "p", "reel", "reels", "explore", "stories", "tv", "watch", "events", "groups",
               "marketplace", "gaming", "photo", "hashtag", "search", "share", "i"}
# Website builders and booking or delivery platforms link their own profiles from the sites they host:
# those are not the business's. Kept the same in engine/jobs.py and scripts/scrape.py (a test checks).
SKIP_HANDLE |= {"squarespace", "wix", "wixsite", "shopify", "wordpress", "wordpressdotcom", "godaddy", "weebly",
                "webflow", "hubspot", "mailchimp", "yelp", "tripadvisor", "opentable", "doordash", "ubereats", "grubhub",
                "toasttab", "squareup"}
DOH = "https://dns.google/resolve?"  # Google's public DNS over HTTPS: no key, no account


_NAT64 = ipaddress.ip_network("64:ff9b::/96")


def _public_ip(ip):
    """True for an address a third-party website may send the fetch to: public, and not an IPv6 form
    that wraps a private IPv4 (::ffff:10.0.0.1, 64:ff9b::7f00:1 through a NAT64, ::127.0.0.1)."""
    try:
        a = ipaddress.ip_address(ip.split("%")[0])
    except ValueError:
        return False
    if a.version == 6:
        inner = a.ipv4_mapped or a.sixtofour
        if inner is None and (a in _NAT64 or 1 < int(a) < 1 << 32):  # NAT64, or the old ::a.b.c.d form
            inner = ipaddress.ip_address(int(a) & 0xFFFFFFFF)
        if inner is not None:
            a = inner
    return a.is_global and not (a.is_multicast or getattr(a, "is_site_local", False))


def _public_host(host):
    """True only if every address of `host` is public. Websites come from third-party listings, so
    one could point the fetch at this machine (the engine itself) or its network."""
    try:
        infos = socket.getaddrinfo(host, None)
    except (OSError, UnicodeError):
        return False
    return bool(infos) and all(_public_ip(i[4][0]) for i in infos)


_FETCH = threading.local()  # per thread: the hard-stop timer of the connection in progress


def _public_connection(address, *args, **kwargs):
    """create_connection that refuses a private peer: catches a DNS answer that changed (rebinding).
    The socket also gets a hard deadline (SITE_HARD s): its timeout is per read, so a website that
    drips a byte at a time would otherwise hold a crawl thread, and the engine's exit, for days."""
    sock = socket.create_connection(address, *args, **kwargs)
    if not _public_ip(sock.getpeername()[0]):
        sock.close()
        raise OSError("refused a non-public address")
    return sock


def _hard_stop(conn):
    """Arm the fetch's deadline (SITE_HARD s from fetch_window's start, so redirect hops share it);
    at the deadline the socket is shut down from a timer thread, which wakes a read blocked on a
    dripping website. fetch_window() disarms it."""
    sock = conn.sock  # taken now: urllib sends Connection: close, so the connection drops its reference
                      # once the response starts; the response's file keeps this same socket object
    def kill():
        try:
            sock.shutdown(socket.SHUT_RDWR)  # wakes a read blocked in the fetch thread with EOF; the fetch
        except OSError:  # thread closes the socket itself (close() here would wait on its buffer lock)
            pass
    _cancel_hard_stop()
    left = getattr(_FETCH, "deadline", None)
    t = threading.Timer(max(0.1, left - time.monotonic()) if left else SITE_HARD, kill)
    t.daemon = True
    t.start()
    _FETCH.timer = t


class fetch_window:
    """`with fetch_window():` around one website or DNS request: one SITE_HARD deadline for all its
    connections (redirect hops too), and the timer disarmed on every exit."""
    def __enter__(self):
        _FETCH.deadline = time.monotonic() + SITE_HARD

    def __exit__(self, *exc):
        _cancel_hard_stop()
        _FETCH.deadline = None


def _cancel_hard_stop():
    t = getattr(_FETCH, "timer", None)
    if t is not None:
        t.cancel()
        _FETCH.timer = None


def _read_capped(resp, cap, seconds):
    """Up to cap bytes of a response, for at most about `seconds`: read1 returns what has arrived
    (read(n) would block for all n bytes), and a connection the hard stop cut keeps what came."""
    out, end = bytearray(), time.monotonic() + seconds
    while len(out) < cap and time.monotonic() < end:
        try:
            chunk = resp.read1(min(1 << 14, cap - len(out)))
        except (OSError, ValueError, http.client.HTTPException):  # cut off, or the TLS object torn down
            break
        if not chunk:
            break
        out += chunk
    return bytes(out)


def ascii_url(url):
    """The URL with a non-ASCII path or query percent-encoded: http.client refuses to send them raw."""
    try:
        p = urllib.parse.urlsplit(url)
        if p.path.isascii() and p.query.isascii():
            return url
        return urllib.parse.urlunsplit((p.scheme, p.netloc, urllib.parse.quote(p.path, safe="/%:@&=+$,;~!*'()"),
                                        urllib.parse.quote(p.query, safe="=&%+/?:@;,!*'()$"), ""))
    except ValueError:
        return url


class _HTTP(http.client.HTTPConnection):
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self._create_connection = _public_connection

    def connect(self):
        super().connect()
        _hard_stop(self)


class _HTTPS(http.client.HTTPSConnection):
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self._create_connection = _public_connection

    def connect(self):
        super().connect()
        _hard_stop(self)


class _HTTPHandler(urllib.request.HTTPHandler):
    def http_open(self, req):
        return self.do_open(_HTTP, req)


class _HTTPSHandler(urllib.request.HTTPSHandler):
    def https_open(self, req):
        return self.do_open(_HTTPS, req, context=self._context)


class _Redirects(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        if urllib.parse.urlsplit(newurl).scheme not in ("http", "https"):
            return None  # urllib's ftp handler would connect past the peer check
        if not _public_host(urllib.parse.urlsplit(newurl).hostname or ""):
            return None
        return super().redirect_request(req, fp, code, msg, headers, newurl)


# No proxy from the environment: these go straight to the business websites.
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}), _HTTPHandler, _HTTPSHandler, _Redirects)


def fetch(url, timeout=SITE_TIMEOUT):
    """Up to 400 KB of a public web page as text; "" on any failure."""
    if not url.startswith(("http://", "https://")):
        url = "http://" + url
    try:
        if not _public_host(urllib.parse.urlsplit(url).hostname or ""):
            return ""
        r = urllib.request.Request(ascii_url(url), headers={"User-Agent": UA, "Accept": "text/html"})
        with fetch_window(), _OPENER.open(r, timeout=timeout) as resp:
            return _read_capped(resp, 400_000, 2 * timeout).decode("utf-8", "replace")
    except Exception:  # noqa: BLE001 - a broken website is just a website without emails
        return ""


def site_host(url):
    try:
        h = (urllib.parse.urlsplit(url if "://" in url else "http://" + url).hostname or "").lower()
    except ValueError:
        return ""
    return h[4:] if h.startswith("www.") else h


def find_socials(text):
    """The first profile link per network on a page, share and login links skipped."""
    out = dict.fromkeys(SOCIALS, "")
    for net, rx in SOCIAL_RE.items():
        for m in rx.finditer(text or ""):
            h = m.group(1).lower()
            if h in SKIP_HANDLE or (net == "facebook" and (h.isdigit() or len(h) < 3)):
                continue
            out[net] = m.group(0).rstrip("\"'/").replace("\\", "")
            break
    return out


_MX = {}  # domain -> answer; a failed lookup is not kept, so the next job asks again


def mx_host(domain):
    """'nxdomain' when the domain does not exist, 'none' when it has no MX record, else who runs its
    mail: 'google_workspace', 'microsoft_365' or 'other'; '' when the lookup failed. Like scrape.py."""
    if domain in _MX:
        return _MX[domain]
    try:
        with fetch_window(), _OPENER.open(urllib.request.Request(
                DOH + urllib.parse.urlencode({"name": domain, "type": "MX"}), headers={"User-Agent": UA}), timeout=8) as r:
            ans = json.load(r)
    except Exception:  # noqa: BLE001 - no answer is not a verdict
        return ""
    mx = " ".join(a.get("data", "") for a in ans.get("Answer") or [] if a.get("type") == 15).lower()
    if ans.get("Status") == 3:
        host = "nxdomain"
    elif not mx:
        host = "none" if ans.get("Status") == 0 else ""
    elif "google.com" in mx or "googlemail.com" in mx:
        host = "google_workspace"
    else:
        host = "microsoft_365" if "outlook.com" in mx else "other"
    if host:
        if len(_MX) > 4096:  # ponytail: forget all at the cap; an LRU would keep the busy domains
            _MX.clear()
        _MX[domain] = host
    return host


def find_emails(text, site=""):
    """Up to 3 addresses from page text, placeholders and image names dropped, the site's own first."""
    found = []
    for e in EMAIL_RE.findall(html.unescape(text or "")):
        e = urllib.parse.unquote(e).lower()
        dom = e.rpartition("@")[2]
        if not EMAIL_RE.fullmatch(e) or e.endswith(JUNK_SUFFIX) or junk_domain(dom) or e in found:
            continue  # "a%[email protected]" unquotes to a newline: not an address
        found.append(e)
    own = lambda e: site and (e.endswith("@" + site) or e.endswith("." + site))
    return sorted(found, key=lambda e: not own(e))[:3]


def add_site_data(rows, seconds, email=True, socials=False, verify=False, get=None, stop=lambda: False, browser_sites=False):
    """HTTP lookups first; optional browser retries use separate workers and the remaining budget."""
    get = get or fetch
    end = time.monotonic() + seconds
    if browser_sites:
        from website_browser import Budget, merge_status

    def extract(r, page):
        out = {}
        if email and not r.get("emails"):
            found = find_emails(page, site_host(r["website"]))
            if verify:
                hosts = [(e, mx_host(e.rpartition("@")[2])) for e in found]
                found = [e for e, h in hosts if h != "nxdomain"]
                out["email_host"] = next((h for e, h in hosts if h != "nxdomain"), "")
            out["emails"] = ", ".join(found)
        if socials:
            out.update(find_socials(page))
        return out

    def http_work(r):
        page = get(r["website"])
        out = extract(r, page)
        if browser_sites:
            out["website_fetch_status"] = "http" if page else "error"
        return out

    enriched = set()
    def run(todo, work, workers, deadline, timeout_status=None):
        pool = concurrent.futures.ThreadPoolExecutor(workers)
        futs = {pool.submit(work, r): r for r in todo}
        pending = set(futs)
        while pending and not stop() and time.monotonic() < deadline:
            _, pending = concurrent.futures.wait(pending, timeout=min(1, max(0, deadline - time.monotonic())))
        done = set(futs) - pending
        if timeout_status:
            for f in pending:
                futs[f]["website_fetch_status"] = merge_status(futs[f].get("website_fetch_status"), timeout_status)
        pool.shutdown(wait=False, cancel_futures=True)
        completed = []
        for f in done:
            if not f.exception():
                got = {k: v for k, v in f.result().items() if v}
                r = futs[f]
                r.update(got)
                completed.append(r)
                if any(k != "website_fetch_status" for k in got):
                    enriched.add(id(r))
        return completed, len(pending)

    todo = [r for r in rows if r.get("website")]
    completed, pending = run(todo, http_work, SITE_WORKERS, end, "timeout" if browser_sites else None)
    if browser_sites and not stop():
        with Budget(max(0, min(30, end - time.monotonic()))) as budget:
            def retry(r):
                rendered = budget.fetch(r["website"], _public_ip)
                out = extract(r, rendered["html"])
                out["website_fetch_status"] = merge_status(r.get("website_fetch_status"), rendered["status"])
                return out
            missing = [r for r in completed if (email and not r.get("emails"))
                       or (socials and not any(r.get(k) for k in SOCIALS))]
            # HTTP already checked these rows: retries the budget cut off are not "not checked".
            run(missing, retry, 2, min(end, time.monotonic() + 30), "budget_exhausted")
    return len(enriched), pending