Skip to content

Free, open-source Google Maps scraper for lead generation (Python, no API key, no Docker). Exports name, phone, email, website, rating and Instagram/Facebook/LinkedIn/TikTok/YouTube/X links to CSV. 100 places with emails in 44 s; --grid beats Google's 120 cap. MX email check, ad-pixel detection. Skill for Claude Code, Codex and Cursor.

91.8 KB
Raw Blame
#!/usr/bin/env python3
"""Offline checks for the engine: job body validation, CSV rows, the sqlite store, the worker
(with a fake Chrome and fake Google results), the HTTP server and its guards, scripts/scrape.py's
own client against that server, the Chrome download's SHA-256 check and the email pass.
No network, no Chrome, no Docker.

    python3 engine/test_engine.py
"""
import csv, hashlib, http.client, importlib.util, io, json, os, queue, re, shutil, socket, sqlite3, subprocess, sys
import tempfile
import threading, time, unittest, unittest.mock, urllib.parse, urllib.request, uuid, zipfile

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import browser, details, jobs, maps, serve  # noqa: E402
from cdp import CDPError  # noqa: E402

GOOD = {"name": "t", "keywords": ["cafes in Austin"], "lang": "en", "lat": "30.2", "lon": "-97.7", "max_time": 60}


def place(n, **kw):
    return dict({"title": f"Biz {n}", "data_id": f"0x1:0x{n:x}", "place_id": f"ChIJ{n}", "cid": str(n),
                 "category": "Cafe", "review_rating": 4.5, "review_count": 10 + n,
                 "latitude": 30.2, "longitude": -97.7}, **kw)


class FakeCDP:
    closed = None


class FakeBrowser:
    """Stands in for browser.Browser: no process; kill() makes the next collect fail like a dead Chrome."""
    made = []

    path = "/usr/bin/google-chrome"

    def __init__(self, **kw):
        self.kw, self.cdp, self.killed, self.contexts = kw, FakeCDP(), threading.Event(), []
        FakeBrowser.made.append(self)

    def context(self, proxy=None):
        self.contexts.append(proxy)
        return f"ctx{len(self.contexts)}"

    def dispose(self, ctx):
        pass

    def kill(self):
        self.cdp.closed = OSError("killed")
        self.killed.set()

    def close(self):
        pass


class FakePage:
    def __init__(self, cdp, context=None):
        pass

    def set_cookie(self, *a):
        pass

    def close(self):
        pass


class FakeAuth:
    """Stands in for browser.ProxyAuth: records which tabs got credentials."""
    logins = []
    refused = False

    def __init__(self, page, user, password):
        FakeAuth.logins.append((user, password))

    def close(self):
        pass


RESULTS = {}  # keyword -> (places, reason) for the fake collect; a list of reasons answers one per search
SEARCHED = []  # the keywords the fake collect was asked for, in order


def fake_collect(page, url, depth=1, max_seconds=0, log=None, stop=lambda: False, on_batch=None):
    for kw, (places, reason) in RESULTS.items():
        if maps.search_url(kw).split("/@")[0].split("?")[0] in url:
            SEARCHED.append(kw)
            if isinstance(reason, list):
                reason = reason.pop(0) if len(reason) > 1 else reason[0]
            if reason == "hang":  # Chrome stops answering until the watchdog kills it
                b = FakeBrowser.made[-1]
                b.killed.wait(10)
                raise CDPError("connection lost: killed")
            if reason == "slow":  # its places stream in, then it scrolls until stopped
                if on_batch and places:
                    on_batch(places, 1)
                end = time.time() + 5
                while not stop() and time.time() < end:
                    time.sleep(0.02)
                return places, "stopped", {"batches": 1, "dom_links": 0}
            return places, reason, {"batches": 1, "dom_links": 0}
    return [], "no-feed", {"batches": 0, "dom_links": 0}


class Fakes(unittest.TestCase):
    def setUp(self):
        self.dir = tempfile.mkdtemp()
        self.saved = (browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth)
        browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth = FakeBrowser, FakePage, fake_collect, FakeAuth
        FakeBrowser.made.clear()
        FakeAuth.logins.clear()
        RESULTS.clear()
        SEARCHED.clear()
        self.store = jobs.Store(os.path.join(self.dir, "data"))
        self.lines = []
        self.worker = jobs.Worker(self.store, self.lines.append)

    def tearDown(self):
        browser.Browser, jobs.Page, maps.collect, browser.ProxyAuth = self.saved
        self.worker.shutdown()
        if self.worker.is_alive():
            self.worker.join(5)
        self.store.db.close()
        shutil.rmtree(self.dir, ignore_errors=True)

    def run_one(self, **body):
        name, data = jobs.validate(dict(GOOD, **body))
        jid = self.store.create(name, data)
        self.worker.run_job(self.store.claim())
        return jid, self.store.get(jid)

    def rows(self, jid):
        return list(csv.DictReader(io.StringIO(self.store.read_csv(jid).decode("utf-8"))))


class Validate(unittest.TestCase):
    def err(self, **body):
        with self.assertRaises(ValueError) as c:
            jobs.validate(body)
        return str(c.exception)

    def test_old_engine_messages(self):
        self.assertEqual(self.err(), "missing name")
        self.assertEqual(self.err(name="x"), "missing keywords")
        self.assertEqual(self.err(name="x", keywords=["a"]), "missing lang")
        self.assertEqual(self.err(name="x", keywords=["a"], lang="eng"), "invalid lang")
        self.assertEqual(self.err(name="x", keywords=["a"], lang="en\n"), "invalid lang")
        self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lat="1"), "missing geo coordinates")
        self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lon="2"), "missing geo coordinates")
        self.assertEqual(self.err(name="x", keywords=["a"], lang="en"), "missing max time")  # neither: allowed
        self.assertEqual(self.err(name="x", keywords=["a"], lang="en", lat="1", lon="2"), "missing max time")

    def test_bad_values(self):
        self.assertIn("invalid geo", self.err(**dict(GOOD, lat="north")))
        self.assertIn("invalid geo", self.err(**dict(GOOD, lat="91")))
        self.assertIn("invalid geo", self.err(**dict(GOOD, lon="nan")))
        self.assertIn("invalid geo", self.err(**dict(GOOD, lat=True)))
        self.assertIn("invalid depth", self.err(**dict(GOOD, depth=0)))
        self.assertIn("invalid depth", self.err(**dict(GOOD, depth="5")))
        self.assertIn("invalid max time", self.err(**dict(GOOD, max_time=-1)))
        self.assertIn("invalid keywords", self.err(**dict(GOOD, keywords=["ok", ""])))
        self.assertIn("SOCKS5", self.err(**dict(GOOD, proxies=["socks5://u:p@h:1"])))
        self.assertIn("invalid country", self.err(**dict(GOOD, country="usa")))
        self.assertIn("invalid reviews_max", self.err(**dict(GOOD, reviews_max=6)))
        self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["ftp://h:1"])))
        self.assertIn("invalid proxy", self.err(**dict(GOOD, proxies=["http://h:99999"])))
        with self.assertRaisesRegex(ValueError, "JSON object"):
            jobs.validate([1])

    def test_defaults_and_numbers(self):
        name, d = jobs.validate(dict(GOOD, lat=30.5, lon=-97, proxies=["socks5://127.0.0.1:1080"]))
        self.assertEqual((name, d["lat"], d["lon"], d["depth"], d["zoom"], d["email"]), ("t", "30.5", "-97", 1, 15, False))
        self.assertEqual(d["proxies"], ["socks5://127.0.0.1:1080"])
        self.assertEqual((d["country"], d["reviews_max"], d["socials"], d["verify_emails"]), ("", 0, False, False))
        d = jobs.validate(dict(GOOD, country="DE", proxies=["http://u%40x:p%3A@h:1"]))[1]
        self.assertEqual((d["country"], jobs.locale(d), d["proxies"]), ("de", "en-DE", ["http://u%40x:p%3A@h:1"]))
        self.assertEqual(jobs.mask("http://u%40x:p%3A@h:1"), "http://u%40x:***@h:1")
        self.assertEqual(jobs.mask("socks5://h:1"), "socks5://h:1")
        self.assertEqual(jobs.mask("http://[abc"), "***")  # a 1.x job's proxy validate() never saw
        self.assertEqual(jobs.validate(dict(GOOD, radius=5_000_000))[1]["radius"], 5_000_000)
        self.assertEqual(self.err(name="x", keywords=["a"], lang="en", max_time=60, lat=10 ** 400, lon=1),
                         "invalid geo coordinates")  # float() overflows: a 422, not a dropped connection


class Rows(unittest.TestCase):
    def test_columns_and_mapping(self):
        r = jobs.to_row(dict(place(1), open_state="Open 24 hours", description="Good coffee",
                             categories="Cafe | Bakery", junk="x"), "cafes")
        self.assertEqual(len(jobs.COLUMNS), 36)
        self.assertEqual(list(r), jobs.COLUMNS + jobs.EXTRA)
        self.assertEqual((r["input_id"], r["status"], r["descriptions"], r["categories"]),
                         ("cafes", "Open 24 hours", "Good coffee", "Cafe | Bakery"))
        self.assertEqual((r["review_rating"], r["review_count"], r["emails"]), ("4.5", "11", ""))
        self.assertTrue(r["link"].startswith("https://www.google.com/maps/search/?api=1&query=Biz%201&query_place_id=ChIJ1"))
        dom = jobs.to_row({"title": "A", "link": "https://www.google.com/maps/place/A/data=!1s0x1:0x2"}, "k")
        self.assertEqual(dom["link"], "https://www.google.com/maps/place/A/data=!1s0x1:0x2")

    def test_csv_utf8_no_bom(self):
        raw = jobs.to_csv([jobs.to_row(place(1, title="Café ☕"), "k")])
        self.assertFalse(raw.startswith(b"\xef\xbb\xbf"))
        self.assertEqual(next(csv.DictReader(io.StringIO(raw.decode())))["title"], "Café ☕")

    def test_lone_surrogate_is_written_not_fatal(self):
        cut = json.loads('"Caf\\ud83d"')  # a name Google cut mid-emoji
        row = jobs.to_row(place(1, title=cut), "k")  # place_id set, so the title is URL-quoted too
        self.assertEqual(row["title"], "Caf?")
        raw = jobs.to_csv([dict(row, user_reviews=cut)])  # reviews are set after to_row
        self.assertIn(b"Caf?", raw)


class Store(Fakes):
    def test_lifecycle(self):
        a = self.store.create("a", {"x": 1})
        b = self.store.create("b", {"x": 2})
        self.assertEqual([j["ID"] for j in self.store.all()], [b, a])  # newest first
        self.assertEqual(self.store.claim()["ID"], a)  # oldest pending first
        self.assertEqual(self.store.get(a)["Status"], "working")
        self.assertEqual(self.store.recover(), (1, 0))  # resumed on the next start
        self.assertEqual((self.store.get(a)["Status"], self.store.get(a)["Progress"]["resumes"]), ("pending", 1))
        self.assertEqual(self.store.append(b, [], jobs.fields({})), len(",".join(jobs.COLUMNS + jobs.EXTRA)) + 2)
        self.assertTrue(self.store.read_csv(b).startswith(b"input_id,link,title"))
        self.assertTrue(self.store.delete(b))
        self.assertIsNone(self.store.read_csv(b))
        self.assertIsNone(self.store.append(b, [], jobs.fields({})))  # a deleted job gets no file back
        self.assertFalse(self.store.finish(b, [], jobs.fields({}), "ok", "", {}))
        self.assertFalse(self.store.delete(b))
        self.assertEqual(set(self.store.get(a)), {"ID", "Name", "Date", "Status", "Data", "Error", "Progress"})
        self.assertRegex(self.store.get(a)["Date"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\+00:00$")

    def test_import_kit1(self):
        # Kit 1.x's data folder as its engine leaves it: jobs.db in WAL mode, rows still in the WAL
        # (the writer is open), <id>.csv beside it.
        old = os.path.join(self.dir, "kit1")
        os.makedirs(old)
        w = sqlite3.connect(os.path.join(old, "jobs.db"))
        w.execute("PRAGMA journal_mode=WAL")
        w.execute("CREATE TABLE jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, status TEXT NOT NULL, "
                  "data TEXT NOT NULL, created_at INT NOT NULL, updated_at INT NOT NULL)")
        done, stuck, nocsv, baddate, big = (str(uuid.uuid4()) for _ in range(5))
        body = json.dumps({"keywords": ["cafe", "bar"], "lang": "en", "max_time": 180 * 10 ** 9})
        with w:
            for jid, status in ((done, "ok"), (stuck, "working"), (nocsv, "ok"), ("../evil", "ok"), (big, "ok")):
                w.execute("INSERT INTO jobs VALUES (?, ?, ?, ?, 1793466309, 1793466322)", (jid, "old " + status, status, body))
            w.execute("INSERT INTO jobs VALUES (?, 'odd', 'ok', ?, 'not a time', 0)", (baddate, body))
        for jid in (done, stuck, "../evil", baddate):
            with open(os.path.join(old, f"{jid}.csv"), "w", newline="") as f:
                f.write("input_id,title\n1,\"A\nB\"\n2,C\n")
        with open(os.path.join(old, f"{big}.csv"), "w", newline="") as f:
            f.write("input_id,descriptions\n1," + "x" * 200_000 + "\n")  # past csv's 128 KB field limit
        logged = []
        try:
            self.assertEqual(self.store.import_kit1(old, logged.append), 3)
        finally:
            w.close()
        self.assertEqual(logged, [f"kit 1.x job {baddate} not imported: ValueError"])  # skipped, the rest kept
        self.assertIsNone(self.store.get(baddate))
        self.assertFalse(os.path.exists(self.store.csv_path(baddate)))
        self.assertEqual(jobs.public(self.store.get(big))["Progress"]["places"], 1)
        a, b = self.store.get(done), self.store.get(stuck)
        self.assertEqual((a["Status"], a["Name"], a["Date"], a["Data"]["max_time"]), ("ok", "old ok", "2026-10-31T17:05:09+00:00", 180))
        self.assertEqual(jobs.public(a)["Progress"], {"keywords_done": 2, "keywords_total": 2, "places": 2, "percent": 100, "stage": "done"})
        self.assertEqual((b["Status"], b["Error"]), ("failed", "kit 1.x job that was working there; its rows are kept"))
        self.assertEqual(self.store.read_csv(done), b"input_id,title\n1,\"A\nB\"\n2,C\n")
        self.assertIsNone(self.store.get(nocsv))
        self.assertIsNone(self.store.get("../evil"))  # not a job id: skipped
        self.store.delete(done)
        self.assertEqual(self.store.import_kit1(old), 0)  # once only: a deleted job stays deleted
        self.assertIsNone(self.store.get(done))
        self.assertEqual(self.store.import_kit1(os.path.join(self.dir, "none")), 0)


class Worker(Fakes):
    def test_keywords_dedupe_into_one_csv(self):
        RESULTS["cafes"] = ([place(1), place(2)], "end-of-list")
        RESULTS["bakeries"] = ([place(2), place(3)], "depth")
        jid, job = self.run_one(keywords=["cafes", "bakeries"], depth=2, lang="de", proxies=["http://10.0.0.1:3128"])
        self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
        rows = self.rows(jid)
        self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2", "Biz 3"])
        self.assertEqual([r["input_id"] for r in rows], ["cafes", "cafes", "bakeries"])
        self.assertEqual(FakeBrowser.made[0].kw, {"lang": "de", "headless": True})
        self.assertEqual(FakeBrowser.made[0].contexts, ["http://10.0.0.1:3128"] * 2)
        self.assertFalse(any(re.search(r"Biz \d", l) for l in self.lines))  # counts only in the log

    def test_no_results_is_ok_with_header(self):
        jid, job = self.run_one(keywords=["nothing here"])
        self.assertEqual(job["Status"], "ok")
        self.assertEqual(self.rows(jid), [])

    def test_no_chrome_fails_with_reason(self):
        def no_chrome(**kw):
            raise browser.BrowserError("No Chrome, Chromium, Edge or Brave found.")
        browser.Browser = no_chrome
        _, job = self.run_one()
        self.assertEqual(job["Status"], "failed")
        self.assertIn("No Chrome", job["Error"])

    def test_no_feed_is_tried_once_more(self):
        RESULTS["cafes in Austin"] = ([place(1)], ["no-feed", "end-of-list"])  # once, then the list
        _, job = self.run_one()
        self.assertEqual((job["Status"], SEARCHED.count("cafes in Austin"), job["Error"]), ("ok", 2, ""))
        self.assertEqual(job["Progress"]["places"], 1)
        SEARCHED.clear()
        RESULTS["cafes in Austin"] = ([], "no-feed")  # twice: a note, no third look
        _, job = self.run_one()
        self.assertEqual((job["Status"], SEARCHED.count("cafes in Austin")), ("ok", 2))
        self.assertIn("no result list twice", job["Error"])

    def test_blocked_fails(self):
        RESULTS["cafes in Austin"] = ([], "blocked")
        _, job = self.run_one()
        self.assertEqual(job["Status"], "failed")
        self.assertIn("unusual-traffic", job["Error"])

    def test_block_stops_the_job(self):
        RESULTS["cafes"] = ([place(1)], "end-of-list")
        RESULTS["bakeries"] = ([], "blocked")
        RESULTS["florists"] = ([place(9)], "end-of-list")
        jid, job = self.run_one(keywords=["cafes", "bakeries", "florists"])
        self.assertEqual(job["Status"], "ok")  # the rows from before the block are kept
        self.assertIn("unusual-traffic", job["Error"])
        self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1"])  # florists never searched

    def test_deleted_before_it_starts(self):
        name, data = jobs.validate(GOOD)
        jid = self.store.create(name, data)
        job = self.store.claim()
        self.store.delete(jid)
        self.worker.run_job(job)
        self.assertEqual(FakeBrowser.made, [])  # no Chrome for a job that is gone

    def test_worker_survives_a_store_error(self):
        calls = []
        def claim():
            calls.append(1)
            if len(calls) == 1:
                raise OSError("disk full")
            return None
        self.store.claim = claim
        self.worker.wake.set()
        self.worker.start()
        deadline = time.time() + 5
        while len(calls) < 2 and time.time() < deadline:
            time.sleep(0.05)
        self.assertGreaterEqual(len(calls), 2)
        self.assertTrue(self.worker.is_alive())

    def test_email_pass_stops_when_asked(self):
        rows = [{"website": f"https://s{i}.com", "emails": ""} for i in range(3)]
        t = time.monotonic()
        jobs.add_site_data(rows, 30, get=lambda url: time.sleep(5) or "", stop=lambda: time.monotonic() - t > 0.3)
        self.assertLess(time.monotonic() - t, 3)

    def test_watchdog_kills_a_hung_chrome(self):
        RESULTS["cafes in Austin"] = ([], "hang")
        saved, jobs.WATCHDOG_GRACE = jobs.WATCHDOG_GRACE, 0.2
        try:
            t = time.monotonic()
            _, job = self.run_one(max_time=1)
        finally:
            jobs.WATCHDOG_GRACE = saved
        self.assertLess(time.monotonic() - t, 5)
        self.assertTrue(FakeBrowser.made[0].killed.is_set())
        self.assertEqual(job["Status"], "failed")

    def test_delete_while_running(self):
        RESULTS["cafes in Austin"] = ([place(1)], "slow")
        name, data = jobs.validate(GOOD)
        jid = self.store.create(name, data)
        t = threading.Thread(target=self.worker.run_job, args=(self.store.claim(),))
        t.start()
        time.sleep(0.2)
        self.worker.cancel(jid)
        self.store.delete(jid)
        t.join(5)
        self.assertFalse(t.is_alive())
        self.assertIsNone(self.store.get(jid))
        self.assertFalse(os.path.exists(self.store.csv_path(jid)))

    def test_email_pass(self):
        RESULTS["cafes in Austin"] = ([place(1, website="https://www.biz1.com"), place(2), place(3, website="x.org")], "end-of-list")
        pages = {"https://www.biz1.com": 'mail <a href="mailto:[email protected]">us</a> or sales&#64;biz1.com',
                 "x.org": "[email protected] nothing"}
        saved = jobs.fetch
        jobs.fetch = lambda url, timeout=0: pages.get(url, "")
        try:
            jid, job = self.run_one(email=True)
        finally:
            jobs.fetch = saved
        self.assertEqual(job["Status"], "ok")
        self.assertEqual([r["emails"] for r in self.rows(jid)], ["[email protected], [email protected]", "", ""])
        self.assertNotIn("instagram", self.rows(jid)[0])  # columns only for the passes a job asked for


class Emails(unittest.TestCase):
    def test_find_emails(self):
        text = "[email protected] x [email protected] [email protected] [email protected] [email protected] [email protected]"
        self.assertEqual(jobs.find_emails(text, "shop.com"), ["[email protected]", "[email protected]"])
        self.assertEqual(jobs.find_emails("[email protected] [email protected] [email protected] [email protected]"),
                         ["[email protected]", "[email protected]"])  # whole labels, not substrings

    def test_public_only(self):
        self.assertFalse(jobs._public_ip("127.0.0.1"))
        self.assertFalse(jobs._public_ip("192.168.1.1"))
        self.assertFalse(jobs._public_ip("169.254.169.254"))
        self.assertFalse(jobs._public_ip("::1"))
        self.assertTrue(jobs._public_ip("8.8.8.8"))
        self.assertFalse(jobs._public_host("localhost"))
        self.assertEqual(jobs.fetch("http://127.0.0.1:9/"), "")
        self.assertEqual(jobs.site_host("https://www.Shop.com/x"), "shop.com")

    def test_private_peer_and_redirect_refused(self):
        srv = socket.socket()
        srv.bind(("127.0.0.1", 0))
        srv.listen(1)
        with self.assertRaisesRegex(OSError, "non-public"):
            jobs._public_connection(srv.getsockname())
        srv.close()
        import urllib.request
        req = urllib.request.Request("https://example.com/")
        self.assertIsNone(jobs._Redirects().redirect_request(req, None, 302, "Found", {}, "http://127.0.0.1:8080/api/v1/jobs"))
        self.assertIsNone(jobs._Redirects().redirect_request(req, None, 302, "Found", {}, "http://169.254.169.254/"))


class Server(Fakes):
    key = ""

    def setUp(self):
        super().setUp()
        self.servers = serve.make_servers(0, self.store, self.worker, self.key)
        self.port = self.servers[0].server_address[1]
        for s in self.servers:
            threading.Thread(target=s.serve_forever, daemon=True).start()

    def tearDown(self):
        for s in self.servers:
            s.shutdown()
            s.server_close()
        super().tearDown()

    def call(self, method, path, body=None, headers=None, raw=None):
        c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
        h = {"Host": f"localhost:{self.port}"}
        h.update(headers or {})
        data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
        c.request(method, path, data, h)
        r = c.getresponse()
        out = (r.status, r.read(), r)
        c.close()
        return out


class Api(Server):
    def test_flow(self):
        code, body, r = self.call("GET", "/api/v1/jobs")
        self.assertEqual((code, body), (200, b"null"))  # the docs' health check prints null
        self.assertTrue(r.getheader("X-Engine").startswith("gmskit-engine/"))
        code, body, _ = self.call("POST", "/api/v1/jobs", {})
        self.assertEqual((code, json.loads(body)), (422, {"code": 422, "message": "missing name"}))
        code, body, _ = self.call("POST", "/api/v1/jobs", GOOD)
        self.assertEqual(code, 201)
        jid = json.loads(body)["id"]
        self.assertTrue(self.worker.wake.is_set())
        job = json.loads(self.call("GET", f"/api/v1/jobs/{jid}")[1])
        self.assertEqual((job["ID"], job["Name"], job["Status"], job["Data"]["keywords"]), (jid, "t", "pending", ["cafes in Austin"]))
        self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download")[0], 404)
        self.store.append(jid, [jobs.to_row(place(1), "k")], jobs.fields({}))
        code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download")
        self.assertEqual((code, r.getheader("Content-Type")), (200, "text/csv; charset=utf-8"))
        self.assertEqual(r.getheader("Content-Disposition"), f'attachment; filename="t-{jid[:8]}.csv"')  # after the search
        self.assertEqual(serve.file_name({"Name": "Bäckerei in München!"}, "c063a815-x"), "backerei-in-munchen-c063a815")
        self.assertEqual(self.call("GET", f"/download?id={jid}")[1], body)
        self.assertEqual(len(json.loads(self.call("GET", "/api/v1/jobs")[1])), 1)
        self.assertEqual(self.call("DELETE", f"/api/v1/jobs/{jid}")[0], 200)
        self.assertEqual(self.call("DELETE", f"/api/v1/jobs/{jid}")[0], 404)
        self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}")[0], 404)
        self.assertEqual(self.call("GET", "/download?id=../../etc/passwd")[0], 404)

    def test_guards(self):
        self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": "evil.com"})[0], 421)
        self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": "localhost.evil.com"})[0], 421)
        self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "https://evil.com"})[0], 403)
        self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "null"})[0], 403)
        self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "http://localhost.evil.com"})[0], 403)
        self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": "http://localhost:3000"})[0], 403)
        self.assertEqual(self.call("POST", "/api/v1/jobs", GOOD, {"Origin": f"http://localhost:{self.port}"})[0], 201)
        self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"Host": f"127.0.0.1:{self.port}"})[0], 200)
        self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"{nope")[0], 400)
        self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"\xff\xfe")[0], 400)
        self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"", headers={"Content-Length": "99999999"})[0], 413)
        self.assertEqual(self.call("POST", "/api/v1/jobs", raw=b"0\r\n\r\n", headers={"Transfer-Encoding": "chunked"})[0], 411)
        self.assertEqual(self.call("POST", "/elsewhere", GOOD)[0], 404)

    def test_refused_post_closes_the_connection(self):
        c = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
        c.request("POST", "/elsewhere", json.dumps(GOOD), {"Host": "localhost"})
        r = c.getresponse()
        r.read()
        self.assertEqual((r.status, r.getheader("Connection")), (404, "close"))
        c.close()

    def test_page_and_docs(self):
        code, body, r = self.call("GET", "/")
        csp = r.getheader("Content-Security-Policy")
        nonce = re.search(r"'nonce-([^']+)'", csp).group(1)
        self.assertIn(f'<script nonce="{nonce}">', body.decode())
        self.assertIn("github.com/Mahanaicoach/google-maps-scraper-kit", body.decode())
        self.assertIn("frame-ancestors 'none'", csp)
        # sponsors.json from raw.githubusercontent.com and the latest release from api.github.com, nothing else
        self.assertIn("connect-src 'self' https://raw.githubusercontent.com https://api.github.com;", csp)
        self.assertIn("assets/sponsors/sponsors.json", body.decode())
        self.assertIn("/releases/latest", body.decode())
        self.assertNotRegex(body.decode(), r"\{[a-z_]+\}")  # every placeholder filled
        code, body, _ = self.call("GET", "/api/openapi.json")
        self.assertEqual(json.loads(body)["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]
                         ["schema"]["required"], ["name", "keywords", "lang", "max_time"])


class ApiKey(Server):
    key = "s3cret"

    def test_key(self):
        self.assertEqual(self.call("GET", "/api/v1/jobs")[0], 401)
        self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"X-API-Key": "wrong"})[0], 401)
        self.assertEqual(self.call("GET", "/api/v1/jobs", headers={"X-API-Key": "s3cret"})[0], 200)
        self.assertEqual(self.call("GET", "/")[0], 200)  # the page holds no data; it asks for the key


class Bind(Fakes):
    """--host 0.0.0.0 is for the Docker image; elsewhere the engine keeps to loopback."""

    def test_any_address_has_no_ipv6_twin(self):
        # Mocked: a real 0.0.0.0 bind would open a port to the network on the machine running the tests.
        with unittest.mock.patch.object(serve, "Server") as v4, unittest.mock.patch.object(serve, "Server6") as v6:
            serve.make_servers(0, self.store, self.worker, host="0.0.0.0")
        v4.assert_called_once_with(("0.0.0.0", 0), serve.Handler)
        v6.assert_not_called()

    def test_any_address_refused_outside_a_container(self):
        argv = ["serve.py", "--host", "0.0.0.0", "--port", "0", "--data", self.dir]
        with unittest.mock.patch.object(serve, "in_container", return_value=False), \
             unittest.mock.patch.object(sys, "argv", argv), self.assertRaises(SystemExit) as cm:
            serve.main()
        self.assertIn("Docker image only", str(cm.exception))


class ChromeStartErrors(unittest.TestCase):
    """scripts/scrape.py names Chrome start errors by their first words (and never retries them):
    the engine's messages must keep starting that way."""

    def test_scrape_recognises_every_start_error(self):
        sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))
        import scrape
        msgs = [browser.start_error("/x/chrome", 127, "chrome: error while loading shared libraries: libnss3.so"),
                browser.start_error("/x/chrome", 1, "[1:2:FATAL:x.cc(1)] boom")]
        saved = browser.find_chrome
        browser.find_chrome = lambda: None
        try:
            for path in (None, os.path.join(tempfile.gettempdir(), "no-such-chrome-binary")):
                with self.assertRaises(browser.BrowserError) as cm:
                    browser.Browser(path=path, timeout=1)
                msgs.append(str(cm.exception))
        finally:
            browser.find_chrome = saved
        for m in msgs:
            self.assertTrue(m.startswith(scrape.CHROME_START), m)


class ScrapeClient(Server):
    """scripts/scrape.py's own create, poll and download against this engine, with the worker running."""

    def test_run_job(self):
        RESULTS["cafes in Austin"] = ([place(1), place(2)], "end-of-list")
        os.environ["SCRAPER_BASE_URL"] = f"http://localhost:{self.port}"
        sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))
        try:
            import scrape
            scrape.BASE = os.environ["SCRAPER_BASE_URL"]
            scrape.POLL_EVERY = 1
            self.worker.start()
            out = io.StringIO()
            saved, sys.stdout = sys.stdout, out
            try:
                jid, rows = scrape.run_job({"name": "t", "keywords": ["cafes in Austin"], "lang": "en",
                                            "lat": "30.2", "lon": "-97.7", "depth": 1, "email": False,
                                            "max_time": 60, "zoom": 15, "fast_mode": False, "radius": 10000})
            finally:
                sys.stdout = saved
        finally:
            del os.environ["SCRAPER_BASE_URL"]
        self.assertEqual([r["title"] for r in rows], ["Biz 1", "Biz 2"])
        self.assertEqual(self.store.get(jid)["Status"], "ok")
        for k in scrape.LEAD:
            self.assertIn(k, rows[0])


class Download(unittest.TestCase):
    def setUp(self):
        self.dir = tempfile.mkdtemp()
        self.saved = (browser.HOME, dict(browser.CFT_SHA256))
        browser.HOME = self.dir

    def tearDown(self):
        browser.HOME = self.saved[0]
        browser.CFT_SHA256.clear()
        browser.CFT_SHA256.update(self.saved[1])
        shutil.rmtree(self.dir, ignore_errors=True)

    @staticmethod
    def zipped():
        buf = io.BytesIO()
        with zipfile.ZipFile(buf, "w") as z:
            info = zipfile.ZipInfo("chrome-headless-shell-linux64/chrome-headless-shell")
            info.external_attr = 0o100755 << 16
            z.writestr(info, b"#!/bin/sh\n")
            z.writestr("chrome-headless-shell-linux64/libfoo.so", b"x")
        return buf.getvalue()

    def opener(self, data):
        return lambda url, timeout=0: io.BytesIO(data)

    def test_platforms(self):
        self.assertEqual(browser.cft_platform("Darwin", "arm64"), "mac-arm64")
        self.assertEqual(browser.cft_platform("Darwin", "x86_64"), "mac-x64")
        self.assertEqual(browser.cft_platform("Linux", "aarch64"), "linux-arm64")
        self.assertEqual(browser.cft_platform("Linux", "x86_64"), "linux64")
        self.assertEqual(browser.cft_platform("Windows", "AMD64"), "win64")
        self.assertEqual(browser.cft_platform("Windows", "ARM64"), "win64")
        self.assertIsNone(browser.cft_platform("Linux", "riscv64"))
        self.assertTrue(all(re.fullmatch(r"[0-9a-f]{64}", h) for h in self.saved[1].values()))

    def test_mismatch_is_deleted_and_never_run(self):
        with self.assertRaises(browser.BrowserError) as c:
            browser.download(lambda m: None, "linux64", self.opener(self.zipped()))
        self.assertIn("does not match the SHA-256", str(c.exception))
        leftovers = [f for _, _, fs in os.walk(self.dir) for f in fs]
        self.assertEqual(leftovers, [])

    def test_verified_download_unpacks(self):
        data = self.zipped()
        browser.CFT_SHA256["linux64"] = hashlib.sha256(data).hexdigest()
        path = browser.download(lambda m: None, "linux64", self.opener(data))
        self.assertEqual(path, browser.shell_path("linux64"))
        self.assertTrue(os.path.isfile(path))
        if os.name != "nt":
            self.assertTrue(os.access(path, os.X_OK))
            self.assertEqual(os.stat(self.dir).st_mode & 0o777, 0o700)
        self.assertFalse(any(f.endswith(".part") for _, _, fs in os.walk(self.dir) for f in fs))

    def test_missing_libraries_hint(self):
        err = "chrome: error while loading shared libraries: libnss3.so: cannot open shared object file\n"
        msg = browser.start_error("/x/chrome", 127, err)
        self.assertIn("libnss3.so", msg)
        self.assertIn("sudo apt-get install -y libnss3", msg)
        self.assertIn("code 3", browser.start_error("/x/chrome", 3, ""))
        trace = "[1:2:FATAL:zygote_host_impl_linux.cc(130)] No usable sandbox!\n#0 0x1 base::debug\n#1 0x2 x\n#2 0x3 y\n"
        self.assertIn("No usable sandbox", browser.start_error("/x/chrome", -5, trace))


FAKE_CHROME = """#!{py}
import os, sys, time
args = sys.argv[1:]
with open(os.environ["FAKE_LOG"], "a") as f:
    f.write(" ".join(args) + "\\n")
if os.environ.get("FAKE_SANDBOX") and "--no-sandbox" not in args:
    sys.stderr.write("[1:2:FATAL:zygote_host_impl_linux.cc(130)] No usable sandbox!\\n#0 0x1 trace\\n")
    sys.exit(1)
profile = [a.split("=", 1)[1] for a in args if a.startswith("--user-data-dir=")][0]
with open(os.path.join(profile, "DevToolsActivePort"), "w") as f:
    f.write("{port}\\n/devtools/browser/x\\n")  # nothing listens there
time.sleep(60)
"""


@unittest.skipIf(os.name == "nt", "the fake Chrome is a script with a shebang")
class LaunchChrome(unittest.TestCase):
    """browser.Browser against a fake Chrome: cleanup on a dead DevTools port, the sandbox relaunch."""

    def setUp(self):
        self.dir = tempfile.mkdtemp()
        s = socket.socket()
        s.bind(("127.0.0.1", 0))
        port = s.getsockname()[1]
        s.close()
        self.exe = os.path.join(self.dir, "fake-chrome")
        with open(self.exe, "w") as f:
            f.write(FAKE_CHROME.format(py=sys.executable, port=port))
        os.chmod(self.exe, 0o755)
        os.environ["FAKE_LOG"] = os.path.join(self.dir, "argv.log")

    def tearDown(self):
        os.environ.pop("FAKE_LOG", None)
        os.environ.pop("FAKE_SANDBOX", None)
        shutil.rmtree(self.dir, ignore_errors=True)

    def launches(self):
        with open(os.environ["FAKE_LOG"]) as f:
            return f.read().splitlines()

    def test_user_agent_keeps_running_version_and_platform(self):
        for native in ("Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/157.0.1.2 Safari/537.36",
                       "Mozilla/5.0 (Macintosh) Chrome/156.0.1.2 Safari/537.36"):
            with unittest.mock.patch.object(browser.Browser, "_launch", return_value=(1, "/test")), \
                 unittest.mock.patch.object(browser, "CDP") as connection:
                connection.return_value.user_agent = None
                connection.return_value.send.return_value = {"userAgent": native}
                with browser.Browser(path=self.exe) as b:
                    expected = native.replace("HeadlessChrome/", "Chrome/") if "HeadlessChrome/" in native else None
                    self.assertEqual(b.cdp.user_agent, expected)
                    b.cdp.send.assert_called_once_with("Browser.getVersion")

    def test_dead_devtools_port_leaves_nothing(self):
        profile = os.path.join(self.dir, "profile")
        os.mkdir(profile)
        b = None
        with self.assertRaisesRegex(browser.BrowserError, "DevTools"):
            b = browser.Browser(path=self.exe, profile=profile, headless=False, timeout=5)
        self.assertIsNone(b)
        self.assertIn("--window-position=-3000,-3000", self.launches()[0])  # a window, out of sight
        self.assertTrue(browser.always_headless(browser.shell_path("linux64")))
        self.assertFalse(browser.always_headless("/usr/bin/google-chrome"))
        self.assertNotIn("--headless", self.launches()[0])
        out = subprocess.run(["ps", "-axo", "command"], capture_output=True, text=True).stdout
        self.assertNotIn(self.exe, out)  # the fake Chrome was stopped

    def test_temp_profile_removed(self):
        tmp = os.path.join(self.dir, "tmp")
        os.mkdir(tmp)
        saved, tempfile.tempdir = tempfile.tempdir, tmp  # Browser's mkdtemp lands here, not in the shared temp
        try:
            with self.assertRaises(browser.BrowserError):
                browser.Browser(path=self.exe, timeout=5)
        finally:
            tempfile.tempdir = saved
        self.assertEqual(os.listdir(tmp), [])

    @unittest.skipIf(getattr(os, "geteuid", lambda: 1)() == 0, "root always runs without the sandbox")
    def test_no_usable_sandbox_relaunches_once_without_it(self):
        os.environ["FAKE_SANDBOX"] = "1"
        with self.assertRaisesRegex(browser.BrowserError, "DevTools"):  # second launch got past the sandbox
            browser.Browser(path=self.exe, timeout=5)
        runs = self.launches()
        self.assertEqual(len(runs), 2)
        self.assertNotIn("--no-sandbox", runs[0])
        self.assertIn("--no-sandbox", runs[1])

    def test_snap_wrapper_skipped(self):
        wrapper = os.path.join(self.dir, "chromium-browser")
        with open(wrapper, "w") as f:
            f.write("#!/bin/sh\nexec /snap/bin/chromium \"$@\"\n")
        self.assertTrue(browser.is_snap(wrapper))
        self.assertTrue(browser.is_snap("/snap/bin/chromium"))
        self.assertFalse(browser.is_snap(self.exe))


def load_engine_cli():
    path = os.path.join(os.path.dirname(HERE), "scripts", "engine.py")
    spec = importlib.util.spec_from_file_location("engine_cli", path)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


class EngineCli(unittest.TestCase):
    def test_port_with_a_wildcard_listener_is_not_free(self):
        cli = load_engine_cli()
        s = socket.socket()
        s.bind(("0.0.0.0", 0))
        s.listen(8)  # both checks connect; a full backlog would refuse the second
        port = s.getsockname()[1]
        try:
            self.assertFalse(cli.port_free(port))
            self.assertTrue(serve.port_taken(port))
        finally:
            s.close()
        self.assertFalse(serve.port_taken(port))

    def test_stop_never_kills_a_stranger(self):
        cli = load_engine_cli()
        p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])
        try:
            self.assertTrue(cli.kill(p.pid))  # "nothing of ours to stop"
            time.sleep(0.3)
            self.assertIsNone(p.poll())
        finally:
            p.kill()
            p.wait()


def wait_until(cond, seconds=5):
    end = time.time() + seconds
    while not cond() and time.time() < end:
        time.sleep(0.02)
    return cond()


class Resume(Fakes):
    """P3: rows stream into the CSV, Progress says how far a job is, and a job the engine was stopped
    in the middle of carries on from its last finished keyword."""

    def start(self, **body):
        name, data = jobs.validate(dict(GOOD, **body))
        jid = self.store.create(name, data)
        t = threading.Thread(target=self.worker.run_job, args=(self.store.claim(),))
        t.start()
        return jid, t

    def test_rows_stream_in_and_a_stopped_engine_resumes(self):
        RESULTS["cafes"] = ([place(1)], "end-of-list")
        RESULTS["bakeries"] = ([place(2), place(3)], "slow")
        RESULTS["florists"] = ([place(4)], "end-of-list")
        jid, t = self.start(keywords=["cafes", "bakeries", "florists"])
        self.assertTrue(wait_until(lambda: self.store.get(jid)["Progress"].get("places") == 3))
        self.assertEqual(len(self.rows(jid)), 3)  # bakeries' places are in before the keyword ends
        prog = jobs.public(self.store.get(jid))["Progress"]
        self.assertEqual((prog["keywords_done"], prog["keywords_total"], prog["places"], prog["stage"]), (1, 3, 3, "search"))
        self.assertTrue(33 < prog["percent"] <= 66)
        self.worker.shutdown(wait=5)  # the engine stops mid-keyword
        t.join(5)
        self.assertEqual(self.store.get(jid)["Status"], "pending")
        RESULTS["bakeries"] = ([place(2), place(3)], "end-of-list")
        SEARCHED.clear()
        jobs.Worker(self.store, self.lines.append).run_job(self.store.claim())  # the next start
        job = self.store.get(jid)
        self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
        self.assertEqual(SEARCHED, ["bakeries", "florists"])  # cafes was finished, bakeries was not
        self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1", "Biz 2", "Biz 3", "Biz 4"])
        self.assertEqual(jobs.public(job)["Progress"], {"keywords_done": 3, "keywords_total": 3, "places": 4,
                                                        "percent": 100, "stage": "done"})

    def crashed(self, keywords, **progress):
        """A job an engine crashed in: one keyword finished, part of the next one appended, a torn line."""
        name, data = jobs.validate(dict(GOOD, keywords=keywords))
        jid = self.store.create(name, data)
        size = self.store.append(jid, [jobs.to_row(place(1), keywords[0])], jobs.fields(data))
        self.store.append(jid, [jobs.to_row(place(2), keywords[1])], jobs.fields(data))
        with open(self.store.csv_path(jid), "ab") as f:
            f.write(b'"cafes,Biz 9,"half a ro')
        self.store.claim()
        self.store.set_progress(jid, dict({"done": 1, "total": len(keywords), "bytes": size, "stage": "search"}, **progress))
        return jid, size

    def test_crash_cuts_the_unfinished_keyword_and_resumes(self):
        jid, size = self.crashed(["cafes", "bakeries"], elapsed=20)
        self.assertEqual(self.store.recover(), (1, 0))
        self.assertEqual(os.path.getsize(self.store.csv_path(jid)), size)  # torn line and bakeries' row gone
        RESULTS["bakeries"] = ([place(2), place(3)], "end-of-list")
        self.worker.run_job(self.store.claim())
        self.assertEqual(SEARCHED, ["bakeries"])
        self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1", "Biz 2", "Biz 3"])
        self.assertTrue(any("resumed at keyword 2/2 with 1 places, 40 s left" in l for l in self.lines))

    def test_too_many_restarts_fail_as_before(self):
        jid, size = self.crashed(["cafes", "bakeries"], resumes=jobs.MAX_RESUMES)
        self.assertEqual(self.store.recover(), (0, 1))
        job = self.store.get(jid)
        self.assertEqual(job["Status"], "failed")
        self.assertIn("the engine stopped while this job ran (4 times); its rows so far are kept", job["Error"])
        self.assertEqual(os.path.getsize(self.store.csv_path(jid)), size)

    def test_no_time_left_ends_without_searching(self):
        jid, _ = self.crashed(["cafes", "bakeries"], elapsed=60)
        self.store.recover()
        self.worker.run_job(self.store.claim())
        job = self.store.get(jid)
        self.assertEqual((job["Status"], SEARCHED, FakeBrowser.made), ("ok", [], []))
        self.assertIn("stopped at max_time", job["Error"])
        self.assertEqual([r["title"] for r in self.rows(jid)], ["Biz 1"])

    def test_old_store_gets_the_progress_column(self):
        folder = os.path.join(self.dir, "old")
        os.makedirs(folder)
        db = sqlite3.connect(os.path.join(folder, "jobs.db"))
        db.execute("CREATE TABLE jobs (id TEXT PRIMARY KEY, name TEXT NOT NULL, date TEXT NOT NULL, "
                   "status TEXT NOT NULL, data TEXT NOT NULL, error TEXT NOT NULL DEFAULT '')")
        db.execute("INSERT INTO jobs VALUES ('x', 'n', 'd', 'ok', '{}', '')")
        db.commit()
        db.close()
        old = jobs.Store(folder)
        self.assertEqual(old.get("x")["Progress"], {})
        old.db.close()


class Backoff(Fakes):
    def setUp(self):
        super().setUp()
        self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)
        jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01

    def tearDown(self):
        jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits
        super().tearDown()

    def test_retry_through_the_next_proxy(self):
        RESULTS["cafes in Austin"] = ([place(1)], ["blocked", "end-of-list"])
        jid, job = self.run_one(proxies=["http://a:1", "http://b:2"])
        self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
        self.assertEqual(FakeBrowser.made[0].contexts, ["http://a:1", "http://b:2"])
        self.assertTrue(any("retry 1/2 in 0.01 s through the next proxy" in l for l in self.lines))

    def test_gives_up_after_bounded_retries(self):
        RESULTS["cafes in Austin"] = ([], ["blocked"])
        _, job = self.run_one()
        self.assertEqual(job["Status"], "failed")
        self.assertIn("unusual-traffic page", job["Error"])  # what scripts/scrape.py looks for
        self.assertIn("Waited and retried 2 time(s)", job["Error"])
        self.assertEqual(SEARCHED, ["cafes in Austin"] * 3)

    def test_a_failed_proxy_hands_the_keyword_to_the_next(self):
        calls = []
        def collect(page, url, **kw):
            calls.append(FakeBrowser.made[-1].contexts[-1])
            if len(calls) == 1:
                raise CDPError("navigate https://x: net::ERR_PROXY_CONNECTION_FAILED")
            return [place(1)], "end-of-list", {"batches": 1, "dom_links": 0}
        maps.collect = collect
        jid, job = self.run_one(proxies=["http://dead:1", "http://ok:2"])
        self.assertEqual((job["Status"], job["Error"], calls), ("ok", "", ["http://dead:1", "http://ok:2"]))
        self.assertEqual(len(self.rows(jid)), 1)

    def test_proxies_take_turns(self):
        for k in ("a", "b", "c"):
            RESULTS[k] = ([], "end-of-list")
        self.run_one(keywords=["a", "b", "c"], proxies=["http://u:pw-s3cret@p1:1", "http://p2:2"])
        self.assertEqual(FakeBrowser.made[0].contexts, ["http://u:pw-s3cret@p1:1", "http://p2:2", "http://u:pw-s3cret@p1:1"])
        self.assertFalse(any("s3cret" in l for l in self.lines))
        self.assertEqual(FakeAuth.logins, [("u", "pw-s3cret")] * 2)  # only the tabs on the proxy with a login


class ProxyLogin(unittest.TestCase):
    """browser.ProxyAuth against a fake DevTools connection: continue every paused request, answer
    the proxy's 407 once with the job's credentials, then give up (wrong credentials)."""

    def test_answers(self):
        class C:
            closed = None
            def __init__(self):
                self.sent = []
            def subscribe(self, method, sid=None, q=None):
                self.q = q or queue.Queue()
                return self.q
            def unsubscribe(self, *a):
                pass
        class P:
            sid = "s"
            def __init__(self):
                self.cdp = C()
            def send(self, method, params=None, timeout=None):
                self.cdp.sent.append((method, params))
                return {}
        page = P()
        auth = browser.ProxyAuth(page, "user", "pw")
        q = page.cdp.q
        q.put({"requestId": "1", "request": {"url": "https://www.google.com/"}})
        q.put({"requestId": "2", "authChallenge": {"source": "Proxy"}})
        q.put({"requestId": "2", "authChallenge": {"source": "Proxy"}})
        q.put({"requestId": "3", "authChallenge": {"source": "Server"}})
        self.assertTrue(wait_until(lambda: len(page.cdp.sent) == 5))
        auth.close()
        sent = page.cdp.sent
        self.assertEqual(sent[0], ("Fetch.enable", {"handleAuthRequests": True, "patterns": [{"urlPattern": "*"}]}))
        self.assertEqual(sent[1], ("Fetch.continueRequest", {"requestId": "1"}))
        self.assertEqual([m[1]["authChallengeResponse"]["response"] for m in sent[2:]],
                         ["ProvideCredentials", "CancelAuth", "Default"])
        self.assertEqual(sent[2][1]["authChallengeResponse"]["username"], "user")
        self.assertTrue(auth.refused)  # the job then says "proxy login refused", not Chrome's vague error
        self.assertIn("proxy login refused", jobs.LOGIN_REFUSED)
        self.assertTrue(jobs.PROXY_FAILED.search(jobs.LOGIN_REFUSED))
        self.assertFalse(jobs.PROXY_FAILED.search("navigate https://x/maps/search/tunnel%20car%20wash: net::ERR_TIMED_OUT"))

    def test_split(self):
        self.assertEqual(browser.split_proxy("http://u%40x:p%3A@h:3128"), ("http://h:3128", "u@x", "p:"))
        self.assertEqual(browser.split_proxy("socks5h://[::1]:1080"), ("socks5://[::1]:1080", "", ""))


def entity(days=7, reviews=False):
    """A place entity shaped like Google's (made-up values) with the full view's fields."""
    e = [None] * 260
    e[11], e[10], e[78] = "Biz", "0x1:0x2", "ChIJx"
    e[4] = [None, None, None, ["https://search.google.com/local/reviews?placeid=ChIJx", "349 reviews"],
            None, None, None, 4.7, 349]
    names = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
    week = [[names[n - 1], n, [2026, 11, n], [["7\u202fAM\u20139\u202fPM", [[7], [21]]]], 0, 1] for n in (6, 7, 1, 2, 3, 4, 5)]
    e[203] = [week[:days], [week[0], 0, 4, None, ["Open \u00b7 Closes 9\u202fPM"]]]
    if reviews:
        def review(stars, text):
            r = [None] * 4
            r[0] = "id"
            r[1] = [None, None, 1, 1, [None, None, ["https://example.com/u"], None, None, ["A Reviewer"]], None, "4 months ago"]
            r[2] = [[stars]] + [None] * 14 + [[[text]]]
            return [r]
        e[175] = [None] * 10
        e[175][3] = [6, 12, 6, 24, 301]
        e[175][9] = [[[review(5, "Great\u202fcoffee"), review(2, "Slow"), review(4, "")]]]
    return e


class ExtraDetailsParser(unittest.TestCase):
    def test_photos_from_search_and_panel_are_valid_json_and_deduplicated(self):
        e = [None] * 204
        photo = [None] * 7
        photo[6] = ["https://lh3.googleusercontent.com/business-photo"]
        e[37] = [[photo]]
        e[72] = [[photo, None, [None] * 7]]
        panel = [None] * 4
        panel[2], panel[3] = "Exterior", [[None] * 6 + [["https://lh3.googleusercontent.com/exterior"]]]
        e[171] = [[panel]]
        out = details.parse_entity(e)
        images = json.loads(out["images"])
        self.assertEqual(images, [{"title": "Exterior", "image": "https://lh3.googleusercontent.com/exterior"},
                                  {"title": "", "image": "https://lh3.googleusercontent.com/business-photo"}])
        row = jobs.to_row(out, "x")
        self.assertEqual(json.loads(row["images"]), images)
        photo[6] = ["javascript:alert(1)"]
        e[171] = None
        self.assertNotIn("images", details.parse_entity(e))
        for bad in ("https://u:[email protected]/x", "https://a.com/x\ny", "https://a.com/" + "x" * 1000):
            photo[6] = [bad]
            self.assertNotIn("images", details.parse_entity(e))
        many = [[None] * 6 + [[f"https://lh3.googleusercontent.com/{n}"]] for n in range(50)]
        e[37] = [many]
        self.assertEqual(len(json.loads(details.parse_entity(e)["images"])), details.IMAGES_MAX)

def batch_details():
    """A made-up entity with the old engine's place-page fields, in the layouts search batches use."""
    e = [None] * 210
    e[9] = [None, None, 47.0000625, 8.0000625]
    e[183] = [None, ["Downtown", "1 Main St", None, "Austin", "78701", "Texas", "US"]]
    opt = lambda name, on, values=(): [None, name, [None, [[on]], None, None,
                                                   [None, [[[[None, None, v] for v in values]]]]]]
    e[100] = [None, [["accessibility", "Accessibility", [opt("Ramp", 1), opt("Ramp", 0), opt("Lift", 0)]],
                     ["payments", "Payments", [opt("Credit cards", 1, ("Visa", "Amex")), opt("Credit cards", 1, ("Visa",))]]]]
    book = [None, None, ["https://www.opentable.com/r/1"]]
    e[46] = [["https://www.opentable.com/r/1", "opentable.com"], ["javascript:alert(1)", "x"],
             ["https://www.opentable.com/r/1", "opentable.com"]]
    e[75] = [[[1, None, [[["opentable.com"], book]]],  # reserve a table: not an order link
              [3, None, [[["doordash.com"], [None, None, ["https://www.doordash.com/store/1"]]],
                         [["bad.com"], [None, None, ["ftp://bad.com/x"]]]]],
              [4, None, None]]]  # Google's own order flow
    e[38] = ["https://example.com/menu", "example.com"]
    return e


class NoCoordinates(Fakes):
    def test_a_job_without_coordinates_searches_by_its_words(self):
        RESULTS["cafes in Austin"] = ([place(1)], "end-of-list")
        urls = []
        maps.collect = lambda page, url, **kw: urls.append(url) or fake_collect(page, url, **kw)
        jid, job = self.run_one(lat="", lon="")
        self.assertEqual((job["Status"], len(self.rows(jid))), ("ok", 1))
        self.assertEqual(urls, ["https://www.google.com/maps/search/cafes%20in%20Austin?hl=en"])
        self.assertEqual((job["Data"]["lat"], job["Data"]["lon"]), ("", ""))


class BatchDetails(Fakes):
    def test_old_engine_columns_from_the_batch(self):
        out = details.parse_entity(batch_details())
        self.assertEqual(json.loads(out["complete_address"]), {"borough": "Downtown", "street": "1 Main St", "city": "Austin",
                                                               "postal_code": "78701", "state": "Texas", "country": "US"})
        self.assertEqual(json.loads(out["about"]), [
            {"id": "accessibility", "name": "Accessibility", "options": [{"name": "Ramp", "enabled": True},
                                                                         {"name": "Lift", "enabled": False}]},
            {"id": "payments", "name": "Payments", "options": [{"name": "Credit cards", "enabled": True,
                                                                "values": ["Visa", "Amex"]}]}])
        self.assertEqual(out["credit_cards_accepted"], "Visa, Amex")
        self.assertEqual(json.loads(out["reservations"]), [{"link": "https://www.opentable.com/r/1", "source": "opentable.com"}])
        self.assertEqual(json.loads(out["order_online"]), [{"link": "https://www.doordash.com/store/1", "source": "doordash.com"}])
        self.assertEqual(json.loads(out["menu"]), {"link": "https://example.com/menu", "source": "example.com"})
        row = jobs.to_row(out, "k")
        self.assertEqual(row["plus_code"], "8FVC2222+22")  # Open Location Code's own example point
        for k in ("complete_address", "about", "credit_cards_accepted", "reservations", "order_online", "menu"):
            self.assertEqual(row[k], out[k])
        odd = [None] * 210  # Google changed a layout: wrong types read as nothing, never an exception
        odd[37], odd[100], odd[75], odd[183], odd[46], odd[38] = [[[None] * 6 + [[["x"]]]]], "s", ["x"], [None, "a"], "y", [["l"], "s"]
        self.assertEqual(details.parse_entity(odd), {})
        for x in (5, True, 1.5):  # a number where a list was: still nothing, never "not iterable"
            odd = [None] * 210
            odd[75], odd[100], odd[203] = x, [None, x], [[["Monday", 1, None, x]]]
            self.assertEqual(details.parse_entity(odd), {})
            odd[100] = [None, [["payments", "Payments", x], ["g", "G", [[None, "Opt", [None, [[1]], None, None, [None, [[x]]]]]]]]]
            self.assertEqual(json.loads(details.parse_entity(odd)["about"])[1]["options"], [{"name": "Opt", "enabled": True}])
        panel = ")]}'\n" + json.dumps([None] * 6 + [[None] * 175 + [[None] * 9 + [[[5]]]]])  # reviews: a number
        self.assertEqual(details.parse_place(panel), {})
        bare = details.parse_entity(entity())
        for k in ("complete_address", "about", "credit_cards_accepted", "reservations", "order_online", "menu", "plus_code"):
            self.assertNotIn(k, bare)

    def test_plus_code(self):
        self.assertEqual(details.plus_code("47.0000625", "8.0000625"), "8FVC2222+22")
        self.assertEqual(details.plus_code(20.3700625, 2.7821875), "7FG49QCJ+2V")  # the spec's example, 10 digits
        self.assertEqual(details.plus_code(-90, -180), "22222222+22")
        self.assertEqual(details.plus_code(90, 180), "C2X2X2X2+X2")  # latitude 90 stays in the top row
        self.assertEqual(details.plus_code(0, 180), details.plus_code(0, -180))
        self.assertEqual(details.plus_code(30.2, -97.7), details.plus_code(30.2, 262.3))  # same meridian
        for bad in ("", None, "x", float("nan"), float("inf")):
            self.assertEqual(details.plus_code(bad, 1), "")
        self.assertEqual(jobs.to_row({"title": "A"}, "k")["plus_code"], "")
        self.assertEqual(jobs.to_row({"title": "A", "plus_code": "6PX4+2V Austin"}, "k")["plus_code"], "6PX4+2V Austin")

    def test_the_panel_fills_only_what_the_batch_lacked(self):
        RESULTS["cafes in Austin"] = ([place(1, about='[{"id": "accessibility"}]', complete_address='{"city": "A"}')],
                                      "end-of-list")
        self.worker.headless = False
        saved = (maps.place_panel, details.parse_place)
        maps.place_panel = lambda page, url: "body"
        details.parse_place = lambda body, limit=5: {"user_reviews": "[]", "about": '[{"id": "payments"}]',
                                                     "credit_cards_accepted": "Visa", "complete_address": '{"city": "B"}'}
        try:
            jid, job = self.run_one(reviews_max=1)
        finally:
            maps.place_panel, details.parse_place = saved
        r = self.rows(jid)[0]
        self.assertEqual({k: job["Progress"].get(k) for k in ("reviews_done", "reviews_total")},
                         {"reviews_done": 1, "reviews_total": 1})  # "reading reviews 1 of 1" while it runs
        self.assertEqual((r["about"], r["credit_cards_accepted"], r["complete_address"], r["user_reviews"]),
                         ('[{"id": "accessibility"}]', "Visa", '{"city": "A"}', "[]"))
        body = ")]}'\n" + json.dumps([None] * 6 + [batch_details()])
        self.assertEqual(details.parse_place(body)["credit_cards_accepted"], "Visa, Amex")


class FullView(unittest.TestCase):
    """details.py fields that only Google's full view carries (a Chrome window), from made-up data."""

    def test_count_link_and_week(self):
        out = details.parse_entity(entity())
        self.assertEqual((out["review_count"], out["review_rating"]), (349, 4.7))
        self.assertEqual(out["reviews_link"], "https://search.google.com/local/reviews?placeid=ChIJx")
        week = json.loads(out["open_hours"])
        self.assertEqual(list(week)[:2], ["Monday", "Tuesday"])
        self.assertEqual(week["Sunday"], ["7 AM\u20139 PM"])
        self.assertEqual(out["open_state"], "Open \u00b7 Closes 9 PM" if False else out["open_state"])

    def test_limited_view_gives_no_week(self):
        self.assertNotIn("open_hours", details.parse_entity(entity(days=1)))  # today only is not a week

    def test_place_panel_reviews_without_names(self):
        body = ")]}'\n" + json.dumps([None] * 6 + [entity(reviews=True)])
        out = details.parse_place(body, limit=2)
        self.assertEqual(json.loads(out["reviews_per_rating"]), {"1": 6, "2": 12, "3": 6, "4": 24, "5": 301})
        self.assertEqual(json.loads(out["user_reviews"]), [{"Rating": 5, "When": "4 months ago", "Description": "Great coffee"},
                                                           {"Rating": 2, "When": "4 months ago", "Description": "Slow"}])
        self.assertNotIn("Reviewer", json.dumps(out))
        self.assertEqual(details.parse_place("not json"), {})


class Locale(unittest.TestCase):
    def test_search_url_and_end_of_list(self):
        self.assertEqual(maps.search_url("a b", 1, 2, 15, "de", "at"), "https://www.google.com/maps/search/a%20b/@1,2,15z?hl=de&gl=at")
        self.assertTrue(maps.search_url("x").endswith("?hl=en"))
        self.assertTrue(maps.ended({"cls": True, "text": ""}))
        self.assertTrue(maps.ended({"cls": False, "text": "… Vous êtes arrivé à la fin de la liste."}))
        self.assertFalse(maps.ended({"cls": False, "text": "Biz 1"}))
        self.assertFalse(maps.ended(None))
        self.assertIn('input[name="set_eom"][value="true"]', maps.JS_CONSENT)  # the reject form, any language
        self.assertIn("tout refuser", maps.JS_CONSENT)

    def test_429_marks_the_batches_limited(self):
        q = queue.Queue()
        class C:
            closed = None
            def subscribe(self, *a, **k):
                return q
        class P:
            sid, cdp = "s", C()
        b = maps.Batches(P())
        q.put({"requestId": "1", "response": {"url": "https://www.google.com/search?tbm=map&pb=x", "status": 429,
                                              "mimeType": "text/html"}})
        self.assertEqual((b.drain(), b.limited, b.pending), ([], 1, {}))


class Websites(unittest.TestCase):
    def test_socials_and_mx(self):
        rows = [{"website": "https://shop.com", "emails": ""}, {"website": "https://gone.com", "emails": ""}]
        pages = {"https://shop.com": '[email protected] <a href="https://www.instagram.com/shopco/">ig</a> '
                                     '<a href="https://facebook.com/sharer">x</a> <a href="https://facebook.com/shopco">fb</a>',
                 "https://gone.com": "[email protected]"}
        hosts = {"shop.com": "google_workspace", "gone-forever.com": "nxdomain"}
        with unittest.mock.patch.object(jobs, "mx_host", side_effect=hosts.get):
            n = jobs.add_site_data(rows, 10, socials=True, verify=True, get=pages.get)
        self.assertEqual(n, (1, 0))
        self.assertEqual((rows[0]["emails"], rows[0]["email_host"]), ("[email protected]", "google_workspace"))
        self.assertEqual((rows[0]["instagram"], rows[0]["facebook"]), ("https://www.instagram.com/shopco", "https://facebook.com/shopco"))
        self.assertEqual(rows[1]["emails"], "")  # its only address is on a domain that does not exist

    def test_mx_answers(self):
        def doh(answer):
            r = unittest.mock.MagicMock()
            r.__enter__.return_value = io.BytesIO(json.dumps(answer).encode())
            return unittest.mock.patch.object(jobs._OPENER, "open", return_value=r)
        jobs._MX.clear()
        with doh({"Status": 3}):
            self.assertEqual(jobs.mx_host("a.test"), "nxdomain")
        with doh({"Status": 0, "Answer": [{"type": 15, "data": "1 aspmx.l.google.com."}]}):
            self.assertEqual(jobs.mx_host("b.test"), "google_workspace")
        with doh({"Status": 0}):
            self.assertEqual(jobs.mx_host("c.test"), "none")
        with doh({"Status": 2}):  # SERVFAIL: no verdict, and not kept for the next job
            self.assertEqual(jobs.mx_host("d.test"), "")
        self.assertNotIn("d.test", jobs._MX)
        jobs._MX.clear()

    def test_same_social_patterns_as_scrape(self):
        sys.path.insert(0, os.path.join(os.path.dirname(HERE), "scripts"))
        import scrape
        self.assertEqual({k: r.pattern for k, r in jobs.SOCIAL_RE.items()}, {k: r.pattern for k, r in scrape.SOCIAL_RE.items()})
        self.assertEqual(jobs.SOCIALS, scrape.SOCIAL_FIELDS)
        self.assertEqual(jobs.SKIP_HANDLE, scrape._SKIP_HANDLE)
        self.assertEqual(jobs.FORMULA.pattern, scrape.FORMULA.pattern)  # the download and the file quote alike
        site = '<a href="https://instagram.com/squarespace">built with</a> <a href="https://instagram.com/realcafe">us</a>'
        self.assertEqual(jobs.find_socials(site)["instagram"], "https://instagram.com/realcafe")  # not the builder's own


class ApiP3(Server):
    def test_progress_json_and_docs(self):
        code, body, _ = self.call("POST", "/api/v1/jobs", dict(GOOD, proxies=["http://u:pw-s3cret@h:1"]))
        jid = json.loads(body)["id"]
        code, body, _ = self.call("GET", f"/api/v1/jobs/{jid}")
        job = json.loads(body)
        self.assertEqual(job["Data"]["proxies"], ["http://u:***@h:1"])
        self.assertNotIn(b"s3cret", body + self.call("GET", "/api/v1/jobs")[1])
        self.assertEqual(job["Progress"], {"keywords_done": 0, "keywords_total": 1, "places": 0, "percent": 0,
                                           "stage": "queued"})
        self.store.append(jid, [jobs.to_row(place(1, title="Café"), "k")], jobs.fields({}))
        code, body, r = self.call("GET", f"/api/v1/jobs/{jid}/download?format=json")
        self.assertEqual((code, r.getheader("Content-Type")), (200, "application/json; charset=utf-8"))
        rows = json.loads(body)
        self.assertEqual((len(rows), rows[0]["title"], rows[0]["review_count"]), (1, "Café", "11"))
        self.assertEqual(self.call("GET", f"/api/v1/jobs/{jid}/download?format=xml")[0], 400)
        self.assertEqual(self.call("GET", f"/download?id={jid}&format=json")[1], body)
        # A formula in scraped text reaches a spreadsheet quoted; the JSON and the stored file keep it as is.
        self.store.append(jid, [jobs.to_row(place(2, title='=HYPERLINK("http://x")', phone="+1 512-555-0100",
                                                  descriptions="line 1\nline 2"), "k")], jobs.fields({}))
        got = list(csv.DictReader(io.StringIO(self.call("GET", f"/api/v1/jobs/{jid}/download")[1].decode())))
        self.assertEqual((got[1]["title"], got[1]["phone"], got[1]["descriptions"], len(got)),
                         ('\'=HYPERLINK("http://x")', "+1 512-555-0100", "line 1\nline 2", 2))
        self.assertEqual(json.loads(self.call("GET", f"/api/v1/jobs/{jid}/download?format=json")[1])[1]["title"],
                         '=HYPERLINK("http://x")')
        self.assertEqual(self.store.read_rows(jid)[1]["title"], '=HYPERLINK("http://x")')
        for s, want in (("@SUM(1)", "'@SUM(1)"), ("-2+3", "'-2+3"), ("\tx", "'\tx"), ("+44 20 7946 0958", "+44 20 7946 0958"),
                        ("Joe's", "Joe's"), ("-", "-"), (None, ""), ("'-x", "''-x"), ("'plain", "'plain"),
                        ("+1 512-555-0100 ext. 12", "+1 512-555-0100 ext. 12"), ("+1 512-555-0100 x12", "+1 512-555-0100 x12")):
            self.assertEqual(jobs.csv_safe(s), want)
        code, body, r = self.call("GET", "/api/docs")
        page, csp = body.decode(), r.getheader("Content-Security-Policy")
        self.assertEqual((code, r.getheader("Content-Type")), (200, "text/html; charset=utf-8"))
        self.assertNotIn("<script", page)
        nonce = re.search(r"nonce-([^']+)'", csp).group(1)
        self.assertIn(f'<style nonce="{nonce}">', page)
        self.assertNotIn("script-src", csp)
        self.assertIn("<code>reviews_max</code>", page)
        spec = json.loads(self.call("GET", "/api/openapi.json")[1])
        self.assertEqual(spec["paths"]["/api/v1/jobs"]["post"]["requestBody"]["content"]["application/json"]
                         ["schema"]["required"], ["name", "keywords", "lang", "max_time"])
        ui = self.call("GET", "/")[1].decode()
        self.assertIn('id="country"', ui)
        self.assertIn('<input id="what" role="combobox"', ui)
        self.assertIn('<input id="where" role="combobox"', ui)
        self.assertIn("const TYPES = " + json.dumps(serve.TYPES) + ";", ui)  # the kit's own types work offline
        self.assertIn(serve.CATEGORIES, ui)  # Google's category names: loaded by the browser from GitHub, pinned
        self.assertRegex(serve.CATEGORIES, r"/[0-9a-f]{40}/")
        front = ui.split("<details>")[0]  # what a beginner sees before opening a section
        self.assertIn('<input type="checkbox" id="email" checked>', front)
        self.assertIn('<input type="checkbox" id="socials" checked>', front)  # socials on by default
        for el in ('id="depth"', 'id="lang"', 'id="max_time"', 'id="proxies"', 'id="reviews"'):
            self.assertNotIn(el, front)
            self.assertIn(el, ui)
        self.assertEqual(ui.count("<details>"), 3)  # location and language, advanced options, proxies
        self.assertNotIn("lobstr.io</a>", ui)
        self.assertIn("socials: $('socials').checked", ui)
        self.assertIn("if (proxies.length) body.proxies = proxies", ui)
        # preset only: Enter in a box never submits; text not in a list is refused; each type is searched in
        # each city, one job per city centred on it; downloads are named after the search; local times; a favicon
        for rule in ("} else if (ev.key === 'Enter') {  // never submits the form", "if (active >= 0) pick(active);",
                     "is not in the list", "is not picked yet", "whats.map(w => w + ' in ' + c.label)",
                     "lat: c.lat, lon: c.lon", "for (const c of wheres)", "'/api/v1/cities?q=' + encodeURIComponent(q)",
                     "stale = true", "!stale) say("):  # Enter picks nothing while a lookup for newer text is pending
            self.assertIn(rule, ui)
        for gone in ('id="keywords"', 'id="coords"', 'id="kwpick"', "<datalist", "Start anyway"):
            self.assertNotIn(gone, ui)
        self.assertIn("a.download = fileName(j) + '.' + fmt", ui)
        self.assertIn("toLocaleString", ui)
        self.assertIn('<link rel="icon" href="data:image/svg+xml,', ui)
        # as gosom: a star button, and a sponsor box that can be closed; paid sponsors in it only when
        # sponsors.json lists one Mahan marked "ui": true, https links only, four at most
        self.assertIn('Star on GitHub</a>', ui)
        self.assertIn('Sponsor on GitHub</a>', ui)
        self.assertIn('id="sp-close"', ui)
        self.assertIn('<div class="sp-grid" id="sp-grid" hidden>', ui)
        for rule in ("s.ui === true", "s.url.startsWith('https://')", ".slice(0, 4)", "noopener noreferrer sponsored"):
            self.assertIn(rule, ui)
        self.assertNotIn(".innerHTML", ui)  # sponsor text is set as text, never parsed as HTML
        # update notice: hidden by default, filled from GitHub's latest release once a day, shown only when newer
        self.assertIn('<div class="update" id="update" role="status" hidden>', ui)
        for rule in ("gmskit-update", "api.github.com/repos/Mahanaicoach/google-maps-scraper-kit/releases/latest",
                     "newer(tag, mine)", "bash scripts/update.sh"):
            self.assertIn(rule, ui)
        script = re.search(r"<script nonce=[^>]+>(.*)</script>", ui, re.S).group(1)
        self.assertNotIn("'\n'", script)  # PAGE must say \\n: a bare newline in a JS string breaks the page


class ApiKeyDocs(Server):
    key = "s3cret"

    def test_docs_need_no_key(self):
        self.assertEqual(self.call("GET", "/api/docs")[0], 200)
        self.assertEqual(self.call("GET", "/api/openapi.json")[0], 200)
        self.assertEqual(self.call("GET", "/api/v1/jobs/00000000-0000-0000-0000-000000000000/download")[0], 401)
        self.assertEqual(self.call("GET", "/api/v1/cities?q=austin")[0], 401)


class Presets(Server):
    def cities(self, q):
        code, body, _ = self.call("GET", "/api/v1/cities?q=" + urllib.parse.quote(q))
        self.assertEqual(code, 200)
        return json.loads(body)

    def test_broken_preset_files_do_not_take_the_engine_down(self):
        tmp = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, tmp, True)
        self.assertEqual(serve.load_types(tmp), [])  # no file
        for bad in ("{not json", '["Dentists", 5]'):
            with open(os.path.join(tmp, "business-types.json"), "w") as f:
                f.write(bad)
            self.assertEqual(serve.load_types(tmp), [])
        with open(os.path.join(tmp, "cities.tsv.gz"), "wb") as f:  # the real file cut short
            with open(os.path.join(serve.PRESETS, "cities.tsv.gz"), "rb") as g:
                f.write(g.read(5000))
        for damage in ("cut", "flip"):  # cut short (EOFError), one byte flipped inside a deflate block (zlib.error)
            if damage == "flip":
                with open(os.path.join(tmp, "cities.tsv.gz"), "r+b") as f:
                    f.seek(1124); b = f.read(1); f.seek(1124); f.write(bytes([b[0] ^ 0xFF]))
            with unittest.mock.patch.object(serve, "PRESETS", tmp), unittest.mock.patch.object(serve, "_cities", None):
                code, body, _ = self.call("GET", "/api/v1/cities?q=aus")
                self.assertEqual(code, 200, damage)
                self.assertIsInstance(json.loads(body), list)
                self.assertEqual(self.call("GET", "/api/v1/jobs")[0], 200)
        self.assertEqual(self.cities("!!!"), [])  # punctuation only is not "nothing typed"

    def test_where(self):
        top = self.cities("")  # nothing typed: the biggest US cities
        self.assertEqual((len(top), top[0]["label"]), (20, "New York City, New York, US"))
        self.assertTrue(all(c["country"] == "US" for c in top))
        self.assertEqual(self.cities("austin")[0], {"label": "Austin, Texas, US", "lat": "30.2672", "lon": "-97.7431",
                                                    "country": "US"})
        self.assertEqual([c["label"] for c in self.cities("Austin, Tex")], ["Austin, Texas, US"])
        self.assertEqual(self.cities("sao paulo")[0]["label"], "São Paulo, BR")  # accents fold
        self.assertEqual(self.cities("lodz")[0]["label"], "Łódź, Łódź Voivodeship, PL")
        self.assertEqual(self.cities("shanghai")[0]["label"], "Shanghai, CN")  # region == name: said once
        self.assertEqual(self.cities("no such place xq"), [])
        self.assertLessEqual(len(self.cities("s")), 20)

    def test_data(self):
        self.assertEqual(len({t.lower() for t in serve.TYPES}), len(serve.TYPES))  # no duplicate business types
        self.assertTrue(all(t == t.strip() and 0 < len(t) <= 80 for t in serve.TYPES))
        labels = set()
        for row in serve.cities():
            key, label, lat, lon, cc = row.split("\t")
            self.assertTrue(-90 <= float(lat) <= 90 and -180 <= float(lon) <= 180, row)
            self.assertRegex(cc, r"^[A-Z]{2}$")
            self.assertEqual(key, serve.fold(label))
            labels.add(label)
        self.assertEqual(len(labels), len(serve.cities()))  # each label names one city
        self.assertGreater(len(labels), 150000)



class LoopOne(Fakes):
    """Audit loop 1: blocks with no places yet, resumes after the search, refused logins, races."""

    def setUp(self):
        super().setUp()
        self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)
        jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01

    def tearDown(self):
        jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits
        super().tearDown()

    def test_real_collect_calls_a_429_blocked(self):
        q = queue.Queue()
        class C:
            closed = None
            def subscribe(self, *a, **k):
                return q
            def unsubscribe(self, *a):
                pass
        class P:
            sid, cdp = "s", C()
            def set_cookie(self, *a):
                pass
            def navigate(self, url):
                q.put({"requestId": "1", "response": {"url": "https://www.google.com/search?tbm=map&pb=x",
                                                      "status": 429, "mimeType": "text/html"}})
            def url(self):
                return "https://www.google.com/maps/search/x"
            def wait_for(self, expr, timeout=15):
                return True
            def eval(self, expr):
                return {"cls": False, "text": ""} if expr == maps.JS_END else []
            def send(self, *a, **k):
                raise CDPError("no body")
        t = time.time()
        real_collect = self.saved[2]  # Fakes swapped maps.collect for the fake one
        places, reason, _ = real_collect(P(), "https://www.google.com/maps/search/x", depth=3, log=lambda m: None)
        self.assertEqual((places, reason), ([], "blocked"))  # not "blocked+dom-fallback"
        self.assertLess(time.time() - t, 5)  # no 10 s wait for a first batch that Google refused

    def test_resume_after_the_reviews_keeps_them(self):
        RESULTS["cafes in Austin"] = ([place(1, website="https://a.com"), place(2)], "end-of-list")
        def reviews(rows, d, deadline, stop, proxies, save=None, notes=None):
            for r in rows:
                r["user_reviews"] = '[{"Rating": 5}]'
            return len(rows)
        self.worker.add_reviews, self.worker.headless = reviews, False  # window mode
        stopping = lambda *a, **k: self.worker.shutdown() or (0, 0)  # the engine stops during the website pass
        with unittest.mock.patch.object(jobs, "add_site_data", side_effect=stopping):
            jid, job = self.run_one(reviews_max=1, email=True)
        self.assertEqual((job["Status"], job["Progress"]["stage"]), ("pending", "websites"))
        w2 = jobs.Worker(self.store, self.lines.append)
        w2.add_reviews = lambda *a, **k: self.fail("the reviews were done already")
        with unittest.mock.patch.object(jobs, "fetch", return_value="[email protected]"):
            w2.run_job(self.store.claim())
        rows = self.rows(jid)
        self.assertEqual(SEARCHED, ["cafes in Austin"])  # searched once, across the restart
        self.assertEqual([r["user_reviews"] for r in rows], ['[{"Rating": 5}]'] * 2)
        self.assertEqual((rows[0]["emails"], self.store.get(jid)["Status"]), ("[email protected]", "ok"))

    def test_blocked_job_resumed_in_its_website_pass_keeps_rows_and_its_note(self):
        RESULTS["cafes in Austin"] = ([place(1, website="https://a.com")], ["slow-blocked"])
        def collect(page, url, on_batch=None, **kw):
            SEARCHED.append("cafes in Austin")
            on_batch([place(1, website="https://a.com")], 1)  # streamed, then Google refuses the rest
            return [], "blocked", {"batches": 1, "dom_links": 0}
        maps.collect = collect
        stopping = lambda *a, **k: self.worker.shutdown() or (0, 0)
        with unittest.mock.patch.object(jobs, "add_site_data", side_effect=stopping):
            jid, job = self.run_one(email=True)
        self.assertEqual(job["Status"], "pending")
        SEARCHED.clear()
        with unittest.mock.patch.object(jobs, "fetch", return_value=""):
            jobs.Worker(self.store, self.lines.append).run_job(self.store.claim())
        job = self.store.get(jid)
        self.assertEqual(SEARCHED, [])  # never back to the block
        self.assertEqual((job["Status"], [r["title"] for r in self.rows(jid)]), ("ok", ["Biz 1"]))
        self.assertIn("unusual-traffic page", job["Error"])
        self.assertIn("Waited and retried 2 time(s)", job["Error"])

    def test_a_refused_login_stops_the_job(self):
        def collect(page, url, **kw):
            SEARCHED.append(url)
            raise CDPError(jobs.LOGIN_REFUSED)
        maps.collect = collect
        _, job = self.run_one(keywords=["a", "b", "c"], proxies=["http://u:bad@h:1"])
        self.assertEqual((job["Status"], len(SEARCHED)), ("failed", 1))
        self.assertEqual(job["Error"], jobs.LOGIN_REFUSED + "; 2 keyword(s) after it were not searched")

    def test_shutdown_before_the_job_starts(self):
        name, data = jobs.validate(GOOD)
        jid = self.store.create(name, data)
        job = self.store.claim()
        self.worker.stopped.set()
        self.worker.run_job(job)
        self.assertEqual((self.store.get(jid)["Status"], FakeBrowser.made), ("pending", []))

    def test_an_engine_that_cannot_bind_leaves_the_jobs_alone(self):
        name, data = jobs.validate(GOOD)
        jid = self.store.create(name, data)
        self.store.claim()  # another engine on this data folder runs it
        s = socket.socket()
        s.bind(("127.0.0.1", 0))
        s.listen(8)
        argv = ["serve.py", "--port", str(s.getsockname()[1]), "--data", os.path.join(self.dir, "data")]
        try:
            with unittest.mock.patch.object(sys, "argv", argv), self.assertRaises(SystemExit):
                serve.main()
        finally:
            s.close()
        self.assertEqual(self.store.get(jid)["Status"], "working")


class LoopTwo(Fakes):
    """Audit loop 2: Chrome dying on its own, dead proxies, headless reviews, masks, notes."""

    def setUp(self):
        super().setUp()
        self.saved_waits = (jobs.BACKOFF, jobs.ROTATE_WAIT)
        jobs.BACKOFF, jobs.ROTATE_WAIT = (0.05, 0.05), 0.01

    def tearDown(self):
        jobs.BACKOFF, jobs.ROTATE_WAIT = self.saved_waits
        super().tearDown()

    def test_chrome_that_dies_is_started_again(self):
        def collect(page, url, **kw):
            SEARCHED.append(url.split("/search/")[1].split("/")[0])
            if len(SEARCHED) == 2:  # Chrome crashes during the second keyword
                FakeBrowser.made[-1].cdp.closed = OSError("gone")
                raise CDPError("connection lost: gone")
            return [place(len(SEARCHED))], "end-of-list", {"batches": 1, "dom_links": 0}
        maps.collect = collect
        jid, job = self.run_one(keywords=["a", "b", "c"])
        self.assertEqual((job["Status"], job["Error"]), ("ok", ""))
        self.assertEqual((SEARCHED, len(FakeBrowser.made)), (["a", "b", "b", "c"], 2))
        self.assertEqual(len(self.rows(jid)), 3)

    def test_chrome_that_keeps_dying_says_what_was_not_searched(self):
        def collect(page, url, **kw):
            FakeBrowser.made[-1].cdp.closed = OSError("gone")
            raise CDPError("connection lost: gone")
        maps.collect = collect
        _, job = self.run_one(keywords=["a", "b", "c"])
        self.assertEqual(job["Status"], "failed")
        self.assertIn("2 keyword(s) after it were not searched", job["Error"])
        self.assertEqual(len(FakeBrowser.made), 1 + jobs.RELAUNCHES)

    def test_a_dead_proxy_is_skipped_after_it_fails(self):
        calls = []
        def collect(page, url, **kw):
            calls.append(FakeBrowser.made[-1].contexts[-1])
            if "dead" in calls[-1]:
                raise CDPError("navigate https://x: net::ERR_PROXY_CONNECTION_FAILED")
            return [], "end-of-list", {"batches": 1, "dom_links": 0}
        maps.collect = collect
        _, job = self.run_one(keywords=["a", "b", "c", "d"], proxies=["http://dead:1", "http://ok:2"])
        self.assertEqual((job["Status"], calls), ("ok", ["http://dead:1"] + ["http://ok:2"] * 4))

    def test_headless_runs_the_reviews_pass(self):
        RESULTS["cafes in Austin"] = ([place(1)], "end-of-list")
        calls = []
        self.worker.add_reviews = lambda *a, **k: calls.append(a[1]["reviews_max"]) or 0
        _, job = self.run_one(reviews_max=2)
        self.assertEqual(job["Status"], "ok")
        self.assertEqual(calls, [2])
        self.assertNotIn("limited view", job["Error"])

    def test_errors_survive_a_restart_mid_search(self):
        name, data = jobs.validate(dict(GOOD, keywords=["a", "b", "c"]))
        jid = self.store.create(name, data)
        self.store.claim()
        self.store.set_progress(jid, {"done": 1, "total": 3, "stage": "search", "failed": 1,
                                      "last_error": "navigate x: net::ERR_NAME_NOT_RESOLVED"})
        self.store.recover()
        maps.collect = lambda page, url, **kw: ([place(2)], "end-of-list", {"batches": 1, "dom_links": 0})
        self.worker.run_job(self.store.claim())
        self.assertIn("1 keyword(s) failed: navigate x: net::ERR_NAME_NOT_RESOLVED", self.store.get(jid)["Error"])

    def test_late_website_pass_is_not_a_max_time_stop(self):
        RESULTS["cafes in Austin"] = ([place(1, website="https://a.com")], "end-of-list")
        def slow_pass(rows, seconds, **kw):
            time.sleep(1.2)  # runs past the job's 1 s max_time
            return 0, 0
        with unittest.mock.patch.object(jobs, "add_site_data", side_effect=slow_pass):
            _, job = self.run_one(email=True, max_time=1)
        self.assertEqual((job["Status"], job["Error"]), ("ok", ""))

    def test_masks_and_deep_json(self):
        self.assertEqual(jobs.mask("http://TOKEN123@h:80"), "http://***@h:80")
        self.assertEqual(jobs.mask("http://u:p@h:80"), "http://u:***@h:80")
        self.assertEqual(jobs.mask("http://h:80"), "http://h:80")
        servers = serve.make_servers(0, self.store, self.worker)
        port = servers[0].server_address[1]
        for srv in servers:
            threading.Thread(target=srv.serve_forever, daemon=True).start()
        try:
            c = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
            c.request("POST", "/api/v1/jobs", b"[" * 100000, {"Host": "localhost"})
            self.assertEqual(c.getresponse().status, 400)
            c.close()
        finally:
            for srv in servers:
                srv.shutdown()
                srv.server_close()


class LoopThree(Fakes):
    """Audit loop 3: passes that run out of time say so; a timeout with the DOM fallback counts."""

    def test_website_pass_out_of_time_says_so(self):
        RESULTS["cafes in Austin"] = ([place(1, website="https://a.com"), place(2, website="https://b.com")], "end-of-list")
        with unittest.mock.patch.object(jobs, "fetch", side_effect=lambda url, timeout=0: time.sleep(3) or ""), \
             unittest.mock.patch.object(jobs, "EMAIL_MIN_SECONDS", 0.3):
            _, job = self.run_one(email=True, max_time=1)
        self.assertEqual(job["Status"], "ok")
        self.assertIn("2 website(s) not checked: ran out of time", job["Error"])

    def test_reviews_out_of_time_say_so(self):
        RESULTS["cafes in Austin"] = ([place(1), place(2), place(3)], "end-of-list")
        self.worker.headless = False
        saved = (maps.place_panel, details.parse_place)
        maps.place_panel = lambda page, url: time.sleep(0.6) or "body"
        details.parse_place = lambda body, limit=5: {"user_reviews": "[]"}
        try:
            _, job = self.run_one(reviews_max=1, max_time=1)
        finally:
            maps.place_panel, details.parse_place = saved
        # Wall clock: 0.6 s per panel against max_time 1. A fast machine reads 2 of 3; a slow CI
        # runner (Windows) has read 0 or 1. The claim under test is that the note says how many.
        self.assertRegex(job["Error"], r"reviews read for [0-2] of 3 places: ran out of time")

    def test_timeout_with_the_dom_fallback_is_a_timeout(self):
        RESULTS["cafes in Austin"] = ([place(1)], "timeout+dom-fallback")
        _, job = self.run_one()
        self.assertIn("stopped at max_time", job["Error"])

    def test_dom_fallback_says_the_rows_are_thin(self):
        for places, said in (([place(1)], "only names, links and coordinates were kept for 1 places"),
                             ([], "no places were read")):
            RESULTS["cafes in Austin"] = (places, "no-growth+dom-fallback")
            _, job = self.run_one()
            self.assertEqual(job["Status"], "ok")
            self.assertIn(f"keyword 1: Google's result data could not be read, so {said}", job["Error"])

def _drip_server(body=True):
    """A local site that sends one byte every 0.3 s, for the deadline tests. Returns its port."""
    srv = socket.socket()
    srv.bind(("127.0.0.1", 0))
    srv.listen(5)

    def serve():
        while True:
            c, _ = srv.accept()
            try:
                c.recv(4096)
                c.sendall(b"HTTP/1.1 200 OK\r\n" + (b"Content-Length: 400000\r\n\r\n" if body else b""))
                while True:
                    c.sendall(b"x")
                    time.sleep(0.3)
            except OSError:
                pass
    threading.Thread(target=serve, daemon=True).start()
    return srv.getsockname()[1]


class AuditGuards(unittest.TestCase):
    """The P5 audit's guards: one check each (see the commit that added them)."""
    OK = {"name": "x", "keywords": ["a"], "lang": "en", "max_time": 5}

    def test_text_fields_reject_surrogates_control_characters_and_string_booleans(self):
        def err(**kw):
            with self.assertRaises(ValueError) as c:
                jobs.validate(dict(self.OK, **kw))
            return str(c.exception)
        self.assertIn("invalid name", err(name="\ud800"))  # sqlite and quote() refuse a lone surrogate
        self.assertIn("invalid keywords", err(keywords=["a\x00b"]))  # csv on 3.9 refuses NUL
        self.assertEqual(err(email="false"), "invalid email: true or false")
        self.assertFalse(jobs.validate(dict(self.OK, email=None, socials=False))[1]["email"])
        self.assertTrue(jobs.validate(dict(self.OK, keywords=["caf\u00e9 in Z\u00fcrich"]))[1]["keywords"])

    def test_public_ip_unwraps_embedded_ipv4_forms(self):
        for ip in ("::ffff:10.0.0.1", "64:ff9b::7f00:1", "::127.0.0.1", "224.0.0.1", "ff02::1", "fec0::1",
                   "2002:7f00:1::", "169.254.169.254"):
            self.assertFalse(jobs._public_ip(ip), ip)
        for ip in ("8.8.8.8", "::ffff:8.8.8.8", "64:ff9b::808:808", "2606:4700::1111"):
            self.assertTrue(jobs._public_ip(ip), ip)

    def test_redirect_to_ftp_refused(self):
        h = jobs._Redirects()
        self.assertIsNone(h.redirect_request(None, None, 302, "", {}, "ftp://93.184.216.34/x"))

    def test_unquoted_email_must_still_be_an_address(self):
        self.assertEqual(jobs.find_emails("a%[email protected] q%[email protected] [email protected]"), ["[email protected]"])

    def test_ascii_url_encodes_non_ascii_path(self):
        self.assertEqual(jobs.ascii_url("http://h/\u043a?q=\u00e4"), "http://h/%D0%BA?q=%C3%A4")
        self.assertEqual(jobs.ascii_url("http://h/plain?x=1"), "http://h/plain?x=1")

    def test_read_capped_stops_at_the_deadline(self):
        """A real dripping socket, with no hard stop: read1 lets the deadline check run between bytes."""
        port = _drip_server()
        t = time.monotonic()
        with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=5) as resp:
            out = jobs._read_capped(resp, 400_000, 1)
        self.assertLess(time.monotonic() - t, 3)
        self.assertTrue(0 < len(out) < 20)

    def test_dripping_website_is_cut_off_by_the_hard_stop(self):
        """Headers that never end: no read deadline can help, SITE_HARD shuts the socket."""
        port = _drip_server(body=False)
        with unittest.mock.patch.object(jobs, "SITE_HARD", 1), unittest.mock.patch.object(jobs, "_public_host", lambda h: True), \
                unittest.mock.patch.object(jobs, "_public_ip", lambda ip: True):
            t = time.monotonic()
            out = jobs.fetch(f"http://127.0.0.1:{port}/", timeout=2)
        self.assertLess(time.monotonic() - t, 4)
        self.assertEqual(out, "")
        self.assertIsNone(jobs._FETCH.timer)
        self.assertIsNone(jobs._FETCH.deadline)

    def test_coordinates_stored_in_plain_decimal_form(self):
        ok = {"name": "x", "keywords": ["a"], "lang": "en", "max_time": 5}
        for lat, lon, want in (("30.2672", "-97.7431", ("30.2672", "-97.7431")), ("1e1", "+2.50", ("10", "2.5")),
                               (0.00001, 5.0, ("0.00001", "5")), ("\u0663\u0660.\u0662", "1", ("30.2", "1"))):
            d = jobs.validate(dict(ok, lat=lat, lon=lon))[1]
            self.assertEqual((d["lat"], d["lon"]), want, lat)

    def test_public_tolerates_odd_proxies_from_a_1x_job(self):
        job = {"ID": "i", "Name": "n", "Date": "d", "Status": "ok", "Error": "", "Progress": {}}
        self.assertEqual(jobs.public(dict(job, Data={"proxies": 5, "keywords": []}))["Data"]["proxies"], [])
        self.assertEqual(jobs.public(dict(job, Data={"proxies": ["http://u:p@h:1", {"a": 1}], "keywords": []}))
                         ["Data"]["proxies"], ["http://u:***@h:1"])

    def test_second_engine_on_the_same_data_is_refused(self):
        with tempfile.TemporaryDirectory() as d:
            held = serve.lock_data(d)
            with self.assertRaises(OSError):
                serve.lock_data(d)
            held.close()
            serve.lock_data(d).close()  # released with the file

    def test_worker_survives_a_log_that_cannot_be_written(self):
        with tempfile.TemporaryDirectory() as d:
            store = jobs.Store(d)
            jid = store.create("n", jobs.validate({"name": "n", "keywords": ["a"], "lang": "en", "max_time": 5})[1])
            calls = []

            def bad_log(msg):
                calls.append(msg)
                raise BrokenPipeError
            w = jobs.Worker(store, bad_log, headless=True)
            w.run_job = lambda job: (_ for _ in ()).throw(RuntimeError("boom"))
            w.stopped = threading.Event()
            w.start()
            time.sleep(0.5)
            w.stopped.set()
            w.wake.set()
            w.join(5)
            self.assertFalse(w.is_alive())
            self.assertEqual(store.get(jid)["Status"], "failed")  # the second try wrote it after the pause
            store.db.close()  # Windows cannot delete the folder while jobs.db is open

    def test_env_value_keeps_a_hash_inside_quotes(self):
        self.assertEqual(serve.env_value('"ab #c"'), "ab #c")
        self.assertEqual(serve.env_value("plain # note"), "plain")
        self.assertEqual(serve.env_value("'q'"), "q")

    def test_odd_entity_values_do_not_raise(self):
        entity = [None] * 100
        entity[11], entity[10] = "Good place", 5  # data_id as a number raised TypeError and failed the whole batch
        self.assertEqual(details.parse_entity(entity).get("title"), "Good place")
        self.assertEqual(jobs.to_row({"title": 7, "place_id": "p", "data_id": "a\x00b"}, "kw")["data_id"], "ab")

if __name__ == "__main__":
    unittest.main(verbosity=1)