Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

30.7 KB
Raw Blame
"""DISCOVER — junk filtering, entity resolution, cost discipline, adapters.

The expensive stage, so the tests care most about the things that cost money
or corrupt the database: that an estimate is honest, that a budget is
enforced before spending rather than after, that a re-run doesn't duplicate
anyone, and that directories never make it in.
"""

import asyncio
import os
from contextlib import closing
import tempfile

import httpx
import pytest
import pytest_asyncio

from harvey.collectors import discover as D
from harvey.config import EnvConfig, load_config
from harvey.signals import SIGNAL_CATALOG, seed_signal_catalog
from harvey.state import StateManager


@pytest_asyncio.fixture
async def state():
    with tempfile.TemporaryDirectory() as tmp:
        sm = StateManager(os.path.join(tmp, "test.db"))
        await sm.init_db()
        await seed_signal_catalog(sm)
        for sig in SIGNAL_CATALOG:
            await sm.set_signal_status(sig["code"], "confirmed")
        yield sm


@pytest.fixture
def config():
    return load_config()


# The real implementations, captured before the autouse fixture stubs them,
# so the geocoding tests can exercise them directly.
REAL_GEOCODE = D.geocode_city
REAL_RESOLVE = D.resolve_coordinates
REAL_SLEEP = asyncio.sleep


@pytest.fixture(autouse=True)
def offline(monkeypatch):
    """No test reaches the network, and none of them wait a real second.

    The geocoding tests call REAL_GEOCODE / REAL_RESOLVE with their own
    mock-transport client.
    """
    async def no_geocode(state, city, radius_km=40, client=None):
        return ""

    async def no_sleep(_seconds):
        return None

    monkeypatch.setattr(D, "geocode_city", no_geocode)
    monkeypatch.setattr(D.asyncio, "sleep", no_sleep)


class FakeProvider(D.DiscoveryProvider):
    """A provider that returns what the test hands it, and spends nothing."""

    key = "fake"
    label = "Fake"
    cost_note = "free"

    def __init__(self, batches, per_query_cost=0.0):
        self.batches = list(batches)
        self.per_query_cost = per_query_cost
        self.calls = 0

    def configured(self, env):
        return True

    def estimate(self, queries):
        return self.per_query_cost * len(queries)

    async def fetch(self, client, env, query):
        self.calls += 1
        self.last_cost = self.per_query_cost
        return self.batches.pop(0) if self.batches else []


@pytest.fixture
def register(monkeypatch):
    """Install a fake provider into the registry for one test."""
    def _install(provider):
        monkeypatch.setitem(D.PROVIDERS, provider.key, provider)
        return provider
    return _install


def biz(**kw):
    kw.setdefault("provider", "fake")
    return D.Business(**kw)


# ── Junk filtering ──


@pytest.mark.parametrize("domain", [
    "yelp.com", "www.yellowpages.com", "angi.com", "homeadvisor.com",
    "facebook.com", "linkedin.com", "thumbtack.com", "houzz.com",
    "en.wikipedia.org", "reddit.com", "indeed.com",
    "miami.edu", "ufhealth.org", "umiamihealth.org",
    "somehospital.com", "mysite.wordpress.com", "shop.shopify.com",
    "192.168.1.1", "localhost",
])
def test_directories_and_institutions_are_junk(domain):
    assert D.is_junk(domain), f"{domain} should have been filtered out"


@pytest.mark.parametrize("domain", [
    "kettlemanroofing.com", "northvaleroofing.com", "sandpiperexteriors.com",
    "quarrylaneroofing.com", "smith-and-sons.co.uk", "acme.dental",
])
def test_real_businesses_survive_the_filter(domain):
    assert not D.is_junk(domain)


def test_listicle_titles_are_junk_even_with_a_clean_domain():
    assert D.is_junk("", "10 Best Roofers in Denver")
    assert D.is_junk("example.com", "Top Roofing Contractors in Dallas")
    assert D.is_junk("example.com", "Roofer Directory")
    assert not D.is_junk("example.com", "Northvale Roofing Company")
    assert D.is_junk("example.net", "Ohio HOA Management Company List")
    assert D.is_junk("example.com", "List of Plumbers in Austin")
    assert not D.is_junk("example.com", "Northvale Management")


def test_normalize_domain_is_the_entity_key():
    for raw in ("https://WWW.Example.com/path?q=1", "www.example.com",
                "http://example.com.", "EXAMPLE.COM"):
        assert D.normalize_domain(raw) == "example.com"
    assert D.normalize_domain("") == ""


# ── Cost discipline ──


def test_estimates_match_published_pricing():
    q = [D.DiscoveryQuery(term="roofer", location="Denver", limit=1000, depth=30)]
    # DataForSEO's own worked example: 1,000 items = $0.012 + $0.36 = $0.372
    assert D.estimate_cost("dataforseo_listings", q) == pytest.approx(0.372)
    # SERP billing is base price x pages; depth 30 is three pages.
    assert D.estimate_cost("dataforseo_serp", q) == pytest.approx(0.0018)
    assert D.estimate_cost("osm", q) == 0.0


@pytest.mark.asyncio
async def test_dry_run_estimates_without_calling_anything(state, config, register):
    fake = register(FakeProvider([[biz(domain="a.com")]], per_query_cost=0.5))
    q = [D.DiscoveryQuery(term="roofer", location="Denver")]
    report = await D.run_discovery(state, config, "fake", q, dry_run=True)
    assert report.estimated_cost == 0.5
    assert fake.calls == 0
    assert report.found == 0
    assert not await state.get_runs()   # a dry run isn't a run


@pytest.mark.asyncio
async def test_budget_cap_refuses_before_spending(state, config, register):
    fake = register(FakeProvider([[biz(domain="a.com")]] * 10, per_query_cost=1.0))
    q = [D.DiscoveryQuery(term="roofer", location=c) for c in "abcdefghij"]
    report = await D.run_discovery(state, config, "fake", q, max_spend=2.0)
    assert report.stopped == "over budget"
    assert fake.calls == 0, "the cap must be checked before the first call, not after"
    assert report.actual_cost == 0.0


@pytest.mark.asyncio
async def test_budget_stops_mid_run_when_actual_exceeds_estimate(state, config, register):
    # Cheap to estimate, expensive in reality — the loop has to notice.
    fake = FakeProvider([[biz(domain=f"{c}.com")] for c in "abcdefghij"],
                        per_query_cost=1.0)
    fake.estimate = lambda queries: 0.0
    register(fake)
    q = [D.DiscoveryQuery(term="roofer", location=c) for c in "abcdefghij"]
    report = await D.run_discovery(state, config, "fake", q, max_spend=3.0)
    assert report.stopped == "budget reached"
    assert fake.calls < 10


@pytest.mark.asyncio
async def test_kill_switch_stops_between_batches(state, config, register):
    await state.set_setting("discovery_paused", "stopped by hand")
    fake = register(FakeProvider([[biz(domain="a.com")]], per_query_cost=0.0))
    q = [D.DiscoveryQuery(term="roofer", location="Denver")]
    report = await D.run_discovery(state, config, "fake", q)
    assert report.stopped == "paused"
    assert fake.calls == 0


@pytest.mark.asyncio
async def test_nothing_runs_until_signals_are_confirmed(state, config, register):
    for sig in SIGNAL_CATALOG:
        await state.set_signal_status(sig["code"], "proposed")
    fake = register(FakeProvider([[biz(domain="a.com")]]))
    report = await D.run_discovery(state, config, "fake",
                                   [D.DiscoveryQuery(term="roofer")])
    assert report.stopped == "no confirmed signals"
    assert fake.calls == 0
    assert "confirm signals" in report.errors[0].lower()


# ── Entity resolution ──


@pytest.mark.asyncio
async def test_rerun_finds_everyone_already_known(state, config, register):
    batch = [
        biz(name="Northvale Roofing", domain="northvaleroofing.com"),
        biz(name="No Site Roofing", external_id="osm:node/1", has_website=False),
    ]
    register(FakeProvider([list(batch), list(batch)]))
    q = [D.DiscoveryQuery(term="roofer", location="Denver")]

    first = await D.run_discovery(state, config, "fake", q)
    assert (first.new_companies, first.known_companies) == (2, 0)

    second = await D.run_discovery(state, config, "fake", q)
    assert (second.new_companies, second.known_companies) == (0, 2), (
        "a re-run must recognise both the domain-keyed and the "
        "external-id-keyed record")


@pytest.mark.asyncio
async def test_websiteless_businesses_are_kept_and_deduped(state, config, register):
    """The best prospect for someone selling websites has no website, so
    `domain` can't be the only identity key."""
    register(FakeProvider([
        [biz(name="A", external_id="osm:node/1", has_website=False),
         biz(name="B", external_id="osm:node/2", has_website=False),
         biz(name="A again", external_id="osm:node/1", has_website=False)],
    ]))
    report = await D.run_discovery(state, config, "fake",
                                   [D.DiscoveryQuery(term="roofer")])
    assert report.new_companies == 2
    assert len(await state.cohort(["NO_WEBSITE"])) == 2


@pytest.mark.asyncio
async def test_duplicates_within_one_run_are_collapsed(state, config, register):
    register(FakeProvider([[
        biz(domain="northvale.com"), biz(domain="northvale.com"), biz(domain="other.com"),
    ]]))
    report = await D.run_discovery(state, config, "fake",
                                   [D.DiscoveryQuery(term="roofer")])
    assert report.found == 3
    assert report.new_companies == 2


@pytest.mark.asyncio
async def test_junk_is_counted_not_silently_dropped(state, config, register):
    register(FakeProvider([[
        biz(domain="yelp.com"), biz(domain="angi.com"), biz(domain="real.com"),
    ]]))
    report = await D.run_discovery(state, config, "fake",
                                   [D.DiscoveryQuery(term="roofer")])
    assert report.junk == 2
    assert report.new_companies == 1


# ── Observations ──


@pytest.mark.asyncio
async def test_every_fact_becomes_an_observation(state, config, register):
    register(FakeProvider([[biz(
        name="Northvale", domain="northvale.com", rank=14, rating=4.6,
        review_count=88, is_claimed=False, has_website=True,
    )]]))
    await D.run_discovery(state, config, "fake", [D.DiscoveryQuery(term="roofer")])

    rows = {o["signal_code"]: o for o in await state.get_observations()}
    assert rows["SERP_RANK"]["value_num"] == 14
    assert rows["REVIEW_RATING"]["value_num"] == pytest.approx(4.6)
    assert rows["REVIEW_COUNT"]["value_num"] == 88
    assert "UNCLAIMED_LISTING" in rows
    assert "NO_WEBSITE" not in rows
    assert rows["FOUND_IN_SERP"]["collector"] == "discover"


@pytest.mark.asyncio
async def test_unconfirmed_signals_are_not_recorded(state, config, register):
    """The confirmation gate has to hold inside the collector, not just in
    the UI that sets it."""
    await state.set_signal_status("REVIEW_RATING", "rejected")
    register(FakeProvider([[biz(domain="northvale.com", rating=4.6, review_count=88)]]))
    await D.run_discovery(state, config, "fake", [D.DiscoveryQuery(term="roofer")])

    codes = {o["signal_code"] for o in await state.get_observations()}
    assert "REVIEW_RATING" not in codes
    assert "REVIEW_COUNT" in codes


@pytest.mark.asyncio
async def test_a_failed_query_does_not_lose_the_successful_ones(state, config, register):
    class Flaky(FakeProvider):
        async def fetch(self, client, env, query):
            self.calls += 1
            if self.calls == 2:
                raise httpx.ConnectError("boom")
            return [biz(domain=f"co{self.calls}.com")]

    register(Flaky([], per_query_cost=0.0))
    q = [D.DiscoveryQuery(term="roofer", location=c) for c in ("a", "b", "c")]
    report = await D.run_discovery(state, config, "fake", q)

    assert report.new_companies == 2
    assert len(report.errors) == 1 and "boom" in report.errors[0]
    runs = await state.get_runs()
    assert runs[0]["status"] == "completed"


@pytest.mark.asyncio
async def test_run_is_logged_with_what_it_actually_cost(state, config, register):
    register(FakeProvider([[biz(domain="a.com")], [biz(domain="b.com")]],
                          per_query_cost=0.25))
    q = [D.DiscoveryQuery(term="roofer", location=c) for c in ("a", "b")]
    report = await D.run_discovery(state, config, "fake", q, max_spend=5.0)

    runs = await state.get_runs()
    assert runs[0]["stage"] == "discover"
    assert runs[0]["records"] == 2
    assert runs[0]["cost_usd"] == pytest.approx(0.5)
    assert report.actual_cost == pytest.approx(0.5)


# ── Provider adapters (parsing, against real response shapes) ──


DFS_LISTINGS = {
    "status_code": 20000,
    "cost": 0.0132,
    "tasks": [{"status_code": 20000, "result": [{"items": [
        {
            "title": "Northvale Roofing Company", "category": "Roofing contractor",
            "place_id": "ChIJabc123", "phone": "+15550100199",
            "url": "https://www.northvaleroofing.com/", "domain": "northvaleroofing.com",
            "address": "1400 Example Row, Raleigh, NC",
            "address_info": {"city": "Raleigh", "region": "North Carolina"},
            "is_claimed": True,
            "rating": {"value": 4.4, "votes_count": 512, "rating_max": 5},
            "check_url": "https://google.com/maps?cid=1",
        },
        {
            "title": "No Website Roofing", "category": "Roofing contractor",
            "place_id": "ChIJxyz789", "phone": "+15550100377",
            "url": None, "domain": None,
            "address_info": {"city": "Denver", "region": "Colorado"},
            "is_claimed": False,
            "rating": {"value": 4.9, "votes_count": 31, "rating_max": 5},
        },
    ]}]}],
}

DFS_SERP = {
    "status_code": 20000,
    "cost": 0.0018,
    "tasks": [{"status_code": 20000, "result": [{"items": [
        {"type": "organic", "rank_group": 1, "rank_absolute": 3,
         "position": "left", "domain": "yelp.com",
         "url": "https://yelp.com/denver", "title": "Best Roofers"},
        {"type": "organic", "rank_group": 14, "rank_absolute": 18,
         "position": "left", "domain": "kettlemanroofing.com",
         "url": "https://kettlemanroofing.com/", "title": "Kettleman Roofing"},
        {"type": "paid", "rank_group": 1, "domain": "ads.example.com"},
        {"type": "people_also_ask", "rank_group": 2},
    ]}]}],
}

SERPER_RESPONSE = {"organic": [
    {"title": "Kettleman Roofing", "link": "https://www.kettlemanroofing.com/",
     "position": 3},
    {"title": "Angi", "link": "https://www.angi.com/companylist/denver/roofing",
     "position": 4},
]}


def _client(handler):
    return httpx.AsyncClient(transport=httpx.MockTransport(handler))


@pytest.mark.asyncio
async def test_dataforseo_listings_adapter():
    provider = D.DataForSEOListings()
    env = {"dataforseo_login": "u", "dataforseo_password": "p"}
    seen = {}

    def handler(request):
        seen["url"] = str(request.url)
        seen["auth"] = request.headers.get("authorization")
        return httpx.Response(200, json=DFS_LISTINGS)

    async with _client(handler) as client:
        out = await provider.fetch(client, env, D.DiscoveryQuery(
            term="roofing contractor", location="Denver, CO",
            coordinate="39.7,-104.9,50"))

    assert seen["auth"].startswith("Basic ")
    assert provider.last_cost == pytest.approx(0.0132)

    northvale, nosite = out
    assert northvale.domain == "northvaleroofing.com"
    assert northvale.phone == "+15550100199"
    assert northvale.location == "Raleigh, North Carolina"
    assert northvale.rating == 4.4 and northvale.review_count == 512
    assert northvale.has_website is True
    assert northvale.external_id == "dataforseo:ChIJabc123"

    # The websiteless record is the valuable one — it must survive intact.
    assert nosite.domain == "" and nosite.has_website is False
    assert nosite.is_claimed is False
    assert nosite.identity() == "dataforseo:ChIJxyz789"


@pytest.mark.asyncio
async def test_dataforseo_serp_adapter_reads_rank_group_not_position():
    """`position` in this payload means left/right column, not rank. Reading
    it as rank would score every result identically."""
    provider = D.DataForSEOSerp()
    async with _client(lambda r: httpx.Response(200, json=DFS_SERP)) as client:
        out = await provider.fetch(
            client, {"dataforseo_login": "u", "dataforseo_password": "p"},
            D.DiscoveryQuery(term="roofer", location="Denver", depth=30))

    assert [b.rank for b in out] == [1, 14], "paid and PAA rows must be dropped"
    assert out[1].domain == "kettlemanroofing.com"


@pytest.mark.asyncio
async def test_serper_adapter():
    provider = D.Serper()
    async with _client(lambda r: httpx.Response(200, json=SERPER_RESPONSE)) as client:
        out = await provider.fetch(client, {"serper_api_key": "k"},
                                   D.DiscoveryQuery(term="roofer", location="Denver"))
    assert out[0].domain == "kettlemanroofing.com" and out[0].rank == 3
    # Junk filtering happens in the run, so the adapter still returns Angi.
    assert out[1].domain == "angi.com"


@pytest.mark.asyncio
async def test_dataforseo_errors_surface_the_message_not_just_the_code():
    """A bare status code costs several wrong guesses; the body usually makes
    the problem obvious."""
    payload = {"status_code": 40501, "status_message": "Invalid Field: 'categories'"}
    provider = D.DataForSEOListings()
    with pytest.raises(RuntimeError, match="Invalid Field"):
        async with _client(lambda r: httpx.Response(200, json=payload)) as client:
            await provider.fetch(client, {"dataforseo_login": "u",
                                          "dataforseo_password": "p"},
                                 D.DiscoveryQuery(term="roofer"))


def test_dataforseo_sandbox_is_never_the_live_host():
    provider = D.DataForSEOListings()
    assert "sandbox" in provider.base_url({"dataforseo_sandbox": "1"})
    assert "sandbox" not in provider.base_url({})


# ── The provider menu the dashboard renders ──


def test_provider_menu_explains_cost_and_configuration():
    menu = {m["key"]: m for m in D.provider_menu({"serper_api_key": "k"})}
    assert menu["osm"]["configured"] is True and not menu["osm"]["needs_key"]
    assert menu["serper"]["configured"] is True
    assert menu["dataforseo_listings"]["configured"] is False
    assert menu["dataforseo_listings"]["env_keys"] == [
        "DATAFORSEO_LOGIN", "DATAFORSEO_PASSWORD"]
    for entry in menu.values():
        assert entry["blurb"] and entry["cost_note"] and entry["signup_url"]


def test_the_default_provider_needs_no_account():
    """Someone should be able to run the whole pipeline before paying anyone."""
    default = D.PROVIDERS[D.DEFAULT_PROVIDER]
    assert default.env_keys == ()
    assert default.caveat, "a free source with thin coverage must say so"


@pytest.mark.asyncio
async def test_unconfigured_provider_says_which_key_is_missing(state, config):
    report = await D.run_discovery(state, config, "dataforseo_listings",
                                   [D.DiscoveryQuery(term="roofer")])
    assert report.stopped == "not configured"
    assert "DATAFORSEO_LOGIN" in report.errors[0]


def test_unknown_provider_fails_loudly():
    with pytest.raises(ValueError, match="unknown discovery provider"):
        import asyncio
        asyncio.run(D.run_discovery(None, None, "nope"))


@pytest.mark.asyncio
async def test_overpass_throttling_is_an_error_not_an_empty_market(monkeypatch):
    """A silent [] on a 504 is indistinguishable from 'no businesses here',
    which is how a throttled run gets mistaken for a clean one."""
    monkeypatch.setattr(D.OpenStreetMap, "BACKOFF", 0.0)
    seen = []

    def handler(request):
        seen.append(str(request.url))
        return httpx.Response(504, text="")

    provider = D.OpenStreetMap()
    with pytest.raises(RuntimeError, match="throttling"):
        async with _client(handler) as client:
            await provider.fetch(client, {}, D.DiscoveryQuery(
                term="roofer", coordinate="39.7,-104.9,40"))
    assert seen == list(D.OpenStreetMap.ENDPOINTS), "every mirror should be tried"


@pytest.mark.asyncio
async def test_overpass_falls_over_to_the_next_mirror(monkeypatch):
    monkeypatch.setattr(D.OpenStreetMap, "BACKOFF", 0.0)
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        if calls["n"] == 1:
            return httpx.Response(504, text="")
        return httpx.Response(200, json={"elements": [{
            "type": "node", "id": 42,
            "tags": {"name": "Sandpiper Exteriors", "craft": "roofer",
                     "phone": "+15550100288", "addr:city": "Denver"},
        }]})

    provider = D.OpenStreetMap()
    async with _client(handler) as client:
        out = await provider.fetch(client, {}, D.DiscoveryQuery(
            term="roofer", coordinate="39.7,-104.9,40"))
    assert len(out) == 1
    assert out[0].name == "Sandpiper Exteriors"
    assert out[0].external_id == "osm:node/42"
    assert out[0].has_website is False


@pytest.mark.asyncio
async def test_overpass_needs_a_coordinate_and_says_so(caplog):
    provider = D.OpenStreetMap()
    async with _client(lambda r: httpx.Response(200, json={})) as client:
        out = await provider.fetch(client, {}, D.DiscoveryQuery(
            term="roofer", location="Denver, CO"))
    assert out == []
    assert "coordinate" in caplog.text.lower()


# ── Geocoding ──


def _geo_client(handler):
    return httpx.AsyncClient(transport=httpx.MockTransport(handler))


@pytest.mark.asyncio
async def test_geocode_is_cached_so_a_city_is_looked_up_once(state):
    """Nominatim permits single place lookups and forbids bulk enumeration.
    Caching forever is what keeps us on the right side of that."""
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        assert "Harvey" in request.headers["user-agent"]
        return httpx.Response(200, json=[{"lat": "39.7392", "lon": "-104.9903"}])

    async with _geo_client(handler) as client:
        assert await REAL_GEOCODE(state, "Denver, CO", client=client) \
            == "39.7392,-104.9903,40"
        assert await REAL_GEOCODE(state, "Denver, CO", client=client) \
            == "39.7392,-104.9903,40"
        assert await REAL_GEOCODE(state, "DENVER, co", client=client) \
            == "39.7392,-104.9903,40"
    assert calls["n"] == 1


@pytest.mark.asyncio
async def test_a_geocode_miss_is_cached_too(state):
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        return httpx.Response(200, json=[])

    async with _geo_client(handler) as client:
        assert await REAL_GEOCODE(state, "Nowhere XYZ", client=client) == ""
        assert await REAL_GEOCODE(state, "Nowhere XYZ", client=client) == ""
    assert calls["n"] == 1, "a miss must not be re-asked on every run"


@pytest.mark.asyncio
async def test_configured_coordinates_win_over_geocoding(state, monkeypatch):
    async def boom(*a, **kw):
        raise AssertionError("should not geocode a query that already has a point")
    monkeypatch.setattr(D, "geocode_city", boom)

    queries = [D.DiscoveryQuery(term="roofer", location="Denver, CO",
                                coordinate="1.0,2.0,25")]
    out = await REAL_RESOLVE(state, queries)
    assert out[0].coordinate == "1.0,2.0,25"


@pytest.mark.asyncio
async def test_one_geocode_per_city_not_per_query(state, monkeypatch):
    """Twelve queries across four cities is four lookups, not twelve."""
    monkeypatch.setattr(D, "geocode_city", REAL_GEOCODE)
    calls = {"n": 0}

    def handler(request):
        calls["n"] += 1
        return httpx.Response(200, json=[{"lat": "1.0", "lon": "2.0"}])

    queries = [D.DiscoveryQuery(term=t, location=c)
               for t in ("roofing", "gutters", "siding")
               for c in ("Denver, CO", "Dallas, TX", "Tampa, FL", "Nashville, TN")]
    async with _geo_client(handler) as client:
        await REAL_RESOLVE(state, queries, client=client)
    assert calls["n"] == 4
    assert all(q.coordinate == "1.0000,2.0000,40" for q in queries)


@pytest.mark.asyncio
async def test_osm_refuses_a_term_it_cannot_map_instead_of_guessing():
    """An earlier fallback to `office=company` answered "roofers in Denver"
    with every registered office in Denver. Returning nothing is correct;
    returning plausible junk is not."""
    provider = D.OpenStreetMap()
    with pytest.raises(RuntimeError, match="no category"):
        async with _client(lambda r: httpx.Response(200, json={})) as client:
            await provider.fetch(client, {}, D.DiscoveryQuery(
                term="storm damage restoration", coordinate="39.7,-104.9,40"))


@pytest.mark.asyncio
async def test_osm_maps_the_trades_it_does_know():
    provider = D.OpenStreetMap()
    assert provider._tags("Roofing contractors") == ["craft=roofer", "shop=roofing"]
    assert provider._tags("Residential roofing") == ["craft=roofer", "shop=roofing"]
    assert provider._tags("Emergency plumber") == ["craft=plumber"]
    assert provider._tags("cosmetic dentist") == ["amenity=dentist"]
    assert provider._tags("B2B SaaS") == []


@pytest.mark.asyncio
async def test_an_unmappable_term_is_reported_not_silently_skipped(
        state, config, register, monkeypatch):
    """The run must surface why a query produced nothing."""
    monkeypatch.setitem(D.PROVIDERS, "osm", D.OpenStreetMap())

    async def fake_fetch(self, client, env, query):
        raise RuntimeError("OpenStreetMap has no category for 'widgets'")
    monkeypatch.setattr(D.OpenStreetMap, "fetch", fake_fetch)

    report = await D.run_discovery(state, config, "osm",
                                   [D.DiscoveryQuery(term="widgets",
                                                     coordinate="1,2,40")])
    assert report.found == 0
    assert report.errors and "no category" in report.errors[0]


# ── business names: SERP titles are page titles, not names ──

@pytest.mark.parametrize("title,domain,expected", [
    ("Northvale Property Management: HOA Management Company in ...", "northvaleproperty.com",
     "Northvale Property Management"),
    ("Home Page - NVCM | Community Association Management", "nvcmnet.com", "NVCM"),
    ("Northvale Association of Community Managers", "nacm.org",
     "Northvale Association of Community Managers"),
    ("Brightwater Texas: Texas HOA Property Management Company", "brightwatertexas.com",
     "Brightwater Texas"),
    # No segment names the business → fall back to the domain, split sensibly.
    ("Denver, CO | HOA Management Firm", "managementharbor.com", "Management Harbor"),
    ("Texas HOA management", "kqresidential.com", "KQ Residential"),
    ("Great Texas Community Management Begins Here", "growwithus.us", "Grow With Us"),
])
def test_business_name_from_serp_title(title, domain, expected):
    assert D.business_name(title, domain) == expected


def test_generic_words_alone_never_match_the_domain():
    """'Management' is in managementharbor.com but says nothing about it."""
    assert not D.name_matches_domain("HOA Management Firm", "managementharbor.com")
    assert D.name_matches_domain("The Management Harbor", "managementharbor.com")


def test_site_name_prefers_og_then_title_and_needs_domain_backing():
    og = '<meta property="og:site_name" content="Grow Community Management" />'
    assert D.site_name_from_html(og, "growwithus.us") == "Grow Community Management"
    title = "<title>Lumen Association Management - HOA Company in TX &amp; CO</title>"
    assert D.site_name_from_html(title, "lumenam.com") == "Lumen Association Management"
    both = ('<meta property="og:site_name" content="Management Harbor">'
            "<title>The Management Harbor: Community Management</title>")
    assert D.site_name_from_html(both, "managementharbor.com") == "The Management Harbor"
    assert D.site_name_from_html("<title>Attention Required! | Cloudflare</title>",
                                 "lumenam.com") == ""


@pytest.mark.asyncio
@pytest.mark.parametrize('provider', ['serper', 'dataforseo_serp'])
async def test_directory_title_still_filtered_after_adapter(provider, tmp_path, monkeypatch):
    title = 'Top 10 Roofing Companies in Denver'
    domain = 'milehighguide.com'
    assert D.is_junk(domain, title)
    item = {'title': title, 'link': f'https://{domain}/roofers',
            'url': f'https://{domain}/roofers', 'domain': domain,
            'position': 1, 'rank_group': 1, 'type': 'organic'}
    valid = dict(item, title='Kettleman Roofing | Denver',
                 domain='kettlemanroofing.com', link='https://kettlemanroofing.com',
                 url='https://kettlemanroofing.com', position=2, rank_group=2)
    payload = ({'organic': [item, valid]} if provider == 'serper' else
               {'status_code': 20000, 'tasks': [{'status_code': 20000,
                 'result': [{'items': [item, valid]}]}]})
    real_client = httpx.AsyncClient
    monkeypatch.setattr(D.httpx, 'AsyncClient', lambda **kw: real_client(
        transport=httpx.MockTransport(lambda request: httpx.Response(200, json=payload)), **kw))
    monkeypatch.setattr('harvey.config.load_env', lambda: EnvConfig(
        serper_api_key='dummy', dataforseo_login='dummy', dataforseo_password='dummy'))
    state = StateManager(str(tmp_path / 'test.db'))
    await state.init_db()
    await seed_signal_catalog(state)
    for signal in SIGNAL_CATALOG:
        await state.set_signal_status(signal['code'], 'confirmed')
    report = await D.run_discovery(state, load_config(), provider_key=provider,
        queries=[D.DiscoveryQuery(term='roofer', location='Denver')])
    assert report.errors == []
    assert report.junk == 1, report.as_dict()
    assert report.found == 2
    assert report.new_companies == 1
    import sqlite3
    with closing(sqlite3.connect(state.db_path)) as db:
        assert db.execute("SELECT domain, name FROM companies").fetchall() == [
            ("kettlemanroofing.com", "Kettleman Roofing")]


@pytest.mark.parametrize('title,domain,expected', [
    ('Contact Us | ContactCraft', 'contactcraft.com', 'ContactCraft'),
    ('Craft | ContactCraft', 'contactcraft.com', 'ContactCraft'),
    ('Roofing Services in Denver | Kettleman Roofing', 'kettlemanroofing.com', 'Kettleman Roofing'),
    ('Plumbing Services in Austin | Acme Plumbing', 'acmeplumbing.com', 'Acme Plumbing'),
])
def test_brand_wins_over_generic_service_segment(title, domain, expected):
    assert D.business_name(title, domain) == expected


@pytest.mark.parametrize("service,brand,domain", [
    ("Landscaping", "Kettleman Landscaping", "kettlemanlandscaping.com"),
    ("Electrical", "Kettleman Electrical", "kettlemanelectrical.com"),
])
def test_service_location_labels_do_not_establish_brand_identity(service, brand, domain):
    label = f"{service} Services in Denver"
    assert not D.name_matches_domain(label, domain)
    assert D.business_name(f"{label} | {brand}", domain) == brand
    html = f'<meta property="og:site_name" content="{brand}"><title>{label}</title>'
    assert D.site_name_from_html(html, domain) == brand
    assert D.site_name_from_html(f"<title>{label}</title>", domain) == ""


def test_homepage_prefers_stronger_brand_over_longer_partial_match():
    html = ('<meta property="og:site_name" content="Kettleman Landscaping">'
            '<title>Landscaping Professionals</title>')
    assert D.site_name_from_html(html, "kettlemanlandscaping.com") == "Kettleman Landscaping"