Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.
fix: retain discovery filtering and prefer verified brand names
Ethan+8944bd6df787
diff --cc harvey/collectors/discover.pyindex 51f824f,1299a80..e2dd85d--- a/harvey/collectors/discover.py+++ b/harvey/collectors/discover.py@@@ -91,110 -87,6 +91,130 @@@ def normalize_domain(url_or_domain: strreturn host[4:] if host.startswith("www.") else host+# ── Business names ────────────────────────────────────────────────────+# SERP providers return a page TITLE, not a business name: "Denver, CO | HOA+# Management Firm", "Home Page - NVCM | Community Association Management".+# Stored as-is it lands in an email as "...at Denver, CO | HOA Management+# Firm". The business is the title segment that shares a DISTINCTIVE word with+# the domain — "management" matches managementharbor.com but says nothing.++_TITLE_SEPARATORS = re.compile(r"\s+[|\-–—·•]\s+|\s*[|•·]\s*|:\s+")+_GENERIC_NAME_WORDS = frozenset("""+home page welcome official site website hoa hoas management manager managers+mgmt mgt company companies co inc llc ltd corp firm group services service+property properties community communities association associations condo+condominium residential homeowners owners real estate the and of in for best+top leading trusted local full arizona phoenix az texas tx florida fl+california ca nevada nv colorado co georgia ga carolina nc usa us america++roofing roof roofers roofer plumbing plumber plumbers contact about+""".split())+++def _words(text: str) -> list[str]:+ return [w for w in re.findall(r"[a-z0-9]+", text.lower()) if w]+++def _domain_stem(domain: str) -> str:+ host = normalize_domain(domain)+ return re.sub(r"[^a-z0-9]", "", host.rsplit(".", 1)[0]) if host else ""+++def _segment_matches_domain(segment: str, stem: str) -> bool:++ # Navigation and service/location labels can share an incidental word++ # with a domain without identifying the business.++ if re.fullmatch(r"(?i)(contact(?: us)?|about(?: us)?|home(?: page)?)", segment.strip()):++ return False++ if re.search(r"(?i)\bservices?\s+(?:in|near)\s+\S", segment):++ return False+ words = _words(segment)+ distinctive = [w for w in words if w not in _GENERIC_NAME_WORDS]+ if any(len(w) >= 3 and w in stem for w in distinctive):+ return True+ # "Northvale Association of Community Managers" ↔ nacm.org+ initials = "".join(w[0] for w in words if w not in ("of", "the", "and", "&"))+ return len(initials) >= 3 and stem.startswith(initials)++++def _segment_domain_evidence(segment: str, stem: str) -> int:++ """Amount of distinctive domain evidence, shared by SERP and homepage names."""++ if not _segment_matches_domain(segment, stem):++ return 0++ return sum(len(w) for w in set(_words(segment))++ if w not in _GENERIC_NAME_WORDS and len(w) >= 3 and w in stem)+++++_SPLIT_WORDS = sorted((w for w in _GENERIC_NAME_WORDS | {"with", "your"}+ if len(w) >= 4), key=len, reverse=True)+++def _split_stem(stem: str) -> list[str]:+ """managementharbor → [management, harbor]; kqresidential → [kq, residential]."""+ parts, buf, i = [], "", 0+ while i < len(stem):+ word = next((w for w in _SPLIT_WORDS if stem.startswith(w, i)), "")+ if word:+ if buf:+ parts.append(buf)+ buf = ""+ parts.append(word)+ i += len(word)+ else:+ buf += stem[i]+ i += 1+ if buf:+ parts.append(buf)+ return parts+++def name_from_domain(domain: str) -> str:+ stem = normalize_domain(domain).rsplit(".", 1)[0]+ words = [w for chunk in re.split(r"[-_.]+", stem) if chunk for w in _split_stem(chunk)]+ # Short vowel-less leftovers are initials: "kq" → "KQ".+ return " ".join(w.upper() if len(w) <= 3 and not re.search(r"[aeiouy]", w)+ else w.capitalize() for w in words)+++def name_matches_domain(name: str, domain: str) -> bool:+ """True when the stored name demonstrably belongs to this domain."""+ stem = _domain_stem(domain)+ return bool(stem and name and _segment_matches_domain(name, stem))+++def site_name_from_html(html: str, domain: str) -> str:+ """The business name a homepage gives itself (og:site_name, then <title>),+ or "" when neither names it in a way the domain backs up."""+ stem = _domain_stem(domain)+ candidates = []+ for pat in (r'<meta[^>]+property=["\']og:site_name["\'][^>]*content=["\']([^"\']+)',+ r'<meta[^>]+content=["\']([^"\']+)["\'][^>]*property=["\']og:site_name',+ r"(?is)<title[^>]*>(.*?)</title>"):+ m = re.search(pat, html or "", re.I)+ if m:+ candidates.append(unescape(re.sub(r"\s+", " ", m.group(1))).strip())- # The fullest name wins: "The Management Harbor" over og's "Management Harbor".++ # Distinctive brand evidence wins; equally backed names prefer the fullest++ # form: "The Management Harbor" over og's "Management Harbor".+ matches = [seg for text in candidates+ for seg in (x.strip(" ,.-") for x in _TITLE_SEPARATORS.split(text))+ if seg and len(seg) <= 60 and _segment_matches_domain(seg, stem)]- return max(matches, key=len, default="")++ return max(matches, key=lambda seg: (_segment_domain_evidence(seg, stem), len(seg)),++ default="")+++def business_name(title: str, domain: str) -> str:+ """The business's name from a SERP title, or one derived from its domain."""+ stem = _domain_stem(domain)+ title = re.sub(r"\s*(\.\.\.|…)\s*$", "", (title or "").strip())+ if stem and title:- for segment in _TITLE_SEPARATORS.split(title):- segment = segment.strip(" ,.-")- if segment and len(segment) <= 60 and _segment_matches_domain(segment, stem):- return segment++ segments = [segment.strip(" ,.-") for segment in _TITLE_SEPARATORS.split(title)]++ matches = [seg for seg in segments++ if seg and len(seg) <= 60 and _segment_matches_domain(seg, stem)]++ # Prefer the segment with the most distinctive domain evidence rather++ # than the first incidental substring. Stable ties preserve title order.++ if matches:++ return max(matches, key=lambda seg: _segment_domain_evidence(seg, stem))+ return name_from_domain(domain) if stem else title++def is_junk(domain: str, name: str = "") -> bool:"""True when this row would pollute the database rather than fill it."""if domain:@@@ -301,6 -193,6 +321,7 @@@ class Businessexternal_id: str = "" # "<provider>:<stable id>"source_url: str = ""provider: str = ""++ raw_title: str = "" # SERP evidence for junk filtering before normalizationdef identity(self) -> str:return self.domain or self.external_id@@@ -495,7 -387,7 +516,8 @@@ class DataForSEOSerp(DiscoveryProvider)if not domain:continueout.append(Business(- name=item.get("title") or domain,+ name=business_name(item.get("title") or "", domain),++ raw_title=item.get("title") or "",domain=domain,website=item.get("url") or "",location=query.location,@@@ -546,7 -438,7 +568,8 @@@ class Serper(DiscoveryProvider)if not domain:continueout.append(Business(- name=item.get("title") or domain,+ name=business_name(item.get("title") or "", domain),++ raw_title=item.get("title") or "",domain=domain,website=item.get("link") or "",location=query.location,@@@ -882,7 -774,7 +905,7 @@@ async def run_discoveryreport.found += len(found)batch = []for biz in found:-- if is_junk(biz.domain, biz.name):++ if is_junk(biz.domain, biz.raw_title or biz.name):report.junk += 1continueident = biz.identity()@@@ -984,3 -876,3 +1007,4 @@@ def provider_menu(env: dict | None = No}for p in PROVIDERS.values()]++diff --cc harvey/collectors/profile.pyindex bf77444,2f9b69c..5c10652--- a/harvey/collectors/profile.py+++ b/harvey/collectors/profile.py@@@ -502,22 -455,6 +502,23 @@@ class ProfileCollectorreturn obs+ async def _improve_name(self, company: dict, domain: str):+ """Replace a name the domain doesn't back up ("Denver, CO | HOA+ Management Firm") or one merely derived from the domain ("Grow With- Us") with the one the homepage gives itself. A real match is kept."""++ Us") with the one the homepage gives itself. Only a distinctive brand match++ is kept; generic industry and navigation text does not establish identity."""+ better = self.site_names.pop(company["id"], "")+ current = company.get("name") or ""+ fallback = current == name_from_domain(domain)+ if not better or better == current or (+ name_matches_domain(current, domain) and not fallback):+ return+ try:+ await self.state.rename_company(company["id"], better)+ logger.info(f"profile: {domain} named '{better}' (was '{current}')")+ except Exception as e:+ logger.debug(f"rename_company failed for {domain}: {e}")+async def _mark_checked(self, company_id: str):try:await self.state.mark_company_profiled(company_id)@@@ -573,3 -509,3 +574,4 @@@ async def profile_companies(state, compexcept Exception as e:await state.finish_run(run_id, status="failed", error=str(e))raise++diff --cc harvey/state.pyindex 6401fb7,a82b4c6..347b14a--- a/harvey/state.py+++ b/harvey/state.py@@@ -1857,3 -1843,3 +1857,4 @@@ class StateManager# Claude budget the rest of the loop is rationing."unprofiled": await self.count_companies_needing_profile(),}++diff --cc tests/test_collectors_profile.pyindex d7d1008,f631c56..99f3d1d--- a/tests/test_collectors_profile.py+++ b/tests/test_collectors_profile.py@@@ -8,6 -8,6 +8,7 @@@ import pytest_asyncifrom harvey.collectors.profile import ProfileCollector, title_looks_like_role, _strip_codefrom harvey.models.company import Company++from harvey.models.prospect import Prospectfrom harvey.signals import SIGNAL_CATALOG, seed_signal_catalogfrom harvey.state import StateManager@@@ -192,36 -155,3 +193,98 @@@ async def test_user_decision_survives_rawait seed_signal_catalog(state) # e.g. after an upgradecodes = {c["code"]: c["status"] for c in await state.get_signal_codes()}assert codes["SERP_RANK"] == "rejected", "re-seeding must not undo a user's choice"+++async def test_profile_upgrades_a_name_the_domain_does_not_back(state, monkeypatch):+ cid = await state.add_company(Company(name="Denver, CO | HOA Management Firm",+ domain="managementharbor.com"))+ keep = await state.add_company(Company(name="Birch Management", domain="birchmgt.com"))+ derived = await state.add_company(Company(name="Grow With Us", domain="growwithus.us"))+ pages = {+ "managementharbor.com": '<meta property="og:site_name" content="Management Harbor">',+ "birchmgt.com": "<title>Denver HOA | Birch Management Group</title>",+ "growwithus.us": '<meta property="og:site_name" content="Grow Community Management">',+ }++ async def fake_resolve(self, domain):+ return f"https://{domain}", pages[domain]++ async def fake_first(self, site, paths):+ return "", ""++ monkeypatch.setattr(ProfileCollector, "_resolve_site", fake_resolve)+ monkeypatch.setattr(ProfileCollector, "_first_present", fake_first)+ async with ProfileCollector(state, {"TECH_STACK"}) as c:+ await c.run([{"id": cid, "name": "Denver, CO | HOA Management Firm",+ "domain": "managementharbor.com"},+ {"id": keep, "name": "Birch Management", "domain": "birchmgt.com"},+ {"id": derived, "name": "Grow With Us", "domain": "growwithus.us"}])+ import sqlite3+ with sqlite3.connect(state.db_path) as db:+ names = dict(db.execute("SELECT domain, name FROM companies"))+ assert names["managementharbor.com"] == "Management Harbor"+ assert names["birchmgt.com"] == "Birch Management" # already right: untouched+ assert names["growwithus.us"] == "Grow Community Management" # was domain-derived++++[email protected]("current,domain,homepage,expected", [++ ("Roofing Services in Denver", "kettlemanroofing.com", "Kettleman Roofing", "Kettleman Roofing"),++ ("Plumbing Services in Austin", "acmeplumbing.com", "Acme Plumbing", "Acme Plumbing"),++ ("Contact Us", "contactcraft.com", "ContactCraft", "ContactCraft"),++ ("Birch Management", "birchmgt.com", "Birch Management Group", "Birch Management"),++ ("NVCM", "nvcmnet.com", "NVCM Group", "NVCM"),++ ("Northvale Association of Community Managers", "nacm.org",++ "Northvale Association of Community Managers Group", "Northvale Association of Community Managers"),++ ("Grow With Us", "growwithus.us", "Grow Community Management", "Grow Community Management"),++ ("Roofing Services in Denver", "kettlemanroofing.com", "", "Roofing Services in Denver"),++])++async def test_profile_repairs_weak_names_and_retains_brands(++ state, monkeypatch, current, domain, homepage, expected):++ cid = await state.add_company(Company(name=current, domain=domain))++ pid = await state.add_prospect(Prospect(company_id=cid, company=current,++ first_name="Test", last_name="Person"))++++ async def fake_resolve(self, domain):++ return f"https://{domain}", f"<title>{homepage}</title>"++++ async def no_sleep(_seconds):++ return None++++ monkeypatch.setattr(ProfileCollector, "_resolve_site", fake_resolve)++ monkeypatch.setattr("harvey.collectors.profile.asyncio.sleep", no_sleep)++ async with ProfileCollector(state, set()) as c:++ await c.run([{"id": cid, "name": current, "domain": domain}])++ import sqlite3++ with sqlite3.connect(state.db_path) as db:++ assert db.execute("SELECT name FROM companies WHERE id = ?", (cid,)).fetchone() == (expected,)++ assert db.execute("SELECT company FROM prospects WHERE id = ?", (pid,)).fetchone() == (expected,)++++[email protected]("industry", ["Landscaping", "Electrical"])[email protected]("use_fallback", [False, True])++async def test_profile_uses_homepage_brand_over_service_location_title(++ state, monkeypatch, industry, use_fallback):++ from harvey.collectors.discover import name_from_domain++ domain = f"kettleman{industry.lower()}.com"++ brand = f"Kettleman {industry}"++ label = f"{industry} Services in Denver"++ current = name_from_domain(domain) if use_fallback else label++ cid = await state.add_company(Company(name=current, domain=domain))++ pid = await state.add_prospect(Prospect(company_id=cid, company=current))++++ async def fake_resolve(self, domain):++ return f"https://{domain}", (++ f'<meta property="og:site_name" content="{brand}"><title>{label}</title>')++++ async def no_sleep(_seconds):++ return None++++ monkeypatch.setattr(ProfileCollector, "_resolve_site", fake_resolve)++ monkeypatch.setattr("harvey.collectors.profile.asyncio.sleep", no_sleep)++ async with ProfileCollector(state, set()) as c:++ await c.run([{"id": cid, "name": current, "domain": domain}])++ assert (await state.get_company_by_domain(domain)).name == brand++ assert (await state.get_prospect(pid)).company == branddiff --cc tests/test_discover.pyindex e04957a,d1344d8..1aad389--- a/tests/test_discover.py+++ b/tests/test_discover.py@@@ -15,7 -15,7 +15,7 @@@ import pytesimport pytest_asynciofrom harvey.collectors import discover as D--from harvey.config import load_config++from harvey.config import EnvConfig, load_configfrom harvey.signals import SIGNAL_CATALOG, seed_signal_catalogfrom harvey.state import StateManager@@@ -661,40 -658,3 +661,106 @@@ async def test_an_unmappable_term_is_recoordinate="1,2,40")])assert report.found == 0assert report.errors and "no category" in report.errors[0]+++# ── business names: SERP titles are page titles, not names ──+[email protected]("title,domain,expected", [+ ("Northvale Property Management: HOA Management Company in ...", "northvaleproperty.com",+ "Northvale Property Management"),+ ("Home Page - NVCM | Community Association Management", "nvcmnet.com", "NVCM"),+ ("Northvale Association of Community Managers", "nacm.org",+ "Northvale Association of Community Managers"),+ ("Brightwater Texas: Texas HOA Property Management Company", "brightwatertexas.com",+ "Brightwater Texas"),+ # No segment names the business → fall back to the domain, split sensibly.+ ("Denver, CO | HOA Management Firm", "managementharbor.com", "Management Harbor"),+ ("Texas HOA management", "kqresidential.com", "KQ Residential"),+ ("Great Texas Community Management Begins Here", "growwithus.us", "Grow With Us"),+])+def test_business_name_from_serp_title(title, domain, expected):+ assert D.business_name(title, domain) == expected+++def test_generic_words_alone_never_match_the_domain():+ """'Management' is in managementharbor.com but says nothing about it."""+ assert not D.name_matches_domain("HOA Management Firm", "managementharbor.com")+ assert D.name_matches_domain("The Management Harbor", "managementharbor.com")+++def test_site_name_prefers_og_then_title_and_needs_domain_backing():+ og = '<meta property="og:site_name" content="Grow Community Management" />'+ assert D.site_name_from_html(og, "growwithus.us") == "Grow Community Management"+ title = "<title>Lumen Association Management - HOA Company in TX & CO</title>"+ assert D.site_name_from_html(title, "lumenam.com") == "Lumen Association Management"+ both = ('<meta property="og:site_name" content="Management Harbor">'+ "<title>The Management Harbor: Community Management</title>")+ assert D.site_name_from_html(both, "managementharbor.com") == "The Management Harbor"+ assert D.site_name_from_html("<title>Attention Required! | Cloudflare</title>",+ "lumenam.com") == ""++++[email protected]('provider', ['serper', 'dataforseo_serp'])++async def test_directory_title_still_filtered_after_adapter(provider, tmp_path, monkeypatch):++ title = 'Top 10 Roofing Companies in Denver'++ domain = 'milehighguide.com'++ assert D.is_junk(domain, title)++ item = {'title': title, 'link': f'https://{domain}/roofers',++ 'url': f'https://{domain}/roofers', 'domain': domain,++ 'position': 1, 'rank_group': 1, 'type': 'organic'}++ valid = dict(item, title='Kettleman Roofing | Denver',++ domain='kettlemanroofing.com', link='https://kettlemanroofing.com',++ url='https://kettlemanroofing.com', position=2, rank_group=2)++ payload = ({'organic': [item, valid]} if provider == 'serper' else++ {'status_code': 20000, 'tasks': [{'status_code': 20000,++ 'result': [{'items': [item, valid]}]}]})++ real_client = httpx.AsyncClient++ monkeypatch.setattr(D.httpx, 'AsyncClient', lambda **kw: real_client(++ transport=httpx.MockTransport(lambda request: httpx.Response(200, json=payload)), **kw))++ monkeypatch.setattr('harvey.config.load_env', lambda: EnvConfig(++ serper_api_key='dummy', dataforseo_login='dummy', dataforseo_password='dummy'))++ state = StateManager(str(tmp_path / 'test.db'))++ await state.init_db()++ await seed_signal_catalog(state)++ for signal in SIGNAL_CATALOG:++ await state.set_signal_status(signal['code'], 'confirmed')++ report = await D.run_discovery(state, load_config(), provider_key=provider,++ queries=[D.DiscoveryQuery(term='roofer', location='Denver')])++ assert report.errors == []++ assert report.junk == 1, report.as_dict()++ assert report.found == 2++ assert report.new_companies == 1++ import sqlite3++ with sqlite3.connect(state.db_path) as db:++ assert db.execute("SELECT domain, name FROM companies").fetchall() == [++ ("kettlemanroofing.com", "Kettleman Roofing")]++++[email protected]('title,domain,expected', [++ ('Contact Us | ContactCraft', 'contactcraft.com', 'ContactCraft'),++ ('Craft | ContactCraft', 'contactcraft.com', 'ContactCraft'),++ ('Roofing Services in Denver | Kettleman Roofing', 'kettlemanroofing.com', 'Kettleman Roofing'),++ ('Plumbing Services in Austin | Acme Plumbing', 'acmeplumbing.com', 'Acme Plumbing'),++])++def test_brand_wins_over_generic_service_segment(title, domain, expected):++ assert D.business_name(title, domain) == expected++++[email protected]("service,brand,domain", [++ ("Landscaping", "Kettleman Landscaping", "kettlemanlandscaping.com"),++ ("Electrical", "Kettleman Electrical", "kettlemanelectrical.com"),++])++def test_service_location_labels_do_not_establish_brand_identity(service, brand, domain):++ label = f"{service} Services in Denver"++ assert not D.name_matches_domain(label, domain)++ assert D.business_name(f"{label} | {brand}", domain) == brand++ html = f'<meta property="og:site_name" content="{brand}"><title>{label}</title>'++ assert D.site_name_from_html(html, domain) == brand++ assert D.site_name_from_html(f"<title>{label}</title>", domain) == ""++++++def test_homepage_prefers_stronger_brand_over_longer_partial_match():++ html = ('<meta property="og:site_name" content="Kettleman Landscaping">'++ '<title>Landscaping Professionals</title>')++ assert D.site_name_from_html(html, "kettlemanlandscaping.com") == "Kettleman Landscaping"