Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

fix: retain discovery filtering and prefer verified brand names

Ethan+8944bd6df787
diff --cc harvey/collectors/discover.py
index 51f824f,1299a80..e2dd85d
--- a/harvey/collectors/discover.py
+++ b/harvey/collectors/discover.py
@@@ -91,110 -87,6 +91,130 @@@ def normalize_domain(url_or_domain: str
return host[4:] if host.startswith("www.") else host
+# ── Business names ────────────────────────────────────────────────────
+# SERP providers return a page TITLE, not a business name: "Denver, CO | HOA
+# Management Firm", "Home Page - NVCM | Community Association Management".
+# Stored as-is it lands in an email as "...at Denver, CO | HOA Management
+# Firm". The business is the title segment that shares a DISTINCTIVE word with
+# the domain — "management" matches managementharbor.com but says nothing.
+
+_TITLE_SEPARATORS = re.compile(r"\s+[|\-–—·•]\s+|\s*[|•·]\s*|:\s+")
+_GENERIC_NAME_WORDS = frozenset("""
+home page welcome official site website hoa hoas management manager managers
+mgmt mgt company companies co inc llc ltd corp firm group services service
+property properties community communities association associations condo
+condominium residential homeowners owners real estate the and of in for best
+top leading trusted local full arizona phoenix az texas tx florida fl
+california ca nevada nv colorado co georgia ga carolina nc usa us america
++roofing roof roofers roofer plumbing plumber plumbers contact about
+""".split())
+
+
+def _words(text: str) -> list[str]:
+ return [w for w in re.findall(r"[a-z0-9]+", text.lower()) if w]
+
+
+def _domain_stem(domain: str) -> str:
+ host = normalize_domain(domain)
+ return re.sub(r"[^a-z0-9]", "", host.rsplit(".", 1)[0]) if host else ""
+
+
+def _segment_matches_domain(segment: str, stem: str) -> bool:
++ # Navigation and service/location labels can share an incidental word
++ # with a domain without identifying the business.
++ if re.fullmatch(r"(?i)(contact(?: us)?|about(?: us)?|home(?: page)?)", segment.strip()):
++ return False
++ if re.search(r"(?i)\bservices?\s+(?:in|near)\s+\S", segment):
++ return False
+ words = _words(segment)
+ distinctive = [w for w in words if w not in _GENERIC_NAME_WORDS]
+ if any(len(w) >= 3 and w in stem for w in distinctive):
+ return True
+ # "Northvale Association of Community Managers" ↔ nacm.org
+ initials = "".join(w[0] for w in words if w not in ("of", "the", "and", "&"))
+ return len(initials) >= 3 and stem.startswith(initials)
+
+
++def _segment_domain_evidence(segment: str, stem: str) -> int:
++ """Amount of distinctive domain evidence, shared by SERP and homepage names."""
++ if not _segment_matches_domain(segment, stem):
++ return 0
++ return sum(len(w) for w in set(_words(segment))
++ if w not in _GENERIC_NAME_WORDS and len(w) >= 3 and w in stem)
++
++
+_SPLIT_WORDS = sorted((w for w in _GENERIC_NAME_WORDS | {"with", "your"}
+ if len(w) >= 4), key=len, reverse=True)
+
+
+def _split_stem(stem: str) -> list[str]:
+ """managementharbor → [management, harbor]; kqresidential → [kq, residential]."""
+ parts, buf, i = [], "", 0
+ while i < len(stem):
+ word = next((w for w in _SPLIT_WORDS if stem.startswith(w, i)), "")
+ if word:
+ if buf:
+ parts.append(buf)
+ buf = ""
+ parts.append(word)
+ i += len(word)
+ else:
+ buf += stem[i]
+ i += 1
+ if buf:
+ parts.append(buf)
+ return parts
+
+
+def name_from_domain(domain: str) -> str:
+ stem = normalize_domain(domain).rsplit(".", 1)[0]
+ words = [w for chunk in re.split(r"[-_.]+", stem) if chunk for w in _split_stem(chunk)]
+ # Short vowel-less leftovers are initials: "kq" → "KQ".
+ return " ".join(w.upper() if len(w) <= 3 and not re.search(r"[aeiouy]", w)
+ else w.capitalize() for w in words)
+
+
+def name_matches_domain(name: str, domain: str) -> bool:
+ """True when the stored name demonstrably belongs to this domain."""
+ stem = _domain_stem(domain)
+ return bool(stem and name and _segment_matches_domain(name, stem))
+
+
+def site_name_from_html(html: str, domain: str) -> str:
+ """The business name a homepage gives itself (og:site_name, then <title>),
+ or "" when neither names it in a way the domain backs up."""
+ stem = _domain_stem(domain)
+ candidates = []
+ for pat in (r'<meta[^>]+property=["\']og:site_name["\'][^>]*content=["\']([^"\']+)',
+ r'<meta[^>]+content=["\']([^"\']+)["\'][^>]*property=["\']og:site_name',
+ r"(?is)<title[^>]*>(.*?)</title>"):
+ m = re.search(pat, html or "", re.I)
+ if m:
+ candidates.append(unescape(re.sub(r"\s+", " ", m.group(1))).strip())
- # The fullest name wins: "The Management Harbor" over og's "Management Harbor".
++ # Distinctive brand evidence wins; equally backed names prefer the fullest
++ # form: "The Management Harbor" over og's "Management Harbor".
+ matches = [seg for text in candidates
+ for seg in (x.strip(" ,.-") for x in _TITLE_SEPARATORS.split(text))
+ if seg and len(seg) <= 60 and _segment_matches_domain(seg, stem)]
- return max(matches, key=len, default="")
++ return max(matches, key=lambda seg: (_segment_domain_evidence(seg, stem), len(seg)),
++ default="")
+
+
+def business_name(title: str, domain: str) -> str:
+ """The business's name from a SERP title, or one derived from its domain."""
+ stem = _domain_stem(domain)
+ title = re.sub(r"\s*(\.\.\.|…)\s*$", "", (title or "").strip())
+ if stem and title:
- for segment in _TITLE_SEPARATORS.split(title):
- segment = segment.strip(" ,.-")
- if segment and len(segment) <= 60 and _segment_matches_domain(segment, stem):
- return segment
++ segments = [segment.strip(" ,.-") for segment in _TITLE_SEPARATORS.split(title)]
++ matches = [seg for seg in segments
++ if seg and len(seg) <= 60 and _segment_matches_domain(seg, stem)]
++ # Prefer the segment with the most distinctive domain evidence rather
++ # than the first incidental substring. Stable ties preserve title order.
++ if matches:
++ return max(matches, key=lambda seg: _segment_domain_evidence(seg, stem))
+ return name_from_domain(domain) if stem else title
+
+
def is_junk(domain: str, name: str = "") -> bool:
"""True when this row would pollute the database rather than fill it."""
if domain:
@@@ -301,6 -193,6 +321,7 @@@ class Business
external_id: str = "" # "<provider>:<stable id>"
source_url: str = ""
provider: str = ""
++ raw_title: str = "" # SERP evidence for junk filtering before normalization
def identity(self) -> str:
return self.domain or self.external_id
@@@ -495,7 -387,7 +516,8 @@@ class DataForSEOSerp(DiscoveryProvider)
if not domain:
continue
out.append(Business(
- name=item.get("title") or domain,
+ name=business_name(item.get("title") or "", domain),
++ raw_title=item.get("title") or "",
domain=domain,
website=item.get("url") or "",
location=query.location,
@@@ -546,7 -438,7 +568,8 @@@ class Serper(DiscoveryProvider)
if not domain:
continue
out.append(Business(
- name=item.get("title") or domain,
+ name=business_name(item.get("title") or "", domain),
++ raw_title=item.get("title") or "",
domain=domain,
website=item.get("link") or "",
location=query.location,
@@@ -882,7 -774,7 +905,7 @@@ async def run_discovery
report.found += len(found)
batch = []
for biz in found:
-- if is_junk(biz.domain, biz.name):
++ if is_junk(biz.domain, biz.raw_title or biz.name):
report.junk += 1
continue
ident = biz.identity()
@@@ -984,3 -876,3 +1007,4 @@@ def provider_menu(env: dict | None = No
}
for p in PROVIDERS.values()
]
++
diff --cc harvey/collectors/profile.py
index bf77444,2f9b69c..5c10652
--- a/harvey/collectors/profile.py
+++ b/harvey/collectors/profile.py
@@@ -502,22 -455,6 +502,23 @@@ class ProfileCollector
return obs
+ async def _improve_name(self, company: dict, domain: str):
+ """Replace a name the domain doesn't back up ("Denver, CO | HOA
+ Management Firm") or one merely derived from the domain ("Grow With
- Us") with the one the homepage gives itself. A real match is kept."""
++ Us") with the one the homepage gives itself. Only a distinctive brand match
++ is kept; generic industry and navigation text does not establish identity."""
+ better = self.site_names.pop(company["id"], "")
+ current = company.get("name") or ""
+ fallback = current == name_from_domain(domain)
+ if not better or better == current or (
+ name_matches_domain(current, domain) and not fallback):
+ return
+ try:
+ await self.state.rename_company(company["id"], better)
+ logger.info(f"profile: {domain} named '{better}' (was '{current}')")
+ except Exception as e:
+ logger.debug(f"rename_company failed for {domain}: {e}")
+
async def _mark_checked(self, company_id: str):
try:
await self.state.mark_company_profiled(company_id)
@@@ -573,3 -509,3 +574,4 @@@ async def profile_companies(state, comp
except Exception as e:
await state.finish_run(run_id, status="failed", error=str(e))
raise
++
diff --cc harvey/state.py
index 6401fb7,a82b4c6..347b14a
--- a/harvey/state.py
+++ b/harvey/state.py
@@@ -1857,3 -1843,3 +1857,4 @@@ class StateManager
# Claude budget the rest of the loop is rationing.
"unprofiled": await self.count_companies_needing_profile(),
}
++
diff --cc tests/test_collectors_profile.py
index d7d1008,f631c56..99f3d1d
--- a/tests/test_collectors_profile.py
+++ b/tests/test_collectors_profile.py
@@@ -8,6 -8,6 +8,7 @@@ import pytest_asynci
from harvey.collectors.profile import ProfileCollector, title_looks_like_role, _strip_code
from harvey.models.company import Company
++from harvey.models.prospect import Prospect
from harvey.signals import SIGNAL_CATALOG, seed_signal_catalog
from harvey.state import StateManager
@@@ -192,36 -155,3 +193,98 @@@ async def test_user_decision_survives_r
await seed_signal_catalog(state) # e.g. after an upgrade
codes = {c["code"]: c["status"] for c in await state.get_signal_codes()}
assert codes["SERP_RANK"] == "rejected", "re-seeding must not undo a user's choice"
+
+
+async def test_profile_upgrades_a_name_the_domain_does_not_back(state, monkeypatch):
+ cid = await state.add_company(Company(name="Denver, CO | HOA Management Firm",
+ domain="managementharbor.com"))
+ keep = await state.add_company(Company(name="Birch Management", domain="birchmgt.com"))
+ derived = await state.add_company(Company(name="Grow With Us", domain="growwithus.us"))
+ pages = {
+ "managementharbor.com": '<meta property="og:site_name" content="Management Harbor">',
+ "birchmgt.com": "<title>Denver HOA | Birch Management Group</title>",
+ "growwithus.us": '<meta property="og:site_name" content="Grow Community Management">',
+ }
+
+ async def fake_resolve(self, domain):
+ return f"https://{domain}", pages[domain]
+
+ async def fake_first(self, site, paths):
+ return "", ""
+
+ monkeypatch.setattr(ProfileCollector, "_resolve_site", fake_resolve)
+ monkeypatch.setattr(ProfileCollector, "_first_present", fake_first)
+ async with ProfileCollector(state, {"TECH_STACK"}) as c:
+ await c.run([{"id": cid, "name": "Denver, CO | HOA Management Firm",
+ "domain": "managementharbor.com"},
+ {"id": keep, "name": "Birch Management", "domain": "birchmgt.com"},
+ {"id": derived, "name": "Grow With Us", "domain": "growwithus.us"}])
+ import sqlite3
+ with sqlite3.connect(state.db_path) as db:
+ names = dict(db.execute("SELECT domain, name FROM companies"))
+ assert names["managementharbor.com"] == "Management Harbor"
+ assert names["birchmgt.com"] == "Birch Management" # already right: untouched
+ assert names["growwithus.us"] == "Grow Community Management" # was domain-derived
++
++
[email protected]("current,domain,homepage,expected", [
++ ("Roofing Services in Denver", "kettlemanroofing.com", "Kettleman Roofing", "Kettleman Roofing"),
++ ("Plumbing Services in Austin", "acmeplumbing.com", "Acme Plumbing", "Acme Plumbing"),
++ ("Contact Us", "contactcraft.com", "ContactCraft", "ContactCraft"),
++ ("Birch Management", "birchmgt.com", "Birch Management Group", "Birch Management"),
++ ("NVCM", "nvcmnet.com", "NVCM Group", "NVCM"),
++ ("Northvale Association of Community Managers", "nacm.org",
++ "Northvale Association of Community Managers Group", "Northvale Association of Community Managers"),
++ ("Grow With Us", "growwithus.us", "Grow Community Management", "Grow Community Management"),
++ ("Roofing Services in Denver", "kettlemanroofing.com", "", "Roofing Services in Denver"),
++])
++async def test_profile_repairs_weak_names_and_retains_brands(
++ state, monkeypatch, current, domain, homepage, expected):
++ cid = await state.add_company(Company(name=current, domain=domain))
++ pid = await state.add_prospect(Prospect(company_id=cid, company=current,
++ first_name="Test", last_name="Person"))
++
++ async def fake_resolve(self, domain):
++ return f"https://{domain}", f"<title>{homepage}</title>"
++
++ async def no_sleep(_seconds):
++ return None
++
++ monkeypatch.setattr(ProfileCollector, "_resolve_site", fake_resolve)
++ monkeypatch.setattr("harvey.collectors.profile.asyncio.sleep", no_sleep)
++ async with ProfileCollector(state, set()) as c:
++ await c.run([{"id": cid, "name": current, "domain": domain}])
++ import sqlite3
++ with sqlite3.connect(state.db_path) as db:
++ assert db.execute("SELECT name FROM companies WHERE id = ?", (cid,)).fetchone() == (expected,)
++ assert db.execute("SELECT company FROM prospects WHERE id = ?", (pid,)).fetchone() == (expected,)
++
++
[email protected]("industry", ["Landscaping", "Electrical"])
[email protected]("use_fallback", [False, True])
++async def test_profile_uses_homepage_brand_over_service_location_title(
++ state, monkeypatch, industry, use_fallback):
++ from harvey.collectors.discover import name_from_domain
++ domain = f"kettleman{industry.lower()}.com"
++ brand = f"Kettleman {industry}"
++ label = f"{industry} Services in Denver"
++ current = name_from_domain(domain) if use_fallback else label
++ cid = await state.add_company(Company(name=current, domain=domain))
++ pid = await state.add_prospect(Prospect(company_id=cid, company=current))
++
++ async def fake_resolve(self, domain):
++ return f"https://{domain}", (
++ f'<meta property="og:site_name" content="{brand}"><title>{label}</title>')
++
++ async def no_sleep(_seconds):
++ return None
++
++ monkeypatch.setattr(ProfileCollector, "_resolve_site", fake_resolve)
++ monkeypatch.setattr("harvey.collectors.profile.asyncio.sleep", no_sleep)
++ async with ProfileCollector(state, set()) as c:
++ await c.run([{"id": cid, "name": current, "domain": domain}])
++ assert (await state.get_company_by_domain(domain)).name == brand
++ assert (await state.get_prospect(pid)).company == brand
diff --cc tests/test_discover.py
index e04957a,d1344d8..1aad389
--- a/tests/test_discover.py
+++ b/tests/test_discover.py
@@@ -15,7 -15,7 +15,7 @@@ import pytes
import pytest_asyncio
from harvey.collectors import discover as D
--from harvey.config import load_config
++from harvey.config import EnvConfig, load_config
from harvey.signals import SIGNAL_CATALOG, seed_signal_catalog
from harvey.state import StateManager
@@@ -661,40 -658,3 +661,106 @@@ async def test_an_unmappable_term_is_re
coordinate="1,2,40")])
assert report.found == 0
assert report.errors and "no category" in report.errors[0]
+
+
+# ── business names: SERP titles are page titles, not names ──
+
[email protected]("title,domain,expected", [
+ ("Northvale Property Management: HOA Management Company in ...", "northvaleproperty.com",
+ "Northvale Property Management"),
+ ("Home Page - NVCM | Community Association Management", "nvcmnet.com", "NVCM"),
+ ("Northvale Association of Community Managers", "nacm.org",
+ "Northvale Association of Community Managers"),
+ ("Brightwater Texas: Texas HOA Property Management Company", "brightwatertexas.com",
+ "Brightwater Texas"),
+ # No segment names the business → fall back to the domain, split sensibly.
+ ("Denver, CO | HOA Management Firm", "managementharbor.com", "Management Harbor"),
+ ("Texas HOA management", "kqresidential.com", "KQ Residential"),
+ ("Great Texas Community Management Begins Here", "growwithus.us", "Grow With Us"),
+])
+def test_business_name_from_serp_title(title, domain, expected):
+ assert D.business_name(title, domain) == expected
+
+
+def test_generic_words_alone_never_match_the_domain():
+ """'Management' is in managementharbor.com but says nothing about it."""
+ assert not D.name_matches_domain("HOA Management Firm", "managementharbor.com")
+ assert D.name_matches_domain("The Management Harbor", "managementharbor.com")
+
+
+def test_site_name_prefers_og_then_title_and_needs_domain_backing():
+ og = '<meta property="og:site_name" content="Grow Community Management" />'
+ assert D.site_name_from_html(og, "growwithus.us") == "Grow Community Management"
+ title = "<title>Lumen Association Management - HOA Company in TX &amp; CO</title>"
+ assert D.site_name_from_html(title, "lumenam.com") == "Lumen Association Management"
+ both = ('<meta property="og:site_name" content="Management Harbor">'
+ "<title>The Management Harbor: Community Management</title>")
+ assert D.site_name_from_html(both, "managementharbor.com") == "The Management Harbor"
+ assert D.site_name_from_html("<title>Attention Required! | Cloudflare</title>",
+ "lumenam.com") == ""
++
++
[email protected]('provider', ['serper', 'dataforseo_serp'])
++async def test_directory_title_still_filtered_after_adapter(provider, tmp_path, monkeypatch):
++ title = 'Top 10 Roofing Companies in Denver'
++ domain = 'milehighguide.com'
++ assert D.is_junk(domain, title)
++ item = {'title': title, 'link': f'https://{domain}/roofers',
++ 'url': f'https://{domain}/roofers', 'domain': domain,
++ 'position': 1, 'rank_group': 1, 'type': 'organic'}
++ valid = dict(item, title='Kettleman Roofing | Denver',
++ domain='kettlemanroofing.com', link='https://kettlemanroofing.com',
++ url='https://kettlemanroofing.com', position=2, rank_group=2)
++ payload = ({'organic': [item, valid]} if provider == 'serper' else
++ {'status_code': 20000, 'tasks': [{'status_code': 20000,
++ 'result': [{'items': [item, valid]}]}]})
++ real_client = httpx.AsyncClient
++ monkeypatch.setattr(D.httpx, 'AsyncClient', lambda **kw: real_client(
++ transport=httpx.MockTransport(lambda request: httpx.Response(200, json=payload)), **kw))
++ monkeypatch.setattr('harvey.config.load_env', lambda: EnvConfig(
++ serper_api_key='dummy', dataforseo_login='dummy', dataforseo_password='dummy'))
++ state = StateManager(str(tmp_path / 'test.db'))
++ await state.init_db()
++ await seed_signal_catalog(state)
++ for signal in SIGNAL_CATALOG:
++ await state.set_signal_status(signal['code'], 'confirmed')
++ report = await D.run_discovery(state, load_config(), provider_key=provider,
++ queries=[D.DiscoveryQuery(term='roofer', location='Denver')])
++ assert report.errors == []
++ assert report.junk == 1, report.as_dict()
++ assert report.found == 2
++ assert report.new_companies == 1
++ import sqlite3
++ with sqlite3.connect(state.db_path) as db:
++ assert db.execute("SELECT domain, name FROM companies").fetchall() == [
++ ("kettlemanroofing.com", "Kettleman Roofing")]
++
++
[email protected]('title,domain,expected', [
++ ('Contact Us | ContactCraft', 'contactcraft.com', 'ContactCraft'),
++ ('Craft | ContactCraft', 'contactcraft.com', 'ContactCraft'),
++ ('Roofing Services in Denver | Kettleman Roofing', 'kettlemanroofing.com', 'Kettleman Roofing'),
++ ('Plumbing Services in Austin | Acme Plumbing', 'acmeplumbing.com', 'Acme Plumbing'),
++])
++def test_brand_wins_over_generic_service_segment(title, domain, expected):
++ assert D.business_name(title, domain) == expected
++
++
[email protected]("service,brand,domain", [
++ ("Landscaping", "Kettleman Landscaping", "kettlemanlandscaping.com"),
++ ("Electrical", "Kettleman Electrical", "kettlemanelectrical.com"),
++])
++def test_service_location_labels_do_not_establish_brand_identity(service, brand, domain):
++ label = f"{service} Services in Denver"
++ assert not D.name_matches_domain(label, domain)
++ assert D.business_name(f"{label} | {brand}", domain) == brand
++ html = f'<meta property="og:site_name" content="{brand}"><title>{label}</title>'
++ assert D.site_name_from_html(html, domain) == brand
++ assert D.site_name_from_html(f"<title>{label}</title>", domain) == ""
++
++
++def test_homepage_prefers_stronger_brand_over_longer_partial_match():
++ html = ('<meta property="og:site_name" content="Kettleman Landscaping">'
++ '<title>Landscaping Professionals</title>')
++ assert D.site_name_from_html(html, "kettlemanlandscaping.com") == "Kettleman Landscaping"