Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.
Prospecting rebuild, part 2: the free PROFILE collector
Ethan9636b58c5455
diff --git a/harvey/collectors/__init__.py b/harvey/collectors/__init__.pynew file mode 100644index 0000000..ebdab12--- /dev/null+++ b/harvey/collectors/__init__.py@@ -0,0 +1,9 @@+"""Collectors — deterministic gatherers that emit observations.++Each collector is independently runnable and re-runnable, states its cost up+front, and writes observations in batches (a long run that dies must not lose+what it already learned). Collectors only emit signals the user has CONFIRMED.++Rule of thumb from the method this implements: if a fact can be obtained+deterministically, do not spend a model call on it.+"""diff --git a/harvey/collectors/profile.py b/harvey/collectors/profile.pynew file mode 100644index 0000000..1c2a165--- /dev/null+++ b/harvey/collectors/profile.py@@ -0,0 +1,485 @@+"""PROFILE collector — what a business is, from three HTTP requests.++Cost: $0. No browser, no model. Homepage + robots.txt + sitemap.xml (+ the+team and careers pages when present), then regex over HTML we already hold.++This is the highest value-per-effort stage in the pipeline: it produces the+signals that actually make an email specific — who their incumbent agency is,+whether they're spending on ads today, what's missing from their site.+"""++import asyncio+import logging+from html import unescape as html_unescape+import random+import re+from datetime import datetime, timedelta, timezone+from urllib.parse import urljoin, urlparse++import httpx++logger = logging.getLogger("harvey.collectors.profile")++TIMEOUT = httpx.Timeout(connect=8.0, read=12.0, write=8.0, pool=8.0)+USER_AGENTS = [+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",+]++# ── Agency detection ──────────────────────────────────────────────────+# Links that appear in footers but are never the site's agency. Matched as+# substrings against the link host.+BENIGN_HOSTS = (+ # social+ "facebook.", "instagram.", "twitter.", "x.com", "linkedin.", "youtube.",+ "tiktok.", "pinterest.", "yelp.", "nextdoor.", "threads.",+ # maps / profiles+ "google.com/maps", "g.page", "maps.app.goo.gl", "goo.gl", "wa.me",+ "bing.com/maps", "apple.com/maps",+ # financing / payments / reviews+ "greensky", "wellsfargo", "synchrony", "affirm", "klarna", "paypal",+ "stripe.com", "bbb.org", "angi.com", "homeadvisor", "houzz.",+ "trustpilot", "birdeye", "podium.com",+ # accessibility / compliance widgets+ "accessibe", "userway", "audioeye", "termly", "iubenda", "onetrust",+ # infra / CDNs / platforms+ "cloudflare", "godaddy", "wordpress.org", "wix.com", "squarespace.com",+ "shopify.com", "webflow.com", "duda.co", "gtranslate", "jquery",+ "gstatic", "googleapis", "cloudfront", "akamai", "w3.org", "schema.org",+ # industry bodies / certs+ "gaf.com", "owenscorning", "certainteed", "nrca.net", "iko.com",+ "energystar.gov", "osha.gov", "nahb.org",+)++# Credit wording → confidence. Only >= 0.75 is recorded: a wrong incumbent in+# an email is worse than saying nothing at all.+STRONG_CREDIT = re.compile(+ r"\b(powered|website|site|web\s*design|designed|developed|built|created|"+ r"marketing)\s+(by|&)\b", re.I)+DESCRIPTIVE_CREDIT = re.compile(+ r"\b(web\s*design|website\s*design|digital\s*marketing|seo|marketing\s*"+ r"agency|web\s*development)\b", re.I)+RECORD_FLOOR = 0.75++# ── Tech / platform fingerprints (name -> regexes over raw HTML) ──+TECH_FINGERPRINTS: dict[str, tuple[str, ...]] = {+ "WordPress": (r"wp-content/", r"wp-includes/"),+ "Shopify": (r"cdn\.shopify\.com", r"myshopify\.com"),+ "Wix": (r"static\.parastorage\.com",),+ "Squarespace": (r"static1\.squarespace\.com",),+ "Webflow": (r"assets\.website-files\.com", r"data-wf-domain"),+ "Duda": (r"irp\.cdn-website\.com", r"d1\.awsstatic"),+ "GoDaddy Website Builder": (r"img1\.wsimg\.com",),+ "HubSpot": (r"js\.hs-scripts\.com", r"js\.hsforms\.net"),+ "Google Analytics": (r"googletagmanager\.com/gtag", r"google-analytics\.com"),+ "Google Tag Manager": (r"googletagmanager\.com/gtm\.js",),+ "Intercom": (r"widget\.intercom\.io",),+ "Drift": (r"js\.driftt\.com",),+ "Podium": (r"connect\.podium\.com",),+ "Birdeye": (r"birdeye\.com",),+ "Calendly": (r"assets\.calendly\.com",),+ "ServiceTitan": (r"servicetitan",),+ "Jobber": (r"getjobber\.com",),+ "Housecall Pro": (r"housecallpro",),+ "CallRail": (r"cdn\.callrail\.com",),+ "Stripe": (r"js\.stripe\.com",),+}++GOOGLE_ADS = (r"googleadservices\.com", r"gtag/js\?id=AW-", r"google_conversion",+ r"googletagmanager\.com/gtag/js\?id=AW-")+META_PIXEL = (r"connect\.facebook\.net/[^\"']*fbevents\.js", r"fbq\s*\(\s*['\"]init")++BOOKING_HINTS = (r"calendly\.com", r"acuityscheduling", r"schedulicity",+ r"squareup\.com/appointments", r"setmore", r"book(ing)?[-_]?now",+ r"schedule[-_]?(an?[-_])?(appointment|estimate|consultation)",+ r"housecallpro", r"getjobber\.com", r"servicetitan")++AI_CRAWLERS = ("gptbot", "google-extended", "ccbot", "anthropic-ai", "claudebot",+ "perplexitybot", "bytespider")++TEAM_PATHS = ("/team", "/about", "/about-us", "/our-team", "/staff",+ "/leadership", "/meet-the-team", "/company/team")+CAREERS_PATHS = ("/careers", "/jobs", "/join-us", "/employment", "/hiring",+ "/about/careers", "/careers/")+CONTACT_PATHS = ("/contact", "/contact-us", "/get-a-quote", "/free-estimate",+ "/request-estimate", "/quote")++# Words that are never part of a person's name. Without this, headings like+# "Request Service", "Baker History" and "Commercial Services" pass a+# Firstname-Lastname regex and become contacts — the single most common+# failure mode when scraping team pages.+NON_NAME_WORDS = frozenset("""+service services repair repairs roofing roof roofs contractor contracting+commercial residential industrial storm damage restoration installation+history about contact quote quotes estimate estimates free request get call+learn more read view our their your team staff careers jobs home page site+company companies inc llc ltd corp group solutions systems partners+gallery projects reviews testimonials financing warranty insurance claims+maintenance inspection replacement gutters siding windows doors metal shingle+shingles tile flat emergency 24 hour hours today now new best top quality+privacy policy terms conditions sitemap blog news events why choose us+areas served locations location city county state north south east west+español espanol menu skip navigation search close open toggle+""".split())++DECISION_MAKER_TITLES = (+ "owner", "founder", "co-founder", "president", "ceo", "principal",+ "partner", "general manager", "operations manager", "practice manager",+ "office manager", "marketing director", "marketing manager",+ "director of marketing", "vice president", "vp ", "managing director",+)+++ROLE_HINT = re.compile(+ r"\b(owner|founder|president|ceo|cfo|coo|cto|principal|partner|manager|"+ r"director|supervisor|superintendent|foreman|estimator|specialist|"+ r"consultant|coordinator|representative|rep|sales|marketing|operations|"+ r"office|project|account|service|technician|installer|inspector|"+ r"vice president|vp|executive|lead|head of|chief)\b", re.I)+++def title_looks_like_role(text: str) -> bool:+ """True when the text following a name reads like a job title.++ A name with no role nearby is usually a nav link or a page heading, not+ a person — requiring a role is the cheapest high-precision filter.+ """+ return bool(ROLE_HINT.search(text[:160]))+++def _strip_code(html: str) -> str:+ """Remove script/style/comments BEFORE any text regex.++ Skipping this is how you end up with contacts named 'return value instan'+ whose title is CEO — minified JS read as prose.+ """+ html = re.sub(r"(?is)<script\b.*?</script>", " ", html)+ html = re.sub(r"(?is)<style\b.*?</style>", " ", html)+ html = re.sub(r"(?s)<!--.*?-->", " ", html)+ return html+++def _text(html: str) -> str:+ return re.sub(r"\s+", " ", re.sub(r"(?s)<[^>]+>", " ", _strip_code(html)))+++class ProfileCollector:+ """Fetches a business's public pages and emits observations.++ Only signals the user CONFIRMED are collected; everything else is skipped+ even if the data is sitting right there in the HTML.+ """++ def __init__(self, state, confirmed: set[str] | None = None, timeout: float = 12.0):+ self.state = state+ self.confirmed = confirmed if confirmed is not None else set()+ self.timeout = timeout+ self._client: httpx.AsyncClient | None = None++ def wants(self, code: str) -> bool:+ return code in self.confirmed++ async def __aenter__(self):+ self._client = httpx.AsyncClient(+ timeout=TIMEOUT, follow_redirects=True,+ headers={"User-Agent": random.choice(USER_AGENTS),+ "Accept": "text/html,application/xhtml+xml,*/*;q=0.8"},+ )+ return self++ async def __aexit__(self, *exc):+ if self._client:+ await self._client.aclose()++ async def _get(self, url: str) -> str:+ try:+ resp = await self._client.get(url)+ except (httpx.HTTPError, ValueError) as e:+ logger.debug(f"fetch failed {url}: {e}")+ return ""+ if resp.status_code != 200:+ return ""+ ctype = resp.headers.get("content-type", "")+ if "html" not in ctype and "xml" not in ctype and "text" not in ctype:+ return ""+ return resp.text++ async def _first_present(self, domain: str, paths) -> tuple[str, str]:+ """Return (url, html) for the first path that returns content."""+ for path in paths:+ url = f"https://{domain}{path}"+ html = await self._get(url)+ if html and len(html) > 500:+ return url, html+ return "", ""++ # ── individual detectors ──++ def detect_agency(self, html: str, domain: str) -> tuple[str, float, str] | None:+ """Find the agency credited in the footer. (name, confidence, url)."""+ cleaned = _strip_code(html)+ m = re.search(r"(?is)<footer\b.*?</footer>", cleaned)+ region = m.group(0) if m else cleaned[-12000:]++ best = None+ for link in re.finditer(r'(?is)<a\b[^>]*href=["\']([^"\']+)["\'][^>]*>(.*?)</a>', region):+ href, anchor = link.group(1), re.sub(r"(?s)<[^>]+>", " ", link.group(2))+ anchor = re.sub(r"\s+", " ", anchor).strip()+ if not href.startswith("http"):+ continue+ host = (urlparse(href).netloc or "").lower().replace("www.", "")+ if not host or host.endswith(domain) or domain.endswith(host):+ continue+ if any(b in href.lower() or b in host for b in BENIGN_HOSTS):+ continue+ if not anchor or len(anchor) > 80:+ continue++ # Context = the ~120 chars before the link, where credits live.+ start = max(0, link.start() - 120)+ context = re.sub(r"(?s)<[^>]+>", " ", region[start:link.start()])+ blob = f"{context} {anchor}"++ if STRONG_CREDIT.search(blob):+ conf = 0.9+ elif DESCRIPTIVE_CREDIT.search(anchor):+ conf = 0.75+ else:+ conf = 0.35 # a bare link — do not record+ if conf >= RECORD_FLOOR and (best is None or conf > best[1]):+ best = (anchor[:80], conf, href)+ return best++ def detect_tech(self, html: str) -> list[str]:+ found = []+ for name, patterns in TECH_FINGERPRINTS.items():+ if any(re.search(p, html, re.I) for p in patterns):+ found.append(name)+ return sorted(found)++ @staticmethod+ def _any(html: str, patterns) -> bool:+ return any(re.search(p, html, re.I) for p in patterns)++ def detect_people(self, html: str) -> list[dict]:+ """Names + titles from a team page. Conservative on purpose."""+ text_html = _strip_code(html)+ people, seen = [], set()+ # Name in a heading, title in the nearby text.+ for m in re.finditer(+ r"(?is)<(h[1-5]|strong|b)[^>]*>\s*([A-Z][a-z'\-]+(?:\s+[A-Z]\.?)?\s+"+ r"[A-Z][a-zA-Z'\-]+)\s*</\1>(?=(.{0,220}))", text_html+ ):+ # NOTE the lookahead above: capturing the trailing context normally+ # would CONSUME it, so finditer would skip every person whose entry+ # fell inside the previous person's 220-char window.+ name = re.sub(r"\s+", " ", m.group(2)).strip()+ # Bound the context at the NEXT heading, in raw HTML, before+ # stripping tags — otherwise one member's role bleeds into the+ # next member's entry.+ raw_after = m.group(3)+ nxt = re.search(r"(?i)<(h[1-5]|strong|b)\b", raw_after)+ if nxt:+ raw_after = raw_after[: nxt.start()]+ after = html_unescape(re.sub(r"(?s)<[^>]+>", " ", raw_after))+ after = re.sub(r"\s+", " ", after).strip()+ if name.lower() in seen or len(name) > 45:+ continue+ # Reject obvious non-names+ if re.search(r"\d|@|\bthe\b|\bour\b|\bwe\b", name, re.I):+ continue+ # Reject headings that merely LOOK like names. Every token must be+ # plausibly a name part; one business word disqualifies the whole+ # match ("Request Service", "Baker History", "Commercial Services").+ tokens = [t.strip(".,'-").lower() for t in name.split()]+ if any(t in NON_NAME_WORDS for t in tokens):+ continue+ # A real contact needs a title that reads like a role; a bare+ # heading with no role text is almost always navigation.+ if not title_looks_like_role(after):+ continue+ title = ""+ low = after.lower()+ for t in DECISION_MAKER_TITLES:+ if t in low:+ idx = low.index(t)+ title = after[idx:idx + 60].strip(" ,.|-—·")+ break+ if not title:+ m2 = re.match(r"[\s,|\-–—·]*([A-Za-z][A-Za-z /&'\-]{2,45})", after)+ if m2:+ title = m2.group(1).strip()+ seen.add(name.lower())+ parts = name.split()+ people.append({+ "first_name": parts[0], "last_name": " ".join(parts[1:]),+ "title": title[:80],+ "is_decision_maker": any(t in title.lower() for t in DECISION_MAKER_TITLES),+ })+ if len(people) >= 12:+ break+ return people++ # ── the stage itself ──++ async def profile_company(self, company_id: str, domain: str) -> list[dict]:+ """Collect every confirmed profile signal for one business."""+ obs: list[dict] = []+ domain = domain.lower().replace("www.", "").strip("/")+ site = f"https://{domain}"++ def add(code, *, num=None, text="", conf=1.0, url=""):+ if not self.wants(code):+ return+ obs.append({+ "signal_code": code, "company_id": company_id, "collector": "profile",+ "value_num": num, "value_text": text, "confidence": conf,+ "evidence_url": url or site,+ })++ home = await self._get(site)+ if not home:+ # A failure IS an observation — silent failures look like clean results.+ logger.info(f"profile: {domain} unreachable")+ return obs++ # Ads (they are spending TODAY — the strongest budget signal here)+ add("RUNNING_GOOGLE_ADS", num=1 if self._any(home, GOOGLE_ADS) else 0)+ add("RUNNING_META_ADS", num=1 if self._any(home, META_PIXEL) else 0)++ # Platform + tooling+ tech = self.detect_tech(home)+ if tech:+ add("TECH_STACK", text=", ".join(tech))++ # Structured data gap+ has_schema = bool(re.search(r'application/ld\+json', home, re.I)+ and re.search(r'"@type"', home))+ add("NO_SCHEMA_MARKUP", num=0 if has_schema else 1)++ # Booking gap+ add("NO_ONLINE_BOOKING", num=0 if self._any(home, BOOKING_HINTS) else 1)++ # Incumbent agency — the compounding signal+ agency = self.detect_agency(home, domain)+ if agency:+ add("INCUMBENT_AGENCY", text=agency[0], conf=agency[1], url=agency[2])++ # robots.txt → AI crawler policy+ if self.wants("BLOCKS_AI_CRAWLERS"):+ robots = (await self._get(f"{site}/robots.txt")).lower()+ blocked = any(+ re.search(rf"user-agent:\s*{re.escape(bot)}[\s\S]{{0,200}}?disallow:\s*/",+ robots)+ for bot in AI_CRAWLERS+ )+ add("BLOCKS_AI_CRAWLERS", num=1 if blocked else 0,+ url=f"{site}/robots.txt")++ # sitemap.xml → size + content freshness+ if self.wants("SITE_PAGE_COUNT") or self.wants("BLOG_STALE"):+ sm = await self._get(f"{site}/sitemap.xml")+ if sm:+ locs = re.findall(r"<loc>\s*([^<]+)\s*</loc>", sm)+ if self.wants("SITE_PAGE_COUNT") and locs:+ add("SITE_PAGE_COUNT", num=len(locs), url=f"{site}/sitemap.xml")+ if self.wants("BLOG_STALE"):+ dates = re.findall(r"<lastmod>\s*(\d{4}-\d{2}-\d{2})", sm)+ if dates:+ newest = max(dates)+ try:+ age = (datetime.now(timezone.utc).date()+ - datetime.strptime(newest, "%Y-%m-%d").date()).days+ add("BLOG_STALE", num=age, text=f"newest content {newest}",+ url=f"{site}/sitemap.xml")+ except ValueError:+ pass++ # contact form — reaches the business without needing an email at all+ if self.wants("CONTACT_FORM_URL"):+ if re.search(r"(?is)<form\b", _strip_code(home)):+ add("CONTACT_FORM_URL", text=site, url=site)+ else:+ curl, chtml = await self._first_present(domain, CONTACT_PATHS)+ if chtml and re.search(r"(?is)<form\b", _strip_code(chtml)):+ add("CONTACT_FORM_URL", text=curl, url=curl)++ # careers page → hiring intent+ if self.wants("HIRING_ROLE"):+ curl, chtml = await self._first_present(domain, CAREERS_PATHS)+ if chtml:+ roles = []+ for m in re.finditer(+ r"(?is)<(h[2-5]|a|li)[^>]*>\s*([^<]{6,70})\s*</\1>", _strip_code(chtml)+ ):+ t = re.sub(r"\s+", " ", m.group(2)).strip()+ if re.search(r"\b(hiring|apply|position|opening|wanted|"+ r"technician|installer|sales|manager|foreman|"+ r"estimator|crew|representative)\b", t, re.I):+ if t.lower() not in [r.lower() for r in roles]:+ roles.append(t)+ if len(roles) >= 3:+ break+ for r in roles:+ add("HIRING_ROLE", text=r, url=curl)++ # team page → people+ if self.wants("CONTACT_FOUND") or self.wants("DECISION_MAKER_TITLE"):+ turl, thtml = await self._first_present(domain, TEAM_PATHS)+ if thtml:+ for person in self.detect_people(thtml):+ label = f"{person['first_name']} {person['last_name']}"+ add("CONTACT_FOUND",+ text=f"{label} — {person['title']}" if person["title"] else label,+ url=turl)+ if person["is_decision_maker"]:+ add("DECISION_MAKER_TITLE",+ text=f"{label} — {person['title']}", url=turl)++ return obs++ async def run(self, companies: list[dict], run_id: str = "") -> int:+ """Profile many businesses, flushing observations after each one.++ Flush-per-item is deliberate: the observations are the asset, and a+ long run that dies must not take them with it.+ """+ total = 0+ for company in companies:+ domain = (company.get("domain") or "").strip()+ if not domain:+ continue+ try:+ obs = await self.profile_company(company["id"], domain)+ except Exception as e:+ logger.warning(f"profile failed for {domain}: {e}")+ continue+ if obs:+ total += await self.state.add_observations(obs, run_id=run_id)+ logger.info(f"profile: {domain} → {len(obs)} observation(s)")+ await asyncio.sleep(random.uniform(0.4, 1.2))+ return total+++async def profile_companies(state, companies: list[dict]) -> tuple[int, str]:+ """Convenience entry point: opens a run, profiles, closes the run."""+ from harvey.signals import seed_signal_catalog+ await seed_signal_catalog(state)+ confirmed = await state.confirmed_signal_codes()+ if not confirmed:+ logger.warning(+ "profile: no signals confirmed yet — confirm signals first "+ "(nothing is collected until you choose what matters)."+ )+ return 0, ""+ run_id = await state.start_run("profile", provider="http")+ try:+ async with ProfileCollector(state, confirmed) as collector:+ n = await collector.run(companies, run_id=run_id)+ await state.finish_run(run_id, records=n, cost_usd=0.0)+ return n, run_id+ except Exception as e:+ await state.finish_run(run_id, status="failed", error=str(e))+ raisediff --git a/tests/test_collectors_profile.py b/tests/test_collectors_profile.pynew file mode 100644index 0000000..f631c56--- /dev/null+++ b/tests/test_collectors_profile.py@@ -0,0 +1,157 @@+"""Tests for the free PROFILE collector and the observation model."""++import os+import tempfile++import pytest+import pytest_asyncio++from harvey.collectors.profile import ProfileCollector, title_looks_like_role, _strip_code+from harvey.models.company import Company+from harvey.signals import SIGNAL_CATALOG, seed_signal_catalog+from harvey.state import StateManager+++@pytest_asyncio.fixture+async def state():+ with tempfile.TemporaryDirectory() as tmp:+ sm = StateManager(os.path.join(tmp, "t.db"))+ await sm.init_db()+ yield sm+++def collector(confirmed=None):+ c = ProfileCollector.__new__(ProfileCollector)+ c.confirmed = confirmed or set()+ return c+++# ── agency detection (the highest-value signal) ──++def test_agency_detected_from_strong_credit():+ html = """<footer><p>© 2026 Acme Roofing.+ Powered by <a href="https://rcmdigital.com">RCM Digital</a></p></footer>"""+ got = collector().detect_agency(html, "acmeroofing.com")+ assert got is not None+ name, conf, url = got+ assert "RCM" in name and conf == 0.9 and "rcmdigital" in url+++def test_agency_descriptive_anchor_scores_lower_but_records():+ html = """<footer><a href="https://xyz.com">Roofing Website Design and Marketing</a></footer>"""+ got = collector().detect_agency(html, "acme.com")+ assert got is not None and got[1] == 0.75+++def test_bare_link_is_not_recorded():+ """A bare footer link is usually the business's own second brand."""+ html = """<footer><a href="https://someplace.com">Someplace</a></footer>"""+ assert collector().detect_agency(html, "acme.com") is None+++def test_benign_footer_links_ignored():+ html = """<footer>+ <a href="https://facebook.com/acme">Facebook</a>+ <a href="https://g.page/acme">Google</a>+ <a href="https://gaf.com">GAF Certified</a>+ <a href="https://accessibe.com">Accessibility</a>+ </footer>"""+ assert collector().detect_agency(html, "acme.com") is None+++def test_self_links_ignored():+ html = """<footer>Powered by <a href="https://acme.com/about">Acme</a></footer>"""+ assert collector().detect_agency(html, "acme.com") is None+++# ── people: real names vs page titles ──++def test_real_people_extracted_with_titles():+ html = """<div><h3>Michael Torres</h3><p>Owner & President</p>+ <h3>Sarah Chen</h3><p>Operations Manager</p>+ <h3>Dave Kowalski</h3><p>Lead Estimator</p></div>"""+ people = collector().detect_people(html)+ assert len(people) == 3, "trailing context must not consume later matches"+ assert people[0]["title"] == "Owner & President" # entities decoded, no bleed+ assert people[1]["title"] == "Operations Manager"+ assert people[0]["is_decision_maker"] is True+ assert people[2]["is_decision_maker"] is False+++def test_page_titles_are_not_people():+ """The classic failure: nav headings that look like Firstname Lastname."""+ html = """<ul><li><h3>Request Service</h3><p>Get a free estimate</p></li>+ <li><h3>Baker History</h3><p>Founded in 1915</p></li>+ <li><h3>Commercial Services</h3><p>Roof repair</p></li></ul>"""+ assert collector().detect_people(html) == []+++def test_title_role_gate():+ assert title_looks_like_role("Owner and founder of the company")+ assert title_looks_like_role("Operations Manager")+ assert not title_looks_like_role("Get a free roof estimate today")+++def test_strip_code_removes_script_before_text_extraction():+ html = '<script>var x = "CEO John Smith";</script><h3>Real Person</h3>'+ assert "CEO John Smith" not in _strip_code(html)+++# ── tech / ads / gaps ──++def test_tech_and_ads_detection():+ c = collector()+ html = """<script src="https://www.googletagmanager.com/gtag/js?id=AW-123"></script>+ <link href="/wp-content/themes/x.css">+ <script src="https://connect.facebook.net/en_US/fbevents.js"></script>"""+ assert "WordPress" in c.detect_tech(html)+ assert c._any(html, __import__("harvey.collectors.profile", fromlist=["GOOGLE_ADS"]).GOOGLE_ADS)+ assert c._any(html, __import__("harvey.collectors.profile", fromlist=["META_PIXEL"]).META_PIXEL)+++def test_clean_page_has_no_false_tech():+ assert collector().detect_tech("<html><body><h1>Acme</h1></body></html>") == []+++# ── confirmation gate ──++async def test_only_confirmed_signals_are_collected(state, monkeypatch):+ await seed_signal_catalog(state)+ await state.set_signal_status("TECH_STACK", "confirmed")+ confirmed = await state.confirmed_signal_codes()+ c = collector(confirmed)++ assert c.wants("TECH_STACK")+ assert not c.wants("INCUMBENT_AGENCY") # proposed, never confirmed++ async def fake_get(url):+ return '<html><link href="/wp-content/x.css"><footer>Powered by <a href="https://ag.com">Agency</a></footer></html>'+ c._get = fake_get+ c.state = state++ cid = await state.add_company(Company(name="Acme", domain="acme.com"))+ obs = await c.profile_company(cid, "acme.com")+ codes = {o["signal_code"] for o in obs}+ assert codes == {"TECH_STACK"}, "unconfirmed signals must not be collected"+++async def test_seeded_catalog_starts_unconfirmed(state):+ n = await seed_signal_catalog(state)+ assert n == len(SIGNAL_CATALOG)+ assert await state.confirmed_signal_codes() == set()+ proposed = await state.get_signal_codes(status="proposed")+ assert len(proposed) == n+ # every proposed signal explains itself to the user+ for sig in proposed:+ assert sig["description"] and sig["cost_note"]+++async def test_user_decision_survives_reseeding(state):+ await seed_signal_catalog(state)+ await state.set_signal_status("SERP_RANK", "rejected")+ await seed_signal_catalog(state) # e.g. after an upgrade+ codes = {c["code"]: c["status"] for c in await state.get_signal_codes()}+ assert codes["SERP_RANK"] == "rejected", "re-seeding must not undo a user's choice"