Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.
Chain PROFILE onto DISCOVER, and make cohorts read the value
Ethan6486fe1d401b
diff --git a/harvey/cli.py b/harvey/cli.pyindex 01c342c..f124b65 100644--- a/harvey/cli.py+++ b/harvey/cli.py@@ -320,6 +320,31 @@ def cmd_outbox(args):asyncio.run(_outbox())+def cmd_profile(args):+ """Read what companies' own websites say about them. Free, no Claude calls."""+ from harvey.state import StateManager+ from harvey.pipeline import run_profile_stage++ async def _profile():+ state = StateManager()+ await state.init_db()++ pending = await state.count_companies_needing_profile(args.stale_days)+ if not pending:+ print("\n Nothing to profile. Every company with a website has "+ "been read recently.\n")+ return++ print(f"\n {pending} companies need profiling. Reading up to "+ f"{args.limit}...\n")+ companies, observations, _ = await run_profile_stage(+ state, limit=args.limit, stale_days=args.stale_days)+ print(f" Read {companies} sites → {observations} observations (free)")+ print("\n See what turned up: harvey dashboard → Signals\n")++ asyncio.run(_profile())++def cmd_discover(args):"""Find businesses. The only stage that spends money, so it estimates first."""from harvey.state import StateManager@@ -375,18 +400,30 @@ def cmd_discover(args):returnprint("\n Running...\n")- report = await run_discovery(state, config, args.provider, queries,- max_spend=args.max_spend)-- r = report- print(f" Found {r.found} results across {r.queries} queries")- print(f" {r.new_companies} new companies, {r.known_companies} already known")- print(f" {r.junk} filtered out as directories/aggregators")- print(f" {r.observations} observations recorded")- print(f" actual cost: ${r.actual_cost:.4f}")- if r.stopped:- print(f"\n Stopped early: {r.stopped}")- for err in r.errors[:5]:+ from harvey.pipeline import run_prospecting++ result = await run_prospecting(+ state, config, args.provider, queries,+ max_spend=args.max_spend, profile=not args.no_profile,+ )+ r = result.discover or {}+ print(f" DISCOVER — {r.get('found', 0)} results across "+ f"{r.get('queries', 0)} queries")+ print(f" {r.get('new_companies', 0)} new companies, "+ f"{r.get('known_companies', 0)} already known")+ print(f" {r.get('junk', 0)} filtered out as directories/aggregators")+ print(f" {r.get('observations', 0)} observations recorded")+ print(f" actual cost: ${r.get('actual_cost', 0):.4f}")+ if r.get("stopped"):+ print(f" stopped early: {r['stopped']}")++ if args.no_profile:+ print("\n PROFILE — skipped (--no-profile)")+ else:+ print(f"\n PROFILE — {result.profiled_companies} sites read, "+ f"{result.profile_observations} observations (free)")++ for err in result.errors[:5]:print(f" ! {err}")print("\n Next: harvey dashboard → Signals → cohort builder\n")@@ -565,8 +602,18 @@ def main():help="Max records per query (default: 100)")sub.add_argument("--max-spend", type=float, default=1.0,help="Hard cap in dollars (default: 1.00)")+ sub.add_argument("--no-profile", action="store_true",+ help="Don't profile what was found (profiling is free)")sub.set_defaults(func=cmd_discover)+ sub = subparsers.add_parser(+ "profile", help="Read discovered companies' websites (free, no Claude)")+ sub.add_argument("--limit", type=int, default=200,+ help="Max sites to read (default: 200)")+ sub.add_argument("--stale-days", type=int, default=90,+ help="Re-read a site after this many days (default: 90)")+ sub.set_defaults(func=cmd_profile)+sub = subparsers.add_parser("signals", help="Review/confirm which signals Harvey prospects against")sub.add_argument("--confirm", metavar="CODES", default="",diff --git a/harvey/collectors/discover.py b/harvey/collectors/discover.pyindex 3531e8f..4ead01f 100644--- a/harvey/collectors/discover.py+++ b/harvey/collectors/discover.py@@ -472,10 +472,17 @@ class OpenStreetMap(DiscoveryProvider):"under 2,000 roofers for the entire US. Great for a free trial ""run or as a cross-reference; not a complete list.")- ENDPOINT = "https://overpass-api.de/api/interpreter"+ # Several public Overpass instances run the same API. The main one+ # throttles readily, so try the mirrors before giving up — a free source+ # that fails half the time is not usable as a default.+ ENDPOINTS = (+ "https://overpass-api.de/api/interpreter",+ "https://overpass.kumi.systems/api/interpreter",+ "https://overpass.private.coffee/api/interpreter",+ )# Overpass etiquette for a shipped tool: modest volume, no parallelism.MIN_INTERVAL = 2.0- BACKOFF = 30.0+ BACKOFF = 5.0# A friendly term maps to the OSM tags that actually carry those businesses.TAG_MAP = {@@ -526,26 +533,32 @@ class OpenStreetMap(DiscoveryProvider):)body = f"[out:json][timeout:40];({clauses});out center {query.limit};"- # Overpass is a free volunteer service and 504s readily under load —- # a cold first request very often does. Retry once with a real- # backoff, then RAISE: returning [] would make a throttled run look- # exactly like a market with no businesses in it.+ # Overpass is a free volunteer service and 429s/504s readily under+ # load. Walk the mirrors, then RAISE: returning [] would make a+ # throttled run look exactly like a market with no businesses in it.headers = {"User-Agent": "Harvey/0.1 (open-source sales agent; ""github.com/ethanplusai/harvey)"}- for attempt in range(2):- r = await client.post(self.ENDPOINT, content=body.encode(),- headers=headers)+ r = None+ for i, endpoint in enumerate(self.ENDPOINTS):+ if i:+ await asyncio.sleep(self.BACKOFF)+ try:+ r = await client.post(endpoint, content=body.encode(),+ headers=headers)+ except httpx.HTTPError as e:+ logger.warning("osm: %s unreachable (%s)", endpoint, e)+ continueif r.status_code not in (429, 504):break- if attempt == 0:- logger.warning("osm: throttled (%s) — backing off %ss",- r.status_code, self.BACKOFF)- await asyncio.sleep(self.BACKOFF)+ logger.warning("osm: %s is throttling (%s) — trying the next mirror",+ endpoint, r.status_code)else:raise RuntimeError(- f"Overpass is throttling (HTTP {r.status_code}). This is a free "- f"community service — wait a minute and re-run, or use a paid "- f"provider for bulk work.")+ "Every public Overpass mirror is throttling right now. This is "+ "free volunteer infrastructure — wait a few minutes and re-run, "+ "or use a paid provider for bulk work.")+ if r is None:+ raise RuntimeError("Could not reach any Overpass mirror.")r.raise_for_status()out = []diff --git a/harvey/dashboard.py b/harvey/dashboard.pyindex 9ac889b..fabbbbb 100644--- a/harvey/dashboard.py+++ b/harvey/dashboard.py@@ -841,8 +841,9 @@ async def start_discovery(request: Request):return JSONResponse({"success": False, "message": "a run is already going"},status_code=409)try:- from harvey.collectors.discover import PROVIDERS, run_discovery+ from harvey.collectors.discover import PROVIDERSfrom harvey.config import load_config+ from harvey.pipeline import run_prospectingbody = await request.json()provider = body.get("provider") or ""@@ -863,9 +864,16 @@ async def start_discovery(request: Request):async def _go():global _discovery_reporttry:- report = await run_discovery(state, config, provider, queries,- max_spend=max_spend)- _discovery_report = report.as_dict()+ # Discovery chains straight into profiling: reading the sites+ # is free, and it is what makes the results worth anything.+ result = await run_prospecting(state, config, provider, queries,+ max_spend=max_spend)+ _discovery_report = {+ **(result.discover or {}),+ "profiled_companies": result.profiled_companies,+ "profile_observations": result.profile_observations,+ "errors": result.errors,+ }except Exception as exc:logger.exception("discovery run failed")_discovery_report = {"errors": [str(exc)], "stopped": "failed"}@@ -877,6 +885,45 @@ async def start_discovery(request: Request):return JSONResponse({"success": False, "message": str(e)}, status_code=500)[email protected]("/api/profile/run")+async def start_profile():+ """Read the websites of everything discovered but not yet looked at.++ Free and model-free, so there is nothing to estimate and no cap to set.+ """+ global _discovery_task, _discovery_report++ if _discovery_task and not _discovery_task.done():+ return JSONResponse({"success": False, "message": "a run is already going"},+ status_code=409)+ try:+ from harvey.pipeline import run_profile_stage++ state = _state()+ await state.init_db()+ pending = await state.count_companies_needing_profile()+ if not pending:+ return {"success": True, "pending": 0}++ async def _go():+ global _discovery_report+ try:+ companies, observations, _ = await run_profile_stage(state, limit=200)+ _discovery_report = {+ "profiled_companies": companies,+ "profile_observations": observations,+ }+ except Exception as exc:+ logger.exception("profile run failed")+ _discovery_report = {"errors": [str(exc)], "stopped": "failed"}++ _discovery_report = None+ _discovery_task = asyncio.create_task(_go())+ return {"success": True, "pending": pending}+ except Exception as e:+ return JSONResponse({"success": False, "message": str(e)}, status_code=500)++@app.post("/api/discover/stop")async def stop_discovery():"""Kill switch. Read between batches, so an in-flight run stops cleanly."""@@ -921,6 +968,7 @@ async def get_today():companies = await query_db("SELECT COUNT(*) AS n FROM companies")n_companies = companies[0]["n"] if companies else 0+ unprofiled_count = await state.count_companies_needing_profile()stats = {"companies": n_companies,"prospects": sum(counts.values()),@@ -928,6 +976,7 @@ async def get_today():"outbox_pending": len(pending),"outbox_approved": len(approved),"open_conversations": open_convos,+ "unprofiled": unprofiled_count,}# Ordered by how much it blocks Harvey from doing anything at all.@@ -977,6 +1026,17 @@ async def get_today():"action": "Find businesses", "tab": "discover",})+ unprofiled = unprofiled_count+ if unprofiled:+ items.append({+ "key": "profile", "tone": "good",+ "title": f"{unprofiled} companies not looked at yet",+ "detail": ("Reading their websites is free and it is what makes "+ "an email specific — who their agency is, what they "+ "are missing, whether they are spending on ads."),+ "action": "Read their sites", "tab": "discover",+ })+if pending:items.append({"key": "outbox", "tone": "warn",diff --git a/harvey/main.py b/harvey/main.pyindex d4d1aac..93d11af 100644--- a/harvey/main.py+++ b/harvey/main.py@@ -10,6 +10,7 @@ import pytzfrom harvey.brain import Brainfrom harvey.config import ConfigError, load_config, load_env, HarveyConfig+from harvey.pipeline import run_profile_stagefrom harvey.state import StateManagerlogging.basicConfig(@@ -201,6 +202,13 @@ async def heartbeat(stop_event: asyncio.Event | None = None):if sender.is_native and not any(n == "send_campaign" for n, _ in tasks):tasks.append(("send_outbox", sender.run()))+ # Profiling rides along every cycle. It is three HTTP requests per+ # business with no model call, so it costs nothing against the+ # Claude budget the rest of this loop is rationing — and it is what+ # turns a name and a domain into something worth writing about.+ if summary.get("unprofiled", 0):+ tasks.append(("profile", run_profile_stage(state, limit=25)))+if len(tasks) > 1:logger.info(f"Running {len(tasks)} agents in parallel: {[t[0] for t in tasks]}")diff --git a/harvey/pipeline.py b/harvey/pipeline.pynew file mode 100644index 0000000..a80f643--- /dev/null+++ b/harvey/pipeline.py@@ -0,0 +1,128 @@+"""The prospecting pipeline: DISCOVER → PROFILE → (people, verification).++Each stage is independently runnable and independently re-runnable. That is+the point of the design, not an accident of it: profiling can re-run without+re-buying discovery, and discovery can widen without re-profiling everyone.++What chains them is a *query*, not a hand-off. PROFILE asks the database+"which companies have I not looked at, or not looked at lately?" rather than+being handed a list by DISCOVER. So a profile run started for any reason+picks up whatever discovery left behind, a crashed run resumes by itself, and+re-observing on a schedule turns the same signal into a time series.++Segment before you enrich: enrichment cost scales with list size and+conversion does not, so profile the businesses you would actually contact,+not every row you hold.+"""++import logging+from dataclasses import dataclass, field++logger = logging.getLogger("harvey.pipeline")++# How long a profile observation stays fresh. Re-reading a site every quarter+# is what makes "they dropped their agency" a visible event.+PROFILE_STALE_DAYS = 90+++@dataclass+class PipelineReport:+ """What a full pass did, stage by stage."""++ discover: dict | None = None+ profiled_companies: int = 0+ profile_observations: int = 0+ profile_run_id: str = ""+ skipped: str = ""+ errors: list[str] = field(default_factory=list)++ def as_dict(self) -> dict:+ return {+ "discover": self.discover,+ "profiled_companies": self.profiled_companies,+ "profile_observations": self.profile_observations,+ "profile_run_id": self.profile_run_id,+ "skipped": self.skipped,+ "errors": self.errors,+ }++ def summary_line(self) -> str:+ parts = []+ if self.discover:+ parts.append(f"{self.discover.get('new_companies', 0)} new companies")+ if self.profiled_companies:+ parts.append(f"{self.profiled_companies} profiled "+ f"({self.profile_observations} observations)")+ return ", ".join(parts) or "nothing to do"+++async def run_profile_stage(+ state,+ limit: int = 100,+ stale_days: int = PROFILE_STALE_DAYS,+) -> tuple[int, int, str]:+ """Profile whatever currently needs it.++ Returns (companies, observations, run_id). Free — three HTTP requests per+ business, no browser, no model call — so it is safe to run on every+ heartbeat without touching the Claude budget.+ """+ from harvey.collectors.profile import profile_companies++ pending = await state.companies_needing_profile(limit=limit,+ stale_days=stale_days)+ if not pending:+ return 0, 0, ""++ logger.info("profile: %d companies need a look", len(pending))+ observations, run_id = await profile_companies(state, pending)+ return len(pending), observations, run_id+++async def run_prospecting(+ state,+ config,+ provider: str | None = None,+ queries=None,+ max_spend: float = 1.0,+ profile: bool = True,+ profile_limit: int = 200,+ dry_run: bool = False,+) -> PipelineReport:+ """Discover businesses, then read what their websites say about them.++ The two stages are deliberately not coupled through a list: profiling+ queries the database for what needs attention, so it also sweeps up+ anything an earlier run left unfinished.+ """+ from harvey.collectors.discover import DEFAULT_PROVIDER, run_discovery++ report = PipelineReport()++ discovery = await run_discovery(+ state, config, provider or DEFAULT_PROVIDER, queries,+ max_spend=max_spend, dry_run=dry_run,+ )+ report.discover = discovery.as_dict()+ report.errors.extend(discovery.errors)++ if dry_run:+ report.skipped = "dry run"+ return report+ if not profile:+ report.skipped = "profiling skipped"+ return report++ # Profile runs even when discovery found nothing new: a previous run may+ # have been interrupted, and observations go stale on a schedule.+ try:+ companies, observations, run_id = await run_profile_stage(+ state, limit=profile_limit)+ report.profiled_companies = companies+ report.profile_observations = observations+ report.profile_run_id = run_id+ except Exception as e:+ logger.exception("profile stage failed")+ report.errors.append(f"profile: {e}")++ return reportdiff --git a/harvey/state.py b/harvey/state.pyindex 5f495c7..ce234e4 100644--- a/harvey/state.py+++ b/harvey/state.py@@ -1062,20 +1062,45 @@ class StateManager:async with db.execute(sql, params) as cursor:return [dict(r) for r in await cursor.fetchall()]+ # A boolean signal is recorded either way: "checked, not running ads" is a+ # real finding, and a later flip from 0 to 1 is a real event. But a COHORT+ # asks who *has* the signal, so it must read the value, not merely the+ # presence of a row. Without this, "companies running Google Ads" silently+ # means "companies we checked for Google Ads" — which is everyone.+ #+ # `latest` is the current view of the world (newest observation per company+ # and signal, with history still on disk underneath). `positive` is the+ # subset where the finding is actually true — value_num IS NULL covers text+ # signals like INCUMBENT_AGENCY, where the row's existence IS the finding.+ _CURRENT_CTE = """+ WITH latest AS (+ SELECT o.* FROM observations o+ JOIN (SELECT company_id, signal_code, MAX(observed_at) AS t+ FROM observations GROUP BY company_id, signal_code) newest+ ON newest.company_id = o.company_id+ AND newest.signal_code = o.signal_code+ AND newest.t = o.observed_at+ ),+ positive AS (+ SELECT * FROM latest WHERE value_num IS NULL OR value_num != 0+ )+ """+async def signal_counts(self) -> list[dict]:- """How many entities carry each signal — the cohort sizes."""+ """How many entities actually carry each signal — the cohort sizes."""async with self._connect() as db:db.row_factory = aiosqlite.Rowasync with db.execute(- """SELECT o.signal_code,- COALESCE(sc.label, o.signal_code) AS label,- sc.status AS status,- COUNT(DISTINCT o.company_id) AS companies,- COUNT(*) AS observations- FROM observations o- LEFT JOIN signal_codes sc ON sc.code = o.signal_code- GROUP BY o.signal_code- ORDER BY companies DESC"""+ self._CURRENT_CTE + """+ SELECT p.signal_code,+ COALESCE(sc.label, p.signal_code) AS label,+ sc.status AS status,+ COUNT(DISTINCT p.company_id) AS companies,+ COUNT(*) AS observations+ FROM positive p+ LEFT JOIN signal_codes sc ON sc.code = p.signal_code+ GROUP BY p.signal_code+ ORDER BY companies DESC""") as cursor:return [dict(r) for r in await cursor.fetchall()]@@ -1094,8 +1119,8 @@ class StateManager:if not require:return []req_ph = ",".join("?" for _ in require)- sql = (- "SELECT company_id FROM observations "+ sql = self._CURRENT_CTE + (+ f"SELECT company_id FROM positive "f"WHERE signal_code IN ({req_ph}) AND company_id != '' ""AND confidence >= ? ""GROUP BY company_id HAVING COUNT(DISTINCT signal_code) = ?"@@ -1105,7 +1130,7 @@ class StateManager:exc_ph = ",".join("?" for _ in exclude)sql += (" AND company_id NOT IN ("- f"SELECT company_id FROM observations WHERE signal_code IN ({exc_ph})"+ f"SELECT company_id FROM positive WHERE signal_code IN ({exc_ph})"")")params.extend(exclude)@@ -1115,6 +1140,54 @@ class StateManager:async with db.execute(sql, params) as cursor:return [row[0] for row in await cursor.fetchall()]+ async def companies_needing_profile(+ self, limit: int = 100, stale_days: int = 90+ ) -> list[dict]:+ """Companies a profile run should visit next.++ Never profiled, or last profiled longer ago than ``stale_days``. The+ staleness window is what makes re-observation a time series rather+ than a duplicate: "they dropped their agency last quarter" is only+ visible if you look again.++ Businesses with no domain are excluded — there is no site to read.+ They are not a failure, they are the NO_WEBSITE cohort.+ """+ async with self._connect() as db:+ db.row_factory = aiosqlite.Row+ async with db.execute(+ """SELECT c.id, c.name, c.domain+ FROM companies c+ LEFT JOIN (+ SELECT company_id, MAX(observed_at) AS last_seen+ FROM observations WHERE collector = 'profile'+ GROUP BY company_id+ ) p ON p.company_id = c.id+ WHERE c.domain != ''+ AND (p.last_seen IS NULL+ OR p.last_seen < datetime('now', ?))+ ORDER BY c.created_at DESC+ LIMIT ?""",+ (f"-{int(stale_days)} days", int(limit)),+ ) as cursor:+ return [dict(r) for r in await cursor.fetchall()]++ async def count_companies_needing_profile(self, stale_days: int = 90) -> int:+ async with self._connect() as db:+ async with db.execute(+ """SELECT COUNT(*) FROM companies c+ LEFT JOIN (+ SELECT company_id, MAX(observed_at) AS last_seen+ FROM observations WHERE collector = 'profile'+ GROUP BY company_id+ ) p ON p.company_id = c.id+ WHERE c.domain != ''+ AND (p.last_seen IS NULL OR p.last_seen < datetime('now', ?))""",+ (f"-{int(stale_days)} days",),+ ) as cursor:+ row = await cursor.fetchone()+ return row[0] if row else 0+# ── Run log (what ran, when, what it produced and cost) ──async def start_run(@@ -1644,4 +1717,8 @@ class StateManager:"active_campaigns": active_campaigns,"open_conversations": open_conversations,"usage_today": usage_today,+ # Companies discovered but not yet read. The heartbeat profiles+ # these every cycle — it is free, so it never competes with the+ # Claude budget the rest of the loop is rationing.+ "unprofiled": await self.count_companies_needing_profile(),}diff --git a/tests/test_discover.py b/tests/test_discover.pyindex 616bfb7..d1344d8 100644--- a/tests/test_discover.py+++ b/tests/test_discover.py@@ -499,10 +499,10 @@ async def test_overpass_throttling_is_an_error_not_an_empty_market(monkeypatch):"""A silent [] on a 504 is indistinguishable from 'no businesses here',which is how a throttled run gets mistaken for a clean one."""monkeypatch.setattr(D.OpenStreetMap, "BACKOFF", 0.0)- calls = {"n": 0}+ seen = []def handler(request):- calls["n"] += 1+ seen.append(str(request.url))return httpx.Response(504, text="")provider = D.OpenStreetMap()@@ -510,11 +510,11 @@ async def test_overpass_throttling_is_an_error_not_an_empty_market(monkeypatch):async with _client(handler) as client:await provider.fetch(client, {}, D.DiscoveryQuery(term="roofer", coordinate="39.7,-104.9,40"))- assert calls["n"] == 2, "it should retry once before giving up"+ assert seen == list(D.OpenStreetMap.ENDPOINTS), "every mirror should be tried"@pytest.mark.asyncio-async def test_overpass_recovers_on_the_retry(monkeypatch):+async def test_overpass_falls_over_to_the_next_mirror(monkeypatch):monkeypatch.setattr(D.OpenStreetMap, "BACKOFF", 0.0)calls = {"n": 0}diff --git a/tests/test_pipeline.py b/tests/test_pipeline.pynew file mode 100644index 0000000..0a6cc0d--- /dev/null+++ b/tests/test_pipeline.py@@ -0,0 +1,289 @@+"""DISCOVER → PROFILE chaining.++The stages are joined by a query, not a hand-off. These tests pin that+property down, because it is what makes a crashed run resumable and+re-observation a time series instead of a duplicate.+"""++import os+import tempfile++import pytest+import pytest_asyncio++from harvey import pipeline as P+from harvey.collectors import discover as D+from harvey.config import load_config+from harvey.models import Company+from harvey.signals import SIGNAL_CATALOG, seed_signal_catalog+from harvey.state import StateManager+++@pytest_asyncio.fixture+async def state():+ with tempfile.TemporaryDirectory() as tmp:+ sm = StateManager(os.path.join(tmp, "test.db"))+ await sm.init_db()+ await seed_signal_catalog(sm)+ for sig in SIGNAL_CATALOG:+ await sm.set_signal_status(sig["code"], "confirmed")+ yield sm+++def config():+ return load_config()+++# ── The join: which companies still need a look ──+++async def test_a_new_company_needs_profiling(state):+ await state.add_company(Company(name="Baker", domain="baker.com"))+ pending = await state.companies_needing_profile()+ assert [c["domain"] for c in pending] == ["baker.com"]+ assert await state.count_companies_needing_profile() == 1+++async def test_a_profiled_company_drops_out(state):+ cid = await state.add_company(Company(name="Baker", domain="baker.com"))+ await state.add_observation("TECH_STACK", company_id=cid,+ collector="profile", value_text="wordpress")+ assert await state.count_companies_needing_profile() == 0+++async def test_a_stale_profile_comes_back_round(state):+ """Re-reading on a schedule is what makes 'they dropped their agency'+ visible. A company profiled last year is not done forever."""+ cid = await state.add_company(Company(name="Baker", domain="baker.com"))+ await state.add_observation("TECH_STACK", company_id=cid, collector="profile",+ value_text="wordpress",+ observed_at="2020-01-01 00:00:00")+ assert await state.count_companies_needing_profile(stale_days=90) == 1+ assert await state.count_companies_needing_profile(stale_days=99999) == 0+++async def test_discovery_observations_do_not_count_as_a_profile(state):+ """DISCOVER writes observations too. Only PROFILE's own count."""+ cid = await state.add_company(Company(name="Baker", domain="baker.com"))+ await state.add_observation("SERP_RANK", company_id=cid,+ collector="discover", value_num=14)+ assert await state.count_companies_needing_profile() == 1+++async def test_websiteless_companies_are_never_queued_for_profiling(state):+ """No site to read. That is the NO_WEBSITE cohort, not a backlog."""+ await state.add_company(Company(name="No Site", external_id="osm:node/1"))+ assert await state.count_companies_needing_profile() == 0+++# ── The chain ──+++class FakeDiscovery(D.DiscoveryProvider):+ key = "fake"+ label = "Fake"++ def __init__(self, businesses):+ self.businesses = businesses++ def configured(self, env):+ return True++ def estimate(self, queries):+ return 0.0++ async def fetch(self, client, env, query):+ return self.businesses+++def chain(monkeypatch):+ """A fake discovery provider and a profile stage that records its input."""+ seen = {"profiled": []}++ def install(businesses):+ monkeypatch.setitem(D.PROVIDERS, "fake", FakeDiscovery(businesses))++ async def fake_profile(state, companies):+ seen["profiled"] = [c["domain"] for c in companies]+ for company in companies:+ await state.add_observation(+ "TECH_STACK", company_id=company["id"],+ collector="profile", value_text="wordpress")+ return len(companies), "run-x"++ monkeypatch.setattr("harvey.collectors.profile.profile_companies",+ fake_profile)+ return seen++ async def offline(state, city, radius_km=40, client=None):+ return ""+ monkeypatch.setattr(D, "geocode_city", offline)+ return install+++async def test_discovery_flows_straight_into_profiling(state, config, chain):+ seen = chain([+ D.Business(name="Baker", domain="baker.com", provider="fake"),+ D.Business(name="Metro", domain="metro.com", provider="fake"),+ ])+ report = await P.run_prospecting(+ state, config, "fake", [D.DiscoveryQuery(term="roofer")])++ assert report.discover["new_companies"] == 2+ assert sorted(seen["profiled"]) == ["baker.com", "metro.com"]+ assert report.profiled_companies == 2+ assert report.profile_observations == 2+++async def test_profiling_sweeps_up_what_an_earlier_run_left(state, config, chain):+ """The stages join through a query, so a run interrupted before profiling+ is finished by the next one — nothing has to remember anything."""+ await state.add_company(Company(name="Left Over", domain="leftover.com"))+ seen = chain([D.Business(name="Baker", domain="baker.com", provider="fake")])++ await P.run_prospecting(state, config, "fake", [D.DiscoveryQuery(term="roofer")])+ assert sorted(seen["profiled"]) == ["baker.com", "leftover.com"]+++async def test_profiling_runs_even_when_discovery_finds_nothing(state, config, chain):+ await state.add_company(Company(name="Left Over", domain="leftover.com"))+ seen = chain([])++ report = await P.run_prospecting(state, config, "fake",+ [D.DiscoveryQuery(term="roofer")])+ assert report.discover["new_companies"] == 0+ assert seen["profiled"] == ["leftover.com"]+++async def test_no_profile_stops_at_discovery(state, config, chain):+ seen = chain([D.Business(name="Baker", domain="baker.com", provider="fake")])+ report = await P.run_prospecting(state, config, "fake",+ [D.DiscoveryQuery(term="roofer")],+ profile=False)+ assert report.discover["new_companies"] == 1+ assert seen["profiled"] == []+ assert report.skipped == "profiling skipped"+++async def test_a_dry_run_neither_discovers_nor_profiles(state, config, chain):+ seen = chain([D.Business(name="Baker", domain="baker.com", provider="fake")])+ report = await P.run_prospecting(state, config, "fake",+ [D.DiscoveryQuery(term="roofer")],+ dry_run=True)+ assert report.skipped == "dry run"+ assert seen["profiled"] == []+ assert await state.count_companies_needing_profile() == 0+++async def test_a_failing_profile_stage_does_not_lose_the_discovery(+ state, config, monkeypatch, chain):+ chain([D.Business(name="Baker", domain="baker.com", provider="fake")])++ async def boom(state, companies):+ raise RuntimeError("network gone")+ monkeypatch.setattr("harvey.collectors.profile.profile_companies", boom)++ report = await P.run_prospecting(state, config, "fake",+ [D.DiscoveryQuery(term="roofer")])+ assert report.discover["new_companies"] == 1, "the companies are still saved"+ assert any("network gone" in e for e in report.errors)+++async def test_profile_stage_is_a_noop_with_nothing_to_do(state):+ companies, observations, run_id = await P.run_profile_stage(state)+ assert (companies, observations, run_id) == (0, 0, "")+++async def test_state_summary_reports_the_profiling_backlog(state):+ """The heartbeat reads this to decide whether to profile this cycle."""+ await state.add_company(Company(name="Baker", domain="baker.com"))+ summary = await state.get_state_summary()+ assert summary["unprofiled"] == 1+++# ── Cohorts read the value, not just the row ──+++async def test_a_false_boolean_is_recorded_but_not_in_the_cohort(state):+ """PROFILE records RUNNING_GOOGLE_ADS=0 for everyone it checks — knowing+ you looked is worth keeping, and a later 0→1 flip is a real event. But+ "companies running Google Ads" must not silently mean "companies we+ checked for Google Ads", which is everyone.+ """+ running = await state.add_company(Company(name="Spender", domain="a.com"))+ checked = await state.add_company(Company(name="Not spending", domain="b.com"))+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=running,+ collector="profile", value_num=1)+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=checked,+ collector="profile", value_num=0)++ assert await state.cohort(["RUNNING_GOOGLE_ADS"]) == [running]++ counts = {c["signal_code"]: c for c in await state.signal_counts()}+ assert counts["RUNNING_GOOGLE_ADS"]["companies"] == 1++ # The negative finding is still on disk.+ rows = await state.get_observations(company_id=checked)+ assert rows[0]["value_num"] == 0+++async def test_text_signals_count_by_their_existence(state):+ """INCUMBENT_AGENCY has no numeric value — the row IS the finding."""+ cid = await state.add_company(Company(name="Baker", domain="a.com"))+ await state.add_observation("INCUMBENT_AGENCY", company_id=cid,+ collector="profile", value_text="Studio3")+ assert await state.cohort(["INCUMBENT_AGENCY"]) == [cid]+++async def test_a_cohort_reflects_the_newest_observation(state):+ """They dropped their agency last quarter. The cohort should say so."""+ cid = await state.add_company(Company(name="Baker", domain="a.com"))+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=cid,+ collector="profile", value_num=1,+ observed_at="2026-01-01 00:00:00")+ assert await state.cohort(["RUNNING_GOOGLE_ADS"]) == [cid]++ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=cid,+ collector="profile", value_num=0,+ observed_at="2026-06-01 00:00:00")+ assert await state.cohort(["RUNNING_GOOGLE_ADS"]) == []++ # Both observations survive — that time series is the durable asset.+ assert len(await state.get_observations(company_id=cid)) == 2+++async def test_exclusions_also_read_the_value(state):+ """Excluding "has online booking" must not exclude everyone who was+ merely checked for it."""+ a = await state.add_company(Company(name="A", domain="a.com"))+ b = await state.add_company(Company(name="B", domain="b.com"))+ for cid, booking in ((a, 1), (b, 0)):+ await state.add_observation("NO_SCHEMA_MARKUP", company_id=cid,+ collector="profile", value_num=1)+ await state.add_observation("NO_ONLINE_BOOKING", company_id=cid,+ collector="profile", value_num=booking)++ both = await state.cohort(["NO_SCHEMA_MARKUP"])+ assert sorted(both) == sorted([a, b])++ # Only A actually lacks booking, so excluding it leaves B.+ assert await state.cohort(["NO_SCHEMA_MARKUP"], exclude=["NO_ONLINE_BOOKING"]) == [b]