Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

Chain PROFILE onto DISCOVER, and make cohorts read the value

Ethan6486fe1d401b
diff --git a/harvey/cli.py b/harvey/cli.py
index 01c342c..f124b65 100644
--- a/harvey/cli.py
+++ b/harvey/cli.py
@@ -320,6 +320,31 @@ def cmd_outbox(args):
asyncio.run(_outbox())
+def cmd_profile(args):
+ """Read what companies' own websites say about them. Free, no Claude calls."""
+ from harvey.state import StateManager
+ from harvey.pipeline import run_profile_stage
+
+ async def _profile():
+ state = StateManager()
+ await state.init_db()
+
+ pending = await state.count_companies_needing_profile(args.stale_days)
+ if not pending:
+ print("\n Nothing to profile. Every company with a website has "
+ "been read recently.\n")
+ return
+
+ print(f"\n {pending} companies need profiling. Reading up to "
+ f"{args.limit}...\n")
+ companies, observations, _ = await run_profile_stage(
+ state, limit=args.limit, stale_days=args.stale_days)
+ print(f" Read {companies} sites → {observations} observations (free)")
+ print("\n See what turned up: harvey dashboard → Signals\n")
+
+ asyncio.run(_profile())
+
+
def cmd_discover(args):
"""Find businesses. The only stage that spends money, so it estimates first."""
from harvey.state import StateManager
@@ -375,18 +400,30 @@ def cmd_discover(args):
return
print("\n Running...\n")
- report = await run_discovery(state, config, args.provider, queries,
- max_spend=args.max_spend)
-
- r = report
- print(f" Found {r.found} results across {r.queries} queries")
- print(f" {r.new_companies} new companies, {r.known_companies} already known")
- print(f" {r.junk} filtered out as directories/aggregators")
- print(f" {r.observations} observations recorded")
- print(f" actual cost: ${r.actual_cost:.4f}")
- if r.stopped:
- print(f"\n Stopped early: {r.stopped}")
- for err in r.errors[:5]:
+ from harvey.pipeline import run_prospecting
+
+ result = await run_prospecting(
+ state, config, args.provider, queries,
+ max_spend=args.max_spend, profile=not args.no_profile,
+ )
+ r = result.discover or {}
+ print(f" DISCOVER — {r.get('found', 0)} results across "
+ f"{r.get('queries', 0)} queries")
+ print(f" {r.get('new_companies', 0)} new companies, "
+ f"{r.get('known_companies', 0)} already known")
+ print(f" {r.get('junk', 0)} filtered out as directories/aggregators")
+ print(f" {r.get('observations', 0)} observations recorded")
+ print(f" actual cost: ${r.get('actual_cost', 0):.4f}")
+ if r.get("stopped"):
+ print(f" stopped early: {r['stopped']}")
+
+ if args.no_profile:
+ print("\n PROFILE — skipped (--no-profile)")
+ else:
+ print(f"\n PROFILE — {result.profiled_companies} sites read, "
+ f"{result.profile_observations} observations (free)")
+
+ for err in result.errors[:5]:
print(f" ! {err}")
print("\n Next: harvey dashboard → Signals → cohort builder\n")
@@ -565,8 +602,18 @@ def main():
help="Max records per query (default: 100)")
sub.add_argument("--max-spend", type=float, default=1.0,
help="Hard cap in dollars (default: 1.00)")
+ sub.add_argument("--no-profile", action="store_true",
+ help="Don't profile what was found (profiling is free)")
sub.set_defaults(func=cmd_discover)
+ sub = subparsers.add_parser(
+ "profile", help="Read discovered companies' websites (free, no Claude)")
+ sub.add_argument("--limit", type=int, default=200,
+ help="Max sites to read (default: 200)")
+ sub.add_argument("--stale-days", type=int, default=90,
+ help="Re-read a site after this many days (default: 90)")
+ sub.set_defaults(func=cmd_profile)
+
sub = subparsers.add_parser(
"signals", help="Review/confirm which signals Harvey prospects against")
sub.add_argument("--confirm", metavar="CODES", default="",
diff --git a/harvey/collectors/discover.py b/harvey/collectors/discover.py
index 3531e8f..4ead01f 100644
--- a/harvey/collectors/discover.py
+++ b/harvey/collectors/discover.py
@@ -472,10 +472,17 @@ class OpenStreetMap(DiscoveryProvider):
"under 2,000 roofers for the entire US. Great for a free trial "
"run or as a cross-reference; not a complete list.")
- ENDPOINT = "https://overpass-api.de/api/interpreter"
+ # Several public Overpass instances run the same API. The main one
+ # throttles readily, so try the mirrors before giving up — a free source
+ # that fails half the time is not usable as a default.
+ ENDPOINTS = (
+ "https://overpass-api.de/api/interpreter",
+ "https://overpass.kumi.systems/api/interpreter",
+ "https://overpass.private.coffee/api/interpreter",
+ )
# Overpass etiquette for a shipped tool: modest volume, no parallelism.
MIN_INTERVAL = 2.0
- BACKOFF = 30.0
+ BACKOFF = 5.0
# A friendly term maps to the OSM tags that actually carry those businesses.
TAG_MAP = {
@@ -526,26 +533,32 @@ class OpenStreetMap(DiscoveryProvider):
)
body = f"[out:json][timeout:40];({clauses});out center {query.limit};"
- # Overpass is a free volunteer service and 504s readily under load —
- # a cold first request very often does. Retry once with a real
- # backoff, then RAISE: returning [] would make a throttled run look
- # exactly like a market with no businesses in it.
+ # Overpass is a free volunteer service and 429s/504s readily under
+ # load. Walk the mirrors, then RAISE: returning [] would make a
+ # throttled run look exactly like a market with no businesses in it.
headers = {"User-Agent": "Harvey/0.1 (open-source sales agent; "
"github.com/ethanplusai/harvey)"}
- for attempt in range(2):
- r = await client.post(self.ENDPOINT, content=body.encode(),
- headers=headers)
+ r = None
+ for i, endpoint in enumerate(self.ENDPOINTS):
+ if i:
+ await asyncio.sleep(self.BACKOFF)
+ try:
+ r = await client.post(endpoint, content=body.encode(),
+ headers=headers)
+ except httpx.HTTPError as e:
+ logger.warning("osm: %s unreachable (%s)", endpoint, e)
+ continue
if r.status_code not in (429, 504):
break
- if attempt == 0:
- logger.warning("osm: throttled (%s) — backing off %ss",
- r.status_code, self.BACKOFF)
- await asyncio.sleep(self.BACKOFF)
+ logger.warning("osm: %s is throttling (%s) — trying the next mirror",
+ endpoint, r.status_code)
else:
raise RuntimeError(
- f"Overpass is throttling (HTTP {r.status_code}). This is a free "
- f"community service — wait a minute and re-run, or use a paid "
- f"provider for bulk work.")
+ "Every public Overpass mirror is throttling right now. This is "
+ "free volunteer infrastructure — wait a few minutes and re-run, "
+ "or use a paid provider for bulk work.")
+ if r is None:
+ raise RuntimeError("Could not reach any Overpass mirror.")
r.raise_for_status()
out = []
diff --git a/harvey/dashboard.py b/harvey/dashboard.py
index 9ac889b..fabbbbb 100644
--- a/harvey/dashboard.py
+++ b/harvey/dashboard.py
@@ -841,8 +841,9 @@ async def start_discovery(request: Request):
return JSONResponse({"success": False, "message": "a run is already going"},
status_code=409)
try:
- from harvey.collectors.discover import PROVIDERS, run_discovery
+ from harvey.collectors.discover import PROVIDERS
from harvey.config import load_config
+ from harvey.pipeline import run_prospecting
body = await request.json()
provider = body.get("provider") or ""
@@ -863,9 +864,16 @@ async def start_discovery(request: Request):
async def _go():
global _discovery_report
try:
- report = await run_discovery(state, config, provider, queries,
- max_spend=max_spend)
- _discovery_report = report.as_dict()
+ # Discovery chains straight into profiling: reading the sites
+ # is free, and it is what makes the results worth anything.
+ result = await run_prospecting(state, config, provider, queries,
+ max_spend=max_spend)
+ _discovery_report = {
+ **(result.discover or {}),
+ "profiled_companies": result.profiled_companies,
+ "profile_observations": result.profile_observations,
+ "errors": result.errors,
+ }
except Exception as exc:
logger.exception("discovery run failed")
_discovery_report = {"errors": [str(exc)], "stopped": "failed"}
@@ -877,6 +885,45 @@ async def start_discovery(request: Request):
return JSONResponse({"success": False, "message": str(e)}, status_code=500)
[email protected]("/api/profile/run")
+async def start_profile():
+ """Read the websites of everything discovered but not yet looked at.
+
+ Free and model-free, so there is nothing to estimate and no cap to set.
+ """
+ global _discovery_task, _discovery_report
+
+ if _discovery_task and not _discovery_task.done():
+ return JSONResponse({"success": False, "message": "a run is already going"},
+ status_code=409)
+ try:
+ from harvey.pipeline import run_profile_stage
+
+ state = _state()
+ await state.init_db()
+ pending = await state.count_companies_needing_profile()
+ if not pending:
+ return {"success": True, "pending": 0}
+
+ async def _go():
+ global _discovery_report
+ try:
+ companies, observations, _ = await run_profile_stage(state, limit=200)
+ _discovery_report = {
+ "profiled_companies": companies,
+ "profile_observations": observations,
+ }
+ except Exception as exc:
+ logger.exception("profile run failed")
+ _discovery_report = {"errors": [str(exc)], "stopped": "failed"}
+
+ _discovery_report = None
+ _discovery_task = asyncio.create_task(_go())
+ return {"success": True, "pending": pending}
+ except Exception as e:
+ return JSONResponse({"success": False, "message": str(e)}, status_code=500)
+
+
@app.post("/api/discover/stop")
async def stop_discovery():
"""Kill switch. Read between batches, so an in-flight run stops cleanly."""
@@ -921,6 +968,7 @@ async def get_today():
companies = await query_db("SELECT COUNT(*) AS n FROM companies")
n_companies = companies[0]["n"] if companies else 0
+ unprofiled_count = await state.count_companies_needing_profile()
stats = {
"companies": n_companies,
"prospects": sum(counts.values()),
@@ -928,6 +976,7 @@ async def get_today():
"outbox_pending": len(pending),
"outbox_approved": len(approved),
"open_conversations": open_convos,
+ "unprofiled": unprofiled_count,
}
# Ordered by how much it blocks Harvey from doing anything at all.
@@ -977,6 +1026,17 @@ async def get_today():
"action": "Find businesses", "tab": "discover",
})
+ unprofiled = unprofiled_count
+ if unprofiled:
+ items.append({
+ "key": "profile", "tone": "good",
+ "title": f"{unprofiled} companies not looked at yet",
+ "detail": ("Reading their websites is free and it is what makes "
+ "an email specific — who their agency is, what they "
+ "are missing, whether they are spending on ads."),
+ "action": "Read their sites", "tab": "discover",
+ })
+
if pending:
items.append({
"key": "outbox", "tone": "warn",
diff --git a/harvey/main.py b/harvey/main.py
index d4d1aac..93d11af 100644
--- a/harvey/main.py
+++ b/harvey/main.py
@@ -10,6 +10,7 @@ import pytz
from harvey.brain import Brain
from harvey.config import ConfigError, load_config, load_env, HarveyConfig
+from harvey.pipeline import run_profile_stage
from harvey.state import StateManager
logging.basicConfig(
@@ -201,6 +202,13 @@ async def heartbeat(stop_event: asyncio.Event | None = None):
if sender.is_native and not any(n == "send_campaign" for n, _ in tasks):
tasks.append(("send_outbox", sender.run()))
+ # Profiling rides along every cycle. It is three HTTP requests per
+ # business with no model call, so it costs nothing against the
+ # Claude budget the rest of this loop is rationing — and it is what
+ # turns a name and a domain into something worth writing about.
+ if summary.get("unprofiled", 0):
+ tasks.append(("profile", run_profile_stage(state, limit=25)))
+
if len(tasks) > 1:
logger.info(f"Running {len(tasks)} agents in parallel: {[t[0] for t in tasks]}")
diff --git a/harvey/pipeline.py b/harvey/pipeline.py
new file mode 100644
index 0000000..a80f643
--- /dev/null
+++ b/harvey/pipeline.py
@@ -0,0 +1,128 @@
+"""The prospecting pipeline: DISCOVER → PROFILE → (people, verification).
+
+Each stage is independently runnable and independently re-runnable. That is
+the point of the design, not an accident of it: profiling can re-run without
+re-buying discovery, and discovery can widen without re-profiling everyone.
+
+What chains them is a *query*, not a hand-off. PROFILE asks the database
+"which companies have I not looked at, or not looked at lately?" rather than
+being handed a list by DISCOVER. So a profile run started for any reason
+picks up whatever discovery left behind, a crashed run resumes by itself, and
+re-observing on a schedule turns the same signal into a time series.
+
+Segment before you enrich: enrichment cost scales with list size and
+conversion does not, so profile the businesses you would actually contact,
+not every row you hold.
+"""
+
+import logging
+from dataclasses import dataclass, field
+
+logger = logging.getLogger("harvey.pipeline")
+
+# How long a profile observation stays fresh. Re-reading a site every quarter
+# is what makes "they dropped their agency" a visible event.
+PROFILE_STALE_DAYS = 90
+
+
+@dataclass
+class PipelineReport:
+ """What a full pass did, stage by stage."""
+
+ discover: dict | None = None
+ profiled_companies: int = 0
+ profile_observations: int = 0
+ profile_run_id: str = ""
+ skipped: str = ""
+ errors: list[str] = field(default_factory=list)
+
+ def as_dict(self) -> dict:
+ return {
+ "discover": self.discover,
+ "profiled_companies": self.profiled_companies,
+ "profile_observations": self.profile_observations,
+ "profile_run_id": self.profile_run_id,
+ "skipped": self.skipped,
+ "errors": self.errors,
+ }
+
+ def summary_line(self) -> str:
+ parts = []
+ if self.discover:
+ parts.append(f"{self.discover.get('new_companies', 0)} new companies")
+ if self.profiled_companies:
+ parts.append(f"{self.profiled_companies} profiled "
+ f"({self.profile_observations} observations)")
+ return ", ".join(parts) or "nothing to do"
+
+
+async def run_profile_stage(
+ state,
+ limit: int = 100,
+ stale_days: int = PROFILE_STALE_DAYS,
+) -> tuple[int, int, str]:
+ """Profile whatever currently needs it.
+
+ Returns (companies, observations, run_id). Free — three HTTP requests per
+ business, no browser, no model call — so it is safe to run on every
+ heartbeat without touching the Claude budget.
+ """
+ from harvey.collectors.profile import profile_companies
+
+ pending = await state.companies_needing_profile(limit=limit,
+ stale_days=stale_days)
+ if not pending:
+ return 0, 0, ""
+
+ logger.info("profile: %d companies need a look", len(pending))
+ observations, run_id = await profile_companies(state, pending)
+ return len(pending), observations, run_id
+
+
+async def run_prospecting(
+ state,
+ config,
+ provider: str | None = None,
+ queries=None,
+ max_spend: float = 1.0,
+ profile: bool = True,
+ profile_limit: int = 200,
+ dry_run: bool = False,
+) -> PipelineReport:
+ """Discover businesses, then read what their websites say about them.
+
+ The two stages are deliberately not coupled through a list: profiling
+ queries the database for what needs attention, so it also sweeps up
+ anything an earlier run left unfinished.
+ """
+ from harvey.collectors.discover import DEFAULT_PROVIDER, run_discovery
+
+ report = PipelineReport()
+
+ discovery = await run_discovery(
+ state, config, provider or DEFAULT_PROVIDER, queries,
+ max_spend=max_spend, dry_run=dry_run,
+ )
+ report.discover = discovery.as_dict()
+ report.errors.extend(discovery.errors)
+
+ if dry_run:
+ report.skipped = "dry run"
+ return report
+ if not profile:
+ report.skipped = "profiling skipped"
+ return report
+
+ # Profile runs even when discovery found nothing new: a previous run may
+ # have been interrupted, and observations go stale on a schedule.
+ try:
+ companies, observations, run_id = await run_profile_stage(
+ state, limit=profile_limit)
+ report.profiled_companies = companies
+ report.profile_observations = observations
+ report.profile_run_id = run_id
+ except Exception as e:
+ logger.exception("profile stage failed")
+ report.errors.append(f"profile: {e}")
+
+ return report
diff --git a/harvey/state.py b/harvey/state.py
index 5f495c7..ce234e4 100644
--- a/harvey/state.py
+++ b/harvey/state.py
@@ -1062,20 +1062,45 @@ class StateManager:
async with db.execute(sql, params) as cursor:
return [dict(r) for r in await cursor.fetchall()]
+ # A boolean signal is recorded either way: "checked, not running ads" is a
+ # real finding, and a later flip from 0 to 1 is a real event. But a COHORT
+ # asks who *has* the signal, so it must read the value, not merely the
+ # presence of a row. Without this, "companies running Google Ads" silently
+ # means "companies we checked for Google Ads" — which is everyone.
+ #
+ # `latest` is the current view of the world (newest observation per company
+ # and signal, with history still on disk underneath). `positive` is the
+ # subset where the finding is actually true — value_num IS NULL covers text
+ # signals like INCUMBENT_AGENCY, where the row's existence IS the finding.
+ _CURRENT_CTE = """
+ WITH latest AS (
+ SELECT o.* FROM observations o
+ JOIN (SELECT company_id, signal_code, MAX(observed_at) AS t
+ FROM observations GROUP BY company_id, signal_code) newest
+ ON newest.company_id = o.company_id
+ AND newest.signal_code = o.signal_code
+ AND newest.t = o.observed_at
+ ),
+ positive AS (
+ SELECT * FROM latest WHERE value_num IS NULL OR value_num != 0
+ )
+ """
+
async def signal_counts(self) -> list[dict]:
- """How many entities carry each signal — the cohort sizes."""
+ """How many entities actually carry each signal — the cohort sizes."""
async with self._connect() as db:
db.row_factory = aiosqlite.Row
async with db.execute(
- """SELECT o.signal_code,
- COALESCE(sc.label, o.signal_code) AS label,
- sc.status AS status,
- COUNT(DISTINCT o.company_id) AS companies,
- COUNT(*) AS observations
- FROM observations o
- LEFT JOIN signal_codes sc ON sc.code = o.signal_code
- GROUP BY o.signal_code
- ORDER BY companies DESC"""
+ self._CURRENT_CTE + """
+ SELECT p.signal_code,
+ COALESCE(sc.label, p.signal_code) AS label,
+ sc.status AS status,
+ COUNT(DISTINCT p.company_id) AS companies,
+ COUNT(*) AS observations
+ FROM positive p
+ LEFT JOIN signal_codes sc ON sc.code = p.signal_code
+ GROUP BY p.signal_code
+ ORDER BY companies DESC"""
) as cursor:
return [dict(r) for r in await cursor.fetchall()]
@@ -1094,8 +1119,8 @@ class StateManager:
if not require:
return []
req_ph = ",".join("?" for _ in require)
- sql = (
- "SELECT company_id FROM observations "
+ sql = self._CURRENT_CTE + (
+ f"SELECT company_id FROM positive "
f"WHERE signal_code IN ({req_ph}) AND company_id != '' "
"AND confidence >= ? "
"GROUP BY company_id HAVING COUNT(DISTINCT signal_code) = ?"
@@ -1105,7 +1130,7 @@ class StateManager:
exc_ph = ",".join("?" for _ in exclude)
sql += (
" AND company_id NOT IN ("
- f"SELECT company_id FROM observations WHERE signal_code IN ({exc_ph})"
+ f"SELECT company_id FROM positive WHERE signal_code IN ({exc_ph})"
")"
)
params.extend(exclude)
@@ -1115,6 +1140,54 @@ class StateManager:
async with db.execute(sql, params) as cursor:
return [row[0] for row in await cursor.fetchall()]
+ async def companies_needing_profile(
+ self, limit: int = 100, stale_days: int = 90
+ ) -> list[dict]:
+ """Companies a profile run should visit next.
+
+ Never profiled, or last profiled longer ago than ``stale_days``. The
+ staleness window is what makes re-observation a time series rather
+ than a duplicate: "they dropped their agency last quarter" is only
+ visible if you look again.
+
+ Businesses with no domain are excluded — there is no site to read.
+ They are not a failure, they are the NO_WEBSITE cohort.
+ """
+ async with self._connect() as db:
+ db.row_factory = aiosqlite.Row
+ async with db.execute(
+ """SELECT c.id, c.name, c.domain
+ FROM companies c
+ LEFT JOIN (
+ SELECT company_id, MAX(observed_at) AS last_seen
+ FROM observations WHERE collector = 'profile'
+ GROUP BY company_id
+ ) p ON p.company_id = c.id
+ WHERE c.domain != ''
+ AND (p.last_seen IS NULL
+ OR p.last_seen < datetime('now', ?))
+ ORDER BY c.created_at DESC
+ LIMIT ?""",
+ (f"-{int(stale_days)} days", int(limit)),
+ ) as cursor:
+ return [dict(r) for r in await cursor.fetchall()]
+
+ async def count_companies_needing_profile(self, stale_days: int = 90) -> int:
+ async with self._connect() as db:
+ async with db.execute(
+ """SELECT COUNT(*) FROM companies c
+ LEFT JOIN (
+ SELECT company_id, MAX(observed_at) AS last_seen
+ FROM observations WHERE collector = 'profile'
+ GROUP BY company_id
+ ) p ON p.company_id = c.id
+ WHERE c.domain != ''
+ AND (p.last_seen IS NULL OR p.last_seen < datetime('now', ?))""",
+ (f"-{int(stale_days)} days",),
+ ) as cursor:
+ row = await cursor.fetchone()
+ return row[0] if row else 0
+
# ── Run log (what ran, when, what it produced and cost) ──
async def start_run(
@@ -1644,4 +1717,8 @@ class StateManager:
"active_campaigns": active_campaigns,
"open_conversations": open_conversations,
"usage_today": usage_today,
+ # Companies discovered but not yet read. The heartbeat profiles
+ # these every cycle — it is free, so it never competes with the
+ # Claude budget the rest of the loop is rationing.
+ "unprofiled": await self.count_companies_needing_profile(),
}
diff --git a/tests/test_discover.py b/tests/test_discover.py
index 616bfb7..d1344d8 100644
--- a/tests/test_discover.py
+++ b/tests/test_discover.py
@@ -499,10 +499,10 @@ async def test_overpass_throttling_is_an_error_not_an_empty_market(monkeypatch):
"""A silent [] on a 504 is indistinguishable from 'no businesses here',
which is how a throttled run gets mistaken for a clean one."""
monkeypatch.setattr(D.OpenStreetMap, "BACKOFF", 0.0)
- calls = {"n": 0}
+ seen = []
def handler(request):
- calls["n"] += 1
+ seen.append(str(request.url))
return httpx.Response(504, text="")
provider = D.OpenStreetMap()
@@ -510,11 +510,11 @@ async def test_overpass_throttling_is_an_error_not_an_empty_market(monkeypatch):
async with _client(handler) as client:
await provider.fetch(client, {}, D.DiscoveryQuery(
term="roofer", coordinate="39.7,-104.9,40"))
- assert calls["n"] == 2, "it should retry once before giving up"
+ assert seen == list(D.OpenStreetMap.ENDPOINTS), "every mirror should be tried"
@pytest.mark.asyncio
-async def test_overpass_recovers_on_the_retry(monkeypatch):
+async def test_overpass_falls_over_to_the_next_mirror(monkeypatch):
monkeypatch.setattr(D.OpenStreetMap, "BACKOFF", 0.0)
calls = {"n": 0}
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
new file mode 100644
index 0000000..0a6cc0d
--- /dev/null
+++ b/tests/test_pipeline.py
@@ -0,0 +1,289 @@
+"""DISCOVER → PROFILE chaining.
+
+The stages are joined by a query, not a hand-off. These tests pin that
+property down, because it is what makes a crashed run resumable and
+re-observation a time series instead of a duplicate.
+"""
+
+import os
+import tempfile
+
+import pytest
+import pytest_asyncio
+
+from harvey import pipeline as P
+from harvey.collectors import discover as D
+from harvey.config import load_config
+from harvey.models import Company
+from harvey.signals import SIGNAL_CATALOG, seed_signal_catalog
+from harvey.state import StateManager
+
+
+@pytest_asyncio.fixture
+async def state():
+ with tempfile.TemporaryDirectory() as tmp:
+ sm = StateManager(os.path.join(tmp, "test.db"))
+ await sm.init_db()
+ await seed_signal_catalog(sm)
+ for sig in SIGNAL_CATALOG:
+ await sm.set_signal_status(sig["code"], "confirmed")
+ yield sm
+
+
+def config():
+ return load_config()
+
+
+# ── The join: which companies still need a look ──
+
+
+async def test_a_new_company_needs_profiling(state):
+ await state.add_company(Company(name="Baker", domain="baker.com"))
+ pending = await state.companies_needing_profile()
+ assert [c["domain"] for c in pending] == ["baker.com"]
+ assert await state.count_companies_needing_profile() == 1
+
+
+async def test_a_profiled_company_drops_out(state):
+ cid = await state.add_company(Company(name="Baker", domain="baker.com"))
+ await state.add_observation("TECH_STACK", company_id=cid,
+ collector="profile", value_text="wordpress")
+ assert await state.count_companies_needing_profile() == 0
+
+
+async def test_a_stale_profile_comes_back_round(state):
+ """Re-reading on a schedule is what makes 'they dropped their agency'
+ visible. A company profiled last year is not done forever."""
+ cid = await state.add_company(Company(name="Baker", domain="baker.com"))
+ await state.add_observation("TECH_STACK", company_id=cid, collector="profile",
+ value_text="wordpress",
+ observed_at="2020-01-01 00:00:00")
+ assert await state.count_companies_needing_profile(stale_days=90) == 1
+ assert await state.count_companies_needing_profile(stale_days=99999) == 0
+
+
+async def test_discovery_observations_do_not_count_as_a_profile(state):
+ """DISCOVER writes observations too. Only PROFILE's own count."""
+ cid = await state.add_company(Company(name="Baker", domain="baker.com"))
+ await state.add_observation("SERP_RANK", company_id=cid,
+ collector="discover", value_num=14)
+ assert await state.count_companies_needing_profile() == 1
+
+
+async def test_websiteless_companies_are_never_queued_for_profiling(state):
+ """No site to read. That is the NO_WEBSITE cohort, not a backlog."""
+ await state.add_company(Company(name="No Site", external_id="osm:node/1"))
+ assert await state.count_companies_needing_profile() == 0
+
+
+# ── The chain ──
+
+
+class FakeDiscovery(D.DiscoveryProvider):
+ key = "fake"
+ label = "Fake"
+
+ def __init__(self, businesses):
+ self.businesses = businesses
+
+ def configured(self, env):
+ return True
+
+ def estimate(self, queries):
+ return 0.0
+
+ async def fetch(self, client, env, query):
+ return self.businesses
+
+
+def chain(monkeypatch):
+ """A fake discovery provider and a profile stage that records its input."""
+ seen = {"profiled": []}
+
+ def install(businesses):
+ monkeypatch.setitem(D.PROVIDERS, "fake", FakeDiscovery(businesses))
+
+ async def fake_profile(state, companies):
+ seen["profiled"] = [c["domain"] for c in companies]
+ for company in companies:
+ await state.add_observation(
+ "TECH_STACK", company_id=company["id"],
+ collector="profile", value_text="wordpress")
+ return len(companies), "run-x"
+
+ monkeypatch.setattr("harvey.collectors.profile.profile_companies",
+ fake_profile)
+ return seen
+
+ async def offline(state, city, radius_km=40, client=None):
+ return ""
+ monkeypatch.setattr(D, "geocode_city", offline)
+ return install
+
+
+async def test_discovery_flows_straight_into_profiling(state, config, chain):
+ seen = chain([
+ D.Business(name="Baker", domain="baker.com", provider="fake"),
+ D.Business(name="Metro", domain="metro.com", provider="fake"),
+ ])
+ report = await P.run_prospecting(
+ state, config, "fake", [D.DiscoveryQuery(term="roofer")])
+
+ assert report.discover["new_companies"] == 2
+ assert sorted(seen["profiled"]) == ["baker.com", "metro.com"]
+ assert report.profiled_companies == 2
+ assert report.profile_observations == 2
+
+
+async def test_profiling_sweeps_up_what_an_earlier_run_left(state, config, chain):
+ """The stages join through a query, so a run interrupted before profiling
+ is finished by the next one — nothing has to remember anything."""
+ await state.add_company(Company(name="Left Over", domain="leftover.com"))
+ seen = chain([D.Business(name="Baker", domain="baker.com", provider="fake")])
+
+ await P.run_prospecting(state, config, "fake", [D.DiscoveryQuery(term="roofer")])
+ assert sorted(seen["profiled"]) == ["baker.com", "leftover.com"]
+
+
+async def test_profiling_runs_even_when_discovery_finds_nothing(state, config, chain):
+ await state.add_company(Company(name="Left Over", domain="leftover.com"))
+ seen = chain([])
+
+ report = await P.run_prospecting(state, config, "fake",
+ [D.DiscoveryQuery(term="roofer")])
+ assert report.discover["new_companies"] == 0
+ assert seen["profiled"] == ["leftover.com"]
+
+
+async def test_no_profile_stops_at_discovery(state, config, chain):
+ seen = chain([D.Business(name="Baker", domain="baker.com", provider="fake")])
+ report = await P.run_prospecting(state, config, "fake",
+ [D.DiscoveryQuery(term="roofer")],
+ profile=False)
+ assert report.discover["new_companies"] == 1
+ assert seen["profiled"] == []
+ assert report.skipped == "profiling skipped"
+
+
+async def test_a_dry_run_neither_discovers_nor_profiles(state, config, chain):
+ seen = chain([D.Business(name="Baker", domain="baker.com", provider="fake")])
+ report = await P.run_prospecting(state, config, "fake",
+ [D.DiscoveryQuery(term="roofer")],
+ dry_run=True)
+ assert report.skipped == "dry run"
+ assert seen["profiled"] == []
+ assert await state.count_companies_needing_profile() == 0
+
+
+async def test_a_failing_profile_stage_does_not_lose_the_discovery(
+ state, config, monkeypatch, chain):
+ chain([D.Business(name="Baker", domain="baker.com", provider="fake")])
+
+ async def boom(state, companies):
+ raise RuntimeError("network gone")
+ monkeypatch.setattr("harvey.collectors.profile.profile_companies", boom)
+
+ report = await P.run_prospecting(state, config, "fake",
+ [D.DiscoveryQuery(term="roofer")])
+ assert report.discover["new_companies"] == 1, "the companies are still saved"
+ assert any("network gone" in e for e in report.errors)
+
+
+async def test_profile_stage_is_a_noop_with_nothing_to_do(state):
+ companies, observations, run_id = await P.run_profile_stage(state)
+ assert (companies, observations, run_id) == (0, 0, "")
+
+
+async def test_state_summary_reports_the_profiling_backlog(state):
+ """The heartbeat reads this to decide whether to profile this cycle."""
+ await state.add_company(Company(name="Baker", domain="baker.com"))
+ summary = await state.get_state_summary()
+ assert summary["unprofiled"] == 1
+
+
+# ── Cohorts read the value, not just the row ──
+
+
+async def test_a_false_boolean_is_recorded_but_not_in_the_cohort(state):
+ """PROFILE records RUNNING_GOOGLE_ADS=0 for everyone it checks — knowing
+ you looked is worth keeping, and a later 0→1 flip is a real event. But
+ "companies running Google Ads" must not silently mean "companies we
+ checked for Google Ads", which is everyone.
+ """
+ running = await state.add_company(Company(name="Spender", domain="a.com"))
+ checked = await state.add_company(Company(name="Not spending", domain="b.com"))
+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=running,
+ collector="profile", value_num=1)
+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=checked,
+ collector="profile", value_num=0)
+
+ assert await state.cohort(["RUNNING_GOOGLE_ADS"]) == [running]
+
+ counts = {c["signal_code"]: c for c in await state.signal_counts()}
+ assert counts["RUNNING_GOOGLE_ADS"]["companies"] == 1
+
+ # The negative finding is still on disk.
+ rows = await state.get_observations(company_id=checked)
+ assert rows[0]["value_num"] == 0
+
+
+async def test_text_signals_count_by_their_existence(state):
+ """INCUMBENT_AGENCY has no numeric value — the row IS the finding."""
+ cid = await state.add_company(Company(name="Baker", domain="a.com"))
+ await state.add_observation("INCUMBENT_AGENCY", company_id=cid,
+ collector="profile", value_text="Studio3")
+ assert await state.cohort(["INCUMBENT_AGENCY"]) == [cid]
+
+
+async def test_a_cohort_reflects_the_newest_observation(state):
+ """They dropped their agency last quarter. The cohort should say so."""
+ cid = await state.add_company(Company(name="Baker", domain="a.com"))
+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=cid,
+ collector="profile", value_num=1,
+ observed_at="2026-01-01 00:00:00")
+ assert await state.cohort(["RUNNING_GOOGLE_ADS"]) == [cid]
+
+ await state.add_observation("RUNNING_GOOGLE_ADS", company_id=cid,
+ collector="profile", value_num=0,
+ observed_at="2026-06-01 00:00:00")
+ assert await state.cohort(["RUNNING_GOOGLE_ADS"]) == []
+
+ # Both observations survive — that time series is the durable asset.
+ assert len(await state.get_observations(company_id=cid)) == 2
+
+
+async def test_exclusions_also_read_the_value(state):
+ """Excluding "has online booking" must not exclude everyone who was
+ merely checked for it."""
+ a = await state.add_company(Company(name="A", domain="a.com"))
+ b = await state.add_company(Company(name="B", domain="b.com"))
+ for cid, booking in ((a, 1), (b, 0)):
+ await state.add_observation("NO_SCHEMA_MARKUP", company_id=cid,
+ collector="profile", value_num=1)
+ await state.add_observation("NO_ONLINE_BOOKING", company_id=cid,
+ collector="profile", value_num=booking)
+
+ both = await state.cohort(["NO_SCHEMA_MARKUP"])
+ assert sorted(both) == sorted([a, b])
+
+ # Only A actually lacks booking, so excluding it leaves B.
+ assert await state.cohort(["NO_SCHEMA_MARKUP"], exclude=["NO_ONLINE_BOOKING"]) == [b]