Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

Fix dashboard config check and add HTTP/www fallback to site fetching

RIG Team213135f3949f
diff --git a/harvey/agents/scout.py b/harvey/agents/scout.py
index 3da53ea..10d2fe9 100644
--- a/harvey/agents/scout.py
+++ b/harvey/agents/scout.py
@@ -1140,13 +1140,34 @@ Respond ONLY with the JSON array."""
)
return await self.state.add_company(company)
+ async def _resolve_base_url(
+ self, domain: str, retries: int = 1
+ ) -> tuple[str, httpx.Response | None]:
+ """Return (working base url, its response) for the first scheme/www
+ variant that actually answers, or ("", None) if none do.
+
+ A bare `https://{domain}` guess marks plenty of live, real small
+ businesses "unreachable" simply because they never got a TLS cert,
+ or only answer on www — costing us the whole company or team-page
+ scrape for a site that was never actually down.
+ """
+ for base in (
+ f"https://{domain}", f"https://www.{domain}",
+ f"http://{domain}", f"http://www.{domain}",
+ ):
+ resp = await self._fetch(base, timeout=SCRAPE_TIMEOUT, retries=retries)
+ if resp is not None:
+ return base, resp
+ return "", None
+
async def _scrape_company_info(self, domain: str) -> dict:
"""Scrape a company's homepage for basic info. Pure Python."""
info = {"name": "", "description": "", "website": f"https://{domain}"}
- resp = await self._fetch(f"https://{domain}", timeout=SCRAPE_TIMEOUT, retries=2)
+ base, resp = await self._resolve_base_url(domain, retries=2)
if resp is None:
return info
+ info["website"] = base
# Keep the raw HTML so signal detection reuses this fetch.
info["_homepage_html"] = resp.text
@@ -1189,8 +1210,12 @@ Respond ONLY with the JSON array."""
"/leadership", "/about/team", "/company/team", "/staff"]
members = []
+ base, _ = await self._resolve_base_url(domain)
+ if not base:
+ return members
+
for path in team_paths:
- url = f"https://{domain}{path}"
+ url = f"{base}{path}"
resp = await self._fetch(url, timeout=SCRAPE_TIMEOUT, retries=1)
if resp is None:
continue
diff --git a/harvey/collectors/profile.py b/harvey/collectors/profile.py
index 1c2a165..521d218 100644
--- a/harvey/collectors/profile.py
+++ b/harvey/collectors/profile.py
@@ -206,10 +206,26 @@ class ProfileCollector:
return ""
return resp.text
- async def _first_present(self, domain: str, paths) -> tuple[str, str]:
+ async def _resolve_site(self, domain: str) -> tuple[str, str]:
+ """Return (working base url, homepage html), or ("", "") if none work.
+
+ Tries https/http and bare/www variants — a plain https guess marks
+ plenty of live, real small-business sites "unreachable" simply
+ because they never got a TLS cert, or only answer on www.
+ """
+ for base in (
+ f"https://{domain}", f"https://www.{domain}",
+ f"http://{domain}", f"http://www.{domain}",
+ ):
+ html = await self._get(base)
+ if html:
+ return base, html
+ return "", ""
+
+ async def _first_present(self, site: str, paths) -> tuple[str, str]:
"""Return (url, html) for the first path that returns content."""
for path in paths:
- url = f"https://{domain}{path}"
+ url = f"{site}{path}"
html = await self._get(url)
if html and len(html) > 500:
return url, html
@@ -328,7 +344,6 @@ class ProfileCollector:
"""Collect every confirmed profile signal for one business."""
obs: list[dict] = []
domain = domain.lower().replace("www.", "").strip("/")
- site = f"https://{domain}"
def add(code, *, num=None, text="", conf=1.0, url=""):
if not self.wants(code):
@@ -339,7 +354,7 @@ class ProfileCollector:
"evidence_url": url or site,
})
- home = await self._get(site)
+ site, home = await self._resolve_site(domain)
if not home:
# A failure IS an observation — silent failures look like clean results.
logger.info(f"profile: {domain} unreachable")
@@ -402,13 +417,13 @@ class ProfileCollector:
if re.search(r"(?is)<form\b", _strip_code(home)):
add("CONTACT_FORM_URL", text=site, url=site)
else:
- curl, chtml = await self._first_present(domain, CONTACT_PATHS)
+ curl, chtml = await self._first_present(site, CONTACT_PATHS)
if chtml and re.search(r"(?is)<form\b", _strip_code(chtml)):
add("CONTACT_FORM_URL", text=curl, url=curl)
# careers page → hiring intent
if self.wants("HIRING_ROLE"):
- curl, chtml = await self._first_present(domain, CAREERS_PATHS)
+ curl, chtml = await self._first_present(site, CAREERS_PATHS)
if chtml:
roles = []
for m in re.finditer(
@@ -427,7 +442,7 @@ class ProfileCollector:
# team page → people
if self.wants("CONTACT_FOUND") or self.wants("DECISION_MAKER_TITLE"):
- turl, thtml = await self._first_present(domain, TEAM_PATHS)
+ turl, thtml = await self._first_present(site, TEAM_PATHS)
if thtml:
for person in self.detect_people(thtml):
label = f"{person['first_name']} {person['last_name']}"
diff --git a/harvey/dashboard.py b/harvey/dashboard.py
index a29bf95..d0a04c3 100644
--- a/harvey/dashboard.py
+++ b/harvey/dashboard.py
@@ -28,6 +28,18 @@ CONFIG_FILE = PROJECT_ROOT / "harvey.yaml"
PID_FILE = PROJECT_ROOT / "data" / "harvey.pid"
LOG_FILE = PROJECT_ROOT / "data" / "harvey.log"
+
+def _resolve_config_file() -> Path:
+ """harvey.local.yaml wins when present — same resolution as config.py's
+ loader, so the dashboard doesn't judge setup complete against the
+ tracked harvey.yaml template while the real trained config sits in the
+ gitignored local override."""
+ from harvey.config import _find_config_file, ConfigFileNotFoundError
+ try:
+ return Path(_find_config_file())
+ except ConfigFileNotFoundError:
+ return CONFIG_FILE
+
app = FastAPI(title="Harvey Dashboard")
# Harvey process tracking
@@ -164,9 +176,10 @@ async def get_setup_status():
# 5. Config valid
config_valid = False
- if CONFIG_FILE.exists():
+ config_path = _resolve_config_file()
+ if config_path.exists():
try:
- with open(CONFIG_FILE) as f:
+ with open(config_path) as f:
cfg = yaml.safe_load(f)
company = cfg.get("persona", {}).get("company", "")
product = cfg.get("product", {}).get("name", "")
@@ -174,7 +187,7 @@ async def get_setup_status():
except Exception:
pass
checks.append({
- "id": "config", "label": "Harvey configured (harvey.yaml)",
+ "id": "config", "label": f"Harvey configured ({config_path.name})",
"done": config_valid,
"required": True,
"help": "Train Harvey on your product. Use the trainer or set up manually through Claude.",
@@ -224,9 +237,9 @@ async def get_setup_status():
def _current_provider() -> str:
- """Read channels.email.provider from harvey.yaml (best-effort)."""
+ """Read channels.email.provider from the active config (best-effort)."""
try:
- with open(CONFIG_FILE) as f:
+ with open(_resolve_config_file()) as f:
cfg = yaml.safe_load(f) or {}
return ((cfg.get("channels") or {}).get("email") or {}).get("provider", "instantly")
except Exception: