Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.
fix: integrate authenticated cohort pagination and complete CSV export
Ethan+78688011f9cc
diff --cc CHANGELOG.mdindex dfe1964,67752a6..b1c5a75--- a/CHANGELOG.md+++ b/CHANGELOG.md@@@ -28,13 -36,6 +36,14 @@@ minor versions can still change behaviogo out. One check per address, cached for 30 days; changing the sendercountry, address source or product description triggers a fresh check. Thereceipt id is logged with each send. Native providers only. Off by default.+- **Cohort builder pagination and CSV export.** The cohort table on the+ Signals tab used to stop at the first 50 matches with no way to see the+ rest. It now pages with Prev / Next ("Showing 51–100 of 147"), and an+ Export CSV button downloads the whole cohort. `POST /api/cohort` accepts+ `offset` and `limit` (default 50, max 200) and orders by name so pages never+ overlap; `GET /api/cohort.csv?require=...&exclude=...` returns every match.- The cohort size is now counted up to 10,000 instead of 1,000.++ Counts and exports include every match; pages use a stable name/id SQL order.++ CSV downloads carry dashboard authorization and escape spreadsheet formulas.## [0.3.0] — 2026-10-13diff --cc harvey/dashboard.pyindex fdbe5cb,b7a1dbd..816c31d--- a/harvey/dashboard.py+++ b/harvey/dashboard.py@@@ -1250,37 -1315,6 +1315,42 @@@ async def set_signals_status(request: Rreturn JSONResponse({"success": False, "message": str(e)}, status_code=500)+COHORT_PAGE_SIZE = 50+COHORT_MAX_PAGE_SIZE = 200- # A cohort bigger than this is a query to narrow, not a list to page through.- COHORT_MAX = 10_000++- async def _cohort_companies(require: list[str], exclude: list[str]) -> list[dict]:- """Every company in the cohort, ordered by name so pages never overlap."""++async def _cohort_companies(require: list[str], exclude: list[str],++ offset: int = 0, limit: int | None = None):+ state = _state()+ await state.init_db()- ids = await state.cohort(require, exclude, limit=COHORT_MAX)- rows: list[dict] = []- # Chunked: older SQLite builds cap a statement at 999 bound variables.- for i in range(0, len(ids), 500):- chunk = ids[i:i + 500]- rows += await query_db(- "SELECT id, name, domain, website, industry, location FROM companies "- f"WHERE id IN ({','.join('?' for _ in chunk)})",- tuple(chunk),- )- rows.sort(key=lambda r: ((r.get("name") or "").lower(), r["id"]))- return rows++ return await state.cohort_companies(require, exclude, offset, limit)+++def _codes(raw) -> list[str]:- """Signal codes from a JSON list or a comma-separated query string."""++ """Accept lists or CSV strings, with bounded SQL parameter counts."""++ if raw is None:++ return []+ if isinstance(raw, str):+ raw = raw.split(",")- return [c.strip() for c in (raw or []) if c and c.strip()]++ if not isinstance(raw, list) or any(not isinstance(c, str) for c in raw):++ raise ValueError("signals must be a list of strings or comma-separated string")++ codes = list(dict.fromkeys(c.strip() for c in raw if c.strip()))++ if len(codes) > 200 or any(len(c) > 128 for c in codes):++ raise ValueError("too many or oversized signal codes")++ return codes++++++def _cohort_integer(raw, default):++ if raw is None:++ return default++ if isinstance(raw, bool) or not isinstance(raw, (int, str)):++ raise ValueError("offset and limit must be integers")++ value = int(raw)++ if abs(value) > 9223372036854775807:++ raise ValueError("offset or limit is too large")++ return value++@app.post("/api/cohort")async def preview_cohort(request: Request):"""How many companies carry ALL these signals and none of those.@@@ -1292,45 -1325,28 +1362,62 @@@"""try:body = await request.json()- require = [c for c in (body.get("require") or []) if c]- exclude = [c for c in (body.get("exclude") or []) if c]++ if not isinstance(body, dict):++ raise ValueError("request body must be an object")+ require = _codes(body.get("require"))+ exclude = _codes(body.get("exclude"))- offset = max(0, int(body.get("offset") or 0))- limit = min(max(1, int(body.get("limit") or COHORT_PAGE_SIZE)),++ offset = max(0, _cohort_integer(body.get("offset"), 0))++ limit = min(max(1, _cohort_integer(body.get("limit"), COHORT_PAGE_SIZE)),+ COHORT_MAX_PAGE_SIZE)if not require:- return {"size": 0, "companies": []}+ return {"size": 0, "offset": 0, "limit": limit, "companies": []}- rows = await _cohort_companies(require, exclude)- return {"size": len(rows), "offset": offset, "limit": limit,- "companies": rows[offset:offset + limit]}- state = _state()- await state.init_db()- ids = await state.cohort(require, exclude, limit=1000)- if not ids:- return {"size": 0, "companies": []}-- placeholders = ",".join("?" for _ in ids[:200])- rows = await query_db(- f"SELECT id, name, domain, industry, location FROM companies "- f"WHERE id IN ({placeholders})",- tuple(ids[:200]),- )- return {"size": len(ids), "companies": rows}++ size, rows = await _cohort_companies(require, exclude, offset, limit)++ return {"size": size, "offset": offset, "limit": limit,++ "companies": rows}++ except (ValueError, TypeError) as e:++ return JSONResponse({"error": str(e)}, status_code=400)except Exception as e:return JSONResponse({"size": 0, "companies": [], "error": str(e)}, status_code=500)++def _csv_cell(value):++ text = str(value or "")++ # Spreadsheet software may ignore leading whitespace before formulas.++ if text.lstrip().startswith(("=", "+", "-", "@")) or text.startswith(("\t", "\r", "\n")):++ return "'" + text++ return text++++[email protected]("/api/cohort.csv")+async def export_cohort(require: str = "", exclude: str = ""):+ """The whole cohort as a CSV download — every match, not just one page."""+ import csv+ import io++ buf = io.StringIO()+ writer = csv.writer(buf)+ writer.writerow(["name", "domain", "website", "industry", "location"])+ try:+ codes = _codes(require)++ excluded = _codes(exclude)+ if codes:- for r in await _cohort_companies(codes, _codes(exclude)):- writer.writerow([r.get(k) or "" for k in++ _, rows = await _cohort_companies(codes, excluded)++ for r in rows:++ writer.writerow([_csv_cell(r.get(k)) for k in+ ("name", "domain", "website", "industry", "location")])++ except ValueError as e:++ return JSONResponse({"error": str(e)}, status_code=400)+ except Exception as e:+ logger.warning("Cohort export failed: %s", e)++ return JSONResponse({"error": "Cohort export failed"}, status_code=500)+ return PlainTextResponse(+ buf.getvalue(),+ media_type="text/csv",+ headers={"Content-Disposition": 'attachment; filename="cohort.csv"'},+ )++@app.get("/api/runs")async def get_runs_api():"""The collector run log — what ran, when, what it produced and cost."""diff --cc harvey/state.pyindex a82b4c6,347b14a..8158211--- a/harvey/state.py+++ b/harvey/state.py@@@ -1199,6 -1199,6 +1199,39 @@@ class StateManagerasync with db.execute(sql, params) as cursor:return [row[0] for row in await cursor.fetchall()]++ async def cohort_companies(self, require: list[str], exclude: list[str],++ offset: int = 0, limit: int | None = None) -> tuple[int, list[dict]]:++ """Count all matches and page company details in one read snapshot."""++ if not require:++ return 0, []++ require = list(dict.fromkeys(require))++ exclude = list(dict.fromkeys(exclude))++ matches = (++ "SELECT company_id FROM positive WHERE company_id != '' AND "++ f"signal_code IN ({','.join('?' for _ in require)}) "++ "AND confidence >= 0 GROUP BY company_id "++ "HAVING COUNT(DISTINCT signal_code) = ?"++ )++ params = [*require, len(require)]++ if exclude:++ matches += (" AND company_id NOT IN (SELECT company_id FROM positive "++ f"WHERE signal_code IN ({','.join('?' for _ in exclude)}))")++ params.extend(exclude)++ base = self._CURRENT_CTE + " SELECT {columns} FROM companies WHERE id IN (" + matches + ")"++ async with self._connect() as db:++ db.row_factory = aiosqlite.Row++ await db.execute("BEGIN")++ async with db.execute(base.format(columns="COUNT(*)"), params) as cursor:++ size = (await cursor.fetchone())[0]++ sql = base.format(columns="id, name, domain, website, industry, location")++ sql += " ORDER BY LOWER(COALESCE(name, '')), id"++ page_params = list(params)++ if limit is not None:++ sql += " LIMIT ? OFFSET ?"++ page_params.extend([limit, offset])++ async with db.execute(sql, page_params) as cursor:++ return size, [dict(row) for row in await cursor.fetchall()]++async def companies_needing_profile(self, limit: int = 100, stale_days: int = 90) -> list[dict]:diff --cc harvey/web/app.jsindex e1c5c80,68df63e..591914a--- a/harvey/web/app.js+++ b/harvey/web/app.js@@@ -582,8 -667,6 +667,9 @@@ function renderDiscoverResult(report, r// ── Cohort builder: a prospect list is a query ──let _cohort = { require: new Set(), exclude: new Set() };+let _cohortOffset = 0;++let _cohortGeneration = 0;+const COHORT_PAGE = 50;function renderCohortBuilder() {const el = document.getElementById('cohort-builder');@@@ -628,17 -710,16 +714,22 @@@ function toggleCohort(key, code, on)}async function runCohort() {++ const generation = ++_cohortGeneration;++ const offset = _cohortOffset;const el = document.getElementById('cohort-result');if (!el) return;const require = [..._cohort.require], exclude = [..._cohort.exclude];if (!require.length) { el.innerHTML = ''; return; }++ // Remove the prior export button as soon as its criteria become stale.++ el.innerHTML = '<p class="muted">Loading cohort…</p>';const data = await api('/api/cohort', {method: 'POST',headers: {'Content-Type': 'application/json'},- body: JSON.stringify({require: require, exclude: exclude}),+ body: JSON.stringify({require: require, exclude: exclude,- offset: _cohortOffset, limit: COHORT_PAGE}),++ offset: offset, limit: COHORT_PAGE}),});++ if (generation !== _cohortGeneration) return;if (!data) { el.innerHTML = ''; return; }let html = '<div style="border-top:1px solid var(--border);margin-top:20px;padding-top:20px">' +@@@ -657,19 -736,9 +746,19 @@@'<td class="muted">' + escHtml(c.industry) + '</td>' +'<td class="muted">' + escHtml(c.location) + '</td></tr>').join('') + '</tbody></table></div>';- if (data.size > 50) {- html += '<p class="muted" style="font-size:12px;margin-top:10px">Showing the first 50.</p>';- }++ const first = data.offset + 1, last = data.offset + data.companies.length;+ const hasPrev = data.offset > 0, hasNext = last < data.size;+ html += '<div style="display:flex;align-items:center;gap:8px;margin-top:10px">' ++ '<span class="muted" style="font-size:12px">Showing ' + first + '–' + last ++ ' of ' + data.size + '</span>' ++ '<span style="flex:1"></span>' ++ '<button class="btn btn-secondary btn-sm" onclick="pageCohort(-1)"' ++ (hasPrev ? '' : ' disabled') + '>Prev</button>' ++ '<button class="btn btn-secondary btn-sm" onclick="pageCohort(1)"' ++ (hasNext ? '' : ' disabled') + '>Next</button>' +- '<a class="btn btn-secondary btn-sm" href="/api/cohort.csv?' + qs + '" download>Export CSV</a>' +++ '<button class="btn btn-secondary btn-sm" onclick="exportCohort(this)">Export CSV</button>' ++ '</div>';} else if (data.size === 0) {html += '<p class="muted" style="font-size:13px;margin-top:12px">' +'No company carries all of those yet. Either loosen the cohort, or run ' +@@@ -682,11 -747,6 +771,39 @@@el.innerHTML = html + '</div>';}++async function exportCohort(button) {++ const qs = new URLSearchParams({require: [..._cohort.require].join(','),++ exclude: [..._cohort.exclude].join(',')});++ button.disabled = true;++ button.textContent = 'Exporting…';++ let url;++ let link;++ try {++ const response = await authFetch('/api/cohort.csv?' + qs);++ if (await lockIfRefused(response)) throw new Error('Dashboard authorization required');++ if (!response.ok) throw new Error('Export failed (HTTP ' + response.status + ')');++ url = URL.createObjectURL(await response.blob());++ link = document.createElement('a');++ link.href = url;++ link.download = 'cohort.csv';++ document.body.appendChild(link);++ link.click();++ button.textContent = 'Export CSV';++ } catch (error) {++ button.textContent = 'Export failed — retry';++ } finally {++ if (link) link.remove();++ // Give the browser time to start consuming the download before revoking.++ if (url) setTimeout(() => URL.revokeObjectURL(url), 1000);++ button.disabled = false;++ }++}+++function pageCohort(dir) {+ _cohortOffset = Math.max(0, _cohortOffset + dir * COHORT_PAGE);+ runCohort();+}+// ── Settings ──async function loadSettings() {diff --cc tests/test_dashboard_signals.pyindex e44990b,dc86a09..bea647c--- a/tests/test_dashboard_signals.py+++ b/tests/test_dashboard_signals.py@@@ -310,3 -258,3 +315,191 @@@ def test_every_signal_has_a_category_thassert sig["category"] in CATEGORY_ORDER, (f"{sig['code']} is in category {sig['category']!r}, which the ""dashboard would silently drop")++++[email protected]('body', [[], None, {'require': [7]}, {'require': {}},++ {'offset': 'bad'}, {'offset': 1.5}, {'limit': True}, {'offset': 2**64},++ {'require': ['x'] * 201 + ['y' * 129]}])++def test_cohort_invalid_inputs_are_client_errors(client, body):++ response = client.post('/api/cohort', json=body) if body is not None else client.post(++ '/api/cohort', content='null', headers={'Content-Type': 'application/json'})++ assert response.status_code == 400++++++def test_cohort_boundaries_and_duplicate_codes(client):++ import asyncio++ asyncio.run(_seed_large_cohort(client.db_path, 3))++ response = client.post('/api/cohort', json={++ 'require': ['RUNNING_GOOGLE_ADS', 'RUNNING_GOOGLE_ADS'], 'offset': -1, 'limit': 0})++ assert response.status_code == 200++ assert response.json()['size'] == 3++ assert response.json()['offset'] == 0++ assert response.json()['limit'] == 1++ assert len(response.json()['companies']) == 1++ assert client.post('/api/cohort', json={++ 'require': ['RUNNING_GOOGLE_ADS'], 'offset': 4}).json()['companies'] == []++++++def test_cohort_over_previous_cap_and_stable_ties(client):++ import asyncio++ import csv++ import io++ import sqlite3++ from contextlib import closing++ async def initialize():++ state = StateManager(str(client.db_path))++ await state.init_db()++ from harvey.signals import seed_signal_catalog++ await seed_signal_catalog(state)++ asyncio.run(initialize())++ # Bulk seed rather than opening 10,001 individual connections.++ with closing(sqlite3.connect(client.db_path)) as db, db:++ db.executemany('INSERT INTO companies (id, name) VALUES (?, ?)',++ [(f'{i:05}', 'Same') for i in range(10001)])++ db.executemany('INSERT INTO observations (id, company_id, signal_code, value_num) '++ 'VALUES (?, ?, ?, 1)',++ [(f'obs{i}', f'{i:05}', 'RUNNING_GOOGLE_ADS') for i in range(10001)])++ page = client.post('/api/cohort', json={++ 'require': ['RUNNING_GOOGLE_ADS'], 'offset': 9999, 'limit': 2}).json()++ assert page['size'] == 10001++ assert [row['id'] for row in page['companies']] == ['09999', '10000']++ response = client.get('/api/cohort.csv', params={'require': 'RUNNING_GOOGLE_ADS'})++ assert response.status_code == 200++ assert len(list(csv.reader(io.StringIO(response.text)))) == 10002++++++def test_cohort_csv_failure_is_not_success(client, monkeypatch):++ async def fail(*args, **kwargs):++ raise RuntimeError('database unavailable')++ monkeypatch.setattr(dash, '_cohort_companies', fail)++ response = client.get('/api/cohort.csv', params={'require': 'RUNNING_GOOGLE_ADS'})++ assert response.status_code == 500++ assert 'text/csv' not in response.headers['content-type']++ assert client.get('/api/cohort.csv').status_code == 200++ assert client.post('/api/cohort', json={'require': ['RUNNING_GOOGLE_ADS']}).status_code == 500++++++def test_cohort_csv_preserves_text_and_neutralizes_formulas(client, monkeypatch):++ import csv++ import io++ async def rows(*args, **kwargs):++ return 1, [dict(name=' =HYPERLINK("evil")', domain='é, "quoted"',++ website='\t=1+1', industry='@SUM(1)', location='line\nnext')]++ monkeypatch.setattr(dash, '_cohort_companies', rows)++ response = client.get('/api/cohort.csv', params={'require': 'RUNNING_GOOGLE_ADS'})++ cells = list(csv.reader(io.StringIO(response.text)))[1]++ assert cells == ['\' =HYPERLINK("evil")', 'é, "quoted"', "'\t=1+1", "'@SUM(1)", 'line\nnext']++++++def test_cohort_csv_requires_authorization(client):++ response = client.get('/api/cohort.csv', headers={'Authorization': ''})++ assert response.status_code == 401++++++def test_cohort_browser_export_uses_auth_and_handles_failures():++ import json++ import shutil++ import subprocess++ node = shutil.which('node')++ if not node:++ pytest.skip('node unavailable')++ web = Path(__file__).resolve().parent.parent / 'harvey' / 'web'++ script = r'''++const fs = require('fs'), vm = require('vm');++const calls = [], revoked = [], links = [];++global.window = {location: {origin: 'http://127.0.0.1:5555', hash: ''}};++global.sessionStorage = {getItem: () => 'T'.repeat(43)};++global.document = {body: {appendChild: x => links.push(x)}, createElement: () =>++ ({click() {this.clicked = true;}, remove() {this.removed = true;}})};++global.setTimeout = fn => fn();++const locks = [];++global.showLock = reason => locks.push(reason);++global.hideLock = () => {};++URL.createObjectURL = () => 'blob:local';++URL.revokeObjectURL = url => revoked.push(url);++let ok = true, status = 200;++global.fetch = async (url, opts) => {calls.push({url, auth: opts.headers.get('Authorization'),++ redirect: opts.redirect}); return {ok, status, blob: async () => ({})};};++vm.runInThisContext(fs.readFileSync(process.argv[1], 'utf8'));++const app = fs.readFileSync(process.argv[2], 'utf8');++vm.runInThisContext('let _locked = false;' + app.slice(app.indexOf('async function lockIfRefused('), app.indexOf('function showLock(')));++vm.runInThisContext("const _cohort = {require: new Set(['A']), exclude: new Set(['B'])};" +++ app.slice(app.indexOf('async function exportCohort('), app.indexOf('function pageCohort(')));++(async () => {++ const button = {};++ await exportCohort(button);++ const success = {...button};++ ok = false; status = 500;++ await exportCohort(button);++ const failure = {...button};++ status = 401;++ await exportCohort(button);++ console.log(JSON.stringify({calls, revoked, links, success, failure, locks, refused: button}));++})().catch(e => {console.error(e); process.exit(1);});++'''++ result = subprocess.run([node, '-e', script, str(web / 'auth.js'), str(web / 'app.js')],++ text=True, capture_output=True, check=True)++ data = json.loads(result.stdout)++ assert all(c['auth'] == 'Bearer ' + 'T' * 43 and c['redirect'] == 'error' for c in data['calls'])++ assert data['calls'][0]['url'] == 'http://127.0.0.1:5555/api/cohort.csv?require=A&exclude=B'++ assert len(data['links']) == 1++ assert data['links'][0]['clicked'] and data['links'][0]['removed']++ assert data['revoked'] == ['blob:local']++ assert data['success'] == {'disabled': False, 'textContent': 'Export CSV'}++ assert data['failure'] == {'disabled': False, 'textContent': 'Export failed — retry'}++ assert data['refused'] == data['failure']++ assert len(data['locks']) == 1 and 'saved token was not accepted' in data['locks'][0]++++++def test_cohort_csv_invalid_codes_are_client_error(client):++ response = client.get('/api/cohort.csv', params={'require': 'x' * 129})++ assert response.status_code == 400++ response = client.get('/api/cohort.csv', params={'exclude': 'x' * 129})++ assert response.status_code == 400++++++def test_cohort_browser_ignores_superseded_pages_and_cleared_filters():++ import json++ import shutil++ import subprocess++ node = shutil.which('node')++ if not node:++ pytest.skip('node unavailable')++ app = Path(__file__).resolve().parent.parent / 'harvey' / 'web' / 'app.js'++ script = r'''++const fs = require('fs'), vm = require('vm');++const app = fs.readFileSync(process.argv[1], 'utf8');++const el = {innerHTML: 'previous export button'}, pending = [];++global.document = {getElementById: () => el};++global.api = (path, opts) => new Promise(resolve => pending.push({resolve, body: JSON.parse(opts.body)}));++global.escHtml = s => s || '';++vm.runInThisContext("const _cohort = {require: new Set(['A']), exclude: new Set()};" +++ "let _cohortOffset = 0, _cohortGeneration = 0; const COHORT_PAGE = 50;" +++ app.slice(app.indexOf('async function runCohort('), app.indexOf('async function exportCohort(')));++const page = (name, offset) => ({size: 100, offset, companies: [{name}]});++(async () => {++ const first = runCohort();++ const loading = el.innerHTML;++ vm.runInThisContext('_cohortOffset = 50');++ const second = runCohort();++ pending[1].resolve(page('LATEST', 50)); await second;++ const latest = el.innerHTML;++ pending[0].resolve(page('STALE', 0)); await first;++ const afterOld = el.innerHTML;++ const third = runCohort();++ vm.runInThisContext('_cohort.require.clear()');++ await runCohort();++ const cleared = el.innerHTML;++ pending[2].resolve(page('STALE_AFTER_CLEAR', 50)); await third;++ console.log(JSON.stringify({loading, latest, afterOld, cleared, afterClear: el.innerHTML,++ offsets: pending.map(x => x.body.offset)}));++})().catch(e => {console.error(e); process.exit(1);});++'''++ response = subprocess.run([node, '-e', script, str(app)], text=True,++ capture_output=True, check=True)++ data = json.loads(response.stdout)++ assert data['offsets'] == [0, 50, 50]++ assert 'Loading cohort' in data['loading'] and 'Export CSV' not in data['loading']++ assert 'LATEST' in data['latest'] and 'STALE' not in data['latest']++ assert data['afterOld'] == data['latest']++ assert data['cleared'] == data['afterClear'] == ''