Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

fix: integrate authenticated cohort pagination and complete CSV export

Ethan+78688011f9cc
diff --cc CHANGELOG.md
index dfe1964,67752a6..b1c5a75
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -28,13 -36,6 +36,14 @@@ minor versions can still change behavio
go out. One check per address, cached for 30 days; changing the sender
country, address source or product description triggers a fresh check. The
receipt id is logged with each send. Native providers only. Off by default.
+- **Cohort builder pagination and CSV export.** The cohort table on the
+ Signals tab used to stop at the first 50 matches with no way to see the
+ rest. It now pages with Prev / Next ("Showing 51–100 of 147"), and an
+ Export CSV button downloads the whole cohort. `POST /api/cohort` accepts
+ `offset` and `limit` (default 50, max 200) and orders by name so pages never
+ overlap; `GET /api/cohort.csv?require=...&exclude=...` returns every match.
- The cohort size is now counted up to 10,000 instead of 1,000.
++ Counts and exports include every match; pages use a stable name/id SQL order.
++ CSV downloads carry dashboard authorization and escape spreadsheet formulas.
## [0.3.0] — 2026-10-13
diff --cc harvey/dashboard.py
index fdbe5cb,b7a1dbd..816c31d
--- a/harvey/dashboard.py
+++ b/harvey/dashboard.py
@@@ -1250,37 -1315,6 +1315,42 @@@ async def set_signals_status(request: R
return JSONResponse({"success": False, "message": str(e)}, status_code=500)
+COHORT_PAGE_SIZE = 50
+COHORT_MAX_PAGE_SIZE = 200
- # A cohort bigger than this is a query to narrow, not a list to page through.
- COHORT_MAX = 10_000
+
+
- async def _cohort_companies(require: list[str], exclude: list[str]) -> list[dict]:
- """Every company in the cohort, ordered by name so pages never overlap."""
++async def _cohort_companies(require: list[str], exclude: list[str],
++ offset: int = 0, limit: int | None = None):
+ state = _state()
+ await state.init_db()
- ids = await state.cohort(require, exclude, limit=COHORT_MAX)
- rows: list[dict] = []
- # Chunked: older SQLite builds cap a statement at 999 bound variables.
- for i in range(0, len(ids), 500):
- chunk = ids[i:i + 500]
- rows += await query_db(
- "SELECT id, name, domain, website, industry, location FROM companies "
- f"WHERE id IN ({','.join('?' for _ in chunk)})",
- tuple(chunk),
- )
- rows.sort(key=lambda r: ((r.get("name") or "").lower(), r["id"]))
- return rows
++ return await state.cohort_companies(require, exclude, offset, limit)
+
+
+def _codes(raw) -> list[str]:
- """Signal codes from a JSON list or a comma-separated query string."""
++ """Accept lists or CSV strings, with bounded SQL parameter counts."""
++ if raw is None:
++ return []
+ if isinstance(raw, str):
+ raw = raw.split(",")
- return [c.strip() for c in (raw or []) if c and c.strip()]
++ if not isinstance(raw, list) or any(not isinstance(c, str) for c in raw):
++ raise ValueError("signals must be a list of strings or comma-separated string")
++ codes = list(dict.fromkeys(c.strip() for c in raw if c.strip()))
++ if len(codes) > 200 or any(len(c) > 128 for c in codes):
++ raise ValueError("too many or oversized signal codes")
++ return codes
++
++
++def _cohort_integer(raw, default):
++ if raw is None:
++ return default
++ if isinstance(raw, bool) or not isinstance(raw, (int, str)):
++ raise ValueError("offset and limit must be integers")
++ value = int(raw)
++ if abs(value) > 9223372036854775807:
++ raise ValueError("offset or limit is too large")
++ return value
+
+
@app.post("/api/cohort")
async def preview_cohort(request: Request):
"""How many companies carry ALL these signals and none of those.
@@@ -1292,45 -1325,28 +1362,62 @@@
"""
try:
body = await request.json()
- require = [c for c in (body.get("require") or []) if c]
- exclude = [c for c in (body.get("exclude") or []) if c]
++ if not isinstance(body, dict):
++ raise ValueError("request body must be an object")
+ require = _codes(body.get("require"))
+ exclude = _codes(body.get("exclude"))
- offset = max(0, int(body.get("offset") or 0))
- limit = min(max(1, int(body.get("limit") or COHORT_PAGE_SIZE)),
++ offset = max(0, _cohort_integer(body.get("offset"), 0))
++ limit = min(max(1, _cohort_integer(body.get("limit"), COHORT_PAGE_SIZE)),
+ COHORT_MAX_PAGE_SIZE)
if not require:
- return {"size": 0, "companies": []}
+ return {"size": 0, "offset": 0, "limit": limit, "companies": []}
- rows = await _cohort_companies(require, exclude)
- return {"size": len(rows), "offset": offset, "limit": limit,
- "companies": rows[offset:offset + limit]}
- state = _state()
- await state.init_db()
- ids = await state.cohort(require, exclude, limit=1000)
- if not ids:
- return {"size": 0, "companies": []}
-
- placeholders = ",".join("?" for _ in ids[:200])
- rows = await query_db(
- f"SELECT id, name, domain, industry, location FROM companies "
- f"WHERE id IN ({placeholders})",
- tuple(ids[:200]),
- )
- return {"size": len(ids), "companies": rows}
++ size, rows = await _cohort_companies(require, exclude, offset, limit)
++ return {"size": size, "offset": offset, "limit": limit,
++ "companies": rows}
++ except (ValueError, TypeError) as e:
++ return JSONResponse({"error": str(e)}, status_code=400)
except Exception as e:
return JSONResponse({"size": 0, "companies": [], "error": str(e)}, status_code=500)
++def _csv_cell(value):
++ text = str(value or "")
++ # Spreadsheet software may ignore leading whitespace before formulas.
++ if text.lstrip().startswith(("=", "+", "-", "@")) or text.startswith(("\t", "\r", "\n")):
++ return "'" + text
++ return text
++
++
[email protected]("/api/cohort.csv")
+async def export_cohort(require: str = "", exclude: str = ""):
+ """The whole cohort as a CSV download — every match, not just one page."""
+ import csv
+ import io
+
+ buf = io.StringIO()
+ writer = csv.writer(buf)
+ writer.writerow(["name", "domain", "website", "industry", "location"])
+ try:
+ codes = _codes(require)
++ excluded = _codes(exclude)
+ if codes:
- for r in await _cohort_companies(codes, _codes(exclude)):
- writer.writerow([r.get(k) or "" for k in
++ _, rows = await _cohort_companies(codes, excluded)
++ for r in rows:
++ writer.writerow([_csv_cell(r.get(k)) for k in
+ ("name", "domain", "website", "industry", "location")])
++ except ValueError as e:
++ return JSONResponse({"error": str(e)}, status_code=400)
+ except Exception as e:
+ logger.warning("Cohort export failed: %s", e)
++ return JSONResponse({"error": "Cohort export failed"}, status_code=500)
+ return PlainTextResponse(
+ buf.getvalue(),
+ media_type="text/csv",
+ headers={"Content-Disposition": 'attachment; filename="cohort.csv"'},
+ )
+
+
@app.get("/api/runs")
async def get_runs_api():
"""The collector run log — what ran, when, what it produced and cost."""
diff --cc harvey/state.py
index a82b4c6,347b14a..8158211
--- a/harvey/state.py
+++ b/harvey/state.py
@@@ -1199,6 -1199,6 +1199,39 @@@ class StateManager
async with db.execute(sql, params) as cursor:
return [row[0] for row in await cursor.fetchall()]
++ async def cohort_companies(self, require: list[str], exclude: list[str],
++ offset: int = 0, limit: int | None = None) -> tuple[int, list[dict]]:
++ """Count all matches and page company details in one read snapshot."""
++ if not require:
++ return 0, []
++ require = list(dict.fromkeys(require))
++ exclude = list(dict.fromkeys(exclude))
++ matches = (
++ "SELECT company_id FROM positive WHERE company_id != '' AND "
++ f"signal_code IN ({','.join('?' for _ in require)}) "
++ "AND confidence >= 0 GROUP BY company_id "
++ "HAVING COUNT(DISTINCT signal_code) = ?"
++ )
++ params = [*require, len(require)]
++ if exclude:
++ matches += (" AND company_id NOT IN (SELECT company_id FROM positive "
++ f"WHERE signal_code IN ({','.join('?' for _ in exclude)}))")
++ params.extend(exclude)
++ base = self._CURRENT_CTE + " SELECT {columns} FROM companies WHERE id IN (" + matches + ")"
++ async with self._connect() as db:
++ db.row_factory = aiosqlite.Row
++ await db.execute("BEGIN")
++ async with db.execute(base.format(columns="COUNT(*)"), params) as cursor:
++ size = (await cursor.fetchone())[0]
++ sql = base.format(columns="id, name, domain, website, industry, location")
++ sql += " ORDER BY LOWER(COALESCE(name, '')), id"
++ page_params = list(params)
++ if limit is not None:
++ sql += " LIMIT ? OFFSET ?"
++ page_params.extend([limit, offset])
++ async with db.execute(sql, page_params) as cursor:
++ return size, [dict(row) for row in await cursor.fetchall()]
++
async def companies_needing_profile(
self, limit: int = 100, stale_days: int = 90
) -> list[dict]:
diff --cc harvey/web/app.js
index e1c5c80,68df63e..591914a
--- a/harvey/web/app.js
+++ b/harvey/web/app.js
@@@ -582,8 -667,6 +667,9 @@@ function renderDiscoverResult(report, r
// ── Cohort builder: a prospect list is a query ──
let _cohort = { require: new Set(), exclude: new Set() };
+let _cohortOffset = 0;
++let _cohortGeneration = 0;
+const COHORT_PAGE = 50;
function renderCohortBuilder() {
const el = document.getElementById('cohort-builder');
@@@ -628,17 -710,16 +714,22 @@@ function toggleCohort(key, code, on)
}
async function runCohort() {
++ const generation = ++_cohortGeneration;
++ const offset = _cohortOffset;
const el = document.getElementById('cohort-result');
if (!el) return;
const require = [..._cohort.require], exclude = [..._cohort.exclude];
if (!require.length) { el.innerHTML = ''; return; }
++ // Remove the prior export button as soon as its criteria become stale.
++ el.innerHTML = '<p class="muted">Loading cohort…</p>';
const data = await api('/api/cohort', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
- body: JSON.stringify({require: require, exclude: exclude}),
+ body: JSON.stringify({require: require, exclude: exclude,
- offset: _cohortOffset, limit: COHORT_PAGE}),
++ offset: offset, limit: COHORT_PAGE}),
});
++ if (generation !== _cohortGeneration) return;
if (!data) { el.innerHTML = ''; return; }
let html = '<div style="border-top:1px solid var(--border);margin-top:20px;padding-top:20px">' +
@@@ -657,19 -736,9 +746,19 @@@
'<td class="muted">' + escHtml(c.industry) + '</td>' +
'<td class="muted">' + escHtml(c.location) + '</td></tr>'
).join('') + '</tbody></table></div>';
- if (data.size > 50) {
- html += '<p class="muted" style="font-size:12px;margin-top:10px">Showing the first 50.</p>';
- }
+
+ const first = data.offset + 1, last = data.offset + data.companies.length;
+ const hasPrev = data.offset > 0, hasNext = last < data.size;
+ html += '<div style="display:flex;align-items:center;gap:8px;margin-top:10px">' +
+ '<span class="muted" style="font-size:12px">Showing ' + first + '&ndash;' + last +
+ ' of ' + data.size + '</span>' +
+ '<span style="flex:1"></span>' +
+ '<button class="btn btn-secondary btn-sm" onclick="pageCohort(-1)"' +
+ (hasPrev ? '' : ' disabled') + '>Prev</button>' +
+ '<button class="btn btn-secondary btn-sm" onclick="pageCohort(1)"' +
+ (hasNext ? '' : ' disabled') + '>Next</button>' +
- '<a class="btn btn-secondary btn-sm" href="/api/cohort.csv?' + qs + '" download>Export CSV</a>' +
++ '<button class="btn btn-secondary btn-sm" onclick="exportCohort(this)">Export CSV</button>' +
+ '</div>';
} else if (data.size === 0) {
html += '<p class="muted" style="font-size:13px;margin-top:12px">' +
'No company carries all of those yet. Either loosen the cohort, or run ' +
@@@ -682,11 -747,6 +771,39 @@@
el.innerHTML = html + '</div>';
}
++async function exportCohort(button) {
++ const qs = new URLSearchParams({require: [..._cohort.require].join(','),
++ exclude: [..._cohort.exclude].join(',')});
++ button.disabled = true;
++ button.textContent = 'Exporting…';
++ let url;
++ let link;
++ try {
++ const response = await authFetch('/api/cohort.csv?' + qs);
++ if (await lockIfRefused(response)) throw new Error('Dashboard authorization required');
++ if (!response.ok) throw new Error('Export failed (HTTP ' + response.status + ')');
++ url = URL.createObjectURL(await response.blob());
++ link = document.createElement('a');
++ link.href = url;
++ link.download = 'cohort.csv';
++ document.body.appendChild(link);
++ link.click();
++ button.textContent = 'Export CSV';
++ } catch (error) {
++ button.textContent = 'Export failed — retry';
++ } finally {
++ if (link) link.remove();
++ // Give the browser time to start consuming the download before revoking.
++ if (url) setTimeout(() => URL.revokeObjectURL(url), 1000);
++ button.disabled = false;
++ }
++}
++
+function pageCohort(dir) {
+ _cohortOffset = Math.max(0, _cohortOffset + dir * COHORT_PAGE);
+ runCohort();
+}
+
// ── Settings ──
async function loadSettings() {
diff --cc tests/test_dashboard_signals.py
index e44990b,dc86a09..bea647c
--- a/tests/test_dashboard_signals.py
+++ b/tests/test_dashboard_signals.py
@@@ -310,3 -258,3 +315,191 @@@ def test_every_signal_has_a_category_th
assert sig["category"] in CATEGORY_ORDER, (
f"{sig['code']} is in category {sig['category']!r}, which the "
"dashboard would silently drop")
++
++
[email protected]('body', [[], None, {'require': [7]}, {'require': {}},
++ {'offset': 'bad'}, {'offset': 1.5}, {'limit': True}, {'offset': 2**64},
++ {'require': ['x'] * 201 + ['y' * 129]}])
++def test_cohort_invalid_inputs_are_client_errors(client, body):
++ response = client.post('/api/cohort', json=body) if body is not None else client.post(
++ '/api/cohort', content='null', headers={'Content-Type': 'application/json'})
++ assert response.status_code == 400
++
++
++def test_cohort_boundaries_and_duplicate_codes(client):
++ import asyncio
++ asyncio.run(_seed_large_cohort(client.db_path, 3))
++ response = client.post('/api/cohort', json={
++ 'require': ['RUNNING_GOOGLE_ADS', 'RUNNING_GOOGLE_ADS'], 'offset': -1, 'limit': 0})
++ assert response.status_code == 200
++ assert response.json()['size'] == 3
++ assert response.json()['offset'] == 0
++ assert response.json()['limit'] == 1
++ assert len(response.json()['companies']) == 1
++ assert client.post('/api/cohort', json={
++ 'require': ['RUNNING_GOOGLE_ADS'], 'offset': 4}).json()['companies'] == []
++
++
++def test_cohort_over_previous_cap_and_stable_ties(client):
++ import asyncio
++ import csv
++ import io
++ import sqlite3
++ from contextlib import closing
++ async def initialize():
++ state = StateManager(str(client.db_path))
++ await state.init_db()
++ from harvey.signals import seed_signal_catalog
++ await seed_signal_catalog(state)
++ asyncio.run(initialize())
++ # Bulk seed rather than opening 10,001 individual connections.
++ with closing(sqlite3.connect(client.db_path)) as db, db:
++ db.executemany('INSERT INTO companies (id, name) VALUES (?, ?)',
++ [(f'{i:05}', 'Same') for i in range(10001)])
++ db.executemany('INSERT INTO observations (id, company_id, signal_code, value_num) '
++ 'VALUES (?, ?, ?, 1)',
++ [(f'obs{i}', f'{i:05}', 'RUNNING_GOOGLE_ADS') for i in range(10001)])
++ page = client.post('/api/cohort', json={
++ 'require': ['RUNNING_GOOGLE_ADS'], 'offset': 9999, 'limit': 2}).json()
++ assert page['size'] == 10001
++ assert [row['id'] for row in page['companies']] == ['09999', '10000']
++ response = client.get('/api/cohort.csv', params={'require': 'RUNNING_GOOGLE_ADS'})
++ assert response.status_code == 200
++ assert len(list(csv.reader(io.StringIO(response.text)))) == 10002
++
++
++def test_cohort_csv_failure_is_not_success(client, monkeypatch):
++ async def fail(*args, **kwargs):
++ raise RuntimeError('database unavailable')
++ monkeypatch.setattr(dash, '_cohort_companies', fail)
++ response = client.get('/api/cohort.csv', params={'require': 'RUNNING_GOOGLE_ADS'})
++ assert response.status_code == 500
++ assert 'text/csv' not in response.headers['content-type']
++ assert client.get('/api/cohort.csv').status_code == 200
++ assert client.post('/api/cohort', json={'require': ['RUNNING_GOOGLE_ADS']}).status_code == 500
++
++
++def test_cohort_csv_preserves_text_and_neutralizes_formulas(client, monkeypatch):
++ import csv
++ import io
++ async def rows(*args, **kwargs):
++ return 1, [dict(name=' =HYPERLINK("evil")', domain='é, "quoted"',
++ website='\t=1+1', industry='@SUM(1)', location='line\nnext')]
++ monkeypatch.setattr(dash, '_cohort_companies', rows)
++ response = client.get('/api/cohort.csv', params={'require': 'RUNNING_GOOGLE_ADS'})
++ cells = list(csv.reader(io.StringIO(response.text)))[1]
++ assert cells == ['\' =HYPERLINK("evil")', 'é, "quoted"', "'\t=1+1", "'@SUM(1)", 'line\nnext']
++
++
++def test_cohort_csv_requires_authorization(client):
++ response = client.get('/api/cohort.csv', headers={'Authorization': ''})
++ assert response.status_code == 401
++
++
++def test_cohort_browser_export_uses_auth_and_handles_failures():
++ import json
++ import shutil
++ import subprocess
++ node = shutil.which('node')
++ if not node:
++ pytest.skip('node unavailable')
++ web = Path(__file__).resolve().parent.parent / 'harvey' / 'web'
++ script = r'''
++const fs = require('fs'), vm = require('vm');
++const calls = [], revoked = [], links = [];
++global.window = {location: {origin: 'http://127.0.0.1:5555', hash: ''}};
++global.sessionStorage = {getItem: () => 'T'.repeat(43)};
++global.document = {body: {appendChild: x => links.push(x)}, createElement: () =>
++ ({click() {this.clicked = true;}, remove() {this.removed = true;}})};
++global.setTimeout = fn => fn();
++const locks = [];
++global.showLock = reason => locks.push(reason);
++global.hideLock = () => {};
++URL.createObjectURL = () => 'blob:local';
++URL.revokeObjectURL = url => revoked.push(url);
++let ok = true, status = 200;
++global.fetch = async (url, opts) => {calls.push({url, auth: opts.headers.get('Authorization'),
++ redirect: opts.redirect}); return {ok, status, blob: async () => ({})};};
++vm.runInThisContext(fs.readFileSync(process.argv[1], 'utf8'));
++const app = fs.readFileSync(process.argv[2], 'utf8');
++vm.runInThisContext('let _locked = false;' + app.slice(app.indexOf('async function lockIfRefused('), app.indexOf('function showLock(')));
++vm.runInThisContext("const _cohort = {require: new Set(['A']), exclude: new Set(['B'])};" +
++ app.slice(app.indexOf('async function exportCohort('), app.indexOf('function pageCohort(')));
++(async () => {
++ const button = {};
++ await exportCohort(button);
++ const success = {...button};
++ ok = false; status = 500;
++ await exportCohort(button);
++ const failure = {...button};
++ status = 401;
++ await exportCohort(button);
++ console.log(JSON.stringify({calls, revoked, links, success, failure, locks, refused: button}));
++})().catch(e => {console.error(e); process.exit(1);});
++'''
++ result = subprocess.run([node, '-e', script, str(web / 'auth.js'), str(web / 'app.js')],
++ text=True, capture_output=True, check=True)
++ data = json.loads(result.stdout)
++ assert all(c['auth'] == 'Bearer ' + 'T' * 43 and c['redirect'] == 'error' for c in data['calls'])
++ assert data['calls'][0]['url'] == 'http://127.0.0.1:5555/api/cohort.csv?require=A&exclude=B'
++ assert len(data['links']) == 1
++ assert data['links'][0]['clicked'] and data['links'][0]['removed']
++ assert data['revoked'] == ['blob:local']
++ assert data['success'] == {'disabled': False, 'textContent': 'Export CSV'}
++ assert data['failure'] == {'disabled': False, 'textContent': 'Export failed — retry'}
++ assert data['refused'] == data['failure']
++ assert len(data['locks']) == 1 and 'saved token was not accepted' in data['locks'][0]
++
++
++def test_cohort_csv_invalid_codes_are_client_error(client):
++ response = client.get('/api/cohort.csv', params={'require': 'x' * 129})
++ assert response.status_code == 400
++ response = client.get('/api/cohort.csv', params={'exclude': 'x' * 129})
++ assert response.status_code == 400
++
++
++def test_cohort_browser_ignores_superseded_pages_and_cleared_filters():
++ import json
++ import shutil
++ import subprocess
++ node = shutil.which('node')
++ if not node:
++ pytest.skip('node unavailable')
++ app = Path(__file__).resolve().parent.parent / 'harvey' / 'web' / 'app.js'
++ script = r'''
++const fs = require('fs'), vm = require('vm');
++const app = fs.readFileSync(process.argv[1], 'utf8');
++const el = {innerHTML: 'previous export button'}, pending = [];
++global.document = {getElementById: () => el};
++global.api = (path, opts) => new Promise(resolve => pending.push({resolve, body: JSON.parse(opts.body)}));
++global.escHtml = s => s || '';
++vm.runInThisContext("const _cohort = {require: new Set(['A']), exclude: new Set()};" +
++ "let _cohortOffset = 0, _cohortGeneration = 0; const COHORT_PAGE = 50;" +
++ app.slice(app.indexOf('async function runCohort('), app.indexOf('async function exportCohort(')));
++const page = (name, offset) => ({size: 100, offset, companies: [{name}]});
++(async () => {
++ const first = runCohort();
++ const loading = el.innerHTML;
++ vm.runInThisContext('_cohortOffset = 50');
++ const second = runCohort();
++ pending[1].resolve(page('LATEST', 50)); await second;
++ const latest = el.innerHTML;
++ pending[0].resolve(page('STALE', 0)); await first;
++ const afterOld = el.innerHTML;
++ const third = runCohort();
++ vm.runInThisContext('_cohort.require.clear()');
++ await runCohort();
++ const cleared = el.innerHTML;
++ pending[2].resolve(page('STALE_AFTER_CLEAR', 50)); await third;
++ console.log(JSON.stringify({loading, latest, afterOld, cleared, afterClear: el.innerHTML,
++ offsets: pending.map(x => x.body.offset)}));
++})().catch(e => {console.error(e); process.exit(1);});
++'''
++ response = subprocess.run([node, '-e', script, str(app)], text=True,
++ capture_output=True, check=True)
++ data = json.loads(response.stdout)
++ assert data['offsets'] == [0, 50, 50]
++ assert 'Loading cohort' in data['loading'] and 'Export CSV' not in data['loading']
++ assert 'LATEST' in data['latest'] and 'STALE' not in data['latest']
++ assert data['afterOld'] == data['latest']
++ assert data['cleared'] == data['afterClear'] == ''