Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix(research): survive all-skipped results; honour [junk] and [fetch] config (#180)

Jordan Gibbs98d014f29f7d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ef0073c..ce3ed7a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,12 @@
- **Codex bench harness rebuilt (`codex_bench/`).** Each query runs in its own git-initialized project set up by `install --target codex`, with the prompt in `research/prompt.txt` and the inline-citation wrapper contract. The session is one `codex exec` under the workspace-write-plus-network sandbox, with JSONL events captured. `result.json` records whether the pipeline ran: vault note count, manifest steps done and not done, tier, subagent spawns, sandbox rejections, and `run verify`. An inline answer that skipped the vault is reported as `PIPELINE_SKIPPED`. `--dry-run` sets up the run dir and prints the exact `codex exec` command without running it. There are no dollar fields.
- **Run spend counters and count ceilings (#92).** `hpr run step --status done` and `hpr run reconcile` count tagged notes and distinct fetched URLs linked to those notes in the vault DB, including HTML and PDFs. Optional `hpr run init --max-sources` / `--max-notes` ceilings block at step boundaries. `run resume` or `run step --status done --force` can clear a count block up to three times; the same count stays clear until it increases, and a fourth clear rejects the step. Agent-reported `estimated_usd` stays unchanged; `--budget` stays inert unless something calls `hpr run spend`. Real runs get count ceilings only when the entry skill passes them to `run init`.
+### Fixed
+
+- **`research` no longer crashes when every search result is skipped.** If all results were login walls or already in the vault, the closing summary indexed `created_notes[-1]` and raised `IndexError`. It now prints "No notes created". `--json` already returned an empty `notes_created` list and still does.
+- **`research` reads the vault's `[junk]` config.** Its login-wall check used the built-in defaults, so `extra_login_signals` and the other `[junk]` thresholds were ignored. `fetch` and `fetch-batch` already applied them.
+- **`research` builds its provider with the vault's `[fetch]` settings and `[junk]` gates.** It called `get_provider` without `settings` / `gates`, so research fetches ignored settings such as `browser_verify_tls` and `allow_private_hosts`. It now passes them the way `fetch` and `fetch-batch` do.
+
## [0.12.0] - 2026-09-24
- **The repo is a Claude Code plugin marketplace.** `/plugin marketplace add jordan-gibbs/hyperresearch` then `/plugin install hyperresearch@hyperresearch` adds one skill, `deep-research`, at `skills/deep-research/SKILL.md`. It checks for the `hyperresearch` CLI, runs `hyperresearch install` in the current project, and hands off to `/hyperresearch`. It holds no pipeline steps; those still come from the Python package.
diff --git a/src/hyperresearch/cli/research.py b/src/hyperresearch/cli/research.py
index 5956972..33d051c 100644
--- a/src/hyperresearch/cli/research.py
+++ b/src/hyperresearch/cli/research.py
@@ -51,6 +51,8 @@ def research(
provider_name or vault.config.web_provider,
profile=vault.config.web_profile,
magic=vault.config.web_magic,
+ settings=vault.config.fetch,
+ gates=vault.config.junk,
)
# Step 1: Search
@@ -227,6 +229,11 @@ def research(
if json_output:
output(success(data, count=len(created_notes), vault=str(vault.root)), json_mode=True)
+ elif not created_notes:
+ console.print(
+ "[yellow]No notes created:[/] every result was skipped "
+ "(already in the vault, or a login wall)."
+ )
else:
console.print(
f"\n[bold]Done:[/] {len(created_notes)} notes created. "
@@ -241,7 +248,7 @@ def _save_result(vault, conn, prov, result, tags, parent) -> dict | None:
url = result.url
# Skip login redirects
- if result.looks_like_login_wall(url):
+ if result.looks_like_login_wall(url, vault.config.junk):
return None
# Skip if already fetched (an orphaned row — note deleted — is not a duplicate)
diff --git a/tests/test_cli/test_research_cmd.py b/tests/test_cli/test_research_cmd.py
new file mode 100644
index 0000000..230a134
--- /dev/null
+++ b/tests/test_cli/test_research_cmd.py
@@ -0,0 +1,108 @@
+"""`research` command robustness — driven offline with a fake provider, no network."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from hyperresearch.cli import app
+from hyperresearch.web.base import WebResult
+
+runner = CliRunner()
+
+_BODY = "A long enough article body about the research topic. " * 40
+
+
+class _FakeProvider:
+ name = "fake"
+
+ def __init__(self, results):
+ self._results = results
+
+ def search(self, query, max_results=5):
+ return self._results
+
+ def fetch(self, url):
+ raise RuntimeError("no fetch in tests")
+
+
+def vault_dir(tmp_path: Path) -> Path:
+ result = runner.invoke(app, ["init", str(tmp_path / "kb"), "--name", "Research Test"])
+ assert result.exit_code == 0
+ return tmp_path / "kb"
+
+
+def _use_provider(monkeypatch, results) -> list[dict]:
+ calls: list[dict] = []
+
+ def fake_get_provider(*args, **kwargs):
+ calls.append(kwargs)
+ return _FakeProvider(results)
+
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", fake_get_provider)
+ return calls
+
+
+def test_research_all_results_skipped_does_not_crash(vault_dir: Path, monkeypatch):
+ """Every search result filtered out (here: login walls) used to hit
+ created_notes[-1] and raise IndexError in the human-readable summary."""
+ os.chdir(vault_dir)
+ _use_provider(
+ monkeypatch,
+ [WebResult(url="https://example.com/login", title="Sign in", content="Please log in")],
+ )
+
+ result = runner.invoke(app, ["research", "some topic"])
+ assert result.exit_code == 0, result.output
+ assert result.exception is None
+ assert "No notes created" in result.output
+
+ result = runner.invoke(app, ["research", "some topic", "--json"])
+ assert result.exit_code == 0
+ data = json.loads(result.output)
+ assert data["ok"] is True
+ assert data["data"]["notes_created"] == []
+
+
+def test_research_login_wall_honours_vault_junk_config(vault_dir: Path, monkeypatch):
+ """A vault-specific [junk] extra_login_signals entry must make research skip
+ the page, as fetch and fetch-batch already do."""
+ os.chdir(vault_dir)
+ cfg = vault_dir / ".hyperresearch" / "config.toml"
+ existing = cfg.read_text(encoding="utf-8")
+ assert "extra_login_signals = []" in existing
+ cfg.write_text(
+ existing.replace("extra_login_signals = []", 'extra_login_signals = ["mitgliedsbereich"]'),
+ encoding="utf-8",
+ )
+ _use_provider(
+ monkeypatch,
+ [WebResult(url="https://example.de/artikel", title="Mitgliedsbereich", content=_BODY)],
+ )
+
+ result = runner.invoke(app, ["research", "some topic", "--json"])
+ assert result.exit_code == 0, result.output
+ data = json.loads(result.output)
+ assert data["data"]["notes_created"] == []
+
+
+def test_research_threads_fetch_settings_and_gates(vault_dir: Path, monkeypatch):
+ """research must build its provider with the vault's [fetch] settings and
+ [junk] gates (TLS verification, SSRF allowlist, ...), like fetch does."""
+ os.chdir(vault_dir)
+ calls = _use_provider(monkeypatch, [])
+
+ result = runner.invoke(app, ["research", "some topic", "--json"])
+ assert result.exit_code == 0, result.output
+
+ from hyperresearch.core.vault import Vault
+
+ vault = Vault.discover()
+ assert len(calls) == 1
+ assert calls[0]["settings"] == vault.config.fetch
+ assert calls[0]["gates"] == vault.config.junk