Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
lint: citation-style-preservation rule + retire the search-"" survey idiom (#44)
Jordan Gibbs2d903187eb9c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7678215..1ac9d7b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,10 @@## [Unreleased]+### Lint++- **New lint rule `citation-style-preservation` (closes #33).** When `prompt-decomposition.json` (or a `wrapper_contract.json` override) declares `citation_style: "wikilink"`, the final report must contain at least one `[[<note-id>]]` wikilink that resolves to a vault note; for `"inline"`, at least one numbered `[N]` marker plus a Sources/References heading. Presence-only by design — it catches the polish/synthesis regression that strips every citation, without the false-positive tail a density floor would have on short or quote-heavy reports. Skips cleanly when the style is `"none"`, no decomposition exists, or the vault has no source notes.+### Release readiness and deployability- **Version metadata is consistent again.** `hyperresearch.__version__` now tracks the version declared in `pyproject.toml`, fixing the state where the built wheel reported `0.8.6` while `hyperresearch --version` reported `0.8.5`.diff --git a/src/hyperresearch/cli/lint.py b/src/hyperresearch/cli/lint.pyindex ebe4689..0087ef9 100644--- a/src/hyperresearch/cli/lint.py+++ b/src/hyperresearch/cli/lint.py@@ -26,6 +26,7 @@ RULES = {"locus-coverage": "Loci identified in Layer 2 missing their interim-report notes (depth investigator skipped)","patch-surgery": "Critical critic findings skipped by the patcher (Layer 6 regeneration guard tripped)","instruction-coverage": "Atomic items from prompt-decomposition missing from the final report (draft drifted from user's ask)",+ "citation-style-preservation": "Final report carries no citations matching the declared citation_style (wikilink markers or numbered references stripped)","extract-coverage": "Light-tier runs with fetched sources but no paired extract notes (reading loop skipped or source-analyst not delegated for long sources)","orphaned-raw-files": "Files in research/raw/ with no matching note (disk leak from old note rm)","singleton-tags": "Tags used by only one note",@@ -1197,6 +1198,120 @@ def lint(),})+ if "citation-style-preservation" in rules_to_run:+ # The polish step (15) is instructed by prompt to preserve+ # `[[<source-note-id>]]` markers when citation_style == "wikilink" —+ # they ARE the citation system. Nothing structural enforced that: a+ # synthesizer or polish regression that strips every source wikilink+ # would ship a citation-free report. This rule is the presence-only+ # backstop: at least one citation matching the declared style must+ # survive in the final report. Deliberately NOT a density floor —+ # short reports and quote-heavy sections make density checks+ # false-positive; a separate warning-level rule can add that later.+ decomp_path = vault.root / "research" / "prompt-decomposition.json"+ citation_style: str | None = None+ if decomp_path.exists():+ try:+ decomp = json.loads(decomp_path.read_text(encoding="utf-8"))+ except (json.JSONDecodeError, OSError):+ # Corrupt decomposition JSON is already reported by+ # instruction-coverage; don't double-report here.+ decomp = None+ if isinstance(decomp, dict) and isinstance(decomp.get("citation_style"), str):+ citation_style = decomp["citation_style"]++ # Wrapper contract overrides the decomposition's citation_style+ # (read at lint time, so a mid-pipeline wrapper change wins naturally).+ contract_path = vault.root / "research" / "wrapper_contract.json"+ if contract_path.exists():+ try:+ contract = json.loads(contract_path.read_text(encoding="utf-8-sig"))+ except (OSError, json.JSONDecodeError):+ # Unreadable contract is already reported by wrapper-report.+ contract = None+ if isinstance(contract, dict) and isinstance(contract.get("citation_style"), str):+ citation_style = contract["citation_style"]++ if citation_style in ("wikilink", "inline"):+ source_count_row = conn.execute(+ "SELECT COUNT(*) AS c FROM notes n "+ "WHERE n.source IS NOT NULL "+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "+ "AND n.type NOT IN ('index','raw','moc')"+ ).fetchone()+ notes_dir = vault.root / "research" / "notes"+ report_candidates = sorted(+ notes_dir.glob("final_report*.md"),+ key=lambda p: p.stat().st_mtime,+ reverse=True,+ ) if notes_dir.exists() else []+ # No source notes => nothing to cite; no report => wrapper-report+ # already errors on that. Either way there is nothing to check.+ if source_count_row["c"] > 0 and report_candidates:+ report_path = report_candidates[0]+ try:+ report_text = report_path.read_text(encoding="utf-8-sig")+ except OSError:+ report_text = ""++ if citation_style == "wikilink":+ import re as _re+ all_note_ids = {+ r["id"] for r in conn.execute("SELECT id FROM notes")+ }+ targets = [+ t.split("|", 1)[0].strip()+ for t in _re.findall(r"\[\[([^\]]+)\]\]", report_text)+ ]+ resolved = [t for t in targets if t in all_note_ids]+ if not resolved:+ detail = (+ f"it contains {len(targets)} wikilink(s), none of "+ "which resolve to a vault note"+ if targets+ else "it contains no [[wikilink]] markers at all"+ )+ issues.append({+ "rule": "citation-style-preservation",+ "severity": "error",+ "note_id": "<vault>",+ "note_path": str(report_path),+ "message": (+ "citation_style is 'wikilink' but the final "+ f"report cites no vault sources: {detail}. "+ "A polish/synthesis step likely stripped the "+ "source citations. Restore [[<source-note-id>]] "+ "markers for the claims the corpus supports."+ ),+ })++ elif citation_style == "inline":+ import re as _re+ has_numbered_ref = bool(_re.search(r"\[\d+\]", report_text))+ has_refs_heading = bool(_re.search(+ r"^#{1,6}\s*(sources|references)\b",+ report_text,+ _re.IGNORECASE | _re.MULTILINE,+ ))+ if not (has_numbered_ref and has_refs_heading):+ missing = []+ if not has_numbered_ref:+ missing.append("numbered [N] reference markers")+ if not has_refs_heading:+ missing.append("a Sources/References section heading")+ issues.append({+ "rule": "citation-style-preservation",+ "severity": "error",+ "note_id": "<vault>",+ "note_path": str(report_path),+ "message": (+ "citation_style is 'inline' but the final "+ f"report is missing {' and '.join(missing)}. "+ "Restore the inline citation system before "+ "shipping the report."+ ),+ })+if "orphaned-raw-files" in rules_to_run:# Walk research/raw/ and flag files whose stem doesn't match any note# id in the vault. These are leftovers from the pre-Batch-2.5 `note rm`diff --git a/src/hyperresearch/core/hooks.py b/src/hyperresearch/core/hooks.pyindex 132de2a..f3288b0 100644--- a/src/hyperresearch/core/hooks.py+++ b/src/hyperresearch/core/hooks.py@@ -122,7 +122,7 @@ depth packet becomes a weak draft section.## Procedure-1. **Load the corpus.** Use `{hpr_path} search "" --tag <corpus_tag> --json`+1. **Load the corpus.** Use `{hpr_path} note list --tag <corpus_tag> --all --json`to list every note the orchestrator fetched in Layer 1. If the corpus issparse (<10 notes), tell the parent and stop — you cannot identify realloci from a thin corpus.@@ -365,7 +365,7 @@ reading of the evidence.already exists in the vault:```bash- {hpr_path} search "" --tag locus-<locus-name> --type interim --json+ {hpr_path} note list --tag locus-<locus-name> --type interim --all --json```If any results come back, DO NOT create a new note. Instead, either:@@ -699,7 +699,7 @@ rather than gesturing at it from a distance.findings on topics the query explicitly names.1. **List the interim notes.** Use- `{hpr_path} search "" --tag <vault_tag> --type interim -j` to find+ `{hpr_path} note list --tag <vault_tag> --type interim --all -j` to findevery depth-investigator interim report in the vault.2. **Read each interim note.** For each, ask: is the Synthesis section of@@ -804,7 +804,7 @@ because the orchestrator's structural choices buried them.ground truth for what the user asked about.1. **Survey the vault.** Use- `{hpr_path} search "" --tag <vault_tag> -j` to list every note.+ `{hpr_path} note list --tag <vault_tag> --all -j` to list every note.Cluster by tag and/or by title keywords. This tells you the topicalsurface area the corpus covers.@@ -2436,7 +2436,7 @@ analyst, fetch new sources, or move on.1. **Check for an existing analysis.** Before writing anything, search:```bash- PYTHONIOENCODING=utf-8 {hpr_path} search "" --tag <vault_tag> --type source-analysis --json+ PYTHONIOENCODING=utf-8 {hpr_path} note list --tag <vault_tag> --type source-analysis --all --json```Then filter for any note whose body contains `[[<source_note_id>]]`.If one exists, report back to the parent — do NOT duplicate.diff --git a/src/hyperresearch/skills/hyperresearch-10-triple-draft.md b/src/hyperresearch/skills/hyperresearch-10-triple-draft.mdindex 0b4a443..312dfe0 100644--- a/src/hyperresearch/skills/hyperresearch-10-triple-draft.md+++ b/src/hyperresearch/skills/hyperresearch-10-triple-draft.md@@ -30,7 +30,7 @@ Read these inputs:- `research/comparisons.md` (full tier) — cross-locus tensions- `research/temp/source-tensions.json` (full tier) — expert disagreements- `research/temp/coverage-gaps.md` (if exists) — items with weak source coverage-- Survey vault: `$HPR search "" --tag <vault_tag> -j` for the evidence landscape+- Survey vault: `$HPR note list --tag <vault_tag> --all -j` for the evidence landscape- Modality calibration (from the scaffold's `modality` field):- **collect** — enumerative coverage, per-entity sections with named fields- **synthesize** — defended thesis with evidence chains, interpretive density@@ -59,7 +59,7 @@ If `pipeline_tier == "light"`: SKIP step 10.1 — 10.3 below and follow this sec**Light tier writes a single draft directly to `research/notes/final_report_<vault_tag>.md`.** No subagents, no triple-draft ensemble, no synthesizer.-1. **Read the vault directly.** Light tier has no `evidence-digest.md` (step 9 was skipped). Survey the vault: `$HPR search "" --tag <vault_tag> -j` and pick the 8–15 most relevant non-deprecated notes. Read each one (`$HPR note show <id1> <id2> ... -j`) before writing.+1. **Read the vault directly.** Light tier has no `evidence-digest.md` (step 9 was skipped). Survey the vault: `$HPR note list --tag <vault_tag> --all -j` and pick the 8–15 most relevant non-deprecated notes. Read each one (`$HPR note show <id1> <id2> ... -j`) before writing.2. **Honor the structural contract.**- Use the literal H2 headings from `required_section_headings` in `research/prompt-decomposition.json`, in order.@@ -101,7 +101,7 @@ Write the 3 angle assignments to `research/temp/draft-angles.md` (for the run lo1. **List all substantive vault notes:**```bash- $HPR search "" --tag <vault_tag> --json+ $HPR note list --tag <vault_tag> --all --json```Filter to non-deprecated notes. You should have 50-100 candidates.diff --git a/src/hyperresearch/skills/hyperresearch-2-width-sweep.md b/src/hyperresearch/skills/hyperresearch-2-width-sweep.mdindex 8fb4934..cf5b5ad 100644--- a/src/hyperresearch/skills/hyperresearch-2-width-sweep.md+++ b/src/hyperresearch/skills/hyperresearch-2-width-sweep.md@@ -165,7 +165,7 @@ Append a few lines with `Edit` or `Write` every 30-60 seconds. Productive thinki**Vault count check** — once every 60 seconds max:```bash-PYTHONIOENCODING=utf-8 $HPR search "" --tag <vault_tag> --json | python -c "import sys,json; d=json.load(sys.stdin); print(f'Notes in vault: {len(d.get(\"data\",{}).get(\"results\",[]))}')"+PYTHONIOENCODING=utf-8 $HPR note list --tag <vault_tag> --all --json | python -c "import sys,json; d=json.load(sys.stdin); print(f'Notes in vault: {len(d.get(\"data\",[]))}')"```The wave is done when the vault note count is ≥80% of total URLs queued.@@ -176,7 +176,7 @@ The wave is done when the vault note count is ≥80% of total URLs queued.After Wave 1 returns, run the coverage check before proceeding:-1. **List fetched sources:** `$HPR search "" --tag <vault_tag> --json` — count substantive (non-deprecated) notes.+1. **List fetched sources:** `$HPR note list --tag <vault_tag> --all --json` — count substantive (non-deprecated) notes.2. **Map sources → atomic items.** For each atomic item in the decomposition, identify which fetched sources serve it. Mark each item as:- **Well-covered** (4+ relevant sources)diff --git a/src/hyperresearch/skills/hyperresearch-4-loci-analysis.md b/src/hyperresearch/skills/hyperresearch-4-loci-analysis.mdindex a4b6f7c..a1c6d6b 100644--- a/src/hyperresearch/skills/hyperresearch-4-loci-analysis.md+++ b/src/hyperresearch/skills/hyperresearch-4-loci-analysis.md@@ -25,7 +25,7 @@ Read these inputs:- `research/temp/contradiction-graph.json` — ranked fight clusters (if step 3 ran)- `research/temp/coverage-gaps.md` — which atomic items have weak coverage-Survey the corpus: `$HPR search "" --tag <vault_tag> -j` to confirm width sweep is complete.+Survey the corpus: `$HPR note list --tag <vault_tag> --all -j` to confirm width sweep is complete.---diff --git a/src/hyperresearch/skills/hyperresearch-5-depth-investigation.md b/src/hyperresearch/skills/hyperresearch-5-depth-investigation.mdindex 47b7f5b..af1541b 100644--- a/src/hyperresearch/skills/hyperresearch-5-depth-investigation.md+++ b/src/hyperresearch/skills/hyperresearch-5-depth-investigation.md@@ -79,7 +79,7 @@ Read these inputs:4. **Read the interim notes.** After all return, list them:```bash- $HPR search "" --tag <vault_tag> --type interim --json+ $HPR note list --tag <vault_tag> --type interim --all --json```Then batch-read them:```bashdiff --git a/src/hyperresearch/skills/hyperresearch-6-cross-locus-reconcile.md b/src/hyperresearch/skills/hyperresearch-6-cross-locus-reconcile.mdindex 612f542..94e6109 100644--- a/src/hyperresearch/skills/hyperresearch-6-cross-locus-reconcile.md+++ b/src/hyperresearch/skills/hyperresearch-6-cross-locus-reconcile.md@@ -25,7 +25,7 @@ description: >Read these inputs:- `research/scaffold.md` — vault_tag- `research/loci.json` — scored loci-- All interim notes: `$HPR search "" --tag <vault_tag> --type interim --json` then `$HPR note show <id1> <id2> ... -j`+- All interim notes: `$HPR note list --tag <vault_tag> --type interim --all --json` then `$HPR note show <id1> <id2> ... -j`You need the `## Committed position` section from every interim note in your context.diff --git a/src/hyperresearch/skills/hyperresearch-7-source-tensions.md b/src/hyperresearch/skills/hyperresearch-7-source-tensions.mdindex 8e7df15..fc648c8 100644--- a/src/hyperresearch/skills/hyperresearch-7-source-tensions.md+++ b/src/hyperresearch/skills/hyperresearch-7-source-tensions.md@@ -26,7 +26,7 @@ Read these inputs:- `research/scaffold.md` — vault_tag- `research/comparisons.md` — cross-locus tensions- `research/temp/contradiction-graph.json` (if step 3 ran)-- Survey vault: `$HPR search "" --tag <vault_tag> -j` for the 15–20 highest-quality non-deprecated sources+- Survey vault: `$HPR note list --tag <vault_tag> --all -j` for the 15–20 highest-quality non-deprecated sources---diff --git a/src/hyperresearch/skills/hyperresearch.md b/src/hyperresearch/skills/hyperresearch.mdindex 3b990d8..8e6cc9c 100644--- a/src/hyperresearch/skills/hyperresearch.md+++ b/src/hyperresearch/skills/hyperresearch.md@@ -161,10 +161,10 @@ Context compaction may eat parts of this conversation. If you're unsure what ste1. **Check the TodoWrite list.** It carries integer step numbers and survives compaction.2. **Check disk artifacts.** Each step writes a canonical artifact:- Step 1: `research/scaffold.md`, `research/prompt-decomposition.json`, `research/temp/coverage-matrix.md`- - Step 2: vault notes tagged with vault_tag (`$HPR search "" --tag <vault_tag> -j`)+ - Step 2: vault notes tagged with vault_tag (`$HPR note list --tag <vault_tag> --all -j`)- Step 3: `research/temp/contradiction-graph.json`, `research/temp/consensus-claims.json`- Step 4: `research/loci.json`- - Step 5: vault notes with `type: interim` (`$HPR search "" --tag <vault_tag> --type interim -j`)+ - Step 5: vault notes with `type: interim` (`$HPR note list --tag <vault_tag> --type interim --all -j`)- Step 6: `research/comparisons.md`- Step 7: `research/temp/source-tensions.json`- Step 8: `research/corpus-critic-gaps.json`, `research/temp/corpus-critic-results.md`diff --git a/tests/test_cli/test_lint.py b/tests/test_cli/test_lint.pyindex 52e8d97..c1dd4a2 100644--- a/tests/test_cli/test_lint.py+++ b/tests/test_cli/test_lint.py@@ -1196,7 +1196,7 @@ def test_patch_surgery_empty_log_without_findings_is_silent(tmp_vault):# ----------------------------------------------------------------------------def _write_decomposition(vault, entities=None, formats=None):+def _write_decomposition(vault, entities=None, formats=None, citation_style=None):import json as _jsonresearch = vault.root / "research"research.mkdir(parents=True, exist_ok=True)@@ -1208,6 +1208,8 @@ def _write_decomposition(vault, entities=None, formats=None):"time_horizons": [],"scope_conditions": [],}+ if citation_style is not None:+ data["citation_style"] = citation_style(research / "prompt-decomposition.json").write_text(_json.dumps(data, ensure_ascii=False, indent=2),encoding="utf-8",@@ -1411,3 +1413,121 @@ def test_extract_coverage_skips_on_hyperresearch_runs(tmp_vault):]# Silent on hyperresearch — locus-coverage handles this modeassert issues == []+++# ---------------------------------------------------------------------------+# citation-style-preservation+# ---------------------------------------------------------------------------+++def _citation_issues(out: str) -> list[dict]:+ import json+ data = json.loads(out)+ return [+ i for i in data.get("data", {}).get("issues", [])+ if i.get("rule") == "citation-style-preservation"+ ]+++def test_citation_preservation_passes_with_resolvable_wikilink(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="wikilink")+ _write_final_report(+ tmp_vault,+ "# Report\n\nSolid-state batteries are improving [[battery-paper]].\n",+ )+ code, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert code == 0+ assert _citation_issues(out) == []+++def test_citation_preservation_flags_report_with_no_wikilinks(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="wikilink")+ _write_final_report(tmp_vault, "# Report\n\nAll citations were stripped.\n")+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ issues = _citation_issues(out)+ assert len(issues) == 1+ assert issues[0]["severity"] == "error"+ assert "no [[wikilink]] markers" in issues[0]["message"]+++def test_citation_preservation_flags_only_unresolvable_wikilinks(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="wikilink")+ _write_final_report(+ tmp_vault,+ "# Report\n\nSee [[does-not-exist]] and [[also-missing|display text]].\n",+ )+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ issues = _citation_issues(out)+ assert len(issues) == 1+ assert "none of which resolve" in issues[0]["message"]+++def test_citation_preservation_wikilink_with_display_pipe_resolves(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="wikilink")+ _write_final_report(+ tmp_vault,+ "# Report\n\nPer [[battery-paper|Chen et al. 2025]], density doubled.\n",+ )+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert _citation_issues(out) == []+++def test_citation_preservation_skips_when_no_source_notes(tmp_vault):+ # Nothing to cite -> nothing to enforce.+ _write_decomposition(tmp_vault, citation_style="wikilink")+ _write_final_report(tmp_vault, "# Report\n\nNo sources exist yet.\n")+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert _citation_issues(out) == []+++def test_citation_preservation_skips_without_decomposition(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_final_report(tmp_vault, "# Report\n\nNo decomposition declared.\n")+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert _citation_issues(out) == []+++def test_citation_preservation_skips_style_none(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="none")+ _write_final_report(tmp_vault, "# Report\n\nDeliberately citation-free.\n")+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert _citation_issues(out) == []+++def test_citation_preservation_inline_passes_with_refs_and_heading(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="inline")+ _write_final_report(+ tmp_vault,+ "# Report\n\nDensity doubled [1].\n\n## Sources\n\n1. Chen et al. 2025\n",+ )+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert _citation_issues(out) == []+++def test_citation_preservation_inline_flags_missing_refs(tmp_vault):+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="inline")+ _write_final_report(tmp_vault, "# Report\n\nNo citations at all.\n")+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ issues = _citation_issues(out)+ assert len(issues) == 1+ assert issues[0]["severity"] == "error"+ assert "numbered [N] reference markers" in issues[0]["message"]+ assert "Sources/References section heading" in issues[0]["message"]+++def test_citation_preservation_wrapper_contract_overrides_style(tmp_vault):+ # Decomposition says wikilink, wrapper overrides to none -> rule skips+ # even though the report has no wikilinks.+ _write_source(tmp_vault, "Battery Paper", "battery-paper")+ _write_decomposition(tmp_vault, citation_style="wikilink")+ _write_wrapper_contract(tmp_vault, citation_style="none")+ _write_final_report(tmp_vault, "# Report\n\nWrapper said no citations.\n")+ _, out = _run_lint(tmp_vault, rule="citation-style-preservation")+ assert _citation_issues(out) == []