Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

Lint rules + data integrity: audit gate, provenance, raw file, note rm

Jordanc5c8c5cb6948
diff --git a/src/hyperresearch/cli/fetch.py b/src/hyperresearch/cli/fetch.py
index 71f2821..9ec78f0 100644
--- a/src/hyperresearch/cli/fetch.py
+++ b/src/hyperresearch/cli/fetch.py
@@ -30,6 +30,178 @@ SKIP_URL_PATTERNS = (
)
+def _append_suggested_by_to_existing(
+ vault_root: Path,
+ note_id: str,
+ suggested_by: list[str],
+ reason: str | None,
+) -> int:
+ """Append `*Suggested by [[src]] — reason*` breadcrumb lines to an existing note.
+
+ Idempotent: if a breadcrumb for a given source id already exists anywhere
+ in the body, it is not re-added. Returns the number of new breadcrumbs
+ actually added.
+ """
+ # Find the note file — scan research/notes/ for `<note-id>.md` or walk subdirs
+ from hyperresearch.core.frontmatter import parse_frontmatter, serialize_frontmatter
+
+ note_path = None
+ for candidate in vault_root.rglob(f"{note_id}.md"):
+ # Require it to be under a notes directory so we don't match index files
+ if "notes" in candidate.parts:
+ note_path = candidate
+ break
+ if note_path is None:
+ return 0
+
+ text = note_path.read_text(encoding="utf-8-sig")
+ meta, body = parse_frontmatter(text)
+
+ reason_suffix = f" — {reason}" if reason else ""
+ added = 0
+ new_lines = []
+ for src_id in suggested_by:
+ src_clean = src_id.strip()
+ if not src_clean:
+ continue
+ # Skip if a breadcrumb for this source id already exists in the body
+ existing_marker = f"[[{src_clean}]]"
+ if existing_marker in body:
+ continue
+ new_lines.append(f"*Suggested by [[{src_clean}]]{reason_suffix}*")
+ added += 1
+
+ if added == 0:
+ return 0
+
+ # Prepend the new breadcrumb lines to the body
+ breadcrumb_block = "\n".join(new_lines) + "\n\n"
+ new_body = breadcrumb_block + body
+ new_text = serialize_frontmatter(meta) + "\n" + new_body
+ note_path.write_text(new_text, encoding="utf-8")
+ return added
+
+
+def _detect_tier(url: str, content_type: str) -> str:
+ """Guess the epistemic tier from URL + content_type. Returns a Tier value.
+
+ This is a coarse first pass. The curation-time agent is expected to refine
+ (e.g. a medium.com post could be practitioner if it's by a named engineer,
+ or commentary if it's a generic explainer — only reading can tell).
+
+ Defaults prioritize reducing the agent's curation load: URLs with strong
+ signal get a non-unknown default so the agent only needs to verify, not
+ classify from scratch.
+ """
+ domain = urlparse(url).netloc.lower()
+
+ # Ground truth: official primary sources — filings, specs, policy text, datasets
+ if domain.endswith(".gov") or ".gov." in domain or domain.endswith(".gov.uk") or domain.endswith(".europa.eu"):
+ return "ground_truth"
+ if content_type == "policy":
+ return "ground_truth"
+ if content_type == "dataset":
+ return "ground_truth"
+ if content_type == "docs":
+ # Official documentation IS the ground truth for a product/standard
+ return "ground_truth"
+ if content_type == "code":
+ # Source code, README, and repo metadata are ground truth for a tool
+ return "ground_truth"
+
+ # Institutional: peer-reviewed scholarship, canonical reference works
+ if content_type == "paper":
+ # Default to institutional; agent may upgrade to ground_truth if the
+ # paper reports original data or downgrade to commentary for derivatives
+ return "institutional"
+ if "wikipedia.org" in domain or "britannica.com" in domain:
+ return "institutional"
+
+ # Practitioner: forums, community threads, hands-on content
+ if content_type == "forum":
+ return "practitioner"
+ if content_type == "review":
+ return "practitioner"
+
+ # Commentary: news articles, op-eds, blog posts without known authority
+ if content_type == "article":
+ return "commentary"
+ if content_type == "blog":
+ return "commentary"
+ if content_type == "transcript":
+ # Could be institutional (conference keynote) or commentary (podcast)
+ # — agent must decide. Start at commentary; promote during curation.
+ return "commentary"
+
+ return "unknown"
+
+
+def _detect_content_type(url: str, raw_content_type: str | None = None) -> str:
+ """Guess the artifact kind from URL + MIME. Returns a ContentType value or 'unknown'.
+
+ This is a coarse first pass. The creation-time agent is expected to refine
+ during curation if the URL/MIME heuristic is wrong (e.g. a medium.com URL
+ could be blog OR article).
+ """
+ if raw_content_type and "pdf" in raw_content_type.lower():
+ return "paper"
+ u = url.lower()
+ domain = urlparse(url).netloc.lower()
+ path = urlparse(url).path.lower()
+
+ # Papers: arxiv, doi, direct PDFs, openreview, ssrn, biorxiv, pubmed
+ if any(d in domain for d in ("arxiv.org", "doi.org", "openreview.net", "ssrn.com", "biorxiv.org", "medrxiv.org", "pubmed.ncbi.nlm.nih.gov", "ncbi.nlm.nih.gov/pmc", "semanticscholar.org", "openalex.org")):
+ return "paper"
+ if path.endswith(".pdf"):
+ return "paper"
+
+ # Code: github, gitlab, bitbucket, pypi
+ if any(d in domain for d in ("github.com", "gitlab.com", "bitbucket.org", "pypi.org", "crates.io", "npmjs.com")):
+ return "code"
+
+ # Dataset portals that happen to live on .gov domains must win first
+ if domain in ("data.gov", "data.gov.uk") or domain.startswith("data.") and (domain.endswith(".gov") or domain.endswith(".gov.uk") or domain.endswith(".europa.eu")):
+ return "dataset"
+
+ # Policy: gov domains, EU, regulator sites
+ if domain.endswith(".gov") or ".gov." in domain or domain.endswith(".gov.uk") or domain.endswith(".europa.eu") or "regulations.gov" in domain:
+ return "policy"
+
+ # Docs: common documentation platforms, docs.* subdomains, /docs paths
+ if any(d in domain for d in ("readthedocs.io", "readthedocs.org", "docs.rs", "developer.mozilla.org")):
+ return "docs"
+ if domain.startswith("docs.") or domain.startswith("documentation.") or ".readthedocs." in domain:
+ return "docs"
+ if "/docs/" in path or path.endswith("/docs") or "/documentation/" in path or "/api/reference" in path or "/reference/" in path:
+ return "docs"
+
+ # Forum / community
+ if any(d in domain for d in ("reddit.com", "news.ycombinator.com", "stackoverflow.com", "stackexchange.com", "lobste.rs", "lemmy.", "discourse.")):
+ return "forum"
+
+ # Blog platforms
+ if any(d in domain for d in ("medium.com", "substack.com", "dev.to", "hashnode.com", "hashnode.dev", "blogspot.com", "wordpress.com", "ghost.io")):
+ return "blog"
+
+ # Transcripts / video
+ if any(d in domain for d in ("youtube.com", "youtu.be", "vimeo.com")):
+ return "transcript"
+
+ # Reference articles
+ if "wikipedia.org" in domain or "wikimedia.org" in domain or "britannica.com" in domain:
+ return "article"
+
+ # Datasets
+ if any(d in domain for d in ("kaggle.com", "data.gov", "data.world", "zenodo.org", "figshare.com")):
+ return "dataset"
+
+ # News / magazine default
+ if any(d in domain for d in ("nytimes.com", "wsj.com", "ft.com", "bloomberg.com", "reuters.com", "bbc.com", "theatlantic.com", "newyorker.com", "economist.com")):
+ return "article"
+
+ return "unknown"
+
+
@app.command("fetch")
def fetch(
url: str = typer.Argument(..., help="URL to fetch and save as a note"),
@@ -39,6 +211,16 @@ def fetch(
provider_name: str | None = typer.Option(None, "--provider", help="Web provider override"),
save_assets: bool = typer.Option(False, "--save-assets", "-a", help="Download images and screenshot"),
visible: bool = typer.Option(False, "--visible", "-V", help="Run browser visibly (for stubborn auth sites)"),
+ suggested_by: list[str] = typer.Option(
+ [],
+ "--suggested-by",
+ help="Source note ID(s) that suggested this URL. Prepends a 'Suggested by [[note-id]]' breadcrumb to the new note's body, which the link extractor picks up as a wiki-link so the fetched note appears in the source's backlinks. Repeat for multiple sources.",
+ ),
+ suggested_by_reason: str | None = typer.Option(
+ None,
+ "--suggested-by-reason",
+ help="One-line justification for why the suggesting source named this URL. Appears next to the [[source-id]] breadcrumb.",
+ ),
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
) -> None:
"""Fetch a URL and save its content as a research note."""
@@ -63,6 +245,38 @@ def fetch(
existing = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
if existing:
note_id = existing["note_id"]
+ # Graceful duplicate handling for the guided reading loop:
+ # if the caller passed --suggested-by, append the breadcrumb to the
+ # existing note instead of erroring out. This lets the main agent
+ # build up provenance on already-fetched URLs without needing to
+ # check for dupes first.
+ if suggested_by:
+ added = _append_suggested_by_to_existing(
+ vault.root, note_id, suggested_by, suggested_by_reason
+ )
+ # Re-sync so the new wiki-link gets picked up by the link extractor
+ plan = compute_sync_plan(vault)
+ if plan.to_update:
+ execute_sync(vault, plan)
+
+ data = {
+ "note_id": note_id,
+ "url": url,
+ "duplicate": True,
+ "backlinks_added": added,
+ "message": (
+ f"URL already fetched; added {added} backlink(s) to existing note"
+ if added
+ else "URL already fetched; breadcrumb(s) already present"
+ ),
+ }
+ if json_output:
+ output(success(data, vault=str(vault.root)), json_mode=True)
+ else:
+ console.print(f"[yellow]Already fetched:[/] {url} → note '{note_id}'")
+ console.print(f" added {added} backlink(s) to existing note")
+ return
+
if json_output:
output(
error(f"URL already fetched as note '{note_id}'", "DUPLICATE_URL"),
@@ -130,6 +344,8 @@ def fetch(
note_title = title or result.title or urlparse(url).path.split("/")[-1] or "Untitled"
domain = result.domain
+ detected_content_type = _detect_content_type(url, result.raw_content_type)
+ detected_tier = _detect_tier(url, detected_content_type)
extra_meta = {
"source": url,
"source_domain": domain,
@@ -139,14 +355,33 @@ def fetch(
if result.metadata.get("author"):
extra_meta["author"] = result.metadata["author"]
+ # Build body with backlink breadcrumb if this fetch was suggested by another note.
+ # Prepending `*Suggested by [[source-id]] — reason*` lines creates wiki-links the
+ # normal link extractor will pick up, so the new note automatically appears in
+ # graph backlinks of its suggester(s). Zero schema change needed.
+ body_content = result.content
+ if suggested_by:
+ breadcrumb_lines = []
+ reason_suffix = f" — {suggested_by_reason}" if suggested_by_reason else ""
+ for src_id in suggested_by:
+ src_id_clean = src_id.strip()
+ if not src_id_clean:
+ continue
+ breadcrumb_lines.append(f"*Suggested by [[{src_id_clean}]]{reason_suffix}*")
+ if breadcrumb_lines:
+ breadcrumb = "\n".join(breadcrumb_lines) + "\n\n"
+ body_content = breadcrumb + body_content
+
note_path = write_note(
vault.notes_dir,
title=note_title,
- body=result.content,
+ body=body_content,
tags=tags,
status="draft",
source=url,
parent=parent,
+ tier=detected_tier,
+ content_type=detected_content_type,
extra_frontmatter=extra_meta,
)
@@ -172,18 +407,17 @@ def fetch(
# Note: tagging and summarization is the agent's job, not an automatic process.
# The agent reads the fetched content and writes meaningful summaries and tags.
- # Add raw_file reference to frontmatter AFTER enrich (enrich rewrites frontmatter)
+ # Persist raw_file reference on the note via NoteMeta — this guarantees the
+ # field survives any future re-serialization (repair, note update, etc.).
+ # The text-injection approach used previously was silently dropped by
+ # NoteMeta.model_config = {"extra": "ignore"} on the next parse.
if raw_file_path:
+ from hyperresearch.core.frontmatter import parse_frontmatter, render_note
note_text = note_path.read_text(encoding="utf-8")
- if note_text.startswith("---") and "raw_file:" not in note_text:
- end_idx = note_text.find("---", 3)
- if end_idx != -1:
- note_text = (
- note_text[:end_idx]
- + f"raw_file: {raw_file_path}\n"
- + note_text[end_idx:]
- )
- note_path.write_text(note_text, encoding="utf-8")
+ meta, body = parse_frontmatter(note_text)
+ if meta.raw_file != raw_file_path:
+ meta.raw_file = raw_file_path
+ note_path.write_text(render_note(meta, body), encoding="utf-8")
# Sync first so the note exists in the notes table (needed for FK on sources/assets)
note_id = note_path.stem
@@ -191,15 +425,44 @@ def fetch(
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
- # Record source in DB
+ # Record source in DB. Use INSERT OR IGNORE to survive a duplicate-URL
+ # race: two parallel fetches on the same URL both pass the earlier
+ # duplicate-check SELECT, both try to INSERT, and without IGNORE the
+ # second crashes with IntegrityError leaving a stray .md file and
+ # notes row but no sources row. With IGNORE the second INSERT is a
+ # no-op and we detect the race by re-selecting the row afterward.
content_hash = hashlib.sha256(result.content.encode("utf-8")).hexdigest()[:16]
conn.execute(
- """INSERT INTO sources (url, note_id, domain, fetched_at, provider, content_hash)
+ """INSERT OR IGNORE INTO sources (url, note_id, domain, fetched_at, provider, content_hash)
VALUES (?, ?, ?, ?, ?, ?)""",
(url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash),
)
conn.commit()
+ # Detect the race: if the committed row's note_id != ours, another
+ # fetch won the race. Clean up our orphan .md file and return the
+ # winner's note_id so the caller can treat this as an idempotent no-op.
+ winner_row = conn.execute(
+ "SELECT note_id FROM sources WHERE url = ?", (url,)
+ ).fetchone()
+ if winner_row and winner_row["note_id"] != note_id:
+ winning_note_id = winner_row["note_id"]
+ # Our write lost the race. Unlink the orphan file and re-sync.
+ if note_path.exists():
+ try:
+ note_path.unlink()
+ except OSError:
+ pass
+ plan2 = compute_sync_plan(vault)
+ if plan2.to_delete:
+ execute_sync(vault, plan2)
+ note_id = winning_note_id
+ winner_path_row = conn.execute(
+ "SELECT path FROM notes WHERE id = ?", (winning_note_id,)
+ ).fetchone()
+ if winner_path_row:
+ note_path = vault.root / winner_path_row["path"]
+
# Save assets (screenshot + images) — only when requested
saved_assets: list[dict] = []
if save_assets:
diff --git a/src/hyperresearch/cli/lint.py b/src/hyperresearch/cli/lint.py
index 3e683f1..4ebcd88 100644
--- a/src/hyperresearch/cli/lint.py
+++ b/src/hyperresearch/cli/lint.py
@@ -15,6 +15,13 @@ RULES = {
"missing-title": "Notes without a title",
"missing-tags": "Notes without any tags",
"missing-summary": "Notes without a summary",
+ "uncurated": "Non-draft notes without tier or content_type classification",
+ "workflow": "Draft or synthesis notes missing paired scaffold/comparison notes",
+ "scaffold-prompt": "Scaffold notes missing the verbatim user prompt as first section (gospel rule)",
+ "audit-gate": "Unresolved CRITICAL findings in research/audit_findings.json block synthesis save",
+ "provenance": "Source notes with no --suggested-by breadcrumb chain (data-flow chain broken)",
+ "analyst-coverage": "Fetched sources without paired extract notes (analyst skipped most sources)",
+ "orphaned-raw-files": "Files in research/raw/ with no matching note (disk leak from old note rm)",
"singleton-tags": "Tags used by only one note",
"broken-links": "Wiki-links that don't resolve",
"orphaned-notes": "Notes with no inbound or outbound links",
@@ -47,6 +54,12 @@ def lint(
rules_to_run = [rule] if rule else list(RULES.keys())
+ # Map from audit-gate CRITICAL finding id -> lint rule name that should be
+ # re-run to verify the fix actually landed. Populated inside the audit-gate
+ # block and consumed at the end of lint() for self-certification detection.
+ # Key words in the finding description are mapped to lint rule names.
+ audit_gate_guards: list[dict] = [] # [{"critical_id", "rule", "description"}]
+
if "missing-title" in rules_to_run:
for row in conn.execute("SELECT id, path FROM notes WHERE title = '' OR title = 'Untitled'"):
issues.append({
@@ -85,6 +98,641 @@ def lint(
"message": "Note has no summary. Add one for better search and listings.",
})
+ if "audit-gate" in rules_to_run:
+ # Block synthesis save unless BOTH:
+ # (a) at least one `conformance` audit run exists in
+ # research/audit_findings.json, AND
+ # (b) every CRITICAL finding in the most recent conformance run has
+ # a non-null `fixed_at` timestamp (applied or explicitly resolved).
+ #
+ # Additionally: surface IMPORTANT findings as info-severity issues so
+ # the agent sees them in the pre-save lint output. They don't block
+ # save, but they do nudge the agent to review them before committing.
+ #
+ # Missing file = no audit has run yet. Gate is OPEN in that case so
+ # early-stage lint runs don't spam errors. But once the file exists,
+ # a missing conformance run is itself an error — the protocol demands
+ # both modes, not just comprehensiveness.
+ import json as _json
+ audit_path = vault.root / "research" / "audit_findings.json"
+ if audit_path.exists():
+ try:
+ audit_data = _json.loads(audit_path.read_text(encoding="utf-8"))
+ except (OSError, _json.JSONDecodeError) as exc:
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"research/audit_findings.json exists but is malformed "
+ f"({type(exc).__name__}). Delete or fix it, then re-run "
+ f"the adversarial audit."
+ ),
+ })
+ audit_data = None
+
+ if isinstance(audit_data, dict):
+ runs = audit_data.get("runs", [])
+ conformance_runs = [r for r in runs if r.get("mode") == "conformance"]
+ comprehensiveness_runs = [r for r in runs if r.get("mode") == "comprehensiveness"]
+
+ # Check (a): a conformance run must exist once any audit has happened.
+ if not conformance_runs:
+ if comprehensiveness_runs:
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"research/audit_findings.json has "
+ f"{len(comprehensiveness_runs)} comprehensiveness run(s) but ZERO "
+ f"conformance runs. Step 11 mandates BOTH modes in parallel. "
+ f"Spawn hyperresearch-auditor with mode=conformance and wait for "
+ f"it to append its findings to audit_findings.json before saving "
+ f"the synthesis."
+ ),
+ })
+ # No runs at all = early stage. Gate stays open.
+ else:
+ # Check (b): no unresolved CRITICALs in the newest conformance run.
+ conformance_runs.sort(key=lambda r: r.get("timestamp", ""))
+ latest = conformance_runs[-1]
+ latest_status = latest.get("status", "unknown")
+ criticals = latest.get("criticals") or []
+ unresolved = [c for c in criticals if not c.get("fixed_at")]
+
+ if unresolved:
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Most recent conformance audit has "
+ f"{len(unresolved)} unresolved CRITICAL finding(s): "
+ + "; ".join(
+ f"[{c.get('id','?')}] {c.get('description','?')[:80]}"
+ for c in unresolved[:5]
+ )
+ + ". Apply the fixes in research/notes/final_report.md, "
+ + "mark each finding with `fixed_at: <ISO>` in "
+ + "research/audit_findings.json, and re-run the conformance "
+ + "auditor to verify."
+ ),
+ })
+ elif latest_status == "failed":
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Most recent conformance audit returned status=failed. "
+ f"Investigate the findings and re-run the auditor."
+ ),
+ })
+
+ # Surface IMPORTANT findings from EITHER mode's newest run. Info
+ # severity = advisory; does not block save, but the agent sees
+ # them in the save-gate lint output and can choose to patch.
+ for mode_runs, mode_label in (
+ (conformance_runs, "conformance"),
+ (comprehensiveness_runs, "comprehensiveness"),
+ ):
+ if not mode_runs:
+ continue
+ mode_runs.sort(key=lambda r: r.get("timestamp", ""))
+ latest_run = mode_runs[-1]
+ important = latest_run.get("important") or []
+ unresolved_important = [i for i in important if not i.get("fixed_at")]
+ if unresolved_important:
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "info",
+ "note_id": "<vault>",
+ "message": (
+ f"{len(unresolved_important)} unresolved IMPORTANT finding(s) in "
+ f"the latest {mode_label} audit (advisory, does not block save): "
+ + "; ".join(
+ f"[{i.get('id','?')}] {i.get('description','?')[:80]}"
+ for i in unresolved_important[:5]
+ )
+ + ". Mark `fixed_at` on each after patching the draft."
+ ),
+ })
+
+ # Build guard-rule map: for each CRITICAL with fixed_at set,
+ # extract the implied lint rule from keywords in its description
+ # and queue that rule for verification. This is how audit-gate
+ # detects self-certification (CRITICAL marked fixed but the
+ # underlying lint rule still fails).
+ #
+ # The keyword list covers the natural-language variations we
+ # see in real auditor outputs across modalities. Keep these
+ # broad — false matches are fine (they cause a re-check
+ # which passes harmlessly); missing a match is the failure
+ # mode we're guarding against.
+ _KW_TO_RULE = [
+ # scaffold-prompt (verbatim prompt gospel rule)
+ ("scaffold-prompt", "scaffold-prompt"),
+ ("scaffold_prompt", "scaffold-prompt"),
+ ("scaffold_extraction_gap", "scaffold-prompt"),
+ ("scaffold extraction gap", "scaffold-prompt"),
+ ("verbatim prompt", "scaffold-prompt"),
+ ("verbatim_prompt", "scaffold-prompt"),
+ ("gospel rule", "scaffold-prompt"),
+ ("user prompt missing", "scaffold-prompt"),
+
+ # analyst-coverage (extract notes per source)
+ ("analyst-coverage", "analyst-coverage"),
+ ("analyst coverage", "analyst-coverage"),
+ ("analyst_coverage", "analyst-coverage"),
+ ("extract coverage", "analyst-coverage"),
+ ("extract ratio", "analyst-coverage"),
+ ("extract notes", "analyst-coverage"),
+ ("fetch:extract", "analyst-coverage"),
+ ("analyst skipped", "analyst-coverage"),
+ ("no extract", "analyst-coverage"),
+
+ # provenance (bouncing reading loop + --suggested-by chain)
+ ("provenance", "provenance"),
+ ("suggested-by", "provenance"),
+ ("suggested_by", "provenance"),
+ ("suggested by", "provenance"),
+ ("bouncing reading loop", "provenance"),
+ ("bouncing loop", "provenance"),
+ ("guided reading loop", "provenance"),
+ ("guided loop", "provenance"),
+ ("reading loop", "provenance"),
+ ("breadcrumb", "provenance"),
+ ("data-flow chain", "provenance"),
+ ("data flow chain", "provenance"),
+ ("data-flow broken", "provenance"),
+ ("rabbit-hole", "provenance"),
+ ("rabbit hole", "provenance"),
+
+ # workflow (scaffold + comparisons + extract artifacts exist)
+ ("workflow", "workflow"),
+ ("missing scaffold", "workflow"),
+ ("missing comparison", "workflow"),
+ ("missing comparisons", "workflow"),
+ ("paired scaffold", "workflow"),
+ ("no scaffold note", "workflow"),
+ ("no comparison note", "workflow"),
+ ("step 7 skipped", "workflow"),
+ ("step 8 skipped", "workflow"),
+
+ # uncurated (tier + content_type + summary metadata)
+ ("uncurated", "uncurated"),
+ ("tier metadata", "uncurated"),
+ ("content_type", "uncurated"),
+ ("content type", "uncurated"),
+ ("tier/content_type", "uncurated"),
+ ("tier and content_type", "uncurated"),
+ ("classification missing", "uncurated"),
+ ("metadata missing", "uncurated"),
+ ("missing tier", "uncurated"),
+ ("missing summary", "uncurated"),
+ ]
+ for mode_runs in (conformance_runs, comprehensiveness_runs):
+ if not mode_runs:
+ continue
+ # We already sorted conformance_runs above; sort comprehensiveness
+ # now too so we pick the newest run of each mode.
+ mode_runs_sorted = sorted(mode_runs, key=lambda r: r.get("timestamp", ""))
+ latest_run = mode_runs_sorted[-1]
+ for c in (latest_run.get("criticals") or []):
+ if not c.get("fixed_at"):
+ continue # unresolved criticals already emitted above
+ desc = (str(c.get("description", "")) + " " +
+ str(c.get("id", ""))).lower()
+ matched = None
+ for kw, rule_name in _KW_TO_RULE:
+ if kw in desc:
+ matched = rule_name
+ break
+ if matched:
+ audit_gate_guards.append({
+ "critical_id": c.get("id", "?"),
+ "rule": matched,
+ "description": c.get("description", "")[:120],
+ })
+ # Ensure the guard rule runs so we can check its
+ # issues in post-processing.
+ if matched not in rules_to_run:
+ rules_to_run.append(matched)
+ else:
+ # CRITICAL marked fixed_at but no known lint rule
+ # maps to its description. The fix is trust-only
+ # — we cannot machine-verify it. Surface a warning
+ # so the user knows this finding was not validated.
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "warning",
+ "note_id": "<vault>",
+ "message": (
+ f"CRITICAL [{c.get('id','?')}] was marked `fixed_at` but "
+ f"its description doesn't map to any known lint rule: "
+ f"'{(c.get('description','') or '')[:100]}'. The fix is "
+ f"agent-self-reported and not machine-verified. Review the "
+ f"draft manually to confirm the issue was actually addressed."
+ ),
+ })
+
+ if "scaffold-prompt" in rules_to_run:
+ # Enforce the gospel rule: every scaffold-tagged note must open with
+ # the user's verbatim prompt as its first section. This is the single
+ # machine-checkable invariant that protects the dispatcher's
+ # "user prompt is gospel" commitment. Without this check, scaffolds
+ # can drift — an agent writes a `## Thesis` opening, never pastes the
+ # verbatim prompt, and every downstream step that re-reads the
+ # scaffold (audit, draft, comparisons) loses its anchor.
+ import re as _re
+ _header_re = _re.compile(
+ r"^\s*##\s+User\s+Prompt\s*\(\s*VERBATIM.*gospel\s*\)",
+ _re.IGNORECASE,
+ )
+ for row in conn.execute("""
+ SELECT n.id, n.path, nc.body
+ FROM notes n
+ JOIN note_content nc ON n.id = nc.note_id
+ WHERE n.id IN (SELECT note_id FROM tags WHERE tag = 'scaffold')
+ """):
+ body_lines = (row["body"] or "").splitlines()
+ # Look for the header within the first 20 non-blank lines
+ header_line_idx = None
+ seen_non_blank = 0
+ for idx, line in enumerate(body_lines):
+ if line.strip():
+ seen_non_blank += 1
+ if _header_re.match(line):
+ header_line_idx = idx
+ break
+ if seen_non_blank >= 20:
+ break
+
+ if header_line_idx is None:
+ issues.append({
+ "rule": "scaffold-prompt",
+ "severity": "error",
+ "note_id": row["id"],
+ "note_path": row["path"],
+ "message": (
+ "Scaffold is missing the verbatim user prompt as its first section. "
+ "Every scaffold MUST open with a `## User Prompt (VERBATIM — gospel)` "
+ "header followed by the user's original question as a blockquote. "
+ "This is the gospel rule — the dispatcher re-reads the prompt from "
+ "this section at every downstream step."
+ ),
+ })
+ continue
+
+ # Check for non-empty blockquote after the header (>= 50 chars total,
+ # which is a loose floor — a real prompt will be much longer).
+ quote_chars = 0
+ for line in body_lines[header_line_idx + 1:]:
+ stripped = line.strip()
+ if not stripped:
+ continue
+ if stripped.startswith(">"):
+ quote_chars += len(stripped.lstrip("> ").strip())
+ elif stripped.startswith("##"):
+ break # hit next header without finding quoted content
+ else:
+ # some scaffolds paste the prompt as plain text without
+ # the blockquote prefix — accept if substantive.
+ quote_chars += len(stripped)
+ break
+
+ if quote_chars < 50:
+ issues.append({
+ "rule": "scaffold-prompt",
+ "severity": "warning",
+ "note_id": row["id"],
+ "note_path": row["path"],
+ "message": (
+ f"Scaffold has the verbatim-prompt header but the content after it "
+ f"is empty or too short ({quote_chars} chars). Paste the user's "
+ f"full prompt as a blockquote under the header."
+ ),
+ })
+
+ if "provenance" in rules_to_run:
+ # Verify the `--suggested-by` data-flow chain forms a rooted tree
+ # (or forest) across all fetched source notes:
+ #
+ # 1. At least one seed source exists (a fetched note with no
+ # breadcrumb) — the loop has a starting point.
+ # 2. Every non-seed source has at least one breadcrumb pointing at
+ # another source note that exists in the vault (reachable from
+ # a seed through one or more breadcrumb hops).
+ # 3. Every breadcrumb's wiki-link target is a real note id — no
+ # dangling backlinks.
+ #
+ # Previous version used `breadcrumb_count < max(2, source_count // 5)`
+ # which was easy to game by backfilling N//5 unrelated breadcrumbs.
+ # The rooted-tree check cannot be satisfied without an actual chain.
+ import re as _re
+ _breadcrumb_re = _re.compile(r"\*Suggested by \[\[([^\]]+)\]\]")
+
+ source_rows = list(conn.execute(
+ "SELECT n.id, n.path, nc.body "
+ "FROM notes n "
+ "JOIN note_content nc ON n.id = nc.note_id "
+ "WHERE n.source IS NOT NULL "
+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "
+ "AND n.type NOT IN ('index','raw','moc')"
+ ))
+
+ if len(source_rows) <= 5:
+ # Small corpora: bouncing loop may not have fired by design.
+ # Skip the structural check; fall back to presence check only.
+ pass
+ else:
+ all_note_ids = {
+ r["id"] for r in conn.execute("SELECT id FROM notes")
+ }
+
+ source_breadcrumbs: dict[str, list[str]] = {}
+ for r in source_rows:
+ targets = _breadcrumb_re.findall(r["body"] or "")
+ # Keep only the wiki-link name before any `|display` pipe.
+ cleaned = [t.split("|", 1)[0].strip() for t in targets]
+ source_breadcrumbs[r["id"]] = cleaned
+
+ seeds = [nid for nid, crumbs in source_breadcrumbs.items() if not crumbs]
+ non_seeds = [nid for nid, crumbs in source_breadcrumbs.items() if crumbs]
+
+ # Condition 1: at least one seed.
+ if not seeds:
+ issues.append({
+ "rule": "provenance",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Provenance graph has no seed: every one of {len(source_rows)} source notes "
+ f"carries a `*Suggested by [[...]]` breadcrumb, which is impossible for a real "
+ f"research session. The guided reading loop must start from at least one seed "
+ f"fetch with no suggester."
+ ),
+ })
+
+ # Condition 2 + 3: verify graph is rooted at seeds and no dangling targets.
+ if non_seeds:
+ dangling: list[tuple[str, str]] = []
+ for nid in non_seeds:
+ for target in source_breadcrumbs[nid]:
+ if target not in all_note_ids:
+ dangling.append((nid, target))
+
+ for src_id, target in dangling[:10]: # cap output
+ issues.append({
+ "rule": "provenance",
+ "severity": "error",
+ "note_id": src_id,
+ "message": (
+ f"Breadcrumb `[[{target}]]` points at a note id that does not exist in the "
+ f"vault. Either the target was deleted, or the breadcrumb was hand-written "
+ f"without a real source. Re-fetch with `--suggested-by <real-note-id>`."
+ ),
+ })
+
+ # BFS from seeds to verify connectivity.
+ reachable = set(seeds)
+ frontier = list(seeds)
+ # Build reverse map: suggester -> notes it sourced
+ suggester_to_sourced: dict[str, list[str]] = {}
+ for nid, crumbs in source_breadcrumbs.items():
+ for t in crumbs:
+ suggester_to_sourced.setdefault(t, []).append(nid)
+ while frontier:
+ current = frontier.pop()
+ for child in suggester_to_sourced.get(current, []):
+ if child not in reachable:
+ reachable.add(child)
+ frontier.append(child)
+
+ unreachable = [nid for nid in non_seeds if nid not in reachable]
+ if unreachable:
+ issues.append({
+ "rule": "provenance",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"{len(unreachable)} source note(s) have breadcrumbs but are not reachable "
+ f"from any seed through the provenance graph — the chain is disconnected. "
+ f"Disconnected islands usually mean an agent fabricated breadcrumbs "
+ f"retroactively without following the guided reading loop. First few: "
+ f"{', '.join(unreachable[:5])}"
+ ),
+ })
+
+ # Condition: coverage. At least half of non-seed sources should have a breadcrumb.
+ # (Cheap heuristic: the count of non-seeds is the count of sources with at least
+ # one breadcrumb; compare to the count of sources that SHOULD have one.)
+ non_seed_ratio = len(non_seeds) / max(len(source_rows), 1)
+ # If there is exactly one seed and no non-seeds, the loop never fired.
+ if len(source_rows) > 5 and len(non_seeds) == 0:
+ issues.append({
+ "rule": "provenance",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Vault has {len(source_rows)} fetched source notes but ZERO "
+ f"`*Suggested by [[...]]` breadcrumbs. The bouncing reading loop never "
+ f"fired — every fetch was a flat batch with no link back to the source "
+ f"that proposed it. Use `$HPR fetch ... --suggested-by <source-note-id> "
+ f"--suggested-by-reason \"<why>\"` for every follow-up fetch."
+ ),
+ })
+ elif non_seed_ratio < 0.3 and len(source_rows) > 10:
+ # Less than 30% of the corpus came from analyst recommendations.
+ # The guided reading loop effectively didn't fire — this is the
+ # exact failure mode both v2 runs exhibited (3/19 and 2/33).
+ # Error severity because it blocks Checkpoint 2.
+ issues.append({
+ "rule": "provenance",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Only {len(non_seeds)}/{len(source_rows)} source notes ({non_seed_ratio:.0%}) "
+ f"have breadcrumbs — the guided reading loop did not fire. The initial batch "
+ f"fetch is not the whole corpus; after fetching seeds you MUST spawn analysts "
+ f"to propose next targets, then fetch those with `--suggested-by`. Target: "
+ f"at least 30% of sources should come from analyst recommendations."
+ ),
+ })
+ elif non_seed_ratio < 0.5 and len(source_rows) > 10:
+ # Between 30% and 50% — under-firing but marginal. Warning only.
+ issues.append({
+ "rule": "provenance",
+ "severity": "warning",
+ "note_id": "<vault>",
+ "message": (
+ f"Only {len(non_seeds)}/{len(source_rows)} source notes ({non_seed_ratio:.0%}) "
+ f"have breadcrumbs. The bouncing reading loop is under-firing — most "
+ f"fetches look like flat seeds rather than analyst-driven discoveries."
+ ),
+ })
+
+ if "analyst-coverage" in rules_to_run:
+ # Detect when source notes were fetched but few were analyzed (i.e.,
+ # spawned a hyperresearch-analyst that wrote an extract note). An
+ # extract note is identified by carrying the `extract` tag.
+ #
+ # Rule applies UNCONDITIONALLY — previous versions gated on
+ # `source_count >= 10` which silently disabled the rule for small
+ # corpora (a compare session with 8 named entities could ship zero
+ # extracts and pass). The gate is removed.
+ source_count_row = conn.execute(
+ "SELECT COUNT(*) as c FROM notes n "
+ "WHERE n.source IS NOT NULL "
+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "
+ "AND n.type NOT IN ('index','raw','moc') "
+ "AND n.id NOT IN (SELECT note_id FROM tags WHERE tag = 'extract') "
+ "AND n.id NOT IN (SELECT note_id FROM tags WHERE tag = 'scaffold') "
+ "AND n.id NOT IN (SELECT note_id FROM tags WHERE tag = 'comparison')"
+ ).fetchone()
+ source_count = source_count_row["c"] if source_count_row else 0
+
+ extract_count_row = conn.execute(
+ "SELECT COUNT(DISTINCT note_id) as c FROM tags WHERE tag = 'extract'"
+ ).fetchone()
+ extract_count = extract_count_row["c"] if extract_count_row else 0
+
+ # Require 1/3 coverage (error floor at 1/4). Even a 2-source session
+ # needs at least 1 extract — the analyst is mandatory, not optional.
+ if source_count >= 1:
+ required_extracts = max(1, source_count // 3)
+ error_floor = max(1, source_count // 4)
+ if extract_count < required_extracts:
+ ratio = extract_count / source_count if source_count else 0
+ issues.append({
+ "rule": "analyst-coverage",
+ "severity": "error" if extract_count < error_floor else "warning",
+ "note_id": "<vault>",
+ "message": (
+ f"Vault has {source_count} fetched source notes but only {extract_count} "
+ f"extract notes ({ratio:.0%} coverage, need ≥{required_extracts}). "
+ f"The analyst was skipped on most sources. Spawn "
+ f"hyperresearch-analyst (mode=extract or mode=guided) on the "
+ f"unanalyzed sources during curation. Target: at least 1 extract per "
+ f"3 sources (floor of 1 for any corpus size)."
+ ),
+ })
+
+ if "orphaned-raw-files" in rules_to_run:
+ # Walk research/raw/ and flag files whose stem doesn't match any note
+ # id in the vault. These are leftovers from the pre-Batch-2.5 `note rm`
+ # which never touched raw files. A cheap disk-leak detector.
+ raw_dir = vault.root / "research" / "raw"
+ if raw_dir.is_dir():
+ note_ids = {r["id"] for r in conn.execute("SELECT id FROM notes")}
+ for raw_file in raw_dir.iterdir():
+ if not raw_file.is_file():
+ continue
+ # Only flag known raw extensions; ignore any README etc.
+ if raw_file.suffix.lower() not in {".pdf", ".png", ".jpg", ".jpeg", ".gif", ".webp"}:
+ continue
+ if raw_file.stem not in note_ids:
+ issues.append({
+ "rule": "orphaned-raw-files",
+ "severity": "warning",
+ "note_id": raw_file.stem,
+ "note_path": str(raw_file.relative_to(vault.root).as_posix()),
+ "message": (
+ f"Raw file {raw_file.name} has no matching note in the vault. "
+ f"Likely a leftover from an old `note rm` that didn't clean up "
+ f"raw files. Delete it manually or let repair handle it."
+ ),
+ })
+
+ if "workflow" in rules_to_run:
+ # Detect research sessions that skipped the mandatory process steps.
+ # Heuristic: a note with a research-output signal (final_report in id,
+ # type=moc, or tag=synthesis) means a research session ran. If that
+ # session didn't produce a scaffold note AND a comparison note, the
+ # protocol was skipped.
+ def _count_by_tag(tag: str) -> int:
+ row = conn.execute(
+ "SELECT COUNT(DISTINCT note_id) as c FROM tags WHERE tag = ?",
+ (tag,),
+ ).fetchone()
+ return row["c"] if row else 0
+
+ has_research_output = conn.execute("""
+ SELECT COUNT(*) as c FROM notes n
+ WHERE n.id LIKE '%final_report%'
+ OR n.type = 'moc'
+ OR n.id IN (SELECT note_id FROM tags WHERE tag = 'synthesis')
+ """).fetchone()["c"]
+
+ scaffold_count = _count_by_tag("scaffold")
+ comparison_count = _count_by_tag("comparison")
+ extract_count = _count_by_tag("extract")
+
+ if has_research_output > 0:
+ if scaffold_count == 0:
+ issues.append({
+ "rule": "workflow",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Vault has {has_research_output} research-output note(s) "
+ f"but 0 scaffold notes. Step 7 was skipped. "
+ f"Research sessions must produce a `scaffold` note before the draft."
+ ),
+ })
+ if comparison_count == 0:
+ issues.append({
+ "rule": "workflow",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"Vault has {has_research_output} research-output note(s) "
+ f"but 0 comparison notes. Step 8 was skipped. "
+ f"Research sessions must produce a `comparison` note before the draft."
+ ),
+ })
+
+ # Flag extract notes that are missing the parent link
+ for row in conn.execute("""
+ SELECT n.id, n.path FROM notes n
+ WHERE n.id IN (SELECT note_id FROM tags WHERE tag = 'extract')
+ AND (n.parent IS NULL OR n.parent = '')
+ """):
+ issues.append({
+ "rule": "workflow",
+ "severity": "warning",
+ "note_id": row["id"],
+ "note_path": row["path"],
+ "message": "Extract note missing --parent source-id. The chain of custody is broken.",
+ })
+
+ if "uncurated" in rules_to_run:
+ # Any note that has moved past draft without tier/content_type classification
+ # is a curation failure. Exempt: draft notes (expected raw), index/raw/moc
+ # types (not sources), and notes whose id starts with _ (auto-generated
+ # index notes like _most-linked, _stale, _orphans).
+ for row in conn.execute(
+ "SELECT n.id, n.path, n.status, n.tier, n.content_type FROM notes n "
+ "WHERE n.type NOT IN ('index','raw','moc') "
+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "
+ "AND n.status != 'draft' "
+ "AND (n.tier IS NULL OR n.tier = 'unknown' "
+ " OR n.content_type IS NULL OR n.content_type = 'unknown')"
+ ):
+ missing = []
+ if not row["tier"] or row["tier"] == "unknown":
+ missing.append("tier")
+ if not row["content_type"] or row["content_type"] == "unknown":
+ missing.append("content_type")
+ issues.append({
+ "rule": "uncurated",
+ "severity": "warning",
+ "note_id": row["id"],
+ "note_path": row["path"],
+ "message": f"Note is {row['status']} but missing {'/'.join(missing)}. Run curation pass.",
+ })
+
if "singleton-tags" in rules_to_run:
for row in conn.execute(
"SELECT tag, COUNT(*) as c FROM tags GROUP BY tag HAVING c = 1"
@@ -182,6 +830,34 @@ def lint(
"message": f"Last reviewed {row['reviewed'][:10]}. Consider re-reviewing.",
})
+ # Audit-gate self-certification post-check. For each CRITICAL finding
+ # that was marked `fixed_at`, we queued its implied lint rule for
+ # re-running via rules_to_run. Now that the full lint pass is done,
+ # check whether the rule still emitted errors. If yes, the agent marked
+ # a CRITICAL resolved without actually fixing the underlying vault state
+ # — emit a self-cert violation error that blocks the save gate.
+ if audit_gate_guards:
+ for guard in audit_gate_guards:
+ rule_errors = [
+ i for i in issues
+ if i.get("rule") == guard["rule"] and i.get("severity") == "error"
+ ]
+ if rule_errors:
+ issues.append({
+ "rule": "audit-gate",
+ "severity": "error",
+ "note_id": "<vault>",
+ "message": (
+ f"SELF-CERTIFICATION VIOLATION: CRITICAL [{guard['critical_id']}] "
+ f"was marked `fixed_at` in research/audit_findings.json, but lint "
+ f"rule `{guard['rule']}` still returns {len(rule_errors)} error(s). "
+ f"The finding was '{guard['description']}'. The draft's `fixed_at` "
+ f"marker does not match the vault's actual state — you must fix the "
+ f"underlying issue (not just the bookkeeping). Run "
+ f"`$HPR lint --rule {guard['rule']} -j` to see what's still broken."
+ ),
+ })
+
summary = {
"errors": sum(1 for i in issues if i.get("severity") == "error"),
"warnings": sum(1 for i in issues if i.get("severity") == "warning"),
diff --git a/src/hyperresearch/cli/note.py b/src/hyperresearch/cli/note.py
index b4f443f..9f668f8 100644
--- a/src/hyperresearch/cli/note.py
+++ b/src/hyperresearch/cli/note.py
@@ -24,6 +24,8 @@ def note_new(
status: str = typer.Option("draft", "--status", "-s", help="Initial status"),
summary: str | None = typer.Option(None, "--summary", help="One-line summary"),
source: str | None = typer.Option(None, "--source", help="Source URL or path"),
+ tier: str | None = typer.Option(None, "--tier", help="Epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),
+ content_type: str | None = typer.Option(None, "--content-type", help="Artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),
template: str | None = typer.Option(None, "--template", "-T", help="Template: note|concept|reference|guide|comparison|moc"),
edit: bool = typer.Option(False, "--edit", "-e", help="Open in $EDITOR"),
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
@@ -38,7 +40,29 @@ def note_new(
from hyperresearch.core.note import write_note
from hyperresearch.core.vault import Vault
- from hyperresearch.models.note import slugify
+ from hyperresearch.models.note import ContentType, Tier, slugify
+
+ # Validate enums up-front so invalid values fail clearly
+ if tier is not None:
+ try:
+ Tier(tier)
+ except ValueError:
+ valid = ", ".join(t.value for t in Tier)
+ if json_output:
+ output(error(f"Invalid --tier '{tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --tier '{tier}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
+ if content_type is not None:
+ try:
+ ContentType(content_type)
+ except ValueError:
+ valid = ", ".join(c.value for c in ContentType)
+ if json_output:
+ output(error(f"Invalid --content-type '{content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --content-type '{content_type}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
vault = Vault.discover()
vault.auto_sync()
@@ -92,11 +116,11 @@ def note_new(
console.print(f"[yellow]Template '{template}' not found, using default.[/]")
path = write_note(vault.notes_dir, title, body=body, tags=tags, status=status,
note_type=note_type, parent=parent, summary=summary,
- source=source)
+ source=source, tier=tier, content_type=content_type)
else:
path = write_note(vault.notes_dir, title, body=body, tags=tags, status=status,
note_type=note_type, parent=parent, summary=summary,
- source=source)
+ source=source, tier=tier, content_type=content_type)
# Read back the note ID (may have been collision-adjusted)
from hyperresearch.core.note import read_note
@@ -154,6 +178,8 @@ def note_show(
data = {
"id": row["id"], "title": row["title"], "path": row["path"],
"status": row["status"], "type": row["type"], "tags": tags,
+ "tier": row["tier"] if "tier" in row.keys() else None,
+ "content_type": row["content_type"] if "content_type" in row.keys() else None,
"created": row["created"], "updated": row["updated"],
"word_count": row["word_count"], "source": row["source"],
"parent": row["parent"], "summary": row["summary"],
@@ -232,6 +258,8 @@ def note_list(
note_type: str | None = typer.Option(None, "--type", help="Filter by type"),
tag: str | None = typer.Option(None, "--tag", "-t", help="Filter by tag"),
parent: str | None = typer.Option(None, "--parent", "-p", help="Filter by parent"),
+ tier: str | None = typer.Option(None, "--tier", help="Filter by epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),
+ content_type: str | None = typer.Option(None, "--content-type", help="Filter by artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),
sort: str = typer.Option("updated", "--sort", help="Sort: created|updated|title|words"),
limit: int = typer.Option(20, "--limit", "-l", help="Max results"),
all_notes: bool = typer.Option(False, "--all", "-a", help="Return all notes (no limit)"),
@@ -239,6 +267,29 @@ def note_list(
) -> None:
"""List notes with optional filters."""
from hyperresearch.core.vault import Vault
+ from hyperresearch.models.note import ContentType, Tier
+
+ # Validate enums up-front
+ if tier is not None:
+ try:
+ Tier(tier)
+ except ValueError:
+ valid = ", ".join(t.value for t in Tier)
+ if json_output:
+ output(error(f"Invalid --tier '{tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --tier '{tier}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
+ if content_type is not None:
+ try:
+ ContentType(content_type)
+ except ValueError:
+ valid = ", ".join(c.value for c in ContentType)
+ if json_output:
+ output(error(f"Invalid --content-type '{content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --content-type '{content_type}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
vault = Vault.discover()
vault.auto_sync()
@@ -258,6 +309,12 @@ def note_list(
if tag:
clauses.append("n.id IN (SELECT note_id FROM tags WHERE tag = ?)")
params.append(tag.lower())
+ if tier:
+ clauses.append("n.tier = ?")
+ params.append(tier)
+ if content_type:
+ clauses.append("n.content_type = ?")
+ params.append(content_type)
where = " AND ".join(clauses) if clauses else "1=1"
@@ -286,6 +343,8 @@ def note_list(
"path": row["path"],
"status": row["status"],
"type": row["type"],
+ "tier": row["tier"] if "tier" in row.keys() else None,
+ "content_type": row["content_type"] if "content_type" in row.keys() else None,
"tags": tag_list,
"word_count": row["word_count"],
"summary": row["summary"],
@@ -334,6 +393,8 @@ def note_update(
set_summary: str | None = typer.Option(None, "--summary", help="Set summary"),
set_parent: str | None = typer.Option(None, "--parent", "-p", help="Set parent topic"),
set_source: str | None = typer.Option(None, "--source", help="Set source URL/path"),
+ set_tier: str | None = typer.Option(None, "--tier", help="Set epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),
+ set_content_type: str | None = typer.Option(None, "--content-type", help="Set artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),
deprecate: bool = typer.Option(False, "--deprecate", help="Mark as deprecated"),
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
) -> None:
@@ -343,6 +404,29 @@ def note_update(
from hyperresearch.core.frontmatter import parse_frontmatter, serialize_frontmatter
from hyperresearch.core.sync import compute_sync_plan, execute_sync
from hyperresearch.core.vault import Vault
+ from hyperresearch.models.note import ContentType, Tier
+
+ # Validate enums up-front
+ if set_tier is not None:
+ try:
+ Tier(set_tier)
+ except ValueError:
+ valid = ", ".join(t.value for t in Tier)
+ if json_output:
+ output(error(f"Invalid --tier '{set_tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --tier '{set_tier}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
+ if set_content_type is not None:
+ try:
+ ContentType(set_content_type)
+ except ValueError:
+ valid = ", ".join(c.value for c in ContentType)
+ if json_output:
+ output(error(f"Invalid --content-type '{set_content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --content-type '{set_content_type}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
vault = Vault.discover()
vault.auto_sync()
@@ -380,6 +464,12 @@ def note_update(
if set_source is not None:
meta.source = set_source
changed.append("source")
+ if set_tier is not None:
+ meta.tier = set_tier
+ changed.append(f"tier={set_tier}")
+ if set_content_type is not None:
+ meta.content_type = set_content_type
+ changed.append(f"content_type={set_content_type}")
if deprecate:
meta.deprecated = True
meta.status = "deprecated"
@@ -448,11 +538,22 @@ def note_rm(
force: bool = typer.Option(False, "--force", "-f", help="Skip confirmation"),
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
) -> None:
- """Delete a note."""
+ """Delete a note and its associated raw file and assets.
+
+ Previous versions only unlinked the `.md` file, leaving raw PDFs under
+ `research/raw/` and assets under `research/assets/<id>/` as orphans.
+ Every fetch-then-delete cycle leaked disk. The current implementation
+ also removes:
+ - the raw file referenced in the note's `raw_file` frontmatter field
+ - any entries in the `assets` table and their files on disk
+ """
+ import shutil
from hyperresearch.core.vault import Vault
vault = Vault.discover()
- row = vault.db.execute("SELECT path FROM notes WHERE id = ?", (note_id,)).fetchone()
+ row = vault.db.execute(
+ "SELECT path FROM notes WHERE id = ?", (note_id,)
+ ).fetchone()
if not row:
if json_output:
output(error(f"Note not found: {note_id}", "NOT_FOUND"), json_mode=True)
@@ -464,6 +565,34 @@ def note_rm(
if not force and not json_output:
typer.confirm(f"Delete {row['path']}?", abort=True)
+ removed_raw: str | None = None
+ removed_assets: list[str] = []
+
+ # Read raw_file straight from the markdown frontmatter — the DB row
+ # may not carry that field if the schema was synced before the column
+ # existed. File parsing is authoritative.
+ if file_path.exists():
+ try:
+ from hyperresearch.core.frontmatter import parse_frontmatter
+ text = file_path.read_text(encoding="utf-8-sig")
+ meta, _ = parse_frontmatter(text)
+ if meta.raw_file:
+ raw_path = vault.root / "research" / meta.raw_file
+ if raw_path.exists():
+ raw_path.unlink()
+ removed_raw = str(raw_path.relative_to(vault.root).as_posix())
+ except Exception:
+ pass
+
+ # Assets directory
+ assets_dir = vault.root / "research" / "assets" / note_id
+ if assets_dir.exists() and assets_dir.is_dir():
+ for asset_file in assets_dir.iterdir():
+ if asset_file.is_file():
+ removed_assets.append(asset_file.name)
+ shutil.rmtree(assets_dir, ignore_errors=True)
+
+ # Finally, unlink the .md file
if file_path.exists():
file_path.unlink()
@@ -473,9 +602,20 @@ def note_rm(
plan = compute_sync_plan(vault)
execute_sync(vault, plan)
+ payload: dict = {"deleted": note_id}
+ if removed_raw:
+ payload["removed_raw"] = removed_raw
+ if removed_assets:
+ payload["removed_assets"] = removed_assets
+
if json_output:
- output(success({"deleted": note_id}, vault=str(vault.root)), json_mode=True)
+ output(success(payload, vault=str(vault.root)), json_mode=True)
else:
- console.print(f"[red]Deleted:[/] {note_id}")
+ msg = f"[red]Deleted:[/] {note_id}"
+ if removed_raw:
+ msg += f"\n raw: {removed_raw}"
+ if removed_assets:
+ msg += f"\n assets: {len(removed_assets)} file(s)"
+ console.print(msg)
diff --git a/src/hyperresearch/core/note.py b/src/hyperresearch/core/note.py
index 9ea8c37..cbccd87 100644
--- a/src/hyperresearch/core/note.py
+++ b/src/hyperresearch/core/note.py
@@ -8,7 +8,12 @@ from datetime import UTC, datetime
from pathlib import Path
from hyperresearch.core.frontmatter import parse_frontmatter, render_note
-from hyperresearch.core.patterns import CODE_BLOCK_RE, INLINE_CODE_RE, WIKI_LINK_RE
+from hyperresearch.core.patterns import (
+ CODE_BLOCK_RE,
+ INLINE_CODE_RE,
+ WIKI_LINK_RE,
+ is_valid_wiki_link_target,
+)
from hyperresearch.models.note import Note, NoteMeta, slugify
@@ -25,14 +30,12 @@ def read_note(file_path: Path, vault_root: Path) -> Note:
if not meta.id:
meta.id = slugify(file_path.stem)
- # Extract outgoing wiki links
+ # Extract outgoing wiki links, filtering citation footnotes and URLs
cleaned = CODE_BLOCK_RE.sub("", body)
cleaned = INLINE_CODE_RE.sub("", cleaned)
raw_links = (m.group(1).strip().rstrip("\\") for m in WIKI_LINK_RE.finditer(cleaned))
outgoing = list(dict.fromkeys(
- ref for ref in raw_links
- if ref and not ref.startswith(("http://", "https://", "ftp://", "mailto:"))
- and not ref.isdigit() # Skip [[100]]-style citation numbers
+ ref for ref in raw_links if is_valid_wiki_link_target(ref)
))
# Word count (simple split)
@@ -60,12 +63,16 @@ def write_note(
parent: str | None = None,
source: str | None = None,
summary: str | None = None,
+ tier: str | None = None,
+ content_type: str | None = None,
extra_frontmatter: dict | None = None,
) -> Path:
"""Create a new note file on disk. Returns the file path.
Args:
notes_dir: The directory to write into (e.g. vault.notes_dir).
+ tier: Epistemic role — ground_truth|institutional|practitioner|commentary|unknown.
+ content_type: Artifact kind — paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown.
extra_frontmatter: Additional fields to set on NoteMeta (e.g. source_domain, fetched_at).
"""
nid = note_id or slugify(title)
@@ -78,6 +85,8 @@ def write_note(
parent=parent,
source=source,
summary=summary,
+ tier=tier,
+ content_type=content_type,
created=datetime.now(UTC),
)
if extra_frontmatter:
diff --git a/src/hyperresearch/core/patterns.py b/src/hyperresearch/core/patterns.py
index bd28465..174b660 100644
--- a/src/hyperresearch/core/patterns.py
+++ b/src/hyperresearch/core/patterns.py
@@ -10,3 +10,84 @@ WIKI_LINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")
# Code block patterns for stripping before link extraction
CODE_BLOCK_RE = re.compile(r"```.*?```", re.DOTALL)
INLINE_CODE_RE = re.compile(r"`[^`]+`")
+
+
+# Citation-footnote patterns that should NOT be treated as wiki-links even
+# though crawl4ai sometimes renders them as [[100]] or [[^100]] in fetched
+# markdown. These originate from Wikipedia-style footnote markup like
+# `[[100]](https://en.wikipedia.org/wiki/Foo#cite_note-100)` where the
+# visible text is the citation number in brackets.
+_CITATION_FOOTNOTE_RE = re.compile(
+ r"^\^?\d+(?:[\s,;\-]+\d+)*$"
+)
+
+# Explicit citation prefixes: [[cite-1]], [[ref_2]], [[fn:3]], [[note-4]]
+_CITE_PREFIX_RE = re.compile(
+ r"^(?:cite|ref|note|fn|footnote|endnote)[-_:]?\d+$",
+ re.IGNORECASE,
+)
+
+# Roman-numeral footnotes: [[iv]], [[xii]], [[ccxliv]] (lowercase or upper).
+# Academic papers use these for appendices and preface footnotes. Uses the
+# standard Roman-numeral grammar (not just "chars in [ivxlcdm]") so real
+# words like "civic" or "mix" don't match by accident. The empty string is
+# explicitly excluded — it would otherwise match M{0}X{0}I{0}.
+_ROMAN_FOOTNOTE_RE = re.compile(
+ r"^(?=[ivxlcdm])M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$",
+ re.IGNORECASE,
+)
+
+# Symbol footnotes: [[*]], [[**]], [[†]], [[‡]], [[§]], [[¶]], [[#]], [[△]].
+# Typography-style footnote markers used in older academic publishing.
+_SYMBOL_FOOTNOTE_RE = re.compile(
+ r"^[\*\u2020\u2021\u00a7\u00b6#\u25b3\u25bd]{1,3}$"
+)
+
+# Figure / table / equation references: [[fig-3]], [[tab-1]], [[eq-2]],
+# [[table1]], [[figure-4b]], [[eq:7]], [[scheme-3]]. These are cross-
+# references within a document, not wiki-links to other notes.
+_DOC_REF_PREFIX_RE = re.compile(
+ r"^(?:fig(?:ure)?|tab(?:le)?|eq(?:uation)?|scheme|chart|plate|box|chart|algo(?:rithm)?)[-_:]?\w*$",
+ re.IGNORECASE,
+)
+
+
+def is_valid_wiki_link_target(ref: str) -> bool:
+ """Return True if a ``[[ref]]`` should be treated as a note reference.
+
+ Filters out content that crawl4ai and other markdown fetchers sometimes
+ produce as double-bracket targets but that should NOT resolve to notes:
+
+ - Empty / whitespace-only refs
+ - URLs (``http://``, ``https://``, ``ftp://``, ``mailto:``)
+ - Anchors (``#section-id``) and absolute paths (``/path/to/thing``)
+ - Pure-numeric citation footnotes: ``[[100]]``, ``[[^100]]``,
+ ``[[100,101]]``, ``[[1-5]]``, ``[[100; 101]]``
+ - Explicit citation prefixes: ``[[cite-1]]``, ``[[ref_2]]``, ``[[fn:3]]``,
+ ``[[note-4]]``, ``[[footnote-5]]``
+ - Roman-numeral footnotes: ``[[iv]]``, ``[[xii]]``
+ - Symbol footnotes: ``[[*]]``, ``[[**]]``, ``[[†]]``, ``[[‡]]``, ``[[§]]``
+ - Document cross-references: ``[[fig-3]]``, ``[[tab-1]]``, ``[[eq-2]]``,
+ ``[[figure-4b]]``, ``[[scheme-3]]``, ``[[algorithm-2]]``
+
+ Valid note references starting with digits (e.g. ``[[10-rules-for-X]]``)
+ are preserved because they contain non-digit characters.
+ """
+ if not ref:
+ return False
+ ref = ref.strip()
+ if not ref:
+ return False
+ if ref.startswith(("http://", "https://", "ftp://", "mailto:", "#", "/")):
+ return False
+ if _CITATION_FOOTNOTE_RE.match(ref):
+ return False
+ if _CITE_PREFIX_RE.match(ref):
+ return False
+ if _ROMAN_FOOTNOTE_RE.match(ref):
+ return False
+ if _SYMBOL_FOOTNOTE_RE.match(ref):
+ return False
+ if _DOC_REF_PREFIX_RE.match(ref):
+ return False
+ return True
diff --git a/src/hyperresearch/core/sync.py b/src/hyperresearch/core/sync.py
index 7df9805..cb9be24 100644
--- a/src/hyperresearch/core/sync.py
+++ b/src/hyperresearch/core/sync.py
@@ -10,7 +10,12 @@ from datetime import UTC, datetime
from pathlib import Path
from hyperresearch.core.note import read_note, strip_markdown
-from hyperresearch.core.patterns import CODE_BLOCK_RE, INLINE_CODE_RE, WIKI_LINK_RE
+from hyperresearch.core.patterns import (
+ CODE_BLOCK_RE,
+ INLINE_CODE_RE,
+ WIKI_LINK_RE,
+ is_valid_wiki_link_target,
+)
@dataclass
@@ -42,13 +47,24 @@ def compute_sync_plan(vault, force: bool = False) -> SyncPlan:
plan = SyncPlan()
# Only scan inside the research directory (notes/, index/)
- # This avoids walking .git/, .venv/, src/, etc. entirely
+ # This avoids walking .git/, .venv/, src/, etc. entirely.
+ #
+ # Files at the research/ root (e.g. research/scaffold.md,
+ # research/comparisons.md, research/synthesis.md) are STAGING files the
+ # agent writes then registers as real notes via `note new --body-file`.
+ # They must NOT be synced as notes themselves — otherwise every run
+ # produces 4 orphan notes and the missing-title/missing-tags/missing-summary
+ # lint rules spam warnings for every research session.
kb_dir = vault.research_dir
if not kb_dir.exists():
return plan
disk_files: dict[str, float] = {}
for md_file in kb_dir.rglob("*.md"):
+ # Skip staging files at the research/ root. Real notes live in
+ # research/notes/** or research/index/**.
+ if md_file.parent == kb_dir:
+ continue
rel = md_file.relative_to(vault.root).as_posix()
disk_files[rel] = md_file.stat().st_mtime
@@ -155,13 +171,14 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Non
conn.execute(
"""
INSERT INTO notes
- (id, title, path, status, type, source, parent,
+ (id, title, path, status, type, tier, content_type, source, parent,
deprecated, reviewed, expires, word_count, summary,
created, updated, file_mtime, content_hash, synced_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
title=excluded.title, path=excluded.path, status=excluded.status,
- type=excluded.type, source=excluded.source, parent=excluded.parent,
+ type=excluded.type, tier=excluded.tier, content_type=excluded.content_type,
+ source=excluded.source, parent=excluded.parent,
deprecated=excluded.deprecated,
reviewed=excluded.reviewed, expires=excluded.expires,
word_count=excluded.word_count, summary=excluded.summary,
@@ -171,6 +188,7 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Non
""",
(
meta.id, meta.title, note.path, meta.status, meta.type,
+ meta.tier, meta.content_type,
meta.source, meta.parent, 1 if meta.deprecated else 0,
reviewed_iso, expires_iso,
note.word_count, meta.summary, created_iso, updated_iso,
@@ -213,16 +231,14 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Non
)
# Update links — extract from original body (not stripped)
+ # Uses the shared filter so this path stays in sync with core.note.read_note
conn.execute("DELETE FROM links WHERE source_id = ?", (meta.id,))
cleaned = CODE_BLOCK_RE.sub("", note.body)
cleaned = INLINE_CODE_RE.sub("", cleaned)
for line_num, line in enumerate(cleaned.split("\n"), 1):
for m in WIKI_LINK_RE.finditer(line):
target_ref = m.group(1).strip().rstrip("\\") # Strip trailing backslash (shell escaping artifact)
- # Skip URLs — they're not note references
- if target_ref.startswith(("http://", "https://", "ftp://", "mailto:")):
- continue
- if not target_ref:
+ if not is_valid_wiki_link_target(target_ref):
continue
conn.execute(
"INSERT OR IGNORE INTO links (source_id, target_ref, line_number, context) "
diff --git a/tests/test_cli/test_lint.py b/tests/test_cli/test_lint.py
new file mode 100644
index 0000000..dace659
--- /dev/null
+++ b/tests/test_cli/test_lint.py
@@ -0,0 +1,517 @@
+"""Tests for lint rules — especially the gospel-enforcing scaffold-prompt rule."""
+
+from __future__ import annotations
+
+import pytest
+from typer.testing import CliRunner
+
+from hyperresearch.cli.lint import app as lint_app
+from hyperresearch.core.note import write_note
+
+
+def _run_lint(vault, rule: str | None = None) -> tuple[int, str]:
+ """Invoke `hyperresearch lint` CLI against a vault. Returns (exit_code, stdout)."""
+ import os
+
+ runner = CliRunner()
+ prev_cwd = os.getcwd()
+ try:
+ os.chdir(vault.root)
+ args = ["--json"]
+ if rule:
+ args = ["--rule", rule, "--json"]
+ result = runner.invoke(lint_app, args, catch_exceptions=False)
+ return result.exit_code, result.output
+ finally:
+ os.chdir(prev_cwd)
+
+
+def _write_scaffold(vault, body: str, note_id: str = "scaffold-test"):
+ write_note(
+ vault.notes_dir,
+ "Scaffold: Test",
+ body=body,
+ tags=["scaffold"],
+ note_id=note_id,
+ )
+ vault.auto_sync()
+
+
+def test_scaffold_prompt_passes_with_verbatim_prompt(tmp_vault):
+ _write_scaffold(
+ tmp_vault,
+ body=(
+ "## User Prompt (VERBATIM — gospel)\n"
+ "> I would like a detailed analysis of the Saint Seiya franchise and its armor classes. "
+ "For each significant character please describe techniques and fate.\n\n"
+ "## What the user explicitly asked for\n"
+ "- Entity enumeration\n"
+ ),
+ )
+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")
+ assert code == 0
+ # Output should contain no scaffold-prompt issues
+ import json
+ data = json.loads(out)
+ issues = data.get("data", {}).get("issues", [])
+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]
+ assert scaffold_issues == []
+
+
+def test_scaffold_prompt_fails_when_header_missing(tmp_vault):
+ _write_scaffold(
+ tmp_vault,
+ body=(
+ "## Thesis\n"
+ "Saint Seiya's armor hierarchy encodes a theology of sacrifice.\n\n"
+ "## Heading progression\n"
+ "1. Cosmology\n"
+ ),
+ )
+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")
+ import json
+ data = json.loads(out)
+ issues = data.get("data", {}).get("issues", [])
+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]
+ assert len(scaffold_issues) == 1
+ assert scaffold_issues[0]["severity"] == "error"
+ assert "verbatim" in scaffold_issues[0]["message"].lower()
+
+
+def test_scaffold_prompt_warns_when_quote_too_short(tmp_vault):
+ _write_scaffold(
+ tmp_vault,
+ body=(
+ "## User Prompt (VERBATIM — gospel)\n"
+ "> hi\n\n"
+ "## What the user explicitly asked for\n"
+ "- thing\n"
+ ),
+ )
+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")
+ import json
+ data = json.loads(out)
+ issues = data.get("data", {}).get("issues", [])
+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]
+ assert len(scaffold_issues) == 1
+ assert scaffold_issues[0]["severity"] == "warning"
+
+
+def test_scaffold_prompt_no_scaffold_notes_is_noop(tmp_vault):
+ # A vault with no scaffold-tagged notes should produce no issues for this rule.
+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")
+ assert code == 0
+ import json
+ data = json.loads(out)
+ issues = data.get("data", {}).get("issues", [])
+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]
+ assert scaffold_issues == []
+
+
+def _write_source(vault, title: str, note_id: str, body: str = "Source content."):
+ write_note(
+ vault.notes_dir,
+ title,
+ body=body,
+ note_id=note_id,
+ source=f"https://example.com/{note_id}",
+ tier="institutional",
+ content_type="article",
+ )
+
+
+def test_provenance_rooted_tree_passes_with_valid_chain(tmp_vault):
+ # Build a valid chain: seed → child1 → child2, plus a few more non-seeds
+ # pointing at the seed or each other. 6+ sources so the rule activates.
+ _write_source(tmp_vault, "Seed Source", "seed-one", body="Seminal paper on X.")
+ _write_source(tmp_vault, "Child A", "child-a", body="*Suggested by [[seed-one]] — follow-up citation*\n\nDerivative work.")
+ _write_source(tmp_vault, "Child B", "child-b", body="*Suggested by [[seed-one]] — cross-reference*\n\nRelated analysis.")
+ _write_source(tmp_vault, "Grandchild", "grandchild", body="*Suggested by [[child-a]] — deeper dive*\n\nMore content.")
+ _write_source(tmp_vault, "Extra seed", "seed-two", body="Another primary source.")
+ _write_source(tmp_vault, "Child C", "child-c", body="*Suggested by [[seed-two]] — reply*\n\nCritical response.")
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="provenance")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]
+ assert issues == [], f"expected no issues, got: {issues}"
+
+
+def test_provenance_fails_when_all_sources_are_seeds(tmp_vault):
+ # 6 sources, zero breadcrumbs — flat batch, rule must fire.
+ for i in range(6):
+ _write_source(tmp_vault, f"Source {i}", f"source-{i}")
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="provenance")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]
+ assert len(issues) >= 1
+ assert any("ZERO" in i["message"] or "no seed" in i["message"].lower() or "bouncing reading loop" in i["message"] for i in issues)
+
+
+def test_provenance_fails_on_dangling_breadcrumb(tmp_vault):
+ # Six sources; one of them references a non-existent note.
+ _write_source(tmp_vault, "Seed", "seed-one")
+ _write_source(tmp_vault, "Child A", "child-a", body="*Suggested by [[seed-one]] — real*\n")
+ _write_source(tmp_vault, "Child B", "child-b", body="*Suggested by [[seed-one]] — real*\n")
+ _write_source(tmp_vault, "Dangler", "dangler", body="*Suggested by [[nonexistent-note]] — fake*\n")
+ _write_source(tmp_vault, "Child C", "child-c", body="*Suggested by [[seed-one]] — real*\n")
+ _write_source(tmp_vault, "Child D", "child-d", body="*Suggested by [[seed-one]] — real*\n")
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="provenance")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]
+ assert any("nonexistent-note" in i["message"] for i in issues), (
+ f"expected dangling breadcrumb issue, got: {issues}"
+ )
+
+
+def test_provenance_small_corpus_is_skipped(tmp_vault):
+ # <= 5 sources, rule should not complain (loop may not have fired by design).
+ for i in range(3):
+ _write_source(tmp_vault, f"Source {i}", f"source-{i}")
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="provenance")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]
+ assert issues == []
+
+
+def test_provenance_errors_on_under_30pct_non_seed_ratio(tmp_vault):
+ """The guided reading loop must actually fire. A large corpus with only
+ a token 1-2 breadcrumbs should ERROR, not just warn. This is the exact
+ failure mode both v2 runs exhibited (3/19 and 2/33 breadcrumbs)."""
+ # 11 sources, 2 of them with breadcrumbs = 18% non-seed (below 30%)
+ _write_source(tmp_vault, "Seed 1", "seed-one")
+ _write_source(tmp_vault, "Seed 2", "seed-two")
+ for i in range(9):
+ # 2 of the 9 have breadcrumbs; rest are seeds
+ if i < 2:
+ body = f"*Suggested by [[seed-one]] — from analyst*\n\nContent {i}"
+ else:
+ body = f"Content {i}"
+ _write_source(tmp_vault, f"Source {i}", f"source-{i}", body=body)
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="provenance")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]
+ errors = [i for i in issues if i.get("severity") == "error"]
+ assert len(errors) >= 1
+ assert "guided reading loop did not fire" in errors[0]["message"]
+
+
+def test_orphaned_raw_files_flags_disk_leak(tmp_vault):
+ """Files in research/raw/ with no matching note should be flagged."""
+ raw_dir = tmp_vault.root / "research" / "raw"
+ raw_dir.mkdir(parents=True, exist_ok=True)
+ # Create a raw file whose stem doesn't match any note.
+ (raw_dir / "orphan-note.pdf").write_bytes(b"%PDF-1.4 dummy")
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="orphaned-raw-files")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "orphaned-raw-files"]
+ assert len(issues) == 1
+ assert "orphan-note" in issues[0]["message"]
+
+
+def _write_audit_findings(vault, data: dict) -> None:
+ import json as _json
+ audit_path = vault.root / "research" / "audit_findings.json"
+ audit_path.parent.mkdir(parents=True, exist_ok=True)
+ audit_path.write_text(_json.dumps(data, indent=2), encoding="utf-8")
+
+
+def test_audit_gate_missing_file_is_open(tmp_vault):
+ """No audit file = gate is OPEN (early-stage research)."""
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ assert code == 0
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ assert issues == []
+
+
+def test_audit_gate_blocks_unresolved_criticals(tmp_vault):
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T10:00:00Z",
+ "status": "needs_fixes",
+ "criticals": [
+ {"id": "C0", "description": "Scaffold missing verbatim prompt", "fixed_at": None},
+ {"id": "C1", "description": "Silver Saints omitted", "fixed_at": None},
+ ],
+ "important": [],
+ "minor": [],
+ }
+ ]
+ })
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ assert len(issues) == 1
+ assert "2 unresolved CRITICAL" in issues[0]["message"]
+
+
+def test_audit_gate_passes_when_all_criticals_fixed(tmp_vault):
+ """A CRITICAL with a vague description that doesn't map to any lint rule
+ should emit a WARNING (unverified agent self-report) but NOT an error —
+ the save gate still opens because warnings don't block."""
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T10:00:00Z",
+ "status": "pass",
+ "criticals": [
+ {"id": "C0", "description": "generic content fix", "fixed_at": "2026-04-14T11:00:00Z"},
+ ],
+ "important": [],
+ "minor": [],
+ }
+ ]
+ })
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ errors = [i for i in issues if i.get("severity") == "error"]
+ warnings = [i for i in issues if i.get("severity") == "warning"]
+ # Save gate stays open (no errors) but emits a warning about the
+ # unverified self-report.
+ assert errors == []
+ assert len(warnings) == 1
+ assert "not machine-verified" in warnings[0]["message"]
+
+
+def test_audit_gate_uses_most_recent_conformance_run(tmp_vault):
+ """If a later run has fixes applied, the gate should pass even if older
+ runs had unresolved findings."""
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T10:00:00Z",
+ "status": "needs_fixes",
+ "criticals": [
+ {"id": "C0", "description": "old finding", "fixed_at": None},
+ ],
+ "important": [], "minor": [],
+ },
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T12:00:00Z",
+ "status": "pass",
+ "criticals": [],
+ "important": [], "minor": [],
+ },
+ ]
+ })
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ assert issues == []
+
+
+def test_audit_gate_fails_when_only_comprehensiveness_run_exists(tmp_vault):
+ """A file with only comprehensiveness runs means the conformance auditor
+ never fired. The save gate must fail in that case."""
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "comprehensiveness",
+ "timestamp": "2026-04-14T10:00:00Z",
+ "status": "needs_fixes",
+ "criticals": [],
+ "important": [
+ {"id": "I1", "description": "some gap", "fixed_at": None},
+ ],
+ "minor": [],
+ }
+ ]
+ })
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ # Expect: one error for missing conformance, one info for important findings.
+ errors = [i for i in issues if i.get("severity") == "error"]
+ infos = [i for i in issues if i.get("severity") == "info"]
+ assert len(errors) == 1
+ assert "ZERO" in errors[0]["message"] and "conformance" in errors[0]["message"]
+ assert len(infos) == 1
+ assert "IMPORTANT" in infos[0]["message"]
+
+
+def test_audit_gate_surfaces_important_findings_as_info(tmp_vault):
+ """Unresolved IMPORTANT findings should surface as info-severity issues
+ (advisory), not block save."""
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T10:00:00Z",
+ "status": "pass",
+ "criticals": [],
+ "important": [
+ {"id": "I1", "description": "Proportional depth uneven", "fixed_at": None},
+ {"id": "I2", "description": "Citation to unfetched source", "fixed_at": None},
+ ],
+ "minor": [],
+ }
+ ]
+ })
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ errors = [i for i in issues if i.get("severity") == "error"]
+ infos = [i for i in issues if i.get("severity") == "info"]
+ assert errors == [] # save is NOT blocked by IMPORTANT alone
+ assert len(infos) == 1
+ assert "2 unresolved IMPORTANT" in infos[0]["message"]
+
+
+def test_audit_gate_catches_self_certification_on_provenance(tmp_vault):
+ """Regression test: when a CRITICAL finding mentions provenance AND is
+ marked `fixed_at`, but the provenance rule still returns errors, the
+ audit-gate must emit a SELF-CERTIFICATION VIOLATION error.
+
+ This is exactly the Q62 failure mode — the agent marked the C1
+ provenance finding as fixed with a justification string, but the
+ vault's actual breadcrumb graph was still broken.
+ """
+ # Build a vault where provenance is genuinely broken (12 sources, none
+ # with breadcrumbs — classic flat batch).
+ for i in range(12):
+ _write_source(tmp_vault, f"Source {i}", f"source-{i}")
+ tmp_vault.auto_sync()
+
+ # Audit findings file: conformance run with a CRITICAL marked fixed_at
+ # that references provenance. The "fix" is a lie — the vault still fails
+ # the provenance rule.
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T20:00:00Z",
+ "status": "pass",
+ "criticals": [
+ {
+ "id": "C1",
+ "description": "Provenance chain broken — no --suggested-by breadcrumbs in corpus",
+ "fixed_at": "2026-04-14T20:20:00Z",
+ }
+ ],
+ "important": [],
+ "minor": [],
+ }
+ ]
+ })
+
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = data.get("data", {}).get("issues", [])
+ audit_errors = [
+ i for i in issues
+ if i.get("rule") == "audit-gate" and i.get("severity") == "error"
+ ]
+ self_cert_errors = [
+ i for i in audit_errors if "SELF-CERTIFICATION VIOLATION" in i["message"]
+ ]
+ assert len(self_cert_errors) == 1, f"expected self-cert violation, got: {audit_errors}"
+ assert "C1" in self_cert_errors[0]["message"]
+ assert "provenance" in self_cert_errors[0]["message"]
+
+
+def test_audit_gate_no_self_cert_when_fix_genuinely_landed(tmp_vault):
+ """Control: when CRITICAL fixed_at IS set AND the underlying lint rule
+ genuinely passes, the audit-gate should NOT emit a self-cert violation."""
+ # Build a vault where provenance is healthy (seed + 6 non-seeds, all
+ # pointing at real notes).
+ _write_source(tmp_vault, "Seed", "seed-one")
+ for i in range(6):
+ _write_source(
+ tmp_vault, f"Child {i}", f"child-{i}",
+ body=f"*Suggested by [[seed-one]] — real*\n\nContent {i}"
+ )
+ tmp_vault.auto_sync()
+
+ _write_audit_findings(tmp_vault, {
+ "runs": [
+ {
+ "mode": "conformance",
+ "timestamp": "2026-04-14T20:00:00Z",
+ "status": "pass",
+ "criticals": [
+ {
+ "id": "C1",
+ "description": "provenance was broken earlier",
+ "fixed_at": "2026-04-14T20:20:00Z",
+ }
+ ],
+ "important": [],
+ "minor": [],
+ }
+ ]
+ })
+
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = data.get("data", {}).get("issues", [])
+ self_cert_errors = [
+ i for i in issues
+ if i.get("rule") == "audit-gate" and "SELF-CERTIFICATION VIOLATION" in i.get("message", "")
+ ]
+ assert self_cert_errors == [], f"false-positive self-cert: {self_cert_errors}"
+
+
+def test_audit_gate_handles_malformed_file(tmp_vault):
+ audit_path = tmp_vault.root / "research" / "audit_findings.json"
+ audit_path.parent.mkdir(parents=True, exist_ok=True)
+ audit_path.write_text("{ not valid json }", encoding="utf-8")
+ code, out = _run_lint(tmp_vault, rule="audit-gate")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]
+ assert len(issues) == 1
+ assert "malformed" in issues[0]["message"]
+
+
+def test_orphaned_raw_files_ignores_matched_raw(tmp_vault):
+ """Raw files whose stem matches a note id are not orphans."""
+ write_note(
+ tmp_vault.notes_dir,
+ "PDF Note",
+ note_id="real-pdf-note",
+ source="https://example.com/paper.pdf",
+ tier="ground_truth",
+ content_type="paper",
+ extra_frontmatter={"raw_file": "raw/real-pdf-note.pdf"},
+ )
+ raw_dir = tmp_vault.root / "research" / "raw"
+ raw_dir.mkdir(parents=True, exist_ok=True)
+ (raw_dir / "real-pdf-note.pdf").write_bytes(b"%PDF-1.4 dummy")
+ tmp_vault.auto_sync()
+
+ code, out = _run_lint(tmp_vault, rule="orphaned-raw-files")
+ import json
+ data = json.loads(out)
+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "orphaned-raw-files"]
+ assert issues == []
diff --git a/tests/test_cli/test_note_ops.py b/tests/test_cli/test_note_ops.py
index bc1be8f..61d9d19 100644
--- a/tests/test_cli/test_note_ops.py
+++ b/tests/test_cli/test_note_ops.py
@@ -38,6 +38,55 @@ def test_note_rm_nonexistent(vault_with_notes):
assert result.exit_code == 1
+def test_note_rm_cleans_raw_file_and_assets(vault_with_notes):
+ """note rm must delete the raw file and assets directory, not just the .md.
+
+ Regression test for Batch 2.5 — prior to this, `note rm` only unlinked
+ the `.md` file and raw/assets leaked on disk forever.
+ """
+ vault_root = vault_with_notes
+
+ # Seed a note with raw_file in its frontmatter and an assets directory.
+ from hyperresearch.core.note import write_note
+ from hyperresearch.core.vault import Vault
+ vault = Vault.discover()
+
+ write_note(
+ vault.notes_dir,
+ "PDF Source",
+ note_id="pdf-source",
+ source="https://example.com/paper.pdf",
+ tier="ground_truth",
+ content_type="paper",
+ extra_frontmatter={"raw_file": "raw/pdf-source.pdf"},
+ )
+
+ raw_dir = vault_root / "research" / "raw"
+ raw_dir.mkdir(parents=True, exist_ok=True)
+ raw_file = raw_dir / "pdf-source.pdf"
+ raw_file.write_bytes(b"%PDF-1.4 test")
+
+ assets_dir = vault_root / "research" / "assets" / "pdf-source"
+ assets_dir.mkdir(parents=True, exist_ok=True)
+ (assets_dir / "screenshot.png").write_bytes(b"PNG fake")
+ (assets_dir / "figure-1.jpg").write_bytes(b"JPG fake")
+
+ runner.invoke(app, ["sync"])
+
+ # Delete it
+ result = runner.invoke(app, ["note", "rm", "pdf-source", "--force", "--json"])
+ assert result.exit_code == 0, result.output
+ data = json.loads(result.output)
+ assert data["ok"] is True
+ assert data["data"]["deleted"] == "pdf-source"
+ assert data["data"].get("removed_raw") == "research/raw/pdf-source.pdf"
+ assert len(data["data"].get("removed_assets", [])) == 2
+
+ # Verify the files are actually gone from disk.
+ assert not raw_file.exists(), "raw file should have been deleted"
+ assert not assets_dir.exists(), "assets directory should have been removed"
+
+
def test_note_mv(vault_with_notes):
result = runner.invoke(
app, ["note", "mv", "beta-note", "notes/moved/beta-note.md", "--json"]
diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.py
index e661377..f9ff2fe 100644
--- a/tests/test_core/test_note.py
+++ b/tests/test_core/test_note.py
@@ -84,6 +84,38 @@ def test_strip_markdown():
assert "[[" not in plain
+def test_raw_file_persists_on_roundtrip(tmp_vault):
+ """raw_file must survive parse + re-serialize (regression test for wipe bug).
+
+ Before Batch 1.2 fix: raw_file was injected into frontmatter as a string
+ AFTER write_note; NoteMeta.model_config = {"extra": "ignore"} silently
+ dropped the field on the next parse, and any re-serialization (repair,
+ note update) wiped it from disk.
+ """
+ from hyperresearch.core.frontmatter import parse_frontmatter, render_note
+
+ path = write_note(
+ tmp_vault.notes_dir,
+ "PDF Note",
+ body="# PDF content\n",
+ extra_frontmatter={"raw_file": "raw/pdf-note.pdf"},
+ )
+ # First read — does NoteMeta capture the field?
+ note = read_note(path, tmp_vault.root)
+ assert note.meta.raw_file == "raw/pdf-note.pdf"
+
+ # Re-serialize (simulates repair.py enrichment or note update).
+ text = path.read_text(encoding="utf-8")
+ meta, body = parse_frontmatter(text)
+ path.write_text(render_note(meta, body), encoding="utf-8")
+
+ # Second read — raw_file must still be present.
+ note2 = read_note(path, tmp_vault.root)
+ assert note2.meta.raw_file == "raw/pdf-note.pdf", (
+ "raw_file was wiped on re-serialize — Batch 1.2 regression"
+ )
+
+
def test_read_utf8_content(tmp_vault):
path = write_note(
tmp_vault.notes_dir,
diff --git a/tests/test_core/test_patterns.py b/tests/test_core/test_patterns.py
new file mode 100644
index 0000000..b233390
--- /dev/null
+++ b/tests/test_core/test_patterns.py
@@ -0,0 +1,133 @@
+"""Tests for wiki-link target validation — citation-footnote edge cases."""
+
+from hyperresearch.core.patterns import is_valid_wiki_link_target
+
+
+def test_empty_and_whitespace_rejected():
+ assert not is_valid_wiki_link_target("")
+ assert not is_valid_wiki_link_target(" ")
+ assert not is_valid_wiki_link_target("\n")
+
+
+def test_urls_rejected():
+ assert not is_valid_wiki_link_target("https://example.com/foo")
+ assert not is_valid_wiki_link_target("http://example.com")
+ assert not is_valid_wiki_link_target("ftp://files.example.com")
+ assert not is_valid_wiki_link_target("mailto:[email protected]")
+
+
+def test_anchors_and_paths_rejected():
+ assert not is_valid_wiki_link_target("#section-1")
+ assert not is_valid_wiki_link_target("/path/to/note")
+
+
+def test_pure_numeric_citation_footnotes_rejected():
+ # The Q91 bug: Wikipedia footnotes rendered as [[100]]
+ assert not is_valid_wiki_link_target("100")
+ assert not is_valid_wiki_link_target("1")
+ assert not is_valid_wiki_link_target("999")
+
+
+def test_caret_prefixed_footnotes_rejected():
+ # Markdown-standard footnote syntax
+ assert not is_valid_wiki_link_target("^1")
+ assert not is_valid_wiki_link_target("^100")
+
+
+def test_multi_number_citations_rejected():
+ # Comma/semicolon/dash-separated citation groups
+ assert not is_valid_wiki_link_target("100,101")
+ assert not is_valid_wiki_link_target("1, 2, 3")
+ assert not is_valid_wiki_link_target("100; 101")
+ assert not is_valid_wiki_link_target("1-5")
+ assert not is_valid_wiki_link_target("100-105")
+
+
+def test_cite_prefix_patterns_rejected():
+ assert not is_valid_wiki_link_target("cite-1")
+ assert not is_valid_wiki_link_target("cite_2")
+ assert not is_valid_wiki_link_target("cite:3")
+ assert not is_valid_wiki_link_target("ref-42")
+ assert not is_valid_wiki_link_target("ref_42")
+ assert not is_valid_wiki_link_target("fn-1")
+ assert not is_valid_wiki_link_target("fn:2")
+ assert not is_valid_wiki_link_target("note-5")
+ assert not is_valid_wiki_link_target("footnote-10")
+ assert not is_valid_wiki_link_target("endnote-3")
+
+
+def test_cite_prefixes_are_case_insensitive():
+ assert not is_valid_wiki_link_target("Cite-1")
+ assert not is_valid_wiki_link_target("REF_2")
+ assert not is_valid_wiki_link_target("Fn-3")
+
+
+def test_valid_note_ids_accepted():
+ # Real note IDs should pass through
+ assert is_valid_wiki_link_target("pegasus-seiya")
+ assert is_valid_wiki_link_target("the-comics-journal-soldier-dream")
+ assert is_valid_wiki_link_target("quantum-computing")
+ assert is_valid_wiki_link_target("scaffold-saint-seiya")
+
+
+def test_note_ids_starting_with_digits_accepted():
+ # [[10-rules-for-X]] or [[1984-novel]] must pass — they contain non-digit chars
+ assert is_valid_wiki_link_target("10-rules-for-typescript")
+ assert is_valid_wiki_link_target("1984-novel")
+ assert is_valid_wiki_link_target("100-greatest-films")
+
+
+def test_note_ids_that_contain_cite_word_accepted():
+ # "citation" is not "cite-N" — should pass
+ assert is_valid_wiki_link_target("citations-are-important")
+ assert is_valid_wiki_link_target("references-guide")
+ assert is_valid_wiki_link_target("note-taking-systems") # plural, no digit
+
+
+def test_whitespace_is_stripped():
+ # Leading/trailing whitespace should be stripped before evaluation
+ assert not is_valid_wiki_link_target(" 100 ")
+ assert not is_valid_wiki_link_target("\t^100\n")
+ assert is_valid_wiki_link_target(" valid-note-id ")
+
+
+def test_roman_numeral_footnotes_rejected():
+ # Academic papers use Roman numerals for appendix/preface footnotes.
+ assert not is_valid_wiki_link_target("i")
+ assert not is_valid_wiki_link_target("iv")
+ assert not is_valid_wiki_link_target("xii")
+ assert not is_valid_wiki_link_target("xxiv")
+ assert not is_valid_wiki_link_target("IV")
+ assert not is_valid_wiki_link_target("XIV")
+
+
+def test_symbol_footnotes_rejected():
+ # Typography-style footnote markers used in older academic publishing.
+ assert not is_valid_wiki_link_target("*")
+ assert not is_valid_wiki_link_target("**")
+ assert not is_valid_wiki_link_target("†")
+ assert not is_valid_wiki_link_target("‡")
+ assert not is_valid_wiki_link_target("§")
+ assert not is_valid_wiki_link_target("¶")
+
+
+def test_document_cross_references_rejected():
+ # Figure / table / equation refs are document-internal, not wiki-links.
+ assert not is_valid_wiki_link_target("fig-3")
+ assert not is_valid_wiki_link_target("figure-4b")
+ assert not is_valid_wiki_link_target("tab-1")
+ assert not is_valid_wiki_link_target("table-2")
+ assert not is_valid_wiki_link_target("eq-2")
+ assert not is_valid_wiki_link_target("equation-7")
+ assert not is_valid_wiki_link_target("scheme-3")
+ assert not is_valid_wiki_link_target("algorithm-2")
+ assert not is_valid_wiki_link_target("Fig-1") # case-insensitive
+
+
+def test_real_words_that_look_like_roman_are_accepted():
+ # "dim" and "mix" look roman-like but are real note ids
+ assert is_valid_wiki_link_target("mix-matrix")
+ assert is_valid_wiki_link_target("div-container")
+ # Longer words containing roman chars are fine
+ assert is_valid_wiki_link_target("civic")
+ assert is_valid_wiki_link_target("minimalism")
diff --git a/tests/test_core/test_sync.py b/tests/test_core/test_sync.py
index a77c416..9d7010a 100644
--- a/tests/test_core/test_sync.py
+++ b/tests/test_core/test_sync.py
@@ -96,3 +96,30 @@ def test_sync_excludes_hyperresearch_dir(tmp_vault):
(tmp_vault.root / ".hyperresearch" / "test.md").write_text("---\ntitle: Bad\n---\nShould not sync")
plan = compute_sync_plan(tmp_vault)
assert all(".hyperresearch" not in str(p) for p in plan.to_add)
+
+
+def test_sync_excludes_research_root_staging_files(tmp_vault):
+ """Files at research/ root (scaffold.md, comparisons.md, synthesis.md)
+ are staging files the agent writes then registers via `note new`. They
+ must NOT appear as orphan notes in the vault index — the current
+ behavior would produce missing-title/tags/summary lint spam on every run.
+ """
+ from hyperresearch.core.note import write_note
+
+ # Real notes under research/notes/
+ write_note(tmp_vault.notes_dir, "Real Note", body="# Real\n")
+
+ # Staging files at research/ root
+ research_root = tmp_vault.research_dir
+ (research_root / "scaffold.md").write_text("# Scaffold staging\n")
+ (research_root / "comparisons.md").write_text("# Comparisons staging\n")
+ (research_root / "synthesis.md").write_text("# Synthesis staging\n")
+
+ plan = compute_sync_plan(tmp_vault)
+
+ # Only the real note should be added.
+ added_names = [p.name for p in plan.to_add]
+ assert "real-note.md" in added_names
+ assert "scaffold.md" not in added_names
+ assert "comparisons.md" not in added_names
+ assert "synthesis.md" not in added_names