Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
Lint rules + data integrity: audit gate, provenance, raw file, note rm
Jordanc5c8c5cb6948
diff --git a/src/hyperresearch/cli/fetch.py b/src/hyperresearch/cli/fetch.pyindex 71f2821..9ec78f0 100644--- a/src/hyperresearch/cli/fetch.py+++ b/src/hyperresearch/cli/fetch.py@@ -30,6 +30,178 @@ SKIP_URL_PATTERNS = ()+def _append_suggested_by_to_existing(+ vault_root: Path,+ note_id: str,+ suggested_by: list[str],+ reason: str | None,+) -> int:+ """Append `*Suggested by [[src]] — reason*` breadcrumb lines to an existing note.++ Idempotent: if a breadcrumb for a given source id already exists anywhere+ in the body, it is not re-added. Returns the number of new breadcrumbs+ actually added.+ """+ # Find the note file — scan research/notes/ for `<note-id>.md` or walk subdirs+ from hyperresearch.core.frontmatter import parse_frontmatter, serialize_frontmatter++ note_path = None+ for candidate in vault_root.rglob(f"{note_id}.md"):+ # Require it to be under a notes directory so we don't match index files+ if "notes" in candidate.parts:+ note_path = candidate+ break+ if note_path is None:+ return 0++ text = note_path.read_text(encoding="utf-8-sig")+ meta, body = parse_frontmatter(text)++ reason_suffix = f" — {reason}" if reason else ""+ added = 0+ new_lines = []+ for src_id in suggested_by:+ src_clean = src_id.strip()+ if not src_clean:+ continue+ # Skip if a breadcrumb for this source id already exists in the body+ existing_marker = f"[[{src_clean}]]"+ if existing_marker in body:+ continue+ new_lines.append(f"*Suggested by [[{src_clean}]]{reason_suffix}*")+ added += 1++ if added == 0:+ return 0++ # Prepend the new breadcrumb lines to the body+ breadcrumb_block = "\n".join(new_lines) + "\n\n"+ new_body = breadcrumb_block + body+ new_text = serialize_frontmatter(meta) + "\n" + new_body+ note_path.write_text(new_text, encoding="utf-8")+ return added+++def _detect_tier(url: str, content_type: str) -> str:+ """Guess the epistemic tier from URL + content_type. Returns a Tier value.++ This is a coarse first pass. The curation-time agent is expected to refine+ (e.g. a medium.com post could be practitioner if it's by a named engineer,+ or commentary if it's a generic explainer — only reading can tell).++ Defaults prioritize reducing the agent's curation load: URLs with strong+ signal get a non-unknown default so the agent only needs to verify, not+ classify from scratch.+ """+ domain = urlparse(url).netloc.lower()++ # Ground truth: official primary sources — filings, specs, policy text, datasets+ if domain.endswith(".gov") or ".gov." in domain or domain.endswith(".gov.uk") or domain.endswith(".europa.eu"):+ return "ground_truth"+ if content_type == "policy":+ return "ground_truth"+ if content_type == "dataset":+ return "ground_truth"+ if content_type == "docs":+ # Official documentation IS the ground truth for a product/standard+ return "ground_truth"+ if content_type == "code":+ # Source code, README, and repo metadata are ground truth for a tool+ return "ground_truth"++ # Institutional: peer-reviewed scholarship, canonical reference works+ if content_type == "paper":+ # Default to institutional; agent may upgrade to ground_truth if the+ # paper reports original data or downgrade to commentary for derivatives+ return "institutional"+ if "wikipedia.org" in domain or "britannica.com" in domain:+ return "institutional"++ # Practitioner: forums, community threads, hands-on content+ if content_type == "forum":+ return "practitioner"+ if content_type == "review":+ return "practitioner"++ # Commentary: news articles, op-eds, blog posts without known authority+ if content_type == "article":+ return "commentary"+ if content_type == "blog":+ return "commentary"+ if content_type == "transcript":+ # Could be institutional (conference keynote) or commentary (podcast)+ # — agent must decide. Start at commentary; promote during curation.+ return "commentary"++ return "unknown"+++def _detect_content_type(url: str, raw_content_type: str | None = None) -> str:+ """Guess the artifact kind from URL + MIME. Returns a ContentType value or 'unknown'.++ This is a coarse first pass. The creation-time agent is expected to refine+ during curation if the URL/MIME heuristic is wrong (e.g. a medium.com URL+ could be blog OR article).+ """+ if raw_content_type and "pdf" in raw_content_type.lower():+ return "paper"+ u = url.lower()+ domain = urlparse(url).netloc.lower()+ path = urlparse(url).path.lower()++ # Papers: arxiv, doi, direct PDFs, openreview, ssrn, biorxiv, pubmed+ if any(d in domain for d in ("arxiv.org", "doi.org", "openreview.net", "ssrn.com", "biorxiv.org", "medrxiv.org", "pubmed.ncbi.nlm.nih.gov", "ncbi.nlm.nih.gov/pmc", "semanticscholar.org", "openalex.org")):+ return "paper"+ if path.endswith(".pdf"):+ return "paper"++ # Code: github, gitlab, bitbucket, pypi+ if any(d in domain for d in ("github.com", "gitlab.com", "bitbucket.org", "pypi.org", "crates.io", "npmjs.com")):+ return "code"++ # Dataset portals that happen to live on .gov domains must win first+ if domain in ("data.gov", "data.gov.uk") or domain.startswith("data.") and (domain.endswith(".gov") or domain.endswith(".gov.uk") or domain.endswith(".europa.eu")):+ return "dataset"++ # Policy: gov domains, EU, regulator sites+ if domain.endswith(".gov") or ".gov." in domain or domain.endswith(".gov.uk") or domain.endswith(".europa.eu") or "regulations.gov" in domain:+ return "policy"++ # Docs: common documentation platforms, docs.* subdomains, /docs paths+ if any(d in domain for d in ("readthedocs.io", "readthedocs.org", "docs.rs", "developer.mozilla.org")):+ return "docs"+ if domain.startswith("docs.") or domain.startswith("documentation.") or ".readthedocs." in domain:+ return "docs"+ if "/docs/" in path or path.endswith("/docs") or "/documentation/" in path or "/api/reference" in path or "/reference/" in path:+ return "docs"++ # Forum / community+ if any(d in domain for d in ("reddit.com", "news.ycombinator.com", "stackoverflow.com", "stackexchange.com", "lobste.rs", "lemmy.", "discourse.")):+ return "forum"++ # Blog platforms+ if any(d in domain for d in ("medium.com", "substack.com", "dev.to", "hashnode.com", "hashnode.dev", "blogspot.com", "wordpress.com", "ghost.io")):+ return "blog"++ # Transcripts / video+ if any(d in domain for d in ("youtube.com", "youtu.be", "vimeo.com")):+ return "transcript"++ # Reference articles+ if "wikipedia.org" in domain or "wikimedia.org" in domain or "britannica.com" in domain:+ return "article"++ # Datasets+ if any(d in domain for d in ("kaggle.com", "data.gov", "data.world", "zenodo.org", "figshare.com")):+ return "dataset"++ # News / magazine default+ if any(d in domain for d in ("nytimes.com", "wsj.com", "ft.com", "bloomberg.com", "reuters.com", "bbc.com", "theatlantic.com", "newyorker.com", "economist.com")):+ return "article"++ return "unknown"++@app.command("fetch")def fetch(url: str = typer.Argument(..., help="URL to fetch and save as a note"),@@ -39,6 +211,16 @@ def fetch(provider_name: str | None = typer.Option(None, "--provider", help="Web provider override"),save_assets: bool = typer.Option(False, "--save-assets", "-a", help="Download images and screenshot"),visible: bool = typer.Option(False, "--visible", "-V", help="Run browser visibly (for stubborn auth sites)"),+ suggested_by: list[str] = typer.Option(+ [],+ "--suggested-by",+ help="Source note ID(s) that suggested this URL. Prepends a 'Suggested by [[note-id]]' breadcrumb to the new note's body, which the link extractor picks up as a wiki-link so the fetched note appears in the source's backlinks. Repeat for multiple sources.",+ ),+ suggested_by_reason: str | None = typer.Option(+ None,+ "--suggested-by-reason",+ help="One-line justification for why the suggesting source named this URL. Appears next to the [[source-id]] breadcrumb.",+ ),json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),) -> None:"""Fetch a URL and save its content as a research note."""@@ -63,6 +245,38 @@ def fetch(existing = conn.execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()if existing:note_id = existing["note_id"]+ # Graceful duplicate handling for the guided reading loop:+ # if the caller passed --suggested-by, append the breadcrumb to the+ # existing note instead of erroring out. This lets the main agent+ # build up provenance on already-fetched URLs without needing to+ # check for dupes first.+ if suggested_by:+ added = _append_suggested_by_to_existing(+ vault.root, note_id, suggested_by, suggested_by_reason+ )+ # Re-sync so the new wiki-link gets picked up by the link extractor+ plan = compute_sync_plan(vault)+ if plan.to_update:+ execute_sync(vault, plan)++ data = {+ "note_id": note_id,+ "url": url,+ "duplicate": True,+ "backlinks_added": added,+ "message": (+ f"URL already fetched; added {added} backlink(s) to existing note"+ if added+ else "URL already fetched; breadcrumb(s) already present"+ ),+ }+ if json_output:+ output(success(data, vault=str(vault.root)), json_mode=True)+ else:+ console.print(f"[yellow]Already fetched:[/] {url} → note '{note_id}'")+ console.print(f" added {added} backlink(s) to existing note")+ return+if json_output:output(error(f"URL already fetched as note '{note_id}'", "DUPLICATE_URL"),@@ -130,6 +344,8 @@ def fetch(note_title = title or result.title or urlparse(url).path.split("/")[-1] or "Untitled"domain = result.domain+ detected_content_type = _detect_content_type(url, result.raw_content_type)+ detected_tier = _detect_tier(url, detected_content_type)extra_meta = {"source": url,"source_domain": domain,@@ -139,14 +355,33 @@ def fetch(if result.metadata.get("author"):extra_meta["author"] = result.metadata["author"]+ # Build body with backlink breadcrumb if this fetch was suggested by another note.+ # Prepending `*Suggested by [[source-id]] — reason*` lines creates wiki-links the+ # normal link extractor will pick up, so the new note automatically appears in+ # graph backlinks of its suggester(s). Zero schema change needed.+ body_content = result.content+ if suggested_by:+ breadcrumb_lines = []+ reason_suffix = f" — {suggested_by_reason}" if suggested_by_reason else ""+ for src_id in suggested_by:+ src_id_clean = src_id.strip()+ if not src_id_clean:+ continue+ breadcrumb_lines.append(f"*Suggested by [[{src_id_clean}]]{reason_suffix}*")+ if breadcrumb_lines:+ breadcrumb = "\n".join(breadcrumb_lines) + "\n\n"+ body_content = breadcrumb + body_content+note_path = write_note(vault.notes_dir,title=note_title,- body=result.content,+ body=body_content,tags=tags,status="draft",source=url,parent=parent,+ tier=detected_tier,+ content_type=detected_content_type,extra_frontmatter=extra_meta,)@@ -172,18 +407,17 @@ def fetch(# Note: tagging and summarization is the agent's job, not an automatic process.# The agent reads the fetched content and writes meaningful summaries and tags.- # Add raw_file reference to frontmatter AFTER enrich (enrich rewrites frontmatter)+ # Persist raw_file reference on the note via NoteMeta — this guarantees the+ # field survives any future re-serialization (repair, note update, etc.).+ # The text-injection approach used previously was silently dropped by+ # NoteMeta.model_config = {"extra": "ignore"} on the next parse.if raw_file_path:+ from hyperresearch.core.frontmatter import parse_frontmatter, render_notenote_text = note_path.read_text(encoding="utf-8")- if note_text.startswith("---") and "raw_file:" not in note_text:- end_idx = note_text.find("---", 3)- if end_idx != -1:- note_text = (- note_text[:end_idx]- + f"raw_file: {raw_file_path}\n"- + note_text[end_idx:]- )- note_path.write_text(note_text, encoding="utf-8")+ meta, body = parse_frontmatter(note_text)+ if meta.raw_file != raw_file_path:+ meta.raw_file = raw_file_path+ note_path.write_text(render_note(meta, body), encoding="utf-8")# Sync first so the note exists in the notes table (needed for FK on sources/assets)note_id = note_path.stem@@ -191,15 +425,44 @@ def fetch(if plan.to_add or plan.to_update:execute_sync(vault, plan)- # Record source in DB+ # Record source in DB. Use INSERT OR IGNORE to survive a duplicate-URL+ # race: two parallel fetches on the same URL both pass the earlier+ # duplicate-check SELECT, both try to INSERT, and without IGNORE the+ # second crashes with IntegrityError leaving a stray .md file and+ # notes row but no sources row. With IGNORE the second INSERT is a+ # no-op and we detect the race by re-selecting the row afterward.content_hash = hashlib.sha256(result.content.encode("utf-8")).hexdigest()[:16]conn.execute(- """INSERT INTO sources (url, note_id, domain, fetched_at, provider, content_hash)+ """INSERT OR IGNORE INTO sources (url, note_id, domain, fetched_at, provider, content_hash)VALUES (?, ?, ?, ?, ?, ?)""",(url, note_id, domain, result.fetched_at.isoformat(), prov.name, content_hash),)conn.commit()+ # Detect the race: if the committed row's note_id != ours, another+ # fetch won the race. Clean up our orphan .md file and return the+ # winner's note_id so the caller can treat this as an idempotent no-op.+ winner_row = conn.execute(+ "SELECT note_id FROM sources WHERE url = ?", (url,)+ ).fetchone()+ if winner_row and winner_row["note_id"] != note_id:+ winning_note_id = winner_row["note_id"]+ # Our write lost the race. Unlink the orphan file and re-sync.+ if note_path.exists():+ try:+ note_path.unlink()+ except OSError:+ pass+ plan2 = compute_sync_plan(vault)+ if plan2.to_delete:+ execute_sync(vault, plan2)+ note_id = winning_note_id+ winner_path_row = conn.execute(+ "SELECT path FROM notes WHERE id = ?", (winning_note_id,)+ ).fetchone()+ if winner_path_row:+ note_path = vault.root / winner_path_row["path"]+# Save assets (screenshot + images) — only when requestedsaved_assets: list[dict] = []if save_assets:diff --git a/src/hyperresearch/cli/lint.py b/src/hyperresearch/cli/lint.pyindex 3e683f1..4ebcd88 100644--- a/src/hyperresearch/cli/lint.py+++ b/src/hyperresearch/cli/lint.py@@ -15,6 +15,13 @@ RULES = {"missing-title": "Notes without a title","missing-tags": "Notes without any tags","missing-summary": "Notes without a summary",+ "uncurated": "Non-draft notes without tier or content_type classification",+ "workflow": "Draft or synthesis notes missing paired scaffold/comparison notes",+ "scaffold-prompt": "Scaffold notes missing the verbatim user prompt as first section (gospel rule)",+ "audit-gate": "Unresolved CRITICAL findings in research/audit_findings.json block synthesis save",+ "provenance": "Source notes with no --suggested-by breadcrumb chain (data-flow chain broken)",+ "analyst-coverage": "Fetched sources without paired extract notes (analyst skipped most sources)",+ "orphaned-raw-files": "Files in research/raw/ with no matching note (disk leak from old note rm)","singleton-tags": "Tags used by only one note","broken-links": "Wiki-links that don't resolve","orphaned-notes": "Notes with no inbound or outbound links",@@ -47,6 +54,12 @@ def lint(rules_to_run = [rule] if rule else list(RULES.keys())+ # Map from audit-gate CRITICAL finding id -> lint rule name that should be+ # re-run to verify the fix actually landed. Populated inside the audit-gate+ # block and consumed at the end of lint() for self-certification detection.+ # Key words in the finding description are mapped to lint rule names.+ audit_gate_guards: list[dict] = [] # [{"critical_id", "rule", "description"}]+if "missing-title" in rules_to_run:for row in conn.execute("SELECT id, path FROM notes WHERE title = '' OR title = 'Untitled'"):issues.append({@@ -85,6 +98,641 @@ def lint("message": "Note has no summary. Add one for better search and listings.",})+ if "audit-gate" in rules_to_run:+ # Block synthesis save unless BOTH:+ # (a) at least one `conformance` audit run exists in+ # research/audit_findings.json, AND+ # (b) every CRITICAL finding in the most recent conformance run has+ # a non-null `fixed_at` timestamp (applied or explicitly resolved).+ #+ # Additionally: surface IMPORTANT findings as info-severity issues so+ # the agent sees them in the pre-save lint output. They don't block+ # save, but they do nudge the agent to review them before committing.+ #+ # Missing file = no audit has run yet. Gate is OPEN in that case so+ # early-stage lint runs don't spam errors. But once the file exists,+ # a missing conformance run is itself an error — the protocol demands+ # both modes, not just comprehensiveness.+ import json as _json+ audit_path = vault.root / "research" / "audit_findings.json"+ if audit_path.exists():+ try:+ audit_data = _json.loads(audit_path.read_text(encoding="utf-8"))+ except (OSError, _json.JSONDecodeError) as exc:+ issues.append({+ "rule": "audit-gate",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"research/audit_findings.json exists but is malformed "+ f"({type(exc).__name__}). Delete or fix it, then re-run "+ f"the adversarial audit."+ ),+ })+ audit_data = None++ if isinstance(audit_data, dict):+ runs = audit_data.get("runs", [])+ conformance_runs = [r for r in runs if r.get("mode") == "conformance"]+ comprehensiveness_runs = [r for r in runs if r.get("mode") == "comprehensiveness"]++ # Check (a): a conformance run must exist once any audit has happened.+ if not conformance_runs:+ if comprehensiveness_runs:+ issues.append({+ "rule": "audit-gate",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"research/audit_findings.json has "+ f"{len(comprehensiveness_runs)} comprehensiveness run(s) but ZERO "+ f"conformance runs. Step 11 mandates BOTH modes in parallel. "+ f"Spawn hyperresearch-auditor with mode=conformance and wait for "+ f"it to append its findings to audit_findings.json before saving "+ f"the synthesis."+ ),+ })+ # No runs at all = early stage. Gate stays open.+ else:+ # Check (b): no unresolved CRITICALs in the newest conformance run.+ conformance_runs.sort(key=lambda r: r.get("timestamp", ""))+ latest = conformance_runs[-1]+ latest_status = latest.get("status", "unknown")+ criticals = latest.get("criticals") or []+ unresolved = [c for c in criticals if not c.get("fixed_at")]++ if unresolved:+ issues.append({+ "rule": "audit-gate",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Most recent conformance audit has "+ f"{len(unresolved)} unresolved CRITICAL finding(s): "+ + "; ".join(+ f"[{c.get('id','?')}] {c.get('description','?')[:80]}"+ for c in unresolved[:5]+ )+ + ". Apply the fixes in research/notes/final_report.md, "+ + "mark each finding with `fixed_at: <ISO>` in "+ + "research/audit_findings.json, and re-run the conformance "+ + "auditor to verify."+ ),+ })+ elif latest_status == "failed":+ issues.append({+ "rule": "audit-gate",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Most recent conformance audit returned status=failed. "+ f"Investigate the findings and re-run the auditor."+ ),+ })++ # Surface IMPORTANT findings from EITHER mode's newest run. Info+ # severity = advisory; does not block save, but the agent sees+ # them in the save-gate lint output and can choose to patch.+ for mode_runs, mode_label in (+ (conformance_runs, "conformance"),+ (comprehensiveness_runs, "comprehensiveness"),+ ):+ if not mode_runs:+ continue+ mode_runs.sort(key=lambda r: r.get("timestamp", ""))+ latest_run = mode_runs[-1]+ important = latest_run.get("important") or []+ unresolved_important = [i for i in important if not i.get("fixed_at")]+ if unresolved_important:+ issues.append({+ "rule": "audit-gate",+ "severity": "info",+ "note_id": "<vault>",+ "message": (+ f"{len(unresolved_important)} unresolved IMPORTANT finding(s) in "+ f"the latest {mode_label} audit (advisory, does not block save): "+ + "; ".join(+ f"[{i.get('id','?')}] {i.get('description','?')[:80]}"+ for i in unresolved_important[:5]+ )+ + ". Mark `fixed_at` on each after patching the draft."+ ),+ })++ # Build guard-rule map: for each CRITICAL with fixed_at set,+ # extract the implied lint rule from keywords in its description+ # and queue that rule for verification. This is how audit-gate+ # detects self-certification (CRITICAL marked fixed but the+ # underlying lint rule still fails).+ #+ # The keyword list covers the natural-language variations we+ # see in real auditor outputs across modalities. Keep these+ # broad — false matches are fine (they cause a re-check+ # which passes harmlessly); missing a match is the failure+ # mode we're guarding against.+ _KW_TO_RULE = [+ # scaffold-prompt (verbatim prompt gospel rule)+ ("scaffold-prompt", "scaffold-prompt"),+ ("scaffold_prompt", "scaffold-prompt"),+ ("scaffold_extraction_gap", "scaffold-prompt"),+ ("scaffold extraction gap", "scaffold-prompt"),+ ("verbatim prompt", "scaffold-prompt"),+ ("verbatim_prompt", "scaffold-prompt"),+ ("gospel rule", "scaffold-prompt"),+ ("user prompt missing", "scaffold-prompt"),++ # analyst-coverage (extract notes per source)+ ("analyst-coverage", "analyst-coverage"),+ ("analyst coverage", "analyst-coverage"),+ ("analyst_coverage", "analyst-coverage"),+ ("extract coverage", "analyst-coverage"),+ ("extract ratio", "analyst-coverage"),+ ("extract notes", "analyst-coverage"),+ ("fetch:extract", "analyst-coverage"),+ ("analyst skipped", "analyst-coverage"),+ ("no extract", "analyst-coverage"),++ # provenance (bouncing reading loop + --suggested-by chain)+ ("provenance", "provenance"),+ ("suggested-by", "provenance"),+ ("suggested_by", "provenance"),+ ("suggested by", "provenance"),+ ("bouncing reading loop", "provenance"),+ ("bouncing loop", "provenance"),+ ("guided reading loop", "provenance"),+ ("guided loop", "provenance"),+ ("reading loop", "provenance"),+ ("breadcrumb", "provenance"),+ ("data-flow chain", "provenance"),+ ("data flow chain", "provenance"),+ ("data-flow broken", "provenance"),+ ("rabbit-hole", "provenance"),+ ("rabbit hole", "provenance"),++ # workflow (scaffold + comparisons + extract artifacts exist)+ ("workflow", "workflow"),+ ("missing scaffold", "workflow"),+ ("missing comparison", "workflow"),+ ("missing comparisons", "workflow"),+ ("paired scaffold", "workflow"),+ ("no scaffold note", "workflow"),+ ("no comparison note", "workflow"),+ ("step 7 skipped", "workflow"),+ ("step 8 skipped", "workflow"),++ # uncurated (tier + content_type + summary metadata)+ ("uncurated", "uncurated"),+ ("tier metadata", "uncurated"),+ ("content_type", "uncurated"),+ ("content type", "uncurated"),+ ("tier/content_type", "uncurated"),+ ("tier and content_type", "uncurated"),+ ("classification missing", "uncurated"),+ ("metadata missing", "uncurated"),+ ("missing tier", "uncurated"),+ ("missing summary", "uncurated"),+ ]+ for mode_runs in (conformance_runs, comprehensiveness_runs):+ if not mode_runs:+ continue+ # We already sorted conformance_runs above; sort comprehensiveness+ # now too so we pick the newest run of each mode.+ mode_runs_sorted = sorted(mode_runs, key=lambda r: r.get("timestamp", ""))+ latest_run = mode_runs_sorted[-1]+ for c in (latest_run.get("criticals") or []):+ if not c.get("fixed_at"):+ continue # unresolved criticals already emitted above+ desc = (str(c.get("description", "")) + " " ++ str(c.get("id", ""))).lower()+ matched = None+ for kw, rule_name in _KW_TO_RULE:+ if kw in desc:+ matched = rule_name+ break+ if matched:+ audit_gate_guards.append({+ "critical_id": c.get("id", "?"),+ "rule": matched,+ "description": c.get("description", "")[:120],+ })+ # Ensure the guard rule runs so we can check its+ # issues in post-processing.+ if matched not in rules_to_run:+ rules_to_run.append(matched)+ else:+ # CRITICAL marked fixed_at but no known lint rule+ # maps to its description. The fix is trust-only+ # — we cannot machine-verify it. Surface a warning+ # so the user knows this finding was not validated.+ issues.append({+ "rule": "audit-gate",+ "severity": "warning",+ "note_id": "<vault>",+ "message": (+ f"CRITICAL [{c.get('id','?')}] was marked `fixed_at` but "+ f"its description doesn't map to any known lint rule: "+ f"'{(c.get('description','') or '')[:100]}'. The fix is "+ f"agent-self-reported and not machine-verified. Review the "+ f"draft manually to confirm the issue was actually addressed."+ ),+ })++ if "scaffold-prompt" in rules_to_run:+ # Enforce the gospel rule: every scaffold-tagged note must open with+ # the user's verbatim prompt as its first section. This is the single+ # machine-checkable invariant that protects the dispatcher's+ # "user prompt is gospel" commitment. Without this check, scaffolds+ # can drift — an agent writes a `## Thesis` opening, never pastes the+ # verbatim prompt, and every downstream step that re-reads the+ # scaffold (audit, draft, comparisons) loses its anchor.+ import re as _re+ _header_re = _re.compile(+ r"^\s*##\s+User\s+Prompt\s*\(\s*VERBATIM.*gospel\s*\)",+ _re.IGNORECASE,+ )+ for row in conn.execute("""+ SELECT n.id, n.path, nc.body+ FROM notes n+ JOIN note_content nc ON n.id = nc.note_id+ WHERE n.id IN (SELECT note_id FROM tags WHERE tag = 'scaffold')+ """):+ body_lines = (row["body"] or "").splitlines()+ # Look for the header within the first 20 non-blank lines+ header_line_idx = None+ seen_non_blank = 0+ for idx, line in enumerate(body_lines):+ if line.strip():+ seen_non_blank += 1+ if _header_re.match(line):+ header_line_idx = idx+ break+ if seen_non_blank >= 20:+ break++ if header_line_idx is None:+ issues.append({+ "rule": "scaffold-prompt",+ "severity": "error",+ "note_id": row["id"],+ "note_path": row["path"],+ "message": (+ "Scaffold is missing the verbatim user prompt as its first section. "+ "Every scaffold MUST open with a `## User Prompt (VERBATIM — gospel)` "+ "header followed by the user's original question as a blockquote. "+ "This is the gospel rule — the dispatcher re-reads the prompt from "+ "this section at every downstream step."+ ),+ })+ continue++ # Check for non-empty blockquote after the header (>= 50 chars total,+ # which is a loose floor — a real prompt will be much longer).+ quote_chars = 0+ for line in body_lines[header_line_idx + 1:]:+ stripped = line.strip()+ if not stripped:+ continue+ if stripped.startswith(">"):+ quote_chars += len(stripped.lstrip("> ").strip())+ elif stripped.startswith("##"):+ break # hit next header without finding quoted content+ else:+ # some scaffolds paste the prompt as plain text without+ # the blockquote prefix — accept if substantive.+ quote_chars += len(stripped)+ break++ if quote_chars < 50:+ issues.append({+ "rule": "scaffold-prompt",+ "severity": "warning",+ "note_id": row["id"],+ "note_path": row["path"],+ "message": (+ f"Scaffold has the verbatim-prompt header but the content after it "+ f"is empty or too short ({quote_chars} chars). Paste the user's "+ f"full prompt as a blockquote under the header."+ ),+ })++ if "provenance" in rules_to_run:+ # Verify the `--suggested-by` data-flow chain forms a rooted tree+ # (or forest) across all fetched source notes:+ #+ # 1. At least one seed source exists (a fetched note with no+ # breadcrumb) — the loop has a starting point.+ # 2. Every non-seed source has at least one breadcrumb pointing at+ # another source note that exists in the vault (reachable from+ # a seed through one or more breadcrumb hops).+ # 3. Every breadcrumb's wiki-link target is a real note id — no+ # dangling backlinks.+ #+ # Previous version used `breadcrumb_count < max(2, source_count // 5)`+ # which was easy to game by backfilling N//5 unrelated breadcrumbs.+ # The rooted-tree check cannot be satisfied without an actual chain.+ import re as _re+ _breadcrumb_re = _re.compile(r"\*Suggested by \[\[([^\]]+)\]\]")++ source_rows = list(conn.execute(+ "SELECT n.id, n.path, nc.body "+ "FROM notes n "+ "JOIN note_content nc ON n.id = nc.note_id "+ "WHERE n.source IS NOT NULL "+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "+ "AND n.type NOT IN ('index','raw','moc')"+ ))++ if len(source_rows) <= 5:+ # Small corpora: bouncing loop may not have fired by design.+ # Skip the structural check; fall back to presence check only.+ pass+ else:+ all_note_ids = {+ r["id"] for r in conn.execute("SELECT id FROM notes")+ }++ source_breadcrumbs: dict[str, list[str]] = {}+ for r in source_rows:+ targets = _breadcrumb_re.findall(r["body"] or "")+ # Keep only the wiki-link name before any `|display` pipe.+ cleaned = [t.split("|", 1)[0].strip() for t in targets]+ source_breadcrumbs[r["id"]] = cleaned++ seeds = [nid for nid, crumbs in source_breadcrumbs.items() if not crumbs]+ non_seeds = [nid for nid, crumbs in source_breadcrumbs.items() if crumbs]++ # Condition 1: at least one seed.+ if not seeds:+ issues.append({+ "rule": "provenance",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Provenance graph has no seed: every one of {len(source_rows)} source notes "+ f"carries a `*Suggested by [[...]]` breadcrumb, which is impossible for a real "+ f"research session. The guided reading loop must start from at least one seed "+ f"fetch with no suggester."+ ),+ })++ # Condition 2 + 3: verify graph is rooted at seeds and no dangling targets.+ if non_seeds:+ dangling: list[tuple[str, str]] = []+ for nid in non_seeds:+ for target in source_breadcrumbs[nid]:+ if target not in all_note_ids:+ dangling.append((nid, target))++ for src_id, target in dangling[:10]: # cap output+ issues.append({+ "rule": "provenance",+ "severity": "error",+ "note_id": src_id,+ "message": (+ f"Breadcrumb `[[{target}]]` points at a note id that does not exist in the "+ f"vault. Either the target was deleted, or the breadcrumb was hand-written "+ f"without a real source. Re-fetch with `--suggested-by <real-note-id>`."+ ),+ })++ # BFS from seeds to verify connectivity.+ reachable = set(seeds)+ frontier = list(seeds)+ # Build reverse map: suggester -> notes it sourced+ suggester_to_sourced: dict[str, list[str]] = {}+ for nid, crumbs in source_breadcrumbs.items():+ for t in crumbs:+ suggester_to_sourced.setdefault(t, []).append(nid)+ while frontier:+ current = frontier.pop()+ for child in suggester_to_sourced.get(current, []):+ if child not in reachable:+ reachable.add(child)+ frontier.append(child)++ unreachable = [nid for nid in non_seeds if nid not in reachable]+ if unreachable:+ issues.append({+ "rule": "provenance",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"{len(unreachable)} source note(s) have breadcrumbs but are not reachable "+ f"from any seed through the provenance graph — the chain is disconnected. "+ f"Disconnected islands usually mean an agent fabricated breadcrumbs "+ f"retroactively without following the guided reading loop. First few: "+ f"{', '.join(unreachable[:5])}"+ ),+ })++ # Condition: coverage. At least half of non-seed sources should have a breadcrumb.+ # (Cheap heuristic: the count of non-seeds is the count of sources with at least+ # one breadcrumb; compare to the count of sources that SHOULD have one.)+ non_seed_ratio = len(non_seeds) / max(len(source_rows), 1)+ # If there is exactly one seed and no non-seeds, the loop never fired.+ if len(source_rows) > 5 and len(non_seeds) == 0:+ issues.append({+ "rule": "provenance",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Vault has {len(source_rows)} fetched source notes but ZERO "+ f"`*Suggested by [[...]]` breadcrumbs. The bouncing reading loop never "+ f"fired — every fetch was a flat batch with no link back to the source "+ f"that proposed it. Use `$HPR fetch ... --suggested-by <source-note-id> "+ f"--suggested-by-reason \"<why>\"` for every follow-up fetch."+ ),+ })+ elif non_seed_ratio < 0.3 and len(source_rows) > 10:+ # Less than 30% of the corpus came from analyst recommendations.+ # The guided reading loop effectively didn't fire — this is the+ # exact failure mode both v2 runs exhibited (3/19 and 2/33).+ # Error severity because it blocks Checkpoint 2.+ issues.append({+ "rule": "provenance",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Only {len(non_seeds)}/{len(source_rows)} source notes ({non_seed_ratio:.0%}) "+ f"have breadcrumbs — the guided reading loop did not fire. The initial batch "+ f"fetch is not the whole corpus; after fetching seeds you MUST spawn analysts "+ f"to propose next targets, then fetch those with `--suggested-by`. Target: "+ f"at least 30% of sources should come from analyst recommendations."+ ),+ })+ elif non_seed_ratio < 0.5 and len(source_rows) > 10:+ # Between 30% and 50% — under-firing but marginal. Warning only.+ issues.append({+ "rule": "provenance",+ "severity": "warning",+ "note_id": "<vault>",+ "message": (+ f"Only {len(non_seeds)}/{len(source_rows)} source notes ({non_seed_ratio:.0%}) "+ f"have breadcrumbs. The bouncing reading loop is under-firing — most "+ f"fetches look like flat seeds rather than analyst-driven discoveries."+ ),+ })++ if "analyst-coverage" in rules_to_run:+ # Detect when source notes were fetched but few were analyzed (i.e.,+ # spawned a hyperresearch-analyst that wrote an extract note). An+ # extract note is identified by carrying the `extract` tag.+ #+ # Rule applies UNCONDITIONALLY — previous versions gated on+ # `source_count >= 10` which silently disabled the rule for small+ # corpora (a compare session with 8 named entities could ship zero+ # extracts and pass). The gate is removed.+ source_count_row = conn.execute(+ "SELECT COUNT(*) as c FROM notes n "+ "WHERE n.source IS NOT NULL "+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "+ "AND n.type NOT IN ('index','raw','moc') "+ "AND n.id NOT IN (SELECT note_id FROM tags WHERE tag = 'extract') "+ "AND n.id NOT IN (SELECT note_id FROM tags WHERE tag = 'scaffold') "+ "AND n.id NOT IN (SELECT note_id FROM tags WHERE tag = 'comparison')"+ ).fetchone()+ source_count = source_count_row["c"] if source_count_row else 0++ extract_count_row = conn.execute(+ "SELECT COUNT(DISTINCT note_id) as c FROM tags WHERE tag = 'extract'"+ ).fetchone()+ extract_count = extract_count_row["c"] if extract_count_row else 0++ # Require 1/3 coverage (error floor at 1/4). Even a 2-source session+ # needs at least 1 extract — the analyst is mandatory, not optional.+ if source_count >= 1:+ required_extracts = max(1, source_count // 3)+ error_floor = max(1, source_count // 4)+ if extract_count < required_extracts:+ ratio = extract_count / source_count if source_count else 0+ issues.append({+ "rule": "analyst-coverage",+ "severity": "error" if extract_count < error_floor else "warning",+ "note_id": "<vault>",+ "message": (+ f"Vault has {source_count} fetched source notes but only {extract_count} "+ f"extract notes ({ratio:.0%} coverage, need ≥{required_extracts}). "+ f"The analyst was skipped on most sources. Spawn "+ f"hyperresearch-analyst (mode=extract or mode=guided) on the "+ f"unanalyzed sources during curation. Target: at least 1 extract per "+ f"3 sources (floor of 1 for any corpus size)."+ ),+ })++ if "orphaned-raw-files" in rules_to_run:+ # Walk research/raw/ and flag files whose stem doesn't match any note+ # id in the vault. These are leftovers from the pre-Batch-2.5 `note rm`+ # which never touched raw files. A cheap disk-leak detector.+ raw_dir = vault.root / "research" / "raw"+ if raw_dir.is_dir():+ note_ids = {r["id"] for r in conn.execute("SELECT id FROM notes")}+ for raw_file in raw_dir.iterdir():+ if not raw_file.is_file():+ continue+ # Only flag known raw extensions; ignore any README etc.+ if raw_file.suffix.lower() not in {".pdf", ".png", ".jpg", ".jpeg", ".gif", ".webp"}:+ continue+ if raw_file.stem not in note_ids:+ issues.append({+ "rule": "orphaned-raw-files",+ "severity": "warning",+ "note_id": raw_file.stem,+ "note_path": str(raw_file.relative_to(vault.root).as_posix()),+ "message": (+ f"Raw file {raw_file.name} has no matching note in the vault. "+ f"Likely a leftover from an old `note rm` that didn't clean up "+ f"raw files. Delete it manually or let repair handle it."+ ),+ })++ if "workflow" in rules_to_run:+ # Detect research sessions that skipped the mandatory process steps.+ # Heuristic: a note with a research-output signal (final_report in id,+ # type=moc, or tag=synthesis) means a research session ran. If that+ # session didn't produce a scaffold note AND a comparison note, the+ # protocol was skipped.+ def _count_by_tag(tag: str) -> int:+ row = conn.execute(+ "SELECT COUNT(DISTINCT note_id) as c FROM tags WHERE tag = ?",+ (tag,),+ ).fetchone()+ return row["c"] if row else 0++ has_research_output = conn.execute("""+ SELECT COUNT(*) as c FROM notes n+ WHERE n.id LIKE '%final_report%'+ OR n.type = 'moc'+ OR n.id IN (SELECT note_id FROM tags WHERE tag = 'synthesis')+ """).fetchone()["c"]++ scaffold_count = _count_by_tag("scaffold")+ comparison_count = _count_by_tag("comparison")+ extract_count = _count_by_tag("extract")++ if has_research_output > 0:+ if scaffold_count == 0:+ issues.append({+ "rule": "workflow",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Vault has {has_research_output} research-output note(s) "+ f"but 0 scaffold notes. Step 7 was skipped. "+ f"Research sessions must produce a `scaffold` note before the draft."+ ),+ })+ if comparison_count == 0:+ issues.append({+ "rule": "workflow",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"Vault has {has_research_output} research-output note(s) "+ f"but 0 comparison notes. Step 8 was skipped. "+ f"Research sessions must produce a `comparison` note before the draft."+ ),+ })++ # Flag extract notes that are missing the parent link+ for row in conn.execute("""+ SELECT n.id, n.path FROM notes n+ WHERE n.id IN (SELECT note_id FROM tags WHERE tag = 'extract')+ AND (n.parent IS NULL OR n.parent = '')+ """):+ issues.append({+ "rule": "workflow",+ "severity": "warning",+ "note_id": row["id"],+ "note_path": row["path"],+ "message": "Extract note missing --parent source-id. The chain of custody is broken.",+ })++ if "uncurated" in rules_to_run:+ # Any note that has moved past draft without tier/content_type classification+ # is a curation failure. Exempt: draft notes (expected raw), index/raw/moc+ # types (not sources), and notes whose id starts with _ (auto-generated+ # index notes like _most-linked, _stale, _orphans).+ for row in conn.execute(+ "SELECT n.id, n.path, n.status, n.tier, n.content_type FROM notes n "+ "WHERE n.type NOT IN ('index','raw','moc') "+ "AND n.id NOT LIKE '\\_%' ESCAPE '\\' "+ "AND n.status != 'draft' "+ "AND (n.tier IS NULL OR n.tier = 'unknown' "+ " OR n.content_type IS NULL OR n.content_type = 'unknown')"+ ):+ missing = []+ if not row["tier"] or row["tier"] == "unknown":+ missing.append("tier")+ if not row["content_type"] or row["content_type"] == "unknown":+ missing.append("content_type")+ issues.append({+ "rule": "uncurated",+ "severity": "warning",+ "note_id": row["id"],+ "note_path": row["path"],+ "message": f"Note is {row['status']} but missing {'/'.join(missing)}. Run curation pass.",+ })+if "singleton-tags" in rules_to_run:for row in conn.execute("SELECT tag, COUNT(*) as c FROM tags GROUP BY tag HAVING c = 1"@@ -182,6 +830,34 @@ def lint("message": f"Last reviewed {row['reviewed'][:10]}. Consider re-reviewing.",})+ # Audit-gate self-certification post-check. For each CRITICAL finding+ # that was marked `fixed_at`, we queued its implied lint rule for+ # re-running via rules_to_run. Now that the full lint pass is done,+ # check whether the rule still emitted errors. If yes, the agent marked+ # a CRITICAL resolved without actually fixing the underlying vault state+ # — emit a self-cert violation error that blocks the save gate.+ if audit_gate_guards:+ for guard in audit_gate_guards:+ rule_errors = [+ i for i in issues+ if i.get("rule") == guard["rule"] and i.get("severity") == "error"+ ]+ if rule_errors:+ issues.append({+ "rule": "audit-gate",+ "severity": "error",+ "note_id": "<vault>",+ "message": (+ f"SELF-CERTIFICATION VIOLATION: CRITICAL [{guard['critical_id']}] "+ f"was marked `fixed_at` in research/audit_findings.json, but lint "+ f"rule `{guard['rule']}` still returns {len(rule_errors)} error(s). "+ f"The finding was '{guard['description']}'. The draft's `fixed_at` "+ f"marker does not match the vault's actual state — you must fix the "+ f"underlying issue (not just the bookkeeping). Run "+ f"`$HPR lint --rule {guard['rule']} -j` to see what's still broken."+ ),+ })+summary = {"errors": sum(1 for i in issues if i.get("severity") == "error"),"warnings": sum(1 for i in issues if i.get("severity") == "warning"),diff --git a/src/hyperresearch/cli/note.py b/src/hyperresearch/cli/note.pyindex b4f443f..9f668f8 100644--- a/src/hyperresearch/cli/note.py+++ b/src/hyperresearch/cli/note.py@@ -24,6 +24,8 @@ def note_new(status: str = typer.Option("draft", "--status", "-s", help="Initial status"),summary: str | None = typer.Option(None, "--summary", help="One-line summary"),source: str | None = typer.Option(None, "--source", help="Source URL or path"),+ tier: str | None = typer.Option(None, "--tier", help="Epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),+ content_type: str | None = typer.Option(None, "--content-type", help="Artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),template: str | None = typer.Option(None, "--template", "-T", help="Template: note|concept|reference|guide|comparison|moc"),edit: bool = typer.Option(False, "--edit", "-e", help="Open in $EDITOR"),json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),@@ -38,7 +40,29 @@ def note_new(from hyperresearch.core.note import write_notefrom hyperresearch.core.vault import Vault- from hyperresearch.models.note import slugify+ from hyperresearch.models.note import ContentType, Tier, slugify++ # Validate enums up-front so invalid values fail clearly+ if tier is not None:+ try:+ Tier(tier)+ except ValueError:+ valid = ", ".join(t.value for t in Tier)+ if json_output:+ output(error(f"Invalid --tier '{tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)+ else:+ console.print(f"[red]Invalid --tier '{tier}'.[/] Must be one of: {valid}")+ raise typer.Exit(1)+ if content_type is not None:+ try:+ ContentType(content_type)+ except ValueError:+ valid = ", ".join(c.value for c in ContentType)+ if json_output:+ output(error(f"Invalid --content-type '{content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)+ else:+ console.print(f"[red]Invalid --content-type '{content_type}'.[/] Must be one of: {valid}")+ raise typer.Exit(1)vault = Vault.discover()vault.auto_sync()@@ -92,11 +116,11 @@ def note_new(console.print(f"[yellow]Template '{template}' not found, using default.[/]")path = write_note(vault.notes_dir, title, body=body, tags=tags, status=status,note_type=note_type, parent=parent, summary=summary,- source=source)+ source=source, tier=tier, content_type=content_type)else:path = write_note(vault.notes_dir, title, body=body, tags=tags, status=status,note_type=note_type, parent=parent, summary=summary,- source=source)+ source=source, tier=tier, content_type=content_type)# Read back the note ID (may have been collision-adjusted)from hyperresearch.core.note import read_note@@ -154,6 +178,8 @@ def note_show(data = {"id": row["id"], "title": row["title"], "path": row["path"],"status": row["status"], "type": row["type"], "tags": tags,+ "tier": row["tier"] if "tier" in row.keys() else None,+ "content_type": row["content_type"] if "content_type" in row.keys() else None,"created": row["created"], "updated": row["updated"],"word_count": row["word_count"], "source": row["source"],"parent": row["parent"], "summary": row["summary"],@@ -232,6 +258,8 @@ def note_list(note_type: str | None = typer.Option(None, "--type", help="Filter by type"),tag: str | None = typer.Option(None, "--tag", "-t", help="Filter by tag"),parent: str | None = typer.Option(None, "--parent", "-p", help="Filter by parent"),+ tier: str | None = typer.Option(None, "--tier", help="Filter by epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),+ content_type: str | None = typer.Option(None, "--content-type", help="Filter by artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),sort: str = typer.Option("updated", "--sort", help="Sort: created|updated|title|words"),limit: int = typer.Option(20, "--limit", "-l", help="Max results"),all_notes: bool = typer.Option(False, "--all", "-a", help="Return all notes (no limit)"),@@ -239,6 +267,29 @@ def note_list() -> None:"""List notes with optional filters."""from hyperresearch.core.vault import Vault+ from hyperresearch.models.note import ContentType, Tier++ # Validate enums up-front+ if tier is not None:+ try:+ Tier(tier)+ except ValueError:+ valid = ", ".join(t.value for t in Tier)+ if json_output:+ output(error(f"Invalid --tier '{tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)+ else:+ console.print(f"[red]Invalid --tier '{tier}'.[/] Must be one of: {valid}")+ raise typer.Exit(1)+ if content_type is not None:+ try:+ ContentType(content_type)+ except ValueError:+ valid = ", ".join(c.value for c in ContentType)+ if json_output:+ output(error(f"Invalid --content-type '{content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)+ else:+ console.print(f"[red]Invalid --content-type '{content_type}'.[/] Must be one of: {valid}")+ raise typer.Exit(1)vault = Vault.discover()vault.auto_sync()@@ -258,6 +309,12 @@ def note_list(if tag:clauses.append("n.id IN (SELECT note_id FROM tags WHERE tag = ?)")params.append(tag.lower())+ if tier:+ clauses.append("n.tier = ?")+ params.append(tier)+ if content_type:+ clauses.append("n.content_type = ?")+ params.append(content_type)where = " AND ".join(clauses) if clauses else "1=1"@@ -286,6 +343,8 @@ def note_list("path": row["path"],"status": row["status"],"type": row["type"],+ "tier": row["tier"] if "tier" in row.keys() else None,+ "content_type": row["content_type"] if "content_type" in row.keys() else None,"tags": tag_list,"word_count": row["word_count"],"summary": row["summary"],@@ -334,6 +393,8 @@ def note_update(set_summary: str | None = typer.Option(None, "--summary", help="Set summary"),set_parent: str | None = typer.Option(None, "--parent", "-p", help="Set parent topic"),set_source: str | None = typer.Option(None, "--source", help="Set source URL/path"),+ set_tier: str | None = typer.Option(None, "--tier", help="Set epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),+ set_content_type: str | None = typer.Option(None, "--content-type", help="Set artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),deprecate: bool = typer.Option(False, "--deprecate", help="Mark as deprecated"),json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),) -> None:@@ -343,6 +404,29 @@ def note_update(from hyperresearch.core.frontmatter import parse_frontmatter, serialize_frontmatterfrom hyperresearch.core.sync import compute_sync_plan, execute_syncfrom hyperresearch.core.vault import Vault+ from hyperresearch.models.note import ContentType, Tier++ # Validate enums up-front+ if set_tier is not None:+ try:+ Tier(set_tier)+ except ValueError:+ valid = ", ".join(t.value for t in Tier)+ if json_output:+ output(error(f"Invalid --tier '{set_tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)+ else:+ console.print(f"[red]Invalid --tier '{set_tier}'.[/] Must be one of: {valid}")+ raise typer.Exit(1)+ if set_content_type is not None:+ try:+ ContentType(set_content_type)+ except ValueError:+ valid = ", ".join(c.value for c in ContentType)+ if json_output:+ output(error(f"Invalid --content-type '{set_content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)+ else:+ console.print(f"[red]Invalid --content-type '{set_content_type}'.[/] Must be one of: {valid}")+ raise typer.Exit(1)vault = Vault.discover()vault.auto_sync()@@ -380,6 +464,12 @@ def note_update(if set_source is not None:meta.source = set_sourcechanged.append("source")+ if set_tier is not None:+ meta.tier = set_tier+ changed.append(f"tier={set_tier}")+ if set_content_type is not None:+ meta.content_type = set_content_type+ changed.append(f"content_type={set_content_type}")if deprecate:meta.deprecated = Truemeta.status = "deprecated"@@ -448,11 +538,22 @@ def note_rm(force: bool = typer.Option(False, "--force", "-f", help="Skip confirmation"),json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),) -> None:- """Delete a note."""+ """Delete a note and its associated raw file and assets.++ Previous versions only unlinked the `.md` file, leaving raw PDFs under+ `research/raw/` and assets under `research/assets/<id>/` as orphans.+ Every fetch-then-delete cycle leaked disk. The current implementation+ also removes:+ - the raw file referenced in the note's `raw_file` frontmatter field+ - any entries in the `assets` table and their files on disk+ """+ import shutilfrom hyperresearch.core.vault import Vaultvault = Vault.discover()- row = vault.db.execute("SELECT path FROM notes WHERE id = ?", (note_id,)).fetchone()+ row = vault.db.execute(+ "SELECT path FROM notes WHERE id = ?", (note_id,)+ ).fetchone()if not row:if json_output:output(error(f"Note not found: {note_id}", "NOT_FOUND"), json_mode=True)@@ -464,6 +565,34 @@ def note_rm(if not force and not json_output:typer.confirm(f"Delete {row['path']}?", abort=True)+ removed_raw: str | None = None+ removed_assets: list[str] = []++ # Read raw_file straight from the markdown frontmatter — the DB row+ # may not carry that field if the schema was synced before the column+ # existed. File parsing is authoritative.+ if file_path.exists():+ try:+ from hyperresearch.core.frontmatter import parse_frontmatter+ text = file_path.read_text(encoding="utf-8-sig")+ meta, _ = parse_frontmatter(text)+ if meta.raw_file:+ raw_path = vault.root / "research" / meta.raw_file+ if raw_path.exists():+ raw_path.unlink()+ removed_raw = str(raw_path.relative_to(vault.root).as_posix())+ except Exception:+ pass++ # Assets directory+ assets_dir = vault.root / "research" / "assets" / note_id+ if assets_dir.exists() and assets_dir.is_dir():+ for asset_file in assets_dir.iterdir():+ if asset_file.is_file():+ removed_assets.append(asset_file.name)+ shutil.rmtree(assets_dir, ignore_errors=True)++ # Finally, unlink the .md fileif file_path.exists():file_path.unlink()@@ -473,9 +602,20 @@ def note_rm(plan = compute_sync_plan(vault)execute_sync(vault, plan)+ payload: dict = {"deleted": note_id}+ if removed_raw:+ payload["removed_raw"] = removed_raw+ if removed_assets:+ payload["removed_assets"] = removed_assets+if json_output:- output(success({"deleted": note_id}, vault=str(vault.root)), json_mode=True)+ output(success(payload, vault=str(vault.root)), json_mode=True)else:- console.print(f"[red]Deleted:[/] {note_id}")+ msg = f"[red]Deleted:[/] {note_id}"+ if removed_raw:+ msg += f"\n raw: {removed_raw}"+ if removed_assets:+ msg += f"\n assets: {len(removed_assets)} file(s)"+ console.print(msg)diff --git a/src/hyperresearch/core/note.py b/src/hyperresearch/core/note.pyindex 9ea8c37..cbccd87 100644--- a/src/hyperresearch/core/note.py+++ b/src/hyperresearch/core/note.py@@ -8,7 +8,12 @@ from datetime import UTC, datetimefrom pathlib import Pathfrom hyperresearch.core.frontmatter import parse_frontmatter, render_note-from hyperresearch.core.patterns import CODE_BLOCK_RE, INLINE_CODE_RE, WIKI_LINK_RE+from hyperresearch.core.patterns import (+ CODE_BLOCK_RE,+ INLINE_CODE_RE,+ WIKI_LINK_RE,+ is_valid_wiki_link_target,+)from hyperresearch.models.note import Note, NoteMeta, slugify@@ -25,14 +30,12 @@ def read_note(file_path: Path, vault_root: Path) -> Note:if not meta.id:meta.id = slugify(file_path.stem)- # Extract outgoing wiki links+ # Extract outgoing wiki links, filtering citation footnotes and URLscleaned = CODE_BLOCK_RE.sub("", body)cleaned = INLINE_CODE_RE.sub("", cleaned)raw_links = (m.group(1).strip().rstrip("\\") for m in WIKI_LINK_RE.finditer(cleaned))outgoing = list(dict.fromkeys(- ref for ref in raw_links- if ref and not ref.startswith(("http://", "https://", "ftp://", "mailto:"))- and not ref.isdigit() # Skip [[100]]-style citation numbers+ ref for ref in raw_links if is_valid_wiki_link_target(ref)))# Word count (simple split)@@ -60,12 +63,16 @@ def write_note(parent: str | None = None,source: str | None = None,summary: str | None = None,+ tier: str | None = None,+ content_type: str | None = None,extra_frontmatter: dict | None = None,) -> Path:"""Create a new note file on disk. Returns the file path.Args:notes_dir: The directory to write into (e.g. vault.notes_dir).+ tier: Epistemic role — ground_truth|institutional|practitioner|commentary|unknown.+ content_type: Artifact kind — paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown.extra_frontmatter: Additional fields to set on NoteMeta (e.g. source_domain, fetched_at)."""nid = note_id or slugify(title)@@ -78,6 +85,8 @@ def write_note(parent=parent,source=source,summary=summary,+ tier=tier,+ content_type=content_type,created=datetime.now(UTC),)if extra_frontmatter:diff --git a/src/hyperresearch/core/patterns.py b/src/hyperresearch/core/patterns.pyindex bd28465..174b660 100644--- a/src/hyperresearch/core/patterns.py+++ b/src/hyperresearch/core/patterns.py@@ -10,3 +10,84 @@ WIKI_LINK_RE = re.compile(r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]")# Code block patterns for stripping before link extractionCODE_BLOCK_RE = re.compile(r"```.*?```", re.DOTALL)INLINE_CODE_RE = re.compile(r"`[^`]+`")+++# Citation-footnote patterns that should NOT be treated as wiki-links even+# though crawl4ai sometimes renders them as [[100]] or [[^100]] in fetched+# markdown. These originate from Wikipedia-style footnote markup like+# `[[100]](https://en.wikipedia.org/wiki/Foo#cite_note-100)` where the+# visible text is the citation number in brackets.+_CITATION_FOOTNOTE_RE = re.compile(+ r"^\^?\d+(?:[\s,;\-]+\d+)*$"+)++# Explicit citation prefixes: [[cite-1]], [[ref_2]], [[fn:3]], [[note-4]]+_CITE_PREFIX_RE = re.compile(+ r"^(?:cite|ref|note|fn|footnote|endnote)[-_:]?\d+$",+ re.IGNORECASE,+)++# Roman-numeral footnotes: [[iv]], [[xii]], [[ccxliv]] (lowercase or upper).+# Academic papers use these for appendices and preface footnotes. Uses the+# standard Roman-numeral grammar (not just "chars in [ivxlcdm]") so real+# words like "civic" or "mix" don't match by accident. The empty string is+# explicitly excluded — it would otherwise match M{0}X{0}I{0}.+_ROMAN_FOOTNOTE_RE = re.compile(+ r"^(?=[ivxlcdm])M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$",+ re.IGNORECASE,+)++# Symbol footnotes: [[*]], [[**]], [[†]], [[‡]], [[§]], [[¶]], [[#]], [[△]].+# Typography-style footnote markers used in older academic publishing.+_SYMBOL_FOOTNOTE_RE = re.compile(+ r"^[\*\u2020\u2021\u00a7\u00b6#\u25b3\u25bd]{1,3}$"+)++# Figure / table / equation references: [[fig-3]], [[tab-1]], [[eq-2]],+# [[table1]], [[figure-4b]], [[eq:7]], [[scheme-3]]. These are cross-+# references within a document, not wiki-links to other notes.+_DOC_REF_PREFIX_RE = re.compile(+ r"^(?:fig(?:ure)?|tab(?:le)?|eq(?:uation)?|scheme|chart|plate|box|chart|algo(?:rithm)?)[-_:]?\w*$",+ re.IGNORECASE,+)+++def is_valid_wiki_link_target(ref: str) -> bool:+ """Return True if a ``[[ref]]`` should be treated as a note reference.++ Filters out content that crawl4ai and other markdown fetchers sometimes+ produce as double-bracket targets but that should NOT resolve to notes:++ - Empty / whitespace-only refs+ - URLs (``http://``, ``https://``, ``ftp://``, ``mailto:``)+ - Anchors (``#section-id``) and absolute paths (``/path/to/thing``)+ - Pure-numeric citation footnotes: ``[[100]]``, ``[[^100]]``,+ ``[[100,101]]``, ``[[1-5]]``, ``[[100; 101]]``+ - Explicit citation prefixes: ``[[cite-1]]``, ``[[ref_2]]``, ``[[fn:3]]``,+ ``[[note-4]]``, ``[[footnote-5]]``+ - Roman-numeral footnotes: ``[[iv]]``, ``[[xii]]``+ - Symbol footnotes: ``[[*]]``, ``[[**]]``, ``[[†]]``, ``[[‡]]``, ``[[§]]``+ - Document cross-references: ``[[fig-3]]``, ``[[tab-1]]``, ``[[eq-2]]``,+ ``[[figure-4b]]``, ``[[scheme-3]]``, ``[[algorithm-2]]``++ Valid note references starting with digits (e.g. ``[[10-rules-for-X]]``)+ are preserved because they contain non-digit characters.+ """+ if not ref:+ return False+ ref = ref.strip()+ if not ref:+ return False+ if ref.startswith(("http://", "https://", "ftp://", "mailto:", "#", "/")):+ return False+ if _CITATION_FOOTNOTE_RE.match(ref):+ return False+ if _CITE_PREFIX_RE.match(ref):+ return False+ if _ROMAN_FOOTNOTE_RE.match(ref):+ return False+ if _SYMBOL_FOOTNOTE_RE.match(ref):+ return False+ if _DOC_REF_PREFIX_RE.match(ref):+ return False+ return Truediff --git a/src/hyperresearch/core/sync.py b/src/hyperresearch/core/sync.pyindex 7df9805..cb9be24 100644--- a/src/hyperresearch/core/sync.py+++ b/src/hyperresearch/core/sync.py@@ -10,7 +10,12 @@ from datetime import UTC, datetimefrom pathlib import Pathfrom hyperresearch.core.note import read_note, strip_markdown-from hyperresearch.core.patterns import CODE_BLOCK_RE, INLINE_CODE_RE, WIKI_LINK_RE+from hyperresearch.core.patterns import (+ CODE_BLOCK_RE,+ INLINE_CODE_RE,+ WIKI_LINK_RE,+ is_valid_wiki_link_target,+)@dataclass@@ -42,13 +47,24 @@ def compute_sync_plan(vault, force: bool = False) -> SyncPlan:plan = SyncPlan()# Only scan inside the research directory (notes/, index/)- # This avoids walking .git/, .venv/, src/, etc. entirely+ # This avoids walking .git/, .venv/, src/, etc. entirely.+ #+ # Files at the research/ root (e.g. research/scaffold.md,+ # research/comparisons.md, research/synthesis.md) are STAGING files the+ # agent writes then registers as real notes via `note new --body-file`.+ # They must NOT be synced as notes themselves — otherwise every run+ # produces 4 orphan notes and the missing-title/missing-tags/missing-summary+ # lint rules spam warnings for every research session.kb_dir = vault.research_dirif not kb_dir.exists():return plandisk_files: dict[str, float] = {}for md_file in kb_dir.rglob("*.md"):+ # Skip staging files at the research/ root. Real notes live in+ # research/notes/** or research/index/**.+ if md_file.parent == kb_dir:+ continuerel = md_file.relative_to(vault.root).as_posix()disk_files[rel] = md_file.stat().st_mtime@@ -155,13 +171,14 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Nonconn.execute("""INSERT INTO notes- (id, title, path, status, type, source, parent,+ (id, title, path, status, type, tier, content_type, source, parent,deprecated, reviewed, expires, word_count, summary,created, updated, file_mtime, content_hash, synced_at)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)ON CONFLICT(id) DO UPDATE SETtitle=excluded.title, path=excluded.path, status=excluded.status,- type=excluded.type, source=excluded.source, parent=excluded.parent,+ type=excluded.type, tier=excluded.tier, content_type=excluded.content_type,+ source=excluded.source, parent=excluded.parent,deprecated=excluded.deprecated,reviewed=excluded.reviewed, expires=excluded.expires,word_count=excluded.word_count, summary=excluded.summary,@@ -171,6 +188,7 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Non""",(meta.id, meta.title, note.path, meta.status, meta.type,+ meta.tier, meta.content_type,meta.source, meta.parent, 1 if meta.deprecated else 0,reviewed_iso, expires_iso,note.word_count, meta.summary, created_iso, updated_iso,@@ -213,16 +231,14 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Non)# Update links — extract from original body (not stripped)+ # Uses the shared filter so this path stays in sync with core.note.read_noteconn.execute("DELETE FROM links WHERE source_id = ?", (meta.id,))cleaned = CODE_BLOCK_RE.sub("", note.body)cleaned = INLINE_CODE_RE.sub("", cleaned)for line_num, line in enumerate(cleaned.split("\n"), 1):for m in WIKI_LINK_RE.finditer(line):target_ref = m.group(1).strip().rstrip("\\") # Strip trailing backslash (shell escaping artifact)- # Skip URLs — they're not note references- if target_ref.startswith(("http://", "https://", "ftp://", "mailto:")):- continue- if not target_ref:+ if not is_valid_wiki_link_target(target_ref):continueconn.execute("INSERT OR IGNORE INTO links (source_id, target_ref, line_number, context) "diff --git a/tests/test_cli/test_lint.py b/tests/test_cli/test_lint.pynew file mode 100644index 0000000..dace659--- /dev/null+++ b/tests/test_cli/test_lint.py@@ -0,0 +1,517 @@+"""Tests for lint rules — especially the gospel-enforcing scaffold-prompt rule."""++from __future__ import annotations++import pytest+from typer.testing import CliRunner++from hyperresearch.cli.lint import app as lint_app+from hyperresearch.core.note import write_note+++def _run_lint(vault, rule: str | None = None) -> tuple[int, str]:+ """Invoke `hyperresearch lint` CLI against a vault. Returns (exit_code, stdout)."""+ import os++ runner = CliRunner()+ prev_cwd = os.getcwd()+ try:+ os.chdir(vault.root)+ args = ["--json"]+ if rule:+ args = ["--rule", rule, "--json"]+ result = runner.invoke(lint_app, args, catch_exceptions=False)+ return result.exit_code, result.output+ finally:+ os.chdir(prev_cwd)+++def _write_scaffold(vault, body: str, note_id: str = "scaffold-test"):+ write_note(+ vault.notes_dir,+ "Scaffold: Test",+ body=body,+ tags=["scaffold"],+ note_id=note_id,+ )+ vault.auto_sync()+++def test_scaffold_prompt_passes_with_verbatim_prompt(tmp_vault):+ _write_scaffold(+ tmp_vault,+ body=(+ "## User Prompt (VERBATIM — gospel)\n"+ "> I would like a detailed analysis of the Saint Seiya franchise and its armor classes. "+ "For each significant character please describe techniques and fate.\n\n"+ "## What the user explicitly asked for\n"+ "- Entity enumeration\n"+ ),+ )+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")+ assert code == 0+ # Output should contain no scaffold-prompt issues+ import json+ data = json.loads(out)+ issues = data.get("data", {}).get("issues", [])+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]+ assert scaffold_issues == []+++def test_scaffold_prompt_fails_when_header_missing(tmp_vault):+ _write_scaffold(+ tmp_vault,+ body=(+ "## Thesis\n"+ "Saint Seiya's armor hierarchy encodes a theology of sacrifice.\n\n"+ "## Heading progression\n"+ "1. Cosmology\n"+ ),+ )+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")+ import json+ data = json.loads(out)+ issues = data.get("data", {}).get("issues", [])+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]+ assert len(scaffold_issues) == 1+ assert scaffold_issues[0]["severity"] == "error"+ assert "verbatim" in scaffold_issues[0]["message"].lower()+++def test_scaffold_prompt_warns_when_quote_too_short(tmp_vault):+ _write_scaffold(+ tmp_vault,+ body=(+ "## User Prompt (VERBATIM — gospel)\n"+ "> hi\n\n"+ "## What the user explicitly asked for\n"+ "- thing\n"+ ),+ )+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")+ import json+ data = json.loads(out)+ issues = data.get("data", {}).get("issues", [])+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]+ assert len(scaffold_issues) == 1+ assert scaffold_issues[0]["severity"] == "warning"+++def test_scaffold_prompt_no_scaffold_notes_is_noop(tmp_vault):+ # A vault with no scaffold-tagged notes should produce no issues for this rule.+ code, out = _run_lint(tmp_vault, rule="scaffold-prompt")+ assert code == 0+ import json+ data = json.loads(out)+ issues = data.get("data", {}).get("issues", [])+ scaffold_issues = [i for i in issues if i.get("rule") == "scaffold-prompt"]+ assert scaffold_issues == []+++def _write_source(vault, title: str, note_id: str, body: str = "Source content."):+ write_note(+ vault.notes_dir,+ title,+ body=body,+ note_id=note_id,+ source=f"https://example.com/{note_id}",+ tier="institutional",+ content_type="article",+ )+++def test_provenance_rooted_tree_passes_with_valid_chain(tmp_vault):+ # Build a valid chain: seed → child1 → child2, plus a few more non-seeds+ # pointing at the seed or each other. 6+ sources so the rule activates.+ _write_source(tmp_vault, "Seed Source", "seed-one", body="Seminal paper on X.")+ _write_source(tmp_vault, "Child A", "child-a", body="*Suggested by [[seed-one]] — follow-up citation*\n\nDerivative work.")+ _write_source(tmp_vault, "Child B", "child-b", body="*Suggested by [[seed-one]] — cross-reference*\n\nRelated analysis.")+ _write_source(tmp_vault, "Grandchild", "grandchild", body="*Suggested by [[child-a]] — deeper dive*\n\nMore content.")+ _write_source(tmp_vault, "Extra seed", "seed-two", body="Another primary source.")+ _write_source(tmp_vault, "Child C", "child-c", body="*Suggested by [[seed-two]] — reply*\n\nCritical response.")+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="provenance")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]+ assert issues == [], f"expected no issues, got: {issues}"+++def test_provenance_fails_when_all_sources_are_seeds(tmp_vault):+ # 6 sources, zero breadcrumbs — flat batch, rule must fire.+ for i in range(6):+ _write_source(tmp_vault, f"Source {i}", f"source-{i}")+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="provenance")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]+ assert len(issues) >= 1+ assert any("ZERO" in i["message"] or "no seed" in i["message"].lower() or "bouncing reading loop" in i["message"] for i in issues)+++def test_provenance_fails_on_dangling_breadcrumb(tmp_vault):+ # Six sources; one of them references a non-existent note.+ _write_source(tmp_vault, "Seed", "seed-one")+ _write_source(tmp_vault, "Child A", "child-a", body="*Suggested by [[seed-one]] — real*\n")+ _write_source(tmp_vault, "Child B", "child-b", body="*Suggested by [[seed-one]] — real*\n")+ _write_source(tmp_vault, "Dangler", "dangler", body="*Suggested by [[nonexistent-note]] — fake*\n")+ _write_source(tmp_vault, "Child C", "child-c", body="*Suggested by [[seed-one]] — real*\n")+ _write_source(tmp_vault, "Child D", "child-d", body="*Suggested by [[seed-one]] — real*\n")+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="provenance")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]+ assert any("nonexistent-note" in i["message"] for i in issues), (+ f"expected dangling breadcrumb issue, got: {issues}"+ )+++def test_provenance_small_corpus_is_skipped(tmp_vault):+ # <= 5 sources, rule should not complain (loop may not have fired by design).+ for i in range(3):+ _write_source(tmp_vault, f"Source {i}", f"source-{i}")+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="provenance")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]+ assert issues == []+++def test_provenance_errors_on_under_30pct_non_seed_ratio(tmp_vault):+ """The guided reading loop must actually fire. A large corpus with only+ a token 1-2 breadcrumbs should ERROR, not just warn. This is the exact+ failure mode both v2 runs exhibited (3/19 and 2/33 breadcrumbs)."""+ # 11 sources, 2 of them with breadcrumbs = 18% non-seed (below 30%)+ _write_source(tmp_vault, "Seed 1", "seed-one")+ _write_source(tmp_vault, "Seed 2", "seed-two")+ for i in range(9):+ # 2 of the 9 have breadcrumbs; rest are seeds+ if i < 2:+ body = f"*Suggested by [[seed-one]] — from analyst*\n\nContent {i}"+ else:+ body = f"Content {i}"+ _write_source(tmp_vault, f"Source {i}", f"source-{i}", body=body)+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="provenance")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "provenance"]+ errors = [i for i in issues if i.get("severity") == "error"]+ assert len(errors) >= 1+ assert "guided reading loop did not fire" in errors[0]["message"]+++def test_orphaned_raw_files_flags_disk_leak(tmp_vault):+ """Files in research/raw/ with no matching note should be flagged."""+ raw_dir = tmp_vault.root / "research" / "raw"+ raw_dir.mkdir(parents=True, exist_ok=True)+ # Create a raw file whose stem doesn't match any note.+ (raw_dir / "orphan-note.pdf").write_bytes(b"%PDF-1.4 dummy")+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="orphaned-raw-files")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "orphaned-raw-files"]+ assert len(issues) == 1+ assert "orphan-note" in issues[0]["message"]+++def _write_audit_findings(vault, data: dict) -> None:+ import json as _json+ audit_path = vault.root / "research" / "audit_findings.json"+ audit_path.parent.mkdir(parents=True, exist_ok=True)+ audit_path.write_text(_json.dumps(data, indent=2), encoding="utf-8")+++def test_audit_gate_missing_file_is_open(tmp_vault):+ """No audit file = gate is OPEN (early-stage research)."""+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ assert code == 0+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ assert issues == []+++def test_audit_gate_blocks_unresolved_criticals(tmp_vault):+ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T10:00:00Z",+ "status": "needs_fixes",+ "criticals": [+ {"id": "C0", "description": "Scaffold missing verbatim prompt", "fixed_at": None},+ {"id": "C1", "description": "Silver Saints omitted", "fixed_at": None},+ ],+ "important": [],+ "minor": [],+ }+ ]+ })+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ assert len(issues) == 1+ assert "2 unresolved CRITICAL" in issues[0]["message"]+++def test_audit_gate_passes_when_all_criticals_fixed(tmp_vault):+ """A CRITICAL with a vague description that doesn't map to any lint rule+ should emit a WARNING (unverified agent self-report) but NOT an error —+ the save gate still opens because warnings don't block."""+ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T10:00:00Z",+ "status": "pass",+ "criticals": [+ {"id": "C0", "description": "generic content fix", "fixed_at": "2026-04-14T11:00:00Z"},+ ],+ "important": [],+ "minor": [],+ }+ ]+ })+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ errors = [i for i in issues if i.get("severity") == "error"]+ warnings = [i for i in issues if i.get("severity") == "warning"]+ # Save gate stays open (no errors) but emits a warning about the+ # unverified self-report.+ assert errors == []+ assert len(warnings) == 1+ assert "not machine-verified" in warnings[0]["message"]+++def test_audit_gate_uses_most_recent_conformance_run(tmp_vault):+ """If a later run has fixes applied, the gate should pass even if older+ runs had unresolved findings."""+ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T10:00:00Z",+ "status": "needs_fixes",+ "criticals": [+ {"id": "C0", "description": "old finding", "fixed_at": None},+ ],+ "important": [], "minor": [],+ },+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T12:00:00Z",+ "status": "pass",+ "criticals": [],+ "important": [], "minor": [],+ },+ ]+ })+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ assert issues == []+++def test_audit_gate_fails_when_only_comprehensiveness_run_exists(tmp_vault):+ """A file with only comprehensiveness runs means the conformance auditor+ never fired. The save gate must fail in that case."""+ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "comprehensiveness",+ "timestamp": "2026-04-14T10:00:00Z",+ "status": "needs_fixes",+ "criticals": [],+ "important": [+ {"id": "I1", "description": "some gap", "fixed_at": None},+ ],+ "minor": [],+ }+ ]+ })+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ # Expect: one error for missing conformance, one info for important findings.+ errors = [i for i in issues if i.get("severity") == "error"]+ infos = [i for i in issues if i.get("severity") == "info"]+ assert len(errors) == 1+ assert "ZERO" in errors[0]["message"] and "conformance" in errors[0]["message"]+ assert len(infos) == 1+ assert "IMPORTANT" in infos[0]["message"]+++def test_audit_gate_surfaces_important_findings_as_info(tmp_vault):+ """Unresolved IMPORTANT findings should surface as info-severity issues+ (advisory), not block save."""+ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T10:00:00Z",+ "status": "pass",+ "criticals": [],+ "important": [+ {"id": "I1", "description": "Proportional depth uneven", "fixed_at": None},+ {"id": "I2", "description": "Citation to unfetched source", "fixed_at": None},+ ],+ "minor": [],+ }+ ]+ })+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ errors = [i for i in issues if i.get("severity") == "error"]+ infos = [i for i in issues if i.get("severity") == "info"]+ assert errors == [] # save is NOT blocked by IMPORTANT alone+ assert len(infos) == 1+ assert "2 unresolved IMPORTANT" in infos[0]["message"]+++def test_audit_gate_catches_self_certification_on_provenance(tmp_vault):+ """Regression test: when a CRITICAL finding mentions provenance AND is+ marked `fixed_at`, but the provenance rule still returns errors, the+ audit-gate must emit a SELF-CERTIFICATION VIOLATION error.++ This is exactly the Q62 failure mode — the agent marked the C1+ provenance finding as fixed with a justification string, but the+ vault's actual breadcrumb graph was still broken.+ """+ # Build a vault where provenance is genuinely broken (12 sources, none+ # with breadcrumbs — classic flat batch).+ for i in range(12):+ _write_source(tmp_vault, f"Source {i}", f"source-{i}")+ tmp_vault.auto_sync()++ # Audit findings file: conformance run with a CRITICAL marked fixed_at+ # that references provenance. The "fix" is a lie — the vault still fails+ # the provenance rule.+ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T20:00:00Z",+ "status": "pass",+ "criticals": [+ {+ "id": "C1",+ "description": "Provenance chain broken — no --suggested-by breadcrumbs in corpus",+ "fixed_at": "2026-04-14T20:20:00Z",+ }+ ],+ "important": [],+ "minor": [],+ }+ ]+ })++ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = data.get("data", {}).get("issues", [])+ audit_errors = [+ i for i in issues+ if i.get("rule") == "audit-gate" and i.get("severity") == "error"+ ]+ self_cert_errors = [+ i for i in audit_errors if "SELF-CERTIFICATION VIOLATION" in i["message"]+ ]+ assert len(self_cert_errors) == 1, f"expected self-cert violation, got: {audit_errors}"+ assert "C1" in self_cert_errors[0]["message"]+ assert "provenance" in self_cert_errors[0]["message"]+++def test_audit_gate_no_self_cert_when_fix_genuinely_landed(tmp_vault):+ """Control: when CRITICAL fixed_at IS set AND the underlying lint rule+ genuinely passes, the audit-gate should NOT emit a self-cert violation."""+ # Build a vault where provenance is healthy (seed + 6 non-seeds, all+ # pointing at real notes).+ _write_source(tmp_vault, "Seed", "seed-one")+ for i in range(6):+ _write_source(+ tmp_vault, f"Child {i}", f"child-{i}",+ body=f"*Suggested by [[seed-one]] — real*\n\nContent {i}"+ )+ tmp_vault.auto_sync()++ _write_audit_findings(tmp_vault, {+ "runs": [+ {+ "mode": "conformance",+ "timestamp": "2026-04-14T20:00:00Z",+ "status": "pass",+ "criticals": [+ {+ "id": "C1",+ "description": "provenance was broken earlier",+ "fixed_at": "2026-04-14T20:20:00Z",+ }+ ],+ "important": [],+ "minor": [],+ }+ ]+ })++ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = data.get("data", {}).get("issues", [])+ self_cert_errors = [+ i for i in issues+ if i.get("rule") == "audit-gate" and "SELF-CERTIFICATION VIOLATION" in i.get("message", "")+ ]+ assert self_cert_errors == [], f"false-positive self-cert: {self_cert_errors}"+++def test_audit_gate_handles_malformed_file(tmp_vault):+ audit_path = tmp_vault.root / "research" / "audit_findings.json"+ audit_path.parent.mkdir(parents=True, exist_ok=True)+ audit_path.write_text("{ not valid json }", encoding="utf-8")+ code, out = _run_lint(tmp_vault, rule="audit-gate")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "audit-gate"]+ assert len(issues) == 1+ assert "malformed" in issues[0]["message"]+++def test_orphaned_raw_files_ignores_matched_raw(tmp_vault):+ """Raw files whose stem matches a note id are not orphans."""+ write_note(+ tmp_vault.notes_dir,+ "PDF Note",+ note_id="real-pdf-note",+ source="https://example.com/paper.pdf",+ tier="ground_truth",+ content_type="paper",+ extra_frontmatter={"raw_file": "raw/real-pdf-note.pdf"},+ )+ raw_dir = tmp_vault.root / "research" / "raw"+ raw_dir.mkdir(parents=True, exist_ok=True)+ (raw_dir / "real-pdf-note.pdf").write_bytes(b"%PDF-1.4 dummy")+ tmp_vault.auto_sync()++ code, out = _run_lint(tmp_vault, rule="orphaned-raw-files")+ import json+ data = json.loads(out)+ issues = [i for i in data.get("data", {}).get("issues", []) if i.get("rule") == "orphaned-raw-files"]+ assert issues == []diff --git a/tests/test_cli/test_note_ops.py b/tests/test_cli/test_note_ops.pyindex bc1be8f..61d9d19 100644--- a/tests/test_cli/test_note_ops.py+++ b/tests/test_cli/test_note_ops.py@@ -38,6 +38,55 @@ def test_note_rm_nonexistent(vault_with_notes):assert result.exit_code == 1+def test_note_rm_cleans_raw_file_and_assets(vault_with_notes):+ """note rm must delete the raw file and assets directory, not just the .md.++ Regression test for Batch 2.5 — prior to this, `note rm` only unlinked+ the `.md` file and raw/assets leaked on disk forever.+ """+ vault_root = vault_with_notes++ # Seed a note with raw_file in its frontmatter and an assets directory.+ from hyperresearch.core.note import write_note+ from hyperresearch.core.vault import Vault+ vault = Vault.discover()++ write_note(+ vault.notes_dir,+ "PDF Source",+ note_id="pdf-source",+ source="https://example.com/paper.pdf",+ tier="ground_truth",+ content_type="paper",+ extra_frontmatter={"raw_file": "raw/pdf-source.pdf"},+ )++ raw_dir = vault_root / "research" / "raw"+ raw_dir.mkdir(parents=True, exist_ok=True)+ raw_file = raw_dir / "pdf-source.pdf"+ raw_file.write_bytes(b"%PDF-1.4 test")++ assets_dir = vault_root / "research" / "assets" / "pdf-source"+ assets_dir.mkdir(parents=True, exist_ok=True)+ (assets_dir / "screenshot.png").write_bytes(b"PNG fake")+ (assets_dir / "figure-1.jpg").write_bytes(b"JPG fake")++ runner.invoke(app, ["sync"])++ # Delete it+ result = runner.invoke(app, ["note", "rm", "pdf-source", "--force", "--json"])+ assert result.exit_code == 0, result.output+ data = json.loads(result.output)+ assert data["ok"] is True+ assert data["data"]["deleted"] == "pdf-source"+ assert data["data"].get("removed_raw") == "research/raw/pdf-source.pdf"+ assert len(data["data"].get("removed_assets", [])) == 2++ # Verify the files are actually gone from disk.+ assert not raw_file.exists(), "raw file should have been deleted"+ assert not assets_dir.exists(), "assets directory should have been removed"++def test_note_mv(vault_with_notes):result = runner.invoke(app, ["note", "mv", "beta-note", "notes/moved/beta-note.md", "--json"]diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.pyindex e661377..f9ff2fe 100644--- a/tests/test_core/test_note.py+++ b/tests/test_core/test_note.py@@ -84,6 +84,38 @@ def test_strip_markdown():assert "[[" not in plain+def test_raw_file_persists_on_roundtrip(tmp_vault):+ """raw_file must survive parse + re-serialize (regression test for wipe bug).++ Before Batch 1.2 fix: raw_file was injected into frontmatter as a string+ AFTER write_note; NoteMeta.model_config = {"extra": "ignore"} silently+ dropped the field on the next parse, and any re-serialization (repair,+ note update) wiped it from disk.+ """+ from hyperresearch.core.frontmatter import parse_frontmatter, render_note++ path = write_note(+ tmp_vault.notes_dir,+ "PDF Note",+ body="# PDF content\n",+ extra_frontmatter={"raw_file": "raw/pdf-note.pdf"},+ )+ # First read — does NoteMeta capture the field?+ note = read_note(path, tmp_vault.root)+ assert note.meta.raw_file == "raw/pdf-note.pdf"++ # Re-serialize (simulates repair.py enrichment or note update).+ text = path.read_text(encoding="utf-8")+ meta, body = parse_frontmatter(text)+ path.write_text(render_note(meta, body), encoding="utf-8")++ # Second read — raw_file must still be present.+ note2 = read_note(path, tmp_vault.root)+ assert note2.meta.raw_file == "raw/pdf-note.pdf", (+ "raw_file was wiped on re-serialize — Batch 1.2 regression"+ )++def test_read_utf8_content(tmp_vault):path = write_note(tmp_vault.notes_dir,diff --git a/tests/test_core/test_patterns.py b/tests/test_core/test_patterns.pynew file mode 100644index 0000000..b233390--- /dev/null+++ b/tests/test_core/test_patterns.py@@ -0,0 +1,133 @@+"""Tests for wiki-link target validation — citation-footnote edge cases."""++from hyperresearch.core.patterns import is_valid_wiki_link_target+++def test_empty_and_whitespace_rejected():+ assert not is_valid_wiki_link_target("")+ assert not is_valid_wiki_link_target(" ")+ assert not is_valid_wiki_link_target("\n")+++def test_urls_rejected():+ assert not is_valid_wiki_link_target("https://example.com/foo")+ assert not is_valid_wiki_link_target("http://example.com")+ assert not is_valid_wiki_link_target("ftp://files.example.com")+ assert not is_valid_wiki_link_target("mailto:[email protected]")+++def test_anchors_and_paths_rejected():+ assert not is_valid_wiki_link_target("#section-1")+ assert not is_valid_wiki_link_target("/path/to/note")+++def test_pure_numeric_citation_footnotes_rejected():+ # The Q91 bug: Wikipedia footnotes rendered as [[100]]+ assert not is_valid_wiki_link_target("100")+ assert not is_valid_wiki_link_target("1")+ assert not is_valid_wiki_link_target("999")+++def test_caret_prefixed_footnotes_rejected():+ # Markdown-standard footnote syntax+ assert not is_valid_wiki_link_target("^1")+ assert not is_valid_wiki_link_target("^100")+++def test_multi_number_citations_rejected():+ # Comma/semicolon/dash-separated citation groups+ assert not is_valid_wiki_link_target("100,101")+ assert not is_valid_wiki_link_target("1, 2, 3")+ assert not is_valid_wiki_link_target("100; 101")+ assert not is_valid_wiki_link_target("1-5")+ assert not is_valid_wiki_link_target("100-105")+++def test_cite_prefix_patterns_rejected():+ assert not is_valid_wiki_link_target("cite-1")+ assert not is_valid_wiki_link_target("cite_2")+ assert not is_valid_wiki_link_target("cite:3")+ assert not is_valid_wiki_link_target("ref-42")+ assert not is_valid_wiki_link_target("ref_42")+ assert not is_valid_wiki_link_target("fn-1")+ assert not is_valid_wiki_link_target("fn:2")+ assert not is_valid_wiki_link_target("note-5")+ assert not is_valid_wiki_link_target("footnote-10")+ assert not is_valid_wiki_link_target("endnote-3")+++def test_cite_prefixes_are_case_insensitive():+ assert not is_valid_wiki_link_target("Cite-1")+ assert not is_valid_wiki_link_target("REF_2")+ assert not is_valid_wiki_link_target("Fn-3")+++def test_valid_note_ids_accepted():+ # Real note IDs should pass through+ assert is_valid_wiki_link_target("pegasus-seiya")+ assert is_valid_wiki_link_target("the-comics-journal-soldier-dream")+ assert is_valid_wiki_link_target("quantum-computing")+ assert is_valid_wiki_link_target("scaffold-saint-seiya")+++def test_note_ids_starting_with_digits_accepted():+ # [[10-rules-for-X]] or [[1984-novel]] must pass — they contain non-digit chars+ assert is_valid_wiki_link_target("10-rules-for-typescript")+ assert is_valid_wiki_link_target("1984-novel")+ assert is_valid_wiki_link_target("100-greatest-films")+++def test_note_ids_that_contain_cite_word_accepted():+ # "citation" is not "cite-N" — should pass+ assert is_valid_wiki_link_target("citations-are-important")+ assert is_valid_wiki_link_target("references-guide")+ assert is_valid_wiki_link_target("note-taking-systems") # plural, no digit+++def test_whitespace_is_stripped():+ # Leading/trailing whitespace should be stripped before evaluation+ assert not is_valid_wiki_link_target(" 100 ")+ assert not is_valid_wiki_link_target("\t^100\n")+ assert is_valid_wiki_link_target(" valid-note-id ")+++def test_roman_numeral_footnotes_rejected():+ # Academic papers use Roman numerals for appendix/preface footnotes.+ assert not is_valid_wiki_link_target("i")+ assert not is_valid_wiki_link_target("iv")+ assert not is_valid_wiki_link_target("xii")+ assert not is_valid_wiki_link_target("xxiv")+ assert not is_valid_wiki_link_target("IV")+ assert not is_valid_wiki_link_target("XIV")+++def test_symbol_footnotes_rejected():+ # Typography-style footnote markers used in older academic publishing.+ assert not is_valid_wiki_link_target("*")+ assert not is_valid_wiki_link_target("**")+ assert not is_valid_wiki_link_target("†")+ assert not is_valid_wiki_link_target("‡")+ assert not is_valid_wiki_link_target("§")+ assert not is_valid_wiki_link_target("¶")+++def test_document_cross_references_rejected():+ # Figure / table / equation refs are document-internal, not wiki-links.+ assert not is_valid_wiki_link_target("fig-3")+ assert not is_valid_wiki_link_target("figure-4b")+ assert not is_valid_wiki_link_target("tab-1")+ assert not is_valid_wiki_link_target("table-2")+ assert not is_valid_wiki_link_target("eq-2")+ assert not is_valid_wiki_link_target("equation-7")+ assert not is_valid_wiki_link_target("scheme-3")+ assert not is_valid_wiki_link_target("algorithm-2")+ assert not is_valid_wiki_link_target("Fig-1") # case-insensitive+++def test_real_words_that_look_like_roman_are_accepted():+ # "dim" and "mix" look roman-like but are real note ids+ assert is_valid_wiki_link_target("mix-matrix")+ assert is_valid_wiki_link_target("div-container")+ # Longer words containing roman chars are fine+ assert is_valid_wiki_link_target("civic")+ assert is_valid_wiki_link_target("minimalism")diff --git a/tests/test_core/test_sync.py b/tests/test_core/test_sync.pyindex a77c416..9d7010a 100644--- a/tests/test_core/test_sync.py+++ b/tests/test_core/test_sync.py@@ -96,3 +96,30 @@ def test_sync_excludes_hyperresearch_dir(tmp_vault):(tmp_vault.root / ".hyperresearch" / "test.md").write_text("---\ntitle: Bad\n---\nShould not sync")plan = compute_sync_plan(tmp_vault)assert all(".hyperresearch" not in str(p) for p in plan.to_add)+++def test_sync_excludes_research_root_staging_files(tmp_vault):+ """Files at research/ root (scaffold.md, comparisons.md, synthesis.md)+ are staging files the agent writes then registers via `note new`. They+ must NOT appear as orphan notes in the vault index — the current+ behavior would produce missing-title/tags/summary lint spam on every run.+ """+ from hyperresearch.core.note import write_note++ # Real notes under research/notes/+ write_note(tmp_vault.notes_dir, "Real Note", body="# Real\n")++ # Staging files at research/ root+ research_root = tmp_vault.research_dir+ (research_root / "scaffold.md").write_text("# Scaffold staging\n")+ (research_root / "comparisons.md").write_text("# Comparisons staging\n")+ (research_root / "synthesis.md").write_text("# Synthesis staging\n")++ plan = compute_sync_plan(tmp_vault)++ # Only the real note should be added.+ added_names = [p.name for p in plan.to_add]+ assert "real-note.md" in added_names+ assert "scaffold.md" not in added_names+ assert "comparisons.md" not in added_names+ assert "synthesis.md" not in added_names