Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
Fix the stored XSS in the wiki viewer and stop the installer deleting skills it didn't write (#75)
Jordan Gibbs27a107f69fb0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e323807..c15c28f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,11 @@## [Unreleased]+### Two safety fixes in the viewer and the installer++- **Stored XSS in `hpr serve` (#72, reported by @letospace).** `_serve_search` escaped every interpolated value except the FTS snippet, and the snippet is note body text — remote page content, for a fetched note. `strip_markdown` was not a defense: its tag regex needs a closing `>`, so an unterminated `<img src=x onerror=...` passed into `body_plain` intact, and the `>` of the `</mark>` the search page injects finished the tag. The snippet is escaped before the markers are substituted now, so the `<mark>` tags are the only markup that survives. Bounded by the server binding `127.0.0.1` and being read-only, but the script ran same-origin with the wiki and could read every note the viewer could reach. Link and image URLs in the renderer also got a scheme allowlist, so a note body can no longer render a `javascript:` or `data:` link.+- **`install --global` deleted `~/.claude/skills/research/` on a name match alone (#73, reported by @letospace).** `research` is an ordinary word and an obvious name for a hand-written personal skill, and `--global` puts the prune in a user-level directory shared across every project, so anyone with one lost it silently on their first upgrade. A marker file cannot fix this — the directories being pruned predate any marker we could have written — so the check is on the content we shipped into them: every `SKILL.md` this project has ever installed names the project. Anything else is left alone and reported in the install output, which previously listed what it installed but never what it deleted.+### Open-access full-text recovery (Unpaywall + Europe PMC)A paywalled paper used to enter the vault as an abstract. `extract_doi` stamped the DOI, the junk gate passed the landing page (an abstract is not junk), and depth investigators then reasoned over ~1,500 characters while the report cited the work as though the paper had been read.diff --git a/src/hyperresearch/core/hooks.py b/src/hyperresearch/core/hooks.pyindex fdea927..b5842f3 100644--- a/src/hyperresearch/core/hooks.py+++ b/src/hyperresearch/core/hooks.py@@ -3679,9 +3679,9 @@ def _prune_global_step_skills(home: Path) -> str | None:suffix = name[len("hyperresearch-"):]if not suffix or not suffix[0].isdigit():continue- for f in child.iterdir():- f.unlink()- child.rmdir()+ if not _is_our_skill_dir(child):+ continue+ _remove_skill_dir(child)pruned.append(name)if not pruned:@@ -3957,6 +3957,38 @@ _RETIRED_SKILL_DIRS: tuple[str, ...] = ("research", # /research alias retired in v0.8.1 — only /hyperresearch now)+# Every SKILL.md this project has ever installed names the project in its+# frontmatter or its body. The retired dirs predate any install marker, so a+# marker file cannot be used to recognize them — the content we shipped is the+# only evidence available after the fact.+_OWNED_SKILL_MARKERS: tuple[str, ...] = ("hyperresearch", "layercake")+++def _is_our_skill_dir(path: Path) -> bool:+ """True when `path` holds a SKILL.md this project wrote.++ A name match alone is not license to delete a directory. `research` is an+ ordinary English word and an obvious name for a hand-written personal+ skill, and `--global` puts the prune in ~/.claude/skills/, which is shared+ across every project the user has. Deleting on name alone destroyed user+ content that was never ours (#73).+ """+ skill_file = path / "SKILL.md"+ if not skill_file.is_file():+ return False+ try:+ text = skill_file.read_text(encoding="utf-8", errors="replace").lower()+ except OSError:+ return False+ return any(marker in text for marker in _OWNED_SKILL_MARKERS)+++def _remove_skill_dir(path: Path) -> None:+ """Delete a skill directory and everything under it."""+ import shutil++ shutil.rmtree(path)+# V1 modality files — left over inside .claude/skills/hyperresearch/ on# vaults that were installed before the V8 alias-based entry skill._RETIRED_HYPERRESEARCH_FILES: tuple[str, ...] = (@@ -3973,8 +4005,13 @@ def _prune_retired_agents(vault_root: Path) -> str | None:Running this on a fresh vault is a no-op. On an upgraded vault, it removesretired agent .md files and the old /research-ensemble + /research-layercakeskill dirs so the installed state matches the current architecture.++ A retired skill dir is only removed when its SKILL.md is recognizably one+ we shipped. Anything else with the same name belongs to the user and is+ reported instead of deleted."""pruned: list[str] = []+ kept: list[str] = []agents_dir = vault_root / ".claude" / "agents"if agents_dir.exists():@@ -3988,16 +4025,13 @@ def _prune_retired_agents(vault_root: Path) -> str | None:if skills_dir.exists():for name in _RETIRED_SKILL_DIRS:p = skills_dir / name- if p.is_dir():- for child in p.iterdir():- if child.is_file():- child.unlink()- elif child.is_dir():- # Should not happen — skills are flat — but be safe- import shutil- shutil.rmtree(child)- p.rmdir()- pruned.append(f"skill dir {name}")+ if not p.is_dir():+ continue+ if not _is_our_skill_dir(p):+ kept.append(name)+ continue+ _remove_skill_dir(p)+ pruned.append(f"skill dir {name}")# V1 modality files (SKILL-collect.md etc.) left inside the# /hyperresearch skill dir from the old multi-file install layout.@@ -4009,9 +4043,17 @@ def _prune_retired_agents(vault_root: Path) -> str | None:p.unlink()pruned.append(f"file hyperresearch/{name}")- if not pruned:+ parts: list[str] = []+ if pruned:+ parts.append("Pruned retired: " + ", ".join(pruned))+ if kept:+ parts.append(+ "Left alone (not ours): "+ + ", ".join(f".claude/skills/{n}" for n in kept)+ )+ if not parts:return None- return "Pruned retired: " + ", ".join(pruned)+ return " | ".join(parts)def _read_skill_source(src_name: str) -> str | None:@@ -4120,9 +4162,9 @@ def _install_hyperresearch_step_skills(vault_root: Path) -> str | None:is_legacy_layercake = child.name.startswith("layercake-")if not (is_stale_hpr or is_legacy_layercake):continue- for f in child.iterdir():- f.unlink()- child.rmdir()+ if not _is_our_skill_dir(child):+ continue+ _remove_skill_dir(child)pruned.append(child.name)if not installed and not pruned:diff --git a/src/hyperresearch/serve/renderer.py b/src/hyperresearch/serve/renderer.pyindex d438af2..84586f3 100644--- a/src/hyperresearch/serve/renderer.py+++ b/src/hyperresearch/serve/renderer.py@@ -5,6 +5,43 @@ from __future__ import annotationsimport htmlimport re+# Schemes a note body is allowed to link out to. Note bodies are fetched page+# content, so a URL in one is attacker-influenced: `javascript:` and `data:`+# both turn a rendered link into script execution in the viewer's origin.+# Relative and anchor targets carry no scheme and are always fine.+SAFE_URL_SCHEMES = frozenset({"http", "https", "mailto", "ftp"})++_SCHEME_RE = re.compile(r"^\s*([a-zA-Z][a-zA-Z0-9+.\-]*)\s*:")+++def _is_safe_url(url: str) -> bool:+ """True unless `url` carries a scheme outside the allowlist.++ HTML entity decoding matters here: the URL has already been through+ `html.escape`, and a browser resolves `javascript:` back to+ `javascript:` when it parses the attribute.+ """+ probe = html.unescape(url).replace("\x00", "")+ # Browsers ignore control characters embedded in a scheme (`java\nscript:`).+ probe = "".join(c for c in probe if ord(c) > 0x20 or c == " ")+ m = _SCHEME_RE.match(probe)+ if m is None:+ return True # relative path, anchor, or protocol-relative URL+ return m.group(1).lower() in SAFE_URL_SCHEMES+++def _link(url: str, text: str) -> str:+ if not _is_safe_url(url):+ return text # keep the label, drop the link+ return f'<a href="{url}">{text}</a>'+++def _image(url: str, alt: str) -> str:+ if not _is_safe_url(url):+ return alt+ return f'<img src="{url}" alt="{alt}">'++# Simple markdown patternsMD_PATTERNS = [# Headers@@ -21,9 +58,9 @@ MD_PATTERNS = [# Code(re.compile(r"`([^`]+)`"), r"<code>\1</code>"),# Links- (re.compile(r"\[([^\]]+)\]\(([^)]+)\)"), r'<a href="\2">\1</a>'),+ (re.compile(r"\[([^\]]+)\]\(([^)]+)\)"), lambda m: _link(m.group(2), m.group(1))),# Images- (re.compile(r"!\[([^\]]*)\]\(([^)]+)\)"), r'<img src="\2" alt="\1">'),+ (re.compile(r"!\[([^\]]*)\]\(([^)]+)\)"), lambda m: _image(m.group(2), m.group(1))),# Horizontal rule(re.compile(r"^---+$", re.MULTILINE), r"<hr>"),# Blockquotesdiff --git a/src/hyperresearch/serve/server.py b/src/hyperresearch/serve/server.pyindex e81b4c9..7ddef3a 100644--- a/src/hyperresearch/serve/server.py+++ b/src/hyperresearch/serve/server.py@@ -517,7 +517,17 @@ class HyperresearchHandler(BaseHTTPRequestHandler):returnbody += f"<p>{len(results)} results</p>\n<div class='results'>\n"for r in results:- snippet = r["snippet"].replace(">>>", "<mark>").replace("<<<", "</mark>")+ # Escape BEFORE substituting the markers, so the <mark> tags are the+ # only markup that survives. The snippet is note body text, which for+ # a fetched note is remote page content, and strip_markdown() does not+ # neutralize it: its tag regex needs a closing ">", so an unterminated+ # "<img src=x onerror=..." passes through whole and the ">" of the+ # </mark> we inject here would finish the tag for it.+ snippet = (+ html_mod.escape(r["snippet"])+ .replace(">>>", "<mark>")+ .replace("<<<", "</mark>")+ )body += (f'<div class="result"><div class="title">'f'<a href="/note/{html_mod.escape(r["id"])}">{html_mod.escape(r["title"])}</a></div>'diff --git a/tests/test_core/test_hooks.py b/tests/test_core/test_hooks.pyindex 41b70a3..bd00491 100644--- a/tests/test_core/test_hooks.py+++ b/tests/test_core/test_hooks.py@@ -325,7 +325,12 @@ def test_prune_retired_agents_removes_old_files(tmp_vault):for name in _RETIRED_SKILL_DIRS:retired_skill = skills_dir / nameretired_skill.mkdir(parents=True, exist_ok=True)- (retired_skill / "SKILL.md").write_text("old skill\n", encoding="utf-8")+ # What we actually shipped into these dirs: the entry skill, renamed.+ # Every version of it named the project in its body.+ (retired_skill / "SKILL.md").write_text(+ f"---\nname: {name}\n---\n\n# Deep research\n\nRun the hyperresearch pipeline.\n",+ encoding="utf-8",+ )result = _prune_retired_agents(tmp_vault.root)assert result is not None@@ -343,6 +348,41 @@ def test_prune_retired_agents_noop_on_clean_vault(tmp_vault):assert result is None+def test_prune_retired_agents_spares_a_users_own_research_skill(tmp_vault):+ """`research` is an obvious name for a hand-written personal skill, and on a+ --global install the prune runs against ~/.claude/skills/. Deleting on name+ alone destroyed user content that was never ours (#73)."""+ skills_dir = tmp_vault.root / ".claude" / "skills"+ mine = skills_dir / "research"+ mine.mkdir(parents=True, exist_ok=True)+ (mine / "SKILL.md").write_text(+ "---\nname: research\ndescription: My own literature workflow\n---\n\nStep 1...\n",+ encoding="utf-8",+ )+ (mine / "reference.md").write_text("notes I wrote\n", encoding="utf-8")++ result = _prune_retired_agents(tmp_vault.root)++ assert (mine / "SKILL.md").read_text(encoding="utf-8").startswith("---\nname: research")+ assert (mine / "reference.md").exists()+ # And the user is told it was left behind, rather than it happening silently.+ assert result is not None+ assert "Left alone (not ours)" in result+ assert ".claude/skills/research" in result+++def test_prune_retired_agents_spares_a_dir_with_no_skill_file(tmp_vault):+ """No SKILL.md means we have no evidence it was ever ours, so leave it."""+ skills_dir = tmp_vault.root / ".claude" / "skills"+ mine = skills_dir / "research"+ mine.mkdir(parents=True, exist_ok=True)+ (mine / "scratch.txt").write_text("something\n", encoding="utf-8")++ _prune_retired_agents(tmp_vault.root)++ assert (mine / "scratch.txt").exists()++# ---------------------------------------------------------------------------# install_hooks — end-to-end integration# ---------------------------------------------------------------------------diff --git a/tests/test_serve/__init__.py b/tests/test_serve/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/tests/test_serve/test_xss.py b/tests/test_serve/test_xss.pynew file mode 100644index 0000000..d6ef895--- /dev/null+++ b/tests/test_serve/test_xss.py@@ -0,0 +1,107 @@+"""The wiki serves note bodies, and note bodies are fetched remote pages.++`core/untrusted.py` already treats those as hostile on the CLI read paths. These+tests hold the viewer to the same assumption (#72).+"""++import html as html_mod++import pytest++from hyperresearch.core.note import strip_markdown+from hyperresearch.serve.renderer import _is_safe_url, render_markdown+++def _render_snippet(raw: str) -> str:+ """The exact transform `_serve_search` applies to an FTS snippet."""+ return (+ html_mod.escape(raw)+ .replace(">>>", "<mark>")+ .replace("<<<", "</mark>")+ )+++class TestSearchSnippet:+ def test_unterminated_tag_cannot_borrow_the_mark_closer(self):+ """strip_markdown's tag regex needs a closing '>', so an unterminated+ tag survives into body_plain intact. Substituting the markers before+ escaping then handed it the '>' it was missing, off the </mark>."""+ body = "trigger <img src=x onerror=alert(document.domain) text"+ assert strip_markdown(body) == body, "premise: strip_markdown lets this through"++ out = _render_snippet(">>>trigger<<< <img src=x onerror=alert(document.domain) text")++ assert "<img" not in out+ assert "onerror" not in out.replace("<img src=x onerror", "")+ assert "<img" in out++ @pytest.mark.parametrize(+ "payload",+ [+ "<script>alert(1)</script>",+ "<svg onload=alert(1)>",+ "\"><script>alert(1)</script>",+ "<iframe src=javascript:alert(1)",+ ],+ )+ def test_no_payload_survives_as_markup(self, payload):+ out = _render_snippet(f">>>hit<<< {payload}")+ assert "<script" not in out+ assert "<svg" not in out+ assert "<iframe" not in out++ def test_markers_still_become_mark_tags(self):+ out = _render_snippet("the >>>term<<< in context")+ assert out == "the <mark>term</mark> in context"++ def test_literal_entity_text_is_not_mistaken_for_a_marker(self):+ """A body containing the literal text '>>>' must not turn into+ a <mark>, since escaping rewrites its '&' first."""+ out = _render_snippet("literal >>> here")+ assert "<mark>" not in out+++class TestUrlSchemes:+ @pytest.mark.parametrize(+ "url",+ [+ "javascript:alert(1)",+ "JaVaScRiPt:alert(1)",+ " javascript:alert(1)",+ "java\tscript:alert(1)",+ "java\nscript:alert(1)",+ "data:text/html;base64,PHNjcmlwdD4=",+ "vbscript:msgbox(1)",+ ],+ )+ def test_dangerous_schemes_render_as_plain_text(self, url):+ assert not _is_safe_url(url)+ out = render_markdown(f"see [click me]({url}) here")+ assert "<a href" not in out+ assert "click me" in out++ def test_entity_encoded_scheme_is_caught(self):+ """A browser resolves `javascript:` back to `javascript:` when it+ parses the attribute. render_markdown happens to escape the '&' before+ the URL reaches here, so this is belt-and-braces for any other caller."""+ assert not _is_safe_url("javascript:alert(1)")++ @pytest.mark.parametrize(+ "url",+ [+ "https://example.com/paper",+ "http://example.com",+ "mailto:[email protected]",+ "/note/some-id",+ "#section",+ "relative/path.md",+ "//cdn.example.com/x.png",+ ],+ )+ def test_ordinary_links_still_render(self, url):+ assert _is_safe_url(url)+ assert f'<a href="{url}">' in render_markdown(f"[label]({url})")++ def test_body_text_is_still_escaped(self):+ out = render_markdown("<script>alert(1)</script>")+ assert "<script>" not in out