Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

Fix the stored XSS in the wiki viewer and stop the installer deleting skills it didn't write (#75)

Jordan Gibbs27a107f69fb0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e323807..c15c28f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
## [Unreleased]
+### Two safety fixes in the viewer and the installer
+
+- **Stored XSS in `hpr serve` (#72, reported by @letospace).** `_serve_search` escaped every interpolated value except the FTS snippet, and the snippet is note body text — remote page content, for a fetched note. `strip_markdown` was not a defense: its tag regex needs a closing `>`, so an unterminated `<img src=x onerror=...` passed into `body_plain` intact, and the `>` of the `</mark>` the search page injects finished the tag. The snippet is escaped before the markers are substituted now, so the `<mark>` tags are the only markup that survives. Bounded by the server binding `127.0.0.1` and being read-only, but the script ran same-origin with the wiki and could read every note the viewer could reach. Link and image URLs in the renderer also got a scheme allowlist, so a note body can no longer render a `javascript:` or `data:` link.
+- **`install --global` deleted `~/.claude/skills/research/` on a name match alone (#73, reported by @letospace).** `research` is an ordinary word and an obvious name for a hand-written personal skill, and `--global` puts the prune in a user-level directory shared across every project, so anyone with one lost it silently on their first upgrade. A marker file cannot fix this — the directories being pruned predate any marker we could have written — so the check is on the content we shipped into them: every `SKILL.md` this project has ever installed names the project. Anything else is left alone and reported in the install output, which previously listed what it installed but never what it deleted.
+
### Open-access full-text recovery (Unpaywall + Europe PMC)
A paywalled paper used to enter the vault as an abstract. `extract_doi` stamped the DOI, the junk gate passed the landing page (an abstract is not junk), and depth investigators then reasoned over ~1,500 characters while the report cited the work as though the paper had been read.
diff --git a/src/hyperresearch/core/hooks.py b/src/hyperresearch/core/hooks.py
index fdea927..b5842f3 100644
--- a/src/hyperresearch/core/hooks.py
+++ b/src/hyperresearch/core/hooks.py
@@ -3679,9 +3679,9 @@ def _prune_global_step_skills(home: Path) -> str | None:
suffix = name[len("hyperresearch-"):]
if not suffix or not suffix[0].isdigit():
continue
- for f in child.iterdir():
- f.unlink()
- child.rmdir()
+ if not _is_our_skill_dir(child):
+ continue
+ _remove_skill_dir(child)
pruned.append(name)
if not pruned:
@@ -3957,6 +3957,38 @@ _RETIRED_SKILL_DIRS: tuple[str, ...] = (
"research", # /research alias retired in v0.8.1 — only /hyperresearch now
)
+# Every SKILL.md this project has ever installed names the project in its
+# frontmatter or its body. The retired dirs predate any install marker, so a
+# marker file cannot be used to recognize them — the content we shipped is the
+# only evidence available after the fact.
+_OWNED_SKILL_MARKERS: tuple[str, ...] = ("hyperresearch", "layercake")
+
+
+def _is_our_skill_dir(path: Path) -> bool:
+ """True when `path` holds a SKILL.md this project wrote.
+
+ A name match alone is not license to delete a directory. `research` is an
+ ordinary English word and an obvious name for a hand-written personal
+ skill, and `--global` puts the prune in ~/.claude/skills/, which is shared
+ across every project the user has. Deleting on name alone destroyed user
+ content that was never ours (#73).
+ """
+ skill_file = path / "SKILL.md"
+ if not skill_file.is_file():
+ return False
+ try:
+ text = skill_file.read_text(encoding="utf-8", errors="replace").lower()
+ except OSError:
+ return False
+ return any(marker in text for marker in _OWNED_SKILL_MARKERS)
+
+
+def _remove_skill_dir(path: Path) -> None:
+ """Delete a skill directory and everything under it."""
+ import shutil
+
+ shutil.rmtree(path)
+
# V1 modality files — left over inside .claude/skills/hyperresearch/ on
# vaults that were installed before the V8 alias-based entry skill.
_RETIRED_HYPERRESEARCH_FILES: tuple[str, ...] = (
@@ -3973,8 +4005,13 @@ def _prune_retired_agents(vault_root: Path) -> str | None:
Running this on a fresh vault is a no-op. On an upgraded vault, it removes
retired agent .md files and the old /research-ensemble + /research-layercake
skill dirs so the installed state matches the current architecture.
+
+ A retired skill dir is only removed when its SKILL.md is recognizably one
+ we shipped. Anything else with the same name belongs to the user and is
+ reported instead of deleted.
"""
pruned: list[str] = []
+ kept: list[str] = []
agents_dir = vault_root / ".claude" / "agents"
if agents_dir.exists():
@@ -3988,16 +4025,13 @@ def _prune_retired_agents(vault_root: Path) -> str | None:
if skills_dir.exists():
for name in _RETIRED_SKILL_DIRS:
p = skills_dir / name
- if p.is_dir():
- for child in p.iterdir():
- if child.is_file():
- child.unlink()
- elif child.is_dir():
- # Should not happen — skills are flat — but be safe
- import shutil
- shutil.rmtree(child)
- p.rmdir()
- pruned.append(f"skill dir {name}")
+ if not p.is_dir():
+ continue
+ if not _is_our_skill_dir(p):
+ kept.append(name)
+ continue
+ _remove_skill_dir(p)
+ pruned.append(f"skill dir {name}")
# V1 modality files (SKILL-collect.md etc.) left inside the
# /hyperresearch skill dir from the old multi-file install layout.
@@ -4009,9 +4043,17 @@ def _prune_retired_agents(vault_root: Path) -> str | None:
p.unlink()
pruned.append(f"file hyperresearch/{name}")
- if not pruned:
+ parts: list[str] = []
+ if pruned:
+ parts.append("Pruned retired: " + ", ".join(pruned))
+ if kept:
+ parts.append(
+ "Left alone (not ours): "
+ + ", ".join(f".claude/skills/{n}" for n in kept)
+ )
+ if not parts:
return None
- return "Pruned retired: " + ", ".join(pruned)
+ return " | ".join(parts)
def _read_skill_source(src_name: str) -> str | None:
@@ -4120,9 +4162,9 @@ def _install_hyperresearch_step_skills(vault_root: Path) -> str | None:
is_legacy_layercake = child.name.startswith("layercake-")
if not (is_stale_hpr or is_legacy_layercake):
continue
- for f in child.iterdir():
- f.unlink()
- child.rmdir()
+ if not _is_our_skill_dir(child):
+ continue
+ _remove_skill_dir(child)
pruned.append(child.name)
if not installed and not pruned:
diff --git a/src/hyperresearch/serve/renderer.py b/src/hyperresearch/serve/renderer.py
index d438af2..84586f3 100644
--- a/src/hyperresearch/serve/renderer.py
+++ b/src/hyperresearch/serve/renderer.py
@@ -5,6 +5,43 @@ from __future__ import annotations
import html
import re
+# Schemes a note body is allowed to link out to. Note bodies are fetched page
+# content, so a URL in one is attacker-influenced: `javascript:` and `data:`
+# both turn a rendered link into script execution in the viewer's origin.
+# Relative and anchor targets carry no scheme and are always fine.
+SAFE_URL_SCHEMES = frozenset({"http", "https", "mailto", "ftp"})
+
+_SCHEME_RE = re.compile(r"^\s*([a-zA-Z][a-zA-Z0-9+.\-]*)\s*:")
+
+
+def _is_safe_url(url: str) -> bool:
+ """True unless `url` carries a scheme outside the allowlist.
+
+ HTML entity decoding matters here: the URL has already been through
+ `html.escape`, and a browser resolves `&#106;avascript:` back to
+ `javascript:` when it parses the attribute.
+ """
+ probe = html.unescape(url).replace("\x00", "")
+ # Browsers ignore control characters embedded in a scheme (`java\nscript:`).
+ probe = "".join(c for c in probe if ord(c) > 0x20 or c == " ")
+ m = _SCHEME_RE.match(probe)
+ if m is None:
+ return True # relative path, anchor, or protocol-relative URL
+ return m.group(1).lower() in SAFE_URL_SCHEMES
+
+
+def _link(url: str, text: str) -> str:
+ if not _is_safe_url(url):
+ return text # keep the label, drop the link
+ return f'<a href="{url}">{text}</a>'
+
+
+def _image(url: str, alt: str) -> str:
+ if not _is_safe_url(url):
+ return alt
+ return f'<img src="{url}" alt="{alt}">'
+
+
# Simple markdown patterns
MD_PATTERNS = [
# Headers
@@ -21,9 +58,9 @@ MD_PATTERNS = [
# Code
(re.compile(r"`([^`]+)`"), r"<code>\1</code>"),
# Links
- (re.compile(r"\[([^\]]+)\]\(([^)]+)\)"), r'<a href="\2">\1</a>'),
+ (re.compile(r"\[([^\]]+)\]\(([^)]+)\)"), lambda m: _link(m.group(2), m.group(1))),
# Images
- (re.compile(r"!\[([^\]]*)\]\(([^)]+)\)"), r'<img src="\2" alt="\1">'),
+ (re.compile(r"!\[([^\]]*)\]\(([^)]+)\)"), lambda m: _image(m.group(2), m.group(1))),
# Horizontal rule
(re.compile(r"^---+$", re.MULTILINE), r"<hr>"),
# Blockquotes
diff --git a/src/hyperresearch/serve/server.py b/src/hyperresearch/serve/server.py
index e81b4c9..7ddef3a 100644
--- a/src/hyperresearch/serve/server.py
+++ b/src/hyperresearch/serve/server.py
@@ -517,7 +517,17 @@ class HyperresearchHandler(BaseHTTPRequestHandler):
return
body += f"<p>{len(results)} results</p>\n<div class='results'>\n"
for r in results:
- snippet = r["snippet"].replace(">>>", "<mark>").replace("<<<", "</mark>")
+ # Escape BEFORE substituting the markers, so the <mark> tags are the
+ # only markup that survives. The snippet is note body text, which for
+ # a fetched note is remote page content, and strip_markdown() does not
+ # neutralize it: its tag regex needs a closing ">", so an unterminated
+ # "<img src=x onerror=..." passes through whole and the ">" of the
+ # </mark> we inject here would finish the tag for it.
+ snippet = (
+ html_mod.escape(r["snippet"])
+ .replace("&gt;&gt;&gt;", "<mark>")
+ .replace("&lt;&lt;&lt;", "</mark>")
+ )
body += (
f'<div class="result"><div class="title">'
f'<a href="/note/{html_mod.escape(r["id"])}">{html_mod.escape(r["title"])}</a></div>'
diff --git a/tests/test_core/test_hooks.py b/tests/test_core/test_hooks.py
index 41b70a3..bd00491 100644
--- a/tests/test_core/test_hooks.py
+++ b/tests/test_core/test_hooks.py
@@ -325,7 +325,12 @@ def test_prune_retired_agents_removes_old_files(tmp_vault):
for name in _RETIRED_SKILL_DIRS:
retired_skill = skills_dir / name
retired_skill.mkdir(parents=True, exist_ok=True)
- (retired_skill / "SKILL.md").write_text("old skill\n", encoding="utf-8")
+ # What we actually shipped into these dirs: the entry skill, renamed.
+ # Every version of it named the project in its body.
+ (retired_skill / "SKILL.md").write_text(
+ f"---\nname: {name}\n---\n\n# Deep research\n\nRun the hyperresearch pipeline.\n",
+ encoding="utf-8",
+ )
result = _prune_retired_agents(tmp_vault.root)
assert result is not None
@@ -343,6 +348,41 @@ def test_prune_retired_agents_noop_on_clean_vault(tmp_vault):
assert result is None
+def test_prune_retired_agents_spares_a_users_own_research_skill(tmp_vault):
+ """`research` is an obvious name for a hand-written personal skill, and on a
+ --global install the prune runs against ~/.claude/skills/. Deleting on name
+ alone destroyed user content that was never ours (#73)."""
+ skills_dir = tmp_vault.root / ".claude" / "skills"
+ mine = skills_dir / "research"
+ mine.mkdir(parents=True, exist_ok=True)
+ (mine / "SKILL.md").write_text(
+ "---\nname: research\ndescription: My own literature workflow\n---\n\nStep 1...\n",
+ encoding="utf-8",
+ )
+ (mine / "reference.md").write_text("notes I wrote\n", encoding="utf-8")
+
+ result = _prune_retired_agents(tmp_vault.root)
+
+ assert (mine / "SKILL.md").read_text(encoding="utf-8").startswith("---\nname: research")
+ assert (mine / "reference.md").exists()
+ # And the user is told it was left behind, rather than it happening silently.
+ assert result is not None
+ assert "Left alone (not ours)" in result
+ assert ".claude/skills/research" in result
+
+
+def test_prune_retired_agents_spares_a_dir_with_no_skill_file(tmp_vault):
+ """No SKILL.md means we have no evidence it was ever ours, so leave it."""
+ skills_dir = tmp_vault.root / ".claude" / "skills"
+ mine = skills_dir / "research"
+ mine.mkdir(parents=True, exist_ok=True)
+ (mine / "scratch.txt").write_text("something\n", encoding="utf-8")
+
+ _prune_retired_agents(tmp_vault.root)
+
+ assert (mine / "scratch.txt").exists()
+
+
# ---------------------------------------------------------------------------
# install_hooks — end-to-end integration
# ---------------------------------------------------------------------------
diff --git a/tests/test_serve/__init__.py b/tests/test_serve/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_serve/test_xss.py b/tests/test_serve/test_xss.py
new file mode 100644
index 0000000..d6ef895
--- /dev/null
+++ b/tests/test_serve/test_xss.py
@@ -0,0 +1,107 @@
+"""The wiki serves note bodies, and note bodies are fetched remote pages.
+
+`core/untrusted.py` already treats those as hostile on the CLI read paths. These
+tests hold the viewer to the same assumption (#72).
+"""
+
+import html as html_mod
+
+import pytest
+
+from hyperresearch.core.note import strip_markdown
+from hyperresearch.serve.renderer import _is_safe_url, render_markdown
+
+
+def _render_snippet(raw: str) -> str:
+ """The exact transform `_serve_search` applies to an FTS snippet."""
+ return (
+ html_mod.escape(raw)
+ .replace("&gt;&gt;&gt;", "<mark>")
+ .replace("&lt;&lt;&lt;", "</mark>")
+ )
+
+
+class TestSearchSnippet:
+ def test_unterminated_tag_cannot_borrow_the_mark_closer(self):
+ """strip_markdown's tag regex needs a closing '>', so an unterminated
+ tag survives into body_plain intact. Substituting the markers before
+ escaping then handed it the '>' it was missing, off the </mark>."""
+ body = "trigger <img src=x onerror=alert(document.domain) text"
+ assert strip_markdown(body) == body, "premise: strip_markdown lets this through"
+
+ out = _render_snippet(">>>trigger<<< <img src=x onerror=alert(document.domain) text")
+
+ assert "<img" not in out
+ assert "onerror" not in out.replace("&lt;img src=x onerror", "")
+ assert "&lt;img" in out
+
+ @pytest.mark.parametrize(
+ "payload",
+ [
+ "<script>alert(1)</script>",
+ "<svg onload=alert(1)>",
+ "\"><script>alert(1)</script>",
+ "<iframe src=javascript:alert(1)",
+ ],
+ )
+ def test_no_payload_survives_as_markup(self, payload):
+ out = _render_snippet(f">>>hit<<< {payload}")
+ assert "<script" not in out
+ assert "<svg" not in out
+ assert "<iframe" not in out
+
+ def test_markers_still_become_mark_tags(self):
+ out = _render_snippet("the >>>term<<< in context")
+ assert out == "the <mark>term</mark> in context"
+
+ def test_literal_entity_text_is_not_mistaken_for_a_marker(self):
+ """A body containing the literal text '&gt;&gt;&gt;' must not turn into
+ a <mark>, since escaping rewrites its '&' first."""
+ out = _render_snippet("literal &gt;&gt;&gt; here")
+ assert "<mark>" not in out
+
+
+class TestUrlSchemes:
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "javascript:alert(1)",
+ "JaVaScRiPt:alert(1)",
+ " javascript:alert(1)",
+ "java\tscript:alert(1)",
+ "java\nscript:alert(1)",
+ "data:text/html;base64,PHNjcmlwdD4=",
+ "vbscript:msgbox(1)",
+ ],
+ )
+ def test_dangerous_schemes_render_as_plain_text(self, url):
+ assert not _is_safe_url(url)
+ out = render_markdown(f"see [click me]({url}) here")
+ assert "<a href" not in out
+ assert "click me" in out
+
+ def test_entity_encoded_scheme_is_caught(self):
+ """A browser resolves `&#106;avascript:` back to `javascript:` when it
+ parses the attribute. render_markdown happens to escape the '&' before
+ the URL reaches here, so this is belt-and-braces for any other caller."""
+ assert not _is_safe_url("&#106;avascript:alert(1)")
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "https://example.com/paper",
+ "http://example.com",
+ "mailto:[email protected]",
+ "/note/some-id",
+ "#section",
+ "relative/path.md",
+ "//cdn.example.com/x.png",
+ ],
+ )
+ def test_ordinary_links_still_render(self, url):
+ assert _is_safe_url(url)
+ assert f'<a href="{url}">' in render_markdown(f"[label]({url})")
+
+ def test_body_text_is_still_escaped(self):
+ out = render_markdown("<script>alert(1)</script>")
+ assert "<script>" not in out