Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: note ids are fixed points of the slugifier; link validator rejects shell fragments and traversal (#126)
Francois du Plessisa26235591f31
diff --git a/src/hyperresearch/cli/repair.py b/src/hyperresearch/cli/repair.pyindex 48a8f2d..7267492 100644--- a/src/hyperresearch/cli/repair.py+++ b/src/hyperresearch/cli/repair.py@@ -69,8 +69,11 @@ def repair(summary=stub_summary(target),)stubs_created += 1- except Exception:- pass+ except Exception as exc:+ # A visible skip, not a silent one: a bare pass here hides+ # exactly the malformed-target failures this loop can hit.+ if not json_output:+ console.print(f" [yellow]stub skipped[/] {target!r}: {exc}")if stubs_created:plan = compute_sync_plan(vault)execute_sync(vault, plan)diff --git a/src/hyperresearch/core/note.py b/src/hyperresearch/core/note.pyindex 55882d3..5a68f4c 100644--- a/src/hyperresearch/core/note.py+++ b/src/hyperresearch/core/note.py@@ -108,7 +108,15 @@ def write_note(content_type: Artifact kind — paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown.extra_frontmatter: Additional fields to set on NoteMeta (e.g. source_domain, fetched_at)."""- nid = note_id or slugify(title)+ # The id must be a fixed point of the slugifier. NoteMeta.ensure_slug+ # already slugifies the FRONTMATTER id, so an unslugified note_id here+ # made the filename diverge from the note's own frontmatter: callers that+ # pass broken-link text verbatim (cli/graph.py stub, cli/repair.py) wrote+ # literal filenames like `! -f "$KERNELDESTINATION".md`, and a `../`+ # shaped ref resolved to a path outside the vault. slugify strips path+ # separators and shell metacharacters and never returns an empty string,+ # so the filename below is always safe.+ nid = slugify(note_id) if note_id else slugify(title)kwargs: dict = dict(title=title,id=nid,diff --git a/src/hyperresearch/core/patterns.py b/src/hyperresearch/core/patterns.pyindex 6d83440..4612d02 100644--- a/src/hyperresearch/core/patterns.py+++ b/src/hyperresearch/core/patterns.py@@ -76,6 +76,15 @@ _DOC_REF_PREFIX_RE = re.compile(# into a note body it must not become a link target (issue #93)._TEMPLATE_PLACEHOLDER_RE = re.compile(r"\{[^{}]*\}")+# Shell/code fragments: characters that code in fetched pages carries but no+# legitimate note id or title does. Bash test syntax ([[ -n "$kernel" ]]) is+# lexically a wiki-link, so when a fetcher flattens <pre> content to prose,+# every shell conditional becomes a broken link that `repair` / `graph stub`+# then materialise as a junk note. Braces are left to the placeholder check+# above, which already rejects `${VAR}`. This is a PARTIAL net by design:+# refs shaped like real ids (`1-d`, `hostid-00000000`) pass.+_SHELL_FRAGMENT_CHARS_RE = re.compile(r'[$"`=\\<>;]')+def is_valid_wiki_link_target(ref: str) -> bool:"""Return True if a ``[[ref]]`` should be treated as a note reference.@@ -97,6 +106,10 @@ def is_valid_wiki_link_target(ref: str) -> bool:- Unsubstituted template placeholders: ``[[{note_id}]]``, ``[[run-{tag}]]``- Unbalanced square brackets: ``[[t.IO[t.Any]]`` (a type annotation whoseclosing ``]`` was eaten by the ``]]`` delimiter)+ - Shell fragments: ``[[-n "$kernel"]]``, ``[[! -f x]]``, ``[[a=b]]`` (bash+ test syntax from flattened code blocks: a leading ``-`` or ``!``, or a+ shell metacharacter no note id or title carries)+ - Path traversal: ``[[../../x]]``, ``[[a/../b]]``, ``[[..]]``Valid note references starting with digits (e.g. ``[[10-rules-for-X]]``)are preserved because they contain non-digit characters.@@ -118,6 +131,15 @@ def is_valid_wiki_link_target(ref: str) -> bool:return Falseif _TEMPLATE_PLACEHOLDER_RE.search(ref):return False+ if _SHELL_FRAGMENT_CHARS_RE.search(ref):+ return False+ # Leading '-' or '!' is option/negation syntax (`-n "$kernel"`,+ # `! -f x`), not a reference. Mid-string they stay legal (titles).+ if ref.startswith(("-", "!")):+ return False+ # Path traversal: never a note reference, whatever writes the file.+ if any(part in ("..", ".") for part in ref.split("/")):+ return Falseif ref.count("[") != ref.count("]"):return Falsereturn not _DOC_REF_PREFIX_RE.match(ref)diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.pyindex b5aa2bb..e8ffa23 100644--- a/tests/test_core/test_note.py+++ b/tests/test_core/test_note.py@@ -240,3 +240,30 @@ def test_read_empty_body(tmp_vault):note = read_note(path, tmp_vault.root)assert note.meta.title == "Empty"assert note.body.strip() == ""+++def test_write_note_id_is_fixed_point_of_slugify(tmp_vault):+ # A raw note_id (broken-link text, cli/graph stub + cli/repair pass it+ # verbatim) must not become a literal filename. Before the fix the+ # frontmatter id (slugified by NoteMeta.ensure_slug) and the filename+ # (built from the raw nid) DIVERGED: shell fragments became names like+ # `! -f "$KERNELDESTINATION".md`.+ from hyperresearch.models.note import slugify++ path = write_note(+ tmp_vault.notes_dir, "Kernel Check", note_id='! -f "$KERNELDESTINATION"'+ )+ assert path.parent == tmp_vault.notes_dir+ assert slugify(path.stem) == path.stem # fixed point on disk+ note = read_note(path, tmp_vault.root)+ assert note.meta.id == path.stem # filename and frontmatter agree+++def test_write_note_traversal_id_stays_in_the_vault(tmp_vault):+ path = write_note(tmp_vault.notes_dir, "Evil", note_id="../../../outside")+ assert path.resolve().is_relative_to(tmp_vault.notes_dir.resolve())+++def test_write_note_clean_id_passes_through_unchanged(tmp_vault):+ path = write_note(tmp_vault.notes_dir, "Whatever Title", note_id="my-note-id")+ assert path.stem == "my-note-id"diff --git a/tests/test_core/test_patterns.py b/tests/test_core/test_patterns.pyindex a5626ad..9c771be 100644--- a/tests/test_core/test_patterns.py+++ b/tests/test_core/test_patterns.py@@ -233,3 +233,38 @@ def test_interior_open_bracket_never_matches():assert _targets("[[[x]]") == ["x"]# Ordinary links and aliases are untouched.assert _targets("[[note-id]] and [[other|Shown]]") == ["note-id", "other"]+++def test_shell_fragments_rejected():+ # When a fetcher flattens <pre> content to prose, bash test syntax+ # reaches this extractor and is lexically a wiki-link; `repair` then+ # materialises junk notes named after shell fragments. Every shape below+ # passed the validator before this check. (`"${HOSTID}"`-style refs are+ # already rejected by the template-placeholder check.)+ assert not is_valid_wiki_link_target('-n "$kernel"')+ assert not is_valid_wiki_link_target('! -f "$KERNELDESTINATION"')+ assert not is_valid_wiki_link_target("$# -lt 1")+ assert not is_valid_wiki_link_target("-z foo")+ assert not is_valid_wiki_link_target("a=b")+ assert not is_valid_wiki_link_target("back`tick")+ assert not is_valid_wiki_link_target("x < y; rm")+++def test_path_traversal_rejected():+ # Never a note reference, whatever downstream writes the file.+ assert not is_valid_wiki_link_target("../../../tmp/pwned")+ assert not is_valid_wiki_link_target("a/../../b")+ assert not is_valid_wiki_link_target("a/..")+ assert not is_valid_wiki_link_target("..")+ assert not is_valid_wiki_link_target(".")+++def test_shell_shaped_but_legitimate_refs_still_accepted():+ # The net is partial BY DESIGN: shapes indistinguishable from real ids+ # pass, and titles with mid-string slashes or punctuation stay legal.+ assert is_valid_wiki_link_target("1-d")+ assert is_valid_wiki_link_target("hostid-00000000")+ assert is_valid_wiki_link_target("UEFI/Secure Boot") # real wiki title shape+ assert is_valid_wiki_link_target("C. elegans")+ assert is_valid_wiki_link_target("yahoo! finance") # '!' mid-string stays legal+ assert is_valid_wiki_link_target("e.g. v1.2")