Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: note ids are fixed points of the slugifier; link validator rejects shell fragments and traversal (#126)

Francois du Plessisa26235591f31
diff --git a/src/hyperresearch/cli/repair.py b/src/hyperresearch/cli/repair.py
index 48a8f2d..7267492 100644
--- a/src/hyperresearch/cli/repair.py
+++ b/src/hyperresearch/cli/repair.py
@@ -69,8 +69,11 @@ def repair(
summary=stub_summary(target),
)
stubs_created += 1
- except Exception:
- pass
+ except Exception as exc:
+ # A visible skip, not a silent one: a bare pass here hides
+ # exactly the malformed-target failures this loop can hit.
+ if not json_output:
+ console.print(f" [yellow]stub skipped[/] {target!r}: {exc}")
if stubs_created:
plan = compute_sync_plan(vault)
execute_sync(vault, plan)
diff --git a/src/hyperresearch/core/note.py b/src/hyperresearch/core/note.py
index 55882d3..5a68f4c 100644
--- a/src/hyperresearch/core/note.py
+++ b/src/hyperresearch/core/note.py
@@ -108,7 +108,15 @@ def write_note(
content_type: Artifact kind — paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown.
extra_frontmatter: Additional fields to set on NoteMeta (e.g. source_domain, fetched_at).
"""
- nid = note_id or slugify(title)
+ # The id must be a fixed point of the slugifier. NoteMeta.ensure_slug
+ # already slugifies the FRONTMATTER id, so an unslugified note_id here
+ # made the filename diverge from the note's own frontmatter: callers that
+ # pass broken-link text verbatim (cli/graph.py stub, cli/repair.py) wrote
+ # literal filenames like `! -f "$KERNELDESTINATION".md`, and a `../`
+ # shaped ref resolved to a path outside the vault. slugify strips path
+ # separators and shell metacharacters and never returns an empty string,
+ # so the filename below is always safe.
+ nid = slugify(note_id) if note_id else slugify(title)
kwargs: dict = dict(
title=title,
id=nid,
diff --git a/src/hyperresearch/core/patterns.py b/src/hyperresearch/core/patterns.py
index 6d83440..4612d02 100644
--- a/src/hyperresearch/core/patterns.py
+++ b/src/hyperresearch/core/patterns.py
@@ -76,6 +76,15 @@ _DOC_REF_PREFIX_RE = re.compile(
# into a note body it must not become a link target (issue #93).
_TEMPLATE_PLACEHOLDER_RE = re.compile(r"\{[^{}]*\}")
+# Shell/code fragments: characters that code in fetched pages carries but no
+# legitimate note id or title does. Bash test syntax ([[ -n "$kernel" ]]) is
+# lexically a wiki-link, so when a fetcher flattens <pre> content to prose,
+# every shell conditional becomes a broken link that `repair` / `graph stub`
+# then materialise as a junk note. Braces are left to the placeholder check
+# above, which already rejects `${VAR}`. This is a PARTIAL net by design:
+# refs shaped like real ids (`1-d`, `hostid-00000000`) pass.
+_SHELL_FRAGMENT_CHARS_RE = re.compile(r'[$"`=\\<>;]')
+
def is_valid_wiki_link_target(ref: str) -> bool:
"""Return True if a ``[[ref]]`` should be treated as a note reference.
@@ -97,6 +106,10 @@ def is_valid_wiki_link_target(ref: str) -> bool:
- Unsubstituted template placeholders: ``[[{note_id}]]``, ``[[run-{tag}]]``
- Unbalanced square brackets: ``[[t.IO[t.Any]]`` (a type annotation whose
closing ``]`` was eaten by the ``]]`` delimiter)
+ - Shell fragments: ``[[-n "$kernel"]]``, ``[[! -f x]]``, ``[[a=b]]`` (bash
+ test syntax from flattened code blocks: a leading ``-`` or ``!``, or a
+ shell metacharacter no note id or title carries)
+ - Path traversal: ``[[../../x]]``, ``[[a/../b]]``, ``[[..]]``
Valid note references starting with digits (e.g. ``[[10-rules-for-X]]``)
are preserved because they contain non-digit characters.
@@ -118,6 +131,15 @@ def is_valid_wiki_link_target(ref: str) -> bool:
return False
if _TEMPLATE_PLACEHOLDER_RE.search(ref):
return False
+ if _SHELL_FRAGMENT_CHARS_RE.search(ref):
+ return False
+ # Leading '-' or '!' is option/negation syntax (`-n "$kernel"`,
+ # `! -f x`), not a reference. Mid-string they stay legal (titles).
+ if ref.startswith(("-", "!")):
+ return False
+ # Path traversal: never a note reference, whatever writes the file.
+ if any(part in ("..", ".") for part in ref.split("/")):
+ return False
if ref.count("[") != ref.count("]"):
return False
return not _DOC_REF_PREFIX_RE.match(ref)
diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.py
index b5aa2bb..e8ffa23 100644
--- a/tests/test_core/test_note.py
+++ b/tests/test_core/test_note.py
@@ -240,3 +240,30 @@ def test_read_empty_body(tmp_vault):
note = read_note(path, tmp_vault.root)
assert note.meta.title == "Empty"
assert note.body.strip() == ""
+
+
+def test_write_note_id_is_fixed_point_of_slugify(tmp_vault):
+ # A raw note_id (broken-link text, cli/graph stub + cli/repair pass it
+ # verbatim) must not become a literal filename. Before the fix the
+ # frontmatter id (slugified by NoteMeta.ensure_slug) and the filename
+ # (built from the raw nid) DIVERGED: shell fragments became names like
+ # `! -f "$KERNELDESTINATION".md`.
+ from hyperresearch.models.note import slugify
+
+ path = write_note(
+ tmp_vault.notes_dir, "Kernel Check", note_id='! -f "$KERNELDESTINATION"'
+ )
+ assert path.parent == tmp_vault.notes_dir
+ assert slugify(path.stem) == path.stem # fixed point on disk
+ note = read_note(path, tmp_vault.root)
+ assert note.meta.id == path.stem # filename and frontmatter agree
+
+
+def test_write_note_traversal_id_stays_in_the_vault(tmp_vault):
+ path = write_note(tmp_vault.notes_dir, "Evil", note_id="../../../outside")
+ assert path.resolve().is_relative_to(tmp_vault.notes_dir.resolve())
+
+
+def test_write_note_clean_id_passes_through_unchanged(tmp_vault):
+ path = write_note(tmp_vault.notes_dir, "Whatever Title", note_id="my-note-id")
+ assert path.stem == "my-note-id"
diff --git a/tests/test_core/test_patterns.py b/tests/test_core/test_patterns.py
index a5626ad..9c771be 100644
--- a/tests/test_core/test_patterns.py
+++ b/tests/test_core/test_patterns.py
@@ -233,3 +233,38 @@ def test_interior_open_bracket_never_matches():
assert _targets("[[[x]]") == ["x"]
# Ordinary links and aliases are untouched.
assert _targets("[[note-id]] and [[other|Shown]]") == ["note-id", "other"]
+
+
+def test_shell_fragments_rejected():
+ # When a fetcher flattens <pre> content to prose, bash test syntax
+ # reaches this extractor and is lexically a wiki-link; `repair` then
+ # materialises junk notes named after shell fragments. Every shape below
+ # passed the validator before this check. (`"${HOSTID}"`-style refs are
+ # already rejected by the template-placeholder check.)
+ assert not is_valid_wiki_link_target('-n "$kernel"')
+ assert not is_valid_wiki_link_target('! -f "$KERNELDESTINATION"')
+ assert not is_valid_wiki_link_target("$# -lt 1")
+ assert not is_valid_wiki_link_target("-z foo")
+ assert not is_valid_wiki_link_target("a=b")
+ assert not is_valid_wiki_link_target("back`tick")
+ assert not is_valid_wiki_link_target("x < y; rm")
+
+
+def test_path_traversal_rejected():
+ # Never a note reference, whatever downstream writes the file.
+ assert not is_valid_wiki_link_target("../../../tmp/pwned")
+ assert not is_valid_wiki_link_target("a/../../b")
+ assert not is_valid_wiki_link_target("a/..")
+ assert not is_valid_wiki_link_target("..")
+ assert not is_valid_wiki_link_target(".")
+
+
+def test_shell_shaped_but_legitimate_refs_still_accepted():
+ # The net is partial BY DESIGN: shapes indistinguishable from real ids
+ # pass, and titles with mid-string slashes or punctuation stay legal.
+ assert is_valid_wiki_link_target("1-d")
+ assert is_valid_wiki_link_target("hostid-00000000")
+ assert is_valid_wiki_link_target("UEFI/Secure Boot") # real wiki title shape
+ assert is_valid_wiki_link_target("C. elegans")
+ assert is_valid_wiki_link_target("yahoo! finance") # '!' mid-string stays legal
+ assert is_valid_wiki_link_target("e.g. v1.2")