Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
"""Tests for note read/write operations."""
import unicodedata
from hyperresearch.core.note import read_note, strip_markdown, write_note
from hyperresearch.models.note import slugify
def test_slugify_basic():
assert slugify("Hello World") == "hello-world"
assert slugify("Python 3.12 Features!") == "python-312-features"
assert slugify(" spaces ") == "spaces"
assert slugify("CamelCase") == "camelcase"
assert slugify("dashes--double") == "dashes-double"
# --- Non-Latin titles must produce meaningful ids ----------------------------
# `flags=re.ASCII` stripped every non-Latin character, so a vault of Japanese
# sources degenerated into wikipedia, wikipedia-2, wikipedia-3 — ids that carry
# no information and make [[wikilink]] citations useless.
def test_slugify_preserves_cjk():
assert slugify("バックライト - Wikipedia") == "バックライト-wikipedia"
assert slugify("液晶ディスプレイ - Wikipedia") == "液晶ディスプレイ-wikipedia"
assert slugify("导光板 - 维基百科") == "导光板-维基百科"
def test_slugify_preserves_other_scripts():
assert slugify("Диффузор (оптика)") == "диффузор-оптика"
assert slugify("Évolution des systèmes") == "évolution-des-systèmes"
def test_distinct_cjk_titles_get_distinct_slugs():
"""The actual bug: different Japanese pages must not collide."""
assert slugify("バックライト - Wikipedia") != slugify("液晶ディスプレイ - Wikipedia")
def test_slugify_ascii_behaviour_is_unchanged():
"""Existing vaults must keep resolving — ASCII slugs must not shift."""
for title in ("Diffuser (optics) - Wikipedia", "Python 3.12 Features!", "CamelCase"):
assert slugify(title).isascii()
def test_slugify_symbol_only_title_falls_back_to_hash():
result = slugify("!!! ??? ***")
assert result.startswith("note-")
def test_slugify_respects_filesystem_byte_limit():
"""One CJK char is 3 UTF-8 bytes; most filesystems cap a name at 255 bytes."""
result = slugify("光" * 200)
assert len(result.encode("utf-8")) <= 200
assert result == unicodedata.normalize("NFC", result), "must not split a character"
def test_slugify_normalises_equivalent_unicode():
"""Composed and decomposed forms of the same title must agree."""
composed = "café" # é as U+00E9
decomposed = "café" # e + combining acute
assert slugify(composed) == slugify(decomposed)
def test_write_and_read_roundtrip(tmp_vault):
path = write_note(
tmp_vault.notes_dir,
"Roundtrip Test",
body="# Content\n\nSome text here.\n",
tags=["test", "roundtrip"],
status="evergreen",
)
assert path.exists()
note = read_note(path, tmp_vault.root)
assert note.meta.title == "Roundtrip Test"
assert note.meta.id == "roundtrip-test"
assert note.meta.tags == ["test", "roundtrip"]
assert note.meta.status == "evergreen"
assert "# Content" in note.body
assert note.word_count > 0
def test_write_source_analysis_note_roundtrip(tmp_vault):
"""A source-analysis note (new v8 NoteType) round-trips through write +
read with its type preserved in frontmatter. Body breadcrumb
`*Suggested by [[source-id]]*` surfaces as an outgoing link that the
sync layer will store as a backlink."""
path = write_note(
tmp_vault.notes_dir,
"Source Analysis — Test Paper",
body="*Suggested by [[test-paper-source]]*\n\n## Thesis\nThe paper argues X.",
note_type="source-analysis",
tags=["source-analysis", "test-run"],
summary="Multi-paragraph summary.\n\nSecond paragraph with specific numbers: 42 pct.",
)
assert path.exists()
note = read_note(path, tmp_vault.root)
assert note.meta.type == "source-analysis"
assert "test-paper-source" in note.outgoing_links
# Multi-line summary preserved through YAML frontmatter serialization
assert "Multi-paragraph summary" in note.meta.summary
assert "42 pct" in note.meta.summary
def test_write_avoids_collision(tmp_vault):
p1 = write_note(tmp_vault.notes_dir, "Same Title")
p2 = write_note(tmp_vault.notes_dir, "Same Title")
assert p1 != p2
assert p1.exists()
assert p2.exists()
# --- Collision ids must survive the frontmatter re-parse ---------------------
# slugify() caps ids at 80 chars / 200 bytes. Appending "-2" to a capped slug
# used to produce an over-cap id that the next parse chopped back to the base
# id: sync refused the duplicate, the sources insert failed its foreign key,
# and the .md file was left orphaned (verbumetecclesia fetches, 2026-07-23).
def test_write_collision_id_survives_reslugify(tmp_vault):
long_title = "x" * 120 # slug hits the 80-char cap
p1 = write_note(tmp_vault.notes_dir, long_title)
p2 = write_note(tmp_vault.notes_dir, long_title)
n1 = read_note(p1, tmp_vault.root)
n2 = read_note(p2, tmp_vault.root)
assert p1.stem == n1.meta.id
assert p2.stem == n2.meta.id # used to collapse back to the base id
assert n1.meta.id != n2.meta.id
assert slugify(n2.meta.id) == n2.meta.id
assert len(p2.stem) <= 80
def test_write_collision_id_survives_reslugify_bytecap(tmp_vault):
long_cjk = "液" * 90 # 3 UTF-8 bytes per char: the 200-byte cap bites
write_note(tmp_vault.notes_dir, long_cjk)
p2 = write_note(tmp_vault.notes_dir, long_cjk)
n2 = read_note(p2, tmp_vault.root)
assert p2.stem == n2.meta.id
assert slugify(n2.meta.id) == n2.meta.id
assert len(p2.stem.encode("utf-8")) <= 200
def test_write_skips_taken_ids_with_no_file(tmp_vault):
"""`note mv` keeps a note's id while renaming its file, so an id can be
taken while `<id>.md` is free. taken_ids skips it through the same
fixed-point collision ids as a file collision."""
p = write_note(tmp_vault.notes_dir, "Same Title", taken_ids={"same-title", "same-title-2"})
assert p.name == "same-title-3.md"
assert read_note(p, tmp_vault.root).meta.id == "same-title-3"
long_title = "x" * 120 # slug hits the 80-char cap
p2 = write_note(tmp_vault.notes_dir, long_title, taken_ids={slugify(long_title)})
n2 = read_note(p2, tmp_vault.root)
assert p2.stem == n2.meta.id != slugify(long_title)
assert slugify(n2.meta.id) == n2.meta.id
assert len(p2.stem) <= 80
def test_write_with_parent(tmp_vault):
"""`parent:` is frontmatter metadata (DB-indexed), NOT a filesystem dir."""
path = write_note(
tmp_vault.notes_dir,
"Child Note",
parent="parent-topic",
)
# Flat layout: note lives directly under notes_dir, not in a parent-slug subdir.
assert path.parent == tmp_vault.notes_dir
assert path.name == "child-note.md"
# parent still appears in the YAML frontmatter for DB filtering.
assert "parent: parent-topic" in path.read_text(encoding="utf-8")
def test_read_extracts_links(tmp_vault):
path = write_note(
tmp_vault.notes_dir,
"Linking Note",
body="See [[note-a]] and [[note-b|display text]].\n",
)
note = read_note(path, tmp_vault.root)
assert "note-a" in note.outgoing_links
assert "note-b" in note.outgoing_links
def test_read_ignores_code_links(tmp_vault):
path = write_note(
tmp_vault.notes_dir,
"Code Note",
body="Real: [[real-link]]\n\n```python\nfake: [[fake-link]]\n```\n\nAlso `[[inline-fake]]`\n",
)
note = read_note(path, tmp_vault.root)
assert "real-link" in note.outgoing_links
assert "fake-link" not in note.outgoing_links
assert "inline-fake" not in note.outgoing_links
def test_strip_markdown():
md = "# Header\n\n**Bold** and *italic*. See [[link|display]] and [url](http://example.com).\n\n```python\ncode\n```\n"
plain = strip_markdown(md)
assert "Header" in plain
assert "Bold" in plain
assert "display" in plain
assert "url" in plain
assert "```" not in plain
assert "**" not in plain
assert "[[" not in plain
def test_strip_markdown_keeps_numeric_comparisons():
"""`<[^>]+>` treated a span between a less-than and a later greater-than as
an HTML tag, so a datasheet sentence like the one below was indexed as
"an input voltage of 23.5 V DC (min. 3 s)": the search text showed a
different threshold from the note."""
md = (
"Battery operation is started without a delay when an input voltage of "
"<22.5 V DC is detected; return threshold UIN >23.5 V DC (min. 3 s)."
)
plain = strip_markdown(md)
assert "<22.5 V DC is detected" in plain
assert "UIN >23.5 V DC" in plain
# A span is a tag only when "<" is followed by a tag-name letter, "/" plus
# a letter, "!" or "?". Excluding just digits after "<" would still eat these.
assert strip_markdown("a < b > c") == "a < b > c"
assert strip_markdown("x <= 5 and y >= 3") == "x <= 5 and y >= 3"
# "</" opens a closing tag only when a letter follows it.
assert strip_markdown("a </3 b> c") == "a </3 b> c"
# Tag names are ASCII, as in HTML.
assert strip_markdown("x <\u00e9 y> z") == "x <\u00e9 y> z"
def test_strip_markdown_keeps_unterminated_tag():
"""The closing ">" stays mandatory: an unterminated tag passes through
whole, which tests/test_serve/test_xss.py asserts as its premise. Making
">" optional would eat everything after "<img"."""
body = "trigger <img src=x onerror=alert(1) text"
assert strip_markdown(body) == body
def test_strip_markdown_still_strips_html_tags():
"""Real tags, closing tags, self-closing tags and comments still go."""
md = 'Some <b>bold</b> text<br/> and <a href="x">a link</a> <!-- note --> done'
assert strip_markdown(md) == "Some bold text and a link done"
# Upper-case names, a doctype, a processing instruction and a tag broken
# across lines are markup too.
md = '<?xml version="1.0"?><!DOCTYPE html><B>bold</B> and<BR> <a\nhref="x">done</a>'
assert strip_markdown(md) == "bold and done"
# Any element name, not a list of known ones: JATS abstracts carry
# namespaced elements, and custom elements are markup as well.
md = "<jats:p>Abstract</jats:p> and <custom-el>more</custom-el>"
assert strip_markdown(md) == "Abstract and more"
def test_raw_file_persists_on_roundtrip(tmp_vault):
"""raw_file must survive parse + re-serialize (regression test for wipe bug).
Before Batch 1.2 fix: raw_file was injected into frontmatter as a string
AFTER write_note; NoteMeta.model_config = {"extra": "ignore"} silently
dropped the field on the next parse, and any re-serialization (repair,
note update) wiped it from disk.
"""
from hyperresearch.core.frontmatter import parse_frontmatter, render_note
path = write_note(
tmp_vault.notes_dir,
"PDF Note",
body="# PDF content\n",
extra_frontmatter={"raw_file": "raw/pdf-note.pdf"},
)
# First read — does NoteMeta capture the field?
note = read_note(path, tmp_vault.root)
assert note.meta.raw_file == "raw/pdf-note.pdf"
# Re-serialize (simulates repair.py enrichment or note update).
text = path.read_text(encoding="utf-8")
meta, body = parse_frontmatter(text)
path.write_text(render_note(meta, body), encoding="utf-8")
# Second read — raw_file must still be present.
note2 = read_note(path, tmp_vault.root)
assert note2.meta.raw_file == "raw/pdf-note.pdf", (
"raw_file was wiped on re-serialize — Batch 1.2 regression"
)
def test_read_utf8_content(tmp_vault):
path = write_note(
tmp_vault.notes_dir,
"Unicode Note",
body="# Umlaute: Aaou. CJK: . Emoji: .\n",
)
note = read_note(path, tmp_vault.root)
assert "Aaou" in note.body
def test_read_empty_body(tmp_vault):
path = tmp_vault.notes_dir / "empty.md"
path.parent.mkdir(exist_ok=True)
path.write_text("---\ntitle: Empty\nid: empty\nstatus: draft\ntype: note\n---\n", encoding="utf-8")
note = read_note(path, tmp_vault.root)
assert note.meta.title == "Empty"
assert note.body.strip() == ""
def test_write_note_id_is_fixed_point_of_slugify(tmp_vault):
# A raw note_id (broken-link text, cli/graph stub + cli/repair pass it
# verbatim) must not become a literal filename. Before the fix the
# frontmatter id (slugified by NoteMeta.ensure_slug) and the filename
# (built from the raw nid) DIVERGED: shell fragments became names like
# `! -f "$KERNELDESTINATION".md`.
from hyperresearch.models.note import slugify
path = write_note(
tmp_vault.notes_dir, "Kernel Check", note_id='! -f "$KERNELDESTINATION"'
)
assert path.parent == tmp_vault.notes_dir
assert slugify(path.stem) == path.stem # fixed point on disk
note = read_note(path, tmp_vault.root)
assert note.meta.id == path.stem # filename and frontmatter agree
def test_write_note_traversal_id_stays_in_the_vault(tmp_vault):
path = write_note(tmp_vault.notes_dir, "Evil", note_id="../../../outside")
assert path.resolve().is_relative_to(tmp_vault.notes_dir.resolve())
def test_write_note_clean_id_passes_through_unchanged(tmp_vault):
path = write_note(tmp_vault.notes_dir, "Whatever Title", note_id="my-note-id")
assert path.stem == "my-note-id"