Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

13.0 KB
Raw Blame
"""Tests for note read/write operations."""

import unicodedata

from hyperresearch.core.note import read_note, strip_markdown, write_note
from hyperresearch.models.note import slugify


def test_slugify_basic():
    assert slugify("Hello World") == "hello-world"
    assert slugify("Python 3.12 Features!") == "python-312-features"
    assert slugify("  spaces  ") == "spaces"
    assert slugify("CamelCase") == "camelcase"
    assert slugify("dashes--double") == "dashes-double"


# --- Non-Latin titles must produce meaningful ids ----------------------------
# `flags=re.ASCII` stripped every non-Latin character, so a vault of Japanese
# sources degenerated into wikipedia, wikipedia-2, wikipedia-3 — ids that carry
# no information and make [[wikilink]] citations useless.


def test_slugify_preserves_cjk():
    assert slugify("バックライト - Wikipedia") == "バックライト-wikipedia"
    assert slugify("液晶ディスプレイ - Wikipedia") == "液晶ディスプレイ-wikipedia"
    assert slugify("导光板 - 维基百科") == "导光板-维基百科"


def test_slugify_preserves_other_scripts():
    assert slugify("Диффузор (оптика)") == "диффузор-оптика"
    assert slugify("Évolution des systèmes") == "évolution-des-systèmes"


def test_distinct_cjk_titles_get_distinct_slugs():
    """The actual bug: different Japanese pages must not collide."""
    assert slugify("バックライト - Wikipedia") != slugify("液晶ディスプレイ - Wikipedia")


def test_slugify_ascii_behaviour_is_unchanged():
    """Existing vaults must keep resolving — ASCII slugs must not shift."""
    for title in ("Diffuser (optics) - Wikipedia", "Python 3.12 Features!", "CamelCase"):
        assert slugify(title).isascii()


def test_slugify_symbol_only_title_falls_back_to_hash():
    result = slugify("!!! ??? ***")
    assert result.startswith("note-")


def test_slugify_respects_filesystem_byte_limit():
    """One CJK char is 3 UTF-8 bytes; most filesystems cap a name at 255 bytes."""
    result = slugify("光" * 200)
    assert len(result.encode("utf-8")) <= 200
    assert result == unicodedata.normalize("NFC", result), "must not split a character"


def test_slugify_normalises_equivalent_unicode():
    """Composed and decomposed forms of the same title must agree."""
    composed = "café"          # é as U+00E9
    decomposed = "café"  # e + combining acute
    assert slugify(composed) == slugify(decomposed)


def test_write_and_read_roundtrip(tmp_vault):
    path = write_note(
        tmp_vault.notes_dir,
        "Roundtrip Test",
        body="# Content\n\nSome text here.\n",
        tags=["test", "roundtrip"],
        status="evergreen",
    )
    assert path.exists()

    note = read_note(path, tmp_vault.root)
    assert note.meta.title == "Roundtrip Test"
    assert note.meta.id == "roundtrip-test"
    assert note.meta.tags == ["test", "roundtrip"]
    assert note.meta.status == "evergreen"
    assert "# Content" in note.body
    assert note.word_count > 0


def test_write_source_analysis_note_roundtrip(tmp_vault):
    """A source-analysis note (new v8 NoteType) round-trips through write +
    read with its type preserved in frontmatter. Body breadcrumb
    `*Suggested by [[source-id]]*` surfaces as an outgoing link that the
    sync layer will store as a backlink."""
    path = write_note(
        tmp_vault.notes_dir,
        "Source Analysis — Test Paper",
        body="*Suggested by [[test-paper-source]]*\n\n## Thesis\nThe paper argues X.",
        note_type="source-analysis",
        tags=["source-analysis", "test-run"],
        summary="Multi-paragraph summary.\n\nSecond paragraph with specific numbers: 42 pct.",
    )
    assert path.exists()

    note = read_note(path, tmp_vault.root)
    assert note.meta.type == "source-analysis"
    assert "test-paper-source" in note.outgoing_links
    # Multi-line summary preserved through YAML frontmatter serialization
    assert "Multi-paragraph summary" in note.meta.summary
    assert "42 pct" in note.meta.summary


def test_write_avoids_collision(tmp_vault):
    p1 = write_note(tmp_vault.notes_dir, "Same Title")
    p2 = write_note(tmp_vault.notes_dir, "Same Title")
    assert p1 != p2
    assert p1.exists()
    assert p2.exists()


# --- Collision ids must survive the frontmatter re-parse ---------------------
# slugify() caps ids at 80 chars / 200 bytes. Appending "-2" to a capped slug
# used to produce an over-cap id that the next parse chopped back to the base
# id: sync refused the duplicate, the sources insert failed its foreign key,
# and the .md file was left orphaned (verbumetecclesia fetches, 2026-07-23).


def test_write_collision_id_survives_reslugify(tmp_vault):
    long_title = "x" * 120  # slug hits the 80-char cap
    p1 = write_note(tmp_vault.notes_dir, long_title)
    p2 = write_note(tmp_vault.notes_dir, long_title)
    n1 = read_note(p1, tmp_vault.root)
    n2 = read_note(p2, tmp_vault.root)
    assert p1.stem == n1.meta.id
    assert p2.stem == n2.meta.id  # used to collapse back to the base id
    assert n1.meta.id != n2.meta.id
    assert slugify(n2.meta.id) == n2.meta.id
    assert len(p2.stem) <= 80


def test_write_collision_id_survives_reslugify_bytecap(tmp_vault):
    long_cjk = "液" * 90  # 3 UTF-8 bytes per char: the 200-byte cap bites
    write_note(tmp_vault.notes_dir, long_cjk)
    p2 = write_note(tmp_vault.notes_dir, long_cjk)
    n2 = read_note(p2, tmp_vault.root)
    assert p2.stem == n2.meta.id
    assert slugify(n2.meta.id) == n2.meta.id
    assert len(p2.stem.encode("utf-8")) <= 200


def test_write_skips_taken_ids_with_no_file(tmp_vault):
    """`note mv` keeps a note's id while renaming its file, so an id can be
    taken while `<id>.md` is free. taken_ids skips it through the same
    fixed-point collision ids as a file collision."""
    p = write_note(tmp_vault.notes_dir, "Same Title", taken_ids={"same-title", "same-title-2"})
    assert p.name == "same-title-3.md"
    assert read_note(p, tmp_vault.root).meta.id == "same-title-3"

    long_title = "x" * 120  # slug hits the 80-char cap
    p2 = write_note(tmp_vault.notes_dir, long_title, taken_ids={slugify(long_title)})
    n2 = read_note(p2, tmp_vault.root)
    assert p2.stem == n2.meta.id != slugify(long_title)
    assert slugify(n2.meta.id) == n2.meta.id
    assert len(p2.stem) <= 80


def test_write_with_parent(tmp_vault):
    """`parent:` is frontmatter metadata (DB-indexed), NOT a filesystem dir."""
    path = write_note(
        tmp_vault.notes_dir,
        "Child Note",
        parent="parent-topic",
    )
    # Flat layout: note lives directly under notes_dir, not in a parent-slug subdir.
    assert path.parent == tmp_vault.notes_dir
    assert path.name == "child-note.md"
    # parent still appears in the YAML frontmatter for DB filtering.
    assert "parent: parent-topic" in path.read_text(encoding="utf-8")


def test_read_extracts_links(tmp_vault):
    path = write_note(
        tmp_vault.notes_dir,
        "Linking Note",
        body="See [[note-a]] and [[note-b|display text]].\n",
    )
    note = read_note(path, tmp_vault.root)
    assert "note-a" in note.outgoing_links
    assert "note-b" in note.outgoing_links


def test_read_ignores_code_links(tmp_vault):
    path = write_note(
        tmp_vault.notes_dir,
        "Code Note",
        body="Real: [[real-link]]\n\n```python\nfake: [[fake-link]]\n```\n\nAlso `[[inline-fake]]`\n",
    )
    note = read_note(path, tmp_vault.root)
    assert "real-link" in note.outgoing_links
    assert "fake-link" not in note.outgoing_links
    assert "inline-fake" not in note.outgoing_links


def test_strip_markdown():
    md = "# Header\n\n**Bold** and *italic*. See [[link|display]] and [url](http://example.com).\n\n```python\ncode\n```\n"
    plain = strip_markdown(md)
    assert "Header" in plain
    assert "Bold" in plain
    assert "display" in plain
    assert "url" in plain
    assert "```" not in plain
    assert "**" not in plain
    assert "[[" not in plain


def test_strip_markdown_keeps_numeric_comparisons():
    """`<[^>]+>` treated a span between a less-than and a later greater-than as
    an HTML tag, so a datasheet sentence like the one below was indexed as
    "an input voltage of 23.5 V DC (min. 3 s)": the search text showed a
    different threshold from the note."""
    md = (
        "Battery operation is started without a delay when an input voltage of "
        "<22.5 V DC is detected; return threshold UIN >23.5 V DC (min. 3 s)."
    )
    plain = strip_markdown(md)
    assert "<22.5 V DC is detected" in plain
    assert "UIN >23.5 V DC" in plain
    # A span is a tag only when "<" is followed by a tag-name letter, "/" plus
    # a letter, "!" or "?". Excluding just digits after "<" would still eat these.
    assert strip_markdown("a < b > c") == "a < b > c"
    assert strip_markdown("x <= 5 and y >= 3") == "x <= 5 and y >= 3"
    # "</" opens a closing tag only when a letter follows it.
    assert strip_markdown("a </3 b> c") == "a </3 b> c"
    # Tag names are ASCII, as in HTML.
    assert strip_markdown("x <\u00e9 y> z") == "x <\u00e9 y> z"


def test_strip_markdown_keeps_unterminated_tag():
    """The closing ">" stays mandatory: an unterminated tag passes through
    whole, which tests/test_serve/test_xss.py asserts as its premise. Making
    ">" optional would eat everything after "<img"."""
    body = "trigger <img src=x onerror=alert(1) text"
    assert strip_markdown(body) == body


def test_strip_markdown_still_strips_html_tags():
    """Real tags, closing tags, self-closing tags and comments still go."""
    md = 'Some <b>bold</b> text<br/> and <a href="x">a link</a> <!-- note --> done'
    assert strip_markdown(md) == "Some bold text and a link done"
    # Upper-case names, a doctype, a processing instruction and a tag broken
    # across lines are markup too.
    md = '<?xml version="1.0"?><!DOCTYPE html><B>bold</B> and<BR> <a\nhref="x">done</a>'
    assert strip_markdown(md) == "bold and done"
    # Any element name, not a list of known ones: JATS abstracts carry
    # namespaced elements, and custom elements are markup as well.
    md = "<jats:p>Abstract</jats:p> and <custom-el>more</custom-el>"
    assert strip_markdown(md) == "Abstract and more"


def test_raw_file_persists_on_roundtrip(tmp_vault):
    """raw_file must survive parse + re-serialize (regression test for wipe bug).

    Before Batch 1.2 fix: raw_file was injected into frontmatter as a string
    AFTER write_note; NoteMeta.model_config = {"extra": "ignore"} silently
    dropped the field on the next parse, and any re-serialization (repair,
    note update) wiped it from disk.
    """
    from hyperresearch.core.frontmatter import parse_frontmatter, render_note

    path = write_note(
        tmp_vault.notes_dir,
        "PDF Note",
        body="# PDF content\n",
        extra_frontmatter={"raw_file": "raw/pdf-note.pdf"},
    )
    # First read — does NoteMeta capture the field?
    note = read_note(path, tmp_vault.root)
    assert note.meta.raw_file == "raw/pdf-note.pdf"

    # Re-serialize (simulates repair.py enrichment or note update).
    text = path.read_text(encoding="utf-8")
    meta, body = parse_frontmatter(text)
    path.write_text(render_note(meta, body), encoding="utf-8")

    # Second read — raw_file must still be present.
    note2 = read_note(path, tmp_vault.root)
    assert note2.meta.raw_file == "raw/pdf-note.pdf", (
        "raw_file was wiped on re-serialize — Batch 1.2 regression"
    )


def test_read_utf8_content(tmp_vault):
    path = write_note(
        tmp_vault.notes_dir,
        "Unicode Note",
        body="# Umlaute: Aaou. CJK: . Emoji: .\n",
    )
    note = read_note(path, tmp_vault.root)
    assert "Aaou" in note.body


def test_read_empty_body(tmp_vault):
    path = tmp_vault.notes_dir / "empty.md"
    path.parent.mkdir(exist_ok=True)
    path.write_text("---\ntitle: Empty\nid: empty\nstatus: draft\ntype: note\n---\n", encoding="utf-8")

    note = read_note(path, tmp_vault.root)
    assert note.meta.title == "Empty"
    assert note.body.strip() == ""


def test_write_note_id_is_fixed_point_of_slugify(tmp_vault):
    # A raw note_id (broken-link text, cli/graph stub + cli/repair pass it
    # verbatim) must not become a literal filename. Before the fix the
    # frontmatter id (slugified by NoteMeta.ensure_slug) and the filename
    # (built from the raw nid) DIVERGED: shell fragments became names like
    # `! -f "$KERNELDESTINATION".md`.
    from hyperresearch.models.note import slugify

    path = write_note(
        tmp_vault.notes_dir, "Kernel Check", note_id='! -f "$KERNELDESTINATION"'
    )
    assert path.parent == tmp_vault.notes_dir
    assert slugify(path.stem) == path.stem  # fixed point on disk
    note = read_note(path, tmp_vault.root)
    assert note.meta.id == path.stem  # filename and frontmatter agree


def test_write_note_traversal_id_stays_in_the_vault(tmp_vault):
    path = write_note(tmp_vault.notes_dir, "Evil", note_id="../../../outside")
    assert path.resolve().is_relative_to(tmp_vault.notes_dir.resolve())


def test_write_note_clean_id_passes_through_unchanged(tmp_vault):
    path = write_note(tmp_vault.notes_dir, "Whatever Title", note_id="my-note-id")
    assert path.stem == "my-note-id"