Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: keep non-Latin characters in note slugs (#46)

synqingbef78da5f427
diff --git a/src/hyperresearch/models/note.py b/src/hyperresearch/models/note.py
index 05d7b2b..7a34b70 100644
--- a/src/hyperresearch/models/note.py
+++ b/src/hyperresearch/models/note.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import re
+import unicodedata
from datetime import UTC, datetime
from enum import StrEnum
@@ -53,20 +54,31 @@ class ContentType(StrEnum):
def slugify(text: str) -> str:
- """Convert text to a URL-friendly slug. Preserves underscores."""
- text = text.lower().strip()
- text = re.sub(r"[^\w\s-]", "", text, flags=re.ASCII)
+ """Convert text to a URL-friendly slug. Preserves underscores.
+
+ Unicode-aware: CJK, Cyrillic, Arabic, and accented Latin characters are kept.
+ Stripping them (the old `flags=re.ASCII` behaviour) collapsed every non-Latin
+ title to whatever ASCII it happened to contain — so `バックライト - Wikipedia`
+ became `wikipedia`, and a vault of Japanese sources degenerated into
+ `wikipedia`, `wikipedia-2`, `wikipedia-3`, with meaningless [[wikilink]] ids.
+ """
+ # NFC-normalise so visually identical titles always yield the same slug.
+ text = unicodedata.normalize("NFC", text).lower().strip()
+ text = re.sub(r"[^\w\s-]", "", text) # Unicode-aware \w — no re.ASCII
text = re.sub(r"\s+", "-", text)
text = re.sub(r"-+", "-", text)
result = text.strip("-")
if not result:
- # Fallback for titles with only special/non-ASCII characters
+ # Fallback for titles with only punctuation/symbols
import hashlib
result = "note-" + hashlib.sha256(text.encode()).hexdigest()[:8]
- # Truncate to avoid Windows MAX_PATH issues (keep well under 260 chars total)
+ # Cap length twice: characters (Windows MAX_PATH) and UTF-8 bytes, since most
+ # filesystems limit a filename to 255 *bytes* and one CJK character costs 3.
if len(result) > 80:
- result = result[:80].rstrip("-")
- return result
+ result = result[:80]
+ if len(result.encode("utf-8")) > 200:
+ result = result.encode("utf-8")[:200].decode("utf-8", errors="ignore")
+ return result.rstrip("-")
class NoteMeta(BaseModel):
diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.py
index 2839469..3a3a65f 100644
--- a/tests/test_core/test_note.py
+++ b/tests/test_core/test_note.py
@@ -1,5 +1,6 @@
"""Tests for note read/write operations."""
+import unicodedata
from hyperresearch.core.note import read_note, strip_markdown, write_note
from hyperresearch.models.note import slugify
@@ -13,6 +14,53 @@ def test_slugify_basic():
assert slugify("dashes--double") == "dashes-double"
+# --- Non-Latin titles must produce meaningful ids ----------------------------
+# `flags=re.ASCII` stripped every non-Latin character, so a vault of Japanese
+# sources degenerated into wikipedia, wikipedia-2, wikipedia-3 — ids that carry
+# no information and make [[wikilink]] citations useless.
+
+
+def test_slugify_preserves_cjk():
+ assert slugify("バックライト - Wikipedia") == "バックライト-wikipedia"
+ assert slugify("液晶ディスプレイ - Wikipedia") == "液晶ディスプレイ-wikipedia"
+ assert slugify("导光板 - 维基百科") == "导光板-维基百科"
+
+
+def test_slugify_preserves_other_scripts():
+ assert slugify("Диффузор (оптика)") == "диффузор-оптика"
+ assert slugify("Évolution des systèmes") == "évolution-des-systèmes"
+
+
+def test_distinct_cjk_titles_get_distinct_slugs():
+ """The actual bug: different Japanese pages must not collide."""
+ assert slugify("バックライト - Wikipedia") != slugify("液晶ディスプレイ - Wikipedia")
+
+
+def test_slugify_ascii_behaviour_is_unchanged():
+ """Existing vaults must keep resolving — ASCII slugs must not shift."""
+ for title in ("Diffuser (optics) - Wikipedia", "Python 3.12 Features!", "CamelCase"):
+ assert slugify(title).isascii()
+
+
+def test_slugify_symbol_only_title_falls_back_to_hash():
+ result = slugify("!!! ??? ***")
+ assert result.startswith("note-")
+
+
+def test_slugify_respects_filesystem_byte_limit():
+ """One CJK char is 3 UTF-8 bytes; most filesystems cap a name at 255 bytes."""
+ result = slugify("光" * 200)
+ assert len(result.encode("utf-8")) <= 200
+ assert result == unicodedata.normalize("NFC", result), "must not split a character"
+
+
+def test_slugify_normalises_equivalent_unicode():
+ """Composed and decomposed forms of the same title must agree."""
+ composed = "café" # é as U+00E9
+ decomposed = "café" # e + combining acute
+ assert slugify(composed) == slugify(decomposed)
+
+
def test_write_and_read_roundtrip(tmp_vault):
path = write_note(
tmp_vault.notes_dir,