Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: keep non-Latin characters in note slugs (#46)
synqingbef78da5f427
diff --git a/src/hyperresearch/models/note.py b/src/hyperresearch/models/note.pyindex 05d7b2b..7a34b70 100644--- a/src/hyperresearch/models/note.py+++ b/src/hyperresearch/models/note.py@@ -3,6 +3,7 @@from __future__ import annotationsimport re+import unicodedatafrom datetime import UTC, datetimefrom enum import StrEnum@@ -53,20 +54,31 @@ class ContentType(StrEnum):def slugify(text: str) -> str:- """Convert text to a URL-friendly slug. Preserves underscores."""- text = text.lower().strip()- text = re.sub(r"[^\w\s-]", "", text, flags=re.ASCII)+ """Convert text to a URL-friendly slug. Preserves underscores.++ Unicode-aware: CJK, Cyrillic, Arabic, and accented Latin characters are kept.+ Stripping them (the old `flags=re.ASCII` behaviour) collapsed every non-Latin+ title to whatever ASCII it happened to contain — so `バックライト - Wikipedia`+ became `wikipedia`, and a vault of Japanese sources degenerated into+ `wikipedia`, `wikipedia-2`, `wikipedia-3`, with meaningless [[wikilink]] ids.+ """+ # NFC-normalise so visually identical titles always yield the same slug.+ text = unicodedata.normalize("NFC", text).lower().strip()+ text = re.sub(r"[^\w\s-]", "", text) # Unicode-aware \w — no re.ASCIItext = re.sub(r"\s+", "-", text)text = re.sub(r"-+", "-", text)result = text.strip("-")if not result:- # Fallback for titles with only special/non-ASCII characters+ # Fallback for titles with only punctuation/symbolsimport hashlibresult = "note-" + hashlib.sha256(text.encode()).hexdigest()[:8]- # Truncate to avoid Windows MAX_PATH issues (keep well under 260 chars total)+ # Cap length twice: characters (Windows MAX_PATH) and UTF-8 bytes, since most+ # filesystems limit a filename to 255 *bytes* and one CJK character costs 3.if len(result) > 80:- result = result[:80].rstrip("-")- return result+ result = result[:80]+ if len(result.encode("utf-8")) > 200:+ result = result.encode("utf-8")[:200].decode("utf-8", errors="ignore")+ return result.rstrip("-")class NoteMeta(BaseModel):diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.pyindex 2839469..3a3a65f 100644--- a/tests/test_core/test_note.py+++ b/tests/test_core/test_note.py@@ -1,5 +1,6 @@"""Tests for note read/write operations."""+import unicodedatafrom hyperresearch.core.note import read_note, strip_markdown, write_notefrom hyperresearch.models.note import slugify@@ -13,6 +14,53 @@ def test_slugify_basic():assert slugify("dashes--double") == "dashes-double"+# --- Non-Latin titles must produce meaningful ids ----------------------------+# `flags=re.ASCII` stripped every non-Latin character, so a vault of Japanese+# sources degenerated into wikipedia, wikipedia-2, wikipedia-3 — ids that carry+# no information and make [[wikilink]] citations useless.+++def test_slugify_preserves_cjk():+ assert slugify("バックライト - Wikipedia") == "バックライト-wikipedia"+ assert slugify("液晶ディスプレイ - Wikipedia") == "液晶ディスプレイ-wikipedia"+ assert slugify("导光板 - 维基百科") == "导光板-维基百科"+++def test_slugify_preserves_other_scripts():+ assert slugify("Диффузор (оптика)") == "диффузор-оптика"+ assert slugify("Évolution des systèmes") == "évolution-des-systèmes"+++def test_distinct_cjk_titles_get_distinct_slugs():+ """The actual bug: different Japanese pages must not collide."""+ assert slugify("バックライト - Wikipedia") != slugify("液晶ディスプレイ - Wikipedia")+++def test_slugify_ascii_behaviour_is_unchanged():+ """Existing vaults must keep resolving — ASCII slugs must not shift."""+ for title in ("Diffuser (optics) - Wikipedia", "Python 3.12 Features!", "CamelCase"):+ assert slugify(title).isascii()+++def test_slugify_symbol_only_title_falls_back_to_hash():+ result = slugify("!!! ??? ***")+ assert result.startswith("note-")+++def test_slugify_respects_filesystem_byte_limit():+ """One CJK char is 3 UTF-8 bytes; most filesystems cap a name at 255 bytes."""+ result = slugify("光" * 200)+ assert len(result.encode("utf-8")) <= 200+ assert result == unicodedata.normalize("NFC", result), "must not split a character"+++def test_slugify_normalises_equivalent_unicode():+ """Composed and decomposed forms of the same title must agree."""+ composed = "café" # é as U+00E9+ decomposed = "café" # e + combining acute+ assert slugify(composed) == slugify(decomposed)++def test_write_and_read_roundtrip(tmp_vault):path = write_note(tmp_vault.notes_dir,