Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(notes): derive a slug for titles the ASCII fold empties (#6494)

Lukas Buck5bb0c40b47f1
diff --git a/changelog.d/6389.bugfix.md b/changelog.d/6389.bugfix.md
new file mode 100644
index 0000000000..8040683bd2
--- /dev/null
+++ b/changelog.d/6389.bugfix.md
@@ -0,0 +1,6 @@
+Notes whose title is written in a script with no ASCII form — CJK, Cyrillic,
+Arabic — or in emoji alone no longer all share the slug `note`. Such a title
+now yields `note-<12 hex>` derived from the title itself, so two differently
+titled notes stay distinguishable in the payload that carries the slug. This
+is the same fallback `security.filename_sanitizer` already uses for an upload
+stem that werkzeug drops. A title with no content at all still returns `note`.
diff --git a/src/local_deep_research/research_library/notes/services/note_service.py b/src/local_deep_research/research_library/notes/services/note_service.py
index d4bfc62648..b0ae36ff89 100644
--- a/src/local_deep_research/research_library/notes/services/note_service.py
+++ b/src/local_deep_research/research_library/notes/services/note_service.py
@@ -436,10 +436,21 @@ class NoteService:
Folds Latin accents to ASCII via stdlib NFKD normalization
(``café`` → ``cafe``). Scripts with no ASCII form (CJK, Cyrillic,
- Arabic) and emoji drop out, so a title that reduces to nothing falls
- back to the ``"note"`` placeholder. A real transliterator is avoided
- on purpose: the licensed options (text-unidecode / python-slugify)
- are copyleft/GPL, and the slug is a non-critical convenience field.
+ Arabic) and emoji drop out at the ``encode("ascii", "ignore")`` step.
+ A real transliterator is avoided on purpose: the licensed options
+ (text-unidecode / python-slugify) are copyleft/GPL, and the slug is a
+ non-critical convenience field.
+
+ A title that reduces to nothing gets ``note-<12 hex>`` derived from
+ the title itself, NOT the bare ``"note"`` placeholder: that constant
+ gave every CJK-, Cyrillic-, Arabic- or emoji-titled note the same
+ slug, so two differently titled notes were indistinguishable in the
+ autocomplete payload that carries it. Same shape as the fallback
+ ``security.filename_sanitizer`` uses for an upload stem werkzeug drops
+ (#6108). Whitespace is collapsed first so the fallback treats padding
+ and runs the way the ASCII branch above already does.
+
+ Only a title with no content at all still returns ``"note"``.
"""
ascii_title = (
unicodedata.normalize("NFKD", title or "")
@@ -451,7 +462,33 @@ class NoteService:
slug = re.sub(r"[^a-z0-9\-]", "", slug)
slug = re.sub(r"-+", "-", slug)
slug = slug.strip("-")
- return (slug or "note")[:500]
+ if slug:
+ return slug[:500]
+
+ collapsed = re.sub(r"\s+", " ", title or "").strip()
+ if not collapsed:
+ return "note"
+ # Normalized and case-folded first, so the fallback answers the way the
+ # ASCII branch above does. That one lowercases and NFKD-folds, so
+ # ``Hello`` and ``hello`` share a slug; hashing the raw text would give
+ # ``Привет`` and ``привет`` two, and a precomposed vs decomposed
+ # spelling of one title two more, for a title that never changed.
+ # ``casefold`` on a lone surrogate is a no-op, so the guard below still
+ # does its job.
+ #
+ # surrogatepass: a title round-tripped through a filesystem or a
+ # lenient client can carry a lone surrogate, which plain UTF-8
+ # encoding raises on. It only feeds the digest — the slug stays ASCII.
+ #
+ # 12 hex characters is 48 bits, deliberately truncated: this is a
+ # convenience label recomputed per request, never stored and never a
+ # lookup key, so a collision costs a duplicate label, not a wrong note.
+ digest = hashlib.sha256(
+ unicodedata.normalize("NFKC", collapsed)
+ .casefold()
+ .encode("utf-8", errors="surrogatepass")
+ ).hexdigest()[:12]
+ return f"note-{digest}"
@staticmethod
def _compute_content_hash(content: str) -> str:
diff --git a/tests/notes/test_note_service.py b/tests/notes/test_note_service.py
index 4def89692d..8fe76f0e7e 100644
--- a/tests/notes/test_note_service.py
+++ b/tests/notes/test_note_service.py
@@ -1,5 +1,6 @@
"""Tests for NoteService linking and versioning methods."""
+import re
import uuid
import pytest
@@ -142,15 +143,20 @@ class TestNoteServiceMethods:
slug = NoteService._generate_slug("")
assert slug == "note"
+ # Whitespace-only has no content to derive from either.
+ assert NoteService._generate_slug(" ") == "note"
def test_generate_slug_only_special_chars(self):
- """Special-char-only titles fall back to ``"note"``."""
+ """Special-char-only titles have content too, so they take the same
+ derived fallback as a CJK title rather than the shared constant.
+ """
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
slug = NoteService._generate_slug("!@#$%^&*()")
- assert slug == "note"
+ assert re.fullmatch(r"note-[0-9a-f]{12}", slug)
+ assert slug != NoteService._generate_slug("?????")
# =========================================================================
# Content Hash Tests
@@ -3814,25 +3820,118 @@ class TestPostReviewRegressions:
"""_generate_slug uses stdlib NFKD (no copyleft transliterator): Latin
diacritics fold to ASCII, but scripts with no ASCII form (CJK,
Cyrillic, Arabic) and emoji drop out. A title that reduces to nothing
- falls back to ``"note"``; ASCII around dropped runs still survives.
+ falls back to a title-derived ``note-<hex>`` rather than the bare
+ ``"note"`` constant (#6389); ASCII around dropped runs still survives.
"""
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
- # CJK-only → nothing survives → placeholder
- assert NoteService._generate_slug("中文笔记") == "note"
- # Cyrillic-only → nothing survives → placeholder
- assert NoteService._generate_slug("Привет") == "note"
+ # CJK-only → nothing survives the ASCII fold → derived fallback
+ assert NoteService._generate_slug("中文笔记").startswith("note-")
+ # Cyrillic-only → same
+ assert NoteService._generate_slug("Привет").startswith("note-")
# Diacritics still collapse to ASCII (NFKD)
assert NoteService._generate_slug("café") == "cafe"
- # Emoji-only falls back
- assert NoteService._generate_slug("🎉🚀") == "note"
+ # Emoji-only → same
+ assert NoteService._generate_slug("🎉🚀").startswith("note-")
# Mixed: the dropped CJK leaves the ASCII half
assert NoteService._generate_slug("Hello 世界") == "hello"
# ASCII-only behaviour unchanged
assert NoteService._generate_slug("Hello World") == "hello-world"
+ def test_generate_slug_fallback_distinguishes_two_non_latin_titles(self):
+ """The defect this replaces: the constant made every CJK note share
+ one slug, so the autocomplete payload could not tell two notes apart.
+ """
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ assert NoteService._generate_slug(
+ "中文笔记"
+ ) != NoteService._generate_slug("日本語")
+ assert NoteService._generate_slug(
+ "Привет"
+ ) != NoteService._generate_slug("🎉🚀")
+
+ def test_generate_slug_fallback_answers_case_and_composition_as_ascii_does(
+ self,
+ ):
+ """The ASCII branch lowercases and NFKD-folds, so ``Hello`` and
+ ``hello`` share a slug. The fallback has to agree: otherwise renaming a
+ note from ``Привет`` to ``привет``, or two clients sending the same
+ title in different Unicode compositions, silently mints a second slug
+ for a title that did not change.
+ """
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ assert NoteService._generate_slug(
+ "Привет"
+ ) == NoteService._generate_slug("привет")
+ # U+0439 precomposed vs U+0438 U+0306 decomposed: one rendered glyph.
+ assert NoteService._generate_slug(
+ "\u0439"
+ ) == NoteService._generate_slug("\u0438\u0306")
+
+ def test_generate_slug_fallback_is_derived_from_content_not_length(self):
+ """``!!!`` and ``???`` are the same length and different text. A
+ fallback keyed on anything but the content would collapse them, which
+ is the collision this replaces.
+ """
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ assert NoteService._generate_slug("!!!") != NoteService._generate_slug(
+ "???"
+ )
+
+ def test_generate_slug_fallback_is_deterministic_and_ascii(self):
+ """A slug goes into a URL and is recomputed per request, so the
+ fallback has to be stable across calls and ASCII-only — a random or
+ time-derived value would satisfy "distinguishable" and break both.
+ """
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ first = NoteService._generate_slug("中文笔记")
+ second = NoteService._generate_slug("中文笔记")
+
+ assert first == second
+ assert first.isascii()
+ assert re.fullmatch(r"note-[0-9a-f]{12}", first)
+
+ def test_generate_slug_fallback_collapses_whitespace_like_the_ascii_branch(
+ self,
+ ):
+ """``" Hello World "`` and ``"Hello World"`` already give one slug.
+ The fallback follows that rule rather than hashing raw bytes, so
+ padding does not mint a second slug for one title.
+ """
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ assert NoteService._generate_slug(" 中文 笔记 ") == (
+ NoteService._generate_slug("中文 笔记")
+ )
+
+ def test_generate_slug_fallback_survives_a_lone_surrogate(self):
+ """A title round-tripped through a filesystem or a lenient client can
+ carry an unpaired surrogate, which plain UTF-8 encoding raises on.
+ """
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ slug = NoteService._generate_slug("\ud800")
+
+ assert re.fullmatch(r"note-[0-9a-f]{12}", slug)
+
def test_summary_executor_drops_task_when_queue_full(self, monkeypatch):
"""When the change-summary worker queue exceeds the soft cap,
_submit_summary_task drops the task rather than letting the
diff --git a/tests/notes/test_note_stress.py b/tests/notes/test_note_stress.py
index 30cdc76204..a97b42f011 100644
--- a/tests/notes/test_note_stress.py
+++ b/tests/notes/test_note_stress.py
@@ -1,6 +1,7 @@
"""Stress and boundary tests for Notes functionality."""
import asyncio
+import re
import uuid
from unittest.mock import AsyncMock, MagicMock
@@ -209,7 +210,6 @@ class TestNoteServiceStress:
("a", "a"),
("A B C", "a-b-c"),
("123", "123"),
- ("---", "note"),
("a" * 1000, "a" * 500), # Truncated
("Hello World", "hello-world"),
(" spaces ", "spaces"),
@@ -218,8 +218,9 @@ class TestNoteServiceStress:
("dot.dot", "dotdot"),
("slash/slash", "slashslash"),
# CJK has no NFKD ASCII form → dropped (we don't ship a copyleft
- # transliterator); a CJK-only title falls back to "note".
- ("日本語", "note"),
+ # transliterator); a CJK-only title takes the title-derived
+ # `note-<hex>` fallback instead of the shared constant (#6389),
+ # so it is checked by shape below rather than by literal.
("Mix日本語Mix", "mixmix"),
]
@@ -227,6 +228,17 @@ class TestNoteServiceStress:
result = NoteService._generate_slug(input_title)
assert result == expected_slug, f"Failed for '{input_title}'"
+ # Titles that leave nothing behind after the ASCII fold still get a
+ # slug of their own, derived from the title (#6389). "---" is in this
+ # set rather than the table above for the same reason: it has content,
+ # so it no longer shares the "note" constant with every other such
+ # title.
+ for empty_slug_title in ("日本語", "---"):
+ assert re.fullmatch(
+ r"note-[0-9a-f]{12}",
+ NoteService._generate_slug(empty_slug_title),
+ ), f"Failed for '{empty_slug_title}'"
+
def test_content_hash_stress(self):
"""Test content hash with various inputs."""
from local_deep_research.research_library.notes.services.note_service import (