~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(notes): derive a slug for titles the ASCII fold empties (#6494)
Lukas Buck5bb0c40b47f1
diff --git a/changelog.d/6389.bugfix.md b/changelog.d/6389.bugfix.mdnew file mode 100644index 0000000000..8040683bd2--- /dev/null+++ b/changelog.d/6389.bugfix.md@@ -0,0 +1,6 @@+Notes whose title is written in a script with no ASCII form — CJK, Cyrillic,+Arabic — or in emoji alone no longer all share the slug `note`. Such a title+now yields `note-<12 hex>` derived from the title itself, so two differently+titled notes stay distinguishable in the payload that carries the slug. This+is the same fallback `security.filename_sanitizer` already uses for an upload+stem that werkzeug drops. A title with no content at all still returns `note`.diff --git a/src/local_deep_research/research_library/notes/services/note_service.py b/src/local_deep_research/research_library/notes/services/note_service.pyindex d4bfc62648..b0ae36ff89 100644--- a/src/local_deep_research/research_library/notes/services/note_service.py+++ b/src/local_deep_research/research_library/notes/services/note_service.py@@ -436,10 +436,21 @@ class NoteService:Folds Latin accents to ASCII via stdlib NFKD normalization(``café`` → ``cafe``). Scripts with no ASCII form (CJK, Cyrillic,- Arabic) and emoji drop out, so a title that reduces to nothing falls- back to the ``"note"`` placeholder. A real transliterator is avoided- on purpose: the licensed options (text-unidecode / python-slugify)- are copyleft/GPL, and the slug is a non-critical convenience field.+ Arabic) and emoji drop out at the ``encode("ascii", "ignore")`` step.+ A real transliterator is avoided on purpose: the licensed options+ (text-unidecode / python-slugify) are copyleft/GPL, and the slug is a+ non-critical convenience field.++ A title that reduces to nothing gets ``note-<12 hex>`` derived from+ the title itself, NOT the bare ``"note"`` placeholder: that constant+ gave every CJK-, Cyrillic-, Arabic- or emoji-titled note the same+ slug, so two differently titled notes were indistinguishable in the+ autocomplete payload that carries it. Same shape as the fallback+ ``security.filename_sanitizer`` uses for an upload stem werkzeug drops+ (#6108). Whitespace is collapsed first so the fallback treats padding+ and runs the way the ASCII branch above already does.++ Only a title with no content at all still returns ``"note"``."""ascii_title = (unicodedata.normalize("NFKD", title or "")@@ -451,7 +462,33 @@ class NoteService:slug = re.sub(r"[^a-z0-9\-]", "", slug)slug = re.sub(r"-+", "-", slug)slug = slug.strip("-")- return (slug or "note")[:500]+ if slug:+ return slug[:500]++ collapsed = re.sub(r"\s+", " ", title or "").strip()+ if not collapsed:+ return "note"+ # Normalized and case-folded first, so the fallback answers the way the+ # ASCII branch above does. That one lowercases and NFKD-folds, so+ # ``Hello`` and ``hello`` share a slug; hashing the raw text would give+ # ``Привет`` and ``привет`` two, and a precomposed vs decomposed+ # spelling of one title two more, for a title that never changed.+ # ``casefold`` on a lone surrogate is a no-op, so the guard below still+ # does its job.+ #+ # surrogatepass: a title round-tripped through a filesystem or a+ # lenient client can carry a lone surrogate, which plain UTF-8+ # encoding raises on. It only feeds the digest — the slug stays ASCII.+ #+ # 12 hex characters is 48 bits, deliberately truncated: this is a+ # convenience label recomputed per request, never stored and never a+ # lookup key, so a collision costs a duplicate label, not a wrong note.+ digest = hashlib.sha256(+ unicodedata.normalize("NFKC", collapsed)+ .casefold()+ .encode("utf-8", errors="surrogatepass")+ ).hexdigest()[:12]+ return f"note-{digest}"@staticmethoddef _compute_content_hash(content: str) -> str:diff --git a/tests/notes/test_note_service.py b/tests/notes/test_note_service.pyindex 4def89692d..8fe76f0e7e 100644--- a/tests/notes/test_note_service.py+++ b/tests/notes/test_note_service.py@@ -1,5 +1,6 @@"""Tests for NoteService linking and versioning methods."""+import reimport uuidimport pytest@@ -142,15 +143,20 @@ class TestNoteServiceMethods:slug = NoteService._generate_slug("")assert slug == "note"+ # Whitespace-only has no content to derive from either.+ assert NoteService._generate_slug(" ") == "note"def test_generate_slug_only_special_chars(self):- """Special-char-only titles fall back to ``"note"``."""+ """Special-char-only titles have content too, so they take the same+ derived fallback as a CJK title rather than the shared constant.+ """from local_deep_research.research_library.notes.services.note_service import (NoteService,)slug = NoteService._generate_slug("!@#$%^&*()")- assert slug == "note"+ assert re.fullmatch(r"note-[0-9a-f]{12}", slug)+ assert slug != NoteService._generate_slug("?????")# =========================================================================# Content Hash Tests@@ -3814,25 +3820,118 @@ class TestPostReviewRegressions:"""_generate_slug uses stdlib NFKD (no copyleft transliterator): Latindiacritics fold to ASCII, but scripts with no ASCII form (CJK,Cyrillic, Arabic) and emoji drop out. A title that reduces to nothing- falls back to ``"note"``; ASCII around dropped runs still survives.+ falls back to a title-derived ``note-<hex>`` rather than the bare+ ``"note"`` constant (#6389); ASCII around dropped runs still survives."""from local_deep_research.research_library.notes.services.note_service import (NoteService,)- # CJK-only → nothing survives → placeholder- assert NoteService._generate_slug("中文笔记") == "note"- # Cyrillic-only → nothing survives → placeholder- assert NoteService._generate_slug("Привет") == "note"+ # CJK-only → nothing survives the ASCII fold → derived fallback+ assert NoteService._generate_slug("中文笔记").startswith("note-")+ # Cyrillic-only → same+ assert NoteService._generate_slug("Привет").startswith("note-")# Diacritics still collapse to ASCII (NFKD)assert NoteService._generate_slug("café") == "cafe"- # Emoji-only falls back- assert NoteService._generate_slug("🎉🚀") == "note"+ # Emoji-only → same+ assert NoteService._generate_slug("🎉🚀").startswith("note-")# Mixed: the dropped CJK leaves the ASCII halfassert NoteService._generate_slug("Hello 世界") == "hello"# ASCII-only behaviour unchangedassert NoteService._generate_slug("Hello World") == "hello-world"+ def test_generate_slug_fallback_distinguishes_two_non_latin_titles(self):+ """The defect this replaces: the constant made every CJK note share+ one slug, so the autocomplete payload could not tell two notes apart.+ """+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ assert NoteService._generate_slug(+ "中文笔记"+ ) != NoteService._generate_slug("日本語")+ assert NoteService._generate_slug(+ "Привет"+ ) != NoteService._generate_slug("🎉🚀")++ def test_generate_slug_fallback_answers_case_and_composition_as_ascii_does(+ self,+ ):+ """The ASCII branch lowercases and NFKD-folds, so ``Hello`` and+ ``hello`` share a slug. The fallback has to agree: otherwise renaming a+ note from ``Привет`` to ``привет``, or two clients sending the same+ title in different Unicode compositions, silently mints a second slug+ for a title that did not change.+ """+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ assert NoteService._generate_slug(+ "Привет"+ ) == NoteService._generate_slug("привет")+ # U+0439 precomposed vs U+0438 U+0306 decomposed: one rendered glyph.+ assert NoteService._generate_slug(+ "\u0439"+ ) == NoteService._generate_slug("\u0438\u0306")++ def test_generate_slug_fallback_is_derived_from_content_not_length(self):+ """``!!!`` and ``???`` are the same length and different text. A+ fallback keyed on anything but the content would collapse them, which+ is the collision this replaces.+ """+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ assert NoteService._generate_slug("!!!") != NoteService._generate_slug(+ "???"+ )++ def test_generate_slug_fallback_is_deterministic_and_ascii(self):+ """A slug goes into a URL and is recomputed per request, so the+ fallback has to be stable across calls and ASCII-only — a random or+ time-derived value would satisfy "distinguishable" and break both.+ """+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ first = NoteService._generate_slug("中文笔记")+ second = NoteService._generate_slug("中文笔记")++ assert first == second+ assert first.isascii()+ assert re.fullmatch(r"note-[0-9a-f]{12}", first)++ def test_generate_slug_fallback_collapses_whitespace_like_the_ascii_branch(+ self,+ ):+ """``" Hello World "`` and ``"Hello World"`` already give one slug.+ The fallback follows that rule rather than hashing raw bytes, so+ padding does not mint a second slug for one title.+ """+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ assert NoteService._generate_slug(" 中文 笔记 ") == (+ NoteService._generate_slug("中文 笔记")+ )++ def test_generate_slug_fallback_survives_a_lone_surrogate(self):+ """A title round-tripped through a filesystem or a lenient client can+ carry an unpaired surrogate, which plain UTF-8 encoding raises on.+ """+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ slug = NoteService._generate_slug("\ud800")++ assert re.fullmatch(r"note-[0-9a-f]{12}", slug)+def test_summary_executor_drops_task_when_queue_full(self, monkeypatch):"""When the change-summary worker queue exceeds the soft cap,_submit_summary_task drops the task rather than letting thediff --git a/tests/notes/test_note_stress.py b/tests/notes/test_note_stress.pyindex 30cdc76204..a97b42f011 100644--- a/tests/notes/test_note_stress.py+++ b/tests/notes/test_note_stress.py@@ -1,6 +1,7 @@"""Stress and boundary tests for Notes functionality."""import asyncio+import reimport uuidfrom unittest.mock import AsyncMock, MagicMock@@ -209,7 +210,6 @@ class TestNoteServiceStress:("a", "a"),("A B C", "a-b-c"),("123", "123"),- ("---", "note"),("a" * 1000, "a" * 500), # Truncated("Hello World", "hello-world"),(" spaces ", "spaces"),@@ -218,8 +218,9 @@ class TestNoteServiceStress:("dot.dot", "dotdot"),("slash/slash", "slashslash"),# CJK has no NFKD ASCII form → dropped (we don't ship a copyleft- # transliterator); a CJK-only title falls back to "note".- ("日本語", "note"),+ # transliterator); a CJK-only title takes the title-derived+ # `note-<hex>` fallback instead of the shared constant (#6389),+ # so it is checked by shape below rather than by literal.("Mix日本語Mix", "mixmix"),]@@ -227,6 +228,17 @@ class TestNoteServiceStress:result = NoteService._generate_slug(input_title)assert result == expected_slug, f"Failed for '{input_title}'"+ # Titles that leave nothing behind after the ASCII fold still get a+ # slug of their own, derived from the title (#6389). "---" is in this+ # set rather than the table above for the same reason: it has content,+ # so it no longer shares the "note" constant with every other such+ # title.+ for empty_slug_title in ("日本語", "---"):+ assert re.fullmatch(+ r"note-[0-9a-f]{12}",+ NoteService._generate_slug(empty_slug_title),+ ), f"Failed for '{empty_slug_title}'"+def test_content_hash_stress(self):"""Test content hash with various inputs."""from local_deep_research.research_library.notes.services.note_service import (