A Q&A platform software for teams at any scales. Whether it's a community forum, help center, or knowledge management platform, you can always count on Apache Answer.
fix: avoid topic fallback for non-Latin titles via pragmatic ASCII transliteration (#1526)
Ahmed Qaside884bb61cb1b
diff --git a/docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt b/docs/release/licenses/LICENSE-mozillazg-go-unidecode.txtnew file mode 100644index 00000000..8a7780fc--- /dev/null+++ b/docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2016 mozillazg++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/go.mod b/go.modindex 89fd7146..11c3a816 100644--- a/go.mod+++ b/go.mod@@ -43,6 +43,7 @@ require (github.com/mark3labs/mcp-go v0.43.2github.com/microcosm-cc/bluemonday v1.0.27github.com/mozillazg/go-pinyin v0.20.0+ github.com/mozillazg/go-unidecode v0.2.0github.com/ory/dockertest/v3 v3.11.0github.com/robfig/cron/v3 v3.0.1github.com/sashabaranov/go-openai v1.41.2diff --git a/go.sum b/go.sumindex bf30d85b..be61e11f 100644--- a/go.sum+++ b/go.sum@@ -462,6 +462,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Ggithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=+github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc=+github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA=github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=diff --git a/pkg/htmltext/htmltext.go b/pkg/htmltext/htmltext.goindex e2e017c8..92908083 100644--- a/pkg/htmltext/htmltext.go+++ b/pkg/htmltext/htmltext.go@@ -25,6 +25,8 @@ import ("net/url""regexp""strings"+ "sync/atomic"+ "unicode""unicode/utf8""github.com/Machiel/slugify"@@ -32,6 +34,7 @@ import ("github.com/apache/answer/pkg/converter"strip "github.com/grokify/html-strip-tags-go""github.com/mozillazg/go-pinyin"+ "github.com/mozillazg/go-unidecode")var (@@ -47,8 +50,27 @@ var ("\r", " ","\t", " ",)++ // Without this, pure non-Latin titles (Arabic, Cyrillic, Hebrew, ...) get+ // stripped by slugify and collapse to the "topic" fallback. Chinese is+ // handled separately by convertChinese.+ transliterateNonLatin atomic.Bool)+func init() {+ transliterateNonLatin.Store(true)+}++// SetTransliterateNonLatin toggles non-Latin script transliteration for URL slugs.+func SetTransliterateNonLatin(enabled bool) {+ transliterateNonLatin.Store(enabled)+}++// IsTransliterateNonLatinEnabled reports whether non-Latin transliteration is on.+func IsTransliterateNonLatinEnabled() bool {+ return transliterateNonLatin.Load()+}+// ClearText clear HTML, get the clear textfunc ClearText(html string) string {if html == "" {@@ -66,6 +88,9 @@ func ClearText(html string) string {func UrlTitle(title string) (text string) {title = convertChinese(title)+ if transliterateNonLatin.Load() {+ title = convertNonLatin(title)+ }title = clearEmoji(title)title = slugify.Slugify(title)title = url.QueryEscape(title)@@ -95,6 +120,30 @@ func convertChinese(content string) string {return strings.Join(pinyin.LazyConvert(content, nil), "-")}+// Short-circuits on Latin-only / Chinese-only input so existing slugs stay byte-identical.+func convertNonLatin(content string) string {+ if !containsNonLatin(content) {+ return content+ }+ return unidecode.Unidecode(content)+}++func containsNonLatin(content string) bool {+ for _, r := range content {+ switch {+ case r < 0x0080: // ASCII+ continue+ case r >= 0x0080 && r <= 0x024F: // Latin-1 Supplement, Latin Extended-A/B+ continue+ case unicode.Is(unicode.Han, r): // handled by convertChinese+ continue+ case unicode.IsLetter(r):+ return true+ }+ }+ return false+}+func cutLongTitle(title string) string {maxBytes := 150if len(title) <= maxBytes {diff --git a/pkg/htmltext/htmltext_test.go b/pkg/htmltext/htmltext_test.goindex 39de9e96..bcedcb3c 100644--- a/pkg/htmltext/htmltext_test.go+++ b/pkg/htmltext/htmltext_test.go@@ -87,6 +87,111 @@ func TestUrlTitle(t *testing.T) {}}+func TestUrlTitleTable(t *testing.T) {+ // Long pure-Arabic title: 50 copies of the same Arabic word, joined by spaces.+ // Unidecode of "كيف" is "kyf", so the slug becomes "kyf-" repeated and+ // exceeds cutLongTitle's 150-byte cap.+ longArabic := strings.Repeat("كيف ", 50)+ wantLongArabic := strings.Repeat("kyf-", 37) + "ky" // 37*4 + 2 = 150 bytes++ cases := []struct {+ name string+ title string+ want string+ }{+ {+ name: "empty",+ title: "",+ want: "topic",+ },+ {+ name: "pure latin unchanged",+ title: "hello world",+ want: "hello-world",+ },+ {+ // Pinyin conversion drops Latin runes by design — matches pre-fix behavior.+ name: "pure chinese unchanged",+ title: "这是一个,标题,title",+ want: "zhe-shi-yi-ge-biao-ti",+ },+ {+ // The fix: previously collapsed to "topic" for all of these scripts.+ // Outputs are an ASCII approximation, not linguistically correct+ // romanization — see PR description.+ name: "arabic transliterated",+ title: "كيف حالك",+ want: "kyf-hlk",+ },+ {+ name: "mixed latin and arabic",+ title: "مرحبا hello",+ want: "mrhb-hello",+ },+ {+ name: "thai transliterated",+ title: "ไทย ไทย",+ want: "aithy-aithy",+ },+ {+ name: "japanese hiragana transliterated",+ title: "こんにちは",+ want: "konnichiha",+ },+ {+ // Japanese with Han-block kanji is caught by the pre-existing pinyin+ // pre-step (Chinese reading, not Japanese), so this path is unchanged+ // by this PR. Pinning to document the existing behavior.+ name: "japanese kanji goes through pinyin path unchanged",+ title: "日本",+ want: "ri-ben",+ },+ {+ name: "korean transliterated",+ title: "안녕하세요",+ want: "annyeonghaseyo",+ },+ {+ name: "hebrew transliterated",+ title: "שלום עולם",+ want: "shlvm-vlm",+ },+ {+ name: "cyrillic transliterated",+ title: "Привет мир",+ want: "privet-mir",+ },+ {+ name: "emoji only falls back to topic",+ title: "😂😂😂",+ want: "topic",+ },+ {+ name: "long arabic truncates at cutLongTitle boundary",+ title: longArabic,+ want: wantLongArabic,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := UrlTitle(tc.title)+ assert.Equal(t, tc.want, got)+ })+ }+}++func TestUrlTitleTransliterationToggle(t *testing.T) {+ defer SetTransliterateNonLatin(true)++ SetTransliterateNonLatin(false)+ // With transliteration off, pure-Arabic titles collapse to the existing+ // "topic" fallback (the pre-fix behavior).+ assert.Equal(t, "topic", UrlTitle("كيف حالك"))++ SetTransliterateNonLatin(true)+ assert.Equal(t, "kyf-hlk", UrlTitle("كيف حالك"))+}+func TestFindFirstMatchedWord(t *testing.T) {var (expectedWord,