Open platform to collect and prioritize feedback
Fixed problems with search in non latin languages.
Albert Nimtz52e34c662267
diff --git a/Makefile b/Makefileindex c21f15893..2be0e4d7a 100644--- a/Makefile+++ b/Makefile@@ -12,9 +12,10 @@ LDFLAGS += -X github.com/getfider/fider/app/pkg/env.version=${VERSION}run: ## Run Fidergodotenv -f .env ./fider-migrate: ## Run all database migrations and Go migrations+migrate: ## Run all database migrationsgodotenv -f .env ./fider migrate-# go run go_migrations/001_backfill_ascii_columns.go++##@ Buildingdiff --git a/app/services/sqlstore/postgres/common.go b/app/services/sqlstore/postgres/common.goindex 28b781638..c815975b5 100644--- a/app/services/sqlstore/postgres/common.go+++ b/app/services/sqlstore/postgres/common.go@@ -11,12 +11,14 @@ import ("github.com/getfider/fider/app/pkg/web")-var onlyalphanumeric = regexp.MustCompile("[^a-zA-Z0-9 |]+")+// allowedTextRunes matches any character that is NOT a Unicode letter (\p{L}), number (\p{N}), space, or pipe.+// This allows both Latin and non-Latin scripts to be preserved for full-text search.+var allowedTextRunes = regexp.MustCompile(`[^\p{L}\p{N} |]+`)var replaceOr = strings.NewReplacer("|", " ")// ToTSQuery converts input to another string that can be safely used for ts_queryfunc ToTSQuery(input string) string {- input = replaceOr.Replace(onlyalphanumeric.ReplaceAllString(input, ""))+ input = replaceOr.Replace(allowedTextRunes.ReplaceAllString(input, ""))return strings.Join(strings.Fields(input), "|")}@@ -26,6 +28,52 @@ func SanitizeString(input string) string {return strings.ToValidUTF8(input, "")}+// mapLocaleToTSConfig maps a tenant's locale short key to the corresponding PostgreSQL text search configuration.+// See the /locale folder for supported locales. Returns 'simple' if no match is found.+func MapLocaleToTSConfig(locale string) string {+ switch locale {+ case "en":+ return "english"+ case "de":+ return "german"+ case "fr":+ return "french"+ case "it":+ return "italian"+ case "es-ES":+ return "spanish"+ case "ru":+ return "russian"+ case "pt-BR":+ return "portuguese"+ case "nl":+ return "dutch"+ case "sv-SE":+ return "swedish"+ case "tr":+ return "turkish"+ case "cs":+ return "czech"+ case "pl":+ return "polish"+ case "ar":+ return "arabic"+ case "el":+ return "greek"+ // No direct mappings or require extensions+ case "sk":+ case "ja":+ case "zh-CN":+ case "ko":+ case "fa":+ case "si-LK":+ default:+ return "simple"+ }++ return "simple"+}+func getViewData(query query.SearchPosts) (string, []enum.PostStatus, string) {var (condition stringdiff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.goindex 6a8cb28dd..251c52fe5 100644--- a/app/services/sqlstore/postgres/post.go+++ b/app/services/sqlstore/postgres/post.go@@ -16,12 +16,10 @@ import ("github.com/getfider/fider/app/pkg/bus""github.com/getfider/fider/app/pkg/env"- "github.com/getfider/fider/app/pkg/log""github.com/getfider/fider/app/models/cmd""github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/errors"- "github.com/mozillazg/go-unidecode")type dbPost struct {@@ -292,12 +290,10 @@ func countPostPerStatus(ctx context.Context, q *query.CountPostPerStatus) errorfunc addNewPost(ctx context.Context, c *cmd.AddNewPost) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {var id int- titleAscii := unidecode.Unidecode(c.Title)- descriptionAscii := unidecode.Unidecode(c.Description)err := trx.Get(&id,- `INSERT INTO posts (title, slug, number, description, title_ascii, description_ascii, tenant_id, user_id, created_at, status)- VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $6), $3, $4, $5, $6, $7, $8, 0)- RETURNING id`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, tenant.ID, user.ID, time.Now())+ `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status)+ VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0)+ RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now())if err != nil {return errors.Wrap(err, "failed add new post")}@@ -318,10 +314,8 @@ func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {func updatePost(ctx context.Context, c *cmd.UpdatePost) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {- titleAscii := unidecode.Unidecode(c.Title)- descriptionAscii := unidecode.Unidecode(c.Description)- _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3, title_ascii = $4, description_ascii = $5- WHERE id = $6 AND tenant_id = $7`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, c.Post.ID, tenant.ID)+ _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3+ WHERE id = $4 AND tenant_id = $5`, c.Title, slug.Make(c.Title), c.Description, c.Post.ID, tenant.ID)if err != nil {return errors.Wrap(err, "failed update post")}@@ -406,23 +400,39 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {if filteredQuery == "" {q.Result = make([]*entity.Post, 0)} else {- normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))+ tsConfig := MapLocaleToTSConfig(tenant.Locale)++ // regexp_replace: replaces spaces with ' & ' for AND search; ':*' enables prefix matching in PostgreSQL full-text search+ tsQuery := fmt.Sprintf("to_tsquery('%s', regexp_replace(unaccent($3), '\\s+', ' & ', 'g') || ':*')", tsConfig)+ // Build tsvector with multiple configurations for better multilingual support using the tenant's locale and english and simple as fallbacks+ tsVector := fmt.Sprintf("to_tsvector('%s', title) || ' ' || to_tsvector('%s', description)", tsConfig, tsConfig)++ if tsConfig != "english" {+ tsVector = fmt.Sprintf("%s || to_tsvector('english', title) || ' ' || to_tsvector('english', description)", tsVector)+ }++ if tsConfig != "simple" {+ tsVector = fmt.Sprintf("%s || to_tsvector('simple', title) || ' ' || to_tsvector('simple', description)", tsVector)+ }++ tsVector = fmt.Sprintf("(%s)", tsVector)+ score := fmt.Sprintf("ts_rank_cd(%s, %s)", tsVector, tsQuery)++ whereParts := fmt.Sprintf(`%s @@ %s OR (title ILIKE '%%' || $3 || '%%') OR (description ILIKE '%%' || $3 || '%%') OR (similarity(title, $3) > 0.3) OR (similarity(description, $3) > 0.3)`, tsVector, tsQuery)- // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it- scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"sql := fmt.Sprintf(`SELECT * FROM (%s) AS q- WHERE %s > 0.5+ WHERE %sORDER BY %s DESCLIMIT 5- `, innerQuery, scoreField, scoreField)+ `, innerQuery, whereParts, score)err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{enum.PostOpen,enum.PostStarted,enum.PostPlanned,enum.PostCompleted,enum.PostDeclined,- }), ToTSQuery(normalizedQuery), normalizedQuery)+ }), ToTSQuery(SanitizeString(q.Query)))}if err != nil {return errors.Wrap(err, "failed to find similar posts")@@ -454,37 +464,43 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {}}- // TODO: REMOVE DEBUG LOG- log.Infof(ctx, "Search query: '@{query}', sanitizeString: '@{sanitizeString}', normalized query: '@{normalizedQuery}', tsquery: '@{tsquery}', tsqueryOriginal: '@{tsqueryOriginal}'", map[string]any{- "query": q.Query,- "sanitizeString": SanitizeString(q.Query),- "normalizedQuery": unidecode.Unidecode(SanitizeString(q.Query)),- "tsquery": ToTSQuery(unidecode.Unidecode(SanitizeString(q.Query))),- "tsqueryOriginal": ToTSQuery(q.Query),- })-var (posts []*dbPosterr error)if q.Query != "" {- normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))+ tsConfig := MapLocaleToTSConfig(tenant.Locale)++ // regexp_replace: replaces spaces with ' & ' for AND search; ':*' enables prefix matching in PostgreSQL full-text search+ tsQuery := fmt.Sprintf("to_tsquery('%s', regexp_replace(unaccent($3), '\\s+', ' & ', 'g') || ':*')", tsConfig)+ tsVector := fmt.Sprintf("to_tsvector('%s', title) || ' ' || to_tsvector('%s', description)", tsConfig, tsConfig)++ if tsConfig != "english" {+ tsVector = fmt.Sprintf("%s || to_tsvector('english', title) || ' ' || to_tsvector('english', description)", tsVector)+ }++ if tsConfig != "simple" {+ tsVector = fmt.Sprintf("%s || to_tsvector('simple', title) || ' ' || to_tsvector('simple', description)", tsVector)+ }++ tsVector = fmt.Sprintf("(%s)", tsVector)+ score := fmt.Sprintf("ts_rank_cd(%s, %s)", tsVector, tsQuery)++ whereParts := fmt.Sprintf(`%s @@ %s OR (title ILIKE '%%' || $3 || '%%') OR (description ILIKE '%%' || $3 || '%%') OR (similarity(title, $3) > 0.3) OR (similarity(description, $3) > 0.3)`, tsVector, tsQuery)- // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it- scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"sql := fmt.Sprintf(`SELECT * FROM (%s) AS q- WHERE %s > 0.4+ WHERE %sORDER BY %s DESCLIMIT %s- `, innerQuery, scoreField, scoreField, q.Limit)+ `, innerQuery, whereParts, score, q.Limit)err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{enum.PostOpen,enum.PostStarted,enum.PostPlanned,enum.PostCompleted,enum.PostDeclined,- }), ToTSQuery(normalizedQuery), normalizedQuery)+ }), ToTSQuery(SanitizeString(q.Query)))} else {condition, statuses, sort := getViewData(*q)diff --git a/go.mod b/go.modindex e995527a5..cd1046a4b 100644--- a/go.mod+++ b/go.mod@@ -142,7 +142,6 @@ require (github.com/mitchellh/go-homedir v1.1.0 // indirectgithub.com/mitchellh/mapstructure v1.5.0 // indirectgithub.com/moricho/tparallel v0.3.1 // indirect- github.com/mozillazg/go-unidecode v0.2.0 // indirectgithub.com/nakabonne/nestif v0.3.1 // indirectgithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirectgithub.com/nishanths/exhaustive v0.12.0 // indirectdiff --git a/go.sum b/go.sumindex a1a200205..cf2eaa545 100644--- a/go.sum+++ b/go.sum@@ -434,8 +434,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmbgithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA=github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI=-github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc=-github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA=github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U=diff --git a/go_migrations/001_backfill_ascii_columns.go b/go_migrations/001_backfill_ascii_columns.godeleted file mode 100644index c6a10e742..000000000--- a/go_migrations/001_backfill_ascii_columns.go+++ /dev/null@@ -1,52 +0,0 @@-package main--import (- "database/sql"- "fmt"- "log"-- "github.com/getfider/fider/app/pkg/dbx"- "github.com/joho/godotenv"- "github.com/mozillazg/go-unidecode"-)--func main() {- godotenv.Load()- db := dbx.Connection()- rows, err := db.Query("SELECT id, title, description FROM posts")- if err != nil {- log.Fatalf("Failed to query posts: %v", err)- }- defer rows.Close()-- for rows.Next() {- var id int- var title, description sql.NullString- if err := rows.Scan(&id, &title, &description); err != nil {- log.Printf("Failed to scan post %d: %v", id, err)- continue- }-- titleAscii := ""- descriptionAscii := ""- if title.Valid {- titleAscii = unidecode.Unidecode(title.String)- }- if description.Valid {- descriptionAscii = unidecode.Unidecode(description.String)- }-- _, err := db.Exec("UPDATE posts SET title_ascii = $1, description_ascii = $2 WHERE id = $3", titleAscii, descriptionAscii, id)- if err != nil {- log.Printf("Failed to update post %d: %v", id, err)- } else {- fmt.Printf("Updated post %d\n", id)- }- }-- if err := rows.Err(); err != nil {- log.Fatalf("Row error: %v", err)- }-- fmt.Println("Backfill complete.")-}diff --git a/migrations/202510101200_add_ascii_columns_to_posts.up.sql b/migrations/202510101200_add_ascii_columns_to_posts.up.sqldeleted file mode 100644index 6ddbf5c48..000000000--- a/migrations/202510101200_add_ascii_columns_to_posts.up.sql+++ /dev/null@@ -1,7 +0,0 @@--- Migration: Add ASCII-normalized columns for search-ALTER TABLE posts ADD COLUMN title_ascii TEXT;-ALTER TABLE posts ADD COLUMN description_ascii TEXT;---- Optional: Backfill existing data-UPDATE posts SET title_ascii = '';-UPDATE posts SET description_ascii = '';diff --git a/migrations/202510131000_add_unaccent.up.sql b/migrations/202510131000_add_unaccent.up.sqlnew file mode 100644index 000000000..c22ec715b--- /dev/null+++ b/migrations/202510131000_add_unaccent.up.sql@@ -0,0 +1 @@+CREATE EXTENSION IF NOT EXISTS unaccent;\ No newline at end of file