Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

Fixed problems with search in non latin languages.

Albert Nimtz52e34c662267
diff --git a/Makefile b/Makefile
index c21f15893..2be0e4d7a 100644
--- a/Makefile
+++ b/Makefile
@@ -12,9 +12,10 @@ LDFLAGS += -X github.com/getfider/fider/app/pkg/env.version=${VERSION}
run: ## Run Fider
godotenv -f .env ./fider
-migrate: ## Run all database migrations and Go migrations
+migrate: ## Run all database migrations
godotenv -f .env ./fider migrate
-# go run go_migrations/001_backfill_ascii_columns.go
+
+
##@ Building
diff --git a/app/services/sqlstore/postgres/common.go b/app/services/sqlstore/postgres/common.go
index 28b781638..c815975b5 100644
--- a/app/services/sqlstore/postgres/common.go
+++ b/app/services/sqlstore/postgres/common.go
@@ -11,12 +11,14 @@ import (
"github.com/getfider/fider/app/pkg/web"
)
-var onlyalphanumeric = regexp.MustCompile("[^a-zA-Z0-9 |]+")
+// allowedTextRunes matches any character that is NOT a Unicode letter (\p{L}), number (\p{N}), space, or pipe.
+// This allows both Latin and non-Latin scripts to be preserved for full-text search.
+var allowedTextRunes = regexp.MustCompile(`[^\p{L}\p{N} |]+`)
var replaceOr = strings.NewReplacer("|", " ")
// ToTSQuery converts input to another string that can be safely used for ts_query
func ToTSQuery(input string) string {
- input = replaceOr.Replace(onlyalphanumeric.ReplaceAllString(input, ""))
+ input = replaceOr.Replace(allowedTextRunes.ReplaceAllString(input, ""))
return strings.Join(strings.Fields(input), "|")
}
@@ -26,6 +28,52 @@ func SanitizeString(input string) string {
return strings.ToValidUTF8(input, "")
}
+// mapLocaleToTSConfig maps a tenant's locale short key to the corresponding PostgreSQL text search configuration.
+// See the /locale folder for supported locales. Returns 'simple' if no match is found.
+func MapLocaleToTSConfig(locale string) string {
+ switch locale {
+ case "en":
+ return "english"
+ case "de":
+ return "german"
+ case "fr":
+ return "french"
+ case "it":
+ return "italian"
+ case "es-ES":
+ return "spanish"
+ case "ru":
+ return "russian"
+ case "pt-BR":
+ return "portuguese"
+ case "nl":
+ return "dutch"
+ case "sv-SE":
+ return "swedish"
+ case "tr":
+ return "turkish"
+ case "cs":
+ return "czech"
+ case "pl":
+ return "polish"
+ case "ar":
+ return "arabic"
+ case "el":
+ return "greek"
+ // No direct mappings or require extensions
+ case "sk":
+ case "ja":
+ case "zh-CN":
+ case "ko":
+ case "fa":
+ case "si-LK":
+ default:
+ return "simple"
+ }
+
+ return "simple"
+}
+
func getViewData(query query.SearchPosts) (string, []enum.PostStatus, string) {
var (
condition string
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.go
index 6a8cb28dd..251c52fe5 100644
--- a/app/services/sqlstore/postgres/post.go
+++ b/app/services/sqlstore/postgres/post.go
@@ -16,12 +16,10 @@ import (
"github.com/getfider/fider/app/pkg/bus"
"github.com/getfider/fider/app/pkg/env"
- "github.com/getfider/fider/app/pkg/log"
"github.com/getfider/fider/app/models/cmd"
"github.com/getfider/fider/app/pkg/dbx"
"github.com/getfider/fider/app/pkg/errors"
- "github.com/mozillazg/go-unidecode"
)
type dbPost struct {
@@ -292,12 +290,10 @@ func countPostPerStatus(ctx context.Context, q *query.CountPostPerStatus) error
func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
var id int
- titleAscii := unidecode.Unidecode(c.Title)
- descriptionAscii := unidecode.Unidecode(c.Description)
err := trx.Get(&id,
- `INSERT INTO posts (title, slug, number, description, title_ascii, description_ascii, tenant_id, user_id, created_at, status)
- VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $6), $3, $4, $5, $6, $7, $8, 0)
- RETURNING id`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, tenant.ID, user.ID, time.Now())
+ `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status)
+ VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0)
+ RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now())
if err != nil {
return errors.Wrap(err, "failed add new post")
}
@@ -318,10 +314,8 @@ func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {
func updatePost(ctx context.Context, c *cmd.UpdatePost) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
- titleAscii := unidecode.Unidecode(c.Title)
- descriptionAscii := unidecode.Unidecode(c.Description)
- _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3, title_ascii = $4, description_ascii = $5
- WHERE id = $6 AND tenant_id = $7`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, c.Post.ID, tenant.ID)
+ _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3
+ WHERE id = $4 AND tenant_id = $5`, c.Title, slug.Make(c.Title), c.Description, c.Post.ID, tenant.ID)
if err != nil {
return errors.Wrap(err, "failed update post")
}
@@ -406,23 +400,39 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {
if filteredQuery == "" {
q.Result = make([]*entity.Post, 0)
} else {
- normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))
+ tsConfig := MapLocaleToTSConfig(tenant.Locale)
+
+ // regexp_replace: replaces spaces with ' & ' for AND search; ':*' enables prefix matching in PostgreSQL full-text search
+ tsQuery := fmt.Sprintf("to_tsquery('%s', regexp_replace(unaccent($3), '\\s+', ' & ', 'g') || ':*')", tsConfig)
+ // Build tsvector with multiple configurations for better multilingual support using the tenant's locale and english and simple as fallbacks
+ tsVector := fmt.Sprintf("to_tsvector('%s', title) || ' ' || to_tsvector('%s', description)", tsConfig, tsConfig)
+
+ if tsConfig != "english" {
+ tsVector = fmt.Sprintf("%s || to_tsvector('english', title) || ' ' || to_tsvector('english', description)", tsVector)
+ }
+
+ if tsConfig != "simple" {
+ tsVector = fmt.Sprintf("%s || to_tsvector('simple', title) || ' ' || to_tsvector('simple', description)", tsVector)
+ }
+
+ tsVector = fmt.Sprintf("(%s)", tsVector)
+ score := fmt.Sprintf("ts_rank_cd(%s, %s)", tsVector, tsQuery)
+
+ whereParts := fmt.Sprintf(`%s @@ %s OR (title ILIKE '%%' || $3 || '%%') OR (description ILIKE '%%' || $3 || '%%') OR (similarity(title, $3) > 0.3) OR (similarity(description, $3) > 0.3)`, tsVector, tsQuery)
- // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it
- scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"
sql := fmt.Sprintf(`
SELECT * FROM (%s) AS q
- WHERE %s > 0.5
+ WHERE %s
ORDER BY %s DESC
LIMIT 5
- `, innerQuery, scoreField, scoreField)
+ `, innerQuery, whereParts, score)
err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{
enum.PostOpen,
enum.PostStarted,
enum.PostPlanned,
enum.PostCompleted,
enum.PostDeclined,
- }), ToTSQuery(normalizedQuery), normalizedQuery)
+ }), ToTSQuery(SanitizeString(q.Query)))
}
if err != nil {
return errors.Wrap(err, "failed to find similar posts")
@@ -454,37 +464,43 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {
}
}
- // TODO: REMOVE DEBUG LOG
- log.Infof(ctx, "Search query: '@{query}', sanitizeString: '@{sanitizeString}', normalized query: '@{normalizedQuery}', tsquery: '@{tsquery}', tsqueryOriginal: '@{tsqueryOriginal}'", map[string]any{
- "query": q.Query,
- "sanitizeString": SanitizeString(q.Query),
- "normalizedQuery": unidecode.Unidecode(SanitizeString(q.Query)),
- "tsquery": ToTSQuery(unidecode.Unidecode(SanitizeString(q.Query))),
- "tsqueryOriginal": ToTSQuery(q.Query),
- })
-
var (
posts []*dbPost
err error
)
if q.Query != "" {
- normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))
+ tsConfig := MapLocaleToTSConfig(tenant.Locale)
+
+ // regexp_replace: replaces spaces with ' & ' for AND search; ':*' enables prefix matching in PostgreSQL full-text search
+ tsQuery := fmt.Sprintf("to_tsquery('%s', regexp_replace(unaccent($3), '\\s+', ' & ', 'g') || ':*')", tsConfig)
+ tsVector := fmt.Sprintf("to_tsvector('%s', title) || ' ' || to_tsvector('%s', description)", tsConfig, tsConfig)
+
+ if tsConfig != "english" {
+ tsVector = fmt.Sprintf("%s || to_tsvector('english', title) || ' ' || to_tsvector('english', description)", tsVector)
+ }
+
+ if tsConfig != "simple" {
+ tsVector = fmt.Sprintf("%s || to_tsvector('simple', title) || ' ' || to_tsvector('simple', description)", tsVector)
+ }
+
+ tsVector = fmt.Sprintf("(%s)", tsVector)
+ score := fmt.Sprintf("ts_rank_cd(%s, %s)", tsVector, tsQuery)
+
+ whereParts := fmt.Sprintf(`%s @@ %s OR (title ILIKE '%%' || $3 || '%%') OR (description ILIKE '%%' || $3 || '%%') OR (similarity(title, $3) > 0.3) OR (similarity(description, $3) > 0.3)`, tsVector, tsQuery)
- // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it
- scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"
sql := fmt.Sprintf(`
SELECT * FROM (%s) AS q
- WHERE %s > 0.4
+ WHERE %s
ORDER BY %s DESC
LIMIT %s
- `, innerQuery, scoreField, scoreField, q.Limit)
+ `, innerQuery, whereParts, score, q.Limit)
err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{
enum.PostOpen,
enum.PostStarted,
enum.PostPlanned,
enum.PostCompleted,
enum.PostDeclined,
- }), ToTSQuery(normalizedQuery), normalizedQuery)
+ }), ToTSQuery(SanitizeString(q.Query)))
} else {
condition, statuses, sort := getViewData(*q)
diff --git a/go.mod b/go.mod
index e995527a5..cd1046a4b 100644
--- a/go.mod
+++ b/go.mod
@@ -142,7 +142,6 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.3.1 // indirect
- github.com/mozillazg/go-unidecode v0.2.0 // indirect
github.com/nakabonne/nestif v0.3.1 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/nishanths/exhaustive v0.12.0 // indirect
diff --git a/go.sum b/go.sum
index a1a200205..cf2eaa545 100644
--- a/go.sum
+++ b/go.sum
@@ -434,8 +434,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA=
github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI=
-github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc=
-github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U=
diff --git a/go_migrations/001_backfill_ascii_columns.go b/go_migrations/001_backfill_ascii_columns.go
deleted file mode 100644
index c6a10e742..000000000
--- a/go_migrations/001_backfill_ascii_columns.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package main
-
-import (
- "database/sql"
- "fmt"
- "log"
-
- "github.com/getfider/fider/app/pkg/dbx"
- "github.com/joho/godotenv"
- "github.com/mozillazg/go-unidecode"
-)
-
-func main() {
- godotenv.Load()
- db := dbx.Connection()
- rows, err := db.Query("SELECT id, title, description FROM posts")
- if err != nil {
- log.Fatalf("Failed to query posts: %v", err)
- }
- defer rows.Close()
-
- for rows.Next() {
- var id int
- var title, description sql.NullString
- if err := rows.Scan(&id, &title, &description); err != nil {
- log.Printf("Failed to scan post %d: %v", id, err)
- continue
- }
-
- titleAscii := ""
- descriptionAscii := ""
- if title.Valid {
- titleAscii = unidecode.Unidecode(title.String)
- }
- if description.Valid {
- descriptionAscii = unidecode.Unidecode(description.String)
- }
-
- _, err := db.Exec("UPDATE posts SET title_ascii = $1, description_ascii = $2 WHERE id = $3", titleAscii, descriptionAscii, id)
- if err != nil {
- log.Printf("Failed to update post %d: %v", id, err)
- } else {
- fmt.Printf("Updated post %d\n", id)
- }
- }
-
- if err := rows.Err(); err != nil {
- log.Fatalf("Row error: %v", err)
- }
-
- fmt.Println("Backfill complete.")
-}
diff --git a/migrations/202510101200_add_ascii_columns_to_posts.up.sql b/migrations/202510101200_add_ascii_columns_to_posts.up.sql
deleted file mode 100644
index 6ddbf5c48..000000000
--- a/migrations/202510101200_add_ascii_columns_to_posts.up.sql
+++ /dev/null
@@ -1,7 +0,0 @@
--- Migration: Add ASCII-normalized columns for search
-ALTER TABLE posts ADD COLUMN title_ascii TEXT;
-ALTER TABLE posts ADD COLUMN description_ascii TEXT;
-
--- Optional: Backfill existing data
-UPDATE posts SET title_ascii = '';
-UPDATE posts SET description_ascii = '';
diff --git a/migrations/202510131000_add_unaccent.up.sql b/migrations/202510131000_add_unaccent.up.sql
new file mode 100644
index 000000000..c22ec715b
--- /dev/null
+++ b/migrations/202510131000_add_unaccent.up.sql
@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS unaccent;
\ No newline at end of file