Open platform to collect and prioritize feedback
Fixed problems with search in latin languages.
Albert Nimtz34f47fd1e751
diff --git a/Makefile b/Makefileindex 2be0e4d7a..c21f15893 100644--- a/Makefile+++ b/Makefile@@ -12,10 +12,9 @@ LDFLAGS += -X github.com/getfider/fider/app/pkg/env.version=${VERSION}run: ## Run Fidergodotenv -f .env ./fider-migrate: ## Run all database migrations+migrate: ## Run all database migrations and Go migrationsgodotenv -f .env ./fider migrate--+# go run go_migrations/001_backfill_ascii_columns.go##@ Buildingdiff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.goindex 001dcc182..6a8cb28dd 100644--- a/app/services/sqlstore/postgres/post.go+++ b/app/services/sqlstore/postgres/post.go@@ -16,10 +16,12 @@ import ("github.com/getfider/fider/app/pkg/bus""github.com/getfider/fider/app/pkg/env"+ "github.com/getfider/fider/app/pkg/log""github.com/getfider/fider/app/models/cmd""github.com/getfider/fider/app/pkg/dbx""github.com/getfider/fider/app/pkg/errors"+ "github.com/mozillazg/go-unidecode")type dbPost struct {@@ -290,10 +292,12 @@ func countPostPerStatus(ctx context.Context, q *query.CountPostPerStatus) errorfunc addNewPost(ctx context.Context, c *cmd.AddNewPost) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {var id int+ titleAscii := unidecode.Unidecode(c.Title)+ descriptionAscii := unidecode.Unidecode(c.Description)err := trx.Get(&id,- `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status)- VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0)- RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now())+ `INSERT INTO posts (title, slug, number, description, title_ascii, description_ascii, tenant_id, user_id, created_at, status)+ VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $6), $3, $4, $5, $6, $7, $8, 0)+ RETURNING id`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, tenant.ID, user.ID, time.Now())if err != nil {return errors.Wrap(err, "failed add new post")}@@ -314,8 +318,10 @@ func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {func updatePost(ctx context.Context, c *cmd.UpdatePost) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {- _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3- WHERE id = $4 AND tenant_id = $5`, c.Title, slug.Make(c.Title), c.Description, c.Post.ID, tenant.ID)+ titleAscii := unidecode.Unidecode(c.Title)+ descriptionAscii := unidecode.Unidecode(c.Description)+ _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3, title_ascii = $4, description_ascii = $5+ WHERE id = $6 AND tenant_id = $7`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, c.Post.ID, tenant.ID)if err != nil {return errors.Wrap(err, "failed update post")}@@ -400,7 +406,10 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {if filteredQuery == "" {q.Result = make([]*entity.Post, 0)} else {- scoreField := "ts_rank(setweight(to_tsvector(title), 'A') || setweight(to_tsvector(description), 'B'), to_tsquery('english', $3)) + similarity(title, $4) + similarity(description, $4)"+ normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))++ // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it+ scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"sql := fmt.Sprintf(`SELECT * FROM (%s) AS qWHERE %s > 0.5@@ -413,7 +422,7 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {enum.PostPlanned,enum.PostCompleted,enum.PostDeclined,- }), ToTSQuery(filteredQuery), SanitizeString(filteredQuery))+ }), ToTSQuery(normalizedQuery), normalizedQuery)}if err != nil {return errors.Wrap(err, "failed to find similar posts")@@ -445,12 +454,24 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {}}+ // TODO: REMOVE DEBUG LOG+ log.Infof(ctx, "Search query: '@{query}', sanitizeString: '@{sanitizeString}', normalized query: '@{normalizedQuery}', tsquery: '@{tsquery}', tsqueryOriginal: '@{tsqueryOriginal}'", map[string]any{+ "query": q.Query,+ "sanitizeString": SanitizeString(q.Query),+ "normalizedQuery": unidecode.Unidecode(SanitizeString(q.Query)),+ "tsquery": ToTSQuery(unidecode.Unidecode(SanitizeString(q.Query))),+ "tsqueryOriginal": ToTSQuery(q.Query),+ })+var (posts []*dbPosterr error)if q.Query != "" {- scoreField := "ts_rank(setweight(to_tsvector(title), 'A') || setweight(to_tsvector(description), 'B'), to_tsquery('english', $3)) + similarity(title, $4) + similarity(description, $4)"+ normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))++ // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it+ scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"sql := fmt.Sprintf(`SELECT * FROM (%s) AS qWHERE %s > 0.4@@ -463,13 +484,12 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {enum.PostPlanned,enum.PostCompleted,enum.PostDeclined,- }), ToTSQuery(q.Query), SanitizeString(q.Query))+ }), ToTSQuery(normalizedQuery), normalizedQuery)} else {condition, statuses, sort := getViewData(*q)if q.MyPostsOnly {condition += " AND user_id = " + strconv.Itoa(user.ID)-}sql := fmt.Sprintf(`diff --git a/go.mod b/go.modindex cd1046a4b..e995527a5 100644--- a/go.mod+++ b/go.mod@@ -142,6 +142,7 @@ require (github.com/mitchellh/go-homedir v1.1.0 // indirectgithub.com/mitchellh/mapstructure v1.5.0 // indirectgithub.com/moricho/tparallel v0.3.1 // indirect+ github.com/mozillazg/go-unidecode v0.2.0 // indirectgithub.com/nakabonne/nestif v0.3.1 // indirectgithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirectgithub.com/nishanths/exhaustive v0.12.0 // indirectdiff --git a/go.sum b/go.sumindex cf2eaa545..a1a200205 100644--- a/go.sum+++ b/go.sum@@ -434,6 +434,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmbgithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA=github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI=+github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc=+github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA=github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U=diff --git a/go_migrations/001_backfill_ascii_columns.go b/go_migrations/001_backfill_ascii_columns.gonew file mode 100644index 000000000..c6a10e742--- /dev/null+++ b/go_migrations/001_backfill_ascii_columns.go@@ -0,0 +1,52 @@+package main++import (+ "database/sql"+ "fmt"+ "log"++ "github.com/getfider/fider/app/pkg/dbx"+ "github.com/joho/godotenv"+ "github.com/mozillazg/go-unidecode"+)++func main() {+ godotenv.Load()+ db := dbx.Connection()+ rows, err := db.Query("SELECT id, title, description FROM posts")+ if err != nil {+ log.Fatalf("Failed to query posts: %v", err)+ }+ defer rows.Close()++ for rows.Next() {+ var id int+ var title, description sql.NullString+ if err := rows.Scan(&id, &title, &description); err != nil {+ log.Printf("Failed to scan post %d: %v", id, err)+ continue+ }++ titleAscii := ""+ descriptionAscii := ""+ if title.Valid {+ titleAscii = unidecode.Unidecode(title.String)+ }+ if description.Valid {+ descriptionAscii = unidecode.Unidecode(description.String)+ }++ _, err := db.Exec("UPDATE posts SET title_ascii = $1, description_ascii = $2 WHERE id = $3", titleAscii, descriptionAscii, id)+ if err != nil {+ log.Printf("Failed to update post %d: %v", id, err)+ } else {+ fmt.Printf("Updated post %d\n", id)+ }+ }++ if err := rows.Err(); err != nil {+ log.Fatalf("Row error: %v", err)+ }++ fmt.Println("Backfill complete.")+}diff --git a/migrations/202510101200_add_ascii_columns_to_posts.up.sql b/migrations/202510101200_add_ascii_columns_to_posts.up.sqlnew file mode 100644index 000000000..6ddbf5c48--- /dev/null+++ b/migrations/202510101200_add_ascii_columns_to_posts.up.sql@@ -0,0 +1,7 @@+-- Migration: Add ASCII-normalized columns for search+ALTER TABLE posts ADD COLUMN title_ascii TEXT;+ALTER TABLE posts ADD COLUMN description_ascii TEXT;++-- Optional: Backfill existing data+UPDATE posts SET title_ascii = '';+UPDATE posts SET description_ascii = '';diff --git a/public/pages/Home/components/PostsContainer.tsx b/public/pages/Home/components/PostsContainer.tsxindex 405930ecc..7f0c0b0ab 100644--- a/public/pages/Home/components/PostsContainer.tsx+++ b/public/pages/Home/components/PostsContainer.tsx@@ -104,7 +104,7 @@ export class PostsContainer extends React.Component<PostsContainerProps, PostsCowindow.clearTimeout(this.timer)this.setState({ posts: reset ? undefined : this.state.posts, loading: true })this.timer = window.setTimeout(() => {- actions.searchPosts({ query, view: view, limit, tags, statuses, myVotes, noTags }).then((response) => {+ actions.searchPosts({ query, view: view, limit, tags, statuses, myVotes, myPosts, noTags }).then((response) => {if (response.ok && this.state.loading) {this.setState({ loading: false, posts: response.data })}