Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

Fixed problems with search in latin languages.

Albert Nimtz34f47fd1e751
diff --git a/Makefile b/Makefile
index 2be0e4d7a..c21f15893 100644
--- a/Makefile
+++ b/Makefile
@@ -12,10 +12,9 @@ LDFLAGS += -X github.com/getfider/fider/app/pkg/env.version=${VERSION}
run: ## Run Fider
godotenv -f .env ./fider
-migrate: ## Run all database migrations
+migrate: ## Run all database migrations and Go migrations
godotenv -f .env ./fider migrate
-
-
+# go run go_migrations/001_backfill_ascii_columns.go
##@ Building
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.go
index 001dcc182..6a8cb28dd 100644
--- a/app/services/sqlstore/postgres/post.go
+++ b/app/services/sqlstore/postgres/post.go
@@ -16,10 +16,12 @@ import (
"github.com/getfider/fider/app/pkg/bus"
"github.com/getfider/fider/app/pkg/env"
+ "github.com/getfider/fider/app/pkg/log"
"github.com/getfider/fider/app/models/cmd"
"github.com/getfider/fider/app/pkg/dbx"
"github.com/getfider/fider/app/pkg/errors"
+ "github.com/mozillazg/go-unidecode"
)
type dbPost struct {
@@ -290,10 +292,12 @@ func countPostPerStatus(ctx context.Context, q *query.CountPostPerStatus) error
func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
var id int
+ titleAscii := unidecode.Unidecode(c.Title)
+ descriptionAscii := unidecode.Unidecode(c.Description)
err := trx.Get(&id,
- `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status)
- VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0)
- RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now())
+ `INSERT INTO posts (title, slug, number, description, title_ascii, description_ascii, tenant_id, user_id, created_at, status)
+ VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $6), $3, $4, $5, $6, $7, $8, 0)
+ RETURNING id`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, tenant.ID, user.ID, time.Now())
if err != nil {
return errors.Wrap(err, "failed add new post")
}
@@ -314,8 +318,10 @@ func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {
func updatePost(ctx context.Context, c *cmd.UpdatePost) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
- _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3
- WHERE id = $4 AND tenant_id = $5`, c.Title, slug.Make(c.Title), c.Description, c.Post.ID, tenant.ID)
+ titleAscii := unidecode.Unidecode(c.Title)
+ descriptionAscii := unidecode.Unidecode(c.Description)
+ _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3, title_ascii = $4, description_ascii = $5
+ WHERE id = $6 AND tenant_id = $7`, c.Title, slug.Make(c.Title), c.Description, titleAscii, descriptionAscii, c.Post.ID, tenant.ID)
if err != nil {
return errors.Wrap(err, "failed update post")
}
@@ -400,7 +406,10 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {
if filteredQuery == "" {
q.Result = make([]*entity.Post, 0)
} else {
- scoreField := "ts_rank(setweight(to_tsvector(title), 'A') || setweight(to_tsvector(description), 'B'), to_tsquery('english', $3)) + similarity(title, $4) + similarity(description, $4)"
+ normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))
+
+ // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it
+ scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"
sql := fmt.Sprintf(`
SELECT * FROM (%s) AS q
WHERE %s > 0.5
@@ -413,7 +422,7 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {
enum.PostPlanned,
enum.PostCompleted,
enum.PostDeclined,
- }), ToTSQuery(filteredQuery), SanitizeString(filteredQuery))
+ }), ToTSQuery(normalizedQuery), normalizedQuery)
}
if err != nil {
return errors.Wrap(err, "failed to find similar posts")
@@ -445,12 +454,24 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {
}
}
+ // TODO: REMOVE DEBUG LOG
+ log.Infof(ctx, "Search query: '@{query}', sanitizeString: '@{sanitizeString}', normalized query: '@{normalizedQuery}', tsquery: '@{tsquery}', tsqueryOriginal: '@{tsqueryOriginal}'", map[string]any{
+ "query": q.Query,
+ "sanitizeString": SanitizeString(q.Query),
+ "normalizedQuery": unidecode.Unidecode(SanitizeString(q.Query)),
+ "tsquery": ToTSQuery(unidecode.Unidecode(SanitizeString(q.Query))),
+ "tsqueryOriginal": ToTSQuery(q.Query),
+ })
+
var (
posts []*dbPost
err error
)
if q.Query != "" {
- scoreField := "ts_rank(setweight(to_tsvector(title), 'A') || setweight(to_tsvector(description), 'B'), to_tsquery('english', $3)) + similarity(title, $4) + similarity(description, $4)"
+ normalizedQuery := unidecode.Unidecode(SanitizeString(q.Query))
+
+ // TODO: implement tenant.Language detection and mapping to psql language names and then replace 'german' with it
+ scoreField := "ts_rank(setweight(to_tsvector('german', title), 'A') || setweight(to_tsvector('german', description), 'B'), to_tsquery('german', $3 || ':*')) + similarity(title, $4) + similarity(description, $4)"
sql := fmt.Sprintf(`
SELECT * FROM (%s) AS q
WHERE %s > 0.4
@@ -463,13 +484,12 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {
enum.PostPlanned,
enum.PostCompleted,
enum.PostDeclined,
- }), ToTSQuery(q.Query), SanitizeString(q.Query))
+ }), ToTSQuery(normalizedQuery), normalizedQuery)
} else {
condition, statuses, sort := getViewData(*q)
if q.MyPostsOnly {
condition += " AND user_id = " + strconv.Itoa(user.ID)
-
}
sql := fmt.Sprintf(`
diff --git a/go.mod b/go.mod
index cd1046a4b..e995527a5 100644
--- a/go.mod
+++ b/go.mod
@@ -142,6 +142,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.3.1 // indirect
+ github.com/mozillazg/go-unidecode v0.2.0 // indirect
github.com/nakabonne/nestif v0.3.1 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/nishanths/exhaustive v0.12.0 // indirect
diff --git a/go.sum b/go.sum
index cf2eaa545..a1a200205 100644
--- a/go.sum
+++ b/go.sum
@@ -434,6 +434,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA=
github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI=
+github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc=
+github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U=
diff --git a/go_migrations/001_backfill_ascii_columns.go b/go_migrations/001_backfill_ascii_columns.go
new file mode 100644
index 000000000..c6a10e742
--- /dev/null
+++ b/go_migrations/001_backfill_ascii_columns.go
@@ -0,0 +1,52 @@
+package main
+
+import (
+ "database/sql"
+ "fmt"
+ "log"
+
+ "github.com/getfider/fider/app/pkg/dbx"
+ "github.com/joho/godotenv"
+ "github.com/mozillazg/go-unidecode"
+)
+
+func main() {
+ godotenv.Load()
+ db := dbx.Connection()
+ rows, err := db.Query("SELECT id, title, description FROM posts")
+ if err != nil {
+ log.Fatalf("Failed to query posts: %v", err)
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var id int
+ var title, description sql.NullString
+ if err := rows.Scan(&id, &title, &description); err != nil {
+ log.Printf("Failed to scan post %d: %v", id, err)
+ continue
+ }
+
+ titleAscii := ""
+ descriptionAscii := ""
+ if title.Valid {
+ titleAscii = unidecode.Unidecode(title.String)
+ }
+ if description.Valid {
+ descriptionAscii = unidecode.Unidecode(description.String)
+ }
+
+ _, err := db.Exec("UPDATE posts SET title_ascii = $1, description_ascii = $2 WHERE id = $3", titleAscii, descriptionAscii, id)
+ if err != nil {
+ log.Printf("Failed to update post %d: %v", id, err)
+ } else {
+ fmt.Printf("Updated post %d\n", id)
+ }
+ }
+
+ if err := rows.Err(); err != nil {
+ log.Fatalf("Row error: %v", err)
+ }
+
+ fmt.Println("Backfill complete.")
+}
diff --git a/migrations/202510101200_add_ascii_columns_to_posts.up.sql b/migrations/202510101200_add_ascii_columns_to_posts.up.sql
new file mode 100644
index 000000000..6ddbf5c48
--- /dev/null
+++ b/migrations/202510101200_add_ascii_columns_to_posts.up.sql
@@ -0,0 +1,7 @@
+-- Migration: Add ASCII-normalized columns for search
+ALTER TABLE posts ADD COLUMN title_ascii TEXT;
+ALTER TABLE posts ADD COLUMN description_ascii TEXT;
+
+-- Optional: Backfill existing data
+UPDATE posts SET title_ascii = '';
+UPDATE posts SET description_ascii = '';
diff --git a/public/pages/Home/components/PostsContainer.tsx b/public/pages/Home/components/PostsContainer.tsx
index 405930ecc..7f0c0b0ab 100644
--- a/public/pages/Home/components/PostsContainer.tsx
+++ b/public/pages/Home/components/PostsContainer.tsx
@@ -104,7 +104,7 @@ export class PostsContainer extends React.Component<PostsContainerProps, PostsCo
window.clearTimeout(this.timer)
this.setState({ posts: reset ? undefined : this.state.posts, loading: true })
this.timer = window.setTimeout(() => {
- actions.searchPosts({ query, view: view, limit, tags, statuses, myVotes, noTags }).then((response) => {
+ actions.searchPosts({ query, view: view, limit, tags, statuses, myVotes, myPosts, noTags }).then((response) => {
if (response.ok && this.state.loading) {
this.setState({ loading: false, posts: response.data })
}