Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

- Added migration script to add search and language columns - added lingua-go for language detection - Fixed wrong fider-language to postgresql-regconfig mappings

Albert Nimtzfc020c805ac7
diff --git a/app/services/sqlstore/postgres/common.go b/app/services/sqlstore/postgres/common.go
index c815975b5..0ac989f2a 100644
--- a/app/services/sqlstore/postgres/common.go
+++ b/app/services/sqlstore/postgres/common.go
@@ -32,41 +32,37 @@ func SanitizeString(input string) string {
// See the /locale folder for supported locales. Returns 'simple' if no match is found.
func MapLocaleToTSConfig(locale string) string {
switch locale {
- case "en":
- return "english"
+ case "ar":
+ return "arabic"
case "de":
return "german"
+ case "en":
+ return "english"
+ case "es-ES":
+ return "spanish"
case "fr":
return "french"
case "it":
return "italian"
- case "es-ES":
- return "spanish"
- case "ru":
- return "russian"
- case "pt-BR":
- return "portuguese"
case "nl":
return "dutch"
+ case "pt-BR":
+ return "portuguese"
+ case "ru":
+ return "russian"
case "sv-SE":
return "swedish"
case "tr":
return "turkish"
case "cs":
- return "czech"
- case "pl":
- return "polish"
- case "ar":
- return "arabic"
case "el":
- return "greek"
- // No direct mappings or require extensions
- case "sk":
+ case "fa":
case "ja":
- case "zh-CN":
case "ko":
- case "fa":
+ case "pl":
case "si-LK":
+ case "sk":
+ case "zh-CN":
default:
return "simple"
}
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.go
index 1117389c8..ff86f0fb9 100644
--- a/app/services/sqlstore/postgres/post.go
+++ b/app/services/sqlstore/postgres/post.go
@@ -8,6 +8,8 @@ import (
"strings"
"time"
+ "github.com/pemistahl/lingua-go"
+
"github.com/getfider/fider/app/models/entity"
"github.com/getfider/fider/app/models/enum"
"github.com/getfider/fider/app/models/query"
@@ -291,10 +293,13 @@ func countPostPerStatus(ctx context.Context, q *query.CountPostPerStatus) error
func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
var id int
+ // Detect language using lingua-go
+ lang := detectPostLanguage(c.Title, c.Description)
+
err := trx.Get(&id,
- `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status)
- VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0)
- RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now())
+ `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status, language)
+ VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0, $7)
+ RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now(), lang)
if err != nil {
return errors.Wrap(err, "failed add new post")
}
@@ -315,8 +320,11 @@ func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {
func updatePost(ctx context.Context, c *cmd.UpdatePost) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
- _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3
- WHERE id = $4 AND tenant_id = $5`, c.Title, slug.Make(c.Title), c.Description, c.Post.ID, tenant.ID)
+ // Detect language using lingua-go
+ lang := detectPostLanguage(c.Title, c.Description)
+ _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3, language = $4
+ WHERE id = $5 AND tenant_id = $6`, c.Title, slug.Make(c.Title), c.Description, lang, c.Post.ID, tenant.ID)
+
if err != nil {
return errors.Wrap(err, "failed update post")
}
@@ -330,6 +338,47 @@ func updatePost(ctx context.Context, c *cmd.UpdatePost) error {
})
}
+// detectPostLanguage uses lingua-go to detect the language of a post and maps it to a PostgreSQL tsvector config or 'simple'.
+func detectPostLanguage(title, description string) string {
+ // Supported lingua languages mapped to locale keys
+ var linguaLangs = []lingua.Language{
+ lingua.English, lingua.German, lingua.French, lingua.Italian, lingua.Dutch, lingua.Polish, lingua.Portuguese, lingua.Russian, lingua.Spanish, lingua.Swedish, lingua.Turkish,
+ lingua.Slovak, lingua.Greek, lingua.Arabic, lingua.Chinese, lingua.Persian, lingua.Japanese,
+ }
+ detector := lingua.NewLanguageDetectorBuilder().FromLanguages(linguaLangs...).Build()
+ text := strings.TrimSpace(title + " " + description)
+ lang, exists := detector.DetectLanguageOf(text)
+ if !exists {
+ return "simple"
+ }
+ // Map lingua-go language to locale code
+ var linguaToLocale = map[lingua.Language]string{
+ lingua.English: "en",
+ lingua.German: "de",
+ lingua.French: "fr",
+ lingua.Italian: "it",
+ lingua.Dutch: "nl",
+ lingua.Polish: "pl",
+ lingua.Portuguese: "pt-BR",
+ lingua.Russian: "ru",
+ lingua.Spanish: "es-ES",
+ lingua.Swedish: "sv-SE",
+ lingua.Turkish: "tr",
+ lingua.Slovak: "sk",
+ lingua.Greek: "el",
+ lingua.Arabic: "ar",
+ lingua.Chinese: "zh-CN",
+ lingua.Persian: "fa",
+ lingua.Japanese: "ja",
+ }
+ locale, ok := linguaToLocale[lang]
+ if !ok {
+ return "simple"
+ }
+ // Map to PostgreSQL tsvector config
+ return MapLocaleToTSConfig(locale)
+}
+
func getPostByID(ctx context.Context, q *query.GetPostByID) error {
return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {
post, err := querySinglePost(ctx, trx, buildPostQuery(user, "p.tenant_id = $1 AND p.id = $2"), tenant.ID, q.PostID)
diff --git a/go.mod b/go.mod
index cd1046a4b..bdc3e3d6b 100644
--- a/go.mod
+++ b/go.mod
@@ -150,6 +150,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+ github.com/pemistahl/lingua-go v1.4.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polyfloyd/go-errorlint v1.5.2 // indirect
github.com/prometheus/common v0.32.1 // indirect
@@ -168,6 +169,7 @@ require (
github.com/sashamelentyev/usestdlibvars v1.26.0 // indirect
github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9 // indirect
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+ github.com/shopspring/decimal v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sivchari/containedctx v1.0.3 // indirect
github.com/sivchari/tenv v1.7.1 // indirect
diff --git a/go.sum b/go.sum
index cf2eaa545..c565fe631 100644
--- a/go.sum
+++ b/go.sum
@@ -466,6 +466,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKBR4rM=
+github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -532,6 +534,8 @@ github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9 h1:rnO6Zp1YMQ
github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9/go.mod h1:dg7lPlu/xK/Ut9SedURCoZbVCR4yC7fM65DtH9/CDHs=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
diff --git a/migrations/202510211300_add_search_column_to_posts.sql b/migrations/202510211300_add_search_column_to_posts.sql
new file mode 100644
index 000000000..4d4648621
--- /dev/null
+++ b/migrations/202510211300_add_search_column_to_posts.sql
@@ -0,0 +1,54 @@
+-- 1. Add language column
+ALTER TABLE posts ADD COLUMN language regconfig;
+
+-- 2. Create language mapping function
+CREATE OR REPLACE FUNCTION map_language_to_tsvector(lang TEXT)
+RETURNS regconfig AS $$
+BEGIN
+ RETURN CASE lang
+ WHEN 'ar' THEN 'arabic'::regconfig
+ WHEN 'cs' THEN 'simple'::regconfig -- Czech not supported, fallback to simple
+ WHEN 'de' THEN 'german'::regconfig
+ WHEN 'el' THEN 'simple'::regconfig -- Greek not supported, fallback to simple
+ WHEN 'en' THEN 'english'::regconfig
+ WHEN 'es-ES' THEN 'spanish'::regconfig
+ WHEN 'fa' THEN 'simple'::regconfig -- Farsi not supported, fallback to simple
+ WHEN 'fr' THEN 'french'::regconfig
+ WHEN 'it' THEN 'italian'::regconfig
+ WHEN 'ja' THEN 'simple'::regconfig -- Japanese not supported, fallback to simple
+ WHEN 'ko' THEN 'simple'::regconfig -- Korean not supported, fallback to simple
+ WHEN 'nl' THEN 'dutch'::regconfig
+ WHEN 'pl' THEN 'simple'::regconfig -- Polish not supported, fallback to simple
+ WHEN 'pt-BR' THEN 'portuguese'::regconfig
+ WHEN 'ru' THEN 'russian'::regconfig
+ WHEN 'sk' THEN 'simple'::regconfig -- Slovak not supported, fallback to simple
+ WHEN 'si-LK' THEN 'simple'::regconfig -- Sinhala not supported, fallback to simple
+ WHEN 'sv-SE' THEN 'swedish'::regconfig
+ WHEN 'tr' THEN 'turkish'::regconfig
+ WHEN 'zh-CN' THEN 'simple'::regconfig -- Chinese not supported, fallback to simple
+ ELSE 'simple'::regconfig
+ END;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- 3. Update language column using tenant's language (assuming you have a way to join or fetch it)
+UPDATE posts
+SET language = map_language_to_tsvector(
+ (SELECT language::TEXT FROM tenants WHERE tenants.id = posts.tenant_id)
+);
+
+-- 4. Add search as a generated column
+ALTER TABLE posts ADD search tsvector GENERATED ALWAYS AS (
+ CASE WHEN language <> 'simple'::regconfig THEN
+ setweight(to_tsvector(language, title::TEXT), 'A') ||
+ setweight(to_tsvector(language, description), 'B') ||
+ setweight(to_tsvector('simple'::regconfig, title::TEXT), 'C') ||
+ setweight(to_tsvector('simple'::regconfig, description), 'D')::tsvector
+ ELSE
+ setweight(to_tsvector('simple'::regconfig, title::TEXT), 'A') ||
+ setweight(to_tsvector('simple'::regconfig, description), 'B')::tsvector
+ END
+) STORED;
+
+-- 5. Create GIN index
+CREATE INDEX idx_posts_search_gin ON posts USING GIN (search);