Open platform to collect and prioritize feedback
- Added migration script to add search and language columns - added lingua-go for language detection - Fixed wrong fider-language to postgresql-regconfig mappings
Albert Nimtzfc020c805ac7
diff --git a/app/services/sqlstore/postgres/common.go b/app/services/sqlstore/postgres/common.goindex c815975b5..0ac989f2a 100644--- a/app/services/sqlstore/postgres/common.go+++ b/app/services/sqlstore/postgres/common.go@@ -32,41 +32,37 @@ func SanitizeString(input string) string {// See the /locale folder for supported locales. Returns 'simple' if no match is found.func MapLocaleToTSConfig(locale string) string {switch locale {- case "en":- return "english"+ case "ar":+ return "arabic"case "de":return "german"+ case "en":+ return "english"+ case "es-ES":+ return "spanish"case "fr":return "french"case "it":return "italian"- case "es-ES":- return "spanish"- case "ru":- return "russian"- case "pt-BR":- return "portuguese"case "nl":return "dutch"+ case "pt-BR":+ return "portuguese"+ case "ru":+ return "russian"case "sv-SE":return "swedish"case "tr":return "turkish"case "cs":- return "czech"- case "pl":- return "polish"- case "ar":- return "arabic"case "el":- return "greek"- // No direct mappings or require extensions- case "sk":+ case "fa":case "ja":- case "zh-CN":case "ko":- case "fa":+ case "pl":case "si-LK":+ case "sk":+ case "zh-CN":default:return "simple"}diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.goindex 1117389c8..ff86f0fb9 100644--- a/app/services/sqlstore/postgres/post.go+++ b/app/services/sqlstore/postgres/post.go@@ -8,6 +8,8 @@ import ("strings""time"+ "github.com/pemistahl/lingua-go"+"github.com/getfider/fider/app/models/entity""github.com/getfider/fider/app/models/enum""github.com/getfider/fider/app/models/query"@@ -291,10 +293,13 @@ func countPostPerStatus(ctx context.Context, q *query.CountPostPerStatus) errorfunc addNewPost(ctx context.Context, c *cmd.AddNewPost) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {var id int+ // Detect language using lingua-go+ lang := detectPostLanguage(c.Title, c.Description)+err := trx.Get(&id,- `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status)- VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0)- RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now())+ `INSERT INTO posts (title, slug, number, description, tenant_id, user_id, created_at, status, language)+ VALUES ($1, $2, (SELECT COALESCE(MAX(number), 0) + 1 FROM posts p WHERE p.tenant_id = $4), $3, $4, $5, $6, 0, $7)+ RETURNING id`, c.Title, slug.Make(c.Title), c.Description, tenant.ID, user.ID, time.Now(), lang)if err != nil {return errors.Wrap(err, "failed add new post")}@@ -315,8 +320,11 @@ func addNewPost(ctx context.Context, c *cmd.AddNewPost) error {func updatePost(ctx context.Context, c *cmd.UpdatePost) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {- _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3- WHERE id = $4 AND tenant_id = $5`, c.Title, slug.Make(c.Title), c.Description, c.Post.ID, tenant.ID)+ // Detect language using lingua-go+ lang := detectPostLanguage(c.Title, c.Description)+ _, err := trx.Execute(`UPDATE posts SET title = $1, slug = $2, description = $3, language = $4+ WHERE id = $5 AND tenant_id = $6`, c.Title, slug.Make(c.Title), c.Description, lang, c.Post.ID, tenant.ID)+if err != nil {return errors.Wrap(err, "failed update post")}@@ -330,6 +338,47 @@ func updatePost(ctx context.Context, c *cmd.UpdatePost) error {})}+// detectPostLanguage uses lingua-go to detect the language of a post and maps it to a PostgreSQL tsvector config or 'simple'.+func detectPostLanguage(title, description string) string {+ // Supported lingua languages mapped to locale keys+ var linguaLangs = []lingua.Language{+ lingua.English, lingua.German, lingua.French, lingua.Italian, lingua.Dutch, lingua.Polish, lingua.Portuguese, lingua.Russian, lingua.Spanish, lingua.Swedish, lingua.Turkish,+ lingua.Slovak, lingua.Greek, lingua.Arabic, lingua.Chinese, lingua.Persian, lingua.Japanese,+ }+ detector := lingua.NewLanguageDetectorBuilder().FromLanguages(linguaLangs...).Build()+ text := strings.TrimSpace(title + " " + description)+ lang, exists := detector.DetectLanguageOf(text)+ if !exists {+ return "simple"+ }+ // Map lingua-go language to locale code+ var linguaToLocale = map[lingua.Language]string{+ lingua.English: "en",+ lingua.German: "de",+ lingua.French: "fr",+ lingua.Italian: "it",+ lingua.Dutch: "nl",+ lingua.Polish: "pl",+ lingua.Portuguese: "pt-BR",+ lingua.Russian: "ru",+ lingua.Spanish: "es-ES",+ lingua.Swedish: "sv-SE",+ lingua.Turkish: "tr",+ lingua.Slovak: "sk",+ lingua.Greek: "el",+ lingua.Arabic: "ar",+ lingua.Chinese: "zh-CN",+ lingua.Persian: "fa",+ lingua.Japanese: "ja",+ }+ locale, ok := linguaToLocale[lang]+ if !ok {+ return "simple"+ }+ // Map to PostgreSQL tsvector config+ return MapLocaleToTSConfig(locale)+}+func getPostByID(ctx context.Context, q *query.GetPostByID) error {return using(ctx, func(trx *dbx.Trx, tenant *entity.Tenant, user *entity.User) error {post, err := querySinglePost(ctx, trx, buildPostQuery(user, "p.tenant_id = $1 AND p.id = $2"), tenant.ID, q.PostID)diff --git a/go.mod b/go.modindex cd1046a4b..bdc3e3d6b 100644--- a/go.mod+++ b/go.mod@@ -150,6 +150,7 @@ require (github.com/olekukonko/tablewriter v0.0.5 // indirectgithub.com/pelletier/go-toml v1.9.5 // indirectgithub.com/pelletier/go-toml/v2 v2.2.2 // indirect+ github.com/pemistahl/lingua-go v1.4.0 // indirectgithub.com/pmezard/go-difflib v1.0.0 // indirectgithub.com/polyfloyd/go-errorlint v1.5.2 // indirectgithub.com/prometheus/common v0.32.1 // indirect@@ -168,6 +169,7 @@ require (github.com/sashamelentyev/usestdlibvars v1.26.0 // indirectgithub.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9 // indirectgithub.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect+ github.com/shopspring/decimal v1.3.1 // indirectgithub.com/sirupsen/logrus v1.9.3 // indirectgithub.com/sivchari/containedctx v1.0.3 // indirectgithub.com/sivchari/tenv v1.7.1 // indirectdiff --git a/go.sum b/go.sumindex cf2eaa545..c565fe631 100644--- a/go.sum+++ b/go.sum@@ -466,6 +466,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3vgithub.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=+github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKBR4rM=+github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=@@ -532,6 +534,8 @@ github.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9 h1:rnO6Zp1YMQgithub.com/securego/gosec/v2 v2.20.1-0.20240525090044-5f0084eb01a9/go.mod h1:dg7lPlu/xK/Ut9SedURCoZbVCR4yC7fM65DtH9/CDHs=github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=diff --git a/migrations/202510211300_add_search_column_to_posts.sql b/migrations/202510211300_add_search_column_to_posts.sqlnew file mode 100644index 000000000..4d4648621--- /dev/null+++ b/migrations/202510211300_add_search_column_to_posts.sql@@ -0,0 +1,54 @@+-- 1. Add language column+ALTER TABLE posts ADD COLUMN language regconfig;++-- 2. Create language mapping function+CREATE OR REPLACE FUNCTION map_language_to_tsvector(lang TEXT)+RETURNS regconfig AS $$+BEGIN+ RETURN CASE lang+ WHEN 'ar' THEN 'arabic'::regconfig+ WHEN 'cs' THEN 'simple'::regconfig -- Czech not supported, fallback to simple+ WHEN 'de' THEN 'german'::regconfig+ WHEN 'el' THEN 'simple'::regconfig -- Greek not supported, fallback to simple+ WHEN 'en' THEN 'english'::regconfig+ WHEN 'es-ES' THEN 'spanish'::regconfig+ WHEN 'fa' THEN 'simple'::regconfig -- Farsi not supported, fallback to simple+ WHEN 'fr' THEN 'french'::regconfig+ WHEN 'it' THEN 'italian'::regconfig+ WHEN 'ja' THEN 'simple'::regconfig -- Japanese not supported, fallback to simple+ WHEN 'ko' THEN 'simple'::regconfig -- Korean not supported, fallback to simple+ WHEN 'nl' THEN 'dutch'::regconfig+ WHEN 'pl' THEN 'simple'::regconfig -- Polish not supported, fallback to simple+ WHEN 'pt-BR' THEN 'portuguese'::regconfig+ WHEN 'ru' THEN 'russian'::regconfig+ WHEN 'sk' THEN 'simple'::regconfig -- Slovak not supported, fallback to simple+ WHEN 'si-LK' THEN 'simple'::regconfig -- Sinhala not supported, fallback to simple+ WHEN 'sv-SE' THEN 'swedish'::regconfig+ WHEN 'tr' THEN 'turkish'::regconfig+ WHEN 'zh-CN' THEN 'simple'::regconfig -- Chinese not supported, fallback to simple+ ELSE 'simple'::regconfig+ END;+END;+$$ LANGUAGE plpgsql IMMUTABLE;++-- 3. Update language column using tenant's language (assuming you have a way to join or fetch it)+UPDATE posts+SET language = map_language_to_tsvector(+ (SELECT language::TEXT FROM tenants WHERE tenants.id = posts.tenant_id)+);++-- 4. Add search as a generated column+ALTER TABLE posts ADD search tsvector GENERATED ALWAYS AS (+ CASE WHEN language <> 'simple'::regconfig THEN+ setweight(to_tsvector(language, title::TEXT), 'A') ||+ setweight(to_tsvector(language, description), 'B') ||+ setweight(to_tsvector('simple'::regconfig, title::TEXT), 'C') ||+ setweight(to_tsvector('simple'::regconfig, description), 'D')::tsvector+ ELSE+ setweight(to_tsvector('simple'::regconfig, title::TEXT), 'A') ||+ setweight(to_tsvector('simple'::regconfig, description), 'B')::tsvector+ END+) STORED;++-- 5. Create GIN index+CREATE INDEX idx_posts_search_gin ON posts USING GIN (search);