Open platform to collect and prioritize feedback
Refactor of languages code
Matt Robertse01ae3943400
diff --git a/app/models/enum/locale.go b/app/models/enum/locale.gonew file mode 100644index 000000000..c938998ae--- /dev/null+++ b/app/models/enum/locale.go@@ -0,0 +1,262 @@+package enum++import (+ "github.com/pemistahl/lingua-go"+)++// Locale represents a supported language/locale in Fider+type Locale struct {+ Code string // Locale code (e.g., "pt-BR", "en")+ Name string // Display name (e.g., "Portuguese (Brazilian)")+ MessageFormatCode string // Culture code for messageformat pluralization (e.g., "pt")+ PostgresConfig string // PostgreSQL text search configuration (e.g., "portuguese", "simple")+ LinguaLanguage lingua.Language // Lingua-go language enum for detection+ IsRTL bool // Right-to-left text direction+}++var (+ // LocaleEnglish represents English+ LocaleEnglish = Locale{+ Code: "en",+ Name: "English",+ MessageFormatCode: "en",+ PostgresConfig: "english",+ LinguaLanguage: lingua.English,+ IsRTL: false,+ }++ // LocalePortugueseBR represents Portuguese (Brazilian)+ LocalePortugueseBR = Locale{+ Code: "pt-BR",+ Name: "Portuguese (Brazilian)",+ MessageFormatCode: "pt",+ PostgresConfig: "portuguese",+ LinguaLanguage: lingua.Portuguese,+ IsRTL: false,+ }++ // LocaleSpanishES represents Spanish+ LocaleSpanishES = Locale{+ Code: "es-ES",+ Name: "Spanish",+ MessageFormatCode: "es",+ PostgresConfig: "spanish",+ LinguaLanguage: lingua.Spanish,+ IsRTL: false,+ }++ // LocaleGerman represents German+ LocaleGerman = Locale{+ Code: "de",+ Name: "German",+ MessageFormatCode: "de",+ PostgresConfig: "german",+ LinguaLanguage: lingua.German,+ IsRTL: false,+ }++ // LocaleFrench represents French+ LocaleFrench = Locale{+ Code: "fr",+ Name: "French",+ MessageFormatCode: "fr",+ PostgresConfig: "french",+ LinguaLanguage: lingua.French,+ IsRTL: false,+ }++ // LocaleSwedishSE represents Swedish+ LocaleSwedishSE = Locale{+ Code: "sv-SE",+ Name: "Swedish",+ MessageFormatCode: "se",+ PostgresConfig: "swedish",+ LinguaLanguage: lingua.Swedish,+ IsRTL: false,+ }++ // LocaleItalian represents Italian+ LocaleItalian = Locale{+ Code: "it",+ Name: "Italian",+ MessageFormatCode: "it",+ PostgresConfig: "italian",+ LinguaLanguage: lingua.Italian,+ IsRTL: false,+ }++ // LocaleJapanese represents Japanese+ LocaleJapanese = Locale{+ Code: "ja",+ Name: "Japanese",+ MessageFormatCode: "ja",+ PostgresConfig: "simple",+ LinguaLanguage: lingua.Japanese,+ IsRTL: false,+ }++ // LocaleDutch represents Dutch+ LocaleDutch = Locale{+ Code: "nl",+ Name: "Dutch",+ MessageFormatCode: "nl",+ PostgresConfig: "dutch",+ LinguaLanguage: lingua.Dutch,+ IsRTL: false,+ }++ // LocalePolish represents Polish+ LocalePolish = Locale{+ Code: "pl",+ Name: "Polish",+ MessageFormatCode: "pl",+ PostgresConfig: "simple",+ LinguaLanguage: lingua.Polish,+ IsRTL: false,+ }++ // LocaleRussian represents Russian+ LocaleRussian = Locale{+ Code: "ru",+ Name: "Russian",+ MessageFormatCode: "ru",+ PostgresConfig: "russian",+ LinguaLanguage: lingua.Russian,+ IsRTL: false,+ }++ // LocaleSlovak represents Slovak+ LocaleSlovak = Locale{+ Code: "sk",+ Name: "Slovak",+ MessageFormatCode: "sk",+ PostgresConfig: "simple",+ LinguaLanguage: lingua.Slovak,+ IsRTL: false,+ }++ // LocaleTurkish represents Turkish+ LocaleTurkish = Locale{+ Code: "tr",+ Name: "Turkish",+ MessageFormatCode: "tr",+ PostgresConfig: "turkish",+ LinguaLanguage: lingua.Turkish,+ IsRTL: false,+ }++ // LocaleGreek represents Greek+ LocaleGreek = Locale{+ Code: "el",+ Name: "Greek",+ MessageFormatCode: "el",+ PostgresConfig: "simple",+ LinguaLanguage: lingua.Greek,+ IsRTL: false,+ }++ // LocaleArabic represents Arabic+ LocaleArabic = Locale{+ Code: "ar",+ Name: "Arabic",+ MessageFormatCode: "ar",+ PostgresConfig: "arabic",+ LinguaLanguage: lingua.Arabic,+ IsRTL: true,+ }++ // LocaleChineseCN represents Chinese (Simplified)+ LocaleChineseCN = Locale{+ Code: "zh-CN",+ Name: "Chinese (Simplified)",+ MessageFormatCode: "zh",+ PostgresConfig: "simple",+ LinguaLanguage: lingua.Chinese,+ IsRTL: false,+ }++ // LocalePersian represents Persian (Farsi)+ LocalePersian = Locale{+ Code: "fa",+ Name: "Persian (پارسی)",+ MessageFormatCode: "fa",+ PostgresConfig: "simple",+ LinguaLanguage: lingua.Persian,+ IsRTL: false,+ }++ // AllLocales contains all supported locales+ AllLocales = []Locale{+ LocaleEnglish,+ LocalePortugueseBR,+ LocaleSpanishES,+ LocaleGerman,+ LocaleFrench,+ LocaleSwedishSE,+ LocaleItalian,+ LocaleJapanese,+ LocaleDutch,+ LocalePolish,+ LocaleRussian,+ LocaleSlovak,+ LocaleTurkish,+ LocaleGreek,+ LocaleArabic,+ LocaleChineseCN,+ LocalePersian,+ }+)++// GetLocaleByCode returns a Locale by its code, or false if not found+func GetLocaleByCode(code string) (Locale, bool) {+ for _, locale := range AllLocales {+ if locale.Code == code {+ return locale, true+ }+ }+ return Locale{}, false+}++// IsValidLocale returns true if the given locale code is valid+func IsValidLocale(code string) bool {+ _, valid := GetLocaleByCode(code)+ return valid+}++// GetLinguaLanguages returns all lingua.Language enums for language detection+func GetLinguaLanguages() []lingua.Language {+ languages := make([]lingua.Language, len(AllLocales))+ for i, locale := range AllLocales {+ languages[i] = locale.LinguaLanguage+ }+ return languages+}++// GetLocaleByLinguaLanguage returns a Locale by its lingua.Language enum+func GetLocaleByLinguaLanguage(lang lingua.Language) (Locale, bool) {+ for _, locale := range AllLocales {+ if locale.LinguaLanguage == lang {+ return locale, true+ }+ }+ return Locale{}, false+}++// MapLocaleToTSConfig maps a locale code to a PostgreSQL text search configuration+// Returns "simple" if the locale is not found or doesn't have native Postgres support+func MapLocaleToTSConfig(code string) string {+ locale, found := GetLocaleByCode(code)+ if !found {+ return "simple"+ }+ return locale.PostgresConfig+}++// GetLocaleCodes returns all locale codes+func GetLocaleCodes() []string {+ codes := make([]string, len(AllLocales))+ for i, locale := range AllLocales {+ codes[i] = locale.Code+ }+ return codes+}diff --git a/app/pkg/i18n/i18n.go b/app/pkg/i18n/i18n.goindex 306f08deb..f8264e566 100644--- a/app/pkg/i18n/i18n.go+++ b/app/pkg/i18n/i18n.go@@ -5,40 +5,15 @@ import ("encoding/json""fmt""os"- "strings""sync""github.com/getfider/fider/app"+ "github.com/getfider/fider/app/models/enum""github.com/getfider/fider/app/pkg/env""github.com/getfider/fider/app/pkg/errors""github.com/gotnospirit/messageformat")-// localeToPlurals maps between Fider locale and gotnospirit/messageformat culture-var localeToPlurals = map[string]string{- "en": "en",- "pt-BR": "pt",- "sv-SE": "se",- "es-ES": "es",- "el": "el",- "nl": "nl",- "de": "de",- "fr": "fr",- "pl": "pl",- "ru": "ru",- "sk": "sk",- "it": "it",- "tr": "tr",- "ja": "ja",- "zh-CN": "zh",- "ar": "ar",- "fa": "fa",-}--var rtlLocales = map[string]bool{- "ar": true,-}-type Params map[string]any// cache for locale parser and file content to prevent excessive disk IO@@ -74,7 +49,12 @@ func getLocaleData(locale string) localeData {panic(errors.Wrap(err, "failed unmarshal to json"))}- parser, err := messageformat.NewWithCulture(localeToPlurals[locale])+ localeInfo, found := enum.GetLocaleByCode(locale)+ if !found {+ panic(errors.New("unknown locale: %s", locale))+ }++ parser, err := messageformat.NewWithCulture(localeInfo.MessageFormatCode)if err != nil {panic(errors.Wrap(err, "failed create parser"))}@@ -106,10 +86,7 @@ func getMessage(locale, key string) (string, *messageformat.Parser) {// IsValidLocale returns true if given locale is validfunc IsValidLocale(locale string) bool {- if _, ok := localeToPlurals[locale]; ok {- return true- }- return false+ return enum.IsValidLocale(locale)}// GetLocale returns the locale defined in context@@ -124,15 +101,12 @@ func GetLocale(ctx context.Context) string {func GetLocaleDirection(ctx context.Context) string {locale := GetLocale(ctx)-- parts := strings.Split(locale, "-")- baseLocale := parts[0]-- // Check if baseLocale is in rtlLocales map- if isRTL, exists := rtlLocales[baseLocale]; exists && isRTL {++ localeInfo, found := enum.GetLocaleByCode(locale)+ if found && localeInfo.IsRTL {return "rtl"}-+return "ltr"}diff --git a/app/services/sqlstore/postgres/common.go b/app/services/sqlstore/postgres/common.goindex 0ac989f2a..f372dddfb 100644--- a/app/services/sqlstore/postgres/common.go+++ b/app/services/sqlstore/postgres/common.go@@ -28,46 +28,11 @@ func SanitizeString(input string) string {return strings.ToValidUTF8(input, "")}-// mapLocaleToTSConfig maps a tenant's locale short key to the corresponding PostgreSQL text search configuration.-// See the /locale folder for supported locales. Returns 'simple' if no match is found.+// MapLocaleToTSConfig maps a tenant's locale short key to the corresponding PostgreSQL text search configuration.+// Returns 'simple' if no match is found or if PostgreSQL doesn't have native support for the language.+// All locale definitions are centralized in app/models/enum/locale.gofunc MapLocaleToTSConfig(locale string) string {- switch locale {- case "ar":- return "arabic"- case "de":- return "german"- case "en":- return "english"- case "es-ES":- return "spanish"- case "fr":- return "french"- case "it":- return "italian"- case "nl":- return "dutch"- case "pt-BR":- return "portuguese"- case "ru":- return "russian"- case "sv-SE":- return "swedish"- case "tr":- return "turkish"- case "cs":- case "el":- case "fa":- case "ja":- case "ko":- case "pl":- case "si-LK":- case "sk":- case "zh-CN":- default:- return "simple"- }-- return "simple"+ return enum.MapLocaleToTSConfig(locale)}func getViewData(query query.SearchPosts) (string, []enum.PostStatus, string) {diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.goindex 1a294bd83..c7ead685e 100644--- a/app/services/sqlstore/postgres/post.go+++ b/app/services/sqlstore/postgres/post.go@@ -340,44 +340,24 @@ func updatePost(ctx context.Context, c *cmd.UpdatePost) error {}// detectPostLanguage uses lingua-go to detect the language of a post and maps it to a PostgreSQL tsvector config or 'simple'.+// All language mappings are centralized in app/models/enum/locale.gofunc detectPostLanguage(title, description string) string {- // Supported lingua languages mapped to locale keys- var linguaLangs = []lingua.Language{- lingua.English, lingua.German, lingua.French, lingua.Italian, lingua.Dutch, lingua.Polish, lingua.Portuguese, lingua.Russian, lingua.Spanish, lingua.Swedish, lingua.Turkish,- lingua.Slovak, lingua.Greek, lingua.Arabic, lingua.Chinese, lingua.Persian, lingua.Japanese,- }+ // Get all supported lingua languages from the centralized locale enum+ linguaLangs := enum.GetLinguaLanguages()detector := lingua.NewLanguageDetectorBuilder().FromLanguages(linguaLangs...).Build()text := strings.TrimSpace(title + " " + description)lang, exists := detector.DetectLanguageOf(text)if !exists {return "simple"}- // Map lingua-go language to locale code- var linguaToLocale = map[lingua.Language]string{- lingua.English: "en",- lingua.German: "de",- lingua.French: "fr",- lingua.Italian: "it",- lingua.Dutch: "nl",- lingua.Polish: "pl",- lingua.Portuguese: "pt-BR",- lingua.Russian: "ru",- lingua.Spanish: "es-ES",- lingua.Swedish: "sv-SE",- lingua.Turkish: "tr",- lingua.Slovak: "sk",- lingua.Greek: "el",- lingua.Arabic: "ar",- lingua.Chinese: "zh-CN",- lingua.Persian: "fa",- lingua.Japanese: "ja",- }- locale, ok := linguaToLocale[lang]++ locale, ok := enum.GetLocaleByLinguaLanguage(lang)if !ok {return "simple"}- // Map to PostgreSQL tsvector config- return MapLocaleToTSConfig(locale)++ // Map to PostgreSQL tsvector config using the locale's PostgresConfig+ return locale.PostgresConfig}func getPostByID(ctx context.Context, q *query.GetPostByID) error {diff --git a/lingui.config.js b/lingui.config.jsindex 58d8808c6..a10bf10b0 100644--- a/lingui.config.js+++ b/lingui.config.js@@ -1,5 +1,8 @@import { formatter } from "@lingui/format-json"+// NOTE: All locale definitions are centralized in app/models/enum/locale.go+// See locale/locales.ts for the complete list of steps when adding a new locale+export default {catalogs: [{diff --git a/locale/locales.ts b/locale/locales.tsindex 3a99eb85b..e1e9b2f7c 100644--- a/locale/locales.ts+++ b/locale/locales.ts@@ -1,3 +1,10 @@+// NOTE: All locale definitions are centralized in app/models/enum/locale.go+// When adding a new locale, update app/models/enum/locale.go first, then:+// 1. Add the locale to this file+// 2. Add it to lingui.config.js+// 3. Add it to public/ssr.tsx+// 4. Generate translation files+interface Locale {text: string}diff --git a/public/ssr.tsx b/public/ssr.tsxindex 539b4c5da..e64e9e499 100644--- a/public/ssr.tsx+++ b/public/ssr.tsx@@ -7,6 +7,8 @@ import { DevBanner, ReadOnlyNotice } from "./components"import { activateI18NSync } from "./services"import { I18nProvider } from "@lingui/react"+// NOTE: All locale definitions are centralized in app/models/enum/locale.go+// See locale/locales.ts for the complete list of steps when adding a new locale// Locale files must be bundled for SSR to work synchronouslyconst messages: { [key: string]: any } = {en: require(`../locale/en/client`),@@ -24,6 +26,8 @@ const messages: { [key: string]: any } = {ru: require(`../locale/ru/client`),sk: require(`../locale/sk/client`),tr: require(`../locale/tr/client`),+ ar: require(`../locale/ar/client`),+ fa: require(`../locale/fa/client`),}// ESBuild doesn't support Dynamic Imports, so we need to map them statically