Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(website): offer every language Weblate reports as translated
Elian Doran5897d741bcb2
diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.ymlindex 8d3bcd4fcb..45c723b8c0 100644--- a/.github/workflows/i18n.yml+++ b/.github/workflows/i18n.yml@@ -7,6 +7,10 @@ on:pull_request:paths:- "apps/client/src/translations/**"+ - "apps/website/src/translations/**"+ - "apps/website/src/locales.ts"+ - "packages/commons/src/lib/i18n.ts"+ - "scripts/translation/**"- ".github/workflows/i18n.yml"permissions:diff --git a/apps/website/src/i18n.spec.ts b/apps/website/src/i18n.spec.tsindex fcc8d790aa..a29638a117 100644--- a/apps/website/src/i18n.spec.ts+++ b/apps/website/src/i18n.spec.ts@@ -1,16 +1,49 @@import { describe, expect, it } from "vitest";-import { extractLocaleFromUrl, mapLocale, swapLocaleInUrl } from "./i18n";+import { extractLocaleFromUrl, LOCALES, mapLocale, swapLocaleInUrl } from "./i18n";describe("mapLocale", () => {it("maps Chinese", () => {expect(mapLocale("zh-TW")).toStrictEqual("zh-Hant");expect(mapLocale("zh-CN")).toStrictEqual("zh-Hans");+ expect(mapLocale("zh-Hant")).toStrictEqual("zh-Hant");+ expect(mapLocale("zh-Hans")).toStrictEqual("zh-Hans");});it("maps languages without countries", () => {expect(mapLocale("ro-RO")).toStrictEqual("ro");expect(mapLocale("ro")).toStrictEqual("ro");});++ it("keeps the region of locales offered per region", () => {+ expect(mapLocale("en-GB")).toStrictEqual("en-GB");+ expect(mapLocale("pt-BR")).toStrictEqual("pt-BR");+ expect(mapLocale("nb-NO")).toStrictEqual("nb-NO");+ });++ it("resolves a bare language to the only region offered for it", () => {+ expect(mapLocale("nb")).toStrictEqual("nb-NO");+ expect(mapLocale("pt")).toStrictEqual("pt-BR");+ });++ it("prefers the exact match over a region sharing the language", () => {+ expect(mapLocale("en")).toStrictEqual("en");+ expect(mapLocale("en-US")).toStrictEqual("en");+ expect(mapLocale("de-AT")).toStrictEqual("de");+ });++ it("falls back to English for languages that are not offered", () => {+ expect(mapLocale("sv")).toStrictEqual("en");+ expect(mapLocale("hu-HU")).toStrictEqual("en");+ expect(mapLocale("")).toStrictEqual("en");+ });++ it("only ever returns a locale the website offers", () => {+ const ids = LOCALES.map(l => l.id);+ const tags = [ "ro-RO", "zh-HK", "pt", "nb", "sv", "en-AU", "ug", "ar-EG" ];+ for (const tag of tags) {+ expect(ids).toContain(mapLocale(tag));+ }+ });});describe("swapLocale", () => {diff --git a/apps/website/src/i18n.ts b/apps/website/src/i18n.tsindex 93931d9365..e8dccbdee7 100644--- a/apps/website/src/i18n.ts+++ b/apps/website/src/i18n.ts@@ -1,16 +1,14 @@import i18next from "i18next";import { initReactI18next } from "react-i18next";-interface Locale {- id: string;- name: string;- rtl?: boolean;-}+import { LOCALES } from "./locales";++export { type Locale, LOCALES } from "./locales";i18next.use(initReactI18next);const localeFiles = import.meta.glob("./translations/*/translation.json", { eager: true });const resources: Record<string, Record<string, Record<string, string>>> = {};-for (const [path, module] of Object.entries(localeFiles)) {+for (const [ path, module ] of Object.entries(localeFiles)) {const id = path.split("/").at(-2);if (!id) continue;@@ -31,32 +29,32 @@ export function initTranslations(lng: string) {});}-export const LOCALES: Locale[] = [- { id: "en", name: "English" },- { id: "ro", name: "Română" },- { id: "zh-Hans", name: "简体中文" },- { id: "zh-Hant", name: "繁體中文" },- { id: "fr", name: "Français" },- { id: "it", name: "Italiano" },- { id: "ja", name: "日本語" },- { id: "pl", name: "Polski" },- { id: "es", name: "Español" },- { id: "ar", name: "اَلْعَرَبِيَّةُ", rtl: true },-].toSorted((a, b) => a.name.localeCompare(b.name));-+/**+ * Resolves a language tag, either a URL segment or `navigator.language`, to a locale+ * from {@link LOCALES}, falling back to `en` for languages the website does not offer.+ */export function mapLocale(locale: string) {- if (!locale) return 'en';+ if (!locale) return "en";const lower = locale.toLowerCase();- if (lower.startsWith('zh')) {- if (lower.includes('tw') || lower.includes('hk') || lower.includes('mo') || lower.includes('hant')) {- return 'zh-Hant';- }- return 'zh-Hans';+ // Chinese is offered by script, which browsers report only through the region.+ if (lower.startsWith("zh")) {+ const traditional = [ "tw", "hk", "mo", "hant" ].some(marker => lower.includes(marker));+ return traditional ? "zh-Hant" : "zh-Hans";}- // Default for everything else- return locale.split('-')[0]; // e.g. "en-US" -> "en"+ const exact = LOCALES.find(l => l.id.toLowerCase() === lower);+ if (exact) return exact.id;++ // "de-AT" falls back to "de". The region-less entry wins over a regional one sharing+ // the language, so "en-US" reaches "en" rather than whichever of "en"/"en-GB" sorts first.+ const language = lower.split("-")[0];+ const regionless = LOCALES.find(l => l.id.toLowerCase() === language);+ if (regionless) return regionless.id;++ // A bare "nb" reaches "nb-NO", the only region offered for it.+ const sameLanguage = LOCALES.find(l => l.id.toLowerCase().split("-")[0] === language);+ return sameLanguage?.id ?? "en";}export function swapLocaleInUrl(url: string, newLocale: string) {@@ -76,7 +74,7 @@ export function swapLocaleInUrl(url: string, newLocale: string) {}export function extractLocaleFromUrl(url: string) {- const localeId = url.split('/')[1];+ const localeId = url.split("/")[1];const correspondingLocale = LOCALES.find(l => l.id === localeId);if (!correspondingLocale) return undefined;return localeId;diff --git a/apps/website/src/locales.ts b/apps/website/src/locales.tsnew file mode 100644index 0000000000..b19bcfc159--- /dev/null+++ b/apps/website/src/locales.ts@@ -0,0 +1,39 @@+export interface Locale {+ id: string;+ name: string;+ rtl?: boolean;+}++/**+ * The languages offered by the language selector, in their own script.+ *+ * `scripts/translation/check-translation-coverage.ts` fails the build when Weblate+ * reports a language above the coverage threshold that is missing here. A locale+ * carrying a region (`pt-BR`, `nb-NO`) also needs `mapLocale()` to resolve it, which+ * it does by matching this list.+ */+export const LOCALES: Locale[] = [+ { id: "ar", name: "اَلْعَرَبِيَّةُ", rtl: true },+ { id: "cs", name: "Čeština" },+ { id: "de", name: "Deutsch" },+ { id: "el", name: "Ελληνικά" },+ { id: "en", name: "English (United States)" },+ { id: "en-GB", name: "English (United Kingdom)" },+ { id: "es", name: "Español" },+ { id: "fr", name: "Français" },+ { id: "ga", name: "Gaeilge" },+ { id: "hi", name: "हिन्दी" },+ { id: "id", name: "Bahasa Indonesia" },+ { id: "it", name: "Italiano" },+ { id: "ja", name: "日本語" },+ { id: "ko", name: "한국어" },+ { id: "nb-NO", name: "Norsk bokmål" },+ { id: "pl", name: "Polski" },+ { id: "pt-BR", name: "Português (Brasil)" },+ { id: "ro", name: "Română" },+ { id: "ru", name: "Русский" },+ { id: "ug", name: "ئۇيغۇرچە", rtl: true },+ { id: "uk", name: "Українська" },+ { id: "zh-Hans", name: "简体中文" },+ { id: "zh-Hant", name: "繁體中文" }+].toSorted((a, b) => a.name.localeCompare(b.name));diff --git a/docs/Developer Guide/Developer Guide/Concepts/Internationalisation Translations/Adding a new locale.md b/docs/Developer Guide/Developer Guide/Concepts/Internationalisation Translations/Adding a new locale.mdindex 88181adb0d..c895702083 100644--- a/docs/Developer Guide/Developer Guide/Concepts/Internationalisation Translations/Adding a new locale.md+++ b/docs/Developer Guide/Developer Guide/Concepts/Internationalisation Translations/Adding a new locale.md@@ -11,4 +11,17 @@ To do so:6. In `apps/client`, look for `widgets/type_widgets/spreadsheet/locales.ts` and modify `UNIVER_LOCALES`, either with a source (if Univer ships a bundle for the language) or with an explicit `null` to fall back to English. Every preset in `SPREADSHEET_PRESET_PACKAGES` must have a bundle for the locale, which `locales.spec.ts` checks.7. Locale mappings for PDF.js might need adjustment. To do so, in `packages/pdfjs-viewer/scripts/build.ts` there is `LOCALE_MAPPINGS`. No entry is needed when the locale's `electronLocale` already names a directory under `packages/pdfjs-viewer/viewer/locale`.-Steps 1 to 6 are keyed by `DISPLAYABLE_LOCALE_IDS`, so `pnpm typecheck` reports each one still missing after step 1.\ No newline at end of file+Steps 1 to 6 are keyed by `DISPLAYABLE_LOCALE_IDS`, so `pnpm typecheck` reports each one still missing after step 1.++## The website++The website tracks its own Weblate component and its own list, so a locale added to the application above is not offered on [triliumnotes.org](https://triliumnotes.org) until it is added there too.++1. In `apps/website`, look for `locales.ts` and add an entry to `LOCALES`, using the language's own name and setting `rtl` where it applies. The id must match the folder under `apps/website/src/translations`, which Weblate creates.+2. A locale carrying a region, such as `pt-BR`, is resolved by `mapLocale()` in `i18n.ts` by matching `LOCALES`, so it needs no further mapping. Chinese is the exception: it is selected by script, which browsers report only through the region, and `mapLocale()` special-cases it.++## The coverage gate++`scripts/translation/check-translation-coverage.ts` fails CI once Weblate reports a language above 50% that is missing from either list. It runs on the `weblate:*` branches, so the pull request that brings the translations in is where it fires.++The gate only catches a language that is translated but not offered. It does not notice one that is offered and has since fallen behind, and it does not check that the locale renders — only that the id is listed.\ No newline at end of filediff --git a/scripts/translation/check-translation-coverage.ts b/scripts/translation/check-translation-coverage.tsindex 0e01040604..ff2d7e588b 100644--- a/scripts/translation/check-translation-coverage.ts+++ b/scripts/translation/check-translation-coverage.ts@@ -1,19 +1,58 @@-import { LOCALES } from "../../packages/commons/src/lib/i18n";-import { getLanguageStats } from "./utils";+import { LOCALES as WEBSITE_LOCALES } from "../../apps/website/src/locales";+import { LOCALES as APP_LOCALES } from "../../packages/commons/src/lib/i18n";+import { getLanguageStats, type WeblateProject } from "./utils";++/**+ * Coverage above which a language must be offered, rather than sitting+ * translated but unreachable.+ */+const MINIMUM_COVERAGE_PERCENT = 50;++interface GatedProject {+ project: WeblateProject;+ /** Names the list a maintainer has to add the locale to, quoted in the failure. */+ listDescription: string;+ localeIds: string[];+}++const GATED_PROJECTS: GatedProject[] = [+ {+ project: "client",+ listDescription: "LOCALES in packages/commons/src/lib/i18n.ts",+ localeIds: APP_LOCALES.map(l => l.id)+ },+ {+ project: "website",+ listDescription: "LOCALES in apps/website/src/locales.ts",+ localeIds: WEBSITE_LOCALES.map(l => l.id)+ }+];async function main() {- const project = "client";- const languageStats = await getLanguageStats(project);- const localesWithCoverage = languageStats.results- .filter(language => language.translated_percent > 50)-- for (const localeData of localesWithCoverage) {- const { language_code: localeId, translated_percent: percentage, language } = localeData;- const locale = LOCALES.find(l => l.id === localeId);- if (!locale) {- console.error(`❌ Language ${language.name} (${localeId}) has a coverage of ${percentage}% in '${project}', but it is not supported by the application.`);- process.exit(1);+ const failures: string[] = [];++ for (const { project, listDescription, localeIds } of GATED_PROJECTS) {+ const languageStats = await getLanguageStats(project);+ for (const localeData of languageStats.results) {+ const localeId = localeData.language_code;+ const percentage = localeData.translated_percent;+ if (percentage <= MINIMUM_COVERAGE_PERCENT || localeIds.includes(localeId)) {+ continue;+ }++ failures.push(+ `❌ ${localeData.language.name} (${localeId}) is ${percentage}% translated `+ + `in '${project}', but is missing from ${listDescription}.`);+ }+ }++ if (failures.length > 0) {+ for (const failure of failures.toSorted((a, b) => a.localeCompare(b))) {+ console.error(failure);}+ console.error(`\n${failures.length} language(s) above `+ + `${MINIMUM_COVERAGE_PERCENT}% are translated but not offered.`);+ process.exit(1);}console.log("✅ Translation coverage check passed.");diff --git a/scripts/translation/utils.ts b/scripts/translation/utils.tsindex 2b2fab50a3..44e7bc81e6 100644--- a/scripts/translation/utils.ts+++ b/scripts/translation/utils.ts@@ -3,10 +3,13 @@ import { join } from "path";const scriptDir = __dirname;+/** The Weblate components under https://hosted.weblate.org/projects/trilium/ that are gated. */+export type WeblateProject = "readme" | "client" | "website";+/** How long a `.language-stats-*.json` cache file is reused before Weblate is queried again. */export const CACHE_MAX_AGE_MS = 24 * 60 * 60 * 1000;-export async function getLanguageStats(project: "readme" | "client") {+export async function getLanguageStats(project: WeblateProject) {const cacheFile = join(scriptDir, `.language-stats-${project}.json`);// Try to read from the cache.diff --git a/scripts/tsconfig.json b/scripts/tsconfig.jsonindex 22100648f5..5430998b69 100644--- a/scripts/tsconfig.json+++ b/scripts/tsconfig.json@@ -15,7 +15,8 @@"include": ["**/*.ts","**/*.mts",- "../packages/commons/src/lib/i18n.ts"+ "../packages/commons/src/lib/i18n.ts",+ "../apps/website/src/locales.ts"],"exclude": ["dist"