The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Move translation files to S3 (#85409)
Chris Vandoodewaard791cba92b744
diff --git a/.github/file-paths.yaml b/.github/file-paths.yamlindex af1b41e7ee3..d114dcd6434 100644--- a/.github/file-paths.yaml+++ b/.github/file-paths.yaml@@ -4,7 +4,6 @@ all_changed:- "**"default: &default- - "locales/**"- "bin/**/!(*.md)"workflows: &workflows@@ -364,6 +363,7 @@ codeql:i18n:- *default+ - "locales/**"- ".github/actions/prepare-frontend/**"- ".github/actions/prepare-backend/**"- ".github/workflows/lint.yml"diff --git a/.github/scripts/file-paths.unit.spec.ts b/.github/scripts/file-paths.unit.spec.tsindex 3fa6305b87a..47893ff5623 100644--- a/.github/scripts/file-paths.unit.spec.ts+++ b/.github/scripts/file-paths.unit.spec.ts@@ -283,7 +283,6 @@ describe("file-paths.yaml", () => {"frontend/build/shared/esbuild/side-effect-free-modules-plugin.js","rspack.main.config.js","bun.lock",- "locales/fr.po","snowplow/events/event.yaml","patches/@cypress+grep+6.0.0.patch",])("runs all E2E specs when %s changes", (file) => {@@ -310,6 +309,16 @@ describe("file-paths.yaml", () => {expect(matches("ci_scripts", file)).toBe(true);});+ it.each(["locales/translations.pin", "locales/fr.po"])(+ "only lints i18n when %s changes",+ (file) => {+ const matched = Object.keys(filters).filter((name) =>+ matches(name, file),+ );+ expect(matched).toEqual(["all_changed", "i18n"]);+ },+ );+it("runs the module cycles test when a cluster is named", () => {expect(matches(diff --git a/.github/workflows/build-scripts.yml b/.github/workflows/build-scripts.ymlindex d616cdefd59..6c6df89f08b 100644--- a/.github/workflows/build-scripts.yml+++ b/.github/workflows/build-scripts.yml@@ -71,6 +71,8 @@ jobs:timeout-minutes: 5uses: ./.github/actions/prepare-backend+ - name: Test translation snapshot scripts+ run: ./bin/i18n/test-translations- name: Run build and release script testsrun: clojure -X:dev:drivers:build:build-dev:build-test:citimeout-minutes: 15diff --git a/.github/workflows/cut-release-branch.yml b/.github/workflows/cut-release-branch.ymlindex 09238333859..5155526a049 100644--- a/.github/workflows/cut-release-branch.yml+++ b/.github/workflows/cut-release-branch.yml@@ -6,6 +6,9 @@ on:jobs:cut:+ permissions:+ contents: read+ id-token: writeruns-on: ${{ vars.DEFAULT_RUNNER_KEY }}timeout-minutes: 5steps:@@ -15,6 +18,7 @@ jobs:sparse-checkout: |release.github+ localestoken: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}- name: Prepare release scripts@@ -56,6 +60,12 @@ jobs:core.setOutput('nextMilestone', String(next + 1));core.setOutput('needsRequirementsUpdate', String(needsRequirementsUpdate));+ - name: Configure AWS credentials+ uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1+ with:+ role-to-assume: ${{ secrets.CI_INFRA_IAM_ROLE }}+ aws-region: ${{ vars.AWS_REGION }}+# Pushing the new branch triggers set-milestone.yml, which does the# full milestone backfill and then dispatches release-notes-preview# to upload the markdown to #writing.@@ -76,7 +86,11 @@ jobs:source .github/actions/create-backport/kondo-ratchets.shwrite_disabled_ratchets .clj-kondo/ratchets.ednwrite_disabled_ratchets .clj-kondo/ratchets-test.edn- git add --sparse .clj-kondo/ratchets.edn .clj-kondo/ratchets-test.edn+ PIN=$(cat locales/translations.pin)+ RELEASE_PIN="v${{ steps.preflight.outputs.next }}/${PIN#*/}"+ aws s3 cp "s3://metabase-ci-infra/translations/$PIN.tar.gz" "s3://metabase-ci-infra/translations/$RELEASE_PIN.tar.gz"+ echo "$RELEASE_PIN" > locales/translations.pin+ git add --sparse .clj-kondo/ratchets.edn .clj-kondo/ratchets-test.edn locales/translations.pingit commit --allow-empty --no-verify -m "Cut $BRANCH branch"git push -u origin "$BRANCH"echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"diff --git a/.github/workflows/translation-update.yml b/.github/workflows/translation-update.ymlindex d1ed490c268..9094df00890 100644--- a/.github/workflows/translation-update.yml+++ b/.github/workflows/translation-update.yml@@ -6,8 +6,13 @@ on:permissions:contents: write+ id-token: writepull-requests: write+concurrency:+ group: translation-update-${{ github.ref }}+ cancel-in-progress: false+jobs:synchronize-translations:if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }}@@ -33,14 +38,26 @@ jobs:- name: install gettextrun: sudo apt-get install gettext+ - name: Configure AWS credentials+ uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1+ with:+ role-to-assume: ${{ secrets.CI_INFRA_IAM_ROLE }}+ aws-region: ${{ vars.AWS_REGION }}++ - name: Prepare translation snapshot+ run: |+ checkout=$(./bin/i18n/prepare-translation-update)+ cp locales/crowdin.yml "$checkout/locales/crowdin.yml"+ echo "METABASE_I18N_CHECKOUT=$checkout" >> "$GITHUB_ENV"+- name: Crowdin downloaduses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2.16.2id: crowdin-downloadwith:- config: locales/crowdin.yml+ config: ${{ env.METABASE_I18N_CHECKOUT }}/locales/crowdin.ymlcommand: downloadcommand_args: |- --config locales/crowdin.yml+ --config ${{ env.METABASE_I18N_CHECKOUT }}/locales/crowdin.yml-e=el-e=et-e=es-MX@@ -62,15 +79,13 @@ jobs:DATE=$(date +'%Y-%m-%d')- ./bin/i18n/update-translation-template-- git add locales/metabase.po- git commit -m "Update translation sources $DATE" --no-verify+ ./bin/i18n/update-translation-template "$METABASE_I18N_CHECKOUT/locales"- ./bin/i18n/build-translation-resources+ ./bin/i18n/build-translation-resources "$METABASE_I18N_CHECKOUT/locales"+ ./bin/i18n/publish-translations "$METABASE_I18N_CHECKOUT"- git add locales/- git commit -a -m "Update translations $Date" --allow-empty --no-verify+ git add locales/translations.pin resources/locales.clj+ git commit -m "Update translations $DATE" --allow-empty --no-verifygit push -u origin "$BRANCH_NAME"- name: Upload i18n violations report@@ -91,9 +106,9 @@ jobs:CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_TOKEN }}with:- config: locales/crowdin.yml+ config: ${{ env.METABASE_I18N_CHECKOUT }}/locales/crowdin.ymlcommand: upload- command_args: --config locales/crowdin.yml+ command_args: --config ${{ env.METABASE_I18N_CHECKOUT }}/locales/crowdin.yml- name: Open PRid: create_prdiff --git a/.gitignore b/.gitignoreindex c54094b6737..c9171f8cbc4 100644--- a/.gitignore+++ b/.gitignore@@ -69,9 +69,8 @@ cypress.env.json/lein-plugins/*/target/local+/locales/metabase-i18n//locales/metabase-*.pot-# Pseudo-locale generated at build time-/locales/en-ZZ.po/logs/modules/drivers/*/resources/namespaces.edn/modules/drivers/*/targetdiff --git a/bin/build/src/build.clj b/bin/build/src/build.cljindex a0bad0fd727..d8f7036d29a 100644--- a/bin/build/src/build.clj+++ b/bin/build/src/build.clj@@ -10,7 +10,6 @@[clojure.tools.build.api :as b][environ.core :as env][flatland.ordered.map :as ordered-map]- [i18n.create-artifacts :as i18n][metabuild-common.core :as u]))(set! *warn-on-reflection* true)@@ -84,7 +83,7 @@:version (fn [{:keys [edition version]}](version-properties/generate-version-properties-file! edition version)):translations (fn [_]- (i18n/create-all-artifacts!))+ (u/sh {:dir u/project-root-directory} "bin/i18n/build-translation-resources")):frontend (fn [{:keys [edition]}](build-frontend! edition)):licenses (fn [{:keys [edition]}]diff --git a/bin/build/src/i18n/common.clj b/bin/build/src/i18n/common.cljindex 80d6e4dd05f..4f5a0cb9e88 100644--- a/bin/build/src/i18n/common.clj+++ b/bin/build/src/i18n/common.clj@@ -9,43 +9,39 @@(set! *warn-on-reflection* true)(defn locales- "Set of all locales for which we have i18n bundles.+ "Set of all locales with a catalog in `directory`.- (locales) ; -> #{\"nl\" \"pt\" \"zh\" \"tr\" \"it\" \"fa\" ...}"- []+ (locales directory) ; -> #{\"nl\" \"pt\" \"zh\" \"tr\" \"it\" \"fa\" ...}"+ [directory](into(sorted-set)- (for [^java.io.File file (.listFiles (io/file (u/filename u/project-root-directory "locales")))+ (for [^java.io.File file (.listFiles (io/file directory)):let [file-name (.getName file)]:when (and (str/ends-with? file-name ".po")(not (str/starts-with? file-name "metabase")))](str/replace file-name #"\.po$" ""))))(defn locale-source-po-filename- "E.g.-- (locale-source-po-filename \"fr\")- ;; =>- \"/home/cam/metabase/locales/fr.po\""- [locale]- (u/filename u/project-root-directory "locales" (format "%s.po" locale)))+ "Path to the catalog for `locale` in `directory`."+ [locale directory]+ (str (io/file directory (format "%s.po" locale))));; see https://github.com/zanata/jgettext/tree/master/src/main/java/org/fedorahosted/tennera/jgettext-(defn- catalog ^Catalog [locale]+(defn- catalog ^Catalog [locale directory](let [parser (PoParser.)]- (.parseCatalog parser (io/file (locale-source-po-filename locale)))))+ (.parseCatalog parser (io/file (locale-source-po-filename locale directory)))))-(defn- po-headers [locale]- (when-let [^Message message (.locateHeader (catalog locale))]+(defn- po-headers [locale directory]+ (when-let [^Message message (.locateHeader (catalog locale directory))](let [header-fields (HeaderFields/wrap (.getMsgstr message))](into {} (for [^String k (.getKeys header-fields)][k (if (= k "Language")locale(.getValue header-fields k))])))))-(defn- po-messages-seq [locale]- (for [^Message message (iterator-seq (.iterator (catalog locale)))+(defn- po-messages-seq [locale directory]+ (for [^Message message (iterator-seq (.iterator (catalog locale directory)));; remove any empty translations:when (not (str/blank? (.getMsgid message)))]{:id (.getMsgid message)@@ -61,9 +57,9 @@(defn po-contents"Contents of the PO file for a `locale`."- [locale]- {:headers (po-headers locale)- :messages (po-messages-seq locale)})+ [locale directory]+ {:headers (po-headers locale directory)+ :messages (po-messages-seq locale directory)})(defn backend-message?"True if `message` (the shape returned by `po-messages-seq`) originates from a Clojure source filediff --git a/bin/build/src/i18n/create_artifacts.clj b/bin/build/src/i18n/create_artifacts.cljindex 4c037b8eb15..1d776d7feda 100644--- a/bin/build/src/i18n/create_artifacts.clj+++ b/bin/build/src/i18n/create_artifacts.clj@@ -14,12 +14,12 @@(set! *warn-on-reflection* true)-(defn- locales-dot-edn []- {:locales (conj (i18n/locales) "en")+(defn- locales-dot-edn [locales]+ {:locales (into (sorted-set "en") locales):packages ["metabase"]:bundle "metabase.Messages"})-(defn- generate-locales-dot-edn! []+(defn- generate-locales-dot-edn! [locales](u/step "Create resources/locales.clj"(let [file (u/filename u/project-root-directory "resources" "locales.clj")](u/delete-file-if-exists! file)@@ -27,7 +27,7 @@(with-out-str(println ";; DO NOT EDIT THIS FILE!")(println ";; It is autogenerated by `i18n.create-artifacts` as part of the release process.")- (pprint/pprint (locales-dot-edn))))+ (pprint/pprint (locales-dot-edn locales))))(u/assert-file-exists file))))(defn- msgids-to-drop@@ -43,9 +43,9 @@"Generate frontend + backend artifacts for one `locale`. Parses the `.po` once, runs the autofixpass, then threads the cleaned contents through the scanner and both builders. Returns theviolations the scanner flagged (post-autofix) for the aggregate report."- [locale]+ [[locale directory]](u/step (format "Create artifacts for locale %s" (pr-str locale))- (let [po-contents (i18n.autofix/autofix-po-contents (i18n/po-contents locale))+ (let [po-contents (i18n.autofix/autofix-po-contents (i18n/po-contents locale directory))violations (i18n.validation/invalid-messages-in-po locale po-contents)drop-msgids (msgids-to-drop violations)](frontend/create-artifact-for-locale! locale drop-msgids po-contents)@@ -103,25 +103,24 @@:when (str/ends-with? (.getName file) ".json")](u/delete-file-if-exists! (.getAbsolutePath file))))-(defn- create-artifacts-for-all-locales! []+(defn- create-artifacts-for-all-locales! [locale->directory];; Empty the targets in case some locales were removed(u/delete-file-if-exists! backend/target-directory)(delete-json-files! frontend/target-directory)- (let [per-locale-violations (doall (pmap create-artifacts-for-locale! (i18n/locales)))+ (let [per-locale-violations (doall (pmap create-artifacts-for-locale! locale->directory))all-violations (->> per-locale-violations(apply concat)(sort-by (juxt :locale (comp str :types) :msgid :plural-index)))](write-violations-report! all-violations)))(defn create-all-artifacts!- "Create backend and frontend i18n artifacts."- ([]- (create-all-artifacts! nil))-- ([_options]- (u/step "Create i18n artifacts"- ;; Generate the `en-ZZ` pseudo-locale .po file- (pseudo-locale/generate-pseudo-locale-po!)- (generate-locales-dot-edn!)- (create-artifacts-for-all-locales!)- (u/announce "Translation resources built successfully."))))+ "Create backend and frontend i18n artifacts from the catalogs in `source-directory`."+ [{:keys [source-directory]}]+ (u/step "Create i18n artifacts"+ (let [pseudo-directory (u/filename u/project-root-directory "target" "i18n")+ locale->directory (assoc (zipmap (i18n/locales source-directory) (repeat source-directory))+ pseudo-locale/pseudo-locale-code pseudo-directory)]+ (pseudo-locale/generate-pseudo-locale-po! source-directory pseudo-directory)+ (generate-locales-dot-edn! (keys locale->directory))+ (create-artifacts-for-all-locales! locale->directory)+ (u/announce "Translation resources built successfully."))))diff --git a/bin/build/src/i18n/create_artifacts/backend.clj b/bin/build/src/i18n/create_artifacts/backend.cljindex 832628edcf4..a0164b549f8 100644--- a/bin/build/src/i18n/create_artifacts/backend.clj+++ b/bin/build/src/i18n/create_artifacts/backend.clj@@ -60,7 +60,7 @@rather than re-parsed here so scanner and writer stay in sync."[locale drop-msgids po-contents](let [target-file (target-filename locale)]- (u/step (format "Create backend artifact %s from %s" target-file (i18n/locale-source-po-filename locale))+ (u/step (format "Create backend artifact %s" target-file)(u/create-directory-unless-exists! target-directory)(u/delete-file-if-exists! target-file)(write-edn-file! po-contents drop-msgids target-file)diff --git a/bin/build/src/i18n/create_artifacts/frontend.clj b/bin/build/src/i18n/create_artifacts/frontend.cljindex 6c7045c3e2f..8e08b83cfa6 100644--- a/bin/build/src/i18n/create_artifacts/frontend.clj+++ b/bin/build/src/i18n/create_artifacts/frontend.clj@@ -78,7 +78,7 @@rather than re-parsed here so scanner and writer stay in sync."[locale drop-msgids po-contents](let [target-file (target-filename locale)]- (u/step (format "Create frontend artifact %s from %s" target-file (i18n/locale-source-po-filename locale))+ (u/step (format "Create frontend artifact %s" target-file)(u/create-directory-unless-exists! target-directory)(u/delete-file-if-exists! target-file)(u/step "Write JSON"diff --git a/bin/build/src/i18n/pseudo_locale.clj b/bin/build/src/i18n/pseudo_locale.cljindex eeb6d47678e..2bb3fe4b6d2 100644--- a/bin/build/src/i18n/pseudo_locale.clj+++ b/bin/build/src/i18n/pseudo_locale.clj@@ -2,13 +2,13 @@"Generates the `en-ZZ` pseudo-locale `.po` file used by E2E tests.This file is the build-time companion to the pseudo-locale strategy described in UXW-3460. Every English source- string from `locales/metabase.po` is transformed into `[zz] {English}`, and the resulting `locales/en-ZZ.po` is- picked up by the normal i18n build pipeline (same as every other real locale).+ string from the pinned `metabase.po` is transformed into `[zz] {English}`, and the resulting `target/i18n/en-ZZ.po`+ is picked up by the normal i18n build pipeline (same as every other real locale).E2E tests can then switch the UI to `en-ZZ` and assert on the predictable `[zz] ...` prefix, without depending onfragile real-language strings that Crowdin may update at any time.- The file is `.gitignore`d — it is regenerated on every build from the current `locales/metabase.po`, so it never+ The file is never committed — it is regenerated on every build from the pinned `metabase.po`, so it neverdrifts. Crowdin does not know about `en-ZZ` (it's not in `locales/crowdin.yml`) so the translation-update workflowleaves it alone.@@ -80,28 +80,26 @@(.addExtractedComment dst comment))dst))-(defn- source-catalog ^Catalog []+(defn- source-catalog ^Catalog [source-directory](let [parser (PoParser.)- source-file (io/file (u/filename u/project-root-directory "locales" "metabase.po"))]+ source-file (io/file source-directory "metabase.po")](.parseCatalog parser source-file)))-(defn- target-file ^java.io.File []- (io/file (u/filename u/project-root-directory "locales" (str pseudo-locale-code ".po"))))-(defn generate-pseudo-locale-po!- "Read `locales/metabase.po` and write `locales/en-ZZ.po` with every msgstr prefixed with `[zz] `. Idempotent: running- this repeatedly produces the same output (modulo jgettext's internal ordering)."- []+ "Read `metabase.po` from `source-directory` and write `en-ZZ.po` to `target-directory` with every msgstr prefixed+ with `[zz] `. Idempotent: running this repeatedly produces the same output (modulo jgettext's internal ordering)."+ [source-directory target-directory](u/step (format "Generate pseudo-locale %s.po" pseudo-locale-code)- (let [src (source-catalog)+ (let [src (source-catalog source-directory)dst (Catalog.)- target (target-file)+ target (io/file target-directory (str pseudo-locale-code ".po"))msg-count (volatile! 0)](.addMessage dst (build-pseudo-header))(doseq [^Message msg (iterator-seq (.iterator src)):when (not (.isHeader msg))](.addMessage dst (build-pseudo-message msg))(vswap! msg-count inc))+ (u/create-directory-unless-exists! (str target-directory))(u/delete-file-if-exists! (.getAbsolutePath target))(.write (PoWriter.) dst target)(u/announce "Wrote %d pseudo-locale messages to %s." @msg-count (.getAbsolutePath target)))))diff --git a/bin/build/test/i18n/common_test.clj b/bin/build/test/i18n/common_test.cljnew file mode 100644index 00000000000..24cbc1e7e6b--- /dev/null+++ b/bin/build/test/i18n/common_test.clj@@ -0,0 +1,41 @@+(ns i18n.common-test+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [i18n.common :as common]+ [i18n.pseudo-locale :as pseudo-locale]+ [metabuild-common.core :as u]))++(set! *warn-on-reflection* true)++(defn- with-catalogs [f]+ (let [^java.io.File directory (u/temporary-file "i18n-test" "")]+ (io/delete-file directory)+ (.mkdir directory)+ (try+ (doseq [locale ["metabase" "fr"]]+ (spit (io/file directory (str locale ".po"))+ (str "msgid \"\"\nmsgstr \"\"\n"+ "\"Content-Type: text/plain; charset=UTF-8\\n\"\n"+ "\"Language: fr\\n\"\n\n"+ "msgid \"Hello\"\nmsgstr \"Bonjour\"\n")))+ (f (str directory))+ (finally+ (doseq [file (reverse (file-seq directory))]+ (io/delete-file file true))))))++(deftest ^:parallel explicit-catalog-directory-test+ (with-catalogs+ (fn [directory]+ (is (= #{"fr"} (common/locales directory)))+ (is (= "Bonjour" (-> (common/po-contents "fr" directory) :messages first :str)))+ (is (= (str (io/file directory "fr.po"))+ (common/locale-source-po-filename "fr" directory))))))++(deftest pseudo-locale-test+ (with-catalogs+ (fn [directory]+ (let [target (str (io/file directory "pseudo"))]+ (pseudo-locale/generate-pseudo-locale-po! directory target)+ (is (= #{"en-ZZ"} (common/locales target)))+ (is (= "[zz] Hello" (-> (common/po-contents "en-ZZ" target) :messages first :str)))))))diff --git a/bin/i18n/README.md b/bin/i18n/README.mdindex ebb7df5e368..ec66b7474c4 100644--- a/bin/i18n/README.md+++ b/bin/i18n/README.md@@ -4,8 +4,8 @@ Metabase's internationalization pipeline has two halves that the scripts in thisorchestrate:1. **[Extraction](#extraction)** — scan Clojure/TypeScript source for `trs` / `tru` / ttag `t`- tagged-template call sites, emit a gettext source template at `locales/metabase.po`.-2. **[Artifact generation](#artifact-generation)** — read back translated `locales/*.po` files,+ tagged-template call sites, emit a gettext source template.+2. **[Artifact generation](#artifact-generation)** — read back the pinned translated `*.po` files,apply [autofixes](#autofix-pass) for unambiguously repairable translator mistakes, validatethe rest, and emit the runtime artifacts at `resources/i18n/*.edn` (backend) and`resources/frontend_client/app/locales/*.json` (frontend). Translations the scanner still@@ -15,19 +15,64 @@ Between the two, [Crowdin](https://crowdin.com/project/metabase-i18n) is where tthe `msgstr` values. The [`.github/workflows/translation-update.yml`](../../.github/workflows/translation-update.yml)workflow runs weekly to pull translations down and open a PR.+## Pinned catalogs++`locales/translations.pin` pins one snapshot of the catalogs, as `<version>/<commit>`. For+example, `v66/516af13bc7e…`. Master uses `latest/<commit>`, and each release branch uses+`vNN/<commit>`. `<commit>` is the Metabase commit that published the snapshot.++Snapshots are `.tar.gz` files in `s3://metabase-ci-infra/translations/<version>/<commit>.tar.gz`.+Builds download them through CloudFront at+`https://d3rmfk6uwjyg5s.cloudfront.net/translations/<version>/<commit>.tar.gz`, which needs no+credentials. A snapshot never changes after upload, so a commit always builds with the same+translations.++The catalogs are downloaded into `locales/metabase-i18n/<version>-<commit>/`, which Git ignores.+Only the current pin is kept: a new pin replaces the old download.++When a release branch is cut, `cut-release-branch.yml` copies the master snapshot to+`vNN/<commit>.tar.gz` and pins it on the new branch.++## Build++Run `./bin/i18n/build-translation-resources` to download the pinned catalogs and build the+frontend JSON and backend EDN resources. `./bin/build.sh` runs it. Frontend builds do not:+without the resources, the app shows English. The build also generates the `en-ZZ`+pseudo-locale into `target/i18n/en-ZZ.po`.++## Publish++The Crowdin workflow publishes updates. To publish by hand, you need AWS credentials that can+write to `s3://metabase-ci-infra/translations/`. Run from the project root:++```sh+checkout=$(./bin/i18n/prepare-translation-update)+./bin/i18n/update-translation-template "$checkout/locales"+./bin/i18n/build-translation-resources "$checkout/locales"+./bin/i18n/publish-translations "$checkout"+rm -rf "$checkout"+```++`publish-translations` uploads a new snapshot named after the current `HEAD` and updates the pin.+It does not overwrite a snapshot that exists, so publish from a new commit each time.+Commit the pin and `resources/locales.clj`.++To merge the catalogs of another branch, run `./bin/i18n/merge-translations <branch> [-o]`.+To test the scripts, run `./bin/i18n/test-translations`.+```mermaidflowchart LRsubgraph Extract ["Phase 1: Extraction"]direction LRSource["source<br/>(.clj/.cljc/.ts/.tsx)"]- Pot[(locales/metabase.po)]+ Pot[(External metabase.po)]Source -->|update-translation-template| Potendsubgraph Sync ["Phase 2: Crowdin sync"]direction LRCrowdin{{Crowdin}}- LocalePo[(locales/*.po)]+ LocalePo[(Pinned S3 snapshot)]Crowdin -->|download| LocalePoend@@ -50,7 +95,7 @@ flowchart LR## Extraction-_Source code → `locales/metabase.po`_+_Source code → external `metabase.po`_### Building the backend pot file@@ -147,9 +192,9 @@ More information in the [grasp issue](https://github.com/borkdude/grasp/issues/2## Artifact generation-_`locales/*.po` → `resources/i18n/*.edn` + `resources/frontend_client/app/locales/*.json`_+_Pinned `*.po` files → `resources/i18n/*.edn` + `resources/frontend_client/app/locales/*.json`_-Once translators have filled in `locales/<locale>.po` through Crowdin, the build pipeline turns+Once translators have filled in the external `<locale>.po` files through Crowdin, the build pipeline turnsthose into the runtime artifacts that Metabase actually ships:- **Backend**: `resources/i18n/<locale>.edn` — consumed by `metabase.util.i18n.impl/translate` via@@ -171,7 +216,7 @@ This calls `clojure -X:build:build/i18n`, which invokestranslator mistakes that can be unambiguously repaired (currently: unescaped apostrophes forbackend strings).3. Run the scanner `i18n.validation/invalid-messages-in-po` on the post-autofix contents to produce- a seq of violation maps for every `msgstr` that *still* fails validation — see+ a seq of violation maps for every `msgstr` that _still_ fails validation — see[violations report](#violations-report) below.4. Derive a drop-set of msgids using `i18n.validation/drop-from-build?` (see[drop policy](#drop-policy)) and pass it, along with the post-autofix `po-contents`, to both@@ -191,7 +236,7 @@ The source of truth for this pipeline lives in `bin/build/src/i18n/`:### Autofix pass-Runs once per locale, *before* validation, on the parsed `po-contents`. Its job is to silently+Runs once per locale, _before_ validation, on the parsed `po-contents`. Its job is to silentlycorrect translator mistakes that can be unambiguously repaired — making translations that wouldotherwise fail the scanner ship correctly. Both the scanner and the artifact writers consume thepost-autofix contents, so they agree on what's being shipped.@@ -204,7 +249,7 @@ ttag has different escaping rules):instead of escape delimiters. Apostrophes that are already doubled, or that sit next to `{` /`}` (i.e. intentional MessageFormat escapes like `'{0}'`), pass through untouched.-This is handled *before* the scanner, so translations that only had apostrophe typos never reach+This is handled _before_ the scanner, so translations that only had apostrophe typos never reachthe violations report. If you ever see `d'aquí`-style entries in the report, it means autofixcouldn't reach them (likely a non-backend context) and you'd need to fix them in Crowdin.@@ -237,20 +282,20 @@ per-type counts and a direct link to the artifact.### Columns-| Column | Meaning |-|-----------------|---------|-| `locale` | The locale the violation was found in (e.g. `fr`, `de`, `pt-BR`, `ar-SA`). |-| `types` | The set of violation types for this row, joined with `+` (e.g. `skipped-arg-index+arg-count-mismatch`). See [violation types](#violation-types) below. |+| Column | Meaning |+| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `locale` | The locale the violation was found in (e.g. `fr`, `de`, `pt-BR`, `ar-SA`). |+| `types` | The set of violation types for this row, joined with `+` (e.g. `skipped-arg-index+arg-count-mismatch`). See [violation types](#violation-types) below. || `dropped` | `true` if this translation was **excluded** from the built `.edn`/`.json` artifact (runtime will fall back to English). `false` if it was kept in the artifact despite imperfection (runtime renders with the issue visible). |-| `backend` | `true` if the msgid originates from Clojure source (`.clj` / `.cljc`), `false` if from frontend source (`.ts` / `.tsx` / `.js` / `.jsx`). This determines which format-system rules were applied. |-| `plural_index` | For singular (non-plural) strings this is empty. For plural strings it's the index of the affected form (`0`, `1`, …) within the locale's `msgstr[N]` array. |-| `msgid` | The English singular source string — the key translators see in Crowdin. |-| `msgid_plural` | The English plural form, if the string has one. Empty for non-plural strings. |-| `msgstr` | The translated string that failed validation — the actual text the translator produced. |-| `error` | Populated only for `invalid-message-format` rows — the `java.text.MessageFormat` exception message (e.g. `Unmatched braces in the pattern.`). Empty for other types. |-| `expected_args` | For `arg-count-mismatch` rows: the acceptable placeholder count(s), derived from `msgid` and `msgid_plural`. Comma-separated (e.g. `0,1` for a plural where singular has 0 placeholders and plural has 1). |-| `actual_args` | For `arg-count-mismatch` rows: the number of distinct placeholder indices actually used in `msgstr`. |-| `source_refs` | Space-separated `path:line` pointers back to the source code where this msgid is used. |+| `backend` | `true` if the msgid originates from Clojure source (`.clj` / `.cljc`), `false` if from frontend source (`.ts` / `.tsx` / `.js` / `.jsx`). This determines which format-system rules were applied. |+| `plural_index` | For singular (non-plural) strings this is empty. For plural strings it's the index of the affected form (`0`, `1`, …) within the locale's `msgstr[N]` array. |+| `msgid` | The English singular source string — the key translators see in Crowdin. |+| `msgid_plural` | The English plural form, if the string has one. Empty for non-plural strings. |+| `msgstr` | The translated string that failed validation — the actual text the translator produced. |+| `error` | Populated only for `invalid-message-format` rows — the `java.text.MessageFormat` exception message (e.g. `Unmatched braces in the pattern.`). Empty for other types. |+| `expected_args` | For `arg-count-mismatch` rows: the acceptable placeholder count(s), derived from `msgid` and `msgid_plural`. Comma-separated (e.g. `0,1` for a plural where singular has 0 placeholders and plural has 1). |+| `actual_args` | For `arg-count-mismatch` rows: the number of distinct placeholder indices actually used in `msgstr`. |+| `source_refs` | Space-separated `path:line` pointers back to the source code where this msgid is used. |### Violation types@@ -322,8 +367,8 @@ tells you which language to open.### Runtime behavior summary-| Violation type | Build-time action | Runtime behavior |-|--------------------------|------------------------|------------------|-| `invalid-message-format` | Dropped from artifact | English fallback |-| `skipped-arg-index` | Kept in artifact | Localized text with literal `{N}` for missing indices |-| `arg-count-mismatch` | Kept in artifact | Localized text with literal `{N}` for missing placeholders, or extra args silently dropped |+| Violation type | Build-time action | Runtime behavior |+| ------------------------ | --------------------- | ------------------------------------------------------------------------------------------ |+| `invalid-message-format` | Dropped from artifact | English fallback |+| `skipped-arg-index` | Kept in artifact | Localized text with literal `{N}` for missing indices |+| `arg-count-mismatch` | Kept in artifact | Localized text with literal `{N}` for missing placeholders, or extra args silently dropped |diff --git a/bin/i18n/build-translation-resources b/bin/i18n/build-translation-resourcesindex 664f19903c6..23c98a41bfa 100755--- a/bin/i18n/build-translation-resources+++ b/bin/i18n/build-translation-resources@@ -2,11 +2,19 @@set -euo pipefail+if [ -n "${1:-}" ]; then+ source_directory=$(cd "$1" && pwd)+fi+# switch to project root directory if we're not already therescript_directory=`dirname "${BASH_SOURCE[0]}"`cd "$script_directory/../.."+if [ -z "${source_directory:-}" ]; then+ source_directory=$(./bin/i18n/fetch-translations)+fi+source "./bin/check-clojure-cli.sh"check_clojure_cli-clojure -X:build:build/i18n+clojure -X:build:build/i18n :source-directory "\"$source_directory\""diff --git a/bin/i18n/common.sh b/bin/i18n/common.shnew file mode 100644index 00000000000..e3779bbc645--- /dev/null+++ b/bin/i18n/common.sh@@ -0,0 +1,28 @@+#!/usr/bin/env bash++I18N_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)+I18N_CACHE="$I18N_ROOT/locales/metabase-i18n"+I18N_PIN="$I18N_ROOT/locales/translations.pin"+I18N_URL="${METABASE_I18N_URL:-https://d3rmfk6uwjyg5s.cloudfront.net/translations}"+I18N_BUCKET="${METABASE_I18N_BUCKET:-metabase-ci-infra}"+I18N_PREFIX="${METABASE_I18N_PREFIX:-translations}"++i18n_pin() {+ local pin="${1:-$(cat "$I18N_PIN")}"+ if [[ ! "$pin" =~ ^(latest|v[0-9]+)/[0-9a-f]{40}$ ]]; then+ echo "Expected <version>/<commit> in $I18N_PIN, got: $pin" >&2+ return 1+ fi+ printf '%s\n' "$pin"+}++i18n_snapshot() {+ printf '%s\n' "$I18N_CACHE/${1/\//-}"+}++i18n_download() {+ local pin="$1" destination="$2"+ mkdir -p "$destination"+ curl -fsSL "$I18N_URL/$pin.tar.gz" | tar -xz -C "$destination"+ test -f "$destination/metabase.po"+}diff --git a/bin/i18n/fetch-translations b/bin/i18n/fetch-translationsnew file mode 100755index 00000000000..f7a3f2b40e1--- /dev/null+++ b/bin/i18n/fetch-translations@@ -0,0 +1,22 @@+#!/usr/bin/env bash++set -euo pipefail+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"++pin=$(i18n_pin "${1:-}")+destination=$(i18n_snapshot "$pin")++if [ ! -d "$destination" ]; then+ mkdir -p "$I18N_CACHE"+ workspace=$(mktemp -d "$I18N_CACHE/fetch.XXXXXX")+ trap 'rm -rf "$workspace"' EXIT+ i18n_download "$pin" "$workspace/locales"+ chmod a-w "$workspace/locales/"*.po+ for snapshot in "$I18N_CACHE"/*; do+ if [[ "$(basename "$snapshot")" =~ ^(latest|v[0-9]+)-[0-9a-f]{40}$ ]]; then+ rm -rf "$snapshot"+ fi+ done+ mv "$workspace/locales" "$destination"+fi+printf '%s\n' "$destination"diff --git a/bin/i18n/merge-translations b/bin/i18n/merge-translationsindex a38e7cae01d..4f7cdb8aa28 100755--- a/bin/i18n/merge-translations+++ b/bin/i18n/merge-translations@@ -12,58 +12,48 @@# Example usage:# ./bin/i18n/merge-translations <branch-name> [-o]#-# Run this script from the root directory since it assumes that .po files exist-# in a locales/ directory.--otherBranch="$1"-prioritizeOtherTranslations=false--shift--usage() {- echo "Usage: $0 <branch> [-o]"- echo " <branch> Specify the branch to merge translations from."- echo " -o Prioritize translations from the other branch, in case of conflicts."- exit 1-}--while getopts "o" opt; do- case ${opt} in- o)- prioritizeOtherTranslations=true- ;;- *)- usage- ;;- esac-done--# Ensure the other branch argument is provided-if [[ -z "$otherBranch" ]]; then- usage+# The script reads the catalogs pinned in locales/translations.pin on both+# branches, publishes the merged catalogs, and updates the pin on this branch.++set -euo pipefail+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"+cd "$I18N_ROOT"++other_branch="${1:?Usage: merge-translations <branch> [-o]}"+prioritize_other=false+case "${2:-}" in+ -o) prioritize_other=true ;;+ "") ;;+ *) echo "Usage: $0 <branch> [-o]" >&2; exit 1 ;;+esac+git rev-parse --verify "$other_branch^{commit}" > /dev/null++checkout=$(./bin/i18n/prepare-translation-update)+trap 'if [ "$?" = 0 ]; then rm -rf "$checkout"; else echo "Update retained at $checkout" >&2; fi' EXIT+source_directory="$checkout/locales"++if other_pin=$(git show "$other_branch:locales/translations.pin" 2>/dev/null); then+ other_directory="$checkout/other"+ i18n_download "$(i18n_pin "$other_pin")" "$other_directory"+else+ other_directory="$checkout/legacy"+ mkdir "$other_directory"+ git archive "$other_branch:locales" -- '*.po' | tar -x -C "$other_directory"fi-remove_unnecessary_wrapping() {- local input=""- while IFS= read -r line; do- input+="$line"$'EscapedNewlineInMergeScript'- done-- echo -E "$input" | sed 's/\(msgid\|msgstr\) ""EscapedNewlineInMergeScript"/\1 "/g' | sed 's/EscapedNewlineInMergeScript/\n/g'-}--for po in locales/*.po; do- set -f-- echo "Merging translations from $otherBranch:$po into $po"-- if [ "$prioritizeOtherTranslations" = true ]; then- msgcat --no-wrap --use-first <(git show "$otherBranch:$po") "$po" | remove_unnecessary_wrapping > "$po.tmp"+for other_po in "$other_directory/"*.po; do+ po="$source_directory/$(basename "$other_po")"+ if [ ! -f "$po" ]; then+ cp "$other_po" "$po"+ chmod u+w "$po"+ elif [ "$prioritize_other" = true ]; then+ msgcat --no-wrap --use-first "$other_po" "$po" > "$po.tmp"+ mv "$po.tmp" "$po"else- msgcat --no-wrap --use-first "$po" <(git show "$otherBranch:$po") | remove_unnecessary_wrapping > "$po.tmp"+ msgcat --no-wrap --use-first "$po" "$other_po" > "$po.tmp"+ mv "$po.tmp" "$po"fi-- # Outputting directly to $po while reading it causes problems, so we output- # to a temporary file, then replace the .po file with it- mv "$po.tmp" "$po"done++./bin/i18n/build-translation-resources "$source_directory"+./bin/i18n/publish-translations "$checkout"diff --git a/bin/i18n/prepare-translation-update b/bin/i18n/prepare-translation-updatenew file mode 100755index 00000000000..0ef243ca181--- /dev/null+++ b/bin/i18n/prepare-translation-update@@ -0,0 +1,13 @@+#!/usr/bin/env bash++set -euo pipefail+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"+cd "$I18N_ROOT"++pin=$(i18n_pin)+snapshot=$(./bin/i18n/fetch-translations "$pin")+checkout=$(mktemp -d "locales/metabase-i18n/update.XXXXXX")+cp -R "$snapshot" "$checkout/locales"+chmod -R u+w "$checkout/locales"+printf '%s\n' "$pin" > "$checkout/base"+printf '%s\n' "$checkout"diff --git a/bin/i18n/publish-translations b/bin/i18n/publish-translationsnew file mode 100755index 00000000000..9354a5317a3--- /dev/null+++ b/bin/i18n/publish-translations@@ -0,0 +1,39 @@+#!/usr/bin/env bash++set -euo pipefail+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"++checkout="${1:?Usage: publish-translations <checkout>}"+base=$(cat "$checkout/base")+if [ "$(i18n_pin)" != "$base" ]; then+ echo "The translation pin changed while this update was being prepared. Start a new update." >&2+ exit 1+fi++catalogs() {+ (cd "$1" && ls -- *.po)+}++unchanged() {+ [ "$(catalogs "$1")" = "$(catalogs "$2")" ] || return 1+ for po in "$1/"*.po; do+ cmp -s "$po" "$2/$(basename "$po")" || return 1+ done+}++if unchanged "$("$I18N_ROOT/bin/i18n/fetch-translations" "$base")" "$checkout/locales"; then+ echo "Translations are unchanged." >&2+ exit 0+fi++pin="${base%%/*}/$(git -C "$I18N_ROOT" rev-parse HEAD)"+tarball="$checkout/translations.tar.gz"+(cd "$checkout/locales" && COPYFILE_DISABLE=1 tar -czf "$tarball" -- *.po)+aws s3api put-object \+ --bucket "$I18N_BUCKET" \+ --key "$I18N_PREFIX/$pin.tar.gz" \+ --body "$tarball" \+ --content-type application/gzip \+ --if-none-match '*' > /dev/null+printf '%s\n' "$pin" > "$I18N_PIN"+printf '%s\n' "$pin"diff --git a/bin/i18n/test-translations b/bin/i18n/test-translationsnew file mode 100755index 00000000000..9ca2fafd9d9--- /dev/null+++ b/bin/i18n/test-translations@@ -0,0 +1,100 @@+#!/usr/bin/env bash++set -euo pipefail++scripts=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)+temp=$(cd "$(mktemp -d "${TMPDIR:-/tmp}/i18n-test.XXXXXX")" && pwd -P)+trap 'chmod -R u+w "$temp"; rm -rf "$temp"' EXIT++export GIT_AUTHOR_NAME=Test [email protected]+export GIT_COMMITTER_NAME=Test [email protected]++fail() {+ echo "FAIL: $*" >&2+ exit 1+}++snapshot() {+ local pin="$1" content="$2"+ local catalogs="$root/catalogs-$content"+ mkdir -p "$catalogs" "$remote/$(dirname "$pin")"+ echo original > "$catalogs/metabase.po"+ echo "$content" > "$catalogs/fr.po"+ (cd "$catalogs" && COPYFILE_DISABLE=1 tar -czf "$remote/$pin.tar.gz" -- *.po)+}++setup() {+ root="$temp/$1"+ project="$root/metabase"+ remote="$root/remote"+ pin="$project/locales/translations.pin"+ first="latest/$(printf '1%.0s' {1..40})"+ second="latest/$(printf '2%.0s' {1..40})"+ export METABASE_I18N_URL="file://$remote"+ mkdir -p "$project/locales" "$project/bin" "$project/stubs"+ cp -R "$scripts" "$project/bin/i18n"+ printf '#!/usr/bin/env bash\nexit 1\n' > "$project/stubs/aws"+ chmod +x "$project/stubs/aws"+ export PATH="$project/stubs:$PATH"+ snapshot "$first" original+ snapshot "$second" newer+ echo "$first" > "$pin"+ cd "$project"+ git init --quiet+ git commit --quiet --allow-empty -m initial+}++test_invalid_pins_fail() {+ setup invalid+ for bad in ../escape latest/abc "v66/$(printf '0%.0s' {1..40})"; do+ echo "$bad" > "$pin"+ ! bin/i18n/fetch-translations > /dev/null 2>&1 || fail "accepted $bad"+ done+ [ -z "$(ls locales/metabase-i18n 2>/dev/null)" ] || fail "cached a bad pin"+}++test_cache_reuse() {+ setup offline+ catalog=$(bin/i18n/fetch-translations)+ [ "$catalog" = "$project/locales/metabase-i18n/${first/\//-}" ] || fail "unexpected catalog $catalog"+ [ "$(cat "$catalog/fr.po")" = original ] || fail "wrong snapshot"+ METABASE_I18N_URL="file://$root/unavailable" bin/i18n/fetch-translations > /dev/null || fail "cache not reused"+}++test_pin_change_replaces_snapshot() {+ setup snapshots+ first_catalog=$(bin/i18n/fetch-translations)+ echo "$second" > "$pin"+ second_catalog=$(bin/i18n/fetch-translations)+ [ ! -e "$first_catalog" ] || fail "old snapshot kept"+ [ "$(cat "$second_catalog/fr.po")" = newer ] || fail "second snapshot wrong"+}++test_failed_upload_keeps_pin() {+ setup failed-upload+ checkout=$(bin/i18n/prepare-translation-update)+ echo updated > "$checkout/locales/fr.po"+ ! bin/i18n/publish-translations "$checkout" > /dev/null 2>&1 || fail "publish succeeded"+ [ "$(cat "$pin")" = "$first" ] || fail "pin advanced"+}++test_unchanged_skips_upload() {+ setup unchanged+ checkout=$(bin/i18n/prepare-translation-update)+ bin/i18n/publish-translations "$checkout" > /dev/null 2>&1 || fail "publish failed"+ [ "$(cat "$pin")" = "$first" ] || fail "pin changed"+}++test_stale_update_is_rejected() {+ setup stale+ checkout=$(bin/i18n/prepare-translation-update)+ echo updated > "$checkout/locales/fr.po"+ echo "$second" > "$pin"+ ! bin/i18n/publish-translations "$checkout" > /dev/null 2>&1 || fail "stale update published"+ [ "$(cat "$pin")" = "$second" ] || fail "pin changed"+}++for test in $(declare -F | awk '$3 ~ /^test_/ { print $3 }'); do+ ( "$test" ) || exit 1+ echo "ok $test"+donediff --git a/bin/i18n/update-translation-template b/bin/i18n/update-translation-templateindex e1a148d4f31..ec071ced5b0 100755--- a/bin/i18n/update-translation-template+++ b/bin/i18n/update-translation-template@@ -2,6 +2,10 @@set -euo pipefail+if [ -n "${1:-}" ]; then+ output_directory=$(cd "$1" && pwd)+fi+# gettext installed via homebrew is "keg-only", add it to the PATHif [ -d "/usr/local/opt/gettext/bin" ]; thenexport PATH="/usr/local/opt/gettext/bin:$PATH"@@ -20,7 +24,10 @@ cd "$script_directory/../.."source "./bin/check-clojure-cli.sh"check_clojure_cli-POT_NAME="locales/metabase.po"+output_directory="${output_directory:-$PWD/target/i18n}"+mkdir -p "$output_directory"++POT_NAME="$output_directory/metabase.po"POT_BACKEND_NAME="locales/metabase-backend.pot"# NOTE: hardcoded in .babelrcPOT_FRONTEND_NAME="locales/metabase-frontend.pot"diff --git a/locales/translations.pin b/locales/translations.pinnew file mode 100644index 00000000000..93d888740e5--- /dev/null+++ b/locales/translations.pin@@ -0,0 +1 @@+latest/511f5da32a2d65e5b09cc305bacdc997580b752e