The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
(ns i18n.create-artifacts
(:require
[clojure.data.csv :as csv]
[clojure.java.io :as io]
[clojure.pprint :as pprint]
[clojure.string :as str]
[i18n.autofix :as i18n.autofix]
[i18n.common :as i18n]
[i18n.create-artifacts.backend :as backend]
[i18n.create-artifacts.frontend :as frontend]
[i18n.pseudo-locale :as pseudo-locale]
[i18n.validation :as i18n.validation]
[metabuild-common.core :as u]))
(set! *warn-on-reflection* true)
(defn- locales-dot-edn [locales]
{:locales (into (sorted-set "en") locales)
:packages ["metabase"]
:bundle "metabase.Messages"})
(defn- generate-locales-dot-edn! [locales]
(u/step "Create resources/locales.clj"
(let [file (u/filename u/project-root-directory "resources" "locales.clj")]
(u/delete-file-if-exists! file)
(spit file
(with-out-str
(println ";; DO NOT EDIT THIS FILE!")
(println ";; It is autogenerated by `i18n.create-artifacts` as part of the release process.")
(pprint/pprint (locales-dot-edn locales))))
(u/assert-file-exists file))))
(defn- msgids-to-drop
"Set of `msgid`s to exclude from build artifacts, given a seq of violations. If any plural form
of a message has a droppable violation, the entire message is excluded."
[violations]
(into #{}
(comp (filter i18n.validation/drop-from-build?)
(map :msgid))
violations))
(defn- create-artifacts-for-locale!
"Generate frontend + backend artifacts for one `locale`. Parses the `.po` once, runs the autofix
pass, then threads the cleaned contents through the scanner and both builders. Returns the
violations the scanner flagged (post-autofix) for the aggregate report."
[[locale directory]]
(u/step (format "Create artifacts for locale %s" (pr-str locale))
(let [po-contents (i18n.autofix/autofix-po-contents (i18n/po-contents locale directory))
violations (i18n.validation/invalid-messages-in-po locale po-contents)
drop-msgids (msgids-to-drop violations)]
(frontend/create-artifact-for-locale! locale drop-msgids po-contents)
(backend/create-artifact-for-locale! locale drop-msgids po-contents)
(when (seq drop-msgids)
(u/announce "Filtered %d invalid translations from %s" (count drop-msgids) locale))
(u/announce "Artifacts for locale %s created successfully." (pr-str locale))
violations)))
(def ^:private violations-report-directory
(u/filename u/project-root-directory "target"))
(def ^:private violations-csv-columns
["locale" "types" "dropped" "backend" "plural_index" "msgid" "msgid_plural" "msgstr"
"error" "expected_args" "actual_args" "source_refs"])
(defn- violation->row [{:keys [locale types backend? plural-index msgid msgid-plural msgstr
error expected-arg-counts actual-arg-count source-refs]
:as violation}]
(->>
[locale
(str/join "+" (sort (map name types)))
(boolean (i18n.validation/drop-from-build? violation))
(boolean backend?)
plural-index
msgid
msgid-plural
msgstr
error
(some->> expected-arg-counts (str/join ","))
actual-arg-count
(str/join " " source-refs)]
(mapv (fnil str ""))))
(defn- write-violations-report! [violations]
(u/step "Write i18n violations report"
(u/create-directory-unless-exists! violations-report-directory)
(let [csv-file (u/filename violations-report-directory "i18n-violations.csv")
edn-file (u/filename violations-report-directory "i18n-violations.edn")]
(with-open [w (io/writer csv-file)]
(csv/write-csv w (cons violations-csv-columns (map violation->row violations))))
(spit edn-file (with-out-str (pprint/pprint (vec violations))))
(let [by-type (frequencies (mapcat :types violations))]
(u/announce "Wrote %d translation violations to %s" (count violations) csv-file)
(doseq [t [:invalid-message-format :skipped-arg-index :arg-count-mismatch]]
(u/announce " %s: %d" (name t) (get by-type t 0)))))))
(defn- delete-json-files!
"Delete the `.json` files in `directory` but keep the directory itself.
`resources/frontend_client/app/locales` is tracked through a `.gitkeep` so the
frontend build can resolve `locales/<locale>.json` against it, and deleting
the directory takes that file with it."
[directory]
(doseq [^java.io.File file (some-> ^java.io.File (io/file directory) .listFiles)
:when (str/ends-with? (.getName file) ".json")]
(u/delete-file-if-exists! (.getAbsolutePath file))))
(defn- create-artifacts-for-all-locales! [locale->directory]
;; Empty the targets in case some locales were removed
(u/delete-file-if-exists! backend/target-directory)
(delete-json-files! frontend/target-directory)
(let [per-locale-violations (doall (pmap create-artifacts-for-locale! locale->directory))
all-violations (->> per-locale-violations
(apply concat)
(sort-by (juxt :locale (comp str :types) :msgid :plural-index)))]
(write-violations-report! all-violations)))
(defn create-all-artifacts!
"Create backend and frontend i18n artifacts from the catalogs in `source-directory`."
[{:keys [source-directory]}]
(u/step "Create i18n artifacts"
(let [pseudo-directory (u/filename u/project-root-directory "target" "i18n")
locale->directory (assoc (zipmap (i18n/locales source-directory) (repeat source-directory))
pseudo-locale/pseudo-locale-code pseudo-directory)]
(pseudo-locale/generate-pseudo-locale-po! source-directory pseudo-directory)
(generate-locales-dot-edn! (keys locale->directory))
(create-artifacts-for-all-locales! locale->directory)
(u/announce "Translation resources built successfully."))))