Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Web page: a translation mechanism, and a language choice in About > Settings

Niko1221d5cca319c962
diff --git a/serve/server.py b/serve/server.py
index 76b5a2c79..ad0e10a90 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -65,6 +65,7 @@ from serve.frontend import (THINK_END, ChatTemplate, Event, OutputParser, anthro
tool_choice_of, unmark_think_literals)
from serve.mcp import McpCancelled, hub_from_config # noqa: E402
from serve import runconfig # noqa: E402
+from serve.webfiles import subtree_file as web_subtree_file # noqa: E402
from strata_control import engine_component # noqa: E402
from strata_control import tls # noqa: E402
@@ -6387,6 +6388,21 @@ def make_handler(svc: Service):
self.end_headers()
self.wfile.write(body)
return
+ sub = web_subtree_file(path)
+ if path.startswith(("/web/locales/", "/web/vendor/")):
+ # the page's translations (serve/web/locales/<lang>.json) and the vendored KaTeX (serve/web/vendor):
+ # only whitelisted names and types, never a path out of those folders
+ if sub is None:
+ self._json(404, {"error": {"message": "not found"}})
+ return
+ body = sub[0].read_bytes()
+ self.send_response(200)
+ self.send_header("Content-Type", sub[1])
+ self.send_header("Cache-Control", "no-cache")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+ return
if path.startswith("/web/"):
# the web app's own files (serve/web): styles, script, icon sprite - same origin, no CDN
name = path[len("/web/"):]
diff --git a/serve/test_web_i18n.js b/serve/test_web_i18n.js
new file mode 100644
index 000000000..a712f6e7a
--- /dev/null
+++ b/serve/test_web_i18n.js
@@ -0,0 +1,92 @@
+// The web page's translation mechanism (serve/web/i18n.js), run in Node with a small stub of the DOM and fetch.
+// node serve/test_web_i18n.js (serve/test_web_i18n.py runs it too and skips when node is missing)
+"use strict";
+const assert = require("node:assert/strict");
+const path = require("node:path");
+
+// a page element that holds data-i18n marks and keeps its attributes
+class El {
+ constructor(attrs, text = "") { this.attrs = {...attrs}; this.textContent = text; this.dataset = {}; }
+ getAttribute(k) { return k in this.attrs ? this.attrs[k] : null; }
+ setAttribute(k, v) { this.attrs[k] = String(v); }
+}
+const marks = [
+ new El({"data-i18n": "a.text"}, "Rewind to here"),
+ new El({"data-i18n-title": "a.title", title: "Light / dark"}),
+ new El({"data-i18n-label": "a.label", "aria-label": "Send"}),
+ new El({"data-i18n-placeholder": "a.hint", placeholder: "Ask anything…"}),
+];
+globalThis.document = {
+ documentElement: {lang: "en"},
+ querySelectorAll: (sel) => marks.filter((e) => sel.slice(1, -1) in e.attrs),
+};
+const catalogs = {
+ de: {"a.text": "Hierher zurückspulen", "a.title": "Hell / Dunkel", "chat.hidden": {one: "{n} Nachricht ausgeblendet", other: "{n} Nachrichten ausgeblendet"}},
+ ru: {"chat.hidden": {one: "{n} сообщение скрыто", few: "{n} сообщения скрыто", many: "{n} сообщений скрыто", other: "{n} сообщения скрыто"}},
+ ja: {"chat.hidden": {other: "{n} 件のメッセージを非表示"}},
+};
+let fetched = [];
+globalThis.fetch = async (url) => {
+ fetched.push(url);
+ const m = /web\/locales\/([\w-]+)\.json$/.exec(url);
+ if (m && catalogs[m[1]]) return {ok: true, json: async () => catalogs[m[1]]};
+ if (m && m[1] === "zh") return {ok: true, json: async () => ["not", "an", "object"]};
+ if (m && m[1] === "es") throw new Error("offline");
+ return {ok: false, json: async () => { throw new Error("404"); }};
+};
+const i18n = require(path.join(__dirname, "web", "i18n.js"));
+const {tr} = i18n;
+const plural = {one: "{n} message hidden", other: "{n} messages hidden"};
+
+(async () => {
+ // English: the text in the code, with its {placeholders}; a key with no text shows the key rather than nothing
+ assert.equal(tr("x.y", "Hello {name}", {name: "Ada"}), "Hello Ada");
+ assert.equal(tr("x.y", "Hello {name}"), "Hello {name}", "an unknown placeholder stays visible");
+ assert.equal(tr("chat.hidden", plural, {n: 1}), "1 message hidden");
+ assert.equal(tr("chat.hidden", plural, {n: 3}), "3 messages hidden");
+ assert.equal(tr("x.y", undefined), "x.y");
+ assert.equal(globalThis.tr, tr, "tr is global for app.js");
+ i18n.apply();
+ assert.equal(marks[0].textContent, "Rewind to here", "English leaves the marked text alone");
+
+ // a catalog replaces the text; a key it does not have falls back to the English text in the code
+ assert.equal(await i18n.setLanguage("de"), "de");
+ assert.equal(document.documentElement.lang, "de");
+ assert.equal(marks[0].textContent, "Hierher zurückspulen");
+ assert.equal(marks[1].attrs.title, "Hell / Dunkel");
+ assert.equal(marks[2].attrs["aria-label"], "Send", "no German text for this key: English stays");
+ assert.equal(tr("chat.hidden", plural, {n: 1}), "1 Nachricht ausgeblendet");
+ assert.equal(tr("chat.hidden", plural, {n: 2}), "2 Nachrichten ausgeblendet");
+ assert.equal(tr("only.english", "Only English"), "Only English");
+
+ // plural forms of the language: Russian has few and many, Japanese only other
+ await i18n.setLanguage("ru");
+ assert.deepEqual([1, 2, 5, 21, 22].map((n) => tr("chat.hidden", plural, {n})),
+ ["1 сообщение скрыто", "2 сообщения скрыто", "5 сообщений скрыто", "21 сообщение скрыто", "22 сообщения скрыто"]);
+ await i18n.setLanguage("ja");
+ assert.equal(tr("chat.hidden", plural, {n: 1}), "1 件のメッセージを非表示");
+
+ // switching back restores the English text that was in the page
+ await i18n.setLanguage("de");
+ await i18n.setLanguage("en");
+ assert.equal(marks[0].textContent, "Rewind to here");
+ assert.equal(marks[1].attrs.title, "Light / dark");
+ assert.equal(marks[3].attrs.placeholder, "Ask anything…");
+ assert.equal(document.documentElement.lang, "en");
+
+ // a language that is not offered, a catalog that is not an object, one that cannot be fetched, a missing file:
+ // all stay English, and never throw
+ fetched = [];
+ assert.equal(await i18n.setLanguage("xx"), "en");
+ assert.deepEqual(fetched, [], "an unknown code is not even requested");
+ assert.equal(await i18n.setLanguage("zh"), "en");
+ assert.equal(await i18n.setLanguage("es"), "en");
+ assert.equal(await i18n.setLanguage("fr"), "en", "French is not in LANGUAGES yet");
+ assert.equal(await i18n.setLanguage("ko"), "en", "the file does not exist in this test");
+ assert.equal(tr("chat.hidden", plural, {n: 3}), "3 messages hidden");
+
+ // the list: English first, every entry [code, own name]
+ assert.equal(i18n.LANGUAGES[0][0], "en");
+ for (const [code, name] of i18n.LANGUAGES) assert.ok(/^[a-z]{2,3}(-[A-Z]{2})?$/.test(code) && name, code);
+ console.log("ok web i18n");
+})().catch((e) => { console.error(e); process.exitCode = 1; });
diff --git a/serve/test_web_i18n.py b/serve/test_web_i18n.py
new file mode 100644
index 000000000..528533739
--- /dev/null
+++ b/serve/test_web_i18n.py
@@ -0,0 +1,108 @@
+"""The web page's translations: the mechanism (node, serve/test_web_i18n.js), the files the server hands out for it
+(serve/web/locales, serve/web/vendor), and the catalogs against the text in the code (tools/web_i18n.py)."""
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+import unittest
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tools"))
+
+import web_i18n # noqa: E402
+from serve.frontend import ChatTemplate # noqa: E402
+from serve.server import ByteTokenizer, MockEngine, Service, serve # noqa: E402
+from serve.webfiles import subtree_file # noqa: E402
+
+
+def run_node(script: str, *args: str) -> None:
+ result = subprocess.run([shutil.which("node"), str(Path(__file__).with_name(script)), *args],
+ capture_output=True, text=True, timeout=60)
+ if result.returncode:
+ raise AssertionError((result.stdout + result.stderr).strip())
+
+
[email protected](shutil.which("node"), "node is not installed")
+class Mechanism(unittest.TestCase):
+ def test_tr_apply_and_set_language(self):
+ run_node("test_web_i18n.js")
+
+
+class WhatTheServerHandsOut(unittest.TestCase):
+ """/web/locales and /web/vendor: whitelisted names and types, nothing outside those folders."""
+
+ def test_names_and_types(self):
+ web = Path(__file__).parent / "web"
+ locales = web / "locales"
+ if (locales / "de.json").is_file():
+ self.assertEqual(subtree_file("/web/locales/de.json")[1], "application/json; charset=utf-8")
+ for bad in ("/web/locales/de.js", "/web/locales/DE.json", "/web/locales/../app.js", "/web/locales/x/y.json",
+ "/web/locales/%2e%2e%2fapp.json", "/web/locales/de.json%00.js", "/web/locales/missing.json",
+ "/web/locales/C:\\x.json", "/web/vendor/katex/../../app.js", "/web/vendor/katex/dist/katex.min.js.map",
+ "/web/vendor/other/dist/katex.min.js", "/web/vendor/katex/dist/fonts/x.ttf", "/web/vendor/katex/dist/"):
+ with self.subTest(bad=bad):
+ self.assertIsNone(subtree_file(bad))
+
+ def test_a_link_out_of_the_folder_is_refused(self):
+ import tempfile
+ with tempfile.TemporaryDirectory() as d:
+ web = Path(d) / "web"
+ (web / "vendor" / "katex").mkdir(parents=True)
+ (web / "locales").mkdir()
+ (web / "secret.js").write_text("secret")
+ (web / "vendor" / "katex" / "ok.js").write_text("ok")
+ self.assertEqual(subtree_file("/web/vendor/katex/ok.js", web)[1], "text/javascript; charset=utf-8")
+ try:
+ (web / "vendor" / "katex" / "link.js").symlink_to(web / "secret.js")
+ except (OSError, NotImplementedError):
+ self.skipTest("symlinks need a privilege here")
+ self.assertIsNone(subtree_file("/web/vendor/katex/link.js", web))
+
+
+class Served(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ tok = ByteTokenizer()
+ cls.svc = Service(MockEngine(tok, "ok"), tok, ChatTemplate(ROOT / "serve" / "chat_template.jinja"))
+ cls.httpd = serve(cls.svc, port=0)
+ cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.httpd.shutdown()
+ cls.httpd.server_close()
+
+ def get(self, path):
+ try:
+ with urllib.request.urlopen(self.base + path, timeout=10) as r:
+ return r.status, r.headers.get("Content-Type", ""), r.read()
+ except urllib.error.HTTPError as e:
+ return e.code, e.headers.get("Content-Type", ""), e.read()
+
+ def test_i18n_script_and_refusals(self):
+ code, ctype, body = self.get("/web/i18n.js")
+ self.assertEqual(code, 200)
+ self.assertIn(b"StrataI18n", body)
+ for path in ("/web/locales/../server.py", "/web/locales/nope.json", "/web/vendor/katex/../../app.js",
+ "/web/vendor/katex/dist/missing.js", "/web/locales/de.json%00"):
+ with self.subTest(path=path):
+ self.assertEqual(self.get(path)[0], 404)
+
+
[email protected]((Path(__file__).parent / "web" / "locales" / "en.json").is_file(), "no catalogs yet")
+class Catalogs(unittest.TestCase):
+ """Every language in serve/web/i18n.js has a catalog with exactly the keys and {placeholders} of the code."""
+
+ def test_the_text_in_the_code_is_plain_literals(self):
+ self.assertTrue(web_i18n.english())
+
+ def test_catalogs_are_complete(self):
+ self.assertEqual(web_i18n.check(), [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/serve/web/app.js b/serve/web/app.js
index ac84f85dc..b3448d92b 100644
--- a/serve/web/app.js
+++ b/serve/web/app.js
@@ -1096,14 +1096,35 @@ $("drawer-close").onclick = () => openDrawer(false);
$("scrim").onclick = () => openDrawer(false);
document.addEventListener("keydown", (e) => { if (e.key === "Escape" && $("drawer").dataset.open === "true") openDrawer(false); });
+// ------------------------------------------------------------------ language (serve/web/i18n.js)
+// The user's own choice, kept in this browser: English until they pick another. /?lang=de shows one for this visit only.
+const urlLang = new URLSearchParams(location.search).get("lang");
+$("lang").innerHTML = StrataI18n.LANGUAGES.map(([code, name]) => `<option value="${esc(code)}">${esc(name)}</option>`).join("");
+function showLanguage() {
+ $("lang").value = StrataI18n.lang;
+ renderChat();
+ setBusy(!!busy);
+ if (lastMetrics) render(lastMetrics);
+}
+$("lang").onchange = async () => {
+ store.set("lang", $("lang").value);
+ await StrataI18n.setLanguage($("lang").value);
+ showLanguage();
+};
+
// ------------------------------------------------------------------ start
-setBusy(false);
-renderChat();
-const startQuestion = new URLSearchParams(location.search).get("q"); // /?q=... starts a chat (a shortcut)
-if (startQuestion) history.replaceState(null, "", location.pathname + location.hash);
-loadHealth().then(loadMcp).then(() => { if (startQuestion) { $("input").value = startQuestion; send(); } });
-showTab(location.hash.slice(1) || "chat");
-poll();
+async function start() {
+ await StrataI18n.setLanguage(urlLang || store.get("lang", "en"));
+ $("lang").value = StrataI18n.lang;
+ setBusy(false);
+ renderChat();
+ const startQuestion = new URLSearchParams(location.search).get("q"); // /?q=... starts a chat (a shortcut)
+ if (startQuestion) history.replaceState(null, "", location.pathname + location.hash);
+ loadHealth().then(loadMcp).then(() => { if (startQuestion) { $("input").value = startQuestion; send(); } });
+ showTab(location.hash.slice(1) || "chat");
+ poll();
+}
+start();
// ------------------------------------------------------------------ images (POST /v1/images/generations)
$("img-form").onsubmit = async (ev) => {
diff --git a/serve/web/i18n.js b/serve/web/i18n.js
new file mode 100644
index 000000000..81defdbb1
--- /dev/null
+++ b/serve/web/i18n.js
@@ -0,0 +1,77 @@
+// serve/web/i18n.js - the web page's translations.
+//
+// The English text sits in the code, next to the key: tr("chat.rewind", "Rewind to here") and
+// tr("chat.hidden", {one: "{n} message hidden", other: "{n} messages hidden"}, {n: 3}). Static page text is marked
+// data-i18n="key" (also data-i18n-title, data-i18n-label, data-i18n-placeholder) and holds its English text in the HTML.
+// So the page reads fine with no catalog at all. Every other language is a catalog, serve/web/locales/<lang>.json
+// (a flat {"key": "text"} object; a plural is {"one": "...", "other": "..."}, the CLDR names of the language).
+// serve/test_web_i18n.py keeps the catalogs complete: same keys as the English text in the code, same {placeholders}.
+//
+// The language is the user's choice (About > Settings), kept in this browser. Nothing is chosen for them from the
+// browser's language: a page that is translated in part would mix two languages.
+"use strict";
+(function (root) {
+ // [code, the language's own name]. A new language is a locales/<code>.json file plus one line here.
+ const LANGUAGES = [["en", "English"], ["de", "Deutsch"], ["es", "Español"], ["ja", "日本語"], ["ko", "한국어"],
+ ["pt", "Português"], ["ru", "Русский"], ["zh", "中文"]];
+ let lang = "en", catalog = null, rules = null;
+
+ const fill = (s, vars) => String(s).replace(/\{(\w+)\}/g, (m, k) => (vars && k in vars ? String(vars[k]) : m));
+ function pick(entry, vars) {
+ if (entry && typeof entry === "object") {
+ const n = vars && vars.n != null ? Number(vars.n) : 1;
+ let cat = "other";
+ try { cat = (rules || new Intl.PluralRules(lang === "en" ? "en" : lang)).select(n); } catch (e) { /* other */ }
+ return entry[cat] != null ? entry[cat] : entry.other != null ? entry.other : entry.one;
+ }
+ return entry;
+ }
+ // tr(key, english, vars): the catalog's text for the chosen language, else the English text in the code
+ function tr(key, english, vars) {
+ const own = catalog && Object.prototype.hasOwnProperty.call(catalog, key) ? catalog[key] : undefined;
+ const text = pick(own != null ? own : english, vars);
+ return fill(text == null ? key : text, vars);
+ }
+
+ // marked page text: the English text is read once from the HTML and kept, so switching back restores it
+ const ATTRS = [["data-i18n", null], ["data-i18n-title", "title"], ["data-i18n-label", "aria-label"],
+ ["data-i18n-placeholder", "placeholder"]];
+ function apply(scope) {
+ const doc = scope || root.document;
+ if (!doc || !doc.querySelectorAll) return;
+ for (const [mark, attr] of ATTRS) {
+ for (const el of doc.querySelectorAll(`[${mark}]`)) {
+ const key = el.getAttribute(mark);
+ const slot = "en_" + mark.replace(/[^a-z]/g, "");
+ if (el.dataset[slot] === undefined) el.dataset[slot] = attr ? (el.getAttribute(attr) || "") : el.textContent;
+ const text = tr(key, el.dataset[slot]);
+ if (attr) el.setAttribute(attr, text); else el.textContent = text;
+ }
+ }
+ if (root.document && root.document.documentElement) root.document.documentElement.lang = lang;
+ }
+
+ const known = (code) => LANGUAGES.some((l) => l[0] === code);
+ // load a catalog; English, or a catalog that cannot be read, leaves the English text in the code
+ async function setLanguage(code) {
+ code = known(code) ? code : "en";
+ let next = null;
+ if (code !== "en") {
+ try {
+ const r = await root.fetch(`web/locales/${code}.json`);
+ const j = r.ok ? await r.json() : null;
+ if (j && typeof j === "object" && !Array.isArray(j)) next = j;
+ } catch (e) { /* the English text stays */ }
+ if (!next) code = "en";
+ }
+ lang = code; catalog = next; rules = null;
+ try { rules = new Intl.PluralRules(lang); } catch (e) { /* other */ }
+ apply();
+ return lang;
+ }
+
+ const api = {LANGUAGES, tr, apply, setLanguage, get lang() { return lang; }, fill, pick};
+ root.StrataI18n = api;
+ root.tr = tr;
+ if (typeof module !== "undefined" && module.exports) module.exports = api;
+})(typeof globalThis !== "undefined" ? globalThis : this);
diff --git a/serve/web/index.html b/serve/web/index.html
index 0c7847f22..0fdb4e6ed 100644
--- a/serve/web/index.html
+++ b/serve/web/index.html
@@ -192,6 +192,11 @@
<label class="st-label" for="api-key">API key <output>only if the server was started with one</output></label>
<input class="st-input" id="api-key" type="password" autocomplete="off" placeholder="Not needed">
</div>
+ <div class="st-field">
+ <label class="st-label" for="lang"><span data-i18n="settings.language">Language</span></label>
+ <select class="st-input" id="lang"></select>
+ <span class="muted small" data-i18n="settings.languageNote">Chat, its settings and the buttons above are translated. The Monitor, Images and About views are still in English.</span>
+ </div>
<div class="toggle-row"><span>Dark theme</span>
<button class="st-toggle" id="dark-toggle" role="switch" aria-checked="false" aria-label="Dark theme"></button></div>
<p class="muted small">Chats, settings and the key are kept in this browser only.
@@ -262,6 +267,7 @@
</aside>
<div class="scrim" id="scrim" hidden></div>
<div class="st-toasts" id="toasts" aria-live="polite"></div>
+<script src="web/i18n.js"></script>
<script src="web/app.js"></script>
</body>
</html>
diff --git a/serve/webfiles.py b/serve/webfiles.py
new file mode 100644
index 000000000..d79b107d4
--- /dev/null
+++ b/serve/webfiles.py
@@ -0,0 +1,35 @@
+"""Which files of serve/web the server hands out below /web/: the page's translations and the vendored KaTeX.
+
+The flat files (app.js, app.css, ...) are served by serve/server.py itself. The two folders here hold more than one
+level, so each has its own whitelist: a name pattern and the types it may have. Nothing outside these folders is ever
+read, a name with a backslash, a colon or a percent sign is refused, and the resolved path must stay inside the folder.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+WEB = Path(__file__).resolve().parent / "web"
+
+# /web/locales/<lang>.json e.g. de.json, pt-BR.json
+LOCALE = re.compile(r"^/web/locales/(?P<name>[a-z]{2,3}(?:-[A-Z]{2})?\.json)$")
+# /web/vendor/katex/dist/... : the script, the style sheet and the fonts only
+VENDOR_TYPES = {".js": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".woff2": "font/woff2"}
+VENDOR_PREFIX = "/web/vendor/katex/"
+
+
+def subtree_file(path: str, web: Path = WEB):
+ """(file, content type) for a /web/locales or /web/vendor path, or None when it is not served."""
+ if any(c in path for c in ("\\", ":", "%", "\0")) or ".." in path:
+ return None
+ m = LOCALE.match(path)
+ if m:
+ f = web / "locales" / m.group("name")
+ return (f, "application/json; charset=utf-8") if f.is_file() else None
+ if path.startswith(VENDOR_PREFIX):
+ vendor = (web / "vendor" / "katex").resolve()
+ f = (vendor / path[len(VENDOR_PREFIX):]).resolve()
+ ctype = VENDOR_TYPES.get(f.suffix)
+ if ctype and f.is_relative_to(vendor) and f.is_file():
+ return f, ctype
+ return None
diff --git a/tools/web_i18n.py b/tools/web_i18n.py
new file mode 100644
index 000000000..747f11019
--- /dev/null
+++ b/tools/web_i18n.py
@@ -0,0 +1,203 @@
+"""The web page's translations: read the English text out of serve/web, write locales/en.json, check the catalogs.
+
+ python tools/web_i18n.py # check every catalog against the text in the code (exit 1 on a problem)
+ python tools/web_i18n.py --write # also rewrite serve/web/locales/en.json from the code
+
+The English text lives in the code (see serve/web/i18n.js):
+ tr("key", "English text", vars) in serve/web/*.js
+ tr("key", {one: "{n} chat", other: "{n} chats"}, vars) a plural
+ <span data-i18n="key">English text</span> in serve/web/index.html (also data-i18n-title, -label, -placeholder)
+Both must be plain string literals (no template strings, no joined strings), so that this reader and a translator see
+the same text. serve/test_web_i18n.py runs the check.
+"""
+from __future__ import annotations
+
+import html.parser
+import json
+import re
+import sys
+from pathlib import Path
+
+WEB = Path(__file__).resolve().parents[1] / "serve" / "web"
+LOCALES = WEB / "locales"
+PLACEHOLDER = re.compile(r"\{(\w+)\}")
+CALL = re.compile(r"(?<![\w.$])tr\(\s*\"([\w.\-]+)\"\s*,\s*")
+PLURAL_CATEGORIES = ("zero", "one", "two", "few", "many", "other")
+
+
+class Problem(Exception):
+ pass
+
+
+def _string(src: str, i: int):
+ """A double-quoted JS string literal at src[i]: (the text, the index after it)."""
+ if src[i] != '"':
+ raise Problem(f"expected a string literal at: {src[i:i + 40]!r}")
+ j = i + 1
+ while j < len(src):
+ if src[j] == "\\":
+ j += 2
+ continue
+ if src[j] == '"':
+ try:
+ return json.loads(src[i:j + 1]), j + 1
+ except ValueError as e:
+ raise Problem(f"not a plain string literal ({e}): {src[i:j + 1]!r}")
+ j += 1
+ raise Problem("unterminated string literal")
+
+
+def _plural(src: str, i: int):
+ """{one: "...", other: "..."} at src[i]."""
+ j = i + 1
+ out = {}
+ while True:
+ m = re.compile(r"\s*(\w+)\s*:\s*").match(src, j)
+ if not m:
+ raise Problem(f"expected a plural form at: {src[j:j + 40]!r}")
+ text, j = _string(src, m.end())
+ out[m.group(1)] = text
+ m = re.compile(r"\s*([,}])").match(src, j)
+ if not m:
+ raise Problem(f"expected , or }} after a plural form at: {src[j:j + 40]!r}")
+ j = m.end()
+ if m.group(1) == "}":
+ break
+ if "other" not in out:
+ raise Problem(f"a plural needs an 'other' form: {out}")
+ return out, j
+
+
+def js_strings(src: str, name: str = "?") -> dict:
+ found = {}
+ for m in CALL.finditer(src):
+ key = m.group(1)
+ i = m.end()
+ try:
+ value, _ = _plural(src, i) if src[i] == "{" else _string(src, i)
+ except Problem as e:
+ line = src.count("\n", 0, m.start()) + 1
+ raise Problem(f"{name}:{line}: tr({key!r}): {e}")
+ found[key] = value
+ return found
+
+
+class _Html(html.parser.HTMLParser):
+ ATTRS = {"data-i18n-title": "title", "data-i18n-label": "aria-label", "data-i18n-placeholder": "placeholder"}
+
+ def __init__(self):
+ super().__init__(convert_charrefs=True)
+ self.found: dict = {}
+ self.open = [] # [(tag, key, [text])]
+
+ def handle_starttag(self, tag, attrs):
+ a = dict(attrs)
+ for mark, attr in self.ATTRS.items():
+ if mark in a:
+ self.found[a[mark]] = a.get(attr) or ""
+ if "data-i18n" in a:
+ self.open.append((tag, a["data-i18n"], []))
+ elif self.open:
+ self.open.append((tag, None, self.open[-1][2]))
+
+ def handle_data(self, data):
+ if self.open:
+ self.open[-1][2].append(data)
+
+ def handle_endtag(self, tag):
+ if self.open and self.open[-1][0] == tag:
+ _, key, text = self.open.pop()
+ if key:
+ self.found[key] = " ".join("".join(text).split())
+
+
+def html_strings(src: str) -> dict:
+ p = _Html()
+ p.feed(src)
+ return p.found
+
+
+def english() -> dict:
+ """Every key in serve/web with its English text; the same key twice must have the same text."""
+ out: dict = {}
+ sources = {p.name: js_strings(p.read_text(encoding="utf-8"), p.name) for p in sorted(WEB.glob("*.js"))}
+ sources["index.html"] = html_strings((WEB / "index.html").read_text(encoding="utf-8"))
+ for name, found in sources.items():
+ for key, value in found.items():
+ if key in out and out[key] != value:
+ raise Problem(f"{key!r} has two different English texts (one is in {name})")
+ out[key] = value
+ return dict(sorted(out.items()))
+
+
+def languages() -> list:
+ src = (WEB / "i18n.js").read_text(encoding="utf-8")
+ block = re.search(r"const LANGUAGES = (\[.*?\]);", src, re.S).group(1)
+ return [m for m in re.findall(r'\["([\w-]+)",', block)]
+
+
+def placeholders(value) -> set:
+ forms = value.values() if isinstance(value, dict) else [value]
+ return {name for form in forms for name in PLACEHOLDER.findall(form)}
+
+
+def check_catalog(code: str, catalog: dict, en: dict) -> list:
+ problems = []
+ for key in sorted(set(en) - set(catalog)):
+ problems.append(f"{code}: missing {key!r}")
+ for key in sorted(set(catalog) - set(en)):
+ problems.append(f"{code}: {key!r} is not in the page (renamed or removed?)")
+ for key, value in catalog.items():
+ if key not in en:
+ continue
+ forms = value if isinstance(value, dict) else {"": value}
+ if isinstance(value, dict):
+ bad = [c for c in value if c not in PLURAL_CATEGORIES]
+ if bad or "other" not in value:
+ problems.append(f"{code}: {key!r}: plural forms must be CLDR names with an 'other' form, got {sorted(value)}")
+ if not isinstance(en[key], dict):
+ problems.append(f"{code}: {key!r} is a plural here but not in the code")
+ elif isinstance(en[key], dict):
+ problems.append(f"{code}: {key!r} is a plain text here but a plural in the code")
+ for form, text in forms.items():
+ if not isinstance(text, str) or not text.strip():
+ problems.append(f"{code}: {key!r} {form} is empty or not a string")
+ if placeholders(value) != placeholders(en[key]) and not (
+ isinstance(value, dict) and isinstance(en[key], dict)
+ and all(placeholders(f) <= placeholders(en[key]) for f in value.values())):
+ problems.append(f"{code}: {key!r} uses {sorted(placeholders(value))}, the code uses {sorted(placeholders(en[key]))}")
+ return problems
+
+
+def check(write: bool = False) -> list:
+ en = english()
+ if write:
+ LOCALES.mkdir(exist_ok=True)
+ (LOCALES / "en.json").write_text(json.dumps(en, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ problems = []
+ on_disk = LOCALES / "en.json"
+ if not on_disk.is_file() or json.loads(on_disk.read_text(encoding="utf-8")) != en:
+ problems.append("en.json is not the text in the code (python tools/web_i18n.py --write)")
+ for code in languages():
+ if code == "en":
+ continue
+ f = LOCALES / f"{code}.json"
+ if not f.is_file():
+ problems.append(f"{code}: serve/web/locales/{code}.json is missing")
+ continue
+ problems += check_catalog(code, json.loads(f.read_text(encoding="utf-8")), en)
+ for f in sorted(LOCALES.glob("*.json")):
+ if f.stem not in languages():
+ problems.append(f"{f.name}: not in LANGUAGES of serve/web/i18n.js")
+ return problems
+
+
+if __name__ == "__main__":
+ try:
+ problems = check(write="--write" in sys.argv)
+ except Problem as e:
+ print(f"web_i18n: {e}")
+ sys.exit(1)
+ for p in problems:
+ print("web_i18n:", p)
+ sys.exit(1 if problems else 0)