Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Web page: a translation mechanism, and a language choice in About > Settings
Niko1221d5cca319c962
diff --git a/serve/server.py b/serve/server.pyindex 76b5a2c79..ad0e10a90 100644--- a/serve/server.py+++ b/serve/server.py@@ -65,6 +65,7 @@ from serve.frontend import (THINK_END, ChatTemplate, Event, OutputParser, anthrotool_choice_of, unmark_think_literals)from serve.mcp import McpCancelled, hub_from_config # noqa: E402from serve import runconfig # noqa: E402+from serve.webfiles import subtree_file as web_subtree_file # noqa: E402from strata_control import engine_component # noqa: E402from strata_control import tls # noqa: E402@@ -6387,6 +6388,21 @@ def make_handler(svc: Service):self.end_headers()self.wfile.write(body)return+ sub = web_subtree_file(path)+ if path.startswith(("/web/locales/", "/web/vendor/")):+ # the page's translations (serve/web/locales/<lang>.json) and the vendored KaTeX (serve/web/vendor):+ # only whitelisted names and types, never a path out of those folders+ if sub is None:+ self._json(404, {"error": {"message": "not found"}})+ return+ body = sub[0].read_bytes()+ self.send_response(200)+ self.send_header("Content-Type", sub[1])+ self.send_header("Cache-Control", "no-cache")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)+ returnif path.startswith("/web/"):# the web app's own files (serve/web): styles, script, icon sprite - same origin, no CDNname = path[len("/web/"):]diff --git a/serve/test_web_i18n.js b/serve/test_web_i18n.jsnew file mode 100644index 000000000..a712f6e7a--- /dev/null+++ b/serve/test_web_i18n.js@@ -0,0 +1,92 @@+// The web page's translation mechanism (serve/web/i18n.js), run in Node with a small stub of the DOM and fetch.+// node serve/test_web_i18n.js (serve/test_web_i18n.py runs it too and skips when node is missing)+"use strict";+const assert = require("node:assert/strict");+const path = require("node:path");++// a page element that holds data-i18n marks and keeps its attributes+class El {+ constructor(attrs, text = "") { this.attrs = {...attrs}; this.textContent = text; this.dataset = {}; }+ getAttribute(k) { return k in this.attrs ? this.attrs[k] : null; }+ setAttribute(k, v) { this.attrs[k] = String(v); }+}+const marks = [+ new El({"data-i18n": "a.text"}, "Rewind to here"),+ new El({"data-i18n-title": "a.title", title: "Light / dark"}),+ new El({"data-i18n-label": "a.label", "aria-label": "Send"}),+ new El({"data-i18n-placeholder": "a.hint", placeholder: "Ask anything…"}),+];+globalThis.document = {+ documentElement: {lang: "en"},+ querySelectorAll: (sel) => marks.filter((e) => sel.slice(1, -1) in e.attrs),+};+const catalogs = {+ de: {"a.text": "Hierher zurückspulen", "a.title": "Hell / Dunkel", "chat.hidden": {one: "{n} Nachricht ausgeblendet", other: "{n} Nachrichten ausgeblendet"}},+ ru: {"chat.hidden": {one: "{n} сообщение скрыто", few: "{n} сообщения скрыто", many: "{n} сообщений скрыто", other: "{n} сообщения скрыто"}},+ ja: {"chat.hidden": {other: "{n} 件のメッセージを非表示"}},+};+let fetched = [];+globalThis.fetch = async (url) => {+ fetched.push(url);+ const m = /web\/locales\/([\w-]+)\.json$/.exec(url);+ if (m && catalogs[m[1]]) return {ok: true, json: async () => catalogs[m[1]]};+ if (m && m[1] === "zh") return {ok: true, json: async () => ["not", "an", "object"]};+ if (m && m[1] === "es") throw new Error("offline");+ return {ok: false, json: async () => { throw new Error("404"); }};+};+const i18n = require(path.join(__dirname, "web", "i18n.js"));+const {tr} = i18n;+const plural = {one: "{n} message hidden", other: "{n} messages hidden"};++(async () => {+ // English: the text in the code, with its {placeholders}; a key with no text shows the key rather than nothing+ assert.equal(tr("x.y", "Hello {name}", {name: "Ada"}), "Hello Ada");+ assert.equal(tr("x.y", "Hello {name}"), "Hello {name}", "an unknown placeholder stays visible");+ assert.equal(tr("chat.hidden", plural, {n: 1}), "1 message hidden");+ assert.equal(tr("chat.hidden", plural, {n: 3}), "3 messages hidden");+ assert.equal(tr("x.y", undefined), "x.y");+ assert.equal(globalThis.tr, tr, "tr is global for app.js");+ i18n.apply();+ assert.equal(marks[0].textContent, "Rewind to here", "English leaves the marked text alone");++ // a catalog replaces the text; a key it does not have falls back to the English text in the code+ assert.equal(await i18n.setLanguage("de"), "de");+ assert.equal(document.documentElement.lang, "de");+ assert.equal(marks[0].textContent, "Hierher zurückspulen");+ assert.equal(marks[1].attrs.title, "Hell / Dunkel");+ assert.equal(marks[2].attrs["aria-label"], "Send", "no German text for this key: English stays");+ assert.equal(tr("chat.hidden", plural, {n: 1}), "1 Nachricht ausgeblendet");+ assert.equal(tr("chat.hidden", plural, {n: 2}), "2 Nachrichten ausgeblendet");+ assert.equal(tr("only.english", "Only English"), "Only English");++ // plural forms of the language: Russian has few and many, Japanese only other+ await i18n.setLanguage("ru");+ assert.deepEqual([1, 2, 5, 21, 22].map((n) => tr("chat.hidden", plural, {n})),+ ["1 сообщение скрыто", "2 сообщения скрыто", "5 сообщений скрыто", "21 сообщение скрыто", "22 сообщения скрыто"]);+ await i18n.setLanguage("ja");+ assert.equal(tr("chat.hidden", plural, {n: 1}), "1 件のメッセージを非表示");++ // switching back restores the English text that was in the page+ await i18n.setLanguage("de");+ await i18n.setLanguage("en");+ assert.equal(marks[0].textContent, "Rewind to here");+ assert.equal(marks[1].attrs.title, "Light / dark");+ assert.equal(marks[3].attrs.placeholder, "Ask anything…");+ assert.equal(document.documentElement.lang, "en");++ // a language that is not offered, a catalog that is not an object, one that cannot be fetched, a missing file:+ // all stay English, and never throw+ fetched = [];+ assert.equal(await i18n.setLanguage("xx"), "en");+ assert.deepEqual(fetched, [], "an unknown code is not even requested");+ assert.equal(await i18n.setLanguage("zh"), "en");+ assert.equal(await i18n.setLanguage("es"), "en");+ assert.equal(await i18n.setLanguage("fr"), "en", "French is not in LANGUAGES yet");+ assert.equal(await i18n.setLanguage("ko"), "en", "the file does not exist in this test");+ assert.equal(tr("chat.hidden", plural, {n: 3}), "3 messages hidden");++ // the list: English first, every entry [code, own name]+ assert.equal(i18n.LANGUAGES[0][0], "en");+ for (const [code, name] of i18n.LANGUAGES) assert.ok(/^[a-z]{2,3}(-[A-Z]{2})?$/.test(code) && name, code);+ console.log("ok web i18n");+})().catch((e) => { console.error(e); process.exitCode = 1; });diff --git a/serve/test_web_i18n.py b/serve/test_web_i18n.pynew file mode 100644index 000000000..528533739--- /dev/null+++ b/serve/test_web_i18n.py@@ -0,0 +1,108 @@+"""The web page's translations: the mechanism (node, serve/test_web_i18n.js), the files the server hands out for it+(serve/web/locales, serve/web/vendor), and the catalogs against the text in the code (tools/web_i18n.py)."""+from __future__ import annotations++import shutil+import subprocess+import sys+import unittest+import urllib.error+import urllib.request+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT / "tools"))++import web_i18n # noqa: E402+from serve.frontend import ChatTemplate # noqa: E402+from serve.server import ByteTokenizer, MockEngine, Service, serve # noqa: E402+from serve.webfiles import subtree_file # noqa: E402+++def run_node(script: str, *args: str) -> None:+ result = subprocess.run([shutil.which("node"), str(Path(__file__).with_name(script)), *args],+ capture_output=True, text=True, timeout=60)+ if result.returncode:+ raise AssertionError((result.stdout + result.stderr).strip())++[email protected](shutil.which("node"), "node is not installed")+class Mechanism(unittest.TestCase):+ def test_tr_apply_and_set_language(self):+ run_node("test_web_i18n.js")+++class WhatTheServerHandsOut(unittest.TestCase):+ """/web/locales and /web/vendor: whitelisted names and types, nothing outside those folders."""++ def test_names_and_types(self):+ web = Path(__file__).parent / "web"+ locales = web / "locales"+ if (locales / "de.json").is_file():+ self.assertEqual(subtree_file("/web/locales/de.json")[1], "application/json; charset=utf-8")+ for bad in ("/web/locales/de.js", "/web/locales/DE.json", "/web/locales/../app.js", "/web/locales/x/y.json",+ "/web/locales/%2e%2e%2fapp.json", "/web/locales/de.json%00.js", "/web/locales/missing.json",+ "/web/locales/C:\\x.json", "/web/vendor/katex/../../app.js", "/web/vendor/katex/dist/katex.min.js.map",+ "/web/vendor/other/dist/katex.min.js", "/web/vendor/katex/dist/fonts/x.ttf", "/web/vendor/katex/dist/"):+ with self.subTest(bad=bad):+ self.assertIsNone(subtree_file(bad))++ def test_a_link_out_of_the_folder_is_refused(self):+ import tempfile+ with tempfile.TemporaryDirectory() as d:+ web = Path(d) / "web"+ (web / "vendor" / "katex").mkdir(parents=True)+ (web / "locales").mkdir()+ (web / "secret.js").write_text("secret")+ (web / "vendor" / "katex" / "ok.js").write_text("ok")+ self.assertEqual(subtree_file("/web/vendor/katex/ok.js", web)[1], "text/javascript; charset=utf-8")+ try:+ (web / "vendor" / "katex" / "link.js").symlink_to(web / "secret.js")+ except (OSError, NotImplementedError):+ self.skipTest("symlinks need a privilege here")+ self.assertIsNone(subtree_file("/web/vendor/katex/link.js", web))+++class Served(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ tok = ByteTokenizer()+ cls.svc = Service(MockEngine(tok, "ok"), tok, ChatTemplate(ROOT / "serve" / "chat_template.jinja"))+ cls.httpd = serve(cls.svc, port=0)+ cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"++ @classmethod+ def tearDownClass(cls):+ cls.httpd.shutdown()+ cls.httpd.server_close()++ def get(self, path):+ try:+ with urllib.request.urlopen(self.base + path, timeout=10) as r:+ return r.status, r.headers.get("Content-Type", ""), r.read()+ except urllib.error.HTTPError as e:+ return e.code, e.headers.get("Content-Type", ""), e.read()++ def test_i18n_script_and_refusals(self):+ code, ctype, body = self.get("/web/i18n.js")+ self.assertEqual(code, 200)+ self.assertIn(b"StrataI18n", body)+ for path in ("/web/locales/../server.py", "/web/locales/nope.json", "/web/vendor/katex/../../app.js",+ "/web/vendor/katex/dist/missing.js", "/web/locales/de.json%00"):+ with self.subTest(path=path):+ self.assertEqual(self.get(path)[0], 404)++[email protected]((Path(__file__).parent / "web" / "locales" / "en.json").is_file(), "no catalogs yet")+class Catalogs(unittest.TestCase):+ """Every language in serve/web/i18n.js has a catalog with exactly the keys and {placeholders} of the code."""++ def test_the_text_in_the_code_is_plain_literals(self):+ self.assertTrue(web_i18n.english())++ def test_catalogs_are_complete(self):+ self.assertEqual(web_i18n.check(), [])+++if __name__ == "__main__":+ unittest.main()diff --git a/serve/web/app.js b/serve/web/app.jsindex ac84f85dc..b3448d92b 100644--- a/serve/web/app.js+++ b/serve/web/app.js@@ -1096,14 +1096,35 @@ $("drawer-close").onclick = () => openDrawer(false);$("scrim").onclick = () => openDrawer(false);document.addEventListener("keydown", (e) => { if (e.key === "Escape" && $("drawer").dataset.open === "true") openDrawer(false); });+// ------------------------------------------------------------------ language (serve/web/i18n.js)+// The user's own choice, kept in this browser: English until they pick another. /?lang=de shows one for this visit only.+const urlLang = new URLSearchParams(location.search).get("lang");+$("lang").innerHTML = StrataI18n.LANGUAGES.map(([code, name]) => `<option value="${esc(code)}">${esc(name)}</option>`).join("");+function showLanguage() {+ $("lang").value = StrataI18n.lang;+ renderChat();+ setBusy(!!busy);+ if (lastMetrics) render(lastMetrics);+}+$("lang").onchange = async () => {+ store.set("lang", $("lang").value);+ await StrataI18n.setLanguage($("lang").value);+ showLanguage();+};+// ------------------------------------------------------------------ start-setBusy(false);-renderChat();-const startQuestion = new URLSearchParams(location.search).get("q"); // /?q=... starts a chat (a shortcut)-if (startQuestion) history.replaceState(null, "", location.pathname + location.hash);-loadHealth().then(loadMcp).then(() => { if (startQuestion) { $("input").value = startQuestion; send(); } });-showTab(location.hash.slice(1) || "chat");-poll();+async function start() {+ await StrataI18n.setLanguage(urlLang || store.get("lang", "en"));+ $("lang").value = StrataI18n.lang;+ setBusy(false);+ renderChat();+ const startQuestion = new URLSearchParams(location.search).get("q"); // /?q=... starts a chat (a shortcut)+ if (startQuestion) history.replaceState(null, "", location.pathname + location.hash);+ loadHealth().then(loadMcp).then(() => { if (startQuestion) { $("input").value = startQuestion; send(); } });+ showTab(location.hash.slice(1) || "chat");+ poll();+}+start();// ------------------------------------------------------------------ images (POST /v1/images/generations)$("img-form").onsubmit = async (ev) => {diff --git a/serve/web/i18n.js b/serve/web/i18n.jsnew file mode 100644index 000000000..81defdbb1--- /dev/null+++ b/serve/web/i18n.js@@ -0,0 +1,77 @@+// serve/web/i18n.js - the web page's translations.+//+// The English text sits in the code, next to the key: tr("chat.rewind", "Rewind to here") and+// tr("chat.hidden", {one: "{n} message hidden", other: "{n} messages hidden"}, {n: 3}). Static page text is marked+// data-i18n="key" (also data-i18n-title, data-i18n-label, data-i18n-placeholder) and holds its English text in the HTML.+// So the page reads fine with no catalog at all. Every other language is a catalog, serve/web/locales/<lang>.json+// (a flat {"key": "text"} object; a plural is {"one": "...", "other": "..."}, the CLDR names of the language).+// serve/test_web_i18n.py keeps the catalogs complete: same keys as the English text in the code, same {placeholders}.+//+// The language is the user's choice (About > Settings), kept in this browser. Nothing is chosen for them from the+// browser's language: a page that is translated in part would mix two languages.+"use strict";+(function (root) {+ // [code, the language's own name]. A new language is a locales/<code>.json file plus one line here.+ const LANGUAGES = [["en", "English"], ["de", "Deutsch"], ["es", "Español"], ["ja", "日本語"], ["ko", "한국어"],+ ["pt", "Português"], ["ru", "Русский"], ["zh", "中文"]];+ let lang = "en", catalog = null, rules = null;++ const fill = (s, vars) => String(s).replace(/\{(\w+)\}/g, (m, k) => (vars && k in vars ? String(vars[k]) : m));+ function pick(entry, vars) {+ if (entry && typeof entry === "object") {+ const n = vars && vars.n != null ? Number(vars.n) : 1;+ let cat = "other";+ try { cat = (rules || new Intl.PluralRules(lang === "en" ? "en" : lang)).select(n); } catch (e) { /* other */ }+ return entry[cat] != null ? entry[cat] : entry.other != null ? entry.other : entry.one;+ }+ return entry;+ }+ // tr(key, english, vars): the catalog's text for the chosen language, else the English text in the code+ function tr(key, english, vars) {+ const own = catalog && Object.prototype.hasOwnProperty.call(catalog, key) ? catalog[key] : undefined;+ const text = pick(own != null ? own : english, vars);+ return fill(text == null ? key : text, vars);+ }++ // marked page text: the English text is read once from the HTML and kept, so switching back restores it+ const ATTRS = [["data-i18n", null], ["data-i18n-title", "title"], ["data-i18n-label", "aria-label"],+ ["data-i18n-placeholder", "placeholder"]];+ function apply(scope) {+ const doc = scope || root.document;+ if (!doc || !doc.querySelectorAll) return;+ for (const [mark, attr] of ATTRS) {+ for (const el of doc.querySelectorAll(`[${mark}]`)) {+ const key = el.getAttribute(mark);+ const slot = "en_" + mark.replace(/[^a-z]/g, "");+ if (el.dataset[slot] === undefined) el.dataset[slot] = attr ? (el.getAttribute(attr) || "") : el.textContent;+ const text = tr(key, el.dataset[slot]);+ if (attr) el.setAttribute(attr, text); else el.textContent = text;+ }+ }+ if (root.document && root.document.documentElement) root.document.documentElement.lang = lang;+ }++ const known = (code) => LANGUAGES.some((l) => l[0] === code);+ // load a catalog; English, or a catalog that cannot be read, leaves the English text in the code+ async function setLanguage(code) {+ code = known(code) ? code : "en";+ let next = null;+ if (code !== "en") {+ try {+ const r = await root.fetch(`web/locales/${code}.json`);+ const j = r.ok ? await r.json() : null;+ if (j && typeof j === "object" && !Array.isArray(j)) next = j;+ } catch (e) { /* the English text stays */ }+ if (!next) code = "en";+ }+ lang = code; catalog = next; rules = null;+ try { rules = new Intl.PluralRules(lang); } catch (e) { /* other */ }+ apply();+ return lang;+ }++ const api = {LANGUAGES, tr, apply, setLanguage, get lang() { return lang; }, fill, pick};+ root.StrataI18n = api;+ root.tr = tr;+ if (typeof module !== "undefined" && module.exports) module.exports = api;+})(typeof globalThis !== "undefined" ? globalThis : this);diff --git a/serve/web/index.html b/serve/web/index.htmlindex 0c7847f22..0fdb4e6ed 100644--- a/serve/web/index.html+++ b/serve/web/index.html@@ -192,6 +192,11 @@<label class="st-label" for="api-key">API key <output>only if the server was started with one</output></label><input class="st-input" id="api-key" type="password" autocomplete="off" placeholder="Not needed"></div>+ <div class="st-field">+ <label class="st-label" for="lang"><span data-i18n="settings.language">Language</span></label>+ <select class="st-input" id="lang"></select>+ <span class="muted small" data-i18n="settings.languageNote">Chat, its settings and the buttons above are translated. The Monitor, Images and About views are still in English.</span>+ </div><div class="toggle-row"><span>Dark theme</span><button class="st-toggle" id="dark-toggle" role="switch" aria-checked="false" aria-label="Dark theme"></button></div><p class="muted small">Chats, settings and the key are kept in this browser only.@@ -262,6 +267,7 @@</aside><div class="scrim" id="scrim" hidden></div><div class="st-toasts" id="toasts" aria-live="polite"></div>+<script src="web/i18n.js"></script><script src="web/app.js"></script></body></html>diff --git a/serve/webfiles.py b/serve/webfiles.pynew file mode 100644index 000000000..d79b107d4--- /dev/null+++ b/serve/webfiles.py@@ -0,0 +1,35 @@+"""Which files of serve/web the server hands out below /web/: the page's translations and the vendored KaTeX.++The flat files (app.js, app.css, ...) are served by serve/server.py itself. The two folders here hold more than one+level, so each has its own whitelist: a name pattern and the types it may have. Nothing outside these folders is ever+read, a name with a backslash, a colon or a percent sign is refused, and the resolved path must stay inside the folder.+"""+from __future__ import annotations++import re+from pathlib import Path++WEB = Path(__file__).resolve().parent / "web"++# /web/locales/<lang>.json e.g. de.json, pt-BR.json+LOCALE = re.compile(r"^/web/locales/(?P<name>[a-z]{2,3}(?:-[A-Z]{2})?\.json)$")+# /web/vendor/katex/dist/... : the script, the style sheet and the fonts only+VENDOR_TYPES = {".js": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".woff2": "font/woff2"}+VENDOR_PREFIX = "/web/vendor/katex/"+++def subtree_file(path: str, web: Path = WEB):+ """(file, content type) for a /web/locales or /web/vendor path, or None when it is not served."""+ if any(c in path for c in ("\\", ":", "%", "\0")) or ".." in path:+ return None+ m = LOCALE.match(path)+ if m:+ f = web / "locales" / m.group("name")+ return (f, "application/json; charset=utf-8") if f.is_file() else None+ if path.startswith(VENDOR_PREFIX):+ vendor = (web / "vendor" / "katex").resolve()+ f = (vendor / path[len(VENDOR_PREFIX):]).resolve()+ ctype = VENDOR_TYPES.get(f.suffix)+ if ctype and f.is_relative_to(vendor) and f.is_file():+ return f, ctype+ return Nonediff --git a/tools/web_i18n.py b/tools/web_i18n.pynew file mode 100644index 000000000..747f11019--- /dev/null+++ b/tools/web_i18n.py@@ -0,0 +1,203 @@+"""The web page's translations: read the English text out of serve/web, write locales/en.json, check the catalogs.++ python tools/web_i18n.py # check every catalog against the text in the code (exit 1 on a problem)+ python tools/web_i18n.py --write # also rewrite serve/web/locales/en.json from the code++The English text lives in the code (see serve/web/i18n.js):+ tr("key", "English text", vars) in serve/web/*.js+ tr("key", {one: "{n} chat", other: "{n} chats"}, vars) a plural+ <span data-i18n="key">English text</span> in serve/web/index.html (also data-i18n-title, -label, -placeholder)+Both must be plain string literals (no template strings, no joined strings), so that this reader and a translator see+the same text. serve/test_web_i18n.py runs the check.+"""+from __future__ import annotations++import html.parser+import json+import re+import sys+from pathlib import Path++WEB = Path(__file__).resolve().parents[1] / "serve" / "web"+LOCALES = WEB / "locales"+PLACEHOLDER = re.compile(r"\{(\w+)\}")+CALL = re.compile(r"(?<![\w.$])tr\(\s*\"([\w.\-]+)\"\s*,\s*")+PLURAL_CATEGORIES = ("zero", "one", "two", "few", "many", "other")+++class Problem(Exception):+ pass+++def _string(src: str, i: int):+ """A double-quoted JS string literal at src[i]: (the text, the index after it)."""+ if src[i] != '"':+ raise Problem(f"expected a string literal at: {src[i:i + 40]!r}")+ j = i + 1+ while j < len(src):+ if src[j] == "\\":+ j += 2+ continue+ if src[j] == '"':+ try:+ return json.loads(src[i:j + 1]), j + 1+ except ValueError as e:+ raise Problem(f"not a plain string literal ({e}): {src[i:j + 1]!r}")+ j += 1+ raise Problem("unterminated string literal")+++def _plural(src: str, i: int):+ """{one: "...", other: "..."} at src[i]."""+ j = i + 1+ out = {}+ while True:+ m = re.compile(r"\s*(\w+)\s*:\s*").match(src, j)+ if not m:+ raise Problem(f"expected a plural form at: {src[j:j + 40]!r}")+ text, j = _string(src, m.end())+ out[m.group(1)] = text+ m = re.compile(r"\s*([,}])").match(src, j)+ if not m:+ raise Problem(f"expected , or }} after a plural form at: {src[j:j + 40]!r}")+ j = m.end()+ if m.group(1) == "}":+ break+ if "other" not in out:+ raise Problem(f"a plural needs an 'other' form: {out}")+ return out, j+++def js_strings(src: str, name: str = "?") -> dict:+ found = {}+ for m in CALL.finditer(src):+ key = m.group(1)+ i = m.end()+ try:+ value, _ = _plural(src, i) if src[i] == "{" else _string(src, i)+ except Problem as e:+ line = src.count("\n", 0, m.start()) + 1+ raise Problem(f"{name}:{line}: tr({key!r}): {e}")+ found[key] = value+ return found+++class _Html(html.parser.HTMLParser):+ ATTRS = {"data-i18n-title": "title", "data-i18n-label": "aria-label", "data-i18n-placeholder": "placeholder"}++ def __init__(self):+ super().__init__(convert_charrefs=True)+ self.found: dict = {}+ self.open = [] # [(tag, key, [text])]++ def handle_starttag(self, tag, attrs):+ a = dict(attrs)+ for mark, attr in self.ATTRS.items():+ if mark in a:+ self.found[a[mark]] = a.get(attr) or ""+ if "data-i18n" in a:+ self.open.append((tag, a["data-i18n"], []))+ elif self.open:+ self.open.append((tag, None, self.open[-1][2]))++ def handle_data(self, data):+ if self.open:+ self.open[-1][2].append(data)++ def handle_endtag(self, tag):+ if self.open and self.open[-1][0] == tag:+ _, key, text = self.open.pop()+ if key:+ self.found[key] = " ".join("".join(text).split())+++def html_strings(src: str) -> dict:+ p = _Html()+ p.feed(src)+ return p.found+++def english() -> dict:+ """Every key in serve/web with its English text; the same key twice must have the same text."""+ out: dict = {}+ sources = {p.name: js_strings(p.read_text(encoding="utf-8"), p.name) for p in sorted(WEB.glob("*.js"))}+ sources["index.html"] = html_strings((WEB / "index.html").read_text(encoding="utf-8"))+ for name, found in sources.items():+ for key, value in found.items():+ if key in out and out[key] != value:+ raise Problem(f"{key!r} has two different English texts (one is in {name})")+ out[key] = value+ return dict(sorted(out.items()))+++def languages() -> list:+ src = (WEB / "i18n.js").read_text(encoding="utf-8")+ block = re.search(r"const LANGUAGES = (\[.*?\]);", src, re.S).group(1)+ return [m for m in re.findall(r'\["([\w-]+)",', block)]+++def placeholders(value) -> set:+ forms = value.values() if isinstance(value, dict) else [value]+ return {name for form in forms for name in PLACEHOLDER.findall(form)}+++def check_catalog(code: str, catalog: dict, en: dict) -> list:+ problems = []+ for key in sorted(set(en) - set(catalog)):+ problems.append(f"{code}: missing {key!r}")+ for key in sorted(set(catalog) - set(en)):+ problems.append(f"{code}: {key!r} is not in the page (renamed or removed?)")+ for key, value in catalog.items():+ if key not in en:+ continue+ forms = value if isinstance(value, dict) else {"": value}+ if isinstance(value, dict):+ bad = [c for c in value if c not in PLURAL_CATEGORIES]+ if bad or "other" not in value:+ problems.append(f"{code}: {key!r}: plural forms must be CLDR names with an 'other' form, got {sorted(value)}")+ if not isinstance(en[key], dict):+ problems.append(f"{code}: {key!r} is a plural here but not in the code")+ elif isinstance(en[key], dict):+ problems.append(f"{code}: {key!r} is a plain text here but a plural in the code")+ for form, text in forms.items():+ if not isinstance(text, str) or not text.strip():+ problems.append(f"{code}: {key!r} {form} is empty or not a string")+ if placeholders(value) != placeholders(en[key]) and not (+ isinstance(value, dict) and isinstance(en[key], dict)+ and all(placeholders(f) <= placeholders(en[key]) for f in value.values())):+ problems.append(f"{code}: {key!r} uses {sorted(placeholders(value))}, the code uses {sorted(placeholders(en[key]))}")+ return problems+++def check(write: bool = False) -> list:+ en = english()+ if write:+ LOCALES.mkdir(exist_ok=True)+ (LOCALES / "en.json").write_text(json.dumps(en, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")+ problems = []+ on_disk = LOCALES / "en.json"+ if not on_disk.is_file() or json.loads(on_disk.read_text(encoding="utf-8")) != en:+ problems.append("en.json is not the text in the code (python tools/web_i18n.py --write)")+ for code in languages():+ if code == "en":+ continue+ f = LOCALES / f"{code}.json"+ if not f.is_file():+ problems.append(f"{code}: serve/web/locales/{code}.json is missing")+ continue+ problems += check_catalog(code, json.loads(f.read_text(encoding="utf-8")), en)+ for f in sorted(LOCALES.glob("*.json")):+ if f.stem not in languages():+ problems.append(f"{f.name}: not in LANGUAGES of serve/web/i18n.js")+ return problems+++if __name__ == "__main__":+ try:+ problems = check(write="--write" in sys.argv)+ except Problem as e:+ print(f"web_i18n: {e}")+ sys.exit(1)+ for p in problems:+ print("web_i18n:", p)+ sys.exit(1 if problems else 0)