One keyboard and mouse for a Mac and a Windows PC over the local network, in either direction.
"""Write locales/beamer.pot from the source, bring every locales/<code>.po in step, and check them.
python3 tools/i18n_extract.py # rewrite beamer.pot, add new strings to each .po, drop gone ones
python3 tools/i18n_extract.py --check # exit 1 if beamer.pot is stale or a translation breaks its {fields}
python3 tools/i18n_extract.py --check --complete # and on any untranslated string: before a release
python3 tools/i18n_extract.py --review it # write docs/i18n/it-review.html for a translator
Standard library only. Strings are found by reading the source, never by importing it, so the
tool runs anywhere, without AppKit or Qt: tr("..."), trn("...", "...", n), trc("context", "...")
and N_("...").
"""
from __future__ import annotations
import argparse
import ast
import html
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from core import i18n # noqa: E402
SOURCES = ("core", "mac_app", "win_app")
LOCALES = ROOT / "locales"
TEMPLATE = LOCALES / "beamer.pot"
FIELD = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)[^{}]*\}")
HEADER = {
"Project-Id-Version": "Beamer",
"MIME-Version": "1.0",
"Content-Type": "text/plain; charset=UTF-8",
"Content-Transfer-Encoding": "8bit",
}
def source_files():
for folder in SOURCES:
for path in sorted((ROOT / folder).glob("*.py")):
yield path
def _literal(node):
return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
def _branches(node) -> list[str]:
"""The literal strings an argument can be: tr("Put back" if out else "Leave out") is two."""
if isinstance(node, ast.IfExp):
return _branches(node.body) + _branches(node.orelse)
text = _literal(node)
return [text] if text else []
def extract(paths=None) -> dict:
"""{(context, msgid, plural): {places: [file:line, ...], notes: [...]}} for every marked string, in source order."""
found: dict = {}
for path in paths or source_files():
text = path.read_text(encoding="utf-8")
lines = text.splitlines()
tree = ast.parse(text, str(path))
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
name = node.func.id if isinstance(node.func, ast.Name) else node.func.attr if isinstance(node.func, ast.Attribute) else ""
args = [_literal(arg) for arg in node.args]
if name in ("tr", "N_") and node.args:
keys = [(None, text, None) for text in _branches(node.args[0])]
elif name == "trn" and len(args) >= 2 and args[0] and args[1]:
keys = [(None, args[0], args[1])]
elif name == "trc" and len(node.args) >= 2 and args[0]:
keys = [(args[0], text, None) for text in _branches(node.args[1])]
else:
continue
# The file, not the line: a line number would make the template stale on every edit above it.
where = path.relative_to(ROOT).as_posix()
note = _translator_note(lines, node.lineno)
for key in keys:
entry = found.setdefault(key, {"places": [], "notes": []})
if where not in entry["places"]:
entry["places"].append(where)
if note and note not in entry["notes"]:
entry["notes"].append(note)
return found
def _translator_note(lines: list[str], lineno: int) -> str:
"""The "# Translators: ..." comment block ending on the line above a call, if there is one."""
block, index = [], lineno - 2
while index >= 0 and lines[index].strip().startswith("#"):
block.insert(0, lines[index].strip().lstrip("#").strip())
index -= 1
for at, line in enumerate(block):
if line.startswith("Translators:"):
return " ".join([line[len("Translators:"):].strip()] + block[at + 1:])
return ""
def _quote(text: str) -> str:
body = text.replace("\\", "\\\\").replace('"', '\\"').replace("\t", "\\t")
if "\n" not in body:
return f'"{body}"'
lines = body.split("\n")
parts = [f'"{line}\\n"' for line in lines[:-1]] + ([f'"{lines[-1]}"'] if lines[-1] else [])
return '""\n' + "\n".join(parts)
def _key(entry) -> tuple:
return (entry.get("msgctxt"), entry["msgid"], entry.get("msgid_plural"))
def render(entries: list[dict], header: dict) -> str:
out = ['msgid ""', 'msgstr ""']
out += [f'"{name}: {value}\\n"' for name, value in header.items()]
for entry in entries:
out.append("")
out += entry.get("comments", [])
if entry.get("flags"):
out.append("#, " + ", ".join(sorted(entry["flags"])))
if entry.get("msgctxt") is not None:
out.append("msgctxt " + _quote(entry["msgctxt"]))
out.append("msgid " + _quote(entry["msgid"]))
if entry.get("msgid_plural") is not None:
out.append("msgid_plural " + _quote(entry["msgid_plural"]))
for index, text in enumerate(entry["msgstr"]):
out.append(f"msgstr[{index}] " + _quote(text))
else:
out.append("msgstr " + _quote(entry["msgstr"][0] if entry["msgstr"] else ""))
return "\n".join(out) + "\n"
def header_of(entries: list[dict]) -> dict:
for entry in entries:
if entry["msgid"] == "" and "msgctxt" not in entry:
lines = entry["msgstr"][0].split("\n") if entry["msgstr"] else []
return dict(line.split(": ", 1) for line in lines if ": " in line)
return {}
def plural_count(header: dict) -> int:
match = re.search(r"nplurals\s*=\s*(\d+)", header.get("Plural-Forms", ""))
return int(match.group(1)) if match else 2
def template_entries(found: dict) -> list[dict]:
entries = []
for (context, msgid, plural), seen in found.items():
entry = {"msgid": msgid, "msgstr": ["", ""] if plural else [""], "flags": set(),
"comments": ["#. " + note for note in seen["notes"]] + ["#: " + " ".join(seen["places"])]}
if "{" in msgid or (plural and "{" in plural):
entry["flags"].add("python-brace-format")
if context is not None:
entry["msgctxt"] = context
if plural is not None:
entry["msgid_plural"] = plural
entries.append(entry)
return entries
def merge(template: list[dict], existing: list[dict], plurals: int) -> list[dict]:
"""The template's entries in its order, each carrying the translation and translator comments
an existing .po already has; strings gone from the source are dropped (git keeps them)."""
known = {_key(entry): entry for entry in existing if entry["msgid"]}
merged = []
for entry in template:
old = known.get(_key(entry))
new = dict(entry, flags=set(entry["flags"]))
if "msgid_plural" in entry:
new["msgstr"] = list(old["msgstr"]) if old else [""] * plurals
else:
new["msgstr"] = list(old["msgstr"][:1]) if old else [""]
if old:
notes = [line for line in old["comments"] if not line.startswith(("#:", "#."))]
new["comments"] = notes + entry["comments"]
new["flags"] |= old["flags"] & {"fuzzy"}
merged.append(new)
return merged
def problems(entries: list[dict], template: list[dict], plurals: int) -> list[str]:
"""Every missing, empty or broken translation, as one line each."""
wanted = {_key(entry) for entry in template}
have = {_key(entry): entry for entry in entries if entry["msgid"]}
found = []
for key in wanted - have.keys():
found.append(f"missing: {key[1]!r}")
for key in have.keys() - wanted:
found.append(f"not in the source any more: {key[1]!r}")
for key, entry in have.items():
if key not in wanted:
continue
sources = [entry["msgid"]] + ([entry["msgid_plural"]] if "msgid_plural" in entry else [])
fields = set().union(*(set(FIELD.findall(text)) for text in sources))
if "msgid_plural" in entry and len(entry["msgstr"]) != plurals:
found.append(f"wrong number of plural forms: {entry['msgid']!r}")
for text in entry["msgstr"]:
if not text.strip():
found.append(f"untranslated: {entry['msgid']!r}")
break
if set(FIELD.findall(text)) - fields:
found.append(f"unknown {{field}} in translation of {entry['msgid']!r}: {text!r}")
missing = fields - set(FIELD.findall(text))
# A singular may leave out {n} ("un computer"); nothing else may go.
if missing - ({"n", "count"} if "msgid_plural" in entry else set()):
found.append(f"translation of {entry['msgid']!r} drops {sorted(missing)}: {text!r}")
try:
text.format(**{name: "x" for name in FIELD.findall(text)})
except (ValueError, IndexError, KeyError) as error:
found.append(f"translation of {entry['msgid']!r} does not format ({error}): {text!r}")
return sorted(found)
def catalogues():
return sorted(path for path in LOCALES.glob("*.po"))
def review_page(code: str, entries: list[dict]) -> str:
"""One page a translator can read and mark up without the code: English, translation, where
it shows, and any note for them."""
name = i18n.LANGUAGES.get(code, code)
rows, new = [], 0
for entry in entries:
if not entry["msgid"]:
continue
english = entry["msgid"] + (f"\n— plural —\n{entry['msgid_plural']}" if "msgid_plural" in entry else "")
target = "\n— plural —\n".join(entry["msgstr"]) if "msgid_plural" in entry else (entry["msgstr"] or [""])[0]
notes = [line[2:].strip() for line in entry["comments"] if line.startswith("#.") or line.startswith("# ")]
places = " ".join(line[2:].strip() for line in entry["comments"] if line.startswith("#:"))
context = entry.get("msgctxt") or ""
# A "# New in <version>: awaiting ..." comment marks a string added since the translator last checked.
fresh = any(line.startswith("# New in ") for line in entry["comments"])
new += fresh
rows.append(
("<tr class=new>" if fresh else "<tr>") + "<td class=n>{n}</td><td>{en}{ctx}</td><td lang={code}>{it}</td><td class=note>{note}<div class=where>{where}</div></td></tr>".format(
n=len(rows) + 1, en=html.escape(english).replace("\n", "<br>"),
ctx=f"<div class=ctx>({html.escape(context)})</div>" if context else "",
code=code, it=html.escape(target).replace("\n", "<br>"),
note=html.escape(" ".join(notes)), where=html.escape(places)))
return f"""<!doctype html>
<html lang=en-GB><meta charset=utf-8><title>Beamer — {html.escape(name)} for checking</title>
<style>
body{{font:15px/1.45 -apple-system,"Segoe UI",system-ui,sans-serif;margin:2rem auto;max-width:72rem;padding:0 1rem;color:#1d1d1f}}
h1{{font-size:1.4rem}} p{{max-width:46rem}}
table{{border-collapse:collapse;width:100%}} td,th{{border-top:1px solid #ddd;padding:.45rem .6rem;vertical-align:top;text-align:left}}
th{{position:sticky;top:0;background:#fff}} td.n{{color:#888;width:2.5rem}} td:nth-child(2),td:nth-child(3){{width:36%}}
.ctx,.where{{color:#888;font-size:.8rem}} .note{{font-size:.85rem;color:#555}}
code{{background:#f3f3f3;padding:0 .2rem}} tr.new td{{background:#fff6d6}}
</style>
<h1>Beamer in {html.escape(name)} — every string, for checking</h1>
<p>Left is the English Beamer shows today; right is the {html.escape(name)} draft. Words in braces, like
<code>{{name}}</code>, are filled in by Beamer (a machine's name, a number) and must stay as they are,
though they can move within the sentence. To correct a line, quote its number with the fix, or edit
<code>locales/{code}.po</code> directly. {len(rows)} strings.{f" The {new} on a yellow background are new and not yet checked." if new else ""}</p>
<table><tr><th>#</th><th>English</th><th>{html.escape(name)}</th><th>Note · where it shows</th></tr>
{chr(10).join(rows)}
</table>
"""
def main() -> int:
# Catalogue lines carry "…" and "→": a Windows console's legacy code page cannot print them.
sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
parser = argparse.ArgumentParser()
parser.add_argument("--check", action="store_true")
parser.add_argument("--complete", action="store_true", help="with --check: fail on any untranslated string too")
parser.add_argument("--review", metavar="CODE")
options = parser.parse_args()
template = template_entries(extract())
pot = render(template, dict(HEADER, **{"Language": "en_GB"}))
if options.review:
path = LOCALES / f"{options.review}.po"
out = ROOT / "docs" / "i18n" / f"{options.review}-review.html"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(review_page(options.review, i18n.parse_po(path.read_text(encoding="utf-8"))), encoding="utf-8")
print(out.relative_to(ROOT))
return 0
if options.check:
# A string not yet translated shows in English, so only --complete (a release) fails on
# one; a stale template or a translation that breaks its {fields} always fails.
failed, pending = [], []
if not TEMPLATE.exists() or TEMPLATE.read_text(encoding="utf-8") != pot:
failed.append(f"{TEMPLATE.relative_to(ROOT)} is stale: run python3 tools/i18n_extract.py")
for path in catalogues():
entries = i18n.parse_po(path.read_text(encoding="utf-8"))
for line in problems(entries, template, plural_count(header_of(entries))):
soft = line.startswith(("missing:", "untranslated:", "not in the source any more:"))
(pending if soft and not options.complete else failed).append(f"{path.relative_to(ROOT)}: {line}")
print("\n".join(failed + pending) or f"{len(template)} strings, catalogues complete")
if pending and not failed:
print(f"{len(pending)} waiting for translation; they show in English until then")
return 1 if failed else 0
LOCALES.mkdir(exist_ok=True)
TEMPLATE.write_text(pot, encoding="utf-8")
for path in catalogues():
entries = i18n.parse_po(path.read_text(encoding="utf-8"))
header = header_of(entries)
path.write_text(render(merge(template, entries, plural_count(header)), header), encoding="utf-8")
print(f"{len(template)} strings")
return 0
if __name__ == "__main__":
sys.exit(main())