A cross platform Audiobookshelf client for Android and iOS
Dart
Add a script that merges Crowdin translations without losing any
Barnabas2c69f1ce2fe5
diff --git a/scripts/crowdin-merge.py b/scripts/crowdin-merge.pynew file mode 100644index 00000000..86b39dfb--- /dev/null+++ b/scripts/crowdin-merge.py@@ -0,0 +1,187 @@+#!/usr/bin/env python3+"""Merge a Crowdin translation branch without losing translations.++Crowdin's ARB export has two recurring faults that break the build or silently+undo work, and neither can be fixed on the Crowdin side (language mapping only+rewrites export paths, never file contents):++ 1. It writes a region @@locale (es-ES, pt-PT, zh-CN) into files named with+ bare codes, which fails gen_l10n with "The locale specified in @@locale+ and the arb filename do not match."+ 2. Keys added to a translated file by hand, after Crowdin's last source+ upload, are missing from its export and disappear on the round trip.++This merges the branch, then repairs both. Nothing is committed and nothing is+pushed - the result is left staged for review.++Usage: python scripts/crowdin-merge.py [branch] (default: origin/l10n_main)+"""++import io+import json+import os+import re+import subprocess+import sys++L10N = 'lib/l10n'+SOURCE = f'{L10N}/app_en.arb'+LOCALE_RE = re.compile(r'("@@locale"\s*:\s*)"[^"]*"')+++def run(args, check=True):+ r = subprocess.run(args, capture_output=True, text=True, encoding='utf-8')+ if check and r.returncode != 0:+ sys.exit(f'failed: {" ".join(args)}\n{r.stderr or r.stdout}')+ return r+++def show(rev, path):+ """File contents at a revision, or None if it does not exist there."""+ r = subprocess.run(['git', 'show', f'{rev}:{path}'], capture_output=True)+ return r.stdout.decode('utf-8') if r.returncode == 0 else None+++def keys_of(text):+ return set(json.loads(text)) if text else set()+++def translation_keys(text):+ return {k for k in keys_of(text) if not k.startswith('@')}+++def restore(theirs, ours, source_keys):+ """Put back keys Crowdin dropped, keeping their file's formatting.++ Only restores a key that still exists in the English source, so a string+ deliberately deleted from app_en.arb stays deleted.+ """+ theirs_keys = keys_of(theirs)+ ours_data = json.loads(ours)++ missing = []+ for k in ours_data:+ if k in theirs_keys or k.startswith('@@'):+ continue+ base = k[1:] if k.startswith('@') else k+ if base in source_keys:+ missing.append(k)++ if not missing:+ return theirs, []++ lines = [' %s: %s' % (json.dumps(k, ensure_ascii=False),+ json.dumps(ours_data[k], ensure_ascii=False))+ for k in missing]++ body = theirs.rstrip()+ assert body.endswith('}')+ body = body[:-1].rstrip()+ if not body.endswith(','):+ body += ','+ result = body + '\n' + ',\n'.join(lines) + '\n}\n'++ json.loads(result)+ return result, [k for k in missing if not k.startswith('@')]+++def main():+ branch = sys.argv[1] if len(sys.argv) > 1 else 'origin/l10n_main'+ os.chdir(run(['git', 'rev-parse', '--show-toplevel']).stdout.strip())++ if run(['git', 'status', '--porcelain', '-uno']).stdout.strip():+ sys.exit('working tree has uncommitted changes - commit or stash first')++ if branch.startswith('origin/'):+ run(['git', 'fetch', 'origin', branch.split('/', 1)[1]])++ base = run(['git', 'merge-base', 'HEAD', branch]).stdout.strip()++ print(f'merging {branch}\n')+ subprocess.run(['git', 'merge', branch, '--no-commit', '--no-ff'],+ capture_output=True)++ source_keys = translation_keys(show('HEAD', SOURCE))+ rows, notes = [], []++ for name in sorted(os.listdir(L10N)):+ if not name.startswith('app_') or not name.endswith('.arb'):+ continue+ path = f'{L10N}/{name}'+ code = name[len('app_'):-len('.arb')]++ if code == 'en':+ # Crowdin must never rewrite the source; excluded_target_languages+ # should prevent it, but restore unconditionally in case it slips.+ # Compare against the merge base, not HEAD - local edits to the+ # source since the branch forked are ours, not Crowdin's.+ if show(base, path) != show(branch, path):+ run(['git', 'checkout', 'HEAD', '--', path])+ notes.append('app_en.arb was modified by the branch and has '+ 'been restored - check excluded_target_languages '+ 'in crowdin.yml')+ continue++ ours, theirs = show('HEAD', path), show(branch, path)+ if theirs is None:+ continue+ if ours is None:+ ours = '{"@@locale": "%s"}' % code++ before = len(translation_keys(ours))++ fixed = LOCALE_RE.sub(r'\1"%s"' % code, theirs)+ locale_fixed = fixed != theirs+ fixed, restored = restore(fixed, ours, source_keys)++ if json.loads(fixed) == json.loads(ours):+ # Nothing gained - keep our copy rather than churn the file. If the+ # export was missing keys, ours is strictly better anyway.+ run(['git', 'checkout', 'HEAD', '--', path])+ rows.append((code, before, before, len(restored), locale_fixed,+ 'kept ours'))+ if restored:+ notes.append('%s: export was missing %d key(s), kept ours - %s'+ % (code, len(restored), ', '.join(restored)))+ continue++ with io.open(path, 'w', encoding='utf-8', newline='\n') as f:+ f.write(fixed)+ run(['git', 'add', '--', path])++ after = len(translation_keys(fixed))+ rows.append((code, before, after, len(restored), locale_fixed,+ '+%d' % (after - before) if after != before else 'same'))+ if restored:+ notes.append('%s: restored %d dropped - %s'+ % (code, len(restored), ', '.join(restored)))++ print('%-6s %8s %8s %9s %8s' % ('lang', 'before', 'after', 'restored', 'locale'))+ for code, before, after, restored, locale_fixed, delta in rows:+ print('%-6s %8d %8d %9s %8s %s'+ % (code, before, after, restored or '-',+ 'fixed' if locale_fixed else '-', delta))++ print(f'\nEnglish source: {len(source_keys)} keys')+ for n in notes:+ print(f' {n}')++ print('\nrunning flutter gen-l10n')+ gen = subprocess.run('flutter gen-l10n', shell=True, capture_output=True,+ text=True, encoding='utf-8')+ warnings = [l for l in (gen.stderr + gen.stdout).splitlines()+ if 'Warning' in l or 'Error' in l]+ if gen.returncode != 0:+ print(gen.stderr or gen.stdout)+ sys.exit('gen-l10n failed - merge left in place for inspection')+ if warnings:+ print(f' {len(warnings)} warning(s), see output of flutter gen-l10n')++ run(['git', 'add', '--', L10N])+ print('\nStaged, not committed. Review with: git diff --cached HEAD')+ print('New language? Add it to _pickLanguage and _languageDisplayName in '+ 'lib/screens/settings_screen.dart')+++if __name__ == '__main__':+ main()