Skip to content

A live bird collage card for your Home Assistant dashboard.

add translation tools to maintainer.py

frederikf210fb1f7486
diff --git a/scripts/utils/maintainer.py b/scripts/utils/maintainer.py
index 17d98a85..4afa880a 100644
--- a/scripts/utils/maintainer.py
+++ b/scripts/utils/maintainer.py
@@ -1,6 +1,17 @@
import os
+import re
+import time
+import requests
-from scripts.utils.helpers import MODEL_PATH, save_language
+from .helpers import MODEL_PATH, save_language, get_language
+
+key_lang = {'af': 'Afrikaans', 'ar': 'Arabic', 'ca': 'Catalan', 'cs': 'Czech', 'da': 'Danish', 'de': 'German', 'en': 'English', 'es': 'Spanish',
+ 'et': 'Estonian', 'fi': 'Finnish', 'fr': 'French', 'hr': 'Croatian', 'hu': 'Hungarian', 'id': 'Indonesian', 'is': 'Icelandic', 'it': 'Italian',
+ 'ja': 'Japanese', 'ko': 'Korean', 'lt': 'Lithuanian', 'lv': 'Latvian', 'nl': 'Dutch', 'no': 'Norwegian', 'pl': 'Polish', 'pt': 'Portuguese',
+ 'ro': 'Romanian', 'ru': 'Russian', 'sk': 'Slovak', 'sl': 'Slovenian', 'sr': 'Serbian', 'sv': 'Swedish', 'th': 'Thai', 'tr': 'Turkish',
+ 'uk': 'Ukrainian', 'vi': 'Vietnamese', 'zh': 'Chinese'}
+languages = ['af', 'ar', 'ca', 'cs', 'da', 'de', 'en', 'es', 'et', 'fi', 'fr', 'hr', 'hu', 'id', 'is', 'it', 'ja',
+ 'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'vi', 'zh']
def get_labels(model, language=None):
@@ -38,3 +49,114 @@ def create_all_languages():
'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'zh']
for language in languages:
create_language(language)
+
+
+def measure_translations(language):
+ en_labels = get_language('en')
+ labels = get_language(language)
+
+ count = 0
+ count_need = 0
+ for en, lab in zip(en_labels.items(), labels.items()):
+ count += 1
+ if en[1] == lab[1]:
+ count_need += 1
+
+ count_trans = len(labels) - count_need
+ return f'| {key_lang[language]} | {count_trans} | {count_trans / len(labels):.1%} |'
+
+
+def measure_all_languages():
+ stats = {}
+ for language in languages:
+ stats[key_lang[language]] = measure_translations(language)
+ print('| Language | Translated species | Translated species (%) |')
+ print('| -------- | ------- | ------ |')
+ for _, stat in sorted(stats.items()):
+ if 'English' in stat:
+ continue
+ print(stat)
+
+
+def scrape_wikipedia(sci_name, language, failed=None):
+ if failed is None:
+ failed = []
+ url = f"https://{language}.wikipedia.org/api/rest_v1/page/summary/{sci_name.replace(" ", "_")}"
+ headers = {'Accept-Encoding': 'gzip', 'User-agent': 'BirdNET-Pi'}
+ try:
+ resp = requests.get(url=url, headers=headers, timeout=10).json()
+ except Exception:
+ time.sleep(1)
+ try:
+ resp = requests.get(url=url, headers=headers, timeout=10).json()
+ except Exception as e:
+ failed.append((language, sci_name, str(e)))
+ return
+
+ type = resp['type']
+ if type == 'Internal error':
+ return
+ if type == 'disambiguation':
+ failed.append((language, sci_name, type))
+ return
+
+ try:
+ com_name = resp['title']
+ except KeyError:
+ return
+
+ if com_name and com_name != sci_name:
+ cleaned = re.sub(r'\(\S+\)', '', com_name).strip()
+ if cleaned != com_name:
+ print(f'*** checkme {com_name}')
+ return com_name
+
+
+def add_translations(language, failed=None):
+ if failed is None:
+ failed = []
+ en_labels = get_language('en')
+ labels = get_language(language)
+ labels_updated = get_language(language)
+
+ count = 0
+ count_trans = 0
+ count_needed = 0
+
+ for en, lab in zip(en_labels.items(), labels.items()):
+ count += 1
+ if en[1] == lab[1] and ' ' in en[0]:
+ count_needed += 1
+ translated = scrape_wikipedia(en[0], language, failed)
+ if translated and translated != en[1]:
+ labels_updated[en[0]] = translated
+ print(f'{lab[1]} -> {translated}')
+ count_trans += 1
+
+ time.sleep(0.1)
+ if (count % 100) == 0:
+ print(f'{count_trans=}', f'{count_needed=}', f'{count / len(labels):.2%}')
+
+ print(f'{count_trans=}', f'{count_trans / len(labels):.2%}', f'{count_needed=}', f'{count_needed / len(labels):.2%}',
+ f'{((count_needed - count_trans) / len(labels)):.2%}')
+ print(failed)
+ save_language(labels_updated, language)
+ return failed
+
+# Remove these
+# ar: (طائر)
+# de: (Vogel)
+# fi: (lintu)
+# is: (fugl)
+# nl: [(vogel), (soort)]
+# sv: (fågel)
+
+
+def update_all_languages():
+ failed = []
+ for language in languages:
+ print(f'start {language}')
+ ret = add_translations(language)
+ print(f'finished {language}')
+ failed.extend(ret)
+ print(failed)