A live bird collage card for your Home Assistant dashboard.
add translation tools to maintainer.py
frederikf210fb1f7486
diff --git a/scripts/utils/maintainer.py b/scripts/utils/maintainer.pyindex 17d98a85..4afa880a 100644--- a/scripts/utils/maintainer.py+++ b/scripts/utils/maintainer.py@@ -1,6 +1,17 @@import os+import re+import time+import requests-from scripts.utils.helpers import MODEL_PATH, save_language+from .helpers import MODEL_PATH, save_language, get_language++key_lang = {'af': 'Afrikaans', 'ar': 'Arabic', 'ca': 'Catalan', 'cs': 'Czech', 'da': 'Danish', 'de': 'German', 'en': 'English', 'es': 'Spanish',+ 'et': 'Estonian', 'fi': 'Finnish', 'fr': 'French', 'hr': 'Croatian', 'hu': 'Hungarian', 'id': 'Indonesian', 'is': 'Icelandic', 'it': 'Italian',+ 'ja': 'Japanese', 'ko': 'Korean', 'lt': 'Lithuanian', 'lv': 'Latvian', 'nl': 'Dutch', 'no': 'Norwegian', 'pl': 'Polish', 'pt': 'Portuguese',+ 'ro': 'Romanian', 'ru': 'Russian', 'sk': 'Slovak', 'sl': 'Slovenian', 'sr': 'Serbian', 'sv': 'Swedish', 'th': 'Thai', 'tr': 'Turkish',+ 'uk': 'Ukrainian', 'vi': 'Vietnamese', 'zh': 'Chinese'}+languages = ['af', 'ar', 'ca', 'cs', 'da', 'de', 'en', 'es', 'et', 'fi', 'fr', 'hr', 'hu', 'id', 'is', 'it', 'ja',+ 'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'vi', 'zh']def get_labels(model, language=None):@@ -38,3 +49,114 @@ def create_all_languages():'ko', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'zh']for language in languages:create_language(language)+++def measure_translations(language):+ en_labels = get_language('en')+ labels = get_language(language)++ count = 0+ count_need = 0+ for en, lab in zip(en_labels.items(), labels.items()):+ count += 1+ if en[1] == lab[1]:+ count_need += 1++ count_trans = len(labels) - count_need+ return f'| {key_lang[language]} | {count_trans} | {count_trans / len(labels):.1%} |'+++def measure_all_languages():+ stats = {}+ for language in languages:+ stats[key_lang[language]] = measure_translations(language)+ print('| Language | Translated species | Translated species (%) |')+ print('| -------- | ------- | ------ |')+ for _, stat in sorted(stats.items()):+ if 'English' in stat:+ continue+ print(stat)+++def scrape_wikipedia(sci_name, language, failed=None):+ if failed is None:+ failed = []+ url = f"https://{language}.wikipedia.org/api/rest_v1/page/summary/{sci_name.replace(" ", "_")}"+ headers = {'Accept-Encoding': 'gzip', 'User-agent': 'BirdNET-Pi'}+ try:+ resp = requests.get(url=url, headers=headers, timeout=10).json()+ except Exception:+ time.sleep(1)+ try:+ resp = requests.get(url=url, headers=headers, timeout=10).json()+ except Exception as e:+ failed.append((language, sci_name, str(e)))+ return++ type = resp['type']+ if type == 'Internal error':+ return+ if type == 'disambiguation':+ failed.append((language, sci_name, type))+ return++ try:+ com_name = resp['title']+ except KeyError:+ return++ if com_name and com_name != sci_name:+ cleaned = re.sub(r'\(\S+\)', '', com_name).strip()+ if cleaned != com_name:+ print(f'*** checkme {com_name}')+ return com_name+++def add_translations(language, failed=None):+ if failed is None:+ failed = []+ en_labels = get_language('en')+ labels = get_language(language)+ labels_updated = get_language(language)++ count = 0+ count_trans = 0+ count_needed = 0++ for en, lab in zip(en_labels.items(), labels.items()):+ count += 1+ if en[1] == lab[1] and ' ' in en[0]:+ count_needed += 1+ translated = scrape_wikipedia(en[0], language, failed)+ if translated and translated != en[1]:+ labels_updated[en[0]] = translated+ print(f'{lab[1]} -> {translated}')+ count_trans += 1++ time.sleep(0.1)+ if (count % 100) == 0:+ print(f'{count_trans=}', f'{count_needed=}', f'{count / len(labels):.2%}')++ print(f'{count_trans=}', f'{count_trans / len(labels):.2%}', f'{count_needed=}', f'{count_needed / len(labels):.2%}',+ f'{((count_needed - count_trans) / len(labels)):.2%}')+ print(failed)+ save_language(labels_updated, language)+ return failed++# Remove these+# ar: (طائر)+# de: (Vogel)+# fi: (lintu)+# is: (fugl)+# nl: [(vogel), (soort)]+# sv: (fågel)+++def update_all_languages():+ failed = []+ for language in languages:+ print(f'start {language}')+ ret = add_translations(language)+ print(f'finished {language}')+ failed.extend(ret)+ print(failed)