Skip to content

Doubao-powered voice input for Linux, Wayland and Omarchy

13.1 KB
Raw Blame
"""Opt-in replay and offline scoring. Never reads the microphone or learns references."""
import argparse
from collections import defaultdict
import json
import math
import os
from pathlib import Path
import threading
import time
import unicodedata
import wave

from doubao_input.personal import PersonalStore, vocabulary_prompt
from doubao_input.personal_corrections import correction_prompt, hotword_context, term_pattern
from doubao_input.settings import write_atomic


def normalized(text):
    return "".join(c for c in unicodedata.normalize("NFC", text).casefold()
                   if not c.isspace() and not unicodedata.category(c).startswith("P"))


def distance(reference, hypothesis):
    previous = list(range(len(hypothesis) + 1))
    for i, left in enumerate(reference, 1):
        current = [i]
        for j, right in enumerate(hypothesis, 1):
            current.append(min(current[-1] + 1, previous[j] + 1,
                               previous[j - 1] + (left != right)))
        previous = current
    return previous[-1]


def read_manifest(path):
    rows = read_rows(path)
    if not rows or len(rows) > 200:
        raise ValueError("Manifest must contain 1–200 samples")
    seen = set()
    for row in rows:
        identifier = row.get("id")
        reference = row.get("reference")
        if (not isinstance(identifier, str) or not identifier.strip() or identifier in seen
                or not isinstance(reference, str) or not normalized(reference) or len(reference) > 4096):
            raise ValueError("Each sample needs a unique id and a nonempty reference of at most 4096 characters")
        seen.add(identifier)
        for key in ("terms", "protected_terms"):
            terms = row.get(key, [])
            if (not isinstance(terms, list) or len(terms) > 30
                    or any(not isinstance(term, str) or not term.strip() or len(term) > 80
                           or not term_pattern(term).search(reference) for term in terms)):
                raise ValueError("Target and protected terms must occur in the reference")
        if "audio" in row:
            if not isinstance(row["audio"], str) or not row["audio"]:
                raise ValueError("Audio must be a WAV path")
            row["audio"] = str((Path(path).resolve().parent / row["audio"]).resolve())
    return rows


def read_rows(path):
    if Path(path).stat().st_size > 10_000_000:
        raise ValueError("Input file exceeds 10 MB")
    rows = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]
    if any(not isinstance(row, dict) for row in rows):
        raise ValueError("Expected JSON objects, one per line")
    return rows


def percentile(values, fraction):
    return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None


def score(samples, results):
    references = {sample["id"]: sample for sample in samples}
    modes = defaultdict(dict)
    for row in results:
        identifier, mode = row.get("id"), row.get("mode")
        if identifier not in references or not isinstance(mode, str) or not mode.strip() or identifier in modes[mode]:
            raise ValueError("Results need known ids and unique id/mode pairs")
        text = row.get("text", "")
        latency = row.get("latency_ms")
        if (not isinstance(text, str) or len(text) > 8192 or
                (latency is not None and (type(latency) not in (int, float) or not math.isfinite(latency) or latency < 0))):
            raise ValueError("Invalid result text or latency")
        modes[mode][identifier] = row
    if not modes:
        raise ValueError("No results to score")
    report = {}
    for mode, rows in modes.items():
        errors = characters = hits = targets = false_corrections = protected = failed = 0
        timings, details = [], []
        for identifier, sample in references.items():
            row = rows.get(identifier)
            ref = normalized(sample["reference"])
            success = row is not None and not row.get("error") and bool(row.get("text", "").strip())
            # Missing/failed requests count as full deletions, never as perfect recognition.
            text = row["text"] if success else ""
            edits = distance(ref, normalized(text))
            errors += edits; characters += len(ref); failed += not success
            for term in sample.get("terms", []):
                targets += 1; hits += bool(term_pattern(term).search(text))
            for term in sample.get("protected_terms", []):
                protected += 1; false_corrections += not bool(term_pattern(term).search(text))
            if success and row.get("latency_ms") is not None:
                timings.append(row["latency_ms"])
            details.append({"id": identifier, "failed": not success, "character_errors": edits})
        report[mode] = {"samples": len(samples), "failed": failed, "character_error_rate": errors / characters,
                        "term_accuracy": hits / targets if targets else None,
                        "protected_term_loss_rate": false_corrections / protected if protected else None,
                        "latency_samples": len(timings), "latency_p50_ms": percentile(timings, .5),
                        "latency_p95_ms": percentile(timings, .95), "details": details}
    return {"normalization": "NFC, case-insensitive, remove whitespace and Unicode punctuation; CER includes failures",
            "latency": "connection + real-time audio + final ASR response; polishing adds its request duration",
            "protected_terms": "loss includes ASR misses and request failures; compare the same samples across modes",
            "modes": report}


def read_audio(path):
    if Path(path).stat().st_size > 2_000_000:
        raise ValueError("Use WAV files of at most 60 seconds")
    with wave.open(str(path), "rb") as audio:
        if (audio.getnchannels(), audio.getframerate(), audio.getsampwidth(), audio.getcomptype()) != (1, 16000, 2, "NONE"):
            raise ValueError("Replay requires mono 16 kHz 16-bit PCM WAV")
        if not 0 < audio.getnframes() <= 960_000:
            raise ValueError("Use WAV files of 0–60 seconds")
        return audio.readframes(audio.getnframes())


def recognize(pcm, credentials, words, *, client_factory=None, timeout=75, cancel=None):
    from doubao_input.doubao.volcengine_asr_client import VolcengineASRClient
    client = (client_factory or VolcengineASRClient)(hotwords=lambda: words)
    opened, finished = threading.Event(), threading.Event()
    result = {"text": ""}
    def failure(*_):
        result["error"] = "asr_failed"; finished.set(); opened.set()
    client.on_open = opened.set
    client.on_result = lambda text: result.update(text=text)
    client.on_finish = finished.set
    client.on_error = client.on_auth_error = failure
    cancel = cancel or threading.Event()
    def wait(event, seconds):
        deadline = time.monotonic() + seconds
        while not cancel.is_set() and time.monotonic() < deadline:
            if event.wait(min(.1, max(0, deadline - time.monotonic()))):
                return True
        return False
    started = time.monotonic()
    try:
        client.prepare(); client.connect(credentials)
        if not wait(opened, 8) or finished.is_set():
            raise TimeoutError("ASR did not open")
        for offset in range(0, len(pcm), 3200):
            if finished.is_set() or cancel.is_set():
                break
            client.send_audio(pcm[offset:offset + 3200])
            cancel.wait(.1)
        client.finish_sending()
        remaining = max(0, timeout - (time.monotonic() - started))
        if not wait(finished, remaining):
            result["error"] = "asr_timeout"
    except (OSError, ValueError, TimeoutError):
        result["error"] = "asr_failed"
    finally:
        client.disconnect()
    result["latency_ms"] = round((time.monotonic() - started) * 1000)
    if not result["text"].strip():
        result.setdefault("error", "asr_empty")
    return result


def replay(samples, words, credentials, *, polisher=None):
    results = []
    for sample in samples:
        pcm = read_audio(sample.get("audio", ""))
        # Reference texts and target labels never enter requests or the hotword list.
        for mode, vocabulary in (("raw", []), ("hotwords", words)):
            row = {"id": sample["id"], "mode": mode, **recognize(pcm, credentials, vocabulary)}
            results.append(row)
            if mode == "hotwords" and polisher is not None:
                polished = {**row, "mode": "hotwords+polish"}
                if not row.get("error"):
                    started = time.monotonic()
                    try:
                        polished["text"] = polisher(row["text"])
                    except (OSError, ValueError, TimeoutError):
                        polished["error"] = "polish_failed"
                    polished["latency_ms"] += round((time.monotonic() - started) * 1000)
                results.append(polished)
    return results


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", required=True)
    source = parser.add_mutually_exclusive_group(required=True)
    source.add_argument("--results", help="Offline JSONL results; no network requests")
    source.add_argument("--replay", action="store_true", help="Upload listed audio to paid official ASR, twice per sample")
    parser.add_argument("--polish", action="store_true", help="Also upload hotword transcripts to configured polishing service")
    parser.add_argument("--personal-db", help="Optional explicit personal SQLite path (Mac/Linux share its schema)")
    parser.add_argument("--out", required=True, help="Private report directory outside the source repository")
    args = parser.parse_args(argv)
    try:
        samples = read_manifest(args.manifest)
        output = Path(args.out).resolve()
        output.mkdir(parents=True, exist_ok=True, mode=0o700)
        if args.polish and not args.replay:
            raise ValueError("--polish requires --replay")
        if args.results:
            results = read_rows(args.results)
            metadata = {"source": "offline", "hotwords": None}
        else:
            from doubao_input.doubao.volcengine_credentials import VolcengineCredentials, VolcengineCredentialsStore
            key = os.environ.get("DOUBAO_ASR_API_KEY")
            credentials = VolcengineCredentials(key) if key else VolcengineCredentialsStore.load()
            if credentials is None:
                raise ValueError("Set DOUBAO_ASR_API_KEY or configure official ASR credentials")
            credentials.validate()
            store = PersonalStore(args.personal_db)
            words, corrections, personal_words = store.recognition_words(), store.corrections(), store.words()
            polisher = None
            if args.polish:
                from doubao_input.polish import ApiKeyStore, PolishClient
                from doubao_input.settings import Settings
                settings = Settings.load()
                key = os.environ.get("DOUBAO_POLISH_API_KEY") or ApiKeyStore.load()
                if not key:
                    raise ValueError("Configure polishing credentials before using --polish")
                settings.polish_base_url = os.environ.get("DOUBAO_POLISH_BASE_URL", settings.polish_base_url)
                settings.polish_model = os.environ.get("DOUBAO_POLISH_MODEL", settings.polish_model)
                settings.validate()
                def polisher(text):
                    # Select language without reading another personal database.
                    import re
                    han = len(re.findall(r"[\u3400-\u4dbf\u4e00-\u9fff]", text))
                    prompt = settings.polish_prompt_zh if han > len(re.findall(r"[A-Za-z]+", text)) else settings.polish_prompt_en
                    prompt = correction_prompt(vocabulary_prompt(prompt, personal_words), text, corrections)
                    return PolishClient().polish(text, base_url=settings.polish_base_url,
                                                 model=settings.polish_model, api_key=key, prompt=prompt)
            # Validate every recording before spending API calls.
            for sample in samples:
                read_audio(sample.get("audio", ""))
            results = replay(samples, words, credentials, polisher=polisher)
            context = hotword_context(words)
            metadata = {"source": "official_asr_replay", "hotwords": json.loads(context) if context else [],
                        "endpoint": "sauc/bigmodel", "personal_vocabulary_count": len(words)}
        report = {**score(samples, results), **metadata}
        write_atomic(output / "results.jsonl", ("\n".join(json.dumps(row, ensure_ascii=False) for row in results) + "\n").encode())
        write_atomic(output / "report.json", (json.dumps(report, ensure_ascii=False, indent=2) + "\n").encode())
    except (OSError, ValueError, TypeError, KeyError, wave.Error):
        parser.exit(2, "Evaluation failed. Check manifest, WAV format, credentials and result schema; no private input is logged.\n")
    print("Evaluation complete. Private results.jsonl and report.json were written to the output directory.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())