Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

11.0 KB
Raw Blame
"""Compare two transcription engines over the same window of a file.

Transcribes a sample range twice, once per engine, and reports where the two
transcripts disagree, windowed every 20s by default. This measures
disagreement between the two outputs, not accuracy against a ground truth.
Neither engine is assumed correct.
"""

from __future__ import annotations

import html
import json
import os
import shutil
import unicodedata
from typing import Any, Optional

# Unicode categories kept when normalizing a word for comparison: letters (L*),
# marks (M*, e.g. combining diacritics), numbers (N*). Punctuation and symbols
# are dropped so "hello," and "hello" agree.
_KEEP_CATEGORY_PREFIXES = ("L", "M", "N")


def normalize_word(word: str) -> str:
    """NFKC-normalize, casefold, and strip everything but letters/marks/numbers."""
    normalized = unicodedata.normalize("NFKC", word or "").casefold()
    return "".join(ch for ch in normalized if unicodedata.category(ch)[0] in _KEEP_CATEGORY_PREFIXES)


def normalize_words(words: list[str]) -> list[str]:
    return [w for w in (normalize_word(w) for w in words) if w]


def word_levenshtein(a: list[str], b: list[str]) -> int:
    """Levenshtein distance over word sequences (insert/delete/substitute, cost 1 each)."""
    if a == b:
        return 0
    if not a:
        return len(b)
    if not b:
        return len(a)
    prev = list(range(len(b) + 1))
    for i, wa in enumerate(a, 1):
        curr = [i] + [0] * len(b)
        for j, wb in enumerate(b, 1):
            cost = 0 if wa == wb else 1
            curr[j] = min(
                prev[j] + 1,       # delete from a
                curr[j - 1] + 1,   # insert into a
                prev[j - 1] + cost,  # substitute
            )
        prev = curr
    return prev[-1]


def disagreement_ratio(text_a: str, text_b: str) -> float:
    """Levenshtein over normalized words, divided by the longer word count.
    0.0 = identical (after normalization), 1.0 = completely different.
    Two empty texts disagree 0.0 (nothing to disagree about), so callers should
    flag that case separately rather than reading it as agreement."""
    words_a = normalize_words(text_a.split())
    words_b = normalize_words(text_b.split())
    denom = max(len(words_a), len(words_b))
    if denom == 0:
        return 0.0
    return word_levenshtein(words_a, words_b) / denom


def _words_in_window(words: list[dict], window_start: float, window_end: float) -> str:
    """Join words whose midpoint falls in [window_start, window_end). Matches
    the midpoint-ownership rule used elsewhere so a word isn't double-counted
    in two adjacent windows."""
    picked = []
    for w in words:
        try:
            start, end = float(w.get("start", 0.0)), float(w.get("end", 0.0))
        except (TypeError, ValueError):
            continue
        midpoint = (start + end) / 2.0
        if window_start <= midpoint < window_end:
            text = str(w.get("word", "")).strip()
            if text:
                picked.append(text)
    return " ".join(picked)


def build_windows(
    words_a: list[dict],
    words_b: list[dict],
    total_duration: float,
    window_seconds: float = 20.0,
) -> list[dict]:
    windows = []
    n_windows = max(1, int(total_duration // window_seconds) + (1 if total_duration % window_seconds else 0))
    for i in range(n_windows):
        start = i * window_seconds
        end = min(start + window_seconds, total_duration)
        text_a = _words_in_window(words_a, start, end)
        text_b = _words_in_window(words_b, start, end)
        both_empty = not text_a and not text_b
        windows.append({
            "start": round(start, 3),
            "end": round(end, 3),
            "text_a": text_a,
            "text_b": text_b,
            "both_empty": both_empty,
            "disagreement": 0.0 if both_empty else round(disagreement_ratio(text_a, text_b), 4),
        })
    return windows


def compare_engines(
    file_path: str,
    engine_a: str,
    engine_b: str,
    *,
    start_seconds: float = 0.0,
    duration_seconds: Optional[float] = None,
    window_seconds: float = 20.0,
    model_size: str = "base",
    language: Optional[str] = None,
    output_dir: Optional[str] = None,
    transcribe_fn=None,
) -> dict:
    """Transcribe the same [start_seconds, start_seconds + duration_seconds)
    window with engine_a and engine_b, and report per-window disagreement.

    transcribe_fn defaults to services.transcription.transcribe_file; tests
    inject a fake to avoid running real engines.
    """
    if transcribe_fn is None:
        from services.transcription import transcribe_file as transcribe_fn

    result_a = transcribe_fn(
        file_path, model_size=model_size, engine=engine_a, language=language,
        enable_diarization=False, start_seconds=start_seconds, duration_seconds=duration_seconds,
    )
    result_b = transcribe_fn(
        file_path, model_size=model_size, engine=engine_b, language=language,
        enable_diarization=False, start_seconds=start_seconds, duration_seconds=duration_seconds,
    )

    duration = max(
        float(result_a.get("duration", 0.0) or 0.0),
        float(result_b.get("duration", 0.0) or 0.0),
        duration_seconds or 0.0,
    )
    windows = build_windows(
        result_a.get("words") or [], result_b.get("words") or [], duration, window_seconds
    )
    scored = [w["disagreement"] for w in windows if not w["both_empty"]]
    overall_disagreement = round(sum(scored) / len(scored), 4) if scored else 0.0
    flagged_empty = sum(1 for w in windows if w["both_empty"])

    report = {
        "file_path": file_path,
        "engine_a": engine_a,
        "engine_b": engine_b,
        "start_seconds": start_seconds,
        "duration_seconds": duration,
        "window_seconds": window_seconds,
        "windows": windows,
        "overall_disagreement": overall_disagreement,
        "both_empty_window_count": flagged_empty,
        "note": "disagreement between the two engines' output, not accuracy against a transcript",
    }

    if output_dir:
        os.makedirs(output_dir, exist_ok=True)
        json_path = os.path.join(output_dir, "comparison.json")
        with open(json_path, "w", encoding="utf-8") as f:
            json.dump(report, f, indent=2, ensure_ascii=False)
        report["json_path"] = json_path

        # A fresh extraction of the same window, purely for the report's
        # <audio> player, independent of whatever transcribe_fn did
        # internally (its own sample wav is already cleaned up by the time
        # we get a result back).
        audio_rel = None
        try:
            from services.audio_extract import extract_wav_16k_mono

            sample_wav = extract_wav_16k_mono(
                file_path, start_seconds=start_seconds, duration_seconds=duration_seconds
            )
            try:
                audio_rel = "sample.wav"
                shutil.copyfile(sample_wav, os.path.join(output_dir, audio_rel))
            finally:
                try:
                    os.unlink(sample_wav)
                except OSError:
                    pass
        except Exception:
            audio_rel = None

        html_path = os.path.join(output_dir, "comparison.html")
        with open(html_path, "w", encoding="utf-8") as f:
            f.write(render_html(report, audio_rel))
        report["html_path"] = html_path

    return report


def _json_for_script_tag(data: Any) -> str:
    """json.dumps, safe to embed inside a <script> tag.

    Replacing only a literal "</script" (case-sensitively) left "</SCRIPT>"
    or "</ScRiPt>" in ASR text able to close the tag early and inject HTML.
    Escaping every "<" closes that regardless of case, and also neutralizes
    "<!--". U+2028/U+2029 are escaped too: valid in a JSON string, but
    treated as line terminators by some JS engines even inside a string
    literal, which can truncate the script.
    """
    encoded = json.dumps(data, ensure_ascii=False)
    return (
        encoded.replace("<", "\\u003c")
        .replace("
", "\\u2028")
        .replace("
", "\\u2029")
    )


def render_html(report: dict, audio_rel: Optional[str]) -> str:
    engine_a = html.escape(str(report["engine_a"]))
    engine_b = html.escape(str(report["engine_b"]))
    file_label = html.escape(os.path.basename(str(report["file_path"])))
    overall = report["overall_disagreement"]
    windows_json = _json_for_script_tag(report["windows"])
    audio_html = (
        f'<audio id="player" controls preload="metadata" src="{html.escape(audio_rel)}"></audio>'
        if audio_rel
        else "<p class=\"no-audio\">No sample audio saved alongside this report.</p>"
    )
    return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Engine comparison: {engine_a} vs {engine_b} ({file_label})</title>
<style>
  body {{ font-family: -apple-system, system-ui, sans-serif; margin: 2rem; color: #1a1a1a; background: #fafafa; }}
  h1 {{ font-size: 1.25rem; }}
  .note {{ color: #666; font-size: 0.9rem; margin-bottom: 1.5rem; }}
  table {{ border-collapse: collapse; width: 100%; }}
  th, td {{ border: 1px solid #ddd; padding: 0.5rem 0.75rem; text-align: left; vertical-align: top; }}
  th {{ background: #f0f0f0; }}
  tr.both-empty {{ background: #fff8e1; }}
  .disagreement {{ font-variant-numeric: tabular-nums; }}
  .disagreement.high {{ color: #b00020; font-weight: 600; }}
  .disagreement.low {{ color: #2e7d32; }}
  button.seek {{ cursor: pointer; }}
  audio {{ width: 100%; margin-bottom: 1.5rem; }}
  .no-audio {{ color: #999; font-style: italic; }}
</style>
</head>
<body>
<h1>Engine comparison: {engine_a} vs {engine_b}</h1>
<p class="note">File: {file_label}. Overall disagreement: <span class="disagreement">{overall}</span>
(word-level edit distance between the two engines' output, normalized; not accuracy against either transcript).</p>
{audio_html}
<table>
  <thead><tr><th>Window</th><th>{engine_a}</th><th>{engine_b}</th><th>Disagreement</th></tr></thead>
  <tbody id="rows"></tbody>
</table>
<script>
const windows = {windows_json};
const rows = document.getElementById("rows");
const player = document.getElementById("player");
function esc(s) {{
  const d = document.createElement("div");
  d.textContent = s;
  return d.innerHTML;
}}
function fmtTime(s) {{
  const m = Math.floor(s / 60);
  const sec = Math.floor(s % 60).toString().padStart(2, "0");
  return m + ":" + sec;
}}
for (const w of windows) {{
  const tr = document.createElement("tr");
  if (w.both_empty) tr.className = "both-empty";
  const level = w.disagreement >= 0.5 ? "high" : w.disagreement <= 0.1 ? "low" : "";
  tr.innerHTML =
    '<td><button class="seek" data-t="' + w.start + '">' + fmtTime(w.start) + '</button>' +
    ' - ' + fmtTime(w.end) + (w.both_empty ? ' <em>(both empty)</em>' : '') + '</td>' +
    '<td>' + esc(w.text_a) + '</td>' +
    '<td>' + esc(w.text_b) + '</td>' +
    '<td class="disagreement ' + level + '">' + w.disagreement.toFixed(2) + '</td>';
  rows.appendChild(tr);
}}
rows.addEventListener("click", (e) => {{
  const btn = e.target.closest("button.seek");
  if (!btn || !player) return;
  player.currentTime = parseFloat(btn.dataset.t);
  player.play();
}});
</script>
</body>
</html>
"""