Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
#!/usr/bin/env python3
"""Local-only, deterministic meeting diarization evaluator.
This tool reads RTTM timelines and optional speaker-attributed transcript JSON. It never uploads,
rewrites, or copies source recordings. Reports contain aggregate metrics and file fingerprints only.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import sys
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import Any
SCHEMA_VERSION = 1
DEPENDENCY_REVISION = "3fd63887eef1dc25edea8263ce4b44aa854d898b"
DIARIZATION_FINGERPRINT = (
"FluidAudio-offline-v1@3fd63887eef1dc25edea8263ce4b44aa854d898b;"
"config=community-default-v1"
)
@dataclass(frozen=True)
class Segment:
speaker: str
start: float
end: float
def resolve(base: Path, value: str) -> Path:
path = Path(os.path.expanduser(value))
return path if path.is_absolute() else (base / path).resolve()
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def read_rttm(path: Path) -> list[Segment]:
segments: list[Segment] = []
for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
fields = line.split()
if len(fields) < 8 or fields[0] != "SPEAKER":
raise ValueError(f"{path}:{line_number}: expected an RTTM SPEAKER row")
start, duration = float(fields[3]), float(fields[4])
if start < 0 or duration <= 0 or not math.isfinite(start + duration):
raise ValueError(f"{path}:{line_number}: invalid start/duration")
segments.append(Segment(fields[7], start, start + duration))
if not segments:
raise ValueError(f"{path}: contains no speaker segments")
return sorted(segments, key=lambda item: (item.start, item.end, item.speaker))
def active(segments: list[Segment], midpoint: float) -> set[str]:
return {segment.speaker for segment in segments if segment.start <= midpoint < segment.end}
def scored_intervals(
reference: list[Segment], hypothesis: list[Segment], collar: float, score_overlap: bool
) -> list[tuple[float, set[str], set[str]]]:
boundaries = sorted({point for segment in reference + hypothesis for point in (segment.start, segment.end)})
reference_boundaries = [point for segment in reference for point in (segment.start, segment.end)]
intervals: list[tuple[float, set[str], set[str]]] = []
for left, right in zip(boundaries, boundaries[1:]):
if right <= left:
continue
midpoint = (left + right) / 2
if collar and any(abs(midpoint - point) < collar for point in reference_boundaries):
continue
ref_active, hyp_active = active(reference, midpoint), active(hypothesis, midpoint)
if not score_overlap and len(ref_active) > 1:
continue
intervals.append((right - left, ref_active, hyp_active))
return intervals
def best_assignment(weights: list[list[float]]) -> list[tuple[int, int]]:
"""Maximum-weight one-to-one assignment; corpus speaker counts are intentionally small."""
rows = len(weights)
columns = max((len(row) for row in weights), default=0)
size = max(rows, columns)
if size > 12:
raise ValueError("more than 12 speakers is unsupported by the deterministic evaluator")
padded = [row + [0.0] * (size - len(row)) for row in weights]
padded += [[0.0] * size for _ in range(size - rows)]
@lru_cache(maxsize=None)
def solve(row: int, used_columns: int) -> tuple[float, tuple[int, ...]]:
if row == size:
return 0.0, ()
best: tuple[float, tuple[int, ...]] | None = None
for column in range(size):
if used_columns & (1 << column):
continue
remaining_score, remaining_columns = solve(row + 1, used_columns | (1 << column))
candidate = (padded[row][column] + remaining_score, (column,) + remaining_columns)
if best is None or candidate[0] > best[0]:
best = candidate
assert best is not None
return best
_, columns_by_row = solve(0, 0)
return [(row, column) for row, column in enumerate(columns_by_row) if row < rows and column < columns]
def overlap_matrix(
intervals: list[tuple[float, set[str], set[str]]], references: list[str], hypotheses: list[str]
) -> list[list[float]]:
matrix = [[0.0 for _ in hypotheses] for _ in references]
ref_index, hyp_index = {v: i for i, v in enumerate(references)}, {v: i for i, v in enumerate(hypotheses)}
for duration, ref_active, hyp_active in intervals:
for ref in ref_active:
for hyp in hyp_active:
matrix[ref_index[ref]][hyp_index[hyp]] += duration
return matrix
def duration_by_speaker(segments: list[Segment]) -> dict[str, float]:
return {
speaker: sum(segment.end - segment.start for segment in segments if segment.speaker == speaker)
for speaker in sorted({segment.speaker for segment in segments})
}
def score_timeline(reference: list[Segment], hypothesis: list[Segment], collar: float, score_overlap: bool) -> dict[str, Any]:
intervals = scored_intervals(reference, hypothesis, collar, score_overlap)
references = sorted({segment.speaker for segment in reference})
hypotheses = sorted({segment.speaker for segment in hypothesis})
matrix = overlap_matrix(intervals, references, hypotheses)
assignment = best_assignment(matrix)
hyp_to_ref = {hypotheses[column]: references[row] for row, column in assignment}
reference_time = miss = false_alarm = confusion = 0.0
for duration, ref_active, hyp_active in intervals:
reference_time += duration * len(ref_active)
miss += duration * max(0, len(ref_active) - len(hyp_active))
false_alarm += duration * max(0, len(hyp_active) - len(ref_active))
correct = sum(1 for hyp in hyp_active if hyp_to_ref.get(hyp) in ref_active)
confusion += duration * max(0, min(len(ref_active), len(hyp_active)) - correct)
ref_durations = duration_by_speaker(reference)
hyp_durations = duration_by_speaker(hypothesis)
mapped_column = {row: column for row, column in assignment}
jaccard_errors, recovered, coverage_and_purity = [], {}, {}
for row, speaker in enumerate(references):
column = mapped_column.get(row)
intersection = matrix[row][column] if column is not None else 0.0
hyp_duration = hyp_durations.get(hypotheses[column], 0.0) if column is not None else 0.0
union = ref_durations[speaker] + hyp_duration - intersection
jaccard_errors.append(1.0 - intersection / union if union else 0.0)
coverage = intersection / ref_durations[speaker] if ref_durations[speaker] else 0.0
purity = intersection / hyp_duration if hyp_duration else 0.0
coverage_and_purity[speaker] = {"coverage": coverage, "purity": purity}
recovered[speaker] = coverage >= 0.5 and purity >= 0.5
buckets = {"<1s": [0, 0], "1-3s": [0, 0], "3-10s": [0, 0], ">10s": [0, 0]}
for speaker, duration in ref_durations.items():
bucket = "<1s" if duration < 1 else "1-3s" if duration < 3 else "3-10s" if duration < 10 else ">10s"
buckets[bucket][1] += 1
buckets[bucket][0] += int(recovered[speaker])
# A meaningful overlap link is at least 250 ms and at least 5% of the shorter speaker.
links: list[tuple[int, int]] = []
for row, ref in enumerate(references):
for column, hyp in enumerate(hypotheses):
threshold = max(0.25, 0.05 * min(ref_durations[ref], hyp_durations[hyp]))
if matrix[row][column] >= threshold:
links.append((row, column))
merge_count = sum(1 for column in range(len(hypotheses)) if len({r for r, c in links if c == column}) > 1)
split_count = sum(1 for row in range(len(references)) if len({c for r, c in links if r == row}) > 1)
denominator = reference_time or 1.0
return {
"collarSeconds": collar,
"overlapScored": score_overlap,
"referenceSpeakers": len(references),
"hypothesisSpeakers": len(hypotheses),
"speakerCountError": len(hypotheses) - len(references),
"mergeCount": merge_count,
"splitCount": split_count,
"recoveryByDuration": {
name: {"recovered": values[0], "total": values[1], "recall": values[0] / values[1] if values[1] else None}
for name, values in buckets.items()
},
"der": (miss + false_alarm + confusion) / denominator,
"derComponents": {
"miss": miss / denominator,
"falseAlarm": false_alarm / denominator,
"confusion": confusion / denominator,
},
"jer": sum(jaccard_errors) / len(jaccard_errors) if jaccard_errors else 0.0,
"speakerCoverageAndPurity": coverage_and_purity,
}
def words(text: str) -> list[str]:
return ["".join(character.lower() for character in token if character.isalnum() or character == "'")
for token in text.split() if any(character.isalnum() for character in token)]
def edit_distance(left: list[str], right: list[str]) -> int:
row = list(range(len(right) + 1))
for left_index, left_word in enumerate(left, 1):
next_row = [left_index]
for right_index, right_word in enumerate(right, 1):
next_row.append(min(next_row[-1] + 1, row[right_index] + 1, row[right_index - 1] + (left_word != right_word)))
row = next_row
return row[-1]
def cpwer(reference_path: Path, hypothesis_path: Path) -> dict[str, Any]:
reference = json.loads(reference_path.read_text(encoding="utf-8"))
hypothesis = json.loads(hypothesis_path.read_text(encoding="utf-8"))
ref_texts = [words(value) for _, value in sorted(reference.items())]
hyp_texts = [words(value) for _, value in sorted(hypothesis.items())]
size = max(len(ref_texts), len(hyp_texts))
costs = []
for ref_index in range(size):
ref_words = ref_texts[ref_index] if ref_index < len(ref_texts) else []
costs.append([
edit_distance(ref_words, hyp_texts[hyp_index] if hyp_index < len(hyp_texts) else [])
for hyp_index in range(size)
])
# Convert minimum-cost assignment into maximum-weight assignment.
maximum = max((value for row in costs for value in row), default=0)
assignment = best_assignment([[maximum - value for value in row] for row in costs])
errors = sum(costs[row][column] for row, column in assignment)
count = sum(len(value) for value in ref_texts)
return {"errors": errors, "referenceWords": count, "cpWER": errors / count if count else 0.0}
def validate_manifest(manifest_path: Path, require_files: bool = True) -> dict[str, Any]:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if manifest.get("schemaVersion") != SCHEMA_VERSION:
raise ValueError(f"schemaVersion must be {SCHEMA_VERSION}")
dependency = manifest.get("dependency", {})
if dependency.get("revision") != DEPENDENCY_REVISION:
raise ValueError("manifest dependency revision does not match the pinned build")
if dependency.get("diarizationFingerprint") != DIARIZATION_FINGERPRINT:
raise ValueError("manifest diarization fingerprint does not match the pinned build")
fixtures = manifest.get("fixtures", [])
if not fixtures:
raise ValueError("manifest must contain at least one fixture")
ids = [fixture.get("id") for fixture in fixtures]
if any(not value for value in ids) or len(ids) != len(set(ids)):
raise ValueError("fixture ids must be present and unique")
if not any(fixture.get("split") == "heldOut" for fixture in fixtures):
raise ValueError("manifest must define a heldOut fixture before tuning")
base = manifest_path.parent
for fixture in fixtures:
if fixture.get("split") not in ("development", "heldOut"):
raise ValueError(f"{fixture['id']}: split must be development or heldOut")
if fixture.get("status", "annotated") == "annotated":
for key in ("referenceRTTM", "hypothesisRTTM"):
if key not in fixture:
raise ValueError(f"{fixture['id']}: annotated fixture is missing {key}")
if require_files and not resolve(base, fixture[key]).is_file():
raise ValueError(f"{fixture['id']}: missing {key} file")
return manifest
def score_manifest(manifest_path: Path) -> dict[str, Any]:
manifest = validate_manifest(manifest_path)
base, reports = manifest_path.parent, []
for fixture in manifest["fixtures"]:
if fixture.get("status", "annotated") != "annotated":
inventory_report: dict[str, Any] = {
"id": fixture["id"],
"split": fixture["split"],
"status": "inventory",
"tags": fixture.get("tags", []),
}
expected = fixture.get("expectedSpeakerCount")
observed = fixture.get("observedRemoteSpeakerCount")
if isinstance(expected, int):
inventory_report["expectedSpeakerCount"] = expected
if isinstance(observed, int):
inventory_report["observedRemoteSpeakerCount"] = observed
if isinstance(expected, int) and isinstance(observed, int):
inventory_report["speakerCountError"] = observed - expected
for manifest_key, report_key in (
("hypothesisRTTM", "hypothesisSHA256"),
("runReport", "runReportSHA256"),
):
if fixture.get(manifest_key):
path = resolve(base, fixture[manifest_key])
if path.is_file():
inventory_report[report_key] = sha256(path)
reports.append(inventory_report)
continue
reference_path = resolve(base, fixture["referenceRTTM"])
hypothesis_path = resolve(base, fixture["hypothesisRTTM"])
reference, hypothesis = read_rttm(reference_path), read_rttm(hypothesis_path)
report: dict[str, Any] = {
"id": fixture["id"], "split": fixture["split"], "tags": fixture.get("tags", []),
"status": "scored", "referenceSHA256": sha256(reference_path),
"hypothesisSHA256": sha256(hypothesis_path),
"comparable": score_timeline(reference, hypothesis, 0.25, False),
"strict": score_timeline(reference, hypothesis, 0.0, True),
}
if fixture.get("referenceSpeakerText") and fixture.get("hypothesisSpeakerText"):
report["cpWER"] = cpwer(resolve(base, fixture["referenceSpeakerText"]), resolve(base, fixture["hypothesisSpeakerText"]))
reports.append(report)
return {
"schemaVersion": SCHEMA_VERSION,
"dependency": manifest["dependency"],
"normalization": manifest.get("normalization", {}),
"fixtures": reports,
}
def self_test() -> None:
reference = [Segment("A", 0, 2), Segment("B", 2, 4)]
perfect = score_timeline(reference, reference, 0, True)
assert perfect["der"] == 0 and perfect["mergeCount"] == 0 and perfect["splitCount"] == 0
merged = score_timeline(reference, [Segment("X", 0, 4)], 0, True)
assert merged["mergeCount"] == 1 and merged["speakerCountError"] == -1 and merged["der"] == 0.5
split = score_timeline([Segment("A", 0, 4)], [Segment("X", 0, 2), Segment("Y", 2, 4)], 0, True)
assert split["splitCount"] == 1 and split["speakerCountError"] == 1 and split["der"] == 0.5
print("meeting_diarization_eval: self-test passed")
def main() -> int:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command", required=True)
subparsers.add_parser("self-test")
validate = subparsers.add_parser("validate")
validate.add_argument("manifest", type=Path)
score = subparsers.add_parser("score")
score.add_argument("manifest", type=Path)
score.add_argument("--output", type=Path)
arguments = parser.parse_args()
if arguments.command == "self-test":
self_test()
return 0
manifest_path = arguments.manifest.resolve()
if arguments.command == "validate":
manifest = validate_manifest(manifest_path)
print(f"valid manifest: {len(manifest['fixtures'])} fixtures")
return 0
report = score_manifest(manifest_path)
encoded = json.dumps(report, indent=2, sort_keys=True) + "\n"
if arguments.output:
arguments.output.parent.mkdir(parents=True, exist_ok=True)
arguments.output.write_text(encoded, encoding="utf-8")
print(arguments.output)
else:
print(encoded, end="")
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except (OSError, ValueError, json.JSONDecodeError) as error:
print(f"error: {error}", file=sys.stderr)
raise SystemExit(2)