Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Tests with the documented WhisperX settings (#409 follow-up)

Murtaza Nasir2d1f3da31cfd
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3f6dc35d..6bf31046 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -57,6 +57,67 @@ jobs:
TEST_DATABASE_URI: postgresql://speakr:speakr@localhost:5432/speakr_test
run: pytest tests/ -q --cov=src --cov-report=term-missing --cov-fail-under=60
+ # Speakr and whisperx-asr-service set up exactly as their docs describe
+ # (config/env.whisperx.example; the service's docker-compose.yml plus its
+ # SETUP_GUIDE .env block), talking over HTTP. The service runs its real code
+ # on stand-in ML packages, so this needs no GPU, secrets or model downloads.
+ # It would have caught #409. Also checks that the model rules vendored in
+ # tests/contract still match the service's own code.
+ e2e-whisperx-defaults:
+ runs-on: ubuntu-latest
+ env:
+ WHISPERX_REF: ${{ vars.WHISPERX_REF || 'main' }}
+
+ steps:
+ - name: Checkout Speakr
+ uses: actions/checkout@v5
+
+ - name: Checkout whisperx-asr-service
+ uses: actions/checkout@v5
+ with:
+ repository: murtaza-nasir/whisperx-asr-service
+ ref: ${{ env.WHISPERX_REF }}
+ path: whisperx
+
+ - name: Set up Python
+ uses: actions/setup-python@v6
+ with:
+ python-version: '3.11'
+ cache: pip
+
+ - name: Install ffmpeg
+ run: sudo apt-get update && sudo apt-get install -y ffmpeg
+
+ - name: Install dependencies
+ run: |
+ pip install -r requirements.txt pytest
+ python -m venv /tmp/wxv
+ /tmp/wxv/bin/pip install -r whisperx/tests/requirements-ci.txt
+
+ - name: Start whisperx-asr-service (documented settings, stand-in models)
+ run: |
+ /tmp/wxv/bin/python whisperx/tests/e2e/stub_server.py --port 9000 \
+ --env-file whisperx/tests/defaults/minimal.env > /tmp/whisperx.log 2>&1 &
+ for i in $(seq 1 60); do curl -fs http://127.0.0.1:9000/health && break; sleep 1; done
+ curl -fs http://127.0.0.1:9000/health
+
+ - name: Contract - vendored model rules match the service
+ env:
+ WHISPERX_REPO: whisperx
+ WHISPERX_PYTHON: /tmp/wxv/bin/python
+ run: pytest tests/contract -q -p no:cacheprovider
+
+ - name: End to end with the documented settings
+ env:
+ SPEAKR_E2E: '1'
+ WHISPERX_URL: http://127.0.0.1:9000
+ WHISPERX_LOG: /tmp/whisperx.log
+ run: pytest tests/e2e -q -p no:cacheprovider -rxs
+
+ - name: Service log
+ if: always()
+ run: tail -n 200 /tmp/whisperx.log || true
+
js-tests:
runs-on: ubuntu-latest
diff --git a/tests/contract/fake_asr_server.py b/tests/contract/fake_asr_server.py
new file mode 100644
index 00000000..cdd5bcb1
--- /dev/null
+++ b/tests/contract/fake_asr_server.py
@@ -0,0 +1,133 @@
+"""A stand-in for whisperx-asr-service's HTTP API, for Speakr's tests.
+
+It runs on a real socket in a background thread, so Speakr's asr_endpoint
+connector talks to it exactly as it talks to the service. Every /asr request
+is recorded and answered with the model decision the real service would make
+(tests/contract/whisperx_model_rules.py): a request the service would refuse
+gets the same status and error body.
+"""
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import parse_qs, urlparse
+
+from tests.contract.whisperx_model_rules import CANONICAL_MODELS, ModelRejected, ServiceModelRules
+
+EMBEDDING_DIM = 256
+
+
+def _embedding(seed):
+ """A fixed, unit-scale voice embedding per speaker label."""
+ return [(((i * 31 + seed * 17) % 97) / 97.0) - 0.5 for i in range(EMBEDDING_DIM)]
+
+
+class FakeASRServer:
+ """Context manager: ``with FakeASRServer(rules) as asr: asr.url``."""
+
+ def __init__(self, rules=None):
+ self.rules = rules or ServiceModelRules()
+ self.requests = []
+ self._lock = threading.Lock()
+ self._server = None
+ self._thread = None
+
+ # -- lifecycle --------------------------------------------------------
+ def __enter__(self):
+ fake = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args): # keep test output quiet
+ pass
+
+ def _json(self, status, body):
+ data = json.dumps(body).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def do_GET(self):
+ path = urlparse(self.path).path
+ if path in ("/health", "/"):
+ return self._json(200, {"status": "healthy"})
+ if path == "/v1/models":
+ return self._json(200, {"object": "list", "data": [
+ {"id": name, "object": "model"} for name in ("whisper-1",) + CANONICAL_MODELS
+ ]})
+ return self._json(404, {"detail": "Not Found"})
+
+ def do_POST(self):
+ url = urlparse(self.path)
+ length = int(self.headers.get("Content-Length") or 0)
+ body = self.rfile.read(length) if length else b""
+ if url.path != "/asr":
+ return self._json(404, {"detail": "Not Found"})
+ query = {k: v[0] for k, v in parse_qs(url.query, keep_blank_values=True).items()}
+ record = {
+ "model_sent": "model" in query,
+ "model": query.get("model"),
+ "params": query,
+ "filename": _filename(body),
+ }
+ try:
+ record["resolved"] = fake.rules.resolve(query.get("model"))
+ record["status"] = 200
+ except ModelRejected as rejected:
+ record["resolved"] = None
+ record["status"] = rejected.status
+ record["detail"] = rejected.detail
+ with fake._lock:
+ fake.requests.append(record)
+ if record["status"] != 200:
+ return self._json(record["status"], {"detail": record["detail"]})
+ return self._json(200, _transcript(query))
+
+ self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+ self._thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self._server.shutdown()
+ self._server.server_close()
+ self._thread.join(timeout=5)
+
+ @property
+ def url(self):
+ host, port = self._server.server_address[:2]
+ return f"http://{host}:{port}"
+
+ def reset(self):
+ with self._lock:
+ self.requests.clear()
+
+
+def _filename(body):
+ """The uploaded file name from a multipart body, for readable failures."""
+ marker = b'filename="'
+ start = body.find(marker)
+ if start < 0:
+ return None
+ start += len(marker)
+ end = body.find(b'"', start)
+ return body[start:end].decode(errors="replace") if end > start else None
+
+
+def _transcript(query):
+ """A small diarized transcript with one speaker, as the service returns it."""
+ segments = [
+ {"start": 0.0, "end": 2.0, "text": "Hello from the test service.", "speaker": "SPEAKER_00",
+ "words": [{"word": "Hello", "start": 0.0, "end": 0.5, "speaker": "SPEAKER_00"}]},
+ {"start": 2.0, "end": 4.0, "text": "This is a second sentence.", "speaker": "SPEAKER_00",
+ "words": [{"word": "This", "start": 2.0, "end": 2.3, "speaker": "SPEAKER_00"}]},
+ ]
+ body = {
+ "text": " ".join(s["text"] for s in segments),
+ "language": query.get("language") or "en",
+ "segments": segments,
+ }
+ if str(query.get("return_speaker_embeddings", "")).lower() == "true":
+ body["speaker_embeddings"] = {"SPEAKER_00": _embedding(0)}
+ return body
diff --git a/tests/contract/test_whisperx_contract.py b/tests/contract/test_whisperx_contract.py
new file mode 100644
index 00000000..9168866b
--- /dev/null
+++ b/tests/contract/test_whisperx_contract.py
@@ -0,0 +1,129 @@
+"""The vendored whisperx model rules agree with the service's own code.
+
+tests/contract/whisperx_model_rules.py is what Speakr's fake ASR server
+answers with. This test imports the real app.pipeline from a
+whisperx-asr-service checkout (WHISPERX_REPO) and compares its decisions with
+the vendored copy on a table of service settings and request models, so a
+change in the service's rules fails Speakr's CI and not a user's install.
+
+Each comparison runs in a subprocess: the service's pipeline needs its
+stand-in ML packages (tests/e2e/stubs in the service repo) or the real ones,
+and neither may leak into Speakr's own test process.
+
+Skipped when WHISPERX_REPO is not set. CI checks the service out, sets it,
+and sets WHISPERX_PYTHON to the interpreter with the service's CI
+requirements installed (tests/requirements-ci.txt in the service repo).
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from tests.contract.whisperx_model_rules import ModelRejected, ServiceModelRules, is_loadable_model
+
+WHISPERX_REPO = os.environ.get("WHISPERX_REPO", "").strip()
+
+pytestmark = pytest.mark.skipif(
+ not WHISPERX_REPO, reason="set WHISPERX_REPO to a whisperx-asr-service checkout")
+
+MODEL_ENV = ("PRELOAD_MODEL", "DEFAULT_MODEL", "OPENAI_WHISPER1_MODEL", "ALLOWED_MODELS",
+ "MAX_LOADED_MODELS", "ASR_BACKEND")
+
+# Service environments as compose delivers them: a key with "" is a variable
+# the .env file does not set.
+SERVICE_ENVS = {
+ "unset": {},
+ "compose-empty (issue #409)": {"PRELOAD_MODEL": ""},
+ "whitespace": {"PRELOAD_MODEL": " "},
+ "documented large-v3": {"PRELOAD_MODEL": "large-v3"},
+ "preload medium": {"PRELOAD_MODEL": "medium"},
+ "no preload, default small": {"PRELOAD_MODEL": "", "DEFAULT_MODEL": "small"},
+ "alias as preload": {"PRELOAD_MODEL": "whisper-large-v2"},
+ "typo as preload": {"PRELOAD_MODEL": "large-v4"},
+ "whisper-1 pointing at itself": {"PRELOAD_MODEL": "", "OPENAI_WHISPER1_MODEL": "whisper-1"},
+ "allowlist": {"PRELOAD_MODEL": "large-v3", "ALLOWED_MODELS": "small, whisper-medium"},
+}
+
+REQUEST_MODELS = [
+ None, "", " ", "large-v3", " small ", "turbo", "distil-large-v3.5", "medium.en",
+ "whisper-1", "whisper-large-v3", "whisper-medium", "whisper-small.en",
+ "Systran/faster-whisper-small", "large-v4", "whisper-2", "org/repo/extra",
+]
+
+_PROBE = r"""
+import json, os, sys
+repo = os.environ["WHISPERX_REPO"]
+sys.path.insert(0, repo)
+try:
+ import whisperx, faster_whisper # noqa: F401 (real packages present: use them)
+except Exception:
+ sys.path.insert(0, os.path.join(repo, "tests", "e2e", "stubs"))
+ for name in [m for m in sys.modules if m.split(".")[0] in ("torch", "whisperx", "faster_whisper")]:
+ del sys.modules[name]
+import app.pipeline as p
+models = json.loads(os.environ["CONTRACT_MODELS"])
+generation = "fixed" if hasattr(p, "InvalidModelError") else "legacy"
+out = {"generation": generation, "default": p.DEFAULT_MODEL, "results": []}
+for model in models:
+ try:
+ out["results"].append({"ok": p.resolve_model_name(model)})
+ except Exception as e:
+ out["results"].append({"error": type(e).__name__})
+print("CONTRACT" + json.dumps(out))
+"""
+
+
+def _real_decisions(env):
+ child_env = {k: v for k, v in os.environ.items() if k not in MODEL_ENV}
+ child_env.update(env)
+ child_env["WHISPERX_REPO"] = os.path.abspath(WHISPERX_REPO)
+ child_env["CONTRACT_MODELS"] = json.dumps(REQUEST_MODELS)
+ child_env.setdefault("DEVICE", "cpu")
+ python = os.environ.get("WHISPERX_PYTHON") or sys.executable
+ proc = subprocess.run([python, "-c", _PROBE], env=child_env, capture_output=True,
+ text=True, timeout=180)
+ lines = [l for l in proc.stdout.splitlines() if l.startswith("CONTRACT")]
+ assert proc.returncode == 0 and lines, f"could not import the service's pipeline:\n{proc.stderr[-3000:]}"
+ return json.loads(lines[-1][len("CONTRACT"):])
+
+
[email protected]("label", list(SERVICE_ENVS))
+def test_vendored_rules_match_the_service(label):
+ env = SERVICE_ENVS[label]
+ real = _real_decisions(env)
+ rules = ServiceModelRules(env, generation=real["generation"])
+ assert rules.default_model == real["default"], (label, "default model")
+
+ for model, decision in zip(REQUEST_MODELS, real["results"]):
+ try:
+ expected = rules.resolve(model)
+ rejected = None
+ except ModelRejected as e:
+ expected, rejected = None, e
+ if real["generation"] == "fixed":
+ if rejected is None:
+ assert decision == {"ok": expected}, (label, model, decision)
+ else:
+ assert decision == {"error": "InvalidModelError"}, (label, model, decision)
+ else:
+ # The legacy service resolves any name and only faster-whisper
+ # refuses it later (HTTP 500), which the vendored rules model as
+ # a rejection of a name that is not loadable.
+ assert "ok" in decision, (label, model, decision)
+ if rejected is None:
+ assert decision["ok"] == expected, (label, model, decision)
+ else:
+ assert rejected.status == 500 and not is_loadable_model(decision["ok"]), (label, model, decision)
+
+
+def test_the_service_default_is_never_empty_on_current_releases():
+ """The #409 regression, checked against the service itself."""
+ real = _real_decisions({"PRELOAD_MODEL": ""})
+ if real["generation"] == "legacy":
+ pytest.skip("service checkout predates 0.4.2; its empty default is the #409 bug")
+ assert real["default"] == "large-v3"
diff --git a/tests/contract/whisperx_model_rules.py b/tests/contract/whisperx_model_rules.py
new file mode 100644
index 00000000..c320f114
--- /dev/null
+++ b/tests/contract/whisperx_model_rules.py
@@ -0,0 +1,143 @@
+"""How whisperx-asr-service chooses the model for a request, vendored for tests.
+
+Speakr's fake ASR server (tests/contract/fake_asr_server.py) answers with these
+rules, so Speakr's tests fail when Speakr sends a request the real service
+would reject. test_whisperx_contract.py checks this copy against the service's
+own code (WHISPERX_REPO) so the two cannot drift apart unnoticed.
+
+Two service generations are modelled:
+
+- ``fixed`` (0.4.2 onward): an empty or whitespace request model uses
+ DEFAULT_MODEL, which is DEFAULT_MODEL / PRELOAD_MODEL / large-v3 and never
+ empty; unknown names are rejected with HTTP 400.
+- ``legacy`` (0.2.0 to 0.4.1): the default is PRELOAD_MODEL as compose passes
+ it, which is an empty string when .env does not set it (Speakr #409), and an
+ unknown name reaches faster-whisper, which fails with HTTP 500.
+"""
+
+import os
+
+BUILTIN_DEFAULT_MODEL = "large-v3"
+
+# faster_whisper.available_models() in the service image (faster-whisper 1.2.1).
+CANONICAL_MODELS = (
+ "tiny.en", "tiny", "base.en", "base", "small.en", "small",
+ "medium.en", "medium", "large-v1", "large-v2", "large-v3", "large",
+ "distil-large-v2", "distil-medium.en", "distil-small.en",
+ "distil-large-v3", "distil-large-v3.5", "large-v3-turbo", "turbo",
+)
+
+STATIC_ALIASES = {
+ "whisper-large-v3": "large-v3",
+ "whisper-large-v2": "large-v2",
+ "whisper-medium": "medium",
+ "whisper-small": "small",
+ "whisper-base": "base",
+ "whisper-tiny": "tiny",
+}
+
+_HF_REPO_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.")
+
+
+class ModelRejected(Exception):
+ """The service would refuse this request; ``status`` is its HTTP code."""
+
+ def __init__(self, status, detail):
+ super().__init__(detail)
+ self.status = status
+ self.detail = detail
+
+
+def _env_or_none(value):
+ if value is None:
+ return None
+ value = value.strip()
+ return value or None
+
+
+def is_hf_repo_id(name):
+ parts = name.split("/")
+ return len(parts) == 2 and all(parts) and all(set(p) <= _HF_REPO_CHARS for p in parts)
+
+
+def is_loadable_model(name):
+ return bool(name) and (name in CANONICAL_MODELS or is_hf_repo_id(name) or os.path.isdir(name))
+
+
+def _strip_whisper_prefix(name):
+ if name.startswith("whisper-") and name[len("whisper-"):] in CANONICAL_MODELS:
+ return name[len("whisper-"):]
+ return name
+
+
+def resolve_default(raw, openai_whisper1=None):
+ """The fixed service's DEFAULT_MODEL for a configured value (never empty)."""
+ name = (raw or "").strip()
+ if name == "whisper-1":
+ name = _env_or_none(openai_whisper1) or BUILTIN_DEFAULT_MODEL
+ name = STATIC_ALIASES.get(name, name)
+ name = _strip_whisper_prefix(name)
+ return name if is_loadable_model(name) else BUILTIN_DEFAULT_MODEL
+
+
+class ServiceModelRules:
+ """The model decision the service makes for one /asr request.
+
+ ``env`` holds the service's container environment as compose delivers it:
+ a key present with value "" is how compose forwards a variable that the
+ .env file does not set.
+ """
+
+ def __init__(self, env=None, generation="fixed"):
+ env = dict(env or {})
+ self.generation = generation
+ if generation == "fixed":
+ preload = _env_or_none(env.get("PRELOAD_MODEL"))
+ configured = _env_or_none(env.get("DEFAULT_MODEL")) or preload or BUILTIN_DEFAULT_MODEL
+ whisper1 = _env_or_none(env.get("OPENAI_WHISPER1_MODEL"))
+ self.default_model = resolve_default(configured, whisper1)
+ self.whisper1 = resolve_default(whisper1) if whisper1 else self.default_model
+ allowed = []
+ for entry in (_env_or_none(env.get("ALLOWED_MODELS")) or "").split(","):
+ entry = entry.strip()
+ if entry:
+ mapped = self._map_alias(entry)
+ if mapped not in allowed:
+ allowed.append(mapped)
+ self.allowed = allowed
+ elif generation == "legacy":
+ # os.getenv("PRELOAD_MODEL", "large-v3"): the fallback only applies
+ # when the variable is absent, not when compose forwards it empty.
+ self.default_model = env["PRELOAD_MODEL"] if "PRELOAD_MODEL" in env else BUILTIN_DEFAULT_MODEL
+ self.whisper1 = env.get("OPENAI_WHISPER1_MODEL", self.default_model)
+ self.allowed = []
+ else:
+ raise ValueError(f"unknown service generation {generation!r}")
+
+ def _map_alias(self, name):
+ if name == "whisper-1":
+ return self.whisper1
+ if name in STATIC_ALIASES:
+ return STATIC_ALIASES[name]
+ return _strip_whisper_prefix(name)
+
+ def resolve(self, model):
+ """Name the service loads for a request's ``model``; raises ModelRejected."""
+ if self.generation == "legacy":
+ # No stripping; an empty model returns DEFAULT_MODEL unmapped.
+ name = self._map_alias(model) if model else self.default_model
+ if not is_loadable_model(name):
+ raise ModelRejected(
+ 500,
+ f"Invalid model size {name!r}, expected one of: {', '.join(CANONICAL_MODELS)}",
+ )
+ return name
+ name = (model or "").strip()
+ if not name:
+ return self.default_model
+ resolved = self._map_alias(name)
+ if not is_loadable_model(resolved):
+ raise ModelRejected(400, f"Unknown model {name!r}.")
+ if self.allowed and resolved not in self.allowed and resolved != self.default_model:
+ raise ModelRejected(400, f"Model {name!r} is not allowed on this server.")
+ return resolved
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
new file mode 100644
index 00000000..d67283ec
--- /dev/null
+++ b/tests/e2e/README.md
@@ -0,0 +1,29 @@
+# Tests with the documented settings
+
+Issue #409 reached users because every test ran with a fully configured
+environment. These tests start from what the docs tell a new user to do.
+
+| Layer | Where | What it checks | When it runs |
+|---|---|---|---|
+| 1 | `tests/test_documented_defaults.py` | Speakr configured only from `config/env.whisperx.example`, no admin settings, against a fake ASR service that answers with whisperx-asr-service's model rules. Every path that sends audio (voice embedding check, upload, reprocess, bulk reprocess, API v1, incognito, upload join), across service versions and Speakr settings. | Every run of the normal suite |
+| 2 | `tests/contract/` | The vendored model rules (`whisperx_model_rules.py`) agree with the service's own `resolve_model_name`. | CI job `e2e-whisperx-defaults` (needs `WHISPERX_REPO`) |
+| 3 | `tests/e2e/test_whisperx_defaults_e2e.py` | Both services as real processes with their documented settings: Speakr under gunicorn, whisperx-asr-service on stand-in ML packages. Startup check, upload with speakers and embeddings, upload join, clean logs. | CI job `e2e-whisperx-defaults` (needs `SPEAKR_E2E=1`) |
+
+No layer needs a GPU, secrets or model downloads.
+
+## Running layers 2 and 3 locally
+
+```bash
+git clone https://github.com/murtaza-nasir/whisperx-asr-service.git /tmp/wx
+python -m venv /tmp/wxv && /tmp/wxv/bin/pip install -r /tmp/wx/tests/requirements-ci.txt
+/tmp/wxv/bin/python /tmp/wx/tests/e2e/stub_server.py --port 9000 \
+ --env-file /tmp/wx/tests/defaults/minimal.env > /tmp/wx.log 2>&1 &
+
+WHISPERX_REPO=/tmp/wx WHISPERX_PYTHON=/tmp/wxv/bin/python pytest tests/contract -q
+SPEAKR_E2E=1 WHISPERX_URL=http://127.0.0.1:9000 WHISPERX_LOG=/tmp/wx.log pytest tests/e2e -q -rxs
+```
+
+To test the service setup behind #409 (no `PRELOAD_MODEL` in the service's
+`.env`), start the stub server with `--set PRELOAD_MODEL=`.
+
+`KEEP_E2E_LOGS=1` keeps Speakr's log and database after the run.
diff --git a/tests/e2e/fake_llm_server.py b/tests/e2e/fake_llm_server.py
new file mode 100644
index 00000000..01161601
--- /dev/null
+++ b/tests/e2e/fake_llm_server.py
@@ -0,0 +1,76 @@
+"""A minimal OpenAI-compatible chat endpoint for the end-to-end test.
+
+The WhisperX example configures a language model for titles and summaries.
+The end-to-end test is about the transcription path, so the model is
+replaced by this server, which answers every chat completion with a fixed
+title or summary (or an empty JSON object when JSON is requested). That keeps
+Speakr's log free of errors that have nothing to do with the ASR service.
+"""
+
+import json
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+class FakeLLMServer:
+ def __init__(self):
+ self.requests = 0
+ self._server = None
+ self._thread = None
+
+ def __enter__(self):
+ fake = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def _send(self, status, body, content_type="application/json"):
+ data = body if isinstance(body, bytes) else json.dumps(body).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def do_GET(self):
+ if self.path.rstrip("/").endswith("/models"):
+ return self._send(200, {"object": "list", "data": [{"id": "test-model", "object": "model"}]})
+ return self._send(404, {"error": "not found"})
+
+ def do_POST(self):
+ length = int(self.headers.get("Content-Length") or 0)
+ request = json.loads(self.rfile.read(length) or b"{}")
+ fake.requests += 1
+ if not self.path.rstrip("/").endswith("/chat/completions"):
+ return self._send(404, {"error": "not found"})
+ wants_json = bool(request.get("response_format"))
+ content = "{}" if wants_json else "Test meeting summary."
+ if request.get("stream"):
+ chunk = {"id": "c1", "object": "chat.completion.chunk", "created": int(time.time()),
+ "model": "test-model",
+ "choices": [{"index": 0, "delta": {"content": content}, "finish_reason": None}]}
+ end = dict(chunk, choices=[{"index": 0, "delta": {}, "finish_reason": "stop"}])
+ body = f"data: {json.dumps(chunk)}\n\ndata: {json.dumps(end)}\n\ndata: [DONE]\n\n".encode()
+ return self._send(200, body, "text/event-stream")
+ return self._send(200, {
+ "id": "c1", "object": "chat.completion", "created": int(time.time()), "model": "test-model",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": content},
+ "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+ })
+
+ self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+ self._thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self._server.shutdown()
+ self._server.server_close()
+
+ @property
+ def url(self):
+ host, port = self._server.server_address[:2]
+ return f"http://{host}:{port}/v1"
diff --git a/tests/e2e/test_whisperx_defaults_e2e.py b/tests/e2e/test_whisperx_defaults_e2e.py
new file mode 100644
index 00000000..13bf21c2
--- /dev/null
+++ b/tests/e2e/test_whisperx_defaults_e2e.py
@@ -0,0 +1,269 @@
+"""End to end: Speakr and whisperx-asr-service as a user sets them up.
+
+Both services run as real processes with their documented configuration and
+nothing else:
+
+- whisperx-asr-service: its docker-compose.yml plus the .env block from its
+ SETUP_GUIDE (tests/defaults/minimal.env), on stand-in ML packages so no GPU
+ or model download is needed. It is started outside this test (CI job, or
+ by hand, see tests/e2e/README.md) and reached at WHISPERX_URL.
+- Speakr: config/env.whisperx.example, pointed at WHISPERX_URL, run by
+ gunicorn exactly as the image runs it, with a fresh SQLite database and
+ the example's admin account created the way docker-entrypoint.sh does. The
+ example's language model is replaced by a local stand-in, since titles and
+ summaries are outside what this test covers.
+
+The test then checks what a new user would see: the startup voice embedding
+check succeeds, an upload is transcribed and diarized with speaker
+embeddings stored, two files joined at upload become one transcribed
+recording, and neither service logged an error.
+
+Runs only with SPEAKR_E2E=1 (the CI job sets it); the normal suite skips it.
+"""
+
+import io
+import json
+import math
+import os
+import re
+import shutil
+import socket
+import sqlite3
+import struct
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+import wave
+
+import httpx
+import pytest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, ROOT)
+
+from tests.e2e.fake_llm_server import FakeLLMServer
+
+pytestmark = pytest.mark.skipif(os.environ.get("SPEAKR_E2E") != "1",
+ reason="end-to-end test; set SPEAKR_E2E=1 with a whisperx stub server running")
+
+WHISPERX_URL = os.environ.get("WHISPERX_URL", "http://127.0.0.1:9000").rstrip("/")
+WHISPERX_LOG = os.environ.get("WHISPERX_LOG", "")
+EXAMPLE = os.path.join(ROOT, "config", "env.whisperx.example")
+ERROR_LINE = re.compile(r"\b(ERROR|CRITICAL)\b|Traceback \(most recent call last\)")
+
+
+def read_example():
+ values = {}
+ with open(EXAMPLE, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ key, value = line.split("=", 1)
+ values[key.strip()] = value.split(" #")[0].strip().strip('"').strip("'")
+ return values
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _wav(seconds, freq):
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(1)
+ w.setsampwidth(2)
+ w.setframerate(16000)
+ w.writeframes(b"".join(struct.pack("<h", int(6000 * math.sin(2 * math.pi * freq * i / 16000)))
+ for i in range(16000 * seconds)))
+ return buf.getvalue()
+
+
[email protected](scope="module")
+def speakr():
+ """Speakr as the WhisperX example configures it, running under gunicorn."""
+ try:
+ httpx.get(f"{WHISPERX_URL}/health", timeout=5).raise_for_status()
+ except Exception as e:
+ pytest.fail(f"whisperx stub server not reachable at {WHISPERX_URL}: {e}")
+
+ work = tempfile.mkdtemp(prefix="speakr-e2e-")
+ port = _free_port()
+ with FakeLLMServer() as llm:
+ env = {k: os.environ[k] for k in ("PATH", "HOME", "LANG", "LC_ALL", "TMPDIR") if k in os.environ}
+ env.update(read_example())
+ env.update({
+ # Where the example's container paths live on this machine.
+ "ASR_BASE_URL": WHISPERX_URL,
+ "SQLALCHEMY_DATABASE_URI": f"sqlite:///{work}/transcriptions.db",
+ "UPLOAD_FOLDER": f"{work}/uploads",
+ "AUTO_EXPORT_DIR": f"{work}/exports",
+ "AUTO_PROCESS_WATCH_DIR": f"{work}/auto-process",
+ "SECRET_KEY": "e2e-" + uuid.uuid4().hex,
+ # The example's language model, replaced (see the module docstring).
+ "TEXT_MODEL_BASE_URL": llm.url,
+ "TEXT_MODEL_API_KEY": "e2e",
+ "PYTHONUNBUFFERED": "1",
+ # The example's admin address is rejected by the domain check (see
+ # test_the_example_admin_account_is_created); skip the check so the
+ # rest of the documented setup can be tested.
+ "SKIP_EMAIL_DOMAIN_CHECK": "true",
+ })
+ for d in ("uploads", "exports", "auto-process"):
+ os.makedirs(os.path.join(work, d), exist_ok=True)
+
+ def run(args):
+ done = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True, timeout=300)
+ if done.returncode != 0:
+ pytest.fail(f"{' '.join(args)} failed:\n{done.stdout[-2000:]}\n{done.stderr[-4000:]}")
+
+ # docker-entrypoint.sh: create the database, then the example's admin.
+ run([sys.executable, "-c", "from src.app import app, db; app.app_context().push(); db.create_all()"])
+ run([sys.executable, "scripts/docker_create_admin.py"])
+
+ log_path = os.path.join(work, "speakr.log")
+ log = open(log_path, "w")
+ proc = subprocess.Popen(
+ [sys.executable, "-m", "gunicorn", "--workers", "1", "--worker-class", "gthread", "--threads", "4",
+ "--bind", f"127.0.0.1:{port}", "--timeout", "600", "src.app:app"],
+ cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT)
+ base = f"http://127.0.0.1:{port}"
+ try:
+ for _ in range(120):
+ try:
+ if httpx.get(f"{base}/login", timeout=2).status_code == 200:
+ break
+ except httpx.HTTPError:
+ pass
+ if proc.poll() is not None:
+ break
+ time.sleep(1)
+ else:
+ pytest.fail("Speakr did not start:\n" + open(log_path).read()[-4000:])
+ if proc.poll() is not None:
+ pytest.fail("Speakr exited:\n" + open(log_path).read()[-4000:])
+ yield {"base": base, "log": log_path, "db": f"{work}/transcriptions.db", "example": env}
+ finally:
+ proc.terminate()
+ try:
+ proc.wait(timeout=20)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ log.close()
+ if os.environ.get("KEEP_E2E_LOGS"):
+ print(f"Speakr log kept at {log_path}")
+ else:
+ shutil.rmtree(work, ignore_errors=True)
+
+
[email protected](scope="module")
+def admin(speakr):
+ """A logged-in session as the example's admin account."""
+ client = httpx.Client(base_url=speakr["base"], timeout=60, follow_redirects=True)
+ page = client.get("/login").text
+ token = re.search(r'name="csrf_token"[^>]*value="([^"]+)"', page) or re.search(
+ r'value="([^"]+)"[^>]*name="csrf_token"', page)
+ r = client.post("/login", data={"email": speakr["example"]["ADMIN_EMAIL"],
+ "password": speakr["example"]["ADMIN_PASSWORD"],
+ "csrf_token": token.group(1)})
+ assert r.status_code == 200 and "/login" not in str(r.url), "could not log in as the example's admin"
+ meta = re.search(r'name="csrf-token" content="([^"]+)"', client.get("/").text)
+ client.headers["X-CSRFToken"] = meta.group(1)
+ yield client
+ client.close()
+
+
+def _wait_for(check, timeout, what):
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ result = check()
+ if result:
+ return result
+ time.sleep(1)
+ pytest.fail(f"timed out waiting for {what}")
+
+
+def _wait_until_done(client, recording_id):
+ def done():
+ status = client.get(f"/recording/{recording_id}/status").json().get("status")
+ return status if status in ("COMPLETED", "FAILED") else None
+ return _wait_for(done, 180, f"recording {recording_id}")
+
+
+def _row(speakr, recording_id):
+ con = sqlite3.connect(speakr["db"])
+ try:
+ con.row_factory = sqlite3.Row
+ return con.execute("SELECT status, transcription, speaker_embeddings, error_message FROM recording"
+ " WHERE id = ?", (recording_id,)).fetchone()
+ finally:
+ con.close()
+
+
+def test_the_startup_voice_embedding_check_succeeds(speakr, admin):
+ def verdict():
+ status = admin.get("/admin/voice-embeddings/status").json()
+ return status if status.get("status") in ("ok", "changed") else None
+ status = _wait_for(verdict, 90, "the startup voice embedding check")
+ assert status["status"] == "ok", status
+
+
+def test_an_upload_is_transcribed_with_speakers_and_embeddings(speakr, admin):
+ r = admin.post("/upload", files={"file": ("meeting.wav", _wav(12, 330), "audio/wav")})
+ assert r.status_code == 202, r.text
+ recording_id = r.json()["id"]
+ assert _wait_until_done(admin, recording_id) == "COMPLETED", dict(_row(speakr, recording_id))
+ row = _row(speakr, recording_id)
+ assert "transcribed by large-v3" in row["transcription"], row["transcription"][:500]
+ embeddings = json.loads(row["speaker_embeddings"] or "{}")
+ assert embeddings and all(len(v) == 256 for v in embeddings.values()), "no speaker embeddings stored"
+
+
+def test_two_files_joined_at_upload_become_one_recording(speakr, admin):
+ group, joined = uuid.uuid4().hex, None
+ for index in range(2):
+ r = admin.post("/upload", data={"join_group": group, "join_index": str(index), "join_count": "2"},
+ files={"file": (f"part{index}.wav", _wav(6, 300 + 150 * index), "audio/wav")})
+ assert r.status_code == 202, r.text
+ joined = r.json().get("id") or joined
+ assert joined, "the join produced no recording"
+ assert _wait_until_done(admin, joined) == "COMPLETED", dict(_row(speakr, joined))
+ assert "transcribed by large-v3" in _row(speakr, joined)["transcription"]
+
+
+def test_neither_service_logged_an_error(speakr, admin):
+ """Runs last in this module, after everything above has used both services."""
+ with open(speakr["log"], encoding="utf-8", errors="replace") as f:
+ speakr_errors = [line.rstrip() for line in f if ERROR_LINE.search(line)]
+ assert not speakr_errors, "Speakr logged errors:\n" + "\n".join(speakr_errors[:40])
+ if WHISPERX_LOG and os.path.exists(WHISPERX_LOG):
+ with open(WHISPERX_LOG, encoding="utf-8", errors="replace") as f:
+ service_errors = [line.rstrip() for line in f if ERROR_LINE.search(line)]
+ assert not service_errors, "whisperx-asr-service logged errors:\n" + "\n".join(service_errors[:40])
+
+
[email protected](strict=True, reason=(
+ "config/env.whisperx.example sets [email protected]; example.com publishes a null MX, so "
+ "scripts/docker_create_admin.py rejects it and docker-entrypoint.sh (set -e) stops. Remove this marker "
+ "once the example uses an address that passes, or sets SKIP_EMAIL_DOMAIN_CHECK."))
+def test_the_example_admin_account_is_created():
+ """The example's admin settings, exactly as a user copies them."""
+ work = tempfile.mkdtemp(prefix="speakr-e2e-admin-")
+ try:
+ env = {k: os.environ[k] for k in ("PATH", "HOME", "LANG") if k in os.environ}
+ env.update(read_example())
+ env.update({"SQLALCHEMY_DATABASE_URI": f"sqlite:///{work}/t.db", "UPLOAD_FOLDER": f"{work}/uploads",
+ "AUTO_EXPORT_DIR": f"{work}/exports", "AUTO_PROCESS_WATCH_DIR": f"{work}/auto",
+ "SECRET_KEY": "e2e", "DISABLE_VOICE_EMBEDDING_CHECK": "true",
+ "ASR_BASE_URL": WHISPERX_URL})
+ subprocess.run([sys.executable, "-c", "from src.app import app, db; app.app_context().push(); db.create_all()"],
+ cwd=ROOT, env=env, check=True, capture_output=True, timeout=300)
+ done = subprocess.run([sys.executable, "scripts/docker_create_admin.py"], cwd=ROOT, env=env,
+ capture_output=True, text=True, timeout=300)
+ assert done.returncode == 0, done.stdout[-1000:]
+ finally:
+ shutil.rmtree(work, ignore_errors=True)
diff --git a/tests/test_documented_defaults.py b/tests/test_documented_defaults.py
new file mode 100644
index 00000000..f5240770
--- /dev/null
+++ b/tests/test_documented_defaults.py
@@ -0,0 +1,405 @@
+"""Speakr with the documented WhisperX settings, against the real service rules.
+
+Issue #409 reached users because every test ran with a fully configured
+environment. These tests start from what the docs tell a new user to do:
+Speakr configured only from config/env.whisperx.example, with no admin
+settings (no default transcription model, no model list), talking over HTTP
+to a stand-in for whisperx-asr-service that answers with the service's own
+model rules (tests/contract/whisperx_model_rules.py, kept in step with the
+service by tests/contract/test_whisperx_contract.py).
+
+Every path that sends audio to the service runs for real: the voice
+embedding check (startup, Check now, re-baseline), upload, reprocess, bulk
+reprocess, API v1 upload and transcribe, incognito and joining two uploads.
+
+The matrix, and the outcome expected in each cell:
+
+ service Speakr admin default outcome
+ current, PRELOAD_MODEL unset none works (service default large-v3)
+ current, PRELOAD_MODEL unset large-v3 works (model named)
+ 0.4.1, PRELOAD_MODEL empty none fails, clearly (#409 on old
+ services; recordings FAILED,
+ the check logs the hint)
+ 0.4.1, PRELOAD_MODEL empty large-v3 works (model named)
+ 0.4.1, PRELOAD_MODEL=large-v3 none works
+
+The voice embedding check also runs with speaker embeddings off, where it
+must send nothing at all.
+
+SHARED-DB: users, recordings, jobs and settings are removed afterwards.
+"""
+
+import io
+import logging
+import math
+import os
+import struct
+import sys
+import uuid
+import wave
+from contextlib import contextmanager
+from datetime import datetime
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+from flask import g
+from flask.testing import FlaskClient
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from src.app import app, db
+from src.models import ProcessingJob, Recording, SystemSetting, User
+from src.services import voice_embedding_check as vec
+from src.services.job_queue import TRANSCRIPTION_JOBS, job_queue
+from src.services.transcription import get_registry
+from tests.contract.fake_asr_server import FakeASRServer
+from tests.contract.whisperx_model_rules import ServiceModelRules
+
+app.config["WTF_CSRF_ENABLED"] = False
+
+EXAMPLE = os.path.join(ROOT, "config", "env.whisperx.example")
+MODEL_SETTINGS = ("transcription_default_model", "transcription_models_visible_json")
+
+# Settings in the example that decide what reaches the ASR service. Anything
+# else the example adds under these prefixes must be handled here too (the
+# guard test below fails otherwise), so the example and these tests cannot
+# drift apart.
+ASR_KEYS = ("ASR_BASE_URL", "ASR_DIARIZE", "ASR_RETURN_SPEAKER_EMBEDDINGS")
+ASR_PREFIXES = ("ASR_", "TRANSCRIPTION_", "WHISPER", "USE_ASR")
+
+SERVICES = {
+ # label: (generation, service environment as compose delivers it)
+ "current, PRELOAD_MODEL unset": ("fixed", {"PRELOAD_MODEL": ""}),
+ "0.4.1, PRELOAD_MODEL empty": ("legacy", {"PRELOAD_MODEL": ""}),
+ "0.4.1, PRELOAD_MODEL=large-v3": ("legacy", {"PRELOAD_MODEL": "large-v3"}),
+}
+
+
+def expected_to_work(service, admin_default):
+ generation, env = SERVICES[service]
+ return bool(admin_default) or bool(ServiceModelRules(env, generation).default_model)
+
+
+def _fake_llm(messages, response_format=None, **kwargs):
+ """The example's language model is out of scope here: titles, summaries
+ and events get a fixed answer so only the ASR path decides the outcome."""
+ content = '{"events": []}' if response_format else "Test meeting"
+ return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=content),
+ finish_reason="stop")], usage=None)
+
+
+def read_example():
+ values = {}
+ with open(EXAMPLE, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ key, value = line.split("=", 1)
+ values[key.strip()] = value.split(" #")[0].strip().strip('"').strip("'")
+ return values
+
+
+def test_the_example_sets_nothing_else_that_reaches_the_asr_service():
+ extra = [k for k in read_example() if k.startswith(ASR_PREFIXES) and k not in ASR_KEYS]
+ assert not extra, f"handle these in tests/test_documented_defaults.py: {extra}"
+
+
+def test_the_example_names_no_transcription_model():
+ """The realistic new install: nothing in Speakr names a model."""
+ assert not any("MODEL" in k and k.startswith(ASR_PREFIXES) for k in read_example())
+
+
+# --------------------------------------------------------------- set-up
+
+class _Client(FlaskClient):
+ def open(self, *args, **kwargs):
+ g.pop("_login_user", None)
+ return super().open(*args, **kwargs)
+
+
+@contextmanager
+def documented_speakr(service, admin_default=None, embeddings=None):
+ """Speakr on the example's ASR settings, pointed at a fake service."""
+ generation, service_env = SERVICES[service]
+ example = read_example()
+ with FakeASRServer(ServiceModelRules(service_env, generation)) as asr, app.app_context():
+ saved_env = {k: os.environ.get(k) for k in ASR_KEYS + ("TRANSCRIPTION_BASE_URL", "TRANSCRIPTION_API_KEY")}
+ saved_settings = {k: SystemSetting.get_setting(k, None) for k in MODEL_SETTINGS}
+ try:
+ for key in ASR_KEYS:
+ os.environ[key] = example.get(key, "")
+ os.environ["ASR_BASE_URL"] = asr.url
+ if embeddings is not None:
+ os.environ["ASR_RETURN_SPEAKER_EMBEDDINGS"] = "true" if embeddings else "false"
+ # The example configures no other transcription service.
+ os.environ["TRANSCRIPTION_BASE_URL"] = ""
+ os.environ["TRANSCRIPTION_API_KEY"] = ""
+ for key in MODEL_SETTINGS:
+ SystemSetting.set_setting(key, "", setting_type="string")
+ if admin_default:
+ SystemSetting.set_setting("transcription_default_model", admin_default, setting_type="string")
+ with patch("src.config.app_config.TRANSCRIPTION_MODELS_AVAILABLE", []), \
+ patch("src.tasks.processing.call_llm_completion", side_effect=_fake_llm):
+ assert get_registry().reinitialize().__class__.__name__ == "ASREndpointConnector"
+ yield asr
+ finally:
+ db.session.rollback()
+ for key, value in saved_env.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+ for key, value in saved_settings.items():
+ SystemSetting.set_setting(key, value or "", setting_type="string")
+ get_registry().reinitialize()
+
+
+def clean_reference():
+ with app.app_context():
+ saved = SystemSetting.get_setting(vec.SETTING_KEY, None)
+ SystemSetting.query.filter_by(key=vec.SETTING_KEY).delete()
+ db.session.commit()
+ yield
+ with app.app_context():
+ SystemSetting.query.filter_by(key=vec.SETTING_KEY).delete()
+ db.session.commit()
+ if saved:
+ SystemSetting.set_setting(vec.SETTING_KEY, saved, setting_type="string")
+
+
+def _user(admin=False):
+ s = uuid.uuid4().hex[:8]
+ u = User(username=f"dd_{s}", email=f"dd_{s}@local.test", password="x", is_admin=admin)
+ db.session.add(u)
+ db.session.commit()
+ return u
+
+
+def _client(user):
+ c = _Client(app, app.response_class)
+ with c.session_transaction() as sess:
+ sess["_user_id"] = str(user.id)
+ sess["_fresh"] = True
+ return c
+
+
+def _wav_bytes(seconds=4, freq=330):
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as w:
+ w.setnchannels(1)
+ w.setsampwidth(2)
+ w.setframerate(16000)
+ w.writeframes(b"".join(struct.pack("<h", int(6000 * math.sin(2 * math.pi * freq * i / 16000)))
+ for i in range(16000 * seconds)))
+ return buf.getvalue()
+
+
+def _drain(recording_ids):
+ """Run the queued transcription-side jobs of these recordings, in order."""
+ for _ in range(12):
+ jobs = (ProcessingJob.query
+ .filter(ProcessingJob.recording_id.in_(recording_ids),
+ ProcessingJob.status == "queued",
+ ProcessingJob.job_type.in_(TRANSCRIPTION_JOBS))
+ .order_by(ProcessingJob.id).all())
+ if not jobs:
+ return
+ for job in jobs:
+ job.status = "processing"
+ job.started_at = datetime.utcnow()
+ db.session.commit()
+ job_queue._process_job(job)
+ db.session.expire_all()
+
+
+def _cleanup(user_ids):
+ from src.models import SpeakerVoiceSample, Speaker
+ from src.models.upload_join import UploadJoinPart
+ rec_ids = [r.id for r in Recording.query.filter(Recording.user_id.in_(user_ids)).all()]
+ if rec_ids:
+ ProcessingJob.query.filter(ProcessingJob.recording_id.in_(rec_ids)).delete(synchronize_session=False)
+ SpeakerVoiceSample.query.filter(SpeakerVoiceSample.recording_id.in_(rec_ids)).delete(synchronize_session=False)
+ UploadJoinPart.query.filter(UploadJoinPart.user_id.in_(user_ids)).delete(synchronize_session=False)
+ if rec_ids:
+ Recording.query.filter(Recording.id.in_(rec_ids)).delete(synchronize_session=False)
+ Speaker.query.filter(Speaker.user_id.in_(user_ids)).delete(synchronize_session=False)
+ User.query.filter(User.id.in_(user_ids)).delete(synchronize_session=False)
+ db.session.commit()
+
+
+def _assert_requests(asr, path, works, admin_default):
+ assert asr.requests, f"{path}: nothing reached the ASR service"
+ for req in asr.requests:
+ if admin_default:
+ assert req["model_sent"] and req["model"] == admin_default, (path, req)
+ else:
+ assert not req["model_sent"], (path, req) # the service decides
+ if works:
+ assert req["status"] == 200, (path, req)
+ else:
+ assert req["status"] != 200, (path, req)
+
+
+def _assert_outcome(recording, path, works):
+ recording = db.session.get(Recording, recording.id)
+ if works:
+ assert recording.status in ("COMPLETED", "SUMMARIZING"), (path, recording.status, recording.error_message)
+ assert "Hello from the test service" in (recording.transcription or ""), path
+ else:
+ assert recording.status == "FAILED", (path, recording.status)
+ message = f"{recording.error_message or ''} {recording.transcription or ''}".lower()
+ assert "model" in message, (path, message)
+
+
+# ------------------------------------------------ voice embedding check
+
+CHECK_CELLS = [(s, a) for s in SERVICES for a in (None, "large-v3")]
+
+
[email protected]("service,admin_default", CHECK_CELLS)
+def test_the_startup_voice_embedding_check(service, admin_default, clean_reference, caplog):
+ works = expected_to_work(service, admin_default)
+ with documented_speakr(service, admin_default) as asr:
+ caplog.clear()
+ with caplog.at_level(logging.INFO):
+ result = vec.check_voice_embeddings(app)
+ _assert_requests(asr, "startup check", works, admin_default)
+ errors = [r.getMessage() for r in caplog.records if r.levelno >= logging.ERROR]
+ if works:
+ assert result["status"] == vec.STATUS_OK, result
+ assert not errors, errors
+ else:
+ assert result["status"] == vec.STATUS_UNKNOWN, result
+ warnings = " ".join(r.getMessage() for r in caplog.records if r.levelno == logging.WARNING)
+ assert "voice embedding check" in warnings.lower() and "PRELOAD_MODEL" in warnings, warnings
+
+
[email protected]("service,admin_default", CHECK_CELLS)
+def test_check_now_and_rebaseline_from_the_admin_page(service, admin_default, clean_reference):
+ works = expected_to_work(service, admin_default)
+ with documented_speakr(service, admin_default) as asr:
+ admin = _user(admin=True)
+ try:
+ client = _client(admin)
+ for path in ("/admin/voice-embeddings/check", "/admin/voice-embeddings/rebaseline"):
+ asr.reset()
+ response = client.post(path)
+ assert response.status_code < 500, (path, response.get_data(as_text=True))
+ _assert_requests(asr, path, works, admin_default)
+ finally:
+ _cleanup([admin.id])
+
+
[email protected]("service", list(SERVICES))
+def test_the_check_sends_nothing_when_speaker_embeddings_are_off(service, clean_reference):
+ with documented_speakr(service, None, embeddings=False) as asr:
+ result = vec.check_voice_embeddings(app)
+ assert result.get("supported") is False
+ assert asr.requests == []
+
+
+# ------------------------------------------------ recordings
+
+UPLOAD_CELLS = [
+ ("current, PRELOAD_MODEL unset", None),
+ ("current, PRELOAD_MODEL unset", "large-v3"),
+ ("0.4.1, PRELOAD_MODEL empty", None),
+ ("0.4.1, PRELOAD_MODEL empty", "large-v3"),
+ ("0.4.1, PRELOAD_MODEL=large-v3", None),
+]
+
+
[email protected]("service,admin_default", UPLOAD_CELLS)
+def test_every_recording_path(service, admin_default):
+ works = expected_to_work(service, admin_default)
+ with documented_speakr(service, admin_default) as asr:
+ user = _user()
+ try:
+ client = _client(user)
+
+ # Upload through the web app.
+ asr.reset()
+ r = client.post("/upload", data={"file": (io.BytesIO(_wav_bytes()), "meeting.wav")},
+ content_type="multipart/form-data")
+ assert r.status_code == 202, r.get_data(as_text=True)
+ recording = db.session.get(Recording, r.get_json()["id"])
+ _drain([recording.id])
+ _assert_requests(asr, "upload", works, admin_default)
+ _assert_outcome(recording, "upload", works)
+ if works:
+ embeddings = db.session.get(Recording, recording.id).speaker_embeddings or {}
+ assert "SPEAKER_00" in embeddings, "speaker embeddings were not stored"
+
+ # Reprocess the transcription.
+ asr.reset()
+ r = client.post(f"/recording/{recording.id}/reprocess_transcription", json={})
+ assert r.status_code < 300, r.get_data(as_text=True)
+ _drain([recording.id])
+ _assert_requests(asr, "reprocess", works, admin_default)
+ _assert_outcome(recording, "reprocess", works)
+
+ # Bulk reprocess.
+ asr.reset()
+ r = client.post("/api/recordings/bulk-reprocess",
+ json={"recording_ids": [recording.id], "type": "transcription"})
+ assert r.status_code < 300, r.get_data(as_text=True)
+ _drain([recording.id])
+ _assert_requests(asr, "bulk reprocess", works, admin_default)
+ _assert_outcome(recording, "bulk reprocess", works)
+
+ # API v1 transcribe.
+ asr.reset()
+ r = client.post(f"/api/v1/recordings/{recording.id}/transcribe", json={})
+ assert r.status_code < 300, r.get_data(as_text=True)
+ _drain([recording.id])
+ _assert_requests(asr, "API v1 transcribe", works, admin_default)
+ _assert_outcome(recording, "API v1 transcribe", works)
+
+ # API v1 upload.
+ asr.reset()
+ r = client.post("/api/v1/recordings/upload",
+ data={"file": (io.BytesIO(_wav_bytes()), "api.wav")},
+ content_type="multipart/form-data")
+ assert r.status_code < 300, r.get_data(as_text=True)
+ body = r.get_json()
+ api_rec = db.session.get(Recording, body.get("id") or body.get("recording_id")
+ or (body.get("recording") or {}).get("id"))
+ _drain([api_rec.id])
+ _assert_requests(asr, "API v1 upload", works, admin_default)
+ _assert_outcome(api_rec, "API v1 upload", works)
+
+ # Two files joined into one recording.
+ asr.reset()
+ group = uuid.uuid4().hex
+ joined = None
+ for index in range(2):
+ r = client.post("/upload", data={
+ "file": (io.BytesIO(_wav_bytes(seconds=2, freq=300 + 100 * index)), f"part{index}.wav"),
+ "join_group": group, "join_index": str(index), "join_count": "2",
+ }, content_type="multipart/form-data")
+ assert r.status_code == 202, r.get_data(as_text=True)
+ joined = r.get_json().get("id") or joined
+ assert joined, "the join produced no recording"
+ _drain([joined])
+ _assert_requests(asr, "upload join", works, admin_default)
+ _assert_outcome(db.session.get(Recording, joined), "upload join", works)
+
+ # Incognito (not enabled by the example; switched on for this step).
+ asr.reset()
+ with patch("src.api.recordings.ENABLE_INCOGNITO_MODE", True):
+ r = client.post("/api/recordings/incognito",
+ data={"file": (io.BytesIO(_wav_bytes()), "private.wav")},
+ content_type="multipart/form-data")
+ _assert_requests(asr, "incognito", works, admin_default)
+ if works:
+ assert r.status_code == 200, r.get_data(as_text=True)
+ assert "Hello from the test service" in r.get_data(as_text=True)
+ else:
+ assert r.status_code >= 400
+ assert "model" in r.get_data(as_text=True).lower()
+ finally:
+ _cleanup([user.id])