Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
21.2 KB
Raw Blame
"""Detecting that the transcription backend changed its voice embedding model (#380).

The failure being guarded against is silent, so most of these tests are about
NOT crying wolf. A backend that is merely unreachable, or a Speakr upgrade that
ships a new canary clip, must not be reported as a changed model: a warning
nobody can act on trains people to ignore the one that matters.

The numbers here come from measuring a real whisperx-asr-service backend:
the same clip re-embedded scores 1.0 and is bit-identical, a different voice
through the same model scores 0.27, and the same voice truncated from 12s to 5s
scores 0.52. The threshold sits at 0.98.
"""

import json
from unittest.mock import patch

import pytest

from src.app import app
from src.database import db
from src.models import SystemSetting
from src.services import voice_embedding_check as vec


REF = [0.10, -0.20, 0.30, 0.40] * 64          # 256 dims, stands in for the baseline
SAME = list(REF)
SCALED = [x * 3 for x in REF]                  # same direction, different magnitude
DIFFERENT = [((i * 37) % 19) / 19 - 0.5 for i in range(256)]
SHORTER = [0.1] * 192                          # what a 192-dim model would return


@pytest.fixture(autouse=True)
def clean_reference():
    """Each test starts with no stored baseline."""
    with app.app_context():
        SystemSetting.query.filter_by(key=vec.SETTING_KEY).delete()
        db.session.commit()
    yield
    with app.app_context():
        SystemSetting.query.filter_by(key=vec.SETTING_KEY).delete()
        db.session.commit()


def _supported():
    return patch.object(vec, 'embeddings_supported', return_value=True)


def _baseline(vector=REF, fingerprint='backend-a'):
    """Record a baseline the way a first startup would."""
    with _supported(), \
            patch.object(vec, 'probe_backend', return_value=vector), \
            patch.object(vec, 'backend_fingerprint', return_value=fingerprint):
        return vec.check_voice_embeddings(app)


# --- the core claim ------------------------------------------------------

def test_a_first_run_records_a_baseline_rather_than_warning():
    with app.app_context():
        result = _baseline()
    assert result['status'] == vec.STATUS_OK
    assert result['dimension'] == 256


def test_the_same_embedding_is_not_reported_as_a_change():
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=SAME), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_OK
    assert result['similarity'] == pytest.approx(1.0)


def test_a_different_embedding_is_reported_as_a_change():
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_CHANGED
    assert result['similarity'] < vec.SIMILARITY_THRESHOLD
    assert 'rebuil' in result['detail'] or 'embeds differently' in result['detail']


def test_magnitude_alone_is_not_a_change():
    """Cosine ignores scale, and it should: a rescaled vector is the same
    direction in the same space, so voice matching still works."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=SCALED), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_OK


def test_a_dimension_change_is_reported_as_dimensions_not_as_similarity():
    """192-dim vs 256-dim is the case from the issue, and cosine cannot even
    be computed for it, so the message has to say so."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=SHORTER), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_CHANGED
    assert result['similarity'] is None
    assert '192' in result['detail'] and '256' in result['detail']


# --- not crying wolf -----------------------------------------------------

def test_an_unreachable_backend_is_not_reported_as_a_changed_model():
    """The most important false positive to avoid. A down ASR service is a
    different problem with a different fix, and telling the user their model
    changed would send them to rebuild profiles for nothing."""
    with app.app_context():
        _baseline()
        with _supported(), \
                patch.object(vec, 'probe_backend',
                             side_effect=vec.CanaryUnavailable('connection refused')), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_OK, 'a down backend must not read as a changed model'
    assert 'could not run' in result['detail']


def test_a_canary_that_diarizes_into_several_speakers_is_unavailable_not_changed():
    """A backend whose VAD splits the single-voice clip gives a result that is
    not comparable, which is missing data rather than evidence of a change."""
    with app.app_context():
        _baseline()

        class _Resp:
            speaker_embeddings = {'SPEAKER_00': REF, 'SPEAKER_01': DIFFERENT}

        class _Conn:
            def transcribe(self, request):
                return _Resp()

        with _supported(), \
                patch('src.services.transcription.registry.get_registry') as reg:
            reg.return_value.get_active_connector.return_value = _Conn()
            with pytest.raises(vec.CanaryUnavailable) as e:
                vec.probe_backend()
        assert 'one speaker' in str(e.value)


def test_a_new_canary_clip_rebaselines_instead_of_warning_everyone():
    """Shipping a different clip must not light up every instance on upgrade.
    A reference taken from the old clip is not comparable with the new one."""
    with app.app_context():
        _baseline()
        stored = json.loads(SystemSetting.get_setting(vec.SETTING_KEY))
        stored['clip_version'] = 'v0'
        SystemSetting.set_setting(vec.SETTING_KEY, json.dumps(stored))

        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
            result = vec.check_voice_embeddings(app)

    assert result['status'] == vec.STATUS_OK
    assert result['clip_version'] == vec.CANARY_CLIP_VERSION


def test_a_connector_without_embeddings_is_unknown_not_broken():
    with app.app_context():
        with patch.object(vec, 'embeddings_supported', return_value=False):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_UNKNOWN
    assert result['supported'] is False


# --- cost ---------------------------------------------------------------

def test_an_unchanged_backend_is_not_re_probed():
    """The probe is a real transcription. Running it every boot would put a
    pointless job through the user's ASR service five times per container."""
    with app.app_context():
        _baseline(fingerprint='backend-a')
        with _supported(), patch.object(vec, 'probe_backend') as probe, \
                patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
            vec.check_voice_embeddings(app)
        probe.assert_not_called()


def test_force_re_probes_even_when_nothing_changed():
    with app.app_context():
        _baseline(fingerprint='backend-a')
        with _supported(), patch.object(vec, 'probe_backend', return_value=SAME) as probe, \
                patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
            vec.check_voice_embeddings(app, force=True)
        probe.assert_called_once()


# --- recovery ------------------------------------------------------------

def test_the_baseline_survives_a_mismatch_so_the_backend_can_be_restored():
    """If a mismatch overwrote the reference, the warning would clear itself on
    the next check and restoring the old backend would then look like a second
    change. The reference is what the stored profiles were built against."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            vec.check_voice_embeddings(app)
        stored = vec.load_reference()
    assert stored['embedding'] == REF, 'the reference was replaced by the mismatching one'


def test_restoring_the_original_backend_clears_the_warning_by_itself():
    """The user should not have to click anything to resolve a change they
    undid."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            assert vec.check_voice_embeddings(app)['status'] == vec.STATUS_CHANGED

        with _supported(), patch.object(vec, 'probe_backend', return_value=SAME), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
            result = vec.check_voice_embeddings(app)
    assert result['status'] == vec.STATUS_OK
    assert result['detail'] is None


def test_rebaselining_accepts_the_new_backend():
    """The other resolution: the change was intended, profiles were rebuilt."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            vec.check_voice_embeddings(app)

        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.rebaseline(app)
    assert result['status'] == vec.STATUS_OK
    with app.app_context():
        assert vec.load_reference()['embedding'] == DIFFERENT


# --- the clip itself -----------------------------------------------------

def test_the_canary_clip_ships_and_is_the_expected_audio():
    """A missing or re-encoded clip silently invalidates every reference, so
    its identity is pinned here rather than assumed."""
    import hashlib
    import os
    import wave

    assert os.path.exists(vec.CANARY_PATH), f'canary clip missing at {vec.CANARY_PATH}'

    with wave.open(vec.CANARY_PATH, 'rb') as w:
        assert w.getnchannels() == 1, 'must be mono'
        assert w.getframerate() == 16000, 'must be 16 kHz'
        assert w.getsampwidth() == 2, 'must be 16-bit PCM'
        duration = w.getnframes() / w.getframerate()
        assert 10 <= duration <= 14, f'expected roughly 12s of audio, got {duration:.1f}s'

    digest = hashlib.sha256(open(vec.CANARY_PATH, 'rb').read()).hexdigest()
    assert digest == '2f4440d771f4206e67d53469002dd0241fd2a19e18db46ba2b79600da51a2741', (
        'the canary clip changed. Every stored reference embedding was taken against the '
        'old bytes and is now meaningless, so bump CANARY_CLIP_VERSION and ship the new '
        'clip under a new filename rather than replacing this one in place.'
    )


def test_cosine_similarity_edge_cases():
    assert vec.cosine_similarity([1, 0], [1, 0]) == pytest.approx(1.0)
    assert vec.cosine_similarity([1, 0], [0, 1]) == pytest.approx(0.0)
    # Lengths that differ cannot be compared; None, not an exception, because
    # the caller distinguishes "cannot compare" from "compares badly".
    assert vec.cosine_similarity([1, 0], [1, 0, 0]) is None
    assert vec.cosine_similarity([0, 0], [1, 0]) is None
    assert vec.cosine_similarity([], []) is None


# --- the matcher's side of it --------------------------------------------
#
# The check above tells the admin the model changed. These cover what happens
# to a user in the meantime, which is where the original silence lived.

def _speaker(user_id, name, vector):
    from src.models import Speaker
    from src.services.speaker_embedding_matcher import serialize_embedding
    return Speaker(user_id=user_id, name=name,
                   average_embedding=serialize_embedding(vector))


@pytest.fixture
def speaker_owner():
    from src.models import User
    import os
    with app.app_context():
        user = User(username=f'vec_owner_{os.getpid()}',
                    email=f'vec_owner_{os.getpid()}@example.com', password='x')
        db.session.add(user)
        db.session.commit()
        uid = user.id
    yield uid
    with app.app_context():
        from src.models import Speaker, User
        Speaker.query.filter_by(user_id=uid).delete()
        User.query.filter_by(id=uid).delete()
        db.session.commit()


def test_profiles_of_another_dimension_are_skipped_and_said_so_in_the_log(speaker_owner, caplog):
    """The original behaviour: a mismatched pair raised inside the loop, the
    blanket `except Exception: continue` treated it as a corrupt profile, and
    the user got an empty match list forever with nothing in the log."""
    from src.services.speaker_embedding_matcher import find_matching_speakers

    with app.app_context():
        db.session.add(_speaker(speaker_owner, 'Old Profile A', REF))
        db.session.add(_speaker(speaker_owner, 'Old Profile B', REF))
        db.session.commit()

        with caplog.at_level('ERROR'):
            matches = find_matching_speakers(SHORTER, speaker_owner, threshold=0.5)

    assert matches == []
    logged = ' '.join(r.getMessage() for r in caplog.records)
    assert '2 of 2' in logged, f'the skip was not reported: {logged!r}'
    assert '256 dimensions' in logged, f'the stored size must be named: {logged!r}'
    assert '192-dimensional' in logged, f'the new size must be named: {logged!r}'
    assert 'rebuilt' in logged or 'restored' in logged, 'the log must say what to do'


def test_matching_still_works_when_the_dimensions_agree(speaker_owner):
    """The guard must not cost normal matching."""
    from src.services.speaker_embedding_matcher import find_matching_speakers

    with app.app_context():
        db.session.add(_speaker(speaker_owner, 'Alice', REF))
        db.session.commit()
        matches = find_matching_speakers(SAME, speaker_owner, threshold=0.7)

    assert [m['name'] for m in matches] == ['Alice']
    assert matches[0]['similarity'] == pytest.approx(100.0, abs=0.1)


def test_a_genuinely_unreadable_profile_is_skipped_and_logged(speaker_owner, caplog):
    """Corruption still has to be tolerated, just not silently."""
    from src.models import Speaker
    from src.services.speaker_embedding_matcher import find_matching_speakers

    with app.app_context():
        broken = Speaker(user_id=speaker_owner, name='Corrupt',
                         average_embedding=b'\x01\x02\x03')  # not a whole float32
        db.session.add(broken)
        db.session.add(_speaker(speaker_owner, 'Alice', REF))
        db.session.commit()

        with caplog.at_level('WARNING'):
            matches = find_matching_speakers(SAME, speaker_owner, threshold=0.7)

    assert [m['name'] for m in matches] == ['Alice'], 'one bad profile must not lose the good ones'


# --- the admin surface ---------------------------------------------------

@pytest.fixture
def admin_client():
    """The endpoints are CSRF-protected, which these tests are not about."""
    import os
    from src.models import User
    previous_csrf = app.config.get('WTF_CSRF_ENABLED')
    app.config['WTF_CSRF_ENABLED'] = False
    with app.app_context():
        admin = User(username=f'vec_admin_{os.getpid()}',
                     email=f'vec_admin_{os.getpid()}@example.com',
                     password='x', is_admin=True)
        plain = User(username=f'vec_plain_{os.getpid()}',
                     email=f'vec_plain_{os.getpid()}@example.com',
                     password='x', is_admin=False)
        db.session.add_all([admin, plain])
        db.session.commit()
        ids = (admin.id, plain.id)
    yield app.test_client(), ids
    app.config['WTF_CSRF_ENABLED'] = previous_csrf
    with app.app_context():
        from src.models import User
        User.query.filter(User.id.in_(ids)).delete(synchronize_session=False)
        db.session.commit()


def _login(client, user_id):
    with client.session_transaction() as sess:
        sess['_user_id'] = str(user_id)
        sess['_fresh'] = True


def test_status_endpoint_never_probes(admin_client):
    """A page load must not put a transcription job through the ASR service."""
    client, (admin_id, _) = admin_client
    _login(client, admin_id)
    with patch.object(vec, 'probe_backend') as probe:
        resp = client.get('/admin/voice-embeddings/status')
    assert resp.status_code == 200
    assert 'status' in resp.get_json()
    probe.assert_not_called()


def test_a_non_admin_cannot_read_or_change_the_verdict(admin_client):
    client, (_, plain_id) = admin_client
    _login(client, plain_id)
    for method, path in (('get', '/admin/voice-embeddings/status'),
                         ('post', '/admin/voice-embeddings/check'),
                         ('post', '/admin/voice-embeddings/rebaseline')):
        resp = getattr(client, method)(path)
        assert resp.status_code == 403, f'{method.upper()} {path} was not refused'


def test_the_check_endpoint_forces_a_probe(admin_client):
    client, (admin_id, _) = admin_client
    _login(client, admin_id)
    with app.app_context():
        _baseline(fingerprint='backend-a')
    with _supported(), patch.object(vec, 'probe_backend', return_value=SAME) as probe, \
            patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
        resp = client.post('/admin/voice-embeddings/check')
    assert resp.status_code == 200
    assert resp.get_json()['status'] == vec.STATUS_OK
    probe.assert_called_once()


# --- the notification must follow the verdict -----------------------------
#
# Found in review: every path that ends in a known-good state has to clear
# the admin notice, or the person who did what the banner told them to do
# keeps seeing the banner.

def _notice_active():
    from src.models import Notification, KIND_VOICE_EMBEDDING_CHANGED
    return Notification.query.filter_by(
        kind=KIND_VOICE_EMBEDDING_CHANGED, resolved_at=None).count()


@pytest.fixture
def an_admin():
    import os
    from src.models import User, Notification
    with app.app_context():
        u = User(username=f'vec_adm_{os.getpid()}', email=f'vec_adm_{os.getpid()}@example.com',
                 password='x', is_admin=True)
        db.session.add(u); db.session.commit(); uid = u.id
    yield uid
    with app.app_context():
        Notification.query.filter_by(user_id=uid).delete()
        User.query.filter_by(id=uid).delete(); db.session.commit()


def test_rebaselining_clears_the_admin_notification(an_admin):
    """rebaseline() deletes the reference and re-checks, landing on the
    fresh-baseline path. That path used to return without resolving the
    notice, so clicking the button the banner points at left the banner up."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            vec.check_voice_embeddings(app)
        assert _notice_active() == 1, 'precondition: the change raised a notice'

        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            result = vec.rebaseline(app)

        assert result['status'] == vec.STATUS_OK
        assert _notice_active() == 0, 'rebaselining left the warning in place'


def test_a_new_clip_version_clears_a_stale_notification(an_admin):
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            vec.check_voice_embeddings(app)
        stored = json.loads(SystemSetting.get_setting(vec.SETTING_KEY))
        stored['clip_version'] = 'v0'
        SystemSetting.set_setting(vec.SETTING_KEY, json.dumps(stored))

        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            vec.check_voice_embeddings(app)
        assert _notice_active() == 0


def test_switching_to_a_connector_without_embeddings_clears_the_notification(an_admin):
    """The old backend's embedding problem cannot be fixed from a backend
    that has no embeddings; the notice would be telling the admin to act on
    something that no longer exists."""
    with app.app_context():
        _baseline()
        with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
                patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
            vec.check_voice_embeddings(app)
        assert _notice_active() == 1

        with patch.object(vec, 'embeddings_supported', return_value=False):
            vec.check_voice_embeddings(app)
        assert _notice_active() == 0