Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
7.8 KB
Raw Blame
"""Configurable sampling temperatures (#411).

Each kind of request (summary, title, chat, event) is sent with the admin value
from the Default Prompts tab, else the built-in default. An invalid saved value
is ignored and the default is used.
"""

import ast
import os
import sys
import uuid
from unittest.mock import patch, MagicMock

import pytest
from flask import g
from flask.testing import FlaskClient

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from src.app import app, db
from src.models import User, SystemSetting
from src.services import llm_settings as ls

app.config["WTF_CSRF_ENABLED"] = False
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


class _Client(FlaskClient):
    def open(self, *args, **kwargs):
        g.pop('_login_user', None)
        return super().open(*args, **kwargs)


@pytest.fixture
def ctx():
    with app.app_context():
        yield
        db.session.rollback()
        for kind in ls.TEMPERATURES:
            SystemSetting.query.filter_by(key=ls.setting_key(kind)).delete()
        db.session.commit()


def _admin_value(kind, value):
    SystemSetting.set_setting(ls.setting_key(kind), value, setting_type='string')
    db.session.commit()


# ---------------------------------------------------------------- resolution

def test_defaults_are_unchanged_from_earlier_releases(ctx):
    assert {k: ls.get_temperature(k) for k in ls.TEMPERATURES} == {'summary': 0.5, 'title': 0.7, 'chat': 0.7, 'event': 0.2}
    assert ls.resolve_temperature('summary') == (0.5, 'default')


def test_admin_value_overrides_the_default(ctx):
    _admin_value('chat', '0.3')
    assert ls.resolve_temperature('chat') == (0.3, 'admin')
    assert ls.resolve_temperature('title') == (0.7, 'default')


def test_environment_variables_are_not_read(ctx, monkeypatch):
    monkeypatch.setenv('SUMMARY_TEMPERATURE', '0')
    assert ls.resolve_temperature('summary') == (0.5, 'default')


@pytest.mark.parametrize('raw', ['abc', '-0.1', '2.5', 'nan', ' ', ''])
def test_invalid_admin_values_fall_back_to_the_default(ctx, raw):
    _admin_value('event', raw)
    assert ls.resolve_temperature('event') == (0.2, 'default')


@pytest.mark.parametrize('raw,expected', [('0', 0.0), ('2', 2.0), ('1.25', 1.25), (0.3, 0.3), (' 0.5 ', 0.5)])
def test_bounds_are_inclusive(raw, expected):
    assert ls.parse_temperature(raw) == expected


def test_an_admin_change_applies_to_the_next_request(ctx):
    assert ls.get_temperature('summary') == 0.5
    _admin_value('summary', '0.1')
    assert ls.get_temperature('summary') == 0.1


# ---------------------------------------------------------------- call sites

CALL_SITES = {
    os.path.join('src', 'tasks', 'processing.py'): {
        '_generate_ai_title': 'title',
        'generate_summary_only_task': 'summary',
        'extract_events_from_transcript': 'event',
        '_generate_incognito_title': 'title',
        'generate_incognito_summary': 'summary',
    },
    os.path.join('src', 'api', 'recordings.py'): {
        'chat_incognito': 'chat',
        'chat_with_transcription': 'chat',
    },
}


def _temperature_kinds(func):
    kinds = []
    for node in ast.walk(func):
        if isinstance(node, ast.Call) and getattr(node.func, 'id', None) == 'get_temperature':
            kinds.append(node.args[0].value)
    return kinds


@pytest.mark.parametrize('path', list(CALL_SITES))
def test_every_call_site_sends_its_configured_temperature(path):
    tree = ast.parse(open(os.path.join(ROOT, path), encoding='utf-8').read())
    funcs = {n.name: n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))}
    for name, kind in CALL_SITES[path].items():
        assert _temperature_kinds(funcs[name]) == [kind], name
        passed = [kw.value for n in ast.walk(funcs[name]) if isinstance(n, ast.Call)
                  for kw in n.keywords if kw.arg == 'temperature']
        assert passed, f"{name} sends no temperature"
        for value in passed:
            # either get_temperature(kind) inline, or a variable assigned from it
            ok = (isinstance(value, ast.Call) and getattr(value.func, 'id', None) == 'get_temperature') or (
                isinstance(value, ast.Name) and any(
                    isinstance(a, ast.Assign) and any(getattr(t, 'id', None) == value.id for t in a.targets)
                    and isinstance(a.value, ast.Call) and getattr(a.value.func, 'id', None) == 'get_temperature'
                    for a in ast.walk(funcs[name])))
            assert ok, f"{name} sends temperature={ast.unparse(value)}"
    literal = [n.lineno for n in ast.walk(tree)
               if isinstance(n, ast.keyword) and n.arg == 'temperature' and isinstance(n.value, ast.Constant)]
    assert literal == [], f"hard-coded temperature in {path} at lines {literal}"


def test_incognito_title_and_summary_use_the_resolved_values(ctx):
    from src.tasks import processing
    _admin_value('title', '0.15')
    _admin_value('summary', '0')
    completion = MagicMock()
    completion.choices = [MagicMock(message=MagicMock(content='A title'))]
    seen = []

    def fake(*args, **kwargs):
        seen.append(kwargs.get('temperature'))
        return completion

    fake_client = MagicMock()
    fake_client.chat.completions.create.side_effect = fake
    with patch.object(processing, 'client', fake_client), patch('src.services.llm.client', fake_client):
        processing._generate_incognito_title('Some transcript text about budgets.')
        processing.generate_incognito_summary('Some transcript text about budgets.')
    assert seen == [0.15, 0.0]


# ---------------------------------------------------------------- admin API

def _client(user):
    c = _Client(app, app.response_class)
    with c.session_transaction() as sess:
        sess["_user_id"] = str(user.id)
        sess["_fresh"] = True
    return c


@pytest.fixture
def users(ctx):
    s = uuid.uuid4().hex[:8]
    admin = User(username=f"tA_{s}", email=f"ta_{s}@local.test", password="x", is_admin=True)
    plain = User(username=f"tU_{s}", email=f"tu_{s}@local.test", password="x")
    db.session.add_all([admin, plain])
    db.session.commit()
    yield admin, plain
    db.session.delete(db.session.get(User, admin.id))
    db.session.delete(db.session.get(User, plain.id))
    db.session.commit()


def test_api_requires_an_admin(users):
    _, plain = users
    c = _client(plain)
    assert c.get('/admin/llm-temperatures').status_code == 403
    assert c.post('/admin/llm-temperatures', json={'values': {'summary': 0}}).status_code == 403


def test_api_reports_values_and_sources(users):
    admin, _ = users
    _admin_value('chat', '0.9')
    data = _client(admin).get('/admin/llm-temperatures').get_json()['temperatures']
    assert data['chat'] == {'value': 0.9, 'source': 'admin', 'default': 0.7}
    assert data['summary'] == {'value': 0.5, 'source': 'default', 'default': 0.5}


def test_api_saves_and_clears(users):
    admin, _ = users
    c = _client(admin)
    r = c.post('/admin/llm-temperatures', json={'values': {'summary': 0, 'event': '0.1'}})
    assert r.status_code == 200, r.get_data(as_text=True)
    assert ls.resolve_temperature('summary') == (0.0, 'admin')
    assert ls.resolve_temperature('event') == (0.1, 'admin')
    r = c.post('/admin/llm-temperatures', json={'values': {'summary': None}})
    assert r.status_code == 200
    assert ls.resolve_temperature('summary') == (0.5, 'default')
    assert ls.resolve_temperature('event') == (0.1, 'admin')


@pytest.mark.parametrize('values', [{'summary': 3}, {'summary': 'hot'}, {'summary': True}, {'nope': 0.1}, {}])
def test_api_rejects_invalid_input_and_saves_nothing(users, values):
    admin, _ = users
    r = _client(admin).post('/admin/llm-temperatures', json={'values': values})
    assert r.status_code == 400
    assert SystemSetting.query.filter(SystemSetting.key.like('llm_temperature_%')).count() == 0