Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(backend) add start-subtitle endpoint
lebaudantoinef48dd5cea130
diff --git a/src/backend/core/api/__init__.py b/src/backend/core/api/__init__.pyindex 5e400ca0..45bfaf6b 100644--- a/src/backend/core/api/__init__.py+++ b/src/backend/core/api/__init__.py@@ -50,6 +50,7 @@ def get_frontend_configuration(request):else None,"default_country": settings.ROOM_TELEPHONY_DEFAULT_COUNTRY,},+ "subtitle": {"enabled": settings.ROOM_SUBTITLE_ENABLED},"livekit": {"url": settings.LIVEKIT_CONFIGURATION["url"],"force_wss_protocol": settings.LIVEKIT_FORCE_WSS_PROTOCOL,diff --git a/src/backend/core/api/permissions.py b/src/backend/core/api/permissions.pyindex 2300e903..68dff17e 100644--- a/src/backend/core/api/permissions.py+++ b/src/backend/core/api/permissions.py@@ -119,3 +119,22 @@ class IsStorageEventEnabled(permissions.BasePermission):def has_permission(self, request, view):"""Determine if access is allowed based on settings."""return settings.RECORDING_STORAGE_EVENT_ENABLE+++class IsSubtitleEnabled(permissions.BasePermission):+ """Check if the subtitle feature is enabled."""++ message = "Access denied, subtitles are disabled."++ def has_permission(self, request, view):+ """Determine if access is allowed based on settings."""+ return settings.ROOM_SUBTITLE_ENABLED+++class HasLiveKitRoomAccess(permissions.BasePermission):+ """Check if authenticated user's LiveKit token is for the specific room."""++ def has_object_permission(self, request, view, obj):+ if not request.auth or not hasattr(request.auth, "video"):+ return False+ return request.auth.video.room == str(obj.id)diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.pyindex 76fdf8ae..b2ef65b4 100644--- a/src/backend/core/api/viewsets.py+++ b/src/backend/core/api/viewsets.py@@ -51,7 +51,9 @@ from core.services.lobby import (LobbyService,)from core.services.room_creation import RoomCreation+from core.services.subtitle import SubtitleException, SubtitleService+from ..authentication.livekit import LiveKitTokenAuthenticationfrom . import permissions, serializers# pylint: disable=too-many-ancestors@@ -530,6 +532,37 @@ class RoomViewSet(status=drf_status.HTTP_200_OK,)+ @decorators.action(+ detail=True,+ methods=["post"],+ url_path="start-subtitle",+ permission_classes=[+ permissions.IsSubtitleEnabled,+ permissions.HasLiveKitRoomAccess,+ ],+ authentication_classes=[LiveKitTokenAuthentication],+ )+ def start_subtitle(self, request, pk=None): # pylint: disable=unused-argument+ """Start realtime transcription for the room.++ Requires valid LiveKit token for room authorization.+ Anonymous users can start subtitles if they have room access tokens.+ """++ room = self.get_object()++ try:+ SubtitleService().start_subtitle(room)+ except SubtitleException:+ return drf_response.Response(+ {"error": f"Subtitles failed to start for room {room.slug}"},+ status=drf_status.HTTP_500_INTERNAL_SERVER_ERROR,+ )++ return drf_response.Response(+ {"status": "success"}, status=drf_status.HTTP_200_OK+ )+class ResourceAccessViewSet(mixins.CreateModelMixin,diff --git a/src/backend/core/authentication/livekit.py b/src/backend/core/authentication/livekit.pynew file mode 100644index 00000000..b236dc54--- /dev/null+++ b/src/backend/core/authentication/livekit.py@@ -0,0 +1,42 @@+"""Authentication using LiveKit token for the Meet core app."""++from django.conf import settings+from django.contrib.auth import get_user_model+from django.contrib.auth.models import AnonymousUser++from livekit.api import TokenVerifier+from rest_framework import authentication, exceptions++UserModel = get_user_model()+++class LiveKitTokenAuthentication(authentication.BaseAuthentication):+ """Authenticate using LiveKit token and load the associated Django user."""++ def authenticate(self, request):+ token = request.data.get("token")+ if not token:+ return None # No authentication attempted++ try:+ verifier = TokenVerifier(+ api_key=settings.LIVEKIT_CONFIGURATION["api_key"],+ api_secret=settings.LIVEKIT_CONFIGURATION["api_secret"],+ )+ claims = verifier.verify(token)++ user_id = claims.identity+ if not user_id:+ raise exceptions.AuthenticationFailed("Token missing user identity")++ try:+ user = UserModel.objects.get(id=user_id)+ except UserModel.DoesNotExist:+ user = AnonymousUser()++ return (user, claims)++ except Exception as e:+ raise exceptions.AuthenticationFailed(+ f"Invalid LiveKit token: {str(e)}"+ ) from ediff --git a/src/backend/core/services/subtitle.py b/src/backend/core/services/subtitle.pynew file mode 100644index 00000000..ea007602--- /dev/null+++ b/src/backend/core/services/subtitle.py@@ -0,0 +1,47 @@+"""Service for managing subtitle agents in LiveKit rooms."""++from logging import getLogger++from django.conf import settings++from asgiref.sync import async_to_sync+from livekit.protocol.agent_dispatch import CreateAgentDispatchRequest++from core import utils++logger = getLogger(__name__)+++class SubtitleException(Exception):+ """Exception raised when subtitle operations fail."""+++class SubtitleService:+ """Service for managing subtitle agents in LiveKit rooms."""++ @async_to_sync+ async def start_subtitle(self, room):+ """Start subtitle agent for the specified room."""++ lkapi = utils.create_livekit_client()++ try:+ # Transcriber agent prevents duplicate subtitle agents per room+ # No error is raised if agent already exists+ await lkapi.agent_dispatch.create_dispatch(+ CreateAgentDispatchRequest(+ agent_name=settings.ROOM_SUBTITLE_AGENT_NAME, room=str(room.id)+ )+ )+ except Exception as e:+ logger.exception("Failed to create agent dispatch for room %s", room.id)+ raise SubtitleException("Failed to create subtitle agent") from e++ finally:+ await lkapi.aclose()++ @async_to_sync+ async def stop_subtitle(self, room) -> None:+ """Stop subtitle agent for the specified room."""++ raise NotImplementedError("Subtitle agent stopping not yet implemented")diff --git a/src/backend/core/tests/rooms/test_api_rooms_subtitle.py b/src/backend/core/tests/rooms/test_api_rooms_subtitle.pynew file mode 100644index 00000000..6f97137d--- /dev/null+++ b/src/backend/core/tests/rooms/test_api_rooms_subtitle.py@@ -0,0 +1,221 @@+"""+Test rooms API endpoints in the Meet core app: start subtitle.+"""+# pylint: disable=W0621++import uuid+from unittest import mock++from django.conf import settings++import pytest+from livekit.api import AccessToken, TwirpError, VideoGrants+from rest_framework.test import APIClient++from ...factories import RoomFactory, UserFactory++pytestmark = pytest.mark.django_db+++def mock_room_id() -> str:+ """Mock room's id."""+ return "d2aeb774-1ecd-4d73-a3ac-3d3530cad7ff"+++def mock_livekit_token(mock_room_id):+ """Mock LiveKit JWT token."""++ video_grants = VideoGrants(+ room=mock_room_id,+ room_join=True,+ room_admin=True,+ can_update_own_metadata=True,+ can_publish_sources=[+ "camera",+ "microphone",+ "screen_share",+ "screen_share_audio",+ ],+ )++ token = (+ AccessToken(+ api_key=settings.LIVEKIT_CONFIGURATION["api_key"],+ api_secret=settings.LIVEKIT_CONFIGURATION["api_secret"],+ )+ .with_grants(video_grants)+ .with_identity(str(uuid.uuid4()))+ )++ return token.to_jwt()+++def mock_livekit_client():+ """Mock LiveKit API client."""+ with mock.patch("core.utils.create_livekit_client") as mock_create:+ mock_client = mock.AsyncMock()+ mock_create.return_value = mock_client+ yield mock_client+++def test_start_subtitle_missing_token_anonymous(settings):+ """Test that anonymous users cannot start subtitles without a valid LiveKit token."""++ settings.ROOM_SUBTITLE_ENABLED = True++ room = RoomFactory()+ client = APIClient()++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ )++ assert response.status_code == 403+ assert response.json() == {+ "detail": "Authentication credentials were not provided."+ }+++def test_start_subtitle_missing_token_authenticated(settings):+ """Test that authenticated users still need a valid LiveKit token to start subtitles."""++ settings.ROOM_SUBTITLE_ENABLED = True++ room = RoomFactory()+ user = UserFactory()+ client = APIClient()+ client.force_login(user)++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ )++ assert response.status_code == 403+ assert response.json() == {+ "detail": "Authentication credentials were not provided."+ }+++def test_start_subtitle_invalid_token():+ """Test that malformed or invalid LiveKit tokens are rejected."""++ room = RoomFactory()+ user = UserFactory()+ client = APIClient()+ client.force_login(user)++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/", {"token": "invalid-token"}+ )++ assert response.status_code == 403+ assert response.json() == {"detail": "Invalid LiveKit token: Not enough segments"}+++def test_start_subtitle_disabled_by_default(mock_livekit_token):+ """Test that subtitle functionality is disabled when feature flag is off."""++ room = RoomFactory()+ user = UserFactory()+ client = APIClient()+ client.force_login(user)++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ {"token": mock_livekit_token},+ )++ assert response.status_code == 403+ assert response.json() == {"detail": "Access denied, subtitles are disabled."}+++def test_start_subtitle_valid_token(+ settings, mock_livekit_client, mock_livekit_token, mock_room_id+):+ """Test successful subtitle initiation with valid token and enabled feature."""++ settings.ROOM_SUBTITLE_ENABLED = True++ room = RoomFactory(id=mock_room_id)+ client = APIClient()++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ {"token": mock_livekit_token},+ )++ assert response.status_code == 200+ assert response.json() == {"status": "success"}++ mock_livekit_client.agent_dispatch.create_dispatch.assert_called_once()++ call_args = mock_livekit_client.agent_dispatch.create_dispatch.call_args[0][0]+ assert call_args.agent_name == "multi-user-transcriber"+ assert call_args.room == "d2aeb774-1ecd-4d73-a3ac-3d3530cad7ff"+++def test_start_subtitle_twirp_error(+ settings, mock_livekit_client, mock_livekit_token, mock_room_id+):+ """Test handling of LiveKit service errors during subtitle initiation."""++ settings.ROOM_SUBTITLE_ENABLED = True++ room = RoomFactory(id=mock_room_id)+ client = APIClient()++ mock_livekit_client.agent_dispatch.create_dispatch.side_effect = TwirpError(+ msg="Internal server error", code=500, status=500+ )++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ {"token": mock_livekit_token},+ )++ assert response.status_code == 500+ assert response.json() == {+ "error": f"Subtitles failed to start for room {room.slug}"+ }+++def test_start_subtitle_wrong_room(settings, mock_livekit_token):+ """Test that tokens are validated against the correct room ID."""++ settings.ROOM_SUBTITLE_ENABLED = True++ room = RoomFactory()+ client = APIClient()++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ {"token": mock_livekit_token},+ )++ assert response.status_code == 403+ assert response.json() == {+ "detail": "You do not have permission to perform this action."+ }+++def test_start_subtitle_wrong_signature(settings, mock_livekit_token):+ """Test that tokens signed with incorrect signature are rejected."""++ settings.ROOM_SUBTITLE_ENABLED = True+ settings.LIVEKIT_CONFIGURATION["api_secret"] = "wrong-secret"++ room = RoomFactory()+ client = APIClient()++ response = client.post(+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",+ {"token": mock_livekit_token},+ )++ assert response.status_code == 403+ assert response.json() == {+ "detail": "Invalid LiveKit token: Signature verification failed"+ }diff --git a/src/backend/core/tests/services/test_subtitle.py b/src/backend/core/tests/services/test_subtitle.pynew file mode 100644index 00000000..16f2c309--- /dev/null+++ b/src/backend/core/tests/services/test_subtitle.py@@ -0,0 +1,48 @@+"""+Test subtitle service.+"""++# pylint: disable=W0621+from unittest import mock++import pytest++from core.factories import RoomFactory+from core.services.subtitle import SubtitleService++pytestmark = pytest.mark.django_db+++def mock_livekit_client():+ """Mock LiveKit API client."""+ with mock.patch("core.utils.create_livekit_client") as mock_create:+ mock_client = mock.AsyncMock()+ mock_create.return_value = mock_client+ yield mock_client+++def test_start_subtitle_settings(mock_livekit_client, settings):+ """Test that start_subtitle uses the configured agent name from Django settings."""++ settings.ROOM_SUBTITLE_AGENT_NAME = "fake-subtitle-agent-name"++ room = RoomFactory(name="my room")+ SubtitleService().start_subtitle(room)++ mock_livekit_client.agent_dispatch.create_dispatch.assert_called_once()++ call_args = mock_livekit_client.agent_dispatch.create_dispatch.call_args[0][0]+ assert call_args.agent_name == "fake-subtitle-agent-name"+ assert call_args.room == str(room.id)+++def test_stop_subtitle_not_implemented():+ """Test that stop_subtitle raises NotImplementedError."""++ room = RoomFactory(name="my room")++ with pytest.raises(+ NotImplementedError, match="Subtitle agent stopping not yet implemented"+ ):+ SubtitleService().stop_subtitle(room)diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.pyindex a7f4af11..3cc95cde 100755--- a/src/backend/meet/settings.py+++ b/src/backend/meet/settings.py@@ -644,6 +644,16 @@ class Base(Configuration):environ_prefix=None,)+ # Subtitles settings+ ROOM_SUBTITLE_ENABLED = values.BooleanValue(+ False, environ_name="ROOM_SUBTITLE_ENABLED", environ_prefix=None+ )+ ROOM_SUBTITLE_AGENT_NAME = values.Value(+ "multi-user-transcriber",+ environ_name="ROOM_SUBTITLE_AGENT_NAME",+ environ_prefix=None,+ )+# pylint: disable=invalid-name@propertydef ENVIRONMENT(self):diff --git a/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl b/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmplindex 346b426b..088c5d8b 100644--- a/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl+++ b/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl@@ -72,6 +72,7 @@ backend:ROOM_TELEPHONY_DEFAULT_COUNTRY: 'FR'ROOM_TELEPHONY_PHONE_NUMBER: '+33901020304'SSL_CERT_FILE: /usr/local/lib/python3.13/site-packages/certifi/cacert.pem+ ROOM_SUBTITLE_ENABLED: Truemigrate: