Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(backend) add start-subtitle endpoint

lebaudantoinef48dd5cea130
diff --git a/src/backend/core/api/__init__.py b/src/backend/core/api/__init__.py
index 5e400ca0..45bfaf6b 100644
--- a/src/backend/core/api/__init__.py
+++ b/src/backend/core/api/__init__.py
@@ -50,6 +50,7 @@ def get_frontend_configuration(request):
else None,
"default_country": settings.ROOM_TELEPHONY_DEFAULT_COUNTRY,
},
+ "subtitle": {"enabled": settings.ROOM_SUBTITLE_ENABLED},
"livekit": {
"url": settings.LIVEKIT_CONFIGURATION["url"],
"force_wss_protocol": settings.LIVEKIT_FORCE_WSS_PROTOCOL,
diff --git a/src/backend/core/api/permissions.py b/src/backend/core/api/permissions.py
index 2300e903..68dff17e 100644
--- a/src/backend/core/api/permissions.py
+++ b/src/backend/core/api/permissions.py
@@ -119,3 +119,22 @@ class IsStorageEventEnabled(permissions.BasePermission):
def has_permission(self, request, view):
"""Determine if access is allowed based on settings."""
return settings.RECORDING_STORAGE_EVENT_ENABLE
+
+
+class IsSubtitleEnabled(permissions.BasePermission):
+ """Check if the subtitle feature is enabled."""
+
+ message = "Access denied, subtitles are disabled."
+
+ def has_permission(self, request, view):
+ """Determine if access is allowed based on settings."""
+ return settings.ROOM_SUBTITLE_ENABLED
+
+
+class HasLiveKitRoomAccess(permissions.BasePermission):
+ """Check if authenticated user's LiveKit token is for the specific room."""
+
+ def has_object_permission(self, request, view, obj):
+ if not request.auth or not hasattr(request.auth, "video"):
+ return False
+ return request.auth.video.room == str(obj.id)
diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py
index 76fdf8ae..b2ef65b4 100644
--- a/src/backend/core/api/viewsets.py
+++ b/src/backend/core/api/viewsets.py
@@ -51,7 +51,9 @@ from core.services.lobby import (
LobbyService,
)
from core.services.room_creation import RoomCreation
+from core.services.subtitle import SubtitleException, SubtitleService
+from ..authentication.livekit import LiveKitTokenAuthentication
from . import permissions, serializers
# pylint: disable=too-many-ancestors
@@ -530,6 +532,37 @@ class RoomViewSet(
status=drf_status.HTTP_200_OK,
)
+ @decorators.action(
+ detail=True,
+ methods=["post"],
+ url_path="start-subtitle",
+ permission_classes=[
+ permissions.IsSubtitleEnabled,
+ permissions.HasLiveKitRoomAccess,
+ ],
+ authentication_classes=[LiveKitTokenAuthentication],
+ )
+ def start_subtitle(self, request, pk=None): # pylint: disable=unused-argument
+ """Start realtime transcription for the room.
+
+ Requires valid LiveKit token for room authorization.
+ Anonymous users can start subtitles if they have room access tokens.
+ """
+
+ room = self.get_object()
+
+ try:
+ SubtitleService().start_subtitle(room)
+ except SubtitleException:
+ return drf_response.Response(
+ {"error": f"Subtitles failed to start for room {room.slug}"},
+ status=drf_status.HTTP_500_INTERNAL_SERVER_ERROR,
+ )
+
+ return drf_response.Response(
+ {"status": "success"}, status=drf_status.HTTP_200_OK
+ )
+
class ResourceAccessViewSet(
mixins.CreateModelMixin,
diff --git a/src/backend/core/authentication/livekit.py b/src/backend/core/authentication/livekit.py
new file mode 100644
index 00000000..b236dc54
--- /dev/null
+++ b/src/backend/core/authentication/livekit.py
@@ -0,0 +1,42 @@
+"""Authentication using LiveKit token for the Meet core app."""
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import AnonymousUser
+
+from livekit.api import TokenVerifier
+from rest_framework import authentication, exceptions
+
+UserModel = get_user_model()
+
+
+class LiveKitTokenAuthentication(authentication.BaseAuthentication):
+ """Authenticate using LiveKit token and load the associated Django user."""
+
+ def authenticate(self, request):
+ token = request.data.get("token")
+ if not token:
+ return None # No authentication attempted
+
+ try:
+ verifier = TokenVerifier(
+ api_key=settings.LIVEKIT_CONFIGURATION["api_key"],
+ api_secret=settings.LIVEKIT_CONFIGURATION["api_secret"],
+ )
+ claims = verifier.verify(token)
+
+ user_id = claims.identity
+ if not user_id:
+ raise exceptions.AuthenticationFailed("Token missing user identity")
+
+ try:
+ user = UserModel.objects.get(id=user_id)
+ except UserModel.DoesNotExist:
+ user = AnonymousUser()
+
+ return (user, claims)
+
+ except Exception as e:
+ raise exceptions.AuthenticationFailed(
+ f"Invalid LiveKit token: {str(e)}"
+ ) from e
diff --git a/src/backend/core/services/subtitle.py b/src/backend/core/services/subtitle.py
new file mode 100644
index 00000000..ea007602
--- /dev/null
+++ b/src/backend/core/services/subtitle.py
@@ -0,0 +1,47 @@
+"""Service for managing subtitle agents in LiveKit rooms."""
+
+from logging import getLogger
+
+from django.conf import settings
+
+from asgiref.sync import async_to_sync
+from livekit.protocol.agent_dispatch import CreateAgentDispatchRequest
+
+from core import utils
+
+logger = getLogger(__name__)
+
+
+class SubtitleException(Exception):
+ """Exception raised when subtitle operations fail."""
+
+
+class SubtitleService:
+ """Service for managing subtitle agents in LiveKit rooms."""
+
+ @async_to_sync
+ async def start_subtitle(self, room):
+ """Start subtitle agent for the specified room."""
+
+ lkapi = utils.create_livekit_client()
+
+ try:
+ # Transcriber agent prevents duplicate subtitle agents per room
+ # No error is raised if agent already exists
+ await lkapi.agent_dispatch.create_dispatch(
+ CreateAgentDispatchRequest(
+ agent_name=settings.ROOM_SUBTITLE_AGENT_NAME, room=str(room.id)
+ )
+ )
+ except Exception as e:
+ logger.exception("Failed to create agent dispatch for room %s", room.id)
+ raise SubtitleException("Failed to create subtitle agent") from e
+
+ finally:
+ await lkapi.aclose()
+
+ @async_to_sync
+ async def stop_subtitle(self, room) -> None:
+ """Stop subtitle agent for the specified room."""
+
+ raise NotImplementedError("Subtitle agent stopping not yet implemented")
diff --git a/src/backend/core/tests/rooms/test_api_rooms_subtitle.py b/src/backend/core/tests/rooms/test_api_rooms_subtitle.py
new file mode 100644
index 00000000..6f97137d
--- /dev/null
+++ b/src/backend/core/tests/rooms/test_api_rooms_subtitle.py
@@ -0,0 +1,221 @@
+"""
+Test rooms API endpoints in the Meet core app: start subtitle.
+"""
+# pylint: disable=W0621
+
+import uuid
+from unittest import mock
+
+from django.conf import settings
+
+import pytest
+from livekit.api import AccessToken, TwirpError, VideoGrants
+from rest_framework.test import APIClient
+
+from ...factories import RoomFactory, UserFactory
+
+pytestmark = pytest.mark.django_db
+
+
+def mock_room_id() -> str:
+ """Mock room's id."""
+ return "d2aeb774-1ecd-4d73-a3ac-3d3530cad7ff"
+
+
+def mock_livekit_token(mock_room_id):
+ """Mock LiveKit JWT token."""
+
+ video_grants = VideoGrants(
+ room=mock_room_id,
+ room_join=True,
+ room_admin=True,
+ can_update_own_metadata=True,
+ can_publish_sources=[
+ "camera",
+ "microphone",
+ "screen_share",
+ "screen_share_audio",
+ ],
+ )
+
+ token = (
+ AccessToken(
+ api_key=settings.LIVEKIT_CONFIGURATION["api_key"],
+ api_secret=settings.LIVEKIT_CONFIGURATION["api_secret"],
+ )
+ .with_grants(video_grants)
+ .with_identity(str(uuid.uuid4()))
+ )
+
+ return token.to_jwt()
+
+
+def mock_livekit_client():
+ """Mock LiveKit API client."""
+ with mock.patch("core.utils.create_livekit_client") as mock_create:
+ mock_client = mock.AsyncMock()
+ mock_create.return_value = mock_client
+ yield mock_client
+
+
+def test_start_subtitle_missing_token_anonymous(settings):
+ """Test that anonymous users cannot start subtitles without a valid LiveKit token."""
+
+ settings.ROOM_SUBTITLE_ENABLED = True
+
+ room = RoomFactory()
+ client = APIClient()
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ )
+
+ assert response.status_code == 403
+ assert response.json() == {
+ "detail": "Authentication credentials were not provided."
+ }
+
+
+def test_start_subtitle_missing_token_authenticated(settings):
+ """Test that authenticated users still need a valid LiveKit token to start subtitles."""
+
+ settings.ROOM_SUBTITLE_ENABLED = True
+
+ room = RoomFactory()
+ user = UserFactory()
+ client = APIClient()
+ client.force_login(user)
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ )
+
+ assert response.status_code == 403
+ assert response.json() == {
+ "detail": "Authentication credentials were not provided."
+ }
+
+
+def test_start_subtitle_invalid_token():
+ """Test that malformed or invalid LiveKit tokens are rejected."""
+
+ room = RoomFactory()
+ user = UserFactory()
+ client = APIClient()
+ client.force_login(user)
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/", {"token": "invalid-token"}
+ )
+
+ assert response.status_code == 403
+ assert response.json() == {"detail": "Invalid LiveKit token: Not enough segments"}
+
+
+def test_start_subtitle_disabled_by_default(mock_livekit_token):
+ """Test that subtitle functionality is disabled when feature flag is off."""
+
+ room = RoomFactory()
+ user = UserFactory()
+ client = APIClient()
+ client.force_login(user)
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ {"token": mock_livekit_token},
+ )
+
+ assert response.status_code == 403
+ assert response.json() == {"detail": "Access denied, subtitles are disabled."}
+
+
+def test_start_subtitle_valid_token(
+ settings, mock_livekit_client, mock_livekit_token, mock_room_id
+):
+ """Test successful subtitle initiation with valid token and enabled feature."""
+
+ settings.ROOM_SUBTITLE_ENABLED = True
+
+ room = RoomFactory(id=mock_room_id)
+ client = APIClient()
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ {"token": mock_livekit_token},
+ )
+
+ assert response.status_code == 200
+ assert response.json() == {"status": "success"}
+
+ mock_livekit_client.agent_dispatch.create_dispatch.assert_called_once()
+
+ call_args = mock_livekit_client.agent_dispatch.create_dispatch.call_args[0][0]
+ assert call_args.agent_name == "multi-user-transcriber"
+ assert call_args.room == "d2aeb774-1ecd-4d73-a3ac-3d3530cad7ff"
+
+
+def test_start_subtitle_twirp_error(
+ settings, mock_livekit_client, mock_livekit_token, mock_room_id
+):
+ """Test handling of LiveKit service errors during subtitle initiation."""
+
+ settings.ROOM_SUBTITLE_ENABLED = True
+
+ room = RoomFactory(id=mock_room_id)
+ client = APIClient()
+
+ mock_livekit_client.agent_dispatch.create_dispatch.side_effect = TwirpError(
+ msg="Internal server error", code=500, status=500
+ )
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ {"token": mock_livekit_token},
+ )
+
+ assert response.status_code == 500
+ assert response.json() == {
+ "error": f"Subtitles failed to start for room {room.slug}"
+ }
+
+
+def test_start_subtitle_wrong_room(settings, mock_livekit_token):
+ """Test that tokens are validated against the correct room ID."""
+
+ settings.ROOM_SUBTITLE_ENABLED = True
+
+ room = RoomFactory()
+ client = APIClient()
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ {"token": mock_livekit_token},
+ )
+
+ assert response.status_code == 403
+ assert response.json() == {
+ "detail": "You do not have permission to perform this action."
+ }
+
+
+def test_start_subtitle_wrong_signature(settings, mock_livekit_token):
+ """Test that tokens signed with incorrect signature are rejected."""
+
+ settings.ROOM_SUBTITLE_ENABLED = True
+ settings.LIVEKIT_CONFIGURATION["api_secret"] = "wrong-secret"
+
+ room = RoomFactory()
+ client = APIClient()
+
+ response = client.post(
+ f"/api/v1.0/rooms/{room.id}/start-subtitle/",
+ {"token": mock_livekit_token},
+ )
+
+ assert response.status_code == 403
+ assert response.json() == {
+ "detail": "Invalid LiveKit token: Signature verification failed"
+ }
diff --git a/src/backend/core/tests/services/test_subtitle.py b/src/backend/core/tests/services/test_subtitle.py
new file mode 100644
index 00000000..16f2c309
--- /dev/null
+++ b/src/backend/core/tests/services/test_subtitle.py
@@ -0,0 +1,48 @@
+"""
+Test subtitle service.
+"""
+
+# pylint: disable=W0621
+from unittest import mock
+
+import pytest
+
+from core.factories import RoomFactory
+from core.services.subtitle import SubtitleService
+
+pytestmark = pytest.mark.django_db
+
+
+def mock_livekit_client():
+ """Mock LiveKit API client."""
+ with mock.patch("core.utils.create_livekit_client") as mock_create:
+ mock_client = mock.AsyncMock()
+ mock_create.return_value = mock_client
+ yield mock_client
+
+
+def test_start_subtitle_settings(mock_livekit_client, settings):
+ """Test that start_subtitle uses the configured agent name from Django settings."""
+
+ settings.ROOM_SUBTITLE_AGENT_NAME = "fake-subtitle-agent-name"
+
+ room = RoomFactory(name="my room")
+ SubtitleService().start_subtitle(room)
+
+ mock_livekit_client.agent_dispatch.create_dispatch.assert_called_once()
+
+ call_args = mock_livekit_client.agent_dispatch.create_dispatch.call_args[0][0]
+ assert call_args.agent_name == "fake-subtitle-agent-name"
+ assert call_args.room == str(room.id)
+
+
+def test_stop_subtitle_not_implemented():
+ """Test that stop_subtitle raises NotImplementedError."""
+
+ room = RoomFactory(name="my room")
+
+ with pytest.raises(
+ NotImplementedError, match="Subtitle agent stopping not yet implemented"
+ ):
+ SubtitleService().stop_subtitle(room)
diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py
index a7f4af11..3cc95cde 100755
--- a/src/backend/meet/settings.py
+++ b/src/backend/meet/settings.py
@@ -644,6 +644,16 @@ class Base(Configuration):
environ_prefix=None,
)
+ # Subtitles settings
+ ROOM_SUBTITLE_ENABLED = values.BooleanValue(
+ False, environ_name="ROOM_SUBTITLE_ENABLED", environ_prefix=None
+ )
+ ROOM_SUBTITLE_AGENT_NAME = values.Value(
+ "multi-user-transcriber",
+ environ_name="ROOM_SUBTITLE_AGENT_NAME",
+ environ_prefix=None,
+ )
+
# pylint: disable=invalid-name
@property
def ENVIRONMENT(self):
diff --git a/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl b/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl
index 346b426b..088c5d8b 100644
--- a/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl
+++ b/src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl
@@ -72,6 +72,7 @@ backend:
ROOM_TELEPHONY_DEFAULT_COUNTRY: 'FR'
ROOM_TELEPHONY_PHONE_NUMBER: '+33901020304'
SSL_CERT_FILE: /usr/local/lib/python3.13/site-packages/certifi/cacert.pem
+ ROOM_SUBTITLE_ENABLED: True
migrate: