Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(backend) keep recordings cut short by LiveKit egress
briquet41f3d1def497
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 68e2075b..e5ab3ffb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,7 @@ and this project adheres to- ⬆️(frontend) upgrade livekit-client to 2.23.0- ✨(backend) expose LiveKit reconnection settings in the config+- ✨(backend) keep recordings cut short by LiveKit egress### Fixeddiff --git a/src/backend/core/recording/enums.py b/src/backend/core/recording/enums.pyindex 40aa54c7..ac10aafc 100644--- a/src/backend/core/recording/enums.py+++ b/src/backend/core/recording/enums.py@@ -27,6 +27,9 @@ class RecordingWorkerEvent(Enum):COMPLETED = "completed"# The recording ended on its configured limit, its media file is available.LIMIT_REACHED = "limit reached"+ # The recording ended early without being stopped, its partial media file+ # is available.+ INTERRUPTED = "interrupted"# The worker stopped before it ever started recording, there is no media file.ABORTED = "aborted"# The worker hit a runtime error once recording had started; its media file@@ -44,6 +47,7 @@ SUCCESSFUL_EVENTS = frozenset({RecordingWorkerEvent.COMPLETED,RecordingWorkerEvent.LIMIT_REACHED,+ RecordingWorkerEvent.INTERRUPTED,})diff --git a/src/backend/core/recording/services/recording_events.py b/src/backend/core/recording/services/recording_events.pyindex cf95084e..d08c8856 100644--- a/src/backend/core/recording/services/recording_events.py+++ b/src/backend/core/recording/services/recording_events.py@@ -33,6 +33,7 @@ NOTIFICATION_PREFIXES = {}NOTIFICATION_SUFFIXES = {RecordingWorkerEvent.LIMIT_REACHED: "LimitReached",+ RecordingWorkerEvent.INTERRUPTED: "Interrupted",RecordingWorkerEvent.FAILED: "Failed",RecordingWorkerEvent.ABORTED: "Aborted",}@@ -61,24 +62,29 @@ class RecordingEventsService:"""@staticmethod- def log_worker_error(recording, event, error=None, error_code=None):- """Log FAILED at error level and expected ABORTED outcomes at info level."""+ def log_worker_error(recording, event, error=None, error_code=None, details=None):+ """Log FAILED at error level, INTERRUPTED at warning level and expected+ ABORTED outcomes at info level."""if event == RecordingWorkerEvent.FAILED:log = logger.error+ elif event == RecordingWorkerEvent.INTERRUPTED:+ log = logger.warningelif event == RecordingWorkerEvent.ABORTED:log = logger.infoelse:returnlog(- "Recording worker reported %s for recording %s (room=%s, mode=%s): %s (error_code=%s)",+ "Recording worker reported %s for recording %s (room=%s, mode=%s): %s "+ "(error_code=%s, details=%s)",event.value,recording.id,recording.room.id,recording.mode,error or "no error reported",error_code or "no error_code reported",+ details or "no details reported",)@staticmethod@@ -196,6 +202,13 @@ class RecordingEventsService:):self._apply_outcome(recording, event, self._handle_limit_reached)+ # Interrupted+ if (+ event == RecordingWorkerEvent.INTERRUPTED+ and recording.status == models.RecordingStatusChoices.ACTIVE+ ):+ self._apply_outcome(recording, event, self._handle_interrupted)+try:self._handle_successful(recording)except RecordingNotSavableError:@@ -224,6 +237,20 @@ class RecordingEventsService:cls._notify_participants(recording, RecordingWorkerEvent.LIMIT_REACHED)+ @classmethod+ def _handle_interrupted(cls, recording: Recording):+ """Stop recording and notify participants that only part of it is saved.++ INTERRUPTED: used when the worker ended the recording early on its own+ because it lost the room or ran out of resources but kept what it had+ recorded so far+ """++ recording.status = models.RecordingStatusChoices.STOPPED+ recording.save()++ cls._notify_participants(recording, RecordingWorkerEvent.INTERRUPTED)+@classmethoddef _handle_failed(cls, recording: Recording):"""Set recording status to failed, matching the worker event, and notify participants.diff --git a/src/backend/core/services/livekit_events.py b/src/backend/core/services/livekit_events.pyindex 850a43f4..4c5eff1c 100644--- a/src/backend/core/services/livekit_events.py+++ b/src/backend/core/services/livekit_events.py@@ -96,6 +96,9 @@ EGRESS_STATUS_TO_RECORDING_EVENT = {api.EgressStatus.EGRESS_FAILED: RecordingWorkerEvent.FAILED,}+EGRESS_ROOM_CONNECTION_FAILED_ERROR = "recording ended early: connection to room failed"+EGRESS_CPU_EXHAUSTED_DETAILS = "End reason: CPU exhausted"+def to_recording_event(egress_status):"""Translate a LiveKit egress status into a recording worker event."""@@ -109,6 +112,34 @@ def to_recording_event(egress_status):return event+def to_ended_recording_event(egress_info):+ """Translate an ended LiveKit egress into a recording worker event.++ This handles partial media file upload where an egress was interrupted+ which can happen when:+ - an transcription egress that lost its room connection uploads an audio file, then ends+ FAILED with the EGRESS_ROOM_CONNECTION_FAILED_ERROR+ - an egress drained because its host ran out of CPU which can transition to COMPLETE state+ with EGRESS_CPU_EXHAUSTED_DETAILS or FAILED state if the drain outlasted its grace period+ and the egress got killed, possibly before the upload.+ """++ status = egress_info.status+ is_cut_short = (+ egress_info.error == EGRESS_ROOM_CONNECTION_FAILED_ERROR+ or egress_info.details == EGRESS_CPU_EXHAUSTED_DETAILS+ )+ has_uploaded_file = any(file.location for file in egress_info.file_results)++ if is_cut_short and (+ status == api.EgressStatus.EGRESS_COMPLETE+ or (status == api.EgressStatus.EGRESS_FAILED and has_uploaded_file)+ ):+ return RecordingWorkerEvent.INTERRUPTED++ return to_recording_event(status)++class LiveKitEventsService:"""Service for processing and handling LiveKit webhook events and notifications."""@@ -221,7 +252,7 @@ class LiveKitEventsService:f"Recording with worker ID {data.egress_info.egress_id} does not exist") from err- event = to_recording_event(data.egress_info.status)+ event = to_ended_recording_event(data.egress_info)self._audit_recording_end(recording, event, data.egress_info.error_code)# Log if/why the recording failed@@ -230,6 +261,7 @@ class LiveKitEventsService:event,error=data.egress_info.error,error_code=data.egress_info.error_code,+ details=data.egress_info.details,)# Update roomdiff --git a/src/backend/core/tests/recording/service/test_recording_events.py b/src/backend/core/tests/recording/service/test_recording_events.pyindex 7c344b6c..7c88f364 100644--- a/src/backend/core/tests/recording/service/test_recording_events.py+++ b/src/backend/core/tests/recording/service/test_recording_events.py@@ -50,6 +50,26 @@ def test_handle_limit_reached_success(mock_notify, mode, notification_type, serv)+ ("mode", "notification_type"),+ (+ ("screen_recording", "screenRecordingInterrupted"),+ ("transcript", "transcriptionInterrupted"),+ ),+)[email protected]("core.utils.notify_participants")+def test_handle_interrupted_success(mock_notify, mode, notification_type, service):+ """Test _handle_interrupted stops recording and notifies participants."""++ recording = RecordingFactory(status="active", mode=mode)+ service._handle_interrupted(recording)++ assert recording.status == "stopped"+ mock_notify.assert_called_once_with(+ room_name=str(recording.room.id), notification_data={"type": notification_type}+ )++@pytest.mark.parametrize(("mode", "notification_type"),(@@ -222,6 +242,7 @@ def test_handle_update_syncs_room_metadata(RecordingWorkerEvent.STARTING,RecordingWorkerEvent.COMPLETED,RecordingWorkerEvent.LIMIT_REACHED,+ RecordingWorkerEvent.INTERRUPTED,RecordingWorkerEvent.ABORTED,RecordingWorkerEvent.FAILED,),@@ -244,6 +265,10 @@ def test_handle_update_ignores_events_without_a_metadata_status((RecordingWorkerEvent.LIMIT_REACHED, "active", "saved", "LimitReached"),(RecordingWorkerEvent.LIMIT_REACHED, "stopped", "saved", None),(RecordingWorkerEvent.LIMIT_REACHED, "saved", "saved", None),+ (RecordingWorkerEvent.INTERRUPTED, "active", "saved", "Interrupted"),+ (RecordingWorkerEvent.INTERRUPTED, "stopped", "saved", None),+ (RecordingWorkerEvent.INTERRUPTED, "saved", "saved", None),+ (RecordingWorkerEvent.INTERRUPTED, "failed_to_stop", "failed_to_stop", None),(RecordingWorkerEvent.ABORTED, "active", "aborted", "Aborted"),(RecordingWorkerEvent.ABORTED, "failed_to_stop", "failed_to_stop", None),(RecordingWorkerEvent.FAILED, "active", "failed", "Failed"),@@ -321,6 +346,7 @@ def test_handle_terminal_event_ignores_non_terminal_events(("event", "expected_status"),((RecordingWorkerEvent.LIMIT_REACHED, "saved"),+ (RecordingWorkerEvent.INTERRUPTED, "saved"),(RecordingWorkerEvent.ABORTED, "aborted"),(RecordingWorkerEvent.FAILED, "failed"),),@@ -407,6 +433,7 @@ def test_handle_update_survives_a_metadata_failure(("event", "expected_level"),((RecordingWorkerEvent.ABORTED, logging.INFO),+ (RecordingWorkerEvent.INTERRUPTED, logging.WARNING),(RecordingWorkerEvent.FAILED, logging.ERROR),),)@@ -419,7 +446,11 @@ def test_log_worker_error_reports_an_unsuccessful_event(with caplog.at_level(logging.INFO):service.log_worker_error(- recording, event, error="could not connect to the room", error_code=500+ recording,+ event,+ error="could not connect to the room",+ error_code=500,+ details="End reason: Failure",)assert (@@ -428,6 +459,7 @@ def test_log_worker_error_reports_an_unsuccessful_event()assert "could not connect to the room" in caplog.textassert "error_code=500" in caplog.text+ assert "details=End reason: Failure" in caplog.textworker_logs = [recordfor record in caplog.records@@ -448,7 +480,7 @@ def test_log_worker_error_reports_an_unsuccessful_event(),)def test_log_worker_error_stays_quiet_on_anything_else(event, service, caplog):- """Test log_worker_error ignores events other than FAILED and ABORTED."""+ """Test log_worker_error ignores events other than FAILED, INTERRUPTED and ABORTED."""recording = RecordingFactory(status="active")diff --git a/src/backend/core/tests/rooms/test_api_rooms_webhook.py b/src/backend/core/tests/rooms/test_api_rooms_webhook.pyindex 2f1fe6c8..286e3113 100644--- a/src/backend/core/tests/rooms/test_api_rooms_webhook.py+++ b/src/backend/core/tests/rooms/test_api_rooms_webhook.py@@ -8,9 +8,12 @@ import hashlibimport jsonfrom unittest import mock+from django.utils import timezone+import pytestfrom livekit import api+from ...factories import RecordingFactoryfrom ...services.livekit_events import ActionFailedError, LiveKitEventsService@@ -184,3 +187,75 @@ def test_action_error(client, mock_livekit_config):content_type="application/json",HTTP_AUTHORIZATION=auth_token,)+++ "core.recording.services.recording_events.notification_service."+ "notify_external_services",+ return_value=False,+)[email protected]("core.utils.notify_participants")[email protected]("core.services.room_management.RoomManagement.update_metadata")+def test_egress_ended_keeps_a_partial_transcript(+ mock_update_metadata,+ mock_notify,+ mock_notify_external_services,+ client,+ mock_livekit_config,+):+ """Should save a transcript whose egress lost the room but uploaded its file.++ LiveKit Egress v1.14 reports it as failed. The payload mirrors the one it+ sends, including a field unknown to the pinned livekit-protocol, which+ must be ignored rather than rejected.+ """+ recording = RecordingFactory(+ worker_id="EG_hycBMAjmt6Ub", mode="transcript", status="active"+ )+ event_data = json.dumps(+ {+ "event": "egress_ended",+ "egressInfo": {+ "egressId": "EG_hycBMAjmt6Ub",+ "roomId": "RM_hycBMAjmt6Ub",+ "roomName": str(recording.room.id),+ "status": "EGRESS_FAILED",+ "error": "recording ended early: connection to room failed",+ "errorCode": 503,+ "details": "End reason: Source closed",+ "fileResults": [+ {+ "filename": f"recordings/{recording.id}.ogg",+ "location": f"https://s3.example.com/recordings/{recording.id}.ogg",+ "size": "123456",+ }+ ],+ },+ "id": "EV_eugWmGhovZmm",+ "createdAt": str(int(timezone.now().timestamp())),+ }+ )+ hash64 = base64.b64encode(hashlib.sha256(event_data.encode()).digest()).decode()+ token = api.AccessToken(+ mock_livekit_config["api_key"], mock_livekit_config["api_secret"]+ )+ token.claims.sha256 = hash64++ response = client.post(+ "/api/v1.0/rooms/webhooks-livekit/",+ data=event_data,+ content_type="application/json",+ HTTP_AUTHORIZATION=token.to_jwt(),+ )++ assert response.status_code == 200+ mock_notify.assert_called_once_with(+ room_name=str(recording.room.id),+ notification_data={"type": "transcriptionInterrupted"},+ )+ mock_notify_external_services.assert_called_once_with(recording)++ recording.refresh_from_db()++ assert recording.status == "saved"diff --git a/src/backend/core/tests/services/test_livekit_events.py b/src/backend/core/tests/services/test_livekit_events.pyindex 2837841b..5cda3c3f 100644--- a/src/backend/core/tests/services/test_livekit_events.py+++ b/src/backend/core/tests/services/test_livekit_events.py@@ -1,7 +1,7 @@"""Test LiveKitEvents service."""-# pylint: disable=W0621,W0613, W0212, E0611, C0302+# pylint: disable=W0621,W0613, W0212, E0611, E1101, C0302import loggingimport uuid@@ -11,6 +11,7 @@ from unittest import mockfrom django.utils import timezoneimport pytest+from faker import Fakerfrom livekit.api import EgressStatusfrom core.audit.testing import find_events@@ -19,12 +20,15 @@ from core.models import Roomfrom core.recording.enums import RecordingWorkerEventfrom core.recording.services.recording_events import RecordingEventsServicefrom core.services.livekit_events import (+ EGRESS_CPU_EXHAUSTED_DETAILS,+ EGRESS_ROOM_CONNECTION_FAILED_ERROR,EGRESS_STATUS_TO_RECORDING_EVENT,ActionFailedError,AuthenticationError,InvalidPayloadError,LiveKitEventsService,api,+ to_ended_recording_event,to_recording_event,)from core.services.lobby import LobbyService@@ -39,6 +43,7 @@ from core.services.sip_management import (from core.utils import NotificationErrorpytestmark = pytest.mark.django_db+fake = Faker()@pytest.fixture@@ -494,6 +499,228 @@ def test_to_recording_event_returns_none_on_unmapped_status(caplog):assert "Unmapped LiveKit egress status" in caplog.text+recording_filename = f"recordings/{fake.file_name(extension='ogg')}"++UPLOADED_FILE = api.FileInfo(+ filename=recording_filename,+ location=f"https://storage.example.com/{recording_filename}",+ size=1048576,+)+NOT_UPLOADED_FILE = api.FileInfo(filename=recording_filename)++ROOM_CONNECTION_FAILED = {+ "status": EgressStatus.EGRESS_FAILED,+ "error": EGRESS_ROOM_CONNECTION_FAILED_ERROR,+ "error_code": 503,+ "details": "End reason: Source closed",+}+CPU_EXHAUSTED_DRAINED = {+ "status": EgressStatus.EGRESS_COMPLETE,+ "details": EGRESS_CPU_EXHAUSTED_DETAILS,+}+CPU_EXHAUSTED_KILLED = {+ "status": EgressStatus.EGRESS_FAILED,+ "error": "CPU exhausted: 8.5 cores used",+ "error_code": 403,+ "details": EGRESS_CPU_EXHAUSTED_DETAILS,+}+++ ("egress_info", "expected_event"),+ (+ (+ api.EgressInfo(**ROOM_CONNECTION_FAILED, file_results=[UPLOADED_FILE]),+ RecordingWorkerEvent.INTERRUPTED,+ ),+ (+ api.EgressInfo(**ROOM_CONNECTION_FAILED, file_results=[NOT_UPLOADED_FILE]),+ RecordingWorkerEvent.FAILED,+ ),+ (+ api.EgressInfo(**CPU_EXHAUSTED_DRAINED, file_results=[UPLOADED_FILE]),+ RecordingWorkerEvent.INTERRUPTED,+ ),+ (+ api.EgressInfo(**CPU_EXHAUSTED_KILLED, file_results=[UPLOADED_FILE]),+ RecordingWorkerEvent.INTERRUPTED,+ ),+ (+ api.EgressInfo(**CPU_EXHAUSTED_KILLED, file_results=[NOT_UPLOADED_FILE]),+ RecordingWorkerEvent.FAILED,+ ),+ # Other failures stay failures, even with an uploaded file.+ (+ api.EgressInfo(+ status=EgressStatus.EGRESS_FAILED,+ error="pipeline frozen",+ error_code=500,+ details="End reason: Source closed",+ file_results=[UPLOADED_FILE],+ ),+ RecordingWorkerEvent.FAILED,+ ),+ (+ api.EgressInfo(+ status=EgressStatus.EGRESS_COMPLETE,+ details="End reason: StopEgress API",+ file_results=[UPLOADED_FILE],+ ),+ RecordingWorkerEvent.COMPLETED,+ ),+ (+ api.EgressInfo(+ status=EgressStatus.EGRESS_ABORTED,+ error="Stop called before pipeline could start",+ error_code=412,+ details="End reason: StopEgress API",+ ),+ RecordingWorkerEvent.ABORTED,+ ),+ (+ api.EgressInfo(+ status=EgressStatus.EGRESS_LIMIT_REACHED,+ error="Session limit reached",+ error_code=413,+ details="End reason: Session limit reached",+ file_results=[UPLOADED_FILE],+ ),+ RecordingWorkerEvent.LIMIT_REACHED,+ ),+ ),+)+def test_to_ended_recording_event_translates_egress_info(egress_info, expected_event):+ """Should tell an egress cut short with a partial file apart from the others."""++ assert to_ended_recording_event(egress_info) == expected_event+++ ("mode", "egress_info_kwargs", "notification_type"),+ (+ ("transcript", ROOM_CONNECTION_FAILED, "transcriptionInterrupted"),+ ("transcript", CPU_EXHAUSTED_DRAINED, "transcriptionInterrupted"),+ ("screen_recording", CPU_EXHAUSTED_DRAINED, "screenRecordingInterrupted"),+ ("screen_recording", CPU_EXHAUSTED_KILLED, "screenRecordingInterrupted"),+ ),+)+ "core.recording.services.recording_events.notification_service."+ "notify_external_services"+)[email protected]("core.utils.notify_participants")[email protected]("core.services.room_management.RoomManagement.update_metadata")+def test_handle_egress_ended_interrupted_saves_partial_recording( # noqa: PLR0913, PLR0917+ mock_update_metadata,+ mock_notify,+ mock_notify_external_services,+ mode,+ egress_info_kwargs,+ notification_type,+ service,+ caplog,+): # pylint: disable=too-many-arguments,too-many-positional-arguments+ """Should save an interrupted recording and tell participants it is partial."""++ mock_notify_external_services.return_value = True++ recording = RecordingFactory(worker_id="worker-1", mode=mode, status="active")+ data = api.WebhookEvent(+ event="egress_ended",+ egress_info=api.EgressInfo(+ egress_id=recording.worker_id,+ file_results=[UPLOADED_FILE],+ **egress_info_kwargs,+ ),+ )++ with caplog.at_level(logging.WARNING):+ service._handle_egress_ended(data)++ mock_notify.assert_called_once_with(+ room_name=str(recording.room.id), notification_data={"type": notification_type}+ )+ mock_notify_external_services.assert_called_once_with(recording)+ mock_update_metadata.assert_called_once_with(+ str(recording.room.id), remove_keys=["recording_mode", "recording_status"]+ )+ assert (+ f"Recording worker reported interrupted for recording {recording.id}"+ in caplog.text+ )++ recording.refresh_from_db()++ assert recording.status == "notification_succeeded"+++ "core.recording.services.recording_events.notification_service."+ "notify_external_services"+)[email protected]("core.utils.notify_participants")[email protected]("core.services.room_management.RoomManagement.update_metadata")+def test_handle_egress_ended_interrupted_after_stop_does_not_notify_participants(+ mock_update_metadata, mock_notify, mock_notify_external_services, service+):+ """Shouldn't notify participants about a recording they already stopped."""++ mock_notify_external_services.return_value = False++ recording = RecordingFactory(worker_id="worker-1", status="stopped")+ data = api.WebhookEvent(+ event="egress_ended",+ egress_info=api.EgressInfo(+ egress_id=recording.worker_id,+ file_results=[UPLOADED_FILE],+ **CPU_EXHAUSTED_DRAINED,+ ),+ )++ service._handle_egress_ended(data)++ mock_notify.assert_not_called()+ mock_notify_external_services.assert_called_once_with(recording)++ recording.refresh_from_db()++ assert recording.status == "saved"++[email protected]("core.utils.notify_participants")[email protected]("core.services.room_management.RoomManagement.update_metadata")+def test_handle_egress_ended_failed_on_another_error_discards_recording(+ mock_update_metadata, mock_notify, service+):+ """Should still flag as failed an egress that failed for another reason."""++ recording = RecordingFactory(+ worker_id="worker-1", mode="transcript", status="active"+ )+ data = api.WebhookEvent(+ event="egress_ended",+ egress_info=api.EgressInfo(+ egress_id=recording.worker_id,+ status=EgressStatus.EGRESS_FAILED,+ error="pipeline frozen",+ error_code=500,+ details="End reason: Source closed",+ file_results=[UPLOADED_FILE],+ ),+ )++ service._handle_egress_ended(data)++ mock_notify.assert_called_once_with(+ room_name=str(recording.room.id),+ notification_data={"type": "transcriptionFailed"},+ )++ recording.refresh_from_db()++ assert recording.status == "failed"++@mock.patch("core.services.room_management.RoomManagement.update_metadata")def test_handle_egress_updated_ignores_unmapped_status(mock_update_metadata, service):"""Shouldn't touch the room's metadata when the egress status is unknown."""