Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(backend) keep recordings cut short by LiveKit egress

briquet41f3d1def497
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68e2075b..e5ab3ffb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ and this project adheres to
- ⬆️(frontend) upgrade livekit-client to 2.23.0
- ✨(backend) expose LiveKit reconnection settings in the config
+- ✨(backend) keep recordings cut short by LiveKit egress
### Fixed
diff --git a/src/backend/core/recording/enums.py b/src/backend/core/recording/enums.py
index 40aa54c7..ac10aafc 100644
--- a/src/backend/core/recording/enums.py
+++ b/src/backend/core/recording/enums.py
@@ -27,6 +27,9 @@ class RecordingWorkerEvent(Enum):
COMPLETED = "completed"
# The recording ended on its configured limit, its media file is available.
LIMIT_REACHED = "limit reached"
+ # The recording ended early without being stopped, its partial media file
+ # is available.
+ INTERRUPTED = "interrupted"
# The worker stopped before it ever started recording, there is no media file.
ABORTED = "aborted"
# The worker hit a runtime error once recording had started; its media file
@@ -44,6 +47,7 @@ SUCCESSFUL_EVENTS = frozenset(
{
RecordingWorkerEvent.COMPLETED,
RecordingWorkerEvent.LIMIT_REACHED,
+ RecordingWorkerEvent.INTERRUPTED,
}
)
diff --git a/src/backend/core/recording/services/recording_events.py b/src/backend/core/recording/services/recording_events.py
index cf95084e..d08c8856 100644
--- a/src/backend/core/recording/services/recording_events.py
+++ b/src/backend/core/recording/services/recording_events.py
@@ -33,6 +33,7 @@ NOTIFICATION_PREFIXES = {
}
NOTIFICATION_SUFFIXES = {
RecordingWorkerEvent.LIMIT_REACHED: "LimitReached",
+ RecordingWorkerEvent.INTERRUPTED: "Interrupted",
RecordingWorkerEvent.FAILED: "Failed",
RecordingWorkerEvent.ABORTED: "Aborted",
}
@@ -61,24 +62,29 @@ class RecordingEventsService:
"""
@staticmethod
- def log_worker_error(recording, event, error=None, error_code=None):
- """Log FAILED at error level and expected ABORTED outcomes at info level."""
+ def log_worker_error(recording, event, error=None, error_code=None, details=None):
+ """Log FAILED at error level, INTERRUPTED at warning level and expected
+ ABORTED outcomes at info level."""
if event == RecordingWorkerEvent.FAILED:
log = logger.error
+ elif event == RecordingWorkerEvent.INTERRUPTED:
+ log = logger.warning
elif event == RecordingWorkerEvent.ABORTED:
log = logger.info
else:
return
log(
- "Recording worker reported %s for recording %s (room=%s, mode=%s): %s (error_code=%s)",
+ "Recording worker reported %s for recording %s (room=%s, mode=%s): %s "
+ "(error_code=%s, details=%s)",
event.value,
recording.id,
recording.room.id,
recording.mode,
error or "no error reported",
error_code or "no error_code reported",
+ details or "no details reported",
)
@staticmethod
@@ -196,6 +202,13 @@ class RecordingEventsService:
):
self._apply_outcome(recording, event, self._handle_limit_reached)
+ # Interrupted
+ if (
+ event == RecordingWorkerEvent.INTERRUPTED
+ and recording.status == models.RecordingStatusChoices.ACTIVE
+ ):
+ self._apply_outcome(recording, event, self._handle_interrupted)
+
try:
self._handle_successful(recording)
except RecordingNotSavableError:
@@ -224,6 +237,20 @@ class RecordingEventsService:
cls._notify_participants(recording, RecordingWorkerEvent.LIMIT_REACHED)
+ @classmethod
+ def _handle_interrupted(cls, recording: Recording):
+ """Stop recording and notify participants that only part of it is saved.
+
+ INTERRUPTED: used when the worker ended the recording early on its own
+ because it lost the room or ran out of resources but kept what it had
+ recorded so far
+ """
+
+ recording.status = models.RecordingStatusChoices.STOPPED
+ recording.save()
+
+ cls._notify_participants(recording, RecordingWorkerEvent.INTERRUPTED)
+
@classmethod
def _handle_failed(cls, recording: Recording):
"""Set recording status to failed, matching the worker event, and notify participants.
diff --git a/src/backend/core/services/livekit_events.py b/src/backend/core/services/livekit_events.py
index 850a43f4..4c5eff1c 100644
--- a/src/backend/core/services/livekit_events.py
+++ b/src/backend/core/services/livekit_events.py
@@ -96,6 +96,9 @@ EGRESS_STATUS_TO_RECORDING_EVENT = {
api.EgressStatus.EGRESS_FAILED: RecordingWorkerEvent.FAILED,
}
+EGRESS_ROOM_CONNECTION_FAILED_ERROR = "recording ended early: connection to room failed"
+EGRESS_CPU_EXHAUSTED_DETAILS = "End reason: CPU exhausted"
+
def to_recording_event(egress_status):
"""Translate a LiveKit egress status into a recording worker event."""
@@ -109,6 +112,34 @@ def to_recording_event(egress_status):
return event
+def to_ended_recording_event(egress_info):
+ """Translate an ended LiveKit egress into a recording worker event.
+
+ This handles partial media file upload where an egress was interrupted
+ which can happen when:
+ - an transcription egress that lost its room connection uploads an audio file, then ends
+ FAILED with the EGRESS_ROOM_CONNECTION_FAILED_ERROR
+ - an egress drained because its host ran out of CPU which can transition to COMPLETE state
+ with EGRESS_CPU_EXHAUSTED_DETAILS or FAILED state if the drain outlasted its grace period
+ and the egress got killed, possibly before the upload.
+ """
+
+ status = egress_info.status
+ is_cut_short = (
+ egress_info.error == EGRESS_ROOM_CONNECTION_FAILED_ERROR
+ or egress_info.details == EGRESS_CPU_EXHAUSTED_DETAILS
+ )
+ has_uploaded_file = any(file.location for file in egress_info.file_results)
+
+ if is_cut_short and (
+ status == api.EgressStatus.EGRESS_COMPLETE
+ or (status == api.EgressStatus.EGRESS_FAILED and has_uploaded_file)
+ ):
+ return RecordingWorkerEvent.INTERRUPTED
+
+ return to_recording_event(status)
+
+
class LiveKitEventsService:
"""Service for processing and handling LiveKit webhook events and notifications."""
@@ -221,7 +252,7 @@ class LiveKitEventsService:
f"Recording with worker ID {data.egress_info.egress_id} does not exist"
) from err
- event = to_recording_event(data.egress_info.status)
+ event = to_ended_recording_event(data.egress_info)
self._audit_recording_end(recording, event, data.egress_info.error_code)
# Log if/why the recording failed
@@ -230,6 +261,7 @@ class LiveKitEventsService:
event,
error=data.egress_info.error,
error_code=data.egress_info.error_code,
+ details=data.egress_info.details,
)
# Update room
diff --git a/src/backend/core/tests/recording/service/test_recording_events.py b/src/backend/core/tests/recording/service/test_recording_events.py
index 7c344b6c..7c88f364 100644
--- a/src/backend/core/tests/recording/service/test_recording_events.py
+++ b/src/backend/core/tests/recording/service/test_recording_events.py
@@ -50,6 +50,26 @@ def test_handle_limit_reached_success(mock_notify, mode, notification_type, serv
)
+ ("mode", "notification_type"),
+ (
+ ("screen_recording", "screenRecordingInterrupted"),
+ ("transcript", "transcriptionInterrupted"),
+ ),
+)
[email protected]("core.utils.notify_participants")
+def test_handle_interrupted_success(mock_notify, mode, notification_type, service):
+ """Test _handle_interrupted stops recording and notifies participants."""
+
+ recording = RecordingFactory(status="active", mode=mode)
+ service._handle_interrupted(recording)
+
+ assert recording.status == "stopped"
+ mock_notify.assert_called_once_with(
+ room_name=str(recording.room.id), notification_data={"type": notification_type}
+ )
+
+
@pytest.mark.parametrize(
("mode", "notification_type"),
(
@@ -222,6 +242,7 @@ def test_handle_update_syncs_room_metadata(
RecordingWorkerEvent.STARTING,
RecordingWorkerEvent.COMPLETED,
RecordingWorkerEvent.LIMIT_REACHED,
+ RecordingWorkerEvent.INTERRUPTED,
RecordingWorkerEvent.ABORTED,
RecordingWorkerEvent.FAILED,
),
@@ -244,6 +265,10 @@ def test_handle_update_ignores_events_without_a_metadata_status(
(RecordingWorkerEvent.LIMIT_REACHED, "active", "saved", "LimitReached"),
(RecordingWorkerEvent.LIMIT_REACHED, "stopped", "saved", None),
(RecordingWorkerEvent.LIMIT_REACHED, "saved", "saved", None),
+ (RecordingWorkerEvent.INTERRUPTED, "active", "saved", "Interrupted"),
+ (RecordingWorkerEvent.INTERRUPTED, "stopped", "saved", None),
+ (RecordingWorkerEvent.INTERRUPTED, "saved", "saved", None),
+ (RecordingWorkerEvent.INTERRUPTED, "failed_to_stop", "failed_to_stop", None),
(RecordingWorkerEvent.ABORTED, "active", "aborted", "Aborted"),
(RecordingWorkerEvent.ABORTED, "failed_to_stop", "failed_to_stop", None),
(RecordingWorkerEvent.FAILED, "active", "failed", "Failed"),
@@ -321,6 +346,7 @@ def test_handle_terminal_event_ignores_non_terminal_events(
("event", "expected_status"),
(
(RecordingWorkerEvent.LIMIT_REACHED, "saved"),
+ (RecordingWorkerEvent.INTERRUPTED, "saved"),
(RecordingWorkerEvent.ABORTED, "aborted"),
(RecordingWorkerEvent.FAILED, "failed"),
),
@@ -407,6 +433,7 @@ def test_handle_update_survives_a_metadata_failure(
("event", "expected_level"),
(
(RecordingWorkerEvent.ABORTED, logging.INFO),
+ (RecordingWorkerEvent.INTERRUPTED, logging.WARNING),
(RecordingWorkerEvent.FAILED, logging.ERROR),
),
)
@@ -419,7 +446,11 @@ def test_log_worker_error_reports_an_unsuccessful_event(
with caplog.at_level(logging.INFO):
service.log_worker_error(
- recording, event, error="could not connect to the room", error_code=500
+ recording,
+ event,
+ error="could not connect to the room",
+ error_code=500,
+ details="End reason: Failure",
)
assert (
@@ -428,6 +459,7 @@ def test_log_worker_error_reports_an_unsuccessful_event(
)
assert "could not connect to the room" in caplog.text
assert "error_code=500" in caplog.text
+ assert "details=End reason: Failure" in caplog.text
worker_logs = [
record
for record in caplog.records
@@ -448,7 +480,7 @@ def test_log_worker_error_reports_an_unsuccessful_event(
),
)
def test_log_worker_error_stays_quiet_on_anything_else(event, service, caplog):
- """Test log_worker_error ignores events other than FAILED and ABORTED."""
+ """Test log_worker_error ignores events other than FAILED, INTERRUPTED and ABORTED."""
recording = RecordingFactory(status="active")
diff --git a/src/backend/core/tests/rooms/test_api_rooms_webhook.py b/src/backend/core/tests/rooms/test_api_rooms_webhook.py
index 2f1fe6c8..286e3113 100644
--- a/src/backend/core/tests/rooms/test_api_rooms_webhook.py
+++ b/src/backend/core/tests/rooms/test_api_rooms_webhook.py
@@ -8,9 +8,12 @@ import hashlib
import json
from unittest import mock
+from django.utils import timezone
+
import pytest
from livekit import api
+from ...factories import RecordingFactory
from ...services.livekit_events import ActionFailedError, LiveKitEventsService
@@ -184,3 +187,75 @@ def test_action_error(client, mock_livekit_config):
content_type="application/json",
HTTP_AUTHORIZATION=auth_token,
)
+
+
+ "core.recording.services.recording_events.notification_service."
+ "notify_external_services",
+ return_value=False,
+)
[email protected]("core.utils.notify_participants")
[email protected]("core.services.room_management.RoomManagement.update_metadata")
+def test_egress_ended_keeps_a_partial_transcript(
+ mock_update_metadata,
+ mock_notify,
+ mock_notify_external_services,
+ client,
+ mock_livekit_config,
+):
+ """Should save a transcript whose egress lost the room but uploaded its file.
+
+ LiveKit Egress v1.14 reports it as failed. The payload mirrors the one it
+ sends, including a field unknown to the pinned livekit-protocol, which
+ must be ignored rather than rejected.
+ """
+ recording = RecordingFactory(
+ worker_id="EG_hycBMAjmt6Ub", mode="transcript", status="active"
+ )
+ event_data = json.dumps(
+ {
+ "event": "egress_ended",
+ "egressInfo": {
+ "egressId": "EG_hycBMAjmt6Ub",
+ "roomId": "RM_hycBMAjmt6Ub",
+ "roomName": str(recording.room.id),
+ "status": "EGRESS_FAILED",
+ "error": "recording ended early: connection to room failed",
+ "errorCode": 503,
+ "details": "End reason: Source closed",
+ "fileResults": [
+ {
+ "filename": f"recordings/{recording.id}.ogg",
+ "location": f"https://s3.example.com/recordings/{recording.id}.ogg",
+ "size": "123456",
+ }
+ ],
+ },
+ "id": "EV_eugWmGhovZmm",
+ "createdAt": str(int(timezone.now().timestamp())),
+ }
+ )
+ hash64 = base64.b64encode(hashlib.sha256(event_data.encode()).digest()).decode()
+ token = api.AccessToken(
+ mock_livekit_config["api_key"], mock_livekit_config["api_secret"]
+ )
+ token.claims.sha256 = hash64
+
+ response = client.post(
+ "/api/v1.0/rooms/webhooks-livekit/",
+ data=event_data,
+ content_type="application/json",
+ HTTP_AUTHORIZATION=token.to_jwt(),
+ )
+
+ assert response.status_code == 200
+ mock_notify.assert_called_once_with(
+ room_name=str(recording.room.id),
+ notification_data={"type": "transcriptionInterrupted"},
+ )
+ mock_notify_external_services.assert_called_once_with(recording)
+
+ recording.refresh_from_db()
+
+ assert recording.status == "saved"
diff --git a/src/backend/core/tests/services/test_livekit_events.py b/src/backend/core/tests/services/test_livekit_events.py
index 2837841b..5cda3c3f 100644
--- a/src/backend/core/tests/services/test_livekit_events.py
+++ b/src/backend/core/tests/services/test_livekit_events.py
@@ -1,7 +1,7 @@
"""
Test LiveKitEvents service.
"""
-# pylint: disable=W0621,W0613, W0212, E0611, C0302
+# pylint: disable=W0621,W0613, W0212, E0611, E1101, C0302
import logging
import uuid
@@ -11,6 +11,7 @@ from unittest import mock
from django.utils import timezone
import pytest
+from faker import Faker
from livekit.api import EgressStatus
from core.audit.testing import find_events
@@ -19,12 +20,15 @@ from core.models import Room
from core.recording.enums import RecordingWorkerEvent
from core.recording.services.recording_events import RecordingEventsService
from core.services.livekit_events import (
+ EGRESS_CPU_EXHAUSTED_DETAILS,
+ EGRESS_ROOM_CONNECTION_FAILED_ERROR,
EGRESS_STATUS_TO_RECORDING_EVENT,
ActionFailedError,
AuthenticationError,
InvalidPayloadError,
LiveKitEventsService,
api,
+ to_ended_recording_event,
to_recording_event,
)
from core.services.lobby import LobbyService
@@ -39,6 +43,7 @@ from core.services.sip_management import (
from core.utils import NotificationError
pytestmark = pytest.mark.django_db
+fake = Faker()
@pytest.fixture
@@ -494,6 +499,228 @@ def test_to_recording_event_returns_none_on_unmapped_status(caplog):
assert "Unmapped LiveKit egress status" in caplog.text
+recording_filename = f"recordings/{fake.file_name(extension='ogg')}"
+
+UPLOADED_FILE = api.FileInfo(
+ filename=recording_filename,
+ location=f"https://storage.example.com/{recording_filename}",
+ size=1048576,
+)
+NOT_UPLOADED_FILE = api.FileInfo(filename=recording_filename)
+
+ROOM_CONNECTION_FAILED = {
+ "status": EgressStatus.EGRESS_FAILED,
+ "error": EGRESS_ROOM_CONNECTION_FAILED_ERROR,
+ "error_code": 503,
+ "details": "End reason: Source closed",
+}
+CPU_EXHAUSTED_DRAINED = {
+ "status": EgressStatus.EGRESS_COMPLETE,
+ "details": EGRESS_CPU_EXHAUSTED_DETAILS,
+}
+CPU_EXHAUSTED_KILLED = {
+ "status": EgressStatus.EGRESS_FAILED,
+ "error": "CPU exhausted: 8.5 cores used",
+ "error_code": 403,
+ "details": EGRESS_CPU_EXHAUSTED_DETAILS,
+}
+
+
+ ("egress_info", "expected_event"),
+ (
+ (
+ api.EgressInfo(**ROOM_CONNECTION_FAILED, file_results=[UPLOADED_FILE]),
+ RecordingWorkerEvent.INTERRUPTED,
+ ),
+ (
+ api.EgressInfo(**ROOM_CONNECTION_FAILED, file_results=[NOT_UPLOADED_FILE]),
+ RecordingWorkerEvent.FAILED,
+ ),
+ (
+ api.EgressInfo(**CPU_EXHAUSTED_DRAINED, file_results=[UPLOADED_FILE]),
+ RecordingWorkerEvent.INTERRUPTED,
+ ),
+ (
+ api.EgressInfo(**CPU_EXHAUSTED_KILLED, file_results=[UPLOADED_FILE]),
+ RecordingWorkerEvent.INTERRUPTED,
+ ),
+ (
+ api.EgressInfo(**CPU_EXHAUSTED_KILLED, file_results=[NOT_UPLOADED_FILE]),
+ RecordingWorkerEvent.FAILED,
+ ),
+ # Other failures stay failures, even with an uploaded file.
+ (
+ api.EgressInfo(
+ status=EgressStatus.EGRESS_FAILED,
+ error="pipeline frozen",
+ error_code=500,
+ details="End reason: Source closed",
+ file_results=[UPLOADED_FILE],
+ ),
+ RecordingWorkerEvent.FAILED,
+ ),
+ (
+ api.EgressInfo(
+ status=EgressStatus.EGRESS_COMPLETE,
+ details="End reason: StopEgress API",
+ file_results=[UPLOADED_FILE],
+ ),
+ RecordingWorkerEvent.COMPLETED,
+ ),
+ (
+ api.EgressInfo(
+ status=EgressStatus.EGRESS_ABORTED,
+ error="Stop called before pipeline could start",
+ error_code=412,
+ details="End reason: StopEgress API",
+ ),
+ RecordingWorkerEvent.ABORTED,
+ ),
+ (
+ api.EgressInfo(
+ status=EgressStatus.EGRESS_LIMIT_REACHED,
+ error="Session limit reached",
+ error_code=413,
+ details="End reason: Session limit reached",
+ file_results=[UPLOADED_FILE],
+ ),
+ RecordingWorkerEvent.LIMIT_REACHED,
+ ),
+ ),
+)
+def test_to_ended_recording_event_translates_egress_info(egress_info, expected_event):
+ """Should tell an egress cut short with a partial file apart from the others."""
+
+ assert to_ended_recording_event(egress_info) == expected_event
+
+
+ ("mode", "egress_info_kwargs", "notification_type"),
+ (
+ ("transcript", ROOM_CONNECTION_FAILED, "transcriptionInterrupted"),
+ ("transcript", CPU_EXHAUSTED_DRAINED, "transcriptionInterrupted"),
+ ("screen_recording", CPU_EXHAUSTED_DRAINED, "screenRecordingInterrupted"),
+ ("screen_recording", CPU_EXHAUSTED_KILLED, "screenRecordingInterrupted"),
+ ),
+)
+ "core.recording.services.recording_events.notification_service."
+ "notify_external_services"
+)
[email protected]("core.utils.notify_participants")
[email protected]("core.services.room_management.RoomManagement.update_metadata")
+def test_handle_egress_ended_interrupted_saves_partial_recording( # noqa: PLR0913, PLR0917
+ mock_update_metadata,
+ mock_notify,
+ mock_notify_external_services,
+ mode,
+ egress_info_kwargs,
+ notification_type,
+ service,
+ caplog,
+): # pylint: disable=too-many-arguments,too-many-positional-arguments
+ """Should save an interrupted recording and tell participants it is partial."""
+
+ mock_notify_external_services.return_value = True
+
+ recording = RecordingFactory(worker_id="worker-1", mode=mode, status="active")
+ data = api.WebhookEvent(
+ event="egress_ended",
+ egress_info=api.EgressInfo(
+ egress_id=recording.worker_id,
+ file_results=[UPLOADED_FILE],
+ **egress_info_kwargs,
+ ),
+ )
+
+ with caplog.at_level(logging.WARNING):
+ service._handle_egress_ended(data)
+
+ mock_notify.assert_called_once_with(
+ room_name=str(recording.room.id), notification_data={"type": notification_type}
+ )
+ mock_notify_external_services.assert_called_once_with(recording)
+ mock_update_metadata.assert_called_once_with(
+ str(recording.room.id), remove_keys=["recording_mode", "recording_status"]
+ )
+ assert (
+ f"Recording worker reported interrupted for recording {recording.id}"
+ in caplog.text
+ )
+
+ recording.refresh_from_db()
+
+ assert recording.status == "notification_succeeded"
+
+
+ "core.recording.services.recording_events.notification_service."
+ "notify_external_services"
+)
[email protected]("core.utils.notify_participants")
[email protected]("core.services.room_management.RoomManagement.update_metadata")
+def test_handle_egress_ended_interrupted_after_stop_does_not_notify_participants(
+ mock_update_metadata, mock_notify, mock_notify_external_services, service
+):
+ """Shouldn't notify participants about a recording they already stopped."""
+
+ mock_notify_external_services.return_value = False
+
+ recording = RecordingFactory(worker_id="worker-1", status="stopped")
+ data = api.WebhookEvent(
+ event="egress_ended",
+ egress_info=api.EgressInfo(
+ egress_id=recording.worker_id,
+ file_results=[UPLOADED_FILE],
+ **CPU_EXHAUSTED_DRAINED,
+ ),
+ )
+
+ service._handle_egress_ended(data)
+
+ mock_notify.assert_not_called()
+ mock_notify_external_services.assert_called_once_with(recording)
+
+ recording.refresh_from_db()
+
+ assert recording.status == "saved"
+
+
[email protected]("core.utils.notify_participants")
[email protected]("core.services.room_management.RoomManagement.update_metadata")
+def test_handle_egress_ended_failed_on_another_error_discards_recording(
+ mock_update_metadata, mock_notify, service
+):
+ """Should still flag as failed an egress that failed for another reason."""
+
+ recording = RecordingFactory(
+ worker_id="worker-1", mode="transcript", status="active"
+ )
+ data = api.WebhookEvent(
+ event="egress_ended",
+ egress_info=api.EgressInfo(
+ egress_id=recording.worker_id,
+ status=EgressStatus.EGRESS_FAILED,
+ error="pipeline frozen",
+ error_code=500,
+ details="End reason: Source closed",
+ file_results=[UPLOADED_FILE],
+ ),
+ )
+
+ service._handle_egress_ended(data)
+
+ mock_notify.assert_called_once_with(
+ room_name=str(recording.room.id),
+ notification_data={"type": "transcriptionFailed"},
+ )
+
+ recording.refresh_from_db()
+
+ assert recording.status == "failed"
+
+
@mock.patch("core.services.room_management.RoomManagement.update_metadata")
def test_handle_egress_updated_ignores_unmapped_status(mock_update_metadata, service):
"""Shouldn't touch the room's metadata when the egress status is unknown."""