Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(backend) add core.recording.event.parsers.S3Parser
ilias04dfb9922ff8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e269a280..7f51832d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,6 +15,7 @@ and this project adheres to- ✨(frontend) make reaction toolbar responsive on small viewports- ✨(frontend) enable reactions on mobile devices- ✨(frontend) introduce picture-in-picture meeting+- ✨(backend) add core.recording.event.parsers.S3Parser### Changeddiff --git a/src/backend/core/recording/event/parsers.py b/src/backend/core/recording/event/parsers.pyindex 9cd93059..02f6db10 100644--- a/src/backend/core/recording/event/parsers.py+++ b/src/backend/core/recording/event/parsers.py@@ -1,6 +1,7 @@"""Meet storage event parser classes."""import logging+import mimetypesimport refrom dataclasses import dataclassfrom functools import lru_cache@@ -18,6 +19,9 @@ from .exceptions import (ParsingEventDataError,)+# Additional MIME type mapping+mimetypes.add_type("audio/ogg", ".ogg")+logger = logging.getLogger(__name__)@@ -74,8 +78,8 @@ def get_parser() -> EventParser:return event_parser_cls(bucket_name=settings.AWS_STORAGE_BUCKET_NAME)-class MinioParser:- """Handle parsing and validation of Minio storage events."""+class BaseS3Parser:+ """Base class for handling parsing and validation of S3-compatible storage events."""def __init__(self, bucket_name: str, allowed_filetypes=None):"""Initialize parser with target bucket name and accepted filetypes."""@@ -91,32 +95,6 @@ class MinioParser:rf"(?P<url_encoded_folder_path>(?:[^%]+%2F)+)?{settings.RECORDING_OUTPUT_FOLDER}%2F(?P<recording_id>{UUID_REGEX})\.(?P<extension>{FILE_EXT_REGEX})")- @staticmethod- def parse(data):- """Convert raw Minio event dictionary to StorageEvent object."""-- if not data:- raise ParsingEventDataError("Received empty data.")-- try:- record = data["Records"][0]- s3 = record["s3"]- bucket_name = s3["bucket"]["name"]- file_object = s3["object"]- filepath = file_object["key"]- filetype = file_object["contentType"]- except (KeyError, IndexError) as e:- raise ParsingEventDataError(f"Missing or malformed key: {e}.") from e- try:- return StorageEvent(- filepath=filepath,- filetype=filetype,- bucket_name=bucket_name,- metadata=None,- )- except TypeError as e:- raise ParsingEventDataError(f"Missing essential data fields: {e}") from e-def validate(self, event_data: StorageEvent) -> str:"""Verify StorageEvent matches bucket, filetype and filepath requirements."""@@ -141,9 +119,56 @@ class MinioParser:return recording_iddef get_recording_id(self, data):- """Extract recording ID from Minio event through parsing and validation."""+ """Extract recording ID from S3 event through parsing and validation."""event_data = self.parse(data)- recording_id = self.validate(event_data)+ return self.validate(event_data)- return recording_id+ def parse(self, data: Dict) -> StorageEvent:+ """To be implemented by subclasses."""+ raise NotImplementedError("Subclasses must implement parse()")+++class MinioParser(BaseS3Parser):+ """Minio specific event parsing."""++ def parse(self, data: Dict) -> StorageEvent:+ if not data:+ raise ParsingEventDataError("Received empty data.")+ try:+ record = data["Records"][0]+ s3 = record["s3"]+ return StorageEvent(+ filepath=s3["object"]["key"],+ filetype=s3["object"]["contentType"], # Minio-specific field+ bucket_name=s3["bucket"]["name"],+ metadata=None,+ )+ except (KeyError, IndexError) as e:+ raise ParsingEventDataError(f"Malformed Minio event: {e}") from e+ except TypeError as e:+ raise ParsingEventDataError(f"Missing essential data fields: {e}") from e+++class S3Parser(BaseS3Parser):+ """AWS S3 specific event parsing."""++ def parse(self, data: Dict) -> StorageEvent:+ if not data:+ raise ParsingEventDataError("Received empty data.")+ try:+ # AWS S3 structure can slightly differ from Minio implementation+ record = data["Records"][0]+ s3 = record["s3"]+ filepath = s3["object"]["key"]+ if not filepath:+ raise ParsingEventDataError("Missing object key name")+ filetype, _ = mimetypes.guess_type(filepath)+ return StorageEvent(+ filepath=filepath,+ filetype=filetype,+ bucket_name=s3["bucket"]["name"],+ metadata=None,+ )+ except (KeyError, IndexError) as e:+ raise ParsingEventDataError(f"Malformed S3 event: {e}") from ediff --git a/src/backend/core/tests/recording/event/test_parsers.py b/src/backend/core/tests/recording/event/test_parsers.pyindex 5071ed54..a7ae1916 100644--- a/src/backend/core/tests/recording/event/test_parsers.py+++ b/src/backend/core/tests/recording/event/test_parsers.py@@ -18,10 +18,13 @@ from core.recording.event.exceptions import ()from core.recording.event.parsers import (MinioParser,+ S3Parser,StorageEvent,get_parser,)+# MinioParser+@pytest.fixturedef valid_minio_event():@@ -47,7 +50,7 @@ def minio_parser():return MinioParser(bucket_name="test-bucket")-def test_parse_valid_event(minio_parser, valid_minio_event):+def test_minio_parse_valid_event(minio_parser, valid_minio_event):"""Test parsing a valid Minio event."""event = minio_parser.parse(valid_minio_event)assert isinstance(event, StorageEvent)@@ -57,13 +60,33 @@ def test_parse_valid_event(minio_parser, valid_minio_event):assert event.metadata is None-def test_parse_empty_data(minio_parser):+def test_minio_parse_with_video_type(minio_parser):+ """Test parsing event with video file type."""+ video_event = {+ "Records": [+ {+ "s3": {+ "bucket": {"name": "test-bucket"},+ "object": {+ "key": "46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",+ "contentType": "video/mp4",+ },+ }+ }+ ]+ }+ event = minio_parser.parse(video_event)+ assert event.filetype == "video/mp4"+ assert event.filepath.endswith(".mp4")+++def test_minio_parse_empty_data(minio_parser):"""Test parsing empty event data raises error."""with pytest.raises(ParsingEventDataError, match="Received empty data."):minio_parser.parse({})-def test_parse_missing_keys(minio_parser):+def test_minio_parse_missing_keys(minio_parser):"""Test parsing event with missing key."""invalid_minio_event = {@@ -77,11 +100,11 @@ def test_parse_missing_keys(minio_parser):]}- with pytest.raises(ParsingEventDataError, match="Missing or malformed key"):+ with pytest.raises(ParsingEventDataError, match="Malformed Minio event:"):minio_parser.parse(invalid_minio_event)-def test_parse_none_key(minio_parser):+def test_minio_parse_none_key(minio_parser):"""Test parsing event with None field."""invalid_minio_event = {@@ -102,7 +125,7 @@ def test_parse_none_key(minio_parser):minio_parser.parse(invalid_minio_event)-def test_validate_invalid_bucket(minio_parser):+def test_minio_validate_invalid_bucket(minio_parser):"""Test validation with wrong bucket name."""event = StorageEvent(filepath="recording%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",@@ -114,7 +137,7 @@ def test_validate_invalid_bucket(minio_parser):minio_parser.validate(event)-def test_validate_invalid_filetype(minio_parser):+def test_minio_validate_invalid_filetype(minio_parser):"""Test validation with unsupported file type."""event = StorageEvent(filepath="recording%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.txt",@@ -139,7 +162,7 @@ def test_validate_invalid_filetype(minio_parser):"folder%2Fuploads%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg", # nested but no recordings/],)-def test_validate_invalid_filepath(invalid_filepath, minio_parser):+def test_minio_validate_invalid_filepath(invalid_filepath, minio_parser):"""Test validation with malformed filepath."""event = StorageEvent(filepath=invalid_filepath,@@ -151,7 +174,7 @@ def test_validate_invalid_filepath(invalid_filepath, minio_parser):minio_parser.validate(event)-def test_validate_valid_event(minio_parser):+def test_minio_validate_valid_event(minio_parser):"""Test validation with valid event data."""event = StorageEvent(filepath="recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",@@ -163,13 +186,13 @@ def test_validate_valid_event(minio_parser):assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"-def test_get_recording_id_success(minio_parser, valid_minio_event):+def test_minio_get_recording_id_success(minio_parser, valid_minio_event):"""Test successful extraction of recording ID."""recording_id = minio_parser.get_recording_id(valid_minio_event)assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"-def test_validate_filepath_with_folder(minio_parser):+def test_minio_validate_filepath_with_folder(minio_parser):"""Test validation of filepath with folder structure."""event = StorageEvent(filepath="parent_folder%2Frecordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",@@ -181,41 +204,21 @@ def test_validate_filepath_with_folder(minio_parser):assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"-def test_parse_with_video_type(minio_parser):- """Test parsing event with video file type."""- video_event = {- "Records": [- {- "s3": {- "bucket": {"name": "test-bucket"},- "object": {- "key": "46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",- "contentType": "video/mp4",- },- }- }- ]- }- event = minio_parser.parse(video_event)- assert event.filetype == "video/mp4"- assert event.filepath.endswith(".mp4")---def test_empty_allowed_filetypes():+def test_minio_empty_allowed_filetypes():"""Test MinioParser with empty allowed_filetypes."""empty_types = set()parser = MinioParser(bucket_name="test-bucket", allowed_filetypes=empty_types)assert parser._allowed_filetypes == {"audio/ogg", "video/mp4"}-def test_custom_allowed_filetypes():+def test_minio_custom_allowed_filetypes():"""Test MinioParser with empty allowed_filetypes."""custom_types = {"audio/mp3", "video/mov"}parser = MinioParser(bucket_name="test-bucket", allowed_filetypes=custom_types)assert parser._allowed_filetypes == {"audio/mp3", "video/mov"}-def test_validate_custom_filetypes():+def test_minio_validate_custom_filetypes():"""Test validation of filepath with folder structure."""parser = MinioParser(bucket_name="test-bucket", allowed_filetypes={"audio/mp3"})@@ -229,18 +232,143 @@ def test_validate_custom_filetypes():parser.validate(event)-def test_constructor_none_bucket():+def test_minio_constructor_none_bucket():"""Test MinioParser constructor with None bucket name."""with pytest.raises(ValueError, match="Bucket name cannot be None or empty"):MinioParser(bucket_name=None)-def test_constructor_empty_bucket():+def test_minio_constructor_empty_bucket():"""Test MinioParser constructor with empty bucket name."""with pytest.raises(ValueError, match="Bucket name cannot be None or empty"):MinioParser(bucket_name="")+# S3Parser+++def valid_s3_event():+ """Mock a valid S3 event."""+ return {+ "Records": [+ {+ "s3": {+ "bucket": {"name": "test-bucket"},+ "object": {+ "key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",+ },+ }+ }+ ]+ }+++def s3_parser():+ """Mock an S3 parser."""+ return S3Parser(bucket_name="test-bucket")+++def test_s3_parse_valid_event(s3_parser, valid_s3_event):+ """Test parsing a valid S3 event."""+ event = s3_parser.parse(valid_s3_event)+ assert isinstance(event, StorageEvent)+ assert event.filepath == "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg"+ assert event.filetype == "audio/ogg"+ assert event.bucket_name == "test-bucket"+ assert event.metadata is None+++def test_s3_parse_empty_data(s3_parser):+ """Test parsing empty S3 event data raises error."""+ with pytest.raises(ParsingEventDataError, match="Received empty data."):+ s3_parser.parse({})+++def test_s3_parse_missing_keys(s3_parser):+ """Test parsing S3 event with missing key."""+ invalid_s3_event = {+ "Records": [+ {+ "s3": {+ "bucket": {"name": "test-bucket"},+ # Missing 'object' key+ }+ }+ ]+ }++ with pytest.raises(ParsingEventDataError, match="Malformed S3 event:"):+ s3_parser.parse(invalid_s3_event)+++def test_s3_parse_none_key(s3_parser):+ """Test parsing S3 event with None field."""+ invalid_s3_event = {+ "Records": [+ {+ "s3": {+ "bucket": {"name": "test-bucket"},+ "object": {+ "key": None,+ },+ }+ }+ ]+ }++ with pytest.raises(ParsingEventDataError, match="Missing object key name"):+ s3_parser.parse(invalid_s3_event)+++def test_s3_parse_with_video_type(s3_parser):+ """Test parsing S3 event with mp4 file extension."""+ video_event = {+ "Records": [+ {+ "s3": {+ "bucket": {"name": "test-bucket"},+ "object": {+ "key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",+ },+ }+ }+ ]+ }+ event = s3_parser.parse(video_event)+ assert event.filetype == "video/mp4"+ assert event.filepath.endswith(".mp4")+++def test_s3_parse_unrecognized_extension(s3_parser):+ """Test parsing S3 event with unrecognized file extension."""+ event_with_unknown_ext = {+ "Records": [+ {+ "s3": {+ "bucket": {"name": "test-bucket"},+ "object": {+ "key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.zzunknown999",+ },+ }+ }+ ]+ }++ with pytest.raises(TypeError, match="filetype cannot be None"):+ s3_parser.parse(event_with_unknown_ext)+++def test_s3_get_recording_id_success(s3_parser, valid_s3_event):+ """Test successful extraction of recording ID from S3 event."""+ recording_id = s3_parser.get_recording_id(valid_s3_event)+ assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"+++# get_parser++@pytest.fixturedef clear_lru_cache():"""Fixture to clear the LRU cache between tests."""