Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(backend) add core.recording.event.parsers.S3Parser

ilias04dfb9922ff8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e269a280..7f51832d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to
- ✨(frontend) make reaction toolbar responsive on small viewports
- ✨(frontend) enable reactions on mobile devices
- ✨(frontend) introduce picture-in-picture meeting
+- ✨(backend) add core.recording.event.parsers.S3Parser
### Changed
diff --git a/src/backend/core/recording/event/parsers.py b/src/backend/core/recording/event/parsers.py
index 9cd93059..02f6db10 100644
--- a/src/backend/core/recording/event/parsers.py
+++ b/src/backend/core/recording/event/parsers.py
@@ -1,6 +1,7 @@
"""Meet storage event parser classes."""
import logging
+import mimetypes
import re
from dataclasses import dataclass
from functools import lru_cache
@@ -18,6 +19,9 @@ from .exceptions import (
ParsingEventDataError,
)
+# Additional MIME type mapping
+mimetypes.add_type("audio/ogg", ".ogg")
+
logger = logging.getLogger(__name__)
@@ -74,8 +78,8 @@ def get_parser() -> EventParser:
return event_parser_cls(bucket_name=settings.AWS_STORAGE_BUCKET_NAME)
-class MinioParser:
- """Handle parsing and validation of Minio storage events."""
+class BaseS3Parser:
+ """Base class for handling parsing and validation of S3-compatible storage events."""
def __init__(self, bucket_name: str, allowed_filetypes=None):
"""Initialize parser with target bucket name and accepted filetypes."""
@@ -91,32 +95,6 @@ class MinioParser:
rf"(?P<url_encoded_folder_path>(?:[^%]+%2F)+)?{settings.RECORDING_OUTPUT_FOLDER}%2F(?P<recording_id>{UUID_REGEX})\.(?P<extension>{FILE_EXT_REGEX})"
)
- @staticmethod
- def parse(data):
- """Convert raw Minio event dictionary to StorageEvent object."""
-
- if not data:
- raise ParsingEventDataError("Received empty data.")
-
- try:
- record = data["Records"][0]
- s3 = record["s3"]
- bucket_name = s3["bucket"]["name"]
- file_object = s3["object"]
- filepath = file_object["key"]
- filetype = file_object["contentType"]
- except (KeyError, IndexError) as e:
- raise ParsingEventDataError(f"Missing or malformed key: {e}.") from e
- try:
- return StorageEvent(
- filepath=filepath,
- filetype=filetype,
- bucket_name=bucket_name,
- metadata=None,
- )
- except TypeError as e:
- raise ParsingEventDataError(f"Missing essential data fields: {e}") from e
-
def validate(self, event_data: StorageEvent) -> str:
"""Verify StorageEvent matches bucket, filetype and filepath requirements."""
@@ -141,9 +119,56 @@ class MinioParser:
return recording_id
def get_recording_id(self, data):
- """Extract recording ID from Minio event through parsing and validation."""
+ """Extract recording ID from S3 event through parsing and validation."""
event_data = self.parse(data)
- recording_id = self.validate(event_data)
+ return self.validate(event_data)
- return recording_id
+ def parse(self, data: Dict) -> StorageEvent:
+ """To be implemented by subclasses."""
+ raise NotImplementedError("Subclasses must implement parse()")
+
+
+class MinioParser(BaseS3Parser):
+ """Minio specific event parsing."""
+
+ def parse(self, data: Dict) -> StorageEvent:
+ if not data:
+ raise ParsingEventDataError("Received empty data.")
+ try:
+ record = data["Records"][0]
+ s3 = record["s3"]
+ return StorageEvent(
+ filepath=s3["object"]["key"],
+ filetype=s3["object"]["contentType"], # Minio-specific field
+ bucket_name=s3["bucket"]["name"],
+ metadata=None,
+ )
+ except (KeyError, IndexError) as e:
+ raise ParsingEventDataError(f"Malformed Minio event: {e}") from e
+ except TypeError as e:
+ raise ParsingEventDataError(f"Missing essential data fields: {e}") from e
+
+
+class S3Parser(BaseS3Parser):
+ """AWS S3 specific event parsing."""
+
+ def parse(self, data: Dict) -> StorageEvent:
+ if not data:
+ raise ParsingEventDataError("Received empty data.")
+ try:
+ # AWS S3 structure can slightly differ from Minio implementation
+ record = data["Records"][0]
+ s3 = record["s3"]
+ filepath = s3["object"]["key"]
+ if not filepath:
+ raise ParsingEventDataError("Missing object key name")
+ filetype, _ = mimetypes.guess_type(filepath)
+ return StorageEvent(
+ filepath=filepath,
+ filetype=filetype,
+ bucket_name=s3["bucket"]["name"],
+ metadata=None,
+ )
+ except (KeyError, IndexError) as e:
+ raise ParsingEventDataError(f"Malformed S3 event: {e}") from e
diff --git a/src/backend/core/tests/recording/event/test_parsers.py b/src/backend/core/tests/recording/event/test_parsers.py
index 5071ed54..a7ae1916 100644
--- a/src/backend/core/tests/recording/event/test_parsers.py
+++ b/src/backend/core/tests/recording/event/test_parsers.py
@@ -18,10 +18,13 @@ from core.recording.event.exceptions import (
)
from core.recording.event.parsers import (
MinioParser,
+ S3Parser,
StorageEvent,
get_parser,
)
+# MinioParser
+
@pytest.fixture
def valid_minio_event():
@@ -47,7 +50,7 @@ def minio_parser():
return MinioParser(bucket_name="test-bucket")
-def test_parse_valid_event(minio_parser, valid_minio_event):
+def test_minio_parse_valid_event(minio_parser, valid_minio_event):
"""Test parsing a valid Minio event."""
event = minio_parser.parse(valid_minio_event)
assert isinstance(event, StorageEvent)
@@ -57,13 +60,33 @@ def test_parse_valid_event(minio_parser, valid_minio_event):
assert event.metadata is None
-def test_parse_empty_data(minio_parser):
+def test_minio_parse_with_video_type(minio_parser):
+ """Test parsing event with video file type."""
+ video_event = {
+ "Records": [
+ {
+ "s3": {
+ "bucket": {"name": "test-bucket"},
+ "object": {
+ "key": "46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",
+ "contentType": "video/mp4",
+ },
+ }
+ }
+ ]
+ }
+ event = minio_parser.parse(video_event)
+ assert event.filetype == "video/mp4"
+ assert event.filepath.endswith(".mp4")
+
+
+def test_minio_parse_empty_data(minio_parser):
"""Test parsing empty event data raises error."""
with pytest.raises(ParsingEventDataError, match="Received empty data."):
minio_parser.parse({})
-def test_parse_missing_keys(minio_parser):
+def test_minio_parse_missing_keys(minio_parser):
"""Test parsing event with missing key."""
invalid_minio_event = {
@@ -77,11 +100,11 @@ def test_parse_missing_keys(minio_parser):
]
}
- with pytest.raises(ParsingEventDataError, match="Missing or malformed key"):
+ with pytest.raises(ParsingEventDataError, match="Malformed Minio event:"):
minio_parser.parse(invalid_minio_event)
-def test_parse_none_key(minio_parser):
+def test_minio_parse_none_key(minio_parser):
"""Test parsing event with None field."""
invalid_minio_event = {
@@ -102,7 +125,7 @@ def test_parse_none_key(minio_parser):
minio_parser.parse(invalid_minio_event)
-def test_validate_invalid_bucket(minio_parser):
+def test_minio_validate_invalid_bucket(minio_parser):
"""Test validation with wrong bucket name."""
event = StorageEvent(
filepath="recording%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
@@ -114,7 +137,7 @@ def test_validate_invalid_bucket(minio_parser):
minio_parser.validate(event)
-def test_validate_invalid_filetype(minio_parser):
+def test_minio_validate_invalid_filetype(minio_parser):
"""Test validation with unsupported file type."""
event = StorageEvent(
filepath="recording%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.txt",
@@ -139,7 +162,7 @@ def test_validate_invalid_filetype(minio_parser):
"folder%2Fuploads%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg", # nested but no recordings/
],
)
-def test_validate_invalid_filepath(invalid_filepath, minio_parser):
+def test_minio_validate_invalid_filepath(invalid_filepath, minio_parser):
"""Test validation with malformed filepath."""
event = StorageEvent(
filepath=invalid_filepath,
@@ -151,7 +174,7 @@ def test_validate_invalid_filepath(invalid_filepath, minio_parser):
minio_parser.validate(event)
-def test_validate_valid_event(minio_parser):
+def test_minio_validate_valid_event(minio_parser):
"""Test validation with valid event data."""
event = StorageEvent(
filepath="recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
@@ -163,13 +186,13 @@ def test_validate_valid_event(minio_parser):
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
-def test_get_recording_id_success(minio_parser, valid_minio_event):
+def test_minio_get_recording_id_success(minio_parser, valid_minio_event):
"""Test successful extraction of recording ID."""
recording_id = minio_parser.get_recording_id(valid_minio_event)
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
-def test_validate_filepath_with_folder(minio_parser):
+def test_minio_validate_filepath_with_folder(minio_parser):
"""Test validation of filepath with folder structure."""
event = StorageEvent(
filepath="parent_folder%2Frecordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
@@ -181,41 +204,21 @@ def test_validate_filepath_with_folder(minio_parser):
assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
-def test_parse_with_video_type(minio_parser):
- """Test parsing event with video file type."""
- video_event = {
- "Records": [
- {
- "s3": {
- "bucket": {"name": "test-bucket"},
- "object": {
- "key": "46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",
- "contentType": "video/mp4",
- },
- }
- }
- ]
- }
- event = minio_parser.parse(video_event)
- assert event.filetype == "video/mp4"
- assert event.filepath.endswith(".mp4")
-
-
-def test_empty_allowed_filetypes():
+def test_minio_empty_allowed_filetypes():
"""Test MinioParser with empty allowed_filetypes."""
empty_types = set()
parser = MinioParser(bucket_name="test-bucket", allowed_filetypes=empty_types)
assert parser._allowed_filetypes == {"audio/ogg", "video/mp4"}
-def test_custom_allowed_filetypes():
+def test_minio_custom_allowed_filetypes():
"""Test MinioParser with empty allowed_filetypes."""
custom_types = {"audio/mp3", "video/mov"}
parser = MinioParser(bucket_name="test-bucket", allowed_filetypes=custom_types)
assert parser._allowed_filetypes == {"audio/mp3", "video/mov"}
-def test_validate_custom_filetypes():
+def test_minio_validate_custom_filetypes():
"""Test validation of filepath with folder structure."""
parser = MinioParser(bucket_name="test-bucket", allowed_filetypes={"audio/mp3"})
@@ -229,18 +232,143 @@ def test_validate_custom_filetypes():
parser.validate(event)
-def test_constructor_none_bucket():
+def test_minio_constructor_none_bucket():
"""Test MinioParser constructor with None bucket name."""
with pytest.raises(ValueError, match="Bucket name cannot be None or empty"):
MinioParser(bucket_name=None)
-def test_constructor_empty_bucket():
+def test_minio_constructor_empty_bucket():
"""Test MinioParser constructor with empty bucket name."""
with pytest.raises(ValueError, match="Bucket name cannot be None or empty"):
MinioParser(bucket_name="")
+# S3Parser
+
+
+def valid_s3_event():
+ """Mock a valid S3 event."""
+ return {
+ "Records": [
+ {
+ "s3": {
+ "bucket": {"name": "test-bucket"},
+ "object": {
+ "key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg",
+ },
+ }
+ }
+ ]
+ }
+
+
+def s3_parser():
+ """Mock an S3 parser."""
+ return S3Parser(bucket_name="test-bucket")
+
+
+def test_s3_parse_valid_event(s3_parser, valid_s3_event):
+ """Test parsing a valid S3 event."""
+ event = s3_parser.parse(valid_s3_event)
+ assert isinstance(event, StorageEvent)
+ assert event.filepath == "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.ogg"
+ assert event.filetype == "audio/ogg"
+ assert event.bucket_name == "test-bucket"
+ assert event.metadata is None
+
+
+def test_s3_parse_empty_data(s3_parser):
+ """Test parsing empty S3 event data raises error."""
+ with pytest.raises(ParsingEventDataError, match="Received empty data."):
+ s3_parser.parse({})
+
+
+def test_s3_parse_missing_keys(s3_parser):
+ """Test parsing S3 event with missing key."""
+ invalid_s3_event = {
+ "Records": [
+ {
+ "s3": {
+ "bucket": {"name": "test-bucket"},
+ # Missing 'object' key
+ }
+ }
+ ]
+ }
+
+ with pytest.raises(ParsingEventDataError, match="Malformed S3 event:"):
+ s3_parser.parse(invalid_s3_event)
+
+
+def test_s3_parse_none_key(s3_parser):
+ """Test parsing S3 event with None field."""
+ invalid_s3_event = {
+ "Records": [
+ {
+ "s3": {
+ "bucket": {"name": "test-bucket"},
+ "object": {
+ "key": None,
+ },
+ }
+ }
+ ]
+ }
+
+ with pytest.raises(ParsingEventDataError, match="Missing object key name"):
+ s3_parser.parse(invalid_s3_event)
+
+
+def test_s3_parse_with_video_type(s3_parser):
+ """Test parsing S3 event with mp4 file extension."""
+ video_event = {
+ "Records": [
+ {
+ "s3": {
+ "bucket": {"name": "test-bucket"},
+ "object": {
+ "key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.mp4",
+ },
+ }
+ }
+ ]
+ }
+ event = s3_parser.parse(video_event)
+ assert event.filetype == "video/mp4"
+ assert event.filepath.endswith(".mp4")
+
+
+def test_s3_parse_unrecognized_extension(s3_parser):
+ """Test parsing S3 event with unrecognized file extension."""
+ event_with_unknown_ext = {
+ "Records": [
+ {
+ "s3": {
+ "bucket": {"name": "test-bucket"},
+ "object": {
+ "key": "recordings%2F46d1a121-2426-484d-8fb3-09b5d886f7a8.zzunknown999",
+ },
+ }
+ }
+ ]
+ }
+
+ with pytest.raises(TypeError, match="filetype cannot be None"):
+ s3_parser.parse(event_with_unknown_ext)
+
+
+def test_s3_get_recording_id_success(s3_parser, valid_s3_event):
+ """Test successful extraction of recording ID from S3 event."""
+ recording_id = s3_parser.get_recording_id(valid_s3_event)
+ assert recording_id == "46d1a121-2426-484d-8fb3-09b5d886f7a8"
+
+
+# get_parser
+
+
@pytest.fixture
def clear_lru_cache():
"""Fixture to clear the LRU cache between tests."""