Open source video conferencing app powered by LiveKit. Built with Django and React.
♻️(summary) enhance file handling in the Celery worker
lebaudantoine4e5032a7a4f6
diff --git a/src/summary/summary/core/celery_worker.py b/src/summary/summary/core/celery_worker.pyindex d99f6a44..66bf5c87 100644--- a/src/summary/summary/core/celery_worker.py+++ b/src/summary/summary/core/celery_worker.py@@ -1,26 +1,22 @@"""Celery workers."""-# ruff: noqa: PLR0913, PLR0915+# ruff: noqa: PLR0913import json-import os-import tempfileimport time-from pathlib import Pathfrom typing import Optionalimport openaiimport sentry_sdkfrom celery import Celery, signalsfrom celery.utils.log import get_task_logger-from minio import Minio-from mutagen import Filefrom requests import Session, exceptionsfrom requests.adapters import HTTPAdapterfrom urllib3.util import Retryfrom summary.core.analytics import MetadataManager, get_analyticsfrom summary.core.config import get_settings+from summary.core.file_service import FileServicefrom summary.core.llm_service import LLMException, LLMObservability, LLMServicefrom summary.core.prompt import (FORMAT_NEXT_STEPS,@@ -59,17 +55,7 @@ if settings.sentry_dsn and settings.sentry_is_enabled:sentry_sdk.init(dsn=settings.sentry_dsn, enable_tracing=True)-class AudioValidationError(Exception):- """Custom exception for audio validation errors."""-- pass---def save_audio_stream(audio_stream, chunk_size=32 * 1024):- """Save an audio stream to a temporary OGG file."""- with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as tmp:- tmp.writelines(audio_stream.stream(chunk_size))- return Path(tmp.name)+file_service = FileService(logger=logger)def create_retry_session():@@ -151,40 +137,6 @@ def process_audio_transcribe_summarize_v2(task_id = self.request.id- logger.info("Download recording | Filename: %s", filename)-- minio_client = Minio(- settings.aws_s3_endpoint_url,- access_key=settings.aws_s3_access_key_id,- secret_key=settings.aws_s3_secret_access_key.get_secret_value(),- secure=settings.aws_s3_secure_access,- )-- logger.debug("Connection to the Minio bucket successful")-- audio_file_stream = minio_client.get_object(- settings.aws_storage_bucket_name, object_name=filename- )-- temp_file_path = save_audio_stream(audio_file_stream)-- logger.info("Recording successfully downloaded")- logger.debug("Recording filepath: %s", temp_file_path)-- audio_file = File(temp_file_path)- metadata_manager.track(task_id, {"audio_length": audio_file.info.length})-- if (- settings.recording_max_duration is not None- and audio_file.info.length > settings.recording_max_duration- ):- error_msg = "Recording too long: %.2fs > %.2fs limit" % (- audio_file.info.length,- settings.recording_max_duration,- )- logger.error(error_msg)- raise AudioValidationError(error_msg)-logger.info("Initiating WhisperX client")whisperx_client = openai.OpenAI(api_key=settings.whisperx_api_key.get_secret_value(),@@ -192,31 +144,31 @@ def process_audio_transcribe_summarize_v2(max_retries=settings.whisperx_max_retries,)- try:+ with (+ file_service.prepare_audio_file(filename) as (audio_file, metadata),+ ):+ metadata_manager.track(task_id, {"audio_length": metadata["duration"]})+logger.info(- "Querying transcription for %s seconds of audio in %s …",- audio_file.info.length,+ "Querying transcription in '%s' language …",language,)+transcription_start_time = time.time()- with open(temp_file_path, "rb") as audio_file:- transcription = whisperx_client.audio.transcriptions.create(- model=settings.whisperx_asr_model,- file=audio_file,- language=language or settings.whisperx_default_language,- )-- transcription_time = round(time.time() - transcription_start_time, 2)- metadata_manager.track(- task_id,- {"transcription_time": transcription_time},- )- logger.info("Transcription received in %s seconds.", transcription_time)- logger.debug("Transcription: \n %s", transcription)- finally:- if os.path.exists(temp_file_path):- os.remove(temp_file_path)- logger.debug("Temporary file removed: %s", temp_file_path)++ transcription = whisperx_client.audio.transcriptions.create(+ model=settings.whisperx_asr_model,+ file=audio_file,+ language=language or settings.whisperx_default_language,+ )++ transcription_time = round(time.time() - transcription_start_time, 2)+ metadata_manager.track(+ task_id,+ {"transcription_time": transcription_time},+ )+ logger.info("Transcription received in %.2f seconds.", transcription_time)+ logger.debug("Transcription: \n %s", transcription)metadata_manager.track_transcription_metadata(task_id, transcription)diff --git a/src/summary/summary/core/config.py b/src/summary/summary/core/config.pyindex ee7654a5..2624cced 100644--- a/src/summary/summary/core/config.py+++ b/src/summary/summary/core/config.py@@ -1,7 +1,7 @@"""Application configuration and settings."""from functools import lru_cache-from typing import Annotated, List, Optional+from typing import Annotated, List, Optional, Setfrom fastapi import Dependsfrom pydantic import SecretStr@@ -19,6 +19,7 @@ class Settings(BaseSettings):# Audio recordingsrecording_max_duration: Optional[int] = None+ recording_allowed_extensions: Set[str] = {".ogg"}# Celery settingscelery_broker_url: str = "redis://redis/0"diff --git a/src/summary/summary/core/file_service.py b/src/summary/summary/core/file_service.pynew file mode 100644index 00000000..69e44629--- /dev/null+++ b/src/summary/summary/core/file_service.py@@ -0,0 +1,133 @@+"""File service to encapsulate files' manipulations."""++import os+import tempfile+from contextlib import contextmanager+from pathlib import Path++import mutagen+from minio import Minio++from summary.core.config import get_settings++settings = get_settings()+++class FileService:+ """Service for downloading and preparing files from MinIO storage."""++ def __init__(self, logger):+ """Initialize FileService with MinIO client and configuration."""+ self._logger = logger++ self._minio_client = Minio(+ settings.aws_s3_endpoint_url,+ access_key=settings.aws_s3_access_key_id,+ secret_key=settings.aws_s3_secret_access_key.get_secret_value(),+ secure=settings.aws_s3_secure_access,+ )++ self._bucket_name = settings.aws_storage_bucket_name+ self._stream_chunk_size = 32 * 1024++ self._allowed_extensions = settings.recording_allowed_extensions+ self._max_duration = settings.recording_max_duration++ def _download_from_minio(self, remote_object_key) -> Path:+ """Download file from MinIO to local temporary file.++ The file is downloaded to a temporary location for local manipulation+ such as validation, conversion, or processing before being used.+ """+ self._logger.info("Download recording | object_key: %s", remote_object_key)++ if not remote_object_key:+ self._logger.warning("Invalid object_key '%s'", remote_object_key)+ raise ValueError("Invalid object_key")++ extension = Path(remote_object_key).suffix.lower()++ if extension not in self._allowed_extensions:+ self._logger.warning("Invalid file extension '%s'", extension)+ raise ValueError(f"Invalid file extension '{extension}'")++ response = None++ try:+ response = self._minio_client.get_object(+ self._bucket_name, remote_object_key+ )++ with tempfile.NamedTemporaryFile(+ suffix=extension, delete=False, prefix="minio_download_"+ ) as tmp:+ for chunk in response.stream(self._stream_chunk_size):+ tmp.write(chunk)++ tmp.flush()+ local_path = Path(tmp.name)++ self._logger.info("Recording successfully downloaded")+ self._logger.debug("Recording local file path: %s", local_path)++ return local_path++ finally:+ if response:+ response.close()++ def _validate_duration(self, local_path: Path) -> float:+ """Validate audio file duration against configured maximum."""+ file_metadata = mutagen.File(local_path).info+ duration = file_metadata.length++ self._logger.info(+ "Recording file duration: %.2f seconds",+ duration,+ )++ if self._max_duration is not None and duration > self._max_duration:+ error_msg = "Recording too long. Limit is %.2fs seconds" % (+ self._max_duration,+ )+ self._logger.error(error_msg)+ raise ValueError(error_msg)++ return duration++ @contextmanager+ def prepare_audio_file(self, remote_object_key: str):+ """Download and prepare audio file for processing.++ Downloads file from MinIO, validates duration, and yields an open+ file handle with metadata. Automatically cleans up temporary files+ when the context exits.+ """+ downloaded_path = None+ file_handle = None++ try:+ downloaded_path = self._download_from_minio(remote_object_key)+ duration = self._validate_duration(downloaded_path)++ extension = downloaded_path.suffix.lower()+ metadata = {"duration": duration, "extension": extension}++ file_handle = open(downloaded_path, "rb")+ yield file_handle, metadata++ finally:+ if file_handle:+ file_handle.close()++ for path in [downloaded_path]:+ if path is None or not os.path.exists(path):+ continue++ try:+ os.remove(path)+ self._logger.debug("Temporary file removed: %s", path)+ except OSError as e:+ self._logger.warning(+ "Failed to remove temporary file %s: %s", path, e+ )