Open source video conferencing app powered by LiveKit. Built with Django and React.
♻️(summary) clean up code and unify logging in preparation for testing
leo14526808ab5b
diff --git a/src/summary/summary/core/analytics.py b/src/summary/summary/core/analytics.pyindex 407f2d39..d88b8e98 100644--- a/src/summary/summary/core/analytics.py+++ b/src/summary/summary/core/analytics.py@@ -112,19 +112,16 @@ class MetadataManager:if self._is_disabled or self.has_task_id(task_id):return- initial_metadata = {- "start_time": time.time(),- "asr_model": settings.whisperx_asr_model,- "retries": 0,- }-_, filename, email, _, received_at, *_ = task_args+ start_time = time.time()initial_metadata = {- **initial_metadata,+ "start_time": start_time,+ "asr_model": settings.whisperx_asr_model,+ "retries": 0,"filename": filename,"email": email,- "queuing_time": round(initial_metadata["start_time"] - received_at, 2),+ "queuing_time": round(start_time - received_at, 2),}self._save_metadata(task_id, initial_metadata)diff --git a/src/summary/summary/core/celery_worker.py b/src/summary/summary/core/celery_worker.pyindex 0f1131a9..8ba10c30 100644--- a/src/summary/summary/core/celery_worker.py+++ b/src/summary/summary/core/celery_worker.py@@ -10,9 +10,7 @@ import openaiimport sentry_sdkfrom celery import Celery, signalsfrom celery.utils.log import get_task_logger-from requests import Session, exceptions-from requests.adapters import HTTPAdapter-from urllib3.util import Retry+from requests import exceptionsfrom summary.core.analytics import MetadataManager, get_analyticsfrom summary.core.config import get_settings@@ -30,6 +28,7 @@ from summary.core.prompt import (PROMPT_USER_PART,)from summary.core.transcript_formatter import TranscriptFormatter+from summary.core.webhook_service import submit_contentsettings = get_settings()analytics = get_analytics()@@ -56,20 +55,89 @@ if settings.sentry_dsn and settings.sentry_is_enabled:sentry_sdk.init(dsn=settings.sentry_dsn, enable_tracing=True)-file_service = FileService(logger=logger)+file_service = FileService()-def create_retry_session():- """Create an HTTP session configured with retry logic."""- session = Session()- retries = Retry(- total=settings.webhook_max_retries,- backoff_factor=settings.webhook_backoff_factor,- status_forcelist=settings.webhook_status_forcelist,- allowed_methods={"POST"},+def transcribe_audio(task_id, filename, language):+ """Transcribe an audio file using WhisperX.++ Downloads the audio from MinIO, sends it to WhisperX for transcription,+ and tracks metadata throughout the process.++ Returns the transcription object, or None if the file could not be retrieved.+ """+ logger.info("Initiating WhisperX client")+ whisperx_client = openai.OpenAI(+ api_key=settings.whisperx_api_key.get_secret_value(),+ base_url=settings.whisperx_base_url,+ max_retries=settings.whisperx_max_retries,+ )++ # Transcription+ try:+ with file_service.prepare_audio_file(filename) as (audio_file, metadata):+ metadata_manager.track(task_id, {"audio_length": metadata["duration"]})++ if language is None:+ language = settings.whisperx_default_language+ logger.info(+ "No language specified, using default from settings: %s",+ (language or "auto-detect"),+ )+ else:+ logger.info(+ "Querying transcription in '%s' language",+ language,+ )++ transcription_start_time = time.time()++ transcription = whisperx_client.audio.transcriptions.create(+ model=settings.whisperx_asr_model, file=audio_file, language=language+ )++ transcription_time = round(time.time() - transcription_start_time, 2)+ metadata_manager.track(+ task_id,+ {"transcription_time": transcription_time},+ )+ logger.info("Transcription received in %.2f seconds.", transcription_time)+ logger.debug("Transcription: \n %s", transcription)++ except FileServiceException:+ logger.exception("Unexpected error for filename: %s", filename)+ return None++ metadata_manager.track_transcription_metadata(task_id, transcription)+ return transcription+++def format_transcript(+ transcription,+ context_language,+ language,+ room,+ recording_date,+ recording_time,+ download_link,+):+ """Format a transcription into readable content with a title.++ Resolves the locale from context_language / language, then uses+ TranscriptFormatter to produce markdown content and a title.++ Returns a (content, title) tuple.+ """+ locale = get_locale(context_language, language)+ formatter = TranscriptFormatter(locale)++ return formatter.format(+ transcription,+ room=room,+ recording_date=recording_date,+ recording_time=recording_time,+ download_link=download_link,)- session.mount("https://", HTTPAdapter(max_retries=retries))- return sessiondef format_actions(llm_output: dict) -> str:@@ -90,20 +158,6 @@ def format_actions(llm_output: dict) -> str:return ""-def post_with_retries(url, data):- """Send POST request with automatic retries."""- session = create_retry_session()- session.headers.update(- {"Authorization": f"Bearer {settings.webhook_api_token.get_secret_value()}"}- )- try:- response = session.post(url, json=data)- response.raise_for_status()- return response- finally:- session.close()--@celery.task(bind=True,autoretry_for=[exceptions.HTTPError],@@ -126,10 +180,11 @@ def process_audio_transcribe_summarize_v2():"""Process an audio file by transcribing it and generating a summary.- This Celery task performs the following operations:- 1. Retrieves the audio file from MinIO storage- 2. Transcribes the audio using WhisperX model- 3. Sends the results via webhook+ This Celery task orchestrates:+ 1. Audio transcription via WhisperX+ 2. Transcript formatting+ 3. Webhook submission+ 4. Conditional summarization queuingArgs:self: Celery task instance (passed on with bind=True)@@ -153,91 +208,21 @@ def process_audio_transcribe_summarize_v2(task_id = self.request.id- logger.info("Initiating WhisperX client")- whisperx_client = openai.OpenAI(- api_key=settings.whisperx_api_key.get_secret_value(),- base_url=settings.whisperx_base_url,- max_retries=settings.whisperx_max_retries,- )-- # Transcription- try:- with (- file_service.prepare_audio_file(filename) as (audio_file, metadata),- ):- metadata_manager.track(task_id, {"audio_length": metadata["duration"]})-- if language is None:- language = settings.whisperx_default_language- logger.info(- "No language specified, using default from settings: %s",- (language or "auto-detect"),- )- else:- logger.info(- "Querying transcription in '%s' language",- language,- )-- transcription_start_time = time.time()-- transcription = whisperx_client.audio.transcriptions.create(- model=settings.whisperx_asr_model, file=audio_file, language=language- )-- transcription_time = round(time.time() - transcription_start_time, 2)- metadata_manager.track(- task_id,- {"transcription_time": transcription_time},- )- logger.info("Transcription received in %.2f seconds.", transcription_time)- logger.debug("Transcription: \n %s", transcription)-- except FileServiceException:- logger.exception("Unexpected error for filename: %s", filename)+ transcription = transcribe_audio(task_id, filename, language)+ if transcription is None:return- metadata_manager.track_transcription_metadata(task_id, transcription)-- # For locale of context, use in decreasing priority context_language,- # language (of meeting), default context language- locale = get_locale(context_language, language)- formatter = TranscriptFormatter(locale)-- content, title = formatter.format(+ content, title = format_transcript(transcription,- room=room,- recording_date=recording_date,- recording_time=recording_time,- download_link=download_link,- )-- data = {- "title": title,- "content": content,- "email": email,- "sub": sub,- }-- logger.debug("Submitting webhook to %s", settings.webhook_url)- logger.debug("Request payload: %s", json.dumps(data, indent=2))-- response = post_with_retries(settings.webhook_url, data)-- try:- response_data = response.json()- document_id = response_data.get("id", "N/A")- except (json.JSONDecodeError, AttributeError):- document_id = "Unable to parse response"- response_data = response.text-- logger.info(- "Webhook success | Document %s submitted (HTTP %s)",- document_id,- response.status_code,+ context_language,+ language,+ room,+ recording_date,+ recording_time,+ download_link,)- logger.debug("Full response: %s", response_data)+ submit_content(content, title, email, sub)metadata_manager.capture(task_id, settings.posthog_event_success)# LLM Summarization@@ -306,12 +291,11 @@ def summarize_transcription(# a singleton client. This is a performance trade-off we accept to ensure per-user# privacy controls in observability traces.llm_observability = LLMObservability(- logger=logger,user_has_tracing_consent=user_has_tracing_consent,session_id=self.request.id,user_id=owner_id,)- llm_service = LLMService(llm_observability=llm_observability, logger=logger)+ llm_service = LLMService(llm_observability=llm_observability)tldr = llm_service.call(PROMPT_SYSTEM_TLDR, transcript, name="tldr")@@ -354,20 +338,9 @@ def summarize_transcription(logger.info("Summary cleaned")summary = tldr + "\n\n" + cleaned_summary + "\n\n" + next_steps+ summary_title = settings.summary_title_template.format(title=title)- data = {- "title": settings.summary_title_template.format(title=title),- "content": summary,- "email": email,- "sub": sub,- }-- logger.debug("Submitting webhook to %s", settings.webhook_url)-- response = post_with_retries(settings.webhook_url, data)-- logger.info("Webhook submitted successfully. Status: %s", response.status_code)- logger.debug("Response body: %s", response.text)+ submit_content(summary, summary_title, email, sub)llm_observability.flush()logger.debug("LLM observability flushed")diff --git a/src/summary/summary/core/file_service.py b/src/summary/summary/core/file_service.pyindex e42e193a..80c63200 100644--- a/src/summary/summary/core/file_service.py+++ b/src/summary/summary/core/file_service.py@@ -1,5 +1,6 @@"""File service to encapsulate files' manipulations."""+import loggingimport osimport subprocessimport tempfile@@ -15,6 +16,9 @@ from summary.core.config import get_settingssettings = get_settings()+logger = logging.getLogger(__name__)++class FileServiceException(Exception):"""Base exception for file service operations."""@@ -24,10 +28,8 @@ class FileServiceException(Exception):class FileService:"""Service for downloading and preparing files from MinIO storage."""- def __init__(self, logger):+ def __init__(self):"""Initialize FileService with MinIO client and configuration."""- self._logger = logger-endpoint = (settings.aws_s3_endpoint_url.removeprefix("https://").removeprefix("http://")@@ -53,16 +55,16 @@ class FileService:The file is downloaded to a temporary location for local manipulationsuch as validation, conversion, or processing before being used."""- self._logger.info("Download recording | object_key: %s", remote_object_key)+ logger.info("Download recording | object_key: %s", remote_object_key)if not remote_object_key:- self._logger.warning("Invalid object_key '%s'", remote_object_key)+ logger.warning("Invalid object_key '%s'", remote_object_key)raise ValueError("Invalid object_key")extension = Path(remote_object_key).suffix.lower()if extension not in self._allowed_extensions:- self._logger.warning("Invalid file extension '%s'", extension)+ logger.warning("Invalid file extension '%s'", extension)raise ValueError(f"Invalid file extension '{extension}'")response = None@@ -81,8 +83,8 @@ class FileService:tmp.flush()local_path = Path(tmp.name)- self._logger.info("Recording successfully downloaded")- self._logger.debug("Recording local file path: %s", local_path)+ logger.info("Recording successfully downloaded")+ logger.debug("Recording local file path: %s", local_path)return local_path@@ -100,7 +102,7 @@ class FileService:file_metadata = mutagen.File(local_path).infoduration = file_metadata.length- self._logger.info(+ logger.info("Recording file duration: %.2f seconds",duration,)@@ -109,14 +111,14 @@ class FileService:error_msg = "Recording too long. Limit is %.2fs seconds" % (self._max_duration,)- self._logger.error(error_msg)+ logger.error(error_msg)raise ValueError(error_msg)return durationdef _extract_audio_from_video(self, video_path: Path) -> Path:"""Extract audio from video file (e.g., MP4) and save as audio file."""- self._logger.info("Extracting audio from video file: %s", video_path)+ logger.info("Extracting audio from video file: %s", video_path)with tempfile.NamedTemporaryFile(suffix=".m4a", delete=False, prefix="audio_extract_"@@ -140,16 +142,16 @@ class FileService:command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)- self._logger.info("Audio successfully extracted to: %s", output_path)+ logger.info("Audio successfully extracted to: %s", output_path)return output_pathexcept FileNotFoundError as e:- self._logger.error("ffmpeg not found. Please install ffmpeg.")+ logger.error("ffmpeg not found. Please install ffmpeg.")if output_path.exists():os.remove(output_path)raise RuntimeError("ffmpeg is not installed or not in PATH") from eexcept subprocess.CalledProcessError as e:- self._logger.error("Audio extraction failed: %s", e.stderr.decode())+ logger.error("Audio extraction failed: %s", e.stderr.decode())if output_path.exists():os.remove(output_path)raise RuntimeError("Failed to extract audio.") from e@@ -173,7 +175,7 @@ class FileService:extension = downloaded_path.suffix.lower()if extension in settings.recording_video_extensions:- self._logger.info("Video file detected, extracting audio...")+ logger.info("Video file detected, extracting audio...")extracted_audio_path = self._extract_audio_from_video(downloaded_path)processed_path = extracted_audio_pathelse:@@ -194,8 +196,6 @@ class FileService:try:os.remove(path)- self._logger.debug("Temporary file removed: %s", path)+ logger.debug("Temporary file removed: %s", path)except OSError as e:- self._logger.warning(- "Failed to remove temporary file %s: %s", path, e- )+ logger.warning("Failed to remove temporary file %s: %s", path, e)diff --git a/src/summary/summary/core/llm_service.py b/src/summary/summary/core/llm_service.pyindex d26d7ad8..e8b52b0c 100644--- a/src/summary/summary/core/llm_service.py+++ b/src/summary/summary/core/llm_service.py@@ -1,5 +1,6 @@"""LLM service to encapsulate LLM's calls."""+import loggingfrom typing import Any, Mapping, Optionalimport openai@@ -10,6 +11,9 @@ from summary.core.config import get_settingssettings = get_settings()+logger = logging.getLogger(__name__)++class LLMObservability:"""Manage observability and tracing for LLM calls using Langfuse.@@ -21,13 +25,11 @@ class LLMObservability:def __init__(self,- logger,session_id: str,user_id: str,user_has_tracing_consent: bool = False,):"""Initialize the LLMObservability client."""- self._logger = loggerself._observability_client: Optional[Langfuse] = Noneself.session_id = session_idself.user_id = user_id@@ -75,7 +77,7 @@ class LLMObservability:}if not self.is_enabled:- self._logger.debug("Using regular OpenAI client (observability disabled)")+ logger.debug("Using regular OpenAI client (observability disabled)")return openai.OpenAI(**base_args)# Langfuse's OpenAI wrapper is imported here to avoid triggering client@@ -83,7 +85,7 @@ class LLMObservability:# is missing. Conditional import ensures Langfuse only initializes when enabled.from langfuse.openai import openai as langfuse_openai # noqa: PLC0415- self._logger.debug("Using LangfuseOpenAI client (observability enabled)")+ logger.debug("Using LangfuseOpenAI client (observability enabled)")return langfuse_openai.OpenAI(**base_args)def flush(self):@@ -99,11 +101,10 @@ class LLMException(Exception):class LLMService:"""Service for performing calls to the LLM configured in the settings."""- def __init__(self, llm_observability, logger):+ def __init__(self, llm_observability):"""Init the LLMService once."""self._client = llm_observability.get_openai_client()self._observability = llm_observability- self._logger = loggerdef call(self,@@ -140,5 +141,5 @@ class LLMService:return response.choices[0].message.contentexcept Exception as e:- self._logger.exception("LLM call failed: %s", e)+ logger.exception("LLM call failed: %s", e)raise LLMException(f"LLM call failed: {e}") from ediff --git a/src/summary/summary/core/webhook_service.py b/src/summary/summary/core/webhook_service.pynew file mode 100644index 00000000..a5ff1cf6--- /dev/null+++ b/src/summary/summary/core/webhook_service.py@@ -0,0 +1,73 @@+"""Service for delivering content to external destinations."""++import json+import logging++from requests import Session+from requests.adapters import HTTPAdapter+from urllib3.util import Retry++from summary.core.config import get_settings++settings = get_settings()++logger = logging.getLogger(__name__)+++def _create_retry_session():+ """Create an HTTP session configured with retry logic."""+ session = Session()+ retries = Retry(+ total=settings.webhook_max_retries,+ backoff_factor=settings.webhook_backoff_factor,+ status_forcelist=settings.webhook_status_forcelist,+ allowed_methods={"POST"},+ )+ session.mount("https://", HTTPAdapter(max_retries=retries))+ return session+++def _post_with_retries(url, data):+ """Send POST request with automatic retries."""+ session = _create_retry_session()+ session.headers.update(+ {"Authorization": f"Bearer {settings.webhook_api_token.get_secret_value()}"}+ )+ try:+ response = session.post(url, json=data)+ response.raise_for_status()+ return response+ finally:+ session.close()+++def submit_content(content, title, email, sub):+ """Submit content to the configured webhook destination.++ Builds the payload, sends it with retries, and logs the outcome.+ """+ data = {+ "title": title,+ "content": content,+ "email": email,+ "sub": sub,+ }++ logger.debug("Submitting to %s", settings.webhook_url)+ logger.debug("Request payload: %s", json.dumps(data, indent=2))++ response = _post_with_retries(settings.webhook_url, data)++ try:+ response_data = response.json()+ document_id = response_data.get("id", "N/A")+ except (json.JSONDecodeError, AttributeError):+ document_id = "Unable to parse response"+ response_data = response.text++ logger.info(+ "Delivery success | Document %s submitted (HTTP %s)",+ document_id,+ response.status_code,+ )+ logger.debug("Full response: %s", response_data)