Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

πŸ›(summary) explicit transcription response format

Florent Chehab9b7c449ca550
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 787a817d..6b1249a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,10 @@ and this project adheres to
- ✨(backend) add command to clean pending and deleted files
- 🧱(helm) run clean files command as cronjob
+### Changed
+
+- ✨(summary) generalized stt api call #1420
+
## [1.21.0] - 2026-06-15
### Added
diff --git a/docs/features/transcription.md b/docs/features/transcription.md
index e5533795..c6ab9323 100644
--- a/docs/features/transcription.md
+++ b/docs/features/transcription.md
@@ -80,7 +80,6 @@ sequenceDiagram
| whisperx_api_key | Secret | β€” | API key for accessing WhisperX. |
| whisperx_base_url | String | `"https://api.whisperx.com/v1"` | Base URL for the WhisperX API. |
| whisperx_asr_model | String | `"whisper-1"` | ASR model used for transcription. |
-| whisperx_max_retries | Integer | `0` | Maximum number of retries for WhisperX API requests. |
| webhook_max_retries | Integer | `2` | Maximum retries for webhook requests. |
| webhook_status_forcelist | List[Int] | `[502, 503, 504]` | HTTP status codes triggering webhook retry. |
| webhook_backoff_factor | Float | `0.1` | Exponential backoff factor for webhook retries. |
diff --git a/src/summary/summary/core/celery_worker.py b/src/summary/summary/core/celery_worker.py
index 944b1d46..ada448b7 100644
--- a/src/summary/summary/core/celery_worker.py
+++ b/src/summary/summary/core/celery_worker.py
@@ -5,11 +5,14 @@
import json
import time
from datetime import datetime
+from typing import Any
+from urllib.parse import urljoin
-import openai
+import requests
import sentry_sdk
from celery import Celery, signals
from celery.utils.log import get_task_logger
+from openai.types.audio import Transcription
from requests import exceptions
from summary.core.analytics import MetadataManager, get_analytics
@@ -101,11 +104,6 @@ def transcribe_audio(
)
logger.info("Initiating WhisperX client")
- whisperx_client = openai.OpenAI(
- api_key=settings.whisperx_api_key.get_secret_value(),
- base_url=settings.whisperx_base_url,
- max_retries=settings.whisperx_max_retries,
- )
# Transcription
try:
@@ -130,8 +128,54 @@ def transcribe_audio(
# Call remote service for transcription
transcription_start_time = time.time()
- transcription = whisperx_client.audio.transcriptions.create(
- model=settings.whisperx_asr_model, file=audio_file, language=language
+
+ api_key = settings.whisperx_api_key.get_secret_value()
+ base_url = settings.whisperx_base_url
+
+ # We use a manual call to the transcripion endpoint, and we do not
+ # directly use the OpenAI lib for this.
+ # This is because, depending on the requested response format,
+ # the OpenAI lib will cast the response to a different dataclass,
+ # which can result in stripping out keys & data that we are interested in.
+ # This is in particular true for word_segments and words.
+ # WhisperX response is slightly different from OpenAI STT endpoints
+ # response.
+ # At the same time "diarized_json" should be the value
+ # provided to STT endpoints in our context.
+ url = urljoin(base_url.rstrip("/") + "/", "audio/transcriptions")
+ res = requests.post(
+ url,
+ data={
+ "model": settings.whisperx_asr_model,
+ "language": language,
+ "timestamp_granularities": ["word", "segment"],
+ "response_format": "diarized_json",
+ },
+ files={"file": audio_file},
+ headers={"Authorization": f"Bearer {api_key}"},
+ # Mimic OpenAI's timeout settings
+ timeout=(60, 10 * 60),
+ )
+ try:
+ res.raise_for_status()
+ except requests.exceptions.HTTPError as e:
+ raise RuntimeError("WhisperX transcription failed, %s", res.text) from e
+
+ transcription_json: dict[str, Any] = res.json()
+ # We remove the "usage" key from the transcription_json dictionary
+ # as it may cause issues with parsing inside the Transcription model
+ # Some API don't share the exact same structure for the "usage" key
+ transcription_json.pop("usage", None)
+
+ # We force the use of the Transcription model here
+ # to avoid changing too much code for now.
+ # Note that it should be WhisperXResponse instead.
+ transcription = Transcription.model_validate(
+ # We add a dummy "text" to make the model validate,
+ # Some API responses lack the "text" key.
+ {"text": "", **transcription_json},
+ extra="allow",
+ strict=False,
)
# Logging
diff --git a/src/summary/summary/core/config.py b/src/summary/summary/core/config.py
index 28a4a44e..b536713e 100644
--- a/src/summary/summary/core/config.py
+++ b/src/summary/summary/core/config.py
@@ -92,7 +92,6 @@ class Settings(BaseSettings):
whisperx_api_key: SecretStr
whisperx_base_url: str = "https://api.openai.com/v1"
whisperx_asr_model: str = "whisper-1"
- whisperx_max_retries: int = 0
# ISO 639-1 language code (e.g., "en", "fr", "es")
whisperx_default_language: Optional[str] = None
whisperx_allowed_languages: Set[str] = {"en", "fr", "de", "nl"}