Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Refuse a file with a video extension that ffprobe cannot read, and a video without an audio track, with 400 audio_decode_failed also under VIDEO_RETENTION and VIDEO_PASSTHROUGH_ASR; only a probe that timed out still trusts the extension (#482)

Murtaza Nasirb42368e43d47
diff --git a/src/api/recordings.py b/src/api/recordings.py
index 85148905..a266c953 100644
--- a/src/api/recordings.py
+++ b/src/api/recordings.py
@@ -58,7 +58,10 @@ from src.file_exporter import export_recording, mark_export_as_deleted
from src.utils.mime import served_media_mime, sandbox_media_response
from src.utils.filenames import shorten_filename
from src.utils.search import LIKE_ESCAPE, contains_pattern, escape_like
-from src.utils.ffprobe import get_codec_info, get_creation_date, get_duration, try_repair_malformed_webm, FFProbeError
+from src.utils.ffprobe import (
+ get_codec_info, get_creation_date, get_duration, try_repair_malformed_webm, FFProbeError,
+ has_audio_stream, probe_rejected_input,
+)
from src.utils.audio_conversion import convert_if_needed
from src.services.storage import get_storage_service
from src.utils.file_hash import compute_file_sha256
@@ -2498,6 +2501,7 @@ def ingest_uploaded_recording(
file_size_mb = os.path.getsize(filepath) / (1024 * 1024)
probe_timeout = max(10, min(60, int(file_size_mb / 10))) # 10s min, scales ~1s per 10MB, 60s max
codec_info = None
+ probe_error = None
try:
codec_info = get_codec_info(filepath, timeout=probe_timeout)
current_app.logger.info(
@@ -2508,6 +2512,7 @@ def ingest_uploaded_recording(
except FFProbeError as e:
current_app.logger.warning(f"Failed to probe {original_filename} (timeout={probe_timeout}s): {e}. Will attempt conversion.")
codec_info = None
+ probe_error = e
# #340: a browser MediaRecorder can produce a .webm whose EBML header
# isn't at the front (chunk-order artifact, seen on crash-recovered
# recordings). If so, trim to the real header and re-probe before we
@@ -2516,6 +2521,7 @@ def ingest_uploaded_recording(
repaired_info = try_repair_malformed_webm(filepath, timeout=probe_timeout)
if repaired_info is not None:
codec_info = repaired_info
+ probe_error = None
current_app.logger.info(f"Recovered malformed WebM header for {original_filename}; probe succeeded after trim")
# Video retention/passthrough: skip conversion for videos, processing pipeline handles extraction
@@ -2544,17 +2550,30 @@ def ingest_uploaded_recording(
effective_limit_mb = regular_limit_mb
is_likely_video_by_ext = False
raise RequestEntityTooLarge()
- has_video = True
- current_app.logger.info(
- f"Probe failed but file extension '{file_ext}' indicates video — "
- f"treating as video for {'VIDEO_PASSTHROUGH_ASR' if VIDEO_PASSTHROUGH_ASR else 'VIDEO_RETENTION'}"
- )
+ if probe_error is not None and probe_rejected_input(probe_error):
+ # ffprobe read the file and could not make sense of it.
+ # Only a probe that did not finish is trusted to the
+ # extension: this file goes through conversion, which
+ # refuses it with audio_decode_failed, as without these
+ # settings (#482).
+ current_app.logger.info(
+ f"ffprobe could not read '{file_ext}' upload; converting it as audio"
+ )
+ else:
+ has_video = True
+ current_app.logger.info(
+ f"Probe failed but file extension '{file_ext}' indicates video — "
+ f"treating as video for {'VIDEO_PASSTHROUGH_ASR' if VIDEO_PASSTHROUGH_ASR else 'VIDEO_RETENTION'}"
+ )
# Per-upload `keep_audio_only` overrides VIDEO_RETENTION for this
# request. VIDEO_PASSTHROUGH_ASR is an admin escape hatch that
# sends video directly to the ASR endpoint — left untouched here.
+ # A video without an audio track has nothing to transcribe: it goes
+ # through conversion, which refuses it, as without these settings.
keep_video_for_this_upload = (
((VIDEO_RETENTION and not keep_audio_only_flag) or VIDEO_PASSTHROUGH_ASR)
and has_video
+ and (codec_info is None or has_audio_stream(codec_info))
)
# An audio file uploaded to be joined is decoded and re-encoded by the
# merge job anyway, so converting it here would only delay the upload
diff --git a/src/file_monitor.py b/src/file_monitor.py
index be11b279..5ab6d858 100644
--- a/src/file_monitor.py
+++ b/src/file_monitor.py
@@ -16,7 +16,7 @@ from datetime import datetime
from pathlib import Path
import mimetypes
from werkzeug.utils import secure_filename
-from src.utils.ffprobe import get_codec_info, get_creation_date, get_duration, FFProbeError
+from src.utils.ffprobe import get_codec_info, get_creation_date, get_duration, FFProbeError, has_audio_stream
from src.utils.ffmpeg_utils import FFmpegError, FFmpegNotFoundError
from src.utils.audio_conversion import convert_if_needed
from src.utils.filenames import shorten_filename
@@ -426,8 +426,10 @@ class FileMonitor:
# Check if this is a video file (for video retention logic)
has_video = codec_info.get('has_video', False) if codec_info else False
- # Video passthrough or retention: skip conversion for videos
- if (VIDEO_PASSTHROUGH_ASR or VIDEO_RETENTION) and has_video:
+ # Video passthrough or retention: skip conversion for videos.
+ # A video without an audio track goes through conversion,
+ # which refuses it, as without these settings.
+ if (VIDEO_PASSTHROUGH_ASR or VIDEO_RETENTION) and has_video and has_audio_stream(codec_info):
self.logger.info(f"Video {'passthrough' if VIDEO_PASSTHROUGH_ASR else 'retention'}: keeping original video, skipping conversion")
final_path = destination_path
else:
diff --git a/src/tasks/processing.py b/src/tasks/processing.py
index ed13ae49..0ca5112d 100644
--- a/src/tasks/processing.py
+++ b/src/tasks/processing.py
@@ -26,7 +26,7 @@ from src.models import Recording, Tag, Event, TranscriptChunk, SystemSetting, Gr
from src.services.embeddings import process_recording_chunks
from src.services.llm import is_using_openai_api, call_llm_completion, format_api_error_message, TEXT_MODEL_NAME, client, http_client_no_proxy, TokenBudgetExceeded
from src.utils import extract_json_object, safe_json_loads
-from src.utils.ffprobe import get_codec_info, is_video_file, is_lossless_audio, FFProbeError
+from src.utils.ffprobe import get_codec_info, is_video_file, is_lossless_audio, FFProbeError, has_audio_stream, probe_rejected_input
from src.utils.ffmpeg_utils import convert_to_mp3, extract_audio_from_video as ffmpeg_extract_audio, compress_audio, FFmpegError, FFmpegNotFoundError, FFmpegTimeoutError, AUDIO_DECODE_FAILED_MESSAGE
from src.utils.audio_conversion import convert_if_needed, ConversionResult
from src.utils.error_formatting import format_error_for_storage
@@ -2543,14 +2543,19 @@ def transcribe_incognito(filepath, original_filename, language=None, min_speaker
actual_content_type = mime_type
# Check if file is video and needs audio extraction
+ # A video without an audio track, and a file ffprobe read and
+ # rejected, go through conversion, which refuses them with
+ # audio_decode_failed (#482). Only a probe that did not finish falls
+ # back to the extension.
is_video = False
try:
- is_video = is_video_file(filepath, timeout=10)
+ _codec_info = get_codec_info(filepath, timeout=10)
+ is_video = bool(_codec_info.get('has_video')) and has_audio_stream(_codec_info)
except FFProbeError as e:
current_app.logger.warning(f"[Incognito] Failed to probe file: {e}")
- # Check by extension
- video_extensions = ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.wmv', '.flv', '.m4v']
- is_video = any(original_filename.lower().endswith(ext) for ext in video_extensions)
+ if not probe_rejected_input(e):
+ video_extensions = ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.wmv', '.flv', '.m4v']
+ is_video = any(original_filename.lower().endswith(ext) for ext in video_extensions)
video_passthrough_active = is_video and VIDEO_PASSTHROUGH_ASR
diff --git a/src/utils/ffprobe.py b/src/utils/ffprobe.py
index 8f6dbac5..28ad1a6c 100644
--- a/src/utils/ffprobe.py
+++ b/src/utils/ffprobe.py
@@ -28,6 +28,33 @@ class FFProbeError(Exception):
pass
+class FFProbeTimeoutError(FFProbeError):
+ """Raised when ffprobe runs past its timeout.
+
+ A timeout says nothing about the input (a large MP4 with its index at the
+ end can take long to probe), unlike a file ffprobe cannot read.
+ """
+ pass
+
+
+class FFProbeNotFoundError(FFProbeError):
+ """Raised when the ffprobe executable is not found."""
+ pass
+
+
+def has_audio_stream(codec_info: Dict[str, Any]) -> bool:
+ """True when the probed file has at least one audio stream."""
+ return bool(codec_info.get('has_audio') or codec_info.get('audio_codec'))
+
+
+def probe_rejected_input(error: FFProbeError) -> bool:
+ """True when ffprobe ran to completion and could not read the file.
+
+ A timeout or a missing ffprobe says nothing about the file itself.
+ """
+ return not isinstance(error, (FFProbeTimeoutError, FFProbeNotFoundError))
+
+
def probe(filename: str, cmd: str = 'ffprobe', timeout: Optional[int] = None) -> Dict[str, Any]:
"""
Run ffprobe on the specified file and return a JSON representation of the output.
@@ -68,9 +95,9 @@ def probe(filename: str, cmd: str = 'ffprobe', timeout: Optional[int] = None) ->
except subprocess.TimeoutExpired:
if p:
p.kill()
- raise FFProbeError(f'ffprobe timed out after {timeout} seconds')
+ raise FFProbeTimeoutError(f'ffprobe timed out after {timeout} seconds')
except FileNotFoundError:
- raise FFProbeError('ffprobe command not found. Please ensure ffmpeg is installed.')
+ raise FFProbeNotFoundError('ffprobe command not found. Please ensure ffmpeg is installed.')
except json.JSONDecodeError as e:
raise FFProbeError(f'Failed to parse ffprobe output: {e}')
diff --git a/tests/test_incognito_settings.py b/tests/test_incognito_settings.py
index 58857346..10616896 100644
--- a/tests/test_incognito_settings.py
+++ b/tests/test_incognito_settings.py
@@ -88,7 +88,7 @@ def test_incognito_decode_failure_reports_a_fixed_message(user, tmp_path):
registry = MagicMock()
registry.get_active_connector.return_value = connector
with patch("src.services.transcription.get_registry", return_value=registry), \
- patch.object(proc, "is_video_file", return_value=True), \
+ patch.object(proc, "get_codec_info", return_value={"has_video": True, "has_audio": True}), \
patch.object(proc, "VIDEO_PASSTHROUGH_ASR", False), \
patch.object(proc, "extract_audio_from_video",
side_effect=FFmpegError("ffmpeg version n8.1.3 ... Error opening input file /tmp/tmpx.mp4")):
diff --git a/tests/test_upload_junk_video.py b/tests/test_upload_junk_video.py
new file mode 100644
index 00000000..ce0290dc
--- /dev/null
+++ b/tests/test_upload_junk_video.py
@@ -0,0 +1,249 @@
+"""Uploads with a video extension under VIDEO_RETENTION / VIDEO_PASSTHROUGH_ASR (#482).
+
+A file named .mp4 that ffprobe cannot read used to be kept as a video and
+fail later in processing. It is now converted like any other upload and
+refused with 400 audio_decode_failed; so is a video without an audio track,
+as without these settings. Only a probe that did not finish still trusts the
+extension. The real ffprobe and ffmpeg binaries are used where they are
+installed.
+"""
+import io
+import os
+import shutil
+import subprocess
+import sys
+import uuid
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.app import app, db
+from src.models import Recording, User
+from src.utils.ffmpeg_utils import AUDIO_DECODE_FAILED_MESSAGE
+from src.utils.ffprobe import (
+ FFProbeError, FFProbeNotFoundError, FFProbeTimeoutError, has_audio_stream,
+ probe, probe_rejected_input,
+)
+import src.tasks.processing as proc
+
+app.config["WTF_CSRF_ENABLED"] = False
+
+needs_ffmpeg = pytest.mark.skipif(
+ not (shutil.which("ffmpeg") and shutil.which("ffprobe")),
+ reason="ffmpeg and ffprobe are not installed",
+)
+
+JUNK = b"This is not a video, only text with a video extension.\n" * 64
+
+
+def _ffmpeg_video(path, with_audio=True):
+ """Write a one-second H.264 MP4, with or without an AAC track."""
+ cmd = ["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc=size=64x64:rate=10:duration=1"]
+ if with_audio:
+ cmd += ["-f", "lavfi", "-i", "sine=frequency=440:duration=1", "-c:a", "aac", "-shortest"]
+ cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", str(path)]
+ subprocess.run(cmd, check=True, capture_output=True)
+ return path.read_bytes()
+
+
+class _Storage:
+ def __init__(self, staging):
+ self.staging = staging
+ self.uploaded = []
+
+ def get_staging_dir(self):
+ os.makedirs(self.staging, exist_ok=True)
+ return self.staging
+
+ def build_recording_key(self, original_filename, recording_id=None, *, now=None):
+ return f"recordings/test/{recording_id}/{original_filename}"
+
+ def upload_local_file(self, local_path, key, *, content_type=None, delete_source=False):
+ self.uploaded.append((local_path, key))
+ return MagicMock(locator=f"local://{key}", key=key)
+
+
+def user():
+ with app.app_context():
+ s = uuid.uuid4().hex[:8]
+ u = User(username=f"junk_{s}", email=f"junk_{s}@local.test", password="x")
+ db.session.add(u)
+ db.session.commit()
+ uid = u.id
+ yield uid
+ with app.app_context():
+ Recording.query.filter_by(user_id=uid).delete()
+ db.session.delete(db.session.get(User, uid))
+ db.session.commit()
+
+
+def _upload(user_id, payload, filename, *, retention=False, passthrough=False, probe_error=None):
+ staging = os.path.join(app.config["UPLOAD_FOLDER"], f"stg_{uuid.uuid4().hex[:6]}")
+ storage = _Storage(staging)
+ client = app.test_client()
+ with client.session_transaction() as sess:
+ sess["_user_id"] = str(user_id)
+ sess["_fresh"] = True
+ extra = []
+ if probe_error is not None:
+ extra.append(patch("src.api.recordings.get_codec_info", side_effect=probe_error))
+ with patch("src.api.recordings.get_storage_service", return_value=storage), \
+ patch("src.api.recordings.VIDEO_RETENTION", retention), \
+ patch("src.api.recordings.VIDEO_PASSTHROUGH_ASR", passthrough), \
+ patch("src.api.recordings.chunking_service", None), \
+ patch("src.services.job_queue.job_queue.enqueue", return_value=1) as enqueue:
+ for p in extra:
+ p.start()
+ try:
+ resp = client.post("/upload", data={"file": (io.BytesIO(payload), filename)},
+ content_type="multipart/form-data")
+ finally:
+ for p in extra:
+ p.stop()
+ leftovers = os.listdir(staging) if os.path.isdir(staging) else []
+ shutil.rmtree(staging, ignore_errors=True)
+ return resp, storage, enqueue, leftovers
+
+
+def _assert_refused(resp, enqueue, leftovers, user_id):
+ assert resp.status_code == 400, resp.get_data(as_text=True)
+ body = resp.get_json()
+ assert body["code"] == "audio_decode_failed"
+ assert body["error"] == AUDIO_DECODE_FAILED_MESSAGE
+ text = resp.get_data(as_text=True).lower()
+ assert "ffmpeg" not in text and "ffprobe" not in text and "_staging" not in text
+ assert leftovers == []
+ assert enqueue.call_count == 0
+ with app.app_context():
+ assert Recording.query.filter_by(user_id=user_id).count() == 0
+
+
+# --- helpers ---------------------------------------------------------------
+
+def test_probe_rejected_input_tells_an_unreadable_file_from_a_probe_that_did_not_finish():
+ assert probe_rejected_input(FFProbeError("ffprobe failed: Invalid data found when processing input"))
+ assert not probe_rejected_input(FFProbeTimeoutError("ffprobe timed out after 10 seconds"))
+ assert not probe_rejected_input(FFProbeNotFoundError("ffprobe command not found."))
+ assert issubclass(FFProbeTimeoutError, FFProbeError) and issubclass(FFProbeNotFoundError, FFProbeError)
+
+
+def test_probe_raises_the_timeout_and_not_found_kinds():
+ with patch("src.utils.ffprobe.subprocess.Popen") as popen:
+ popen.return_value.communicate.side_effect = subprocess.TimeoutExpired("ffprobe", 1)
+ with pytest.raises(FFProbeTimeoutError):
+ probe("x.mp4", timeout=1)
+ with patch("src.utils.ffprobe.subprocess.Popen", side_effect=FileNotFoundError):
+ with pytest.raises(FFProbeNotFoundError):
+ probe("x.mp4", timeout=1)
+
+
+def test_has_audio_stream():
+ assert has_audio_stream({"has_audio": True})
+ assert has_audio_stream({"audio_codec": "aac"})
+ assert not has_audio_stream({"has_audio": False, "audio_codec": None, "has_video": True})
+
+
+# --- /upload -----------------------------------------------------------------
+
+@needs_ffmpeg
[email protected]("retention,passthrough", [(True, False), (False, True), (False, False)])
+def test_junk_with_a_video_extension_is_refused(user, retention, passthrough):
+ resp, storage, enqueue, leftovers = _upload(user, JUNK, "meeting.mp4",
+ retention=retention, passthrough=passthrough)
+ _assert_refused(resp, enqueue, leftovers, user)
+ assert storage.uploaded == []
+
+
+@needs_ffmpeg
[email protected]("retention,passthrough", [(True, False), (False, True), (False, False)])
+def test_video_without_an_audio_track_is_refused_as_without_retention(user, tmp_path, retention, passthrough):
+ payload = _ffmpeg_video(tmp_path / "silent.mp4", with_audio=False)
+ resp, storage, enqueue, leftovers = _upload(user, payload, "silent.mp4",
+ retention=retention, passthrough=passthrough)
+ _assert_refused(resp, enqueue, leftovers, user)
+
+
+@needs_ffmpeg
[email protected]("retention,passthrough", [(True, False), (False, True)])
+def test_real_video_with_audio_is_kept_as_video(user, tmp_path, retention, passthrough):
+ payload = _ffmpeg_video(tmp_path / "talk.mp4", with_audio=True)
+ resp, storage, enqueue, leftovers = _upload(user, payload, "talk.mp4",
+ retention=retention, passthrough=passthrough)
+ assert resp.status_code in (200, 202), resp.get_data(as_text=True)
+ assert enqueue.call_count == 1
+ assert len(storage.uploaded) == 1
+ assert storage.uploaded[0][0].endswith(".mp4") # the video itself, not extracted audio
+
+
+@needs_ffmpeg
+def test_real_video_with_audio_has_its_audio_extracted_without_retention(user, tmp_path):
+ payload = _ffmpeg_video(tmp_path / "talk.mp4", with_audio=True)
+ resp, storage, enqueue, leftovers = _upload(user, payload, "talk.mp4")
+ assert resp.status_code in (200, 202), resp.get_data(as_text=True)
+ assert enqueue.call_count == 1
+ assert not storage.uploaded[0][0].endswith(".mp4") # the extracted audio
+
+
+def test_a_probe_that_timed_out_still_trusts_the_video_extension(user):
+ """A large MP4 can be slow to probe: it is kept as a video, as before."""
+ with patch("src.api.recordings.convert_if_needed") as convert, \
+ patch("src.api.recordings.get_duration", return_value=12.0), \
+ patch("src.api.recordings.get_creation_date", return_value=None):
+ resp, storage, enqueue, leftovers = _upload(
+ user, JUNK, "slow.mp4", retention=True,
+ probe_error=FFProbeTimeoutError("ffprobe timed out after 10 seconds"))
+ assert resp.status_code in (200, 202), resp.get_data(as_text=True)
+ assert convert.call_count == 0
+ assert enqueue.call_count == 1
+
+
+# --- incognito ---------------------------------------------------------------
+
+def _incognito(user_id, payload, filename):
+ connector = MagicMock(PROVIDER_NAME="test", supports_diarization=False, default_diarize=False)
+ connector.specifications = MagicMock(max_duration_seconds=1800, handles_chunking_internally=False,
+ recommended_chunk_seconds=600, unsupported_codecs=[],
+ preserve_original_audio=False)
+ response = MagicMock(text="hello world transcript", segments=None, speakers=None, speaker_embeddings=None)
+ response.has_diarization.return_value = False
+ connector.transcribe.return_value = response
+ registry = MagicMock()
+ registry.get_active_connector.return_value = connector
+ client = app.test_client()
+ with client.session_transaction() as sess:
+ sess["_user_id"] = str(user_id)
+ with patch("src.api.recordings.ENABLE_INCOGNITO_MODE", True), \
+ patch("src.services.transcription.get_registry", return_value=registry), \
+ patch.object(proc, "VIDEO_PASSTHROUGH_ASR", True), \
+ patch.object(proc, "chunking_service", None), patch.object(proc, "client", None):
+ resp = client.post("/api/recordings/incognito",
+ data={"file": (io.BytesIO(payload), filename)},
+ content_type="multipart/form-data")
+ return resp, connector
+
+
+@needs_ffmpeg
+def test_incognito_passthrough_refuses_junk_with_a_video_extension(user):
+ resp, connector = _incognito(user, JUNK, "meeting.mp4")
+ assert resp.status_code == 400, resp.get_data(as_text=True)
+ assert resp.get_json()["code"] == "audio_decode_failed"
+ assert "ffmpeg" not in resp.get_data(as_text=True).lower()
+ assert connector.transcribe.call_count == 0
+
+
+@needs_ffmpeg
+def test_incognito_passthrough_refuses_a_video_without_audio(user, tmp_path):
+ resp, connector = _incognito(user, _ffmpeg_video(tmp_path / "s.mp4", with_audio=False), "s.mp4")
+ assert resp.status_code == 400, resp.get_data(as_text=True)
+ assert resp.get_json()["code"] == "audio_decode_failed"
+ assert connector.transcribe.call_count == 0
+
+
+@needs_ffmpeg
+def test_incognito_passthrough_sends_a_real_video(user, tmp_path):
+ resp, connector = _incognito(user, _ffmpeg_video(tmp_path / "t.mp4"), "t.mp4")
+ assert resp.status_code == 200, resp.get_data(as_text=True)
+ assert connector.transcribe.call_count == 1