Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Refuse a file with a video extension that ffprobe cannot read, and a video without an audio track, with 400 audio_decode_failed also under VIDEO_RETENTION and VIDEO_PASSTHROUGH_ASR; only a probe that timed out still trusts the extension (#482)
Murtaza Nasirb42368e43d47
diff --git a/src/api/recordings.py b/src/api/recordings.pyindex 85148905..a266c953 100644--- a/src/api/recordings.py+++ b/src/api/recordings.py@@ -58,7 +58,10 @@ from src.file_exporter import export_recording, mark_export_as_deletedfrom src.utils.mime import served_media_mime, sandbox_media_responsefrom src.utils.filenames import shorten_filenamefrom src.utils.search import LIKE_ESCAPE, contains_pattern, escape_like-from src.utils.ffprobe import get_codec_info, get_creation_date, get_duration, try_repair_malformed_webm, FFProbeError+from src.utils.ffprobe import (+ get_codec_info, get_creation_date, get_duration, try_repair_malformed_webm, FFProbeError,+ has_audio_stream, probe_rejected_input,+)from src.utils.audio_conversion import convert_if_neededfrom src.services.storage import get_storage_servicefrom src.utils.file_hash import compute_file_sha256@@ -2498,6 +2501,7 @@ def ingest_uploaded_recording(file_size_mb = os.path.getsize(filepath) / (1024 * 1024)probe_timeout = max(10, min(60, int(file_size_mb / 10))) # 10s min, scales ~1s per 10MB, 60s maxcodec_info = None+ probe_error = Nonetry:codec_info = get_codec_info(filepath, timeout=probe_timeout)current_app.logger.info(@@ -2508,6 +2512,7 @@ def ingest_uploaded_recording(except FFProbeError as e:current_app.logger.warning(f"Failed to probe {original_filename} (timeout={probe_timeout}s): {e}. Will attempt conversion.")codec_info = None+ probe_error = e# #340: a browser MediaRecorder can produce a .webm whose EBML header# isn't at the front (chunk-order artifact, seen on crash-recovered# recordings). If so, trim to the real header and re-probe before we@@ -2516,6 +2521,7 @@ def ingest_uploaded_recording(repaired_info = try_repair_malformed_webm(filepath, timeout=probe_timeout)if repaired_info is not None:codec_info = repaired_info+ probe_error = Nonecurrent_app.logger.info(f"Recovered malformed WebM header for {original_filename}; probe succeeded after trim")# Video retention/passthrough: skip conversion for videos, processing pipeline handles extraction@@ -2544,17 +2550,30 @@ def ingest_uploaded_recording(effective_limit_mb = regular_limit_mbis_likely_video_by_ext = Falseraise RequestEntityTooLarge()- has_video = True- current_app.logger.info(- f"Probe failed but file extension '{file_ext}' indicates video — "- f"treating as video for {'VIDEO_PASSTHROUGH_ASR' if VIDEO_PASSTHROUGH_ASR else 'VIDEO_RETENTION'}"- )+ if probe_error is not None and probe_rejected_input(probe_error):+ # ffprobe read the file and could not make sense of it.+ # Only a probe that did not finish is trusted to the+ # extension: this file goes through conversion, which+ # refuses it with audio_decode_failed, as without these+ # settings (#482).+ current_app.logger.info(+ f"ffprobe could not read '{file_ext}' upload; converting it as audio"+ )+ else:+ has_video = True+ current_app.logger.info(+ f"Probe failed but file extension '{file_ext}' indicates video — "+ f"treating as video for {'VIDEO_PASSTHROUGH_ASR' if VIDEO_PASSTHROUGH_ASR else 'VIDEO_RETENTION'}"+ )# Per-upload `keep_audio_only` overrides VIDEO_RETENTION for this# request. VIDEO_PASSTHROUGH_ASR is an admin escape hatch that# sends video directly to the ASR endpoint — left untouched here.+ # A video without an audio track has nothing to transcribe: it goes+ # through conversion, which refuses it, as without these settings.keep_video_for_this_upload = (((VIDEO_RETENTION and not keep_audio_only_flag) or VIDEO_PASSTHROUGH_ASR)and has_video+ and (codec_info is None or has_audio_stream(codec_info)))# An audio file uploaded to be joined is decoded and re-encoded by the# merge job anyway, so converting it here would only delay the uploaddiff --git a/src/file_monitor.py b/src/file_monitor.pyindex be11b279..5ab6d858 100644--- a/src/file_monitor.py+++ b/src/file_monitor.py@@ -16,7 +16,7 @@ from datetime import datetimefrom pathlib import Pathimport mimetypesfrom werkzeug.utils import secure_filename-from src.utils.ffprobe import get_codec_info, get_creation_date, get_duration, FFProbeError+from src.utils.ffprobe import get_codec_info, get_creation_date, get_duration, FFProbeError, has_audio_streamfrom src.utils.ffmpeg_utils import FFmpegError, FFmpegNotFoundErrorfrom src.utils.audio_conversion import convert_if_neededfrom src.utils.filenames import shorten_filename@@ -426,8 +426,10 @@ class FileMonitor:# Check if this is a video file (for video retention logic)has_video = codec_info.get('has_video', False) if codec_info else False- # Video passthrough or retention: skip conversion for videos- if (VIDEO_PASSTHROUGH_ASR or VIDEO_RETENTION) and has_video:+ # Video passthrough or retention: skip conversion for videos.+ # A video without an audio track goes through conversion,+ # which refuses it, as without these settings.+ if (VIDEO_PASSTHROUGH_ASR or VIDEO_RETENTION) and has_video and has_audio_stream(codec_info):self.logger.info(f"Video {'passthrough' if VIDEO_PASSTHROUGH_ASR else 'retention'}: keeping original video, skipping conversion")final_path = destination_pathelse:diff --git a/src/tasks/processing.py b/src/tasks/processing.pyindex ed13ae49..0ca5112d 100644--- a/src/tasks/processing.py+++ b/src/tasks/processing.py@@ -26,7 +26,7 @@ from src.models import Recording, Tag, Event, TranscriptChunk, SystemSetting, Grfrom src.services.embeddings import process_recording_chunksfrom src.services.llm import is_using_openai_api, call_llm_completion, format_api_error_message, TEXT_MODEL_NAME, client, http_client_no_proxy, TokenBudgetExceededfrom src.utils import extract_json_object, safe_json_loads-from src.utils.ffprobe import get_codec_info, is_video_file, is_lossless_audio, FFProbeError+from src.utils.ffprobe import get_codec_info, is_video_file, is_lossless_audio, FFProbeError, has_audio_stream, probe_rejected_inputfrom src.utils.ffmpeg_utils import convert_to_mp3, extract_audio_from_video as ffmpeg_extract_audio, compress_audio, FFmpegError, FFmpegNotFoundError, FFmpegTimeoutError, AUDIO_DECODE_FAILED_MESSAGEfrom src.utils.audio_conversion import convert_if_needed, ConversionResultfrom src.utils.error_formatting import format_error_for_storage@@ -2543,14 +2543,19 @@ def transcribe_incognito(filepath, original_filename, language=None, min_speakeractual_content_type = mime_type# Check if file is video and needs audio extraction+ # A video without an audio track, and a file ffprobe read and+ # rejected, go through conversion, which refuses them with+ # audio_decode_failed (#482). Only a probe that did not finish falls+ # back to the extension.is_video = Falsetry:- is_video = is_video_file(filepath, timeout=10)+ _codec_info = get_codec_info(filepath, timeout=10)+ is_video = bool(_codec_info.get('has_video')) and has_audio_stream(_codec_info)except FFProbeError as e:current_app.logger.warning(f"[Incognito] Failed to probe file: {e}")- # Check by extension- video_extensions = ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.wmv', '.flv', '.m4v']- is_video = any(original_filename.lower().endswith(ext) for ext in video_extensions)+ if not probe_rejected_input(e):+ video_extensions = ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.wmv', '.flv', '.m4v']+ is_video = any(original_filename.lower().endswith(ext) for ext in video_extensions)video_passthrough_active = is_video and VIDEO_PASSTHROUGH_ASRdiff --git a/src/utils/ffprobe.py b/src/utils/ffprobe.pyindex 8f6dbac5..28ad1a6c 100644--- a/src/utils/ffprobe.py+++ b/src/utils/ffprobe.py@@ -28,6 +28,33 @@ class FFProbeError(Exception):pass+class FFProbeTimeoutError(FFProbeError):+ """Raised when ffprobe runs past its timeout.++ A timeout says nothing about the input (a large MP4 with its index at the+ end can take long to probe), unlike a file ffprobe cannot read.+ """+ pass+++class FFProbeNotFoundError(FFProbeError):+ """Raised when the ffprobe executable is not found."""+ pass+++def has_audio_stream(codec_info: Dict[str, Any]) -> bool:+ """True when the probed file has at least one audio stream."""+ return bool(codec_info.get('has_audio') or codec_info.get('audio_codec'))+++def probe_rejected_input(error: FFProbeError) -> bool:+ """True when ffprobe ran to completion and could not read the file.++ A timeout or a missing ffprobe says nothing about the file itself.+ """+ return not isinstance(error, (FFProbeTimeoutError, FFProbeNotFoundError))++def probe(filename: str, cmd: str = 'ffprobe', timeout: Optional[int] = None) -> Dict[str, Any]:"""Run ffprobe on the specified file and return a JSON representation of the output.@@ -68,9 +95,9 @@ def probe(filename: str, cmd: str = 'ffprobe', timeout: Optional[int] = None) ->except subprocess.TimeoutExpired:if p:p.kill()- raise FFProbeError(f'ffprobe timed out after {timeout} seconds')+ raise FFProbeTimeoutError(f'ffprobe timed out after {timeout} seconds')except FileNotFoundError:- raise FFProbeError('ffprobe command not found. Please ensure ffmpeg is installed.')+ raise FFProbeNotFoundError('ffprobe command not found. Please ensure ffmpeg is installed.')except json.JSONDecodeError as e:raise FFProbeError(f'Failed to parse ffprobe output: {e}')diff --git a/tests/test_incognito_settings.py b/tests/test_incognito_settings.pyindex 58857346..10616896 100644--- a/tests/test_incognito_settings.py+++ b/tests/test_incognito_settings.py@@ -88,7 +88,7 @@ def test_incognito_decode_failure_reports_a_fixed_message(user, tmp_path):registry = MagicMock()registry.get_active_connector.return_value = connectorwith patch("src.services.transcription.get_registry", return_value=registry), \- patch.object(proc, "is_video_file", return_value=True), \+ patch.object(proc, "get_codec_info", return_value={"has_video": True, "has_audio": True}), \patch.object(proc, "VIDEO_PASSTHROUGH_ASR", False), \patch.object(proc, "extract_audio_from_video",side_effect=FFmpegError("ffmpeg version n8.1.3 ... Error opening input file /tmp/tmpx.mp4")):diff --git a/tests/test_upload_junk_video.py b/tests/test_upload_junk_video.pynew file mode 100644index 00000000..ce0290dc--- /dev/null+++ b/tests/test_upload_junk_video.py@@ -0,0 +1,249 @@+"""Uploads with a video extension under VIDEO_RETENTION / VIDEO_PASSTHROUGH_ASR (#482).++A file named .mp4 that ffprobe cannot read used to be kept as a video and+fail later in processing. It is now converted like any other upload and+refused with 400 audio_decode_failed; so is a video without an audio track,+as without these settings. Only a probe that did not finish still trusts the+extension. The real ffprobe and ffmpeg binaries are used where they are+installed.+"""+import io+import os+import shutil+import subprocess+import sys+import uuid+from unittest.mock import MagicMock, patch++import pytest++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from src.app import app, db+from src.models import Recording, User+from src.utils.ffmpeg_utils import AUDIO_DECODE_FAILED_MESSAGE+from src.utils.ffprobe import (+ FFProbeError, FFProbeNotFoundError, FFProbeTimeoutError, has_audio_stream,+ probe, probe_rejected_input,+)+import src.tasks.processing as proc++app.config["WTF_CSRF_ENABLED"] = False++needs_ffmpeg = pytest.mark.skipif(+ not (shutil.which("ffmpeg") and shutil.which("ffprobe")),+ reason="ffmpeg and ffprobe are not installed",+)++JUNK = b"This is not a video, only text with a video extension.\n" * 64+++def _ffmpeg_video(path, with_audio=True):+ """Write a one-second H.264 MP4, with or without an AAC track."""+ cmd = ["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", "testsrc=size=64x64:rate=10:duration=1"]+ if with_audio:+ cmd += ["-f", "lavfi", "-i", "sine=frequency=440:duration=1", "-c:a", "aac", "-shortest"]+ cmd += ["-c:v", "libx264", "-pix_fmt", "yuv420p", str(path)]+ subprocess.run(cmd, check=True, capture_output=True)+ return path.read_bytes()+++class _Storage:+ def __init__(self, staging):+ self.staging = staging+ self.uploaded = []++ def get_staging_dir(self):+ os.makedirs(self.staging, exist_ok=True)+ return self.staging++ def build_recording_key(self, original_filename, recording_id=None, *, now=None):+ return f"recordings/test/{recording_id}/{original_filename}"++ def upload_local_file(self, local_path, key, *, content_type=None, delete_source=False):+ self.uploaded.append((local_path, key))+ return MagicMock(locator=f"local://{key}", key=key)+++def user():+ with app.app_context():+ s = uuid.uuid4().hex[:8]+ u = User(username=f"junk_{s}", email=f"junk_{s}@local.test", password="x")+ db.session.add(u)+ db.session.commit()+ uid = u.id+ yield uid+ with app.app_context():+ Recording.query.filter_by(user_id=uid).delete()+ db.session.delete(db.session.get(User, uid))+ db.session.commit()+++def _upload(user_id, payload, filename, *, retention=False, passthrough=False, probe_error=None):+ staging = os.path.join(app.config["UPLOAD_FOLDER"], f"stg_{uuid.uuid4().hex[:6]}")+ storage = _Storage(staging)+ client = app.test_client()+ with client.session_transaction() as sess:+ sess["_user_id"] = str(user_id)+ sess["_fresh"] = True+ extra = []+ if probe_error is not None:+ extra.append(patch("src.api.recordings.get_codec_info", side_effect=probe_error))+ with patch("src.api.recordings.get_storage_service", return_value=storage), \+ patch("src.api.recordings.VIDEO_RETENTION", retention), \+ patch("src.api.recordings.VIDEO_PASSTHROUGH_ASR", passthrough), \+ patch("src.api.recordings.chunking_service", None), \+ patch("src.services.job_queue.job_queue.enqueue", return_value=1) as enqueue:+ for p in extra:+ p.start()+ try:+ resp = client.post("/upload", data={"file": (io.BytesIO(payload), filename)},+ content_type="multipart/form-data")+ finally:+ for p in extra:+ p.stop()+ leftovers = os.listdir(staging) if os.path.isdir(staging) else []+ shutil.rmtree(staging, ignore_errors=True)+ return resp, storage, enqueue, leftovers+++def _assert_refused(resp, enqueue, leftovers, user_id):+ assert resp.status_code == 400, resp.get_data(as_text=True)+ body = resp.get_json()+ assert body["code"] == "audio_decode_failed"+ assert body["error"] == AUDIO_DECODE_FAILED_MESSAGE+ text = resp.get_data(as_text=True).lower()+ assert "ffmpeg" not in text and "ffprobe" not in text and "_staging" not in text+ assert leftovers == []+ assert enqueue.call_count == 0+ with app.app_context():+ assert Recording.query.filter_by(user_id=user_id).count() == 0+++# --- helpers ---------------------------------------------------------------++def test_probe_rejected_input_tells_an_unreadable_file_from_a_probe_that_did_not_finish():+ assert probe_rejected_input(FFProbeError("ffprobe failed: Invalid data found when processing input"))+ assert not probe_rejected_input(FFProbeTimeoutError("ffprobe timed out after 10 seconds"))+ assert not probe_rejected_input(FFProbeNotFoundError("ffprobe command not found."))+ assert issubclass(FFProbeTimeoutError, FFProbeError) and issubclass(FFProbeNotFoundError, FFProbeError)+++def test_probe_raises_the_timeout_and_not_found_kinds():+ with patch("src.utils.ffprobe.subprocess.Popen") as popen:+ popen.return_value.communicate.side_effect = subprocess.TimeoutExpired("ffprobe", 1)+ with pytest.raises(FFProbeTimeoutError):+ probe("x.mp4", timeout=1)+ with patch("src.utils.ffprobe.subprocess.Popen", side_effect=FileNotFoundError):+ with pytest.raises(FFProbeNotFoundError):+ probe("x.mp4", timeout=1)+++def test_has_audio_stream():+ assert has_audio_stream({"has_audio": True})+ assert has_audio_stream({"audio_codec": "aac"})+ assert not has_audio_stream({"has_audio": False, "audio_codec": None, "has_video": True})+++# --- /upload -----------------------------------------------------------------++@needs_ffmpeg[email protected]("retention,passthrough", [(True, False), (False, True), (False, False)])+def test_junk_with_a_video_extension_is_refused(user, retention, passthrough):+ resp, storage, enqueue, leftovers = _upload(user, JUNK, "meeting.mp4",+ retention=retention, passthrough=passthrough)+ _assert_refused(resp, enqueue, leftovers, user)+ assert storage.uploaded == []+++@needs_ffmpeg[email protected]("retention,passthrough", [(True, False), (False, True), (False, False)])+def test_video_without_an_audio_track_is_refused_as_without_retention(user, tmp_path, retention, passthrough):+ payload = _ffmpeg_video(tmp_path / "silent.mp4", with_audio=False)+ resp, storage, enqueue, leftovers = _upload(user, payload, "silent.mp4",+ retention=retention, passthrough=passthrough)+ _assert_refused(resp, enqueue, leftovers, user)+++@needs_ffmpeg[email protected]("retention,passthrough", [(True, False), (False, True)])+def test_real_video_with_audio_is_kept_as_video(user, tmp_path, retention, passthrough):+ payload = _ffmpeg_video(tmp_path / "talk.mp4", with_audio=True)+ resp, storage, enqueue, leftovers = _upload(user, payload, "talk.mp4",+ retention=retention, passthrough=passthrough)+ assert resp.status_code in (200, 202), resp.get_data(as_text=True)+ assert enqueue.call_count == 1+ assert len(storage.uploaded) == 1+ assert storage.uploaded[0][0].endswith(".mp4") # the video itself, not extracted audio+++@needs_ffmpeg+def test_real_video_with_audio_has_its_audio_extracted_without_retention(user, tmp_path):+ payload = _ffmpeg_video(tmp_path / "talk.mp4", with_audio=True)+ resp, storage, enqueue, leftovers = _upload(user, payload, "talk.mp4")+ assert resp.status_code in (200, 202), resp.get_data(as_text=True)+ assert enqueue.call_count == 1+ assert not storage.uploaded[0][0].endswith(".mp4") # the extracted audio+++def test_a_probe_that_timed_out_still_trusts_the_video_extension(user):+ """A large MP4 can be slow to probe: it is kept as a video, as before."""+ with patch("src.api.recordings.convert_if_needed") as convert, \+ patch("src.api.recordings.get_duration", return_value=12.0), \+ patch("src.api.recordings.get_creation_date", return_value=None):+ resp, storage, enqueue, leftovers = _upload(+ user, JUNK, "slow.mp4", retention=True,+ probe_error=FFProbeTimeoutError("ffprobe timed out after 10 seconds"))+ assert resp.status_code in (200, 202), resp.get_data(as_text=True)+ assert convert.call_count == 0+ assert enqueue.call_count == 1+++# --- incognito ---------------------------------------------------------------++def _incognito(user_id, payload, filename):+ connector = MagicMock(PROVIDER_NAME="test", supports_diarization=False, default_diarize=False)+ connector.specifications = MagicMock(max_duration_seconds=1800, handles_chunking_internally=False,+ recommended_chunk_seconds=600, unsupported_codecs=[],+ preserve_original_audio=False)+ response = MagicMock(text="hello world transcript", segments=None, speakers=None, speaker_embeddings=None)+ response.has_diarization.return_value = False+ connector.transcribe.return_value = response+ registry = MagicMock()+ registry.get_active_connector.return_value = connector+ client = app.test_client()+ with client.session_transaction() as sess:+ sess["_user_id"] = str(user_id)+ with patch("src.api.recordings.ENABLE_INCOGNITO_MODE", True), \+ patch("src.services.transcription.get_registry", return_value=registry), \+ patch.object(proc, "VIDEO_PASSTHROUGH_ASR", True), \+ patch.object(proc, "chunking_service", None), patch.object(proc, "client", None):+ resp = client.post("/api/recordings/incognito",+ data={"file": (io.BytesIO(payload), filename)},+ content_type="multipart/form-data")+ return resp, connector+++@needs_ffmpeg+def test_incognito_passthrough_refuses_junk_with_a_video_extension(user):+ resp, connector = _incognito(user, JUNK, "meeting.mp4")+ assert resp.status_code == 400, resp.get_data(as_text=True)+ assert resp.get_json()["code"] == "audio_decode_failed"+ assert "ffmpeg" not in resp.get_data(as_text=True).lower()+ assert connector.transcribe.call_count == 0+++@needs_ffmpeg+def test_incognito_passthrough_refuses_a_video_without_audio(user, tmp_path):+ resp, connector = _incognito(user, _ffmpeg_video(tmp_path / "s.mp4", with_audio=False), "s.mp4")+ assert resp.status_code == 400, resp.get_data(as_text=True)+ assert resp.get_json()["code"] == "audio_decode_failed"+ assert connector.transcribe.call_count == 0+++@needs_ffmpeg+def test_incognito_passthrough_sends_a_real_video(user, tmp_path):+ resp, connector = _incognito(user, _ffmpeg_video(tmp_path / "t.mp4"), "t.mp4")+ assert resp.status_code == 200, resp.get_data(as_text=True)+ assert connector.transcribe.call_count == 1