Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.
Python agent-infrastructureai-agentai-searchautomationbilibiliclaude-codeclicursorfree-apillm-toolsmcppythonreddit-scrapertwitter-scraperweb-scraperxiaohongshuyoutube-transcript
fix(transcribe): reject oversized media before processing
Pnantf3fef517c504
diff --git a/agent_reach/transcribe.py b/agent_reach/transcribe.pyindex a5a28c8c..1643a090 100644--- a/agent_reach/transcribe.py+++ b/agent_reach/transcribe.py@@ -14,6 +14,7 @@ Designed to be importable from channels (e.g. YouTubeChannel.transcribe).from __future__ import annotationsimport ipaddress+import mathimport shutilimport subprocessimport tempfile@@ -31,6 +32,8 @@ CHUNK_SECONDS = 600 # 10 min — small enough that boundary cuts rarely lose meMAX_SOURCE_BYTES = 512 * 1024 * 1024MAX_CHUNKS = 24 # 4 hours at the standard 10-minute segment sizeMAX_TOTAL_CHUNK_BYTES = 96 * 1024 * 1024+MAX_AUDIO_SECONDS = MAX_CHUNKS * CHUNK_SECONDS+FFPROBE_TIMEOUT_SECONDS = 30PROVIDERS = {"groq": {@@ -78,6 +81,68 @@ def _require_size_at_most(path: Path, limit: int, label: str) -> int:return size+def _probe_audio_duration(path: Path) -> float:+ """Return duration in seconds or fail closed before media generation."""+ _require("ffprobe")+ cmd = [+ "ffprobe",+ "-v",+ "error",+ "-show_entries",+ "format=duration",+ "-of",+ "default=noprint_wrappers=1:nokey=1",+ "-i",+ str(path),+ ]+ try:+ proc = subprocess.run(+ cmd,+ capture_output=True,+ text=True,+ timeout=FFPROBE_TIMEOUT_SECONDS,+ )+ except subprocess.TimeoutExpired:+ raise TranscribeError(+ "ffprobe timed out while reading audio duration "+ f"after {FFPROBE_TIMEOUT_SECONDS}s"+ ) from None+ except OSError as exc:+ raise TranscribeError(+ f"ffprobe could not read audio duration: {exc}"+ ) from exc++ if proc.returncode != 0:+ detail = proc.stderr.strip()[:300] or "unknown ffprobe error"+ raise TranscribeError(+ f"ffprobe failed while reading audio duration: {detail}"+ )++ raw_duration = proc.stdout.strip()+ try:+ duration = float(raw_duration)+ except (TypeError, ValueError):+ raise TranscribeError(+ "ffprobe could not parse a valid audio duration"+ ) from None+ if not math.isfinite(duration) or duration <= 0:+ raise TranscribeError(+ "ffprobe could not parse a valid positive audio duration"+ )+ return duration+++def _require_duration_within_budget(path: Path) -> float:+ """Reject audio that cannot fit within the bounded chunk budget."""+ duration = _probe_audio_duration(path)+ if duration > MAX_AUDIO_SECONDS:+ max_minutes = MAX_AUDIO_SECONDS // 60+ raise TranscribeError(+ f"audio duration exceeds safety limit of {max_minutes} minutes"+ )+ return duration++def _run(cmd: List[str], timeout: int = 600) -> None:"""Run a subprocess, raising TranscribeError on nonzero exit or timeout.@@ -180,6 +245,8 @@ def compress_audio(src: Path, out_dir: Path) -> Path:"-y","-i",str(src),+ "-t",+ str(MAX_AUDIO_SECONDS),"-vn","-ac","1",@@ -195,6 +262,17 @@ def compress_audio(src: Path, out_dir: Path) -> Path:def chunk_audio(src: Path, out_dir: Path, segment_seconds: int = CHUNK_SECONDS) -> List[Path]:"""Split src into segments. Re-encodes each segment so cuts align to keyframes."""+ if segment_seconds <= 0:+ raise TranscribeError("chunk segment duration must be positive")+ possible_chunks = (+ MAX_AUDIO_SECONDS + segment_seconds - 1+ ) // segment_seconds+ if possible_chunks > MAX_CHUNKS:+ raise TranscribeError(+ f"chunk generation safety limit is {MAX_CHUNKS}; "+ f"segment duration {segment_seconds}s could create "+ f"{possible_chunks} chunks"+ )_require("ffmpeg")pattern = out_dir / "chunk_%03d.m4a"_run(@@ -205,6 +283,8 @@ def chunk_audio(src: Path, out_dir: Path, segment_seconds: int = CHUNK_SECONDS)"-y","-i",str(src),+ "-t",+ str(MAX_AUDIO_SECONDS),"-f","segment","-segment_time",@@ -311,6 +391,7 @@ def _transcribe_in_dir(source: str, order: List[str], cfg: Config, work_dir: Pataudio = download_audio(source, work_dir)_require_size_at_most(audio, MAX_SOURCE_BYTES, "source")+ _require_duration_within_budget(audio)compressed = compress_audio(audio, work_dir)if compressed.stat().st_size <= SIZE_LIMIT_BYTES:chunks = [compressed]diff --git a/tests/test_transcribe.py b/tests/test_transcribe.pyindex d4b43ff3..863c688a 100644--- a/tests/test_transcribe.py+++ b/tests/test_transcribe.py@@ -1,6 +1,7 @@# -*- coding: utf-8 -*-"""Tests for agent_reach.transcribe — provider routing, fallback, and errors."""+import subprocessfrom pathlib import Pathfrom typing import List@@ -29,6 +30,12 @@ def chunk_file(tmp_path):return p+def bounded_audio_duration(monkeypatch):+ """Treat synthetic fixture bytes as a short valid audio stream."""+ monkeypatch.setattr(tr, "_probe_audio_duration", lambda _path: 60.0)++class FakeResponse:def __init__(self, status_code: int, text: str = ""):self.status_code = status_code@@ -161,6 +168,96 @@ class TestFallback:class TestOrchestrator:+ def test_rejects_overlong_audio_before_compression(+ self, monkeypatch, fake_config, chunk_file+ ):+ fake_config.set("groq_api_key", "gsk_test")+ events = []++ def fake_run(cmd, **_kwargs):+ events.append(cmd[0])+ return subprocess.CompletedProcess(+ cmd,+ 0,+ stdout=str(tr.MAX_AUDIO_SECONDS + 1),+ stderr="",+ )++ monkeypatch.setattr(tr, "_require", lambda _binary: None)+ monkeypatch.setattr(tr.subprocess, "run", fake_run)+ monkeypatch.setattr(+ tr,+ "compress_audio",+ lambda *_args: (_ for _ in ()).throw(+ AssertionError("overlong audio must fail before compression")+ ),+ )++ with pytest.raises(tr.TranscribeError, match="duration.*limit"):+ tr.transcribe(str(chunk_file), config=fake_config)++ assert events == ["ffprobe"]++ def test_duration_probe_timeout_fails_before_compression(+ self, monkeypatch, fake_config, chunk_file+ ):+ fake_config.set("groq_api_key", "gsk_test")+ observed = {}++ def timeout_probe(cmd, **_kwargs):+ observed["timeout"] = _kwargs.get("timeout")+ raise subprocess.TimeoutExpired(+ cmd,+ timeout=tr.FFPROBE_TIMEOUT_SECONDS,+ )++ monkeypatch.setattr(tr, "_require", lambda _binary: None)+ monkeypatch.setattr(tr.subprocess, "run", timeout_probe)+ monkeypatch.setattr(+ tr,+ "compress_audio",+ lambda *_args: (_ for _ in ()).throw(+ AssertionError("timed-out probe must fail before compression")+ ),+ )++ with pytest.raises(+ tr.TranscribeError,+ match=r"ffprobe timed out.*30s",+ ):+ tr.transcribe(str(chunk_file), config=fake_config)++ assert observed["timeout"] == tr.FFPROBE_TIMEOUT_SECONDS++ def test_unparseable_duration_fails_before_compression(+ self, monkeypatch, fake_config, chunk_file+ ):+ fake_config.set("groq_api_key", "gsk_test")+ monkeypatch.setattr(tr, "_require", lambda _binary: None)+ monkeypatch.setattr(+ tr.subprocess,+ "run",+ lambda cmd, **_kwargs: subprocess.CompletedProcess(+ cmd,+ 0,+ stdout="N/A\n",+ stderr="",+ ),+ )+ monkeypatch.setattr(+ tr,+ "compress_audio",+ lambda *_args: (_ for _ in ()).throw(+ AssertionError("invalid duration must fail before compression")+ ),+ )++ with pytest.raises(+ tr.TranscribeError,+ match=r"ffprobe could not parse.*duration",+ ):+ tr.transcribe(str(chunk_file), config=fake_config)+def test_rejects_oversized_source_before_compression(self, monkeypatch, fake_config, chunk_file):@@ -177,7 +274,14 @@ class TestOrchestrator:with pytest.raises(tr.TranscribeError, match="source.*limit"):tr.transcribe(str(chunk_file), config=fake_config)- def test_local_file_skips_yt_dlp(self, monkeypatch, fake_config, tmp_path, chunk_file):+ def test_local_file_skips_yt_dlp(+ self,+ monkeypatch,+ fake_config,+ tmp_path,+ chunk_file,+ bounded_audio_duration,+ ):fake_config.set("groq_api_key", "gsk_test")def boom_download(*a, **k):@@ -206,7 +310,12 @@ class TestOrchestrator:assert text == "transcript text"def test_chunks_concatenated_with_newlines(- self, monkeypatch, fake_config, tmp_path, chunk_file+ self,+ monkeypatch,+ fake_config,+ tmp_path,+ chunk_file,+ bounded_audio_duration,):fake_config.set("groq_api_key", "gsk_test")# Force the "needs chunking" path by writing a file above the size limit.@@ -234,7 +343,12 @@ class TestOrchestrator:assert text == "part one\npart two"def test_rejects_too_many_chunks_before_any_provider_call(- self, monkeypatch, fake_config, tmp_path, chunk_file+ self,+ monkeypatch,+ fake_config,+ tmp_path,+ chunk_file,+ bounded_audio_duration,):fake_config.set("groq_api_key", "gsk_test")monkeypatch.setattr(tr, "SIZE_LIMIT_BYTES", 1)@@ -262,7 +376,12 @@ class TestOrchestrator:assert provider_calls == []def test_rejects_excessive_total_chunk_bytes_before_provider_calls(- self, monkeypatch, fake_config, tmp_path, chunk_file+ self,+ monkeypatch,+ fake_config,+ tmp_path,+ chunk_file,+ bounded_audio_duration,):fake_config.set("groq_api_key", "gsk_test")monkeypatch.setattr(tr, "SIZE_LIMIT_BYTES", 10)@@ -297,7 +416,13 @@ class TestOrchestrator:with pytest.raises(tr.TranscribeError, match="unknown provider"):tr.transcribe(str(chunk_file), provider="azure", config=fake_config)- def test_auto_temp_dir_is_cleaned_up(self, monkeypatch, fake_config, tmp_path):+ def test_auto_temp_dir_is_cleaned_up(+ self,+ monkeypatch,+ fake_config,+ tmp_path,+ bounded_audio_duration,+ ):fake_config.set("groq_api_key", "gsk_test")created_work_dirs = []@@ -341,7 +466,13 @@ class TestOrchestrator:assert created_work_dirsassert not created_work_dirs[0].exists()- def test_explicit_out_dir_is_preserved(self, monkeypatch, fake_config, tmp_path):+ def test_explicit_out_dir_is_preserved(+ self,+ monkeypatch,+ fake_config,+ tmp_path,+ bounded_audio_duration,+ ):fake_config.set("groq_api_key", "gsk_test")work = tmp_path / "caller-owned"@@ -453,6 +584,61 @@ class TestDownloadAudioSafety:assert captured["cmd"][-1] == "https://youtu.be/abc123"+class TestMediaGenerationBudget:+ def test_compression_has_hard_duration_cap(+ self, monkeypatch, tmp_path, chunk_file+ ):+ captured = {}++ def fake_run(cmd, timeout=600):+ captured["cmd"] = cmd+ (tmp_path / "compressed.m4a").write_bytes(b"compressed")++ monkeypatch.setattr(tr, "_require", lambda _binary: None)+ monkeypatch.setattr(tr, "_run", fake_run)++ tr.compress_audio(chunk_file, tmp_path)++ duration_index = captured["cmd"].index("-t")+ assert captured["cmd"][duration_index + 1] == str(tr.MAX_AUDIO_SECONDS)++ def test_chunk_generation_has_hard_duration_cap(+ self, monkeypatch, tmp_path, chunk_file+ ):+ captured = {}++ def fake_run(cmd, timeout=600):+ captured["cmd"] = cmd+ (tmp_path / "chunk_000.m4a").write_bytes(b"chunk")++ monkeypatch.setattr(tr, "_require", lambda _binary: None)+ monkeypatch.setattr(tr, "_run", fake_run)++ tr.chunk_audio(chunk_file, tmp_path)++ duration_index = captured["cmd"].index("-t")+ assert captured["cmd"][duration_index + 1] == str(tr.MAX_AUDIO_SECONDS)++ def test_chunk_generation_rejects_segment_size_that_can_exceed_budget(+ self, monkeypatch, tmp_path, chunk_file+ ):+ monkeypatch.setattr(tr, "_require", lambda _binary: None)+ monkeypatch.setattr(+ tr,+ "_run",+ lambda *_args, **_kwargs: pytest.fail(+ "unsafe chunk budget must fail before ffmpeg"+ ),+ )++ with pytest.raises(tr.TranscribeError, match=r"chunk.*limit.*24"):+ tr.chunk_audio(+ chunk_file,+ tmp_path,+ segment_seconds=tr.CHUNK_SECONDS - 1,+ )++# --- YouTubeChannel integration --------------------------------------- #