AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(youtube): avoid unnecessary transcript fallback spend
terry-li-hmf780817a5083
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2315262a..47187304 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed++- YouTube transcript fetches now reuse a completed VTT left on disk when yt-dlp times out, honor `.env` values for caption languages, and allow keyed runs to tune the 12-second fast-fail timeout.+## [3.18.0] - 2026-07-21### Changeddiff --git a/CONFIGURATION.md b/CONFIGURATION.mdindex f52c999b..d3fda47a 100644--- a/CONFIGURATION.md+++ b/CONFIGURATION.md@@ -160,6 +160,8 @@ python3 skills/last30days/scripts/last30days.py "MCP servers" \| Jobs / careers pages | none for public ATS pages; web backend improves fallback discovery | `--hiring-signals` and strong Hiring Signals in standard company reports | yes || Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |+**YouTube transcript tuning.** `LAST30DAYS_YT_SUB_LANGS` controls the comma-separated caption-language priority passed to yt-dlp and defaults to `en,es,pt`. When `SCRAPECREATORS_API_KEY` is available, yt-dlp uses one fast attempt before the paid fallback; set `LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT` to the number of seconds allowed for that attempt when a throttled host needs longer than the 12-second default. A VTT completed before the timeout is reused rather than discarded.+**X on cookie-less hosts.** Bird (the free X source) scrapes X using your logged-in browser cookies (`AUTH_TOKEN`/`CT0`), which agent hosts like OpenClaw, CI, or headless runs often can't supply — and scraping carries some account risk. On those, set `XQUIK_API_KEY` (or `XAI_API_KEY`) for full, ranked X coverage from a single API key: the same engagement-based ranking, first-party authorship, and handle (from/mentions) lanes the native X source gets. `--diagnose` reports whether the key is working (and flags an unpaid key).**Example `.env` skeleton** (placeholders only - replace with your own values):diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.pyindex 32925554..eb57d3b6 100644--- a/skills/last30days/scripts/lib/env.py+++ b/skills/last30days/scripts/lib/env.py@@ -564,6 +564,7 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:# resolved above via openai_auth).('GROQ_API_KEY', None),('LAST30DAYS_YT_SUB_LANGS', 'en,es,pt'),+ ('LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT', None),('GITHUB_TOKEN', None),]@@ -576,6 +577,19 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:if config.get('LAST30DAYS_DEBUG'):os.environ.setdefault('LAST30DAYS_DEBUG', config['LAST30DAYS_DEBUG'])+ # youtube_yt reads these tuning knobs lazily from os.environ, so values+ # loaded from .env must be exported into the current engine process.+ if config.get('LAST30DAYS_YT_SUB_LANGS'):+ os.environ.setdefault(+ 'LAST30DAYS_YT_SUB_LANGS',+ config['LAST30DAYS_YT_SUB_LANGS'],+ )+ if config.get('LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT'):+ os.environ.setdefault(+ 'LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT',+ config['LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT'],+ )+# Backward-compat: ScrapeCreators' own examples and tutorials use the# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and# CREATORS). Accept that form too so users who follow the vendor's docsdiff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.pyindex dda63a25..ea2f0a86 100644--- a/skills/last30days/scripts/lib/youtube_yt.py+++ b/skills/last30days/scripts/lib/youtube_yt.py@@ -591,6 +591,18 @@ def _ytdlp_sub_langs() -> str:return ",".join(code.strip().lower() for code in raw.split(",") if code.strip()) or "en,es,pt"+def _transcript_fast_timeout() -> float:+ """Return the keyed-run yt-dlp timeout, preserving the 12s default."""+ raw = os.environ.get("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", "").strip()+ try:+ timeout = float(raw) if raw else float(_TRANSCRIPT_FAST_TIMEOUT)+ except ValueError:+ return float(_TRANSCRIPT_FAST_TIMEOUT)+ if not math.isfinite(timeout) or timeout <= 0:+ return float(_TRANSCRIPT_FAST_TIMEOUT)+ return timeout++def _pick_ytdlp_vtt(video_id: str, temp_dir: str, priority: List[str]) -> Optional[Path]:"""Return the best on-disk VTT match for video_id, preferring priority order."""matches = list(Path(temp_dir).glob(f"{video_id}*.vtt"))@@ -669,7 +681,7 @@ def _fetch_transcript_ytdlp(f"https://www.youtube.com/watch?v={video_id}",]- timeout = _TRANSCRIPT_FAST_TIMEOUT if fast_fail else _TRANSCRIPT_TIMEOUT+ timeout = _transcript_fast_timeout() if fast_fail else _TRANSCRIPT_TIMEOUTattempts = 1 if fast_fail else _TRANSCRIPT_MAX_RETRIES + 1last_reason: Optional[str] = Nonefor attempt in range(attempts):@@ -679,6 +691,12 @@ def _fetch_transcript_ytdlp(last_reason = f"timed out after {timeout}s"_log(f"yt-dlp transcript timed out after {timeout}s for {video_id} "f"(attempt {attempt + 1}/{attempts})")+ # yt-dlp downloads requested languages sequentially. A timeout can+ # therefore leave a complete first-choice VTT on disk; keep it+ # instead of spending a ScrapeCreators fallback credit.+ partial_vtt = _read_vtt(video_id, temp_dir)+ if partial_vtt is not None:+ return partial_vttif attempt < attempts - 1:time.sleep(_transcript_backoff(video_id, attempt))continuediff --git a/tests/test_youtube_transcript_fallback_spend.py b/tests/test_youtube_transcript_fallback_spend.pynew file mode 100644index 00000000..107aec93--- /dev/null+++ b/tests/test_youtube_transcript_fallback_spend.py@@ -0,0 +1,98 @@+from pathlib import Path+from unittest import mock++import pytest+from lib import env, youtube_yt+++def test_youtube_tuning_loaded_from_env_file_reaches_lazy_readers(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ config_file = tmp_path / ".env"+ config_file.write_text(+ "LAST30DAYS_YT_SUB_LANGS=en\nLAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT=25\n",+ encoding="utf-8",+ )+ config_file.chmod(0o600)+ monkeypatch.setattr(env, "CONFIG_DIR", tmp_path)+ monkeypatch.setattr(env, "CONFIG_FILE", config_file)+ monkeypatch.setenv("LAST30DAYS_CONFIG_DIR", str(tmp_path))+ monkeypatch.delenv("LAST30DAYS_YT_SUB_LANGS", raising=False)+ monkeypatch.delenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", raising=False)+ monkeypatch.chdir(tmp_path)++ with (+ mock.patch.object(env, "_load_keychain", return_value={}),+ mock.patch.object(env, "_load_pass", return_value={}),+ ):+ config = env.get_config()++ assert config["LAST30DAYS_YT_SUB_LANGS"] == "en"+ assert config["LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT"] == "25"+ assert youtube_yt._ytdlp_sub_langs() == "en"+ assert youtube_yt._transcript_fast_timeout() == 25.0+++ "value",+ ["", "not-a-number", "0", "-1", "nan", "inf"],+)+def test_invalid_or_empty_fast_timeout_uses_default(+ value: str,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", value)++ assert youtube_yt._transcript_fast_timeout() == 12.0+++def test_timeout_reuses_completed_vtt_without_paid_fallback(+ tmp_path: Path,+) -> None:+ (tmp_path / "video123.en.vtt").write_text(+ "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nusable transcript\n",+ encoding="utf-8",+ )+ status: dict[str, str] = {}++ with mock.patch.object(+ youtube_yt.subproc,+ "run_with_timeout",+ side_effect=youtube_yt.subproc.SubprocTimeout,+ ) as run_mock:+ transcript = youtube_yt._fetch_transcript_ytdlp(+ "video123",+ str(tmp_path),+ status=status,+ fast_fail=True,+ )++ assert transcript is not None+ assert "usable transcript" in transcript+ assert "ytdlp_error" not in status+ assert run_mock.call_count == 1+++def test_fast_timeout_override_controls_keyed_attempt(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", "25")+ status: dict[str, str] = {}++ with mock.patch.object(+ youtube_yt.subproc,+ "run_with_timeout",+ side_effect=youtube_yt.subproc.SubprocTimeout,+ ) as run_mock:+ transcript = youtube_yt._fetch_transcript_ytdlp(+ "video123",+ str(tmp_path),+ status=status,+ fast_fail=True,+ )++ assert transcript is None+ assert run_mock.call_args.kwargs["timeout"] == 25.0+ assert status["ytdlp_error"] == "timed out after 25.0s"diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.pyindex d2fe2e8e..e8cf5e02 100644--- a/tests/test_youtube_yt.py+++ b/tests/test_youtube_yt.py@@ -957,7 +957,8 @@ class TestYtdlpFastFail(unittest.TestCase):"""token present -> one attempt, shortened timeout, no retry sleeps."""with tempfile.TemporaryDirectory() as temp_dir:status = {}- with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \+ with mock.patch.dict(os.environ, {"LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT": ""}), \+ mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \mock.patch.object(youtube_yt.subproc, "run_with_timeout",return_value=self._transient_fail()) as run_mock, \mock.patch.object(youtube_yt.time, "sleep") as sleep_mock: