Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(youtube): avoid unnecessary transcript fallback spend

terry-li-hmf780817a5083
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2315262a..47187304 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+
+- YouTube transcript fetches now reuse a completed VTT left on disk when yt-dlp times out, honor `.env` values for caption languages, and allow keyed runs to tune the 12-second fast-fail timeout.
+
## [3.18.0] - 2026-07-21
### Changed
diff --git a/CONFIGURATION.md b/CONFIGURATION.md
index f52c999b..d3fda47a 100644
--- a/CONFIGURATION.md
+++ b/CONFIGURATION.md
@@ -160,6 +160,8 @@ python3 skills/last30days/scripts/last30days.py "MCP servers" \
| Jobs / careers pages | none for public ATS pages; web backend improves fallback discovery | `--hiring-signals` and strong Hiring Signals in standard company reports | yes |
| Apify (alternate scraper) | `APIFY_API_TOKEN` | fallback for Reddit/TikTok/Instagram when ScrapeCreators is exhausted | yes (limited) |
+**YouTube transcript tuning.** `LAST30DAYS_YT_SUB_LANGS` controls the comma-separated caption-language priority passed to yt-dlp and defaults to `en,es,pt`. When `SCRAPECREATORS_API_KEY` is available, yt-dlp uses one fast attempt before the paid fallback; set `LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT` to the number of seconds allowed for that attempt when a throttled host needs longer than the 12-second default. A VTT completed before the timeout is reused rather than discarded.
+
**X on cookie-less hosts.** Bird (the free X source) scrapes X using your logged-in browser cookies (`AUTH_TOKEN`/`CT0`), which agent hosts like OpenClaw, CI, or headless runs often can't supply — and scraping carries some account risk. On those, set `XQUIK_API_KEY` (or `XAI_API_KEY`) for full, ranked X coverage from a single API key: the same engagement-based ranking, first-party authorship, and handle (from/mentions) lanes the native X source gets. `--diagnose` reports whether the key is working (and flags an unpaid key).
**Example `.env` skeleton** (placeholders only - replace with your own values):
diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py
index 32925554..eb57d3b6 100644
--- a/skills/last30days/scripts/lib/env.py
+++ b/skills/last30days/scripts/lib/env.py
@@ -564,6 +564,7 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:
# resolved above via openai_auth).
('GROQ_API_KEY', None),
('LAST30DAYS_YT_SUB_LANGS', 'en,es,pt'),
+ ('LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT', None),
('GITHUB_TOKEN', None),
]
@@ -576,6 +577,19 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:
if config.get('LAST30DAYS_DEBUG'):
os.environ.setdefault('LAST30DAYS_DEBUG', config['LAST30DAYS_DEBUG'])
+ # youtube_yt reads these tuning knobs lazily from os.environ, so values
+ # loaded from .env must be exported into the current engine process.
+ if config.get('LAST30DAYS_YT_SUB_LANGS'):
+ os.environ.setdefault(
+ 'LAST30DAYS_YT_SUB_LANGS',
+ config['LAST30DAYS_YT_SUB_LANGS'],
+ )
+ if config.get('LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT'):
+ os.environ.setdefault(
+ 'LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT',
+ config['LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT'],
+ )
+
# Backward-compat: ScrapeCreators' own examples and tutorials use the
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
# CREATORS). Accept that form too so users who follow the vendor's docs
diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py
index dda63a25..ea2f0a86 100644
--- a/skills/last30days/scripts/lib/youtube_yt.py
+++ b/skills/last30days/scripts/lib/youtube_yt.py
@@ -591,6 +591,18 @@ def _ytdlp_sub_langs() -> str:
return ",".join(code.strip().lower() for code in raw.split(",") if code.strip()) or "en,es,pt"
+def _transcript_fast_timeout() -> float:
+ """Return the keyed-run yt-dlp timeout, preserving the 12s default."""
+ raw = os.environ.get("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", "").strip()
+ try:
+ timeout = float(raw) if raw else float(_TRANSCRIPT_FAST_TIMEOUT)
+ except ValueError:
+ return float(_TRANSCRIPT_FAST_TIMEOUT)
+ if not math.isfinite(timeout) or timeout <= 0:
+ return float(_TRANSCRIPT_FAST_TIMEOUT)
+ return timeout
+
+
def _pick_ytdlp_vtt(video_id: str, temp_dir: str, priority: List[str]) -> Optional[Path]:
"""Return the best on-disk VTT match for video_id, preferring priority order."""
matches = list(Path(temp_dir).glob(f"{video_id}*.vtt"))
@@ -669,7 +681,7 @@ def _fetch_transcript_ytdlp(
f"https://www.youtube.com/watch?v={video_id}",
]
- timeout = _TRANSCRIPT_FAST_TIMEOUT if fast_fail else _TRANSCRIPT_TIMEOUT
+ timeout = _transcript_fast_timeout() if fast_fail else _TRANSCRIPT_TIMEOUT
attempts = 1 if fast_fail else _TRANSCRIPT_MAX_RETRIES + 1
last_reason: Optional[str] = None
for attempt in range(attempts):
@@ -679,6 +691,12 @@ def _fetch_transcript_ytdlp(
last_reason = f"timed out after {timeout}s"
_log(f"yt-dlp transcript timed out after {timeout}s for {video_id} "
f"(attempt {attempt + 1}/{attempts})")
+ # yt-dlp downloads requested languages sequentially. A timeout can
+ # therefore leave a complete first-choice VTT on disk; keep it
+ # instead of spending a ScrapeCreators fallback credit.
+ partial_vtt = _read_vtt(video_id, temp_dir)
+ if partial_vtt is not None:
+ return partial_vtt
if attempt < attempts - 1:
time.sleep(_transcript_backoff(video_id, attempt))
continue
diff --git a/tests/test_youtube_transcript_fallback_spend.py b/tests/test_youtube_transcript_fallback_spend.py
new file mode 100644
index 00000000..107aec93
--- /dev/null
+++ b/tests/test_youtube_transcript_fallback_spend.py
@@ -0,0 +1,98 @@
+from pathlib import Path
+from unittest import mock
+
+import pytest
+from lib import env, youtube_yt
+
+
+def test_youtube_tuning_loaded_from_env_file_reaches_lazy_readers(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ config_file = tmp_path / ".env"
+ config_file.write_text(
+ "LAST30DAYS_YT_SUB_LANGS=en\nLAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT=25\n",
+ encoding="utf-8",
+ )
+ config_file.chmod(0o600)
+ monkeypatch.setattr(env, "CONFIG_DIR", tmp_path)
+ monkeypatch.setattr(env, "CONFIG_FILE", config_file)
+ monkeypatch.setenv("LAST30DAYS_CONFIG_DIR", str(tmp_path))
+ monkeypatch.delenv("LAST30DAYS_YT_SUB_LANGS", raising=False)
+ monkeypatch.delenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", raising=False)
+ monkeypatch.chdir(tmp_path)
+
+ with (
+ mock.patch.object(env, "_load_keychain", return_value={}),
+ mock.patch.object(env, "_load_pass", return_value={}),
+ ):
+ config = env.get_config()
+
+ assert config["LAST30DAYS_YT_SUB_LANGS"] == "en"
+ assert config["LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT"] == "25"
+ assert youtube_yt._ytdlp_sub_langs() == "en"
+ assert youtube_yt._transcript_fast_timeout() == 25.0
+
+
+ "value",
+ ["", "not-a-number", "0", "-1", "nan", "inf"],
+)
+def test_invalid_or_empty_fast_timeout_uses_default(
+ value: str,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", value)
+
+ assert youtube_yt._transcript_fast_timeout() == 12.0
+
+
+def test_timeout_reuses_completed_vtt_without_paid_fallback(
+ tmp_path: Path,
+) -> None:
+ (tmp_path / "video123.en.vtt").write_text(
+ "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nusable transcript\n",
+ encoding="utf-8",
+ )
+ status: dict[str, str] = {}
+
+ with mock.patch.object(
+ youtube_yt.subproc,
+ "run_with_timeout",
+ side_effect=youtube_yt.subproc.SubprocTimeout,
+ ) as run_mock:
+ transcript = youtube_yt._fetch_transcript_ytdlp(
+ "video123",
+ str(tmp_path),
+ status=status,
+ fast_fail=True,
+ )
+
+ assert transcript is not None
+ assert "usable transcript" in transcript
+ assert "ytdlp_error" not in status
+ assert run_mock.call_count == 1
+
+
+def test_fast_timeout_override_controls_keyed_attempt(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", "25")
+ status: dict[str, str] = {}
+
+ with mock.patch.object(
+ youtube_yt.subproc,
+ "run_with_timeout",
+ side_effect=youtube_yt.subproc.SubprocTimeout,
+ ) as run_mock:
+ transcript = youtube_yt._fetch_transcript_ytdlp(
+ "video123",
+ str(tmp_path),
+ status=status,
+ fast_fail=True,
+ )
+
+ assert transcript is None
+ assert run_mock.call_args.kwargs["timeout"] == 25.0
+ assert status["ytdlp_error"] == "timed out after 25.0s"
diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py
index d2fe2e8e..e8cf5e02 100644
--- a/tests/test_youtube_yt.py
+++ b/tests/test_youtube_yt.py
@@ -957,7 +957,8 @@ class TestYtdlpFastFail(unittest.TestCase):
"""token present -> one attempt, shortened timeout, no retry sleeps."""
with tempfile.TemporaryDirectory() as temp_dir:
status = {}
- with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
+ with mock.patch.dict(os.environ, {"LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT": ""}), \
+ mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \
mock.patch.object(youtube_yt.subproc, "run_with_timeout",
return_value=self._transient_fail()) as run_mock, \
mock.patch.object(youtube_yt.time, "sleep") as sleep_mock: