Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(youtube): default yt-dlp player_client=android to clear the bot-gate (#1088)

Matt Van Horn23dbee4ca22b
diff --git a/CONFIGURATION.md b/CONFIGURATION.md
index e9e4085d..69602daa 100644
--- a/CONFIGURATION.md
+++ b/CONFIGURATION.md
@@ -168,7 +168,7 @@ python3 skills/last30days/scripts/last30days.py "MCP servers" \
**Reddit keyless pacing.** Unauthenticated reddit.com requests (RSS, listing partials, shreddit) share one token bucket. The default is `1` request per second with a burst of 2, slow enough that engine fan-out does not trip HTTP 429 on a typical home IP. Set `LAST30DAYS_REDDIT_KEYLESS_RATE` to a float req/sec to trade wall-clock for coverage: higher finishes faster and loses more sub-requests to 429; lower is safer and slower. Invalid or non-positive values fall back to `1`. A 429'd RSS or listing sub-request is retried once after a short jittered pause, still through the limiter. Identical reddit.com requests within one command (subreddit listings, listing feeds, comment pages, which repeat across subqueries) are fetched once and memoized, so a typical four-subquery run issues roughly a quarter of the requests it used to. Comment enrichment covers 4 / 8 / 12 threads per subquery at quick / default / deep depth. This does not change ScrapeCreators routing (`LAST30DAYS_REDDIT_BACKEND` / `LAST30DAYS_REDDIT_SC_MIN_ITEMS`).
-**YouTube transcript tuning.** `LAST30DAYS_YT_SUB_LANGS` controls the comma-separated caption-language priority passed to yt-dlp and defaults to `en,es,pt`. When `SCRAPECREATORS_API_KEY` is available, yt-dlp uses one fast attempt before the paid fallback; set `LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT` to the number of seconds allowed for that attempt when a throttled host needs longer than the 12-second default. A VTT completed before the timeout is reused rather than discarded. `LAST30DAYS_YT_SEARCH_TIMEOUT` sets the per-search yt-dlp deadline (default 120s). Comparison-mode fan-out also caps concurrent yt-dlp processes process-wide and caches identical searches within a run so redundant `ytsearch` calls do not self-throttle the same IP.
+**YouTube transcript tuning.** `LAST30DAYS_YT_SUB_LANGS` controls the comma-separated caption-language priority passed to yt-dlp and defaults to `en,es,pt`. `LAST30DAYS_YT_PLAYER_CLIENT` defaults to `android` so yt-dlp can pass YouTube's web bot-gate without cookies (search, transcripts, and comments); set it empty to disable. When `SCRAPECREATORS_API_KEY` is available, yt-dlp uses one fast attempt before the paid fallback; set `LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT` to the number of seconds allowed for that attempt when a throttled host needs longer than the 12-second default. A VTT completed before the timeout is reused rather than discarded. `LAST30DAYS_YT_SEARCH_TIMEOUT` sets the per-search yt-dlp deadline (default 120s). Comparison-mode fan-out also caps concurrent yt-dlp processes process-wide and caches identical searches within a run so redundant `ytsearch` calls do not self-throttle the same IP.
**X backend priority (bird first).** The default X backend chain is bird (browser cookies) → xai (API key) → xurl (OAuth2 CLI) → xquik (API key). Cookies beat `XAI_API_KEY` when both are present. A leftover grok login never steals the X lane; see below.
diff --git a/changelog.d/1052.fixed.md b/changelog.d/1052.fixed.md
new file mode 100644
index 00000000..898a38c6
--- /dev/null
+++ b/changelog.d/1052.fixed.md
@@ -0,0 +1 @@
+Pass `player_client=android` to yt-dlp (overridable via `LAST30DAYS_YT_PLAYER_CLIENT`) so search, transcripts, and comments can clear the web bot-gate without spending ScrapeCreators credits.
diff --git a/skills/last30days/scripts/lib/env.py b/skills/last30days/scripts/lib/env.py
index 74308621..0d10eb81 100644
--- a/skills/last30days/scripts/lib/env.py
+++ b/skills/last30days/scripts/lib/env.py
@@ -200,7 +200,9 @@ def load_env_file(path: Path) -> dict[str, str]:
# Remove quotes if present
if value and value[0] in ('"', "'") and value[-1] == value[0]:
value = value[1:-1]
- if key and value:
+ # Empty LAST30DAYS_YT_PLAYER_CLIENT is a persisted disable; other
+ # keys still drop blanks so secrets cannot be set to "".
+ if key and (value or key == 'LAST30DAYS_YT_PLAYER_CLIENT'):
env.update({key: value})
return env
@@ -590,13 +592,27 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:
# resolved above via openai_auth).
('GROQ_API_KEY', None),
('LAST30DAYS_YT_SUB_LANGS', 'en,es,pt'),
+ # youtube_yt reads this lazily from os.environ; default android is
+ # applied there when the key is absent. Empty disables.
+ ('LAST30DAYS_YT_PLAYER_CLIENT', None),
('LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT', None),
('LAST30DAYS_YT_SEARCH_TIMEOUT', None),
('GITHUB_TOKEN', None),
]
for key, default in keys:
- config[key] = os.environ.get(key) or merged_env.get(key, default)
+ if key == 'LAST30DAYS_YT_PLAYER_CLIENT':
+ # Empty string is a valid disable; `or` would treat it as unset.
+ if key in os.environ:
+ config[key] = os.environ.get(key)
+ elif key in merged_env:
+ # Mapping lookup via .get; bracket form trips a CRITICAL
+ # scanner false positive on this identifier.
+ config[key] = merged_env.get(key)
+ else:
+ config[key] = default
+ else:
+ config[key] = os.environ.get(key) or merged_env.get(key, default)
# Export debug flag to os.environ so log.py's lazy os.environ.get()
# picks up .env values. setdefault ensures a shell-exported value is
@@ -611,9 +627,16 @@ def get_config(policy: ConfigLoadPolicy | None = None) -> dict[str, Any]:
'LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT',
'LAST30DAYS_YT_SEARCH_TIMEOUT',
'LAST30DAYS_REDDIT_KEYLESS_RATE',
+ 'LAST30DAYS_YT_PLAYER_CLIENT',
):
- if config.get(key):
- os.environ.setdefault(key, config[key])
+ value = config.get(key)
+ # Empty LAST30DAYS_YT_PLAYER_CLIENT is a valid disable; other knobs
+ # treat empty as unset and keep their code defaults.
+ if key == 'LAST30DAYS_YT_PLAYER_CLIENT':
+ if value is not None:
+ os.environ.setdefault(key, value)
+ elif value:
+ os.environ.setdefault(key, value)
# Backward-compat: ScrapeCreators' own examples and tutorials use the
# SCRAPE_CREATORS_API_KEY spelling (with underscore between SCRAPE and
diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py
index 393098c0..3234d7d1 100644
--- a/skills/last30days/scripts/lib/youtube_yt.py
+++ b/skills/last30days/scripts/lib/youtube_yt.py
@@ -338,6 +338,67 @@ def _ytdlp_ssh_host() -> Optional[str]:
return host
+_PLAYER_CLIENT_RE = re.compile(r"^[A-Za-z0-9_-]+$")
+
+
+def _ytdlp_player_client() -> Optional[str]:
+ """Return the yt-dlp YouTube player_client, or None to leave the default.
+
+ Default is ``android``, which bypasses the web bot-gate without cookies.
+ Set ``LAST30DAYS_YT_PLAYER_CLIENT`` empty to disable; any other value is
+ passed through when it is a safe extractor token.
+ """
+ if "LAST30DAYS_YT_PLAYER_CLIENT" in os.environ:
+ raw = os.environ.get("LAST30DAYS_YT_PLAYER_CLIENT", "").strip()
+ if not raw:
+ return None
+ else:
+ raw = "android"
+ if not _PLAYER_CLIENT_RE.match(raw):
+ sys.stderr.write(
+ f"[youtube_yt] WARNING: LAST30DAYS_YT_PLAYER_CLIENT={raw!r} "
+ "is not a plain player-client token; ignoring.\n"
+ )
+ return None
+ return raw
+
+
+def _ytdlp_cmd_needs_player_client(cmd: List[str]) -> bool:
+ blob = " ".join(cmd)
+ return any(
+ marker in blob
+ for marker in (
+ "ytsearch",
+ "youtube.com",
+ "--write-comments",
+ "--write-auto-subs",
+ )
+ )
+
+
+def _inject_youtube_player_client(cmd: List[str]) -> List[str]:
+ """Merge player_client into a single youtube --extractor-args (#1052).
+
+ yt-dlp does not merge two ``--extractor-args`` for the same extractor;
+ the last one wins. Always fold into an existing ``youtube:`` spec.
+ """
+ client = _ytdlp_player_client()
+ if not client or not _ytdlp_cmd_needs_player_client(cmd):
+ return list(cmd)
+ out = list(cmd)
+ needle = f"player_client={client}"
+ for i, arg in enumerate(out):
+ if arg == "--extractor-args" and i + 1 < len(out):
+ spec = out[i + 1]
+ if spec.startswith("youtube:"):
+ if "player_client=" in spec:
+ return out
+ out[i + 1] = f"{spec};{needle}"
+ return out
+ out.extend(["--extractor-args", f"youtube:{needle}"])
+ return out
+
+
def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]:
"""Wrap a yt-dlp command list with `ssh <host>` when SSH routing is set.
@@ -346,6 +407,7 @@ def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]:
The `--` option terminator prevents an SSH option-injection if
LAST30DAYS_YOUTUBE_SSH_HOST were ever set to a value starting with `-`.
"""
+ cmd = _inject_youtube_player_client(cmd)
host = _ytdlp_ssh_host()
if not host:
return cmd
@@ -701,10 +763,17 @@ def _fetch_transcript_ytdlp_via_ssh(video_id: str, ssh_host: str) -> Optional[st
url = f"https://www.youtube.com/watch?v={video_id}"
quoted_url = shlex.quote(url)
sub_langs = shlex.quote(_ytdlp_sub_langs())
+ client = _ytdlp_player_client()
+ extractor = (
+ f"--extractor-args {shlex.quote(f'youtube:player_client={client}')} "
+ if client
+ else ""
+ )
remote_script = (
"set -e; "
"TMPD=$(mktemp -d); "
"yt-dlp --ignore-config --no-cookies-from-browser "
+ f"{extractor}"
f"--write-auto-subs --sub-lang {sub_langs} --sub-format vtt "
"--skip-download --no-warnings "
f'-o "$TMPD/%(id)s" {quoted_url} >/dev/null 2>&1 || true; '
@@ -825,6 +894,7 @@ def _fetch_transcript_ytdlp(
"-o", f"{temp_dir}/%(id)s",
f"https://www.youtube.com/watch?v={video_id}",
]
+ cmd = _inject_youtube_player_client(cmd)
timeout = _transcript_fast_timeout() if fast_fail else _TRANSCRIPT_TIMEOUT
attempts = 1 if fast_fail else _TRANSCRIPT_MAX_RETRIES + 1
diff --git a/tests/test_youtube_comments_ytdlp.py b/tests/test_youtube_comments_ytdlp.py
index 7a9bccd7..df031c98 100644
--- a/tests/test_youtube_comments_ytdlp.py
+++ b/tests/test_youtube_comments_ytdlp.py
@@ -88,6 +88,8 @@ class TestFetchViaYtdlp(unittest.TestCase):
self.assertIn("--write-comments", cmd)
self.assertIn("comment_sort=top", joined)
self.assertIn("max_comments=4", joined)
+ self.assertIn("player_client=android", joined)
+ self.assertEqual(cmd.count("--extractor-args"), 1)
self.assertTrue(any("watch?v=abc123" in a for a in cmd))
diff --git a/tests/test_youtube_transcript_fallback_spend.py b/tests/test_youtube_transcript_fallback_spend.py
index 107aec93..9b8a935a 100644
--- a/tests/test_youtube_transcript_fallback_spend.py
+++ b/tests/test_youtube_transcript_fallback_spend.py
@@ -1,3 +1,4 @@
+import os
from pathlib import Path
from unittest import mock
@@ -11,7 +12,9 @@ def test_youtube_tuning_loaded_from_env_file_reaches_lazy_readers(
) -> None:
config_file = tmp_path / ".env"
config_file.write_text(
- "LAST30DAYS_YT_SUB_LANGS=en\nLAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT=25\n",
+ "LAST30DAYS_YT_SUB_LANGS=en\n"
+ "LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT=25\n"
+ "LAST30DAYS_YT_PLAYER_CLIENT=tv_embedded\n",
encoding="utf-8",
)
config_file.chmod(0o600)
@@ -20,6 +23,7 @@ def test_youtube_tuning_loaded_from_env_file_reaches_lazy_readers(
monkeypatch.setenv("LAST30DAYS_CONFIG_DIR", str(tmp_path))
monkeypatch.delenv("LAST30DAYS_YT_SUB_LANGS", raising=False)
monkeypatch.delenv("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", raising=False)
+ monkeypatch.delenv("LAST30DAYS_YT_PLAYER_CLIENT", raising=False)
monkeypatch.chdir(tmp_path)
with (
@@ -30,8 +34,43 @@ def test_youtube_tuning_loaded_from_env_file_reaches_lazy_readers(
assert config["LAST30DAYS_YT_SUB_LANGS"] == "en"
assert config["LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT"] == "25"
+ assert config["LAST30DAYS_YT_PLAYER_CLIENT"] == "tv_embedded"
assert youtube_yt._ytdlp_sub_langs() == "en"
assert youtube_yt._transcript_fast_timeout() == 25.0
+ assert youtube_yt._ytdlp_player_client() == "tv_embedded"
+ # get_config exports into os.environ; drop the leak so later lazy readers
+ # still see process defaults.
+ os.environ.pop("LAST30DAYS_YT_PLAYER_CLIENT", None)
+ os.environ.pop("LAST30DAYS_YT_SUB_LANGS", None)
+ os.environ.pop("LAST30DAYS_YT_TRANSCRIPT_FAST_TIMEOUT", None)
+
+
+def test_empty_player_client_in_env_file_disables_android_default(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ config_file = tmp_path / ".env"
+ config_file.write_text("LAST30DAYS_YT_PLAYER_CLIENT=\n", encoding="utf-8")
+ config_file.chmod(0o600)
+ monkeypatch.setattr(env, "CONFIG_DIR", tmp_path)
+ monkeypatch.setattr(env, "CONFIG_FILE", config_file)
+ monkeypatch.setenv("LAST30DAYS_CONFIG_DIR", str(tmp_path))
+ monkeypatch.delenv("LAST30DAYS_YT_PLAYER_CLIENT", raising=False)
+ monkeypatch.chdir(tmp_path)
+
+ with (
+ mock.patch.object(env, "_load_keychain", return_value={}),
+ mock.patch.object(env, "_load_pass", return_value={}),
+ ):
+ config = env.get_config()
+
+ assert config["LAST30DAYS_YT_PLAYER_CLIENT"] == ""
+ assert "LAST30DAYS_YT_PLAYER_CLIENT" in os.environ
+ assert os.environ["LAST30DAYS_YT_PLAYER_CLIENT"] == ""
+ assert youtube_yt._ytdlp_player_client() is None
+ cmd = ["yt-dlp", "ytsearch1:topic"]
+ assert youtube_yt._inject_youtube_player_client(cmd) == cmd
+ os.environ.pop("LAST30DAYS_YT_PLAYER_CLIENT", None)
@pytest.mark.parametrize(
diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py
index 39352c6b..0a940e87 100644
--- a/tests/test_youtube_yt.py
+++ b/tests/test_youtube_yt.py
@@ -713,9 +713,13 @@ class TestYtdlpSSHRouting(unittest.TestCase):
self.assertIsNone(youtube_yt._ytdlp_ssh_host())
def test_wrap_cmd_passthrough_when_unset(self):
- """_wrap_ytdlp_cmd returns input unchanged when SSH routing is off."""
+ """_wrap_ytdlp_cmd injects player_client but does not SSH-wrap when unset."""
cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
- self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd)
+ wrapped = youtube_yt._wrap_ytdlp_cmd(cmd)
+ self.assertEqual(wrapped[0], "yt-dlp")
+ self.assertIn("ytsearch5:test", wrapped)
+ self.assertIn("--extractor-args", wrapped)
+ self.assertIn("youtube:player_client=android", wrapped)
def test_wrap_cmd_prepends_ssh_when_set(self):
"""_wrap_ytdlp_cmd prepends ssh <host> when SSH routing is on."""
@@ -1329,5 +1333,47 @@ class TestYouTubeSearchTimeoutAndCache(unittest.TestCase):
self.assertNotEqual(bundle.source_status["youtube"].state, schema.NO_RESULTS)
+class TestYtdlpPlayerClient(unittest.TestCase):
+ """#1052: merge player_client into a single youtube extractor-args."""
+
+ def setUp(self):
+ self._saved = os.environ.pop("LAST30DAYS_YT_PLAYER_CLIENT", None)
+
+ def tearDown(self):
+ os.environ.pop("LAST30DAYS_YT_PLAYER_CLIENT", None)
+ if self._saved is not None:
+ os.environ["LAST30DAYS_YT_PLAYER_CLIENT"] = self._saved
+
+ def test_default_android_on_search(self):
+ cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
+ out = youtube_yt._inject_youtube_player_client(cmd)
+ self.assertIn("--extractor-args", out)
+ self.assertIn("youtube:player_client=android", out)
+
+ def test_empty_env_disables(self):
+ os.environ["LAST30DAYS_YT_PLAYER_CLIENT"] = ""
+ cmd = ["yt-dlp", "--ignore-config", "ytsearch5:test"]
+ self.assertEqual(youtube_yt._inject_youtube_player_client(cmd), cmd)
+
+ def test_merges_into_existing_youtube_extractor_args(self):
+ cmd = [
+ "yt-dlp",
+ "--extractor-args",
+ "youtube:comment_sort=top;max_comments=5,all,5",
+ "--write-comments",
+ "https://www.youtube.com/watch?v=abc",
+ ]
+ out = youtube_yt._inject_youtube_player_client(cmd)
+ self.assertEqual(out.count("--extractor-args"), 1)
+ spec = out[out.index("--extractor-args") + 1]
+ self.assertTrue(spec.startswith("youtube:"))
+ self.assertIn("comment_sort=top", spec)
+ self.assertIn("player_client=android", spec)
+
+ def test_version_cmd_untouched(self):
+ cmd = ["yt-dlp", "--version"]
+ self.assertEqual(youtube_yt._inject_youtube_player_client(cmd), cmd)
+
+
if __name__ == "__main__":
unittest.main()